Enable writing histogram-set-json-format.

gni/angle.gni

@@ -142,6 +142,10 @@ declare_args() {
   angle_enable_perf_counter_output = false
 }
 
+declare_args() {
+  angle_has_histograms = angle_has_build
+}
+
 if (!defined(angle_zlib_compression_utils_dir)) {
   angle_zlib_compression_utils_dir = "//third_party/zlib/google"
 }
@@ -206,7 +210,7 @@ set_defaults("angle_test") {
   configs = angle_common_configs - [
               "$angle_root:constructor_and_destructor_warnings",
               "$angle_root:extra_warnings",
-            ]
+            ] + [ "$angle_root/src/tests:angle_maybe_has_histograms" ]
 
   public_configs = []
   public_deps = []
@@ -328,10 +332,20 @@ template("angle_test") {
            ]
 
     sources += [
+      "$angle_root/src/tests/test_utils/runner/HistogramWriter.h",
       "$angle_root/src/tests/test_utils/runner/TestSuite.cpp",
       "$angle_root/src/tests/test_utils/runner/TestSuite.h",
     ]
 
+    if (angle_has_histograms) {
+      sources +=
+          [ "$angle_root/src/tests/test_utils/runner/HistogramWriter.cpp" ]
+      deps += [
+        "//third_party/catapult/tracing/tracing:histogram",
+        "//third_party/catapult/tracing/tracing:reserved_infos",
+      ]
+    }
+
     if ((is_linux && !is_chromeos) ||
         (build_with_chromium && chromeos_is_browser_only)) {
       use_xvfb = true

src/common/debug.h

@@ -407,6 +407,15 @@ std::ostream &FmtHex(std::ostream &os, T value)
 #endif
 
 #if defined(__clang__)
+#    define ANGLE_DISABLE_EXTRA_SEMI_STMT_WARNING \
+        _Pragma("clang diagnostic push") _Pragma("clang diagnostic ignored \"-Wextra-semi-stmt\"")
+#    define ANGLE_REENABLE_EXTRA_SEMI_STMT_WARNING _Pragma("clang diagnostic pop")
+#else
+#    define ANGLE_DISABLE_EXTRA_SEMI_STMT_WARNING
+#    define ANGLE_REENABLE_EXTRA_SEMI_STMT_WARNING
+#endif
+
+#if defined(__clang__)
 #    define ANGLE_DISABLE_SHADOWING_WARNING \
         _Pragma("clang diagnostic push") _Pragma("clang diagnostic ignored \"-Wshadow-field\"")
 #    define ANGLE_REENABLE_SHADOWING_WARNING _Pragma("clang diagnostic pop")

src/tests/BUILD.gn

@@ -16,10 +16,6 @@ declare_args() {
       is_win || is_linux || is_chromeos || is_android || is_mac || is_fuchsia
 }
 
-declare_args() {
-  angle_has_histograms = build_angle_perftests && angle_has_build
-}
-
 angle_test("test_utils_unittest_helper") {
   sources = [
     "../../util/test_utils_unittest_helper.cpp",
@@ -226,6 +222,13 @@ if (is_win || is_linux || is_chromeos || is_mac || is_android) {
   }
 }
 
+config("angle_maybe_has_histograms") {
+  if (angle_has_histograms) {
+    defines = [ "ANGLE_HAS_HISTOGRAMS" ]
+    include_dirs = [ "$root_gen_dir" ]
+  }
+}
+
 template("angle_perftests_common") {
   assert(defined(invoker.test_utils))
 
@@ -243,14 +246,15 @@ template("angle_perftests_common") {
       "perf_tests/third_party/perf/perf_result_reporter.h",
       "perf_tests/third_party/perf/perf_test.cc",
       "perf_tests/third_party/perf/perf_test.h",
+      "test_utils/runner/HistogramWriter.h",
+      "test_utils/runner/TestSuite.h",
     ]
     deps = [ "$angle_jsoncpp_dir:jsoncpp" ]
-    public_deps = [ "${invoker.test_utils}" ]
+    public_deps = [
+      "$angle_root/third_party/rapidjson:rapidjson",
+      "${invoker.test_utils}",
+    ]
     public_configs += [ "${angle_root}:libANGLE_config" ]
-
-    if (angle_has_histograms) {
-      deps += [ "//third_party/catapult/tracing/tracing:histogram" ]
-    }
   }
 }
 

src/tests/perf_tests/ANGLEPerfTest.cpp

@@ -14,6 +14,7 @@
 #include "common/platform.h"
 #include "common/system_utils.h"
 #include "common/utilities.h"
+#include "test_utils/runner/TestSuite.h"
 #include "third_party/perf/perf_test.h"
 #include "third_party/trace_event/trace_event.h"
 #include "util/shader_utils.h"
@@ -359,6 +360,11 @@ double ANGLEPerfTest::printResults()
         printf("Ran %0.2lf iterations per second\n", fps);
     }
 
+    // Output histogram JSON set format if enabled.
+    double secondsPerStep      = elapsedTimeSeconds[0] / static_cast<double>(mNumStepsPerformed);
+    double secondsPerIteration = secondsPerStep / static_cast<double>(mIterationsPerStep);
+    TestSuite::GetInstance()->addHistogramSample(
+        mName + mBackend, mStory, secondsPerIteration * kMilliSecondsPerSecond, "msBestFitFormat");
     return retValue;
 }
 

src/tests/test_utils/runner/HistogramWriter.cpp

+//
+// Copyright 2020 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// HistogramWriter:
+//   Helper class for writing histogram-json-set-format files to JSON.
+
+#include "HistogramWriter.h"
+
+#include "common/debug.h"
+
+#include <rapidjson/document.h>
+
+#if !defined(ANGLE_HAS_HISTOGRAMS)
+#    error "Must have histograms enabled"
+#endif  // !defined(ANGLE_HAS_HISTOGRAMS)
+
+ANGLE_DISABLE_EXTRA_SEMI_WARNING
+ANGLE_DISABLE_EXTRA_SEMI_STMT_WARNING
+ANGLE_DISABLE_DESTRUCTOR_OVERRIDE_WARNING
+ANGLE_DISABLE_SUGGEST_OVERRIDE_WARNINGS
+#include "tracing/tracing/value/diagnostics/reserved_infos.h"
+#include "tracing/tracing/value/histogram.h"
+ANGLE_REENABLE_SUGGEST_OVERRIDE_WARNINGS
+ANGLE_REENABLE_DESTRUCTOR_OVERRIDE_WARNING
+ANGLE_REENABLE_EXTRA_SEMI_STMT_WARNING
+ANGLE_REENABLE_EXTRA_SEMI_WARNING
+
+namespace js    = rapidjson;
+namespace proto = catapult::tracing::tracing::proto;
+
+namespace angle
+{
+namespace
+{
+std::string AsJsonString(const std::string string)
+{
+    return "\"" + string + "\"";
+}
+
+std::string GetUnitAndDirection(proto::UnitAndDirection unit)
+{
+    ASSERT(unit.improvement_direction() == proto::SMALLER_IS_BETTER);
+    ASSERT(unit.unit() == proto::MS_BEST_FIT_FORMAT);
+    return "msBestFitFormat";
+}
+}  // namespace
+
+HistogramWriter::HistogramWriter() = default;
+
+HistogramWriter::~HistogramWriter() = default;
+
+void HistogramWriter::addSample(const std::string &measurement,
+                                const std::string &story,
+                                double value,
+                                const std::string &units)
+{
+    std::string measurementAndStory = measurement + story;
+    if (mHistograms.count(measurementAndStory) == 0)
+    {
+        proto::UnitAndDirection unitAndDirection;
+        unitAndDirection.set_improvement_direction(proto::SMALLER_IS_BETTER);
+        unitAndDirection.set_unit(proto::MS_BEST_FIT_FORMAT);
+
+        std::unique_ptr<catapult::HistogramBuilder> builder =
+            std::make_unique<catapult::HistogramBuilder>(measurement, unitAndDirection);
+
+        // Set all summary options as false - we don't want to generate metric_std, metric_count,
+        // and so on for all metrics.
+        builder->SetSummaryOptions(proto::SummaryOptions());
+        mHistograms[measurementAndStory] = std::move(builder);
+
+        proto::Diagnostic stories;
+        proto::GenericSet *genericSet = stories.mutable_generic_set();
+        genericSet->add_values(AsJsonString(story));
+        mHistograms[measurementAndStory]->AddDiagnostic(catapult::kStoriesDiagnostic, stories);
+    }
+
+    mHistograms[measurementAndStory]->AddSample(value);
+}
+
+void HistogramWriter::getAsJSON(js::Document *doc) const
+{
+    proto::HistogramSet histogramSet;
+
+    for (const auto &histogram : mHistograms)
+    {
+        std::unique_ptr<proto::Histogram> proto = histogram.second->toProto();
+        histogramSet.mutable_histograms()->AddAllocated(proto.release());
+    }
+
+    // Custom JSON serialization for histogram-json-set.
+    doc->SetArray();
+
+    js::Document::AllocatorType &allocator = doc->GetAllocator();
+
+    for (int histogramIndex = 0; histogramIndex < histogramSet.histograms_size(); ++histogramIndex)
+    {
+        const proto::Histogram &histogram = histogramSet.histograms(histogramIndex);
+
+        js::Value obj(js::kObjectType);
+
+        js::Value name(histogram.name(), allocator);
+        obj.AddMember("name", name, allocator);
+
+        js::Value description(histogram.description(), allocator);
+        obj.AddMember("description", description, allocator);
+
+        js::Value unitAndDirection(GetUnitAndDirection(histogram.unit()), allocator);
+        obj.AddMember("unit", unitAndDirection, allocator);
+
+        if (histogram.has_diagnostics())
+        {
+            js::Value diags(js::kObjectType);
+
+            for (const auto &mapIter : histogram.diagnostics().diagnostic_map())
+            {
+                js::Value key(mapIter.first, allocator);
+                const proto::Diagnostic &diagnostic = mapIter.second;
+
+                if (!diagnostic.shared_diagnostic_guid().empty())
+                {
+                    js::Value guid(diagnostic.shared_diagnostic_guid(), allocator);
+                    diags.AddMember(key, guid, allocator);
+                }
+                else if (diagnostic.has_generic_set())
+                {
+                    const proto::GenericSet genericSet = diagnostic.generic_set();
+
+                    js::Value setObj(js::kObjectType);
+                    setObj.AddMember("type", "GenericSet", allocator);
+
+                    js::Value values(js::kArrayType);
+
+                    for (const std::string &value : genericSet.values())
+                    {
+                        js::Value valueStr(value, allocator);
+                        values.PushBack(valueStr, allocator);
+                    }
+
+                    setObj.AddMember("values", values, allocator);
+
+                    diags.AddMember(key, setObj, allocator);
+                }
+                else
+                {
+                    UNREACHABLE();
+                }
+            }
+
+            obj.AddMember("diagnostics", diags, allocator);
+        }
+
+        js::Value sampleValues(js::kArrayType);
+
+        for (int sampleIndex = 0; sampleIndex < histogram.sample_values_size(); ++sampleIndex)
+        {
+            js::Value sample(histogram.sample_values(sampleIndex));
+            sampleValues.PushBack(sample, allocator);
+        }
+
+        obj.AddMember("sampleValues", sampleValues, allocator);
+
+        js::Value maxNumSamplesValues(histogram.max_num_sample_values());
+        obj.AddMember("maxNumSamplesValues", maxNumSamplesValues, allocator);
+
+        if (histogram.has_bin_boundaries())
+        {
+            js::Value binBoundaries(js::kArrayType);
+
+            const proto::BinBoundaries &boundaries = histogram.bin_boundaries();
+            for (int binIndex = 0; binIndex < boundaries.bin_specs_size(); ++binIndex)
+            {
+                js::Value binSpec(boundaries.bin_specs(binIndex).bin_boundary());
+                binBoundaries.PushBack(binSpec, allocator);
+            }
+
+            obj.AddMember("binBoundaries", binBoundaries, allocator);
+        }
+
+        if (histogram.has_summary_options())
+        {
+            const proto::SummaryOptions &options = histogram.summary_options();
+
+            js::Value summary(js::kObjectType);
+
+            js::Value avg(options.avg());
+            js::Value count(options.count());
+            js::Value max(options.max());
+            js::Value min(options.min());
+            js::Value std(options.std());
+            js::Value sum(options.sum());
+
+            summary.AddMember("avg", avg, allocator);
+            summary.AddMember("count", count, allocator);
+            summary.AddMember("max", max, allocator);
+            summary.AddMember("min", min, allocator);
+            summary.AddMember("std", std, allocator);
+            summary.AddMember("sum", sum, allocator);
+
+            obj.AddMember("summaryOptions", summary, allocator);
+        }
+
+        if (histogram.has_running())
+        {
+            const proto::RunningStatistics &running = histogram.running();
+
+            js::Value stats(js::kArrayType);
+
+            js::Value count(running.count());
+            js::Value max(running.max());
+            js::Value meanlogs(running.meanlogs());
+            js::Value mean(running.mean());
+            js::Value min(running.min());
+            js::Value sum(running.sum());
+            js::Value variance(running.variance());
+
+            stats.PushBack(count, allocator);
+            stats.PushBack(max, allocator);
+            stats.PushBack(meanlogs, allocator);
+            stats.PushBack(mean, allocator);
+            stats.PushBack(min, allocator);
+            stats.PushBack(sum, allocator);
+            stats.PushBack(variance, allocator);
+
+            obj.AddMember("running", stats, allocator);
+        }
+
+        doc->PushBack(obj, allocator);
+    }
+
+    for (const auto &diagnosticIt : histogramSet.shared_diagnostics())
+    {
+        const proto::Diagnostic &diagnostic = diagnosticIt.second;
+
+        js::Value obj(js::kObjectType);
+
+        js::Value name(diagnosticIt.first, allocator);
+        obj.AddMember("name", name, allocator);
+
+        switch (diagnostic.diagnostic_oneof_case())
+        {
+            case proto::Diagnostic::kGenericSet:
+            {
+                js::Value type("GenericSet", allocator);
+                obj.AddMember("type", type, allocator);
+
+                const proto::GenericSet &genericSet = diagnostic.generic_set();
+
+                js::Value values(js::kArrayType);
+
+                for (const std::string &value : genericSet.values())
+                {
+                    js::Value valueStr(value, allocator);
+                    values.PushBack(valueStr, allocator);
+                }
+
+                obj.AddMember("values", values, allocator);
+                break;
+            }
+
+            default:
+                UNREACHABLE();
+        }
+
+        doc->PushBack(obj, allocator);
+    }
+}
+}  // namespace angle

src/tests/test_utils/runner/HistogramWriter.h

+//
+// Copyright 2020 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// HistogramWriter:
+//   Helper class for writing histogram-json-set-format files to JSON.
+
+#ifndef ANGLE_TESTS_TEST_UTILS_HISTOGRAM_WRITER_H_
+#define ANGLE_TESTS_TEST_UTILS_HISTOGRAM_WRITER_H_
+
+#include <map>
+#include <memory>
+#include <string>
+
+// Include forward delcarations for rapidjson types.
+#include <rapidjson/fwd.h>
+
+namespace catapult
+{
+class HistogramBuilder;
+}  // namespace catapult
+
+namespace angle
+{
+class HistogramWriter
+{
+  public:
+    HistogramWriter();
+    ~HistogramWriter();
+
+    void addSample(const std::string &measurement,
+                   const std::string &story,
+                   double value,
+                   const std::string &units);
+
+    void getAsJSON(rapidjson::Document *doc) const;
+
+  private:
+#if defined(ANGLE_HAS_HISTOGRAMS)
+    std::map<std::string, std::unique_ptr<catapult::HistogramBuilder>> mHistograms;
+#endif  // defined(ANGLE_HAS_HISTOGRAMS)
+};
+
+// Define a stub implementation when histograms are compiled out.
+#if !defined(ANGLE_HAS_HISTOGRAMS)
+inline HistogramWriter::HistogramWriter()  = default;
+inline HistogramWriter::~HistogramWriter() = default;
+inline void HistogramWriter::addSample(const std::string &measurement,
+                                       const std::string &story,
+                                       double value,
+                                       const std::string &units)
+{}
+inline void HistogramWriter::getAsJSON(rapidjson::Document *doc) const {}
+#endif  // !defined(ANGLE_HAS_HISTOGRAMS)
+
+}  // namespace angle
+
+#endif  // ANGLE_TESTS_TEST_UTILS_HISTOGRAM_WRITER_H_

src/tests/test_utils/runner/TestSuite.cpp

@@ -315,14 +315,14 @@ void WriteResultsFile(bool interrupted,
     }
 }
 
-void WriteHistogramJson(const TestResults &testResults,
+void WriteHistogramJson(const HistogramWriter &histogramWriter,
                         const std::string &outputFile,
                         const char *testSuiteName)
 {
     js::Document doc;
     doc.SetArray();
 
-    // TODO: http://anglebug.com/4769 - Implement histogram output.
+    histogramWriter.getAsJSON(&doc);
 
     printf("Writing histogram json to %s\n", outputFile.c_str());
 
@@ -335,6 +335,7 @@ void WriteHistogramJson(const TestResults &testResults,
 void WriteOutputFiles(bool interrupted,
                       const TestResults &testResults,
                       const std::string &resultsFile,
+                      const HistogramWriter &histogramWriter,
                       const std::string &histogramJsonOutputFile,
                       const char *testSuiteName)
 {
@@ -345,7 +346,7 @@ void WriteOutputFiles(bool interrupted,
 
     if (!histogramJsonOutputFile.empty())
     {
-        WriteHistogramJson(testResults, histogramJsonOutputFile, testSuiteName);
+        WriteHistogramJson(histogramWriter, histogramJsonOutputFile, testSuiteName);
     }
 }
 
@@ -382,11 +383,13 @@ class TestEventListener : public testing::EmptyTestEventListener
     TestEventListener(const std::string &resultsFile,
                       const std::string &histogramJsonFile,
                       const char *testSuiteName,
-                      TestResults *testResults)
+                      TestResults *testResults,
+                      HistogramWriter *histogramWriter)
         : mResultsFile(resultsFile),
           mHistogramJsonFile(histogramJsonFile),
           mTestSuiteName(testSuiteName),
-          mTestResults(testResults)
+          mTestResults(testResults),
+          mHistogramWriter(histogramWriter)
     {}
 
     void OnTestStart(const testing::TestInfo &testInfo) override
@@ -409,7 +412,8 @@ class TestEventListener : public testing::EmptyTestEventListener
     {
         std::lock_guard<std::mutex> guard(mTestResults->currentTestMutex);
         mTestResults->allDone = true;
-        WriteOutputFiles(false, *mTestResults, mResultsFile, mHistogramJsonFile, mTestSuiteName);
+        WriteOutputFiles(false, *mTestResults, mResultsFile, *mHistogramWriter, mHistogramJsonFile,
+                         mTestSuiteName);
     }
 
   private:
@@ -417,6 +421,7 @@ class TestEventListener : public testing::EmptyTestEventListener
     std::string mHistogramJsonFile;
     const char *mTestSuiteName;
     TestResults *mTestResults;
+    HistogramWriter *mHistogramWriter;
 };
 
 bool IsTestDisabled(const testing::TestInfo &testInfo)
@@ -816,6 +821,9 @@ void GTestListTests(const std::map<TestIdentifier, TestResult> &resultsMap)
 }
 }  // namespace
 
+// static
+TestSuite *TestSuite::mInstance = nullptr;
+
 TestIdentifier::TestIdentifier() = default;
 
 TestIdentifier::TestIdentifier(const std::string &suiteNameIn, const std::string &nameIn)
@@ -889,6 +897,9 @@ TestSuite::TestSuite(int *argc, char **argv)
       mBatchId(-1),
       mFlakyRetries(0)
 {
+    ASSERT(mInstance == nullptr);
+    mInstance = this;
+
     Optional<int> filterArgIndex;
     bool alsoRunDisabledTests = false;
 
@@ -1091,7 +1102,8 @@ TestSuite::TestSuite(int *argc, char **argv)
     {
         testing::TestEventListeners &listeners = testing::UnitTest::GetInstance()->listeners();
         listeners.Append(new TestEventListener(mResultsFile, mHistogramJsonFile,
-                                               mTestSuiteName.c_str(), &mTestResults));
+                                               mTestSuiteName.c_str(), &mTestResults,
+                                               &mHistogramWriter));
 
         for (const TestIdentifier &id : testSet)
         {
@@ -1151,7 +1163,8 @@ void TestSuite::onCrashOrTimeout(TestResultType crashOrTimeout)
         return;
     }
 
-    WriteOutputFiles(true, mTestResults, mResultsFile, mHistogramJsonFile, mTestSuiteName.c_str());
+    WriteOutputFiles(true, mTestResults, mResultsFile, mHistogramWriter, mHistogramJsonFile,
+                     mTestSuiteName.c_str());
 }
 
 bool TestSuite::launchChildTestProcess(uint32_t batchId,
@@ -1522,7 +1535,8 @@ int TestSuite::run()
     }
 
     // Dump combined results.
-    WriteOutputFiles(false, mTestResults, mResultsFile, mHistogramJsonFile, mTestSuiteName.c_str());
+    WriteOutputFiles(false, mTestResults, mResultsFile, mHistogramWriter, mHistogramJsonFile,
+                     mTestSuiteName.c_str());
 
     totalRunTime.stop();
     printf("Tests completed in %lf seconds\n", totalRunTime.getElapsedTime());
@@ -1587,6 +1601,14 @@ void TestSuite::startWatchdog()
     mWatchdogThread = std::thread(watchdogMain);
 }
 
+void TestSuite::addHistogramSample(const std::string &measurement,
+                                   const std::string &story,
+                                   double value,
+                                   const std::string &units)
+{
+    mHistogramWriter.addSample(measurement, story, value, units);
+}
+
 bool GetTestResultsFromFile(const char *fileName, TestResults *resultsOut)
 {
     std::ifstream ifs(fileName);

src/tests/test_utils/runner/TestSuite.h

@@ -6,6 +6,9 @@
 // TestSuite:
 //   Basic implementation of a test harness in ANGLE.
 
+#ifndef ANGLE_TESTS_TEST_UTILS_TEST_SUITE_H_
+#define ANGLE_TESTS_TEST_UTILS_TEST_SUITE_H_
+
 #include <map>
 #include <memory>
 #include <mutex>
@@ -13,6 +16,7 @@
 #include <string>
 #include <thread>
 
+#include "HistogramWriter.h"
 #include "util/test_utils.h"
 
 namespace angle
@@ -122,6 +126,12 @@ class TestSuite
 
     int run();
     void onCrashOrTimeout(TestResultType crashOrTimeout);
+    void addHistogramSample(const std::string &measurement,
+                            const std::string &story,
+                            double value,
+                            const std::string &units);
+
+    static TestSuite *GetInstance() { return mInstance; }
 
   private:
     bool parseSingleArg(const char *argument);
@@ -130,6 +140,8 @@ class TestSuite
     int printFailuresAndReturnCount() const;
     void startWatchdog();
 
+    static TestSuite *mInstance;
+
     std::string mTestExecutableName;
     std::string mTestSuiteName;
     TestQueue mTestQueue;
@@ -160,7 +172,10 @@ class TestSuite
     std::map<TestIdentifier, FileLine> mTestFileLines;
     std::vector<ProcessInfo> mCurrentProcesses;
     std::thread mWatchdogThread;
+    HistogramWriter mHistogramWriter;
 };
 
 bool GetTestResultsFromFile(const char *fileName, TestResults *resultsOut);
 }  // namespace angle
+
+#endif  // ANGLE_TESTS_TEST_UTILS_TEST_SUITE_H_