Add glmark2 to angle_perftests

src/tests/BUILD.gn

@@ -51,13 +51,6 @@ if (build_with_chromium) {
     if (build_angle_gles1_conform_tests) {
       deps += [ "//src/tests:angle_gles1_conformance_tests" ]
     }
-
-    if (is_win) {
-      deps += [
-        "//third_party/glmark2:glmark2_angle",
-        "//third_party/glmark2:glmark2_wgl",
-      ]
-    }
   }
 }
 
@@ -282,6 +275,16 @@ if (is_win || is_linux || is_android || is_mac || is_fuchsia) {
     if (is_android) {
       use_native_activity = true
     }
+
+    if (is_win || is_linux) {
+      data_deps += [ "${angle_root}/third_party/glmark2:glmark2_angle" ]
+    }
+    if (is_win) {
+      data_deps += [ "${angle_root}/third_party/glmark2:glmark2_wgl" ]
+    }
+    if (is_linux) {
+      data_deps += [ "${angle_root}/third_party/glmark2:glmark2_glx" ]
+    }
   }
 }
 

src/tests/angle_perftests.gni

@@ -5,6 +5,8 @@
 angle_perf_tests_sources = [
   "perf_tests/ANGLEPerfTest.cpp",
   "perf_tests/ANGLEPerfTest.h",
+  "perf_tests/ANGLEPerfTestArgs.cpp",
+  "perf_tests/ANGLEPerfTestArgs.h",
   "perf_tests/BlitFramebufferPerf.cpp",
   "perf_tests/BindingPerf.cpp",
   "perf_tests/BufferSubData.cpp",
@@ -26,6 +28,7 @@ angle_perf_tests_sources = [
   "perf_tests/TexturesPerf.cpp",
   "perf_tests/UniformsPerf.cpp",
   "perf_tests/VulkanBarriersPerf.cpp",
+  "perf_tests/glmark2Benchmark.cpp",
   "perf_tests/third_party/perf/perf_test.cc",
   "perf_tests/third_party/perf/perf_test.h",
   "test_utils/angle_test_configs.cpp",
@@ -41,6 +44,8 @@ angle_white_box_perf_tests_sources = [
   "angle_unittests_utils.h",
   "perf_tests/ANGLEPerfTest.cpp",
   "perf_tests/ANGLEPerfTest.h",
+  "perf_tests/ANGLEPerfTestArgs.cpp",
+  "perf_tests/ANGLEPerfTestArgs.h",
   "perf_tests/BitSetIteratorPerf.cpp",
   "perf_tests/CompilerPerf.cpp",
   "perf_tests/EGLInitializePerf.cpp",  # Uses ANGLEGetDisplayPlatform, a non-standard EP.

src/tests/perf_tests/ANGLEPerfTest.cpp

@@ -9,6 +9,7 @@
 
 #include "ANGLEPerfTest.h"
 
+#include "ANGLEPerfTestArgs.h"
 #include "common/platform.h"
 #include "third_party/perf/perf_test.h"
 #include "util/shader_utils.h"
@@ -26,6 +27,8 @@
 #    include "util/windows/WGLWindow.h"
 #endif  // defined(ANGLE_USE_UTIL_LOADER) &&defined(ANGLE_PLATFORM_WINDOWS)
 
+using namespace angle;
+
 namespace
 {
 constexpr size_t kInitialTraceEventBufferSize = 50000;
@@ -35,11 +38,6 @@ constexpr double kCalibrationRunTimeSeconds   = 1.0;
 constexpr double kMaximumRunTimeSeconds       = 10.0;
 constexpr unsigned int kNumTrials             = 3;
 
-bool gCalibration = false;
-Optional<unsigned int> gStepsToRunOverride;
-bool gEnableTrace      = false;
-const char *gTraceFile = "ANGLETrace.json";
-
 struct TraceCategory
 {
     unsigned char enabled;
@@ -157,11 +155,6 @@ void DumpTraceEventsToJSONFile(const std::vector<TraceEvent> &traceEvents,
 
     outFile.close();
 }
-
-bool OneFrame()
-{
-    return gStepsToRunOverride.valid() && gStepsToRunOverride.value() == 1;
-}
 }  // anonymous namespace
 
 ANGLEPerfTest::ANGLEPerfTest(const std::string &name,
@@ -614,46 +607,3 @@ EGLWindow *ANGLERenderTest::createEGLWindow(const RenderTestParams &testParams)
     return EGLWindow::New(testParams.majorVersion, testParams.minorVersion,
                           testParams.eglParameters);
 }
-
-void ANGLEProcessPerfTestArgs(int *argc, char **argv)
-{
-    int argcOutCount = 0;
-
-    for (int argIndex = 0; argIndex < *argc; argIndex++)
-    {
-        if (strcmp("--one-frame-only", argv[argIndex]) == 0)
-        {
-            gStepsToRunOverride = 1;
-        }
-        else if (strcmp("--enable-trace", argv[argIndex]) == 0)
-        {
-            gEnableTrace = true;
-        }
-        else if (strcmp("--trace-file", argv[argIndex]) == 0 && argIndex < *argc - 1)
-        {
-            gTraceFile = argv[argIndex];
-            // Skip an additional argument.
-            argIndex++;
-        }
-        else if (strcmp("--calibration", argv[argIndex]) == 0)
-        {
-            gCalibration = true;
-        }
-        else if (strcmp("--steps", argv[argIndex]) == 0 && argIndex < *argc - 1)
-        {
-            unsigned int stepsToRun = 0;
-            std::stringstream strstr;
-            strstr << argv[argIndex + 1];
-            strstr >> stepsToRun;
-            gStepsToRunOverride = stepsToRun;
-            // Skip an additional argument.
-            argIndex++;
-        }
-        else
-        {
-            argv[argcOutCount++] = argv[argIndex];
-        }
-    }
-
-    *argc = argcOutCount;
-}

src/tests/perf_tests/ANGLEPerfTestArgs.cpp

+//
+// Copyright 2019 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// ANGLEPerfTestArgs.cpp:
+//   Parse command line arguments for angle_perftests.
+//
+
+#include "ANGLEPerfTestArgs.h"
+#include <string.h>
+#include <sstream>
+
+namespace angle
+{
+bool gCalibration = false;
+Optional<unsigned int> gStepsToRunOverride;
+bool gEnableTrace      = false;
+const char *gTraceFile = "ANGLETrace.json";
+}  // namespace angle
+
+using namespace angle;
+
+void ANGLEProcessPerfTestArgs(int *argc, char **argv)
+{
+    int argcOutCount = 0;
+
+    for (int argIndex = 0; argIndex < *argc; argIndex++)
+    {
+        if (strcmp("--one-frame-only", argv[argIndex]) == 0)
+        {
+            gStepsToRunOverride = 1;
+        }
+        else if (strcmp("--enable-trace", argv[argIndex]) == 0)
+        {
+            gEnableTrace = true;
+        }
+        else if (strcmp("--trace-file", argv[argIndex]) == 0 && argIndex < *argc - 1)
+        {
+            gTraceFile = argv[argIndex];
+            // Skip an additional argument.
+            argIndex++;
+        }
+        else if (strcmp("--calibration", argv[argIndex]) == 0)
+        {
+            gCalibration = true;
+        }
+        else if (strcmp("--steps", argv[argIndex]) == 0 && argIndex < *argc - 1)
+        {
+            unsigned int stepsToRun = 0;
+            std::stringstream strstr;
+            strstr << argv[argIndex + 1];
+            strstr >> stepsToRun;
+            gStepsToRunOverride = stepsToRun;
+            // Skip an additional argument.
+            argIndex++;
+        }
+        else
+        {
+            argv[argcOutCount++] = argv[argIndex];
+        }
+    }
+
+    *argc = argcOutCount;
+}

src/tests/perf_tests/ANGLEPerfTestArgs.h

+//
+// Copyright 2019 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// ANGLEPerfTestArgs.h:
+//   Command line arguments for angle_perftests.
+//
+
+#ifndef TESTS_PERF_TESTS_ANGLE_PERF_TEST_ARGS_H_
+#define TESTS_PERF_TESTS_ANGLE_PERF_TEST_ARGS_H_
+
+#include "common/Optional.h"
+
+namespace angle
+{
+extern bool gCalibration;
+extern Optional<unsigned int> gStepsToRunOverride;
+extern bool gEnableTrace;
+extern const char *gTraceFile;
+
+inline bool OneFrame()
+{
+    return gStepsToRunOverride.valid() && gStepsToRunOverride.value() == 1;
+}
+}  // namespace angle
+
+#endif  // TESTS_PERF_TESTS_ANGLE_PERF_TEST_ARGS_H_

src/tests/perf_tests/README.md

@@ -39,5 +39,6 @@ ANGLE implements a no-op driver for OpenGL, D3D11 and Vulkan. To run on these co
 * [`TextureSamplingBenchmark`](TextureSampling.cpp): Tests Texture sampling performance.
 * [`TextureBenchmark`](TexturesPerf.cpp): Tests Texture state change performance.
 * [`LinkProgramBenchmark`](LinkProgramPerfTest.cpp): Tests performance of `glLinkProgram`.
+* [`glmark2`](glmark2.cpp): Runs the glmark2 benchmark.
 
 Many other tests can be found that have documentation in their classes.

src/tests/perf_tests/glmark2Benchmark.cpp

+//
+// Copyright 2019 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// glmark2Benchmark:
+//   Runs the glmark2 benchmark.
+//
+
+#include <gtest/gtest.h>
+
+#include <stdio.h>
+#include <sstream>
+
+#include "../perf_tests/third_party/perf/perf_test.h"
+#include "ANGLEPerfTestArgs.h"
+#include "common/platform.h"
+#include "common/system_utils.h"
+#include "test_utils/angle_test_configs.h"
+#include "test_utils/angle_test_instantiate.h"
+
+using namespace angle;
+
+namespace
+{
+
+struct BenchmarkInfo
+{
+    const char *glmark2Config;
+    const char *gtestSuffix;
+};
+
+// Each glmark2 scene is individually benchmarked.  If glmark2 is run without a specific benchmark,
+// it can produce an aggregate score, which is not interesting at the moment.  Adding an empty
+// string ("") to this list will enable a test where glmark2 is run with the default scenes and the
+// score for each test as well as the overall score is output.
+constexpr BenchmarkInfo kBenchmarks[] = {
+    {"build:use-vbo=false", "build"},
+    {"build:use-vbo=true", "build_vbo"},
+    {"texture:texture-filter=nearest", "texture_nearest"},
+    {"texture:texture-filter=linear", "texture_linear"},
+    {"texture:texture-filter=mipmap", "texture_mipmap"},
+    {"shading:shading=gouraud", "shading_gouraud"},
+    {"shading:shading=blinn-phong-inf", "shading_blinn_phong"},
+    {"shading:shading=phong", "shading_phong"},
+    {"shading:shading=cel", "shading_cel"},
+    {"bump:bump-render=high-poly", "bump_high_poly"},
+    {"bump:bump-render=normals", "bump_normals"},
+    {"bump:bump-render=height", "bump_height"},
+    {"effect2d:kernel=0,1,0;1,-4,1;0,1,0;", "effect2d_edge"},
+    {"effect2d:kernel=1,1,1,1,1;1,1,1,1,1;1,1,1,1,1;", "effect2d_blur"},
+    {"pulsar:light=false:quads=5:texture=false", "pulsar"},
+    {"desktop:blur-radius=5:effect=blur:passes=1:separable=true:windows=4", "desktop_blur"},
+    {"desktop:effect=shadow:windows=4", "desktop_shadow"},
+    {"buffer:columns=200:interleave=false:update-dispersion=0.9:update-fraction=0.5:update-method="
+     "map",
+     "buffer_map"},
+    {"buffer:columns=200:interleave=false:update-dispersion=0.9:update-fraction=0.5:update-method="
+     "subdata",
+     "buffer_subdata"},
+    {"buffer:columns=200:interleave=true:update-dispersion=0.9:update-fraction=0.5:update-method="
+     "map",
+     "buffer_map_interleave"},
+    {"ideas:speed=duration", "ideas"},
+    {"jellyfish", "jellyfish"},
+    {"terrain", "terrain"},
+    {"shadow", "shadow"},
+    {"refract", "refract"},
+    {"conditionals:fragment-steps=0:vertex-steps=0", "conditionals"},
+    {"conditionals:fragment-steps=5:vertex-steps=0", "conditionals_fragment"},
+    {"conditionals:fragment-steps=0:vertex-steps=5", "conditionals_vertex"},
+    {"function:fragment-complexity=low:fragment-steps=5", "function"},
+    {"function:fragment-complexity=medium:fragment-steps=5", "function_complex"},
+    {"loop:fragment-loop=false:fragment-steps=5:vertex-steps=5", "loop_no_fsloop"},
+    {"loop:fragment-steps=5:fragment-uniform=false:vertex-steps=5", "loop_no_uniform"},
+    {"loop:fragment-steps=5:fragment-uniform=true:vertex-steps=5", "loop"},
+};
+
+using GLMark2BenchmarkTestParams = std::tuple<PlatformParameters, size_t>;
+std::string GLMark2BenchmarkPrint(
+    const ::testing::TestParamInfo<GLMark2BenchmarkTestParams> &paramsInfo)
+{
+    const GLMark2BenchmarkTestParams &params = paramsInfo.param;
+    std::ostringstream out;
+
+    out << std::get<0>(params) << '_';
+    out << kBenchmarks[std::get<1>(params)].gtestSuffix;
+
+    return out.str();
+}
+
+class GLMark2Benchmark : public testing::TestWithParam<GLMark2BenchmarkTestParams>
+{
+  public:
+    GLMark2Benchmark()
+    {
+        switch (std::get<0>(GetParam()).getRenderer())
+        {
+            case EGL_PLATFORM_ANGLE_TYPE_D3D11_ANGLE:
+                mBackend = "d3d11";
+                break;
+            case EGL_PLATFORM_ANGLE_TYPE_OPENGLES_ANGLE:
+                mBackend = "gl";
+                break;
+            case EGL_PLATFORM_ANGLE_TYPE_VULKAN_ANGLE:
+                mBackend = "vulkan";
+                break;
+            default:
+                break;
+        }
+    }
+
+    void run()
+    {
+        // Only supported on Linux and Windows at the moment.
+        if (!IsLinux() && !IsWindows())
+        {
+            return;
+        }
+
+        const char *benchmark = kBenchmarks[std::get<1>(GetParam())].glmark2Config;
+        bool completeRun      = benchmark == nullptr || benchmark[0] == '\0';
+
+        Optional<std::string> cwd = GetCWD();
+
+        // Set the current working directory to the executable's, as the data path of glmark2 is
+        // set relative to that path.
+        std::string executableDir = GetExecutableDirectory();
+        SetCWD(executableDir.c_str());
+        SetEnvironmentVar("ANGLE_DEFAULT_PLATFORM", mBackend);
+
+        std::vector<const char *> args = {
+            "glmark2_angle",
+        };
+        if (OneFrame())
+        {
+            args.push_back("--validate");
+        }
+        if (!completeRun)
+        {
+            args.push_back("--benchmark");
+            args.push_back(benchmark);
+            fprintf(stderr, "Running benchmark: %s\n", benchmark);
+        }
+        args.push_back(nullptr);
+
+        std::string output;
+        int exitCode;
+
+        bool success = RunApp(args, &output, nullptr, &exitCode);
+
+        // Restore the current working directory for the next tests.
+        if (cwd.valid())
+        {
+            SetCWD(cwd.value().c_str());
+        }
+
+        ASSERT_TRUE(success);
+        ASSERT_EQ(EXIT_SUCCESS, exitCode);
+
+        if (!OneFrame())
+        {
+            parseOutput(output, completeRun);
+        }
+    }
+
+  private:
+    void parseOutput(const std::string &output, bool completeRun)
+    {
+        // Output is in the following format:
+        //
+        // =======================================================
+        //     glmark2 2017.07
+        // =======================================================
+        //     OpenGL Information
+        //     GL_VENDOR:     ...
+        //     GL_RENDERER:   ...
+        //     GL_VERSION:    ...
+        // =======================================================
+        // [test] config: FPS: uint FrameTime: float ms
+        // [test] config: Not Supported
+        // ...
+        // =======================================================
+        //                                   glmark2 Score: uint
+        // =======================================================
+        //
+        // This function skips the header, prints one line for each test/config line where there's
+        // an FPS value, and finally prints the overall score.
+        std::istringstream glmark2Output(output);
+        std::string line;
+
+        // Expect ==== at the top of the header
+        std::getline(glmark2Output, line);
+        ASSERT_EQ('=', line[0]);
+
+        // Skip one line
+        std::getline(glmark2Output, line);
+
+        // Expect ==== in the middle of the header
+        std::getline(glmark2Output, line);
+        ASSERT_EQ('=', line[0]);
+
+        // Skip four lines
+        std::getline(glmark2Output, line);
+        std::getline(glmark2Output, line);
+        std::getline(glmark2Output, line);
+        std::getline(glmark2Output, line);
+
+        // The fourth line is the GL_VERSION.  Expect it to include ANGLE, otherwise we are not
+        // running against ANGLE.
+        ASSERT_NE(line.find("ANGLE"), std::string::npos);
+
+        // Expect ==== at the bottom of the header
+        std::getline(glmark2Output, line);
+        ASSERT_EQ('=', line[0]);
+
+        // Read configs until the top of the footer is reached
+        while (std::getline(glmark2Output, line) && line[0] != '=')
+        {
+            // Parse the line
+            std::istringstream lin(line);
+
+            std::string testName, testConfig;
+            lin >> testName >> testConfig;
+            EXPECT_TRUE(lin);
+
+            std::string fpsTag, frametimeTag;
+            size_t fps;
+            float frametime;
+
+            lin >> fpsTag >> fps >> frametimeTag >> frametime;
+
+            // If the line is not in `FPS: uint FrameTime: Float ms` format, the test is not
+            // supported.  It will be skipped.
+            if (!lin)
+            {
+                continue;
+            }
+
+            EXPECT_EQ("FPS:", fpsTag);
+            EXPECT_EQ("FrameTime:", frametimeTag);
+
+            perf_test::PrintResult(testName + testConfig, mBackend, "fps", fps, "", true);
+        }
+
+        // Get the score line: `glmark2 Score: uint`
+        std::string glmark2Tag, scoreTag;
+        size_t score;
+        glmark2Output >> glmark2Tag >> scoreTag >> score;
+        EXPECT_TRUE(glmark2Output);
+        EXPECT_EQ("glmark2", glmark2Tag);
+        EXPECT_EQ("Score:", scoreTag);
+
+        if (!completeRun)
+        {
+            perf_test::PrintResult("glmark2", mBackend, "score", score, "", true);
+        }
+    }
+
+    const char *mBackend = "invalid";
+};
+
+TEST_P(GLMark2Benchmark, Run)
+{
+    run();
+}
+
+ANGLE_INSTANTIATE_TEST_COMBINE_1(GLMark2Benchmark,
+                                 GLMark2BenchmarkPrint,
+                                 testing::Range(static_cast<size_t>(0), ArraySize(kBenchmarks)),
+                                 ES2_D3D11(),
+                                 ES2_OPENGLES(),
+                                 ES2_VULKAN());
+
+}  // namespace

src/tests/test_utils/angle_test_instantiate.h

@@ -87,6 +87,10 @@ struct CombinedPrintToStringParamName
 // Instantiate the test for a combination of N parameters and the enumeration of platforms in the
 // extra args, similar to ANGLE_INSTANTIATE_TEST.  The macros are defined only for the Ns currently
 // in use, and can be expanded as necessary.
+#define ANGLE_INSTANTIATE_TEST_COMBINE_1(testName, print, combine1, first, ...) \
+    const decltype(first) testName##params[] = {first, ##__VA_ARGS__};          \
+    INSTANTIATE_TEST_SUITE_P(                                                   \
+        , testName, testing::Combine(ANGLE_INSTANTIATE_TEST_PLATFORMS(testName), combine1), print)
 #define ANGLE_INSTANTIATE_TEST_COMBINE_5(testName, print, combine1, combine2, combine3, combine4, \
                                          combine5, first, ...)                                    \
     const decltype(first) testName##params[] = {first, ##__VA_ARGS__};                            \

third_party/glmark2/BUILD.gn

@@ -109,6 +109,9 @@ source_set("glmark2_headers") {
     "src/src/texture.h",
   ]
 
+  if (is_linux) {
+    sources += [ "src/src/gl-state-glx.h" ]
+  }
   if (is_win) {
     sources += [
       "src/src/gl-state-wgl.h",
@@ -377,6 +380,26 @@ glmark2_exe("glmark2_angle") {
     "EGL_EGL_PROTOTYPES=1",
     "GLMARK2_USE_EGL",
   ]
+
+  if (is_linux && !is_component_build) {
+    # Set rpath to find shared libs in a non-component build.
+    public_configs = [ "//build/config/gcc:rpath_for_built_shared_libraries" ]
+  }
+}
+
+if (is_linux) {
+  glmark2_exe("glmark2_glx") {
+    sources = [
+      "src/src/gl-state-glx.cpp",
+      "src/src/glad/include/glad/glx.h",
+      "src/src/glad/src/glx.c",
+    ]
+    deps = [
+      ":glmark2_common_gl",
+    ]
+
+    defines = [ "GLMARK2_USE_GLX" ]
+  }
 }
 
 if (is_win) {