Tests: Add a perf test using traces

DEPS

@@ -78,12 +78,6 @@ deps = {
     'condition': 'not build_with_chromium',
   },
 
-  # Closed-source ANGLE capture files and tests
-  'src/tests/internal_capture_tests': {
-    'url': '{chrome_internal_git}/angle/angle-captures.git@e2fa6fa80de07f784da7214e933c8565ebfc34da',
-    'condition': 'checkout_angle_internal',
-  },
-
   'testing': {
     'url': '{chromium_git}/chromium/src/testing@c1b508625d47f26cfc04eab47747a792a5ee765b',
     'condition': 'not build_with_chromium',
@@ -417,6 +411,20 @@ hooks = [
                 '-d', 'tools/flex-bison/windows/',
     ],
   },
+
+  # Download internal captures for perf tests
+  {
+    'name': 'restricted_traces',
+    'pattern': '\\.sha1',
+    'condition': 'checkout_angle_internal',
+    'action': [ 'download_from_google_storage',
+                '--directory',
+                '--recursive',
+                '--extract',
+                '--bucket', 'chrome-angle-capture-binaries',
+                './src/tests/perf_tests/restricted_traces',
+    ]
+  }
 ]
 
 recursedeps = [

src/tests/BUILD.gn

@@ -11,6 +11,7 @@ declare_args() {
   # Don't build dEQP by default.
   build_angle_deqp_tests = false
   build_angle_gles1_conform_tests = false
+  build_angle_trace_perf_tests = false
 }
 
 angle_test("test_utils_unittest_helper") {
@@ -261,6 +262,13 @@ if (is_win || is_linux || is_android || is_mac || is_fuchsia) {
     if (is_linux) {
       data_deps += [ "${angle_root}/third_party/glmark2:glmark2_glx" ]
     }
+
+    if (build_angle_trace_perf_tests) {
+      import("perf_tests/restricted_traces/angle_trace_perf_tests.gni")
+      sources += angle_trace_perf_sources
+      defines += angle_trace_perf_defines
+      data += angle_trace_perf_data
+    }
   }
 }
 

src/tests/perf_tests/ANGLEPerfTest.cpp

@@ -639,6 +639,11 @@ OSWindow *ANGLERenderTest::getWindow()
     return mOSWindow;
 }
 
+GLWindowBase *ANGLERenderTest::getGLWindow()
+{
+    return mGLWindow;
+}
+
 bool ANGLERenderTest::areExtensionPrerequisitesFulfilled() const
 {
     for (const char *extension : mExtensionPrerequisites)

src/tests/perf_tests/ANGLEPerfTest.h

@@ -131,6 +131,7 @@ class ANGLERenderTest : public ANGLEPerfTest
     bool popEvent(Event *event);
 
     OSWindow *getWindow();
+    GLWindowBase *getGLWindow();
 
     std::vector<TraceEvent> &getTraceEventBuffer();
 

src/tests/perf_tests/README.md

@@ -40,5 +40,6 @@ ANGLE implements a no-op driver for OpenGL, D3D11 and Vulkan. To run on these co
 * [`TextureBenchmark`](TexturesPerf.cpp): Tests Texture state change performance.
 * [`LinkProgramBenchmark`](LinkProgramPerfTest.cpp): Tests performance of `glLinkProgram`.
 * [`glmark2`](glmark2.cpp): Runs the glmark2 benchmark.
+* [`TracePerfTest`](TracePerfTest.cpp): Runs replays of restricted traces, not available publicly. To enable, read more in [`RestrictedTraceTests`](restricted_traces/RestrictedTraceTests.md)
 
 Many other tests can be found that have documentation in their classes.

src/tests/perf_tests/TracePerfTest.cpp

+//
+// Copyright 2020 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// TracePerf:
+//   Performance test for ANGLE replaying traces.
+//
+
+#include <gtest/gtest.h>
+#include "common/system_utils.h"
+#include "tests/perf_tests/ANGLEPerfTest.h"
+#include "util/egl_loader_autogen.h"
+
+#include "restricted_traces/trex_1300_1310/trex_1300_1310_capture_context1.h"
+#include "restricted_traces/trex_200_210/trex_200_210_capture_context1.h"
+#include "restricted_traces/trex_800_810/trex_800_810_capture_context1.h"
+#include "restricted_traces/trex_900_910/trex_900_910_capture_context1.h"
+
+#include <cassert>
+#include <functional>
+#include <sstream>
+
+using namespace angle;
+using namespace egl_platform;
+
+namespace
+{
+
+enum class TracePerfTestID
+{
+    TRex200,
+    TRex800,
+    TRex900,
+    TRex1300,
+};
+
+struct TracePerfParams final : public RenderTestParams
+{
+    // Common default options
+    TracePerfParams()
+    {
+        majorVersion = 2;
+        minorVersion = 0;
+        windowWidth  = 1920;
+        windowHeight = 1080;
+        trackGpuTime = true;
+
+        // Display the frame after every drawBenchmark invocation
+        iterationsPerStep = 1;
+    }
+
+    std::string story() const override
+    {
+        std::stringstream strstr;
+
+        strstr << RenderTestParams::story();
+
+        switch (testID)
+        {
+            case TracePerfTestID::TRex200:
+                strstr << "_trex_200";
+                break;
+            case TracePerfTestID::TRex800:
+                strstr << "_trex_800";
+                break;
+            case TracePerfTestID::TRex900:
+                strstr << "_trex_900";
+                break;
+            case TracePerfTestID::TRex1300:
+                strstr << "_trex_1300";
+                break;
+            default:
+                assert(0);
+                break;
+        }
+
+        return strstr.str();
+    }
+
+    TracePerfTestID testID;
+};
+
+std::ostream &operator<<(std::ostream &os, const TracePerfParams &params)
+{
+    os << params.backendAndStory().substr(1);
+    return os;
+}
+
+class TracePerfTest : public ANGLERenderTest, public ::testing::WithParamInterface<TracePerfParams>
+{
+  public:
+    TracePerfTest();
+
+    void initializeBenchmark() override;
+    void destroyBenchmark() override;
+    void drawBenchmark() override;
+
+    uint32_t mStartFrame;
+    uint32_t mEndFrame;
+    std::function<void(uint32_t)> mReplayFunc;
+};
+
+TracePerfTest::TracePerfTest()
+    : ANGLERenderTest("TracePerf", GetParam()), mStartFrame(0), mEndFrame(0)
+{}
+
+void TracePerfTest::initializeBenchmark()
+{
+    const auto &params = GetParam();
+
+    // TODO: Note the start and end frames in the trace
+    //       i.e. mStartFrame = trex_200_210::kReplayFrameStart
+    switch (params.testID)
+    {
+        // For each case, bootstrap the trace
+        case TracePerfTestID::TRex200:
+            mStartFrame = 200;
+            mEndFrame   = 210;
+            mReplayFunc = trex_200_210::ReplayContext1Frame;
+            trex_200_210::SetBinaryDataDir(ANGLE_TRACE_DATA_DIR_trex_200_210);
+            trex_200_210::SetupContext1Replay();
+            break;
+        case TracePerfTestID::TRex800:
+            mStartFrame = 800;
+            mEndFrame   = 810;
+            mReplayFunc = trex_800_810::ReplayContext1Frame;
+            trex_800_810::SetBinaryDataDir(ANGLE_TRACE_DATA_DIR_trex_800_810);
+            trex_800_810::SetupContext1Replay();
+            break;
+        case TracePerfTestID::TRex900:
+            mStartFrame = 900;
+            mEndFrame   = 910;
+            mReplayFunc = trex_900_910::ReplayContext1Frame;
+            trex_900_910::SetBinaryDataDir(ANGLE_TRACE_DATA_DIR_trex_900_910);
+            trex_900_910::SetupContext1Replay();
+            break;
+        case TracePerfTestID::TRex1300:
+            mStartFrame = 1300;
+            mEndFrame   = 1310;
+            mReplayFunc = trex_1300_1310::ReplayContext1Frame;
+            trex_1300_1310::SetBinaryDataDir(ANGLE_TRACE_DATA_DIR_trex_1300_1310);
+            trex_1300_1310::SetupContext1Replay();
+            break;
+        default:
+            assert(0);
+            break;
+    }
+
+    ASSERT_TRUE(mEndFrame > mStartFrame);
+
+    getWindow()->setVisible(true);
+}
+
+void TracePerfTest::destroyBenchmark() {}
+
+void TracePerfTest::drawBenchmark()
+{
+    startGpuTimer();
+
+    for (uint32_t frame = mStartFrame; frame < mEndFrame; ++frame)
+    {
+        mReplayFunc(frame);
+        getGLWindow()->swap();
+    }
+
+    stopGpuTimer();
+}
+
+TracePerfParams TRexReplayPerfOpenGLOrGLESParams_200_210()
+{
+    TracePerfParams params;
+    params.eglParameters = OPENGL_OR_GLES();
+    params.testID        = TracePerfTestID::TRex200;
+    return params;
+}
+
+TracePerfParams TRexReplayPerfOpenGLOrGLESParams_800_810()
+{
+    TracePerfParams params;
+    params.eglParameters = OPENGL_OR_GLES();
+    params.testID        = TracePerfTestID::TRex800;
+    return params;
+}
+
+TracePerfParams TRexReplayPerfOpenGLOrGLESParams_900_910()
+{
+    TracePerfParams params;
+    params.eglParameters = OPENGL_OR_GLES();
+    params.testID        = TracePerfTestID::TRex900;
+    return params;
+}
+
+TracePerfParams TRexReplayPerfOpenGLOrGLESParams_1300_1310()
+{
+    TracePerfParams params;
+    params.eglParameters = OPENGL_OR_GLES();
+    params.testID        = TracePerfTestID::TRex1300;
+    return params;
+}
+
+TracePerfParams TRexReplayPerfVulkanParams_200_210()
+{
+    TracePerfParams params;
+    params.eglParameters = VULKAN();
+    params.testID        = TracePerfTestID::TRex200;
+    return params;
+}
+
+TracePerfParams TRexReplayPerfVulkanParams_800_810()
+{
+    TracePerfParams params;
+    params.eglParameters = VULKAN();
+    params.testID        = TracePerfTestID::TRex800;
+    return params;
+}
+
+TracePerfParams TRexReplayPerfVulkanParams_900_910()
+{
+    TracePerfParams params;
+    params.eglParameters = VULKAN();
+    params.testID        = TracePerfTestID::TRex900;
+    return params;
+}
+
+TracePerfParams TRexReplayPerfVulkanParams_1300_1310()
+{
+    TracePerfParams params;
+    params.eglParameters = VULKAN();
+    params.testID        = TracePerfTestID::TRex1300;
+    return params;
+}
+
+TEST_P(TracePerfTest, Run)
+{
+    run();
+}
+
+ANGLE_INSTANTIATE_TEST(TracePerfTest,
+                       TRexReplayPerfOpenGLOrGLESParams_200_210(),
+                       TRexReplayPerfOpenGLOrGLESParams_800_810(),
+                       TRexReplayPerfOpenGLOrGLESParams_900_910(),
+                       TRexReplayPerfOpenGLOrGLESParams_1300_1310(),
+                       TRexReplayPerfVulkanParams_200_210(),
+                       TRexReplayPerfVulkanParams_800_810(),
+                       TRexReplayPerfVulkanParams_900_910(),
+                       TRexReplayPerfVulkanParams_1300_1310());
+
+}  // anonymous namespace

src/tests/perf_tests/restricted_traces/RestrictedTraceTests.md

+# ANGLE Restricted Traces
+
+The files in this directory are traces of real applications.  We host them
+internally because they may contain third party IP which we don't want
+to share publicly.
+
+## Setup
+
+In order to compile and run with these, you must be granted access by Google,
+then authenticate with the cloud with your @google account:
+```
+gcloud auth login
+```
+```
+download_from_google_storage --config
+<enter 0 for the project ID>
+```
+Add the following to ANGLE's .gclient file:
+```
+    "custom_vars": {
+      "checkout_angle_internal":"True"
+    },
+```
+Then use gclient to pull down binary files from a cloud storage bucket.
+```
+gclient runhooks
+```
+To build the tests, add the following GN argument:
+```
+build_angle_trace_perf_tests = true
+```
+Build the angle_perftests:
+```
+autoninja -C out/Release angle_perftests
+```
+On desktop, run them like so:
+```
+out/Release/angle_perftests --gtest_filter=TracePerfTest*
+```
+On Android, run them like so:
+```
+out/Release/bin/run_angle_perftests --gtest_filter=TracePerfTest*
+```
+
+## Updating traces
+
+The current TRex traces were captured on Windows with an Nvidia driver.
+
+Update START and END for each range.
+
+### Windows
+
+```
+cd gfxbench\out\install\vs2017-x64\lib
+python ..\..\..\..\scripts\refresh_angle_libs.py --verbose
+set TFW_PACKAGE_DIR=../../../build/vs2017-x64/testfw/tfw-dev
+set ANGLE_DEFAULT_PLATFORM=vulkan
+
+set START=200
+set END=210
+set LABEL=trex
+mkdir ..\..\..\..\..\angle\src\tests\perf_tests\restricted_traces\%LABEL%_%START%_%END%
+set ANGLE_CAPTURE_LABEL=%LABEL%_%START%_%END%
+set ANGLE_CAPTURE_FRAME_START=%START%
+set ANGLE_CAPTURE_FRAME_END=%END%
+set ANGLE_CAPTURE_OUT_DIR=..\..\..\..\..\angle\src\tests\perf_tests\restricted_traces\%LABEL%_%START%_%END%
+..\bin\testfw_app.exe -b ../../../build/vs2017-x64/testfw/tfw-dev --gfx egl -w 1920 -h 1080 -t gl_trex --ei -frame_step_time=40
+```
+
+### Linux
+
+```
+cd gfxbench/out/install/linux/lib
+python ../../../../scripts/refresh_angle_libs.py --verbose
+export PLATFORM=linux
+export TFW_PACKAGE_DIR=../../../build/linux/testfw_Release/tfw-dev
+export ANGLE_DEFAULT_PLATFORM=vulkan
+export LD_LIBRARY_PATH=.
+
+export START=200
+export END=210
+export LABEL=trex
+mkdir -p ../../../../../angle/src/tests/perf_tests/restricted_traces/${LABEL}_${START}_${END}
+export ANGLE_CAPTURE_LABEL=${LABEL}_${START}_${END}
+export ANGLE_CAPTURE_FRAME_START=$START
+export ANGLE_CAPTURE_FRAME_END=$END
+export ANGLE_CAPTURE_OUT_DIR=../../../../../angle/src/tests/perf_tests/restricted_traces/${LABEL}_${START}_${END}
+../bin/testfw_app -b $TFW_PACKAGE_DIR --gfx egl -w 512 -h 512 -t gl_trex --ei -frame_step_time=40
+```
+
+## Upload to the cloud
+
+```
+cd ~/chromium/src/third_party/angle/src/tests/perf_tests/restricted_traces
+upload_to_google_storage.py --bucket chrome-angle-capture-binaries --archive trex_200_210
+upload_to_google_storage.py --bucket chrome-angle-capture-binaries --archive trex_800_810
+upload_to_google_storage.py --bucket chrome-angle-capture-binaries --archive trex_900_910
+upload_to_google_storage.py --bucket chrome-angle-capture-binaries --archive trex_1300_1310
+```
+
+## Adding new tests
+Once you are able to capture a set of frames from an application using the steps above, you'll need to:
+
+* Add the tests to restricted_traces/angle_trace_perf_tests.gni
+* Update TracePerfTest.cpp to include the headers from the test, add it to the list of TracePerfTestID, and support it throughout the file using the namespace created in the trace.
+* Upload the traces to the cloud using the steps above.
+* Add the sha1 files created by the upload and submit them with your changes.

src/tests/perf_tests/restricted_traces/angle_trace_perf_tests.gni

+# Copyright 2020 The ANGLE Project Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This include iterates through a list of tests and rebases the files so they
+# can be consumed by tests/BUILD.gn for use with TracePerfTest.cpp.
+
+import("../../../../gni/angle.gni")
+
+_tests = [
+  "trex_200_210",
+  "trex_800_810",
+  "trex_900_910",
+  "trex_1300_1310",
+]
+
+angle_trace_perf_sources = [ "perf_tests/TracePerfTest.cpp" ]
+angle_trace_perf_data =
+    [ "${angle_root}/src/tests/perf_tests/restricted_traces" ]
+angle_trace_perf_defines = []
+
+foreach(_test, _tests) {
+  _test_dir = "${_test}"
+  _test_ctx = "${_test_dir}/${_test}_capture_context1"
+
+  # Similar to capture replay sample, use the file index for sources
+  angle_trace_perf_sources +=
+      rebase_path(read_file("${_test_ctx}_files.txt", "list lines"),
+                  ".",
+                  "perf_tests/restricted_traces/${_test_dir}") +
+      [
+        "perf_tests/restricted_traces/${_test_ctx}.cpp",
+        "perf_tests/restricted_traces/${_test_ctx}.h",
+      ]
+
+  if (is_android) {
+    _test_data_path = "/sdcard/chromium_tests_root/third_party/angle/src/tests/perf_tests/restricted_traces/${_test}"
+  } else {
+    _test_data_path = rebase_path("${_test_dir}")
+  }
+
+  angle_trace_perf_defines +=
+      [ "ANGLE_TRACE_DATA_DIR_${_test}=\"${_test_data_path}\"" ]
+}

src/tests/perf_tests/restricted_traces/trex_1300_1310.tar.gz.sha1

+f0aaa3443918791c75076cbe183c0dc01335656b
\ No newline at end of file

src/tests/perf_tests/restricted_traces/trex_200_210.tar.gz.sha1

+4bd3274e80b8b27358598d90c3510d220ea38112
\ No newline at end of file

src/tests/perf_tests/restricted_traces/trex_800_810.tar.gz.sha1

+caceb0f4be19599130a11ebd9d1de45371b0c2d8
\ No newline at end of file

src/tests/perf_tests/restricted_traces/trex_900_910.tar.gz.sha1

+f3c22df8b9364478e7fb5f8823928e7c9abf3960
\ No newline at end of file