Automate testing of capture_replay

samples/capture_replay/CaptureReplay.cpp

@@ -11,11 +11,6 @@
 
 #include "util/frame_capture_utils.h"
 
-#define ANGLE_MACRO_STRINGIZE_AUX(a) #a
-#define ANGLE_MACRO_STRINGIZE(a) ANGLE_MACRO_STRINGIZE_AUX(a)
-#define ANGLE_MACRO_CONCAT_AUX(a, b) a##b
-#define ANGLE_MACRO_CONCAT(a, b) ANGLE_MACRO_CONCAT_AUX(a, b)
-
 // Build the right context header based on replay ID
 // This will expand to "angle_capture_context<#>.h"
 #include ANGLE_MACRO_STRINGIZE(ANGLE_CAPTURE_REPLAY_SAMPLE_HEADER)

src/tests/BUILD.gn

@@ -1343,6 +1343,7 @@ group("angle_tests") {
     ":angle_end2end_tests",
     ":angle_perftests",
     ":angle_unittests",
+    "capture_replay_tests",
   ]
   if (!is_fuchsia) {
     deps += [

src/tests/capture_replay_tests.py

+#
+# Copyright 2020 The ANGLE Project Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+"""
+Script testing capture_replay with angle_end2end_tests
+"""
+
+# Automation script will:
+# 1. Build all tests in angle_end2end with frame capture enabled
+# 2. Run each test with frame capture
+# 3. Build CaptureReplayTest with cpp trace files
+# 4. Run CaptureReplayTest
+# 5. Output the number of test successes and failures. A test succeeds if no error occurs during
+# its capture and replay, and the GL states at the end of two runs match. Any unexpected failure
+# will return non-zero exit code
+
+# Run this script with Python to test capture replay on angle_end2end tests
+# python path/to/capture_replay_tests.py
+# Command line arguments:
+# --build_dir: specifies build directory relative to angle folder.
+# Default is out/CaptureReplayTestsDebug
+# --verbose: off by default
+# --use_goma: uses goma for compiling and linking test. Off by default
+# --gtest_filter: same as gtest_filter of Google's test framework. Default is */ES2_Vulkan
+# --test_suite: test suite to execute on. Default is angle_end2end_tests
+
+import argparse
+import distutils.util
+import os
+import shutil
+import subprocess
+
+from sys import platform
+
+DEFAULT_BUILD_DIR = "out/CaptureReplayTestsDebug"  # relative to angle folder
+DEFAULT_FILTER = "*/ES2_Vulkan"
+DEFAULT_TEST_SUITE = "angle_end2end_tests"
+
+
+class Logger():
+    verbose = False
+
+    @staticmethod
+    def log(msg):
+        if Logger.verbose:
+            print(msg)
+
+
+def RunGnGen(build_dir, arguments, is_log_showed=False):
+    command = 'gn gen --args="'
+    is_first_argument = True
+    for argument in arguments:
+        if is_first_argument:
+            is_first_argument = False
+        else:
+            command += ' '
+        command += argument[0]
+        command += '='
+        command += argument[1]
+    command += '" '
+    command += build_dir
+    if is_log_showed:
+        subprocess.call(command, shell=True)
+    else:
+        subprocess.check_output(command, shell=True)
+
+
+def RunAutoninja(build_dir, target, is_log_showed=False):
+    command = "autoninja "
+    command += target
+    command += " -C "
+    command += build_dir
+    if is_log_showed:
+        subprocess.call(command, shell=True)
+    else:
+        subprocess.check_output(command, shell=True)
+
+
+# return a list of tests and their params in the form
+# [(test1, params1), (test2, params2),...]
+def GetTestNamesAndParams(test_exec_path, filter="*"):
+    output = subprocess.check_output(
+        '"' + test_exec_path + '" --gtest_list_tests --gtest_filter=' + filter,
+        shell=True,
+        stderr=subprocess.PIPE).splitlines()
+
+    tests = []
+    last_testcase_name = ""
+    test_name_splitter = "# GetParam() ="
+    for line in output:
+        if test_name_splitter in line:
+            # must be a test name line
+            test_name_and_params = line.split(test_name_splitter)
+            tests.append((last_testcase_name + test_name_and_params[0].strip(), \
+                test_name_and_params[1].strip()))
+        else:
+            # gtest_list_tests returns the test in this format
+            # test case
+            #    test name1
+            #    test name2
+            # Need to remember the last test case name to append to the test name
+            last_testcase_name = line
+    return tests
+
+
+class Test():
+
+    def __init__(self, full_test_name, params, use_goma):
+        self.full_test_name = full_test_name
+        self.params = params
+        self.use_goma = use_goma
+
+    def __str__(self):
+        return self.full_test_name + " Params: " + self.params
+
+    def Run(self, test_exe_path):
+        try:
+            output = subprocess.check_output(
+                '"' + test_exe_path + '" --gtest_filter=' + self.full_test_name + \
+                ' --use-angle=vulkan',
+                shell=True,
+                stderr=subprocess.PIPE)
+            Logger.log("Ran " + self.full_test_name + " with capture")
+            return (0, output)
+        except subprocess.CalledProcessError as e:
+            return (e.returncode, e.output)
+
+    def BuildReplay(self, build_dir, replay_exec):
+        RunGnGen(build_dir, [("use_goma", self.use_goma),
+                             ("angle_with_capture_by_default", "true"),
+                             ("angle_build_capture_replay_tests", "true")])
+        RunAutoninja(build_dir, replay_exec)
+        Logger.log("Built replay of " + self.full_test_name)
+
+    def RunReplay(self, build_dir, replay_exec):
+        try:
+            output = subprocess.check_output(
+                '"' + build_dir + '/' + replay_exec + '" --use-angle=vulkan',
+                shell=True,
+                stderr=subprocess.PIPE)
+            Logger.log("Ran replay of " + self.full_test_name)
+            return (0, output)
+        except subprocess.CalledProcessError as e:
+            return (e.returncode, e.output)
+
+
+def ClearFolderContent(path):
+    all_files = []
+    for f in os.listdir(path):
+        if os.path.isfile(path + "/" + f) and f.startswith("angle_capture_context"):
+            os.remove(path + "/" + f)
+
+
+def CanRunReplay(path):
+    files = [
+        "angle_capture_context1.h", "angle_capture_context1.cpp",
+        "angle_capture_context1_files.txt", "angle_capture_context1_frame000.cpp"
+    ]
+    for file in files:
+        if not os.path.isfile(path + "/" + file):
+            return False
+    return True
+
+
+def SetCWDToAngleFolder():
+    angle_folder = "angle"
+    cwd = os.path.dirname(os.path.abspath(__file__))
+    cwd = cwd.split(angle_folder)[0] + angle_folder
+    os.chdir(cwd)
+    return cwd
+
+
+def main(build_dir, verbose, use_goma, gtest_filter, test_exec):
+    Logger.verbose = verbose
+    cwd = SetCWDToAngleFolder()
+    capture_out_dir = "src/tests/capture_replay_tests/traces"  # relative to ANGLE folder
+    if not os.path.isdir(capture_out_dir):
+        os.mkdir(capture_out_dir)
+    environment_vars = [("ANGLE_CAPTURE_FRAME_END", "0"),
+                        ("ANGLE_CAPTURE_OUT_DIR", capture_out_dir)]
+    replay_exec = "capture_replay_tests"
+    if platform == "win32":
+        test_exec += ".exe"
+        replay_exec += ".exe"
+    # generate gn files
+    RunGnGen(build_dir, [("use_goma", use_goma), ("angle_with_capture_by_default", "true")], True)
+    # build angle_end2end
+    RunAutoninja(build_dir, test_exec, True)
+    # get a list of tests
+    test_names_and_params = GetTestNamesAndParams(build_dir + '/' + test_exec, gtest_filter)
+    all_tests = []
+    for test_name_and_params in test_names_and_params:
+        all_tests.append(Test(test_name_and_params[0], test_name_and_params[1], use_goma))
+
+    for environment_var in environment_vars:
+        os.environ[environment_var[0]] = environment_var[1]
+
+    for test in all_tests:
+        if verbose:
+            print("*" * 30)
+        ClearFolderContent(capture_out_dir)
+        os.environ["ANGLE_CAPTURE_ENABLED"] = "1"
+        run_output = test.Run(build_dir + "/" + test_exec)
+        if run_output[0] == 0 and CanRunReplay(capture_out_dir):
+            os.environ["ANGLE_CAPTURE_ENABLED"] = "0"
+            test.BuildReplay(build_dir, replay_exec)
+            replay_output = test.RunReplay(build_dir, replay_exec)
+            if replay_output[0] != 0:
+                print("Failed: " + test.full_test_name)
+                print(replay_output[1])
+            else:
+                print("Passed: " + test.full_test_name)
+        else:
+            print("Skipped: " + test.full_test_name + ". Skipping replay since capture" + \
+                "didn't produce appropriate files or has crashed")
+    for environment_var in environment_vars:
+        del os.environ[environment_var[0]]
+
+    if os.path.isdir(capture_out_dir):
+        shutil.rmtree(capture_out_dir)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--build_dir', default=DEFAULT_BUILD_DIR)
+    parser.add_argument('--verbose', default="False")
+    parser.add_argument('--use_goma', default="false")
+    parser.add_argument('--gtest_filter', default=DEFAULT_FILTER)
+    parser.add_argument('--test_suite', default=DEFAULT_TEST_SUITE)
+    args = parser.parse_args()
+    main(args.build_dir, distutils.util.strtobool(args.verbose), args.use_goma, args.gtest_filter,
+         args.test_suite)

src/tests/capture_replay_tests/.gitignore

+angle_capture_context*
\ No newline at end of file

src/tests/capture_replay_tests/BUILD.gn

+# Copyright 2020 The ANGLE Project Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("../../../gni/angle.gni")
+declare_args() {
+  # Determines if we build the capture_replay_tests. Off by default.
+  angle_build_capture_replay_tests = false
+
+  # Decide which context to replay, starting with desktop default
+  angle_capture_replay_test_context_id = 1
+}
+
+if (angle_build_capture_replay_tests) {
+  # TODO (nguyenmh): http://anglebug.com/4758:
+  # turn angle_executable into angle_test when adding android support
+  angle_executable("capture_replay_tests") {
+    testonly = true
+    _contextid = angle_capture_replay_test_context_id
+    _trace_sources =
+        read_file("traces/angle_capture_context${_contextid}_files.txt",
+                  "list lines") +
+        [
+          "angle_capture_context${_contextid}.cpp",
+          "angle_capture_context${_contextid}.h",
+        ]
+    sources =
+        rebase_path(_trace_sources, ".", "traces") + [ "CaptureReplayTest.cpp" ]
+    deps = [
+      "$angle_root:angle_compression",
+      "../../../:angle_common",
+      "../../../util:angle_util",
+      "../../../util:angle_util_loader_headers",
+    ]
+    configs += [ "../../../:library_name_config" ]
+
+    suppressed_configs += [ "$angle_root:constructor_and_destructor_warnings" ]
+
+    # Disable optimization to avoid optimizing huge files.
+    if (!is_debug) {
+      suppressed_configs += [ "//build/config/compiler:default_optimization" ]
+      configs += [ "//build/config/compiler:no_optimize" ]
+    }
+    _data_path = rebase_path("traces", root_out_dir)
+    defines = [
+      "ANGLE_CAPTURE_REPLAY_TEST_DATA_DIR=\"${_data_path}\"",
+      "ANGLE_CAPTURE_REPLAY_TEST_CONTEXT_ID=${_contextid}",
+      "ANGLE_CAPTURE_REPLAY_TEST_HEADER=traces/angle_capture_context${_contextid}.h",
+    ]
+  }
+} else {
+  group("capture_replay_tests") {
+  }
+}

src/tests/capture_replay_tests/CaptureReplayTest.cpp

+//
+// Copyright 2020 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// CaptureReplayTest.cpp:
+//   Application that runs replay for testing of capture replay
+//
+
+#include "common/system_utils.h"
+#include "util/EGLPlatformParameters.h"
+#include "util/EGLWindow.h"
+#include "util/OSWindow.h"
+#include "util/egl_loader_autogen.h"
+#include "util/gles_loader_autogen.h"
+
+#include <stdint.h>
+#include <string.h>
+#include <functional>
+#include <list>
+#include <memory>
+#include <string>
+#include <utility>
+
+#include "util/frame_capture_utils.h"
+
+// Build the right context header based on replay ID
+// This will expand to "angle_capture_context<#>.h"
+#include ANGLE_MACRO_STRINGIZE(ANGLE_CAPTURE_REPLAY_TEST_HEADER)
+
+// Assign the context numbered functions based on GN arg selecting replay ID
+std::function<void()> SetupContextReplay = reinterpret_cast<void (*)()>(
+    ANGLE_MACRO_CONCAT(SetupContext,
+                       ANGLE_MACRO_CONCAT(ANGLE_CAPTURE_REPLAY_TEST_CONTEXT_ID, Replay)));
+std::function<void(int)> ReplayContextFrame = reinterpret_cast<void (*)(int)>(
+    ANGLE_MACRO_CONCAT(ReplayContext,
+                       ANGLE_MACRO_CONCAT(ANGLE_CAPTURE_REPLAY_TEST_CONTEXT_ID, Frame)));
+std::function<void()> ResetContextReplay = reinterpret_cast<void (*)()>(
+    ANGLE_MACRO_CONCAT(ResetContext,
+                       ANGLE_MACRO_CONCAT(ANGLE_CAPTURE_REPLAY_TEST_CONTEXT_ID, Replay)));
+
+class CaptureReplayTest
+{
+  public:
+    CaptureReplayTest(int width, int height, EGLint glesMajorVersion, EGLint glesMinorVersion)
+        : mWidth(width), mHeight(height), mOSWindow(nullptr), mEGLWindow(nullptr)
+
+    {
+        mPlatformParams.renderer   = EGL_PLATFORM_ANGLE_TYPE_VULKAN_ANGLE;
+        mPlatformParams.deviceType = EGL_PLATFORM_ANGLE_DEVICE_TYPE_HARDWARE_ANGLE;
+
+        // Load EGL library so we can initialize the display.
+        mEntryPointsLib.reset(
+            angle::OpenSharedLibrary(ANGLE_EGL_LIBRARY_NAME, angle::SearchType::ApplicationDir));
+
+        mEGLWindow = EGLWindow::New(glesMajorVersion, glesMinorVersion);
+        mOSWindow  = OSWindow::New();
+    }
+
+    ~CaptureReplayTest()
+    {
+        EGLWindow::Delete(&mEGLWindow);
+        OSWindow::Delete(&mOSWindow);
+    }
+
+    bool initialize()
+    {
+        // Set CWD to executable directory.
+        std::string exeDir = angle::GetExecutableDirectory();
+        if (!angle::SetCWD(exeDir.c_str()))
+            return false;
+        if (kIsBinaryDataCompressed)
+        {
+            SetBinaryDataDecompressCallback(angle::DecompressBinaryData);
+        }
+        SetBinaryDataDir(ANGLE_CAPTURE_REPLAY_TEST_DATA_DIR);
+        SetupContextReplay();
+
+        return true;
+    }
+
+    void draw() { ReplayContextFrame(0); }
+
+    void swap() { mEGLWindow->swap(); }
+
+    int run()
+    {
+        if (!mOSWindow->initialize("Capture Replay Test", mWidth, mHeight))
+        {
+            return -1;
+        }
+
+        mOSWindow->setVisible(true);
+
+        ConfigParameters configParams;
+        configParams.redBits     = 8;
+        configParams.greenBits   = 8;
+        configParams.blueBits    = 8;
+        configParams.alphaBits   = 8;
+        configParams.depthBits   = 24;
+        configParams.stencilBits = 8;
+
+        if (!mEGLWindow->initializeGL(mOSWindow, mEntryPointsLib.get(),
+                                      angle::GLESDriverType::AngleEGL, mPlatformParams,
+                                      configParams))
+        {
+            return -1;
+        }
+
+        // Disable vsync
+        if (!mEGLWindow->setSwapInterval(0))
+        {
+            return -1;
+        }
+
+        angle::LoadGLES(eglGetProcAddress);
+
+        int result = 0;
+
+        if (!initialize())
+        {
+            result = -1;
+        }
+
+        draw();
+        swap();
+
+        mEGLWindow->destroyGL();
+        mOSWindow->destroy();
+
+        return result;
+    }
+
+  private:
+    uint32_t mWidth;
+    uint32_t mHeight;
+    OSWindow *mOSWindow;
+    EGLWindow *mEGLWindow;
+
+    EGLPlatformParameters mPlatformParams;
+
+    // Handle to the entry point binding library.
+    std::unique_ptr<angle::Library> mEntryPointsLib;
+};
+
+int main(int argc, char **argv)
+{
+    // TODO (nguyenmh): http://anglebug.com/4759: initialize app with arguments taken from cmdline
+    const int width               = 128;
+    const int height              = 128;
+    const EGLint glesMajorVersion = 2;
+    const GLint glesMinorVersion  = 0;
+    CaptureReplayTest app(width, height, glesMajorVersion, glesMinorVersion);
+    return app.run();
+}

util/frame_capture_utils.h

@@ -19,6 +19,11 @@
 #define USE_SYSTEM_ZLIB
 #include "compression_utils_portable.h"
 
+#define ANGLE_MACRO_STRINGIZE_AUX(a) #a
+#define ANGLE_MACRO_STRINGIZE(a) ANGLE_MACRO_STRINGIZE_AUX(a)
+#define ANGLE_MACRO_CONCAT_AUX(a, b) a##b
+#define ANGLE_MACRO_CONCAT(a, b) ANGLE_MACRO_CONCAT_AUX(a, b)
+
 namespace angle
 {