Add new test runner harness.

src/tests/BUILD.gn

@@ -72,15 +72,18 @@ template("angle_tests_main") {
 
     main = invoker.main
 
-    if (build_with_chromium) {
-      sources = [
-        "//gpu/$main.cc",
-      ]
-    } else {
-      sources = [
-        "$main.cpp",
-      ]
-    }
+    sources = [
+      "$main.cpp",
+      "test_utils/runner/TestSuite.cpp",
+      "test_utils/runner/TestSuite.h",
+    ]
+    deps += [ "$angle_root/third_party/rapidjson:rapidjson" ]
+    data = [
+      "//testing/scripts/common.py",
+      "//testing/test_env.py",
+      "//testing/xvfb.py",
+      "test_utils/runner/run_gtest_angle_test.py",
+    ]
   }
 }
 

src/tests/angle_end2end_tests_main.cpp

@@ -5,12 +5,13 @@
 //
 
 #include "gtest/gtest.h"
+#include "test_utils/runner/TestSuite.h"
 
 void ANGLEProcessTestArgs(int *argc, char *argv[]);
 
 int main(int argc, char **argv)
 {
-    testing::InitGoogleTest(&argc, argv);
+    tr::TestSuite testSuite(&argc, argv);
     ANGLEProcessTestArgs(&argc, argv);
     int rt = RUN_ALL_TESTS();
     return rt;

src/tests/angle_unittest_main.cpp

@@ -6,6 +6,7 @@
 
 #include "GLSLANG/ShaderLang.h"
 #include "gtest/gtest.h"
+#include "test_utils/runner/TestSuite.h"
 
 class CompilerTestEnvironment : public testing::Environment
 {
@@ -29,7 +30,7 @@ class CompilerTestEnvironment : public testing::Environment
 
 int main(int argc, char **argv)
 {
-    testing::InitGoogleTest(&argc, argv);
+    tr::TestSuite testSuite(&argc, argv);
     testing::AddGlobalTestEnvironment(new CompilerTestEnvironment());
     int rt = RUN_ALL_TESTS();
     return rt;

src/tests/test_utils/runner/TestSuite.cpp

+//
+// Copyright 2019 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// TestSuite:
+//   Basic implementation of a test harness in ANGLE.
+
+#include "TestSuite.h"
+
+#include <time.h>
+
+#include <gtest/gtest.h>
+#include <rapidjson/document.h>
+#include <rapidjson/filewritestream.h>
+#include <rapidjson/prettywriter.h>
+
+// We directly call into a function to register the parameterized tests. This saves spinning up
+// a subprocess with a new gtest filter.
+#include "third_party/googletest/src/googletest/src/gtest-internal-inl.h"
+
+namespace js = rapidjson;
+
+namespace tr
+{
+
+namespace
+{
+// FIXME: non-Windows paths
+constexpr char kPathSeparator = '\\';
+// FIXME: non-Windows environment vars
+
+const char *ParseFlagValue(const char *flag, const char *argument)
+{
+    if (strstr(argument, flag) == argument)
+    {
+        return argument + strlen(flag);
+    }
+
+    return nullptr;
+}
+
+bool ParseIntFlag(const char *flag, const char *argument, int *valueOut)
+{
+    const char *value = ParseFlagValue(flag, argument);
+    if (!value)
+    {
+        return false;
+    }
+
+    char *end            = nullptr;
+    const long longValue = strtol(value, &end, 10);
+
+    if (*end != '\0')
+    {
+        printf("Error parsing integer flag value.\n");
+        exit(1);
+    }
+
+    if (longValue == LONG_MAX || longValue == LONG_MIN || static_cast<int>(longValue) != longValue)
+    {
+        printf("OVerflow when parsing integer flag value.\n");
+        exit(1);
+    }
+
+    *valueOut = static_cast<int>(longValue);
+    return true;
+}
+
+bool ParseStringFlag(const char *flag, const char *argument, const char **valueOut)
+{
+    const char *value = ParseFlagValue(flag, argument);
+    if (!value)
+    {
+        return false;
+    }
+
+    *valueOut = value;
+    return true;
+}
+
+void DeleteArg(int *argc, char **argv, int argIndex)
+{
+    // Shift the remainder of the argv list left by one.  Note that argv has (*argc + 1) elements,
+    // the last one always being NULL.  The following loop moves the trailing NULL element as well.
+    for (int index = argIndex; index < *argc; ++index)
+    {
+        argv[index] = argv[index + 1];
+    }
+    (*argc)--;
+}
+
+void AddArg(int *argc, char **argv, const char *arg)
+{
+    // This unsafe const_cast is necessary to work around gtest limitations.
+    argv[*argc]     = const_cast<char *>(arg);
+    argv[*argc + 1] = nullptr;
+    (*argc)++;
+}
+}  // namespace
+
+class TestSuite::TestEventListener : public testing::EmptyTestEventListener
+{
+  public:
+    TestEventListener(const char *outputDirectory, const char *testSuiteName)
+        : mOutputDirectory(outputDirectory), mTestSuiteName(testSuiteName)
+    {}
+
+    void OnTestProgramEnd(const testing::UnitTest &testProgramInfo) override
+    {
+        time_t ltime;
+        time(&ltime);
+        struct tm *timeinfo = gmtime(&ltime);
+        ltime               = mktime(timeinfo);
+
+        js::Document doc;
+        doc.SetObject();
+
+        js::Document::AllocatorType &allocator = doc.GetAllocator();
+
+        int passed = testProgramInfo.successful_test_count();
+        int failed = testProgramInfo.failed_test_count();
+
+        js::Value numFailuresByType;
+        numFailuresByType.SetObject();
+        numFailuresByType.AddMember("PASS", passed, allocator);
+        numFailuresByType.AddMember("FAIL", failed, allocator);
+
+        doc.AddMember("interrupted", false, allocator);
+        doc.AddMember("path_delimiter", ".", allocator);
+        doc.AddMember("version", 3, allocator);
+        doc.AddMember("seconds_since_epoch", ltime, allocator);
+        doc.AddMember("num_failures_by_type", numFailuresByType, allocator);
+
+        js::Value testSuite;
+        testSuite.SetObject();
+
+        for (int i = 0; i < testProgramInfo.total_test_case_count(); ++i)
+        {
+            const testing::TestCase *testCase = testProgramInfo.GetTestCase(i);
+            for (int j = 0; j < testCase->total_test_count(); ++j)
+            {
+                const testing::TestInfo *testInfo = testCase->GetTestInfo(j);
+                const testing::TestResult *result = testInfo->result();
+
+                // Avoid recording info for tests that are not part of the shard.
+                if (!testInfo->should_run())
+                    continue;
+
+                js::Value jsResult;
+                jsResult.SetObject();
+                if (result->Passed())
+                {
+                    jsResult.AddMember("actual", "PASS", allocator);
+                }
+                else if (result->Failed())
+                {
+                    jsResult.AddMember("actual", "FAIL", allocator);
+                }
+
+                jsResult.AddMember("expected", "PASS", allocator);
+
+                double timeInSeconds = static_cast<double>(result->elapsed_time()) / 1000.0;
+
+                js::Value times;
+                times.SetArray();
+                times.PushBack(timeInSeconds, allocator);
+
+                jsResult.AddMember("times", times, allocator);
+
+                char testName[500];
+                sprintf(testName, "%s.%s", testInfo->test_case_name(), testInfo->name());
+                js::Value jsName;
+                jsName.SetString(testName, allocator);
+
+                testSuite.AddMember(jsName, jsResult, allocator);
+            }
+        }
+
+        js::Value tests;
+        tests.SetObject();
+        tests.AddMember(js::StringRef(mTestSuiteName), testSuite, allocator);
+
+        doc.AddMember("tests", tests, allocator);
+
+        char buf[500];
+        sprintf(buf, "%s%c%s", mOutputDirectory, kPathSeparator, "output.json");
+
+        printf("opening %s\n", buf);
+
+        FILE *fp = fopen(buf, "w");
+
+        constexpr size_t kBufferSize = 0xFFFF;
+        std::vector<char> writeBuffer(kBufferSize);
+        js::FileWriteStream os(fp, writeBuffer.data(), kBufferSize);
+        js::PrettyWriter<js::FileWriteStream> writer(os);
+        doc.Accept(writer);
+
+        fclose(fp);
+    }
+
+  private:
+    const char *mOutputDirectory;
+    const char *mTestSuiteName;
+};
+
+struct TestIdentifier
+{
+    TestIdentifier() {}
+    TestIdentifier(const TestIdentifier &other) = default;
+
+    const char *testCaseName;
+    const char *testName;
+    const char *file;
+    int line = 0;
+};
+
+std::vector<TestIdentifier> GetCompiledInTests()
+{
+    testing::UnitTest *const unitTest = testing::UnitTest::GetInstance();
+
+    std::vector<TestIdentifier> tests;
+    for (int i = 0; i < unitTest->total_test_case_count(); ++i)
+    {
+        const testing::TestCase *testCase = unitTest->GetTestCase(i);
+        for (int j = 0; j < testCase->total_test_count(); ++j)
+        {
+            const testing::TestInfo *testInfo = testCase->GetTestInfo(j);
+            TestIdentifier testData;
+            testData.testCaseName = testCase->name();
+            testData.testName     = testInfo->name();
+            testData.file         = testInfo->file();
+            testData.line         = testInfo->line();
+            tests.push_back(testData);
+        }
+    }
+    return tests;
+}
+
+std::string GetTestFilterForShard(const std::vector<TestIdentifier> &tests,
+                                  int shardIndex,
+                                  int shardCount)
+{
+    std::stringstream filterStream;
+    int testIndex = shardIndex;
+
+    filterStream << "--gtest_filter=";
+
+    while (testIndex < static_cast<int>(tests.size()))
+    {
+        filterStream << tests[testIndex].testCaseName;
+        filterStream << ".";
+        filterStream << tests[testIndex].testName;
+        testIndex += shardCount;
+        if (testIndex < static_cast<int>(tests.size()))
+        {
+            filterStream << ":";
+        }
+    }
+
+    return filterStream.str();
+}
+
+TestSuite::TestSuite(int *argc, char **argv)
+    : mResultsDirectory(nullptr), mShardCount(-1), mShardIndex(-1)
+{
+    bool hasFilter = false;
+
+    if (*argc <= 0)
+    {
+        printf("Missing test arguments.\n");
+        exit(1);
+    }
+
+    parseTestSuiteName(argv[0]);
+
+    for (int argIndex = 1; argIndex < *argc;)
+    {
+        if (parseTestSuiteFlag(argv[argIndex]))
+        {
+            DeleteArg(argc, argv, argIndex);
+        }
+        else
+        {
+            if (ParseFlagValue("--gtest_filter=", argv[argIndex]))
+            {
+                hasFilter = true;
+            }
+            ++argIndex;
+        }
+    }
+
+    if ((mShardIndex >= 0) != (mShardCount > 1))
+    {
+        printf("Shard index and shard count must be specified together.\n");
+        exit(1);
+    }
+
+    if (mShardCount > 0)
+    {
+        if (hasFilter)
+        {
+            printf("Cannot use gtest_filter in conjunction with sharding parameters.\n");
+            exit(1);
+        }
+
+        testing::internal::UnitTestImpl *impl = testing::internal::GetUnitTestImpl();
+        impl->RegisterParameterizedTests();
+
+        std::vector<TestIdentifier> tests = GetCompiledInTests();
+        mFilterString                     = GetTestFilterForShard(tests, mShardIndex, mShardCount);
+
+        // Note that we only add a filter string if we previously deleted a shader index/count
+        // argument. So we will have space for the new filter string in argv.
+        AddArg(argc, argv, mFilterString.c_str());
+    }
+
+    if (mResultsDirectory)
+    {
+        testing::TestEventListeners &listeners = testing::UnitTest::GetInstance()->listeners();
+        listeners.Append(new TestEventListener(mResultsDirectory, mTestSuiteName.c_str()));
+    }
+
+    testing::InitGoogleTest(argc, argv);
+}
+
+TestSuite::~TestSuite() = default;
+
+void TestSuite::parseTestSuiteName(const char *executable)
+{
+    const char *baseNameStart = strrchr(executable, kPathSeparator);
+    if (!baseNameStart)
+    {
+        baseNameStart = executable;
+    }
+    else
+    {
+        baseNameStart++;
+    }
+
+    const char kSuffix[]       = ".exe";
+    const char *baseNameSuffix = strstr(baseNameStart, kSuffix);
+    if (baseNameSuffix == (baseNameStart + strlen(baseNameStart) - strlen(kSuffix)))
+    {
+        mTestSuiteName.insert(mTestSuiteName.begin(), baseNameStart, baseNameSuffix);
+    }
+    else
+    {
+        mTestSuiteName = baseNameStart;
+    }
+}
+
+bool TestSuite::parseTestSuiteFlag(const char *argument)
+{
+    return (ParseIntFlag("--shard-count=", argument, &mShardCount) ||
+            ParseIntFlag("--shard-index=", argument, &mShardIndex) ||
+            ParseStringFlag("--results-directory=", argument, &mResultsDirectory));
+}
+}  // namespace tr

src/tests/test_utils/runner/TestSuite.h

+//
+// Copyright 2019 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// TestSuite:
+//   Basic implementation of a test harness in ANGLE.
+
+#include <memory>
+#include <string>
+
+namespace tr
+{
+
+class TestSuite
+{
+  public:
+    TestSuite(int *argc, char **argv);
+    ~TestSuite();
+
+  private:
+    void parseTestSuiteName(const char *executable);
+    bool parseTestSuiteFlag(const char *argument);
+
+    class TestEventListener;
+
+    std::string mTestSuiteName;
+    std::string mFilterString;
+    const char *mResultsDirectory;
+    int mShardCount;
+    int mShardIndex;
+};
+
+}  // namespace tr

src/tests/test_utils/runner/run_gtest_angle_test.py

+#!/usr/bin/env python
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Runs an isolated non-Telemetry ANGLE test .
+
+The main contract is that the caller passes the arguments:
+
+  --isolated-script-test-output=[FILENAME]
+json is written to that file in the format produced by
+common.parse_common_test_results.
+
+Optional argument:
+
+  --isolated-script-test-filter=[TEST_NAMES]
+
+is a double-colon-separated ("::") list of test names, to run just that subset
+of tests. This list is parsed by this harness and sent down via the
+--gtest_filter argument.
+
+This script is intended to be the base command invoked by the isolate,
+followed by a subsequent non-python executable.  It is modeled after
+run_gtest_perf_test.py
+"""
+
+import argparse
+import json
+import os
+import shutil
+import sys
+import tempfile
+import traceback
+
+def GetSrcDir():
+  dirs = [
+    os.path.join(os.path.abspath(__file__), '..', '..', '..', '..', '..'),
+    os.path.join(os.path.abspath(__file__), '..', '..', '..', '..', '..', '..', '..'),
+  ]
+
+  for dir in dirs:
+    if os.path.isdir(os.path.join(dir, 'testing')):
+      return dir
+  raise Exception('failed to find testing directory')
+
+def GetScriptsDir():
+  return os.path.join(GetSrcDir(), 'testing', 'scripts')
+
+# Add src/testing/ into sys.path for importing xvfb and test_env.
+sys.path.append(GetScriptsDir())
+
+import common
+
+
+import xvfb
+import test_env
+
+# Unfortunately we need to copy these variables from ../test_env.py.
+# Importing it and using its get_sandbox_env breaks test runs on Linux
+# (it seems to unset DISPLAY).
+CHROME_SANDBOX_ENV = 'CHROME_DEVEL_SANDBOX'
+CHROME_SANDBOX_PATH = '/opt/chromium/chrome_sandbox'
+
+
+def IsWindows():
+  return sys.platform == 'cygwin' or sys.platform.startswith('win')
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('executable', help='Test executable.')
+  parser.add_argument(
+      '--isolated-script-test-output', type=str,
+      required=True)
+  parser.add_argument(
+      '--isolated-script-test-filter', type=str, required=False)
+  parser.add_argument('--xvfb', help='Start xvfb.', action='store_true')
+
+  args, extra_flags = parser.parse_known_args()
+
+  env = os.environ.copy()
+
+  total_shards = None
+  shard_index = None
+  if 'GTEST_TOTAL_SHARDS' in env:
+    extra_flags += ['--shard-count=%d' % env['GTEST_TOTAL_SHARDS']]
+  if 'GTEST_SHARD_INDEX' in env:
+    extra_flags += ['--shard-index=%d' % env['GTEST_SHARD_INDEX']]
+
+  # Assume we want to set up the sandbox environment variables all the
+  # time; doing so is harmless on non-Linux platforms and is needed
+  # all the time on Linux.
+  env[CHROME_SANDBOX_ENV] = CHROME_SANDBOX_PATH
+
+  rc = 0
+  try:
+    # Consider adding stdio control flags.
+    if args.isolated_script_test_output:
+      extra_flags.append('--results-directory=%s' %
+        os.path.dirname(args.isolated_script_test_output))
+
+    if args.isolated_script_test_filter:
+      filter_list = common.extract_filter_list(
+        args.isolated_script_test_filter)
+      extra_flags.append('--gtest_filter=' + ':'.join(filter_list))
+
+    if IsWindows():
+      args.executable = '.\\%s.exe' % args.executable
+    else:
+      args.executable = './%s' % args.executable
+    with common.temporary_file() as tempfile_path:
+      env['CHROME_HEADLESS'] = '1'
+      cmd = [args.executable] + extra_flags
+
+      if args.xvfb:
+        rc = xvfb.run_executable(cmd, env, stdoutfile=tempfile_path)
+      else:
+        rc = test_env.run_command_with_output(cmd, env=env,
+                                              stdoutfile=tempfile_path)
+
+  except Exception:
+    traceback.print_exc()
+    rc = 1
+
+  valid = (rc == 0)
+  failures = [] if valid else ['(entire test suite)']
+  output_json = {
+      'valid': valid,
+      'failures': failures,
+    }
+
+  return rc
+
+
+# This is not really a "script test" so does not need to manually add
+# any additional compile targets.
+def main_compile_targets(args):
+  json.dump([], args.output)
+
+
+if __name__ == '__main__':
+  # Conform minimally to the protocol defined by ScriptTest.
+  if 'compile_targets' in sys.argv:
+    funcs = {
+      'run': None,
+      'compile_targets': main_compile_targets,
+    }
+    sys.exit(common.run_script(sys.argv[1:], funcs))
+  sys.exit(main())
+