Test Runner: Add test expectations parser.

gni/angle.gni

@@ -408,8 +408,9 @@ if (angle_standalone || build_with_chromium) {
       public_deps = invoker.public_deps + [
                       "$angle_root:angle_common",
                       "$angle_root:includes",
-                      "$angle_root/util:angle_test_utils",
+                      "$angle_root/src/tests:angle_test_expectations",
                       "$angle_root/third_party/rapidjson:rapidjson",
+                      "$angle_root/util:angle_test_utils",
                       "//testing/gmock",
                       "//testing/gtest",
                       "//third_party/googletest:gmock",

src/tests/BUILD.gn

@@ -195,6 +195,7 @@ if (is_win || is_linux || is_chromeos || is_android || is_fuchsia || is_apple) {
 
     sources = angle_end2end_tests_sources + [ "angle_end2end_tests_main.cpp" ]
     libs = []
+    data = [ "angle_end2end_tests_expectations.txt" ]
 
     if (is_mac) {
       sources += angle_end2end_tests_mac_sources
@@ -222,7 +223,6 @@ if (is_win || is_linux || is_chromeos || is_android || is_fuchsia || is_apple) {
 
     deps = [
       ":angle_common_test_utils_shared",
-      ":angle_test_expectations",
       "$angle_root:angle_image_util",
     ]
 
@@ -275,7 +275,7 @@ if (is_win || is_linux || is_chromeos || is_android || is_fuchsia || is_apple) {
 
   angle_test("angle_white_box_tests") {
     include_dirs = [ "." ]
-    sources = angle_white_box_tests_sources + [ "angle_end2end_tests_main.cpp" ]
+    sources = angle_white_box_tests_sources + [ "angle_generic_tests_main.cpp" ]
 
     if (is_win && angle_enable_d3d11) {
       sources += angle_white_box_tests_win_sources
@@ -333,6 +333,7 @@ template("angle_perftests_common") {
       "test_utils/runner/TestSuite.h",
     ]
     public_deps = [
+      "$angle_root/src/tests:angle_test_expectations",
       "$angle_root/third_party/rapidjson:rapidjson",
       "${invoker.test_utils}",
     ]
@@ -648,7 +649,7 @@ if (build_angle_gles1_conform_tests) {
     include_dirs = [ "." ]
 
     sources = [
-      "angle_end2end_tests_main.cpp",
+      "angle_generic_tests_main.cpp",
       "gles1_conformance_tests/ConformanceTests.cpp",
       "test_utils/ANGLETest.cpp",
       "test_utils/ANGLETest.h",
@@ -725,7 +726,7 @@ if (build_angle_gles1_conform_tests) {
 
       sources = [
         "$gles1_conform_root/conform/covgl/shell.c",
-        "angle_end2end_tests_main.cpp",
+        "angle_generic_tests_main.cpp",
         "gles1_conformance_tests/CovglTests.cpp",
         "test_utils/ANGLETest.cpp",
         "test_utils/ANGLETest.h",
@@ -795,7 +796,7 @@ if (build_angle_gles1_conform_tests) {
     include_dirs = [ "$gles1_conform_root/conform" ]
 
     sources = [
-      "angle_end2end_tests_main.cpp",
+      "angle_generic_tests_main.cpp",
       "gles1_conformance_tests/PrimtestTests.cpp",
       "test_utils/ANGLETest.cpp",
       "test_utils/ANGLETest.h",
@@ -1263,7 +1264,6 @@ if (build_angle_deqp_tests && !is_fuchsia) {
     angle_test("angle_deqp_${_api}_tests") {
       deps = [
         ":${shared_library_name}",
-        ":angle_test_expectations",
         "$angle_root:angle_common",
         "$angle_root/util:angle_util",
       ]

src/tests/angle_end2end_tests_expectations.txt

+// Copyright 2021 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// See deqp_support/README.md for format.
+
+3786 WIN NVIDIA D3D11 : BufferDataOverflowTest.VertexBufferIntegerOverflow/ES3_D3D11 = SKIP
+4092 WIN VULKAN : BufferDataOverflowTest.VertexBufferIntegerOverflow/ES3_Vulkan* = SKIP
+4092 WIN OPENGL : BufferDataOverflowTest.VertexBufferIntegerOverflow/ES3_OpenGL = SKIP
+4092 WIN GLES : BufferDataOverflowTest.VertexBufferIntegerOverflow/ES3_OpenGLES = SKIP
+

src/tests/angle_end2end_tests_main.cpp

@@ -6,6 +6,7 @@
 
 #include "gtest/gtest.h"
 #include "test_utils/runner/TestSuite.h"
+#include "util/OSWindow.h"
 
 void ANGLEProcessTestArgs(int *argc, char *argv[]);
 
@@ -21,8 +22,9 @@ void RegisterContextCompatibilityTests();
 // If we ever move to a text-based expectations format, we should move this list in that file.
 namespace
 {
-const char *kSlowTests[] = {"GLSLTest.VerifyMaxVertexUniformVectors*",
+const char *kSlowTests[]               = {"GLSLTest.VerifyMaxVertexUniformVectors*",
                             "MultiThreadingTest.MultiContextDeleteDraw*"};
+constexpr char kTestExpectationsPath[] = "src/tests/angle_end2end_tests_expectations.txt";
 }  // namespace
 
 int main(int argc, char **argv)
@@ -31,5 +33,19 @@ int main(int argc, char **argv)
     ANGLEProcessTestArgs(&argc, argv);
     RegisterContextCompatibilityTests();
     testSuite.registerSlowTests(kSlowTests, ArraySize(kSlowTests));
+
+    constexpr size_t kMaxPath = 512;
+    std::array<char, kMaxPath> foundDataPath;
+    if (!angle::FindTestDataPath(kTestExpectationsPath, foundDataPath.data(), foundDataPath.size()))
+    {
+        std::cerr << "Unable to find test expectations path (" << kTestExpectationsPath << ")\n";
+        return EXIT_FAILURE;
+    }
+
+    if (!testSuite.loadAllTestExpectationsFromFile(std::string(foundDataPath.data())))
+    {
+        return EXIT_FAILURE;
+    }
+
     return testSuite.run();
 }

src/tests/angle_generic_tests_main.cpp

+//
+// Copyright 2021 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// Generic entry point for test suites.
+
+#include "gtest/gtest.h"
+#include "test_utils/runner/TestSuite.h"
+
+void ANGLEProcessTestArgs(int *argc, char *argv[]);
+
+int main(int argc, char **argv)
+{
+    angle::TestSuite testSuite(&argc, argv);
+    ANGLEProcessTestArgs(&argc, argv);
+    return testSuite.run();
+}

src/tests/deqp_support/angle_deqp_gtest.cpp

@@ -21,8 +21,6 @@
 #include "common/string_utils.h"
 #include "common/system_utils.h"
 #include "platform/PlatformMethods.h"
-#include "tests/test_expectations/GPUTestConfig.h"
-#include "tests/test_expectations/GPUTestExpectationsParser.h"
 #include "tests/test_utils/runner/TestSuite.h"
 #include "util/OSWindow.h"
 #include "util/test_utils.h"
@@ -244,7 +242,6 @@ class dEQPCaseList
 
   private:
     std::vector<CaseInfo> mCaseInfoList;
-    GPUTestExpectationsParser mTestExpectationsParser;
     size_t mTestModuleIndex;
     bool mInitialized = false;
 };
@@ -296,16 +293,11 @@ void dEQPCaseList::initialize()
     }
 #endif
 
-    if (!mTestExpectationsParser.loadTestExpectationsFromFile(testConfig,
-                                                              testExpectationsPath.value()))
-    {
-        std::stringstream errorMsgStream;
-        for (const auto &message : mTestExpectationsParser.getErrorMessages())
-        {
-            errorMsgStream << std::endl << " " << message;
-        }
+    TestSuite *testSuite = TestSuite::GetInstance();
 
-        std::cerr << "Failed to load test expectations." << errorMsgStream.str() << std::endl;
+    if (!testSuite->loadTestExpectationsFromFileWithConfig(testConfig,
+                                                           testExpectationsPath.value()))
+    {
         Die();
     }
 
@@ -328,20 +320,12 @@ void dEQPCaseList::initialize()
         if (gTestName.empty())
             continue;
 
-        int expectation = mTestExpectationsParser.getTestExpectation(dEQPName);
+        int expectation = testSuite->getTestExpectation(dEQPName);
         mCaseInfoList.push_back(CaseInfo(dEQPName, gTestName, expectation));
     }
 
-    std::stringstream unusedMsgStream;
-    bool anyUnused = false;
-    for (const auto &message : mTestExpectationsParser.getUnusedExpectationsMessages())
-    {
-        anyUnused = true;
-        unusedMsgStream << std::endl << " " << message;
-    }
-    if (anyUnused)
+    if (testSuite->logAnyUnusedTestExpectations())
     {
-        std::cerr << "Failed to validate test expectations." << unusedMsgStream.str() << std::endl;
         Die();
     }
 }

src/tests/gl_tests/BufferDataTest.cpp

@@ -987,11 +987,6 @@ class BufferDataOverflowTest : public ANGLETest
 // See description above.
 TEST_P(BufferDataOverflowTest, VertexBufferIntegerOverflow)
 {
-    // http://anglebug.com/3786: flaky timeout on Win10 FYI x64 Release (NVIDIA GeForce GTX 1660)
-    ANGLE_SKIP_TEST_IF(IsWindows() && IsNVIDIA() && IsD3D11());
-    // http://anglebug.com/4092
-    ANGLE_SKIP_TEST_IF(IsWindows() && (IsVulkan() || IsOpenGL()));
-
     // These values are special, to trigger the rounding bug.
     unsigned int numItems       = 0x7FFFFFE;
     constexpr GLsizei bufferCnt = 8;

src/tests/test_expectations/GPUTestExpectationsParser.h

@@ -17,7 +17,6 @@
 
 namespace angle
 {
-
 struct GPUTestConfig;
 
 class GPUTestExpectationsParser

src/tests/test_utils/ANGLETest.cpp

@@ -14,6 +14,7 @@
 
 #include "common/platform.h"
 #include "gpu_info_util/SystemInfo.h"
+#include "test_utils/runner/TestSuite.h"
 #include "util/EGLWindow.h"
 #include "util/OSWindow.h"
 #include "util/random_utils.h"
@@ -183,6 +184,35 @@ bool ShouldAlwaysForceNewDisplay()
     SystemInfo *systemInfo = GetTestSystemInfo();
     return (!systemInfo || !IsWindows() || systemInfo->hasAMDGPU());
 }
+
+GPUTestConfig::API GetTestConfigAPIFromRenderer(EGLenum renderer, EGLenum deviceType)
+{
+    switch (renderer)
+    {
+        case EGL_PLATFORM_ANGLE_TYPE_D3D11_ANGLE:
+            return GPUTestConfig::kAPID3D11;
+        case EGL_PLATFORM_ANGLE_TYPE_D3D9_ANGLE:
+            return GPUTestConfig::kAPID3D9;
+        case EGL_PLATFORM_ANGLE_TYPE_OPENGL_ANGLE:
+            return GPUTestConfig::kAPIGLDesktop;
+        case EGL_PLATFORM_ANGLE_TYPE_OPENGLES_ANGLE:
+            return GPUTestConfig::kAPIGLES;
+        case EGL_PLATFORM_ANGLE_TYPE_VULKAN_ANGLE:
+            if (deviceType == EGL_PLATFORM_ANGLE_DEVICE_TYPE_SWIFTSHADER_ANGLE)
+            {
+                return GPUTestConfig::kAPISwiftShader;
+            }
+            else
+            {
+                return GPUTestConfig::kAPIVulkan;
+            }
+        case EGL_PLATFORM_ANGLE_TYPE_METAL_ANGLE:
+            return GPUTestConfig::kAPIMetal;
+        default:
+            std::cerr << "Unknown Renderer enum: 0x%X\n" << renderer;
+            return GPUTestConfig::kAPIUnknown;
+    }
+}
 }  // anonymous namespace
 
 GLColorRGB::GLColorRGB(const Vector3 &floatColor)
@@ -518,10 +548,30 @@ void ANGLETestBase::ANGLETestSetUp()
     gPlatformContext.warningsAsErrors = false;
     gPlatformContext.currentTest      = this;
 
+    const testing::TestInfo *testInfo = testing::UnitTest::GetInstance()->current_test_info();
+
+    // Check the skip list.
+
+    angle::GPUTestConfig::API api = GetTestConfigAPIFromRenderer(mCurrentParams->getRenderer(),
+                                                                 mCurrentParams->getDeviceType());
+    GPUTestConfig testConfig      = GPUTestConfig(api, 0);
+
+    std::stringstream fullTestNameStr;
+    fullTestNameStr << testInfo->test_case_name() << "." << testInfo->name();
+    std::string fullTestName = fullTestNameStr.str();
+
+    TestSuite *testSuite    = TestSuite::GetInstance();
+    int32_t testExpectation = testSuite->getTestExpectationWithConfig(testConfig, fullTestName);
+
+    if (testExpectation == GPUTestExpectationsParser::kGpuTestSkip)
+    {
+        GTEST_SKIP() << "Test skipped on this config";
+        return;
+    }
+
     if (IsWindows())
     {
-        const auto &info = testing::UnitTest::GetInstance()->current_test_info();
-        WriteDebugMessage("Entering %s.%s\n", info->test_case_name(), info->name());
+        WriteDebugMessage("Entering %s\n", fullTestName.c_str());
     }
 
     if (mCurrentParams->noFixture)

src/tests/test_utils/runner/TestSuite.cpp

@@ -1865,6 +1865,66 @@ bool GetTestResultsFromFile(const char *fileName, TestResults *resultsOut)
     return true;
 }
 
+void TestSuite::dumpTestExpectationsErrorMessages()
+{
+    std::stringstream errorMsgStream;
+    for (const auto &message : mTestExpectationsParser.getErrorMessages())
+    {
+        errorMsgStream << std::endl << " " << message;
+    }
+
+    std::cerr << "Failed to load test expectations." << errorMsgStream.str() << std::endl;
+}
+
+bool TestSuite::loadTestExpectationsFromFileWithConfig(const GPUTestConfig &config,
+                                                       const std::string &fileName)
+{
+    if (!mTestExpectationsParser.loadTestExpectationsFromFile(config, fileName))
+    {
+        dumpTestExpectationsErrorMessages();
+        return false;
+    }
+    return true;
+}
+
+bool TestSuite::loadAllTestExpectationsFromFile(const std::string &fileName)
+{
+    if (!mTestExpectationsParser.loadAllTestExpectationsFromFile(fileName))
+    {
+        dumpTestExpectationsErrorMessages();
+        return false;
+    }
+    return true;
+}
+
+bool TestSuite::logAnyUnusedTestExpectations()
+{
+    std::stringstream unusedMsgStream;
+    bool anyUnused = false;
+    for (const auto &message : mTestExpectationsParser.getUnusedExpectationsMessages())
+    {
+        anyUnused = true;
+        unusedMsgStream << std::endl << " " << message;
+    }
+    if (anyUnused)
+    {
+        std::cerr << "Failed to validate test expectations." << unusedMsgStream.str() << std::endl;
+        return true;
+    }
+    return false;
+}
+
+int32_t TestSuite::getTestExpectation(const std::string &testName)
+{
+    return mTestExpectationsParser.getTestExpectation(testName);
+}
+
+int32_t TestSuite::getTestExpectationWithConfig(const GPUTestConfig &config,
+                                                const std::string &testName)
+{
+    return mTestExpectationsParser.getTestExpectationWithConfig(config, testName);
+}
+
 const char *TestResultTypeToString(TestResultType type)
 {
     switch (type)

src/tests/test_utils/runner/TestSuite.h

@@ -17,6 +17,7 @@
 #include <thread>
 
 #include "HistogramWriter.h"
+#include "tests/test_expectations/GPUTestExpectationsParser.h"
 #include "util/test_utils.h"
 
 namespace angle
@@ -144,12 +145,21 @@ class TestSuite
     int getShardIndex() const { return mShardIndex; }
     int getBatchId() const { return mBatchId; }
 
+    // Test expectation processing.
+    bool loadTestExpectationsFromFileWithConfig(const GPUTestConfig &config,
+                                                const std::string &fileName);
+    bool loadAllTestExpectationsFromFile(const std::string &fileName);
+    int32_t getTestExpectation(const std::string &testName);
+    int32_t getTestExpectationWithConfig(const GPUTestConfig &config, const std::string &testName);
+    bool logAnyUnusedTestExpectations();
+
   private:
     bool parseSingleArg(const char *argument);
     bool launchChildTestProcess(uint32_t batchId, const std::vector<TestIdentifier> &testsInBatch);
     bool finishProcess(ProcessInfo *processInfo);
     int printFailuresAndReturnCount() const;
     void startWatchdog();
+    void dumpTestExpectationsErrorMessages();
 
     static TestSuite *mInstance;
 
@@ -186,6 +196,7 @@ class TestSuite
     HistogramWriter mHistogramWriter;
     std::vector<std::string> mSlowTests;
     std::string mTestArtifactDirectory;
+    GPUTestExpectationsParser mTestExpectationsParser;
 };
 
 bool GetTestResultsFromFile(const char *fileName, TestResults *resultsOut);