Test Runner: Add a slow tests list.

src/common/string_utils.cpp

@@ -251,4 +251,45 @@ std::vector<std::string> GetCachedStringsFromEnvironmentVarOrAndroidProperty(
     std::string environment = GetEnvironmentVarOrAndroidProperty(varName, propertyName);
     return SplitString(environment, separator, TRIM_WHITESPACE, SPLIT_WANT_NONEMPTY);
 }
+
+// reference name can have *.
+bool NamesMatchWithWildcard(const char *ref, const char *testName)
+{
+    // Find the first * in ref.
+    const char *firstWildcard = strchr(ref, '*');
+
+    // If there are no wildcards, match the strings precisely.
+    if (firstWildcard == nullptr)
+    {
+        return strcmp(ref, testName) == 0;
+    }
+
+    // Otherwise, match up to the wildcard first.
+    size_t preWildcardLen = firstWildcard - ref;
+    if (strncmp(ref, testName, preWildcardLen) != 0)
+    {
+        return false;
+    }
+
+    const char *postWildcardRef = ref + preWildcardLen + 1;
+
+    // As a small optimization, if the wildcard is the last character in ref, accept the match
+    // already.
+    if (postWildcardRef[0] == '\0')
+    {
+        return true;
+    }
+
+    // Try to match the wildcard with a number of characters.
+    for (size_t matchSize = 0; testName[matchSize] != '\0'; ++matchSize)
+    {
+        if (NamesMatchWithWildcard(postWildcardRef, testName + matchSize))
+        {
+            return true;
+        }
+    }
+
+    return false;
+}
+
 }  // namespace angle

src/common/string_utils.h

@@ -105,6 +105,8 @@ std::vector<std::string> GetCachedStringsFromEnvironmentVarOrAndroidProperty(
     const char *propertyName,
     const char *separator);
 
+// reference name can have *.
+bool NamesMatchWithWildcard(const char *ref, const char *testName);
 }  // namespace angle
 
 #endif  // LIBANGLE_STRING_UTILS_H_

src/common/string_utils_unittest.cpp

@@ -136,6 +136,33 @@ TEST(StringUtilsTest, HexStringToUIntBasic)
     EXPECT_FALSE(HexStringToUInt(testStringD, &uintValue));
 }
 
+// Basic functionality for NamesMatchWithWildcard.
+TEST(StringUtilsTest, NamesMatchWithWildcard)
+{
+    EXPECT_TRUE(NamesMatchWithWildcard("ASDF", "ASDF"));
+    EXPECT_TRUE(NamesMatchWithWildcard("A*", "ASDF"));
+    EXPECT_TRUE(NamesMatchWithWildcard("AS*", "ASDF"));
+    EXPECT_TRUE(NamesMatchWithWildcard("ASD*", "ASDF"));
+    EXPECT_TRUE(NamesMatchWithWildcard("ASDF*", "ASDF"));
+
+    EXPECT_TRUE(NamesMatchWithWildcard("*F", "ASDF"));
+    EXPECT_TRUE(NamesMatchWithWildcard("*DF", "ASDF"));
+    EXPECT_TRUE(NamesMatchWithWildcard("*SDF", "ASDF"));
+    EXPECT_TRUE(NamesMatchWithWildcard("*ASDF", "ASDF"));
+
+    EXPECT_TRUE(NamesMatchWithWildcard("AS**", "ASDF"));
+    EXPECT_TRUE(NamesMatchWithWildcard("AS***", "ASDF"));
+    EXPECT_TRUE(NamesMatchWithWildcard("**DF", "ASDF"));
+    EXPECT_TRUE(NamesMatchWithWildcard("***DF", "ASDF"));
+
+    EXPECT_TRUE(NamesMatchWithWildcard("A*F", "ASDF"));
+    EXPECT_TRUE(NamesMatchWithWildcard("A**F", "ASDF"));
+    EXPECT_TRUE(NamesMatchWithWildcard("*SD*", "ASDF"));
+    EXPECT_TRUE(NamesMatchWithWildcard("*S*D*", "ASDF"));
+
+    EXPECT_TRUE(NamesMatchWithWildcard("ASD*", "ASDF*"));
+}
+
 // Note: ReadFileToString is harder to test
 
 class BeginsWithTest : public testing::Test

src/tests/angle_end2end_tests_main.cpp

@@ -18,10 +18,17 @@ void ANGLEProcessTestArgs(int *argc, char *argv[]);
 // likely specialize more register functions more like dEQP instead of relying on static init.
 void RegisterContextCompatibilityTests();
 
+// If we ever move to a text-based expectations format, we should move this list in that file.
+namespace
+{
+const char *kSlowTests[] = {"GLSLTest.VerifyMaxVertexUniformVectors*"};
+}  // namespace
+
 int main(int argc, char **argv)
 {
     angle::TestSuite testSuite(&argc, argv);
     ANGLEProcessTestArgs(&argc, argv);
     RegisterContextCompatibilityTests();
+    testSuite.registerSlowTests(kSlowTests, ArraySize(kSlowTests));
     return testSuite.run();
 }

src/tests/gl_tests/GLSLTest.cpp

@@ -2113,9 +2113,6 @@ void main()
 // can actually be used.
 TEST_P(GLSLTest, VerifyMaxVertexUniformVectors)
 {
-    // Times out on D3D11 on test infra. http://anglebug.com/5076
-    ANGLE_SKIP_TEST_IF(IsD3D11() && IsIntel());
-
     // crbug.com/680631
     ANGLE_SKIP_TEST_IF(IsOzone() && IsIntel());
 

src/tests/test_expectations/GPUTestExpectationsParser.cpp

@@ -238,47 +238,6 @@ inline Token ParseToken(const std::string &word)
     }
     return kTokenWord;
 }
-
-// reference name can have *.
-inline bool NamesMatching(const char *ref, const char *testName)
-{
-    // Find the first * in ref.
-    const char *firstWildcard = strchr(ref, '*');
-
-    // If there are no wildcards, match the strings precisely.
-    if (firstWildcard == nullptr)
-    {
-        return strcmp(ref, testName) == 0;
-    }
-
-    // Otherwise, match up to the wildcard first.
-    size_t preWildcardLen = firstWildcard - ref;
-    if (strncmp(ref, testName, preWildcardLen) != 0)
-    {
-        return false;
-    }
-
-    const char *postWildcardRef = ref + preWildcardLen + 1;
-
-    // As a small optimization, if the wildcard is the last character in ref, accept the match
-    // already.
-    if (postWildcardRef[0] == '\0')
-    {
-        return true;
-    }
-
-    // Try to match the wildcard with a number of characters.
-    for (size_t matchSize = 0; testName[matchSize] != '\0'; ++matchSize)
-    {
-        if (NamesMatching(postWildcardRef, testName + matchSize))
-        {
-            return true;
-        }
-    }
-
-    return false;
-}
-
 }  // anonymous namespace
 
 const char *GetConditionName(uint32_t condition)
@@ -356,7 +315,7 @@ int32_t GPUTestExpectationsParser::getTestExpectation(const std::string &testNam
     GPUTestExpectationEntry *foundEntry = nullptr;
     for (size_t i = 0; i < mEntries.size(); ++i)
     {
-        if (NamesMatching(mEntries[i].testName.c_str(), testName.c_str()))
+        if (NamesMatchWithWildcard(mEntries[i].testName.c_str(), testName.c_str()))
         {
             size_t expectationLen = mEntries[i].testName.length();
             // The longest/most specific matching expectation overrides any others.

src/tests/test_utils/runner/TestSuite.cpp

@@ -374,17 +374,39 @@ TestIdentifier GetTestIdentifier(const testing::TestInfo &testInfo)
     return {testInfo.test_suite_name(), testInfo.name()};
 }
 
+bool IsSlowTest(const std::vector<std::string> &slowTests, const TestIdentifier &testID)
+{
+    char buffer[200] = {};
+    testID.sprintfName(buffer);
+
+    for (const std::string &slowTest : slowTests)
+    {
+        if (NamesMatchWithWildcard(slowTest.c_str(), buffer))
+        {
+            return true;
+        }
+    }
+
+    return false;
+}
+
 class TestEventListener : public testing::EmptyTestEventListener
 {
   public:
     // Note: TestResults is owned by the TestSuite. It should outlive TestEventListener.
     TestEventListener(const std::string &resultsFile,
                       const std::string &histogramJsonFile,
+                      const std::vector<std::string> &slowTests,
+                      double fastTestTimeout,
+                      double slowTestTimeout,
                       const char *testSuiteName,
                       TestResults *testResults,
                       HistogramWriter *histogramWriter)
         : mResultsFile(resultsFile),
           mHistogramJsonFile(histogramJsonFile),
+          mSlowTests(slowTests),
+          mFastTestTimeout(fastTestTimeout),
+          mSlowTestTimeout(slowTestTimeout),
           mTestSuiteName(testSuiteName),
           mTestResults(testResults),
           mHistogramWriter(histogramWriter)
@@ -395,6 +417,8 @@ class TestEventListener : public testing::EmptyTestEventListener
         std::lock_guard<std::mutex> guard(mTestResults->currentTestMutex);
         mTestResults->currentTest = GetTestIdentifier(testInfo);
         mTestResults->currentTestTimer.start();
+        mTestResults->currentTestTimeout =
+            IsSlowTest(mSlowTests, mTestResults->currentTest) ? mSlowTestTimeout : mFastTestTimeout;
     }
 
     void OnTestEnd(const testing::TestInfo &testInfo) override
@@ -403,7 +427,8 @@ class TestEventListener : public testing::EmptyTestEventListener
         mTestResults->currentTestTimer.stop();
         const testing::TestResult &resultIn = *testInfo.result();
         UpdateCurrentTestResult(resultIn, mTestResults);
-        mTestResults->currentTest = TestIdentifier();
+        mTestResults->currentTest        = TestIdentifier();
+        mTestResults->currentTestTimeout = 0.0;
     }
 
     void OnTestProgramEnd(const testing::UnitTest &testProgramInfo) override
@@ -417,6 +442,9 @@ class TestEventListener : public testing::EmptyTestEventListener
   private:
     std::string mResultsFile;
     std::string mHistogramJsonFile;
+    const std::vector<std::string> &mSlowTests;
+    double mFastTestTimeout;
+    double mSlowTestTimeout;
     const char *mTestSuiteName;
     TestResults *mTestResults;
     HistogramWriter *mHistogramWriter;
@@ -1107,9 +1135,9 @@ TestSuite::TestSuite(int *argc, char **argv)
     if (!mBotMode)
     {
         testing::TestEventListeners &listeners = testing::UnitTest::GetInstance()->listeners();
-        listeners.Append(new TestEventListener(mResultsFile, mHistogramJsonFile,
-                                               mTestSuiteName.c_str(), &mTestResults,
-                                               &mHistogramWriter));
+        listeners.Append(new TestEventListener(
+            mResultsFile, mHistogramJsonFile, mSlowTests, mTestTimeout, mTestTimeout * 3.0,
+            mTestSuiteName.c_str(), &mTestResults, &mHistogramWriter));
 
         for (const TestIdentifier &id : testSet)
         {
@@ -1590,7 +1618,7 @@ void TestSuite::startWatchdog()
             {
                 std::lock_guard<std::mutex> guard(mTestResults.currentTestMutex);
                 if (mTestResults.currentTestTimer.getElapsedTime() >
-                    static_cast<double>(mTestTimeout))
+                    mTestResults.currentTestTimeout)
                 {
                     break;
                 }
@@ -1615,6 +1643,14 @@ void TestSuite::addHistogramSample(const std::string &measurement,
     mHistogramWriter.addSample(measurement, story, value, units);
 }
 
+void TestSuite::registerSlowTests(const char *slowTests[], size_t numSlowTests)
+{
+    for (size_t slowTestIndex = 0; slowTestIndex < numSlowTests; ++slowTestIndex)
+    {
+        mSlowTests.push_back(slowTests[slowTestIndex]);
+    }
+}
+
 bool GetTestResultsFromFile(const char *fileName, TestResults *resultsOut)
 {
     std::ifstream ifs(fileName);

src/tests/test_utils/runner/TestSuite.h

@@ -92,7 +92,8 @@ struct TestResults
     std::mutex currentTestMutex;
     TestIdentifier currentTest;
     Timer currentTestTimer;
-    bool allDone = false;
+    double currentTestTimeout = 0.0;
+    bool allDone              = false;
 };
 
 struct FileLine
@@ -130,6 +131,7 @@ class TestSuite
                             const std::string &story,
                             double value,
                             const std::string &units);
+    void registerSlowTests(const char *slowTests[], size_t numSlowTests);
 
     static TestSuite *GetInstance() { return mInstance; }
 
@@ -173,6 +175,7 @@ class TestSuite
     std::vector<ProcessInfo> mCurrentProcesses;
     std::thread mWatchdogThread;
     HistogramWriter mHistogramWriter;
+    std::vector<std::string> mSlowTests;
 };
 
 bool GetTestResultsFromFile(const char *fileName, TestResults *resultsOut);