Test Runner: Add ability to retry flaky tests.

src/tests/test_utils/runner/README.md

@@ -24,6 +24,7 @@ following additional command-line arguments:
  * `--results-file` specifies a location for the JSON test result output
  * `--shard-count` and `--shard-index` control the test sharding
  * `--test-timeout` limits the amount of time spent in each test
+ * `--flaky-retries` allows for tests to fail a fixed number of times and still pass
 
 `--isolated-script-test-output` and `--isolated-script-perf-test-output` mirror `--results-file`
 and `--histogram-json-file` respectively.

src/tests/test_utils/runner/TestSuite.h

@@ -54,7 +54,7 @@ enum class TestResultType
 {
     Crash,
     Fail,
-    Skip,
+    NoResult,
     Pass,
     Timeout,
     Unknown,
@@ -64,8 +64,9 @@ const char *TestResultTypeToString(TestResultType type);
 
 struct TestResult
 {
-    TestResultType type       = TestResultType::Skip;
+    TestResultType type       = TestResultType::NoResult;
     double elapsedTimeSeconds = 0.0;
+    uint32_t flakyFailures    = 0;
 };
 
 inline bool operator==(const TestResult &a, const TestResult &b)
@@ -153,6 +154,7 @@ class TestSuite
     int mTestTimeout;
     int mBatchTimeout;
     int mBatchId;
+    int mFlakyRetries;
     std::vector<std::string> mChildProcessArgs;
     std::map<TestIdentifier, FileLine> mTestFileLines;
     std::vector<ProcessInfo> mCurrentProcesses;

src/tests/test_utils/runner/TestSuite_unittest.cpp

@@ -24,6 +24,10 @@ namespace js = rapidjson;
 namespace
 {
 constexpr char kTestHelperExecutable[] = "test_utils_unittest_helper";
+constexpr int kFlakyRetries            = 3;
+
+// Enable this for debugging.
+constexpr bool kDebugOutput = false;
 
 class TestSuiteTest : public testing::Test
 {
@@ -36,48 +40,70 @@ class TestSuiteTest : public testing::Test
         }
     }
 
-    std::string mTempFileName;
-};
+    bool runTestSuite(const std::vector<std::string> &extraArgs, TestResults *actualResults)
+    {
+        std::string executablePath = GetExecutableDirectory();
+        EXPECT_NE(executablePath, "");
+        executablePath += std::string("/") + kTestHelperExecutable + GetExecutableExtension();
 
-// Tests the ANGLE standalone testing harness. Runs four tests with different ending conditions.
-// Verifies that Pass, Fail, Crash and Timeout are all handled correctly.
-TEST_F(TestSuiteTest, RunMockTests)
-{
-    std::string executablePath = GetExecutableDirectory();
-    EXPECT_NE(executablePath, "");
-    executablePath += std::string("/") + kTestHelperExecutable + GetExecutableExtension();
+        constexpr uint32_t kMaxTempDirLen = 100;
+        char tempDirName[kMaxTempDirLen * 2];
 
-    constexpr uint32_t kMaxTempDirLen = 100;
-    char tempDirName[kMaxTempDirLen * 2];
-    ASSERT_TRUE(GetTempDir(tempDirName, kMaxTempDirLen));
+        if (!GetTempDir(tempDirName, kMaxTempDirLen))
+        {
+            return false;
+        }
 
-    std::stringstream tempFNameStream;
-    tempFNameStream << tempDirName << GetPathSeparator() << "test_temp_" << rand() << ".json";
-    mTempFileName = tempFNameStream.str();
+        std::stringstream tempFNameStream;
+        tempFNameStream << tempDirName << GetPathSeparator() << "test_temp_" << rand() << ".json";
+        mTempFileName = tempFNameStream.str();
 
-    std::string resultsFileName = "--results-file=" + mTempFileName;
+        std::string resultsFileName = "--results-file=" + mTempFileName;
 
-    std::vector<const char *> args = {executablePath.c_str(),
-                                      kRunTestSuite,
-                                      "--gtest_filter=MockTestSuiteTest.DISABLED_*",
-                                      "--gtest_also_run_disabled_tests",
-                                      "--bot-mode",
-                                      "--test-timeout=5",
-                                      resultsFileName.c_str()};
+        std::vector<const char *> args = {
+            executablePath.c_str(), kRunTestSuite,      "--gtest_also_run_disabled_tests",
+            "--bot-mode",           "--test-timeout=5", resultsFileName.c_str()};
 
-    ProcessHandle process(args, true, true);
-    EXPECT_TRUE(process->started());
-    EXPECT_TRUE(process->finish());
-    EXPECT_TRUE(process->finished());
-    EXPECT_EQ(process->getStderr(), "");
+        for (const std::string &arg : extraArgs)
+        {
+            args.push_back(arg.c_str());
+        }
 
-    // Uncomment this for debugging.
-    // printf("stdout:\n%s\n", process->getStdout().c_str());
+        if (kDebugOutput)
+        {
+            printf("Test arguments:\n");
+            for (const char *arg : args)
+            {
+                printf("%s ", arg);
+            }
+            printf("\n");
+        }
+
+        ProcessHandle process(args, true, true);
+        EXPECT_TRUE(process->started());
+        EXPECT_TRUE(process->finish());
+        EXPECT_TRUE(process->finished());
+        EXPECT_EQ(process->getStderr(), "");
+
+        if (kDebugOutput)
+        {
+            printf("stdout:\n%s\n", process->getStdout().c_str());
+        }
+
+        return GetTestResultsFromFile(mTempFileName.c_str(), actualResults);
+    }
+
+    std::string mTempFileName;
+};
+
+// Tests the ANGLE standalone testing harness. Runs four tests with different ending conditions.
+// Verifies that Pass, Fail, Crash and Timeout are all handled correctly.
+TEST_F(TestSuiteTest, RunMockTests)
+{
+    std::vector<std::string> extraArgs = {"--gtest_filter=MockTestSuiteTest.DISABLED_*"};
 
     TestResults actual;
-    ASSERT_TRUE(GetTestResultsFromFile(mTempFileName.c_str(), &actual));
-    EXPECT_TRUE(DeleteFile(mTempFileName.c_str()));
-    mTempFileName.clear();
+    ASSERT_TRUE(runTestSuite(extraArgs, &actual));
 
     std::map<TestIdentifier, TestResult> expectedResults = {
         {{"MockTestSuiteTest", "DISABLED_Pass"}, {TestResultType::Pass, 0.0}},
@@ -89,6 +115,22 @@ TEST_F(TestSuiteTest, RunMockTests)
     EXPECT_EQ(expectedResults, actual.results);
 }
 
+// Verifies the flaky retry feature works as expected.
+TEST_F(TestSuiteTest, RunFlakyTests)
+{
+    std::vector<std::string> extraArgs = {"--gtest_filter=MockFlakyTestSuiteTest.DISABLED_Flaky",
+                                          "--flaky-retries=" + std::to_string(kFlakyRetries)};
+
+    TestResults actual;
+    ASSERT_TRUE(runTestSuite(extraArgs, &actual));
+
+    std::map<TestIdentifier, TestResult> expectedResults = {
+        {{"MockFlakyTestSuiteTest", "DISABLED_Flaky"},
+         {TestResultType::Pass, 0.0, kFlakyRetries - 1}}};
+
+    EXPECT_EQ(expectedResults, actual.results);
+}
+
 // Normal passing test.
 TEST(MockTestSuiteTest, DISABLED_Pass)
 {
@@ -107,6 +149,43 @@ TEST(MockTestSuiteTest, DISABLED_Timeout)
     angle::Sleep(20000);
 }
 
+// Trigger a flaky test failure.
+TEST(MockFlakyTestSuiteTest, DISABLED_Flaky)
+{
+    constexpr uint32_t kMaxTempDirLen = 100;
+    char tempDirName[kMaxTempDirLen * 2];
+    ASSERT_TRUE(GetTempDir(tempDirName, kMaxTempDirLen));
+
+    std::stringstream tempFNameStream;
+    tempFNameStream << tempDirName << GetPathSeparator() << "flaky_temp.txt";
+    std::string tempFileName = tempFNameStream.str();
+
+    int fails = 0;
+    {
+        FILE *fp = fopen(tempFileName.c_str(), "r");
+        if (fp)
+        {
+            ASSERT_EQ(fscanf(fp, "%d", &fails), 1);
+            fclose(fp);
+        }
+    }
+
+    if (fails >= kFlakyRetries - 1)
+    {
+        angle::DeleteFile(tempFileName.c_str());
+    }
+    else
+    {
+        EXPECT_TRUE(false);
+
+        FILE *fp = fopen(tempFileName.c_str(), "w");
+        ASSERT_NE(fp, nullptr);
+
+        fprintf(fp, "%d", fails + 1);
+        fclose(fp);
+    }
+}
+
 // Trigger a test crash.
 // TEST(MockTestSuiteTest, DISABLED_Crash)
 // {