Test Runner: Add maximum failure count.

src/tests/test_utils/runner/README.md

@@ -27,6 +27,7 @@ following additional command-line arguments:
  * `--flaky-retries` allows for tests to fail a fixed number of times and still pass
  * `--disable-crash-handler` forces off OS-level crash handling
  * `--isolated-outdir` specifies a test artifacts directory
+ * `--max-failures` specifies a count of failures after which the harness early exits.
 
 `--isolated-script-test-output` and `--isolated-script-perf-test-output` mirror `--results-file`
 and `--histogram-json-file` respectively.

src/tests/test_utils/runner/TestSuite.cpp

@@ -47,6 +47,7 @@ constexpr char kResultFileArg[]        = "--results-file=";
 constexpr char kTestTimeoutArg[]       = "--test-timeout=";
 constexpr char kDisableCrashHandler[]  = "--disable-crash-handler";
 constexpr char kIsolatedOutDir[]       = "--isolated-outdir=";
+constexpr char kMaxFailures[]          = "--max-failures=";
 
 constexpr char kStartedTestString[] = "[ RUN      ] ";
 constexpr char kPassedTestString[]  = "[       OK ] ";
@@ -68,6 +69,7 @@ constexpr int kDefaultBatchTimeout = 600;
 constexpr int kDefaultBatchSize      = 256;
 constexpr double kIdleMessageTimeout = 15.0;
 constexpr int kDefaultMaxProcesses   = 16;
+constexpr int kDefaultMaxFailures    = 100;
 
 const char *ParseFlagValue(const char *flag, const char *argument)
 {
@@ -1056,7 +1058,9 @@ TestSuite::TestSuite(int *argc, char **argv)
       mTestTimeout(kDefaultTestTimeout),
       mBatchTimeout(kDefaultBatchTimeout),
       mBatchId(-1),
-      mFlakyRetries(0)
+      mFlakyRetries(0),
+      mMaxFailures(kDefaultMaxFailures),
+      mFailureCount(0)
 {
     ASSERT(mInstance == nullptr);
     mInstance = this;
@@ -1308,6 +1312,7 @@ bool TestSuite::parseSingleArg(const char *argument)
             ParseIntArg(kTestTimeoutArg, argument, &mTestTimeout) ||
             ParseIntArg("--batch-timeout=", argument, &mBatchTimeout) ||
             ParseIntArg(kFlakyRetries, argument, &mFlakyRetries) ||
+            ParseIntArg(kMaxFailures, argument, &mMaxFailures) ||
             // Other test functions consume the batch ID, so keep it in the list.
             ParseIntArgNoDelete(kBatchId, argument, &mBatchId) ||
             ParseStringArg("--results-directory=", argument, &mResultsDirectory) ||
@@ -1569,10 +1574,12 @@ bool TestSuite::finishProcess(ProcessInfo *processInfo)
         else if (result.type == TestResultType::Timeout)
         {
             printf(" (TIMEOUT in %0.1lf s)\n", result.elapsedTimeSeconds);
+            mFailureCount++;
         }
         else
         {
             printf(" (%s)\n", ResultTypeToString(result.type));
+            mFailureCount++;
 
             const std::string &batchStdout = processInfo->process->getStdout();
             PrintTestOutputSnippet(id, result, batchStdout);
@@ -1702,6 +1709,7 @@ int TestSuite::run()
                 {
                     // Because the whole batch failed we can't know how long each test took.
                     mTestResults.results[testIdentifier].type = TestResultType::Timeout;
+                    mFailureCount++;
                 }
 
                 processIter = mCurrentProcesses.erase(processIter);
@@ -1727,6 +1735,14 @@ int TestSuite::run()
             messageTimer.start();
         }
 
+        // Early exit if we passed the maximum failure threshold. Still wait for current tests.
+        if (mFailureCount > mMaxFailures)
+        {
+            printf("Reached maximum failure count (%d), clearing test queue.\n", mMaxFailures);
+            TestQueue emptyTestQueue;
+            std::swap(mTestQueue, emptyTestQueue);
+        }
+
         // Sleep briefly and continue.
         angle::Sleep(100);
     }

src/tests/test_utils/runner/TestSuite.h

@@ -193,6 +193,8 @@ class TestSuite
     int mBatchTimeout;
     int mBatchId;
     int mFlakyRetries;
+    int mMaxFailures;
+    int mFailureCount;
     std::vector<std::string> mChildProcessArgs;
     std::map<TestIdentifier, FileLine> mTestFileLines;
     std::vector<ProcessInfo> mCurrentProcesses;