Test Runner: Omit results files when too many failures.

This will cause the test shards to be re-tried due to an invalid result. The goal is to retry flaky configs where whole shards fail. Bug: angleproject:6075 Change-Id: I76d0f08da29ffc4bcad7f282f7be38618c384bce Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/2982500Reviewed-by: Shahbaz Youssefi <syoussefi@chromium.org> Reviewed-by: Jonah Ryan-Davis <jonahr@google.com> Commit-Queue: Jamie Madill <jmadill@chromium.org>

Test Runner: Omit results files when too many failures.
15623ada · Jamie Madill · Angle LUCI CQ · 41f86f35 · 15623ada
Commit 15623ada authored Jun 23, 2021 by Jamie Madill Committed by Angle LUCI CQ Jun 23, 2021
Hide whitespace changes
Inline Side-by-side

Showing with 13 additions and 3 deletions

TestSuite.cpp src/tests/test_utils/runner/TestSuite.cpp +13 -3

No files found.
--- a/src/tests/test_utils/runner/TestSuite.cpp
+++ b/src/tests/test_utils/runner/TestSuite.cpp
@@ -1736,7 +1736,7 @@ int TestSuite::run()
        }

        // Early exit if we passed the maximum failure threshold. Still wait for current tests.
-        if (mFailureCount > mMaxFailures)
+        if (mFailureCount > mMaxFailures && !mTestQueue.empty())
        {
            printf("Reached maximum failure count (%d), clearing test queue.\n", mMaxFailures);
            TestQueue emptyTestQueue;
@@ -1748,8 +1748,18 @@ int TestSuite::run()
    }

    // Dump combined results.
-    WriteOutputFiles(false, mTestResults, mResultsFile, mHistogramWriter, mHistogramJsonFile,
-                     mTestSuiteName.c_str());
+    if (mFailureCount > mMaxFailures)
+    {
+        printf(
+            "Omitted results files because the failure count (%d) exceeded the maximum number of "
+            "failures (%d).\n",
+            mFailureCount, mMaxFailures);
+    }
+    else
+    {
+        WriteOutputFiles(false, mTestResults, mResultsFile, mHistogramWriter, mHistogramJsonFile,
+                         mTestSuiteName.c_str());
+    }

    totalRunTime.stop();
    printf("Tests completed in %lf seconds\n", totalRunTime.getElapsedTime());