Test Runner: Fixes to skipped tests.

Uses a different token for skipped and not run tests in the json. The token isn't standard but should only be visible when batches report to the parent process that a test wasn't run. It shouldn't be visible to the scripts that consume test results on the bots. Also fixes the expected result for skipped tests to also be "SKIP". Bug: angleproject:5951 Change-Id: I0bd7d13b2b25b583f9c42718786cb588bacd4bc6 Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/2894524Reviewed-by: Yuly Novikov <ynovikov@chromium.org> Commit-Queue: Jamie Madill <jmadill@chromium.org>

Test Runner: Fixes to skipped tests.
1d019f49 · Jamie Madill · Commit Bot · b300dc52 · 1d019f49 · 1d019f49
Commit 1d019f49 authored May 13, 2021 by Jamie Madill Committed by Commit Bot May 13, 2021
Hide whitespace changes
Inline Side-by-side

Showing with 17 additions and 1 deletion

TestSuite.cpp src/tests/test_utils/runner/TestSuite.cpp +8 -1

TestSuite_unittest.cpp src/tests/test_utils/runner/TestSuite_unittest.cpp +9 -0

No files found.
--- a/src/tests/test_utils/runner/TestSuite.cpp
+++ b/src/tests/test_utils/runner/TestSuite.cpp
@@ -161,10 +161,11 @@ const char *ResultTypeToString(TestResultType type)
            return "CRASH";
        case TestResultType::Fail:
            return "FAIL";
+        case TestResultType::NoResult:
+            return "NOTRUN";
        case TestResultType::Pass:
            return "PASS";
        case TestResultType::Skip:
-        case TestResultType::NoResult:
            return "SKIP";
        case TestResultType::Timeout:
            return "TIMEOUT";
@@ -181,6 +182,8 @@ TestResultType GetResultTypeFromString(const std::string &str)
        return TestResultType::Fail;
    if (str == "PASS")
        return TestResultType::Pass;
+    if (str == "NOTRUN")
+        return TestResultType::NoResult;
    if (str == "SKIP")
        return TestResultType::Skip;
    if (str == "TIMEOUT")
@@ -308,6 +311,10 @@ void WriteResultsFile(bool interrupted,
        actualResult += ResultTypeToString(result.type);

        std::string expectedResult = "PASS";
+        if (result.type == TestResultType::Skip)
+        {
+            expectedResult = "SKIP";
+        }

        // Handle flaky passing tests.
        if (result.flakyFailures > 0 && result.type == TestResultType::Pass)

--- a/src/tests/test_utils/runner/TestSuite_unittest.cpp
+++ b/src/tests/test_utils/runner/TestSuite_unittest.cpp
@@ -143,6 +143,7 @@ TEST_F(TestSuiteTest, RunCrashingTests)
 {
    std::vector<std::string> extraArgs = {
        "--gtest_filter=MockTestSuiteTest.DISABLED_Pass:MockTestSuiteTest.DISABLED_Fail:"
+        "MockTestSuiteTest.DISABLED_Skip:"
        "MockCrashTestSuiteTest.DISABLED_*",
        "--disable-crash-handler"};

@@ -152,8 +153,10 @@ TEST_F(TestSuiteTest, RunCrashingTests)
    std::map<TestIdentifier, TestResult> expectedResults = {
        {{"MockTestSuiteTest", "DISABLED_Pass"}, {TestResultType::Pass, 0.0}},
        {{"MockTestSuiteTest", "DISABLED_Fail"}, {TestResultType::Fail, 0.0}},
+        {{"MockTestSuiteTest", "DISABLED_Skip"}, {TestResultType::Skip, 0.0}},
        {{"MockCrashTestSuiteTest", "DISABLED_Crash"}, {TestResultType::Crash, 0.0}},
        {{"MockCrashTestSuiteTest", "DISABLED_PassAfterCrash"}, {TestResultType::Pass, 0.0}},
+        {{"MockCrashTestSuiteTest", "DISABLED_SkipAfterCrash"}, {TestResultType::Skip, 0.0}},
    };

    EXPECT_EQ(expectedResults, actual.results);
@@ -231,4 +234,10 @@ TEST(MockCrashTestSuiteTest, DISABLED_PassAfterCrash)
 {
    EXPECT_TRUE(true);
 }
+
+// This test runs after the crash test.
+TEST(MockCrashTestSuiteTest, DISABLED_SkipAfterCrash)
+{
+    GTEST_SKIP() << "Test skipped.";
+}
 }  // namespace