Skip to content

A
angle
Commit 1d019f49 by Jamie Madill Committed by Commit Bot
Options

Test Runner: Fixes to skipped tests.

Uses a different token for skipped and not run tests in the json. The token isn't standard but should only be visible when batches report to the parent process that a test wasn't run. It shouldn't be visible to the scripts that consume test results on the bots. Also fixes the expected result for skipped tests to also be "SKIP". Bug: angleproject:5951 Change-Id: I0bd7d13b2b25b583f9c42718786cb588bacd4bc6 Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/2894524Reviewed-by: 's avatarYuly Novikov <ynovikov@chromium.org> Commit-Queue: Jamie Madill <jmadill@chromium.org>
parent b300dc52
Showing with 17 additions and 1 deletion
src/tests/test_utils/runner/TestSuite.cpp
...@@ -161,10 +161,11 @@ const char *ResultTypeToString(TestResultType type) ...@@ -161,10 +161,11 @@ const char *ResultTypeToString(TestResultType type)
return "CRASH"; return "CRASH";
case TestResultType::Fail: case TestResultType::Fail:
return "FAIL"; return "FAIL";
case TestResultType::NoResult:
return "NOTRUN";
case TestResultType::Pass: case TestResultType::Pass:
return "PASS"; return "PASS";
case TestResultType::Skip: case TestResultType::Skip:
case TestResultType::NoResult:
return "SKIP"; return "SKIP";
case TestResultType::Timeout: case TestResultType::Timeout:
return "TIMEOUT"; return "TIMEOUT";
...@@ -181,6 +182,8 @@ TestResultType GetResultTypeFromString(const std::string &str) ...@@ -181,6 +182,8 @@ TestResultType GetResultTypeFromString(const std::string &str)
return TestResultType::Fail; return TestResultType::Fail;
if (str == "PASS") if (str == "PASS")
return TestResultType::Pass; return TestResultType::Pass;
if (str == "NOTRUN")
return TestResultType::NoResult;
if (str == "SKIP") if (str == "SKIP")
return TestResultType::Skip; return TestResultType::Skip;
if (str == "TIMEOUT") if (str == "TIMEOUT")
...@@ -308,6 +311,10 @@ void WriteResultsFile(bool interrupted, ...@@ -308,6 +311,10 @@ void WriteResultsFile(bool interrupted,
actualResult += ResultTypeToString(result.type); actualResult += ResultTypeToString(result.type);
std::string expectedResult = "PASS"; std::string expectedResult = "PASS";
if (result.type == TestResultType::Skip)
{
expectedResult = "SKIP";
}
// Handle flaky passing tests. // Handle flaky passing tests.
if (result.flakyFailures > 0 && result.type == TestResultType::Pass) if (result.flakyFailures > 0 && result.type == TestResultType::Pass)
......
src/tests/test_utils/runner/TestSuite_unittest.cpp
...@@ -143,6 +143,7 @@ TEST_F(TestSuiteTest, RunCrashingTests) ...@@ -143,6 +143,7 @@ TEST_F(TestSuiteTest, RunCrashingTests)
{ {
std::vector<std::string> extraArgs = { std::vector<std::string> extraArgs = {
"--gtest_filter=MockTestSuiteTest.DISABLED_Pass:MockTestSuiteTest.DISABLED_Fail:" "--gtest_filter=MockTestSuiteTest.DISABLED_Pass:MockTestSuiteTest.DISABLED_Fail:"
"MockTestSuiteTest.DISABLED_Skip:"
"MockCrashTestSuiteTest.DISABLED_*", "MockCrashTestSuiteTest.DISABLED_*",
"--disable-crash-handler"}; "--disable-crash-handler"};
...@@ -152,8 +153,10 @@ TEST_F(TestSuiteTest, RunCrashingTests) ...@@ -152,8 +153,10 @@ TEST_F(TestSuiteTest, RunCrashingTests)
std::map<TestIdentifier, TestResult> expectedResults = { std::map<TestIdentifier, TestResult> expectedResults = {
{{"MockTestSuiteTest", "DISABLED_Pass"}, {TestResultType::Pass, 0.0}}, {{"MockTestSuiteTest", "DISABLED_Pass"}, {TestResultType::Pass, 0.0}},
{{"MockTestSuiteTest", "DISABLED_Fail"}, {TestResultType::Fail, 0.0}}, {{"MockTestSuiteTest", "DISABLED_Fail"}, {TestResultType::Fail, 0.0}},
{{"MockTestSuiteTest", "DISABLED_Skip"}, {TestResultType::Skip, 0.0}},
{{"MockCrashTestSuiteTest", "DISABLED_Crash"}, {TestResultType::Crash, 0.0}}, {{"MockCrashTestSuiteTest", "DISABLED_Crash"}, {TestResultType::Crash, 0.0}},
{{"MockCrashTestSuiteTest", "DISABLED_PassAfterCrash"}, {TestResultType::Pass, 0.0}}, {{"MockCrashTestSuiteTest", "DISABLED_PassAfterCrash"}, {TestResultType::Pass, 0.0}},
{{"MockCrashTestSuiteTest", "DISABLED_SkipAfterCrash"}, {TestResultType::Skip, 0.0}},
}; };
EXPECT_EQ(expectedResults, actual.results); EXPECT_EQ(expectedResults, actual.results);
...@@ -231,4 +234,10 @@ TEST(MockCrashTestSuiteTest, DISABLED_PassAfterCrash) ...@@ -231,4 +234,10 @@ TEST(MockCrashTestSuiteTest, DISABLED_PassAfterCrash)
{ {
EXPECT_TRUE(true); EXPECT_TRUE(true);
} }
// This test runs after the crash test.
TEST(MockCrashTestSuiteTest, DISABLED_SkipAfterCrash)
{
GTEST_SKIP() << "Test skipped.";
}
} // namespace } // namespace
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment