Update test result output from ANGLE dEQP gtests.

src/tests/deqp_support/angle_deqp_gtest.cpp

@@ -26,13 +26,15 @@
 
 namespace angle
 {
-
 namespace
 {
-
 bool gGlobalError = false;
 bool gExpectError = false;
 
+// Stored as globals to work around a Clang bug. http://crbug.com/951458
+std::vector<std::string> gUnexpectedFailed;
+std::vector<std::string> gUnexpectedPasses;
+
 void HandlePlatformError(angle::PlatformMethods *platform, const char *errorMessage)
 {
     if (!gExpectError)
@@ -220,12 +222,10 @@ class dEQPCaseList
     std::vector<CaseInfo> mCaseInfoList;
     angle::GPUTestExpectationsParser mTestExpectationsParser;
     size_t mTestModuleIndex;
-    bool mInitialized;
+    bool mInitialized = false;
 };
 
-dEQPCaseList::dEQPCaseList(size_t testModuleIndex)
-    : mTestModuleIndex(testModuleIndex), mInitialized(false)
-{}
+dEQPCaseList::dEQPCaseList(size_t testModuleIndex) : mTestModuleIndex(testModuleIndex) {}
 
 void dEQPCaseList::initialize()
 {
@@ -294,27 +294,7 @@ void dEQPCaseList::initialize()
             continue;
 
         int expectation = mTestExpectationsParser.getTestExpectation(dEQPName);
-        if (expectation != angle::GPUTestExpectationsParser::kGpuTestSkip)
-        {
-            mCaseInfoList.push_back(CaseInfo(dEQPName, gTestName, expectation));
-        }
-    }
-}
-
-bool TestPassed(TestResult result)
-{
-    switch (result)
-    {
-        case TestResult::Pass:
-            return true;
-        case TestResult::Fail:
-            return false;
-        case TestResult::NotSupported:
-            return true;
-        case TestResult::Exception:
-            return false;
-        default:
-            return false;
+        mCaseInfoList.push_back(CaseInfo(dEQPName, gTestName, expectation));
     }
 }
 
@@ -346,7 +326,7 @@ class dEQPTest : public testing::TestWithParam<size_t>
   protected:
     void runTest() const
     {
-        if (sExceptions > 1)
+        if (sTestExceptionCount > 1)
         {
             std::cout << "Too many exceptions, skipping all remaining tests." << std::endl;
             return;
@@ -355,62 +335,132 @@ class dEQPTest : public testing::TestWithParam<size_t>
         const auto &caseInfo = GetCaseList().getCaseInfo(GetParam());
         std::cout << caseInfo.mDEQPName << std::endl;
 
+        // Tests that crash exit the harness before collecting the result. To tally the number of
+        // crashed tests we track how many tests we "tried" to run.
+        sTestCount++;
+
+        if (caseInfo.mExpectation == angle::GPUTestExpectationsParser::kGpuTestSkip)
+        {
+            std::cout << "Test skipped.\n";
+            return;
+        }
+
         gExpectError = (caseInfo.mExpectation != angle::GPUTestExpectationsParser::kGpuTestPass);
         TestResult result = deqp_libtester_run(caseInfo.mDEQPName.c_str());
 
-        bool testPassed = TestPassed(result);
+        bool testSucceeded = countTestResultAndReturnSuccess(result);
 
         // Check the global error flag for unexpected platform errors.
         if (gGlobalError)
         {
-            testPassed   = false;
-            gGlobalError = false;
+            testSucceeded = false;
+            gGlobalError  = false;
         }
 
         if (caseInfo.mExpectation == angle::GPUTestExpectationsParser::kGpuTestPass)
         {
-            EXPECT_TRUE(testPassed);
-            sPasses += (testPassed ? 1u : 0u);
-            sFails += (!testPassed ? 1u : 0u);
+            EXPECT_TRUE(testSucceeded);
+
+            if (!testSucceeded)
+            {
+                gUnexpectedFailed.push_back(caseInfo.mDEQPName);
+            }
         }
-        else if (testPassed)
+        else if (testSucceeded)
         {
             std::cout << "Test expected to fail but passed!" << std::endl;
-            sUnexpectedPasses++;
+            gUnexpectedPasses.push_back(caseInfo.mDEQPName);
         }
-        else
+    }
+
+    bool countTestResultAndReturnSuccess(TestResult result) const
+    {
+        switch (result)
+        {
+            case TestResult::Pass:
+                sPassedTestCount++;
+                return true;
+            case TestResult::Fail:
+                sFailedTestCount++;
+                return false;
+            case TestResult::NotSupported:
+                sNotSupportedTestCount++;
+                return true;
+            case TestResult::Exception:
+                sTestExceptionCount++;
+                return false;
+            default:
+                std::cerr << "Unexpected test result code: " << static_cast<int>(result) << "\n";
+                return false;
+        }
+    }
+
+    static void PrintTestStats()
+    {
+        uint32_t crashedCount =
+            sTestCount - (sPassedTestCount + sFailedTestCount + sNotSupportedTestCount +
+                          sTestExceptionCount + sSkippedTestCount);
+
+        std::cout << "Total: " << sTestCount << " tests.\n";
+        std::cout << "Passed: " << sPassedTestCount << " tests.\n";
+        std::cout << "Failed: " << sFailedTestCount << " tests.\n";
+        std::cout << "Skipped: " << sSkippedTestCount << " tests.\n";
+        std::cout << "Not Supported: " << sNotSupportedTestCount << " tests.\n";
+        std::cout << "Exception: " << sTestExceptionCount << " tests.\n";
+        std::cout << "Crashed: " << crashedCount << " tests.\n";
+
+        if (!gUnexpectedPasses.empty())
         {
-            sFails++;
+            std::cout << "\nSome tests unexpectedly passed:\n";
+            for (const std::string &testName : gUnexpectedPasses)
+            {
+                std::cout << testName << " unexpectedly passed.\n";
+            }
         }
 
-        if (result == TestResult::Exception)
+        if (!gUnexpectedFailed.empty())
         {
-            sExceptions++;
+            std::cout << "\nSome tests unexpectedly failed:\n";
+            for (const std::string &testName : gUnexpectedFailed)
+            {
+                std::cout << testName << " unexpectedly failed.\n";
+            }
         }
     }
 
-    static unsigned int sPasses;
-    static unsigned int sFails;
-    static unsigned int sUnexpectedPasses;
-    static unsigned int sExceptions;
+    static uint32_t sTestCount;
+    static uint32_t sPassedTestCount;
+    static uint32_t sFailedTestCount;
+    static uint32_t sTestExceptionCount;
+    static uint32_t sNotSupportedTestCount;
+    static uint32_t sSkippedTestCount;
 };
 
 template <size_t TestModuleIndex>
-unsigned int dEQPTest<TestModuleIndex>::sPasses = 0;
+uint32_t dEQPTest<TestModuleIndex>::sTestCount = 0;
 template <size_t TestModuleIndex>
-unsigned int dEQPTest<TestModuleIndex>::sFails = 0;
+uint32_t dEQPTest<TestModuleIndex>::sPassedTestCount = 0;
 template <size_t TestModuleIndex>
-unsigned int dEQPTest<TestModuleIndex>::sUnexpectedPasses = 0;
+uint32_t dEQPTest<TestModuleIndex>::sFailedTestCount = 0;
 template <size_t TestModuleIndex>
-unsigned int dEQPTest<TestModuleIndex>::sExceptions = 0;
+uint32_t dEQPTest<TestModuleIndex>::sTestExceptionCount = 0;
+template <size_t TestModuleIndex>
+uint32_t dEQPTest<TestModuleIndex>::sNotSupportedTestCount = 0;
+template <size_t TestModuleIndex>
+uint32_t dEQPTest<TestModuleIndex>::sSkippedTestCount = 0;
 
 // static
 template <size_t TestModuleIndex>
 void dEQPTest<TestModuleIndex>::SetUpTestCase()
 {
-    sPasses           = 0;
-    sFails            = 0;
-    sUnexpectedPasses = 0;
+    sPassedTestCount       = 0;
+    sFailedTestCount       = 0;
+    sNotSupportedTestCount = 0;
+    sTestExceptionCount    = 0;
+    sTestCount             = 0;
+    sSkippedTestCount      = 0;
+    gUnexpectedPasses.clear();
+    gUnexpectedFailed.clear();
 
     std::vector<const char *> argv;
 
@@ -440,21 +490,7 @@ void dEQPTest<TestModuleIndex>::SetUpTestCase()
 template <size_t TestModuleIndex>
 void dEQPTest<TestModuleIndex>::TearDownTestCase()
 {
-    unsigned int total = sPasses + sFails;
-    float passFrac     = static_cast<float>(sPasses) / static_cast<float>(total) * 100.0f;
-    float failFrac     = static_cast<float>(sFails) / static_cast<float>(total) * 100.0f;
-    std::cout << "Passed: " << sPasses << "/" << total << " tests. (" << passFrac << "%)"
-              << std::endl;
-    if (sFails > 0)
-    {
-        std::cout << "Failed: " << sFails << "/" << total << " tests. (" << failFrac << "%)"
-                  << std::endl;
-    }
-    if (sUnexpectedPasses > 0)
-    {
-        std::cout << sUnexpectedPasses << " tests unexpectedly passed." << std::endl;
-    }
-
+    PrintTestStats();
     deqp_libtester_shutdown_platform();
 }
 
@@ -542,7 +578,6 @@ void DeleteArg(int *argc, int argIndex, char **argv)
         argv[moveIndex] = argv[moveIndex + 1];
     }
 }
-
 }  // anonymous namespace
 
 // Called from main() to process command-line arguments.