Log dEQP QPA files as test artifacts.

scripts/run_gtest_angle_test.py

@@ -78,6 +78,9 @@ def main():
     if 'GTEST_SHARD_INDEX' in env:
         extra_flags += ['--shard-index=' + env['GTEST_SHARD_INDEX']]
         env.pop('GTEST_SHARD_INDEX')
+    if 'ISOLATED_OUTDIR' in env:
+        extra_flags += ['--isolated-outdir=' + env['ISOLATED_OUTDIR']]
+        env.pop('ISOLATED_OUTDIR')
 
     # Assume we want to set up the sandbox environment variables all the
     # time; doing so is harmless on non-Linux platforms and is needed

src/tests/deqp_support/angle_deqp_gtest.cpp

@@ -23,6 +23,7 @@
 #include "platform/PlatformMethods.h"
 #include "tests/test_expectations/GPUTestConfig.h"
 #include "tests/test_expectations/GPUTestExpectationsParser.h"
+#include "tests/test_utils/runner/TestSuite.h"
 #include "util/OSWindow.h"
 #include "util/test_utils.h"
 
@@ -30,11 +31,19 @@ namespace angle
 {
 namespace
 {
+#if !defined(NDEBUG)
+constexpr bool kIsDebug = true;
+#else
+constexpr bool kIsDebug                = false;
+#endif  // !defined(NDEBUG)
+
 bool gGlobalError = false;
 bool gExpectError = false;
-uint32_t gBatchId = 0;
 bool gVerbose     = false;
 
+// Set this to true temporarily to enable image logging in release. Useful for diagnosing errors.
+bool gLogImages = kIsDebug;
+
 constexpr char kInfoTag[] = "*RESULT";
 
 void HandlePlatformError(PlatformMethods *platform, const char *errorMessage)
@@ -108,7 +117,6 @@ constexpr char kdEQPEGLString[]    = "--deqp-egl-display-type=";
 constexpr char kANGLEEGLString[]   = "--use-angle=";
 constexpr char kANGLEPreRotation[] = "--emulated-pre-rotation=";
 constexpr char kdEQPCaseString[]   = "--deqp-case=";
-constexpr char kBatchIdString[]    = "--batch-id=";
 constexpr char kVerboseString[]    = "--verbose";
 
 std::array<char, 500> gCaseStringBuffer;
@@ -127,7 +135,8 @@ constexpr uint32_t kDefaultPreRotation = 0;
 const APIInfo *gInitAPI = nullptr;
 uint32_t gPreRotation   = kDefaultPreRotation;
 
-constexpr const char *gdEQPEGLConfigNameString = "--deqp-gl-config-name=";
+constexpr const char gdEQPEGLConfigNameString[] = "--deqp-gl-config-name=";
+constexpr const char gdEQPLogImagesString[]     = "--deqp-log-images=";
 
 // Default the config to RGBA8
 const char *gEGLConfigName = "rgba8888d24s8";
@@ -386,8 +395,8 @@ class dEQPTest : public testing::TestWithParam<size_t>
             return;
         }
 
-        gExpectError      = (caseInfo.mExpectation != GPUTestExpectationsParser::kGpuTestPass);
-        TestResult result = deqp_libtester_run(caseInfo.mDEQPName.c_str());
+        gExpectError          = (caseInfo.mExpectation != GPUTestExpectationsParser::kGpuTestPass);
+        dEQPTestResult result = deqp_libtester_run(caseInfo.mDEQPName.c_str());
 
         bool testSucceeded = countTestResultAndReturnSuccess(result);
 
@@ -414,20 +423,20 @@ class dEQPTest : public testing::TestWithParam<size_t>
         }
     }
 
-    bool countTestResultAndReturnSuccess(TestResult result) const
+    bool countTestResultAndReturnSuccess(dEQPTestResult result) const
     {
         switch (result)
         {
-            case TestResult::Pass:
+            case dEQPTestResult::Pass:
                 sPassedTestCount++;
                 return true;
-            case TestResult::Fail:
+            case dEQPTestResult::Fail:
                 sFailedTestCount++;
                 return false;
-            case TestResult::NotSupported:
+            case dEQPTestResult::NotSupported:
                 sNotSupportedTestCount++;
                 return true;
-            case TestResult::Exception:
+            case dEQPTestResult::Exception:
                 sTestExceptionCount++;
                 return false;
             default:
@@ -533,16 +542,30 @@ void dEQPTest<TestModuleIndex>::SetUpTestCase()
         argv.push_back("--deqp-visibility=hidden");
     }
 
-    std::string logNameString;
-    if (gBatchId != 0)
+    TestSuite *testSuite = TestSuite::GetInstance();
+
+    std::stringstream logNameStream;
+    logNameStream << "TestResults";
+    if (testSuite->getBatchId() != -1)
     {
-        std::stringstream logNameStream;
-        logNameStream << "--deqp-log-filename=test-results-batch-" << std::setfill('0')
-                      << std::setw(3) << gBatchId << ".qpa";
-        logNameString = logNameStream.str();
-        argv.push_back(logNameString.c_str());
+        logNameStream << "-Batch" << std::setfill('0') << std::setw(3) << testSuite->getBatchId();
+    }
+    logNameStream << ".qpa";
+
+    std::stringstream logArgStream;
+    logArgStream << "--deqp-log-filename=" << testSuite->addTestArtifact(logNameStream.str());
 
-        // Flushing during multi-process execution punishes HDDs. http://anglebug.com/5157
+    std::string logNameString = logArgStream.str();
+    argv.push_back(logNameString.c_str());
+
+    if (!gLogImages)
+    {
+        argv.push_back("--deqp-log-images=disable");
+    }
+
+    // Flushing during multi-process execution punishes HDDs. http://anglebug.com/5157
+    if (testSuite->getBatchId() != -1)
+    {
         argv.push_back("--deqp-log-flush=disable");
     }
 
@@ -697,10 +720,21 @@ void HandleCaseName(const char *caseString, int *argc, int argIndex, char **argv
     argv[argIndex] = gCaseStringBuffer.data();
 }
 
-void HandleBatchId(const char *batchIdString)
+void HandleLogImages(const char *logImagesString)
 {
-    std::stringstream batchIdStream(batchIdString);
-    batchIdStream >> gBatchId;
+    if (strcmp(logImagesString, "enable") == 0)
+    {
+        gLogImages = true;
+    }
+    else if (strcmp(logImagesString, "disable") == 0)
+    {
+        gLogImages = false;
+    }
+    else
+    {
+        std::cout << "Error parsing log images setting. Use enable/disable.";
+        exit(1);
+    }
 }
 }  // anonymous namespace
 
@@ -731,15 +765,15 @@ void InitTestHarness(int *argc, char **argv)
         {
             HandleCaseName(argv[argIndex] + strlen(kdEQPCaseString), argc, argIndex, argv);
         }
-        else if (strncmp(argv[argIndex], kBatchIdString, strlen(kBatchIdString)) == 0)
-        {
-            HandleBatchId(argv[argIndex] + strlen(kBatchIdString));
-        }
         else if (strncmp(argv[argIndex], kVerboseString, strlen(kVerboseString)) == 0 ||
                  strcmp(argv[argIndex], "-v") == 0)
         {
             gVerbose = true;
         }
+        else if (strncmp(argv[argIndex], gdEQPLogImagesString, strlen(gdEQPLogImagesString)) == 0)
+        {
+            HandleLogImages(argv[argIndex] + strlen(gdEQPLogImagesString));
+        }
         argIndex++;
     }
 

src/tests/deqp_support/angle_deqp_libtester.h

@@ -29,7 +29,7 @@
 #endif
 
 // Possible results of deqp_libtester_run
-enum class TestResult
+enum class dEQPTestResult
 {
     Pass,
     Fail,
@@ -44,6 +44,6 @@ ANGLE_LIBTESTER_EXPORT bool deqp_libtester_init_platform(int argc,
                                                          void *logErrorFunc,
                                                          uint32_t preRotation);
 ANGLE_LIBTESTER_EXPORT void deqp_libtester_shutdown_platform();
-ANGLE_LIBTESTER_EXPORT TestResult deqp_libtester_run(const char *caseName);
+ANGLE_LIBTESTER_EXPORT dEQPTestResult deqp_libtester_run(const char *caseName);
 
 #endif  // ANGLE_DEQP_LIBTESTER_H_

src/tests/deqp_support/angle_deqp_libtester_main.cpp

@@ -139,7 +139,7 @@ ANGLE_LIBTESTER_EXPORT void deqp_libtester_shutdown_platform()
     g_platform = nullptr;
 }
 
-ANGLE_LIBTESTER_EXPORT TestResult deqp_libtester_run(const char *caseName)
+ANGLE_LIBTESTER_EXPORT dEQPTestResult deqp_libtester_run(const char *caseName)
 {
     const char *emptyString = "";
     if (g_platform == nullptr)
@@ -158,18 +158,18 @@ ANGLE_LIBTESTER_EXPORT TestResult deqp_libtester_run(const char *caseName)
             switch (result.getCode())
             {
                 case QP_TEST_RESULT_PASS:
-                    return TestResult::Pass;
+                    return dEQPTestResult::Pass;
                 case QP_TEST_RESULT_NOT_SUPPORTED:
                     std::cout << "Not supported! " << result.getDescription() << std::endl;
-                    return TestResult::NotSupported;
+                    return dEQPTestResult::NotSupported;
                 case QP_TEST_RESULT_QUALITY_WARNING:
                     std::cout << "Quality warning! " << result.getDescription() << std::endl;
-                    return TestResult::Pass;
+                    return dEQPTestResult::Pass;
                 case QP_TEST_RESULT_COMPATIBILITY_WARNING:
                     std::cout << "Compatiblity warning! " << result.getDescription() << std::endl;
-                    return TestResult::Pass;
+                    return dEQPTestResult::Pass;
                 default:
-                    return TestResult::Fail;
+                    return dEQPTestResult::Fail;
             }
         }
         else
@@ -180,8 +180,8 @@ ANGLE_LIBTESTER_EXPORT TestResult deqp_libtester_run(const char *caseName)
     catch (const std::exception &e)
     {
         std::cout << "Exception running test: " << e.what() << std::endl;
-        return TestResult::Exception;
+        return dEQPTestResult::Exception;
     }
 
-    return TestResult::Fail;
+    return dEQPTestResult::Fail;
 }

src/tests/test_utils/runner/README.md

@@ -26,6 +26,7 @@ following additional command-line arguments:
  * `--test-timeout` limits the amount of time spent in each test
  * `--flaky-retries` allows for tests to fail a fixed number of times and still pass
  * `--disable-crash-handler` forces off OS-level crash handling
+ * `--isolated-outdir` specifies a test artifacts directory
 
 `--isolated-script-test-output` and `--isolated-script-perf-test-output` mirror `--results-file`
 and `--histogram-json-file` respectively.

src/tests/test_utils/runner/TestSuite.h

@@ -94,6 +94,8 @@ struct TestResults
     Timer currentTestTimer;
     double currentTestTimeout = 0.0;
     bool allDone              = false;
+    std::string testArtifactsFakeTestName;
+    std::vector<std::string> testArtifactPaths;
 };
 
 struct FileLine
@@ -135,6 +137,12 @@ class TestSuite
 
     static TestSuite *GetInstance() { return mInstance; }
 
+    // Returns the path to the artifact in the output directory.
+    std::string addTestArtifact(const std::string &artifactName);
+
+    int getShardIndex() const { return mShardIndex; }
+    int getBatchId() const { return mBatchId; }
+
   private:
     bool parseSingleArg(const char *argument);
     bool launchChildTestProcess(uint32_t batchId, const std::vector<TestIdentifier> &testsInBatch);
@@ -176,6 +184,7 @@ class TestSuite
     std::thread mWatchdogThread;
     HistogramWriter mHistogramWriter;
     std::vector<std::string> mSlowTests;
+    std::string mTestArtifactDirectory;
 };
 
 bool GetTestResultsFromFile(const char *fileName, TestResults *resultsOut);