Add option to run each test config in a separate process.

src/common/system_utils.h

@@ -28,6 +28,8 @@ bool PrependPathToEnvironmentVar(const char *variableName, const char *path);
 // Run an application and get the output.  Gets a nullptr-terminated set of args to execute the
 // application with, and returns the stdout and stderr outputs as well as the exit code.
 //
+// Pass nullptr for stdoutOut/stderrOut if you don't need to capture. exitCodeOut is required.
+//
 // Returns false if it fails to actually execute the application.
 bool RunApp(const std::vector<const char *> &args,
             std::string *stdoutOut,

src/common/system_utils_win.cpp

@@ -193,7 +193,11 @@ bool RunApp(const std::vector<const char *> &args,
     {
         startInfo.hStdError = GetStdHandle(STD_ERROR_HANDLE);
     }
-    startInfo.dwFlags |= STARTF_USESTDHANDLES;
+
+    if (stderrOut || stdoutOut)
+    {
+        startInfo.dwFlags |= STARTF_USESTDHANDLES;
+    }
 
     // Create the child process.
     PROCESS_INFORMATION processInfo = {};

src/tests/test_utils/ANGLETest.cpp

@@ -285,36 +285,85 @@ GLColor32F ReadColor32F(GLint x, GLint y)
 }
 }  // namespace angle
 
+using namespace angle;
+
 namespace
 {
-angle::PlatformMethods gDefaultPlatformMethods;
+PlatformMethods gDefaultPlatformMethods;
 TestPlatformContext gPlatformContext;
 
 // After a fixed number of iterations we reset the test window. This works around some driver bugs.
 constexpr uint32_t kWindowReuseLimit = 50;
 
-constexpr char kUseConfig[] = "--use-config=";
+constexpr char kUseConfig[]                = "--use-config=";
+constexpr char kSeparateProcessPerConfig[] = "--separate-process-per-config";
+
+bool RunSeparateProcessesForEachConfig(int *argc, char *argv[])
+{
+    std::vector<const char *> commonArgs;
+    for (int argIndex = 0; argIndex < *argc; ++argIndex)
+    {
+        if (strncmp(argv[argIndex], kSeparateProcessPerConfig, strlen(kSeparateProcessPerConfig)) !=
+            0)
+        {
+            commonArgs.push_back(argv[argIndex]);
+        }
+    }
+
+    // Force GoogleTest init now so that we hit the test config init in angle_test_instantiate.cpp.
+    // After instantiation is finished we can gather a full list of enabled configs. Then we can
+    // iterate the list of configs to spawn a child process for each enabled config.
+    testing::InitGoogleTest(argc, argv);
+
+    std::vector<std::string> configNames = GetAvailableTestPlatformNames();
+
+    bool success = true;
+
+    for (const std::string &config : configNames)
+    {
+        std::stringstream strstr;
+        strstr << kUseConfig << config;
+
+        std::string configStr = strstr.str();
+
+        std::vector<const char *> childArgs = commonArgs;
+        childArgs.push_back(configStr.c_str());
+
+        int exitCode = 0;
+        if (!RunApp(childArgs, nullptr, nullptr, &exitCode))
+        {
+            std::cerr << "Launching child config " << config << " failed.\n";
+        }
+        else if (exitCode != 0)
+        {
+            std::cerr << "Child config " << config << " failed with exit code " << exitCode
+                      << ".\n";
+            success = false;
+        }
+    }
+    return success;
+}
 }  // anonymous namespace
 
 // static
-std::array<angle::Vector3, 6> ANGLETestBase::GetQuadVertices()
+std::array<Vector3, 6> ANGLETestBase::GetQuadVertices()
 {
-    return angle::kQuadVertices;
+    return kQuadVertices;
 }
 
 // static
 std::array<GLushort, 6> ANGLETestBase::GetQuadIndices()
 {
-    return angle::kIndexedQuadIndices;
+    return kIndexedQuadIndices;
 }
 
 // static
-std::array<angle::Vector3, 4> ANGLETestBase::GetIndexedQuadVertices()
+std::array<Vector3, 4> ANGLETestBase::GetIndexedQuadVertices()
 {
-    return angle::kIndexedQuadVertices;
+    return kIndexedQuadVertices;
 }
 
-ANGLETestBase::ANGLETestBase(const angle::PlatformParameters &params)
+ANGLETestBase::ANGLETestBase(const PlatformParameters &params)
     : mWidth(16),
       mHeight(16),
       mIgnoreD3D11SDKLayersWarnings(false),
@@ -323,13 +372,13 @@ ANGLETestBase::ANGLETestBase(const angle::PlatformParameters &params)
       m2DTexturedQuadProgram(0),
       m3DTexturedQuadProgram(0),
       mDeferContextInit(false),
-      mAlwaysForceNewDisplay(angle::ShouldAlwaysForceNewDisplay()),
+      mAlwaysForceNewDisplay(ShouldAlwaysForceNewDisplay()),
       mForceNewDisplay(mAlwaysForceNewDisplay),
       mCurrentParams(nullptr),
       mFixture(nullptr)
 {
     // Override the default platform methods with the ANGLE test methods pointer.
-    angle::PlatformParameters withMethods     = params;
+    PlatformParameters withMethods            = params;
     withMethods.eglParameters.platformMethods = &gDefaultPlatformMethods;
 
     auto iter = gFixtures.find(withMethods);
@@ -379,24 +428,24 @@ void ANGLETestBase::initOSWindow()
     }
 
     // On Linux we must keep the test windows visible. On Windows it doesn't seem to need it.
-    mFixture->osWindow->setVisible(!angle::IsWindows());
+    mFixture->osWindow->setVisible(!IsWindows());
 
     switch (mCurrentParams->driver)
     {
-        case angle::GLESDriverType::AngleEGL:
+        case GLESDriverType::AngleEGL:
         {
             mFixture->eglWindow =
                 EGLWindow::New(mCurrentParams->majorVersion, mCurrentParams->minorVersion);
             break;
         }
 
-        case angle::GLESDriverType::SystemEGL:
+        case GLESDriverType::SystemEGL:
         {
             std::cerr << "Unsupported driver." << std::endl;
             break;
         }
 
-        case angle::GLESDriverType::SystemWGL:
+        case GLESDriverType::SystemWGL:
         {
             // WGL tests are currently disabled.
             std::cerr << "Unsupported driver." << std::endl;
@@ -427,21 +476,21 @@ ANGLETestBase::~ANGLETestBase()
 
 void ANGLETestBase::ANGLETestSetUp()
 {
-    gDefaultPlatformMethods.overrideWorkaroundsD3D = angle::TestPlatform_overrideWorkaroundsD3D;
-    gDefaultPlatformMethods.overrideFeaturesVk     = angle::TestPlatform_overrideFeaturesVk;
-    gDefaultPlatformMethods.logError               = angle::TestPlatform_logError;
-    gDefaultPlatformMethods.logWarning             = angle::TestPlatform_logWarning;
-    gDefaultPlatformMethods.logInfo                = angle::TestPlatform_logInfo;
+    gDefaultPlatformMethods.overrideWorkaroundsD3D = TestPlatform_overrideWorkaroundsD3D;
+    gDefaultPlatformMethods.overrideFeaturesVk     = TestPlatform_overrideFeaturesVk;
+    gDefaultPlatformMethods.logError               = TestPlatform_logError;
+    gDefaultPlatformMethods.logWarning             = TestPlatform_logWarning;
+    gDefaultPlatformMethods.logInfo                = TestPlatform_logInfo;
     gDefaultPlatformMethods.context                = &gPlatformContext;
 
     gPlatformContext.ignoreMessages   = false;
     gPlatformContext.warningsAsErrors = false;
     gPlatformContext.currentTest      = this;
 
-    if (angle::IsWindows())
+    if (IsWindows())
     {
         const auto &info = testing::UnitTest::GetInstance()->current_test_info();
-        angle::WriteDebugMessage("Entering %s.%s\n", info->test_case_name(), info->name());
+        WriteDebugMessage("Entering %s.%s\n", info->test_case_name(), info->name());
     }
 
     if (mCurrentParams->noFixture)
@@ -451,8 +500,8 @@ void ANGLETestBase::ANGLETestSetUp()
         ANGLETestEnvironment::GetEGLLibrary()->getAs("eglGetProcAddress", &getProcAddress);
         ASSERT_NE(nullptr, getProcAddress);
 
-        angle::LoadEGL(getProcAddress);
-        angle::LoadGLES(getProcAddress);
+        LoadEGL(getProcAddress);
+        LoadGLES(getProcAddress);
 #endif  // defined(ANGLE_USE_UTIL_LOADER)
         return;
     }
@@ -521,10 +570,10 @@ void ANGLETestBase::ANGLETestTearDown()
 {
     gPlatformContext.currentTest = nullptr;
 
-    if (angle::IsWindows())
+    if (IsWindows())
     {
         const testing::TestInfo *info = testing::UnitTest::GetInstance()->current_test_info();
-        angle::WriteDebugMessage("Exiting %s.%s\n", info->test_case_name(), info->name());
+        WriteDebugMessage("Exiting %s.%s\n", info->test_case_name(), info->name());
     }
 
     if (mCurrentParams->noFixture)
@@ -583,7 +632,7 @@ void ANGLETestBase::setupQuadVertexBuffer(GLfloat positionAttribZ, GLfloat posit
     }
 
     auto quadVertices = GetQuadVertices();
-    for (angle::Vector3 &vertex : quadVertices)
+    for (Vector3 &vertex : quadVertices)
     {
         vertex.x() *= positionAttribXYScale;
         vertex.y() *= positionAttribXYScale;
@@ -602,8 +651,8 @@ void ANGLETestBase::setupIndexedQuadVertexBuffer(GLfloat positionAttribZ,
         glGenBuffers(1, &mQuadVertexBuffer);
     }
 
-    auto quadVertices = angle::kIndexedQuadVertices;
-    for (angle::Vector3 &vertex : quadVertices)
+    auto quadVertices = kIndexedQuadVertices;
+    for (Vector3 &vertex : quadVertices)
     {
         vertex.x() *= positionAttribXYScale;
         vertex.y() *= positionAttribXYScale;
@@ -622,8 +671,8 @@ void ANGLETestBase::setupIndexedQuadIndexBuffer()
     }
 
     glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, mQuadIndexBuffer);
-    glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(angle::kIndexedQuadIndices),
-                 angle::kIndexedQuadIndices.data(), GL_STATIC_DRAW);
+    glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(kIndexedQuadIndices), kIndexedQuadIndices.data(),
+                 GL_STATIC_DRAW);
 }
 
 // static
@@ -684,7 +733,7 @@ void ANGLETestBase::drawQuad(GLuint program,
 
     GLint positionLocation = glGetAttribLocation(program, positionAttribName.c_str());
 
-    std::array<angle::Vector3, 6> quadVertices;
+    std::array<Vector3, 6> quadVertices;
 
     if (useVertexBuffer)
     {
@@ -695,7 +744,7 @@ void ANGLETestBase::drawQuad(GLuint program,
     else
     {
         quadVertices = GetQuadVertices();
-        for (angle::Vector3 &vertex : quadVertices)
+        for (Vector3 &vertex : quadVertices)
         {
             vertex.x() *= positionAttribXYScale;
             vertex.y() *= positionAttribXYScale;
@@ -794,7 +843,7 @@ void ANGLETestBase::drawIndexedQuad(GLuint program,
     }
     else
     {
-        indices = angle::kIndexedQuadIndices.data();
+        indices = kIndexedQuadIndices.data();
     }
 
     if (!restrictedRange)
@@ -1096,8 +1145,7 @@ void ANGLETestBase::setRobustResourceInit(bool enabled)
     mFixture->configParams.robustResourceInit = enabled;
 }
 
-void ANGLETestBase::setContextProgramCacheEnabled(bool enabled,
-                                                  angle::CacheProgramFunc cacheProgramFunc)
+void ANGLETestBase::setContextProgramCacheEnabled(bool enabled, CacheProgramFunc cacheProgramFunc)
 {
     mFixture->configParams.contextProgramCacheEnabled = enabled;
     gDefaultPlatformMethods.cacheProgram              = cacheProgramFunc;
@@ -1284,30 +1332,30 @@ OSWindow *ANGLETestBase::mOSWindowSingleton = nullptr;
 std::map<angle::PlatformParameters, ANGLETestBase::TestFixture> ANGLETestBase::gFixtures;
 Optional<EGLint> ANGLETestBase::mLastRendererType;
 
-std::unique_ptr<angle::Library> ANGLETestEnvironment::gEGLLibrary;
-std::unique_ptr<angle::Library> ANGLETestEnvironment::gWGLLibrary;
+std::unique_ptr<Library> ANGLETestEnvironment::gEGLLibrary;
+std::unique_ptr<Library> ANGLETestEnvironment::gWGLLibrary;
 
 void ANGLETestEnvironment::SetUp() {}
 
 void ANGLETestEnvironment::TearDown() {}
 
-angle::Library *ANGLETestEnvironment::GetEGLLibrary()
+Library *ANGLETestEnvironment::GetEGLLibrary()
 {
 #if defined(ANGLE_USE_UTIL_LOADER)
     if (!gEGLLibrary)
     {
-        gEGLLibrary.reset(angle::OpenSharedLibrary(ANGLE_EGL_LIBRARY_NAME));
+        gEGLLibrary.reset(OpenSharedLibrary(ANGLE_EGL_LIBRARY_NAME));
     }
 #endif  // defined(ANGLE_USE_UTIL_LOADER)
     return gEGLLibrary.get();
 }
 
-angle::Library *ANGLETestEnvironment::GetWGLLibrary()
+Library *ANGLETestEnvironment::GetWGLLibrary()
 {
 #if defined(ANGLE_USE_UTIL_LOADER) && defined(ANGLE_PLATFORM_WINDOWS)
     if (!gWGLLibrary)
     {
-        gWGLLibrary.reset(angle::OpenSharedLibrary("opengl32"));
+        gWGLLibrary.reset(OpenSharedLibrary("opengl32"));
     }
 #endif  // defined(ANGLE_USE_UTIL_LOADER) && defined(ANGLE_PLATFORM_WINDOWS)
     return gWGLLibrary.get();
@@ -1321,7 +1369,31 @@ void ANGLEProcessTestArgs(int *argc, char *argv[])
     {
         if (strncmp(argv[argIndex], kUseConfig, strlen(kUseConfig)) == 0)
         {
-            angle::gSelectedConfig = std::string(argv[argIndex] + strlen(kUseConfig));
+            gSelectedConfig = std::string(argv[argIndex] + strlen(kUseConfig));
+        }
+        if (strncmp(argv[argIndex], kSeparateProcessPerConfig, strlen(kSeparateProcessPerConfig)) ==
+            0)
+        {
+            gSeparateProcessPerConfig = true;
+        }
+    }
+
+    if (gSeparateProcessPerConfig)
+    {
+        if (!gSelectedConfig.empty())
+        {
+            std::cout << "Cannot use both a single test config and separate processes.\n";
+            exit(1);
+        }
+
+        if (RunSeparateProcessesForEachConfig(argc, argv))
+        {
+            exit(0);
+        }
+        else
+        {
+            std::cout << "Some subprocesses failed.\n";
+            exit(1);
         }
     }
 }

src/tests/test_utils/angle_test_instantiate.cpp

@@ -67,9 +67,12 @@ bool IsNativeConfigSupported(const PlatformParameters &param, OSWindow *osWindow
     // Not yet implemented.
     return false;
 }
+
+std::map<PlatformParameters, bool> gParamAvailabilityCache;
 }  // namespace
 
 std::string gSelectedConfig;
+bool gSeparateProcessPerConfig = false;
 
 SystemInfo *GetTestSystemInfo()
 {
@@ -84,7 +87,8 @@ SystemInfo *GetTestSystemInfo()
 
         // Print complete system info when available.
         // Seems to trip up Android test expectation parsing.
-        if (!IsAndroid())
+        // Also don't print info when a config is selected to prevent test spam.
+        if (!IsAndroid() && gSelectedConfig.empty())
         {
             PrintSystemInfo(*sSystemInfo);
         }
@@ -408,48 +412,80 @@ bool IsPlatformAvailable(const PlatformParameters &param)
             return false;
     }
 
-    static std::map<PlatformParameters, bool> paramAvailabilityCache;
-    auto iter = paramAvailabilityCache.find(param);
-    if (iter != paramAvailabilityCache.end())
-    {
-        return iter->second;
-    }
-
     bool result = false;
 
-    if (!gSelectedConfig.empty())
+    auto iter = gParamAvailabilityCache.find(param);
+    if (iter != gParamAvailabilityCache.end())
     {
-        std::stringstream strstr;
-        strstr << param;
-        if (strstr.str() == gSelectedConfig)
-        {
-            result = true;
-        }
+        result = iter->second;
     }
     else
     {
-        const SystemInfo *systemInfo = GetTestSystemInfo();
-
-        if (systemInfo)
+        if (!gSelectedConfig.empty())
         {
-            result = IsConfigWhitelisted(*systemInfo, param);
+            std::stringstream strstr;
+            strstr << param;
+            if (strstr.str() == gSelectedConfig)
+            {
+                result = true;
+            }
         }
         else
         {
-            result = IsConfigSupported(param);
+            const SystemInfo *systemInfo = GetTestSystemInfo();
+
+            if (systemInfo)
+            {
+                result = IsConfigWhitelisted(*systemInfo, param);
+            }
+            else
+            {
+                result = IsConfigSupported(param);
+            }
         }
+
+        gParamAvailabilityCache[param] = result;
+
+        // Enable this unconditionally to print available platforms.
+        if (!gSelectedConfig.empty())
+        {
+            if (result)
+            {
+                std::cout << "Test Config: " << param << "\n";
+            }
+        }
+        else if (!result)
+        {
+            std::cout << "Skipping tests using configuration " << param
+                      << " because it is not available.\n";
+        }
+    }
+
+    // Disable all tests in the parent process when running child processes.
+    if (gSeparateProcessPerConfig)
+    {
+        return false;
     }
+    return result;
+}
 
-    paramAvailabilityCache[param] = result;
+std::vector<std::string> GetAvailableTestPlatformNames()
+{
+    std::vector<std::string> platformNames;
 
-    if (!result)
+    for (const auto &iter : gParamAvailabilityCache)
     {
-        std::cout << "Skipping tests using configuration " << param
-                  << " because it is not available." << std::endl;
+        if (iter.second)
+        {
+            std::stringstream strstr;
+            strstr << iter.first;
+            platformNames.push_back(strstr.str());
+        }
     }
 
-    // Uncomment this to print available platforms.
-    // std::cout << "Platform: " << param << " (" << paramAvailabilityCache.size() << ")\n";
-    return result;
+    // Keep the list sorted.
+    std::sort(platformNames.begin(), platformNames.end());
+
+    return platformNames;
 }
 }  // namespace angle

src/tests/test_utils/angle_test_instantiate.h

@@ -122,8 +122,15 @@ bool IsConfigSupported(const PlatformParameters &param);
 // Returns shared test system information. Can be used globally in the tests.
 SystemInfo *GetTestSystemInfo();
 
+// Returns a list of all enabled test platform names. For use in configuration enumeration.
+std::vector<std::string> GetAvailableTestPlatformNames();
+
 // Active config (e.g. ES2_Vulkan).
 extern std::string gSelectedConfig;
+
+// Use a separate isolated process per test config. This works around driver flakiness when using
+// multiple APIs/windows/etc in the same process.
+extern bool gSeparateProcessPerConfig;
 }  // namespace angle
 
 #endif  // ANGLE_TEST_INSTANTIATE_H_