Add option to run each test config in a separate process.

src/common/system_utils.h

@@ -28,6 +28,8 @@ bool PrependPathToEnvironmentVar(const char *variableName, const char *path);
 // Run an application and get the output.  Gets a nullptr-terminated set of args to execute the
 // application with, and returns the stdout and stderr outputs as well as the exit code.
 //
+// Pass nullptr for stdoutOut/stderrOut if you don't need to capture. exitCodeOut is required.
+//
 // Returns false if it fails to actually execute the application.
 bool RunApp(const std::vector<const char *> &args,
             std::string *stdoutOut,

src/common/system_utils_win.cpp

@@ -193,7 +193,11 @@ bool RunApp(const std::vector<const char *> &args,
     {
         startInfo.hStdError = GetStdHandle(STD_ERROR_HANDLE);
     }
-    startInfo.dwFlags |= STARTF_USESTDHANDLES;
+
+    if (stderrOut || stdoutOut)
+    {
+        startInfo.dwFlags |= STARTF_USESTDHANDLES;
+    }
 
     // Create the child process.
     PROCESS_INFORMATION processInfo = {};

src/tests/test_utils/angle_test_instantiate.cpp

@@ -67,9 +67,12 @@ bool IsNativeConfigSupported(const PlatformParameters &param, OSWindow *osWindow
     // Not yet implemented.
     return false;
 }
+
+std::map<PlatformParameters, bool> gParamAvailabilityCache;
 }  // namespace
 
 std::string gSelectedConfig;
+bool gSeparateProcessPerConfig = false;
 
 SystemInfo *GetTestSystemInfo()
 {
@@ -84,7 +87,8 @@ SystemInfo *GetTestSystemInfo()
 
         // Print complete system info when available.
         // Seems to trip up Android test expectation parsing.
-        if (!IsAndroid())
+        // Also don't print info when a config is selected to prevent test spam.
+        if (!IsAndroid() && gSelectedConfig.empty())
         {
             PrintSystemInfo(*sSystemInfo);
         }
@@ -408,48 +412,80 @@ bool IsPlatformAvailable(const PlatformParameters &param)
             return false;
     }
 
-    static std::map<PlatformParameters, bool> paramAvailabilityCache;
-    auto iter = paramAvailabilityCache.find(param);
-    if (iter != paramAvailabilityCache.end())
-    {
-        return iter->second;
-    }
-
     bool result = false;
 
-    if (!gSelectedConfig.empty())
+    auto iter = gParamAvailabilityCache.find(param);
+    if (iter != gParamAvailabilityCache.end())
     {
-        std::stringstream strstr;
-        strstr << param;
-        if (strstr.str() == gSelectedConfig)
-        {
-            result = true;
-        }
+        result = iter->second;
     }
     else
     {
-        const SystemInfo *systemInfo = GetTestSystemInfo();
-
-        if (systemInfo)
+        if (!gSelectedConfig.empty())
         {
-            result = IsConfigWhitelisted(*systemInfo, param);
+            std::stringstream strstr;
+            strstr << param;
+            if (strstr.str() == gSelectedConfig)
+            {
+                result = true;
+            }
         }
         else
         {
-            result = IsConfigSupported(param);
+            const SystemInfo *systemInfo = GetTestSystemInfo();
+
+            if (systemInfo)
+            {
+                result = IsConfigWhitelisted(*systemInfo, param);
+            }
+            else
+            {
+                result = IsConfigSupported(param);
+            }
         }
+
+        gParamAvailabilityCache[param] = result;
+
+        // Enable this unconditionally to print available platforms.
+        if (!gSelectedConfig.empty())
+        {
+            if (result)
+            {
+                std::cout << "Test Config: " << param << "\n";
+            }
+        }
+        else if (!result)
+        {
+            std::cout << "Skipping tests using configuration " << param
+                      << " because it is not available.\n";
+        }
+    }
+
+    // Disable all tests in the parent process when running child processes.
+    if (gSeparateProcessPerConfig)
+    {
+        return false;
     }
+    return result;
+}
 
-    paramAvailabilityCache[param] = result;
+std::vector<std::string> GetAvailableTestPlatformNames()
+{
+    std::vector<std::string> platformNames;
 
-    if (!result)
+    for (const auto &iter : gParamAvailabilityCache)
     {
-        std::cout << "Skipping tests using configuration " << param
-                  << " because it is not available." << std::endl;
+        if (iter.second)
+        {
+            std::stringstream strstr;
+            strstr << iter.first;
+            platformNames.push_back(strstr.str());
+        }
     }
 
-    // Uncomment this to print available platforms.
-    // std::cout << "Platform: " << param << " (" << paramAvailabilityCache.size() << ")\n";
-    return result;
+    // Keep the list sorted.
+    std::sort(platformNames.begin(), platformNames.end());
+
+    return platformNames;
 }
 }  // namespace angle

src/tests/test_utils/angle_test_instantiate.h

@@ -122,8 +122,15 @@ bool IsConfigSupported(const PlatformParameters &param);
 // Returns shared test system information. Can be used globally in the tests.
 SystemInfo *GetTestSystemInfo();
 
+// Returns a list of all enabled test platform names. For use in configuration enumeration.
+std::vector<std::string> GetAvailableTestPlatformNames();
+
 // Active config (e.g. ES2_Vulkan).
 extern std::string gSelectedConfig;
+
+// Use a separate isolated process per test config. This works around driver flakiness when using
+// multiple APIs/windows/etc in the same process.
+extern bool gSeparateProcessPerConfig;
 }  // namespace angle
 
 #endif  // ANGLE_TEST_INSTANTIATE_H_