Vulkan: parallelize internal shader compilation

scripts/run_code_generation_hashes.json

@@ -192,7 +192,7 @@
   "Vulkan format:src/libANGLE/renderer/vulkan/vk_format_table_autogen.cpp":
     "ef0ea80cf33e60f76391bcfed10b3c0a",
   "Vulkan internal shader programs:src/libANGLE/renderer/vulkan/gen_vk_internal_shaders.py":
-    "d9a4742e5cba2adefb9608e6439914c9",
+    "1262e5e903c7dad214ded83625f9d3c4",
   "Vulkan internal shader programs:src/libANGLE/renderer/vulkan/shaders/gen/BufferUtils.comp.00000000.inc":
     "caa03e84d757844a099d0e408a162c7e",
   "Vulkan internal shader programs:src/libANGLE/renderer/vulkan/shaders/gen/BufferUtils.comp.00000001.inc":

src/libANGLE/renderer/vulkan/gen_vk_internal_shaders.py

@@ -12,6 +12,7 @@
 from datetime import date
 import io
 import json
+import multiprocessing
 import os
 import platform
 import re
@@ -252,7 +253,116 @@ compact_newlines_regex = re.compile(r"\n\s*\n", re.MULTILINE)
 def cleanup_preprocessed_shader(shader_text):
     return compact_newlines_regex.sub('\n\n', shader_text.strip())
 
-def compile_variation(glslang_path, shader_file, shader_basename, flags, enums,
+class CompileQueue:
+    class AppendPreprocessorOutput:
+        def __init__(self, shader_file, preprocessor_args, output_path):
+            # Asynchronously launch the preprocessor job.
+            self.process = subprocess.Popen(preprocessor_args,
+                                            stdout=subprocess.PIPE,
+                                            stderr=subprocess.PIPE)
+            # Store the file name for output to be appended to.
+            self.output_path = output_path
+            # Store info for error description.
+            self.shader_file = shader_file
+
+        def wait(self, queue):
+            (out, err) = self.process.communicate()
+            if self.process.returncode == 0:
+                # Append preprocessor output to the output file.
+                with open(self.output_path, 'ab') as incfile:
+                    incfile.write('\n\n#if 0  // Generated from:\n')
+                    incfile.write(cleanup_preprocessed_shader(out.replace('\r\n', '\n')))
+                    incfile.write('\n#endif  // Preprocessed code\n')
+                out = None
+            return (out, err, self.process.returncode, None,
+                    "Error running preprocessor on " + self.shader_file)
+
+    class CompileToSPIRV:
+        def __init__(self, shader_file, shader_basename, variation_string, output_path,
+                     compile_args, preprocessor_args):
+            # Asynchronously launch the compile job.
+            self.process = subprocess.Popen(compile_args,
+                                            stdout=subprocess.PIPE,
+                                            stderr=subprocess.PIPE)
+            # Store info for launching the preprocessor.
+            self.preprocessor_args = preprocessor_args
+            self.output_path = output_path
+            # Store info for job and error description.
+            self.shader_file = shader_file
+            self.shader_basename = shader_basename
+            self.variation_string = variation_string
+
+        def wait(self, queue):
+            (out, err) = self.process.communicate()
+            if self.process.returncode == 0:
+                # Insert the preprocessor job in the queue.
+                queue.append(CompileQueue.AppendPreprocessorOutput(self.shader_file,
+                                                                   self.preprocessor_args,
+                                                                   self.output_path))
+            # If all the output says is the source file name, don't bother printing it.
+            if out.strip() == self.shader_file:
+                out = None
+            description = self.output_path + ': ' + self.shader_basename + self.variation_string
+            return (out, err, self.process.returncode, description,
+                    "Error compiling " + self.shader_file)
+
+    def __init__(self):
+        # Compile with as many CPU threads are detected.  Once a shader is compiled, another job is
+        # automatically added to the queue to append the preprocessor output to the generated file.
+        self.queue = []
+        self.thread_count = multiprocessing.cpu_count()
+
+    def _wait_first(self, ignore_output=False):
+        (out, err, returncode, description, exception_description) = self.queue[0].wait(self.queue)
+        self.queue.pop(0)
+        if not ignore_output:
+            if description:
+                print description
+            if out and out.strip():
+                print out.strip()
+            if err and err.strip():
+                print err
+            if returncode != 0:
+                return exception_description
+        return None
+
+    # Wait for all pending tasks.  If called after error is detected, ignore_output can be used to
+    # make sure errors in later jobs are suppressed to avoid cluttering the output.  This is
+    # because the same compile error is likely present in other variations of the same shader and
+    # outputting the same error multiple times is not useful.
+    def _wait_all(self, ignore_output=False):
+        exception_description = None
+        while len(self.queue) > 0:
+            this_job_exception = self._wait_first(ignore_output)
+            # If encountered an error, keep it to be raised, ignoring errors from following jobs.
+            if this_job_exception and not ignore_output:
+                exception_description = this_job_exception
+                ignore_output = True
+
+        return exception_description
+
+    def add_job(self, shader_file, shader_basename, variation_string, output_path,
+                compile_args, preprocessor_args):
+        # If the queue is full, wait until there is at least one slot available.
+        while len(self.queue) >= self.thread_count:
+            exception = self._wait_first(False)
+            # If encountered an exception, cleanup following jobs and raise it.
+            if exception:
+                self._wait_all(True)
+                raise Exception(exception)
+
+        # Add a compile job
+        self.queue.append(CompileQueue.CompileToSPIRV(shader_file, shader_basename,
+                                                      variation_string, output_path,
+                                                      compile_args, preprocessor_args))
+
+    def finish(self):
+        exception = self._wait_all(False)
+        # If encountered an exception, cleanup following jobs and raise it.
+        if exception is not None:
+            raise Exception(exception)
+
+def compile_variation(glslang_path, compile_queue, shader_file, shader_basename, flags, enums,
         flags_active, enum_indices, flags_bits, enum_bits, output_shaders):
 
     glslang_args = [glslang_path]
@@ -295,17 +405,8 @@ def compile_variation(glslang_path, shader_file, shader_basename, flags, enums,
         glslang_args += ['-o', output_path]                            # Output file
         glslang_args.append(shader_file)                               # Input GLSL shader
 
-        print output_path + ': ' + shader_basename + variation_string
-        result = subprocess.call(glslang_args)
-        if result != 0:
-            raise Exception("Error compiling " + shader_file)
-
-        with open(output_path, 'ab') as incfile:
-            shader_text = subprocess.check_output(glslang_preprocessor_output_args)
-
-            incfile.write('\n\n#if 0  // Generated from:\n')
-            incfile.write(cleanup_preprocessed_shader(shader_text.replace('\r\n', '\n')))
-            incfile.write('\n#endif  // Preprocessed code\n')
+        compile_queue.add_job(shader_file, shader_basename, variation_string, output_path,
+                              glslang_args, glslang_preprocessor_output_args)
 
 class ShaderAndVariations:
     def __init__(self, shader_file):
@@ -483,6 +584,8 @@ def main():
 
     input_shaders_and_variations = [ShaderAndVariations(shader_file) for shader_file in input_shaders]
 
+    compile_queue = CompileQueue()
+
     for shader_and_variation in input_shaders_and_variations:
         shader_file = shader_and_variation.shader_file
         flags = shader_and_variation.flags
@@ -501,8 +604,8 @@ def main():
             # a number where each bit says whether a flag is active or not,
             # with values in [0, 2^len(flags))
             for flags_active in range(1 << len(flags)):
-                compile_variation(glslang_path, shader_file, output_name, flags, enums,
-                        flags_active, enum_indices, flags_bits, enum_bits, output_shaders)
+                compile_variation(glslang_path, compile_queue, shader_file, output_name, flags,
+                        enums, flags_active, enum_indices, flags_bits, enum_bits, output_shaders)
 
             if not next_enum_variation(enums, enum_indices):
                 break
@@ -514,6 +617,8 @@ def main():
         print(','.join(outputs))
         return 0
 
+    compile_queue.finish()
+
     # STEP 2: Consolidate the .inc files into an auto-generated cpp/h library.
     with open(out_file_cpp, 'w') as outfile:
         includes = "\n".join([gen_shader_include(shader) for shader in output_shaders])