Capture/Replay tests: Fixes and improvements.

src/tests/capture_replay_tests.py

@@ -30,7 +30,6 @@ import multiprocessing
 import os
 import queue
 import re
-import shlex
 import shutil
 import subprocess
 import sys
@@ -181,19 +180,14 @@ def info(str):
 class SubProcess():
 
     def __init__(self, command, env=os.environ, pipe_stdout=PIPE_STDOUT):
-        parsed_command = shlex.split(command)
         # shell=False so that only 1 subprocess is spawned.
-        # if shell=True, a shell probess is spawned, which in turn spawns the process running
+        # if shell=True, a shell process is spawned, which in turn spawns the process running
         # the command. Since we do not have a handle to the 2nd process, we cannot terminate it.
         if pipe_stdout:
             self.proc_handle = subprocess.Popen(
-                parsed_command,
-                env=env,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT,
-                shell=False)
+                command, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False)
         else:
-            self.proc_handle = subprocess.Popen(parsed_command, env=env, shell=False)
+            self.proc_handle = subprocess.Popen(command, env=env, shell=False)
 
     def Join(self, timeout):
         debug('Joining with subprocess %d, timeout %s' % (self.Pid(), str(timeout)))
@@ -216,15 +210,34 @@ class SubProcess():
 # should have this. This object is created inside the main process, and each worker process.
 class ChildProcessesManager():
 
+    @classmethod
+    def _GetGnAndAutoninjaAbsolutePaths(self):
+        # get gn/autoninja absolute path because subprocess with shell=False doesn't look
+        # into the PATH environment variable on Windows
+        depot_tools_name = "depot_tools"
+        if platform == "win32":
+            paths = os.environ["PATH"].split(";")
+        else:
+            paths = os.environ["PATH"].split(":")
+        for path in paths:
+            if path.endswith(depot_tools_name):
+                if platform == "win32":
+                    return os.path.join(path, "gn.bat"), os.path.join(path, "autoninja.bat")
+                else:
+                    return os.path.join(path, "gn"), os.path.join(path, "autoninja")
+        logging.exception("No gn or autoninja found on system")
+
     def __init__(self):
         # a dictionary of Subprocess, with pid as key
         self.subprocesses = {}
         # list of Python multiprocess.Process handles
         self.workers = []
 
+        self._gn_path, self._autoninja_path = self._GetGnAndAutoninjaAbsolutePaths()
+
     def CreateSubprocess(self, command, env=None, pipe_stdout=True):
         subprocess = SubProcess(command, env, pipe_stdout)
-        debug('Creating subprocess: %s with pid %d' % (str(command), subprocess.Pid()))
+        debug('Creating subprocess: %s with pid %d' % (' '.join(command), subprocess.Pid()))
         self.subprocesses[subprocess.Pid()] = subprocess
         return subprocess.Pid()
 
@@ -279,37 +292,17 @@ class ChildProcessesManager():
                 count += 1
         return count
 
+    def RunGNGenProcess(self, build_dir, gn_args, pipe_stdout):
+        debug('Calling GN gen with %s' % str(gn_args))
+        args_str = ' '.join(['%s=%s' % (k, v) for (k, v) in gn_args])
+        cmd = [self._gn_path, 'gen', '--args=%s' % args_str, build_dir]
+        pid = self.CreateSubprocess(cmd, pipe_stdout=pipe_stdout)
+        return self.JoinSubprocess(pid)
 
-def CreateGNGenCommand(gn_path, build_dir, arguments):
-    debug('Calling GN gen with %s' % str(arguments))
-    args_str = ' '.join(['%s=%s' % (k, v) for (k, v) in arguments])
-    command = '"%s" gen --args="%s" %s' % (gn_path, args_str, build_dir)
-    return command
-
-
-def CreateAutoninjaCommand(autoninja_path, build_dir, target):
-    command = '"' + autoninja_path + '" '
-    command += target
-    command += " -C "
-    command += build_dir
-    return command
-
-
-def GetGnAndAutoninjaAbsolutePaths():
-    # get gn/autoninja absolute path because subprocess with shell=False doesn't look
-    # into the PATH environment variable on Windows
-    depot_tools_name = "depot_tools"
-    if platform == "win32":
-        paths = os.environ["PATH"].split(";")
-    else:
-        paths = os.environ["PATH"].split(":")
-    for path in paths:
-        if path.endswith(depot_tools_name):
-            if platform == "win32":
-                return os.path.join(path, "gn.bat"), os.path.join(path, "autoninja.bat")
-            else:
-                return os.path.join(path, "gn"), os.path.join(path, "autoninja")
-    return "", ""
+    def RunAutoninjaProcess(self, build_dir, target, pipe_stdout):
+        cmd = [self._autoninja_path, '-C', build_dir, target]
+        pid = self.CreateSubprocess(cmd, pipe_stdout=pipe_stdout)
+        return self.JoinSubprocess(pid)
 
 
 def GetTestsListForFilter(test_path, filter):
@@ -397,7 +390,7 @@ def WriteAngleTraceGLHeader():
 
 class GroupedResult():
     Passed = "Passed"
-    Failed = "Failed"
+    Failed = "Comparison Failed"
     TimedOut = "Timeout"
     Crashed = "Crashed"
     CompileFailed = "CompileFailed"
@@ -556,7 +549,7 @@ class TestBatch():
         if not self.keep_temp_files:
             ClearFolderContent(trace_folder_path)
         filt = ':'.join([test.full_test_name for test in self.tests])
-        cmd = '"%s" --gtest_filter=%s --angle-per-test-capture-label' % (test_exe_path, filt)
+        cmd = [test_exe_path, '--gtest_filter=%s' % filt, '--angle-per-test-capture-label']
         capture_proc_id = child_processes_manager.CreateSubprocess(cmd, env)
         returncode, output = child_processes_manager.JoinSubprocess(capture_proc_id,
                                                                     SUBPROCESS_TIMEOUT)
@@ -584,8 +577,8 @@ class TestBatch():
                     skipped_tests))
         return continued_tests
 
-    def BuildReplay(self, gn_path, autoninja_path, replay_build_dir, trace_dir, trace_folder_path,
-                    composite_file_id, tests, child_processes_manager):
+    def BuildReplay(self, replay_build_dir, trace_dir, trace_folder_path, composite_file_id, tests,
+                    child_processes_manager):
         # write gni file that holds all the traces files in a list
         self.CreateGNIFile(trace_folder_path, composite_file_id, tests)
         # write header and cpp composite files, which glue the trace files with
@@ -593,22 +586,20 @@ class TestBatch():
         self.CreateTestsCompositeFiles(trace_folder_path, composite_file_id, tests)
         gn_args = [("use_goma", str(self.use_goma).lower()),
                    ("angle_build_capture_replay_tests", "true"),
-                   ("angle_capture_replay_test_trace_dir", '\\"' + trace_dir + '\\"'),
+                   ("angle_capture_replay_test_trace_dir", '"%s"' % trace_dir),
                    ("angle_with_capture_by_default", "false"),
                    ("angle_capture_replay_composite_file_id", str(composite_file_id))]
         if self.goma_dir:
             gn_args.append(("goma_dir", self.goma_dir))
-        gn_command = CreateGNGenCommand(gn_path, replay_build_dir, gn_args)
-        gn_proc_id = child_processes_manager.CreateSubprocess(gn_command)
-        returncode, output = child_processes_manager.JoinSubprocess(gn_proc_id)
+        returncode, output = child_processes_manager.RunGNGenProcess(replay_build_dir, gn_args,
+                                                                     True)
         if returncode != 0:
             self.results.append(
                 GroupedResult(GroupedResult.CompileFailed, "Build replay failed at gn generation",
                               output, tests))
             return False
-        autoninja_command = CreateAutoninjaCommand(autoninja_path, replay_build_dir, REPLAY_BINARY)
-        autoninja_proc_id = child_processes_manager.CreateSubprocess(autoninja_command)
-        returncode, output = child_processes_manager.JoinSubprocess(autoninja_proc_id)
+        returncode, output = child_processes_manager.RunAutoninjaProcess(
+            replay_build_dir, REPLAY_BINARY, True)
         if returncode != 0:
             self.results.append(
                 GroupedResult(GroupedResult.CompileFailed, "Build replay failed at autoninja",
@@ -619,8 +610,7 @@ class TestBatch():
     def RunReplay(self, replay_exe_path, child_processes_manager, tests):
         env = os.environ.copy()
         env['ANGLE_CAPTURE_ENABLED'] = '0'
-        command = '"' + replay_exe_path + '"'
-        replay_proc_id = child_processes_manager.CreateSubprocess(command, env)
+        replay_proc_id = child_processes_manager.CreateSubprocess([replay_exe_path], env)
         returncode, output = child_processes_manager.JoinSubprocess(replay_proc_id,
                                                                     SUBPROCESS_TIMEOUT)
         if returncode == -1:
@@ -631,6 +621,7 @@ class TestBatch():
             self.results.append(
                 GroupedResult(GroupedResult.TimedOut, "Replay run timed out", "", tests))
             return
+
         output_lines = output.splitlines()
         passes = []
         fails = []
@@ -762,8 +753,7 @@ def SetCWDToAngleFolder():
     return cwd
 
 
-def RunTests(args, worker_id, job_queue, gn_path, autoninja_path, trace_dir, result_list,
-             message_queue):
+def RunTests(args, worker_id, job_queue, trace_dir, result_list, message_queue):
     trace_folder_path = os.path.join(REPLAY_SAMPLE_FOLDER, trace_dir)
     test_exec_path = os.path.join(args.out_dir, 'Capture', args.test_suite)
     replay_build_dir = os.path.join(args.out_dir, 'Replay%d' % worker_id)
@@ -790,8 +780,8 @@ def RunTests(args, worker_id, job_queue, gn_path, autoninja_path, trace_dir, res
                 result_list.append(test_batch.GetResults())
                 message_queue.put(str(test_batch.GetResults()))
                 continue
-            success = test_batch.BuildReplay(gn_path, autoninja_path, replay_build_dir, trace_dir,
-                                             trace_folder_path, composite_file_id, continued_tests,
+            success = test_batch.BuildReplay(replay_build_dir, trace_dir, trace_folder_path,
+                                             composite_file_id, continued_tests,
                                              child_processes_manager)
             if test_batch.keep_temp_files:
                 composite_file_id += 1
@@ -863,29 +853,21 @@ def main(args):
 
         CreateTraceFolders(worker_count, trace_folder)
         WriteAngleTraceGLHeader()
-        gn_path, autoninja_path = GetGnAndAutoninjaAbsolutePaths()
-        if gn_path == "" or autoninja_path == "":
-            logging.error("No gn or autoninja found on system")
-            return 1
         # generate gn files
         gn_args = [("use_goma", str(args.use_goma).lower()),
                    ("angle_with_capture_by_default", "true")]
         if args.goma_dir:
             gn_args.append(("goma_dir", args.goma_dir))
-        capture_build_dir = os.path.join(args.out_dir, 'Capture')
-        gn_command = CreateGNGenCommand(gn_path, capture_build_dir, gn_args)
-        gn_proc_id = child_processes_manager.CreateSubprocess(gn_command, pipe_stdout=False)
-        returncode, output = child_processes_manager.JoinSubprocess(gn_proc_id)
+        capture_build_dir = os.path.normpath(r"%s/Capture" % args.out_dir)
+        returncode, output = child_processes_manager.RunGNGenProcess(capture_build_dir, gn_args,
+                                                                     False)
         if returncode != 0:
             logging.error(output)
             child_processes_manager.KillAll()
             return 1
         # run autoninja to build all tests
-        autoninja_command = CreateAutoninjaCommand(autoninja_path, capture_build_dir,
-                                                   args.test_suite)
-        autoninja_proc_id = child_processes_manager.CreateSubprocess(
-            autoninja_command, pipe_stdout=False)
-        returncode, output = child_processes_manager.JoinSubprocess(autoninja_proc_id)
+        returncode, output = child_processes_manager.RunAutoninjaProcess(
+            capture_build_dir, args.test_suite, False)
         if returncode != 0:
             logging.error(output)
             child_processes_manager.KillAll()
@@ -938,8 +920,8 @@ def main(args):
         for worker_id in range(worker_count):
             proc = multiprocessing.Process(
                 target=RunTests,
-                args=(args, worker_id, job_queue, gn_path, autoninja_path,
-                      trace_folder + str(worker_id), result_list, message_queue))
+                args=(args, worker_id, job_queue, trace_folder + str(worker_id), result_list,
+                      message_queue))
             child_processes_manager.AddWorker(proc)
             proc.start()
 
@@ -991,11 +973,12 @@ def main(args):
 
         print("\n\n")
         print("Elapsed time: %.2lf seconds" % (end_time - start_time))
-        print("Passed: %d, Failed: %d, Crashed: %d, CompileFailed %d, Skipped: %d, Timeout: %d" %
-              (passed_count, failed_count, crashed_count, compile_failed_count, skipped_count,
+        print(
+            "Passed: %d, Comparison Failed: %d, Crashed: %d, CompileFailed %d, Skipped: %d, Timeout: %d"
+            % (passed_count, failed_count, crashed_count, compile_failed_count, skipped_count,
                timedout_count))
         if len(failed_tests):
-            print("Failed tests:")
+            print("Comparison Failed tests:")
             for failed_test in sorted(failed_tests):
                 print("\t" + failed_test)
         if len(crashed_tests):

src/tests/capture_replay_tests/BUILD.gn

@@ -53,6 +53,7 @@ if (angle_build_capture_replay_tests) {
       "ANGLE_CAPTURE_REPLAY_COMPOSITE_TESTS_HEADER=" +
           angle_capture_replay_test_trace_dir +
           "/CompositeTests${angle_capture_replay_composite_file_id}.h",
+      "ANGLE_REPLAY_EXPORT=",
     ]
     include_dirs = [ "." ]
   }