Subzero: Run cross tests as a much more configurable python script.

Makefile.standalone

@@ -224,7 +224,13 @@ check: check-lit check-unit runtime
 	@echo "Crosstests ignored, minimal build"
 else
 check: check-lit check-unit runtime
-	(cd crosstest; ./runtests.sh)
+       # Do all native/sse2 tests, but only test_vector_ops for native/sse4.1.
+       # For (slow) sandboxed tests, limit to Om1/sse4.1.
+	./pydir/crosstest_generator.py -v --lit \
+	  -i native,sse2 -i native,sse4.1,test_vector_ops \
+	  -i sandbox,sse4.1,Om1
+	LLVM_BIN_PATH=$(LLVM_BIN_PATH) \
+	$(LLVM_SRC_PATH)/utils/lit/lit.py -sv crosstest/Output
 endif
 
 FORMAT_BLACKLIST =

crosstest/crosstest.cfg

+[simple_loop]
+driver: simple_loop_main.c
+test: simple_loop.c
+
+[mem_intrin]
+driver: mem_intrin_main.cpp
+test: mem_intrin.cpp
+
+[test_arith]
+driver: test_arith_main.cpp
+test: test_arith.cpp test_arith_frem.ll test_arith_sqrt.ll
+
+[test_bitmanip]
+driver: test_bitmanip_main.cpp
+test: test_bitmanip.cpp test_bitmanip_intrin.ll
+
+[test_calling_conv]
+driver: test_calling_conv_main.cpp
+test: test_calling_conv.cpp
+
+[test_cast]
+driver: test_cast_main.cpp
+test: test_cast.cpp test_cast_to_u1.ll test_cast_vectors.ll
+
+[test_fcmp]
+driver: test_fcmp_main.cpp
+test: test_fcmp.pnacl.ll
+
+[test_global]
+driver: test_global_main.cpp
+test: test_global.cpp
+
+[test_icmp]
+driver: test_icmp_main.cpp
+test: test_icmp.cpp test_icmp_i1vec.ll
+
+[test_select]
+driver: test_select_main.cpp
+test: test_select.ll
+
+[test_stacksave]
+driver: test_stacksave_main.c
+test: test_stacksave.c
+
+[test_sync_atomic]
+driver: test_sync_atomic_main.cpp
+# Compile the non-Subzero object files straight from source since the native
+# LLVM backend does not understand how to lower NaCl-specific intrinsics.
+flags: --crosstest-bitcode=0
+test: test_sync_atomic.cpp
+
+[test_vector_ops]
+driver: test_vector_ops_main.cpp
+test: test_vector_ops.ll

crosstest/lit.cfg

+import os
+import re
+import lit.formats
+
+config.name = 'subzero_crosstests'
+config.test_format = lit.formats.ShTest()
+config.suffixes = ['.xtest']
+config.test_source_root = os.path.dirname(__file__)
+config.test_exec_root = config.test_source_root
+
+llvmbintools = [r"\bFileCheck\b"]
+llvmbinpath = os.path.abspath(os.environ.get('LLVM_BIN_PATH'))
+
+for tool in llvmbintools:
+  # The re.sub() line is adapted from one of LLVM's lit.cfg files.
+  # Extract the tool name from the pattern.  This relies on the tool
+  # name being surrounded by \b word match operators.  If the
+  # pattern starts with "| ", include it in the string to be
+  # substituted.
+  substitution = re.sub(r"^(\\)?((\| )?)\W+b([0-9A-Za-z-_]+)\\b\W*$",
+                        r"\2" + llvmbinpath + "/" + r"\4",
+                        tool)
+  config.substitutions.append((tool, substitution))

crosstest/runtests.sh

-#!/bin/sh
-
-# TODO: Retire this script and move the individual tests into the lit
-# framework, to leverage parallel testing and other lit goodness.
-
-set -eux
-
-# TODO(stichnot): Use FindBaseNaCl so that the script can be run from
-# anywhere within the native_client directory.
-PATH="../pydir:${PATH}"
-OPTLEVELS="m1 2"
-ATTRIBUTES="sse2 sse4.1"
-SANDBOX="0 1"
-OUTDIR=Output
-# Clean the output directory to avoid reusing stale results.
-rm -rf "${OUTDIR}"
-mkdir -p "${OUTDIR}"
-
-for sb in ${SANDBOX} ; do
-  for optlevel in ${OPTLEVELS} ; do
-    for attribute in ${ATTRIBUTES} ; do
-
-      crosstest.py -O${optlevel} --mattr ${attribute} \
-        --prefix=Subzero_ \
-        --target=x8632 \
-        --sandbox=${sb} \
-        --dir="${OUTDIR}" \
-        --test=simple_loop.c \
-        --driver=simple_loop_main.c \
-        --output=simple_loop_sb${sb}_O${optlevel}_${attribute}
-
-      crosstest.py -O${optlevel} --mattr ${attribute} \
-        --prefix=Subzero_ \
-        --target=x8632 \
-        --sandbox=${sb} \
-        --dir="${OUTDIR}" \
-        --test=mem_intrin.cpp \
-        --driver=mem_intrin_main.cpp \
-        --output=mem_intrin_sb${sb}_O${optlevel}_${attribute}
-
-      crosstest.py -O${optlevel} --mattr ${attribute} \
-        --prefix=Subzero_ \
-        --target=x8632 \
-        --sandbox=${sb} \
-        --dir="${OUTDIR}" \
-        --test=test_arith.cpp \
-        --test=test_arith_frem.ll \
-        --test=test_arith_sqrt.ll \
-        --driver=test_arith_main.cpp \
-        --output=test_arith_sb${sb}_O${optlevel}_${attribute}
-
-      crosstest.py -O${optlevel} --mattr ${attribute} \
-        --prefix=Subzero_ \
-        --target=x8632 \
-        --sandbox=${sb} \
-        --dir="${OUTDIR}" \
-        --test=test_bitmanip.cpp --test=test_bitmanip_intrin.ll \
-        --driver=test_bitmanip_main.cpp \
-        --output=test_bitmanip_sb${sb}_O${optlevel}_${attribute}
-
-      crosstest.py -O${optlevel} --mattr ${attribute} \
-        --prefix=Subzero_ --target=x8632 \
-        --sandbox=${sb} \
-        --dir="${OUTDIR}" \
-        --test=test_calling_conv.cpp \
-        --driver=test_calling_conv_main.cpp \
-        --output=test_calling_conv_sb${sb}_O${optlevel}_${attribute}
-
-      crosstest.py -O${optlevel} --mattr ${attribute} \
-        --prefix=Subzero_ \
-        --target=x8632 \
-        --sandbox=${sb} \
-        --dir="${OUTDIR}" \
-        --test=test_cast.cpp --test=test_cast_to_u1.ll \
-        --test=test_cast_vectors.ll \
-        --driver=test_cast_main.cpp \
-        --output=test_cast_sb${sb}_O${optlevel}_${attribute}
-
-      crosstest.py -O${optlevel} --mattr ${attribute} \
-        --prefix=Subzero_ \
-        --target=x8632 \
-        --sandbox=${sb} \
-        --dir="${OUTDIR}" \
-        --test=test_fcmp.pnacl.ll \
-        --driver=test_fcmp_main.cpp \
-        --output=test_fcmp_sb${sb}_O${optlevel}_${attribute}
-
-      crosstest.py -O${optlevel} --mattr ${attribute} \
-        --prefix=Subzero_ \
-        --target=x8632 \
-        --sandbox=${sb} \
-        --dir="${OUTDIR}" \
-        --test=test_global.cpp \
-        --driver=test_global_main.cpp \
-        --output=test_global_sb${sb}_O${optlevel}_${attribute}
-
-      crosstest.py -O${optlevel} --mattr ${attribute} \
-        --prefix=Subzero_ \
-        --target=x8632 \
-        --sandbox=${sb} \
-        --dir="${OUTDIR}" \
-        --test=test_icmp.cpp --test=test_icmp_i1vec.ll \
-        --driver=test_icmp_main.cpp \
-        --output=test_icmp_sb${sb}_O${optlevel}_${attribute}
-
-      crosstest.py -O${optlevel} --mattr ${attribute} \
-        --prefix=Subzero_ \
-        --target=x8632 \
-        --sandbox=${sb} \
-        --dir="${OUTDIR}" \
-        --test=test_select.ll \
-        --driver=test_select_main.cpp \
-        --output=test_select_sb${sb}_O${optlevel}_${attribute}
-
-      crosstest.py -O${optlevel} --mattr ${attribute} \
-        --prefix=Subzero_ \
-        --target=x8632 \
-        --sandbox=${sb} \
-        --dir="${OUTDIR}" \
-        --test=test_stacksave.c \
-        --driver=test_stacksave_main.c \
-        --output=test_stacksave_sb${sb}_O${optlevel}_${attribute}
-
-      # Compile the non-subzero object files straight from source
-      # since the native LLVM backend does not understand how to
-      # lower NaCl-specific intrinsics.
-      crosstest.py -O${optlevel} --mattr ${attribute} \
-        --prefix=Subzero_ \
-        --target=x8632 \
-        --sandbox=${sb} \
-        --dir="${OUTDIR}" \
-        --test=test_sync_atomic.cpp \
-        --crosstest-bitcode=0 \
-        --driver=test_sync_atomic_main.cpp \
-        --output=test_sync_atomic_sb${sb}_O${optlevel}_${attribute}
-
-      crosstest.py -O${optlevel} --mattr ${attribute} \
-        --prefix=Subzero_ \
-        --target=x8632 \
-        --sandbox=${sb} \
-        --dir="${OUTDIR}" \
-        --test=test_vector_ops.ll \
-        --driver=test_vector_ops_main.cpp \
-        --output=test_vector_ops_sb${sb}_O${optlevel}_${attribute}
-
-    done
-  done
-done
-
-for sb in ${SANDBOX} ; do
-  if [ $sb = 0 ] ; then
-    PREFIX=
-  else
-    PREFIX="../../../../run.py -q"
-  fi
-  for optlevel in ${OPTLEVELS} ; do
-    for attribute in ${ATTRIBUTES}; do
-      ${PREFIX} "${OUTDIR}"/simple_loop_sb${sb}_O${optlevel}_${attribute}
-      ${PREFIX} "${OUTDIR}"/mem_intrin_sb${sb}_O${optlevel}_${attribute}
-      ${PREFIX} "${OUTDIR}"/test_arith_sb${sb}_O${optlevel}_${attribute}
-      ${PREFIX} "${OUTDIR}"/test_bitmanip_sb${sb}_O${optlevel}_${attribute}
-      ${PREFIX} "${OUTDIR}"/test_calling_conv_sb${sb}_O${optlevel}_${attribute}
-      ${PREFIX} "${OUTDIR}"/test_cast_sb${sb}_O${optlevel}_${attribute}
-      ${PREFIX} "${OUTDIR}"/test_fcmp_sb${sb}_O${optlevel}_${attribute}
-      ${PREFIX} "${OUTDIR}"/test_global_sb${sb}_O${optlevel}_${attribute}
-      ${PREFIX} "${OUTDIR}"/test_icmp_sb${sb}_O${optlevel}_${attribute}
-      ${PREFIX} "${OUTDIR}"/test_select_sb${sb}_O${optlevel}_${attribute}
-      ${PREFIX} "${OUTDIR}"/test_stacksave_sb${sb}_O${optlevel}_${attribute}
-      ${PREFIX} "${OUTDIR}"/test_sync_atomic_sb${sb}_O${optlevel}_${attribute}
-      ${PREFIX} "${OUTDIR}"/test_vector_ops_sb${sb}_O${optlevel}_${attribute}
-    done
-  done
-done

pydir/crosstest.py

@@ -10,18 +10,17 @@ from utils import shellcmd
 from utils import FindBaseNaCl
 
 def main():
-    """Builds a cross-test binary that allows functions translated by
-    Subzero and llc to be compared.
+    """Builds a cross-test binary for comparing Subzero and llc translation.
 
-    Each --test argument is compiled once by llc and once by Subzero.
-    C/C++ tests are first compiled down to PNaCl bitcode by the
-    build-pnacl-ir.py script.  The --prefix argument ensures that
-    symbol names are different between the two object files, to avoid
-    linking errors.
+    Each --test argument is compiled once by llc and once by Subzero.  C/C++
+    tests are first compiled down to PNaCl bitcode using pnacl-clang and
+    pnacl-opt.  The --prefix argument ensures that symbol names are different
+    between the two object files, to avoid linking errors.
+
+    There is also a --driver argument that specifies the C/C++ file that calls
+    the test functions with a variety of interesting inputs and compares their
+    results.
 
-    There is also a --driver argument that specifies the C/C++ file
-    that calls the test functions with a variety of interesting inputs
-    and compares their results.
     """
     # arch_map maps a Subzero target string to an llvm-mc -triple string.
     arch_map = { 'x8632':'i686', 'x8664':'x86_64', 'arm':'armv7a' }
@@ -76,24 +75,35 @@ def main():
     bindir = ('{root}/toolchain/linux_x86/pnacl_newlib/bin'
               .format(root=nacl_root))
     triple = arch_map[args.target] + ('-nacl' if args.sandbox else '')
+    mypath = os.path.abspath(os.path.dirname(sys.argv[0]))
 
     objs = []
     for arg in args.test:
+        # Construct a "unique key" for each test so that tests can be run in
+        # parallel without race conditions on temporary file creation.
+        key = '{target}.{sb}.O{opt}.{attr}'.format(
+            target=args.target, sb='sb' if args.sandbox else 'nat',
+            opt=args.optlevel, attr=args.attr)
         base, ext = os.path.splitext(arg)
         if ext == '.ll':
             bitcode = arg
         else:
-            bitcode = os.path.join(args.dir, base + '.pnacl.ll')
-            shellcmd(['../pydir/build-pnacl-ir.py', '--disable-verify',
-                      '--dir', args.dir, arg])
+            # Use pnacl-clang and pnacl-opt to produce PNaCl bitcode.
+            bitcode_nonfinal = os.path.join(args.dir, base + '.' + key + '.bc')
+            bitcode = os.path.join(args.dir, base + '.' + key + '.pnacl.ll')
+            shellcmd(['{bin}/pnacl-clang'.format(bin=bindir),
+                      '-O2', '-c', arg, '-o', bitcode_nonfinal])
+            shellcmd(['{bin}/pnacl-opt'.format(bin=bindir),
+                      '-pnacl-abi-simplify-preopt',
+                      '-pnacl-abi-simplify-postopt',
+                      '-pnaclabi-allow-debug-metadata',
+                      bitcode_nonfinal, '-S', '-o', bitcode])
 
-        base_sz = '{base}.{sb}.O{opt}.{attr}.{target}'.format(
-            base=base, sb='sb' if args.sandbox else 'nat', opt=args.optlevel,
-            attr=args.attr, target=args.target)
+        base_sz = '{base}.{key}'.format(base=base, key=key)
         asm_sz = os.path.join(args.dir, base_sz + '.sz.s')
         obj_sz = os.path.join(args.dir, base_sz + '.sz.o')
         obj_llc = os.path.join(args.dir, base_sz + '.llc.o')
-        shellcmd(['../pnacl-sz',
+        shellcmd(['{path}/pnacl-sz'.format(path=os.path.dirname(mypath)),
                   '-O' + args.optlevel,
                   '-mattr=' + args.attr,
                   '--target=' + args.target,

pydir/crosstest_generator.py

+#!/usr/bin/env python2
+
+import argparse
+import ConfigParser
+import os
+import shutil
+import sys
+
+from utils import shellcmd
+from utils import FindBaseNaCl
+
+def Match(desc, includes, excludes, default_match):
+  """Determines whether desc is a match against includes and excludes.
+
+  'desc' is a set of attributes, and 'includes' and 'excludes' are lists of sets
+  of attributes.
+
+  If 'desc' matches any element from 'excludes', the result is False.
+  Otherwise, if 'desc' matches any element from 'includes', the result is True.
+  Otherwise, the 'default_match' value is returned.
+  """
+  for exclude in excludes:
+    if exclude < desc:
+      return False
+  for include in includes:
+    if include < desc:
+      return True
+  return default_match
+
+def main():
+  """Framework for cross test generation and execution.
+
+  Builds and executes cross tests from the space of all possible attribute
+  combinations.  The space can be restricted by providing subsets of attributes
+  to specifically include or exclude.
+  """
+  # pypath is where to find other Subzero python scripts.
+  pypath = os.path.abspath(os.path.dirname(sys.argv[0]))
+  root = FindBaseNaCl()
+
+  # The rest of the attribute sets.
+  targets = [ 'x8632' ]
+  sandboxing = [ 'native', 'sandbox' ]
+  opt_levels = [ 'Om1', 'O2' ]
+  arch_attrs = [ 'sse2', 'sse4.1' ]
+  # all_keys is only used in the help text.
+  all_keys = '; '.join([' '.join(targets), ' '.join(sandboxing),
+                        ' '.join(opt_levels), ' '.join(arch_attrs)])
+
+  argparser = argparse.ArgumentParser(
+    description='  ' + main.__doc__ +
+    'The set of attributes is the set of tests plus the following:\n' +
+    all_keys, formatter_class=argparse.RawTextHelpFormatter)
+  argparser.add_argument('--config', default='crosstest.cfg', dest='config',
+                         metavar='FILE', help='Test configuration file')
+  argparser.add_argument('--print-tests', default=False, action='store_true',
+                         help='Print the set of test names and exit')
+  argparser.add_argument('--include', '-i', default=[], dest='include',
+                         action='append', metavar='ATTR_LIST',
+                         help='Attributes to include (comma-separated). ' +
+                              'Can be used multiple times.')
+  argparser.add_argument('--exclude', '-e', default=[], dest='exclude',
+                         action='append', metavar='ATTR_LIST',
+                         help='Attributes to include (comma-separated). ' +
+                              'Can be used multiple times.')
+  argparser.add_argument('--verbose', '-v', default=False, action='store_true',
+                         help='Use verbose output')
+  argparser.add_argument('--defer', default=False, action='store_true',
+                         help='Defer execution until all executables are built')
+  argparser.add_argument('--no-compile', '-n', default=False,
+                         action='store_true',
+                         help="Don't build; reuse binaries from the last run")
+  argparser.add_argument('--dir', dest='dir', metavar='DIRECTORY',
+                         default=('{root}/toolchain_build/src/subzero/' +
+                                  'crosstest/Output').format(root=root),
+                         help='Output directory')
+  argparser.add_argument('--lit', default=False, action='store_true',
+                         help='Generate files for lit testing')
+  args = argparser.parse_args()
+
+  # Run from the crosstest directory to make it easy to grab inputs.
+  crosstest_dir = '{root}/toolchain_build/src/subzero/crosstest'.format(
+    root=root)
+  os.chdir(crosstest_dir)
+
+  tests = ConfigParser.RawConfigParser()
+  tests.read('crosstest.cfg')
+
+  if args.print_tests:
+    print 'Test name attributes: ' + ' '.join(sorted(tests.sections()))
+    sys.exit(0)
+
+  # includes and excludes are both lists of sets.
+  includes = [ set(item.split(',')) for item in args.include ]
+  excludes = [ set(item.split(',')) for item in args.exclude ]
+  # If any --include args are provided, the default is to not match.
+  default_match = not args.include
+
+  # Delete and recreate the output directory, unless --no-compile was specified.
+  if not args.no_compile:
+    if os.path.exists(args.dir):
+      if os.path.isdir(args.dir):
+        shutil.rmtree(args.dir)
+      else:
+        os.remove(args.dir)
+    if not os.path.exists(args.dir):
+      os.makedirs(args.dir)
+
+  # If --defer is specified, collect the run commands into deferred_cmds for
+  # later execution.
+  deferred_cmds = []
+  for test in sorted(tests.sections()):
+    for target in targets:
+      for sb in sandboxing:
+        for opt in opt_levels:
+          for attr in arch_attrs:
+            desc = [ test, target, sb, opt, attr ]
+            if Match(set(desc), includes, excludes, default_match):
+              exe = '{test}_{target}_{sb}_{opt}_{attr}'.format(
+                test=test, target=target, sb=sb, opt=opt,
+                attr=attr)
+              extra = (tests.get(test, 'flags').split(' ')
+                       if tests.has_option(test, 'flags') else [])
+              # Generate the compile command.
+              cmp_cmd = ['{path}/crosstest.py'.format(path=pypath),
+                         '-{opt}'.format(opt=opt),
+                         '--mattr={attr}'.format(attr=attr),
+                         '--prefix=Subzero_',
+                         '--target={target}'.format(target=target),
+                         '--sandbox={sb}'.format(sb='1' if sb=='sandbox'
+                                                 else '0'),
+                         '--dir={dir}'.format(dir=args.dir),
+                         '--output={exe}'.format(exe=exe),
+                         '--driver={drv}'.format(drv=tests.get(test, 'driver')),
+                        ] + extra + \
+                        [ '--test=' + t
+                          for t in tests.get(test, 'test').split(' ') ]
+              run_cmd_base = os.path.join(args.dir, exe)
+              # Generate the run command.
+              run_cmd = run_cmd_base
+              if sb == 'sandbox':
+                run_cmd = '{root}/run.py -q '.format(root=root) + run_cmd
+              if args.lit:
+                # Create a file to drive the lit test.
+                with open(run_cmd_base + '.xtest', 'w') as f:
+                  f.write('# RUN: sh %s | FileCheck %s\n')
+                  f.write('cd ' + crosstest_dir + ' && \\\n')
+                  f.write(' '.join(cmp_cmd) + ' && \\\n')
+                  f.write(run_cmd + '\n')
+                  f.write('echo Recreate a failure using ' + __file__ +
+                          ' --include=' + ','.join(desc) + '\n')
+                  f.write('# CHECK: Failures=0\n')
+              else:
+                if not args.no_compile:
+                  shellcmd(cmp_cmd,
+                           echo=args.verbose)
+                if (args.defer):
+                  deferred_cmds.append(run_cmd)
+                else:
+                  shellcmd(run_cmd, echo=True)
+  for run_cmd in deferred_cmds:
+    shellcmd(run_cmd, echo=True)
+
+if __name__ == '__main__':
+  main()