Skip to content

S
swiftshader
Commit dc7c597e by Jim Stichnoth
Options

Subzero: Run cross tests as a much more configurable python script.

The runtests.sh script is removed and replaced with crosstest_generator.py. "make check" limits to a relevant subset of cross tests to control the combinatorial explosion. We cut the native tests almost in half, and the sandboxed tests down to a quarter. The --include and --exclude logic is copied/adapted from szbuild.py. The script works by running through every possible test in the combinatorial explosion, and if the test is a match against the --include and --exclude arguments, the test is built and run. The script includes lit support, which is the most likely way it will be run. When run with the --lit argument, it sprays the output directory with lit test files in the form of shell scripts, and "make check" runs lit on that directory. BUG= https://code.google.com/p/nativeclient/issues/detail?id=4085 R=jvoung@chromium.org, mtrofin@chromium.org Review URL: https://codereview.chromium.org/987503004
parent c1f07ea7
Showing with 276 additions and 191 deletions
Makefile.standalone
......@@ -224,7 +224,13 @@ check: check-lit check-unit runtime
@echo "Crosstests ignored, minimal build"
else
check: check-lit check-unit runtime
(cd crosstest; ./runtests.sh)
# Do all native/sse2 tests, but only test_vector_ops for native/sse4.1.
# For (slow) sandboxed tests, limit to Om1/sse4.1.
./pydir/crosstest_generator.py -v --lit \
-i native,sse2 -i native,sse4.1,test_vector_ops \
-i sandbox,sse4.1,Om1
LLVM_BIN_PATH=$(LLVM_BIN_PATH) \
$(LLVM_SRC_PATH)/utils/lit/lit.py -sv crosstest/Output
endif
FORMAT_BLACKLIST =
......
crosstest/crosstest.cfg 0 → 100644
[simple_loop]
driver: simple_loop_main.c
test: simple_loop.c
[mem_intrin]
driver: mem_intrin_main.cpp
test: mem_intrin.cpp
[test_arith]
driver: test_arith_main.cpp
test: test_arith.cpp test_arith_frem.ll test_arith_sqrt.ll
[test_bitmanip]
driver: test_bitmanip_main.cpp
test: test_bitmanip.cpp test_bitmanip_intrin.ll
[test_calling_conv]
driver: test_calling_conv_main.cpp
test: test_calling_conv.cpp
[test_cast]
driver: test_cast_main.cpp
test: test_cast.cpp test_cast_to_u1.ll test_cast_vectors.ll
[test_fcmp]
driver: test_fcmp_main.cpp
test: test_fcmp.pnacl.ll
[test_global]
driver: test_global_main.cpp
test: test_global.cpp
[test_icmp]
driver: test_icmp_main.cpp
test: test_icmp.cpp test_icmp_i1vec.ll
[test_select]
driver: test_select_main.cpp
test: test_select.ll
[test_stacksave]
driver: test_stacksave_main.c
test: test_stacksave.c
[test_sync_atomic]
driver: test_sync_atomic_main.cpp
# Compile the non-Subzero object files straight from source since the native
# LLVM backend does not understand how to lower NaCl-specific intrinsics.
flags: --crosstest-bitcode=0
test: test_sync_atomic.cpp
[test_vector_ops]
driver: test_vector_ops_main.cpp
test: test_vector_ops.ll
crosstest/lit.cfg 0 → 100644
import os
import re
import lit.formats
config.name = 'subzero_crosstests'
config.test_format = lit.formats.ShTest()
config.suffixes = ['.xtest']
config.test_source_root = os.path.dirname(__file__)
config.test_exec_root = config.test_source_root
llvmbintools = [r"\bFileCheck\b"]
llvmbinpath = os.path.abspath(os.environ.get('LLVM_BIN_PATH'))
for tool in llvmbintools:
# The re.sub() line is adapted from one of LLVM's lit.cfg files.
# Extract the tool name from the pattern. This relies on the tool
# name being surrounded by \b word match operators. If the
# pattern starts with "| ", include it in the string to be
# substituted.
substitution = re.sub(r"^(\\)?((\| )?)\W+b([0-9A-Za-z-_]+)\\b\W*$",
r"\2" + llvmbinpath + "/" + r"\4",
tool)
config.substitutions.append((tool, substitution))
crosstest/runtests.sh deleted 100755 → 0
#!/bin/sh
# TODO: Retire this script and move the individual tests into the lit
# framework, to leverage parallel testing and other lit goodness.
set -eux
# TODO(stichnot): Use FindBaseNaCl so that the script can be run from
# anywhere within the native_client directory.
PATH="../pydir:${PATH}"
OPTLEVELS="m1 2"
ATTRIBUTES="sse2 sse4.1"
SANDBOX="0 1"
OUTDIR=Output
# Clean the output directory to avoid reusing stale results.
rm -rf "${OUTDIR}"
mkdir -p "${OUTDIR}"
for sb in ${SANDBOX} ; do
for optlevel in ${OPTLEVELS} ; do
for attribute in ${ATTRIBUTES} ; do
crosstest.py -O${optlevel} --mattr ${attribute} \
--prefix=Subzero_ \
--target=x8632 \
--sandbox=${sb} \
--dir="${OUTDIR}" \
--test=simple_loop.c \
--driver=simple_loop_main.c \
--output=simple_loop_sb${sb}_O${optlevel}_${attribute}
crosstest.py -O${optlevel} --mattr ${attribute} \
--prefix=Subzero_ \
--target=x8632 \
--sandbox=${sb} \
--dir="${OUTDIR}" \
--test=mem_intrin.cpp \
--driver=mem_intrin_main.cpp \
--output=mem_intrin_sb${sb}_O${optlevel}_${attribute}
crosstest.py -O${optlevel} --mattr ${attribute} \
--prefix=Subzero_ \
--target=x8632 \
--sandbox=${sb} \
--dir="${OUTDIR}" \
--test=test_arith.cpp \
--test=test_arith_frem.ll \
--test=test_arith_sqrt.ll \
--driver=test_arith_main.cpp \
--output=test_arith_sb${sb}_O${optlevel}_${attribute}
crosstest.py -O${optlevel} --mattr ${attribute} \
--prefix=Subzero_ \
--target=x8632 \
--sandbox=${sb} \
--dir="${OUTDIR}" \
--test=test_bitmanip.cpp --test=test_bitmanip_intrin.ll \
--driver=test_bitmanip_main.cpp \
--output=test_bitmanip_sb${sb}_O${optlevel}_${attribute}
crosstest.py -O${optlevel} --mattr ${attribute} \
--prefix=Subzero_ --target=x8632 \
--sandbox=${sb} \
--dir="${OUTDIR}" \
--test=test_calling_conv.cpp \
--driver=test_calling_conv_main.cpp \
--output=test_calling_conv_sb${sb}_O${optlevel}_${attribute}
crosstest.py -O${optlevel} --mattr ${attribute} \
--prefix=Subzero_ \
--target=x8632 \
--sandbox=${sb} \
--dir="${OUTDIR}" \
--test=test_cast.cpp --test=test_cast_to_u1.ll \
--test=test_cast_vectors.ll \
--driver=test_cast_main.cpp \
--output=test_cast_sb${sb}_O${optlevel}_${attribute}
crosstest.py -O${optlevel} --mattr ${attribute} \
--prefix=Subzero_ \
--target=x8632 \
--sandbox=${sb} \
--dir="${OUTDIR}" \
--test=test_fcmp.pnacl.ll \
--driver=test_fcmp_main.cpp \
--output=test_fcmp_sb${sb}_O${optlevel}_${attribute}
crosstest.py -O${optlevel} --mattr ${attribute} \
--prefix=Subzero_ \
--target=x8632 \
--sandbox=${sb} \
--dir="${OUTDIR}" \
--test=test_global.cpp \
--driver=test_global_main.cpp \
--output=test_global_sb${sb}_O${optlevel}_${attribute}
crosstest.py -O${optlevel} --mattr ${attribute} \
--prefix=Subzero_ \
--target=x8632 \
--sandbox=${sb} \
--dir="${OUTDIR}" \
--test=test_icmp.cpp --test=test_icmp_i1vec.ll \
--driver=test_icmp_main.cpp \
--output=test_icmp_sb${sb}_O${optlevel}_${attribute}
crosstest.py -O${optlevel} --mattr ${attribute} \
--prefix=Subzero_ \
--target=x8632 \
--sandbox=${sb} \
--dir="${OUTDIR}" \
--test=test_select.ll \
--driver=test_select_main.cpp \
--output=test_select_sb${sb}_O${optlevel}_${attribute}
crosstest.py -O${optlevel} --mattr ${attribute} \
--prefix=Subzero_ \
--target=x8632 \
--sandbox=${sb} \
--dir="${OUTDIR}" \
--test=test_stacksave.c \
--driver=test_stacksave_main.c \
--output=test_stacksave_sb${sb}_O${optlevel}_${attribute}
# Compile the non-subzero object files straight from source
# since the native LLVM backend does not understand how to
# lower NaCl-specific intrinsics.
crosstest.py -O${optlevel} --mattr ${attribute} \
--prefix=Subzero_ \
--target=x8632 \
--sandbox=${sb} \
--dir="${OUTDIR}" \
--test=test_sync_atomic.cpp \
--crosstest-bitcode=0 \
--driver=test_sync_atomic_main.cpp \
--output=test_sync_atomic_sb${sb}_O${optlevel}_${attribute}
crosstest.py -O${optlevel} --mattr ${attribute} \
--prefix=Subzero_ \
--target=x8632 \
--sandbox=${sb} \
--dir="${OUTDIR}" \
--test=test_vector_ops.ll \
--driver=test_vector_ops_main.cpp \
--output=test_vector_ops_sb${sb}_O${optlevel}_${attribute}
done
done
done
for sb in ${SANDBOX} ; do
if [ $sb = 0 ] ; then
PREFIX=
else
PREFIX="../../../../run.py -q"
fi
for optlevel in ${OPTLEVELS} ; do
for attribute in ${ATTRIBUTES}; do
${PREFIX} "${OUTDIR}"/simple_loop_sb${sb}_O${optlevel}_${attribute}
${PREFIX} "${OUTDIR}"/mem_intrin_sb${sb}_O${optlevel}_${attribute}
${PREFIX} "${OUTDIR}"/test_arith_sb${sb}_O${optlevel}_${attribute}
${PREFIX} "${OUTDIR}"/test_bitmanip_sb${sb}_O${optlevel}_${attribute}
${PREFIX} "${OUTDIR}"/test_calling_conv_sb${sb}_O${optlevel}_${attribute}
${PREFIX} "${OUTDIR}"/test_cast_sb${sb}_O${optlevel}_${attribute}
${PREFIX} "${OUTDIR}"/test_fcmp_sb${sb}_O${optlevel}_${attribute}
${PREFIX} "${OUTDIR}"/test_global_sb${sb}_O${optlevel}_${attribute}
${PREFIX} "${OUTDIR}"/test_icmp_sb${sb}_O${optlevel}_${attribute}
${PREFIX} "${OUTDIR}"/test_select_sb${sb}_O${optlevel}_${attribute}
${PREFIX} "${OUTDIR}"/test_stacksave_sb${sb}_O${optlevel}_${attribute}
${PREFIX} "${OUTDIR}"/test_sync_atomic_sb${sb}_O${optlevel}_${attribute}
${PREFIX} "${OUTDIR}"/test_vector_ops_sb${sb}_O${optlevel}_${attribute}
done
done
done
pydir/crosstest.py
......@@ -10,18 +10,17 @@ from utils import shellcmd
from utils import FindBaseNaCl
def main():
"""Builds a cross-test binary that allows functions translated by
Subzero and llc to be compared.
"""Builds a cross-test binary for comparing Subzero and llc translation.
Each --test argument is compiled once by llc and once by Subzero.
C/C++ tests are first compiled down to PNaCl bitcode by the
build-pnacl-ir.py script. The --prefix argument ensures that
symbol names are different between the two object files, to avoid
linking errors.
Each --test argument is compiled once by llc and once by Subzero. C/C++
tests are first compiled down to PNaCl bitcode using pnacl-clang and
pnacl-opt. The --prefix argument ensures that symbol names are different
between the two object files, to avoid linking errors.
There is also a --driver argument that specifies the C/C++ file that calls
the test functions with a variety of interesting inputs and compares their
results.
There is also a --driver argument that specifies the C/C++ file
that calls the test functions with a variety of interesting inputs
and compares their results.
"""
# arch_map maps a Subzero target string to an llvm-mc -triple string.
arch_map = { 'x8632':'i686', 'x8664':'x86_64', 'arm':'armv7a' }
......@@ -76,24 +75,35 @@ def main():
bindir = ('{root}/toolchain/linux_x86/pnacl_newlib/bin'
.format(root=nacl_root))
triple = arch_map[args.target] + ('-nacl' if args.sandbox else '')
mypath = os.path.abspath(os.path.dirname(sys.argv[0]))
objs = []
for arg in args.test:
# Construct a "unique key" for each test so that tests can be run in
# parallel without race conditions on temporary file creation.
key = '{target}.{sb}.O{opt}.{attr}'.format(
target=args.target, sb='sb' if args.sandbox else 'nat',
opt=args.optlevel, attr=args.attr)
base, ext = os.path.splitext(arg)
if ext == '.ll':
bitcode = arg
else:
bitcode = os.path.join(args.dir, base + '.pnacl.ll')
shellcmd(['../pydir/build-pnacl-ir.py', '--disable-verify',
'--dir', args.dir, arg])
# Use pnacl-clang and pnacl-opt to produce PNaCl bitcode.
bitcode_nonfinal = os.path.join(args.dir, base + '.' + key + '.bc')
bitcode = os.path.join(args.dir, base + '.' + key + '.pnacl.ll')
shellcmd(['{bin}/pnacl-clang'.format(bin=bindir),
'-O2', '-c', arg, '-o', bitcode_nonfinal])
shellcmd(['{bin}/pnacl-opt'.format(bin=bindir),
'-pnacl-abi-simplify-preopt',
'-pnacl-abi-simplify-postopt',
'-pnaclabi-allow-debug-metadata',
bitcode_nonfinal, '-S', '-o', bitcode])
base_sz = '{base}.{sb}.O{opt}.{attr}.{target}'.format(
base=base, sb='sb' if args.sandbox else 'nat', opt=args.optlevel,
attr=args.attr, target=args.target)
base_sz = '{base}.{key}'.format(base=base, key=key)
asm_sz = os.path.join(args.dir, base_sz + '.sz.s')
obj_sz = os.path.join(args.dir, base_sz + '.sz.o')
obj_llc = os.path.join(args.dir, base_sz + '.llc.o')
shellcmd(['../pnacl-sz',
shellcmd(['{path}/pnacl-sz'.format(path=os.path.dirname(mypath)),
'-O' + args.optlevel,
'-mattr=' + args.attr,
'--target=' + args.target,
......
pydir/crosstest_generator.py 0 → 100755
#!/usr/bin/env python2
import argparse
import ConfigParser
import os
import shutil
import sys
from utils import shellcmd
from utils import FindBaseNaCl
def Match(desc, includes, excludes, default_match):
"""Determines whether desc is a match against includes and excludes.
'desc' is a set of attributes, and 'includes' and 'excludes' are lists of sets
of attributes.
If 'desc' matches any element from 'excludes', the result is False.
Otherwise, if 'desc' matches any element from 'includes', the result is True.
Otherwise, the 'default_match' value is returned.
"""
for exclude in excludes:
if exclude < desc:
return False
for include in includes:
if include < desc:
return True
return default_match
def main():
"""Framework for cross test generation and execution.
Builds and executes cross tests from the space of all possible attribute
combinations. The space can be restricted by providing subsets of attributes
to specifically include or exclude.
"""
# pypath is where to find other Subzero python scripts.
pypath = os.path.abspath(os.path.dirname(sys.argv[0]))
root = FindBaseNaCl()
# The rest of the attribute sets.
targets = [ 'x8632' ]
sandboxing = [ 'native', 'sandbox' ]
opt_levels = [ 'Om1', 'O2' ]
arch_attrs = [ 'sse2', 'sse4.1' ]
# all_keys is only used in the help text.
all_keys = '; '.join([' '.join(targets), ' '.join(sandboxing),
' '.join(opt_levels), ' '.join(arch_attrs)])
argparser = argparse.ArgumentParser(
description=' ' + main.__doc__ +
'The set of attributes is the set of tests plus the following:\n' +
all_keys, formatter_class=argparse.RawTextHelpFormatter)
argparser.add_argument('--config', default='crosstest.cfg', dest='config',
metavar='FILE', help='Test configuration file')
argparser.add_argument('--print-tests', default=False, action='store_true',
help='Print the set of test names and exit')
argparser.add_argument('--include', '-i', default=[], dest='include',
action='append', metavar='ATTR_LIST',
help='Attributes to include (comma-separated). ' +
'Can be used multiple times.')
argparser.add_argument('--exclude', '-e', default=[], dest='exclude',
action='append', metavar='ATTR_LIST',
help='Attributes to include (comma-separated). ' +
'Can be used multiple times.')
argparser.add_argument('--verbose', '-v', default=False, action='store_true',
help='Use verbose output')
argparser.add_argument('--defer', default=False, action='store_true',
help='Defer execution until all executables are built')
argparser.add_argument('--no-compile', '-n', default=False,
action='store_true',
help="Don't build; reuse binaries from the last run")
argparser.add_argument('--dir', dest='dir', metavar='DIRECTORY',
default=('{root}/toolchain_build/src/subzero/' +
'crosstest/Output').format(root=root),
help='Output directory')
argparser.add_argument('--lit', default=False, action='store_true',
help='Generate files for lit testing')
args = argparser.parse_args()
# Run from the crosstest directory to make it easy to grab inputs.
crosstest_dir = '{root}/toolchain_build/src/subzero/crosstest'.format(
root=root)
os.chdir(crosstest_dir)
tests = ConfigParser.RawConfigParser()
tests.read('crosstest.cfg')
if args.print_tests:
print 'Test name attributes: ' + ' '.join(sorted(tests.sections()))
sys.exit(0)
# includes and excludes are both lists of sets.
includes = [ set(item.split(',')) for item in args.include ]
excludes = [ set(item.split(',')) for item in args.exclude ]
# If any --include args are provided, the default is to not match.
default_match = not args.include
# Delete and recreate the output directory, unless --no-compile was specified.
if not args.no_compile:
if os.path.exists(args.dir):
if os.path.isdir(args.dir):
shutil.rmtree(args.dir)
else:
os.remove(args.dir)
if not os.path.exists(args.dir):
os.makedirs(args.dir)
# If --defer is specified, collect the run commands into deferred_cmds for
# later execution.
deferred_cmds = []
for test in sorted(tests.sections()):
for target in targets:
for sb in sandboxing:
for opt in opt_levels:
for attr in arch_attrs:
desc = [ test, target, sb, opt, attr ]
if Match(set(desc), includes, excludes, default_match):
exe = '{test}_{target}_{sb}_{opt}_{attr}'.format(
test=test, target=target, sb=sb, opt=opt,
attr=attr)
extra = (tests.get(test, 'flags').split(' ')
if tests.has_option(test, 'flags') else [])
# Generate the compile command.
cmp_cmd = ['{path}/crosstest.py'.format(path=pypath),
'-{opt}'.format(opt=opt),
'--mattr={attr}'.format(attr=attr),
'--prefix=Subzero_',
'--target={target}'.format(target=target),
'--sandbox={sb}'.format(sb='1' if sb=='sandbox'
else '0'),
'--dir={dir}'.format(dir=args.dir),
'--output={exe}'.format(exe=exe),
'--driver={drv}'.format(drv=tests.get(test, 'driver')),
] + extra + \
[ '--test=' + t
for t in tests.get(test, 'test').split(' ') ]
run_cmd_base = os.path.join(args.dir, exe)
# Generate the run command.
run_cmd = run_cmd_base
if sb == 'sandbox':
run_cmd = '{root}/run.py -q '.format(root=root) + run_cmd
if args.lit:
# Create a file to drive the lit test.
with open(run_cmd_base + '.xtest', 'w') as f:
f.write('# RUN: sh %s | FileCheck %s\n')
f.write('cd ' + crosstest_dir + ' && \\\n')
f.write(' '.join(cmp_cmd) + ' && \\\n')
f.write(run_cmd + '\n')
f.write('echo Recreate a failure using ' + __file__ +
' --include=' + ','.join(desc) + '\n')
f.write('# CHECK: Failures=0\n')
else:
if not args.no_compile:
shellcmd(cmp_cmd,
echo=args.verbose)
if (args.defer):
deferred_cmds.append(run_cmd)
else:
shellcmd(run_cmd, echo=True)
for run_cmd in deferred_cmds:
shellcmd(run_cmd, echo=True)
if __name__ == '__main__':
main()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment