Regres: Simplify running of local tests

tests/regres/cmd/run_testlist/main.go

@@ -24,14 +24,14 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"io/ioutil"
+	"log"
 	"os"
+	"path/filepath"
+	"regexp"
 	"runtime"
-	"sort"
 	"strings"
 	"time"
 
-	"../../cause"
 	"../../deqp"
 	"../../shell"
 	"../../testlist"
@@ -43,27 +43,32 @@ var (
 	numThreads    = flag.Int("num-threads", runtime.NumCPU(), "number of parallel test runner processes")
 	maxProcMemory = flag.Uint64("max-proc-mem", shell.MaxProcMemory, "maximum virtual memory per child process")
 	output        = flag.String("output", "results.json", "path to an output JSON results file")
+	filter        = flag.String("filter", "", "filter for test names. Start with a '/' to indicate regex")
 )
 
 const testTimeout = time.Minute * 2
 
-func runTests() error {
-	tests, err := ioutil.ReadFile(*testList)
-	if err != nil {
-		return cause.Wrap(err, "Couldn't read '%s'", *testList)
-	}
+func run() error {
 	group := testlist.Group{
 		Name: "",
-		File: "",
+		File: *testList,
 		API:  testlist.Vulkan,
 	}
-	for _, line := range strings.Split(string(tests), "\n") {
-		line = strings.TrimSpace(line)
-		if line != "" && !strings.HasPrefix(line, "#") {
-			group.Tests = append(group.Tests, line)
+	if err := group.Load(); err != nil {
+		return err
+	}
+
+	if *filter != "" {
+		if strings.HasPrefix(*filter, "/") {
+			re := regexp.MustCompile((*filter)[1:])
+			group = group.Filter(re.MatchString)
+		} else {
+			group = group.Filter(func(name string) bool {
+				ok, _ := filepath.Match(*filter, name)
+				return ok
+			})
 		}
 	}
-	sort.Strings(group.Tests)
 
 	testLists := testlist.Lists{group}
 
@@ -80,11 +85,23 @@ func runTests() error {
 		TestTimeout:      testTimeout,
 	}
 
+	log.Printf("Running %d tests...\n", len(group.Tests))
+
 	res, err := config.Run()
 	if err != nil {
 		return err
 	}
 
+	counts := map[testlist.Status]int{}
+	for _, r := range res.Tests {
+		counts[r.Status] = counts[r.Status] + 1
+	}
+	for _, s := range testlist.Statuses {
+		if count := counts[s]; count > 0 {
+			log.Printf("%s: %d\n", string(s), count)
+		}
+	}
+
 	err = res.Save(*output)
 	if err != nil {
 		return err
@@ -96,7 +113,7 @@ func runTests() error {
 func main() {
 	flag.ErrHelp = errors.New("regres is a tool to detect regressions between versions of SwiftShader")
 	flag.Parse()
-	if err := runTests(); err != nil {
+	if err := run(); err != nil {
 		_, _ = fmt.Fprintln(os.Stderr, err)
 		os.Exit(-1)
 	}

tests/regres/run_testlist.bat

-go run %~dp0cmd\run_testlist\main.go %*
+go run %~dp0cmd\run_testlist\main.go --test-list=%~dp0testlists\vk-master.txt %*
\ No newline at end of file

tests/regres/run_testlist.sh

+#!/bin/bash
+
+ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )"
+
+go run $ROOT_DIR/cmd/run_testlist/main.go --test-list=$ROOT_DIR/testlists/vk-master.txt $@

tests/regres/testlist/testlist.go

@@ -48,6 +48,22 @@ type Group struct {
 	Tests []string
 }
 
+// Load loads the test list file and appends all tests to the Group.
+func (g *Group) Load() error {
+	tests, err := ioutil.ReadFile(g.File)
+	if err != nil {
+		return cause.Wrap(err, "Couldn't read '%s'", tests)
+	}
+	for _, line := range strings.Split(string(tests), "\n") {
+		line = strings.TrimSpace(line)
+		if line != "" && !strings.HasPrefix(line, "#") {
+			g.Tests = append(g.Tests, line)
+		}
+	}
+	sort.Strings(g.Tests)
+	return nil
+}
+
 // Filter returns a new Group that contains only tests that match the predicate.
 func (g Group) Filter(pred func(string) bool) Group {
 	out := Group{
@@ -117,27 +133,22 @@ func Load(root, jsonPath string) (Lists, error) {
 
 	out := make(Lists, len(jsonGroups))
 	for i, jsonGroup := range jsonGroups {
-		path := filepath.Join(dir, jsonGroup.TestFile)
-		tests, err := ioutil.ReadFile(path)
-		if err != nil {
-			return nil, cause.Wrap(err, "Couldn't read '%s'", tests)
-		}
-		relPath, err := filepath.Rel(root, path)
-		if err != nil {
-			return nil, cause.Wrap(err, "Couldn't get relative path for '%s'", path)
-		}
 		group := Group{
 			Name: jsonGroup.Name,
-			File: relPath,
+			File: filepath.Join(dir, jsonGroup.TestFile),
 			API:  API(jsonGroup.API),
 		}
-		for _, line := range strings.Split(string(tests), "\n") {
-			line = strings.TrimSpace(line)
-			if line != "" && !strings.HasPrefix(line, "#") {
-				group.Tests = append(group.Tests, line)
-			}
+		if err := group.Load(); err != nil {
+			return nil, err
 		}
-		sort.Strings(group.Tests)
+
+		// Make the path relative before displaying it to the world.
+		relPath, err := filepath.Rel(root, group.File)
+		if err != nil {
+			return nil, cause.Wrap(err, "Couldn't get relative path for '%s'", group.File)
+		}
+		group.File = relPath
+
 		out[i] = group
 	}