Regres: add cmd/run_testlist

tests/regres/cmd/run_testlist/main.go

+// Copyright 2019 The SwiftShader Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// run_testlist is a tool runs a dEQP test list, using multiple sand-boxed
+// processes.
+//
+// Unlike simply running deqp with its --deqp-caselist-file flag, run_testlist
+// uses multiple sand-boxed processes, which greatly reduces testing time, and
+// gracefully handles crashing processes.
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"runtime"
+	"sort"
+	"strings"
+	"time"
+
+	"../../cause"
+	"../../deqp"
+	"../../shell"
+	"../../testlist"
+)
+
+var (
+	deqpVkBinary  = flag.String("deqp-vk", "deqp-vk", "path to the deqp-vk binary")
+	testList      = flag.String("test-list", "vk-master-PASS.txt", "path to a test list file")
+	numThreads    = flag.Int("num-threads", runtime.NumCPU(), "number of parallel test runner processes")
+	maxProcMemory = flag.Uint64("max-proc-mem", shell.MaxProcMemory, "maximum virtual memory per child process")
+	output        = flag.String("output", "results.json", "path to an output JSON results file")
+)
+
+const testTimeout = time.Minute * 2
+
+func runTests() error {
+	tests, err := ioutil.ReadFile(*testList)
+	if err != nil {
+		return cause.Wrap(err, "Couldn't read '%s'", *testList)
+	}
+	group := testlist.Group{
+		Name: "",
+		File: "",
+		API:  testlist.Vulkan,
+	}
+	for _, line := range strings.Split(string(tests), "\n") {
+		line = strings.TrimSpace(line)
+		if line != "" && !strings.HasPrefix(line, "#") {
+			group.Tests = append(group.Tests, line)
+		}
+	}
+	sort.Strings(group.Tests)
+
+	testLists := testlist.Lists{group}
+
+	shell.MaxProcMemory = *maxProcMemory
+
+	config := deqp.Config{
+		ExeEgl:           "",
+		ExeGles2:         "",
+		ExeGles3:         "",
+		ExeVulkan:        *deqpVkBinary,
+		Env:              os.Environ(),
+		NumParallelTests: *numThreads,
+		TestLists:        testLists,
+		TestTimeout:      testTimeout,
+	}
+
+	res, err := config.Run()
+	if err != nil {
+		return err
+	}
+
+	err = res.Save(*output)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func main() {
+	if runtime.GOOS != "linux" {
+		log.Fatal("regres only currently runs on linux")
+	}
+
+	flag.ErrHelp = errors.New("regres is a tool to detect regressions between versions of SwiftShader")
+	flag.Parse()
+	if err := runTests(); err != nil {
+		_, _ = fmt.Fprintln(os.Stderr, err)
+		os.Exit(-1)
+	}
+}