Regres: Generate coverage data on nightly runs

tests/regres/cmd/run_testlist/main.go

@@ -132,7 +132,7 @@ func run() error {
 	}
 
 	if *genCoverage {
-		if err := ioutil.WriteFile("coverage.json", []byte(res.Coverage.JSON()), 0666); err != nil {
+		if err := ioutil.WriteFile("coverage.json", []byte(res.Coverage.JSON("master")), 0666); err != nil {
 			return err
 		}
 	}

tests/regres/cov/coverage.go

@@ -85,24 +85,24 @@ func (e Env) Import(profrawPath string) (*Coverage, error) {
 	}
 	defer os.Remove(profdata)
 
-	args := []string{
-		"export",
-		e.ExePath,
-		"-instr-profile=" + profdata,
-		"-format=text",
-	}
-	if e.LLVM.Version.GreaterEqual(llvm.Version{Major: 9}) {
-		// LLVM 9 has new flags that omit stuff we don't care about.
-		args = append(args,
-			"-skip-expansions",
-			"-skip-functions",
-		)
-	}
-
 	if e.TurboCov == "" {
+		args := []string{
+			"export",
+			e.ExePath,
+			"-instr-profile=" + profdata,
+			"-format=text",
+		}
+		if e.LLVM.Version.GreaterEqual(llvm.Version{Major: 9}) {
+			// LLVM 9 has new flags that omit stuff we don't care about.
+			args = append(args,
+				"-skip-expansions",
+				"-skip-functions",
+			)
+		}
+
 		data, err := exec.Command(e.LLVM.Cov(), args...).Output()
 		if err != nil {
-			return nil, cause.Wrap(err, "llvm-cov errored: %v", string(data))
+			return nil, cause.Wrap(err, "llvm-cov errored: %v", string(err.(*exec.ExitError).Stderr))
 		}
 		cov, err := e.parseCov(data)
 		if err != nil {
@@ -113,7 +113,7 @@ func (e Env) Import(profrawPath string) (*Coverage, error) {
 
 	data, err := exec.Command(e.TurboCov, e.ExePath, profdata).Output()
 	if err != nil {
-		return nil, cause.Wrap(err, "turbo-cov errored: %v", string(data))
+		return nil, cause.Wrap(err, "turbo-cov errored: %v", string(err.(*exec.ExitError).Stderr))
 	}
 	cov, err := e.parseTurboCov(data)
 	if err != nil {
@@ -566,12 +566,15 @@ func indent(s string) string {
 }
 
 // JSON returns the full test tree serialized to JSON.
-func (t *Tree) JSON() string {
+func (t *Tree) JSON(revision string) string {
 	sb := &strings.Builder{}
 	sb.WriteString(`{`)
 
+	// write the revision
+	sb.WriteString(`"r":"` + revision + `"`)
+
 	// write the strings
-	sb.WriteString(`"n":[`)
+	sb.WriteString(`,"n":[`)
 	for i, s := range t.strings.s {
 		if i > 0 {
 			sb.WriteString(`,`)

tests/regres/cov/turbo-cov/main.cpp

@@ -47,7 +47,7 @@ int main(int argc, const char **argv)
 {
 	if(argc < 3)
 	{
-		fprintf(stderr, "llvm-cov-bin <exe> <profdata>\n");
+		fprintf(stderr, "turbo-cov <exe> <profdata>\n");
 		return 1;
 	}
 

tests/regres/deqp/deqp.go

@@ -158,7 +158,8 @@ func (c *Config) Run() (*Results, error) {
 	// For each API that we are testing
 	for _, list := range c.TestLists {
 		// Resolve the test runner
-		var exe string
+		exe, supportsCoverage := "", false
+
 		switch list.API {
 		case testlist.EGL:
 			exe = c.ExeEgl
@@ -167,7 +168,7 @@ func (c *Config) Run() (*Results, error) {
 		case testlist.GLES3:
 			exe = c.ExeGles3
 		case testlist.Vulkan:
-			exe = c.ExeVulkan
+			exe, supportsCoverage = c.ExeVulkan, true
 		default:
 			return nil, fmt.Errorf("Unknown API '%v'", list.API)
 		}
@@ -182,7 +183,7 @@ func (c *Config) Run() (*Results, error) {
 		wg.Add(c.NumParallelTests)
 		for i := 0; i < c.NumParallelTests; i++ {
 			go func(index int) {
-				c.TestRoutine(exe, tests, results, index)
+				c.TestRoutine(exe, tests, results, index, supportsCoverage)
 				wg.Done()
 			}(goroutineIndex)
 			goroutineIndex++
@@ -255,7 +256,7 @@ func (c *Config) Run() (*Results, error) {
 // is written to results.
 // TestRoutine only returns once the tests chan has been closed.
 // TestRoutine does not close the results chan.
-func (c *Config) TestRoutine(exe string, tests <-chan string, results chan<- TestResult, goroutineIndex int) {
+func (c *Config) TestRoutine(exe string, tests <-chan string, results chan<- TestResult, goroutineIndex int, supportsCoverage bool) {
 	// Context for the GCOV_PREFIX environment variable:
 	// If you compile SwiftShader with gcc and the --coverage flag, the build will contain coverage instrumentation.
 	// We can use this to get the code coverage of SwiftShader from running dEQP.
@@ -286,8 +287,10 @@ func (c *Config) TestRoutine(exe string, tests <-chan string, results chan<- Tes
 	}
 
 	coverageFile := filepath.Join(c.TempDir, fmt.Sprintf("%v.profraw", goroutineIndex))
-	if c.CoverageEnv != nil {
-		env = cov.AppendRuntimeEnv(env, coverageFile)
+	if supportsCoverage {
+		if c.CoverageEnv != nil {
+			env = cov.AppendRuntimeEnv(env, coverageFile)
+		}
 	}
 	logPath := filepath.Join(c.TempDir, fmt.Sprintf("%v.log", goroutineIndex))
 
@@ -312,10 +315,10 @@ nextTest:
 		}
 
 		var coverage *cov.Coverage
-		if c.CoverageEnv != nil {
+		if c.CoverageEnv != nil && supportsCoverage { // IsFile() check here is for GLES tests that don't emit coverage.
 			coverage, err = c.CoverageEnv.Import(coverageFile)
 			if err != nil {
-				log.Printf("Warning: Failed to get test coverage for test '%v'. %v", name, err)
+				log.Printf("Warning: Failed to process test coverage for test '%v'. %v", name, err)
 			}
 			os.Remove(coverageFile)
 		}

tests/regres/git/git.go

@@ -120,10 +120,8 @@ func CheckoutRemoteBranch(path, url string, branch string) error {
 	for _, cmds := range [][]string{
 		{"init"},
 		{"remote", "add", "origin", url},
-		// Note: this depth is here to prevent massive dEQP checkouts that can
-		// take all day. If the commit cannot be found in the checked out branch
-		// then this limit may need to be increased.
-		{"fetch", "origin", "--depth=99", branch},
+		{"fetch", "origin", "--depth=1", branch},
+		{"checkout", branch},
 	} {
 		if err := shell.Shell(gitTimeout, exe, path, cmds...); err != nil {
 			os.RemoveAll(path)

tests/regres/run.sh

@@ -2,4 +2,4 @@
 
 ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )"
 
-go run $ROOT_DIR/cmd/regres/main.go $@
+go run $ROOT_DIR/cmd/regres/main.go $@ 2>&1 | tee regres-log.txt

tests/regres/testlist/testlist.go

@@ -79,6 +79,20 @@ func (g Group) Filter(pred func(string) bool) Group {
 	return out
 }
 
+// Limit returns a new Group that contains a maximum of limit tests.
+func (g Group) Limit(limit int) Group {
+	out := Group{
+		Name:  g.Name,
+		File:  g.File,
+		API:   g.API,
+		Tests: g.Tests,
+	}
+	if len(g.Tests) > limit {
+		out.Tests = g.Tests[:limit]
+	}
+	return out
+}
+
 // Lists is the full list of tests to be run.
 type Lists []Group