Regres: Add support for generating collated coverage info

tests/regres/cmd/regres/main.go

@@ -147,8 +147,8 @@ func (r *regres) resolveDirs() error {
 	}
 
 	for _, path := range allDirs {
-		if _, err := os.Stat(*path); err != nil {
-			return cause.Wrap(err, "Couldn't find path '%v'", *path)
+		if !util.IsDir(*path) {
+			return fmt.Errorf("Couldn't find path '%v'", *path)
 		}
 	}
 
@@ -917,12 +917,12 @@ func (t *test) run(testLists testlist.Lists, d deqpBuild) (*deqp.Results, error)
 	log.Printf("Running tests for '%s'\n", t.commit)
 
 	swiftshaderICDSo := filepath.Join(t.buildDir, "libvk_swiftshader.so")
-	if _, err := os.Stat(swiftshaderICDSo); err != nil {
+	if !util.IsFile(swiftshaderICDSo) {
 		return nil, fmt.Errorf("Couldn't find '%s'", swiftshaderICDSo)
 	}
 
 	swiftshaderICDJSON := filepath.Join(t.buildDir, "Linux", "vk_swiftshader_icd.json")
-	if _, err := os.Stat(swiftshaderICDJSON); err != nil {
+	if !util.IsFile(swiftshaderICDJSON) {
 		return nil, fmt.Errorf("Couldn't find '%s'", swiftshaderICDJSON)
 	}
 

tests/regres/cmd/run_testlist/main.go

@@ -21,9 +21,12 @@
 package main
 
 import (
+	"bytes"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"math/rand"
 	"os"
@@ -33,9 +36,12 @@ import (
 	"strings"
 	"time"
 
+	"../../cov"
 	"../../deqp"
+	"../../llvm"
 	"../../shell"
 	"../../testlist"
+	"../../util"
 )
 
 var (
@@ -48,6 +54,7 @@ var (
 	limit         = flag.Int("limit", 0, "only run a maximum of this number of tests")
 	shuffle       = flag.Bool("shuffle", false, "shuffle tests")
 	noResults     = flag.Bool("no-results", false, "disable generation of results.json file")
+	genCoverage   = flag.Bool("coverage", false, "generate test coverage")
 )
 
 const testTimeout = time.Minute * 2
@@ -98,6 +105,15 @@ func run() error {
 		TestTimeout:      testTimeout,
 	}
 
+	if *genCoverage {
+		icdPath := findSwiftshaderICD()
+		config.CoverageEnv = &cov.Env{
+			LLVM:    findLLVMToolchain(icdPath),
+			RootDir: projectRootDir(),
+			ExePath: findSwiftshaderSO(icdPath),
+		}
+	}
+
 	res, err := config.Run()
 	if err != nil {
 		return err
@@ -113,6 +129,12 @@ func run() error {
 		}
 	}
 
+	if *genCoverage {
+		if err := ioutil.WriteFile("coverage.json", []byte(res.Coverage.JSON()), 0666); err != nil {
+			return err
+		}
+	}
+
 	if !*noResults {
 		err = res.Save(*output)
 		if err != nil {
@@ -123,6 +145,80 @@ func run() error {
 	return nil
 }
 
+func findSwiftshaderICD() string {
+	icdPaths := strings.Split(os.Getenv("VK_ICD_FILENAMES"), ";")
+	for _, icdPath := range icdPaths {
+		_, file := filepath.Split(icdPath)
+		if file == "vk_swiftshader_icd.json" {
+			return icdPath
+		}
+	}
+	panic("Cannot find vk_swiftshader_icd.json in VK_ICD_FILENAMES")
+}
+
+func findSwiftshaderSO(vkSwiftshaderICD string) string {
+	root := struct {
+		ICD struct {
+			Path string `json:"library_path"`
+		}
+	}{}
+
+	icd, err := ioutil.ReadFile(vkSwiftshaderICD)
+	if err != nil {
+		panic(fmt.Errorf("Could not read '%v'. %v", vkSwiftshaderICD, err))
+	}
+
+	if err := json.NewDecoder(bytes.NewReader(icd)).Decode(&root); err != nil {
+		panic(fmt.Errorf("Could not parse '%v'. %v", vkSwiftshaderICD, err))
+	}
+
+	if util.IsFile(root.ICD.Path) {
+		return root.ICD.Path
+	}
+	dir := filepath.Dir(vkSwiftshaderICD)
+	path, err := filepath.Abs(filepath.Join(dir, root.ICD.Path))
+	if err != nil {
+		panic(fmt.Errorf("Could not locate ICD so at '%v'. %v", root.ICD.Path, err))
+	}
+
+	return path
+}
+
+func findLLVMToolchain(vkSwiftshaderICD string) llvm.Toolchain {
+	minVersion := llvm.Version{Major: 8}
+
+	// Try finding the llvm toolchain via the CMake generated
+	// coverage-toolchain.txt file that sits next to vk_swiftshader_icd.json.
+	dir := filepath.Dir(vkSwiftshaderICD)
+	toolchainInfoPath := filepath.Join(dir, "coverage-toolchain.txt")
+	if util.IsFile(toolchainInfoPath) {
+		if body, err := ioutil.ReadFile(toolchainInfoPath); err == nil {
+			toolchain := llvm.Search(string(body)).FindAtLeast(minVersion)
+			if toolchain != nil {
+				return *toolchain
+			}
+		}
+	}
+
+	// Fallback, try searching PATH.
+	toolchain := llvm.Search().FindAtLeast(llvm.Version{Major: 8})
+	if toolchain == nil {
+		log.Fatal("Could not find LLVM toolchain")
+	}
+
+	return *toolchain
+}
+
+func projectRootDir() string {
+	_, thisFile, _, _ := runtime.Caller(1)
+	thisDir := filepath.Dir(thisFile)
+	root, err := filepath.Abs(filepath.Join(thisDir, "../../../.."))
+	if err != nil {
+		panic(err)
+	}
+	return root
+}
+
 func main() {
 	flag.ErrHelp = errors.New("regres is a tool to detect regressions between versions of SwiftShader")
 	flag.Parse()

tests/regres/cov/coverage_test.go

+// Copyright 2020 The SwiftShader Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cov_test
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+
+	cov "."
+)
+
+var (
+	fileA = "coverage/file/a"
+	fileB = "coverage/file/b"
+	fileC = "coverage/file/c"
+	fileD = "coverage/file/c"
+
+	span0 = cov.Span{cov.Location{3, 2}, cov.Location{3, 9}}
+	span1 = cov.Span{cov.Location{4, 1}, cov.Location{5, 1}}
+	span2 = cov.Span{cov.Location{5, 5}, cov.Location{5, 7}}
+	span3 = cov.Span{cov.Location{7, 2}, cov.Location{7, 7}}
+)
+
+//                a
+//        ╭───────┴───────╮
+//        b               c
+//    ╭───┴───╮       ╭───┴───╮
+//    d       e       f       g
+//  ╭─┴─╮   ╭─┴─╮   ╭─┴─╮   ╭─┴─╮
+//  h   i   j   k   l   m   n   o
+//     ╭┴╮ ╭┴╮ ╭┴╮ ╭┴╮ ╭┴╮ ╭╯
+//     p q r s t u v w x y z
+//
+
+func TestTree(t *testing.T) {
+	tree := &cov.Tree{}
+
+	t.Log("Add 'b' with the coverage [0,1]")
+	tree.Add(cov.Path{"a", "b"}, coverage(fileA, span0, span1))
+
+	//           [0,1]
+	//            (a)
+	//       ╭─────╯
+	//       b
+
+	checkSpans(t, tree.Spans(), span0, span1)
+	checkTests(t, tree, `{a:{b}}`)
+	checkCoverage(t, tree, fileA, `a:{[0,1]}`)
+
+	t.Log("Add 'i' with the coverage [0,1]")
+	tree.Add(cov.Path{"a", "b", "d", "i"}, coverage(fileA, span0, span1))
+
+	//           [0,1]
+	//            (a)
+	//       ╭─────╯
+	//       b
+	//    ╭──╯
+	//    d
+	//    ╰─╮
+	//      i
+	checkSpans(t, tree.Spans(), span0, span1)
+	checkTests(t, tree, `{a:{b:{d:{i}}}}`)
+	checkCoverage(t, tree, fileA, `a:{[0,1]}`)
+
+	t.Log("Add 'e' with the coverage [0,1,2]")
+	tree.Add(cov.Path{"a", "b", "e"}, coverage(fileA, span0, span1, span2))
+
+	//           [0,1]
+	//            (a)
+	//       ┏━━━━━┛
+	//      (b)
+	//    ╭──┺━━┓
+	//    d    (e)[2]
+	//    ╰─╮
+	//      i
+	checkSpans(t, tree.Spans(), span0, span1, span2)
+	checkTests(t, tree, `{a:{b:{d:{i} e}}}`)
+	checkCoverage(t, tree, fileA, `a:{[0,1] b:{[] e:{[2]}}}`)
+
+	t.Log("Add 'n' with the coverage [0,3]")
+	tree.Add(cov.Path{"a", "c", "g", "n"}, coverage(fileA, span0, span3))
+
+	//            [0]
+	//            (a)
+	//       ┏━━━━━┻━━━━━┓
+	//   [1](b)         (c)[3]
+	//    ╭──┺━━┓        ╰──╮
+	//    d    (e)[2]       g
+	//    ╰─╮             ╭─╯
+	//      i             n
+	checkSpans(t, tree.Spans(), span0, span1, span2, span3)
+	checkTests(t, tree, `{a:{b:{d:{i}e}c:{g:{n}}}}`)
+	checkCoverage(t, tree, fileA, `a:{[0] b:{[1] e:{[2]}} c:{[3]}}`)
+
+	t.Log("Add 'o' with the coverage [0, 3]")
+	tree.Add(cov.Path{"a", "c", "g", "o"}, coverage(fileA, span0, span3))
+
+	//              [0]
+	//              (a)
+	//       ┏━━━━━━━┻━━━━━━━┓
+	//   [1](b)             (c)[3]
+	//    ╭──┺━━┓            ╰──╮
+	//    d    (e)[2]           g
+	//    ╰─╮                 ╭─┴─╮
+	//      i                 n   o
+	checkSpans(t, tree.Spans(), span0, span1, span2, span3)
+	checkTests(t, tree, `{a:{b:{d:{i}e}c:{g:{n o}}}}`)
+	checkCoverage(t, tree, fileA, `a:{[0] b:{[1] e:{[2]}} c:{[3]}}`)
+
+	t.Log("Add 'f' with the coverage [1]")
+	tree.Add(cov.Path{"a", "c", "f"}, coverage(fileA, span1))
+
+	//               (a)
+	//       ┏━━━━━━━━┻━━━━━━━━┓
+	// [0,1](b)               (c)
+	//    ╭──┺━━┓           ┏━━┻━━┓
+	//    d    (e)[2]   [1](f)   (g)[0,3]
+	//    ╰─╮                   ╭─┴─╮
+	//      i                   n   o
+	checkSpans(t, tree.Spans(), span0, span1, span2, span3)
+	checkTests(t, tree, `{a:{b:{d:{i} e} c:{f g:{n o}}}}`)
+	checkCoverage(t, tree, fileA, `a:{[] b:{[0,1] e:{[2]}} c:{[] f:{[1]} g:{[0,3]}}}`)
+
+	t.Log("Add 'j' with the coverage [3]")
+	tree.Add(cov.Path{"a", "b", "e", "j"}, coverage(fileA, span3))
+
+	//                   (a)
+	//           ┏━━━━━━━━┻━━━━━━━━┓
+	//          (b)               (c)
+	//       ┏━━━┻━━━┓          ┏━━┻━━┓
+	// [0,1](d)     (e)[3]  [1](f)   (g)[0,3]
+	//       ╰─╮   ╭─╯              ╭─┴─╮
+	//         i   j                n   o
+	checkSpans(t, tree.Spans(), span0, span1, span2, span3)
+	checkTests(t, tree, `{a:{b:{d:{i} e:{j}} c:{f g:{n o}}}}`)
+	checkCoverage(t, tree, fileA, `a:{[] b:{[] d:{[0,1]} e:{[3]}} c:{[] f:{[1]} g:{[0,3]}}}`)
+
+	t.Log("Add 'k' with the coverage [3]")
+	tree.Add(cov.Path{"a", "b", "e", "k"}, coverage(fileA, span3))
+
+	//                   (a)
+	//           ┏━━━━━━━━┻━━━━━━━━┓
+	//          (b)               (c)
+	//       ┏━━━┻━━━┓          ┏━━┻━━┓
+	// [0,1](d)     (e)[3]  [1](f)   (g)[0,3]
+	//       ╰─╮   ╭─┴─╮            ╭─┴─╮
+	//         i   j   k            n   o
+	checkSpans(t, tree.Spans(), span0, span1, span2, span3)
+	checkTests(t, tree, `{a:{b:{d:{i} e:{j k}} c:{f g:{n o}}}}`)
+	checkCoverage(t, tree, fileA, `a:{[] b:{[] d:{[0,1]} e:{[3]}} c:{[] f:{[1]} g:{[0,3]}}}`)
+
+	t.Log("Add 'v' with the coverage [1,2]")
+	tree.Add(cov.Path{"a", "c", "f", "l", "v"}, coverage(fileA, span1, span2))
+
+	//                   (a)
+	//           ┏━━━━━━━━┻━━━━━━━━━━┓
+	//          (b)                 (c)
+	//       ┏━━━┻━━━┓            ┏━━┻━━┓
+	// [0,1](d)     (e)[3]  [1,2](f)   (g)[0,3]
+	//       ╰─╮   ╭─┴─╮        ╭─╯   ╭─┴─╮
+	//         i   j   k        l     n   o
+	//                         ╭╯
+	//                         v
+	checkSpans(t, tree.Spans(), span0, span1, span2, span3)
+	checkTests(t, tree, `{a:{b:{d:{i} e:{j k}} c:{f:{l:{v}} g:{n o}}}}`)
+	checkCoverage(t, tree, fileA, `a:{[] b:{[] d:{[0,1]} e:{[3]}} c:{[] f:{[1,2]} g:{[0,3]}}}`)
+
+	t.Log("Add 'x' with the coverage [1,2]")
+	tree.Add(cov.Path{"a", "c", "f", "l", "x"}, coverage(fileA, span1, span2))
+
+	//                   (a)
+	//           ┏━━━━━━━━┻━━━━━━━━━━┓
+	//          (b)                 (c)
+	//       ┏━━━┻━━━┓            ┏━━┻━━┓
+	// [0,1](d)     (e)[3]  [1,2](f)   (g)[0,3]
+	//       ╰─╮   ╭─┴─╮        ╭─╯   ╭─┴─╮
+	//         i   j   k        l     n   o
+	//                         ╭┴╮
+	//                         v x
+	checkSpans(t, tree.Spans(), span0, span1, span2, span3)
+	checkTests(t, tree, `{a:{b:{d:{i} e:{j k}} c:{f:{l:{v x}} g:{n o}}}}`)
+	checkCoverage(t, tree, fileA, `a:{[] b:{[] d:{[0,1]} e:{[3]}} c:{[] f:{[1,2]} g:{[0,3]}}}`)
+
+	t.Log("Add 'z' with the coverage [2]")
+	tree.Add(cov.Path{"a", "c", "g", "n", "z"}, coverage(fileA, span2))
+
+	//                   (a)
+	//           ┏━━━━━━━━┻━━━━━━━━━━━━┓
+	//          (b)                   (c)
+	//       ┏━━━┻━━━┓            ┏━━━━┻━━━━┓
+	// [0,1](d)     (e)[3]  [1,2](f)       (g)
+	//       ╰─╮   ╭─┴─╮        ╭─╯       ┏━┻━┓
+	//         i   j   k        l    [2](n) (o)[0,3]
+	//                         ╭┴╮      ╭╯
+	//                         v x      z
+	checkSpans(t, tree.Spans(), span0, span1, span2, span3)
+	checkTests(t, tree, `{a:{b:{d:{i} e:{j k}} c:{f:{l:{v x}} g:{n: {z} o}}}}`)
+	checkCoverage(t, tree, fileA, `a:{[] b:{[] d:{[0,1]} e:{[3]}} c:{[] f:{[1,2]} g:{[] n:{[2]} o:{[0,3]}}}}`)
+}
+
+func checkSpans(t *testing.T, got []cov.Span, expect ...cov.Span) {
+	if !reflect.DeepEqual(got, expect) {
+		t.Errorf("Spans not as expected.\nGot:    %+v\nExpect: %+v", got, expect)
+	}
+}
+
+func checkTests(t *testing.T, tree *cov.Tree, expect string) {
+	g, e := tree.Tests().String(tree.Strings()), expect
+	if tg, te := trimWS(g), trimWS(e); tg != te {
+		t.Errorf("Tests not as expected.\nGot:\n%v\nExpect:\n%v\n------\nGot:    %v\nExpect: %v", g, e, tg, te)
+	}
+}
+
+func checkCoverage(t *testing.T, tree *cov.Tree, file string, expect string) {
+	g, e := tree.File(file).String(tree.Tests(), tree.Strings()), expect
+	if tg, te := trimWS(g), trimWS(e); tg != te {
+		t.Errorf("Coverage not as expected.\nGot:\n%v\nExpect:\n%v\n------\nGot:    %v\nExpect: %v", g, e, tg, te)
+	}
+}
+
+func trimWS(s string) string {
+	s = strings.ReplaceAll(s, " ", "")
+	s = strings.ReplaceAll(s, "\n", "")
+	return s
+}
+
+func coverage(file string, spans ...cov.Span) *cov.Coverage {
+	return &cov.Coverage{
+		[]cov.File{
+			cov.File{
+				Path:  file,
+				Spans: spans,
+			},
+		},
+	}
+}
+
+func spans(ids ...cov.SpanID) cov.SpanSet {
+	out := make(cov.SpanSet, len(ids))
+	for _, id := range ids {
+		out[id] = struct{}{}
+	}
+	return out
+}

tests/regres/deqp/deqp.go

@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"math/rand"
 	"os"
@@ -31,6 +32,7 @@ import (
 	"time"
 
 	"../cause"
+	"../cov"
 	"../shell"
 	"../testlist"
 	"../util"
@@ -59,10 +61,12 @@ type Config struct {
 	ExeGles2         string
 	ExeGles3         string
 	ExeVulkan        string
+	TempDir          string // Directory for temporary log files, coverage output.
 	TestLists        testlist.Lists
 	Env              []string
 	LogReplacements  map[string]string
 	NumParallelTests int
+	CoverageEnv      *cov.Env
 	TestTimeout      time.Duration
 }
 
@@ -72,6 +76,7 @@ type Results struct {
 	Version  int
 	Error    string
 	Tests    map[string]TestResult
+	Coverage *cov.Tree
 	Duration time.Duration
 }
 
@@ -81,6 +86,7 @@ type TestResult struct {
 	Status    testlist.Status
 	TimeTaken time.Duration
 	Err       string `json:",omitempty"`
+	Coverage  *cov.Coverage
 }
 
 func (r TestResult) String() string {
@@ -131,9 +137,16 @@ func (r *Results) Save(path string) error {
 
 // Run runs all the tests.
 func (c *Config) Run() (*Results, error) {
-
 	start := time.Now()
 
+	if c.TempDir == "" {
+		dir, err := ioutil.TempDir("", "deqp")
+		if err != nil {
+			return nil, cause.Wrap(err, "Could not generate temporary directory")
+		}
+		c.TempDir = dir
+	}
+
 	// Wait group that completes once all the tests have finished.
 	wg := sync.WaitGroup{}
 	results := make(chan TestResult, 256)
@@ -200,6 +213,10 @@ func (c *Config) Run() (*Results, error) {
 		Tests:   map[string]TestResult{},
 	}
 
+	if c.CoverageEnv != nil {
+		out.Coverage = &cov.Tree{}
+	}
+
 	// Collect the results.
 	finished := make(chan struct{})
 	lastUpdate := time.Now()
@@ -207,7 +224,6 @@ func (c *Config) Run() (*Results, error) {
 		start, i := time.Now(), 0
 		for r := range results {
 			i++
-			out.Tests[r.Test] = r
 			if time.Since(lastUpdate) > time.Minute {
 				lastUpdate = time.Now()
 				remaining := numTests - i
@@ -215,6 +231,12 @@ func (c *Config) Run() (*Results, error) {
 					i, numTests, util.Percent(i, numTests),
 					(time.Since(start)/time.Duration(i))*time.Duration(remaining))
 			}
+			out.Tests[r.Test] = r
+			if r.Coverage != nil {
+				path := strings.Split(r.Test, ".")
+				out.Coverage.Add(cov.Path(path), r.Coverage)
+				r.Coverage = nil // Free memory
+			}
 		}
 		close(finished)
 	}()
@@ -263,6 +285,12 @@ func (c *Config) TestRoutine(exe string, tests <-chan string, results chan<- Tes
 		env = append(env, v)
 	}
 
+	coverageFile := filepath.Join(c.TempDir, fmt.Sprintf("%v.profraw", goroutineIndex))
+	if c.CoverageEnv != nil {
+		env = cov.AppendRuntimeEnv(env, coverageFile)
+	}
+	logPath := filepath.Join(c.TempDir, fmt.Sprintf("%v.log", goroutineIndex))
+
 nextTest:
 	for name := range tests {
 		// log.Printf("Running test '%s'\n", name)
@@ -274,6 +302,7 @@ nextTest:
 			"--deqp-log-images=disable",
 			"--deqp-log-shader-sources=disable",
 			"--deqp-log-flush=disable",
+			"--deqp-log-filename="+logPath,
 			"-n="+name)
 		duration := time.Since(start)
 		out := string(outRaw)
@@ -282,6 +311,15 @@ nextTest:
 			out = strings.ReplaceAll(out, k, v)
 		}
 
+		var coverage *cov.Coverage
+		if c.CoverageEnv != nil {
+			coverage, err = c.CoverageEnv.Import(coverageFile)
+			if err != nil {
+				log.Printf("Warning: Failed to get test coverage for test '%v'. %v", name, err)
+			}
+			os.Remove(coverageFile)
+		}
+
 		for _, test := range []struct {
 			re *regexp.Regexp
 			s  testlist.Status
@@ -298,6 +336,7 @@ nextTest:
 					Status:    test.s,
 					TimeTaken: duration,
 					Err:       s,
+					Coverage:  coverage,
 				}
 				continue nextTest
 			}
@@ -319,6 +358,7 @@ nextTest:
 				Status:    testlist.Crash,
 				TimeTaken: duration,
 				Err:       out,
+				Coverage:  coverage,
 			}
 		case shell.ErrTimeout:
 			log.Printf("Timeout for test '%v'\n", name)
@@ -326,34 +366,35 @@ nextTest:
 				Test:      name,
 				Status:    testlist.Timeout,
 				TimeTaken: duration,
+				Coverage:  coverage,
 			}
 		case nil:
 			toks := deqpRE.FindStringSubmatch(out)
 			if len(toks) < 3 {
 				err := fmt.Sprintf("Couldn't parse test '%v' output:\n%s", name, out)
 				log.Println("Warning: ", err)
-				results <- TestResult{Test: name, Status: testlist.Fail, Err: err}
+				results <- TestResult{Test: name, Status: testlist.Fail, Err: err, Coverage: coverage}
 				continue
 			}
 			switch toks[1] {
 			case "Pass":
-				results <- TestResult{Test: name, Status: testlist.Pass, TimeTaken: duration}
+				results <- TestResult{Test: name, Status: testlist.Pass, TimeTaken: duration, Coverage: coverage}
 			case "NotSupported":
-				results <- TestResult{Test: name, Status: testlist.NotSupported, TimeTaken: duration}
+				results <- TestResult{Test: name, Status: testlist.NotSupported, TimeTaken: duration, Coverage: coverage}
 			case "CompatibilityWarning":
-				results <- TestResult{Test: name, Status: testlist.CompatibilityWarning, TimeTaken: duration}
+				results <- TestResult{Test: name, Status: testlist.CompatibilityWarning, TimeTaken: duration, Coverage: coverage}
 			case "QualityWarning":
-				results <- TestResult{Test: name, Status: testlist.QualityWarning, TimeTaken: duration}
+				results <- TestResult{Test: name, Status: testlist.QualityWarning, TimeTaken: duration, Coverage: coverage}
 			case "Fail":
 				var err string
 				if toks[2] != "Fail" {
 					err = toks[2]
 				}
-				results <- TestResult{Test: name, Status: testlist.Fail, Err: err, TimeTaken: duration}
+				results <- TestResult{Test: name, Status: testlist.Fail, Err: err, TimeTaken: duration, Coverage: coverage}
 			default:
 				err := fmt.Sprintf("Couldn't parse test output:\n%s", out)
 				log.Println("Warning: ", err)
-				results <- TestResult{Test: name, Status: testlist.Fail, Err: err, TimeTaken: duration}
+				results <- TestResult{Test: name, Status: testlist.Fail, Err: err, TimeTaken: duration, Coverage: coverage}
 			}
 		}
 	}

tests/regres/llvm/llvm.go

+// Copyright 2020 The SwiftShader Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package llvm provides functions and types for locating and using the llvm
+// toolchains.
+package llvm
+
+import (
+	"fmt"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"sort"
+	"strconv"
+
+	"../util"
+)
+
+const maxLLVMVersion = 10
+
+// Version holds the build version information of an LLVM toolchain.
+type Version struct {
+	Major, Minor, Point int
+}
+
+// GreaterEqual returns true if v >= rhs.
+func (v Version) GreaterEqual(rhs Version) bool {
+	if v.Major > rhs.Major {
+		return true
+	}
+	if v.Major < rhs.Major {
+		return false
+	}
+	if v.Minor > rhs.Minor {
+		return true
+	}
+	if v.Minor < rhs.Minor {
+		return false
+	}
+	return v.Point >= rhs.Point
+}
+
+// Toolchain holds the paths and version information about an LLVM toolchain.
+type Toolchain struct {
+	Version Version
+	BinDir  string
+}
+
+// Toolchains is a list of Toolchain
+type Toolchains []Toolchain
+
+// FindAtLeast looks for a toolchain with the given version, returning the highest found version.
+func (l Toolchains) FindAtLeast(v Version) *Toolchain {
+	out := (*Toolchain)(nil)
+	for _, t := range l {
+		if t.Version.GreaterEqual(v) && (out == nil || out.Version.GreaterEqual(t.Version)) {
+			t := t
+			out = &t
+		}
+	}
+	return out
+}
+
+// Search looks for llvm toolchains in paths.
+// If paths is empty, then PATH is searched.
+func Search(paths ...string) Toolchains {
+	toolchains := map[Version]Toolchain{}
+	search := func(name string) {
+		if len(paths) > 0 {
+			for _, path := range paths {
+				if util.IsFile(path) {
+					path = filepath.Dir(path)
+				}
+				if t := toolchain(path); t != nil {
+					toolchains[t.Version] = *t
+					continue
+				}
+				if t := toolchain(filepath.Join(path, "bin")); t != nil {
+					toolchains[t.Version] = *t
+					continue
+				}
+			}
+		} else {
+			path, err := exec.LookPath(name)
+			if err == nil {
+				if t := toolchain(filepath.Dir(path)); t != nil {
+					toolchains[t.Version] = *t
+				}
+			}
+		}
+	}
+
+	search("clang")
+	for i := 8; i < maxLLVMVersion; i++ {
+		search(fmt.Sprintf("clang-%d", i))
+	}
+
+	out := make([]Toolchain, 0, len(toolchains))
+	for _, t := range toolchains {
+		out = append(out, t)
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Version.GreaterEqual(out[j].Version) })
+
+	return out
+}
+
+// Cov returns the path to the llvm-cov executable.
+func (t Toolchain) Cov() string {
+	return filepath.Join(t.BinDir, "llvm-cov"+exeExt())
+}
+
+// Profdata returns the path to the llvm-profdata executable.
+func (t Toolchain) Profdata() string {
+	return filepath.Join(t.BinDir, "llvm-profdata"+exeExt())
+}
+
+func toolchain(dir string) *Toolchain {
+	t := Toolchain{BinDir: dir}
+	if t.resolve() {
+		return &t
+	}
+	return nil
+}
+
+func (t *Toolchain) resolve() bool {
+	if !util.IsFile(t.Profdata()) { // llvm-profdata doesn't have --version flag
+		return false
+	}
+	version, ok := parseVersion(t.Cov())
+	t.Version = version
+	return ok
+}
+
+func exeExt() string {
+	switch runtime.GOOS {
+	case "windows":
+		return ".exe"
+	default:
+		return ""
+	}
+}
+
+var versionRE = regexp.MustCompile(`(?:clang|LLVM) version ([0-9]+)\.([0-9]+)\.([0-9]+)`)
+
+func parseVersion(tool string) (Version, bool) {
+	out, err := exec.Command(tool, "--version").Output()
+	if err != nil {
+		return Version{}, false
+	}
+	matches := versionRE.FindStringSubmatch(string(out))
+	if len(matches) < 4 {
+		return Version{}, false
+	}
+	major, majorErr := strconv.Atoi(matches[1])
+	minor, minorErr := strconv.Atoi(matches[2])
+	point, pointErr := strconv.Atoi(matches[3])
+	if majorErr != nil || minorErr != nil || pointErr != nil {
+		return Version{}, false
+	}
+	return Version{major, minor, point}, true
+}