Regres: Add support for generating collated coverage info

tests/regres/cmd/regres/main.go

@@ -147,8 +147,8 @@ func (r *regres) resolveDirs() error {
 	}
 
 	for _, path := range allDirs {
-		if _, err := os.Stat(*path); err != nil {
-			return cause.Wrap(err, "Couldn't find path '%v'", *path)
+		if !util.IsDir(*path) {
+			return fmt.Errorf("Couldn't find path '%v'", *path)
 		}
 	}
 
@@ -917,12 +917,12 @@ func (t *test) run(testLists testlist.Lists, d deqpBuild) (*deqp.Results, error)
 	log.Printf("Running tests for '%s'\n", t.commit)
 
 	swiftshaderICDSo := filepath.Join(t.buildDir, "libvk_swiftshader.so")
-	if _, err := os.Stat(swiftshaderICDSo); err != nil {
+	if !util.IsFile(swiftshaderICDSo) {
 		return nil, fmt.Errorf("Couldn't find '%s'", swiftshaderICDSo)
 	}
 
 	swiftshaderICDJSON := filepath.Join(t.buildDir, "Linux", "vk_swiftshader_icd.json")
-	if _, err := os.Stat(swiftshaderICDJSON); err != nil {
+	if !util.IsFile(swiftshaderICDJSON) {
 		return nil, fmt.Errorf("Couldn't find '%s'", swiftshaderICDJSON)
 	}
 

tests/regres/cmd/run_testlist/main.go

@@ -21,9 +21,12 @@
 package main
 
 import (
+	"bytes"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"math/rand"
 	"os"
@@ -33,9 +36,12 @@ import (
 	"strings"
 	"time"
 
+	"../../cov"
 	"../../deqp"
+	"../../llvm"
 	"../../shell"
 	"../../testlist"
+	"../../util"
 )
 
 var (
@@ -48,6 +54,7 @@ var (
 	limit         = flag.Int("limit", 0, "only run a maximum of this number of tests")
 	shuffle       = flag.Bool("shuffle", false, "shuffle tests")
 	noResults     = flag.Bool("no-results", false, "disable generation of results.json file")
+	genCoverage   = flag.Bool("coverage", false, "generate test coverage")
 )
 
 const testTimeout = time.Minute * 2
@@ -98,6 +105,15 @@ func run() error {
 		TestTimeout:      testTimeout,
 	}
 
+	if *genCoverage {
+		icdPath := findSwiftshaderICD()
+		config.CoverageEnv = &cov.Env{
+			LLVM:    findLLVMToolchain(icdPath),
+			RootDir: projectRootDir(),
+			ExePath: findSwiftshaderSO(icdPath),
+		}
+	}
+
 	res, err := config.Run()
 	if err != nil {
 		return err
@@ -113,6 +129,12 @@ func run() error {
 		}
 	}
 
+	if *genCoverage {
+		if err := ioutil.WriteFile("coverage.json", []byte(res.Coverage.JSON()), 0666); err != nil {
+			return err
+		}
+	}
+
 	if !*noResults {
 		err = res.Save(*output)
 		if err != nil {
@@ -123,6 +145,80 @@ func run() error {
 	return nil
 }
 
+func findSwiftshaderICD() string {
+	icdPaths := strings.Split(os.Getenv("VK_ICD_FILENAMES"), ";")
+	for _, icdPath := range icdPaths {
+		_, file := filepath.Split(icdPath)
+		if file == "vk_swiftshader_icd.json" {
+			return icdPath
+		}
+	}
+	panic("Cannot find vk_swiftshader_icd.json in VK_ICD_FILENAMES")
+}
+
+func findSwiftshaderSO(vkSwiftshaderICD string) string {
+	root := struct {
+		ICD struct {
+			Path string `json:"library_path"`
+		}
+	}{}
+
+	icd, err := ioutil.ReadFile(vkSwiftshaderICD)
+	if err != nil {
+		panic(fmt.Errorf("Could not read '%v'. %v", vkSwiftshaderICD, err))
+	}
+
+	if err := json.NewDecoder(bytes.NewReader(icd)).Decode(&root); err != nil {
+		panic(fmt.Errorf("Could not parse '%v'. %v", vkSwiftshaderICD, err))
+	}
+
+	if util.IsFile(root.ICD.Path) {
+		return root.ICD.Path
+	}
+	dir := filepath.Dir(vkSwiftshaderICD)
+	path, err := filepath.Abs(filepath.Join(dir, root.ICD.Path))
+	if err != nil {
+		panic(fmt.Errorf("Could not locate ICD so at '%v'. %v", root.ICD.Path, err))
+	}
+
+	return path
+}
+
+func findLLVMToolchain(vkSwiftshaderICD string) llvm.Toolchain {
+	minVersion := llvm.Version{Major: 8}
+
+	// Try finding the llvm toolchain via the CMake generated
+	// coverage-toolchain.txt file that sits next to vk_swiftshader_icd.json.
+	dir := filepath.Dir(vkSwiftshaderICD)
+	toolchainInfoPath := filepath.Join(dir, "coverage-toolchain.txt")
+	if util.IsFile(toolchainInfoPath) {
+		if body, err := ioutil.ReadFile(toolchainInfoPath); err == nil {
+			toolchain := llvm.Search(string(body)).FindAtLeast(minVersion)
+			if toolchain != nil {
+				return *toolchain
+			}
+		}
+	}
+
+	// Fallback, try searching PATH.
+	toolchain := llvm.Search().FindAtLeast(llvm.Version{Major: 8})
+	if toolchain == nil {
+		log.Fatal("Could not find LLVM toolchain")
+	}
+
+	return *toolchain
+}
+
+func projectRootDir() string {
+	_, thisFile, _, _ := runtime.Caller(1)
+	thisDir := filepath.Dir(thisFile)
+	root, err := filepath.Abs(filepath.Join(thisDir, "../../../.."))
+	if err != nil {
+		panic(err)
+	}
+	return root
+}
+
 func main() {
 	flag.ErrHelp = errors.New("regres is a tool to detect regressions between versions of SwiftShader")
 	flag.Parse()

tests/regres/cov/coverage.go

+// Copyright 2020 The SwiftShader Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package cov provides functions for consuming and combining llvm coverage
+// information from multiple processes.
+package cov
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"../cause"
+	"../llvm"
+)
+
+// Location describes a single line-column position in a source file.
+type Location struct {
+	Line, Column int
+}
+
+func (l Location) String() string {
+	return fmt.Sprintf("%v:%v", l.Line, l.Column)
+}
+
+// Span describes a start and end interval in a source file.
+type Span struct {
+	Start, End Location
+}
+
+func (l Span) String() string {
+	return fmt.Sprintf("%v-%v", l.Start, l.End)
+}
+
+// File describes the coverage spans in a single source file.
+type File struct {
+	Path  string
+	Spans []Span
+}
+
+// Coverage describes the coverage spans for all the source files for a single
+// process invocation.
+type Coverage struct {
+	Files []File
+}
+
+// Env holds the enviroment settings for performing coverage processing.
+type Env struct {
+	LLVM    llvm.Toolchain
+	RootDir string // path to SwiftShader git root directory
+	ExePath string // path to the executable binary
+}
+
+// AppendRuntimeEnv returns the environment variables env with the
+// LLVM_PROFILE_FILE environment variable appended.
+func AppendRuntimeEnv(env []string, coverageFile string) []string {
+	return append(env, "LLVM_PROFILE_FILE="+coverageFile)
+}
+
+// Import uses the llvm-profdata and llvm-cov tools to import the coverage
+// information from a .profraw file.
+func (e Env) Import(profrawPath string) (*Coverage, error) {
+	profdata := profrawPath + ".profdata"
+
+	if err := exec.Command(e.LLVM.Profdata(), "merge", "-sparse", profrawPath, "-o", profdata).Run(); err != nil {
+		return nil, cause.Wrap(err, "llvm-profdata errored")
+	}
+	defer os.Remove(profdata)
+
+	args := []string{
+		"export",
+		e.ExePath,
+		"-instr-profile=" + profdata,
+		"-format=text",
+	}
+	if e.LLVM.Version.GreaterEqual(llvm.Version{Major: 9}) {
+		// LLVM 9 has new flags that omit stuff we don't care about.
+		args = append(args,
+			"-skip-expansions",
+			"-skip-functions",
+		)
+	}
+	data, err := exec.Command(e.LLVM.Cov(), args...).Output()
+	if err != nil {
+		return nil, cause.Wrap(err, "llvm-cov errored")
+	}
+
+	c, err := e.parse(data)
+	if err != nil {
+		return nil, cause.Wrap(err, "Couldn't parse coverage json data")
+	}
+
+	return c, nil
+}
+
+// https://clang.llvm.org/docs/SourceBasedCodeCoverage.html
+// https://stackoverflow.com/a/56792192
+func (e Env) parse(raw []byte) (*Coverage, error) {
+	// line int, col int, count int64, hasCount bool, isRegionEntry bool
+	type segment []interface{}
+
+	type file struct {
+		// expansions ignored
+		Name     string    `json:"filename"`
+		Segments []segment `json:"segments"`
+		// summary ignored
+	}
+
+	type data struct {
+		Files []file `json:"files"`
+	}
+
+	root := struct {
+		Data []data `json:"data"`
+	}{}
+	err := json.NewDecoder(bytes.NewReader(raw)).Decode(&root)
+	if err != nil {
+		return nil, err
+	}
+
+	c := &Coverage{Files: make([]File, 0, len(root.Data[0].Files))}
+	for _, f := range root.Data[0].Files {
+		relpath, err := filepath.Rel(e.RootDir, f.Name)
+		if err != nil {
+			return nil, err
+		}
+		if strings.HasPrefix(relpath, "..") {
+			continue
+		}
+		file := File{Path: relpath}
+		for sIdx := 0; sIdx+1 < len(f.Segments); sIdx++ {
+			start := Location{(int)(f.Segments[sIdx][0].(float64)), (int)(f.Segments[sIdx][1].(float64))}
+			end := Location{(int)(f.Segments[sIdx+1][0].(float64)), (int)(f.Segments[sIdx+1][1].(float64))}
+			covered := f.Segments[sIdx][2].(float64) != 0
+			if covered {
+				if c := len(file.Spans); c > 0 && file.Spans[c-1].End == start {
+					file.Spans[c-1].End = end
+				} else {
+					file.Spans = append(file.Spans, Span{start, end})
+				}
+			}
+		}
+		if len(file.Spans) > 0 {
+			c.Files = append(c.Files, file)
+		}
+	}
+	return c, nil
+}
+
+// Path is a tree node path formed from a list of strings
+type Path []string
+
+// Tree represents source code coverage across a tree of different processes.
+// Each tree node is addressed by a Path.
+type Tree struct {
+	initialized bool
+	strings     Strings
+	spans       map[Span]SpanID
+	testRoot    Test
+	files       map[string]TestCoverageMap
+}
+
+func (t *Tree) init() {
+	if !t.initialized {
+		t.strings.m = map[string]StringID{}
+		t.spans = map[Span]SpanID{}
+		t.testRoot = newTest()
+		t.files = map[string]TestCoverageMap{}
+		t.initialized = true
+	}
+}
+
+// Spans returns all the spans used by the tree
+func (t *Tree) Spans() []Span {
+	out := make([]Span, 0, len(t.spans))
+	for span := range t.spans {
+		out = append(out, span)
+	}
+	sort.Slice(out, func(i, j int) bool {
+		if out[i].Start.Line < out[j].Start.Line {
+			return true
+		}
+		if out[i].Start.Line > out[j].Start.Line {
+			return false
+		}
+		return out[i].Start.Column < out[j].Start.Column
+	})
+	return out
+}
+
+// File returns the TestCoverageMap for the given file
+func (t *Tree) File(path string) TestCoverageMap {
+	return t.files[path]
+}
+
+// Tests returns the root test
+func (t *Tree) Tests() *Test { return &t.testRoot }
+
+// Strings returns the string table
+func (t *Tree) Strings() Strings { return t.strings }
+
+type indexedTest struct {
+	index   TestIndex
+	created bool
+}
+
+func (t *Tree) index(path Path) []indexedTest {
+	out := make([]indexedTest, len(path))
+	test := &t.testRoot
+	for i, p := range path {
+		name := t.strings.index(p)
+		idx, ok := test.indices[name]
+		if !ok {
+			idx = TestIndex(len(test.children))
+			test.children = append(test.children, newTest())
+			test.indices[name] = idx
+		}
+		out[i] = indexedTest{idx, !ok}
+		test = &test.children[idx]
+	}
+	return out
+}
+
+func (t *Tree) addSpans(spans []Span) SpanSet {
+	out := make(SpanSet, len(spans))
+	for _, s := range spans {
+		id, ok := t.spans[s]
+		if !ok {
+			id = SpanID(len(t.spans))
+			t.spans[s] = id
+		}
+		out[id] = struct{}{}
+	}
+	return out
+}
+
+// Add adds the coverage information cov to the tree node addressed by path.
+func (t *Tree) Add(path Path, cov *Coverage) {
+	t.init()
+
+	tests := t.index(path)
+
+nextFile:
+	// For each file with coverage...
+	for _, file := range cov.Files {
+		// Lookup or create the file's test coverage map
+		tcm, ok := t.files[file.Path]
+		if !ok {
+			tcm = TestCoverageMap{}
+			t.files[file.Path] = tcm
+		}
+
+		// Add all the spans to the map, get the span ids
+		spans := t.addSpans(file.Spans)
+
+		// Starting from the test root, walk down the test tree.
+		test := t.testRoot
+		parent := (*TestCoverage)(nil)
+		for _, indexedTest := range tests {
+			if indexedTest.created {
+				if parent != nil && len(test.children) == 1 {
+					parent.Spans = parent.Spans.add(spans)
+					delete(parent.Children, indexedTest.index)
+				} else {
+					tc := tcm.index(indexedTest.index)
+					tc.Spans = spans
+				}
+				continue nextFile
+			}
+
+			test = test.children[indexedTest.index]
+			tc := tcm.index(indexedTest.index)
+
+			// If the tree node contains spans that are not in this new test,
+			// we need to push those spans down to all the other children.
+			if lower := tc.Spans.sub(spans); len(lower) > 0 {
+				// push into each child node
+				for i := range test.children {
+					child := tc.Children.index(TestIndex(i))
+					child.Spans = child.Spans.add(lower)
+				}
+				// remove from node
+				tc.Spans = tc.Spans.sub(lower)
+			}
+
+			// The spans that are in the new test, but are not part of the tree
+			// node carry propagating down.
+			spans = spans.sub(tc.Spans)
+			if len(spans) == 0 {
+				continue nextFile
+			}
+
+			tcm = tc.Children
+			parent = tc
+		}
+	}
+}
+
+// StringID is an identifier of a string
+type StringID int
+
+// Strings holds a map of string to identifier
+type Strings struct {
+	m map[string]StringID
+	s []string
+}
+
+func (s *Strings) index(str string) StringID {
+	i, ok := s.m[str]
+	if !ok {
+		i = StringID(len(s.s))
+		s.s = append(s.s, str)
+		s.m[str] = i
+	}
+	return i
+}
+
+// TestIndex is an child test index
+type TestIndex int
+
+// Test is an collection of named sub-tests
+type Test struct {
+	indices  map[StringID]TestIndex
+	children []Test
+}
+
+func newTest() Test {
+	return Test{
+		indices: map[StringID]TestIndex{},
+	}
+}
+
+type namedIndex struct {
+	name string
+	idx  TestIndex
+}
+
+func (t Test) byName(s Strings) []namedIndex {
+	out := make([]namedIndex, len(t.children))
+	for id, idx := range t.indices {
+		out[idx] = namedIndex{s.s[id], idx}
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].name < out[j].name })
+	return out
+}
+
+func (t Test) String(s Strings) string {
+	sb := strings.Builder{}
+	for i, n := range t.byName(s) {
+		child := t.children[n.idx]
+		if i > 0 {
+			sb.WriteString(" ")
+		}
+		sb.WriteString(n.name)
+		if len(child.children) > 0 {
+			sb.WriteString(fmt.Sprintf(":%v", child.String(s)))
+		}
+	}
+	return "{" + sb.String() + "}"
+}
+
+// TestCoverage holds the coverage information for a deqp test group / leaf.
+// For example:
+// The deqp test group may hold spans that are common for all children, and may
+// also optionally hold child nodes that describe coverage that differs per
+// child test.
+type TestCoverage struct {
+	Spans    SpanSet
+	Children TestCoverageMap
+}
+
+func (tc TestCoverage) String(t *Test, s Strings) string {
+	sb := strings.Builder{}
+	sb.WriteString(fmt.Sprintf("{%v", tc.Spans))
+	if len(tc.Children) > 0 {
+		sb.WriteString(" ")
+		sb.WriteString(tc.Children.String(t, s))
+	}
+	sb.WriteString("}")
+	return sb.String()
+}
+
+// TestCoverageMap is a map of TestIndex to *TestCoverage.
+type TestCoverageMap map[TestIndex]*TestCoverage
+
+func (tcm TestCoverageMap) String(t *Test, s Strings) string {
+	sb := strings.Builder{}
+	for _, n := range t.byName(s) {
+		if child, ok := tcm[n.idx]; ok {
+			sb.WriteString(fmt.Sprintf("\n%v: %v", n.name, child.String(&t.children[n.idx], s)))
+		}
+	}
+	if sb.Len() > 0 {
+		sb.WriteString("\n")
+	}
+	return indent(sb.String())
+}
+
+func newTestCoverage() *TestCoverage {
+	return &TestCoverage{
+		Children: TestCoverageMap{},
+		Spans:    SpanSet{},
+	}
+}
+
+func (tcm TestCoverageMap) index(idx TestIndex) *TestCoverage {
+	tc, ok := tcm[idx]
+	if !ok {
+		tc = newTestCoverage()
+		tcm[idx] = tc
+	}
+	return tc
+}
+
+// SpanID is an identifier of a span in a Tree.
+type SpanID int
+
+// SpanSet is a set of SpanIDs.
+type SpanSet map[SpanID]struct{}
+
+// List returns the full list of sorted span ids.
+func (s SpanSet) List() []SpanID {
+	out := make([]SpanID, 0, len(s))
+	for span := range s {
+		out = append(out, span)
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i] < out[j] })
+	return out
+}
+
+func (s SpanSet) String() string {
+	sb := strings.Builder{}
+	sb.WriteString(`[`)
+	l := s.List()
+	for i, span := range l {
+		if i > 0 {
+			sb.WriteString(`, `)
+		}
+		sb.WriteString(fmt.Sprintf("%v", span))
+	}
+	sb.WriteString(`]`)
+	return sb.String()
+}
+
+func (s SpanSet) sub(rhs SpanSet) SpanSet {
+	out := make(SpanSet, len(s))
+	for span := range s {
+		if _, found := rhs[span]; !found {
+			out[span] = struct{}{}
+		}
+	}
+	return out
+}
+
+func (s SpanSet) add(rhs SpanSet) SpanSet {
+	out := make(SpanSet, len(s)+len(rhs))
+	for span := range s {
+		out[span] = struct{}{}
+	}
+	for span := range rhs {
+		out[span] = struct{}{}
+	}
+	return out
+}
+
+func indent(s string) string {
+	return strings.TrimSuffix(strings.ReplaceAll(s, "\n", "\n  "), "  ")
+}
+
+// JSON returns the full test tree serialized to JSON.
+func (t *Tree) JSON() string {
+	sb := &strings.Builder{}
+	sb.WriteString(`{`)
+
+	// write the strings
+	sb.WriteString(`"n":[`)
+	for i, s := range t.strings.s {
+		if i > 0 {
+			sb.WriteString(`,`)
+		}
+		sb.WriteString(`"`)
+		sb.WriteString(s)
+		sb.WriteString(`"`)
+	}
+	sb.WriteString(`]`)
+
+	// write the tests
+	sb.WriteString(`,"t":`)
+	t.writeTestJSON(&t.testRoot, sb)
+
+	// write the spans
+	sb.WriteString(`,"s":`)
+	t.writeSpansJSON(sb)
+
+	// write the files
+	sb.WriteString(`,"f":`)
+	t.writeFilesJSON(sb)
+
+	sb.WriteString(`}`)
+	return sb.String()
+}
+
+func (t *Tree) writeTestJSON(test *Test, sb *strings.Builder) {
+	names := map[int]StringID{}
+	for name, idx := range test.indices {
+		names[int(idx)] = name
+	}
+
+	sb.WriteString(`[`)
+	for i, child := range test.children {
+		if i > 0 {
+			sb.WriteString(`,`)
+		}
+		sb.WriteString(`[`)
+		sb.WriteString(fmt.Sprintf("%v,", names[i]))
+		t.writeTestJSON(&child, sb)
+		sb.WriteString(`]`)
+	}
+
+	sb.WriteString(`]`)
+}
+
+func (t *Tree) writeSpansJSON(sb *strings.Builder) {
+	type spanAndID struct {
+		span Span
+		id   SpanID
+	}
+	spans := make([]spanAndID, 0, len(t.spans))
+	for span, id := range t.spans {
+		spans = append(spans, spanAndID{span, id})
+	}
+	sort.Slice(spans, func(i, j int) bool { return spans[i].id < spans[j].id })
+
+	sb.WriteString(`[`)
+	for i, s := range spans {
+		if i > 0 {
+			sb.WriteString(`,`)
+		}
+		span := s.span
+		sb.WriteString(fmt.Sprintf("[%v,%v,%v,%v]",
+			span.Start.Line, span.Start.Column,
+			span.End.Line, span.End.Column))
+	}
+
+	sb.WriteString(`]`)
+}
+
+func (t *Tree) writeFilesJSON(sb *strings.Builder) {
+	paths := make([]string, 0, len(t.files))
+	for path := range t.files {
+		paths = append(paths, path)
+	}
+	sort.Strings(paths)
+
+	sb.WriteString(`{`)
+	for i, path := range paths {
+		if i > 0 {
+			sb.WriteString(`,`)
+		}
+		sb.WriteString(`"`)
+		sb.WriteString(path)
+		sb.WriteString(`":`)
+		t.writeCoverageMapJSON(t.files[path], sb)
+	}
+
+	sb.WriteString(`}`)
+}
+
+func (t *Tree) writeCoverageMapJSON(c TestCoverageMap, sb *strings.Builder) {
+	ids := make([]TestIndex, 0, len(c))
+	for id := range c {
+		ids = append(ids, id)
+	}
+	sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] })
+
+	sb.WriteString(`[`)
+	for i, id := range ids {
+		if i > 0 {
+			sb.WriteString(`,`)
+		}
+
+		sb.WriteString(`[`)
+		sb.WriteString(fmt.Sprintf("%v", id))
+		sb.WriteString(`,`)
+		t.writeCoverageJSON(c[id], sb)
+		sb.WriteString(`]`)
+	}
+	sb.WriteString(`]`)
+}
+
+func (t *Tree) writeCoverageJSON(c *TestCoverage, sb *strings.Builder) {
+	sb.WriteString(`{`)
+	comma := false
+	if len(c.Spans) > 0 {
+		sb.WriteString(`"s":[`)
+		for i, spanID := range c.Spans.List() {
+			if i > 0 {
+				sb.WriteString(`,`)
+			}
+			sb.WriteString(fmt.Sprintf("%v", spanID))
+		}
+		sb.WriteString(`]`)
+		comma = true
+	}
+	if len(c.Children) > 0 {
+		if comma {
+			sb.WriteString(`,`)
+		}
+		sb.WriteString(`"c":`)
+		t.writeCoverageMapJSON(c.Children, sb)
+	}
+	sb.WriteString(`}`)
+}

tests/regres/cov/coverage_test.go

+// Copyright 2020 The SwiftShader Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cov_test
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+
+	cov "."
+)
+
+var (
+	fileA = "coverage/file/a"
+	fileB = "coverage/file/b"
+	fileC = "coverage/file/c"
+	fileD = "coverage/file/c"
+
+	span0 = cov.Span{cov.Location{3, 2}, cov.Location{3, 9}}
+	span1 = cov.Span{cov.Location{4, 1}, cov.Location{5, 1}}
+	span2 = cov.Span{cov.Location{5, 5}, cov.Location{5, 7}}
+	span3 = cov.Span{cov.Location{7, 2}, cov.Location{7, 7}}
+)
+
+//                a
+//        ╭───────┴───────╮
+//        b               c
+//    ╭───┴───╮       ╭───┴───╮
+//    d       e       f       g
+//  ╭─┴─╮   ╭─┴─╮   ╭─┴─╮   ╭─┴─╮
+//  h   i   j   k   l   m   n   o
+//     ╭┴╮ ╭┴╮ ╭┴╮ ╭┴╮ ╭┴╮ ╭╯
+//     p q r s t u v w x y z
+//
+
+func TestTree(t *testing.T) {
+	tree := &cov.Tree{}
+
+	t.Log("Add 'b' with the coverage [0,1]")
+	tree.Add(cov.Path{"a", "b"}, coverage(fileA, span0, span1))
+
+	//           [0,1]
+	//            (a)
+	//       ╭─────╯
+	//       b
+
+	checkSpans(t, tree.Spans(), span0, span1)
+	checkTests(t, tree, `{a:{b}}`)
+	checkCoverage(t, tree, fileA, `a:{[0,1]}`)
+
+	t.Log("Add 'i' with the coverage [0,1]")
+	tree.Add(cov.Path{"a", "b", "d", "i"}, coverage(fileA, span0, span1))
+
+	//           [0,1]
+	//            (a)
+	//       ╭─────╯
+	//       b
+	//    ╭──╯
+	//    d
+	//    ╰─╮
+	//      i
+	checkSpans(t, tree.Spans(), span0, span1)
+	checkTests(t, tree, `{a:{b:{d:{i}}}}`)
+	checkCoverage(t, tree, fileA, `a:{[0,1]}`)
+
+	t.Log("Add 'e' with the coverage [0,1,2]")
+	tree.Add(cov.Path{"a", "b", "e"}, coverage(fileA, span0, span1, span2))
+
+	//           [0,1]
+	//            (a)
+	//       ┏━━━━━┛
+	//      (b)
+	//    ╭──┺━━┓
+	//    d    (e)[2]
+	//    ╰─╮
+	//      i
+	checkSpans(t, tree.Spans(), span0, span1, span2)
+	checkTests(t, tree, `{a:{b:{d:{i} e}}}`)
+	checkCoverage(t, tree, fileA, `a:{[0,1] b:{[] e:{[2]}}}`)
+
+	t.Log("Add 'n' with the coverage [0,3]")
+	tree.Add(cov.Path{"a", "c", "g", "n"}, coverage(fileA, span0, span3))
+
+	//            [0]
+	//            (a)
+	//       ┏━━━━━┻━━━━━┓
+	//   [1](b)         (c)[3]
+	//    ╭──┺━━┓        ╰──╮
+	//    d    (e)[2]       g
+	//    ╰─╮             ╭─╯
+	//      i             n
+	checkSpans(t, tree.Spans(), span0, span1, span2, span3)
+	checkTests(t, tree, `{a:{b:{d:{i}e}c:{g:{n}}}}`)
+	checkCoverage(t, tree, fileA, `a:{[0] b:{[1] e:{[2]}} c:{[3]}}`)
+
+	t.Log("Add 'o' with the coverage [0, 3]")
+	tree.Add(cov.Path{"a", "c", "g", "o"}, coverage(fileA, span0, span3))
+
+	//              [0]
+	//              (a)
+	//       ┏━━━━━━━┻━━━━━━━┓
+	//   [1](b)             (c)[3]
+	//    ╭──┺━━┓            ╰──╮
+	//    d    (e)[2]           g
+	//    ╰─╮                 ╭─┴─╮
+	//      i                 n   o
+	checkSpans(t, tree.Spans(), span0, span1, span2, span3)
+	checkTests(t, tree, `{a:{b:{d:{i}e}c:{g:{n o}}}}`)
+	checkCoverage(t, tree, fileA, `a:{[0] b:{[1] e:{[2]}} c:{[3]}}`)
+
+	t.Log("Add 'f' with the coverage [1]")
+	tree.Add(cov.Path{"a", "c", "f"}, coverage(fileA, span1))
+
+	//               (a)
+	//       ┏━━━━━━━━┻━━━━━━━━┓
+	// [0,1](b)               (c)
+	//    ╭──┺━━┓           ┏━━┻━━┓
+	//    d    (e)[2]   [1](f)   (g)[0,3]
+	//    ╰─╮                   ╭─┴─╮
+	//      i                   n   o
+	checkSpans(t, tree.Spans(), span0, span1, span2, span3)
+	checkTests(t, tree, `{a:{b:{d:{i} e} c:{f g:{n o}}}}`)
+	checkCoverage(t, tree, fileA, `a:{[] b:{[0,1] e:{[2]}} c:{[] f:{[1]} g:{[0,3]}}}`)
+
+	t.Log("Add 'j' with the coverage [3]")
+	tree.Add(cov.Path{"a", "b", "e", "j"}, coverage(fileA, span3))
+
+	//                   (a)
+	//           ┏━━━━━━━━┻━━━━━━━━┓
+	//          (b)               (c)
+	//       ┏━━━┻━━━┓          ┏━━┻━━┓
+	// [0,1](d)     (e)[3]  [1](f)   (g)[0,3]
+	//       ╰─╮   ╭─╯              ╭─┴─╮
+	//         i   j                n   o
+	checkSpans(t, tree.Spans(), span0, span1, span2, span3)
+	checkTests(t, tree, `{a:{b:{d:{i} e:{j}} c:{f g:{n o}}}}`)
+	checkCoverage(t, tree, fileA, `a:{[] b:{[] d:{[0,1]} e:{[3]}} c:{[] f:{[1]} g:{[0,3]}}}`)
+
+	t.Log("Add 'k' with the coverage [3]")
+	tree.Add(cov.Path{"a", "b", "e", "k"}, coverage(fileA, span3))
+
+	//                   (a)
+	//           ┏━━━━━━━━┻━━━━━━━━┓
+	//          (b)               (c)
+	//       ┏━━━┻━━━┓          ┏━━┻━━┓
+	// [0,1](d)     (e)[3]  [1](f)   (g)[0,3]
+	//       ╰─╮   ╭─┴─╮            ╭─┴─╮
+	//         i   j   k            n   o
+	checkSpans(t, tree.Spans(), span0, span1, span2, span3)
+	checkTests(t, tree, `{a:{b:{d:{i} e:{j k}} c:{f g:{n o}}}}`)
+	checkCoverage(t, tree, fileA, `a:{[] b:{[] d:{[0,1]} e:{[3]}} c:{[] f:{[1]} g:{[0,3]}}}`)
+
+	t.Log("Add 'v' with the coverage [1,2]")
+	tree.Add(cov.Path{"a", "c", "f", "l", "v"}, coverage(fileA, span1, span2))
+
+	//                   (a)
+	//           ┏━━━━━━━━┻━━━━━━━━━━┓
+	//          (b)                 (c)
+	//       ┏━━━┻━━━┓            ┏━━┻━━┓
+	// [0,1](d)     (e)[3]  [1,2](f)   (g)[0,3]
+	//       ╰─╮   ╭─┴─╮        ╭─╯   ╭─┴─╮
+	//         i   j   k        l     n   o
+	//                         ╭╯
+	//                         v
+	checkSpans(t, tree.Spans(), span0, span1, span2, span3)
+	checkTests(t, tree, `{a:{b:{d:{i} e:{j k}} c:{f:{l:{v}} g:{n o}}}}`)
+	checkCoverage(t, tree, fileA, `a:{[] b:{[] d:{[0,1]} e:{[3]}} c:{[] f:{[1,2]} g:{[0,3]}}}`)
+
+	t.Log("Add 'x' with the coverage [1,2]")
+	tree.Add(cov.Path{"a", "c", "f", "l", "x"}, coverage(fileA, span1, span2))
+
+	//                   (a)
+	//           ┏━━━━━━━━┻━━━━━━━━━━┓
+	//          (b)                 (c)
+	//       ┏━━━┻━━━┓            ┏━━┻━━┓
+	// [0,1](d)     (e)[3]  [1,2](f)   (g)[0,3]
+	//       ╰─╮   ╭─┴─╮        ╭─╯   ╭─┴─╮
+	//         i   j   k        l     n   o
+	//                         ╭┴╮
+	//                         v x
+	checkSpans(t, tree.Spans(), span0, span1, span2, span3)
+	checkTests(t, tree, `{a:{b:{d:{i} e:{j k}} c:{f:{l:{v x}} g:{n o}}}}`)
+	checkCoverage(t, tree, fileA, `a:{[] b:{[] d:{[0,1]} e:{[3]}} c:{[] f:{[1,2]} g:{[0,3]}}}`)
+
+	t.Log("Add 'z' with the coverage [2]")
+	tree.Add(cov.Path{"a", "c", "g", "n", "z"}, coverage(fileA, span2))
+
+	//                   (a)
+	//           ┏━━━━━━━━┻━━━━━━━━━━━━┓
+	//          (b)                   (c)
+	//       ┏━━━┻━━━┓            ┏━━━━┻━━━━┓
+	// [0,1](d)     (e)[3]  [1,2](f)       (g)
+	//       ╰─╮   ╭─┴─╮        ╭─╯       ┏━┻━┓
+	//         i   j   k        l    [2](n) (o)[0,3]
+	//                         ╭┴╮      ╭╯
+	//                         v x      z
+	checkSpans(t, tree.Spans(), span0, span1, span2, span3)
+	checkTests(t, tree, `{a:{b:{d:{i} e:{j k}} c:{f:{l:{v x}} g:{n: {z} o}}}}`)
+	checkCoverage(t, tree, fileA, `a:{[] b:{[] d:{[0,1]} e:{[3]}} c:{[] f:{[1,2]} g:{[] n:{[2]} o:{[0,3]}}}}`)
+}
+
+func checkSpans(t *testing.T, got []cov.Span, expect ...cov.Span) {
+	if !reflect.DeepEqual(got, expect) {
+		t.Errorf("Spans not as expected.\nGot:    %+v\nExpect: %+v", got, expect)
+	}
+}
+
+func checkTests(t *testing.T, tree *cov.Tree, expect string) {
+	g, e := tree.Tests().String(tree.Strings()), expect
+	if tg, te := trimWS(g), trimWS(e); tg != te {
+		t.Errorf("Tests not as expected.\nGot:\n%v\nExpect:\n%v\n------\nGot:    %v\nExpect: %v", g, e, tg, te)
+	}
+}
+
+func checkCoverage(t *testing.T, tree *cov.Tree, file string, expect string) {
+	g, e := tree.File(file).String(tree.Tests(), tree.Strings()), expect
+	if tg, te := trimWS(g), trimWS(e); tg != te {
+		t.Errorf("Coverage not as expected.\nGot:\n%v\nExpect:\n%v\n------\nGot:    %v\nExpect: %v", g, e, tg, te)
+	}
+}
+
+func trimWS(s string) string {
+	s = strings.ReplaceAll(s, " ", "")
+	s = strings.ReplaceAll(s, "\n", "")
+	return s
+}
+
+func coverage(file string, spans ...cov.Span) *cov.Coverage {
+	return &cov.Coverage{
+		[]cov.File{
+			cov.File{
+				Path:  file,
+				Spans: spans,
+			},
+		},
+	}
+}
+
+func spans(ids ...cov.SpanID) cov.SpanSet {
+	out := make(cov.SpanSet, len(ids))
+	for _, id := range ids {
+		out[id] = struct{}{}
+	}
+	return out
+}

tests/regres/deqp/deqp.go

@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"math/rand"
 	"os"
@@ -31,6 +32,7 @@ import (
 	"time"
 
 	"../cause"
+	"../cov"
 	"../shell"
 	"../testlist"
 	"../util"
@@ -59,10 +61,12 @@ type Config struct {
 	ExeGles2         string
 	ExeGles3         string
 	ExeVulkan        string
+	TempDir          string // Directory for temporary log files, coverage output.
 	TestLists        testlist.Lists
 	Env              []string
 	LogReplacements  map[string]string
 	NumParallelTests int
+	CoverageEnv      *cov.Env
 	TestTimeout      time.Duration
 }
 
@@ -72,6 +76,7 @@ type Results struct {
 	Version  int
 	Error    string
 	Tests    map[string]TestResult
+	Coverage *cov.Tree
 	Duration time.Duration
 }
 
@@ -81,6 +86,7 @@ type TestResult struct {
 	Status    testlist.Status
 	TimeTaken time.Duration
 	Err       string `json:",omitempty"`
+	Coverage  *cov.Coverage
 }
 
 func (r TestResult) String() string {
@@ -131,9 +137,16 @@ func (r *Results) Save(path string) error {
 
 // Run runs all the tests.
 func (c *Config) Run() (*Results, error) {
-
 	start := time.Now()
 
+	if c.TempDir == "" {
+		dir, err := ioutil.TempDir("", "deqp")
+		if err != nil {
+			return nil, cause.Wrap(err, "Could not generate temporary directory")
+		}
+		c.TempDir = dir
+	}
+
 	// Wait group that completes once all the tests have finished.
 	wg := sync.WaitGroup{}
 	results := make(chan TestResult, 256)
@@ -200,6 +213,10 @@ func (c *Config) Run() (*Results, error) {
 		Tests:   map[string]TestResult{},
 	}
 
+	if c.CoverageEnv != nil {
+		out.Coverage = &cov.Tree{}
+	}
+
 	// Collect the results.
 	finished := make(chan struct{})
 	lastUpdate := time.Now()
@@ -207,7 +224,6 @@ func (c *Config) Run() (*Results, error) {
 		start, i := time.Now(), 0
 		for r := range results {
 			i++
-			out.Tests[r.Test] = r
 			if time.Since(lastUpdate) > time.Minute {
 				lastUpdate = time.Now()
 				remaining := numTests - i
@@ -215,6 +231,12 @@ func (c *Config) Run() (*Results, error) {
 					i, numTests, util.Percent(i, numTests),
 					(time.Since(start)/time.Duration(i))*time.Duration(remaining))
 			}
+			out.Tests[r.Test] = r
+			if r.Coverage != nil {
+				path := strings.Split(r.Test, ".")
+				out.Coverage.Add(cov.Path(path), r.Coverage)
+				r.Coverage = nil // Free memory
+			}
 		}
 		close(finished)
 	}()
@@ -263,6 +285,12 @@ func (c *Config) TestRoutine(exe string, tests <-chan string, results chan<- Tes
 		env = append(env, v)
 	}
 
+	coverageFile := filepath.Join(c.TempDir, fmt.Sprintf("%v.profraw", goroutineIndex))
+	if c.CoverageEnv != nil {
+		env = cov.AppendRuntimeEnv(env, coverageFile)
+	}
+	logPath := filepath.Join(c.TempDir, fmt.Sprintf("%v.log", goroutineIndex))
+
 nextTest:
 	for name := range tests {
 		// log.Printf("Running test '%s'\n", name)
@@ -274,6 +302,7 @@ nextTest:
 			"--deqp-log-images=disable",
 			"--deqp-log-shader-sources=disable",
 			"--deqp-log-flush=disable",
+			"--deqp-log-filename="+logPath,
 			"-n="+name)
 		duration := time.Since(start)
 		out := string(outRaw)
@@ -282,6 +311,15 @@ nextTest:
 			out = strings.ReplaceAll(out, k, v)
 		}
 
+		var coverage *cov.Coverage
+		if c.CoverageEnv != nil {
+			coverage, err = c.CoverageEnv.Import(coverageFile)
+			if err != nil {
+				log.Printf("Warning: Failed to get test coverage for test '%v'. %v", name, err)
+			}
+			os.Remove(coverageFile)
+		}
+
 		for _, test := range []struct {
 			re *regexp.Regexp
 			s  testlist.Status
@@ -298,6 +336,7 @@ nextTest:
 					Status:    test.s,
 					TimeTaken: duration,
 					Err:       s,
+					Coverage:  coverage,
 				}
 				continue nextTest
 			}
@@ -319,6 +358,7 @@ nextTest:
 				Status:    testlist.Crash,
 				TimeTaken: duration,
 				Err:       out,
+				Coverage:  coverage,
 			}
 		case shell.ErrTimeout:
 			log.Printf("Timeout for test '%v'\n", name)
@@ -326,34 +366,35 @@ nextTest:
 				Test:      name,
 				Status:    testlist.Timeout,
 				TimeTaken: duration,
+				Coverage:  coverage,
 			}
 		case nil:
 			toks := deqpRE.FindStringSubmatch(out)
 			if len(toks) < 3 {
 				err := fmt.Sprintf("Couldn't parse test '%v' output:\n%s", name, out)
 				log.Println("Warning: ", err)
-				results <- TestResult{Test: name, Status: testlist.Fail, Err: err}
+				results <- TestResult{Test: name, Status: testlist.Fail, Err: err, Coverage: coverage}
 				continue
 			}
 			switch toks[1] {
 			case "Pass":
-				results <- TestResult{Test: name, Status: testlist.Pass, TimeTaken: duration}
+				results <- TestResult{Test: name, Status: testlist.Pass, TimeTaken: duration, Coverage: coverage}
 			case "NotSupported":
-				results <- TestResult{Test: name, Status: testlist.NotSupported, TimeTaken: duration}
+				results <- TestResult{Test: name, Status: testlist.NotSupported, TimeTaken: duration, Coverage: coverage}
 			case "CompatibilityWarning":
-				results <- TestResult{Test: name, Status: testlist.CompatibilityWarning, TimeTaken: duration}
+				results <- TestResult{Test: name, Status: testlist.CompatibilityWarning, TimeTaken: duration, Coverage: coverage}
 			case "QualityWarning":
-				results <- TestResult{Test: name, Status: testlist.QualityWarning, TimeTaken: duration}
+				results <- TestResult{Test: name, Status: testlist.QualityWarning, TimeTaken: duration, Coverage: coverage}
 			case "Fail":
 				var err string
 				if toks[2] != "Fail" {
 					err = toks[2]
 				}
-				results <- TestResult{Test: name, Status: testlist.Fail, Err: err, TimeTaken: duration}
+				results <- TestResult{Test: name, Status: testlist.Fail, Err: err, TimeTaken: duration, Coverage: coverage}
 			default:
 				err := fmt.Sprintf("Couldn't parse test output:\n%s", out)
 				log.Println("Warning: ", err)
-				results <- TestResult{Test: name, Status: testlist.Fail, Err: err, TimeTaken: duration}
+				results <- TestResult{Test: name, Status: testlist.Fail, Err: err, TimeTaken: duration, Coverage: coverage}
 			}
 		}
 	}

tests/regres/llvm/llvm.go

+// Copyright 2020 The SwiftShader Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package llvm provides functions and types for locating and using the llvm
+// toolchains.
+package llvm
+
+import (
+	"fmt"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"sort"
+	"strconv"
+
+	"../util"
+)
+
+const maxLLVMVersion = 10
+
+// Version holds the build version information of an LLVM toolchain.
+type Version struct {
+	Major, Minor, Point int
+}
+
+// GreaterEqual returns true if v >= rhs.
+func (v Version) GreaterEqual(rhs Version) bool {
+	if v.Major > rhs.Major {
+		return true
+	}
+	if v.Major < rhs.Major {
+		return false
+	}
+	if v.Minor > rhs.Minor {
+		return true
+	}
+	if v.Minor < rhs.Minor {
+		return false
+	}
+	return v.Point >= rhs.Point
+}
+
+// Toolchain holds the paths and version information about an LLVM toolchain.
+type Toolchain struct {
+	Version Version
+	BinDir  string
+}
+
+// Toolchains is a list of Toolchain
+type Toolchains []Toolchain
+
+// FindAtLeast looks for a toolchain with the given version, returning the highest found version.
+func (l Toolchains) FindAtLeast(v Version) *Toolchain {
+	out := (*Toolchain)(nil)
+	for _, t := range l {
+		if t.Version.GreaterEqual(v) && (out == nil || out.Version.GreaterEqual(t.Version)) {
+			t := t
+			out = &t
+		}
+	}
+	return out
+}
+
+// Search looks for llvm toolchains in paths.
+// If paths is empty, then PATH is searched.
+func Search(paths ...string) Toolchains {
+	toolchains := map[Version]Toolchain{}
+	search := func(name string) {
+		if len(paths) > 0 {
+			for _, path := range paths {
+				if util.IsFile(path) {
+					path = filepath.Dir(path)
+				}
+				if t := toolchain(path); t != nil {
+					toolchains[t.Version] = *t
+					continue
+				}
+				if t := toolchain(filepath.Join(path, "bin")); t != nil {
+					toolchains[t.Version] = *t
+					continue
+				}
+			}
+		} else {
+			path, err := exec.LookPath(name)
+			if err == nil {
+				if t := toolchain(filepath.Dir(path)); t != nil {
+					toolchains[t.Version] = *t
+				}
+			}
+		}
+	}
+
+	search("clang")
+	for i := 8; i < maxLLVMVersion; i++ {
+		search(fmt.Sprintf("clang-%d", i))
+	}
+
+	out := make([]Toolchain, 0, len(toolchains))
+	for _, t := range toolchains {
+		out = append(out, t)
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Version.GreaterEqual(out[j].Version) })
+
+	return out
+}
+
+// Cov returns the path to the llvm-cov executable.
+func (t Toolchain) Cov() string {
+	return filepath.Join(t.BinDir, "llvm-cov"+exeExt())
+}
+
+// Profdata returns the path to the llvm-profdata executable.
+func (t Toolchain) Profdata() string {
+	return filepath.Join(t.BinDir, "llvm-profdata"+exeExt())
+}
+
+func toolchain(dir string) *Toolchain {
+	t := Toolchain{BinDir: dir}
+	if t.resolve() {
+		return &t
+	}
+	return nil
+}
+
+func (t *Toolchain) resolve() bool {
+	if !util.IsFile(t.Profdata()) { // llvm-profdata doesn't have --version flag
+		return false
+	}
+	version, ok := parseVersion(t.Cov())
+	t.Version = version
+	return ok
+}
+
+func exeExt() string {
+	switch runtime.GOOS {
+	case "windows":
+		return ".exe"
+	default:
+		return ""
+	}
+}
+
+var versionRE = regexp.MustCompile(`(?:clang|LLVM) version ([0-9]+)\.([0-9]+)\.([0-9]+)`)
+
+func parseVersion(tool string) (Version, bool) {
+	out, err := exec.Command(tool, "--version").Output()
+	if err != nil {
+		return Version{}, false
+	}
+	matches := versionRE.FindStringSubmatch(string(out))
+	if len(matches) < 4 {
+		return Version{}, false
+	}
+	major, majorErr := strconv.Atoi(matches[1])
+	minor, minorErr := strconv.Atoi(matches[2])
+	point, pointErr := strconv.Atoi(matches[3])
+	if majorErr != nil || minorErr != nil || pointErr != nil {
+		return Version{}, false
+	}
+	return Version{major, minor, point}, true
+}