Add test/regres/export_to_sheets.go

tests/regres/consts/consts.go

+// Copyright 2019 The SwiftShader Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package consts holds constants shared between the regres tools.
+package consts
+
+const (
+	// TestListUpdateCommitSubjectPrefix is the commit message prefix on commits
+	// that update the full test lists results.
+	TestListUpdateCommitSubjectPrefix = "Regres: Update test lists @ "
+)

tests/regres/export_to_sheets.go

+// Copyright 2019 The SwiftShader Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// export_to_sheets updates a Google sheets document with the latest test
+// results
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"./cause"
+	"./consts"
+	"./git"
+	"./testlist"
+
+	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/google"
+	"google.golang.org/api/sheets/v4"
+)
+
+var (
+	authdir       = flag.String("authdir", "~/.regres-auth", "directory to hold credentials.json and generated token")
+	projectPath   = flag.String("projpath", ".", "project path")
+	testListPath  = flag.String("testlist", "tests/regres/full-tests.json", "project relative path to the test list .json file")
+	spreadsheetID = flag.String("spreadsheet", "1RCxbqtKNDG9rVMe_xHMapMBgzOCp24mumab73SbHtfw", "identifier of the spreadsheet to update")
+)
+
+const (
+	columnGitHash = "GIT_HASH"
+	columnGitDate = "GIT_DATE"
+)
+
+func main() {
+	flag.Parse()
+
+	if err := run(); err != nil {
+		log.Fatalln(err)
+	}
+}
+
+func run() error {
+	// Load the full test list. We use this to find the test file names.
+	lists, err := testlist.Load(".", *testListPath)
+	if err != nil {
+		return cause.Wrap(err, "Unable to load test list")
+	}
+
+	// Load the creditials used for editing the Google Sheets spreadsheet.
+	srv, err := createSheetsService(*authdir)
+	if err != nil {
+		return cause.Wrap(err, "Unable to authenticate")
+	}
+
+	// Ensure that there is a sheet for each of the test lists.
+	if err := createTestListSheets(srv, lists); err != nil {
+		return cause.Wrap(err, "Unable to create sheets")
+	}
+
+	spreadsheet, err := srv.Spreadsheets.Get(*spreadsheetID).Do()
+	if err != nil {
+		return cause.Wrap(err, "Unable to get spreadsheet")
+	}
+
+	req := sheets.BatchUpdateValuesRequest{
+		ValueInputOption: "RAW",
+	}
+
+	testListDir := filepath.Dir(filepath.Join(*projectPath, *testListPath))
+	changes, err := git.Log(testListDir, 100)
+	if err != nil {
+		return cause.Wrap(err, "Couldn't get git changes for '%v'", testListDir)
+	}
+
+	for _, group := range lists {
+		sheetName := group.Name
+		fmt.Println("Processing sheet", sheetName)
+		sheet := getSheet(spreadsheet, sheetName)
+		if sheet == nil {
+			return cause.Wrap(err, "Sheet '%v' not found", sheetName)
+		}
+
+		columnHeaders, err := fetchRow(srv, spreadsheet, sheet, 0)
+		if err != nil {
+			return cause.Wrap(err, "Couldn't get sheet '%v' column headers", sheetName)
+		}
+
+		columnIndices := listToMap(columnHeaders)
+
+		hashColumnIndex, found := columnIndices[columnGitHash]
+		if !found {
+			return cause.Wrap(err, "Couldn't find sheet '%v' column header '%v'", sheetName, columnGitHash)
+		}
+
+		hashValues, err := fetchColumn(srv, spreadsheet, sheet, hashColumnIndex)
+		if err != nil {
+			return cause.Wrap(err, "Couldn't get sheet '%v' column headers", sheetName)
+		}
+		hashValues = hashValues[1:] // Skip header
+
+		hashIndices := listToMap(hashValues)
+		rowValues := map[string]interface{}{}
+
+		rowInsertionPoint := 1 + len(hashValues)
+
+		for i := len(changes) - 1; i > 0; i-- {
+			change := changes[i]
+			if !strings.HasPrefix(change.Subject, consts.TestListUpdateCommitSubjectPrefix) {
+				continue
+			}
+
+			hash := change.Hash.String()
+			if _, found := hashIndices[hash]; found {
+				continue // Already in the sheet
+			}
+
+			rowValues[columnGitHash] = change.Hash.String()
+			rowValues[columnGitDate] = change.Date.Format("2006-01-02")
+
+			path := filepath.Join(*projectPath, group.File)
+			hasData := false
+			for _, status := range testlist.Statuses {
+				path := testlist.FilePathWithStatus(path, status)
+				data, err := git.Show(path, hash)
+				if err != nil {
+					continue
+				}
+				lines, err := countLines(data)
+				if err != nil {
+					return cause.Wrap(err, "Couldn't count lines in file '%s'", path)
+				}
+
+				rowValues[string(status)] = lines
+				hasData = true
+			}
+
+			if !hasData {
+				continue
+			}
+
+			data, err := mapToList(columnIndices, rowValues)
+			if err != nil {
+				return cause.Wrap(err, "Couldn't map row values to column for sheet %v. Column headers: [%+v]", sheetName, columnHeaders)
+			}
+
+			req.Data = append(req.Data, &sheets.ValueRange{
+				Range:  rowRange(rowInsertionPoint, sheet),
+				Values: [][]interface{}{data},
+			})
+			rowInsertionPoint++
+
+			fmt.Printf("Adding test data at %v to %v\n", hash[:8], sheetName)
+		}
+	}
+
+	if _, err := srv.Spreadsheets.Values.BatchUpdate(*spreadsheetID, &req).Do(); err != nil {
+		return cause.Wrap(err, "Values BatchUpdate failed")
+	}
+
+	return nil
+}
+
+// listToMap returns the list l as a map where the key is the stringification
+// of the element, and the value is the element index.
+func listToMap(l []interface{}) map[string]int {
+	out := map[string]int{}
+	for i, v := range l {
+		out[fmt.Sprint(v)] = i
+	}
+	return out
+}
+
+// mapToList transforms the two maps into a single slice of values.
+// indices is a map of identifier to output slice element index.
+// values is a map of identifier to value.
+func mapToList(indices map[string]int, values map[string]interface{}) ([]interface{}, error) {
+	out := []interface{}{}
+	for name, value := range values {
+		index, ok := indices[name]
+		if !ok {
+			return nil, fmt.Errorf("No index for '%v'", name)
+		}
+		for len(out) <= index {
+			out = append(out, nil)
+		}
+		out[index] = value
+	}
+	return out, nil
+}
+
+// countLines returns the number of new lines in the byte slice data.
+func countLines(data []byte) (int, error) {
+	scanner := bufio.NewScanner(bytes.NewReader(data))
+	lines := 0
+	for scanner.Scan() {
+		lines++
+	}
+	return lines, nil
+}
+
+// getSheet returns the sheet with the given title name, or nil if the sheet
+// cannot be found.
+func getSheet(spreadsheet *sheets.Spreadsheet, name string) *sheets.Sheet {
+	for _, sheet := range spreadsheet.Sheets {
+		if sheet.Properties.Title == name {
+			return sheet
+		}
+	}
+	return nil
+}
+
+// rowRange returns a sheets range ("name!Ai:i") for the entire row with the
+// given index.
+func rowRange(index int, sheet *sheets.Sheet) string {
+	return fmt.Sprintf("%v!A%v:%v", sheet.Properties.Title, index+1, index+1)
+}
+
+// columnRange returns a sheets range ("name!i1:i") for the entire column with
+// the given index.
+func columnRange(index int, sheet *sheets.Sheet) string {
+	col := 'A' + index
+	if index > 25 {
+		panic("UNIMPLEMENTED")
+	}
+	return fmt.Sprintf("%v!%c1:%c", sheet.Properties.Title, col, col)
+}
+
+// fetchRow returns all the values in the given sheet's row.
+func fetchRow(srv *sheets.Service, spreadsheet *sheets.Spreadsheet, sheet *sheets.Sheet, row int) ([]interface{}, error) {
+	rng := rowRange(row, sheet)
+	data, err := srv.Spreadsheets.Values.Get(spreadsheet.SpreadsheetId, rng).Do()
+	if err != nil {
+		return nil, cause.Wrap(err, "Couldn't fetch %v", rng)
+	}
+	return data.Values[0], nil
+}
+
+// fetchColumn returns all the values in the given sheet's column.
+func fetchColumn(srv *sheets.Service, spreadsheet *sheets.Spreadsheet, sheet *sheets.Sheet, row int) ([]interface{}, error) {
+	rng := columnRange(row, sheet)
+	data, err := srv.Spreadsheets.Values.Get(spreadsheet.SpreadsheetId, rng).Do()
+	if err != nil {
+		return nil, cause.Wrap(err, "Couldn't fetch %v", rng)
+	}
+	out := make([]interface{}, len(data.Values))
+	for i, l := range data.Values {
+		if len(l) > 0 {
+			out[i] = l[0]
+		}
+	}
+	return out, nil
+}
+
+// insertRows inserts blank rows into the given sheet.
+func insertRows(srv *sheets.Service, spreadsheet *sheets.Spreadsheet, sheet *sheets.Sheet, aboveRow, count int) error {
+	req := sheets.BatchUpdateSpreadsheetRequest{
+		Requests: []*sheets.Request{{
+			InsertRange: &sheets.InsertRangeRequest{
+				Range: &sheets.GridRange{
+					SheetId:       sheet.Properties.SheetId,
+					StartRowIndex: int64(aboveRow),
+					EndRowIndex:   int64(aboveRow + count),
+				},
+				ShiftDimension: "ROWS",
+			}},
+		},
+	}
+	if _, err := srv.Spreadsheets.BatchUpdate(*spreadsheetID, &req).Do(); err != nil {
+		return cause.Wrap(err, "Values BatchUpdate failed")
+	}
+	return nil
+}
+
+// createTestListSheets adds a new sheet for each of the test lists, if they
+// do not already exist. These new sheets are populated with column headers.
+func createTestListSheets(srv *sheets.Service, testlists testlist.Lists) error {
+	spreadsheet, err := srv.Spreadsheets.Get(*spreadsheetID).Do()
+	if err != nil {
+		return cause.Wrap(err, "Unable to get spreadsheet")
+	}
+
+	spreadsheetReq := sheets.BatchUpdateSpreadsheetRequest{}
+	updateReq := sheets.BatchUpdateValuesRequest{ValueInputOption: "RAW"}
+	headers := []interface{}{columnGitHash, columnGitDate}
+	for _, s := range testlist.Statuses {
+		headers = append(headers, string(s))
+	}
+
+	for _, group := range testlists {
+		name := group.Name
+		if getSheet(spreadsheet, name) == nil {
+			spreadsheetReq.Requests = append(spreadsheetReq.Requests, &sheets.Request{
+				AddSheet: &sheets.AddSheetRequest{
+					Properties: &sheets.SheetProperties{
+						Title: name,
+					},
+				},
+			})
+			updateReq.Data = append(updateReq.Data,
+				&sheets.ValueRange{
+					Range:  name + "!A1:Z",
+					Values: [][]interface{}{headers},
+				},
+			)
+		}
+	}
+
+	if len(spreadsheetReq.Requests) > 0 {
+		if _, err := srv.Spreadsheets.BatchUpdate(*spreadsheetID, &spreadsheetReq).Do(); err != nil {
+			return cause.Wrap(err, "Spreadsheets BatchUpdate failed")
+		}
+	}
+	if len(updateReq.Data) > 0 {
+		if _, err := srv.Spreadsheets.Values.BatchUpdate(*spreadsheetID, &updateReq).Do(); err != nil {
+			return cause.Wrap(err, "Values BatchUpdate failed")
+		}
+	}
+
+	return nil
+}
+
+// createSheetsService creates a new Google Sheets service using the credentials
+// in the credentials.json file.
+func createSheetsService(authdir string) (*sheets.Service, error) {
+	authdir = os.ExpandEnv(authdir)
+	if home, err := os.UserHomeDir(); err == nil {
+		authdir = strings.ReplaceAll(authdir, "~", home)
+	}
+
+	os.MkdirAll(authdir, 0777)
+
+	credentialsPath := filepath.Join(authdir, "credentials.json")
+	b, err := ioutil.ReadFile(credentialsPath)
+	if err != nil {
+		return nil, cause.Wrap(err, "Unable to read client secret file '%v'\n"+
+			"Obtain this file from: https://console.developers.google.com/apis/credentials", credentialsPath)
+	}
+
+	config, err := google.ConfigFromJSON(b, "https://www.googleapis.com/auth/spreadsheets")
+	if err != nil {
+		return nil, cause.Wrap(err, "Unable to parse client secret file to config")
+	}
+
+	client, err := getClient(authdir, config)
+	if err != nil {
+		return nil, cause.Wrap(err, "Unable obtain client")
+	}
+
+	srv, err := sheets.New(client)
+	if err != nil {
+		return nil, cause.Wrap(err, "Unable to retrieve Sheets client")
+	}
+	return srv, nil
+}
+
+// Retrieve a token, saves the token, then returns the generated client.
+func getClient(authdir string, config *oauth2.Config) (*http.Client, error) {
+	// The file token.json stores the user's access and refresh tokens, and is
+	// created automatically when the authorization flow completes for the first
+	// time.
+	tokFile := filepath.Join(authdir, "token.json")
+	tok, err := tokenFromFile(tokFile)
+	if err != nil {
+		tok, err = getTokenFromWeb(config)
+		if err != nil {
+			return nil, cause.Wrap(err, "Unable to get token from web")
+		}
+		if err := saveToken(tokFile, tok); err != nil {
+			log.Println("Warning: failed to write token: %v", err)
+		}
+	}
+	return config.Client(context.Background(), tok), nil
+}
+
+// Request a token from the web, then returns the retrieved token.
+func getTokenFromWeb(config *oauth2.Config) (*oauth2.Token, error) {
+	authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline)
+	fmt.Printf("Go to the following link in your browser then type the "+
+		"authorization code: \n%v\n", authURL)
+
+	var authCode string
+	if _, err := fmt.Scan(&authCode); err != nil {
+		return nil, cause.Wrap(err, "Unable to read authorization code")
+	}
+
+	tok, err := config.Exchange(context.TODO(), authCode)
+	if err != nil {
+		return nil, cause.Wrap(err, "Unable to retrieve token from web")
+	}
+	return tok, nil
+}
+
+// Retrieves a token from a local file.
+func tokenFromFile(path string) (*oauth2.Token, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	tok := &oauth2.Token{}
+	err = json.NewDecoder(f).Decode(tok)
+	return tok, err
+}
+
+// Saves a token to a file path.
+func saveToken(path string, token *oauth2.Token) error {
+	fmt.Printf("Saving credential file to: %s\n", path)
+	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return cause.Wrap(err, "Unable to cache oauth token")
+	}
+	defer f.Close()
+	json.NewEncoder(f).Encode(token)
+	return nil
+}

tests/regres/git/git.go

@@ -22,6 +22,7 @@ import (
 	"net/url"
 	"os"
 	"os/exec"
+	"strings"
 	"time"
 
 	"../cause"
@@ -139,3 +140,83 @@ func FetchRefHash(ref, url string) (Hash, error) {
 	}
 	return ParseHash(string(out)), nil
 }
+
+type ChangeList struct {
+	Hash        Hash
+	Date        time.Time
+	Author      string
+	Subject     string
+	Description string
+}
+
+// Log returns the top count ChangeLists at HEAD.
+func Log(path string, count int) ([]ChangeList, error) {
+	return LogFrom(path, "HEAD", count)
+}
+
+// LogFrom returns the top count ChangeList starting from at.
+func LogFrom(path, at string, count int) ([]ChangeList, error) {
+	if at == "" {
+		at = "HEAD"
+	}
+	out, err := shell.Exec(gitTimeout, exe, "", nil, "log", at, "--pretty=format:"+prettyFormat, fmt.Sprintf("-%d", count), path)
+	if err != nil {
+		return nil, err
+	}
+	return parseLog(string(out)), nil
+}
+
+// Parent returns the parent ChangeList for cl.
+func Parent(cl ChangeList) (ChangeList, error) {
+	out, err := shell.Exec(gitTimeout, exe, "", nil, "log", "--pretty=format:"+prettyFormat, fmt.Sprintf("%v^", cl.Hash))
+	if err != nil {
+		return ChangeList{}, err
+	}
+	cls := parseLog(string(out))
+	if len(cls) == 0 {
+		return ChangeList{}, fmt.Errorf("Unexpected output")
+	}
+	return cls[0], nil
+}
+
+// HeadCL returns the HEAD ChangeList at the given commit/tag/branch.
+func HeadCL(path string) (ChangeList, error) {
+	cls, err := LogFrom(path, "HEAD", 1)
+	if err != nil {
+		return ChangeList{}, err
+	}
+	if len(cls) == 0 {
+		return ChangeList{}, fmt.Errorf("No commits found")
+	}
+	return cls[0], nil
+}
+
+// Show content of the file at path for the given commit/tag/branch.
+func Show(path, at string) ([]byte, error) {
+	return shell.Exec(gitTimeout, exe, "", nil, "show", at+":"+path)
+}
+
+const prettyFormat = "ǁ%Hǀ%cIǀ%an <%ae>ǀ%sǀ%b"
+
+func parseLog(str string) []ChangeList {
+	msgs := strings.Split(str, "ǁ")
+	cls := make([]ChangeList, 0, len(msgs))
+	for _, s := range msgs {
+		if parts := strings.Split(s, "ǀ"); len(parts) == 5 {
+			cl := ChangeList{
+				Hash:        ParseHash(parts[0]),
+				Author:      strings.TrimSpace(parts[2]),
+				Subject:     strings.TrimSpace(parts[3]),
+				Description: strings.TrimSpace(parts[4]),
+			}
+			date, err := time.Parse(time.RFC3339, parts[1])
+			if err != nil {
+				panic(err)
+			}
+			cl.Date = date
+
+			cls = append(cls, cl)
+		}
+	}
+	return cls
+}

tests/regres/main.go

@@ -42,6 +42,7 @@ import (
 	"time"
 
 	"./cause"
+	"./consts"
 	"./git"
 	"./shell"
 	"./testlist"
@@ -444,7 +445,7 @@ func (r *regres) updateTestLists(client *gerrit.Client) error {
 	}
 
 	commitMsg := strings.Builder{}
-	commitMsg.WriteString("Regres: Update test lists @ " + headHash.String()[:8])
+	commitMsg.WriteString(consts.TestListUpdateCommitSubjectPrefix + headHash.String()[:8])
 	if results != nil && len(*changes) > 0 {
 		// Reuse gerrit change ID if there's already a change up for review.
 		id := (*changes)[0].ChangeID
@@ -739,11 +740,9 @@ func (t *test) writeTestListsByStatus(testLists testlist.Lists, results *CommitT
 	out := []string{}
 
 	for _, list := range testLists {
-		files := map[Status]*os.File{}
-		ext := filepath.Ext(list.File)
-		name := list.File[:len(list.File)-len(ext)]
-		for _, status := range Statuses {
-			path := filepath.Join(t.srcDir, name+"-"+string(status)+ext)
+		files := map[testlist.Status]*os.File{}
+		for _, status := range testlist.Statuses {
+			path := testlist.FilePathWithStatus(filepath.Join(t.srcDir, list.File), status)
 			dir := filepath.Dir(path)
 			os.MkdirAll(dir, 0777)
 			f, err := os.Create(path)
@@ -772,50 +771,6 @@ func (t *test) resultsCachePath(testLists testlist.Lists) string {
 	return filepath.Join(t.resDir, testLists.Hash())
 }
 
-// Status is an enumerator of test results.
-type Status string
-
-const (
-	// Pass is the status of a successful test.
-	Pass = Status("PASS")
-	// Fail is the status of a failed test.
-	Fail = Status("FAIL")
-	// Timeout is the status of a test that failed to complete in the alloted
-	// time.
-	Timeout = Status("TIMEOUT")
-	// Crash is the status of a test that crashed.
-	Crash = Status("CRASH")
-	// NotSupported is the status of a test feature not supported by the driver.
-	NotSupported = Status("NOT_SUPPORTED")
-	// CompatibilityWarning is the status passing test with a warning.
-	CompatibilityWarning = Status("COMPATIBILITY_WARNING")
-	// QualityWarning is the status passing test with a warning.
-	QualityWarning = Status("QUALITY_WARNING")
-)
-
-// Statuses is the full list of status types
-var Statuses = []Status{Pass, Fail, Timeout, Crash, NotSupported, CompatibilityWarning, QualityWarning}
-
-// Failing returns true if the task status requires fixing.
-func (s Status) Failing() bool {
-	switch s {
-	case Fail, Timeout, Crash:
-		return true
-	default:
-		return false
-	}
-}
-
-// Passing returns true if the task status is considered a pass.
-func (s Status) Passing() bool {
-	switch s {
-	case Pass, CompatibilityWarning, QualityWarning:
-		return true
-	default:
-		return false
-	}
-}
-
 // CommitTestResults holds the results the tests across all APIs for a given
 // commit. The CommitTestResults structure may be serialized to cache the
 // results.
@@ -874,7 +829,7 @@ func compare(old, new *CommitTestResults) string {
 		return "Build now fixed. Cannot compare against broken parent."
 	}
 
-	oldStatusCounts, newStatusCounts := map[Status]int{}, map[Status]int{}
+	oldStatusCounts, newStatusCounts := map[testlist.Status]int{}, map[testlist.Status]int{}
 	totalTests := 0
 
 	broken, fixed, failing, removed, changed := []string{}, []string{}, []string{}, []string{}, []string{}
@@ -934,15 +889,15 @@ func compare(old, new *CommitTestResults) string {
 	sb.WriteString(fmt.Sprintf("          Total tests: %d\n", totalTests))
 	for _, s := range []struct {
 		label  string
-		status Status
+		status testlist.Status
 	}{
-		{"                 Pass", Pass},
-		{"                 Fail", Fail},
-		{"              Timeout", Timeout},
-		{"                Crash", Crash},
-		{"        Not Supported", NotSupported},
-		{"Compatibility Warning", CompatibilityWarning},
-		{"      Quality Warning", QualityWarning},
+		{"                 Pass", testlist.Pass},
+		{"                 Fail", testlist.Fail},
+		{"              Timeout", testlist.Timeout},
+		{"                Crash", testlist.Crash},
+		{"        Not Supported", testlist.NotSupported},
+		{"Compatibility Warning", testlist.CompatibilityWarning},
+		{"      Quality Warning", testlist.QualityWarning},
 	} {
 		old, new := oldStatusCounts[s.status], newStatusCounts[s.status]
 		if old == 0 && new == 0 {
@@ -1003,7 +958,7 @@ func compare(old, new *CommitTestResults) string {
 // TestResult holds the results of a single API test.
 type TestResult struct {
 	Test   string
-	Status Status
+	Status testlist.Status
 	Err    string `json:",omitempty"`
 }
 
@@ -1037,13 +992,13 @@ func (t *test) deqpTestRoutine(exe string, tests <-chan string, results chan<- T
 		default:
 			results <- TestResult{
 				Test:   name,
-				Status: Crash,
+				Status: testlist.Crash,
 				Err:    cause.Wrap(err, string(out)).Error(),
 			}
 		case shell.ErrTimeout:
 			results <- TestResult{
 				Test:   name,
-				Status: Timeout,
+				Status: testlist.Timeout,
 				Err:    cause.Wrap(err, string(out)).Error(),
 			}
 		case nil:
@@ -1051,28 +1006,28 @@ func (t *test) deqpTestRoutine(exe string, tests <-chan string, results chan<- T
 			if len(toks) < 3 {
 				err := fmt.Sprintf("Couldn't parse test '%v' output:\n%s", name, string(out))
 				log.Println("Warning: ", err)
-				results <- TestResult{Test: name, Status: Fail, Err: err}
+				results <- TestResult{Test: name, Status: testlist.Fail, Err: err}
 				continue
 			}
 			switch toks[1] {
 			case "Pass":
-				results <- TestResult{Test: name, Status: Pass}
+				results <- TestResult{Test: name, Status: testlist.Pass}
 			case "NotSupported":
-				results <- TestResult{Test: name, Status: NotSupported}
+				results <- TestResult{Test: name, Status: testlist.NotSupported}
 			case "CompatibilityWarning":
-				results <- TestResult{Test: name, Status: CompatibilityWarning}
+				results <- TestResult{Test: name, Status: testlist.CompatibilityWarning}
 			case "QualityWarning":
-				results <- TestResult{Test: name, Status: QualityWarning}
+				results <- TestResult{Test: name, Status: testlist.QualityWarning}
 			case "Fail":
 				var err string
 				if toks[2] != "Fail" {
 					err = toks[2]
 				}
-				results <- TestResult{Test: name, Status: Fail, Err: err}
+				results <- TestResult{Test: name, Status: testlist.Fail, Err: err}
 			default:
 				err := fmt.Sprintf("Couldn't parse test output:\n%s", string(out))
 				log.Println("Warning: ", err)
-				results <- TestResult{Test: name, Status: Fail, Err: err}
+				results <- TestResult{Test: name, Status: testlist.Fail, Err: err}
 			}
 		}
 	}

tests/regres/testlist/testlist.go

@@ -145,3 +145,55 @@ func Load(root, jsonPath string) (Lists, error) {
 
 	return out, nil
 }
+
+// Status is an enumerator of test results.
+type Status string
+
+const (
+	// Pass is the status of a successful test.
+	Pass = Status("PASS")
+	// Fail is the status of a failed test.
+	Fail = Status("FAIL")
+	// Timeout is the status of a test that failed to complete in the alloted
+	// time.
+	Timeout = Status("TIMEOUT")
+	// Crash is the status of a test that crashed.
+	Crash = Status("CRASH")
+	// NotSupported is the status of a test feature not supported by the driver.
+	NotSupported = Status("NOT_SUPPORTED")
+	// CompatibilityWarning is the status passing test with a warning.
+	CompatibilityWarning = Status("COMPATIBILITY_WARNING")
+	// QualityWarning is the status passing test with a warning.
+	QualityWarning = Status("QUALITY_WARNING")
+)
+
+// Statuses is the full list of status types
+var Statuses = []Status{Pass, Fail, Timeout, Crash, NotSupported, CompatibilityWarning, QualityWarning}
+
+// Failing returns true if the task status requires fixing.
+func (s Status) Failing() bool {
+	switch s {
+	case Fail, Timeout, Crash:
+		return true
+	default:
+		return false
+	}
+}
+
+// Passing returns true if the task status is considered a pass.
+func (s Status) Passing() bool {
+	switch s {
+	case Pass, CompatibilityWarning, QualityWarning:
+		return true
+	default:
+		return false
+	}
+}
+
+// FilePathWithStatus returns the path to the test list file with the status
+// appended before the file extension.
+func FilePathWithStatus(listPath string, status Status) string {
+	ext := filepath.Ext(listPath)
+	name := listPath[:len(listPath)-len(ext)]
+	return name + "-" + string(status) + ext
+}