Skip to content

S
swiftshader
Commit a27f6975 by Ben Clayton
Options

Regres: Run latest PS testlists on parent.

This is better than running potentially two different test lists for both. Requires caching the parent test results with the testlist's hash as a cache key. Change-Id: Ic153e65395fc1c3f02a66bbb1df60731ce28901b Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/26557Tested-by: 's avatarBen Clayton <bclayton@google.com> Reviewed-by: 's avatarNicolas Capens <nicolascapens@google.com>
parent 03aee541
Showing with 373 additions and 253 deletions
tests/regres/main.go
......@@ -26,12 +26,10 @@
package main
import (
"bytes"
"encoding/json"
"errors"
"flag"
"fmt"
"io/ioutil"
"log"
"os"
"os/exec"
......@@ -46,6 +44,7 @@ import (
"./cause"
"./git"
"./shell"
"./testlist"
gerrit "github.com/andygrunwald/go-gerrit"
)
......@@ -60,8 +59,8 @@ const (
testTimeout = time.Minute * 5 // timeout for a single test
buildTimeout = time.Minute * 10 // timeout for a build
dailyUpdateTestListHour = 5 // 5am
fullTestList = "tests/regres/full-tests.json"
ciTestList = "tests/regres/ci-tests.json"
fullTestListRelPath = "tests/regres/full-tests.json"
ciTestListRelPath = "tests/regres/ci-tests.json"
)
var (
......@@ -119,19 +118,29 @@ type regres struct {
// resolveDirs ensures that the necessary directories used can be found, and
// expands them to absolute paths.
func (r *regres) resolveDirs() error {
for _, path := range []*string{
allDirs := []*string{
&r.deqpBuild,
&r.cacheRoot,
} {
}
for _, path := range allDirs {
abs, err := filepath.Abs(*path)
if err != nil {
return cause.Wrap(err, "Couldn't find path '%v'", *path)
}
if _, err := os.Stat(abs); err != nil {
return cause.Wrap(err, "Couldn't find path '%v'", abs)
}
*path = abs
}
if err := os.MkdirAll(r.cacheRoot, 0777); err != nil {
return cause.Wrap(err, "Couldn't create cache root directory")
}
for _, path := range allDirs {
if _, err := os.Stat(*path); err != nil {
return cause.Wrap(err, "Couldn't find path '%v'", *path)
}
}
return nil
}
......@@ -233,25 +242,14 @@ func (r *regres) run() error {
log.Printf("Testing change '%s'\n", change.id)
// Get the test results for the latest patchset in the change.
latest, err := r.newTest(ciTestList, change.latest).lazyRun()
// Test the latest patchset in the change, diff against parent change.
msg, err := r.test(change)
if err != nil {
log.Println(cause.Wrap(err, "Failed to test changelist '%s'", change.latest))
time.Sleep(time.Minute)
continue
}
// Get the test results for the changes's parent changelist.
parent, err := r.newTest(ciTestList, change.parent).lazyRun()
if err != nil {
log.Println(cause.Wrap(err, "Failed to test changelist '%s'", change.parent))
time.Sleep(time.Minute)
continue
}
// Compare the latest patchset to the change's parent commit.
msg := compare(parent, latest)
// Always include the reportHeader in the message.
// changeInfo.update() uses this header to detect whether a patchset has
// already got a test result.
......@@ -273,6 +271,97 @@ func (r *regres) run() error {
}
}
func (r *regres) test(change *changeInfo) (string, error) {
log.Printf("Testing latest patchset for change '%s'\n", change.id)
latest, testlists, err := r.testLatest(change)
if err != nil {
return "", cause.Wrap(err, "Failed to test latest change of '%v'", change.id)
}
log.Printf("Testing parent of change '%s'\n", change.id)
parent, err := r.testParent(change, testlists)
if err != nil {
return "", cause.Wrap(err, "Failed to test parent change of '%v'", change.id)
}
log.Println("Comparing latest patchset's results with parent")
msg := compare(parent, latest)
return msg, nil
}
func (r *regres) testLatest(change *changeInfo) (*CommitTestResults, testlist.Lists, error) {
// Get the test results for the latest patchset in the change.
test := r.newTest(change.latest)
defer test.cleanup()
if err := test.checkout(); err != nil {
return nil, nil, cause.Wrap(err, "Failed to checkout '%s'", change.latest)
}
testlists, err := test.loadTestLists(ciTestListRelPath)
if err != nil {
return nil, nil, cause.Wrap(err, "Failed to load '%s'", change.latest)
}
cachePath := test.resultsCachePath(testlists)
if results, err := loadCommitTestResults(cachePath); err == nil {
return results, testlists, nil // Use cached results
}
if err := test.build(); err != nil {
return nil, nil, cause.Wrap(err, "Failed to build '%s'", change.latest)
}
results, err := test.run(testlists)
if err != nil {
return nil, nil, cause.Wrap(err, "Failed to test '%s'", change.latest)
}
// Cache the results for future tests
if err := results.save(cachePath); err != nil {
log.Printf("Warning: Couldn't save results of test to '%v'\n", cachePath)
}
return results, testlists, nil
}
func (r *regres) testParent(change *changeInfo, testlists testlist.Lists) (*CommitTestResults, error) {
// Get the test results for the changes's parent changelist.
test := r.newTest(change.parent)
defer test.cleanup()
cachePath := test.resultsCachePath(testlists)
if results, err := loadCommitTestResults(cachePath); err == nil {
return results, nil // Use cached results
}
// Couldn't load cached results. Have to build them.
if err := test.checkout(); err != nil {
return nil, cause.Wrap(err, "Failed to checkout '%s'", change.parent)
}
// Build the parent change.
if err := test.build(); err != nil {
return nil, cause.Wrap(err, "Failed to build '%s'", change.parent)
}
// Run the tests on the parent change.
results, err := test.run(testlists)
if err != nil {
return nil, cause.Wrap(err, "Failed to test '%s'", change.parent)
}
// Store the results of the parent change to the cache.
if err := results.save(cachePath); err != nil {
log.Printf("Warning: Couldn't save results of test to '%v'\n", cachePath)
}
return results, nil
}
func (r *regres) updateTestLists(client *gerrit.Client) error {
log.Println("Updating test lists")
......@@ -282,27 +371,44 @@ func (r *regres) updateTestLists(client *gerrit.Client) error {
}
// Get the full test results for latest master.
t := r.newTest(fullTestList, headHash)
test := r.newTest(headHash)
defer test.cleanup()
testLists, err := test.loadTestLists(fullTestListRelPath)
if err != nil {
return cause.Wrap(err, "Failed to load full test lists for '%s'", headHash)
}
// Couldn't load cached results. Have to build them.
if err := test.checkout(); err != nil {
return cause.Wrap(err, "Failed to checkout '%s'", headHash)
}
// Build the change.
if err := test.build(); err != nil {
return cause.Wrap(err, "Failed to build '%s'", headHash)
}
// Keep the checked out directory after the test is run. We want this so
// we can build a new patchset containing the updated test lists.
t.keepCheckouts = true
if !r.keepCheckouts {
defer os.RemoveAll(t.srcDir)
// Run the tests on the change.
results, err := test.run(testLists)
if err != nil {
return cause.Wrap(err, "Failed to test '%s'", headHash)
}
if _, err := t.run(); err != nil {
return cause.Wrap(err, "Failed to test changelist '%s'", headHash)
// Write out the test list status files.
filePaths, err := test.writeTestListsByStatus(testLists, results)
if err != nil {
return cause.Wrap(err, "Failed to write test lists by status")
}
// Stage all the updated test files.
for _, path := range t.writtenTestLists {
for _, path := range filePaths {
log.Println("Staging", path)
git.Add(t.srcDir, path)
git.Add(test.srcDir, path)
}
log.Println("Checking for existing test list")
results, _, err := client.Changes.QueryChanges(&gerrit.QueryChangeOptions{
changes, _, err := client.Changes.QueryChanges(&gerrit.QueryChangeOptions{
QueryOptions: gerrit.QueryOptions{
Query: []string{fmt.Sprintf(`status:open+owner:"%v"`, r.gerritEmail)},
Limit: 1,
......@@ -314,14 +420,14 @@ func (r *regres) updateTestLists(client *gerrit.Client) error {
commitMsg := strings.Builder{}
commitMsg.WriteString("Regres: Update test lists @ " + headHash.String()[:8])
if results != nil && len(*results) > 0 {
if results != nil && len(*changes) > 0 {
// Reuse gerrit change ID if there's already a change up for review.
id := (*results)[0].ChangeID
id := (*changes)[0].ChangeID
commitMsg.WriteString("\n\n")
commitMsg.WriteString("Change-Id: " + id)
}
if err := git.Commit(t.srcDir, commitMsg.String(), git.CommitFlags{
if err := git.Commit(test.srcDir, commitMsg.String(), git.CommitFlags{
Name: "SwiftShader Regression Bot",
Email: r.gerritEmail,
}); err != nil {
......@@ -332,7 +438,7 @@ func (r *regres) updateTestLists(client *gerrit.Client) error {
log.Printf("DRY RUN: post results for review")
} else {
log.Println("Pushing test results for review")
if err := git.Push(t.srcDir, gitURL, "HEAD", "refs/for/master", git.PushFlags{
if err := git.Push(test.srcDir, gitURL, "HEAD", "refs/for/master", git.PushFlags{
Username: r.gerritUser,
Password: r.gerritPass,
}); err != nil {
......@@ -347,12 +453,13 @@ func (r *regres) updateTestLists(client *gerrit.Client) error {
// changeInfo holds the important information about a single, open change in
// gerrit.
type changeInfo struct {
id string // Gerrit change ID.
pending bool // Is this change waiting a test for the latest patchset?
priority int // Calculated priority based on Gerrit labels.
latest git.Hash // Git hash of the latest patchset in the change.
parent git.Hash // Git hash of the changelist this change is based on.
lastUpdated time.Time // Time the change was last fetched.
id string // Gerrit change ID.
pending bool // Is this change waiting a test for the latest patchset?
priority int // Calculated priority based on Gerrit labels.
latest git.Hash // Git hash of the latest patchset in the change.
parent git.Hash // Git hash of the changelist this change is based on.
lastUpdated time.Time // Time the change was last fetched.
commitMessage string
}
// queryChanges updates the changes map by querying gerrit for the latest open
......@@ -441,21 +548,21 @@ func (c *changeInfo) update(client *gerrit.Client) error {
c.pending = canTest
c.latest = git.ParseHash(change.CurrentRevision)
c.parent = git.ParseHash(current.Commit.Parents[0].Commit)
c.commitMessage = current.Commit.Message
return nil
}
func (r *regres) newTest(testListPath string, commit git.Hash) *test {
func (r *regres) newTest(commit git.Hash) *test {
srcDir := filepath.Join(r.cacheRoot, "src", commit.String())
resDir := filepath.Join(r.cacheRoot, "res", commit.String())
return &test{
r: r,
commit: commit,
srcDir: srcDir,
resDir: resDir,
outDir: filepath.Join(srcDir, "out"),
buildDir: filepath.Join(srcDir, "build"),
testListPath: testListPath,
r: r,
commit: commit,
srcDir: srcDir,
resDir: resDir,
outDir: filepath.Join(srcDir, "out"),
buildDir: filepath.Join(srcDir, "build"),
}
}
......@@ -467,129 +574,28 @@ type test struct {
outDir string // directory for SwiftShader output
buildDir string // directory for SwiftShader build
keepCheckouts bool // don't delete source & build checkouts after testing
testListPath string // relative path to the test list .json file
writtenTestLists []string // paths to test updated lists that have been written
}
// lazyRun lazily runs the test t.
// If the test results are not already cached, then test will setup the test
// environment, and call t.run().
// The results of the test will be cached into r.cacheRoot.
func (t *test) lazyRun() (*CommitTestResults, error) {
load := func(data []byte) (interface{}, error) {
var res CommitTestResults
if err := json.NewDecoder(bytes.NewReader(data)).Decode(&res); err != nil {
return nil, err
}
if res.Version != dataVersion {
return nil, errors.New("Data is from an old version")
}
return &res, nil
}
build := func() ([]byte, interface{}, error) {
res, err := t.run()
if err != nil {
return nil, nil, err
}
b := bytes.Buffer{}
enc := json.NewEncoder(&b)
enc.SetIndent("", " ")
if err := enc.Encode(res); err != nil {
return nil, nil, err
}
return b.Bytes(), res, nil
}
res, err := loadOrBuild(filepath.Join(t.resDir, "results.json"), load, build)
if err != nil {
return nil, err
// cleanup removes any temporary files used by the test.
func (t *test) cleanup() {
if t.srcDir != "" && !t.keepCheckouts {
os.RemoveAll(t.srcDir)
}
return res.(*CommitTestResults), nil
}
// run executes the tests for the test environment t.
// If the source is not cached, run will fetch the commit to be tested,
// before building it, and then run the required tests.
func (t *test) run() (*CommitTestResults, error) {
// checkout clones the test's source commit into t.src.
func (t *test) checkout() error {
if isDir(t.srcDir) && t.keepCheckouts {
log.Printf("Reusing source cache for commit '%s'\n", t.commit)
} else {
log.Printf("Checking out '%s'\n", t.commit)
os.RemoveAll(t.srcDir)
if err := git.Checkout(t.srcDir, gitURL, t.commit); err != nil {
return nil, cause.Wrap(err, "Checking out commit '%s'", t.commit)
}
log.Printf("Checked out commit '%s'\n", t.commit)
if !t.keepCheckouts {
defer os.RemoveAll(t.srcDir)
}
}
if err := t.build(); err != nil {
log.Printf("Warning: Commit '%s' failed to build. %v", t.commit, err)
return &CommitTestResults{Version: dataVersion, Built: false}, nil
}
log.Printf("Built '%s'\n", t.commit)
// Load the list of tests that need executing.
// Note: this list may vary by each commit.
testLists, err := t.loadTestList()
if err != nil {
return nil, cause.Wrap(err, "Loading test lists")
}
results, err := t.runTests(testLists)
if err != nil {
return nil, cause.Wrap(err, "Running tests")
}
log.Printf("Ran tests for '%s'\n", t.commit)
if t.keepCheckouts {
if err := t.writeTestListsByStatus(testLists, results); err != nil {
return nil, cause.Wrap(err, "Writing test lists by status")
}
}
return results, nil
}
// loadOrBuild is a helper for building a lazy resolved cache.
// loadOrBuild attempts to load the file at path. If the file exists and loaded
// successfully, then load() is called with the file data, and the the result
// object from load() is returned.
// If the file does not exist, the file cannot be loaded, or load() returns an
// error, then build() is called and the byte slice is saved to path, and the
// object is returned.
func loadOrBuild(path string,
load func([]byte) (interface{}, error),
build func() ([]byte, interface{}, error)) (interface{}, error) {
if data, err := ioutil.ReadFile(path); err == nil {
out, err := load(data)
if err == nil {
return out, nil
}
log.Printf("Warning: Failed to load '%s': %v", path, err)
os.Remove(path) // Delete and rebuild.
return nil
}
data, obj, err := build()
if err != nil {
return nil, err
log.Printf("Checking out '%s'\n", t.commit)
os.RemoveAll(t.srcDir)
if err := git.Checkout(t.srcDir, gitURL, t.commit); err != nil {
return cause.Wrap(err, "Checking out commit '%s'", t.commit)
}
os.MkdirAll(filepath.Dir(path), 0777)
if err := ioutil.WriteFile(path, data, 0777); err != nil {
log.Printf("Warning: Failed to write to '%s': %v", path, err)
}
return obj, nil
log.Printf("Checked out commit '%s'\n", t.commit)
return nil
}
// build builds the SwiftShader source into t.buildDir.
......@@ -614,8 +620,8 @@ func (t *test) build() error {
return nil
}
// runTests runs all the tests.
func (t *test) runTests(testLists []TestList) (*CommitTestResults, error) {
// run runs all the tests.
func (t *test) run(testLists testlist.Lists) (*CommitTestResults, error) {
log.Printf("Running tests for '%s'\n", t.commit)
start := time.Now()
......@@ -630,13 +636,13 @@ func (t *test) runTests(testLists []TestList) (*CommitTestResults, error) {
// Resolve the test runner
var exe string
switch list.API {
case egl:
case testlist.EGL:
exe = filepath.Join(t.r.deqpBuild, "modules", "egl", "deqp-egl")
case gles2:
case testlist.GLES2:
exe = filepath.Join(t.r.deqpBuild, "modules", "gles2", "deqp-gles2")
case gles3:
case testlist.GLES3:
exe = filepath.Join(t.r.deqpBuild, "modules", "gles3", "deqp-gles3")
case vulkan:
case testlist.Vulkan:
exe = filepath.Join(t.r.deqpBuild, "external", "vulkancts", "modules", "vulkan", "deqp-vk")
default:
return nil, fmt.Errorf("Unknown API '%v'", list.API)
......@@ -704,7 +710,9 @@ func (t *test) runTests(testLists []TestList) (*CommitTestResults, error) {
return &out, nil
}
func (t *test) writeTestListsByStatus(testLists []TestList, results *CommitTestResults) error {
func (t *test) writeTestListsByStatus(testLists testlist.Lists, results *CommitTestResults) ([]string, error) {
out := []string{}
for _, list := range testLists {
files := map[Status]*os.File{}
ext := filepath.Ext(list.File)
......@@ -715,12 +723,12 @@ func (t *test) writeTestListsByStatus(testLists []TestList, results *CommitTestR
os.MkdirAll(dir, 0777)
f, err := os.Create(path)
if err != nil {
return cause.Wrap(err, "Couldn't create file '%v'", path)
return nil, cause.Wrap(err, "Couldn't create file '%v'", path)
}
defer f.Close()
files[status] = f
t.writtenTestLists = append(t.writtenTestLists, path)
out = append(out, path)
}
for _, testName := range list.Tests {
......@@ -730,7 +738,13 @@ func (t *test) writeTestListsByStatus(testLists []TestList, results *CommitTestR
}
}
return nil
return out, nil
}
// resultsCachePath returns the path to the cache results file for the given
// test and testlists.
func (t *test) resultsCachePath(testLists testlist.Lists) string {
return filepath.Join(t.resDir, testLists.Hash())
}
// Status is an enumerator of test results.
......@@ -767,6 +781,16 @@ func (s Status) Failing() bool {
}
}
// Passing returns true if the task status is considered a pass.
func (s Status) Passing() bool {
switch s {
case Pass, CompatibilityWarning, QualityWarning:
return true
default:
return false
}
}
// CommitTestResults holds the results the tests across all APIs for a given
// commit. The CommitTestResults structure may be serialized to cache the
// results.
......@@ -777,6 +801,41 @@ type CommitTestResults struct {
Duration time.Duration
}
func loadCommitTestResults(path string) (*CommitTestResults, error) {
f, err := os.Open(path)
if err != nil {
return nil, cause.Wrap(err, "Couldn't open '%s' for loading test results", path)
}
defer f.Close()
var out CommitTestResults
if err := json.NewDecoder(f).Decode(&out); err != nil {
return nil, err
}
if out.Version != dataVersion {
return nil, errors.New("Data is from an old version")
}
return &out, nil
}
func (r *CommitTestResults) save(path string) error {
os.MkdirAll(filepath.Dir(path), 0777)
f, err := os.Create(path)
if err != nil {
return cause.Wrap(err, "Couldn't open '%s' for saving test results", path)
}
defer f.Close()
enc := json.NewEncoder(f)
enc.SetIndent("", " ")
if err := enc.Encode(r); err != nil {
return cause.Wrap(err, "Couldn't encode test results")
}
return nil
}
// compare returns a string describing all differences between two
// CommitTestResults. This string is used as the report message posted to the
// gerrit code review.
......@@ -798,9 +857,9 @@ func compare(old, new *CommitTestResults) string {
for test, new := range new.Tests {
old, found := old.Tests[test]
switch {
case (!found || old.Status == Pass) && new.Status.Failing():
case found && old.Status.Passing() && new.Status.Failing():
broken = append(broken, test)
case (found && old.Status.Failing()) && new.Status == Pass:
case found && old.Status.Failing() && new.Status.Passing():
fixed = append(fixed, test)
case found && old.Status != new.Status:
changed = append(changed, test)
......@@ -865,9 +924,9 @@ func compare(old, new *CommitTestResults) string {
case old == new:
sb.WriteString(fmt.Sprintf("%s: %v\n", s.label, new))
case change == 0:
sb.WriteString(fmt.Sprintf("%s: %v -> %v\n", s.label, old, new))
sb.WriteString(fmt.Sprintf("%s: %v -> %v (%+d)\n", s.label, old, new, new-old))
default:
sb.WriteString(fmt.Sprintf("%s: %v -> %v (%+d%%)\n", s.label, old, new, change))
sb.WriteString(fmt.Sprintf("%s: %v -> %v (%+d %+d%%)\n", s.label, old, new, new-old, change))
}
}
......@@ -990,89 +1049,30 @@ func (t *test) deqpTestRoutine(exe string, tests <-chan string, results chan<- T
}
}
// API is an enumerator of graphics APIs.
type API string
const (
egl = API("egl")
gles2 = API("gles2")
gles3 = API("gles3")
vulkan = API("vulkan")
)
// TestList is a list of tests to be run for a given API.
type TestList struct {
Name string
File string
API API
Tests []string
}
// loadTestList loads the test list json file.
// The file is first searched at {Commit}/{t.testListPath}
// loadTestLists loads the full test lists from the json file.
// The file is first searched at {t.srcDir}/{relPath}
// If this cannot be found, then the file is searched at the fallback path
// {CWD}/{t.testListPath}
// {CWD}/{relPath}
// This allows CLs to alter the list of tests to be run, as well as providing
// a default set.
func (t *test) loadTestList() ([]TestList, error) {
// find the test.json file in {SwiftShader}/tests/regres
root := t.srcDir
if isFile(filepath.Join(root, t.testListPath)) {
log.Println("Using test list from commit")
} else {
// Not found there. Search locally.
root, _ = os.Getwd()
if isFile(filepath.Join(root, t.testListPath)) {
log.Println("Using test list from regres")
} else {
return nil, fmt.Errorf("Could not find test list file '%v'", t.testListPath)
}
func (t *test) loadTestLists(relPath string) (testlist.Lists, error) {
// Seach for the test.json file in the checked out source directory.
if path := filepath.Join(t.srcDir, relPath); isFile(path) {
log.Printf("Loading test list '%v' from commit\n", relPath)
return testlist.Load(t.srcDir, path)
}
jsonPath := filepath.Join(root, t.testListPath)
i, err := ioutil.ReadFile(jsonPath)
// Not found there. Search locally.
wd, err := os.Getwd()
if err != nil {
return nil, cause.Wrap(err, "Couldn't read test list from '%s'", jsonPath)
return testlist.Lists{}, cause.Wrap(err, "Couldn't get current working directory")
}
var groups []struct {
Name string
API string
TestFile string `json:"tests"`
if path := filepath.Join(wd, relPath); isFile(path) {
log.Printf("Loading test list '%v' from regres\n", relPath)
return testlist.Load(wd, relPath)
}
if err := json.NewDecoder(bytes.NewReader(i)).Decode(&groups); err != nil {
return nil, cause.Wrap(err, "Couldn't parse '%s'", jsonPath)
}
dir := filepath.Dir(jsonPath)
out := make([]TestList, len(groups))
for i, group := range groups {
path := filepath.Join(dir, group.TestFile)
tests, err := ioutil.ReadFile(path)
if err != nil {
return nil, cause.Wrap(err, "Couldn't read '%s'", tests)
}
relPath, err := filepath.Rel(root, path)
if err != nil {
return nil, cause.Wrap(err, "Couldn't get relative path for '%s'", path)
}
list := TestList{
Name: group.Name,
File: relPath,
API: API(group.API),
}
for _, line := range strings.Split(string(tests), "\n") {
line = strings.TrimSpace(line)
if line != "" && !strings.HasPrefix(line, "#") {
list.Tests = append(list.Tests, line)
}
}
sort.Strings(list.Tests)
out[i] = list
}
return out, nil
return nil, errors.New("Couldn't find a test list file")
}
// isDir returns true if path is a file.
......
tests/regres/testlist/testlist.go 0 → 100644
// Copyright 2019 The SwiftShader Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package testlist provides utilities for handling test lists.
package testlist
import (
"bytes"
"crypto/sha1"
"encoding/gob"
"encoding/hex"
"encoding/json"
"io/ioutil"
"path/filepath"
"sort"
"strings"
"../cause"
)
// API is an enumerator of graphics APIs.
type API string
// Graphics APIs.
const (
EGL = API("egl")
GLES2 = API("gles2")
GLES3 = API("gles3")
Vulkan = API("vulkan")
)
// Group is a list of tests to be run for a single API.
type Group struct {
Name string
File string
API API
Tests []string
}
// Lists is the full list of tests to be run.
type Lists []Group
// Hash returns a SHA1 hash of the set of tests.
func (l Lists) Hash() string {
h := sha1.New()
if err := gob.NewEncoder(h).Encode(l); err != nil {
panic(cause.Wrap(err, "Could not encode testlist to produce hash"))
}
var hash [20]byte
copy(hash[:], h.Sum(nil))
return hex.EncodeToString(hash[:])
}
// Load loads the test list json file and returns the full set of tests.
func Load(root, jsonPath string) (Lists, error) {
root, err := filepath.Abs(root)
if err != nil {
return nil, cause.Wrap(err, "Couldn't get absolute path of '%s'", root)
}
jsonPath, err = filepath.Abs(jsonPath)
if err != nil {
return nil, cause.Wrap(err, "Couldn't get absolute path of '%s'", jsonPath)
}
i, err := ioutil.ReadFile(jsonPath)
if err != nil {
return nil, cause.Wrap(err, "Couldn't read test list from '%s'", jsonPath)
}
var jsonGroups []struct {
Name string
API string
TestFile string `json:"tests"`
}
if err := json.NewDecoder(bytes.NewReader(i)).Decode(&jsonGroups); err != nil {
return nil, cause.Wrap(err, "Couldn't parse '%s'", jsonPath)
}
dir := filepath.Dir(jsonPath)
out := make(Lists, len(jsonGroups))
for i, jsonGroup := range jsonGroups {
path := filepath.Join(dir, jsonGroup.TestFile)
tests, err := ioutil.ReadFile(path)
if err != nil {
return nil, cause.Wrap(err, "Couldn't read '%s'", tests)
}
relPath, err := filepath.Rel(root, path)
if err != nil {
return nil, cause.Wrap(err, "Couldn't get relative path for '%s'", path)
}
group := Group{
Name: jsonGroup.Name,
File: relPath,
API: API(jsonGroup.API),
}
for _, line := range strings.Split(string(tests), "\n") {
line = strings.TrimSpace(line)
if line != "" && !strings.HasPrefix(line, "#") {
group.Tests = append(group.Tests, line)
}
}
sort.Strings(group.Tests)
out[i] = group
}
return out, nil
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment