Add support for internal error status checking in regres

tests/regres/cmd/run_testlist/main.go

@@ -53,16 +53,17 @@ func min(a, b int) int {
 }
 
 var (
-	deqpVkBinary  = flag.String("deqp-vk", "deqp-vk", "path to the deqp-vk binary")
-	testList      = flag.String("test-list", "vk-master-PASS.txt", "path to a test list file")
-	numThreads    = flag.Int("num-threads", min(runtime.NumCPU(), 100), "number of parallel test runner processes")
-	maxProcMemory = flag.Uint64("max-proc-mem", shell.MaxProcMemory, "maximum virtual memory per child process")
-	output        = flag.String("output", "results.json", "path to an output JSON results file")
-	filter        = flag.String("filter", "", "filter for test names. Start with a '/' to indicate regex")
-	limit         = flag.Int("limit", 0, "only run a maximum of this number of tests")
-	shuffle       = flag.Bool("shuffle", false, "shuffle tests")
-	noResults     = flag.Bool("no-results", false, "disable generation of results.json file")
-	genCoverage   = flag.Bool("coverage", false, "generate test coverage")
+	deqpVkBinary     = flag.String("deqp-vk", "deqp-vk", "path to the deqp-vk binary")
+	testList         = flag.String("test-list", "vk-master-PASS.txt", "path to a test list file")
+	numThreads       = flag.Int("num-threads", min(runtime.NumCPU(), 100), "number of parallel test runner processes")
+	maxProcMemory    = flag.Uint64("max-proc-mem", shell.MaxProcMemory, "maximum virtual memory per child process")
+	output           = flag.String("output", "results.json", "path to an output JSON results file")
+	filter           = flag.String("filter", "", "filter for test names. Start with a '/' to indicate regex")
+	limit            = flag.Int("limit", 0, "only run a maximum of this number of tests")
+	shuffle          = flag.Bool("shuffle", false, "shuffle tests")
+	noResults        = flag.Bool("no-results", false, "disable generation of results.json file")
+	genCoverage      = flag.Bool("coverage", false, "generate test coverage")
+	enableValidation = flag.Bool("validation", false, "run deqp-vk with Vulkan validation layers")
 )
 
 const testTimeout = time.Minute * 2
@@ -111,6 +112,7 @@ func run() error {
 		NumParallelTests: *numThreads,
 		TestLists:        testlist.Lists{group},
 		TestTimeout:      testTimeout,
+		ValidationLayer:  *enableValidation,
 	}
 
 	if *genCoverage {

tests/regres/deqp/deqp.go

@@ -42,7 +42,7 @@ const dataVersion = 1
 
 var (
 	// Regular expression to parse the output of a dEQP test.
-	deqpRE = regexp.MustCompile(`(Fail|Pass|NotSupported|CompatibilityWarning|QualityWarning) \(([^\)]*)\)`)
+	deqpRE = regexp.MustCompile(`(Fail|Pass|NotSupported|CompatibilityWarning|QualityWarning|InternalError) \(([^\)]*)\)`)
 	// Regular expression to parse a test that failed due to UNIMPLEMENTED()
 	unimplementedRE = regexp.MustCompile(`[^\n]*UNIMPLEMENTED:[^\n]*`)
 	// Regular expression to parse a test that failed due to UNSUPPORTED()
@@ -68,6 +68,7 @@ type Config struct {
 	NumParallelTests int
 	CoverageEnv      *cov.Env
 	TestTimeout      time.Duration
+	ValidationLayer  bool
 }
 
 // Results holds the results of tests across all APIs.
@@ -316,7 +317,14 @@ nextTest:
 		// log.Printf("Running test '%s'\n", name)
 
 		start := time.Now()
+		// Set validation layer according to flag.
+		validation := "disable"
+		if c.ValidationLayer {
+			validation = "enable"
+		}
+
 		outRaw, err := shell.Exec(c.TestTimeout, exe, filepath.Dir(exe), env,
+			"--deqp-validation="+validation,
 			"--deqp-surface-type=pbuffer",
 			"--deqp-shadercache=disable",
 			"--deqp-log-images=disable",
@@ -411,6 +419,12 @@ nextTest:
 					err = toks[2]
 				}
 				results <- TestResult{Test: name, Status: testlist.Fail, Err: err, TimeTaken: duration, Coverage: coverage}
+			case "InternalError":
+				var err string
+				if toks[2] != "InternalError" {
+					err = toks[2]
+				}
+				results <- TestResult{Test: name, Status: testlist.InternalError, Err: err, TimeTaken: duration, Coverage: coverage}
 			default:
 				err := fmt.Sprintf("Couldn't parse test output:\n%s", out)
 				log.Println("Warning: ", err)

tests/regres/testlist/testlist.go

@@ -198,6 +198,8 @@ const (
 	CompatibilityWarning = Status("COMPATIBILITY_WARNING")
 	// QualityWarning is the status passing test with a warning.
 	QualityWarning = Status("QUALITY_WARNING")
+	// InternalError is the status of a test that failed on an API usage error.
+	InternalError = Status("INTERNAL_ERROR")
 )
 
 // Statuses is the full list of status types
@@ -214,12 +216,13 @@ var Statuses = []Status{
 	NotSupported,
 	CompatibilityWarning,
 	QualityWarning,
+	InternalError,
 }
 
 // Failing returns true if the task status requires fixing.
 func (s Status) Failing() bool {
 	switch s {
-	case Fail, Timeout, Crash, Unimplemented, Unreachable, Assert, Abort:
+	case Fail, Timeout, Crash, Unimplemented, Unreachable, Assert, Abort, InternalError:
 		return true
 	case Unsupported:
 		// This may seem surprising that this should be a failure, however these