dEQP test_expectations parser should output an error on unused lines.

src/tests/deqp_support/angle_deqp_gtest.cpp

@@ -296,6 +296,19 @@ void dEQPCaseList::initialize()
         int expectation = mTestExpectationsParser.getTestExpectation(dEQPName);
         mCaseInfoList.push_back(CaseInfo(dEQPName, gTestName, expectation));
     }
+
+    std::stringstream unusedMsgStream;
+    bool anyUnused = false;
+    for (const auto &message : mTestExpectationsParser.getUnusedExpectationsMessages())
+    {
+        anyUnused = true;
+        unusedMsgStream << std::endl << " " << message;
+    }
+    if (anyUnused)
+    {
+        std::cerr << "Failed to validate test expectations." << unusedMsgStream.str() << std::endl;
+        Die();
+    }
 }
 
 template <size_t TestModuleIndex>

src/tests/deqp_support/deqp_egl_test_expectations.txt

@@ -84,7 +84,6 @@
 1340 WIN : dEQP-EGL.functional.render.multi_context.gles2_gles3.rgba5551_window = FAIL
 1340 WIN : dEQP-EGL.functional.render.multi_context.gles2_gles3.rgba5551_pbuffer = FAIL
 1340 WIN : dEQP-EGL.functional.negative_api.choose_config = SKIP
-1340 WIN : dEQP-EGL.functional.negative_api.get_display = FAIL
 1340 WIN : dEQP-EGL.functional.negative_api.surface_attrib = SKIP
 1340 WIN : dEQP-EGL.functional.negative_api.swap_interval = FAIL
 2382 WIN : dEQP-EGL.functional.native_color_mapping.native_window.* = SKIP
@@ -142,7 +141,6 @@
 2546 LINUX : dEQP-EGL.functional.query_surface.simple.pixmap.rgba8888_depth_stencil = SKIP
 2546 LINUX : dEQP-EGL.functional.query_surface.simple.pixmap.rgba8888_no_depth_no_stencil = SKIP
 2546 LINUX : dEQP-EGL.functional.render.multi_context.* = SKIP
-2546 LINUX : dEQP-EGL.functional.render.multi_context.gles2.rgba8888_pixmap = SKIP
 2546 LINUX : dEQP-EGL.functional.render.single_context.gles2.rgba8888_pixmap = SKIP
 2546 LINUX : dEQP-EGL.functional.render.single_context.gles3.rgba8888_pixmap = SKIP
 2546 LINUX : dEQP-EGL.functional.thread_cleanup.* = SKIP
@@ -220,13 +218,10 @@
 // ES 1 tests
 2306 WIN : dEQP-EGL.functional.color_clears.single_context.gles1* = FAIL
 2306 WIN : dEQP-EGL.functional.color_clears.multi_context.gles1* = FAIL
-2306 WIN : dEQP-EGL.functional.color_clears.multi_thread.gles1* = FAIL
 2306 MAC : dEQP-EGL.functional.color_clears.single_context.gles1* = FAIL
 2306 MAC : dEQP-EGL.functional.color_clears.multi_context.gles1* = FAIL
-2306 MAC : dEQP-EGL.functional.color_clears.multi_thread.gles1* = FAIL
 2306 LINUX : dEQP-EGL.functional.color_clears.single_context.gles1* = FAIL
 2306 LINUX : dEQP-EGL.functional.color_clears.multi_context.gles1* = FAIL
-2306 LINUX : dEQP-EGL.functional.color_clears.multi_thread.gles1* = FAIL
 
 // Tests failing since the Jan 2018 roll of dEQP
 2341 : dEQP-EGL.functional.swap_buffers.* = SKIP

src/tests/deqp_support/deqp_gles2_test_expectations.txt

@@ -32,10 +32,6 @@
 // Skip these tests due to timeouts
 1034 : dEQP-GLES2.functional.flush_finish.* = SKIP
 
-// Don't run these tests for faster turnover
-998 : dEQP-GLES2.performance.* = SKIP
-998 : dEQP-GLES2.stress.* = SKIP
-
 // Failures related to not supporting separate depth/stencil masks on D3D11.
 1655 D3D11 : dEQP-GLES2.functional.fragment_ops.depth_stencil.stencil_depth_funcs.stencil_* = FAIL
 1655 D3D11 : dEQP-GLES2.functional.fragment_ops.depth_stencil.stencil_ops.* = FAIL
@@ -292,12 +288,19 @@
 3243 VULKAN : dEQP-GLES2.functional.texture.mipmap.cube.projected.linear_linear = FAIL
 3243 VULKAN : dEQP-GLES2.functional.texture.mipmap.cube.bias.linear_nearest = FAIL
 3243 VULKAN : dEQP-GLES2.functional.texture.mipmap.cube.bias.linear_linear = FAIL
-3243 D3D11 : dEQP-GLES2.functional.texture.mipmap.cube.basic.linear_nearest = FAIL
-3243 D3D11 : dEQP-GLES2.functional.texture.mipmap.cube.basic.linear_linear = FAIL
-3243 D3D11 : dEQP-GLES2.functional.texture.mipmap.cube.projected.linear_nearest = FAIL
-3243 D3D11 : dEQP-GLES2.functional.texture.mipmap.cube.projected.linear_linear = FAIL
-3243 D3D11 : dEQP-GLES2.functional.texture.mipmap.cube.bias.linear_nearest = FAIL
-3243 D3D11 : dEQP-GLES2.functional.texture.mipmap.cube.bias.linear_linear = FAIL
+// D3D11 AMD already covered by Line 148
+3243 D3D11 INTEL : dEQP-GLES2.functional.texture.mipmap.cube.basic.linear_nearest = FAIL
+3243 D3D11 INTEL : dEQP-GLES2.functional.texture.mipmap.cube.basic.linear_linear = FAIL
+3243 D3D11 INTEL : dEQP-GLES2.functional.texture.mipmap.cube.projected.linear_nearest = FAIL
+3243 D3D11 INTEL : dEQP-GLES2.functional.texture.mipmap.cube.projected.linear_linear = FAIL
+3243 D3D11 INTEL : dEQP-GLES2.functional.texture.mipmap.cube.bias.linear_nearest = FAIL
+3243 D3D11 INTEL : dEQP-GLES2.functional.texture.mipmap.cube.bias.linear_linear = FAIL
+3243 D3D11 NVIDIA : dEQP-GLES2.functional.texture.mipmap.cube.basic.linear_nearest = FAIL
+3243 D3D11 NVIDIA : dEQP-GLES2.functional.texture.mipmap.cube.basic.linear_linear = FAIL
+3243 D3D11 NVIDIA : dEQP-GLES2.functional.texture.mipmap.cube.projected.linear_nearest = FAIL
+3243 D3D11 NVIDIA : dEQP-GLES2.functional.texture.mipmap.cube.projected.linear_linear = FAIL
+3243 D3D11 NVIDIA : dEQP-GLES2.functional.texture.mipmap.cube.bias.linear_nearest = FAIL
+3243 D3D11 NVIDIA : dEQP-GLES2.functional.texture.mipmap.cube.bias.linear_linear = FAIL
 
 // Fail with very tiny pixel differences
 3240 VULKAN : dEQP-GLES2.functional.texture.vertex.cube.filtering.linear_mipmap_linear_nearest_clamp = FAIL

src/tests/deqp_support/deqp_gles31_test_expectations.txt

@@ -51,18 +51,6 @@
 1442 : dEQP-GLES31.functional.program_interface_query.transform_feedback_varying.type.vertex_fragment.struct.* = SKIP
 1442 : dEQP-GLES31.functional.image_load_store.* = SKIP
 
-1442 D3D11 : dEQP-GLES31.functional.stencil_texturing.* = SKIP
-// TODO(xinghua.cao@intel.com): FAIL expectation instead of SKIP should be sufficient for OpenGL, but the
-// test expectations parser doesn't support having FAIL for GL and SKIP for D3D with the same test filter.
-
-1729 D3D11 : dEQP-GLES31.functional.atomic_counter.* = SKIP
-1951 D3D11 : dEQP-GLES31.functional.layout_binding.ssbo.* = SKIP
-1951 D3D11 : dEQP-GLES31.functional.shaders.opaque_type_indexing.ssbo.* = SKIP
-1442 D3D11 : dEQP-GLES31.functional.program_interface_query.* = SKIP
-1442 D3D11 : dEQP-GLES31.functional.synchronization.* = SKIP
-1442 D3D11 : dEQP-GLES31.functional.layout_binding.image.* = SKIP
-// This case is skipped since it uses atomic counter builtin functions which haven't been implemented.
-1729 D3D11 : dEQP-GLES31.functional.state_query.program.active_atomic_counter_buffers_get_programiv = SKIP
 
 // D3D11 Failing Tests
 // Below cases are failed due to binding multi-UAVs to a single buffer is not supported in D3D backend.
@@ -605,3 +593,16 @@
 1951 D3D11 : dEQP-GLES31.functional.debug.negative_coverage.callbacks.compute.exceed_shared_memory_size_limit = FAIL
 1951 D3D11 : dEQP-GLES31.functional.debug.negative_coverage.get_error.compute.exceed_shared_memory_size_limit = FAIL
 1951 D3D11 : dEQP-GLES31.functional.debug.negative_coverage.log.compute.exceed_shared_memory_size_limit = FAIL
+
+1442 D3D11 : dEQP-GLES31.functional.stencil_texturing.* = SKIP
+// TODO(xinghua.cao@intel.com): FAIL expectation instead of SKIP should be sufficient for OpenGL, but the
+// test expectations parser doesn't support having FAIL for GL and SKIP for D3D with the same test filter.
+
+1729 D3D11 : dEQP-GLES31.functional.atomic_counter.* = SKIP
+1951 D3D11 : dEQP-GLES31.functional.layout_binding.ssbo.* = SKIP
+1951 D3D11 : dEQP-GLES31.functional.shaders.opaque_type_indexing.ssbo.* = SKIP
+1442 D3D11 : dEQP-GLES31.functional.program_interface_query.* = SKIP
+1442 D3D11 : dEQP-GLES31.functional.synchronization.* = SKIP
+1442 D3D11 : dEQP-GLES31.functional.layout_binding.image.* = SKIP
+// This case is skipped since it uses atomic counter builtin functions which haven't been implemented.
+1729 D3D11 : dEQP-GLES31.functional.state_query.program.active_atomic_counter_buffers_get_programiv = SKIP

src/tests/deqp_support/deqp_gles3_test_expectations.txt

@@ -30,8 +30,6 @@
 //  91531 NEXUS5X GLES : conformance_more_* = SKIP
 
 // Don't run these tests for faster turnover
-998 : dEQP-GLES3.performance.* = SKIP
-998 : dEQP-GLES3.stress.* = SKIP
 1101 : dEQP-GLES3.functional.flush_finish.* = SKIP
 
 // Figure out why this caused this regression in Chrome:
@@ -121,10 +119,6 @@
 1095 LINUX : dEQP-GLES3.functional.texture.mipmap.2d.projected.* = FAIL
 
 // Windows only failure
-1092 WIN D3D11 : dEQP-GLES3.functional.shaders.texture_functions.textureoffset.sampler3d_fixed_fragment = FAIL
-1092 WIN D3D11 : dEQP-GLES3.functional.shaders.texture_functions.textureoffset.sampler3d_float_fragment = FAIL
-1092 WIN D3D11 : dEQP-GLES3.functional.shaders.texture_functions.textureprojoffset.sampler3d_fixed_fragment = FAIL
-1092 WIN D3D11 : dEQP-GLES3.functional.shaders.texture_functions.textureprojoffset.sampler3d_float_fragment = FAIL
 1096 WIN D3D11 : dEQP-GLES3.functional.fragment_ops.depth_stencil.stencil_depth_funcs.stencil_* = FAIL
 1096 WIN D3D11 : dEQP-GLES3.functional.fragment_ops.depth_stencil.stencil_ops.* = FAIL
 1096 WIN D3D11 : dEQP-GLES3.functional.fragment_ops.depth_stencil.write_mask.* = FAIL
@@ -274,12 +268,6 @@
 1323 LINUX OPENGL : dEQP-GLES3.functional.fbo.depth.depth_write_clamp.depth_component32f = FAIL
 1323 LINUX OPENGL : dEQP-GLES3.functional.fbo.depth.depth_write_clamp.depth32f_stencil8 = FAIL
 1323 LINUX OPENGL : dEQP-GLES3.functional.rasterization.fbo.rbo_multisample_max.primitives.points = FAIL
-1323 LINUX OPENGL : dEQP-GLES3.functional.texture.mipmap.2d.projected.linear_linear_clamp = FAIL
-1323 LINUX OPENGL : dEQP-GLES3.functional.texture.mipmap.2d.projected.linear_linear_mirror = FAIL
-1323 LINUX OPENGL : dEQP-GLES3.functional.texture.mipmap.2d.projected.linear_linear_repeat = FAIL
-1323 LINUX OPENGL : dEQP-GLES3.functional.texture.mipmap.2d.projected.nearest_linear_clamp = FAIL
-1323 LINUX OPENGL : dEQP-GLES3.functional.texture.mipmap.2d.projected.nearest_linear_mirror = FAIL
-1323 LINUX OPENGL : dEQP-GLES3.functional.texture.mipmap.2d.projected.nearest_linear_repeat = FAIL
 
 // While loops with 'continue' seem flaky on Linux. (Possibly all GL renderers)
 1324 LINUX OPENGL : dEQP-GLES3.functional.shaders.loops.while_constant_iterations.conditional_continue_vertex = FAIL
@@ -700,9 +688,6 @@
 2950 VULKAN : dEQP-GLES3.functional.state_query.integers.max_samples_get* = FAIL
 2950 VULKAN : dEQP-GLES3.functional.state_query.integers.min_program_texel_offset_get* = FAIL
 2950 VULKAN : dEQP-GLES3.functional.state_query.integers.compressed_texture_formats_get* = FAIL
-2950 VULKAN : dEQP-GLES3.functional.state_query.integers.max_uniform_block_size_get* = FAIL
-2950 VULKAN : dEQP-GLES3.functional.state_query.integers.max_combined_vertex_uniform_components_get* = FAIL
-2950 VULKAN : dEQP-GLES3.functional.state_query.integers.max_combined_fragment_uniform_components_get* = FAIL
 2950 VULKAN : dEQP-GLES3.functional.state_query.indexed.uniform_buffer* = FAIL
 2950 VULKAN : dEQP-GLES3.functional.state_query.integers64.max_uniform_block_size_get* = FAIL
 2950 VULKAN : dEQP-GLES3.functional.state_query.integers64.max_combined_vertex_uniform_components_get* = FAIL

src/tests/test_expectations/GPUTestExpectationsParser.cpp

@@ -272,12 +272,27 @@ bool GPUTestExpectationsParser::loadTestExpectationsFromFile(const GPUTestConfig
     return loadTestExpectations(config, data);
 }
 
-int32_t GPUTestExpectationsParser::getTestExpectation(const std::string &testName) const
+int32_t GPUTestExpectationsParser::getTestExpectation(const std::string &testName)
 {
+    size_t maxExpectationLen            = 0;
+    GPUTestExpectationEntry *foundEntry = nullptr;
     for (size_t i = 0; i < mEntries.size(); ++i)
     {
-        if (NamesMatching(mEntries[i].mTestName, testName))
-            return mEntries[i].mTestExpectation;
+        if (NamesMatching(mEntries[i].testName, testName))
+        {
+            size_t expectationLen = mEntries[i].testName.length();
+            // The longest/most specific matching expectation overrides any others.
+            if (expectationLen > maxExpectationLen)
+            {
+                maxExpectationLen = expectationLen;
+                foundEntry        = &mEntries[i];
+            }
+        }
+    }
+    if (foundEntry != nullptr)
+    {
+        foundEntry->used = true;
+        return foundEntry->testExpectation;
     }
     return kGpuTestPass;
 }
@@ -287,6 +302,20 @@ const std::vector<std::string> &GPUTestExpectationsParser::getErrorMessages() co
     return mErrorMessages;
 }
 
+std::vector<std::string> GPUTestExpectationsParser::getUnusedExpectationsMessages() const
+{
+    std::vector<std::string> messages;
+    std::vector<GPUTestExpectationsParser::GPUTestExpectationEntry> unusedExpectations =
+        getUnusedExpectations();
+    for (size_t i = 0; i < unusedExpectations.size(); ++i)
+    {
+        std::string message =
+            "Line " + ToString(unusedExpectations[i].lineNumber) + ": expectation was unused.";
+        messages.push_back(message);
+    }
+    return messages;
+}
+
 bool GPUTestExpectationsParser::parseLine(const GPUTestConfig &config,
                                           const std::string &lineData,
                                           size_t lineNumber)
@@ -295,7 +324,8 @@ bool GPUTestExpectationsParser::parseLine(const GPUTestConfig &config,
         SplitString(lineData, kWhitespaceASCII, KEEP_WHITESPACE, SPLIT_WANT_NONEMPTY);
     int32_t stage = kLineParserBegin;
     GPUTestExpectationEntry entry;
-    entry.mLineNumber = lineNumber;
+    entry.lineNumber  = lineNumber;
+    entry.used        = false;
     bool skipLine     = false;
     for (size_t i = 0; i < tokens.size() && !skipLine; ++i)
     {
@@ -392,7 +422,7 @@ bool GPUTestExpectationsParser::parseLine(const GPUTestConfig &config,
                 }
                 else if (stage == kLineParserColon)
                 {
-                    entry.mTestName = tokens[i];
+                    entry.testName = tokens[i];
                 }
                 else
                 {
@@ -412,13 +442,13 @@ bool GPUTestExpectationsParser::parseLine(const GPUTestConfig &config,
                     pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber);
                     return false;
                 }
-                if (entry.mTestExpectation != 0)
+                if (entry.testExpectation != 0)
                 {
                     pushErrorMessage(kErrorMessage[kErrorEntryWithExpectationConflicts],
                                      lineNumber);
                     return false;
                 }
-                entry.mTestExpectation = kTokenData[token].expectation;
+                entry.testExpectation = kTokenData[token].expectation;
                 if (stage == kLineParserEqual)
                     stage++;
                 break;
@@ -472,10 +502,10 @@ bool GPUTestExpectationsParser::detectConflictsBetweenEntries()
     {
         for (size_t j = i + 1; j < mEntries.size(); ++j)
         {
-            if (mEntries[i].mTestName == mEntries[j].mTestName)
+            if (mEntries[i].testName == mEntries[j].testName)
             {
-                pushErrorMessage(kErrorMessage[kErrorEntriesOverlap], mEntries[i].mLineNumber,
-                                 mEntries[j].mLineNumber);
+                pushErrorMessage(kErrorMessage[kErrorEntriesOverlap], mEntries[i].lineNumber,
+                                 mEntries[j].lineNumber);
                 rt = true;
             }
         }
@@ -483,6 +513,20 @@ bool GPUTestExpectationsParser::detectConflictsBetweenEntries()
     return rt;
 }
 
+std::vector<GPUTestExpectationsParser::GPUTestExpectationEntry>
+GPUTestExpectationsParser::getUnusedExpectations() const
+{
+    std::vector<GPUTestExpectationsParser::GPUTestExpectationEntry> unusedExpectations;
+    for (size_t i = 0; i < mEntries.size(); ++i)
+    {
+        if (!mEntries[i].used)
+        {
+            unusedExpectations.push_back(mEntries[i]);
+        }
+    }
+    return unusedExpectations;
+}
+
 void GPUTestExpectationsParser::pushErrorMessage(const std::string &message, size_t lineNumber)
 {
     mErrorMessages.push_back("Line " + ToString(lineNumber) + " : " + message.c_str());
@@ -497,7 +541,7 @@ void GPUTestExpectationsParser::pushErrorMessage(const std::string &message,
 }
 
 GPUTestExpectationsParser::GPUTestExpectationEntry::GPUTestExpectationEntry()
-    : mTestExpectation(0), mLineNumber(0)
+    : testExpectation(0), lineNumber(0)
 {}
 
 }  // namespace angle

src/tests/test_expectations/GPUTestExpectationsParser.h

@@ -42,17 +42,21 @@ class GPUTestExpectationsParser
     // Query error messages from the last LoadTestExpectations() call.
     const std::vector<std::string> &getErrorMessages() const;
 
+    // Query error messages from any expectations that weren't used before being queried.
+    std::vector<std::string> getUnusedExpectationsMessages() const;
+
     // Get the test expectation of a given test on a given bot.
-    int32_t getTestExpectation(const std::string &testName) const;
+    int32_t getTestExpectation(const std::string &testName);
 
   private:
     struct GPUTestExpectationEntry
     {
         GPUTestExpectationEntry();
 
-        std::string mTestName;
-        int32_t mTestExpectation;
-        size_t mLineNumber;
+        std::string testName;
+        int32_t testExpectation;
+        size_t lineNumber;
+        bool used;
     };
 
     // Parse a line of text. If we have a valid entry, save it; otherwise,
@@ -69,6 +73,9 @@ class GPUTestExpectationsParser
     // error message.
     bool detectConflictsBetweenEntries();
 
+    // Query a list of any expectations that were's used before being queried.
+    std::vector<GPUTestExpectationEntry> getUnusedExpectations() const;
+
     // Save an error message, which can be queried later.
     void pushErrorMessage(const std::string &message, size_t lineNumber);
     void pushErrorMessage(const std::string &message,

src/tests/test_expectations/GPUTestExpectationsParser_unittest.cpp

@@ -352,4 +352,73 @@ TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserComment)
         GPUTestExpectationsParser::kGpuTestPass);
 }
 
+// A misspelled expectation should not be matched from getTestExpectation, and should lead to an
+// unused expectation when later queried.
+TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserMisspelledExpectation)
+{
+    GPUTestConfigTester config;
+    std::string line =
+        R"(100 : dEQP-GLES31.functionaal.layout_binding.ubo.* = SKIP)";  // "functionaal"
+    GPUTestExpectationsParser parser;
+    EXPECT_TRUE(parser.loadTestExpectations(config, line));
+    EXPECT_TRUE(parser.getErrorMessages().empty());
+    // Default behavior is to let missing tests pass
+    EXPECT_EQ(
+        parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
+        GPUTestExpectationsParser::kGpuTestPass);
+    EXPECT_EQ(parser.getUnusedExpectationsMessages().size(), 1u);
+    if (parser.getUnusedExpectationsMessages().size() >= 1)
+        EXPECT_EQ(parser.getUnusedExpectationsMessages()[0], "Line 1: expectation was unused.");
+}
+
+// Wild characters that match groups of expectations can be overridden with more specific lines.
+// The parse should still compute correctly which lines were used and which were unused.
+TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserOverrideExpectation)
+{
+    GPUTestConfigTester config;
+    // Fail all layout_binding tests, but skip the layout_binding.ubo subset.
+    std::string line = R"(100 : dEQP-GLES31.functional.layout_binding.* = FAIL
+100 : dEQP-GLES31.functional.layout_binding.ubo.* = SKIP)";
+    GPUTestExpectationsParser parser;
+    EXPECT_TRUE(parser.loadTestExpectations(config, line));
+    EXPECT_TRUE(parser.getErrorMessages().empty());
+    // Default behavior is to let missing tests pass
+    EXPECT_EQ(
+        parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
+        GPUTestExpectationsParser::kGpuTestSkip);
+    // The FAIL expectation was unused because it was overridden.
+    EXPECT_EQ(parser.getUnusedExpectationsMessages().size(), 1u);
+    if (parser.getUnusedExpectationsMessages().size() >= 1)
+        EXPECT_EQ(parser.getUnusedExpectationsMessages()[0], "Line 1: expectation was unused.");
+    // Now try a test that doesn't match the override criteria
+    EXPECT_EQ(parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.image.test"),
+              GPUTestExpectationsParser::kGpuTestFail);
+    EXPECT_TRUE(parser.getUnusedExpectationsMessages().empty());
+}
+
+// This test is the same as GPUTestExpectationsParserOverrideExpectation, but verifying the order
+// doesn't matter when overriding.
+TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserOverrideExpectationOtherOrder)
+{
+    GPUTestConfigTester config;
+    // Fail all layout_binding tests, but skip the layout_binding.ubo subset.
+    std::string line = R"(100 : dEQP-GLES31.functional.layout_binding.ubo.* = SKIP
+100 : dEQP-GLES31.functional.layout_binding.* = FAIL)";
+    GPUTestExpectationsParser parser;
+    EXPECT_TRUE(parser.loadTestExpectations(config, line));
+    EXPECT_TRUE(parser.getErrorMessages().empty());
+    // Default behavior is to let missing tests pass
+    EXPECT_EQ(
+        parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
+        GPUTestExpectationsParser::kGpuTestSkip);
+    // The FAIL expectation was unused because it was overridden.
+    EXPECT_EQ(parser.getUnusedExpectationsMessages().size(), 1u);
+    if (parser.getUnusedExpectationsMessages().size() >= 1)
+        EXPECT_EQ(parser.getUnusedExpectationsMessages()[0], "Line 2: expectation was unused.");
+    // Now try a test that doesn't match the override criteria
+    EXPECT_EQ(parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.image.test"),
+              GPUTestExpectationsParser::kGpuTestFail);
+    EXPECT_TRUE(parser.getUnusedExpectationsMessages().empty());
+}
+
 }  // anonymous namespace