Skip to content

A
angle
Commit e601181d by Jamie Madill Committed by Commit Bot
Options

Allow specifying GPU config on expectation check.

When we load the expectations we can now specify the config at a later point. This will make it easier to run test suites that use multiple APIs, like angle_end2end_tests. We can instead specify the config when we check the expectation. Bug: angleproject:5951 Change-Id: I9607db093e4a459550a7cd6864b17adfa55e17f2 Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/2892273 Commit-Queue: Jamie Madill <jmadill@chromium.org> Reviewed-by: 's avatarJonah Ryan-Davis <jonahr@google.com> Reviewed-by: 's avatarGeoff Lang <geofflang@chromium.org>
parent c5e344b1
Showing with 239 additions and 183 deletions
src/tests/test_expectations/GPUTestConfig.cpp
......@@ -330,7 +330,7 @@ inline bool GetGPUTestSystemInfo(SystemInfo **sysInfo)
sSystemInfo = new SystemInfo;
if (!GetSystemInfo(sSystemInfo))
{
std::cout << "Error populating SystemInfo for dEQP tests." << std::endl;
std::cout << "Error populating SystemInfo." << std::endl;
}
else
{
......
src/tests/test_expectations/GPUTestExpectationsParser.cpp
......@@ -14,8 +14,6 @@
#include "common/debug.h"
#include "common/string_utils.h"
#include "GPUTestConfig.h"
namespace angle
{
......@@ -272,8 +270,8 @@ GPUTestExpectationsParser::GPUTestExpectationsParser()
GPUTestExpectationsParser::~GPUTestExpectationsParser() = default;
bool GPUTestExpectationsParser::loadTestExpectations(const GPUTestConfig &config,
const std::string &data)
bool GPUTestExpectationsParser::loadTestExpectationsImpl(const GPUTestConfig *config,
const std::string &data)
{
mEntries.clear();
mErrorMessages.clear();
......@@ -294,8 +292,19 @@ bool GPUTestExpectationsParser::loadTestExpectations(const GPUTestConfig &config
return rt;
}
bool GPUTestExpectationsParser::loadTestExpectationsFromFile(const GPUTestConfig &config,
const std::string &path)
bool GPUTestExpectationsParser::loadTestExpectations(const GPUTestConfig &config,
const std::string &data)
{
return loadTestExpectationsImpl(&config, data);
}
bool GPUTestExpectationsParser::loadAllTestExpectations(const std::string &data)
{
return loadTestExpectationsImpl(nullptr, data);
}
bool GPUTestExpectationsParser::loadTestExpectationsFromFileImpl(const GPUTestConfig *config,
const std::string &path)
{
mEntries.clear();
mErrorMessages.clear();
......@@ -306,23 +315,50 @@ bool GPUTestExpectationsParser::loadTestExpectationsFromFile(const GPUTestConfig
mErrorMessages.push_back(kErrorMessage[kErrorFileIO]);
return false;
}
return loadTestExpectations(config, data);
return loadTestExpectationsImpl(config, data);
}
int32_t GPUTestExpectationsParser::getTestExpectation(const std::string &testName)
bool GPUTestExpectationsParser::loadTestExpectationsFromFile(const GPUTestConfig &config,
const std::string &path)
{
return loadTestExpectationsFromFileImpl(&config, path);
}
bool GPUTestExpectationsParser::loadAllTestExpectationsFromFile(const std::string &path)
{
return loadTestExpectationsFromFileImpl(nullptr, path);
}
int32_t GPUTestExpectationsParser::getTestExpectationImpl(const GPUTestConfig *config,
const std::string &testName)
{
size_t maxExpectationLen = 0;
GPUTestExpectationEntry *foundEntry = nullptr;
for (size_t i = 0; i < mEntries.size(); ++i)
for (GPUTestExpectationEntry &entry : mEntries)
{
if (NamesMatchWithWildcard(mEntries[i].testName.c_str(), testName.c_str()))
if (NamesMatchWithWildcard(entry.testName.c_str(), testName.c_str()))
{
size_t expectationLen = mEntries[i].testName.length();
size_t expectationLen = entry.testName.length();
// Filter by condition first.
bool satisfiesConditions = true;
if (config)
{
for (size_t condition : entry.conditions)
{
if (!config->getConditions()[condition])
{
satisfiesConditions = false;
break;
}
}
}
// The longest/most specific matching expectation overrides any others.
if (expectationLen > maxExpectationLen)
if (satisfiesConditions && expectationLen > maxExpectationLen)
{
maxExpectationLen = expectationLen;
foundEntry = &mEntries[i];
foundEntry = &entry;
}
}
}
......@@ -334,6 +370,17 @@ int32_t GPUTestExpectationsParser::getTestExpectation(const std::string &testNam
return kGpuTestPass;
}
int32_t GPUTestExpectationsParser::getTestExpectation(const std::string &testName)
{
return getTestExpectationImpl(nullptr, testName);
}
int32_t GPUTestExpectationsParser::getTestExpectationWithConfig(const GPUTestConfig &config,
const std::string &testName)
{
return getTestExpectationImpl(&config, testName);
}
const std::vector<std::string> &GPUTestExpectationsParser::getErrorMessages() const
{
return mErrorMessages;
......@@ -353,7 +400,7 @@ std::vector<std::string> GPUTestExpectationsParser::getUnusedExpectationsMessage
return messages;
}
bool GPUTestExpectationsParser::parseLine(const GPUTestConfig &config,
bool GPUTestExpectationsParser::parseLine(const GPUTestConfig *config,
const std::string &lineData,
size_t lineNumber)
{
......@@ -421,9 +468,17 @@ bool GPUTestExpectationsParser::parseLine(const GPUTestConfig &config,
}
{
bool err = false;
if (!checkTokenCondition(config, err, token, lineNumber))
if (config)
{
if (!checkTokenCondition(*config, err, token, lineNumber))
{
skipLine = true; // Move to the next line without adding this one.
}
}
else
{
skipLine = true; // Move to the next line without adding this one.
// Store the conditions for later comparison if we don't have a config.
entry.conditions[kTokenData[token].condition] = true;
}
if (err)
{
......
src/tests/test_expectations/GPUTestExpectationsParser.h
......@@ -13,6 +13,8 @@
#include <string>
#include <vector>
#include "GPUTestConfig.h"
namespace angle
{
......@@ -38,6 +40,8 @@ class GPUTestExpectationsParser
// Return true if parsing succeeds.
bool loadTestExpectations(const GPUTestConfig &config, const std::string &data);
bool loadTestExpectationsFromFile(const GPUTestConfig &config, const std::string &path);
bool loadAllTestExpectations(const std::string &data);
bool loadAllTestExpectationsFromFile(const std::string &path);
// Query error messages from the last LoadTestExpectations() call.
const std::vector<std::string> &getErrorMessages() const;
......@@ -47,6 +51,7 @@ class GPUTestExpectationsParser
// Get the test expectation of a given test on a given bot.
int32_t getTestExpectation(const std::string &testName);
int32_t getTestExpectationWithConfig(const GPUTestConfig &config, const std::string &testName);
private:
struct GPUTestExpectationEntry
......@@ -57,11 +62,12 @@ class GPUTestExpectationsParser
int32_t testExpectation;
size_t lineNumber;
bool used;
GPUTestConfig::ConditionArray conditions;
};
// Parse a line of text. If we have a valid entry, save it; otherwise,
// generate error messages.
bool parseLine(const GPUTestConfig &config, const std::string &lineData, size_t lineNumber);
bool parseLine(const GPUTestConfig *config, const std::string &lineData, size_t lineNumber);
// Check a the condition assigned to a particular token.
bool checkTokenCondition(const GPUTestConfig &config,
......@@ -82,6 +88,12 @@ class GPUTestExpectationsParser
size_t entry1LineNumber,
size_t entry2LineNumber);
// Config is optional.
bool loadTestExpectationsFromFileImpl(const GPUTestConfig *config, const std::string &path);
bool loadTestExpectationsImpl(const GPUTestConfig *config, const std::string &data);
int32_t getTestExpectationImpl(const GPUTestConfig *config, const std::string &testName);
std::vector<GPUTestExpectationEntry> mEntries;
std::vector<std::string> mErrorMessages;
};
......
src/tests/test_expectations/GPUTestExpectationsParser_unittest.cpp
......@@ -14,6 +14,11 @@ using namespace angle;
namespace
{
enum class ConditionTestType
{
OnLoad,
OnGet,
};
class GPUTestConfigTester : public GPUTestConfig
{
......@@ -27,316 +32,297 @@ class GPUTestConfigTester : public GPUTestConfig
}
};
class GPUTestExpectationsParserTest : public testing::TestWithParam<ConditionTestType>
{
public:
bool load(const std::string &line)
{
if (GetParam() == ConditionTestType::OnLoad)
{
return parser.loadTestExpectations(config, line);
}
else
{
return parser.loadAllTestExpectations(line);
}
}
int32_t get(const std::string &testName)
{
if (GetParam() == ConditionTestType::OnLoad)
{
return parser.getTestExpectation(testName);
}
else
{
return parser.getTestExpectationWithConfig(config, testName);
}
}
GPUTestConfigTester config;
GPUTestExpectationsParser parser;
};
// A correct entry with a test that's skipped on all platforms should not lead
// to any errors, and should properly return the expectation SKIP.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserSkip)
TEST_P(GPUTestExpectationsParserTest, GPUTestExpectationsParserSkip)
{
GPUTestConfigTester config;
std::string line =
R"(100 : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = SKIP)";
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.loadTestExpectations(config, line));
EXPECT_TRUE(load(line));
EXPECT_TRUE(parser.getErrorMessages().empty());
EXPECT_EQ(
parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestSkip);
EXPECT_EQ(get("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestSkip);
}
// A correct entry with a test that's failed on all platforms should not lead
// to any errors, and should properly return the expectation FAIL.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserFail)
TEST_P(GPUTestExpectationsParserTest, GPUTestExpectationsParserFail)
{
GPUTestConfigTester config;
std::string line =
R"(100 : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = FAIL)";
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.loadTestExpectations(config, line));
EXPECT_TRUE(load(line));
EXPECT_TRUE(parser.getErrorMessages().empty());
EXPECT_EQ(
parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestFail);
EXPECT_EQ(get("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestFail);
}
// A correct entry with a test that's passed on all platforms should not lead
// to any errors, and should properly return the expectation PASS.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserPass)
TEST_P(GPUTestExpectationsParserTest, GPUTestExpectationsParserPass)
{
GPUTestConfigTester config;
std::string line =
R"(100 : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = PASS)";
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.loadTestExpectations(config, line));
EXPECT_TRUE(load(line));
EXPECT_TRUE(parser.getErrorMessages().empty());
EXPECT_EQ(
parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestPass);
EXPECT_EQ(get("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestPass);
}
// A correct entry with a test that's timed out on all platforms should not lead
// to any errors, and should properly return the expectation TIMEOUT.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserTimeout)
TEST_P(GPUTestExpectationsParserTest, GPUTestExpectationsParserTimeout)
{
GPUTestConfigTester config;
std::string line =
R"(100 : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = TIMEOUT)";
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.loadTestExpectations(config, line));
EXPECT_TRUE(load(line));
EXPECT_TRUE(parser.getErrorMessages().empty());
EXPECT_EQ(
parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestTimeout);
EXPECT_EQ(get("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestTimeout);
}
// A correct entry with a test that's flaky on all platforms should not lead
// to any errors, and should properly return the expectation FLAKY.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserFlaky)
TEST_P(GPUTestExpectationsParserTest, GPUTestExpectationsParserFlaky)
{
GPUTestConfigTester config;
std::string line =
R"(100 : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = FLAKY)";
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.loadTestExpectations(config, line));
EXPECT_TRUE(load(line));
EXPECT_TRUE(parser.getErrorMessages().empty());
EXPECT_EQ(
parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestFlaky);
EXPECT_EQ(get("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestFlaky);
}
// A correct entry with a test that's skipped on windows should not lead
// to any errors, and should properly return the expectation SKIP on this
// tester.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserSingleLineWin)
TEST_P(GPUTestExpectationsParserTest, GPUTestExpectationsParserSingleLineWin)
{
GPUTestConfigTester config;
std::string line =
R"(100 WIN : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = SKIP)";
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.loadTestExpectations(config, line));
EXPECT_TRUE(load(line));
EXPECT_TRUE(parser.getErrorMessages().empty());
EXPECT_EQ(
parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestSkip);
EXPECT_EQ(get("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestSkip);
}
// A correct entry with a test that's skipped on windows/NVIDIA should not lead
// to any errors, and should properly return the expectation SKIP on this
// tester.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserSingleLineWinNVIDIA)
TEST_P(GPUTestExpectationsParserTest, GPUTestExpectationsParserSingleLineWinNVIDIA)
{
GPUTestConfigTester config;
std::string line =
R"(100 WIN NVIDIA : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = SKIP)";
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.loadTestExpectations(config, line));
EXPECT_TRUE(load(line));
EXPECT_TRUE(parser.getErrorMessages().empty());
EXPECT_EQ(
parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestSkip);
EXPECT_EQ(get("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestSkip);
}
// A correct entry with a test that's skipped on windows/NVIDIA/D3D11 should not
// lead to any errors, and should properly return the expectation SKIP on this
// tester.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserSingleLineWinNVIDIAD3D11)
TEST_P(GPUTestExpectationsParserTest, GPUTestExpectationsParserSingleLineWinNVIDIAD3D11)
{
GPUTestConfigTester config;
std::string line =
R"(100 WIN NVIDIA D3D11 : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = SKIP)";
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.loadTestExpectations(config, line));
EXPECT_TRUE(load(line));
EXPECT_TRUE(parser.getErrorMessages().empty());
EXPECT_EQ(
parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestSkip);
EXPECT_EQ(get("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestSkip);
}
// Same as GPUTestExpectationsParserSingleLineWinNVIDIAD3D11, but verifying that the order
// of these conditions doesn't matter
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserSingleLineWinNVIDIAD3D11OtherOrder)
TEST_P(GPUTestExpectationsParserTest, GPUTestExpectationsParserSingleLineWinNVIDIAD3D11OtherOrder)
{
GPUTestConfigTester config;
std::string line =
R"(100 D3D11 NVIDIA WIN : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = SKIP)";
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.loadTestExpectations(config, line));
EXPECT_TRUE(load(line));
EXPECT_TRUE(parser.getErrorMessages().empty());
EXPECT_EQ(
parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestSkip);
EXPECT_EQ(get("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestSkip);
}
// A correct entry with a test that's skipped on mac should not lead
// to any errors, and should default to PASS on this tester (windows).
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserSingleLineMac)
TEST_P(GPUTestExpectationsParserTest, GPUTestExpectationsParserSingleLineMac)
{
GPUTestConfigTester config;
std::string line =
R"(100 MAC : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = SKIP)";
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.loadTestExpectations(config, line));
EXPECT_TRUE(load(line));
EXPECT_TRUE(parser.getErrorMessages().empty());
EXPECT_EQ(
parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestPass);
EXPECT_EQ(get("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestPass);
}
// A correct entry with a test that has conflicting entries should not lead
// to any errors, and should default to PASS.
// (https:anglebug.com/3368) In the future, this condition should cause an
// error
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserSingleLineConflict)
TEST_P(GPUTestExpectationsParserTest, GPUTestExpectationsParserSingleLineConflict)
{
GPUTestConfigTester config;
std::string line =
R"(100 WIN MAC : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = SKIP)";
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.loadTestExpectations(config, line));
EXPECT_TRUE(load(line));
EXPECT_TRUE(parser.getErrorMessages().empty());
EXPECT_EQ(
parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestPass);
EXPECT_EQ(get("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestPass);
}
// A line without a bug ID should return an error and not add the expectation.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserMissingBugId)
TEST_P(GPUTestExpectationsParserTest, GPUTestExpectationsParserMissingBugId)
{
GPUTestConfigTester config;
std::string line = R"( : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = SKIP)";
GPUTestExpectationsParser parser;
EXPECT_FALSE(parser.loadTestExpectations(config, line));
EXPECT_FALSE(load(line));
EXPECT_EQ(parser.getErrorMessages().size(), 1u);
if (parser.getErrorMessages().size() >= 1)
{
EXPECT_EQ(parser.getErrorMessages()[0], "Line 1 : entry with wrong format");
}
// Default behavior is to let missing tests pass
EXPECT_EQ(
parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestPass);
EXPECT_EQ(get("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestPass);
}
// A line without a bug ID should return an error and not add the expectation, (even if
// the line contains conditions that might be mistaken for a bug id)
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserMissingBugIdWithConditions)
TEST_P(GPUTestExpectationsParserTest, GPUTestExpectationsParserMissingBugIdWithConditions)
{
GPUTestConfigTester config;
std::string line =
R"(WIN D3D11 : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = SKIP)";
GPUTestExpectationsParser parser;
EXPECT_FALSE(parser.loadTestExpectations(config, line));
EXPECT_FALSE(load(line));
EXPECT_EQ(parser.getErrorMessages().size(), 1u);
if (parser.getErrorMessages().size() >= 1)
{
EXPECT_EQ(parser.getErrorMessages()[0], "Line 1 : entry with wrong format");
}
// Default behavior is to let missing tests pass
EXPECT_EQ(
parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestPass);
EXPECT_EQ(get("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestPass);
}
// A line without a colon should return an error and not add the expectation.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserMissingColon)
TEST_P(GPUTestExpectationsParserTest, GPUTestExpectationsParserMissingColon)
{
GPUTestConfigTester config;
std::string line = R"(100 dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = SKIP)";
GPUTestExpectationsParser parser;
EXPECT_FALSE(parser.loadTestExpectations(config, line));
EXPECT_FALSE(load(line));
EXPECT_EQ(parser.getErrorMessages().size(), 1u);
if (parser.getErrorMessages().size() >= 1)
{
EXPECT_EQ(parser.getErrorMessages()[0], "Line 1 : entry with wrong format");
}
// Default behavior is to let missing tests pass
EXPECT_EQ(
parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestPass);
EXPECT_EQ(get("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestPass);
}
// A wild character (*) at the end of a line should match any expectations that are a subset of that
// line. It should not greedily match to omany expectations that aren't in that subset.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserWildChar)
TEST_P(GPUTestExpectationsParserTest, GPUTestExpectationsParserWildChar)
{
GPUTestConfigTester config;
std::string line = R"(100 : dEQP-GLES31.functional.layout_binding.ubo.* = SKIP)";
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.loadTestExpectations(config, line));
EXPECT_TRUE(load(line));
EXPECT_TRUE(parser.getErrorMessages().empty());
EXPECT_EQ(
parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestSkip);
EXPECT_EQ(get("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestSkip);
// Also ensure the wild char is not too wild, only covers tests that are more specific
EXPECT_EQ(parser.getTestExpectation(
"dEQP-GLES31.functional.program_interface_query.transform_feedback_varying."
EXPECT_EQ(get("dEQP-GLES31.functional.program_interface_query.transform_feedback_varying."
"resource_list.vertex_fragment.builtin_gl_position"),
GPUTestExpectationsParser::kGpuTestPass);
}
// A line without an equals should return an error and not add the expectation.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserMissingEquals)
TEST_P(GPUTestExpectationsParserTest, GPUTestExpectationsParserMissingEquals)
{
GPUTestConfigTester config;
std::string line = R"(100 : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max SKIP)";
GPUTestExpectationsParser parser;
EXPECT_FALSE(parser.loadTestExpectations(config, line));
EXPECT_FALSE(load(line));
EXPECT_EQ(parser.getErrorMessages().size(), 1u);
if (parser.getErrorMessages().size() >= 1)
{
EXPECT_EQ(parser.getErrorMessages()[0], "Line 1 : entry with wrong format");
}
// Default behavior is to let missing tests pass
EXPECT_EQ(
parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestPass);
EXPECT_EQ(get("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestPass);
}
// A line without an expectation should return an error and not add the expectation.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserMissingExpectation)
TEST_P(GPUTestExpectationsParserTest, GPUTestExpectationsParserMissingExpectation)
{
GPUTestConfigTester config;
std::string line = R"(100 : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max =)";
GPUTestExpectationsParser parser;
EXPECT_FALSE(parser.loadTestExpectations(config, line));
EXPECT_FALSE(load(line));
EXPECT_EQ(parser.getErrorMessages().size(), 1u);
if (parser.getErrorMessages().size() >= 1)
{
EXPECT_EQ(parser.getErrorMessages()[0], "Line 1 : entry with wrong format");
}
// Default behavior is to let missing tests pass
EXPECT_EQ(
parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestPass);
EXPECT_EQ(get("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestPass);
}
// A line with an expectation that doesn't exist should return an error and not add the expectation.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserInvalidExpectation)
TEST_P(GPUTestExpectationsParserTest, GPUTestExpectationsParserInvalidExpectation)
{
GPUTestConfigTester config;
std::string line =
R"(100 : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = WRONG)";
GPUTestExpectationsParser parser;
EXPECT_FALSE(parser.loadTestExpectations(config, line));
EXPECT_FALSE(load(line));
EXPECT_EQ(parser.getErrorMessages().size(), 1u);
if (parser.getErrorMessages().size() >= 1)
{
EXPECT_EQ(parser.getErrorMessages()[0], "Line 1 : entry with wrong format");
}
// Default behavior is to let missing tests pass
EXPECT_EQ(
parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestPass);
EXPECT_EQ(get("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestPass);
}
// ChromeOS is reserved as a token, but doesn't actually check any conditions. Any tokens that
// do not check conditions should return an error and not add the expectation
// (https://anglebug.com/3363) Remove/update this test when ChromeOS is supported
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserUnimplementedCondition)
TEST_P(GPUTestExpectationsParserTest, GPUTestExpectationsParserUnimplementedCondition)
{
GPUTestConfigTester config;
// Does not apply when loading all expectations and not checking the config.
if (GetParam() == ConditionTestType::OnGet)
{
GTEST_SKIP() << "Test does not apply when loading all expectations.";
return;
}
std::string line =
R"(100 CHROMEOS : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = SKIP)";
GPUTestExpectationsParser parser;
EXPECT_FALSE(parser.loadTestExpectations(config, line));
EXPECT_FALSE(load(line));
EXPECT_EQ(parser.getErrorMessages().size(), 1u);
if (parser.getErrorMessages().size() >= 1)
{
......@@ -344,39 +330,32 @@ TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserUnimplementedCondit
"Line 1 : entry invalid, likely unimplemented modifiers");
}
// Default behavior is to let missing tests pass
EXPECT_EQ(
parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestPass);
EXPECT_EQ(get("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestPass);
}
// If a line starts with a comment, it's ignored and should not be added to the list.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserComment)
TEST_P(GPUTestExpectationsParserTest, GPUTestExpectationsParserComment)
{
GPUTestConfigTester config;
std::string line =
R"(//100 : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = SKIP)";
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.loadTestExpectations(config, line));
EXPECT_TRUE(load(line));
EXPECT_TRUE(parser.getErrorMessages().empty());
EXPECT_EQ(
parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestPass);
EXPECT_EQ(get("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestPass);
}
// A misspelled expectation should not be matched from getTestExpectation, and should lead to an
// unused expectation when later queried.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserMisspelledExpectation)
TEST_P(GPUTestExpectationsParserTest, GPUTestExpectationsParserMisspelledExpectation)
{
GPUTestConfigTester config;
std::string line =
R"(100 : dEQP-GLES31.functionaal.layout_binding.ubo.* = SKIP)"; // "functionaal"
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.loadTestExpectations(config, line));
EXPECT_TRUE(load(line));
EXPECT_TRUE(parser.getErrorMessages().empty());
// Default behavior is to let missing tests pass
EXPECT_EQ(
parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestPass);
EXPECT_EQ(get("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestPass);
EXPECT_EQ(parser.getUnusedExpectationsMessages().size(), 1u);
if (parser.getUnusedExpectationsMessages().size() >= 1)
{
......@@ -386,19 +365,16 @@ TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserMisspelledExpectati
// Wild characters that match groups of expectations can be overridden with more specific lines.
// The parse should still compute correctly which lines were used and which were unused.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserOverrideExpectation)
TEST_P(GPUTestExpectationsParserTest, GPUTestExpectationsParserOverrideExpectation)
{
GPUTestConfigTester config;
// Fail all layout_binding tests, but skip the layout_binding.ubo subset.
std::string line = R"(100 : dEQP-GLES31.functional.layout_binding.* = FAIL
100 : dEQP-GLES31.functional.layout_binding.ubo.* = SKIP)";
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.loadTestExpectations(config, line));
EXPECT_TRUE(load(line));
EXPECT_TRUE(parser.getErrorMessages().empty());
// Default behavior is to let missing tests pass
EXPECT_EQ(
parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestSkip);
EXPECT_EQ(get("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestSkip);
// The FAIL expectation was unused because it was overridden.
EXPECT_EQ(parser.getUnusedExpectationsMessages().size(), 1u);
if (parser.getUnusedExpectationsMessages().size() >= 1)
......@@ -406,26 +382,23 @@ TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserOverrideExpectation
EXPECT_EQ(parser.getUnusedExpectationsMessages()[0], "Line 1: expectation was unused.");
}
// Now try a test that doesn't match the override criteria
EXPECT_EQ(parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.image.test"),
EXPECT_EQ(get("dEQP-GLES31.functional.layout_binding.image.test"),
GPUTestExpectationsParser::kGpuTestFail);
EXPECT_TRUE(parser.getUnusedExpectationsMessages().empty());
}
// This test is the same as GPUTestExpectationsParserOverrideExpectation, but verifying the order
// doesn't matter when overriding.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserOverrideExpectationOtherOrder)
TEST_P(GPUTestExpectationsParserTest, GPUTestExpectationsParserOverrideExpectationOtherOrder)
{
GPUTestConfigTester config;
// Fail all layout_binding tests, but skip the layout_binding.ubo subset.
std::string line = R"(100 : dEQP-GLES31.functional.layout_binding.ubo.* = SKIP
100 : dEQP-GLES31.functional.layout_binding.* = FAIL)";
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.loadTestExpectations(config, line));
EXPECT_TRUE(load(line));
EXPECT_TRUE(parser.getErrorMessages().empty());
// Default behavior is to let missing tests pass
EXPECT_EQ(
parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestSkip);
EXPECT_EQ(get("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
GPUTestExpectationsParser::kGpuTestSkip);
// The FAIL expectation was unused because it was overridden.
EXPECT_EQ(parser.getUnusedExpectationsMessages().size(), 1u);
if (parser.getUnusedExpectationsMessages().size() >= 1)
......@@ -433,9 +406,25 @@ TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserOverrideExpectation
EXPECT_EQ(parser.getUnusedExpectationsMessages()[0], "Line 2: expectation was unused.");
}
// Now try a test that doesn't match the override criteria
EXPECT_EQ(parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.image.test"),
EXPECT_EQ(get("dEQP-GLES31.functional.layout_binding.image.test"),
GPUTestExpectationsParser::kGpuTestFail);
EXPECT_TRUE(parser.getUnusedExpectationsMessages().empty());
}
std::string ConditionTestTypeName(testing::TestParamInfo<ConditionTestType> testParamInfo)
{
if (testParamInfo.param == ConditionTestType::OnLoad)
{
return "OnLoad";
}
else
{
return "OnGet";
}
}
INSTANTIATE_TEST_SUITE_P(,
GPUTestExpectationsParserTest,
testing::Values(ConditionTestType::OnGet, ConditionTestType::OnLoad),
ConditionTestTypeName);
} // anonymous namespace
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment