Skip to content

A
angle
Commit e20560fa by Jamie Madill Committed by Commit Bot
Options

Re-land "Add new test runner harness."

Re-land changes: * Unit test is suppressed in ASAN * --deqp-case is fixed * Debug layer errors should correctly work with failure expectations Original message: The ANGLE test harness is a harness around GoogleTest that provides functionality similar to the Chromium test harness. It supports: * splitting a test set into shards * catching and reporting crashes and timeouts * outputting to the Chromium JSON test results format * multi-process execution Unit tests are added in test_utils_unittest.cpp. Bug: angleproject:3162 Change-Id: I841f2b5dfe51f7f44dac68324bdf6afd418b8bfb Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/1948240Reviewed-by: 's avatarJonah Ryan-Davis <jonahr@google.com> Reviewed-by: 's avatarYuly Novikov <ynovikov@chromium.org> Commit-Queue: Jamie Madill <jmadill@chromium.org>
parent af52f9c6
Showing with 545 additions and 147 deletions
gni/angle.gni
...@@ -187,11 +187,14 @@ set_defaults("angle_test") { ...@@ -187,11 +187,14 @@ set_defaults("angle_test") {
public_deps = [] public_deps = []
sources = [] sources = []
data = [] data = []
defines = []
main = "" main = ""
suppressed_configs = angle_remove_configs suppressed_configs = angle_remove_configs
# TODO(jmadill): Migrate to standalone harness. http://anglebug.com/3162 # By default use the Chromium harness in Chromium. Can be overriden in a target.
if (build_with_chromium) { standalone_harness = !build_with_chromium
if (!standalone_harness) {
suppressed_configs -= [ "//build/config/compiler:default_include_dirs" ] suppressed_configs -= [ "//build/config/compiler:default_include_dirs" ]
} }
...@@ -284,18 +287,6 @@ template("angle_static_library") { ...@@ -284,18 +287,6 @@ template("angle_static_library") {
} }
template("angle_test") { template("angle_test") {
_googletest_deps = [
"//testing/gmock",
"//testing/gtest",
"//third_party/googletest:gmock",
"//third_party/googletest:gtest",
]
# TODO(jmadill): Migrate to standalone harness. http://anglebug.com/3162
if (build_with_chromium) {
_googletest_deps += [ "//base/test:test_support" ]
}
test(target_name) { test(target_name) {
forward_variables_from(invoker, forward_variables_from(invoker,
"*", "*",
...@@ -309,9 +300,10 @@ template("angle_test") { ...@@ -309,9 +300,10 @@ template("angle_test") {
forward_variables_from(invoker, [ "visibility" ]) forward_variables_from(invoker, [ "visibility" ])
configs += invoker.configs configs += invoker.configs
configs -= invoker.suppressed_configs configs -= invoker.suppressed_configs + [
configs -= [ angle_root + ":constructor_and_destructor_warnings" ] "$angle_root:constructor_and_destructor_warnings",
configs -= [ angle_root + ":extra_warnings" ] "$angle_root:extra_warnings",
]
if (is_linux && !is_component_build) { if (is_linux && !is_component_build) {
# Set rpath to find shared libs in a non-component build. # Set rpath to find shared libs in a non-component build.
...@@ -319,21 +311,39 @@ template("angle_test") { ...@@ -319,21 +311,39 @@ template("angle_test") {
} }
if (is_android) { if (is_android) {
configs += [ angle_root + ":build_id_config" ] configs += [ "$angle_root:build_id_config" ]
if (build_with_chromium) {
configs -= [ "//build/config/android:hide_all_but_jni" ]
}
} }
deps += _googletest_deps + [ deps += [
"$angle_root:angle_common", "$angle_root:angle_common",
"$angle_root:includes", "$angle_root:includes",
] "$angle_root/third_party/rapidjson:rapidjson",
"//testing/gmock",
if (build_with_chromium) { "//testing/gtest",
sources += [ "//gpu/${invoker.main}.cc" ] "//third_party/googletest:gmock",
"//third_party/googletest:gtest",
]
sources += [
"$angle_root/src/tests/test_utils/runner/TestSuite.cpp",
"$angle_root/src/tests/test_utils/runner/TestSuite.h",
]
# To use the Chromium test infrastructure we must currently use the //base test launcher.
# Eventually we could switch to using standalone testing. See http://crbug.com/837741
if (standalone_harness) {
if (invoker.main != "") {
sources += [ "${invoker.main}.cpp" ]
}
} else { } else {
sources += [ "${invoker.main}.cpp" ] if (invoker.main != "") {
sources += [ "//gpu/${invoker.main}.cc" ]
}
deps += [ "//base/test:test_support" ]
if (is_android) {
configs -= [ "//build/config/android:hide_all_but_jni" ]
}
} }
} }
} }
src/common/platform.h
...@@ -123,4 +123,11 @@ ...@@ -123,4 +123,11 @@
# endif # endif
#endif #endif
// Define ANGLE_WITH_ASAN macro.
#if defined(__has_feature)
# if __has_feature(address_sanitizer)
# define ANGLE_WITH_ASAN 1
# endif
#endif
#endif // COMMON_PLATFORM_H_ #endif // COMMON_PLATFORM_H_
src/common/system_utils.h
...@@ -17,6 +17,8 @@ namespace angle ...@@ -17,6 +17,8 @@ namespace angle
std::string GetExecutablePath(); std::string GetExecutablePath();
std::string GetExecutableDirectory(); std::string GetExecutableDirectory();
const char *GetSharedLibraryExtension(); const char *GetSharedLibraryExtension();
const char *GetExecutableExtension();
char GetPathSeparator();
Optional<std::string> GetCWD(); Optional<std::string> GetCWD();
bool SetCWD(const char *dirName); bool SetCWD(const char *dirName);
bool SetEnvironmentVar(const char *variableName, const char *value); bool SetEnvironmentVar(const char *variableName, const char *value);
......
src/common/system_utils_posix.cpp
...@@ -125,4 +125,14 @@ void BreakDebugger() ...@@ -125,4 +125,14 @@ void BreakDebugger()
// See https://cs.chromium.org/chromium/src/base/debug/debugger_posix.cc // See https://cs.chromium.org/chromium/src/base/debug/debugger_posix.cc
abort(); abort();
} }
const char *GetExecutableExtension()
{
return "";
}
char GetPathSeparator()
{
return '/';
}
} // namespace angle } // namespace angle
src/common/system_utils_win.cpp
...@@ -166,4 +166,13 @@ void BreakDebugger() ...@@ -166,4 +166,13 @@ void BreakDebugger()
__debugbreak(); __debugbreak();
} }
const char *GetExecutableExtension()
{
return ".exe";
}
char GetPathSeparator()
{
return '\\';
}
} // namespace angle } // namespace angle
src/tests/BUILD.gn
...@@ -13,11 +13,19 @@ declare_args() { ...@@ -13,11 +13,19 @@ declare_args() {
build_angle_gles1_conform_tests = false build_angle_gles1_conform_tests = false
} }
angle_executable("test_utils_unittest_helper") { angle_test("test_utils_unittest_helper") {
sources = test_utils_unittest_helper_sources standalone_harness = true
sources = [
"../../util/test_utils_unittest_helper.cpp",
"../../util/test_utils_unittest_helper.h",
"test_utils/angle_test_instantiate.h",
"test_utils/runner/TestSuite_unittest.cpp",
]
deps = [ deps = [
"${angle_root}:angle_common", "$angle_root:angle_common",
"$angle_root:angle_util",
] ]
} }
......
src/tests/angle_deqp_tests_main.cpp
...@@ -8,6 +8,8 @@ ...@@ -8,6 +8,8 @@
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include "test_utils/runner/TestSuite.h"
// Defined in angle_deqp_gtest.cpp. Declared here so we don't need to make a header that we import // Defined in angle_deqp_gtest.cpp. Declared here so we don't need to make a header that we import
// in Chromium. // in Chromium.
namespace angle namespace angle
...@@ -18,7 +20,7 @@ void InitTestHarness(int *argc, char **argv); ...@@ -18,7 +20,7 @@ void InitTestHarness(int *argc, char **argv);
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
angle::InitTestHarness(&argc, argv); angle::InitTestHarness(&argc, argv);
testing::InitGoogleTest(&argc, argv); angle::TestSuite testSuite(&argc, argv);
int rt = RUN_ALL_TESTS(); int rt = RUN_ALL_TESTS();
return rt; return rt;
} }
src/tests/angle_end2end_tests_main.cpp
...@@ -5,13 +5,14 @@ ...@@ -5,13 +5,14 @@
// //
#include "gtest/gtest.h" #include "gtest/gtest.h"
#include "test_utils/runner/TestSuite.h"
void ANGLEProcessTestArgs(int *argc, char *argv[]); void ANGLEProcessTestArgs(int *argc, char *argv[]);
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
angle::TestSuite testSuite(&argc, argv);
ANGLEProcessTestArgs(&argc, argv); ANGLEProcessTestArgs(&argc, argv);
testing::InitGoogleTest(&argc, argv);
int rt = RUN_ALL_TESTS(); int rt = RUN_ALL_TESTS();
return rt; return rt;
} }
src/tests/angle_perftests_main.cpp
...@@ -9,13 +9,14 @@ ...@@ -9,13 +9,14 @@
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include "test_utils/runner/TestSuite.h"
void ANGLEProcessPerfTestArgs(int *argc, char **argv); void ANGLEProcessPerfTestArgs(int *argc, char **argv);
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
angle::TestSuite testSuite(&argc, argv);
ANGLEProcessPerfTestArgs(&argc, argv); ANGLEProcessPerfTestArgs(&argc, argv);
testing::InitGoogleTest(&argc, argv);
testing::AddGlobalTestEnvironment(new testing::Environment());
int rt = RUN_ALL_TESTS(); int rt = RUN_ALL_TESTS();
return rt; return rt;
} }
src/tests/angle_unittest_main.cpp
...@@ -6,6 +6,7 @@ ...@@ -6,6 +6,7 @@
#include "GLSLANG/ShaderLang.h" #include "GLSLANG/ShaderLang.h"
#include "gtest/gtest.h" #include "gtest/gtest.h"
#include "test_utils/runner/TestSuite.h"
class CompilerTestEnvironment : public testing::Environment class CompilerTestEnvironment : public testing::Environment
{ {
...@@ -29,8 +30,7 @@ class CompilerTestEnvironment : public testing::Environment ...@@ -29,8 +30,7 @@ class CompilerTestEnvironment : public testing::Environment
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
testing::InitGoogleTest(&argc, argv); angle::TestSuite testSuite(&argc, argv);
testing::AddGlobalTestEnvironment(new CompilerTestEnvironment()); testing::AddGlobalTestEnvironment(new CompilerTestEnvironment());
int rt = RUN_ALL_TESTS(); return testSuite.run();
return rt;
} }
src/tests/angle_unittests.gni
...@@ -44,91 +44,92 @@ angle_unittests_sources = [ ...@@ -44,91 +44,92 @@ angle_unittests_sources = [
"../libANGLE/renderer/ImageImpl_mock.h", "../libANGLE/renderer/ImageImpl_mock.h",
"../libANGLE/renderer/TextureImpl_mock.h", "../libANGLE/renderer/TextureImpl_mock.h",
"../libANGLE/renderer/TransformFeedbackImpl_mock.h", "../libANGLE/renderer/TransformFeedbackImpl_mock.h",
"../tests/angle_unittests_utils.h", "angle_unittests_utils.h",
"../tests/compiler_tests/API_test.cpp", "compiler_tests/API_test.cpp",
"../tests/compiler_tests/AppendixALimitations_test.cpp", "compiler_tests/AppendixALimitations_test.cpp",
"../tests/compiler_tests/ARB_texture_rectangle_test.cpp", "compiler_tests/ARB_texture_rectangle_test.cpp",
"../tests/compiler_tests/AtomicCounter_test.cpp", "compiler_tests/AtomicCounter_test.cpp",
"../tests/compiler_tests/BufferVariables_test.cpp", "compiler_tests/BufferVariables_test.cpp",
"../tests/compiler_tests/CollectVariables_test.cpp", "compiler_tests/CollectVariables_test.cpp",
"../tests/compiler_tests/ConstantFolding_test.cpp", "compiler_tests/ConstantFolding_test.cpp",
"../tests/compiler_tests/ConstantFoldingNaN_test.cpp", "compiler_tests/ConstantFoldingNaN_test.cpp",
"../tests/compiler_tests/ConstantFoldingOverflow_test.cpp", "compiler_tests/ConstantFoldingOverflow_test.cpp",
"../tests/compiler_tests/ConstructCompiler_test.cpp", "compiler_tests/ConstructCompiler_test.cpp",
"../tests/compiler_tests/DebugShaderPrecision_test.cpp", "compiler_tests/DebugShaderPrecision_test.cpp",
"../tests/compiler_tests/EmulateGLBaseVertexBaseInstance_test.cpp", "compiler_tests/EmulateGLBaseVertexBaseInstance_test.cpp",
"../tests/compiler_tests/EmulateGLDrawID_test.cpp", "compiler_tests/EmulateGLDrawID_test.cpp",
"../tests/compiler_tests/EmulateGLFragColorBroadcast_test.cpp", "compiler_tests/EmulateGLFragColorBroadcast_test.cpp",
"../tests/compiler_tests/ExpressionLimit_test.cpp", "compiler_tests/ExpressionLimit_test.cpp",
"../tests/compiler_tests/EXT_YUV_target_test.cpp", "compiler_tests/EXT_YUV_target_test.cpp",
"../tests/compiler_tests/EXT_blend_func_extended_test.cpp", "compiler_tests/EXT_blend_func_extended_test.cpp",
"../tests/compiler_tests/EXT_frag_depth_test.cpp", "compiler_tests/EXT_frag_depth_test.cpp",
"../tests/compiler_tests/EXT_shader_texture_lod_test.cpp", "compiler_tests/EXT_shader_texture_lod_test.cpp",
"../tests/compiler_tests/ExtensionDirective_test.cpp", "compiler_tests/ExtensionDirective_test.cpp",
"../tests/compiler_tests/FloatLex_test.cpp", "compiler_tests/FloatLex_test.cpp",
"../tests/compiler_tests/FragDepth_test.cpp", "compiler_tests/FragDepth_test.cpp",
"../tests/compiler_tests/GLSLCompatibilityOutput_test.cpp", "compiler_tests/GLSLCompatibilityOutput_test.cpp",
"../tests/compiler_tests/GlFragDataNotModified_test.cpp", "compiler_tests/GlFragDataNotModified_test.cpp",
"../tests/compiler_tests/GeometryShader_test.cpp", "compiler_tests/GeometryShader_test.cpp",
"../tests/compiler_tests/ImmutableString_test.cpp", "compiler_tests/ImmutableString_test.cpp",
"../tests/compiler_tests/InitOutputVariables_test.cpp", "compiler_tests/InitOutputVariables_test.cpp",
"../tests/compiler_tests/IntermNode_test.cpp", "compiler_tests/IntermNode_test.cpp",
"../tests/compiler_tests/NV_draw_buffers_test.cpp", "compiler_tests/NV_draw_buffers_test.cpp",
"../tests/compiler_tests/OES_standard_derivatives_test.cpp", "compiler_tests/OES_standard_derivatives_test.cpp",
"../tests/compiler_tests/Pack_Unpack_test.cpp", "compiler_tests/Pack_Unpack_test.cpp",
"../tests/compiler_tests/PruneEmptyCases_test.cpp", "compiler_tests/PruneEmptyCases_test.cpp",
"../tests/compiler_tests/PruneEmptyDeclarations_test.cpp", "compiler_tests/PruneEmptyDeclarations_test.cpp",
"../tests/compiler_tests/PrunePureLiteralStatements_test.cpp", "compiler_tests/PrunePureLiteralStatements_test.cpp",
"../tests/compiler_tests/PruneUnusedFunctions_test.cpp", "compiler_tests/PruneUnusedFunctions_test.cpp",
"../tests/compiler_tests/QualificationOrderESSL31_test.cpp", "compiler_tests/QualificationOrderESSL31_test.cpp",
"../tests/compiler_tests/QualificationOrder_test.cpp", "compiler_tests/QualificationOrder_test.cpp",
"../tests/compiler_tests/RecordConstantPrecision_test.cpp", "compiler_tests/RecordConstantPrecision_test.cpp",
"../tests/compiler_tests/RegenerateStructNames_test.cpp", "compiler_tests/RegenerateStructNames_test.cpp",
"../tests/compiler_tests/RemovePow_test.cpp", "compiler_tests/RemovePow_test.cpp",
"../tests/compiler_tests/RemoveUnreferencedVariables_test.cpp", "compiler_tests/RemoveUnreferencedVariables_test.cpp",
"../tests/compiler_tests/RewriteDoWhile_test.cpp", "compiler_tests/RewriteDoWhile_test.cpp",
"../tests/compiler_tests/SamplerMultisample_test.cpp", "compiler_tests/SamplerMultisample_test.cpp",
"../tests/compiler_tests/ScalarizeVecAndMatConstructorArgs_test.cpp", "compiler_tests/ScalarizeVecAndMatConstructorArgs_test.cpp",
"../tests/compiler_tests/ShaderImage_test.cpp", "compiler_tests/ShaderImage_test.cpp",
"../tests/compiler_tests/ShaderValidation_test.cpp", "compiler_tests/ShaderValidation_test.cpp",
"../tests/compiler_tests/ShaderVariable_test.cpp", "compiler_tests/ShaderVariable_test.cpp",
"../tests/compiler_tests/ShCompile_test.cpp", "compiler_tests/ShCompile_test.cpp",
"../tests/compiler_tests/TextureFunction_test.cpp", "compiler_tests/TextureFunction_test.cpp",
"../tests/compiler_tests/Type_test.cpp", "compiler_tests/Type_test.cpp",
"../tests/compiler_tests/TypeTracking_test.cpp", "compiler_tests/TypeTracking_test.cpp",
"../tests/compiler_tests/UnfoldShortCircuitAST_test.cpp", "compiler_tests/UnfoldShortCircuitAST_test.cpp",
"../tests/compiler_tests/VariablePacker_test.cpp", "compiler_tests/VariablePacker_test.cpp",
"../tests/compiler_tests/VectorizeVectorScalarArithmetic_test.cpp", "compiler_tests/VectorizeVectorScalarArithmetic_test.cpp",
"../tests/compiler_tests/OVR_multiview_test.cpp", "compiler_tests/OVR_multiview_test.cpp",
"../tests/compiler_tests/OVR_multiview2_test.cpp", "compiler_tests/OVR_multiview2_test.cpp",
"../tests/compiler_tests/WorkGroupSize_test.cpp", "compiler_tests/WorkGroupSize_test.cpp",
"../tests/test_expectations/GPUTestExpectationsParser_unittest.cpp", "test_expectations/GPUTestExpectationsParser_unittest.cpp",
"../tests/preprocessor_tests/char_test.cpp", "preprocessor_tests/char_test.cpp",
"../tests/preprocessor_tests/comment_test.cpp", "preprocessor_tests/comment_test.cpp",
"../tests/preprocessor_tests/define_test.cpp", "preprocessor_tests/define_test.cpp",
"../tests/preprocessor_tests/error_test.cpp", "preprocessor_tests/error_test.cpp",
"../tests/preprocessor_tests/extension_test.cpp", "preprocessor_tests/extension_test.cpp",
"../tests/preprocessor_tests/identifier_test.cpp", "preprocessor_tests/identifier_test.cpp",
"../tests/preprocessor_tests/if_test.cpp", "preprocessor_tests/if_test.cpp",
"../tests/preprocessor_tests/input_test.cpp", "preprocessor_tests/input_test.cpp",
"../tests/preprocessor_tests/location_test.cpp", "preprocessor_tests/location_test.cpp",
"../tests/preprocessor_tests/MockDiagnostics.h", "preprocessor_tests/MockDiagnostics.h",
"../tests/preprocessor_tests/MockDirectiveHandler.h", "preprocessor_tests/MockDirectiveHandler.h",
"../tests/preprocessor_tests/number_test.cpp", "preprocessor_tests/number_test.cpp",
"../tests/preprocessor_tests/operator_test.cpp", "preprocessor_tests/operator_test.cpp",
"../tests/preprocessor_tests/pragma_test.cpp", "preprocessor_tests/pragma_test.cpp",
"../tests/preprocessor_tests/PreprocessorTest.cpp", "preprocessor_tests/PreprocessorTest.cpp",
"../tests/preprocessor_tests/PreprocessorTest.h", "preprocessor_tests/PreprocessorTest.h",
"../tests/preprocessor_tests/space_test.cpp", "preprocessor_tests/space_test.cpp",
"../tests/preprocessor_tests/token_test.cpp", "preprocessor_tests/token_test.cpp",
"../tests/preprocessor_tests/version_test.cpp", "preprocessor_tests/version_test.cpp",
"../tests/test_utils/compiler_test.cpp", "test_utils/angle_test_instantiate.h",
"../tests/test_utils/compiler_test.h", "test_utils/compiler_test.cpp",
"../tests/test_utils/ConstantFoldingTest.h", "test_utils/compiler_test.h",
"../tests/test_utils/ConstantFoldingTest.cpp", "test_utils/ConstantFoldingTest.h",
"../tests/test_utils/ShaderCompileTreeTest.h", "test_utils/ConstantFoldingTest.cpp",
"../tests/test_utils/ShaderCompileTreeTest.cpp", "test_utils/ShaderCompileTreeTest.h",
"../tests/test_utils/ShaderExtensionTest.h", "test_utils/ShaderCompileTreeTest.cpp",
"test_utils/ShaderExtensionTest.h",
"../../util/test_utils_unittest.cpp", "../../util/test_utils_unittest.cpp",
"../../util/test_utils_unittest_helper.h", "../../util/test_utils_unittest_helper.h",
] ]
...@@ -139,11 +140,6 @@ angle_unittests_hlsl_sources = [ ...@@ -139,11 +140,6 @@ angle_unittests_hlsl_sources = [
"../tests/compiler_tests/UnrollFlatten_test.cpp", "../tests/compiler_tests/UnrollFlatten_test.cpp",
] ]
test_utils_unittest_helper_sources = [
"../../util/test_utils_unittest_helper.cpp",
"../../util/test_utils_unittest_helper.h",
]
if (is_android) { if (is_android) {
angle_unittests_sources += angle_unittests_sources +=
[ "../tests/compiler_tests/ImmutableString_test_ESSL_autogen.cpp" ] [ "../tests/compiler_tests/ImmutableString_test_ESSL_autogen.cpp" ]
...@@ -151,3 +147,8 @@ if (is_android) { ...@@ -151,3 +147,8 @@ if (is_android) {
angle_unittests_sources += angle_unittests_sources +=
[ "../tests/compiler_tests/ImmutableString_test_autogen.cpp" ] [ "../tests/compiler_tests/ImmutableString_test_autogen.cpp" ]
} }
if (!is_android && !is_fuchsia) {
angle_unittests_sources +=
[ "../tests/test_utils/runner/TestSuite_unittest.cpp" ]
}
src/tests/angle_white_box_tests_main.cpp
...@@ -6,10 +6,11 @@ ...@@ -6,10 +6,11 @@
#include "gtest/gtest.h" #include "gtest/gtest.h"
#include "test_utils/ANGLETest.h" #include "test_utils/ANGLETest.h"
#include "test_utils/runner/TestSuite.h"
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
testing::InitGoogleTest(&argc, argv); angle::TestSuite testSuite(&argc, argv);
testing::AddGlobalTestEnvironment(new ANGLETestEnvironment()); testing::AddGlobalTestEnvironment(new ANGLETestEnvironment());
int rt = RUN_ALL_TESTS(); int rt = RUN_ALL_TESTS();
return rt; return rt;
......
src/tests/test_utils/ANGLETest.cpp
...@@ -58,9 +58,7 @@ void TestPlatform_logError(PlatformMethods *platform, const char *errorMessage) ...@@ -58,9 +58,7 @@ void TestPlatform_logError(PlatformMethods *platform, const char *errorMessage)
GTEST_NONFATAL_FAILURE_(errorMessage); GTEST_NONFATAL_FAILURE_(errorMessage);
// Print the stack and stop any crash handling to prevent duplicate reports.
PrintStackBacktrace(); PrintStackBacktrace();
TerminateCrashHandler();
} }
void TestPlatform_logWarning(PlatformMethods *platform, const char *warningMessage) void TestPlatform_logWarning(PlatformMethods *platform, const char *warningMessage)
...@@ -481,8 +479,6 @@ void ANGLETestBase::ANGLETestSetUp() ...@@ -481,8 +479,6 @@ void ANGLETestBase::ANGLETestSetUp()
{ {
mSetUpCalled = true; mSetUpCalled = true;
InitCrashHandler(nullptr);
gDefaultPlatformMethods.overrideWorkaroundsD3D = TestPlatform_overrideWorkaroundsD3D; gDefaultPlatformMethods.overrideWorkaroundsD3D = TestPlatform_overrideWorkaroundsD3D;
gDefaultPlatformMethods.overrideFeaturesVk = TestPlatform_overrideFeaturesVk; gDefaultPlatformMethods.overrideFeaturesVk = TestPlatform_overrideFeaturesVk;
gDefaultPlatformMethods.logError = TestPlatform_logError; gDefaultPlatformMethods.logError = TestPlatform_logError;
...@@ -616,8 +612,6 @@ void ANGLETestBase::ANGLETestTearDown() ...@@ -616,8 +612,6 @@ void ANGLETestBase::ANGLETestTearDown()
mFixture->eglWindow->destroySurface(); mFixture->eglWindow->destroySurface();
} }
TerminateCrashHandler();
// Check for quit message // Check for quit message
Event myEvent; Event myEvent;
while (mFixture->osWindow->popEvent(&myEvent)) while (mFixture->osWindow->popEvent(&myEvent))
......
src/tests/test_utils/ANGLETest.h
...@@ -625,14 +625,4 @@ bool IsGLExtensionRequestable(const std::string &extName); ...@@ -625,14 +625,4 @@ bool IsGLExtensionRequestable(const std::string &extName);
extern angle::PlatformMethods gDefaultPlatformMethods; extern angle::PlatformMethods gDefaultPlatformMethods;
#define ANGLE_SKIP_TEST_IF(COND) \
do \
{ \
if (COND) \
{ \
std::cout << "Test skipped: " #COND "." << std::endl; \
return; \
} \
} while (0)
#endif // ANGLE_TESTS_ANGLE_TEST_H_ #endif // ANGLE_TESTS_ANGLE_TEST_H_
src/tests/test_utils/angle_test_instantiate.h
...@@ -12,6 +12,8 @@ ...@@ -12,6 +12,8 @@
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include "common/platform.h"
namespace angle namespace angle
{ {
struct SystemInfo; struct SystemInfo;
...@@ -40,6 +42,15 @@ bool IsIntel(); ...@@ -40,6 +42,15 @@ bool IsIntel();
bool IsAMD(); bool IsAMD();
bool IsNVIDIA(); bool IsNVIDIA();
inline bool IsASan()
{
#if defined(ANGLE_WITH_ASAN)
return true;
#else
return false;
#endif // defined(ANGLE_WITH_ASAN)
}
bool IsPlatformAvailable(const PlatformParameters &param); bool IsPlatformAvailable(const PlatformParameters &param);
// This functions is used to filter which tests should be registered, // This functions is used to filter which tests should be registered,
...@@ -208,4 +219,14 @@ extern std::string gSelectedConfig; ...@@ -208,4 +219,14 @@ extern std::string gSelectedConfig;
extern bool gSeparateProcessPerConfig; extern bool gSeparateProcessPerConfig;
} // namespace angle } // namespace angle
#define ANGLE_SKIP_TEST_IF(COND) \
do \
{ \
if (COND) \
{ \
std::cout << "Test skipped: " #COND "." << std::endl; \
return; \
} \
} while (0)
#endif // ANGLE_TEST_INSTANTIATE_H_ #endif // ANGLE_TEST_INSTANTIATE_H_
src/tests/test_utils/runner/README.md 0 → 100644
# ANGLE Test Harness
The ANGLE test harness is a harness around GoogleTest that provides functionality similar to the
[Chromium test harness][BaseTest]. It features:
* splitting a test set into shards
* catching and reporting crashes and timeouts
* outputting to the Chromium [JSON test results format][JSONFormat]
* multi-process execution
## Command-Line Arguments
The ANGLE test harness accepts all standard GoogleTest arguments. The harness also accepts the
following additional command-line arguments:
* `--shard-count` and `--shard-index` control the test sharding
* `--bot-mode` enables multi-process execution and test batching
* `--batch-size` limits the number of tests to run in each batch
* `--batch-timeout` limits the amount of time spent in each batch
* `--max-processes` limits the number of simuntaneous processes
* `--test-timeout` limits the amount of time spent in each test
* `--results-file` specifies a location for the JSON test result output
* `--results-directory` specifies a directory to write test results to
* `--filter-file` allows passing a larget `gtest_filter` via a file
## Implementation Notes
* The test harness only requires `angle_common` and `angle_util`.
* It does not depend on any Chromium browser code. This allows us to compile on other non-Clang platforms.
* It uses rapidjson to read and write JSON files.
* Timeouts are detected via a watchdog thread.
* Crashes are handled via ANGLE's test crash handling code.
* Currently it does not entirely support Android or Fuchsia.
* Test execution is not currently deterministic in multi-process mode.
* We capture stdout to output test failure reasons.
See the source code for more details: [TestSuite.h](TestSuite.h) and [TestSuite.cpp](TestSuite.cpp).
## Potential Areas of Improvement
* Deterministic test execution.
* Using sockets to communicate with test children. Similar to dEQP's test harness.
* Closer integration with ANGLE's test expectations and system config libraries.
* Supporting a GoogleTest-free integration.
[BaseTest]: https://chromium.googlesource.com/chromium/src/+/refs/heads/master/base/test/
[JSONFormat]: https://chromium.googlesource.com/chromium/src/+/master/docs/testing/json_test_results_format.md
src/tests/test_utils/runner/TestSuite.h 0 → 100644
//
// Copyright 2019 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// TestSuite:
// Basic implementation of a test harness in ANGLE.
#include <map>
#include <memory>
#include <mutex>
#include <string>
#include <thread>
#include "util/test_utils.h"
namespace angle
{
struct TestIdentifier
{
TestIdentifier();
TestIdentifier(const std::string &suiteNameIn, const std::string &nameIn);
TestIdentifier(const TestIdentifier &other);
~TestIdentifier();
TestIdentifier &operator=(const TestIdentifier &other);
static bool ParseFromString(const std::string &str, TestIdentifier *idOut);
bool valid() const { return !testName.empty(); }
void sprintfName(char *outBuffer) const;
std::string testSuiteName;
std::string testName;
};
inline bool operator<(const TestIdentifier &a, const TestIdentifier &b)
{
return std::tie(a.testSuiteName, a.testName) < std::tie(b.testSuiteName, b.testName);
}
inline bool operator==(const TestIdentifier &a, const TestIdentifier &b)
{
return std::tie(a.testSuiteName, a.testName) == std::tie(b.testSuiteName, b.testName);
}
inline std::ostream &operator<<(std::ostream &os, const TestIdentifier &id)
{
return os << id.testSuiteName << "." << id.testName;
}
enum class TestResultType
{
Crash,
Fail,
Skip,
Pass,
Timeout,
Unknown,
};
const char *TestResultTypeToString(TestResultType type);
struct TestResult
{
TestResultType type = TestResultType::Skip;
double elapsedTimeSeconds = 0.0;
};
inline bool operator==(const TestResult &a, const TestResult &b)
{
return a.type == b.type;
}
inline std::ostream &operator<<(std::ostream &os, const TestResult &result)
{
return os << TestResultTypeToString(result.type);
}
struct TestResults
{
TestResults();
~TestResults();
std::map<TestIdentifier, TestResult> results;
std::mutex currentTestMutex;
TestIdentifier currentTest;
Timer currentTestTimer;
bool allDone = false;
};
struct FileLine
{
const char *file;
int line;
};
struct ProcessInfo : angle::NonCopyable
{
ProcessInfo();
~ProcessInfo();
ProcessInfo(ProcessInfo &&other);
ProcessInfo &operator=(ProcessInfo &&rhs);
ProcessHandle process;
std::vector<TestIdentifier> testsInBatch;
std::string resultsFileName;
std::string filterFileName;
std::string commandLine;
};
class TestSuite
{
public:
TestSuite(int *argc, char **argv);
~TestSuite();
int run();
void onCrashOrTimeout(TestResultType crashOrTimeout);
private:
bool parseSingleArg(const char *argument);
bool launchChildTestProcess(const std::vector<TestIdentifier> &testsInBatch);
bool finishProcess(ProcessInfo *processInfo);
int printFailuresAndReturnCount() const;
void startWatchdog();
std::string mTestExecutableName;
std::string mTestSuiteName;
std::vector<TestIdentifier> mTestQueue;
std::string mFilterString;
std::string mFilterFile;
std::string mResultsDirectory;
std::string mResultsFile;
int mShardCount;
int mShardIndex;
angle::CrashCallback mCrashCallback;
TestResults mTestResults;
bool mBotMode;
int mBatchSize;
int mCurrentResultCount;
int mTotalResultCount;
int mMaxProcesses;
int mTestTimeout;
int mBatchTimeout;
std::vector<std::string> mGoogleTestCommandLineArgs;
std::map<TestIdentifier, FileLine> mTestFileLines;
std::vector<ProcessInfo> mCurrentProcesses;
std::thread mWatchdogThread;
};
bool GetTestResultsFromFile(const char *fileName, TestResults *resultsOut);
} // namespace angle
src/tests/test_utils/runner/TestSuite_unittest.cpp 0 → 100644
//
// Copyright 2019 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// TestSuite_unittest.cpp: Unit tests for ANGLE's test harness.
//
#include <gtest/gtest.h>
#include "../angle_test_instantiate.h"
#include "TestSuite.h"
#include "common/debug.h"
#include "common/system_utils.h"
#include "util/test_utils.h"
#include "util/test_utils_unittest_helper.h"
#include <rapidjson/document.h>
using namespace angle;
namespace js = rapidjson;
namespace
{
constexpr char kTestHelperExecutable[] = "test_utils_unittest_helper";
class TestSuiteTest : public testing::Test
{
protected:
void TearDown() override
{
if (!mTempFileName.empty())
{
angle::DeleteFile(mTempFileName.c_str());
}
}
std::string mTempFileName;
};
// Tests the ANGLE standalone testing harness. Runs four tests with different ending conditions.
// Verifies that Pass, Fail, Crash and Timeout are all handled correctly.
TEST_F(TestSuiteTest, RunMockTests)
{
// Fails in ASan when trying to communicate with child processes. http://crbug.com/1030192
ANGLE_SKIP_TEST_IF(IsASan());
std::string executablePath = GetExecutableDirectory();
EXPECT_NE(executablePath, "");
executablePath += std::string("/") + kTestHelperExecutable + GetExecutableExtension();
constexpr uint32_t kMaxTempDirLen = 100;
char tempFileName[kMaxTempDirLen * 2];
ASSERT_TRUE(GetTempDir(tempFileName, kMaxTempDirLen));
std::stringstream tempFNameStream;
tempFNameStream << tempFileName << "/test_temp_" << rand() << ".json";
mTempFileName = tempFNameStream.str();
std::string resultsFileName = "--results-file=" + mTempFileName;
std::vector<const char *> args = {executablePath.c_str(),
kRunTestSuite,
"--gtest_filter=MockTestSuiteTest.DISABLED_*",
"--gtest_also_run_disabled_tests",
"--bot-mode",
"--test-timeout=10",
resultsFileName.c_str()};
ProcessHandle process(args, true, true);
EXPECT_TRUE(process->started());
EXPECT_TRUE(process->finish());
EXPECT_TRUE(process->finished());
EXPECT_EQ(process->getStderr(), "");
TestResults actual;
ASSERT_TRUE(GetTestResultsFromFile(mTempFileName.c_str(), &actual));
EXPECT_TRUE(DeleteFile(mTempFileName.c_str()));
mTempFileName.clear();
std::map<TestIdentifier, TestResult> expectedResults = {
{{"MockTestSuiteTest", "DISABLED_Pass"}, {TestResultType::Pass, 0.0}},
{{"MockTestSuiteTest", "DISABLED_Fail"}, {TestResultType::Fail, 0.0}},
{{"MockTestSuiteTest", "DISABLED_Timeout"}, {TestResultType::Timeout, 0.0}},
{{"MockTestSuiteTest", "DISABLED_Crash"}, {TestResultType::Crash, 0.0}},
};
EXPECT_EQ(expectedResults, actual.results);
}
// Normal passing test.
TEST(MockTestSuiteTest, DISABLED_Pass)
{
EXPECT_TRUE(true);
}
// Normal failing test.
TEST(MockTestSuiteTest, DISABLED_Fail)
{
EXPECT_TRUE(false);
}
// Trigger a test timeout.
TEST(MockTestSuiteTest, DISABLED_Timeout)
{
angle::Sleep(30000);
}
// Trigger a test crash.
TEST(MockTestSuiteTest, DISABLED_Crash)
{
ANGLE_CRASH();
}
} // namespace
util/posix/crash_handler_posix.cpp
...@@ -35,7 +35,6 @@ ...@@ -35,7 +35,6 @@
namespace angle namespace angle
{ {
#if defined(ANGLE_PLATFORM_ANDROID) || defined(ANGLE_PLATFORM_FUCHSIA) #if defined(ANGLE_PLATFORM_ANDROID) || defined(ANGLE_PLATFORM_FUCHSIA)
void PrintStackBacktrace() void PrintStackBacktrace()
...@@ -54,6 +53,10 @@ void TerminateCrashHandler() ...@@ -54,6 +53,10 @@ void TerminateCrashHandler()
} }
#else #else
namespace
{
CrashCallback *gCrashHandlerCallback;
} // namespace
# if defined(ANGLE_PLATFORM_APPLE) # if defined(ANGLE_PLATFORM_APPLE)
...@@ -85,6 +88,11 @@ void PrintStackBacktrace() ...@@ -85,6 +88,11 @@ void PrintStackBacktrace()
static void Handler(int sig) static void Handler(int sig)
{ {
if (gCrashHandlerCallback)
{
(*gCrashHandlerCallback)();
}
printf("\nSignal %d:\n", sig); printf("\nSignal %d:\n", sig);
PrintStackBacktrace(); PrintStackBacktrace();
...@@ -127,6 +135,11 @@ void PrintStackBacktrace() ...@@ -127,6 +135,11 @@ void PrintStackBacktrace()
static void Handler(int sig) static void Handler(int sig)
{ {
if (gCrashHandlerCallback)
{
(*gCrashHandlerCallback)();
}
printf("\nSignal %d [%s]:\n", sig, strsignal(sig)); printf("\nSignal %d [%s]:\n", sig, strsignal(sig));
PrintStackBacktrace(); PrintStackBacktrace();
...@@ -142,6 +155,7 @@ static constexpr int kSignals[] = { ...@@ -142,6 +155,7 @@ static constexpr int kSignals[] = {
void InitCrashHandler(CrashCallback *callback) void InitCrashHandler(CrashCallback *callback)
{ {
gCrashHandlerCallback = callback;
for (int sig : kSignals) for (int sig : kSignals)
{ {
// Register our signal handler unless something's already done so (e.g. catchsegv). // Register our signal handler unless something's already done so (e.g. catchsegv).
...@@ -155,6 +169,7 @@ void InitCrashHandler(CrashCallback *callback) ...@@ -155,6 +169,7 @@ void InitCrashHandler(CrashCallback *callback)
void TerminateCrashHandler() void TerminateCrashHandler()
{ {
gCrashHandlerCallback = nullptr;
for (int sig : kSignals) for (int sig : kSignals)
{ {
void (*prev)(int) = signal(sig, SIG_DFL); void (*prev)(int) = signal(sig, SIG_DFL);
......
util/test_utils_unittest_helper.cpp
...@@ -7,12 +7,22 @@ ...@@ -7,12 +7,22 @@
#include "test_utils_unittest_helper.h" #include "test_utils_unittest_helper.h"
#include "../src/tests/test_utils/runner/TestSuite.h"
#include "common/system_utils.h" #include "common/system_utils.h"
#include <string.h> #include <string.h>
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
for (int argIndex = 1; argIndex < argc; ++argIndex)
{
if (strcmp(argv[argIndex], kRunTestSuite) == 0)
{
angle::TestSuite testSuite(&argc, argv);
return testSuite.run();
}
}
if (argc != 3 || strcmp(argv[1], kRunAppTestArg1) != 0 || strcmp(argv[2], kRunAppTestArg2) != 0) if (argc != 3 || strcmp(argv[1], kRunAppTestArg1) != 0 || strcmp(argv[2], kRunAppTestArg2) != 0)
{ {
fprintf(stderr, "Expected command line:\n%s %s %s\n", argv[0], kRunAppTestArg1, fprintf(stderr, "Expected command line:\n%s %s %s\n", argv[0], kRunAppTestArg1,
......
util/test_utils_unittest_helper.h
...@@ -16,6 +16,7 @@ constexpr char kRunAppTestStdout[] = "RunAppTest stdout test\n"; ...@@ -16,6 +16,7 @@ constexpr char kRunAppTestStdout[] = "RunAppTest stdout test\n";
constexpr char kRunAppTestStderr[] = "RunAppTest stderr test\n .. that expands multiple lines\n"; constexpr char kRunAppTestStderr[] = "RunAppTest stderr test\n .. that expands multiple lines\n";
constexpr char kRunAppTestArg1[] = "--expected-arg1"; constexpr char kRunAppTestArg1[] = "--expected-arg1";
constexpr char kRunAppTestArg2[] = "expected_arg2"; constexpr char kRunAppTestArg2[] = "expected_arg2";
constexpr char kRunTestSuite[] = "--run-test-suite";
} // anonymous namespace } // anonymous namespace
#endif // COMMON_SYSTEM_UTILS_UNITTEST_HELPER_H_ #endif // COMMON_SYSTEM_UTILS_UNITTEST_HELPER_H_
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment