Skip to content

A
angle
Commit 5407aaa0 by Jamie Madill Committed by Commit Bot
Options

Re-land "Add new test runner harness." (#2)

Re-land #2 changes: * export labels are fixed for the CFI build * crash test disabled because of flakiness and issues with asan Re-land changes: * Unit test is suppressed in ASAN * --deqp-case is fixed * Debug layer errors should correctly work with failure expectations Original message: The ANGLE test harness is a harness around GoogleTest that provides functionality similar to the Chromium test harness. It supports: * splitting a test set into shards * catching and reporting crashes and timeouts * outputting to the Chromium JSON test results format * multi-process execution Unit tests are added in test_utils_unittest.cpp. Bug: angleproject:3162 Bug: chromium:1030192 Change-Id: I71d66a407ea0e53d73cbe75b5b4bfb9e73791534 Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/1965091 Commit-Queue: Jamie Madill <jmadill@chromium.org> Reviewed-by: 's avatarJonah Ryan-Davis <jonahr@google.com>
parent 5cfab195
Showing with 1729 additions and 148 deletions
gni/angle.gni
......@@ -189,11 +189,14 @@ set_defaults("angle_test") {
public_deps = []
sources = []
data = []
defines = []
main = ""
suppressed_configs = angle_remove_configs
# TODO(jmadill): Migrate to standalone harness. http://anglebug.com/3162
if (build_with_chromium) {
# By default use the Chromium harness in Chromium. Can be overriden in a target.
standalone_harness = !build_with_chromium
if (!standalone_harness) {
suppressed_configs -= [ "//build/config/compiler:default_include_dirs" ]
}
......@@ -286,18 +289,6 @@ template("angle_static_library") {
}
template("angle_test") {
_googletest_deps = [
"//testing/gmock",
"//testing/gtest",
"//third_party/googletest:gmock",
"//third_party/googletest:gtest",
]
# TODO(jmadill): Migrate to standalone harness. http://anglebug.com/3162
if (build_with_chromium) {
_googletest_deps += [ "//base/test:test_support" ]
}
test(target_name) {
forward_variables_from(invoker,
"*",
......@@ -311,9 +302,10 @@ template("angle_test") {
forward_variables_from(invoker, [ "visibility" ])
configs += invoker.configs
configs -= invoker.suppressed_configs
configs -= [ angle_root + ":constructor_and_destructor_warnings" ]
configs -= [ angle_root + ":extra_warnings" ]
configs -= invoker.suppressed_configs + [
"$angle_root:constructor_and_destructor_warnings",
"$angle_root:extra_warnings",
]
if (is_linux && !is_component_build) {
# Set rpath to find shared libs in a non-component build.
......@@ -321,22 +313,40 @@ template("angle_test") {
}
if (is_android) {
configs += [ angle_root + ":build_id_config" ]
if (build_with_chromium) {
configs -= [ "//build/config/android:hide_all_but_jni" ]
}
configs += [ "$angle_root:build_id_config" ]
}
deps += _googletest_deps + [
"$angle_root:angle_common",
"$angle_root:includes",
"$angle_root/util:angle_test_utils",
]
if (build_with_chromium) {
sources += [ "//gpu/${invoker.main}.cc" ]
deps += [
"$angle_root:angle_common",
"$angle_root:includes",
"$angle_root/third_party/rapidjson:rapidjson",
"$angle_root/util:angle_test_utils",
"//testing/gmock",
"//testing/gtest",
"//third_party/googletest:gmock",
"//third_party/googletest:gtest",
]
sources += [
"$angle_root/src/tests/test_utils/runner/TestSuite.cpp",
"$angle_root/src/tests/test_utils/runner/TestSuite.h",
]
# To use the Chromium test infrastructure we must currently use the //base test launcher.
# Eventually we could switch to using standalone testing. See http://crbug.com/837741
if (standalone_harness) {
if (invoker.main != "") {
sources += [ "${invoker.main}.cpp" ]
}
} else {
sources += [ "${invoker.main}.cpp" ]
if (invoker.main != "") {
sources += [ "//gpu/${invoker.main}.cc" ]
}
deps += [ "//base/test:test_support" ]
if (is_android) {
configs -= [ "//build/config/android:hide_all_but_jni" ]
}
}
}
}
src/common/platform.h
......@@ -123,4 +123,11 @@
# endif
#endif
// Define ANGLE_WITH_ASAN macro.
#if defined(__has_feature)
# if __has_feature(address_sanitizer)
# define ANGLE_WITH_ASAN 1
# endif
#endif
#endif // COMMON_PLATFORM_H_
src/common/system_utils.h
......@@ -17,6 +17,8 @@ namespace angle
std::string GetExecutablePath();
std::string GetExecutableDirectory();
const char *GetSharedLibraryExtension();
const char *GetExecutableExtension();
char GetPathSeparator();
Optional<std::string> GetCWD();
bool SetCWD(const char *dirName);
bool SetEnvironmentVar(const char *variableName, const char *value);
......
src/common/system_utils_posix.cpp
......@@ -125,4 +125,14 @@ void BreakDebugger()
// See https://cs.chromium.org/chromium/src/base/debug/debugger_posix.cc
abort();
}
const char *GetExecutableExtension()
{
return "";
}
char GetPathSeparator()
{
return '/';
}
} // namespace angle
src/common/system_utils_win.cpp
......@@ -166,4 +166,13 @@ void BreakDebugger()
__debugbreak();
}
const char *GetExecutableExtension()
{
return ".exe";
}
char GetPathSeparator()
{
return '\\';
}
} // namespace angle
src/tests/BUILD.gn
......@@ -13,11 +13,18 @@ declare_args() {
build_angle_gles1_conform_tests = false
}
angle_executable("test_utils_unittest_helper") {
sources = test_utils_unittest_helper_sources
angle_test("test_utils_unittest_helper") {
standalone_harness = true
sources = [
"../../util/test_utils_unittest_helper.cpp",
"../../util/test_utils_unittest_helper.h",
"test_utils/angle_test_instantiate.h",
"test_utils/runner/TestSuite_unittest.cpp",
]
deps = [
"${angle_root}:angle_common",
"$angle_root:angle_common",
]
}
......
src/tests/angle_deqp_tests_main.cpp
......@@ -8,6 +8,8 @@
#include <gtest/gtest.h>
#include "test_utils/runner/TestSuite.h"
// Defined in angle_deqp_gtest.cpp. Declared here so we don't need to make a header that we import
// in Chromium.
namespace angle
......@@ -18,7 +20,7 @@ void InitTestHarness(int *argc, char **argv);
int main(int argc, char **argv)
{
angle::InitTestHarness(&argc, argv);
testing::InitGoogleTest(&argc, argv);
angle::TestSuite testSuite(&argc, argv);
int rt = RUN_ALL_TESTS();
return rt;
}
src/tests/angle_end2end_tests_main.cpp
......@@ -5,13 +5,14 @@
//
#include "gtest/gtest.h"
#include "test_utils/runner/TestSuite.h"
void ANGLEProcessTestArgs(int *argc, char *argv[]);
int main(int argc, char **argv)
{
angle::TestSuite testSuite(&argc, argv);
ANGLEProcessTestArgs(&argc, argv);
testing::InitGoogleTest(&argc, argv);
int rt = RUN_ALL_TESTS();
return rt;
}
src/tests/angle_perftests_main.cpp
......@@ -9,13 +9,14 @@
#include <gtest/gtest.h>
#include "test_utils/runner/TestSuite.h"
void ANGLEProcessPerfTestArgs(int *argc, char **argv);
int main(int argc, char **argv)
{
angle::TestSuite testSuite(&argc, argv);
ANGLEProcessPerfTestArgs(&argc, argv);
testing::InitGoogleTest(&argc, argv);
testing::AddGlobalTestEnvironment(new testing::Environment());
int rt = RUN_ALL_TESTS();
return rt;
}
src/tests/angle_unittest_main.cpp
......@@ -6,6 +6,7 @@
#include "GLSLANG/ShaderLang.h"
#include "gtest/gtest.h"
#include "test_utils/runner/TestSuite.h"
class CompilerTestEnvironment : public testing::Environment
{
......@@ -29,8 +30,7 @@ class CompilerTestEnvironment : public testing::Environment
int main(int argc, char **argv)
{
testing::InitGoogleTest(&argc, argv);
angle::TestSuite testSuite(&argc, argv);
testing::AddGlobalTestEnvironment(new CompilerTestEnvironment());
int rt = RUN_ALL_TESTS();
return rt;
return testSuite.run();
}
src/tests/angle_unittests.gni
......@@ -44,91 +44,92 @@ angle_unittests_sources = [
"../libANGLE/renderer/ImageImpl_mock.h",
"../libANGLE/renderer/TextureImpl_mock.h",
"../libANGLE/renderer/TransformFeedbackImpl_mock.h",
"../tests/angle_unittests_utils.h",
"../tests/compiler_tests/API_test.cpp",
"../tests/compiler_tests/AppendixALimitations_test.cpp",
"../tests/compiler_tests/ARB_texture_rectangle_test.cpp",
"../tests/compiler_tests/AtomicCounter_test.cpp",
"../tests/compiler_tests/BufferVariables_test.cpp",
"../tests/compiler_tests/CollectVariables_test.cpp",
"../tests/compiler_tests/ConstantFolding_test.cpp",
"../tests/compiler_tests/ConstantFoldingNaN_test.cpp",
"../tests/compiler_tests/ConstantFoldingOverflow_test.cpp",
"../tests/compiler_tests/ConstructCompiler_test.cpp",
"../tests/compiler_tests/DebugShaderPrecision_test.cpp",
"../tests/compiler_tests/EmulateGLBaseVertexBaseInstance_test.cpp",
"../tests/compiler_tests/EmulateGLDrawID_test.cpp",
"../tests/compiler_tests/EmulateGLFragColorBroadcast_test.cpp",
"../tests/compiler_tests/ExpressionLimit_test.cpp",
"../tests/compiler_tests/EXT_YUV_target_test.cpp",
"../tests/compiler_tests/EXT_blend_func_extended_test.cpp",
"../tests/compiler_tests/EXT_frag_depth_test.cpp",
"../tests/compiler_tests/EXT_shader_texture_lod_test.cpp",
"../tests/compiler_tests/ExtensionDirective_test.cpp",
"../tests/compiler_tests/FloatLex_test.cpp",
"../tests/compiler_tests/FragDepth_test.cpp",
"../tests/compiler_tests/GLSLCompatibilityOutput_test.cpp",
"../tests/compiler_tests/GlFragDataNotModified_test.cpp",
"../tests/compiler_tests/GeometryShader_test.cpp",
"../tests/compiler_tests/ImmutableString_test.cpp",
"../tests/compiler_tests/InitOutputVariables_test.cpp",
"../tests/compiler_tests/IntermNode_test.cpp",
"../tests/compiler_tests/NV_draw_buffers_test.cpp",
"../tests/compiler_tests/OES_standard_derivatives_test.cpp",
"../tests/compiler_tests/Pack_Unpack_test.cpp",
"../tests/compiler_tests/PruneEmptyCases_test.cpp",
"../tests/compiler_tests/PruneEmptyDeclarations_test.cpp",
"../tests/compiler_tests/PrunePureLiteralStatements_test.cpp",
"../tests/compiler_tests/PruneUnusedFunctions_test.cpp",
"../tests/compiler_tests/QualificationOrderESSL31_test.cpp",
"../tests/compiler_tests/QualificationOrder_test.cpp",
"../tests/compiler_tests/RecordConstantPrecision_test.cpp",
"../tests/compiler_tests/RegenerateStructNames_test.cpp",
"../tests/compiler_tests/RemovePow_test.cpp",
"../tests/compiler_tests/RemoveUnreferencedVariables_test.cpp",
"../tests/compiler_tests/RewriteDoWhile_test.cpp",
"../tests/compiler_tests/SamplerMultisample_test.cpp",
"../tests/compiler_tests/ScalarizeVecAndMatConstructorArgs_test.cpp",
"../tests/compiler_tests/ShaderImage_test.cpp",
"../tests/compiler_tests/ShaderValidation_test.cpp",
"../tests/compiler_tests/ShaderVariable_test.cpp",
"../tests/compiler_tests/ShCompile_test.cpp",
"../tests/compiler_tests/TextureFunction_test.cpp",
"../tests/compiler_tests/Type_test.cpp",
"../tests/compiler_tests/TypeTracking_test.cpp",
"../tests/compiler_tests/UnfoldShortCircuitAST_test.cpp",
"../tests/compiler_tests/VariablePacker_test.cpp",
"../tests/compiler_tests/VectorizeVectorScalarArithmetic_test.cpp",
"../tests/compiler_tests/OVR_multiview_test.cpp",
"../tests/compiler_tests/OVR_multiview2_test.cpp",
"../tests/compiler_tests/WorkGroupSize_test.cpp",
"../tests/test_expectations/GPUTestExpectationsParser_unittest.cpp",
"../tests/preprocessor_tests/char_test.cpp",
"../tests/preprocessor_tests/comment_test.cpp",
"../tests/preprocessor_tests/define_test.cpp",
"../tests/preprocessor_tests/error_test.cpp",
"../tests/preprocessor_tests/extension_test.cpp",
"../tests/preprocessor_tests/identifier_test.cpp",
"../tests/preprocessor_tests/if_test.cpp",
"../tests/preprocessor_tests/input_test.cpp",
"../tests/preprocessor_tests/location_test.cpp",
"../tests/preprocessor_tests/MockDiagnostics.h",
"../tests/preprocessor_tests/MockDirectiveHandler.h",
"../tests/preprocessor_tests/number_test.cpp",
"../tests/preprocessor_tests/operator_test.cpp",
"../tests/preprocessor_tests/pragma_test.cpp",
"../tests/preprocessor_tests/PreprocessorTest.cpp",
"../tests/preprocessor_tests/PreprocessorTest.h",
"../tests/preprocessor_tests/space_test.cpp",
"../tests/preprocessor_tests/token_test.cpp",
"../tests/preprocessor_tests/version_test.cpp",
"../tests/test_utils/compiler_test.cpp",
"../tests/test_utils/compiler_test.h",
"../tests/test_utils/ConstantFoldingTest.h",
"../tests/test_utils/ConstantFoldingTest.cpp",
"../tests/test_utils/ShaderCompileTreeTest.h",
"../tests/test_utils/ShaderCompileTreeTest.cpp",
"../tests/test_utils/ShaderExtensionTest.h",
"angle_unittests_utils.h",
"compiler_tests/API_test.cpp",
"compiler_tests/AppendixALimitations_test.cpp",
"compiler_tests/ARB_texture_rectangle_test.cpp",
"compiler_tests/AtomicCounter_test.cpp",
"compiler_tests/BufferVariables_test.cpp",
"compiler_tests/CollectVariables_test.cpp",
"compiler_tests/ConstantFolding_test.cpp",
"compiler_tests/ConstantFoldingNaN_test.cpp",
"compiler_tests/ConstantFoldingOverflow_test.cpp",
"compiler_tests/ConstructCompiler_test.cpp",
"compiler_tests/DebugShaderPrecision_test.cpp",
"compiler_tests/EmulateGLBaseVertexBaseInstance_test.cpp",
"compiler_tests/EmulateGLDrawID_test.cpp",
"compiler_tests/EmulateGLFragColorBroadcast_test.cpp",
"compiler_tests/ExpressionLimit_test.cpp",
"compiler_tests/EXT_YUV_target_test.cpp",
"compiler_tests/EXT_blend_func_extended_test.cpp",
"compiler_tests/EXT_frag_depth_test.cpp",
"compiler_tests/EXT_shader_texture_lod_test.cpp",
"compiler_tests/ExtensionDirective_test.cpp",
"compiler_tests/FloatLex_test.cpp",
"compiler_tests/FragDepth_test.cpp",
"compiler_tests/GLSLCompatibilityOutput_test.cpp",
"compiler_tests/GlFragDataNotModified_test.cpp",
"compiler_tests/GeometryShader_test.cpp",
"compiler_tests/ImmutableString_test.cpp",
"compiler_tests/InitOutputVariables_test.cpp",
"compiler_tests/IntermNode_test.cpp",
"compiler_tests/NV_draw_buffers_test.cpp",
"compiler_tests/OES_standard_derivatives_test.cpp",
"compiler_tests/Pack_Unpack_test.cpp",
"compiler_tests/PruneEmptyCases_test.cpp",
"compiler_tests/PruneEmptyDeclarations_test.cpp",
"compiler_tests/PrunePureLiteralStatements_test.cpp",
"compiler_tests/PruneUnusedFunctions_test.cpp",
"compiler_tests/QualificationOrderESSL31_test.cpp",
"compiler_tests/QualificationOrder_test.cpp",
"compiler_tests/RecordConstantPrecision_test.cpp",
"compiler_tests/RegenerateStructNames_test.cpp",
"compiler_tests/RemovePow_test.cpp",
"compiler_tests/RemoveUnreferencedVariables_test.cpp",
"compiler_tests/RewriteDoWhile_test.cpp",
"compiler_tests/SamplerMultisample_test.cpp",
"compiler_tests/ScalarizeVecAndMatConstructorArgs_test.cpp",
"compiler_tests/ShaderImage_test.cpp",
"compiler_tests/ShaderValidation_test.cpp",
"compiler_tests/ShaderVariable_test.cpp",
"compiler_tests/ShCompile_test.cpp",
"compiler_tests/TextureFunction_test.cpp",
"compiler_tests/Type_test.cpp",
"compiler_tests/TypeTracking_test.cpp",
"compiler_tests/UnfoldShortCircuitAST_test.cpp",
"compiler_tests/VariablePacker_test.cpp",
"compiler_tests/VectorizeVectorScalarArithmetic_test.cpp",
"compiler_tests/OVR_multiview_test.cpp",
"compiler_tests/OVR_multiview2_test.cpp",
"compiler_tests/WorkGroupSize_test.cpp",
"test_expectations/GPUTestExpectationsParser_unittest.cpp",
"preprocessor_tests/char_test.cpp",
"preprocessor_tests/comment_test.cpp",
"preprocessor_tests/define_test.cpp",
"preprocessor_tests/error_test.cpp",
"preprocessor_tests/extension_test.cpp",
"preprocessor_tests/identifier_test.cpp",
"preprocessor_tests/if_test.cpp",
"preprocessor_tests/input_test.cpp",
"preprocessor_tests/location_test.cpp",
"preprocessor_tests/MockDiagnostics.h",
"preprocessor_tests/MockDirectiveHandler.h",
"preprocessor_tests/number_test.cpp",
"preprocessor_tests/operator_test.cpp",
"preprocessor_tests/pragma_test.cpp",
"preprocessor_tests/PreprocessorTest.cpp",
"preprocessor_tests/PreprocessorTest.h",
"preprocessor_tests/space_test.cpp",
"preprocessor_tests/token_test.cpp",
"preprocessor_tests/version_test.cpp",
"test_utils/angle_test_instantiate.h",
"test_utils/compiler_test.cpp",
"test_utils/compiler_test.h",
"test_utils/ConstantFoldingTest.h",
"test_utils/ConstantFoldingTest.cpp",
"test_utils/ShaderCompileTreeTest.h",
"test_utils/ShaderCompileTreeTest.cpp",
"test_utils/ShaderExtensionTest.h",
"../../util/test_utils_unittest.cpp",
"../../util/test_utils_unittest_helper.h",
]
......@@ -139,11 +140,6 @@ angle_unittests_hlsl_sources = [
"../tests/compiler_tests/UnrollFlatten_test.cpp",
]
test_utils_unittest_helper_sources = [
"../../util/test_utils_unittest_helper.cpp",
"../../util/test_utils_unittest_helper.h",
]
if (is_android) {
angle_unittests_sources +=
[ "../tests/compiler_tests/ImmutableString_test_ESSL_autogen.cpp" ]
......@@ -151,3 +147,8 @@ if (is_android) {
angle_unittests_sources +=
[ "../tests/compiler_tests/ImmutableString_test_autogen.cpp" ]
}
if (!is_android && !is_fuchsia) {
angle_unittests_sources +=
[ "../tests/test_utils/runner/TestSuite_unittest.cpp" ]
}
src/tests/angle_white_box_tests_main.cpp
......@@ -6,10 +6,11 @@
#include "gtest/gtest.h"
#include "test_utils/ANGLETest.h"
#include "test_utils/runner/TestSuite.h"
int main(int argc, char **argv)
{
testing::InitGoogleTest(&argc, argv);
angle::TestSuite testSuite(&argc, argv);
testing::AddGlobalTestEnvironment(new ANGLETestEnvironment());
int rt = RUN_ALL_TESTS();
return rt;
......
src/tests/test_utils/ANGLETest.cpp
......@@ -58,9 +58,7 @@ void TestPlatform_logError(PlatformMethods *platform, const char *errorMessage)
GTEST_NONFATAL_FAILURE_(errorMessage);
// Print the stack and stop any crash handling to prevent duplicate reports.
PrintStackBacktrace();
TerminateCrashHandler();
}
void TestPlatform_logWarning(PlatformMethods *platform, const char *warningMessage)
......@@ -481,8 +479,6 @@ void ANGLETestBase::ANGLETestSetUp()
{
mSetUpCalled = true;
InitCrashHandler(nullptr);
gDefaultPlatformMethods.overrideWorkaroundsD3D = TestPlatform_overrideWorkaroundsD3D;
gDefaultPlatformMethods.overrideFeaturesVk = TestPlatform_overrideFeaturesVk;
gDefaultPlatformMethods.logError = TestPlatform_logError;
......@@ -616,8 +612,6 @@ void ANGLETestBase::ANGLETestTearDown()
mFixture->eglWindow->destroySurface();
}
TerminateCrashHandler();
// Check for quit message
Event myEvent;
while (mFixture->osWindow->popEvent(&myEvent))
......
src/tests/test_utils/ANGLETest.h
......@@ -625,14 +625,4 @@ bool IsGLExtensionRequestable(const std::string &extName);
extern angle::PlatformMethods gDefaultPlatformMethods;
#define ANGLE_SKIP_TEST_IF(COND) \
do \
{ \
if (COND) \
{ \
std::cout << "Test skipped: " #COND "." << std::endl; \
return; \
} \
} while (0)
#endif // ANGLE_TESTS_ANGLE_TEST_H_
src/tests/test_utils/angle_test_instantiate.h
......@@ -12,6 +12,8 @@
#include <gtest/gtest.h>
#include "common/platform.h"
namespace angle
{
struct SystemInfo;
......@@ -40,6 +42,15 @@ bool IsIntel();
bool IsAMD();
bool IsNVIDIA();
inline bool IsASan()
{
#if defined(ANGLE_WITH_ASAN)
return true;
#else
return false;
#endif // defined(ANGLE_WITH_ASAN)
}
bool IsPlatformAvailable(const PlatformParameters &param);
// This functions is used to filter which tests should be registered,
......@@ -208,4 +219,14 @@ extern std::string gSelectedConfig;
extern bool gSeparateProcessPerConfig;
} // namespace angle
#define ANGLE_SKIP_TEST_IF(COND) \
do \
{ \
if (COND) \
{ \
std::cout << "Test skipped: " #COND "." << std::endl; \
return; \
} \
} while (0)
#endif // ANGLE_TEST_INSTANTIATE_H_
src/tests/test_utils/runner/README.md 0 → 100644
# ANGLE Test Harness
The ANGLE test harness is a harness around GoogleTest that provides functionality similar to the
[Chromium test harness][BaseTest]. It features:
* splitting a test set into shards
* catching and reporting crashes and timeouts
* outputting to the Chromium [JSON test results format][JSONFormat]
* multi-process execution
## Command-Line Arguments
The ANGLE test harness accepts all standard GoogleTest arguments. The harness also accepts the
following additional command-line arguments:
* `--shard-count` and `--shard-index` control the test sharding
* `--bot-mode` enables multi-process execution and test batching
* `--batch-size` limits the number of tests to run in each batch
* `--batch-timeout` limits the amount of time spent in each batch
* `--max-processes` limits the number of simuntaneous processes
* `--test-timeout` limits the amount of time spent in each test
* `--results-file` specifies a location for the JSON test result output
* `--results-directory` specifies a directory to write test results to
* `--filter-file` allows passing a larget `gtest_filter` via a file
## Implementation Notes
* The test harness only requires `angle_common` and `angle_util`.
* It does not depend on any Chromium browser code. This allows us to compile on other non-Clang platforms.
* It uses rapidjson to read and write JSON files.
* Timeouts are detected via a watchdog thread.
* Crashes are handled via ANGLE's test crash handling code.
* Currently it does not entirely support Android or Fuchsia.
* Test execution is not currently deterministic in multi-process mode.
* We capture stdout to output test failure reasons.
See the source code for more details: [TestSuite.h](TestSuite.h) and [TestSuite.cpp](TestSuite.cpp).
## Potential Areas of Improvement
* Deterministic test execution.
* Using sockets to communicate with test children. Similar to dEQP's test harness.
* Closer integration with ANGLE's test expectations and system config libraries.
* Supporting a GoogleTest-free integration.
[BaseTest]: https://chromium.googlesource.com/chromium/src/+/refs/heads/master/base/test/
[JSONFormat]: https://chromium.googlesource.com/chromium/src/+/master/docs/testing/json_test_results_format.md
src/tests/test_utils/runner/TestSuite.cpp 0 → 100644
//
// Copyright 2019 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// TestSuite:
// Basic implementation of a test harness in ANGLE.
#include "TestSuite.h"
#include "common/debug.h"
#include "common/platform.h"
#include "common/system_utils.h"
#include "util/Timer.h"
#include <time.h>
#include <fstream>
#include <gtest/gtest.h>
#include <rapidjson/document.h>
#include <rapidjson/filewritestream.h>
#include <rapidjson/istreamwrapper.h>
#include <rapidjson/prettywriter.h>
// We directly call into a function to register the parameterized tests. This saves spinning up
// a subprocess with a new gtest filter.
#include "third_party/googletest/src/googletest/src/gtest-internal-inl.h"
namespace js = rapidjson;
namespace angle
{
namespace
{
constexpr char kTestTimeoutArg[] = "--test-timeout=";
constexpr char kFilterFileArg[] = "--filter-file=";
constexpr char kResultFileArg[] = "--results-file=";
constexpr int kDefaultTestTimeout = 10;
constexpr int kDefaultBatchSize = 1000;
const char *ParseFlagValue(const char *flag, const char *argument)
{
if (strstr(argument, flag) == argument)
{
return argument + strlen(flag);
}
return nullptr;
}
bool ParseIntArg(const char *flag, const char *argument, int *valueOut)
{
const char *value = ParseFlagValue(flag, argument);
if (!value)
{
return false;
}
char *end = nullptr;
const long longValue = strtol(value, &end, 10);
if (*end != '\0')
{
printf("Error parsing integer flag value.\n");
exit(1);
}
if (longValue == LONG_MAX || longValue == LONG_MIN || static_cast<int>(longValue) != longValue)
{
printf("Overflow when parsing integer flag value.\n");
exit(1);
}
*valueOut = static_cast<int>(longValue);
return true;
}
bool ParseFlag(const char *expected, const char *actual, bool *flagOut)
{
if (strcmp(expected, actual) == 0)
{
*flagOut = true;
return true;
}
return false;
}
bool ParseStringArg(const char *flag, const char *argument, std::string *valueOut)
{
const char *value = ParseFlagValue(flag, argument);
if (!value)
{
return false;
}
*valueOut = value;
return true;
}
void DeleteArg(int *argc, char **argv, int argIndex)
{
// Shift the remainder of the argv list left by one. Note that argv has (*argc + 1) elements,
// the last one always being NULL. The following loop moves the trailing NULL element as well.
for (int index = argIndex; index < *argc; ++index)
{
argv[index] = argv[index + 1];
}
(*argc)--;
}
void AddArg(int *argc, char **argv, const char *arg)
{
// This unsafe const_cast is necessary to work around gtest limitations.
argv[*argc] = const_cast<char *>(arg);
argv[*argc + 1] = nullptr;
(*argc)++;
}
const char *ResultTypeToString(TestResultType type)
{
switch (type)
{
case TestResultType::Crash:
return "CRASH";
case TestResultType::Fail:
return "FAIL";
case TestResultType::Pass:
return "PASS";
case TestResultType::Skip:
return "SKIP";
case TestResultType::Timeout:
return "TIMEOUT";
case TestResultType::Unknown:
return "UNKNOWN";
}
}
TestResultType GetResultTypeFromString(const std::string &str)
{
if (str == "CRASH")
return TestResultType::Crash;
if (str == "FAIL")
return TestResultType::Fail;
if (str == "PASS")
return TestResultType::Pass;
if (str == "SKIP")
return TestResultType::Skip;
if (str == "TIMEOUT")
return TestResultType::Timeout;
return TestResultType::Unknown;
}
js::Value ResultTypeToJSString(TestResultType type, js::Document::AllocatorType *allocator)
{
js::Value jsName;
jsName.SetString(ResultTypeToString(type), *allocator);
return jsName;
}
// Writes out a TestResults to the Chromium JSON Test Results format.
// https://chromium.googlesource.com/chromium/src.git/+/master/docs/testing/json_test_results_format.md
void WriteTestResults(bool interrupted,
const TestResults &testResults,
const std::string &outputFile,
const char *testSuiteName)
{
time_t ltime;
time(&ltime);
struct tm *timeinfo = gmtime(&ltime);
ltime = mktime(timeinfo);
uint64_t secondsSinceEpoch = static_cast<uint64_t>(ltime);
js::Document doc;
doc.SetObject();
js::Document::AllocatorType &allocator = doc.GetAllocator();
doc.AddMember("interrupted", interrupted, allocator);
doc.AddMember("path_delimiter", ".", allocator);
doc.AddMember("version", 3, allocator);
doc.AddMember("seconds_since_epoch", secondsSinceEpoch, allocator);
js::Value testSuite;
testSuite.SetObject();
std::map<TestResultType, uint32_t> counts;
for (const auto &resultIter : testResults.results)
{
const TestIdentifier &id = resultIter.first;
const TestResult &result = resultIter.second;
js::Value jsResult;
jsResult.SetObject();
counts[result.type]++;
jsResult.AddMember("expected", "PASS", allocator);
jsResult.AddMember("actual", ResultTypeToJSString(result.type, &allocator), allocator);
js::Value times;
times.SetArray();
times.PushBack(result.elapsedTimeSeconds, allocator);
jsResult.AddMember("times", times, allocator);
char testName[500];
id.sprintfName(testName);
js::Value jsName;
jsName.SetString(testName, allocator);
testSuite.AddMember(jsName, jsResult, allocator);
}
js::Value numFailuresByType;
numFailuresByType.SetObject();
for (const auto &countIter : counts)
{
TestResultType type = countIter.first;
uint32_t count = countIter.second;
js::Value jsCount(count);
numFailuresByType.AddMember(ResultTypeToJSString(type, &allocator), jsCount, allocator);
}
doc.AddMember("num_failures_by_type", numFailuresByType, allocator);
js::Value tests;
tests.SetObject();
tests.AddMember(js::StringRef(testSuiteName), testSuite, allocator);
doc.AddMember("tests", tests, allocator);
printf("Writing test results to %s\n", outputFile.c_str());
FILE *fp = fopen(outputFile.c_str(), "w");
constexpr size_t kBufferSize = 0xFFFF;
std::vector<char> writeBuffer(kBufferSize);
js::FileWriteStream os(fp, writeBuffer.data(), kBufferSize);
js::PrettyWriter<js::FileWriteStream> writer(os);
doc.Accept(writer);
fclose(fp);
}
void UpdateCurrentTestResult(const testing::TestResult &resultIn, TestResults *resultsOut)
{
TestResult &resultOut = resultsOut->results[resultsOut->currentTest];
// Note: Crashes and Timeouts are detected by the crash handler and a watchdog thread.
if (resultIn.Skipped())
{
resultOut.type = TestResultType::Skip;
}
else if (resultIn.Failed())
{
resultOut.type = TestResultType::Fail;
}
else
{
resultOut.type = TestResultType::Pass;
}
resultOut.elapsedTimeSeconds = resultsOut->currentTestTimer.getElapsedTime();
}
TestIdentifier GetTestIdentifier(const testing::TestInfo &testInfo)
{
return {testInfo.test_suite_name(), testInfo.name()};
}
class TestEventListener : public testing::EmptyTestEventListener
{
public:
// Note: TestResults is owned by the TestSuite. It should outlive TestEventListener.
TestEventListener(const std::string &outputFile,
const char *testSuiteName,
TestResults *testResults)
: mResultsFile(outputFile), mTestSuiteName(testSuiteName), mTestResults(testResults)
{}
void OnTestStart(const testing::TestInfo &testInfo) override
{
std::lock_guard<std::mutex> guard(mTestResults->currentTestMutex);
mTestResults->currentTest = GetTestIdentifier(testInfo);
mTestResults->currentTestTimer.start();
}
void OnTestEnd(const testing::TestInfo &testInfo) override
{
std::lock_guard<std::mutex> guard(mTestResults->currentTestMutex);
mTestResults->currentTestTimer.stop();
const testing::TestResult &resultIn = *testInfo.result();
UpdateCurrentTestResult(resultIn, mTestResults);
mTestResults->currentTest = TestIdentifier();
}
void OnTestProgramEnd(const testing::UnitTest &testProgramInfo) override
{
std::lock_guard<std::mutex> guard(mTestResults->currentTestMutex);
mTestResults->allDone = true;
WriteTestResults(false, *mTestResults, mResultsFile, mTestSuiteName);
}
private:
std::string mResultsFile;
const char *mTestSuiteName;
TestResults *mTestResults;
};
using TestIdentifierFilter = std::function<bool(const TestIdentifier &id)>;
std::vector<TestIdentifier> FilterTests(std::map<TestIdentifier, FileLine> *fileLinesOut,
TestIdentifierFilter filter)
{
std::vector<TestIdentifier> tests;
const testing::UnitTest &testProgramInfo = *testing::UnitTest::GetInstance();
for (int suiteIndex = 0; suiteIndex < testProgramInfo.total_test_suite_count(); ++suiteIndex)
{
const testing::TestSuite &testSuite = *testProgramInfo.GetTestSuite(suiteIndex);
for (int testIndex = 0; testIndex < testSuite.total_test_count(); ++testIndex)
{
const testing::TestInfo &testInfo = *testSuite.GetTestInfo(testIndex);
TestIdentifier id = GetTestIdentifier(testInfo);
if (filter(id))
{
tests.emplace_back(id);
if (fileLinesOut)
{
(*fileLinesOut)[id] = {testInfo.file(), testInfo.line()};
}
}
}
}
return tests;
}
std::vector<TestIdentifier> GetFilteredTests(std::map<TestIdentifier, FileLine> *fileLinesOut)
{
TestIdentifierFilter gtestIDFilter = [](const TestIdentifier &id) {
return testing::internal::UnitTestOptions::FilterMatchesTest(id.testSuiteName, id.testName);
};
return FilterTests(fileLinesOut, gtestIDFilter);
}
std::vector<TestIdentifier> GetCompiledInTests(std::map<TestIdentifier, FileLine> *fileLinesOut)
{
TestIdentifierFilter passthroughFilter = [](const TestIdentifier &id) { return true; };
return FilterTests(fileLinesOut, passthroughFilter);
}
std::vector<TestIdentifier> GetShardTests(int shardIndex,
int shardCount,
std::map<TestIdentifier, FileLine> *fileLinesOut)
{
std::vector<TestIdentifier> allTests = GetCompiledInTests(fileLinesOut);
std::vector<TestIdentifier> shardTests;
for (int testIndex = shardIndex; testIndex < static_cast<int>(allTests.size());
testIndex += shardCount)
{
shardTests.emplace_back(allTests[testIndex]);
}
return shardTests;
}
std::string GetTestFilter(const std::vector<TestIdentifier> &tests)
{
std::stringstream filterStream;
filterStream << "--gtest_filter=";
for (size_t testIndex = 0; testIndex < tests.size(); ++testIndex)
{
if (testIndex != 0)
{
filterStream << ":";
}
filterStream << tests[testIndex];
}
return filterStream.str();
}
std::string ParseTestSuiteName(const char *executable)
{
const char *baseNameStart = strrchr(executable, GetPathSeparator());
if (!baseNameStart)
{
baseNameStart = executable;
}
else
{
baseNameStart++;
}
const char *suffix = GetExecutableExtension();
size_t suffixLen = strlen(suffix);
if (suffixLen == 0)
{
return baseNameStart;
}
const char *baseNameSuffix = strstr(baseNameStart, suffix);
ASSERT(baseNameSuffix == (baseNameStart + strlen(baseNameStart) - suffixLen));
return std::string(baseNameStart, baseNameSuffix);
}
bool GetTestResultsFromJSON(const js::Document &document, TestResults *resultsOut)
{
if (!document.HasMember("tests") || !document["tests"].IsObject())
{
return false;
}
const js::Value::ConstObject &tests = document["tests"].GetObject();
if (tests.MemberCount() != 1)
{
return false;
}
const js::Value::Member &suite = *tests.MemberBegin();
if (!suite.value.IsObject())
{
return false;
}
const js::Value::ConstObject &actual = suite.value.GetObject();
for (auto iter = actual.MemberBegin(); iter != actual.MemberEnd(); ++iter)
{
// Get test identifier.
const js::Value &name = iter->name;
if (!name.IsString())
{
return false;
}
TestIdentifier id;
if (!TestIdentifier::ParseFromString(name.GetString(), &id))
{
return false;
}
// Get test result.
const js::Value &value = iter->value;
if (!value.IsObject())
{
return false;
}
const js::Value::ConstObject &obj = value.GetObject();
if (!obj.HasMember("expected") || !obj.HasMember("actual"))
{
return false;
}
const js::Value &expected = obj["expected"];
const js::Value &actual = obj["actual"];
if (!expected.IsString() || !actual.IsString())
{
return false;
}
const std::string expectedStr = expected.GetString();
const std::string actualStr = actual.GetString();
if (expectedStr != "PASS")
{
return false;
}
TestResultType resultType = GetResultTypeFromString(actualStr);
if (resultType == TestResultType::Unknown)
{
return false;
}
double elapsedTimeSeconds = 0.0;
if (obj.HasMember("times"))
{
const js::Value &times = obj["times"];
if (!times.IsArray())
{
return false;
}
const js::Value::ConstArray &timesArray = times.GetArray();
if (timesArray.Size() != 1 || !timesArray[0].IsDouble())
{
return false;
}
elapsedTimeSeconds = timesArray[0].GetDouble();
}
TestResult &result = resultsOut->results[id];
result.elapsedTimeSeconds = elapsedTimeSeconds;
result.type = resultType;
}
return true;
}
bool MergeTestResults(const TestResults &input, TestResults *output)
{
for (const auto &resultsIter : input.results)
{
const TestIdentifier &id = resultsIter.first;
const TestResult &inputResult = resultsIter.second;
TestResult &outputResult = output->results[id];
// This should probably handle situations where a test is run more than once.
if (inputResult.type != TestResultType::Skip)
{
if (outputResult.type != TestResultType::Skip)
{
printf("Warning: duplicate entry for %s.%s.\n", id.testSuiteName.c_str(),
id.testName.c_str());
return false;
}
outputResult.elapsedTimeSeconds = inputResult.elapsedTimeSeconds;
outputResult.type = inputResult.type;
}
}
return true;
}
void PrintTestOutputSnippet(const TestIdentifier &id,
const TestResult &result,
const std::string &fullOutput)
{
std::stringstream nameStream;
nameStream << id;
std::string fullName = nameStream.str();
size_t runPos = fullOutput.find(std::string("[ RUN ] ") + fullName);
if (runPos == std::string::npos)
{
printf("Cannot locate test output snippet.\n");
return;
}
size_t endPos = fullOutput.find(std::string("[ FAILED ] ") + fullName, runPos);
// Only clip the snippet to the "OK" message if the test really
// succeeded. It still might have e.g. crashed after printing it.
if (endPos == std::string::npos && result.type == TestResultType::Pass)
{
endPos = fullOutput.find(std::string("[ OK ] ") + fullName, runPos);
}
if (endPos != std::string::npos)
{
size_t newline_pos = fullOutput.find("\n", endPos);
if (newline_pos != std::string::npos)
endPos = newline_pos + 1;
}
std::cout << "\n";
if (endPos != std::string::npos)
{
std::cout << fullOutput.substr(runPos, endPos - runPos);
}
else
{
std::cout << fullOutput.substr(runPos);
}
std::cout << "\n";
}
} // namespace
TestIdentifier::TestIdentifier() = default;
TestIdentifier::TestIdentifier(const std::string &suiteNameIn, const std::string &nameIn)
: testSuiteName(suiteNameIn), testName(nameIn)
{}
TestIdentifier::TestIdentifier(const TestIdentifier &other) = default;
TestIdentifier::~TestIdentifier() = default;
TestIdentifier &TestIdentifier::operator=(const TestIdentifier &other) = default;
void TestIdentifier::sprintfName(char *outBuffer) const
{
sprintf(outBuffer, "%s.%s", testSuiteName.c_str(), testName.c_str());
}
// static
bool TestIdentifier::ParseFromString(const std::string &str, TestIdentifier *idOut)
{
size_t separator = str.find(".");
if (separator == std::string::npos)
{
return false;
}
idOut->testSuiteName = str.substr(0, separator);
idOut->testName = str.substr(separator + 1, str.length() - separator - 1);
return true;
}
TestResults::TestResults() = default;
TestResults::~TestResults() = default;
ProcessInfo::ProcessInfo() = default;
ProcessInfo &ProcessInfo::operator=(ProcessInfo &&rhs)
{
process = std::move(rhs.process);
testsInBatch = std::move(rhs.testsInBatch);
resultsFileName = std::move(rhs.resultsFileName);
filterFileName = std::move(rhs.filterFileName);
commandLine = std::move(rhs.commandLine);
return *this;
}
ProcessInfo::~ProcessInfo() = default;
ProcessInfo::ProcessInfo(ProcessInfo &&other)
{
*this = std::move(other);
}
TestSuite::TestSuite(int *argc, char **argv)
: mShardCount(-1),
mShardIndex(-1),
mBotMode(false),
mBatchSize(kDefaultBatchSize),
mCurrentResultCount(0),
mTotalResultCount(0),
mMaxProcesses(NumberOfProcessors()),
mTestTimeout(kDefaultTestTimeout),
mBatchTimeout(60)
{
bool hasFilter = false;
#if defined(ANGLE_PLATFORM_WINDOWS)
testing::GTEST_FLAG(catch_exceptions) = false;
#endif
// Note that the crash callback must be owned and not use global constructors.
mCrashCallback = [this]() { onCrashOrTimeout(TestResultType::Crash); };
InitCrashHandler(&mCrashCallback);
if (*argc <= 0)
{
printf("Missing test arguments.\n");
exit(1);
}
mTestExecutableName = argv[0];
mTestSuiteName = ParseTestSuiteName(mTestExecutableName.c_str());
for (int argIndex = 1; argIndex < *argc;)
{
if (parseSingleArg(argv[argIndex]))
{
DeleteArg(argc, argv, argIndex);
continue;
}
if (ParseFlagValue("--gtest_filter=", argv[argIndex]))
{
hasFilter = true;
}
else
{
mGoogleTestCommandLineArgs.push_back(argv[argIndex]);
}
++argIndex;
}
if ((mShardIndex >= 0) != (mShardCount > 1))
{
printf("Shard index and shard count must be specified together.\n");
exit(1);
}
if (!mFilterFile.empty())
{
if (hasFilter)
{
printf("Cannot use gtest_filter in conjunction with a filter file.\n");
exit(1);
}
if (mShardCount > 0)
{
printf("Cannot use filter file in conjunction with sharding parameters.\n");
exit(1);
}
uint32_t fileSize = 0;
if (!GetFileSize(mFilterFile.c_str(), &fileSize))
{
printf("Error getting filter file size: %s\n", mFilterFile.c_str());
exit(1);
}
std::vector<char> fileContents(fileSize + 1, 0);
if (!ReadEntireFileToString(mFilterFile.c_str(), fileContents.data(), fileSize))
{
printf("Error loading filter file: %s\n", mFilterFile.c_str());
exit(1);
}
mFilterString.assign(fileContents.data());
if (mFilterString.substr(0, strlen("--gtest_filter=")) != std::string("--gtest_filter="))
{
printf("Filter file must start with \"--gtest_filter=\".");
exit(1);
}
// Note that we only add a filter string if we previously deleted a shader filter file
// argument. So we will have space for the new filter string in argv.
AddArg(argc, argv, mFilterString.c_str());
}
// Call into gtest internals to force parameterized test name registration.
// TODO(jmadill): Clean this up so we don't need to call it.
testing::internal::UnitTestImpl *impl = testing::internal::GetUnitTestImpl();
impl->RegisterParameterizedTests();
if (mShardCount > 0)
{
if (hasFilter)
{
printf("Cannot use gtest_filter in conjunction with sharding parameters.\n");
exit(1);
}
mTestQueue = GetShardTests(mShardIndex, mShardCount, &mTestFileLines);
mFilterString = GetTestFilter(mTestQueue);
// Note that we only add a filter string if we previously deleted a shader index/count
// argument. So we will have space for the new filter string in argv.
AddArg(argc, argv, mFilterString.c_str());
}
testing::InitGoogleTest(argc, argv);
if (mShardCount <= 0)
{
mTestQueue = GetFilteredTests(&mTestFileLines);
}
mTotalResultCount = mTestQueue.size();
if ((mBotMode || !mResultsDirectory.empty()) && mResultsFile.empty())
{
// Create a default output file in bot mode.
mResultsFile = "output.json";
}
if (!mResultsDirectory.empty())
{
std::stringstream resultFileName;
resultFileName << mResultsDirectory << GetPathSeparator() << mResultsFile;
mResultsFile = resultFileName.str();
}
if (!mResultsFile.empty())
{
testing::TestEventListeners &listeners = testing::UnitTest::GetInstance()->listeners();
listeners.Append(
new TestEventListener(mResultsFile, mTestSuiteName.c_str(), &mTestResults));
std::vector<TestIdentifier> testList = GetFilteredTests(nullptr);
for (const TestIdentifier &id : testList)
{
mTestResults.results[id].type = TestResultType::Skip;
}
}
}
TestSuite::~TestSuite()
{
if (mWatchdogThread.joinable())
{
mWatchdogThread.detach();
}
TerminateCrashHandler();
}
bool TestSuite::parseSingleArg(const char *argument)
{
return (ParseIntArg("--shard-count=", argument, &mShardCount) ||
ParseIntArg("--shard-index=", argument, &mShardIndex) ||
ParseIntArg("--batch-size=", argument, &mBatchSize) ||
ParseIntArg("--max-processes=", argument, &mMaxProcesses) ||
ParseIntArg(kTestTimeoutArg, argument, &mTestTimeout) ||
ParseIntArg("--batch-timeout=", argument, &mBatchTimeout) ||
ParseStringArg("--results-directory=", argument, &mResultsDirectory) ||
ParseStringArg(kResultFileArg, argument, &mResultsFile) ||
ParseStringArg(kFilterFileArg, argument, &mFilterFile) ||
ParseFlag("--bot-mode", argument, &mBotMode));
}
void TestSuite::onCrashOrTimeout(TestResultType crashOrTimeout)
{
if (mTestResults.currentTest.valid())
{
TestResult &result = mTestResults.results[mTestResults.currentTest];
result.type = crashOrTimeout;
result.elapsedTimeSeconds = mTestResults.currentTestTimer.getElapsedTime();
}
if (mResultsFile.empty())
{
printf("No results file specified.\n");
return;
}
WriteTestResults(true, mTestResults, mResultsFile, mTestSuiteName.c_str());
}
bool TestSuite::launchChildTestProcess(const std::vector<TestIdentifier> &testsInBatch)
{
constexpr uint32_t kMaxPath = 1000;
// Create a temporary file to store the test list
ProcessInfo processInfo;
char filterBuffer[kMaxPath] = {};
if (!CreateTemporaryFile(filterBuffer, kMaxPath))
{
std::cerr << "Error creating temporary file for test list.\n";
return false;
}
processInfo.filterFileName.assign(filterBuffer);
std::string filterString = GetTestFilter(testsInBatch);
FILE *fp = fopen(processInfo.filterFileName.c_str(), "w");
if (!fp)
{
std::cerr << "Error opening temporary file for test list.\n";
return false;
}
fprintf(fp, "%s", filterString.c_str());
fclose(fp);
std::string filterFileArg = kFilterFileArg + processInfo.filterFileName;
// Create a temporary file to store the test output.
char resultsBuffer[kMaxPath] = {};
if (!CreateTemporaryFile(resultsBuffer, kMaxPath))
{
std::cerr << "Error creating temporary file for test list.\n";
return false;
}
processInfo.resultsFileName.assign(resultsBuffer);
std::string resultsFileArg = kResultFileArg + processInfo.resultsFileName;
// Construct commandline for child process.
std::vector<const char *> args;
args.push_back(mTestExecutableName.c_str());
args.push_back(filterFileArg.c_str());
args.push_back(resultsFileArg.c_str());
for (const std::string &arg : mGoogleTestCommandLineArgs)
{
args.push_back(arg.c_str());
}
std::string timeoutStr;
if (mTestTimeout != kDefaultTestTimeout)
{
std::stringstream timeoutStream;
timeoutStream << kTestTimeoutArg << mTestTimeout;
timeoutStr = timeoutStream.str();
args.push_back(timeoutStr.c_str());
}
// Launch child process and wait for completion.
processInfo.process = LaunchProcess(args, true, true);
if (!processInfo.process->started())
{
std::cerr << "Error launching child process.\n";
return false;
}
std::stringstream commandLineStr;
for (const char *arg : args)
{
commandLineStr << arg << " ";
}
processInfo.commandLine = commandLineStr.str();
processInfo.testsInBatch = testsInBatch;
mCurrentProcesses.emplace_back(std::move(processInfo));
return true;
}
bool TestSuite::finishProcess(ProcessInfo *processInfo)
{
// Get test results and merge into master list.
TestResults batchResults;
if (!GetTestResultsFromFile(processInfo->resultsFileName.c_str(), &batchResults))
{
std::cerr << "Error reading test results from child process.\n";
return false;
}
if (!MergeTestResults(batchResults, &mTestResults))
{
std::cerr << "Error merging batch test results.\n";
return false;
}
// Process results and print unexpected errors.
for (const auto &resultIter : batchResults.results)
{
const TestIdentifier &id = resultIter.first;
const TestResult &result = resultIter.second;
// Skip results aren't procesed since they're added back to the test queue below.
if (result.type == TestResultType::Skip)
{
continue;
}
mCurrentResultCount++;
printf("[%d/%d] %s.%s", mCurrentResultCount, mTotalResultCount, id.testSuiteName.c_str(),
id.testName.c_str());
if (result.type == TestResultType::Pass)
{
printf(" (%g ms)\n", result.elapsedTimeSeconds * 1000.0);
}
else
{
printf(" (%s)\n", ResultTypeToString(result.type));
const std::string &batchStdout = processInfo->process->getStdout();
PrintTestOutputSnippet(id, result, batchStdout);
}
}
// On unexpected exit, re-queue any unfinished tests.
if (processInfo->process->getExitCode() != 0)
{
for (const auto &resultIter : batchResults.results)
{
const TestIdentifier &id = resultIter.first;
const TestResult &result = resultIter.second;
if (result.type == TestResultType::Skip)
{
mTestQueue.emplace_back(id);
}
}
}
// Clean up any dirty temporary files.
for (const std::string &tempFile : {processInfo->filterFileName, processInfo->resultsFileName})
{
// Note: we should be aware that this cleanup won't happen if the harness itself crashes.
// If this situation comes up in the future we should add crash cleanup to the harness.
if (!angle::DeleteFile(tempFile.c_str()))
{
std::cerr << "Warning: Error cleaning up temp file: " << tempFile << "\n";
}
}
processInfo->process.reset();
return true;
}
int TestSuite::run()
{
// Run tests serially.
if (!mBotMode)
{
startWatchdog();
return RUN_ALL_TESTS();
}
constexpr double kIdleMessageTimeout = 5.0;
Timer messageTimer;
messageTimer.start();
while (!mTestQueue.empty() || !mCurrentProcesses.empty())
{
bool progress = false;
// Spawn a process if needed and possible.
while (static_cast<int>(mCurrentProcesses.size()) < mMaxProcesses && !mTestQueue.empty())
{
int numTests = std::min<int>(mTestQueue.size(), mBatchSize);
std::vector<TestIdentifier> testsInBatch;
testsInBatch.assign(mTestQueue.begin(), mTestQueue.begin() + numTests);
mTestQueue.erase(mTestQueue.begin(), mTestQueue.begin() + numTests);
if (!launchChildTestProcess(testsInBatch))
{
return 1;
}
progress = true;
}
// Check for process completion.
for (auto processIter = mCurrentProcesses.begin(); processIter != mCurrentProcesses.end();)
{
ProcessInfo &processInfo = *processIter;
if (processInfo.process->finished())
{
if (!finishProcess(&processInfo))
{
return 1;
}
processIter = mCurrentProcesses.erase(processIter);
progress = true;
}
else if (processInfo.process->getElapsedTimeSeconds() > mBatchTimeout)
{
// Terminate the process and record timeouts for the batch.
// Because we can't determine which sub-test caused a timeout, record the whole
// batch as a timeout failure. Can be improved by using socket message passing.
if (!processInfo.process->kill())
{
return 1;
}
for (const TestIdentifier &testIdentifier : processInfo.testsInBatch)
{
// Because the whole batch failed we can't know how long each test took.
mTestResults.results[testIdentifier].type = TestResultType::Timeout;
}
processIter = mCurrentProcesses.erase(processIter);
progress = true;
}
else
{
processIter++;
}
}
if (!progress && messageTimer.getElapsedTime() > kIdleMessageTimeout)
{
for (const ProcessInfo &processInfo : mCurrentProcesses)
{
double processTime = processInfo.process->getElapsedTimeSeconds();
if (processTime > kIdleMessageTimeout)
{
printf("Running for %d seconds: %s\n", static_cast<int>(processTime),
processInfo.commandLine.c_str());
}
}
messageTimer.start();
}
// Sleep briefly and continue.
angle::Sleep(10);
}
// Dump combined results.
WriteTestResults(true, mTestResults, mResultsFile, mTestSuiteName.c_str());
return printFailuresAndReturnCount() == 0;
}
int TestSuite::printFailuresAndReturnCount() const
{
std::vector<std::string> failures;
for (const auto &resultIter : mTestResults.results)
{
const TestIdentifier &id = resultIter.first;
const TestResult &result = resultIter.second;
if (result.type != TestResultType::Pass)
{
const FileLine &fileLine = mTestFileLines.find(id)->second;
std::stringstream failureMessage;
failureMessage << id << " (" << fileLine.file << ":" << fileLine.line << ") ("
<< ResultTypeToString(result.type) << ")";
failures.emplace_back(failureMessage.str());
}
}
if (failures.empty())
return 0;
printf("%zu test%s failed:\n", failures.size(), failures.size() > 1 ? "s" : "");
for (const std::string &failure : failures)
{
printf(" %s\n", failure.c_str());
}
return static_cast<int>(failures.size());
}
void TestSuite::startWatchdog()
{
auto watchdogMain = [this]() {
do
{
{
std::lock_guard<std::mutex> guard(mTestResults.currentTestMutex);
if (mTestResults.currentTestTimer.getElapsedTime() >
static_cast<double>(mTestTimeout))
{
onCrashOrTimeout(TestResultType::Timeout);
exit(2);
}
if (mTestResults.allDone)
return;
}
angle::Sleep(1000);
} while (true);
};
mWatchdogThread = std::thread(watchdogMain);
}
bool GetTestResultsFromFile(const char *fileName, TestResults *resultsOut)
{
std::ifstream ifs(fileName);
if (!ifs.is_open())
{
std::cerr << "Error opening " << fileName << "\n";
return false;
}
js::IStreamWrapper ifsWrapper(ifs);
js::Document document;
document.ParseStream(ifsWrapper);
if (document.HasParseError())
{
std::cerr << "Parse error reading JSON document: " << document.GetParseError() << "\n";
return false;
}
if (!GetTestResultsFromJSON(document, resultsOut))
{
std::cerr << "Error getting test results from JSON.\n";
return false;
}
return true;
}
const char *TestResultTypeToString(TestResultType type)
{
switch (type)
{
case TestResultType::Crash:
return "Crash";
case TestResultType::Fail:
return "Fail";
case TestResultType::Skip:
return "Skip";
case TestResultType::Pass:
return "Pass";
case TestResultType::Timeout:
return "Timeout";
case TestResultType::Unknown:
return "Unknown";
}
}
} // namespace angle
src/tests/test_utils/runner/TestSuite.h 0 → 100644
//
// Copyright 2019 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// TestSuite:
// Basic implementation of a test harness in ANGLE.
#include <map>
#include <memory>
#include <mutex>
#include <string>
#include <thread>
#include "util/test_utils.h"
namespace angle
{
struct TestIdentifier
{
TestIdentifier();
TestIdentifier(const std::string &suiteNameIn, const std::string &nameIn);
TestIdentifier(const TestIdentifier &other);
~TestIdentifier();
TestIdentifier &operator=(const TestIdentifier &other);
static bool ParseFromString(const std::string &str, TestIdentifier *idOut);
bool valid() const { return !testName.empty(); }
void sprintfName(char *outBuffer) const;
std::string testSuiteName;
std::string testName;
};
inline bool operator<(const TestIdentifier &a, const TestIdentifier &b)
{
return std::tie(a.testSuiteName, a.testName) < std::tie(b.testSuiteName, b.testName);
}
inline bool operator==(const TestIdentifier &a, const TestIdentifier &b)
{
return std::tie(a.testSuiteName, a.testName) == std::tie(b.testSuiteName, b.testName);
}
inline std::ostream &operator<<(std::ostream &os, const TestIdentifier &id)
{
return os << id.testSuiteName << "." << id.testName;
}
enum class TestResultType
{
Crash,
Fail,
Skip,
Pass,
Timeout,
Unknown,
};
const char *TestResultTypeToString(TestResultType type);
struct TestResult
{
TestResultType type = TestResultType::Skip;
double elapsedTimeSeconds = 0.0;
};
inline bool operator==(const TestResult &a, const TestResult &b)
{
return a.type == b.type;
}
inline std::ostream &operator<<(std::ostream &os, const TestResult &result)
{
return os << TestResultTypeToString(result.type);
}
struct TestResults
{
TestResults();
~TestResults();
std::map<TestIdentifier, TestResult> results;
std::mutex currentTestMutex;
TestIdentifier currentTest;
Timer currentTestTimer;
bool allDone = false;
};
struct FileLine
{
const char *file;
int line;
};
struct ProcessInfo : angle::NonCopyable
{
ProcessInfo();
~ProcessInfo();
ProcessInfo(ProcessInfo &&other);
ProcessInfo &operator=(ProcessInfo &&rhs);
ProcessHandle process;
std::vector<TestIdentifier> testsInBatch;
std::string resultsFileName;
std::string filterFileName;
std::string commandLine;
};
class TestSuite
{
public:
TestSuite(int *argc, char **argv);
~TestSuite();
int run();
void onCrashOrTimeout(TestResultType crashOrTimeout);
private:
bool parseSingleArg(const char *argument);
bool launchChildTestProcess(const std::vector<TestIdentifier> &testsInBatch);
bool finishProcess(ProcessInfo *processInfo);
int printFailuresAndReturnCount() const;
void startWatchdog();
std::string mTestExecutableName;
std::string mTestSuiteName;
std::vector<TestIdentifier> mTestQueue;
std::string mFilterString;
std::string mFilterFile;
std::string mResultsDirectory;
std::string mResultsFile;
int mShardCount;
int mShardIndex;
angle::CrashCallback mCrashCallback;
TestResults mTestResults;
bool mBotMode;
int mBatchSize;
int mCurrentResultCount;
int mTotalResultCount;
int mMaxProcesses;
int mTestTimeout;
int mBatchTimeout;
std::vector<std::string> mGoogleTestCommandLineArgs;
std::map<TestIdentifier, FileLine> mTestFileLines;
std::vector<ProcessInfo> mCurrentProcesses;
std::thread mWatchdogThread;
};
bool GetTestResultsFromFile(const char *fileName, TestResults *resultsOut);
} // namespace angle
src/tests/test_utils/runner/TestSuite_unittest.cpp 0 → 100644
//
// Copyright 2019 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// TestSuite_unittest.cpp: Unit tests for ANGLE's test harness.
//
#include <gtest/gtest.h>
#include "../angle_test_instantiate.h"
#include "TestSuite.h"
#include "common/debug.h"
#include "common/system_utils.h"
#include "util/test_utils.h"
#include "util/test_utils_unittest_helper.h"
#include <rapidjson/document.h>
using namespace angle;
namespace js = rapidjson;
namespace
{
constexpr char kTestHelperExecutable[] = "test_utils_unittest_helper";
class TestSuiteTest : public testing::Test
{
protected:
void TearDown() override
{
if (!mTempFileName.empty())
{
angle::DeleteFile(mTempFileName.c_str());
}
}
std::string mTempFileName;
};
// Tests the ANGLE standalone testing harness. Runs four tests with different ending conditions.
// Verifies that Pass, Fail, Crash and Timeout are all handled correctly.
TEST_F(TestSuiteTest, RunMockTests)
{
std::string executablePath = GetExecutableDirectory();
EXPECT_NE(executablePath, "");
executablePath += std::string("/") + kTestHelperExecutable + GetExecutableExtension();
constexpr uint32_t kMaxTempDirLen = 100;
char tempFileName[kMaxTempDirLen * 2];
ASSERT_TRUE(GetTempDir(tempFileName, kMaxTempDirLen));
std::stringstream tempFNameStream;
tempFNameStream << tempFileName << "/test_temp_" << rand() << ".json";
mTempFileName = tempFNameStream.str();
std::string resultsFileName = "--results-file=" + mTempFileName;
std::vector<const char *> args = {executablePath.c_str(),
kRunTestSuite,
"--gtest_filter=MockTestSuiteTest.DISABLED_*",
"--gtest_also_run_disabled_tests",
"--bot-mode",
"--test-timeout=10",
resultsFileName.c_str()};
ProcessHandle process(args, true, true);
EXPECT_TRUE(process->started());
EXPECT_TRUE(process->finish());
EXPECT_TRUE(process->finished());
EXPECT_EQ(process->getStderr(), "");
TestResults actual;
ASSERT_TRUE(GetTestResultsFromFile(mTempFileName.c_str(), &actual));
EXPECT_TRUE(DeleteFile(mTempFileName.c_str()));
mTempFileName.clear();
std::map<TestIdentifier, TestResult> expectedResults = {
{{"MockTestSuiteTest", "DISABLED_Pass"}, {TestResultType::Pass, 0.0}},
{{"MockTestSuiteTest", "DISABLED_Fail"}, {TestResultType::Fail, 0.0}},
{{"MockTestSuiteTest", "DISABLED_Timeout"}, {TestResultType::Timeout, 0.0}},
// {{"MockTestSuiteTest", "DISABLED_Crash"}, {TestResultType::Crash, 0.0}},
};
EXPECT_EQ(expectedResults, actual.results);
}
// Normal passing test.
TEST(MockTestSuiteTest, DISABLED_Pass)
{
EXPECT_TRUE(true);
}
// Normal failing test.
TEST(MockTestSuiteTest, DISABLED_Fail)
{
EXPECT_TRUE(false);
}
// Trigger a test timeout.
TEST(MockTestSuiteTest, DISABLED_Timeout)
{
angle::Sleep(30000);
}
// Trigger a test crash.
// TEST(MockTestSuiteTest, DISABLED_Crash)
// {
// ANGLE_CRASH();
// }
} // namespace
util/posix/crash_handler_posix.cpp
......@@ -35,7 +35,6 @@
namespace angle
{
#if defined(ANGLE_PLATFORM_ANDROID) || defined(ANGLE_PLATFORM_FUCHSIA)
void PrintStackBacktrace()
......@@ -54,6 +53,10 @@ void TerminateCrashHandler()
}
#else
namespace
{
CrashCallback *gCrashHandlerCallback;
} // namespace
# if defined(ANGLE_PLATFORM_APPLE)
......@@ -85,6 +88,11 @@ void PrintStackBacktrace()
static void Handler(int sig)
{
if (gCrashHandlerCallback)
{
(*gCrashHandlerCallback)();
}
printf("\nSignal %d:\n", sig);
PrintStackBacktrace();
......@@ -127,6 +135,11 @@ void PrintStackBacktrace()
static void Handler(int sig)
{
if (gCrashHandlerCallback)
{
(*gCrashHandlerCallback)();
}
printf("\nSignal %d [%s]:\n", sig, strsignal(sig));
PrintStackBacktrace();
......@@ -142,6 +155,7 @@ static constexpr int kSignals[] = {
void InitCrashHandler(CrashCallback *callback)
{
gCrashHandlerCallback = callback;
for (int sig : kSignals)
{
// Register our signal handler unless something's already done so (e.g. catchsegv).
......@@ -155,6 +169,7 @@ void InitCrashHandler(CrashCallback *callback)
void TerminateCrashHandler()
{
gCrashHandlerCallback = nullptr;
for (int sig : kSignals)
{
void (*prev)(int) = signal(sig, SIG_DFL);
......
util/test_utils_unittest_helper.cpp
......@@ -7,12 +7,22 @@
#include "test_utils_unittest_helper.h"
#include "../src/tests/test_utils/runner/TestSuite.h"
#include "common/system_utils.h"
#include <string.h>
int main(int argc, char **argv)
{
for (int argIndex = 1; argIndex < argc; ++argIndex)
{
if (strcmp(argv[argIndex], kRunTestSuite) == 0)
{
angle::TestSuite testSuite(&argc, argv);
return testSuite.run();
}
}
if (argc != 3 || strcmp(argv[1], kRunAppTestArg1) != 0 || strcmp(argv[2], kRunAppTestArg2) != 0)
{
fprintf(stderr, "Expected command line:\n%s %s %s\n", argv[0], kRunAppTestArg1,
......
util/test_utils_unittest_helper.h
......@@ -16,6 +16,7 @@ constexpr char kRunAppTestStdout[] = "RunAppTest stdout test\n";
constexpr char kRunAppTestStderr[] = "RunAppTest stderr test\n .. that expands multiple lines\n";
constexpr char kRunAppTestArg1[] = "--expected-arg1";
constexpr char kRunAppTestArg2[] = "expected_arg2";
constexpr char kRunTestSuite[] = "--run-test-suite";
} // anonymous namespace
#endif // COMMON_SYSTEM_UTILS_UNITTEST_HELPER_H_
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment