Commit 61629b49 by Abseil Team Committed by Xiaoyi Zhang

Export Test - Do Not Merge

Add --gtest_fail_fast support to googletest. - Analogous functionality to to golang -test.failfast and python --failfast - Stops test execution upon first test failure. - Also add support Bazel equivalent env var (TESTBRIDGE_TEST_RUNNER_FAIL_FAST) PiperOrigin-RevId: 302488880
parent 749148f1
......@@ -2116,6 +2116,15 @@ For example:
everything in test suite `FooTest` except `FooTest.Bar` and everything in
test suite `BarTest` except `BarTest.Foo`.
#### Stop test execution upon first failure
By default, a googletest program runs all tests the user has defined. In some
cases (e.g. iterative test development & execution) it may be desirable stop
test execution upon first failure (trading improved latency for completeness).
If `GTEST_FAIL_FAST` environment variable or `--gtest_fail_fast` flag is set,
the test runner will stop execution as soon as the first test failure is
found.
#### Temporarily Disabling Tests
If you have a broken test that you cannot fix right away, you can add the
......
......@@ -101,6 +101,10 @@ GTEST_DECLARE_bool_(catch_exceptions);
// to let Google Test decide.
GTEST_DECLARE_string_(color);
// This flag controls whether the test runner should continue execution past
// first failure.
GTEST_DECLARE_bool_(fail_fast);
// This flag sets up the filter to select by name using a glob pattern
// the tests to run. If the filter is not given all tests are executed.
GTEST_DECLARE_string_(filter);
......@@ -795,6 +799,9 @@ class GTEST_API_ TestInfo {
// deletes it.
void Run();
// Skip and records the test result for this object.
void Skip();
static void ClearTestResult(TestInfo* test_info) {
test_info->result_.Clear();
}
......@@ -943,6 +950,9 @@ class GTEST_API_ TestSuite {
// Runs every test in this TestSuite.
void Run();
// Skips the execution of tests under this TestSuite
void Skip();
// Runs SetUpTestSuite() for this TestSuite. This wrapper is needed
// for catching exceptions thrown from SetUpTestSuite().
void RunSetUpTestSuite() {
......
......@@ -84,6 +84,7 @@ const char kAlsoRunDisabledTestsFlag[] = "also_run_disabled_tests";
const char kBreakOnFailureFlag[] = "break_on_failure";
const char kCatchExceptionsFlag[] = "catch_exceptions";
const char kColorFlag[] = "color";
const char kFailFast[] = "fail_fast";
const char kFilterFlag[] = "filter";
const char kListTestsFlag[] = "list_tests";
const char kOutputFlag[] = "output";
......@@ -164,6 +165,7 @@ class GTestFlagSaver {
color_ = GTEST_FLAG(color);
death_test_style_ = GTEST_FLAG(death_test_style);
death_test_use_fork_ = GTEST_FLAG(death_test_use_fork);
fail_fast_ = GTEST_FLAG(fail_fast);
filter_ = GTEST_FLAG(filter);
internal_run_death_test_ = GTEST_FLAG(internal_run_death_test);
list_tests_ = GTEST_FLAG(list_tests);
......@@ -187,6 +189,7 @@ class GTestFlagSaver {
GTEST_FLAG(death_test_style) = death_test_style_;
GTEST_FLAG(death_test_use_fork) = death_test_use_fork_;
GTEST_FLAG(filter) = filter_;
GTEST_FLAG(fail_fast) = fail_fast_;
GTEST_FLAG(internal_run_death_test) = internal_run_death_test_;
GTEST_FLAG(list_tests) = list_tests_;
GTEST_FLAG(output) = output_;
......@@ -208,6 +211,7 @@ class GTestFlagSaver {
std::string color_;
std::string death_test_style_;
bool death_test_use_fork_;
bool fail_fast_;
std::string filter_;
std::string internal_run_death_test_;
bool list_tests_;
......
......@@ -213,6 +213,21 @@ static const char* GetDefaultFilter() {
return kUniversalFilter;
}
// Bazel passes in the argument to '--test_runner_fail_fast' via the
// TESTBRIDGE_TEST_RUNNER_FAIL_FAST environment variable.
static bool GetDefaultFailFast() {
const char* const testbridge_test_runner_fail_fast =
internal::posix::GetEnv("TESTBRIDGE_TEST_RUNNER_FAIL_FAST");
if (testbridge_test_runner_fail_fast != nullptr) {
return strcmp(testbridge_test_runner_fail_fast, "1") == 0;
}
return false;
}
GTEST_DEFINE_bool_(
fail_fast, internal::BoolFromGTestEnv("fail_fast", GetDefaultFailFast()),
"True if and only if a test failure should stop further test execution.");
GTEST_DEFINE_bool_(
also_run_disabled_tests,
internal::BoolFromGTestEnv("also_run_disabled_tests", false),
......@@ -2863,6 +2878,28 @@ void TestInfo::Run() {
impl->set_current_test_info(nullptr);
}
// Skip and records a skipped test result for this object.
void TestInfo::Skip() {
if (!should_run_) return;
internal::UnitTestImpl* const impl = internal::GetUnitTestImpl();
impl->set_current_test_info(this);
TestEventListener* repeater = UnitTest::GetInstance()->listeners().repeater();
// Notifies the unit test event listeners that a test is about to start.
repeater->OnTestStart(*this);
const TestPartResult test_part_result =
TestPartResult(TestPartResult::kSkip, this->file(), this->line(), "");
impl->GetTestPartResultReporterForCurrentThread()->ReportTestPartResult(
test_part_result);
// Notifies the unit test event listener that a test has just finished.
repeater->OnTestEnd(*this);
impl->set_current_test_info(nullptr);
}
// class TestSuite
// Gets the number of successful tests in this test suite.
......@@ -2975,6 +3012,12 @@ void TestSuite::Run() {
start_timestamp_ = internal::GetTimeInMillis();
for (int i = 0; i < total_test_count(); i++) {
GetMutableTestInfo(i)->Run();
if (GTEST_FLAG(fail_fast) && GetMutableTestInfo(i)->result()->Failed()) {
for (int j = i + 1; j < total_test_count(); j++) {
GetMutableTestInfo(j)->Skip();
}
break;
}
}
elapsed_time_ = internal::GetTimeInMillis() - start_timestamp_;
......@@ -2992,6 +3035,36 @@ void TestSuite::Run() {
impl->set_current_test_suite(nullptr);
}
// Skips all tests under this TestSuite.
void TestSuite::Skip() {
if (!should_run_) return;
internal::UnitTestImpl* const impl = internal::GetUnitTestImpl();
impl->set_current_test_suite(this);
TestEventListener* repeater = UnitTest::GetInstance()->listeners().repeater();
// Call both legacy and the new API
repeater->OnTestSuiteStart(*this);
// Legacy API is deprecated but still available
#ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI
repeater->OnTestCaseStart(*this);
#endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI
for (int i = 0; i < total_test_count(); i++) {
GetMutableTestInfo(i)->Skip();
}
// Call both legacy and the new API
repeater->OnTestSuiteEnd(*this);
// Legacy API is deprecated but still available
#ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI
repeater->OnTestCaseEnd(*this);
#endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI
impl->set_current_test_suite(nullptr);
}
// Clears the results of all tests in this test suite.
void TestSuite::ClearResult() {
ad_hoc_test_result_.Clear();
......@@ -5523,6 +5596,13 @@ bool UnitTestImpl::RunAllTests() {
for (int test_index = 0; test_index < total_test_suite_count();
test_index++) {
GetMutableSuiteCase(test_index)->Run();
if (GTEST_FLAG(fail_fast) &&
GetMutableSuiteCase(test_index)->Failed()) {
for (int j = test_index + 1; j < total_test_suite_count(); j++) {
GetMutableSuiteCase(j)->Skip();
}
break;
}
}
}
......@@ -6136,6 +6216,7 @@ static bool ParseGoogleTestFlag(const char* const arg) {
&GTEST_FLAG(death_test_style)) ||
ParseBoolFlag(arg, kDeathTestUseFork,
&GTEST_FLAG(death_test_use_fork)) ||
ParseBoolFlag(arg, kFailFast, &GTEST_FLAG(fail_fast)) ||
ParseStringFlag(arg, kFilterFlag, &GTEST_FLAG(filter)) ||
ParseStringFlag(arg, kInternalRunDeathTestFlag,
&GTEST_FLAG(internal_run_death_test)) ||
......@@ -6150,8 +6231,7 @@ static bool ParseGoogleTestFlag(const char* const arg) {
&GTEST_FLAG(stack_trace_depth)) ||
ParseStringFlag(arg, kStreamResultToFlag,
&GTEST_FLAG(stream_result_to)) ||
ParseBoolFlag(arg, kThrowOnFailureFlag,
&GTEST_FLAG(throw_on_failure));
ParseBoolFlag(arg, kThrowOnFailureFlag, &GTEST_FLAG(throw_on_failure));
}
#if GTEST_USE_OWN_FLAGFILE_FLAG_
......
......@@ -58,6 +58,7 @@ cc_test(
"googletest-catch-exceptions-test_.cc",
"googletest-color-test_.cc",
"googletest-env-var-test_.cc",
"googletest-failfast-unittest_.cc",
"googletest-filter-unittest_.cc",
"googletest-break-on-failure-unittest_.cc",
"googletest-listener-test.cc",
......@@ -223,6 +224,21 @@ py_test(
)
cc_binary(
name = "googletest-failfast-unittest_",
testonly = 1,
srcs = ["googletest-failfast-unittest_.cc"],
deps = ["//:gtest"],
)
py_test(
name = "googletest-failfast-unittest",
size = "medium",
srcs = ["googletest-failfast-unittest.py"],
data = [":googletest-failfast-unittest_"],
deps = [":gtest_test_utils"],
)
cc_binary(
name = "googletest-filter-unittest_",
testonly = 1,
srcs = ["googletest-filter-unittest_.cc"],
......
......@@ -85,6 +85,8 @@ class GTestEnvVarTest(gtest_test_utils.TestCase):
TestFlag('break_on_failure', '1', '0')
TestFlag('color', 'yes', 'auto')
SetEnvVar('TESTBRIDGE_TEST_RUNNER_FAIL_FAST', None) # For 'fail_fast' test
TestFlag('fail_fast', '1', '0')
TestFlag('filter', 'FooTest.Bar', '*')
SetEnvVar('XML_OUTPUT_FILE', None) # For 'output' test
TestFlag('output', 'xml:tmp/foo.xml', '')
......
......@@ -72,6 +72,11 @@ void PrintFlag(const char* flag) {
return;
}
if (strcmp(flag, "fail_fast") == 0) {
cout << GTEST_FLAG(fail_fast);
return;
}
if (strcmp(flag, "filter") == 0) {
cout << GTEST_FLAG(filter);
return;
......
// Copyright 2005, Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Unit test for Google Test test filters.
//
// A user can specify which test(s) in a Google Test program to run via
// either the GTEST_FILTER environment variable or the --gtest_filter
// flag. This is used for testing such functionality.
//
// The program will be invoked from a Python unit test. Don't run it
// directly.
#include "gtest/gtest.h"
namespace {
// Test HasFixtureTest.
class HasFixtureTest : public testing::Test {};
TEST_F(HasFixtureTest, Test0) {}
TEST_F(HasFixtureTest, Test1) { FAIL() << "Expected failure."; }
TEST_F(HasFixtureTest, Test2) { FAIL() << "Expected failure."; }
TEST_F(HasFixtureTest, Test3) { FAIL() << "Expected failure."; }
TEST_F(HasFixtureTest, Test4) { FAIL() << "Expected failure."; }
// Test HasSimpleTest.
TEST(HasSimpleTest, Test0) {}
TEST(HasSimpleTest, Test1) { FAIL() << "Expected failure."; }
TEST(HasSimpleTest, Test2) { FAIL() << "Expected failure."; }
TEST(HasSimpleTest, Test3) { FAIL() << "Expected failure."; }
TEST(HasSimpleTest, Test4) { FAIL() << "Expected failure."; }
// Test HasDisabledTest.
TEST(HasDisabledTest, Test0) {}
TEST(HasDisabledTest, DISABLED_Test1) { FAIL() << "Expected failure."; }
TEST(HasDisabledTest, Test2) { FAIL() << "Expected failure."; }
TEST(HasDisabledTest, Test3) { FAIL() << "Expected failure."; }
TEST(HasDisabledTest, Test4) { FAIL() << "Expected failure."; }
// Test HasDeathTest
TEST(HasDeathTest, Test0) { EXPECT_DEATH_IF_SUPPORTED(exit(1), ".*"); }
TEST(HasDeathTest, Test1) {
EXPECT_DEATH_IF_SUPPORTED(FAIL() << "Expected failure.", ".*");
}
TEST(HasDeathTest, Test2) {
EXPECT_DEATH_IF_SUPPORTED(FAIL() << "Expected failure.", ".*");
}
TEST(HasDeathTest, Test3) {
EXPECT_DEATH_IF_SUPPORTED(FAIL() << "Expected failure.", ".*");
}
TEST(HasDeathTest, Test4) {
EXPECT_DEATH_IF_SUPPORTED(FAIL() << "Expected failure.", ".*");
}
// Test DISABLED_HasDisabledSuite
TEST(DISABLED_HasDisabledSuite, Test0) {}
TEST(DISABLED_HasDisabledSuite, Test1) { FAIL() << "Expected failure."; }
TEST(DISABLED_HasDisabledSuite, Test2) { FAIL() << "Expected failure."; }
TEST(DISABLED_HasDisabledSuite, Test3) { FAIL() << "Expected failure."; }
TEST(DISABLED_HasDisabledSuite, Test4) { FAIL() << "Expected failure."; }
// Test HasParametersTest
class HasParametersTest : public testing::TestWithParam<int> {};
TEST_P(HasParametersTest, Test1) { FAIL() << "Expected failure."; }
TEST_P(HasParametersTest, Test2) { FAIL() << "Expected failure."; }
INSTANTIATE_TEST_SUITE_P(HasParametersSuite, HasParametersTest,
testing::Values(1, 2));
class MyTestListener : public ::testing::EmptyTestEventListener {
void OnTestSuiteStart(const ::testing::TestSuite& test_suite) override {
printf("We are in OnTestSuiteStart of %s.\n", test_suite.name());
}
void OnTestStart(const ::testing::TestInfo& test_info) override {
printf("We are in OnTestStart of %s.%s.\n", test_info.test_suite_name(),
test_info.name());
}
void OnTestPartResult(
const ::testing::TestPartResult& test_part_result) override {
printf("We are in OnTestPartResult %s:%d.\n", test_part_result.file_name(),
test_part_result.line_number());
}
void OnTestEnd(const ::testing::TestInfo& test_info) override {
printf("We are in OnTestEnd of %s.%s.\n", test_info.test_suite_name(),
test_info.name());
}
void OnTestSuiteEnd(const ::testing::TestSuite& test_suite) override {
printf("We are in OnTestSuiteEnd of %s.\n", test_suite.name());
}
};
TEST(HasSkipTest, Test0) { SUCCEED() << "Expected success."; }
TEST(HasSkipTest, Test1) { GTEST_SKIP() << "Expected skip."; }
TEST(HasSkipTest, Test2) { FAIL() << "Expected failure."; }
TEST(HasSkipTest, Test3) { FAIL() << "Expected failure."; }
TEST(HasSkipTest, Test4) { FAIL() << "Expected failure."; }
} // namespace
int main(int argc, char **argv) {
::testing::InitGoogleTest(&argc, argv);
::testing::UnitTest::GetInstance()->listeners().Append(new MyTestListener());
return RUN_ALL_TESTS();
}
......@@ -37,21 +37,22 @@
// code once "gtest.h" has been #included.
// Do not move it after other gtest #includes.
TEST(CommandLineFlagsTest, CanBeAccessedInCodeOnceGTestHIsIncluded) {
bool dummy = testing::GTEST_FLAG(also_run_disabled_tests)
|| testing::GTEST_FLAG(break_on_failure)
|| testing::GTEST_FLAG(catch_exceptions)
|| testing::GTEST_FLAG(color) != "unknown"
|| testing::GTEST_FLAG(filter) != "unknown"
|| testing::GTEST_FLAG(list_tests)
|| testing::GTEST_FLAG(output) != "unknown"
|| testing::GTEST_FLAG(print_time)
|| testing::GTEST_FLAG(random_seed)
|| testing::GTEST_FLAG(repeat) > 0
|| testing::GTEST_FLAG(show_internal_stack_frames)
|| testing::GTEST_FLAG(shuffle)
|| testing::GTEST_FLAG(stack_trace_depth) > 0
|| testing::GTEST_FLAG(stream_result_to) != "unknown"
|| testing::GTEST_FLAG(throw_on_failure);
bool dummy = testing::GTEST_FLAG(also_run_disabled_tests) ||
testing::GTEST_FLAG(break_on_failure) ||
testing::GTEST_FLAG(catch_exceptions) ||
testing::GTEST_FLAG(color) != "unknown" ||
testing::GTEST_FLAG(fail_fast) ||
testing::GTEST_FLAG(filter) != "unknown" ||
testing::GTEST_FLAG(list_tests) ||
testing::GTEST_FLAG(output) != "unknown" ||
testing::GTEST_FLAG(print_time) ||
testing::GTEST_FLAG(random_seed) ||
testing::GTEST_FLAG(repeat) > 0 ||
testing::GTEST_FLAG(show_internal_stack_frames) ||
testing::GTEST_FLAG(shuffle) ||
testing::GTEST_FLAG(stack_trace_depth) > 0 ||
testing::GTEST_FLAG(stream_result_to) != "unknown" ||
testing::GTEST_FLAG(throw_on_failure);
EXPECT_TRUE(dummy || !dummy); // Suppresses warning that dummy is unused.
}
......@@ -202,6 +203,7 @@ using testing::GTEST_FLAG(break_on_failure);
using testing::GTEST_FLAG(catch_exceptions);
using testing::GTEST_FLAG(color);
using testing::GTEST_FLAG(death_test_use_fork);
using testing::GTEST_FLAG(fail_fast);
using testing::GTEST_FLAG(filter);
using testing::GTEST_FLAG(list_tests);
using testing::GTEST_FLAG(output);
......@@ -1598,6 +1600,7 @@ class GTestFlagSaverTest : public Test {
GTEST_FLAG(catch_exceptions) = false;
GTEST_FLAG(death_test_use_fork) = false;
GTEST_FLAG(color) = "auto";
GTEST_FLAG(fail_fast) = false;
GTEST_FLAG(filter) = "";
GTEST_FLAG(list_tests) = false;
GTEST_FLAG(output) = "";
......@@ -1625,6 +1628,7 @@ class GTestFlagSaverTest : public Test {
EXPECT_FALSE(GTEST_FLAG(catch_exceptions));
EXPECT_STREQ("auto", GTEST_FLAG(color).c_str());
EXPECT_FALSE(GTEST_FLAG(death_test_use_fork));
EXPECT_FALSE(GTEST_FLAG(fail_fast));
EXPECT_STREQ("", GTEST_FLAG(filter).c_str());
EXPECT_FALSE(GTEST_FLAG(list_tests));
EXPECT_STREQ("", GTEST_FLAG(output).c_str());
......@@ -1641,6 +1645,7 @@ class GTestFlagSaverTest : public Test {
GTEST_FLAG(catch_exceptions) = true;
GTEST_FLAG(color) = "no";
GTEST_FLAG(death_test_use_fork) = true;
GTEST_FLAG(fail_fast) = true;
GTEST_FLAG(filter) = "abc";
GTEST_FLAG(list_tests) = true;
GTEST_FLAG(output) = "xml:foo.xml";
......@@ -5495,10 +5500,12 @@ TEST_F(SetUpTestSuiteTest, TestSetupTestSuite2) {
// The Flags struct stores a copy of all Google Test flags.
struct Flags {
// Constructs a Flags struct where each flag has its default value.
Flags() : also_run_disabled_tests(false),
Flags()
: also_run_disabled_tests(false),
break_on_failure(false),
catch_exceptions(false),
death_test_use_fork(false),
fail_fast(false),
filter(""),
list_tests(false),
output(""),
......@@ -5544,6 +5551,14 @@ struct Flags {
return flags;
}
// Creates a Flags struct where the gtest_fail_fast flag has
// the given value.
static Flags FailFast(bool fail_fast) {
Flags flags;
flags.fail_fast = fail_fast;
return flags;
}
// Creates a Flags struct where the gtest_filter flag has the given
// value.
static Flags Filter(const char* filter) {
......@@ -5629,6 +5644,7 @@ struct Flags {
bool break_on_failure;
bool catch_exceptions;
bool death_test_use_fork;
bool fail_fast;
const char* filter;
bool list_tests;
const char* output;
......@@ -5650,6 +5666,7 @@ class ParseFlagsTest : public Test {
GTEST_FLAG(break_on_failure) = false;
GTEST_FLAG(catch_exceptions) = false;
GTEST_FLAG(death_test_use_fork) = false;
GTEST_FLAG(fail_fast) = false;
GTEST_FLAG(filter) = "";
GTEST_FLAG(list_tests) = false;
GTEST_FLAG(output) = "";
......@@ -5680,6 +5697,7 @@ class ParseFlagsTest : public Test {
EXPECT_EQ(expected.break_on_failure, GTEST_FLAG(break_on_failure));
EXPECT_EQ(expected.catch_exceptions, GTEST_FLAG(catch_exceptions));
EXPECT_EQ(expected.death_test_use_fork, GTEST_FLAG(death_test_use_fork));
EXPECT_EQ(expected.fail_fast, GTEST_FLAG(fail_fast));
EXPECT_STREQ(expected.filter, GTEST_FLAG(filter).c_str());
EXPECT_EQ(expected.list_tests, GTEST_FLAG(list_tests));
EXPECT_STREQ(expected.output, GTEST_FLAG(output).c_str());
......@@ -5766,6 +5784,15 @@ TEST_F(ParseFlagsTest, NoFlag) {
GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags(), false);
}
// Tests parsing --gtest_fail_fast.
TEST_F(ParseFlagsTest, FailFast) {
const char* argv[] = {"foo.exe", "--gtest_fail_fast", nullptr};
const char* argv2[] = {"foo.exe", nullptr};
GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::FailFast(true), false);
}
// Tests parsing a bad --gtest_filter flag.
TEST_F(ParseFlagsTest, FilterBad) {
const char* argv[] = {"foo.exe", "--gtest_filter", nullptr};
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment