Skip to content

G
googletest
Commit fe186c38 by shiqian
Options

Implements --gtest_also_run_disabled_tests. By Eric Roman.

parent 53e0dc40
Showing with 259 additions and 17 deletions
src/gtest-internal-inl.h
......@@ -63,6 +63,7 @@ namespace testing {
// We don't want the users to modify these flags in the code, but want
// Google Test's own unit tests to be able to access them. Therefore we
// declare them here as opposed to in gtest.h.
GTEST_DECLARE_bool_(also_run_disabled_tests);
GTEST_DECLARE_bool_(break_on_failure);
GTEST_DECLARE_bool_(catch_exceptions);
GTEST_DECLARE_string_(color);
......@@ -72,8 +73,8 @@ GTEST_DECLARE_bool_(list_tests);
GTEST_DECLARE_string_(output);
GTEST_DECLARE_bool_(print_time);
GTEST_DECLARE_int32_(repeat);
GTEST_DECLARE_int32_(stack_trace_depth);
GTEST_DECLARE_bool_(show_internal_stack_frames);
GTEST_DECLARE_int32_(stack_trace_depth);
namespace internal {
......@@ -82,6 +83,7 @@ namespace internal {
extern const TypeId kTestTypeIdInGoogleTest;
// Names of the flags (needed for parsing Google Test flags).
const char kAlsoRunDisabledTestsFlag[] = "also_run_disabled_tests";
const char kBreakOnFailureFlag[] = "break_on_failure";
const char kCatchExceptionsFlag[] = "catch_exceptions";
const char kColorFlag[] = "color";
......@@ -97,6 +99,7 @@ class GTestFlagSaver {
public:
// The c'tor.
GTestFlagSaver() {
also_run_disabled_tests_ = GTEST_FLAG(also_run_disabled_tests);
break_on_failure_ = GTEST_FLAG(break_on_failure);
catch_exceptions_ = GTEST_FLAG(catch_exceptions);
color_ = GTEST_FLAG(color);
......@@ -112,6 +115,7 @@ class GTestFlagSaver {
// The d'tor is not virtual. DO NOT INHERIT FROM THIS CLASS.
~GTestFlagSaver() {
GTEST_FLAG(also_run_disabled_tests) = also_run_disabled_tests_;
GTEST_FLAG(break_on_failure) = break_on_failure_;
GTEST_FLAG(catch_exceptions) = catch_exceptions_;
GTEST_FLAG(color) = color_;
......@@ -126,6 +130,7 @@ class GTestFlagSaver {
}
private:
// Fields for saving the original values of flags.
bool also_run_disabled_tests_;
bool break_on_failure_;
bool catch_exceptions_;
String color_;
......
src/gtest.cc
......@@ -154,6 +154,11 @@ const char kStackTraceMarker[] = "\nStack trace:\n";
} // namespace internal
GTEST_DEFINE_bool_(
also_run_disabled_tests,
internal::BoolFromGTestEnv("also_run_disabled_tests", false),
"Run disabled tests too, in addition to the tests normally being run.");
GTEST_DEFINE_bool_(
break_on_failure,
internal::BoolFromGTestEnv("break_on_failure", false),
"True iff a failed assertion should be a debugger break-point.");
......@@ -1610,7 +1615,7 @@ bool String::CaseInsensitiveWideCStringEquals(const wchar_t* lhs,
right = towlower(*rhs++);
} while (left && left == right);
return left == right;
#endif // OS selector
#endif // OS selector
}
// Constructs a String by copying a given number of chars from a
......@@ -2736,7 +2741,7 @@ void PrettyUnitTestResultPrinter::OnUnitTestEnd(
}
int num_disabled = impl->disabled_test_count();
if (num_disabled) {
if (num_disabled && !GTEST_FLAG(also_run_disabled_tests)) {
if (!num_failures) {
printf("\n"); // Add a spacer if no FAILURE banner is displayed.
}
......@@ -3602,7 +3607,8 @@ int UnitTestImpl::FilterTests() {
kDisableTestFilter);
test_info->impl()->set_is_disabled(is_disabled);
const bool should_run = !is_disabled &&
const bool should_run =
(GTEST_FLAG(also_run_disabled_tests) || !is_disabled) &&
internal::UnitTestOptions::FilterMatchesTest(test_case_name,
test_name);
test_info->impl()->set_should_run(should_run);
......@@ -3860,7 +3866,9 @@ void ParseGoogleTestFlagsOnlyImpl(int* argc, CharType** argv) {
using internal::ParseStringFlag;
// Do we see a Google Test flag?
if (ParseBoolFlag(arg, kBreakOnFailureFlag,
if (ParseBoolFlag(arg, kAlsoRunDisabledTestsFlag,
&GTEST_FLAG(also_run_disabled_tests)) ||
ParseBoolFlag(arg, kBreakOnFailureFlag,
&GTEST_FLAG(break_on_failure)) ||
ParseBoolFlag(arg, kCatchExceptionsFlag,
&GTEST_FLAG(catch_exceptions)) ||
......
test/gtest_filter_unittest.py
......@@ -40,12 +40,11 @@ environments and command line flags.
__author__ = 'wan@google.com (Zhanyong Wan)'
import gtest_test_utils
import os
import re
import sets
import sys
import unittest
import gtest_test_utils
# Constants.
......@@ -55,6 +54,9 @@ FILTER_ENV_VAR = 'GTEST_FILTER'
# The command line flag for specifying the test filters.
FILTER_FLAG = 'gtest_filter'
# The command line flag for including disabled tests.
ALSO_RUN_DISABED_TESTS_FLAG = 'gtest_also_run_disabled_tests'
# Command to run the gtest_filter_unittest_ program.
COMMAND = os.path.join(gtest_test_utils.GetBuildDir(),
'gtest_filter_unittest_')
......@@ -80,7 +82,17 @@ PARAM_TESTS = [
'SeqQ/ParamTest.TestY/1',
]
ALL_TESTS = [
DISABLED_TESTS = [
'BarTest.DISABLED_TestFour',
'BarTest.DISABLED_TestFive',
'BazTest.DISABLED_TestC',
'DISABLED_FoobarTest.Test1',
'DISABLED_FoobarTest.DISABLED_Test2',
'DISABLED_FoobarbazTest.TestA',
]
# All the non-disabled tests.
ACTIVE_TESTS = [
'FooTest.Abc',
'FooTest.Xyz',
......@@ -176,6 +188,18 @@ class GTestFilterUnitTest(unittest.TestCase):
tests_run = Run(command)
self.AssertSetEqual(tests_run, tests_to_run)
def RunAndVerifyAllowingDisabled(self, gtest_filter, tests_to_run):
"""Runs gtest_flag_unittest_ with the given filter, and enables
disabled tests. Verifies that the right set of tests were run.
"""
# Construct the command line.
command = '%s --%s' % (COMMAND, ALSO_RUN_DISABED_TESTS_FLAG)
if gtest_filter is not None:
command = '%s --%s=%s' % (command, FILTER_FLAG, gtest_filter)
tests_run = Run(command)
self.AssertSetEqual(tests_run, tests_to_run)
def setUp(self):
"""Sets up test case. Determines whether value-parameterized tests are
enabled in the binary and sets flags accordingly.
......@@ -188,7 +212,7 @@ class GTestFilterUnitTest(unittest.TestCase):
def testDefaultBehavior(self):
"""Tests the behavior of not specifying the filter."""
self.RunAndVerify(None, ALL_TESTS)
self.RunAndVerify(None, ACTIVE_TESTS)
def testEmptyFilter(self):
"""Tests an empty filter."""
......@@ -199,28 +223,62 @@ class GTestFilterUnitTest(unittest.TestCase):
"""Tests a filter that matches nothing."""
self.RunAndVerify('BadFilter', [])
self.RunAndVerifyAllowingDisabled('BadFilter', [])
def testFullName(self):
"""Tests filtering by full name."""
self.RunAndVerify('FooTest.Xyz', ['FooTest.Xyz'])
self.RunAndVerifyAllowingDisabled('FooTest.Xyz', ['FooTest.Xyz'])
def testUniversalFilters(self):
"""Tests filters that match everything."""
self.RunAndVerify('*', ALL_TESTS)
self.RunAndVerify('*.*', ALL_TESTS)
self.RunAndVerify('*', ACTIVE_TESTS)
self.RunAndVerify('*.*', ACTIVE_TESTS)
self.RunAndVerifyAllowingDisabled('*', ACTIVE_TESTS + DISABLED_TESTS)
self.RunAndVerifyAllowingDisabled('*.*', ACTIVE_TESTS + DISABLED_TESTS)
def testFilterByTestCase(self):
"""Tests filtering by test case name."""
self.RunAndVerify('FooTest.*', ['FooTest.Abc', 'FooTest.Xyz'])
BAZ_TESTS = ['BazTest.TestOne', 'BazTest.TestA', 'BazTest.TestB']
self.RunAndVerify('BazTest.*', BAZ_TESTS)
self.RunAndVerifyAllowingDisabled('BazTest.*',
BAZ_TESTS + ['BazTest.DISABLED_TestC'])
def testFilterByTest(self):
"""Tests filtering by test name."""
self.RunAndVerify('*.TestOne', ['BarTest.TestOne', 'BazTest.TestOne'])
def testFilterDisabledTests(self):
"""Select only the disabled tests to run."""
self.RunAndVerify('DISABLED_FoobarTest.Test1', [])
self.RunAndVerifyAllowingDisabled('DISABLED_FoobarTest.Test1',
['DISABLED_FoobarTest.Test1'])
self.RunAndVerify('*DISABLED_*', [])
self.RunAndVerifyAllowingDisabled('*DISABLED_*', DISABLED_TESTS)
self.RunAndVerify('*.DISABLED_*', [])
self.RunAndVerifyAllowingDisabled('*.DISABLED_*', [
'BarTest.DISABLED_TestFour',
'BarTest.DISABLED_TestFive',
'BazTest.DISABLED_TestC',
'DISABLED_FoobarTest.DISABLED_Test2',
])
self.RunAndVerify('DISABLED_*', [])
self.RunAndVerifyAllowingDisabled('DISABLED_*', [
'DISABLED_FoobarTest.Test1',
'DISABLED_FoobarTest.DISABLED_Test2',
'DISABLED_FoobarbazTest.TestA',
])
def testWildcardInTestCaseName(self):
"""Tests using wildcard in the test case name."""
......@@ -231,7 +289,7 @@ class GTestFilterUnitTest(unittest.TestCase):
'BazTest.TestOne',
'BazTest.TestA',
'BazTest.TestB',] + PARAM_TESTS)
'BazTest.TestB' ] + PARAM_TESTS)
def testWildcardInTestName(self):
"""Tests using wildcard in the test name."""
......
test/gtest_filter_unittest_.cc
......@@ -67,6 +67,13 @@ TEST(BarTest, TestTwo) {
TEST(BarTest, TestThree) {
}
TEST(BarTest, DISABLED_TestFour) {
FAIL() << "Expected failure.";
}
TEST(BarTest, DISABLED_TestFive) {
FAIL() << "Expected failure.";
}
// Test case BazTest.
......@@ -80,6 +87,26 @@ TEST(BazTest, TestA) {
TEST(BazTest, TestB) {
}
TEST(BazTest, DISABLED_TestC) {
FAIL() << "Expected failure.";
}
// Test case FoobarTest
TEST(DISABLED_FoobarTest, Test1) {
FAIL() << "Expected failure.";
}
TEST(DISABLED_FoobarTest, DISABLED_Test2) {
FAIL() << "Expected failure.";
}
// Test case FoobarbazTest
TEST(DISABLED_FoobarbazTest, TestA) {
FAIL() << "Expected failure.";
}
#ifdef GTEST_HAS_PARAM_TEST
class ParamTest : public testing::TestWithParam<int> {
};
......
test/gtest_output_test.py
......@@ -40,12 +40,12 @@ SYNOPSIS
__author__ = 'wan@google.com (Zhanyong Wan)'
import gtest_test_utils
import os
import re
import string
import sys
import unittest
import gtest_test_utils
# The flag for generating the golden file
......@@ -64,10 +64,13 @@ PROGRAM_PATH = os.path.join(gtest_test_utils.GetBuildDir(), PROGRAM)
COMMAND_WITH_COLOR = PROGRAM_PATH + ' --gtest_color=yes'
COMMAND_WITH_TIME = (PROGRAM_PATH + ' --gtest_print_time '
+ '--gtest_filter="FatalFailureTest.*:LoggingTest.*"')
COMMAND_WITH_DISABLED = (PROGRAM_PATH + ' --gtest_also_run_disabled_tests '
+ '--gtest_filter="*DISABLED_*"')
GOLDEN_PATH = os.path.join(gtest_test_utils.GetSourceDir(),
GOLDEN_NAME)
def ToUnixLineEnding(s):
"""Changes all Windows/Mac line endings in s to UNIX line endings."""
......@@ -191,7 +194,8 @@ def GetCommandOutput(cmd):
class GTestOutputTest(unittest.TestCase):
def testOutput(self):
output = (GetCommandOutput(COMMAND_WITH_COLOR) +
GetCommandOutput(COMMAND_WITH_TIME))
GetCommandOutput(COMMAND_WITH_TIME) +
GetCommandOutput(COMMAND_WITH_DISABLED))
golden_file = open(GOLDEN_PATH, 'rb')
golden = golden_file.read()
golden_file.close()
......@@ -206,7 +210,8 @@ class GTestOutputTest(unittest.TestCase):
if __name__ == '__main__':
if sys.argv[1:] == [GENGOLDEN_FLAG]:
output = (GetCommandOutput(COMMAND_WITH_COLOR) +
GetCommandOutput(COMMAND_WITH_TIME))
GetCommandOutput(COMMAND_WITH_TIME) +
GetCommandOutput(COMMAND_WITH_DISABLED))
golden_file = open(GOLDEN_PATH, 'wb')
golden_file.write(output)
golden_file.close()
......
test/gtest_output_test_.cc
......@@ -212,6 +212,14 @@ TEST(SCOPED_TRACETest, CanBeRepeated) {
<< "trace point A, B, and D.";
}
TEST(DisabledTestsWarningTest,
DISABLED_AlsoRunDisabledTestsFlagSuppressesWarning) {
// This test body is intentionally empty. Its sole purpose is for
// verifying that the --gtest_also_run_disabled_tests flag
// suppresses the "YOU HAVE 12 DISABLED TESTS" warning at the end of
// the test output.
}
// Tests using assertions outside of TEST and TEST_F.
//
// This function creates two failures intentionally.
......
test/gtest_output_test_golden_lin.txt
......@@ -534,7 +534,9 @@ Expected fatal failure.
[ FAILED ] ExpectFailureTest.ExpectNonFatalFailureOnAllThreads
33 FAILED TESTS
The non-test part of the code is expected to have 2 failures.
 YOU HAVE 1 DISABLED TEST
The non-test part of the code is expected to have 2 failures.
gtest_output_test_.cc:#: Failure
Value of: false
......@@ -604,3 +606,36 @@ Expected fatal failure.
[ FAILED ] LoggingTest.InterleavingLoggingAndAssertions
4 FAILED TESTS
YOU HAVE 1 DISABLED TEST
The non-test part of the code is expected to have 2 failures.
gtest_output_test_.cc:#: Failure
Value of: false
Actual: false
Expected: true
gtest_output_test_.cc:#: Failure
Value of: 3
Expected: 2
Note: Google Test filter = *DISABLED_*
[==========] Running 1 test from 1 test case.
[----------] Global test environment set-up.
FooEnvironment::SetUp() called.
BarEnvironment::SetUp() called.
[----------] 1 test from DisabledTestsWarningTest
[ RUN ] DisabledTestsWarningTest.DISABLED_AlsoRunDisabledTestsFlagSuppressesWarning
[ OK ] DisabledTestsWarningTest.DISABLED_AlsoRunDisabledTestsFlagSuppressesWarning
[----------] Global test environment tear-down
BarEnvironment::TearDown() called.
gtest_output_test_.cc:#: Failure
Failed
Expected non-fatal failure.
FooEnvironment::TearDown() called.
gtest_output_test_.cc:#: Failure
Failed
Expected fatal failure.
[==========] 1 test from 1 test case ran.
[ PASSED ] 1 test.
[ FAILED ] 0 tests, listed below:
0 FAILED TESTS
test/gtest_output_test_golden_win.txt
......@@ -481,6 +481,8 @@ Expected fatal failure.
[ FAILED ] ExpectFailureTest.ExpectNonFatalFailureOnAllThreads
36 FAILED TESTS
YOU HAVE 1 DISABLED TEST
The non-test part of the code is expected to have 2 failures.
gtest_output_test_.cc:#: error: Value of: false
......@@ -542,3 +544,32 @@ Expected fatal failure.
[ FAILED ] LoggingTest.InterleavingLoggingAndAssertions
4 FAILED TESTS
YOU HAVE 1 DISABLED TEST
The non-test part of the code is expected to have 2 failures.
gtest_output_test_.cc:#: error: Value of: false
Actual: false
Expected: true
gtest_output_test_.cc:#: error: Value of: 3
Expected: 2
Note: Google Test filter = *DISABLED_*
[==========] Running 1 test from 1 test case.
[----------] Global test environment set-up.
FooEnvironment::SetUp() called.
BarEnvironment::SetUp() called.
[----------] 1 test from DisabledTestsWarningTest
[ RUN ] DisabledTestsWarningTest.DISABLED_AlsoRunDisabledTestsFlagSuppressesWarning
[ OK ] DisabledTestsWarningTest.DISABLED_AlsoRunDisabledTestsFlagSuppressesWarning
[----------] Global test environment tear-down
BarEnvironment::TearDown() called.
gtest_output_test_.cc:#: error: Failed
Expected non-fatal failure.
FooEnvironment::TearDown() called.
gtest_output_test_.cc:#: error: Failed
Expected fatal failure.
[==========] 1 test from 1 test case ran.
[ PASSED ] 1 test.
[ FAILED ] 0 tests, listed below:
0 FAILED TESTS
test/gtest_unittest.cc
......@@ -84,6 +84,7 @@ using testing::AssertionResult;
using testing::AssertionSuccess;
using testing::DoubleLE;
using testing::FloatLE;
using testing::GTEST_FLAG(also_run_disabled_tests);
using testing::GTEST_FLAG(break_on_failure);
using testing::GTEST_FLAG(catch_exceptions);
using testing::GTEST_FLAG(death_test_use_fork);
......@@ -1128,6 +1129,7 @@ class GTestFlagSaverTest : public Test {
static void SetUpTestCase() {
saver_ = new GTestFlagSaver;
GTEST_FLAG(also_run_disabled_tests) = false;
GTEST_FLAG(break_on_failure) = false;
GTEST_FLAG(catch_exceptions) = false;
GTEST_FLAG(death_test_use_fork) = false;
......@@ -1149,6 +1151,7 @@ class GTestFlagSaverTest : public Test {
// Verifies that the Google Test flags have their default values, and then
// modifies each of them.
void VerifyAndModifyFlags() {
EXPECT_FALSE(GTEST_FLAG(also_run_disabled_tests));
EXPECT_FALSE(GTEST_FLAG(break_on_failure));
EXPECT_FALSE(GTEST_FLAG(catch_exceptions));
EXPECT_STREQ("auto", GTEST_FLAG(color).c_str());
......@@ -1159,6 +1162,7 @@ class GTestFlagSaverTest : public Test {
EXPECT_FALSE(GTEST_FLAG(print_time));
EXPECT_EQ(1, GTEST_FLAG(repeat));
GTEST_FLAG(also_run_disabled_tests) = true;
GTEST_FLAG(break_on_failure) = true;
GTEST_FLAG(catch_exceptions) = true;
GTEST_FLAG(color) = "no";
......@@ -4067,7 +4071,8 @@ TEST_F(SetUpTestCaseTest, Test2) {
// The Flags struct stores a copy of all Google Test flags.
struct Flags {
// Constructs a Flags struct where each flag has its default value.
Flags() : break_on_failure(false),
Flags() : also_run_disabled_tests(false),
break_on_failure(false),
catch_exceptions(false),
death_test_use_fork(false),
filter(""),
......@@ -4078,6 +4083,14 @@ struct Flags {
// Factory methods.
// Creates a Flags struct where the gtest_also_run_disabled_tests flag has
// the given value.
static Flags AlsoRunDisabledTests(bool also_run_disabled_tests) {
Flags flags;
flags.also_run_disabled_tests = also_run_disabled_tests;
return flags;
}
// Creates a Flags struct where the gtest_break_on_failure flag has
// the given value.
static Flags BreakOnFailure(bool break_on_failure) {
......@@ -4143,6 +4156,7 @@ struct Flags {
}
// These fields store the flag values.
bool also_run_disabled_tests;
bool break_on_failure;
bool catch_exceptions;
bool death_test_use_fork;
......@@ -4158,6 +4172,7 @@ class InitGoogleTestTest : public Test {
protected:
// Clears the flags before each test.
virtual void SetUp() {
GTEST_FLAG(also_run_disabled_tests) = false;
GTEST_FLAG(break_on_failure) = false;
GTEST_FLAG(catch_exceptions) = false;
GTEST_FLAG(death_test_use_fork) = false;
......@@ -4181,6 +4196,8 @@ class InitGoogleTestTest : public Test {
// Verifies that the flag values match the expected values.
static void CheckFlags(const Flags& expected) {
EXPECT_EQ(expected.also_run_disabled_tests,
GTEST_FLAG(also_run_disabled_tests));
EXPECT_EQ(expected.break_on_failure, GTEST_FLAG(break_on_failure));
EXPECT_EQ(expected.catch_exceptions, GTEST_FLAG(catch_exceptions));
EXPECT_EQ(expected.death_test_use_fork, GTEST_FLAG(death_test_use_fork));
......@@ -4687,6 +4704,54 @@ TEST_F(InitGoogleTestTest, Repeat) {
TEST_PARSING_FLAGS(argv, argv2, Flags::Repeat(1000));
}
// Tests having a --gtest_also_run_disabled_tests flag
TEST_F(InitGoogleTestTest, AlsoRunDisabledTestsFlag) {
const char* argv[] = {
"foo.exe",
"--gtest_also_run_disabled_tests",
NULL
};
const char* argv2[] = {
"foo.exe",
NULL
};
TEST_PARSING_FLAGS(argv, argv2, Flags::AlsoRunDisabledTests(true));
}
// Tests having a --gtest_also_run_disabled_tests flag with a "true" value
TEST_F(InitGoogleTestTest, AlsoRunDisabledTestsTrue) {
const char* argv[] = {
"foo.exe",
"--gtest_also_run_disabled_tests=1",
NULL
};
const char* argv2[] = {
"foo.exe",
NULL
};
TEST_PARSING_FLAGS(argv, argv2, Flags::AlsoRunDisabledTests(true));
}
// Tests having a --gtest_also_run_disabled_tests flag with a "false" value
TEST_F(InitGoogleTestTest, AlsoRunDisabledTestsFalse) {
const char* argv[] = {
"foo.exe",
"--gtest_also_run_disabled_tests=0",
NULL
};
const char* argv2[] = {
"foo.exe",
NULL
};
TEST_PARSING_FLAGS(argv, argv2, Flags::AlsoRunDisabledTests(false));
}
#ifdef GTEST_OS_WINDOWS
// Tests parsing wide strings.
TEST_F(InitGoogleTestTest, WideStrings) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment