Skip to content

B
benchmark
Commit 1ce286f6 by Joao Paulo Magalhaes
Options

Avoid compiler-specific pragmas in result check macros.

- Epsilon is now understood as relative to expected value. - Improve error messages for epsilon checks.
parent da69e5de
Showing with 55 additions and 45 deletions
test/output_test.h
......@@ -148,40 +148,50 @@ struct ResultsCheckerEntry {
// checked.
size_t AddChecker(const char* bm_name, ResultsCheckFn fn);
#ifdef __clang__
/* NOTE: using , ## __VA_ARGS__ to deal with zero-args calls to
* variadic macros is not portable, but works in clang, gcc, msvc, icc.
* clang requires switching off compiler warnings for pedantic mode.
* @see http://stackoverflow.com/questions/32047685/variadic-macro-without-arguments */
# pragma clang diagnostic push
// warning: token pasting of ',' and __VA_ARGS__ is a GNU extension
# pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments"
#elif defined(__GNUC__)
/* GCC also issues a warning for zero-args calls to variadic macros.
* This warning is switched on with -pedantic and apparently there is no
* easy way to turn it off as with clang. But marking this as a system
* header works.
* @see https://gcc.gnu.org/onlinedocs/cpp/System-Headers.html
* @see http://stackoverflow.com/questions/35587137/ */
# pragma GCC system_header
#endif
#define _CHECK_RESULT_VALUE(entry, getfn, var_type, var_name, relationship, value, ...) \
CONCAT(CHECK_, relationship)(entry.getfn< var_type >(var_name), (value), ## __VA_ARGS__) \
<< "\n" << __FILE__ << ":" << __LINE__ << ": " \
<< entry.name << ": expected (" << #var_type << ")" \
<< var_name << "=" << entry.getfn< var_type >(var_name) \
<< " to be " #relationship " to " << (value);
#define CHECK_RESULT_VALUE(entry, var_type, var_name, relationship, value, ...) \
_CHECK_RESULT_VALUE(entry, GetAs, var_type, var_name, relationship, value, ## __VA_ARGS__)
#define CHECK_COUNTER_VALUE(entry, var_type, var_name, relationship, value, ...) \
_CHECK_RESULT_VALUE(entry, GetCounterAs, var_type, var_name, relationship, value, ## __VA_ARGS__)
#ifdef __clang__
# pragma clang diagnostic pop
#endif
//----------------------------------
// Macros to help in result checking. Do not use them with arguments causing
// side-effects.
#define _CHECK_RESULT_VALUE(entry, getfn, var_type, var_name, relationship, value) \
CONCAT(CHECK_, relationship) \
(entry.getfn< var_type >(var_name), (value)) << "\n" \
<< __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \
<< __FILE__ << ":" << __LINE__ << ": " \
<< "expected (" << #var_type << ")" << (var_name) \
<< "=" << (entry).getfn< var_type >(var_name) \
<< " to be " #relationship " to " << (value) << "\n"
// check with tolerance. eps_factor is the tolerance window, which will be
// interpreted relative to value.
#define _CHECK_RESULT_VALUE_EPS(entry, getfn, var_type, var_name, relationship, value, eps_factor) \
CONCAT(CHECK_, relationship) \
(entry.getfn< var_type >(var_name), (value), (eps_factor) * (value)) << "\n" \
<< __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \
<< __FILE__ << ":" << __LINE__ << ": " \
<< "expected (" << #var_type << ")" << (var_name) \
<< "=" << (entry).getfn< var_type >(var_name) \
<< " to be " #relationship " to " << (value) << "\n" \
<< __FILE__ << ":" << __LINE__ << ": " \
<< "with tolerance of " << (eps_factor) * (value) \
<< " (" << (eps_factor)*100. << "%), " \
<< "but delta was " << ((entry).getfn< var_type >(var_name) - (value)) \
<< " (" << (((entry).getfn< var_type >(var_name) - (value)) \
/ \
((value) > 1.e-5 || value < -1.e-5 ? value : 1.e-5)*100.) \
<< "%)"
#define CHECK_RESULT_VALUE(entry, var_type, var_name, relationship, value) \
_CHECK_RESULT_VALUE(entry, GetAs, var_type, var_name, relationship, value)
#define CHECK_COUNTER_VALUE(entry, var_type, var_name, relationship, value) \
_CHECK_RESULT_VALUE(entry, GetCounterAs, var_type, var_name, relationship, value)
#define CHECK_RESULT_VALUE_EPS(entry, var_name, relationship, value, eps_factor) \
_CHECK_RESULT_VALUE_EPS(entry, GetAs, double, var_name, relationship, value, eps_factor)
#define CHECK_COUNTER_VALUE_EPS(entry, var_name, relationship, value, eps_factor) \
_CHECK_RESULT_VALUE_EPS(entry, GetCounterAs, double, var_name, relationship, value, eps_factor)
#define CHECK_BENCHMARK_RESULTS(bm_name, checker_function) \
size_t CONCAT(dummy, __LINE__) = AddChecker(bm_name, checker_function)
......
test/user_counters_test.cc
......@@ -24,7 +24,7 @@ void BM_Counters_Simple(benchmark::State& state) {
state.counters["foo"] = 1;
state.counters["bar"] = 2 * state.iterations();
}
BENCHMARK(BM_Counters_Simple);//->ThreadRange(1, 32);
BENCHMARK(BM_Counters_Simple);
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"},
{"\"iterations\": %int,$", MR_Next},
......@@ -39,7 +39,7 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_Simple", [](ResultsCheckerEntry const& e) {
double its = e.GetAs< double >("iterations");
CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1);
// check that the value of bar is within 0.1% of the expected value
CHECK_COUNTER_VALUE(e, double, "bar", EQ_EPS, 2. * its, 0.001 * its);
CHECK_COUNTER_VALUE_EPS(e, "bar", EQ_EPS, 2.*its, 0.001);
});
// ========================================================================= //
......@@ -55,10 +55,10 @@ void BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) {
state.SetBytesProcessed(364);
state.SetItemsProcessed(150);
}
BENCHMARK(BM_Counters_WithBytesAndItemsPSec);//->ThreadRange(1, 32);
BENCHMARK(BM_Counters_WithBytesAndItemsPSec);
ADD_CASES(TC_ConsoleOut,
{{"^BM_Counters_WithBytesAndItemsPSec %console_report "
"bar=%hrfloat foo=%hrfloat +%floatB/s +%float items/s$"}});
"bar=%hrfloat foo=%hrfloat +%hrfloatB/s +%hrfloat items/s$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %int,$", MR_Next},
......@@ -73,12 +73,12 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_WithBytesAndItemsPSec\","
"%csv_bytes_items_report,%float,%float$"}});
CHECK_BENCHMARK_RESULTS("BM_Counters_WithBytesAndItemsPSec",
[](ResultsCheckerEntry const& e) {
double t = e.DurationCPUTime(); // this (and not real time) is the time used
CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1);
CHECK_COUNTER_VALUE(e, int, "bar", EQ, num_calls1);
// check that the values are within 0.1% of the expected values
double t = e.DurationCPUTime(); // this (and not real time) is the time used
CHECK_RESULT_VALUE(e, double, "bytes_per_second", EQ_EPS, 364. / t, 0.001 * t);
CHECK_RESULT_VALUE(e, double, "items_per_second", EQ_EPS, 150. / t, 0.001 * t);
CHECK_RESULT_VALUE_EPS(e, "bytes_per_second", EQ_EPS, 364./t, 0.001);
CHECK_RESULT_VALUE_EPS(e, "items_per_second", EQ_EPS, 150./t, 0.001);
});
// ========================================================================= //
......@@ -92,7 +92,7 @@ void BM_Counters_Rate(benchmark::State& state) {
state.counters["foo"] = bm::Counter{1, bm::Counter::kIsRate};
state.counters["bar"] = bm::Counter{2, bm::Counter::kIsRate};
}
BENCHMARK(BM_Counters_Rate);//->ThreadRange(1, 32);
BENCHMARK(BM_Counters_Rate);
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Rate %console_report bar=%hrfloat foo=%hrfloat$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"},
{"\"iterations\": %int,$", MR_Next},
......@@ -105,10 +105,10 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"},
ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Rate\",%csv_report,%float,%float$"}});
CHECK_BENCHMARK_RESULTS("BM_Counters_Rate",
[](ResultsCheckerEntry const& e) {
// check that the values are within 0.1% of the expected values
double t = e.DurationCPUTime(); // this (and not real time) is the time used
CHECK_COUNTER_VALUE(e, double, "foo", EQ_EPS, 5. / t, 0.001 * t);
CHECK_COUNTER_VALUE(e, double, "bar", EQ_EPS, 2. / t, 0.001 * t);
// check that the values are within 0.1% of the expected values
CHECK_COUNTER_VALUE_EPS(e, "foo", EQ_EPS, 1./t, 0.001);
CHECK_COUNTER_VALUE_EPS(e, "bar", EQ_EPS, 2./t, 0.001);
});
// ========================================================================= //
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment