Avoid compiler-specific pragmas in result check macros.

test/output_test.h

@@ -148,40 +148,50 @@ struct ResultsCheckerEntry {
 // checked.
 size_t AddChecker(const char* bm_name, ResultsCheckFn fn);
 
-#ifdef __clang__
-    /* NOTE: using , ## __VA_ARGS__ to deal with zero-args calls to
-     * variadic macros is not portable, but works in clang, gcc, msvc, icc.
-     * clang requires switching off compiler warnings for pedantic mode.
-     * @see http://stackoverflow.com/questions/32047685/variadic-macro-without-arguments */
-#   pragma clang diagnostic push
-    // warning: token pasting of ',' and __VA_ARGS__ is a GNU extension
-#   pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments"
-#elif defined(__GNUC__)
-    /* GCC also issues a warning for zero-args calls to variadic macros.
-     * This warning is switched on with -pedantic and apparently there is no
-     * easy way to turn it off as with clang. But marking this as a system
-     * header works.
-     * @see https://gcc.gnu.org/onlinedocs/cpp/System-Headers.html
-     * @see http://stackoverflow.com/questions/35587137/ */
-#   pragma GCC system_header
-#endif
-
-#define _CHECK_RESULT_VALUE(entry, getfn, var_type, var_name, relationship, value, ...) \
-    CONCAT(CHECK_, relationship)(entry.getfn< var_type >(var_name), (value), ## __VA_ARGS__) \
-                                << "\n" << __FILE__ << ":" << __LINE__ << ": "  \
-                                << entry.name << ": expected (" << #var_type << ")" \
-                                << var_name << "=" << entry.getfn< var_type >(var_name) \
-                                << " to be " #relationship " to " << (value);
-
-#define CHECK_RESULT_VALUE(entry, var_type, var_name, relationship, value, ...) \
-    _CHECK_RESULT_VALUE(entry, GetAs, var_type, var_name, relationship, value, ## __VA_ARGS__)
-
-#define CHECK_COUNTER_VALUE(entry, var_type, var_name, relationship, value, ...) \
-    _CHECK_RESULT_VALUE(entry, GetCounterAs, var_type, var_name, relationship, value, ## __VA_ARGS__)
-
-#ifdef __clang__
-#   pragma clang diagnostic pop
-#endif
+//----------------------------------
+// Macros to help in result checking. Do not use them with arguments causing
+// side-effects.
+
+#define _CHECK_RESULT_VALUE(entry, getfn, var_type, var_name, relationship, value) \
+    CONCAT(CHECK_, relationship)                                        \
+    (entry.getfn< var_type >(var_name), (value)) << "\n"                \
+    << __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n"     \
+    << __FILE__ << ":" << __LINE__ << ": "                              \
+    << "expected (" << #var_type << ")" << (var_name)                   \
+    << "=" << (entry).getfn< var_type >(var_name)                       \
+    << " to be " #relationship " to " << (value) << "\n"
+
+// check with tolerance. eps_factor is the tolerance window, which will be
+// interpreted relative to value.
+#define _CHECK_RESULT_VALUE_EPS(entry, getfn, var_type, var_name, relationship, value, eps_factor) \
+    CONCAT(CHECK_, relationship)                                        \
+    (entry.getfn< var_type >(var_name), (value), (eps_factor) * (value)) << "\n" \
+    << __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n"     \
+    << __FILE__ << ":" << __LINE__ << ": "                              \
+    << "expected (" << #var_type << ")" << (var_name)                   \
+    << "=" << (entry).getfn< var_type >(var_name)                       \
+    << " to be " #relationship " to " << (value) << "\n"                \
+    << __FILE__ << ":" << __LINE__ << ": "                              \
+    << "with tolerance of " << (eps_factor) * (value)                   \
+    << " (" << (eps_factor)*100. << "%), "                              \
+    << "but delta was " << ((entry).getfn< var_type >(var_name) - (value)) \
+    << " (" << (((entry).getfn< var_type >(var_name) - (value))         \
+               /                                                        \
+               ((value) > 1.e-5 || value < -1.e-5 ? value : 1.e-5)*100.) \
+    << "%)"
+
+#define CHECK_RESULT_VALUE(entry, var_type, var_name, relationship, value) \
+    _CHECK_RESULT_VALUE(entry, GetAs, var_type, var_name, relationship, value)
+
+#define CHECK_COUNTER_VALUE(entry, var_type, var_name, relationship, value) \
+    _CHECK_RESULT_VALUE(entry, GetCounterAs, var_type, var_name, relationship, value)
+
+#define CHECK_RESULT_VALUE_EPS(entry, var_name, relationship, value, eps_factor) \
+    _CHECK_RESULT_VALUE_EPS(entry, GetAs, double, var_name, relationship, value, eps_factor)
+
+#define CHECK_COUNTER_VALUE_EPS(entry, var_name, relationship, value, eps_factor) \
+    _CHECK_RESULT_VALUE_EPS(entry, GetCounterAs, double, var_name, relationship, value, eps_factor)
+
 
 #define CHECK_BENCHMARK_RESULTS(bm_name, checker_function)              \
     size_t CONCAT(dummy, __LINE__) = AddChecker(bm_name, checker_function)

test/user_counters_test.cc

@@ -24,7 +24,7 @@ void BM_Counters_Simple(benchmark::State& state) {
   state.counters["foo"] = 1;
   state.counters["bar"] = 2 * state.iterations();
 }
-BENCHMARK(BM_Counters_Simple);//->ThreadRange(1, 32);
+BENCHMARK(BM_Counters_Simple);
 ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}});
 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"},
                        {"\"iterations\": %int,$", MR_Next},
@@ -39,7 +39,7 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_Simple", [](ResultsCheckerEntry const& e) {
   double its = e.GetAs< double >("iterations");
   CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1);
   // check that the value of bar is within 0.1% of the expected value
-  CHECK_COUNTER_VALUE(e, double, "bar", EQ_EPS, 2. * its, 0.001 * its);
+  CHECK_COUNTER_VALUE_EPS(e, "bar", EQ_EPS, 2.*its, 0.001);
 });
 
 // ========================================================================= //
@@ -55,10 +55,10 @@ void BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) {
   state.SetBytesProcessed(364);
   state.SetItemsProcessed(150);
 }
-BENCHMARK(BM_Counters_WithBytesAndItemsPSec);//->ThreadRange(1, 32);
+BENCHMARK(BM_Counters_WithBytesAndItemsPSec);
 ADD_CASES(TC_ConsoleOut,
           {{"^BM_Counters_WithBytesAndItemsPSec %console_report "
-            "bar=%hrfloat foo=%hrfloat +%floatB/s +%float items/s$"}});
+            "bar=%hrfloat foo=%hrfloat +%hrfloatB/s +%hrfloat items/s$"}});
 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"},
                        {"\"iterations\": %int,$", MR_Next},
                        {"\"real_time\": %int,$", MR_Next},
@@ -73,12 +73,12 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_WithBytesAndItemsPSec\","
                        "%csv_bytes_items_report,%float,%float$"}});
 CHECK_BENCHMARK_RESULTS("BM_Counters_WithBytesAndItemsPSec",
                         [](ResultsCheckerEntry const& e) {
+  double t = e.DurationCPUTime(); // this (and not real time) is the time used
   CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1);
   CHECK_COUNTER_VALUE(e, int, "bar", EQ, num_calls1);
   // check that the values are within 0.1% of the expected values
-  double t = e.DurationCPUTime(); // this (and not real time) is the time used
-  CHECK_RESULT_VALUE(e, double, "bytes_per_second", EQ_EPS, 364. / t, 0.001 * t);
-  CHECK_RESULT_VALUE(e, double, "items_per_second", EQ_EPS, 150. / t, 0.001 * t);
+  CHECK_RESULT_VALUE_EPS(e, "bytes_per_second", EQ_EPS, 364./t, 0.001);
+  CHECK_RESULT_VALUE_EPS(e, "items_per_second", EQ_EPS, 150./t, 0.001);
 });
 
 // ========================================================================= //
@@ -92,7 +92,7 @@ void BM_Counters_Rate(benchmark::State& state) {
   state.counters["foo"] = bm::Counter{1, bm::Counter::kIsRate};
   state.counters["bar"] = bm::Counter{2, bm::Counter::kIsRate};
 }
-BENCHMARK(BM_Counters_Rate);//->ThreadRange(1, 32);
+BENCHMARK(BM_Counters_Rate);
 ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Rate %console_report bar=%hrfloat foo=%hrfloat$"}});
 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"},
                        {"\"iterations\": %int,$", MR_Next},
@@ -105,10 +105,10 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"},
 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Rate\",%csv_report,%float,%float$"}});
 CHECK_BENCHMARK_RESULTS("BM_Counters_Rate",
                         [](ResultsCheckerEntry const& e) {
-  // check that the values are within 0.1% of the expected values
   double t = e.DurationCPUTime(); // this (and not real time) is the time used
-  CHECK_COUNTER_VALUE(e, double, "foo", EQ_EPS, 5. / t, 0.001 * t);
-  CHECK_COUNTER_VALUE(e, double, "bar", EQ_EPS, 2. / t, 0.001 * t);
+  // check that the values are within 0.1% of the expected values
+  CHECK_COUNTER_VALUE_EPS(e, "foo", EQ_EPS, 1./t, 0.001);
+  CHECK_COUNTER_VALUE_EPS(e, "bar", EQ_EPS, 2./t, 0.001);
 });
 
 // ========================================================================= //