Skip to content

B
benchmark
Unverified Commit caa2fcb1 by Roman Lebedev Committed by GitHub
Options

Counter(): add 'one thousand' param. (#657)

* Counter(): add 'one thousand' param. Needed for https://github.com/google/benchmark/pull/654 Custom user counters are quite custom. It is not guaranteed that the user *always* expects for these to have 1k == 1000. If the counter represents bytes/memory/etc, 1k should be 1024. Some bikeshedding points: 1. Is this sufficient, or do we really want to go full on into custom types with names? I think just the '1000' is sufficient for now. 2. Should there be a helper benchmark::Counter::Counter{1000,1024}() static 'constructor' functions, since these two, by far, will be the most used? 3. In the future, we should be somehow encoding this info into JSON. * Counter(): use std::pair<> to represent 'one thousand' * Counter(): just use a new enum with two values 1000 vs 1024. Simpler is better. If someone comes up with a real reason to need something more advanced, it can be added later on. * Counter: just store the 1000 or 1024 in the One_K values directly * Counter: s/One_K/OneK/
parent d9cab612
Showing with 178 additions and 6 deletions
README.md
...@@ -657,9 +657,12 @@ In multithreaded benchmarks, each counter is set on the calling thread only. ...@@ -657,9 +657,12 @@ In multithreaded benchmarks, each counter is set on the calling thread only.
When the benchmark finishes, the counters from each thread will be summed; When the benchmark finishes, the counters from each thread will be summed;
the resulting sum is the value which will be shown for the benchmark. the resulting sum is the value which will be shown for the benchmark.
The `Counter` constructor accepts two parameters: the value as a `double` The `Counter` constructor accepts three parameters: the value as a `double`
and a bit flag which allows you to show counters as rates and/or as ; a bit flag which allows you to show counters as rates, and/or as per-thread
per-thread averages: iteration, and/or as per-thread averages, and/or iteration invariants;
and a flag specifying the 'unit' - i.e. is 1k a 1000 (default,
`benchmark::Counter::OneK::kIs1000`), or 1024
(`benchmark::Counter::OneK::kIs1024`)?
```c++ ```c++
// sets a simple counter // sets a simple counter
...@@ -675,6 +678,9 @@ per-thread averages: ...@@ -675,6 +678,9 @@ per-thread averages:
// There's also a combined flag: // There's also a combined flag:
state.counters["FooAvgRate"] = Counter(numFoos,benchmark::Counter::kAvgThreadsRate); state.counters["FooAvgRate"] = Counter(numFoos,benchmark::Counter::kAvgThreadsRate);
// This says that we process with the rate of state.range(0) bytes every iteration:
state.counters["BytesProcessed"] = Counter(state.range(0), benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::OneK::kIs1024);
``` ```
When you're compiling in C++11 mode or later you can use `insert()` with When you're compiling in C++11 mode or later you can use `insert()` with
......
include/benchmark/benchmark.h
...@@ -373,11 +373,20 @@ class Counter { ...@@ -373,11 +373,20 @@ class Counter {
kAvgIterationsRate = kIsRate | kAvgIterations kAvgIterationsRate = kIsRate | kAvgIterations
}; };
enum OneK {
// 1'000 items per 1k
kIs1000 = 1000,
// 1'024 items per 1k
kIs1024 = 1024
};
double value; double value;
Flags flags; Flags flags;
OneK oneK;
BENCHMARK_ALWAYS_INLINE BENCHMARK_ALWAYS_INLINE
Counter(double v = 0., Flags f = kDefaults) : value(v), flags(f) {} Counter(double v = 0., Flags f = kDefaults, OneK k = kIs1000)
: value(v), flags(f), oneK(k) {}
BENCHMARK_ALWAYS_INLINE operator double const&() const { return value; } BENCHMARK_ALWAYS_INLINE operator double const&() const { return value; }
BENCHMARK_ALWAYS_INLINE operator double&() { return value; } BENCHMARK_ALWAYS_INLINE operator double&() { return value; }
......
src/console_reporter.cc
...@@ -150,7 +150,7 @@ void ConsoleReporter::PrintRunData(const Run& result) { ...@@ -150,7 +150,7 @@ void ConsoleReporter::PrintRunData(const Run& result) {
for (auto& c : result.counters) { for (auto& c : result.counters) {
const std::size_t cNameLen = std::max(std::string::size_type(10), const std::size_t cNameLen = std::max(std::string::size_type(10),
c.first.length()); c.first.length());
auto const& s = HumanReadableNumber(c.second.value, 1000); auto const& s = HumanReadableNumber(c.second.value, c.second.oneK);
if (output_options_ & OO_Tabular) { if (output_options_ & OO_Tabular) {
if (c.second.flags & Counter::kIsRate) { if (c.second.flags & Counter::kIsRate) {
printer(Out, COLOR_DEFAULT, " %*s/s", cNameLen - 2, s.c_str()); printer(Out, COLOR_DEFAULT, " %*s/s", cNameLen - 2, s.c_str());
......
src/statistics.cc
...@@ -165,7 +165,8 @@ std::vector<BenchmarkReporter::Run> ComputeStats( ...@@ -165,7 +165,8 @@ std::vector<BenchmarkReporter::Run> ComputeStats(
// user counters // user counters
for (auto const& kv : counter_stats) { for (auto const& kv : counter_stats) {
const auto uc_stat = Stat.compute_(kv.second.s); const auto uc_stat = Stat.compute_(kv.second.s);
auto c = Counter(uc_stat, counter_stats[kv.first].c.flags); auto c = Counter(uc_stat, counter_stats[kv.first].c.flags,
counter_stats[kv.first].c.oneK);
data.counters[kv.first] = c; data.counters[kv.first] = c;
} }
......
test/CMakeLists.txt
...@@ -128,6 +128,9 @@ add_test(user_counters_test user_counters_test --benchmark_min_time=0.01) ...@@ -128,6 +128,9 @@ add_test(user_counters_test user_counters_test --benchmark_min_time=0.01)
compile_output_test(user_counters_tabular_test) compile_output_test(user_counters_tabular_test)
add_test(user_counters_tabular_test user_counters_tabular_test --benchmark_counters_tabular=true --benchmark_min_time=0.01) add_test(user_counters_tabular_test user_counters_tabular_test --benchmark_counters_tabular=true --benchmark_min_time=0.01)
compile_output_test(user_counters_thousands_test)
add_test(user_counters_thousands_test user_counters_thousands_test --benchmark_min_time=0.01)
check_cxx_compiler_flag(-std=c++03 BENCHMARK_HAS_CXX03_FLAG) check_cxx_compiler_flag(-std=c++03 BENCHMARK_HAS_CXX03_FLAG)
if (BENCHMARK_HAS_CXX03_FLAG) if (BENCHMARK_HAS_CXX03_FLAG)
compile_benchmark_test(cxx03_test) compile_benchmark_test(cxx03_test)
......
test/user_counters_thousands_test.cc 0 → 100644
#undef NDEBUG
#include "benchmark/benchmark.h"
#include "output_test.h"
// ========================================================================= //
// ------------------------ Thousands Customisation ------------------------ //
// ========================================================================= //
void BM_Counters_Thousands(benchmark::State& state) {
for (auto _ : state) {
}
namespace bm = benchmark;
state.counters.insert({
{"t0_1000000DefaultBase",
bm::Counter(1000 * 1000, bm::Counter::kDefaults)},
{"t1_1000000Base1000", bm::Counter(1000 * 1000, bm::Counter::kDefaults,
benchmark::Counter::OneK::kIs1000)},
{"t2_1000000Base1024", bm::Counter(1000 * 1000, bm::Counter::kDefaults,
benchmark::Counter::OneK::kIs1024)},
{"t3_1048576Base1000", bm::Counter(1024 * 1024, bm::Counter::kDefaults,
benchmark::Counter::OneK::kIs1000)},
{"t4_1048576Base1024", bm::Counter(1024 * 1024, bm::Counter::kDefaults,
benchmark::Counter::OneK::kIs1024)},
});
}
BENCHMARK(BM_Counters_Thousands)->Repetitions(2);
ADD_CASES(
TC_ConsoleOut,
{
{"^BM_Counters_Thousands/repeats:2 %console_report "
"t0_1000000DefaultBase=1000k "
"t1_1000000Base1000=1000k t2_1000000Base1024=976.56[23]k "
"t3_1048576Base1000=1048.58k t4_1048576Base1024=1024k$"},
{"^BM_Counters_Thousands/repeats:2 %console_report "
"t0_1000000DefaultBase=1000k "
"t1_1000000Base1000=1000k t2_1000000Base1024=976.56[23]k "
"t3_1048576Base1000=1048.58k t4_1048576Base1024=1024k$"},
{"^BM_Counters_Thousands/repeats:2_mean %console_report "
"t0_1000000DefaultBase=1000k t1_1000000Base1000=1000k "
"t2_1000000Base1024=976.56[23]k t3_1048576Base1000=1048.58k "
"t4_1048576Base1024=1024k$"},
{"^BM_Counters_Thousands/repeats:2_median %console_report "
"t0_1000000DefaultBase=1000k t1_1000000Base1000=1000k "
"t2_1000000Base1024=976.56[23]k t3_1048576Base1000=1048.58k "
"t4_1048576Base1024=1024k$"},
{"^BM_Counters_Thousands/repeats:2_stddev %console_report "
"t0_1000000DefaultBase=0 t1_1000000Base1000=0 t2_1000000Base1024=0 "
"t3_1048576Base1000=0 t4_1048576Base1024=0$"},
});
ADD_CASES(TC_JSONOut,
{{"\"name\": \"BM_Counters_Thousands/repeats:2\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\",$", MR_Next},
{"\"t0_1000000DefaultBase\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
{"\"t1_1000000Base1000\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
{"\"t2_1000000Base1024\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
{"\"t3_1048576Base1000\": 1\\.048576(0)*e\\+(0)*6,$", MR_Next},
{"\"t4_1048576Base1024\": 1\\.048576(0)*e\\+(0)*6$", MR_Next},
{"}", MR_Next}});
ADD_CASES(TC_JSONOut,
{{"\"name\": \"BM_Counters_Thousands/repeats:2\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\",$", MR_Next},
{"\"t0_1000000DefaultBase\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
{"\"t1_1000000Base1000\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
{"\"t2_1000000Base1024\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
{"\"t3_1048576Base1000\": 1\\.048576(0)*e\\+(0)*6,$", MR_Next},
{"\"t4_1048576Base1024\": 1\\.048576(0)*e\\+(0)*6$", MR_Next},
{"}", MR_Next}});
ADD_CASES(TC_JSONOut,
{{"\"name\": \"BM_Counters_Thousands/repeats:2_mean\",$"},
{"\"run_type\": \"aggregate\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\",$", MR_Next},
{"\"t0_1000000DefaultBase\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
{"\"t1_1000000Base1000\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
{"\"t2_1000000Base1024\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
{"\"t3_1048576Base1000\": 1\\.048576(0)*e\\+(0)*6,$", MR_Next},
{"\"t4_1048576Base1024\": 1\\.048576(0)*e\\+(0)*6$", MR_Next},
{"}", MR_Next}});
ADD_CASES(TC_JSONOut,
{{"\"name\": \"BM_Counters_Thousands/repeats:2_median\",$"},
{"\"run_type\": \"aggregate\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\",$", MR_Next},
{"\"t0_1000000DefaultBase\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
{"\"t1_1000000Base1000\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
{"\"t2_1000000Base1024\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
{"\"t3_1048576Base1000\": 1\\.048576(0)*e\\+(0)*6,$", MR_Next},
{"\"t4_1048576Base1024\": 1\\.048576(0)*e\\+(0)*6$", MR_Next},
{"}", MR_Next}});
ADD_CASES(TC_JSONOut,
{{"\"name\": \"BM_Counters_Thousands/repeats:2_stddev\",$"},
{"\"run_type\": \"aggregate\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\",$", MR_Next},
{"\"t0_1000000DefaultBase\": 0\\.(0)*e\\+(0)*,$", MR_Next},
{"\"t1_1000000Base1000\": 0\\.(0)*e\\+(0)*,$", MR_Next},
{"\"t2_1000000Base1024\": 0\\.(0)*e\\+(0)*,$", MR_Next},
{"\"t3_1048576Base1000\": 0\\.(0)*e\\+(0)*,$", MR_Next},
{"\"t4_1048576Base1024\": 0\\.(0)*e\\+(0)*$", MR_Next},
{"}", MR_Next}});
ADD_CASES(
TC_CSVOut,
{{"^\"BM_Counters_Thousands/"
"repeats:2\",%csv_report,1e\\+(0)*6,1e\\+(0)*6,1e\\+(0)*6,1\\.04858e\\+("
"0)*6,1\\.04858e\\+(0)*6$"},
{"^\"BM_Counters_Thousands/"
"repeats:2\",%csv_report,1e\\+(0)*6,1e\\+(0)*6,1e\\+(0)*6,1\\.04858e\\+("
"0)*6,1\\.04858e\\+(0)*6$"},
{"^\"BM_Counters_Thousands/"
"repeats:2_mean\",%csv_report,1e\\+(0)*6,1e\\+(0)*6,1e\\+(0)*6,1\\."
"04858e\\+(0)*6,1\\.04858e\\+(0)*6$"},
{"^\"BM_Counters_Thousands/"
"repeats:2_median\",%csv_report,1e\\+(0)*6,1e\\+(0)*6,1e\\+(0)*6,1\\."
"04858e\\+(0)*6,1\\.04858e\\+(0)*6$"},
{"^\"BM_Counters_Thousands/repeats:2_stddev\",%csv_report,0,0,0,0,0$"}});
// VS2013 does not allow this function to be passed as a lambda argument
// to CHECK_BENCHMARK_RESULTS()
void CheckThousands(Results const& e) {
if (e.name != "BM_Counters_Thousands/repeats:2")
return; // Do not check the aggregates!
// check that the values are within 0.01% of the expected values
CHECK_FLOAT_COUNTER_VALUE(e, "t0_1000000DefaultBase", EQ, 1000 * 1000,
0.0001);
CHECK_FLOAT_COUNTER_VALUE(e, "t1_1000000Base1000", EQ, 1000 * 1000, 0.0001);
CHECK_FLOAT_COUNTER_VALUE(e, "t2_1000000Base1024", EQ, 1000 * 1000, 0.0001);
CHECK_FLOAT_COUNTER_VALUE(e, "t3_1048576Base1000", EQ, 1024 * 1024, 0.0001);
CHECK_FLOAT_COUNTER_VALUE(e, "t4_1048576Base1024", EQ, 1024 * 1024, 0.0001);
}
CHECK_BENCHMARK_RESULTS("BM_Counters_Thousands", &CheckThousands);
// ========================================================================= //
// --------------------------- TEST CASES END ------------------------------ //
// ========================================================================= //
int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment