Skip to content

B
benchmark
Unverified Commit 507c06e6 by Roman Lebedev Committed by GitHub
Options

Aggregates: use non-aggregate count as iteration count. (#706)

It is incorrect to say that an aggregate is computed over run's iterations, because those iterations already got averaged. Similarly, if there are N repetitions with 1 iterations each, an aggregate will be computed over N measurements, not 1. Thus it is best to simply use the count of separate reports. Fixes #586.
parent 99d1356c
Showing with 151 additions and 68 deletions
src/statistics.cc
...@@ -141,6 +141,9 @@ std::vector<BenchmarkReporter::Run> ComputeStats( ...@@ -141,6 +141,9 @@ std::vector<BenchmarkReporter::Run> ComputeStats(
} }
} }
const double iteration_rescale_factor =
double(reports.size()) / double(run_iterations);
for (const auto& Stat : *reports[0].statistics) { for (const auto& Stat : *reports[0].statistics) {
// Get the data from the accumulator to BenchmarkReporter::Run's. // Get the data from the accumulator to BenchmarkReporter::Run's.
Run data; Run data;
...@@ -148,15 +151,30 @@ std::vector<BenchmarkReporter::Run> ComputeStats( ...@@ -148,15 +151,30 @@ std::vector<BenchmarkReporter::Run> ComputeStats(
data.run_type = BenchmarkReporter::Run::RT_Aggregate; data.run_type = BenchmarkReporter::Run::RT_Aggregate;
data.aggregate_name = Stat.name_; data.aggregate_name = Stat.name_;
data.report_label = report_label; data.report_label = report_label;
data.iterations = run_iterations;
// It is incorrect to say that an aggregate is computed over
// run's iterations, because those iterations already got averaged.
// Similarly, if there are N repetitions with 1 iterations each,
// an aggregate will be computed over N measurements, not 1.
// Thus it is best to simply use the count of separate reports.
data.iterations = reports.size();
data.real_accumulated_time = Stat.compute_(real_accumulated_time_stat); data.real_accumulated_time = Stat.compute_(real_accumulated_time_stat);
data.cpu_accumulated_time = Stat.compute_(cpu_accumulated_time_stat); data.cpu_accumulated_time = Stat.compute_(cpu_accumulated_time_stat);
// We will divide these times by data.iterations when reporting, but the
// data.iterations is not nessesairly the scale of these measurements,
// because in each repetition, these timers are sum over all the iterations.
// And if we want to say that the stats are over N repetitions and not
// M iterations, we need to multiply these by (N/M).
data.real_accumulated_time *= iteration_rescale_factor;
data.cpu_accumulated_time *= iteration_rescale_factor;
data.time_unit = reports[0].time_unit; data.time_unit = reports[0].time_unit;
// user counters // user counters
for (auto const& kv : counter_stats) { for (auto const& kv : counter_stats) {
// Do *NOT* rescale the custom counters. They are already properly scaled.
const auto uc_stat = Stat.compute_(kv.second.s); const auto uc_stat = Stat.compute_(kv.second.s);
auto c = Counter(uc_stat, counter_stats[kv.first].c.flags, auto c = Counter(uc_stat, counter_stats[kv.first].c.flags,
counter_stats[kv.first].c.oneK); counter_stats[kv.first].c.oneK);
......
test/output_test_helper.cc
...@@ -44,9 +44,11 @@ SubMap& GetSubstitutions() { ...@@ -44,9 +44,11 @@ SubMap& GetSubstitutions() {
{"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"}, {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"},
{"%int", "[ ]*[0-9]+"}, {"%int", "[ ]*[0-9]+"},
{" %s ", "[ ]+"}, {" %s ", "[ ]+"},
{"%time", "[ ]*[0-9]{1,6} ns"}, {"%time", "[ ]*[0-9]+ ns"},
{"%console_report", "[ ]*[0-9]{1,6} ns [ ]*[0-9]{1,6} ns [ ]*[0-9]+"}, {"%console_report", "[ ]*[0-9]+ ns [ ]*[0-9]+ ns [ ]*[0-9]+"},
{"%console_us_report", "[ ]*[0-9] us [ ]*[0-9] us [ ]*[0-9]+"}, {"%console_time_only_report", "[ ]*[0-9]+ ns [ ]*[0-9]+ ns"},
{"%console_us_report", "[ ]*[0-9]+ us [ ]*[0-9]+ us [ ]*[0-9]+"},
{"%console_us_time_only_report", "[ ]*[0-9]+ us [ ]*[0-9]+ us"},
{"%csv_header", {"%csv_header",
"name,iterations,real_time,cpu_time,time_unit,bytes_per_second," "name,iterations,real_time,cpu_time,time_unit,bytes_per_second,"
"items_per_second,label,error_occurred,error_message"}, "items_per_second,label,error_occurred,error_message"},
......
test/reporter_output_test.cc
...@@ -239,11 +239,12 @@ void BM_Repeat(benchmark::State& state) { ...@@ -239,11 +239,12 @@ void BM_Repeat(benchmark::State& state) {
} }
// need two repetitions min to be able to output any aggregate output // need two repetitions min to be able to output any aggregate output
BENCHMARK(BM_Repeat)->Repetitions(2); BENCHMARK(BM_Repeat)->Repetitions(2);
ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:2 %console_report$"}, ADD_CASES(TC_ConsoleOut,
{{"^BM_Repeat/repeats:2 %console_report$"},
{"^BM_Repeat/repeats:2 %console_report$"}, {"^BM_Repeat/repeats:2 %console_report$"},
{"^BM_Repeat/repeats:2_mean %console_report$"}, {"^BM_Repeat/repeats:2_mean %console_time_only_report [ ]*2$"},
{"^BM_Repeat/repeats:2_median %console_report$"}, {"^BM_Repeat/repeats:2_median %console_time_only_report [ ]*2$"},
{"^BM_Repeat/repeats:2_stddev %console_report$"}}); {"^BM_Repeat/repeats:2_stddev %console_time_only_report [ ]*2$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"},
{"\"run_name\": \"BM_Repeat/repeats:2\"", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:2\"", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next},
...@@ -254,14 +255,17 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"}, ...@@ -254,14 +255,17 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"},
{"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:2_median\",$"}, {"\"name\": \"BM_Repeat/repeats:2_median\",$"},
{"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:2_stddev\",$"}, {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"},
{"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next}}); {"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"}, ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"},
{"^\"BM_Repeat/repeats:2\",%csv_report$"}, {"^\"BM_Repeat/repeats:2\",%csv_report$"},
{"^\"BM_Repeat/repeats:2_mean\",%csv_report$"}, {"^\"BM_Repeat/repeats:2_mean\",%csv_report$"},
...@@ -269,12 +273,13 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"}, ...@@ -269,12 +273,13 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"},
{"^\"BM_Repeat/repeats:2_stddev\",%csv_report$"}}); {"^\"BM_Repeat/repeats:2_stddev\",%csv_report$"}});
// but for two repetitions, mean and median is the same, so let's repeat.. // but for two repetitions, mean and median is the same, so let's repeat..
BENCHMARK(BM_Repeat)->Repetitions(3); BENCHMARK(BM_Repeat)->Repetitions(3);
ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:3 %console_report$"}, ADD_CASES(TC_ConsoleOut,
{{"^BM_Repeat/repeats:3 %console_report$"},
{"^BM_Repeat/repeats:3 %console_report$"}, {"^BM_Repeat/repeats:3 %console_report$"},
{"^BM_Repeat/repeats:3 %console_report$"}, {"^BM_Repeat/repeats:3 %console_report$"},
{"^BM_Repeat/repeats:3_mean %console_report$"}, {"^BM_Repeat/repeats:3_mean %console_time_only_report [ ]*3$"},
{"^BM_Repeat/repeats:3_median %console_report$"}, {"^BM_Repeat/repeats:3_median %console_time_only_report [ ]*3$"},
{"^BM_Repeat/repeats:3_stddev %console_report$"}}); {"^BM_Repeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
{"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next},
...@@ -288,14 +293,17 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, ...@@ -288,14 +293,17 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
{"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:3_median\",$"}, {"\"name\": \"BM_Repeat/repeats:3_median\",$"},
{"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}, {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"},
{"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next}}); {"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"}, ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
{"^\"BM_Repeat/repeats:3\",%csv_report$"}, {"^\"BM_Repeat/repeats:3\",%csv_report$"},
{"^\"BM_Repeat/repeats:3\",%csv_report$"}, {"^\"BM_Repeat/repeats:3\",%csv_report$"},
...@@ -304,13 +312,14 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"}, ...@@ -304,13 +312,14 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
{"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}}); {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}});
// median differs between even/odd number of repetitions, so just to be sure // median differs between even/odd number of repetitions, so just to be sure
BENCHMARK(BM_Repeat)->Repetitions(4); BENCHMARK(BM_Repeat)->Repetitions(4);
ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:4 %console_report$"}, ADD_CASES(TC_ConsoleOut,
{{"^BM_Repeat/repeats:4 %console_report$"},
{"^BM_Repeat/repeats:4 %console_report$"}, {"^BM_Repeat/repeats:4 %console_report$"},
{"^BM_Repeat/repeats:4 %console_report$"}, {"^BM_Repeat/repeats:4 %console_report$"},
{"^BM_Repeat/repeats:4 %console_report$"}, {"^BM_Repeat/repeats:4 %console_report$"},
{"^BM_Repeat/repeats:4_mean %console_report$"}, {"^BM_Repeat/repeats:4_mean %console_time_only_report [ ]*4$"},
{"^BM_Repeat/repeats:4_median %console_report$"}, {"^BM_Repeat/repeats:4_median %console_time_only_report [ ]*4$"},
{"^BM_Repeat/repeats:4_stddev %console_report$"}}); {"^BM_Repeat/repeats:4_stddev %console_time_only_report [ ]*4$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"},
{"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next},
...@@ -327,14 +336,17 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"}, ...@@ -327,14 +336,17 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"},
{"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"iterations\": 4,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:4_median\",$"}, {"\"name\": \"BM_Repeat/repeats:4_median\",$"},
{"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next},
{"\"iterations\": 4,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:4_stddev\",$"}, {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"},
{"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next}}); {"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"iterations\": 4,$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"}, ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"},
{"^\"BM_Repeat/repeats:4\",%csv_report$"}, {"^\"BM_Repeat/repeats:4\",%csv_report$"},
{"^\"BM_Repeat/repeats:4\",%csv_report$"}, {"^\"BM_Repeat/repeats:4\",%csv_report$"},
...@@ -362,25 +374,29 @@ void BM_SummaryRepeat(benchmark::State& state) { ...@@ -362,25 +374,29 @@ void BM_SummaryRepeat(benchmark::State& state) {
} }
} }
BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly(); BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();
ADD_CASES(TC_ConsoleOut, ADD_CASES(
TC_ConsoleOut,
{{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
{"^BM_SummaryRepeat/repeats:3_mean %console_report$"}, {"^BM_SummaryRepeat/repeats:3_mean %console_time_only_report [ ]*3$"},
{"^BM_SummaryRepeat/repeats:3_median %console_report$"}, {"^BM_SummaryRepeat/repeats:3_median %console_time_only_report [ ]*3$"},
{"^BM_SummaryRepeat/repeats:3_stddev %console_report$"}}); {"^BM_SummaryRepeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});
ADD_CASES(TC_JSONOut, ADD_CASES(TC_JSONOut,
{{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
{"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"}, {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
{"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"}, {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"},
{"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}, {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"},
{"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next}}); {"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next}});
ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
{"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"}, {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},
{"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"}, {"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"},
...@@ -394,25 +410,29 @@ void BM_SummaryDisplay(benchmark::State& state) { ...@@ -394,25 +410,29 @@ void BM_SummaryDisplay(benchmark::State& state) {
} }
} }
BENCHMARK(BM_SummaryDisplay)->Repetitions(2)->DisplayAggregatesOnly(); BENCHMARK(BM_SummaryDisplay)->Repetitions(2)->DisplayAggregatesOnly();
ADD_CASES(TC_ConsoleOut, ADD_CASES(
TC_ConsoleOut,
{{".*BM_SummaryDisplay/repeats:2 ", MR_Not}, {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
{"^BM_SummaryDisplay/repeats:2_mean %console_report$"}, {"^BM_SummaryDisplay/repeats:2_mean %console_time_only_report [ ]*2$"},
{"^BM_SummaryDisplay/repeats:2_median %console_report$"}, {"^BM_SummaryDisplay/repeats:2_median %console_time_only_report [ ]*2$"},
{"^BM_SummaryDisplay/repeats:2_stddev %console_report$"}}); {"^BM_SummaryDisplay/repeats:2_stddev %console_time_only_report [ ]*2$"}});
ADD_CASES(TC_JSONOut, ADD_CASES(TC_JSONOut,
{{".*BM_SummaryDisplay/repeats:2 ", MR_Not}, {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
{"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"}, {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"},
{"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next},
{"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"}, {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"},
{"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next},
{"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"}, {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"},
{"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next}}); {"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next}});
ADD_CASES(TC_CSVOut, ADD_CASES(TC_CSVOut,
{{".*BM_SummaryDisplay/repeats:2 ", MR_Not}, {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
{"^\"BM_SummaryDisplay/repeats:2_mean\",%csv_report$"}, {"^\"BM_SummaryDisplay/repeats:2_mean\",%csv_report$"},
...@@ -428,27 +448,33 @@ BENCHMARK(BM_RepeatTimeUnit) ...@@ -428,27 +448,33 @@ BENCHMARK(BM_RepeatTimeUnit)
->Repetitions(3) ->Repetitions(3)
->ReportAggregatesOnly() ->ReportAggregatesOnly()
->Unit(benchmark::kMicrosecond); ->Unit(benchmark::kMicrosecond);
ADD_CASES(TC_ConsoleOut, ADD_CASES(
TC_ConsoleOut,
{{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
{"^BM_RepeatTimeUnit/repeats:3_mean %console_us_report$"}, {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_time_only_report [ ]*3$"},
{"^BM_RepeatTimeUnit/repeats:3_median %console_us_report$"}, {"^BM_RepeatTimeUnit/repeats:3_median %console_us_time_only_report [ "
{"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_report$"}}); "]*3$"},
{"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_time_only_report [ "
"]*3$"}});
ADD_CASES(TC_JSONOut, ADD_CASES(TC_JSONOut,
{{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
{"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"},
{"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"time_unit\": \"us\",?$"}, {"\"time_unit\": \"us\",?$"},
{"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"},
{"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"time_unit\": \"us\",?$"}, {"\"time_unit\": \"us\",?$"},
{"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"},
{"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"time_unit\": \"us\",?$"}}); {"\"time_unit\": \"us\",?$"}});
ADD_CASES(TC_CSVOut, ADD_CASES(TC_CSVOut,
{{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
...@@ -465,55 +491,92 @@ const auto UserStatistics = [](const std::vector<double>& v) { ...@@ -465,55 +491,92 @@ const auto UserStatistics = [](const std::vector<double>& v) {
}; };
void BM_UserStats(benchmark::State& state) { void BM_UserStats(benchmark::State& state) {
for (auto _ : state) { for (auto _ : state) {
state.SetIterationTime(150 / 10e8);
} }
} }
// clang-format off // clang-format off
BENCHMARK(BM_UserStats) BENCHMARK(BM_UserStats)
->Repetitions(3) ->Repetitions(3)
->Iterations(5)
->UseManualTime()
->ComputeStatistics("", UserStatistics); ->ComputeStatistics("", UserStatistics);
// clang-format on // clang-format on
// check that user-provided stats is calculated, and is after the default-ones // check that user-provided stats is calculated, and is after the default-ones
// empty string as name is intentional, it would sort before anything else // empty string as name is intentional, it would sort before anything else
ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/repeats:3 %console_report$"}, ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
{"^BM_UserStats/repeats:3 %console_report$"}, "]* 150 ns %time [ ]*5$"},
{"^BM_UserStats/repeats:3 %console_report$"}, {"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
{"^BM_UserStats/repeats:3_mean %console_report$"}, "]* 150 ns %time [ ]*5$"},
{"^BM_UserStats/repeats:3_median %console_report$"}, {"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
{"^BM_UserStats/repeats:3_stddev %console_report$"}, "]* 150 ns %time [ ]*5$"},
{"^BM_UserStats/repeats:3_ %console_report$"}}); {"^BM_UserStats/iterations:5/repeats:3/"
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_UserStats/repeats:3\",$"}, "manual_time_mean [ ]* 150 ns %time [ ]*3$"},
{"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next}, {"^BM_UserStats/iterations:5/repeats:3/"
"manual_time_median [ ]* 150 ns %time [ ]*3$"},
{"^BM_UserStats/iterations:5/repeats:3/"
"manual_time_stddev [ ]* 0 ns %time [ ]*3$"},
{"^BM_UserStats/iterations:5/repeats:3/manual_time_ "
"[ ]* 150 ns %time [ ]*3$"}});
ADD_CASES(
TC_JSONOut,
{{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
{"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next},
{"\"name\": \"BM_UserStats/repeats:3\",$"}, {"\"iterations\": 5,$", MR_Next},
{"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
{"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next},
{"\"name\": \"BM_UserStats/repeats:3\",$"}, {"\"iterations\": 5,$", MR_Next},
{"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
{"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next},
{"\"name\": \"BM_UserStats/repeats:3_mean\",$"}, {"\"iterations\": 5,$", MR_Next},
{"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",$"},
{"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"name\": \"BM_UserStats/repeats:3_median\",$"}, {"\"iterations\": 3,$", MR_Next},
{"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_median\",$"},
{"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next},
{"\"name\": \"BM_UserStats/repeats:3_stddev\",$"}, {"\"iterations\": 3,$", MR_Next},
{"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_stddev\",$"},
{"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"name\": \"BM_UserStats/repeats:3_\",$"}, {"\"iterations\": 3,$", MR_Next},
{"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_\",$"},
{"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"aggregate_name\": \"\",$", MR_Next}}); {"\"aggregate_name\": \"\",$", MR_Next},
ADD_CASES(TC_CSVOut, {{"^\"BM_UserStats/repeats:3\",%csv_report$"}, {"\"iterations\": 3,$", MR_Next},
{"^\"BM_UserStats/repeats:3\",%csv_report$"}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}});
{"^\"BM_UserStats/repeats:3\",%csv_report$"}, ADD_CASES(
{"^\"BM_UserStats/repeats:3_mean\",%csv_report$"}, TC_CSVOut,
{"^\"BM_UserStats/repeats:3_median\",%csv_report$"}, {{"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
{"^\"BM_UserStats/repeats:3_stddev\",%csv_report$"}, {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
{"^\"BM_UserStats/repeats:3_\",%csv_report$"}}); {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
{"^\"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",%csv_report$"},
{"^\"BM_UserStats/iterations:5/repeats:3/"
"manual_time_median\",%csv_report$"},
{"^\"BM_UserStats/iterations:5/repeats:3/"
"manual_time_stddev\",%csv_report$"},
{"^\"BM_UserStats/iterations:5/repeats:3/manual_time_\",%csv_report$"}});
// ========================================================================= // // ========================================================================= //
// --------------------------- TEST CASES END ------------------------------ // // --------------------------- TEST CASES END ------------------------------ //
......
test/user_counters_thousands_test.cc
...@@ -45,9 +45,9 @@ ADD_CASES( ...@@ -45,9 +45,9 @@ ADD_CASES(
"t0_1000000DefaultBase=1000k t1_1000000Base1000=1000k " "t0_1000000DefaultBase=1000k t1_1000000Base1000=1000k "
"t2_1000000Base1024=976.56[23]k t3_1048576Base1000=1048.58k " "t2_1000000Base1024=976.56[23]k t3_1048576Base1000=1048.58k "
"t4_1048576Base1024=1024k$"}, "t4_1048576Base1024=1024k$"},
{"^BM_Counters_Thousands/repeats:2_stddev %console_report " {"^BM_Counters_Thousands/repeats:2_stddev %console_time_only_report [ "
"t0_1000000DefaultBase=0 t1_1000000Base1000=0 t2_1000000Base1024=0 " "]*2 t0_1000000DefaultBase=0 t1_1000000Base1000=0 "
"t3_1048576Base1000=0 t4_1048576Base1024=0$"}, "t2_1000000Base1024=0 t3_1048576Base1000=0 t4_1048576Base1024=0$"},
}); });
ADD_CASES(TC_JSONOut, ADD_CASES(TC_JSONOut,
{{"\"name\": \"BM_Counters_Thousands/repeats:2\",$"}, {{"\"name\": \"BM_Counters_Thousands/repeats:2\",$"},
...@@ -82,7 +82,7 @@ ADD_CASES(TC_JSONOut, ...@@ -82,7 +82,7 @@ ADD_CASES(TC_JSONOut,
{"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": 2,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\",$", MR_Next}, {"\"time_unit\": \"ns\",$", MR_Next},
...@@ -97,7 +97,7 @@ ADD_CASES(TC_JSONOut, ...@@ -97,7 +97,7 @@ ADD_CASES(TC_JSONOut,
{"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": 2,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\",$", MR_Next}, {"\"time_unit\": \"ns\",$", MR_Next},
...@@ -112,7 +112,7 @@ ADD_CASES(TC_JSONOut, ...@@ -112,7 +112,7 @@ ADD_CASES(TC_JSONOut,
{"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": 2,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\",$", MR_Next}, {"\"time_unit\": \"ns\",$", MR_Next},
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment