Skip to content

B
benchmark
Unverified Commit 8688c5c4 by Roman Lebedev Committed by GitHub
Options

Track 'type' of the run - is it an actual measurement, or an aggregate. (#658)

This is *only* exposed in the JSON. Not in CSV, which is deprecated. This *only* supposed to track these two states. An additional field could later track which aggregate this is, specifically (statistic name, rms, bigo, ...) The motivation is that we already have ReportAggregatesOnly, but it affects the entire reports, both the display, and the reporters (json files), which isn't ideal. It would be very useful to have a 'display aggregates only' option, both in the library's console reporter, and the python tooling, This will be especially needed for the 'store separate iterations'.
parent 9a179cb9
Showing with 35 additions and 1 deletion
include/benchmark/benchmark.h
...@@ -1276,8 +1276,11 @@ class BenchmarkReporter { ...@@ -1276,8 +1276,11 @@ class BenchmarkReporter {
}; };
struct Run { struct Run {
enum RunType { RT_Iteration, RT_Aggregate };
Run() Run()
: error_occurred(false), : run_type(RT_Iteration),
error_occurred(false),
iterations(1), iterations(1),
time_unit(kNanosecond), time_unit(kNanosecond),
real_accumulated_time(0), real_accumulated_time(0),
...@@ -1296,6 +1299,7 @@ class BenchmarkReporter { ...@@ -1296,6 +1299,7 @@ class BenchmarkReporter {
max_bytes_used(0) {} max_bytes_used(0) {}
std::string benchmark_name; std::string benchmark_name;
RunType run_type; // is this a measurement, or an aggregate?
std::string report_label; // Empty if not set by benchmark. std::string report_label; // Empty if not set by benchmark.
bool error_occurred; bool error_occurred;
std::string error_message; std::string error_message;
......
src/complexity.cc
...@@ -187,6 +187,7 @@ std::vector<BenchmarkReporter::Run> ComputeBigO( ...@@ -187,6 +187,7 @@ std::vector<BenchmarkReporter::Run> ComputeBigO(
// Get the data from the accumulator to BenchmarkReporter::Run's. // Get the data from the accumulator to BenchmarkReporter::Run's.
Run big_o; Run big_o;
big_o.run_type = BenchmarkReporter::Run::RT_Aggregate;
big_o.benchmark_name = benchmark_name + "_BigO"; big_o.benchmark_name = benchmark_name + "_BigO";
big_o.iterations = 0; big_o.iterations = 0;
big_o.real_accumulated_time = result_real.coef; big_o.real_accumulated_time = result_real.coef;
...@@ -204,6 +205,7 @@ std::vector<BenchmarkReporter::Run> ComputeBigO( ...@@ -204,6 +205,7 @@ std::vector<BenchmarkReporter::Run> ComputeBigO(
// Only add label to mean/stddev if it is same for all runs // Only add label to mean/stddev if it is same for all runs
Run rms; Run rms;
big_o.report_label = reports[0].report_label; big_o.report_label = reports[0].report_label;
rms.run_type = BenchmarkReporter::Run::RT_Aggregate;
rms.benchmark_name = benchmark_name + "_RMS"; rms.benchmark_name = benchmark_name + "_RMS";
rms.report_label = big_o.report_label; rms.report_label = big_o.report_label;
rms.iterations = 0; rms.iterations = 0;
......
src/json_reporter.cc
...@@ -160,6 +160,15 @@ void JSONReporter::PrintRunData(Run const& run) { ...@@ -160,6 +160,15 @@ void JSONReporter::PrintRunData(Run const& run) {
std::string indent(6, ' '); std::string indent(6, ' ');
std::ostream& out = GetOutputStream(); std::ostream& out = GetOutputStream();
out << indent << FormatKV("name", run.benchmark_name) << ",\n"; out << indent << FormatKV("name", run.benchmark_name) << ",\n";
out << indent << FormatKV("run_type", [&run]() -> const char* {
switch (run.run_type) {
case BenchmarkReporter::Run::RT_Iteration:
return "iteration";
case BenchmarkReporter::Run::RT_Aggregate:
return "aggregate";
}
BENCHMARK_UNREACHABLE();
}()) << ",\n";
if (run.error_occurred) { if (run.error_occurred) {
out << indent << FormatKV("error_occurred", run.error_occurred) << ",\n"; out << indent << FormatKV("error_occurred", run.error_occurred) << ",\n";
out << indent << FormatKV("error_message", run.error_message) << ",\n"; out << indent << FormatKV("error_message", run.error_message) << ",\n";
......
src/statistics.cc
...@@ -150,6 +150,7 @@ std::vector<BenchmarkReporter::Run> ComputeStats( ...@@ -150,6 +150,7 @@ std::vector<BenchmarkReporter::Run> ComputeStats(
for (const auto& Stat : *reports[0].statistics) { for (const auto& Stat : *reports[0].statistics) {
// Get the data from the accumulator to BenchmarkReporter::Run's. // Get the data from the accumulator to BenchmarkReporter::Run's.
Run data; Run data;
data.run_type = BenchmarkReporter::Run::RT_Aggregate;
data.benchmark_name = reports[0].benchmark_name + "_" + Stat.name_; data.benchmark_name = reports[0].benchmark_name + "_" + Stat.name_;
data.report_label = report_label; data.report_label = report_label;
data.iterations = run_iterations; data.iterations = run_iterations;
......
test/complexity_test.cc
...@@ -25,12 +25,14 @@ int AddComplexityTest(std::string big_o_test_name, std::string rms_test_name, ...@@ -25,12 +25,14 @@ int AddComplexityTest(std::string big_o_test_name, std::string rms_test_name,
{"^%bigo_name", MR_Not}, // Assert we we didn't only matched a name. {"^%bigo_name", MR_Not}, // Assert we we didn't only matched a name.
{"^%rms_name %rms %rms[ ]*$", MR_Next}}); {"^%rms_name %rms %rms[ ]*$", MR_Next}});
AddCases(TC_JSONOut, {{"\"name\": \"%bigo_name\",$"}, AddCases(TC_JSONOut, {{"\"name\": \"%bigo_name\",$"},
{"\"run_type\": \"aggregate\",$", MR_Next},
{"\"cpu_coefficient\": %float,$", MR_Next}, {"\"cpu_coefficient\": %float,$", MR_Next},
{"\"real_coefficient\": %float,$", MR_Next}, {"\"real_coefficient\": %float,$", MR_Next},
{"\"big_o\": \"%bigo\",$", MR_Next}, {"\"big_o\": \"%bigo\",$", MR_Next},
{"\"time_unit\": \"ns\"$", MR_Next}, {"\"time_unit\": \"ns\"$", MR_Next},
{"}", MR_Next}, {"}", MR_Next},
{"\"name\": \"%rms_name\",$"}, {"\"name\": \"%rms_name\",$"},
{"\"run_type\": \"aggregate\",$", MR_Next},
{"\"rms\": %float$", MR_Next}, {"\"rms\": %float$", MR_Next},
{"}", MR_Next}}); {"}", MR_Next}});
AddCases(TC_CSVOut, {{"^\"%bigo_name\",,%float,%float,%bigo,,,,,$"}, AddCases(TC_CSVOut, {{"^\"%bigo_name\",,%float,%float,%bigo,,,,,$"},
......
test/memory_manager_test.cc
...@@ -21,6 +21,7 @@ BENCHMARK(BM_empty); ...@@ -21,6 +21,7 @@ BENCHMARK(BM_empty);
ADD_CASES(TC_ConsoleOut, {{"^BM_empty %console_report$"}}); ADD_CASES(TC_ConsoleOut, {{"^BM_empty %console_report$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_empty\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_empty\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
......
test/user_counters_tabular_test.cc
...@@ -70,6 +70,7 @@ void BM_Counters_Tabular(benchmark::State& state) { ...@@ -70,6 +70,7 @@ void BM_Counters_Tabular(benchmark::State& state) {
} }
BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 16); BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 16);
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
...@@ -114,6 +115,7 @@ void BM_CounterRates_Tabular(benchmark::State& state) { ...@@ -114,6 +115,7 @@ void BM_CounterRates_Tabular(benchmark::State& state) {
} }
BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16); BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16);
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
...@@ -158,6 +160,7 @@ void BM_CounterSet0_Tabular(benchmark::State& state) { ...@@ -158,6 +160,7 @@ void BM_CounterSet0_Tabular(benchmark::State& state) {
} }
BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16); BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16);
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
...@@ -190,6 +193,7 @@ void BM_CounterSet1_Tabular(benchmark::State& state) { ...@@ -190,6 +193,7 @@ void BM_CounterSet1_Tabular(benchmark::State& state) {
} }
BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16); BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16);
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
...@@ -226,6 +230,7 @@ void BM_CounterSet2_Tabular(benchmark::State& state) { ...@@ -226,6 +230,7 @@ void BM_CounterSet2_Tabular(benchmark::State& state) {
} }
BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16); BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16);
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
......
test/user_counters_test.cc
...@@ -32,6 +32,7 @@ BENCHMARK(BM_Counters_Simple); ...@@ -32,6 +32,7 @@ BENCHMARK(BM_Counters_Simple);
ADD_CASES(TC_ConsoleOut, ADD_CASES(TC_ConsoleOut,
{{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}}); {{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
...@@ -70,6 +71,7 @@ ADD_CASES(TC_ConsoleOut, ...@@ -70,6 +71,7 @@ ADD_CASES(TC_ConsoleOut,
{{"^BM_Counters_WithBytesAndItemsPSec %console_report " {{"^BM_Counters_WithBytesAndItemsPSec %console_report "
"bar=%hrfloat foo=%hrfloat +%hrfloatB/s +%hrfloat items/s$"}}); "bar=%hrfloat foo=%hrfloat +%hrfloatB/s +%hrfloat items/s$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
...@@ -110,6 +112,7 @@ ADD_CASES( ...@@ -110,6 +112,7 @@ ADD_CASES(
TC_ConsoleOut, TC_ConsoleOut,
{{"^BM_Counters_Rate %console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); {{"^BM_Counters_Rate %console_report bar=%hrfloat/s foo=%hrfloat/s$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
...@@ -142,6 +145,7 @@ BENCHMARK(BM_Counters_Threads)->ThreadRange(1, 8); ...@@ -142,6 +145,7 @@ BENCHMARK(BM_Counters_Threads)->ThreadRange(1, 8);
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report " ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report "
"bar=%hrfloat foo=%hrfloat$"}}); "bar=%hrfloat foo=%hrfloat$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
...@@ -175,6 +179,7 @@ BENCHMARK(BM_Counters_AvgThreads)->ThreadRange(1, 8); ...@@ -175,6 +179,7 @@ BENCHMARK(BM_Counters_AvgThreads)->ThreadRange(1, 8);
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int " ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int "
"%console_report bar=%hrfloat foo=%hrfloat$"}}); "%console_report bar=%hrfloat foo=%hrfloat$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
...@@ -210,6 +215,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int " ...@@ -210,6 +215,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int "
"%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}});
ADD_CASES(TC_JSONOut, ADD_CASES(TC_JSONOut,
{{"\"name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$"}, {{"\"name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
...@@ -243,6 +249,7 @@ BENCHMARK(BM_Counters_IterationInvariant); ...@@ -243,6 +249,7 @@ BENCHMARK(BM_Counters_IterationInvariant);
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_IterationInvariant %console_report " ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_IterationInvariant %console_report "
"bar=%hrfloat foo=%hrfloat$"}}); "bar=%hrfloat foo=%hrfloat$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_IterationInvariant\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_IterationInvariant\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
...@@ -281,6 +288,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kIsIterationInvariantRate " ...@@ -281,6 +288,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kIsIterationInvariantRate "
"%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}});
ADD_CASES(TC_JSONOut, ADD_CASES(TC_JSONOut,
{{"\"name\": \"BM_Counters_kIsIterationInvariantRate\",$"}, {{"\"name\": \"BM_Counters_kIsIterationInvariantRate\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
...@@ -317,6 +325,7 @@ BENCHMARK(BM_Counters_AvgIterations); ...@@ -317,6 +325,7 @@ BENCHMARK(BM_Counters_AvgIterations);
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgIterations %console_report " ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgIterations %console_report "
"bar=%hrfloat foo=%hrfloat$"}}); "bar=%hrfloat foo=%hrfloat$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgIterations\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgIterations\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
...@@ -352,6 +361,7 @@ BENCHMARK(BM_Counters_kAvgIterationsRate); ...@@ -352,6 +361,7 @@ BENCHMARK(BM_Counters_kAvgIterationsRate);
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kAvgIterationsRate " ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kAvgIterationsRate "
"%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_kAvgIterationsRate\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_kAvgIterationsRate\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment