Skip to content

B
benchmark
Commit 020bac98 by Joao Paulo Magalhaes
Options

Extend tabular counter tests to different counter sets.

parent 25060449
Showing with 144 additions and 50 deletions
test/user_counters_tabular_test.cc
...@@ -4,16 +4,50 @@ ...@@ -4,16 +4,50 @@
#include "benchmark/benchmark.h" #include "benchmark/benchmark.h"
#include "output_test.h" #include "output_test.h"
// ========================================================================= // // @todo: <jpmag> this checks the full output at once; the rule for
// ---------------------- Testing Prologue Output -------------------------- // // CounterSet1 was failing because it was not matching "^[-]+$".
// ========================================================================= // // @todo: <jpmag> check that the counters are vertically aligned.
ADD_CASES(TC_ConsoleOut, {
ADD_CASES(TC_ConsoleOut, // keeping these lines long improves readability, so:
{{"^[-]+$", MR_Next}, // clang-format off
{"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Bat %s Baz %s Foo %s Frob %s Lob$", {"^[-]+$", MR_Next},
MR_Next}, {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Bat %s Baz %s Foo %s Frob %s Lob$", MR_Next},
{"^[-]+$", MR_Next}}); {"^[-]+$", MR_Next},
ADD_CASES(TC_CSVOut, {{"%csv_header,\"Bar\",\"Bat\",\"Baz\",\"Foo\",\"Frob\",\"Lob\""}}); {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
{"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
{"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
{"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
{"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
{"^[-]+$", MR_Next},
{"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Baz %s Foo$", MR_Next},
{"^[-]+$", MR_Next},
{"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^[-]+$", MR_Next},
{"^Benchmark %s Time %s CPU %s Iterations %s Bat %s Baz %s Foo$", MR_Next},
{"^[-]+$", MR_Next},
{"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$"},
// clang-format on
});
ADD_CASES(TC_CSVOut, {{"%csv_header,"
"\"Bar\",\"Bat\",\"Baz\",\"Foo\",\"Frob\",\"Lob\""}});
// ========================================================================= // // ========================================================================= //
// ------------------------- Tabular Counters Output ----------------------- // // ------------------------- Tabular Counters Output ----------------------- //
...@@ -22,20 +56,18 @@ ADD_CASES(TC_CSVOut, {{"%csv_header,\"Bar\",\"Bat\",\"Baz\",\"Foo\",\"Frob\",\"L ...@@ -22,20 +56,18 @@ ADD_CASES(TC_CSVOut, {{"%csv_header,\"Bar\",\"Bat\",\"Baz\",\"Foo\",\"Frob\",\"L
void BM_Counters_Tabular(benchmark::State& state) { void BM_Counters_Tabular(benchmark::State& state) {
while (state.KeepRunning()) { while (state.KeepRunning()) {
} }
namespace bm = benchmark;
state.counters.insert({ state.counters.insert({
{"Foo", 1}, {"Foo", { 1, bm::Counter::kAvgThreads}},
{"Bar", 2}, {"Bar", { 2, bm::Counter::kAvgThreads}},
{"Baz", 4}, {"Baz", { 4, bm::Counter::kAvgThreads}},
{"Bat", 8}, {"Bat", { 8, bm::Counter::kAvgThreads}},
{"Frob", 16}, {"Frob", {16, bm::Counter::kAvgThreads}},
{"Lob", 32}, {"Lob", {32, bm::Counter::kAvgThreads}},
}); });
} }
BENCHMARK(BM_Counters_Tabular); BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 16);
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Tabular %console_report " ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"},
"[ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat "
"[ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Tabular\",$"},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %int,$", MR_Next}, {"\"real_time\": %int,$", MR_Next},
{"\"cpu_time\": %int,$", MR_Next}, {"\"cpu_time\": %int,$", MR_Next},
...@@ -47,7 +79,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Tabular\",$"}, ...@@ -47,7 +79,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Tabular\",$"},
{"\"Frob\": %float,$", MR_Next}, {"\"Frob\": %float,$", MR_Next},
{"\"Lob\": %float$", MR_Next}, {"\"Lob\": %float$", MR_Next},
{"}", MR_Next}}); {"}", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular\",%csv_report," ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/threads:%int\",%csv_report,"
"%float,%float,%float,%float,%float,%float$"}}); "%float,%float,%float,%float,%float,%float$"}});
// VS2013 does not allow this function to be passed as a lambda argument // VS2013 does not allow this function to be passed as a lambda argument
// to CHECK_BENCHMARK_RESULTS() // to CHECK_BENCHMARK_RESULTS()
...@@ -59,7 +91,7 @@ void CheckTabular(Results const& e) { ...@@ -59,7 +91,7 @@ void CheckTabular(Results const& e) {
CHECK_COUNTER_VALUE(e, int, "Frob", EQ, 16); CHECK_COUNTER_VALUE(e, int, "Frob", EQ, 16);
CHECK_COUNTER_VALUE(e, int, "Lob", EQ, 32); CHECK_COUNTER_VALUE(e, int, "Lob", EQ, 32);
} }
CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular", &CheckTabular); CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/threads:%int", &CheckTabular);
// ========================================================================= // // ========================================================================= //
// -------------------- Tabular+Rate Counters Output ----------------------- // // -------------------- Tabular+Rate Counters Output ----------------------- //
...@@ -70,19 +102,16 @@ void BM_CounterRates_Tabular(benchmark::State& state) { ...@@ -70,19 +102,16 @@ void BM_CounterRates_Tabular(benchmark::State& state) {
} }
namespace bm = benchmark; namespace bm = benchmark;
state.counters.insert({ state.counters.insert({
{"Foo", { 1, bm::Counter::kIsRate}}, {"Foo", { 1, bm::Counter::kAvgThreadsRate}},
{"Bar", { 2, bm::Counter::kIsRate}}, {"Bar", { 2, bm::Counter::kAvgThreadsRate}},
{"Baz", { 4, bm::Counter::kIsRate}}, {"Baz", { 4, bm::Counter::kAvgThreadsRate}},
{"Bat", { 8, bm::Counter::kIsRate}}, {"Bat", { 8, bm::Counter::kAvgThreadsRate}},
{"Frob", {16, bm::Counter::kIsRate}}, {"Frob", {16, bm::Counter::kAvgThreadsRate}},
{"Lob", {32, bm::Counter::kIsRate}}, {"Lob", {32, bm::Counter::kAvgThreadsRate}},
}); });
} }
BENCHMARK(BM_CounterRates_Tabular); BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16);
ADD_CASES(TC_ConsoleOut, {{"^BM_CounterRates_Tabular %console_report " ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"},
"[ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s "
"[ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterRates_Tabular\",$"},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %int,$", MR_Next}, {"\"real_time\": %int,$", MR_Next},
{"\"cpu_time\": %int,$", MR_Next}, {"\"cpu_time\": %int,$", MR_Next},
...@@ -94,7 +123,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterRates_Tabular\",$"}, ...@@ -94,7 +123,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterRates_Tabular\",$"},
{"\"Frob\": %float,$", MR_Next}, {"\"Frob\": %float,$", MR_Next},
{"\"Lob\": %float$", MR_Next}, {"\"Lob\": %float$", MR_Next},
{"}", MR_Next}}); {"}", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_CounterRates_Tabular\",%csv_report," ADD_CASES(TC_CSVOut, {{"^\"BM_CounterRates_Tabular/threads:%int\",%csv_report,"
"%float,%float,%float,%float,%float,%float$"}}); "%float,%float,%float,%float,%float,%float$"}});
// VS2013 does not allow this function to be passed as a lambda argument // VS2013 does not allow this function to be passed as a lambda argument
// to CHECK_BENCHMARK_RESULTS() // to CHECK_BENCHMARK_RESULTS()
...@@ -107,29 +136,26 @@ void CheckTabularRate(Results const& e) { ...@@ -107,29 +136,26 @@ void CheckTabularRate(Results const& e) {
CHECK_FLOAT_COUNTER_VALUE(e, "Frob", EQ, 16./t, 0.001); CHECK_FLOAT_COUNTER_VALUE(e, "Frob", EQ, 16./t, 0.001);
CHECK_FLOAT_COUNTER_VALUE(e, "Lob", EQ, 32./t, 0.001); CHECK_FLOAT_COUNTER_VALUE(e, "Lob", EQ, 32./t, 0.001);
} }
CHECK_BENCHMARK_RESULTS("BM_CounterRates_Tabular", &CheckTabularRate); CHECK_BENCHMARK_RESULTS("BM_CounterRates_Tabular/threads:%int",
&CheckTabularRate);
// ========================================================================= // // ========================================================================= //
// ------------------------- Tabular Counters Output ----------------------- // // ------------------------- Tabular Counters Output ----------------------- //
// ========================================================================= // // ========================================================================= //
// set only some of the counters // set only some of the counters
void BM_CounterSet_Tabular(benchmark::State& state) { void BM_CounterSet0_Tabular(benchmark::State& state) {
while (state.KeepRunning()) { while (state.KeepRunning()) {
} }
namespace bm = benchmark;
state.counters.insert({ state.counters.insert({
{"Foo", 10}, {"Foo", {10, bm::Counter::kAvgThreads}},
{"Bar", 20}, {"Bar", {20, bm::Counter::kAvgThreads}},
{"Baz", 40}, {"Baz", {40, bm::Counter::kAvgThreads}},
}); });
} }
BENCHMARK(BM_CounterSet_Tabular); BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16);
ADD_CASES(TC_ConsoleOut, { ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"},
{"^[-]+$", MR_Next},
{"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Baz %s Foo$", MR_Next},
{"^[-]+$", MR_Next},
{"^BM_CounterSet_Tabular %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet_Tabular\",$"},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %int,$", MR_Next}, {"\"real_time\": %int,$", MR_Next},
{"\"cpu_time\": %int,$", MR_Next}, {"\"cpu_time\": %int,$", MR_Next},
...@@ -138,16 +164,84 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet_Tabular\",$"}, ...@@ -138,16 +164,84 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet_Tabular\",$"},
{"\"Baz\": %float,$", MR_Next}, {"\"Baz\": %float,$", MR_Next},
{"\"Foo\": %float$", MR_Next}, {"\"Foo\": %float$", MR_Next},
{"}", MR_Next}}); {"}", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet_Tabular\",%csv_report," ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet0_Tabular/threads:%int\",%csv_report,"
"%float,,%float,%float,,"}}); "%float,,%float,%float,,"}});
// VS2013 does not allow this function to be passed as a lambda argument // VS2013 does not allow this function to be passed as a lambda argument
// to CHECK_BENCHMARK_RESULTS() // to CHECK_BENCHMARK_RESULTS()
void CheckSet(Results const& e) { void CheckSet0(Results const& e) {
CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10); CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10);
CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 20); CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 20);
CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40); CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40);
} }
CHECK_BENCHMARK_RESULTS("BM_CounterSet_Tabular", &CheckSet); CHECK_BENCHMARK_RESULTS("BM_CounterSet0_Tabular", &CheckSet0);
// again.
void BM_CounterSet1_Tabular(benchmark::State& state) {
while (state.KeepRunning()) {
}
namespace bm = benchmark;
state.counters.insert({
{"Foo", {15, bm::Counter::kAvgThreads}},
{"Bar", {25, bm::Counter::kAvgThreads}},
{"Baz", {45, bm::Counter::kAvgThreads}},
});
}
BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16);
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %int,$", MR_Next},
{"\"cpu_time\": %int,$", MR_Next},
{"\"time_unit\": \"ns\",$", MR_Next},
{"\"Bar\": %float,$", MR_Next},
{"\"Baz\": %float,$", MR_Next},
{"\"Foo\": %float$", MR_Next},
{"}", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet1_Tabular/threads:%int\",%csv_report,"
"%float,,%float,%float,,"}});
// VS2013 does not allow this function to be passed as a lambda argument
// to CHECK_BENCHMARK_RESULTS()
void CheckSet1(Results const& e) {
CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 15);
CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 25);
CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 45);
}
CHECK_BENCHMARK_RESULTS("BM_CounterSet1_Tabular/threads:%int", &CheckSet1);
// ========================================================================= //
// ------------------------- Tabular Counters Output ----------------------- //
// ========================================================================= //
// set only some of the counters, different set now.
void BM_CounterSet2_Tabular(benchmark::State& state) {
while (state.KeepRunning()) {
}
namespace bm = benchmark;
state.counters.insert({
{"Foo", {10, bm::Counter::kAvgThreads}},
{"Bat", {30, bm::Counter::kAvgThreads}},
{"Baz", {40, bm::Counter::kAvgThreads}},
});
}
BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16);
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %int,$", MR_Next},
{"\"cpu_time\": %int,$", MR_Next},
{"\"time_unit\": \"ns\",$", MR_Next},
{"\"Bat\": %float,$", MR_Next},
{"\"Baz\": %float,$", MR_Next},
{"\"Foo\": %float$", MR_Next},
{"}", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet2_Tabular/threads:%int\",%csv_report,"
",%float,%float,%float,,"}});
// VS2013 does not allow this function to be passed as a lambda argument
// to CHECK_BENCHMARK_RESULTS()
void CheckSet2(Results const& e) {
CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10);
CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 30);
CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40);
}
CHECK_BENCHMARK_RESULTS("BM_CounterSet2_Tabular", &CheckSet2);
// ========================================================================= // // ========================================================================= //
// --------------------------- TEST CASES END ------------------------------ // // --------------------------- TEST CASES END ------------------------------ //
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment