Refactor output test runner into standalone module. (#277)

test/CMakeLists.txt

@@ -2,11 +2,20 @@
 
 find_package(Threads REQUIRED)
 
+add_library(output_test_helper STATIC output_test_helper.cc)
+
 macro(compile_benchmark_test name)
   add_executable(${name} "${name}.cc")
   target_link_libraries(${name} benchmark ${CMAKE_THREAD_LIBS_INIT})
 endmacro(compile_benchmark_test)
 
+
+macro(compile_output_test name)
+  add_executable(${name} "${name}.cc")
+  target_link_libraries(${name} output_test_helper benchmark ${CMAKE_THREAD_LIBS_INIT})
+endmacro(compile_output_test)
+
+
 # Demonstration executable
 compile_benchmark_test(benchmark_test)
 add_test(benchmark benchmark_test --benchmark_min_time=0.01)
@@ -54,7 +63,7 @@ add_test(map_test map_test --benchmark_min_time=0.01)
 compile_benchmark_test(multiple_ranges_test)
 add_test(multiple_ranges_test multiple_ranges_test --benchmark_min_time=0.01)
 
-compile_benchmark_test(reporter_output_test)
+compile_output_test(reporter_output_test)
 add_test(reporter_output_test reporter_output_test --benchmark_min_time=0.01)
 
 check_cxx_compiler_flag(-std=c++03 BENCHMARK_HAS_CXX03_FLAG)
@@ -69,7 +78,7 @@ if (BENCHMARK_HAS_CXX03_FLAG)
   add_test(cxx03 cxx03_test --benchmark_min_time=0.01)
 endif()
 
-compile_benchmark_test(complexity_test)
+compile_output_test(complexity_test)
 add_test(complexity_benchmark complexity_test --benchmark_min_time=0.01)
 
 # Add the coverage command(s)

test/output_test.h

+#ifndef TEST_OUTPUT_TEST_H
+#define TEST_OUTPUT_TEST_H
+
+#undef NDEBUG
+#include "benchmark/benchmark.h"
+#include "../src/re.h"
+#include <vector>
+#include <string>
+#include <initializer_list>
+#include <memory>
+#include <utility>
+
+#define CONCAT2(x, y) x##y
+#define CONCAT(x, y) CONCAT2(x, y)
+
+#define ADD_CASES(...) \
+    int CONCAT(dummy, __LINE__) = ::AddCases(__VA_ARGS__)
+
+#define SET_SUBSTITUTIONS(...) \
+    int CONCAT(dummy, __LINE__) = ::SetSubstitutions(__VA_ARGS__)
+
+enum MatchRules {
+  MR_Default, // Skip non-matching lines until a match is found.
+  MR_Next,    // Match must occur on the next line.
+  MR_Not      // No line between the current position and the next match matches
+              // the regex
+};
+
+struct TestCase {
+  TestCase(std::string re, int rule = MR_Default);
+
+  std::string regex_str;
+  int match_rule;
+  std::string substituted_regex;
+  std::shared_ptr<benchmark::Regex> regex;
+};
+
+enum TestCaseID {
+  TC_ConsoleOut,
+  TC_ConsoleErr,
+  TC_JSONOut,
+  TC_JSONErr,
+  TC_CSVOut,
+  TC_CSVErr,
+
+  TC_NumID // PRIVATE
+};
+
+// Add a list of test cases to be run against the output specified by
+// 'ID'
+int AddCases(TestCaseID ID, std::initializer_list<TestCase> il);
+
+// Add or set a list of substitutions to be performed on constructed regex's
+// See 'output_test_helper.cc' for a list of default substitutions.
+int SetSubstitutions(
+    std::initializer_list<std::pair<std::string, std::string>> il);
+
+// Run all output tests.
+void RunOutputTests(int argc, char* argv[]);
+
+// ========================================================================= //
+// --------------------------- Misc Utilities ------------------------------ //
+// ========================================================================= //
+
+namespace {
+
+const char* const dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
+
+} //  end namespace
+
+
+#endif // TEST_OUTPUT_TEST_H
\ No newline at end of file

test/output_test_helper.cc

+#include "output_test.h"
+#include "../src/check.h" // NOTE: check.h is for internal use only!
+#include "../src/re.h" // NOTE: re.h is for internal use only
+#include <memory>
+#include <map>
+#include <iostream>
+#include <sstream>
+
+
+// ========================================================================= //
+// ------------------------------ Internals -------------------------------- //
+// ========================================================================= //
+namespace internal { namespace {
+
+using TestCaseList = std::vector<TestCase>;
+
+// Use a vector because the order elements are added matters during iteration.
+// std::map/unordered_map don't guarantee that.
+// For example:
+//  SetSubstitutions({{"%HelloWorld", "Hello"}, {"%Hello", "Hi"}});
+//     Substitute("%HelloWorld") // Always expands to Hello.
+using SubMap = std::vector<std::pair<std::string, std::string>>;
+
+TestCaseList& GetTestCaseList(TestCaseID ID) {
+    // Uses function-local statics to ensure initialization occurs
+    // before first use.
+    static TestCaseList lists[TC_NumID];
+    return lists[ID];
+}
+
+SubMap& GetSubstitutions() {
+    // Don't use 'dec_re' from header because it may not yet be initialized.
+    static std::string dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
+    static SubMap map = {
+        {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"},
+        {"%int", "[ ]*[0-9]+"},
+        {" %s ", "[ ]+"},
+        {"%time", "[ ]*[0-9]{1,5} ns"},
+        {"%console_report", "[ ]*[0-9]{1,5} ns [ ]*[0-9]{1,5} ns [ ]*[0-9]+"},
+        {"%csv_report", "[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,"}
+    };
+    return map;
+}
+
+std::string PerformSubstitutions(std::string source) {
+    SubMap const& subs = GetSubstitutions();
+    using SizeT = std::string::size_type;
+    for (auto const& KV : subs) {
+        SizeT pos;
+        SizeT next_start = 0;
+        while ((pos = source.find(KV.first, next_start)) != std::string::npos) {
+            next_start = pos + KV.second.size();
+            source.replace(pos, KV.first.size(), KV.second);
+        }
+    }
+    return source;
+}
+
+void CheckCase(std::stringstream& remaining_output, TestCase const& TC,
+               TestCaseList const& not_checks)
+{
+    std::string first_line;
+    bool on_first = true;
+    std::string line;
+    while (remaining_output.eof() == false) {
+        CHECK(remaining_output.good());
+        std::getline(remaining_output, line);
+        if (on_first) {
+            first_line = line;
+            on_first = false;
+        }
+        for (auto& NC : not_checks) {
+            CHECK(!NC.regex->Match(line)) << "Unexpected match for line \""
+                                          << line << "\" for MR_Not regex \""
+                                          << NC.regex_str << "\"";
+        }
+        if (TC.regex->Match(line)) return;
+        CHECK(TC.match_rule != MR_Next) << "Expected line \"" << line
+                                     << "\" to match regex \"" << TC.regex_str << "\"";
+    }
+    CHECK(remaining_output.eof() == false)
+        << "End of output reached before match for regex \"" << TC.regex_str
+        << "\" was found"
+        << "\n    actual regex string \"" << TC.substituted_regex << "\""
+        << "\n    started matching near: " << first_line;
+}
+
+
+void CheckCases(TestCaseList const& checks, std::stringstream& output) {
+    std::vector<TestCase> not_checks;
+    for (size_t i=0; i < checks.size(); ++i) {
+        const auto& TC = checks[i];
+        if (TC.match_rule == MR_Not) {
+            not_checks.push_back(TC);
+            continue;
+        }
+        CheckCase(output, TC, not_checks);
+        not_checks.clear();
+    }
+}
+
+class TestReporter : public benchmark::BenchmarkReporter {
+public:
+  TestReporter(std::vector<benchmark::BenchmarkReporter*> reps)
+      : reporters_(reps)  {}
+
+  virtual bool ReportContext(const Context& context) {
+    bool last_ret = false;
+    bool first = true;
+    for (auto rep : reporters_) {
+      bool new_ret = rep->ReportContext(context);
+      CHECK(first || new_ret == last_ret)
+          << "Reports return different values for ReportContext";
+      first = false;
+      last_ret = new_ret;
+    }
+    return last_ret;
+  }
+
+  void ReportRuns(const std::vector<Run>& report)
+    { for (auto rep : reporters_) rep->ReportRuns(report); }
+  void Finalize() { for (auto rep : reporters_) rep->Finalize(); }
+
+private:
+  std::vector<benchmark::BenchmarkReporter*> reporters_;
+};
+
+}} // end namespace internal
+
+// ========================================================================= //
+// -------------------------- Public API Definitions------------------------ //
+// ========================================================================= //
+
+TestCase::TestCase(std::string re, int rule)
+    : regex_str(std::move(re)), match_rule(rule),
+      substituted_regex(internal::PerformSubstitutions(regex_str)),
+      regex(std::make_shared<benchmark::Regex>())
+{
+    std::string err_str;
+    regex->Init(substituted_regex, &err_str);
+    CHECK(err_str.empty())
+        << "Could not construct regex \"" << substituted_regex << "\""
+        << "\n    originally \"" << regex_str << "\""
+        << "\n    got error: " << err_str;
+}
+
+int AddCases(TestCaseID ID, std::initializer_list<TestCase> il) {
+    auto& L = internal::GetTestCaseList(ID);
+    L.insert(L.end(), il);
+    return 0;
+}
+
+int SetSubstitutions(std::initializer_list<std::pair<std::string, std::string>> il) {
+    auto& subs = internal::GetSubstitutions();
+    for (auto const& KV : il) {
+        bool exists = false;
+        for (auto& EKV : subs) {
+            if (EKV.first == KV.first) {
+                EKV.second = KV.second;
+                exists = true;
+                break;
+            }
+        }
+        if (!exists) subs.push_back(KV);
+    }
+    return 0;
+}
+
+void RunOutputTests(int argc, char* argv[]) {
+  using internal::GetTestCaseList;
+  benchmark::Initialize(&argc, argv);
+  benchmark::ConsoleReporter CR(benchmark::ConsoleReporter::OO_None);
+  benchmark::JSONReporter JR;
+  benchmark::CSVReporter CSVR;
+  struct ReporterTest {
+    const char* name;
+    std::vector<TestCase>& output_cases;
+    std::vector<TestCase>& error_cases;
+    benchmark::BenchmarkReporter& reporter;
+    std::stringstream out_stream;
+    std::stringstream err_stream;
+
+    ReporterTest(const char* n,
+                 std::vector<TestCase>& out_tc,
+                 std::vector<TestCase>& err_tc,
+                 benchmark::BenchmarkReporter& br)
+        : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) {
+        reporter.SetOutputStream(&out_stream);
+        reporter.SetErrorStream(&err_stream);
+    }
+  } TestCases[] = {
+      {"ConsoleReporter", GetTestCaseList(TC_ConsoleOut),
+                          GetTestCaseList(TC_ConsoleErr), CR},
+      {"JSONReporter",    GetTestCaseList(TC_JSONOut),
+                          GetTestCaseList(TC_JSONErr), JR},
+      {"CSVReporter",     GetTestCaseList(TC_CSVOut),
+                          GetTestCaseList(TC_CSVErr), CSVR},
+  };
+
+  // Create the test reporter and run the benchmarks.
+  std::cout << "Running benchmarks...\n";
+  internal::TestReporter test_rep({&CR, &JR, &CSVR});
+  benchmark::RunSpecifiedBenchmarks(&test_rep);
+
+  for (auto& rep_test : TestCases) {
+      std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n";
+      std::string banner(msg.size() - 1, '-');
+      std::cout << banner << msg << banner << "\n";
+
+      std::cerr << rep_test.err_stream.str();
+      std::cout << rep_test.out_stream.str();
+
+      internal::CheckCases(rep_test.error_cases,rep_test.err_stream);
+      internal::CheckCases(rep_test.output_cases, rep_test.out_stream);
+
+      std::cout << "\n";
+  }
+}
+
+