Begin JSON reporter change: Allow users to specify arbitrary JSON input/output. (#499)

.travis.yml

@@ -11,25 +11,50 @@ matrix:
     - compiler: gcc
       addons:
         apt:
+          sources:
+            ubuntu-toolchain-r-test
           packages:
             - lcov
-      env: COMPILER=g++ C_COMPILER=gcc BUILD_TYPE=Coverage
+            - g++-4.9
+      env: COMPILER=g++-4.9 C_COMPILER=gcc-4.9 BUILD_TYPE=Coverage
     - compiler: gcc
-      env: COMPILER=g++ C_COMPILER=gcc BUILD_TYPE=Debug
+      addons:
+        apt:
+          sources:
+            ubuntu-toolchain-r-test
+          packages:
+            - g++-4.9
+      env: COMPILER=g++-4.9 C_COMPILER=gcc-4.9 BUILD_TYPE=Debug
     - compiler: gcc
-      env: COMPILER=g++ C_COMPILER=gcc BUILD_TYPE=Release
+      addons:
+        apt:
+          sources:
+            ubuntu-toolchain-r-test
+          packages:
+            - g++-4.9
+      env: COMPILER=g++-4.9 C_COMPILER=gcc-4.9 BUILD_TYPE=Release
     - compiler: gcc
       addons:
         apt:
+          sources:
+            ubuntu-toolchain-r-test
           packages:
-            - g++-multilib
-      env: COMPILER=g++ C_COMPILER=gcc BUILD_TYPE=Debug BUILD_32_BITS=ON
+            - g++-4.9
+            - gcc-4.9-multilib
+            - g++-4.9-multilib
+            - gcc-multilib
+      env: COMPILER=g++-4.9 C_COMPILER=gcc-4.9 BUILD_TYPE=Debug BUILD_32_BITS=ON
     - compiler: gcc
       addons:
         apt:
+          sources:
+            ubuntu-toolchain-r-test
           packages:
-            - g++-multilib
-      env: COMPILER=g++ C_COMPILER=gcc BUILD_TYPE=Release BUILD_32_BITS=ON
+            - g++-4.9
+            - gcc-4.9-multilib
+            - g++-4.9-multilib
+            - gcc-multilib
+      env: COMPILER=g++-4.9 C_COMPILER=gcc-4.9 BUILD_TYPE=Release BUILD_32_BITS=ON
     - compiler: gcc
       addons:
         apt:
@@ -41,8 +66,20 @@ matrix:
         - COMPILER=g++-6 C_COMPILER=gcc-6  BUILD_TYPE=Debug
         - EXTRA_FLAGS="-fno-omit-frame-pointer -g -O2 -fsanitize=undefined,address -fuse-ld=gold"
     - compiler: clang
+      addons:
+        apt:
+          sources:
+            ubuntu-toolchain-r-test
+          packages:
+            - g++-4.9
       env: COMPILER=clang++ C_COMPILER=clang BUILD_TYPE=Debug
     - compiler: clang
+      addons:
+        apt:
+          sources:
+            ubuntu-toolchain-r-test
+          packages:
+            - g++-4.9
       env: COMPILER=clang++ C_COMPILER=clang BUILD_TYPE=Release
     # Clang w/ libc++
     - compiler: clang
@@ -123,12 +160,12 @@ matrix:
       osx_image: xcode8.3
       compiler: clang
       env:
-        - COMPILER=clang++ BUILD_TYPE=Debug
+        - COMPILER=clang++ C_COMPILER=clang BUILD_TYPE=Debug
     - os: osx
       osx_image: xcode8.3
       compiler: clang
       env:
-        - COMPILER=clang++ BUILD_TYPE=Release
+        - COMPILER=clang++ C_COMPILER=clang BUILD_TYPE=Release
     - os: osx
       osx_image: xcode8.3
       compiler: gcc
@@ -157,6 +194,7 @@ install:
     fi
 
 script:
+  - export CC=${C_COMPILER} CXX=${COMPILER}
   - cmake -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${COMPILER} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_CXX_FLAGS="${EXTRA_FLAGS}" -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON -DBENCHMARK_BUILD_32_BITS=${BUILD_32_BITS} ..
   - make
   - ctest -C ${BUILD_TYPE} --output-on-failure

CMakeLists.txt

@@ -200,6 +200,8 @@ find_package(Threads REQUIRED)
 
 # Set up directories
 include_directories(${PROJECT_SOURCE_DIR}/include)
+set(JSON_INCLUDE_DIR "${PROJECT_SOURCE_DIR}/third_party/json/src")
+include_directories(${JSON_INCLUDE_DIR})
 
 # Build the targets
 add_subdirectory(src)

README.md

@@ -189,6 +189,23 @@ pair.
 BENCHMARK(BM_SetInsert)->Ranges({{1<<10, 8<<10}, {128, 512}});
 ```
 
+### Specifying JSON Input Arguments.
+Benchmarks can also accept arbitrary JSON arguments as input using the `WithInput`
+function. See [Working With JSON Objects for more information](#working-with-json-objects).
+
+```c++
+BENCHMARK(BM_Foo)
+->WithInput({
+  {"name", "case1"},
+  {"size", 42},
+  {"input_list", {1, 2, 3, 4}}
+})
+->WithInput({
+  {"name", "case2"},
+  {"use_random_size", true}
+});
+```
+
 For more complex patterns of inputs, passing a custom function to `Apply` allows
 programmatic specification of an arbitrary set of arguments on which to run the
 benchmark. The following example enumerates a dense range on one parameter,
@@ -801,6 +818,46 @@ BM_memcpy/32          12 ns         12 ns   54687500
 BM_memcpy/32k       1834 ns       1837 ns     357143
 ```
 
+## Working with JSON Objects
+
+The benchmark library uses JSON as it's primary input and output format. Specifically
+Google benchmark uses [nlohmann's JSON library](https://github.com/nlohmann/json),
+except with the `nlohmann` namespace changed into `benchmark` to avoid conflict
+with other external users of the header.
+
+For examples and in-depth documentation about using the JSON library, please
+refer to the [original documentation](https://github.com/nlohmann/json#examples).
+
+Because compiling the JSON header can be quite slow, Google Benchmark provides
+the option `BENCHMARK_HAS_NO_JSON_HEADER` which can be defined to to disable
+including it when building benchmarks. However, this also disables usage
+of all API's which traffic in json types.
+
+## Reporting Arbitrary Benchmark Results using JSON.
+
+The benchmark library allows benchmarks to report arbitrary user output as JSON
+using `State::operator[](const std::string &)`. For example:
+
+```c++
+void BM_JSONOutput(benchmark::State& st) {
+  int my_count = 0;
+  bool saw_failure = false;
+  std::vector<int> inputs = GenerateInputs();
+  for (auto _ : st) {
+    /* ... */
+  }
+  st["my_count"] = my_count;
+  st["saw_failure"] = saw_failure;
+  st["inputs"] = inputs;
+}
+```
+
+Currently, only the JSON reporter displays custom user output; but this will
+be fixed in future revisions. Additionally, future revisions will allow
+user counters to be represented and reported using the JSON interface.
+
+Additionally, for multi-threaded benchmarks, if two threads report a field
+with the same name, the first reported value will be chosen.
 
 ## Output Formats
 The library supports multiple output formats. Use the
@@ -904,9 +961,9 @@ a modern C++ toolchain, both compiler and standard library.
 
 The following minimum versions are strongly recommended build the library:
 
-* GCC 4.8
+* GCC 4.9
 * Clang 3.4
-* Visual Studio 2013
+* Visual Studio 2015
 * Intel 2015 Update 1
 
 Anything older *may* work.

appveyor.yml

@@ -20,12 +20,6 @@ environment:
     - compiler: msvc-14-seh
       generator: "Visual Studio 14 2015 Win64"
 
-    - compiler: msvc-12-seh
-      generator: "Visual Studio 12 2013"
-
-    - compiler: msvc-12-seh
-      generator: "Visual Studio 12 2013 Win64"
-
     - compiler: gcc-5.3.0-posix
       generator: "MinGW Makefiles"
       cxx_path: 'C:\mingw-w64\i686-5.3.0-posix-dwarf-rt_v4-rev0\mingw32\bin'

include/benchmark/benchmark.h

@@ -184,6 +184,14 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond);
 #include <type_traits>
 #include <initializer_list>
 #include <utility>
+#else
+#ifndef BENCHMARK_HAS_NO_JSON_HEADER
+#define BENCHMARK_HAS_NO_JSON_HEADER
+#endif
+#endif
+
+#if !defined(BENCHMARK_HAS_NO_JSON_HEADER)
+#include "json.hpp"
 #endif
 
 #if defined(_MSC_VER)
@@ -272,11 +280,55 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* console_reporter,
 // TODO(dominic)
 // void MemoryUsage();
 
+#ifndef BENCHMARK_HAS_NO_JSON_HEADER
+using json = nlohmann::json;
+#endif
+
 namespace internal {
 class Benchmark;
 class BenchmarkImp;
 class BenchmarkFamilies;
 
+/// This class allows objects to store references to json object even
+/// when the header is not currently available. Technically this class
+/// causes ODR violations, but it's unlikely to cause problems.
+class JSONPointer {
+ public:
+  JSONPointer();
+  ~JSONPointer();
+#ifndef BENCHMARK_HAS_NO_JSON_HEADER
+  explicit JSONPointer(json J);
+  inline JSONPointer(JSONPointer&& JP) noexcept;
+  inline JSONPointer& operator=(JSONPointer&& JP) noexcept;
+  inline json& get() const;
+  inline json* operator->() const;
+  json& operator*() const { return get(); }
+#endif
+ private:
+  struct PIMPL;
+  PIMPL* ptr_;
+#ifndef BENCHMARK_HAS_CXX11
+  BENCHMARK_DISALLOW_COPY_AND_ASSIGN(JSONPointer);
+#endif
+};
+
+#ifndef BENCHMARK_HAS_NO_JSON_HEADER
+struct JSONPointer::PIMPL {
+  json value;
+};
+inline JSONPointer::JSONPointer(JSONPointer&& JP) noexcept : ptr_(JP.ptr_) {
+  JP.ptr_ = nullptr;
+}
+inline JSONPointer& JSONPointer::operator=(JSONPointer&& JP) noexcept {
+  PIMPL* tmp = ptr_;
+  ptr_ = JP.ptr_;
+  JP.ptr_ = tmp;
+  return *this;
+}
+inline json& JSONPointer::get() const { return ptr_->value; }
+inline json* JSONPointer::operator->() const { return &ptr_->value; }
+#endif  // BENCHMARK_HAS_NO_JSON_HEADER
+
 void UseCharPointer(char const volatile*);
 
 // Take ownership of the pointer and register the benchmark. Return the
@@ -551,6 +603,24 @@ class State {
     this->SetLabel(str.c_str());
   }
 
+#ifndef BENCHMARK_HAS_NO_JSON_HEADER
+  // Return the JSON input specified when registering the benchmark, if any;
+  // otherwise, return an empty JSON object.
+  BENCHMARK_ALWAYS_INLINE
+  json::const_reference GetInput() const { return json_input_.get(); }
+
+  // Return the JSON object which is used to store output from running the
+  // benchmark.
+  json::const_reference GetOutput() const { return json_output_.get(); }
+
+  // Get or create a JSON object with the specified key and return a reference
+  // to that object. This can be used to specify arbitrary JSON output which
+  // will be reported by the JSON reporter.
+  json::reference operator[](json::object_t::key_type const& key) {
+    return json_output_.get()[key];
+  }
+#endif
+
   // Range arguments for this run. CHECKs if the argument has been set.
   BENCHMARK_ALWAYS_INLINE
   int range(std::size_t pos = 0) const {
@@ -573,6 +643,8 @@ class State {
   size_t total_iterations_;
 
   std::vector<int> range_;
+  internal::JSONPointer json_input_;
+  internal::JSONPointer json_output_;
 
   size_t bytes_processed_;
   size_t items_processed_;
@@ -591,9 +663,9 @@ class State {
   const size_t max_iterations;
 
   // TODO(EricWF) make me private
-  State(size_t max_iters, const std::vector<int>& ranges, int thread_i,
-        int n_threads, internal::ThreadTimer* timer,
-        internal::ThreadManager* manager);
+  State(size_t max_iters, const std::vector<int>& ranges,
+        internal::JSONPointer json_ptr, int thread_i, int n_threads,
+        internal::ThreadTimer* timer, internal::ThreadManager* manager);
 
  private:
   void StartKeepRunning();
@@ -811,6 +883,17 @@ class Benchmark {
   // Equivalent to ThreadRange(NumCPUs(), NumCPUs())
   Benchmark* ThreadPerCpu();
 
+#ifndef BENCHMARK_HAS_NO_JSON_HEADER
+  // Run this benchmark once with "input_value" as the extra arguments passed
+  // to the function, which can be accessed using 'State::GetInput()'.
+  //
+  // The name of the benchmark for the specified input value will be appended
+  // with "/<name>" if the input contains a "name" field; otherwise each of
+  // the fields will be appended as "/<key>:<value>" when value is a primitive,
+  // and "/<key>" otherwise.
+  Benchmark* WithInput(json input_value);
+#endif
+
   virtual void Run(State& state) = 0;
 
   // Used inside the benchmark implementation
@@ -832,6 +915,7 @@ class Benchmark {
   ReportMode report_mode_;
   std::vector<std::string> arg_names_;   // Args for all benchmark runs
   std::vector<std::vector<int> > args_;  // Args for all benchmark runs
+  std::vector<JSONPointer> json_args_;
   TimeUnit time_unit_;
   int range_multiplier_;
   double min_time_;
@@ -1152,9 +1236,9 @@ class Fixture : public internal::Benchmark {
 
 // ------------------------------------------------------
 // Benchmark Reporters
-
 namespace benchmark {
 
+#ifndef BENCHMARK_HAS_NO_JSON_HEADER
 struct CPUInfo {
   struct CacheInfo {
     std::string type;
@@ -1249,6 +1333,8 @@ class BenchmarkReporter {
     bool report_rms;
 
     UserCounters counters;
+
+    json json_output;
   };
 
   // Construct a BenchmarkReporter with the output stream set to 'std::cout'
@@ -1359,6 +1445,7 @@ class CSVReporter : public BenchmarkReporter {
   bool printed_header_;
   std::set< std::string > user_counter_names_;
 };
+#endif  // BENCHMARK_HAS_NO_JSON_HEADER
 
 inline const char* GetTimeUnitString(TimeUnit unit) {
   switch (unit) {

src/CMakeLists.txt

@@ -10,6 +10,7 @@ file(GLOB
   SOURCE_FILES
     *.cc
     ${PROJECT_SOURCE_DIR}/include/benchmark/*.h
+    ${JSON_INCLUDE_DIR}/*.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/*.h)
 
 add_library(benchmark ${SOURCE_FILES})
@@ -20,6 +21,7 @@ set_target_properties(benchmark PROPERTIES
 )
 target_include_directories(benchmark PUBLIC
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../include>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../third_party/json/src>
     )
 
 # Link threads.
@@ -71,6 +73,9 @@ if (BENCHMARK_ENABLE_INSTALL)
     DIRECTORY "${PROJECT_SOURCE_DIR}/include/benchmark"
     DESTINATION ${include_install_dir}
     FILES_MATCHING PATTERN "*.*h")
+  install(
+    FILES "${JSON_INCLUDE_DIR}/json.hpp"
+    DESTINATION ${include_install_dir}/benchmark)
 
   install(
       FILES "${project_config}" "${version_config}"

src/benchmark.cc

@@ -108,6 +108,15 @@ namespace internal {
 
 void UseCharPointer(char const volatile*) {}
 
+JSONPointer::JSONPointer() : JSONPointer(json::object_t{}) {}
+
+JSONPointer::JSONPointer(json val)
+    : ptr_(new JSONPointer::PIMPL{std::move(val)}) {}
+
+JSONPointer::~JSONPointer() {
+  if (ptr_) delete ptr_;
+}
+
 class ThreadManager {
  public:
   ThreadManager(int num_threads)
@@ -146,6 +155,7 @@ class ThreadManager {
     std::string report_label_;
     std::string error_message_;
     bool has_error_ = false;
+    json json_output = json::object_t{};
     UserCounters counters;
   };
   GUARDED_BY(GetBenchmarkMutex()) Result results;
@@ -256,6 +266,7 @@ BenchmarkReporter::Run CreateRunReport(
     report.statistics = b.statistics;
     report.counters = results.counters;
     internal::Finish(&report.counters, seconds, b.threads);
+    report.json_output = results.json_output;
   }
   return report;
 }
@@ -266,7 +277,8 @@ void RunInThread(const benchmark::internal::Benchmark::Instance* b,
                  size_t iters, int thread_id,
                  internal::ThreadManager* manager) {
   internal::ThreadTimer timer;
-  State st(iters, b->arg, thread_id, b->threads, &timer, manager);
+  State st(iters, b->arg, internal::JSONPointer(b->json_arg), thread_id,
+           b->threads, &timer, manager);
   b->benchmark->Run(st);
   CHECK(st.iterations() == st.max_iterations)
       << "Benchmark returned before State::KeepRunning() returned false!";
@@ -279,6 +291,14 @@ void RunInThread(const benchmark::internal::Benchmark::Instance* b,
     results.bytes_processed += st.bytes_processed();
     results.items_processed += st.items_processed();
     results.complexity_n += st.complexity_length_n();
+    for (auto It = st.GetOutput().begin(); It != st.GetOutput().end(); ++It) {
+      auto Key = It.key();
+      if (results.json_output.count(Key) == 0) {
+        results.json_output[Key] = It.value();
+        continue;
+      }
+      // FIXME: Figure out how to merge JSON values.
+    }
     internal::Increment(&results.counters, st.counters);
   }
   manager->NotifyThreadComplete();
@@ -394,13 +414,17 @@ std::vector<BenchmarkReporter::Run> RunBenchmark(
 }  // namespace
 }  // namespace internal
 
-State::State(size_t max_iters, const std::vector<int>& ranges, int thread_i,
-             int n_threads, internal::ThreadTimer* timer,
+State::State(size_t max_iters, const std::vector<int>& ranges,
+             internal::JSONPointer json_ptr,
+
+             int thread_i, int n_threads, internal::ThreadTimer* timer,
              internal::ThreadManager* manager)
     : started_(false),
       finished_(false),
       total_iterations_(max_iters + 1),
       range_(ranges),
+      json_input_(std::move(json_ptr)),
+      json_output_(json::object_t{}),
       bytes_processed_(0),
       items_processed_(0),
       complexity_n_(0),

src/benchmark_api_internal.h

@@ -18,6 +18,7 @@ struct Benchmark::Instance {
   Benchmark* benchmark;
   ReportMode report_mode;
   std::vector<int> arg;
+  json json_arg;
   TimeUnit time_unit;
   int range_multiplier;
   bool use_real_time;

src/benchmark_register.cc

@@ -131,6 +131,14 @@ bool BenchmarkFamilies::FindBenchmarks(
         (family->thread_counts_.empty()
              ? &one_thread
              : &static_cast<const std::vector<int>&>(family->thread_counts_));
+    std::vector<JSONPointer> one_json_arg;
+    one_json_arg.emplace_back(json{});
+    using JSONPointerVect = std::vector<JSONPointer> const*;
+    bool has_json_args = !family->json_args_.empty();
+    JSONPointerVect json_args =
+        (!has_json_args ? &one_json_arg
+                        : &static_cast<const std::vector<JSONPointer>&>(
+                              family->json_args_));
     const size_t family_size = family->args_.size() * thread_counts->size();
     // The benchmark will be run at least 'family_size' different inputs.
     // If 'family_size' is very large warn the user.
@@ -142,63 +150,80 @@ bool BenchmarkFamilies::FindBenchmarks(
     // family size.
     if (spec == ".") benchmarks->reserve(family_size);
 
-    for (auto const& args : family->args_) {
-      for (int num_threads : *thread_counts) {
-        Benchmark::Instance instance;
-        instance.name = family->name_;
-        instance.benchmark = family.get();
-        instance.report_mode = family->report_mode_;
-        instance.arg = args;
-        instance.time_unit = family->time_unit_;
-        instance.range_multiplier = family->range_multiplier_;
-        instance.min_time = family->min_time_;
-        instance.iterations = family->iterations_;
-        instance.repetitions = family->repetitions_;
-        instance.use_real_time = family->use_real_time_;
-        instance.use_manual_time = family->use_manual_time_;
-        instance.complexity = family->complexity_;
-        instance.complexity_lambda = family->complexity_lambda_;
-        instance.statistics = &family->statistics_;
-        instance.threads = num_threads;
-
-        // Add arguments to instance name
-        size_t arg_i = 0;
-        for (auto const& arg : args) {
-          instance.name += "/";
-
-          if (arg_i < family->arg_names_.size()) {
-            const auto& arg_name = family->arg_names_[arg_i];
-            if (!arg_name.empty()) {
-              instance.name +=
-                  StringPrintF("%s:", family->arg_names_[arg_i].c_str());
+    for (auto const& json_arg : *json_args) {
+      for (auto const& args : family->args_) {
+        for (int num_threads : *thread_counts) {
+          Benchmark::Instance instance;
+          instance.name = family->name_;
+          instance.benchmark = family.get();
+          instance.report_mode = family->report_mode_;
+          instance.arg = args;
+          instance.json_arg = json_arg.get();
+          instance.time_unit = family->time_unit_;
+          instance.range_multiplier = family->range_multiplier_;
+          instance.min_time = family->min_time_;
+          instance.iterations = family->iterations_;
+          instance.repetitions = family->repetitions_;
+          instance.use_real_time = family->use_real_time_;
+          instance.use_manual_time = family->use_manual_time_;
+          instance.complexity = family->complexity_;
+          instance.complexity_lambda = family->complexity_lambda_;
+          instance.statistics = &family->statistics_;
+          instance.threads = num_threads;
+
+          if (has_json_args) {
+            json& arg = json_arg.get();
+            if (arg.count("name") != 0) {
+              instance.name += "/" + arg.at("name").get<std::string>();
+            } else {
+              for (auto It = arg.begin(); It != arg.end(); ++It) {
+                instance.name += "/" + It.key();
+                if (It.value().is_primitive()) {
+                  instance.name += ":" + It.value().dump();
+                }
+              }
             }
           }
-          
-          instance.name += StringPrintF("%d", arg);
-          ++arg_i;
-        }
+          // Add arguments to instance name
+          size_t arg_i = 0;
+          for (auto const& arg : args) {
+            instance.name += "/";
+
+            if (arg_i < family->arg_names_.size()) {
+              const auto& arg_name = family->arg_names_[arg_i];
+              if (!arg_name.empty()) {
+                instance.name +=
+                    StringPrintF("%s:", family->arg_names_[arg_i].c_str());
+              }
+            }
 
-        if (!IsZero(family->min_time_))
-          instance.name += StringPrintF("/min_time:%0.3f", family->min_time_);
-        if (family->iterations_ != 0)
-          instance.name += StringPrintF("/iterations:%d", family->iterations_);
-        if (family->repetitions_ != 0)
-          instance.name += StringPrintF("/repeats:%d", family->repetitions_);
-
-        if (family->use_manual_time_) {
-          instance.name += "/manual_time";
-        } else if (family->use_real_time_) {
-          instance.name += "/real_time";
-        }
+            instance.name += StringPrintF("%d", arg);
+            ++arg_i;
+          }
 
-        // Add the number of threads used to the name
-        if (!family->thread_counts_.empty()) {
-          instance.name += StringPrintF("/threads:%d", instance.threads);
-        }
+          if (!IsZero(family->min_time_))
+            instance.name += StringPrintF("/min_time:%0.3f", family->min_time_);
+          if (family->iterations_ != 0)
+            instance.name +=
+                StringPrintF("/iterations:%d", family->iterations_);
+          if (family->repetitions_ != 0)
+            instance.name += StringPrintF("/repeats:%d", family->repetitions_);
+
+          if (family->use_manual_time_) {
+            instance.name += "/manual_time";
+          } else if (family->use_real_time_) {
+            instance.name += "/real_time";
+          }
 
-        if (re.Match(instance.name)) {
-          instance.last_benchmark_instance = (&args == &family->args_.back());
-          benchmarks->push_back(std::move(instance));
+          // Add the number of threads used to the name
+          if (!family->thread_counts_.empty()) {
+            instance.name += StringPrintF("/threads:%d", instance.threads);
+          }
+
+          if (re.Match(instance.name)) {
+            instance.last_benchmark_instance = (&args == &family->args_.back());
+            benchmarks->push_back(std::move(instance));
+          }
         }
       }
     }
@@ -451,6 +476,11 @@ Benchmark* Benchmark::ThreadPerCpu() {
   return this;
 }
 
+Benchmark* Benchmark::WithInput(json obj) {
+  json_args_.emplace_back(std::move(obj));
+  return this;
+}
+
 void Benchmark::SetName(const char* name) { name_ = name; }
 
 int Benchmark::ArgsCnt() const {
@@ -469,6 +499,7 @@ void FunctionBenchmark::Run(State& st) { func_(st); }
 
 }  // end namespace internal
 
+
 void ClearRegisteredBenchmarks() {
   internal::BenchmarkFamilies::GetInstance()->ClearBenchmarks();
 }

src/json_reporter.cc

@@ -195,6 +195,12 @@ void JSONReporter::PrintRunData(Run const& run) {
   if (!run.report_label.empty()) {
     out << ",\n" << indent << FormatKV("label", run.report_label);
   }
+  if (!run.json_output.empty()) {
+    out << ",\n"
+        << indent
+        << "\"json_output\": " << std::setw(static_cast<int>(indent.size()) + 2)
+        << run.json_output;
+  }
   out << '\n';
 }
 

test/CMakeLists.txt

@@ -106,6 +106,7 @@ add_test(user_counters_test user_counters_test --benchmark_min_time=0.01)
 compile_output_test(user_counters_tabular_test)
 add_test(user_counters_tabular_test user_counters_tabular_test --benchmark_counters_tabular=true --benchmark_min_time=0.01)
 
+
 check_cxx_compiler_flag(-std=c++03 BENCHMARK_HAS_CXX03_FLAG)
 if (BENCHMARK_HAS_CXX03_FLAG)
   compile_benchmark_test(cxx03_test)
@@ -134,6 +135,8 @@ endif()
 compile_output_test(complexity_test)
 add_test(complexity_benchmark complexity_test --benchmark_min_time=${COMPLEXITY_MIN_TIME})
 
+compile_benchmark_test(disable_json_test)
+add_test(disable_json disable_json_test)
 ###############################################################################
 # GoogleTest Unit Tests
 ###############################################################################
@@ -154,6 +157,7 @@ if (BENCHMARK_ENABLE_GTEST_TESTS)
   endmacro()
 
   add_gtest(statistics_test)
+  add_gtest(json_test)
 endif(BENCHMARK_ENABLE_GTEST_TESTS)
 
 
@@ -175,6 +179,7 @@ if (${CMAKE_BUILD_TYPE_LOWER} MATCHES "coverage")
       COMMAND ${LCOV} -q --no-external -c -b "${CMAKE_SOURCE_DIR}" -d . -o after.lcov
       COMMAND ${LCOV} -q -a before.lcov -a after.lcov --output-file final.lcov
       COMMAND ${LCOV} -q -r final.lcov "'${CMAKE_SOURCE_DIR}/test/*'" -o final.lcov
+      COMMAND ${LCOV} -q -r final.lcov "${JSON_INCLUDE_DIR}/json.hpp" -o final.lcov
       COMMAND ${GENHTML} final.lcov -o lcov --demangle-cpp --sort -p "${CMAKE_BINARY_DIR}" -t benchmark
       DEPENDS filter_test benchmark_test options_test basic_test fixture_test cxx03_test complexity_test
       WORKING_DIRECTORY ${CMAKE_BINARY_DIR}

test/disable_json_test.cc

+//===---------------------------------------------------------------------===//
+// disable_json_test - Test that the JSON header can be disabled by defining
+//  BENCHMARK_HAS_NO_JSON_HEADER.
+//===---------------------------------------------------------------------===//
+#define BENCHMARK_HAS_NO_JSON_HEADER
+#include "benchmark/benchmark.h"
+
+#ifdef BENCHMARK_JSON_H
+#error json.h should not be included.
+#endif
+
+namespace benchmark {
+struct json {
+  json() = delete;
+};  // attempt to cause a duplicate definition error.
+}  // namespace benchmark
+
+int main() {}

test/json_test.cc

+//===---------------------------------------------------------------------===//
+// json_test - Unit tests for benchmark/json.h
+//===---------------------------------------------------------------------===//
+
+#include "benchmark/benchmark.h"
+#include "gtest/gtest.h"
+
+namespace {
+TEST(JSONTest, BreathingTest) {
+  using benchmark::json;
+  using namespace benchmark;
+  json obj{{"name", "foo"}, {"value", 42}, {"list", {1, 2, 3}}};
+  auto expect_json = R"(
+  {
+    "name": "foo",
+    "value": 42,
+    "list": [1, 2, 3]
+  })"_json;
+  EXPECT_EQ(obj, expect_json);
+  EXPECT_EQ(obj.at("name"), "foo");
+  EXPECT_EQ(obj.at("value"), 42);
+  EXPECT_EQ(obj.at("list"), (json::array_t{1, 2, 3}));
+  auto list = obj.at("list");
+  auto expect_list = expect_json.at("list");
+  for (auto It = list.begin(), EIt = expect_list.begin(); It != list.end();
+       ++It, ++EIt) {
+    EXPECT_EQ(*It, *EIt);
+  }
+}
+
+TEST(JSONTest, JSONInputTest) {
+  using namespace benchmark;
+  RegisterBenchmark("test1",
+                    [](State& st) {
+                      json obj = st.GetInput();
+                      assert(!obj.is_null());
+                      switch (obj.at("case").get<int>()) {
+                        case 1:
+                          assert(obj.at("name") == "foo");
+                          assert(obj.at("a") == 42);
+                          break;
+                        case 2:
+                          assert(obj.count("name") == 0);
+                          assert(obj.at("b") == 101);
+                          break;
+                        default:
+                          assert(false && "in default case");
+                      }
+                      for (auto _ : st) {
+                      }
+                    })
+      ->WithInput({{"case", 1}, {"name", "foo"}, {"a", 42}})
+      ->WithInput({{"case", 2}, {"b", 101}});
+  RunSpecifiedBenchmarks();
+}
+
+}  // end namespace

test/reporter_output_test.cc

@@ -373,6 +373,41 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_UserStats/repeats:3\",%csv_report$"},
                       {"^\"BM_UserStats/repeats:3_stddev\",%csv_report$"},
                       {"^\"BM_UserStats/repeats:3_\",%csv_report$"}});
 
+void BM_JSONInputTest(benchmark::State& st) {
+  for (auto _ : st) {
+  }
+  st["foo"] = 42;
+  st["baz"] = "abc";
+  st["list"] = {1, 2, 3};
+}
+BENCHMARK(BM_JSONInputTest)
+    ->WithInput({{"name", "case1"}, {"a", 42}})
+    ->WithInput({{"b", 42}, {"c", 101}});
+ADD_CASES(TC_ConsoleOut, {{"^BM_JSONInputTest/case1 %console_report$"},
+                          {"^BM_JSONInputTest/b:42/c:101 %console_report$"}});
+ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_JSONInputTest/case1\",$"},
+                       {"\"name\": \"BM_JSONInputTest/b:42/c:101\",$"}});
+ADD_CASES(TC_CSVOut, {{"^\"BM_JSONInputTest/case1\",%csv_report$"},
+                      {"^\"BM_JSONInputTest/b:42/c:101\",%csv_report$"}});
+
+void BM_JSONOutputTest(benchmark::State& st) {
+  for (auto _ : st) {
+  }
+  st["foo"] = 42;
+  st["baz"] = "abc";
+  st["list"] = {1, 2, 3};
+}
+BENCHMARK(BM_JSONOutputTest);
+ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_JSONOutputTest\",$"},
+                       {"\"json_output\": \\{$"},
+                       {"\"baz\": \"abc\",$"},
+                       {"\"foo\": 42,$"},
+                       {"\"list\": \\[$"},
+                       {"1,$"},
+                       {"2,$"},
+                       {"3$"},
+                       {"]$"},
+                       {"}$"}});
 // ========================================================================= //
 // --------------------------- TEST CASES END ------------------------------ //
 // ========================================================================= //

third_party/json/.gitignore

+json_unit
+json_benchmarks
+json_benchmarks_simple
+fuzz-testing
+
+*.dSYM
+*.o
+*.gcno
+*.gcda
+
+build
+build_coverage
+
+doc/xml
+doc/html
+me.nlohmann.json.docset
+
+benchmarks/files/numbers/*.json
+
+.idea
+cmake-build-debug
+
+test/test-*

third_party/json/LICENSE.MIT

+MIT License 
+
+Copyright (c) 2013-2017 Niels Lohmann
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.