added lambdas to complexity report

include/benchmark/benchmark_api.h

@@ -152,6 +152,7 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond);
 #include <assert.h>
 #include <stddef.h>
 #include <stdint.h>
+#include <functional>
 
 #include "macros.h"
 
@@ -247,15 +248,20 @@ enum BigO {
   oNCubed,
   oLogN,
   oNLogN,
-  oAuto
+  oAuto,
+  oLambda
 };
 
+// BigOFunc is passed to a benchmark in order to specify the asymptotic 
+// computational complexity for the benchmark.
+typedef double(BigOFunc)(size_t);
+
 // State is passed to a running Benchmark and contains state for the
 // benchmark to use.
 class State {
 public:
   State(size_t max_iters, bool has_x, int x, bool has_y, int y,
-        int thread_i, int n_threads);
+    int thread_i, int n_threads);
 
   // Returns true iff the benchmark should continue through another iteration.
   // NOTE: A benchmark may not return from the test until KeepRunning() has
@@ -268,13 +274,13 @@ public:
     }
     bool const res = total_iterations_++ < max_iterations;
     if (BENCHMARK_BUILTIN_EXPECT(!res, false)) {
-        assert(started_ && (!finished_ || error_occurred_));
-        if (!error_occurred_) {
-            PauseTiming();
-        }
-        // Total iterations now is one greater than max iterations. Fix this.
-        total_iterations_ = max_iterations;
-        finished_ = true;
+      assert(started_ && (!finished_ || error_occurred_));
+      if (!error_occurred_) {
+        PauseTiming();
+      }
+      // Total iterations now is one greater than max iterations. Fix this.
+      total_iterations_ = max_iterations;
+      finished_ = true;
     }
     return res;
   }
@@ -359,7 +365,7 @@ public:
   // represent the length of N.
   BENCHMARK_ALWAYS_INLINE
   void SetComplexityN(size_t complexity_n) {
-    complexity_n_ = complexity_n;
+	  complexity_n_ = complexity_n;
   }
 
   BENCHMARK_ALWAYS_INLINE
@@ -533,10 +539,14 @@ public:
   // to control how many iterations are run, and in the printing of items/second
   // or MB/second values.
   Benchmark* UseManualTime();
-
+  
   // Set the asymptotic computational complexity for the benchmark. If called
   // the asymptotic computational complexity will be shown on the output. 
   Benchmark* Complexity(BigO complexity = benchmark::oAuto);
+  
+  // Set the asymptotic computational complexity for the benchmark. If called
+  // the asymptotic computational complexity will be shown on the output.
+  Benchmark* Complexity(BigOFunc* complexity);
 
   // Support for running multiple copies of the same benchmark concurrently
   // in multiple threads.  This may be useful when measuring the scaling

include/benchmark/reporter.h

@@ -83,11 +83,12 @@ class BenchmarkReporter {
 
     // This is set to 0.0 if memory tracing is not enabled.
     double max_heapbytes_used;
-
+    
     // Keep track of arguments to compute asymptotic complexity
-    BigO   complexity;
-    int complexity_n;
-
+    BigO complexity;
+    BigOFunc* complexity_lambda;
+    size_t complexity_n;
+    
     // Inform print function whether the current run is a complexity report
     bool report_big_o;
     bool report_rms;
@@ -113,7 +114,7 @@ class BenchmarkReporter {
   // 'reports' contains additional entries representing the asymptotic
   // complexity and RMS of that benchmark family.
   virtual void ReportRuns(const std::vector<Run>& report) = 0;
-
+  
   // Called once and only once after ever group of benchmarks is run and
   // reported.
   virtual void Finalize() {}
@@ -159,7 +160,7 @@ class ConsoleReporter : public BenchmarkReporter {
   virtual bool ReportContext(const Context& context);
   virtual void ReportRuns(const std::vector<Run>& reports);
 
-protected:
+ protected:
   virtual void PrintRunData(const Run& report);
 
   size_t name_field_width_;
@@ -189,25 +190,25 @@ private:
 
 inline const char* GetTimeUnitString(TimeUnit unit) {
   switch (unit) {
-    case kMillisecond:
-      return "ms";
-    case kMicrosecond:
-      return "us";
-    case kNanosecond:
-    default:
-      return "ns";
+  case kMillisecond:
+    return "ms";
+  case kMicrosecond:
+    return "us";
+  case kNanosecond:
+  default:
+    return "ns";
   }
 }
 
 inline double GetTimeUnitMultiplier(TimeUnit unit) {
-    switch (unit) {
-    case kMillisecond:
-      return 1e3;
-    case kMicrosecond:
-      return 1e6;
-    case kNanosecond:
-    default:
-      return 1e9;
+  switch (unit) {
+  case kMillisecond:
+    return 1e3;
+  case kMicrosecond:
+    return 1e6;
+  case kNanosecond:
+  default:
+    return 1e9;
   }
 }
 

src/benchmark.cc

@@ -124,7 +124,7 @@ struct ThreadStats {
     ThreadStats() : bytes_processed(0), items_processed(0), complexity_n(0) {}
     int64_t bytes_processed;
     int64_t items_processed;
-    int     complexity_n;
+    size_t  complexity_n;
 };
 
 // Timer management class
@@ -140,7 +140,7 @@ class TimerManager {
         manual_time_used_(0),
         num_finalized_(0),
         phase_number_(0),
-        entered_(0)
+        entered_(0) 
   {
   }
 
@@ -277,11 +277,11 @@ class TimerManager {
       int phase_number_cp = phase_number_;
       auto cb = [this, phase_number_cp]() {
         return this->phase_number_ > phase_number_cp ||
-               entered_ == running_threads_; // A thread has aborted in error
+          entered_ == running_threads_; // A thread has aborted in error
       };
       phase_condition_.wait(ml.native_handle(), cb);
       if (phase_number_ > phase_number_cp)
-          return false;
+        return false;
       // else (running_threads_ == entered_) and we are the last thread.
     }
     // Last thread has reached the barrier
@@ -311,6 +311,7 @@ struct Benchmark::Instance {
   bool           use_real_time;
   bool           use_manual_time;
   BigO           complexity;
+  BigOFunc*      complexity_lambda;
   bool           last_benchmark_instance;
   int            repetitions;
   double         min_time;
@@ -356,6 +357,7 @@ public:
   void UseRealTime();
   void UseManualTime();
   void Complexity(BigO complexity);
+  void ComplexityLambda(BigOFunc* complexity);
   void Threads(int t);
   void ThreadRange(int min_threads, int max_threads);
   void ThreadPerCpu();
@@ -376,6 +378,7 @@ private:
   bool use_real_time_;
   bool use_manual_time_;
   BigO complexity_;
+  BigOFunc* complexity_lambda_;
   std::vector<int> thread_counts_;
 
   BenchmarkImp& operator=(BenchmarkImp const&);
@@ -440,6 +443,7 @@ bool BenchmarkFamilies::FindBenchmarks(
         instance.use_real_time = family->use_real_time_;
         instance.use_manual_time = family->use_manual_time_;
         instance.complexity = family->complexity_;
+        instance.complexity_lambda = family->complexity_lambda_;
         instance.threads = num_threads;
         instance.multithreaded = !(family->thread_counts_.empty());
 
@@ -567,6 +571,10 @@ void BenchmarkImp::Complexity(BigO complexity){
   complexity_ = complexity;
 }
 
+void BenchmarkImp::ComplexityLambda(BigOFunc* complexity) {
+  complexity_lambda_ = complexity;
+}
+
 void BenchmarkImp::Threads(int t) {
   CHECK_GT(t, 0);
   thread_counts_.push_back(t);
@@ -691,6 +699,12 @@ Benchmark* Benchmark::Complexity(BigO complexity) {
   return this;
 }
 
+Benchmark* Benchmark::Complexity(BigOFunc* complexity) {
+  imp_->Complexity(oLambda);
+  imp_->ComplexityLambda(complexity);
+  return this;
+}
+
 Benchmark* Benchmark::Threads(int t) {
   imp_->Threads(t);
   return this;
@@ -717,7 +731,7 @@ void FunctionBenchmark::Run(State& st) {
 } // end namespace internal
 
 namespace {
-
+  
 // Execute one thread of benchmark b for the specified number of iterations.
 // Adds the stats collected for the thread into *total.
 void RunInThread(const benchmark::internal::Benchmark::Instance* b,
@@ -731,15 +745,15 @@ void RunInThread(const benchmark::internal::Benchmark::Instance* b,
     MutexLock l(GetBenchmarkLock());
     total->bytes_processed += st.bytes_processed();
     total->items_processed += st.items_processed();
-    total->complexity_n += st.complexity_length_n();
+	total->complexity_n += st.complexity_length_n();
   }
 
   timer_manager->Finalize();
 }
 
 void RunBenchmark(const benchmark::internal::Benchmark::Instance& b,
-                  BenchmarkReporter* br,
-                  std::vector<BenchmarkReporter::Run>& complexity_reports)
+  BenchmarkReporter* br,
+  std::vector<BenchmarkReporter::Run>& complexity_reports)
   EXCLUDES(GetBenchmarkLock()) {
   size_t iters = 1;
 
@@ -750,7 +764,7 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b,
     pool.resize(b.threads);
 
   const int repeats = b.repetitions != 0 ? b.repetitions
-                                         : FLAGS_benchmark_repetitions;
+    : FLAGS_benchmark_repetitions;
   for (int i = 0; i < repeats; i++) {
     std::string mem;
     for (;;) {
@@ -830,27 +844,28 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b,
         report.time_unit = b.time_unit;
 
         if (!report.error_occurred) {
-          double bytes_per_second = 0;
-          if (total.bytes_processed > 0 && seconds > 0.0) {
-            bytes_per_second = (total.bytes_processed / seconds);
-          }
-          double items_per_second = 0;
-          if (total.items_processed > 0 && seconds > 0.0) {
-            items_per_second = (total.items_processed / seconds);
-          }
-
-          if (b.use_manual_time) {
-            report.real_accumulated_time = manual_accumulated_time;
-          } else {
-            report.real_accumulated_time = real_accumulated_time;
-          }
-          report.cpu_accumulated_time = cpu_accumulated_time;
-          report.bytes_per_second = bytes_per_second;
-          report.items_per_second = items_per_second;
-          report.complexity_n = total.complexity_n;
-          report.complexity = b.complexity;
-          if(report.complexity != oNone)
-            complexity_reports.push_back(report);
+        double bytes_per_second = 0;
+        if (total.bytes_processed > 0 && seconds > 0.0) {
+          bytes_per_second = (total.bytes_processed / seconds);
+        }
+        double items_per_second = 0;
+        if (total.items_processed > 0 && seconds > 0.0) {
+          items_per_second = (total.items_processed / seconds);
+        }
+
+        if (b.use_manual_time) {
+          report.real_accumulated_time = manual_accumulated_time;
+        } else {
+          report.real_accumulated_time = real_accumulated_time;
+        }
+        report.cpu_accumulated_time = cpu_accumulated_time;
+        report.bytes_per_second = bytes_per_second;
+        report.items_per_second = items_per_second;
+        report.complexity_n = total.complexity_n;
+        report.complexity = b.complexity;
+        report.complexity_lambda = b.complexity_lambda;
+        if(report.complexity != oNone)
+          complexity_reports.push_back(report);
         }
 
         reports.push_back(report);
@@ -878,17 +893,17 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b,
   }
   std::vector<BenchmarkReporter::Run> additional_run_stats = ComputeStats(reports);
   reports.insert(reports.end(), additional_run_stats.begin(),
-                    additional_run_stats.end());
+    additional_run_stats.end());
 
   if((b.complexity != oNone) && b.last_benchmark_instance) {
     additional_run_stats = ComputeBigO(complexity_reports);
     reports.insert(reports.end(), additional_run_stats.begin(),
-                   additional_run_stats.end());
+      additional_run_stats.end());
     complexity_reports.clear();
   }
 
   br->ReportRuns(reports);
-
+  
   if (b.multithreaded) {
     for (std::thread& thread : pool)
       thread.join();
@@ -949,56 +964,56 @@ void State::SetLabel(const char* label) {
 }
 
 namespace internal {
-namespace {
-
-void RunMatchingBenchmarks(const std::vector<Benchmark::Instance>& benchmarks,
-                           BenchmarkReporter* reporter) {
-  CHECK(reporter != nullptr);
-
-  // Determine the width of the name field using a minimum width of 10.
-  bool has_repetitions = FLAGS_benchmark_repetitions > 1;
-  size_t name_field_width = 10;
-  for (const Benchmark::Instance& benchmark : benchmarks) {
-    name_field_width =
-        std::max<size_t>(name_field_width, benchmark.name.size());
-    has_repetitions |= benchmark.repetitions > 1;
-  }
-  if (has_repetitions)
-    name_field_width += std::strlen("_stddev");
+  namespace {
+
+    void RunMatchingBenchmarks(const std::vector<Benchmark::Instance>& benchmarks,
+      BenchmarkReporter* reporter) {
+      CHECK(reporter != nullptr);
+
+      // Determine the width of the name field using a minimum width of 10.
+      bool has_repetitions = FLAGS_benchmark_repetitions > 1;
+      size_t name_field_width = 10;
+      for (const Benchmark::Instance& benchmark : benchmarks) {
+        name_field_width =
+          std::max<size_t>(name_field_width, benchmark.name.size());
+        has_repetitions |= benchmark.repetitions > 1;
+      }
+      if (has_repetitions)
+        name_field_width += std::strlen("_stddev");
 
-  // Print header here
-  BenchmarkReporter::Context context;
-  context.num_cpus = NumCPUs();
-  context.mhz_per_cpu = CyclesPerSecond() / 1000000.0f;
+      // Print header here
+      BenchmarkReporter::Context context;
+      context.num_cpus = NumCPUs();
+      context.mhz_per_cpu = CyclesPerSecond() / 1000000.0f;
 
-  context.cpu_scaling_enabled = CpuScalingEnabled();
-  context.name_field_width = name_field_width;
+      context.cpu_scaling_enabled = CpuScalingEnabled();
+      context.name_field_width = name_field_width;
 
-  // Keep track of runing times of all instances of current benchmark
-  std::vector<BenchmarkReporter::Run> complexity_reports;
+      // Keep track of runing times of all instances of current benchmark
+      std::vector<BenchmarkReporter::Run> complexity_reports;
 
-  if (reporter->ReportContext(context)) {
-    for (const auto& benchmark : benchmarks) {
-      RunBenchmark(benchmark, reporter, complexity_reports);
+      if (reporter->ReportContext(context)) {
+        for (const auto& benchmark : benchmarks) {
+          RunBenchmark(benchmark, reporter, complexity_reports);
+        }
+      }
     }
-  }
-}
 
-std::unique_ptr<BenchmarkReporter> GetDefaultReporter() {
-  typedef std::unique_ptr<BenchmarkReporter> PtrType;
-  if (FLAGS_benchmark_format == "console") {
-    return PtrType(new ConsoleReporter);
-  } else if (FLAGS_benchmark_format == "json") {
-    return PtrType(new JSONReporter);
-  } else if (FLAGS_benchmark_format == "csv") {
-    return PtrType(new CSVReporter);
-  } else {
-    std::cerr << "Unexpected format: '" << FLAGS_benchmark_format << "'\n";
-    std::exit(1);
-  }
-}
+    std::unique_ptr<BenchmarkReporter> GetDefaultReporter() {
+      typedef std::unique_ptr<BenchmarkReporter> PtrType;
+      if (FLAGS_benchmark_format == "console") {
+        return PtrType(new ConsoleReporter);
+      } else if (FLAGS_benchmark_format == "json") {
+        return PtrType(new JSONReporter);
+      } else if (FLAGS_benchmark_format == "csv") {
+        return PtrType(new CSVReporter);
+      } else {
+        std::cerr << "Unexpected format: '" << FLAGS_benchmark_format << "'\n";
+        std::exit(1);
+      }
+    }
 
-} // end namespace
+  } // end namespace
 } // end namespace internal
 
 size_t RunSpecifiedBenchmarks() {

src/complexity.cc

@@ -25,43 +25,43 @@
 #include <functional>
 
 namespace benchmark {
-  
+
 // Internal function to calculate the different scalability forms
-std::function<double(int)> FittingCurve(BigO complexity) {
+BigOFunc* FittingCurve(BigO complexity) {
   switch (complexity) {
-    case oN:
-      return [](int n) {return n; };
-    case oNSquared:
-      return [](int n) {return n*n; };
-    case oNCubed:
-      return [](int n) {return n*n*n; };
-    case oLogN:
-      return [](int n) {return log2(n); };
-    case oNLogN:
-      return [](int n) {return n * log2(n); };
-    case o1:
-    default:
-      return [](int) {return 1; };
+  case oN:
+    return [](size_t n) -> double {return n; };
+  case oNSquared:
+    return [](size_t n) -> double {return n * n; };
+  case oNCubed:
+    return [](size_t n) -> double {return n * n * n; };
+  case oLogN:
+    return [](size_t n) {return log2(n); };
+  case oNLogN:
+    return [](size_t n) {return n * log2(n); };
+  case o1:
+  default:
+    return [](size_t) {return 1.0; };
   }
 }
 
 // Function to return an string for the calculated complexity
 std::string GetBigOString(BigO complexity) {
   switch (complexity) {
-    case oN:
-      return "* N";
-    case oNSquared:
-      return "* N**2";
-    case oNCubed:
-      return "* N**3";
-    case oLogN:
-      return "* lgN";
-    case oNLogN:
-      return "* NlgN";
-    case o1:
-      return "* 1";
-    default:
-      return "";
+  case oN:
+    return "N";
+  case oNSquared:
+    return "N^2";
+  case oNCubed:
+    return "N^3";
+  case oLogN:
+    return "lgN";
+  case oNLogN:
+    return "NlgN";
+  case o1:
+    return "(1)";
+  default:
+    return "f(N)";
   }
 }
 
@@ -75,21 +75,9 @@ std::string GetBigOString(BigO complexity) {
 // For a deeper explanation on the algorithm logic, look the README file at
 // http://github.com/ismaelJimenez/Minimal-Cpp-Least-Squared-Fit
 
-// This interface is currently not used from the oustide, but it has been 
-// provided for future upgrades. If in the future it is not needed to support 
-// Cxx03, then all the calculations could be upgraded to use lambdas because 
-// they are more powerful and provide a cleaner inferface than enumerators, 
-// but complete implementation with lambdas will not work for Cxx03 
-// (e.g. lack of std::function).
-// In case lambdas are implemented, the interface would be like :
-//   -> Complexity([](int n) {return n;};)
-// and any arbitrary and valid  equation would be allowed, but the option to 
-// calculate the best fit to the most common scalability curves will still 
-// be kept.
-
-LeastSq CalculateLeastSq(const std::vector<int>& n, 
-                         const std::vector<double>& time, 
-                         std::function<double(int)> fitting_curve) {
+LeastSq MinimalLeastSq(const std::vector<int>& n,
+                       const std::vector<double>& time,
+                       BigOFunc* fitting_curve) {
   double sigma_gn = 0.0;
   double sigma_gn_squared = 0.0;
   double sigma_time = 0.0;
@@ -105,6 +93,7 @@ LeastSq CalculateLeastSq(const std::vector<int>& n,
   }
 
   LeastSq result;
+  result.complexity = oLambda;
 
   // Calculate complexity.
   result.coef = sigma_time_gn / sigma_gn_squared;
@@ -144,19 +133,19 @@ LeastSq MinimalLeastSq(const std::vector<int>& n,
       oLogN, oN, oNLogN, oNSquared, oNCubed };
 
     // Take o1 as default best fitting curve
-    best_fit = CalculateLeastSq(n, time, FittingCurve(o1));
+    best_fit = MinimalLeastSq(n, time, FittingCurve(o1));
     best_fit.complexity = o1;
 
     // Compute all possible fitting curves and stick to the best one
     for (const auto& fit : fit_curves) {
-      LeastSq current_fit = CalculateLeastSq(n, time, FittingCurve(fit));
+      LeastSq current_fit = MinimalLeastSq(n, time, FittingCurve(fit));
       if (current_fit.rms < best_fit.rms) {
         best_fit = current_fit;
         best_fit.complexity = fit;
       }
     }
   } else {
-    best_fit = CalculateLeastSq(n, time, FittingCurve(complexity));
+    best_fit = MinimalLeastSq(n, time, FittingCurve(complexity));
     best_fit.complexity = complexity;
   }
 
@@ -164,14 +153,14 @@ LeastSq MinimalLeastSq(const std::vector<int>& n,
 }
 
 std::vector<BenchmarkReporter::Run> ComputeStats(
-    const std::vector<BenchmarkReporter::Run>& reports)
+  const std::vector<BenchmarkReporter::Run>& reports)
 {
   typedef BenchmarkReporter::Run Run;
   std::vector<Run> results;
 
   auto error_count = std::count_if(
-      reports.begin(), reports.end(),
-      [](Run const& run) {return run.error_occurred;});
+    reports.begin(), reports.end(),
+    [](Run const& run) {return run.error_occurred;});
 
   if (reports.size() - error_count < 2) {
     // We don't report aggregated data if there was a single run.
@@ -193,9 +182,9 @@ std::vector<BenchmarkReporter::Run> ComputeStats(
     if (run.error_occurred)
       continue;
     real_accumulated_time_stat +=
-        Stat1_d(run.real_accumulated_time/run.iterations, run.iterations);
+      Stat1_d(run.real_accumulated_time/run.iterations, run.iterations);
     cpu_accumulated_time_stat +=
-        Stat1_d(run.cpu_accumulated_time/run.iterations, run.iterations);
+      Stat1_d(run.cpu_accumulated_time/run.iterations, run.iterations);
     items_per_second_stat += Stat1_d(run.items_per_second, run.iterations);
     bytes_per_second_stat += Stat1_d(run.bytes_per_second, run.iterations);
   }
@@ -205,9 +194,9 @@ std::vector<BenchmarkReporter::Run> ComputeStats(
   mean_data.benchmark_name = reports[0].benchmark_name + "_mean";
   mean_data.iterations = run_iterations;
   mean_data.real_accumulated_time = real_accumulated_time_stat.Mean() *
-                                     run_iterations;
+    run_iterations;
   mean_data.cpu_accumulated_time = cpu_accumulated_time_stat.Mean() *
-                                    run_iterations;
+    run_iterations;
   mean_data.bytes_per_second = bytes_per_second_stat.Mean();
   mean_data.items_per_second = items_per_second_stat.Mean();
 
@@ -225,9 +214,9 @@ std::vector<BenchmarkReporter::Run> ComputeStats(
   stddev_data.report_label = mean_data.report_label;
   stddev_data.iterations = 0;
   stddev_data.real_accumulated_time =
-      real_accumulated_time_stat.StdDev();
+    real_accumulated_time_stat.StdDev();
   stddev_data.cpu_accumulated_time =
-      cpu_accumulated_time_stat.StdDev();
+    cpu_accumulated_time_stat.StdDev();
   stddev_data.bytes_per_second = bytes_per_second_stat.StdDev();
   stddev_data.items_per_second = items_per_second_stat.StdDev();
 
@@ -237,7 +226,7 @@ std::vector<BenchmarkReporter::Run> ComputeStats(
 }
 
 std::vector<BenchmarkReporter::Run> ComputeBigO(
-    const std::vector<BenchmarkReporter::Run>& reports)
+  const std::vector<BenchmarkReporter::Run>& reports)
 {
   typedef BenchmarkReporter::Run Run;
   std::vector<Run> results;
@@ -256,14 +245,16 @@ std::vector<BenchmarkReporter::Run> ComputeBigO(
     cpu_time.push_back(run.cpu_accumulated_time/run.iterations);
   }
 
-  LeastSq result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity);
-
-  // result_cpu.complexity is passed as parameter to result_real because in case
-  // reports[0].complexity is oAuto, the noise on the measured data could make
-  // the best fit function of Cpu and Real differ. In order to solve this, we
-  // take the best fitting function for the Cpu, and apply it to Real data.
-  LeastSq result_real = MinimalLeastSq(n, real_time, result_cpu.complexity);
+  LeastSq result_cpu;
+  LeastSq result_real;
 
+  if (reports[0].complexity != oLambda) {
+    result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity);
+    result_real = MinimalLeastSq(n, real_time, result_cpu.complexity);
+  } else {
+    result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity_lambda);
+    result_real = MinimalLeastSq(n, real_time, reports[0].complexity_lambda);
+  }
   std::string benchmark_name = reports[0].benchmark_name.substr(0, reports[0].benchmark_name.find('/'));
 
   // Get the data from the accumulator to BenchmarkReporter::Run's.

src/complexity.h

@@ -26,15 +26,15 @@
 
 namespace benchmark {
 
-// Return a vector containing the mean and standard devation information for
-// the specified list of reports. If 'reports' contains less than two
-// non-errored runs an empty vector is returned
-std::vector<BenchmarkReporter::Run> ComputeStats(
+  // Return a vector containing the mean and standard devation information for
+  // the specified list of reports. If 'reports' contains less than two
+  // non-errored runs an empty vector is returned
+  std::vector<BenchmarkReporter::Run> ComputeStats(
     const std::vector<BenchmarkReporter::Run>& reports);
 
-// Return a vector containing the bigO and RMS information for the specified
-// list of reports. If 'reports.size() < 2' an empty vector is returned.
-std::vector<BenchmarkReporter::Run> ComputeBigO(
+  // Return a vector containing the bigO and RMS information for the specified
+  // list of reports. If 'reports.size() < 2' an empty vector is returned.
+  std::vector<BenchmarkReporter::Run> ComputeBigO(
     const std::vector<BenchmarkReporter::Run>& reports);
 
 // This data structure will contain the result returned by MinimalLeastSq
@@ -60,11 +60,5 @@ struct LeastSq {
 // Function to return an string for the calculated complexity
 std::string GetBigOString(BigO complexity);
 
-// Find the coefficient for the high-order term in the running time, by
-// minimizing the sum of squares of relative error.
-LeastSq MinimalLeastSq(const std::vector<int>& n,
-                       const std::vector<double>& time,
-                       const BigO complexity = oAuto);
-
 } // end namespace benchmark
 #endif // COMPLEXITY_H_

src/console_reporter.cc

@@ -90,8 +90,8 @@ void ConsoleReporter::PrintRunData(const Run& result) {
   const double cpu_time = result.GetAdjustedCPUTime();
 
   if(result.report_big_o) {
-    std::string big_o = result.report_big_o ? GetBigOString(result.complexity) : "";
-    ColorPrintf(Out, COLOR_YELLOW, "%10.4f %s %10.4f %s ",
+    std::string big_o = GetBigOString(result.complexity);
+    ColorPrintf(Out, COLOR_YELLOW, "%10.2f %s %10.2f %s ",
                 real_time, big_o.c_str(), cpu_time, big_o.c_str());
   } else if(result.report_rms) {
     ColorPrintf(Out, COLOR_YELLOW, "%10.0f %% %10.0f %% ",

src/csv_reporter.cc

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #include "benchmark/reporter.h"
+#include "complexity.h"
 
 #include <cstdint>
 #include <algorithm>
@@ -87,8 +88,10 @@ void CSVReporter::PrintRunData(const Run & run) {
   Out << run.GetAdjustedRealTime() << ",";
   Out << run.GetAdjustedCPUTime() << ",";
 
-  // Do not print timeLabel on RMS report
-  if(!run.report_rms) {
+  // Do not print timeLabel on bigO and RMS report
+  if(run.report_big_o) {
+    Out << GetBigOString(run.complexity);
+  } else if(!run.report_rms){
     Out << GetTimeUnitString(run.time_unit);
   }
   Out << ",";

src/json_reporter.cc

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #include "benchmark/reporter.h"
+#include "complexity.h"
 
 #include <cstdint>
 #include <algorithm>
@@ -132,15 +133,29 @@ void JSONReporter::PrintRunData(Run const& run) {
         out << indent
             << FormatKV("iterations", run.iterations)
             << ",\n";
-    }
-    out << indent
-        << FormatKV("real_time", RoundDouble(run.GetAdjustedRealTime()))
-        << ",\n";
-    out << indent
-        << FormatKV("cpu_time", RoundDouble(run.GetAdjustedCPUTime()));
-    if(!run.report_rms) {
+        out << indent
+            << FormatKV("real_time", RoundDouble(run.GetAdjustedRealTime()))
+            << ",\n";
+        out << indent
+            << FormatKV("cpu_time", RoundDouble(run.GetAdjustedCPUTime()));
         out << ",\n" << indent
             << FormatKV("time_unit", GetTimeUnitString(run.time_unit));
+    } else if(run.report_big_o) {
+        out << indent
+            << FormatKV("cpu_coefficient", RoundDouble(run.GetAdjustedCPUTime()))
+            << ",\n";
+        out << indent
+            << FormatKV("real_coefficient", RoundDouble(run.GetAdjustedRealTime()))
+            << ",\n";
+        out << indent
+            << FormatKV("big_o", GetBigOString(run.complexity))
+            << ",\n";
+        out << indent
+            << FormatKV("time_unit", GetTimeUnitString(run.time_unit));
+    } else if(run.report_rms) {
+        out << indent
+            << FormatKV("rms", RoundDouble(run.GetAdjustedCPUTime()*100))
+            << "%";
     }
     if (run.bytes_per_second > 0.0) {
         out << ",\n" << indent

test/complexity_test.cc

 
-#include "benchmark/benchmark_api.h"
-
-#include <cstdlib>
-#include <string>
+#undef NDEBUG
+#include "benchmark/benchmark.h"
+#include "../src/check.h" // NOTE: check.h is for internal use only!
+#include "../src/re.h"    // NOTE: re.h is for internal use only
+#include <cassert>
+#include <cstring>
+#include <iostream>
+#include <sstream>
 #include <vector>
-#include <map>
+#include <utility>
 #include <algorithm>
 
-std::vector<int> ConstructRandomVector(int size) {
-  std::vector<int> v;
-  v.reserve(size);
-  for (int i = 0; i < size; ++i) {
-    v.push_back(rand() % size);
+namespace {
+
+// ========================================================================= //
+// -------------------------- Testing Case --------------------------------- //
+// ========================================================================= //
+
+enum MatchRules {
+  MR_Default, // Skip non-matching lines until a match is found.
+  MR_Next    // Match must occur on the next line.
+};
+
+struct TestCase {
+  std::string regex;
+  int match_rule;
+
+  TestCase(std::string re, int rule = MR_Default) : regex(re), match_rule(rule) {}
+
+  void Check(std::stringstream& remaining_output) const {
+    benchmark::Regex r;
+    std::string err_str;
+    r.Init(regex, &err_str);
+    CHECK(err_str.empty()) << "Could not construct regex \"" << regex << "\""
+                           << " got Error: " << err_str;
+
+    std::string line;
+    while (remaining_output.eof() == false) {
+        CHECK(remaining_output.good());
+        std::getline(remaining_output, line);
+        if (r.Match(line)) return;
+        CHECK(match_rule != MR_Next) << "Expected line \"" << line
+                                     << "\" to match regex \"" << regex << "\"";
+    }
+
+    CHECK(remaining_output.eof() == false)
+        << "End of output reached before match for regex \"" << regex
+        << "\" was found";
   }
-  return v;
-}
+};
 
-std::map<int, int> ConstructRandomMap(int size) {
-  std::map<int, int> m;
-  for (int i = 0; i < size; ++i) {
-    m.insert(std::make_pair(rand() % size, rand() % size));
+std::vector<TestCase> ConsoleOutputTests;
+std::vector<TestCase> JSONOutputTests;
+std::vector<TestCase> CSVOutputTests;
+
+// ========================================================================= //
+// -------------------------- Test Helpers --------------------------------- //
+// ========================================================================= //
+
+class TestReporter : public benchmark::BenchmarkReporter {
+public:
+  TestReporter(std::vector<benchmark::BenchmarkReporter*> reps)
+      : reporters_(reps)  {}
+
+  virtual bool ReportContext(const Context& context) {
+    bool last_ret = false;
+    bool first = true;
+    for (auto rep : reporters_) {
+      bool new_ret = rep->ReportContext(context);
+      CHECK(first || new_ret == last_ret)
+          << "Reports return different values for ReportContext";
+      first = false;
+      last_ret = new_ret;
+    }
+    return last_ret;
   }
-  return m;
+
+  virtual void ReportRuns(const std::vector<Run>& report) {
+    for (auto rep : reporters_)
+      rep->ReportRuns(report);
+  }
+
+  virtual void Finalize() {
+      for (auto rep : reporters_)
+        rep->Finalize();
+  }
+
+private:
+  std::vector<benchmark::BenchmarkReporter*> reporters_;
+};
+
+
+#define CONCAT2(x, y) x##y
+#define CONCAT(x, y) CONCAT2(x, y)
+
+#define ADD_CASES(...) \
+    int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__)
+
+int AddCases(std::vector<TestCase>* out, std::initializer_list<TestCase> const& v) {
+  for (auto const& TC : v)
+    out->push_back(TC);
+  return 0;
+}
+
+template <class First>
+std::string join(First f) { return f; }
+
+template <class First, class ...Args>
+std::string join(First f, Args&&... args) {
+    return std::string(std::move(f)) + "[ ]+" + join(std::forward<Args>(args)...);
 }
 
+std::string dec_re = "[0-9]+\\.[0-9]+";
+
+#define ADD_COMPLEXITY_CASES(...) \
+    int CONCAT(dummy, __LINE__) = AddComplexityTest(__VA_ARGS__)
+
+int AddComplexityTest(std::vector<TestCase>* console_out, std::vector<TestCase>* json_out,
+                      std::vector<TestCase>* csv_out, std::string big_o_test_name, 
+                      std::string rms_test_name, std::string big_o) {
+  std::string big_o_str = dec_re + " " + big_o;
+  AddCases(console_out, {
+    {join("^" + big_o_test_name + "", big_o_str, big_o_str) + "[ ]*$"},
+    {join("^" + rms_test_name + "", "[0-9]+ %", "[0-9]+ %") + "[ ]*$"}
+  });
+  AddCases(json_out, {
+    {"\"name\": \"" + big_o_test_name + "\",$"},
+    {"\"cpu_coefficient\": [0-9]+,$", MR_Next},
+    {"\"real_coefficient\": [0-9]{1,5},$", MR_Next},
+    {"\"big_o\": \"" + big_o + "\",$", MR_Next},
+    {"\"time_unit\": \"ns\"$", MR_Next},
+    {"}", MR_Next},
+    {"\"name\": \"" + rms_test_name + "\",$"},
+    {"\"rms\": [0-9]+%$", MR_Next},
+    {"}", MR_Next}
+  });
+  AddCases(csv_out, {
+    {"^\"" + big_o_test_name + "\",," + dec_re + "," + dec_re + "," + big_o + ",,,,,$"},
+    {"^\"" + rms_test_name + "\",," + dec_re + "," + dec_re + ",,,,,,$"}
+  });
+  return 0;
+}
+
+}  // end namespace
+
+// ========================================================================= //
+// --------------------------- Testing BigO O(1) --------------------------- //
+// ========================================================================= //
+
 void BM_Complexity_O1(benchmark::State& state) {
   while (state.KeepRunning()) {
   }
   state.SetComplexityN(state.range_x());
 }
-BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity(benchmark::o1);
+BENCHMARK(BM_Complexity_O1)->Range(1, 1<<18)->Complexity();
+BENCHMARK(BM_Complexity_O1)->Range(1, 1<<18)->Complexity(benchmark::o1);
+BENCHMARK(BM_Complexity_O1)->Range(1, 1<<18)->Complexity([](size_t){return 1.0; });
+
+std::string big_o_1_test_name = "BM_Complexity_O1_BigO";
+std::string rms_o_1_test_name = "BM_Complexity_O1_RMS";
+std::string enum_auto_big_o_1 = "\\([0-9]+\\)";
+std::string lambda_big_o_1 = "f\\(N\\)";
+
+// Add automatic tests
+ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, 
+                     big_o_1_test_name, rms_o_1_test_name, enum_auto_big_o_1);
+
+// Add enum tests
+ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, 
+                     big_o_1_test_name, rms_o_1_test_name, enum_auto_big_o_1);
+
+// Add lambda tests
+ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, 
+                     big_o_1_test_name, rms_o_1_test_name, lambda_big_o_1);
+
+// ========================================================================= //
+// --------------------------- Testing BigO O(N) --------------------------- //
+// ========================================================================= //
+
+std::vector<int> ConstructRandomVector(int size) {
+  std::vector<int> v;
+  v.reserve(size);
+  for (int i = 0; i < size; ++i) {
+    v.push_back(rand() % size);
+  }
+  return v;
+}
 
-static void BM_Complexity_O_N(benchmark::State& state) {
+void BM_Complexity_O_N(benchmark::State& state) {
   auto v = ConstructRandomVector(state.range_x());
   const int item_not_in_vector = state.range_x()*2; // Test worst case scenario (item not in vector)
   while (state.KeepRunning()) {
@@ -39,51 +195,30 @@ static void BM_Complexity_O_N(benchmark::State& state) {
   }
   state.SetComplexityN(state.range_x());
 }
-BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oN);
 BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity();
+BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oN);
+BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity([](size_t n) -> double{return n; });
 
-static void BM_Complexity_O_N_Squared(benchmark::State& state) {
-  std::string s1(state.range_x(), '-');
-  std::string s2(state.range_x(), '-');
-  state.SetComplexityN(state.range_x());
-  while (state.KeepRunning())
-    for(char& c1 : s1) {
-        for(char& c2 : s2) {
-            benchmark::DoNotOptimize(c1 = 'a');
-            benchmark::DoNotOptimize(c2 = 'b');
-        }
-    }
-}
-BENCHMARK(BM_Complexity_O_N_Squared) -> Range(1, 1<<8) -> Complexity(benchmark::oNSquared);
+std::string big_o_n_test_name = "BM_Complexity_O_N_BigO";
+std::string rms_o_n_test_name = "BM_Complexity_O_N_RMS";
+std::string enum_auto_big_o_n = "N";
+std::string lambda_big_o_n = "f\\(N\\)";
 
-static void BM_Complexity_O_N_Cubed(benchmark::State& state) {
-  std::string s1(state.range_x(), '-');
-  std::string s2(state.range_x(), '-');
-  std::string s3(state.range_x(), '-');
-  state.SetComplexityN(state.range_x());
-  while (state.KeepRunning())
-    for(char& c1 : s1) {
-        for(char& c2 : s2) {
-            for(char& c3 : s3) {
-                benchmark::DoNotOptimize(c1 = 'a');
-                benchmark::DoNotOptimize(c2 = 'b');
-                benchmark::DoNotOptimize(c3 = 'c');
-            }
-        }
-    }
-}
-BENCHMARK(BM_Complexity_O_N_Cubed) -> DenseRange(1, 8) -> Complexity(benchmark::oNCubed);
+// Add automatic tests
+ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, 
+                     big_o_n_test_name, rms_o_n_test_name, enum_auto_big_o_n);
 
-static void BM_Complexity_O_log_N(benchmark::State& state) {
-  auto m = ConstructRandomMap(state.range_x());
-  const int item_not_in_vector = state.range_x()*2; // Test worst case scenario (item not in vector)
-  while (state.KeepRunning()) {
-      benchmark::DoNotOptimize(m.find(item_not_in_vector));
-  }
-  state.SetComplexityN(state.range_x());
-}
-BENCHMARK(BM_Complexity_O_log_N)
-    -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oLogN);
+// Add enum tests
+ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, 
+                     big_o_n_test_name, rms_o_n_test_name, enum_auto_big_o_n);
+
+// Add lambda tests
+ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, 
+                     big_o_n_test_name, rms_o_n_test_name, lambda_big_o_n);
+
+// ========================================================================= //
+// ------------------------- Testing BigO O(N*lgN) ------------------------- //
+// ========================================================================= //
 
 static void BM_Complexity_O_N_log_N(benchmark::State& state) {
   auto v = ConstructRandomVector(state.range_x());
@@ -92,15 +227,82 @@ static void BM_Complexity_O_N_log_N(benchmark::State& state) {
   }
   state.SetComplexityN(state.range_x());
 }
-BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oNLogN);
 BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity();
+BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oNLogN);
+BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity([](size_t n) {return n * log2(n); });
 
-// Test benchmark with no range and check no complexity is calculated.
-void BM_Extreme_Cases(benchmark::State& state) {
-  while (state.KeepRunning()) {
+std::string big_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_BigO";
+std::string rms_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_RMS";
+std::string enum_auto_big_o_n_lg_n = "NlgN";
+std::string lambda_big_o_n_lg_n = "f\\(N\\)";
+
+// Add automatic tests
+ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, 
+                     big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n);
+
+// Add enum tests
+ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, 
+                     big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n);
+
+// Add lambda tests
+ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, 
+                     big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n);
+
+
+// ========================================================================= //
+// --------------------------- TEST CASES END ------------------------------ //
+// ========================================================================= //
+
+
+int main(int argc, char* argv[]) {
+  // Add --color_print=false to argv since we don't want to match color codes.
+  char new_arg[64];
+  char* new_argv[64];
+  std::copy(argv, argv + argc, new_argv);
+  new_argv[argc++] = std::strcpy(new_arg, "--color_print=false");
+  benchmark::Initialize(&argc, new_argv);
+
+  benchmark::ConsoleReporter CR;
+  benchmark::JSONReporter JR;
+  benchmark::CSVReporter CSVR;
+  struct ReporterTest {
+    const char* name;
+    std::vector<TestCase>& output_cases;
+    benchmark::BenchmarkReporter& reporter;
+    std::stringstream out_stream;
+    std::stringstream err_stream;
+
+    ReporterTest(const char* n,
+                 std::vector<TestCase>& out_tc,
+                 benchmark::BenchmarkReporter& br)
+        : name(n), output_cases(out_tc), reporter(br) {
+        reporter.SetOutputStream(&out_stream);
+        reporter.SetErrorStream(&err_stream);
+    }
+  } TestCases[] = {
+      {"ConsoleReporter", ConsoleOutputTests, CR},
+      {"JSONReporter", JSONOutputTests, JR},
+      {"CSVReporter", CSVOutputTests, CSVR}
+  };
+
+  // Create the test reporter and run the benchmarks.
+  std::cout << "Running benchmarks...\n";
+  TestReporter test_rep({&CR, &JR, &CSVR});
+  benchmark::RunSpecifiedBenchmarks(&test_rep);
+
+  for (auto& rep_test : TestCases) {
+      std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n";
+      std::string banner(msg.size() - 1, '-');
+      std::cout << banner << msg << banner << "\n";
+
+      std::cerr << rep_test.err_stream.str();
+      std::cout << rep_test.out_stream.str();
+
+      for (const auto& TC : rep_test.output_cases)
+        TC.Check(rep_test.out_stream);
+
+      std::cout << "\n";
   }
+  return 0;
 }
-BENCHMARK(BM_Extreme_Cases) -> Complexity(benchmark::oNLogN);
-BENCHMARK(BM_Extreme_Cases) -> Arg(42) -> Complexity();
 
-BENCHMARK_MAIN()