[Tooling] 'display aggregates only' support. (#674)

docs/tools.md

@@ -6,6 +6,15 @@ The `compare.py` can be used to compare the result of benchmarks.
 
 **NOTE**: the utility relies on the scipy package which can be installed using [these instructions](https://www.scipy.org/install.html).
 
+### Displaying aggregates only
+
+The switch `-a` / `--display_aggregates_only` can be used to control the
+displayment of the normal iterations vs the aggregates. When passed, it will
+be passthrough to the benchmark binaries to be run, and will be accounted for
+in the tool itself; only the aggregates will be displayed, but not normal runs.
+It only affects the display, the separate runs will still be used to calculate
+the U test.
+
 ### Modes of operation
 
 There are three modes of operation:

tools/compare.py

@@ -36,6 +36,17 @@ def create_parser():
     parser = ArgumentParser(
         description='versatile benchmark output compare tool')
 
+    parser.add_argument(
+        '-a',
+        '--display_aggregates_only',
+        dest='display_aggregates_only',
+        action="store_true",
+        help="If there are repetitions, by default, we display everything - the"
+             " actual runs, and the aggregates computed. Sometimes, it is "
+             "desirable to only view the aggregates. E.g. when there are a lot "
+             "of repetitions. Do note that only the display is affected. "
+             "Internally, all the actual runs are still used, e.g. for U test.")
+
     utest = parser.add_argument_group()
     utest.add_argument(
         '--no-utest',
@@ -200,6 +211,9 @@ def main():
 
     check_inputs(test_baseline, test_contender, benchmark_options)
 
+    if args.display_aggregates_only:
+        benchmark_options += ['--benchmark_display_aggregates_only=true']
+
     options_baseline = []
     options_contender = []
 
@@ -223,7 +237,8 @@ def main():
 
     # Diff and output
     output_lines = gbench.report.generate_difference_report(
-        json1, json2, args.utest, args.utest_alpha)
+        json1, json2, args.display_aggregates_only,
+        args.utest, args.utest_alpha)
     print(description)
     for ln in output_lines:
         print(ln)
@@ -246,6 +261,7 @@ class TestParser(unittest.TestCase):
     def test_benchmarks_basic(self):
         parsed = self.parser.parse_args(
             ['benchmarks', self.testInput0, self.testInput1])
+        self.assertFalse(parsed.display_aggregates_only)
         self.assertTrue(parsed.utest)
         self.assertEqual(parsed.mode, 'benchmarks')
         self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
@@ -255,6 +271,7 @@ class TestParser(unittest.TestCase):
     def test_benchmarks_basic_without_utest(self):
         parsed = self.parser.parse_args(
             ['--no-utest', 'benchmarks', self.testInput0, self.testInput1])
+        self.assertFalse(parsed.display_aggregates_only)
         self.assertFalse(parsed.utest)
         self.assertEqual(parsed.utest_alpha, 0.05)
         self.assertEqual(parsed.mode, 'benchmarks')
@@ -262,9 +279,20 @@ class TestParser(unittest.TestCase):
         self.assertEqual(parsed.test_contender[0].name, self.testInput1)
         self.assertFalse(parsed.benchmark_options)
 
+    def test_benchmarks_basic_display_aggregates_only(self):
+        parsed = self.parser.parse_args(
+            ['-a', 'benchmarks', self.testInput0, self.testInput1])
+        self.assertTrue(parsed.display_aggregates_only)
+        self.assertTrue(parsed.utest)
+        self.assertEqual(parsed.mode, 'benchmarks')
+        self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
+        self.assertEqual(parsed.test_contender[0].name, self.testInput1)
+        self.assertFalse(parsed.benchmark_options)
+
     def test_benchmarks_basic_with_utest_alpha(self):
         parsed = self.parser.parse_args(
             ['--alpha=0.314', 'benchmarks', self.testInput0, self.testInput1])
+        self.assertFalse(parsed.display_aggregates_only)
         self.assertTrue(parsed.utest)
         self.assertEqual(parsed.utest_alpha, 0.314)
         self.assertEqual(parsed.mode, 'benchmarks')
@@ -275,6 +303,7 @@ class TestParser(unittest.TestCase):
     def test_benchmarks_basic_without_utest_with_utest_alpha(self):
         parsed = self.parser.parse_args(
             ['--no-utest', '--alpha=0.314', 'benchmarks', self.testInput0, self.testInput1])
+        self.assertFalse(parsed.display_aggregates_only)
         self.assertFalse(parsed.utest)
         self.assertEqual(parsed.utest_alpha, 0.314)
         self.assertEqual(parsed.mode, 'benchmarks')
@@ -285,6 +314,7 @@ class TestParser(unittest.TestCase):
     def test_benchmarks_with_remainder(self):
         parsed = self.parser.parse_args(
             ['benchmarks', self.testInput0, self.testInput1, 'd'])
+        self.assertFalse(parsed.display_aggregates_only)
         self.assertTrue(parsed.utest)
         self.assertEqual(parsed.mode, 'benchmarks')
         self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
@@ -294,6 +324,7 @@ class TestParser(unittest.TestCase):
     def test_benchmarks_with_remainder_after_doubleminus(self):
         parsed = self.parser.parse_args(
             ['benchmarks', self.testInput0, self.testInput1, '--', 'e'])
+        self.assertFalse(parsed.display_aggregates_only)
         self.assertTrue(parsed.utest)
         self.assertEqual(parsed.mode, 'benchmarks')
         self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
@@ -303,6 +334,7 @@ class TestParser(unittest.TestCase):
     def test_filters_basic(self):
         parsed = self.parser.parse_args(
             ['filters', self.testInput0, 'c', 'd'])
+        self.assertFalse(parsed.display_aggregates_only)
         self.assertTrue(parsed.utest)
         self.assertEqual(parsed.mode, 'filters')
         self.assertEqual(parsed.test[0].name, self.testInput0)
@@ -313,6 +345,7 @@ class TestParser(unittest.TestCase):
     def test_filters_with_remainder(self):
         parsed = self.parser.parse_args(
             ['filters', self.testInput0, 'c', 'd', 'e'])
+        self.assertFalse(parsed.display_aggregates_only)
         self.assertTrue(parsed.utest)
         self.assertEqual(parsed.mode, 'filters')
         self.assertEqual(parsed.test[0].name, self.testInput0)
@@ -323,6 +356,7 @@ class TestParser(unittest.TestCase):
     def test_filters_with_remainder_after_doubleminus(self):
         parsed = self.parser.parse_args(
             ['filters', self.testInput0, 'c', 'd', '--', 'f'])
+        self.assertFalse(parsed.display_aggregates_only)
         self.assertTrue(parsed.utest)
         self.assertEqual(parsed.mode, 'filters')
         self.assertEqual(parsed.test[0].name, self.testInput0)
@@ -333,6 +367,7 @@ class TestParser(unittest.TestCase):
     def test_benchmarksfiltered_basic(self):
         parsed = self.parser.parse_args(
             ['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e'])
+        self.assertFalse(parsed.display_aggregates_only)
         self.assertTrue(parsed.utest)
         self.assertEqual(parsed.mode, 'benchmarksfiltered')
         self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
@@ -344,6 +379,7 @@ class TestParser(unittest.TestCase):
     def test_benchmarksfiltered_with_remainder(self):
         parsed = self.parser.parse_args(
             ['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e', 'f'])
+        self.assertFalse(parsed.display_aggregates_only)
         self.assertTrue(parsed.utest)
         self.assertEqual(parsed.mode, 'benchmarksfiltered')
         self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
@@ -355,6 +391,7 @@ class TestParser(unittest.TestCase):
     def test_benchmarksfiltered_with_remainder_after_doubleminus(self):
         parsed = self.parser.parse_args(
             ['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e', '--', 'g'])
+        self.assertFalse(parsed.display_aggregates_only)
         self.assertTrue(parsed.utest)
         self.assertEqual(parsed.mode, 'benchmarksfiltered')
         self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
@@ -365,7 +402,7 @@ class TestParser(unittest.TestCase):
 
 
 if __name__ == '__main__':
-    # unittest.main()
+    #unittest.main()
     main()
 
 # vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4

tools/gbench/Inputs/test3_run0.json

@@ -9,6 +9,7 @@
   "benchmarks": [
     {
       "name": "BM_One",
+      "run_type": "aggregate",
       "iterations": 1000,
       "real_time": 10,
       "cpu_time": 100,
@@ -30,6 +31,15 @@
     },
     {
       "name": "short",
+      "run_type": "aggregate",
+      "iterations": 1000,
+      "real_time": 8,
+      "cpu_time": 80,
+      "time_unit": "ns"
+    },
+    {
+      "name": "medium",
+      "run_type": "iteration",
       "iterations": 1000,
       "real_time": 8,
       "cpu_time": 80,

tools/gbench/Inputs/test3_run1.json

@@ -16,6 +16,7 @@
     },
     {
       "name": "BM_Two",
+      "run_type": "aggregate",
       "iterations": 1000,
       "real_time": 10,
       "cpu_time": 89,
@@ -30,10 +31,19 @@
     },
     {
       "name": "short",
+      "run_type": "aggregate",
       "iterations": 1000,
       "real_time": 8,
       "cpu_time": 80,
       "time_unit": "ns"
+    },
+    {
+      "name": "medium",
+      "run_type": "iteration",
+      "iterations": 1200,
+      "real_time": 5,
+      "cpu_time": 53,
+      "time_unit": "ns"
     }
   ]
 }

tools/gbench/report.py

@@ -96,6 +96,7 @@ def filter_benchmark(json_orig, family, replacement=""):
 def generate_difference_report(
         json1,
         json2,
+        display_aggregates_only=False,
         utest=False,
         utest_alpha=0.05,
         use_color=True):
@@ -191,6 +192,13 @@ def generate_difference_report(
         timings_cpu[0].append(bn['cpu_time'])
         timings_cpu[1].append(other_bench['cpu_time'])
 
+        # *After* recording this run for u test, *if* we were asked to only
+        # display aggregates, and if it is non-aggregate, then skip it.
+        if display_aggregates_only and 'run_type' in bn and 'run_type' in other_bench:
+            assert bn['run_type'] == other_bench['run_type']
+            if bn['run_type'] != 'aggregate':
+                continue;
+
         tres = calculate_change(timings_time[0][-1], timings_time[1][-1])
         cpures = calculate_change(timings_cpu[0][-1], timings_cpu[1][-1])
         output_strs += [color_format(use_color,
@@ -331,10 +339,60 @@ class TestReportDifferenceWithUTest(unittest.TestCase):
              'repetitions',
              'recommended.'],
             ['short', '+0.0000', '+0.0000', '8', '8', '80', '80'],
+            ['medium', '-0.3750', '-0.3375', '8', '5', '80', '53'],
+        ]
+        json1, json2 = self.load_results()
+        output_lines_with_header = generate_difference_report(
+            json1, json2, utest=True, utest_alpha=0.05, use_color=False)
+        output_lines = output_lines_with_header[2:]
+        print("\n")
+        print("\n".join(output_lines_with_header))
+        self.assertEqual(len(output_lines), len(expect_lines))
+        for i in range(0, len(output_lines)):
+            parts = [x for x in output_lines[i].split(' ') if x]
+            self.assertEqual(parts, expect_lines[i])
+
+
+class TestReportDifferenceWithUTestWhileDisplayingAggregatesOnly(unittest.TestCase):
+    def load_results(self):
+        import json
+        testInputs = os.path.join(
+            os.path.dirname(
+                os.path.realpath(__file__)),
+            'Inputs')
+        testOutput1 = os.path.join(testInputs, 'test3_run0.json')
+        testOutput2 = os.path.join(testInputs, 'test3_run1.json')
+        with open(testOutput1, 'r') as f:
+            json1 = json.load(f)
+        with open(testOutput2, 'r') as f:
+            json2 = json.load(f)
+        return json1, json2
+
+    def test_utest(self):
+        expect_lines = []
+        expect_lines = [
+            ['BM_One', '-0.1000', '+0.1000', '10', '9', '100', '110'],
+            ['BM_Two', '+0.1111', '-0.0111', '9', '10', '90', '89'],
+            ['BM_Two', '+0.2500', '+0.1125', '8', '10', '80', '89'],
+            ['BM_Two_pvalue',
+             '0.2207',
+             '0.6831',
+             'U',
+             'Test,',
+             'Repetitions:',
+             '2.',
+             'WARNING:',
+             'Results',
+             'unreliable!',
+             '9+',
+             'repetitions',
+             'recommended.'],
+            ['short', '+0.0000', '+0.0000', '8', '8', '80', '80'],
         ]
         json1, json2 = self.load_results()
         output_lines_with_header = generate_difference_report(
-            json1, json2, True, 0.05, use_color=False)
+            json1, json2, display_aggregates_only=True,
+            utest=True, utest_alpha=0.05, use_color=False)
         output_lines = output_lines_with_header[2:]
         print("\n")
         print("\n".join(output_lines_with_header))