Merge pull request #1479 from petrhosek/json

googletest/CMakeLists.txt

@@ -304,7 +304,9 @@ if (gtest_build_tests)
   cxx_executable(gtest_xml_outfile1_test_ test gtest_main)
   cxx_executable(gtest_xml_outfile2_test_ test gtest_main)
   py_test(gtest_xml_outfiles_test)
+  py_test(gtest_json_outfiles_test)
 
   cxx_executable(gtest_xml_output_unittest_ test gtest)
   py_test(gtest_xml_output_unittest)
+  py_test(gtest_json_output_unittest)
 endif()

googletest/docs/AdvancedGuide.md

@@ -2060,6 +2060,207 @@ Things to note:
 
 _Availability:_ Linux, Windows, Mac.
 
+#### Generating an JSON Report {#JsonReport}
+
+gUnit can also emit a JSON report as an alternative format to XML. To generate
+the JSON report, set the `GUNIT_OUTPUT` environment variable or the
+`--gunit_output` flag to the string `"json:path_to_output_file"`, which will
+create the file at the given location. You can also just use the string
+`"json"`, in which case the output can be found in the `test_detail.json` file
+in the current directory.
+
+The report format conforms to the following JSON Schema:
+
+```json
+{
+  "$schema": "http://json-schema.org/schema#",
+  "type": "object",
+  "definitions": {
+    "TestCase": {
+      "type": "object",
+      "properties": {
+        "name": { "type": "string" },
+        "tests": { "type": "integer" },
+        "failures": { "type": "integer" },
+        "disabled": { "type": "integer" },
+        "time": { "type": "string" },
+        "testsuite": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/TestInfo"
+          }
+        }
+      }
+    },
+    "TestInfo": {
+      "type": "object",
+      "properties": {
+        "name": { "type": "string" },
+        "status": {
+          "type": "string",
+          "enum": ["RUN", "NOTRUN"]
+        },
+        "time": { "type": "string" },
+        "classname": { "type": "string" },
+        "failures": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/Failure"
+          }
+        }
+      }
+    },
+    "Failure": {
+      "type": "object",
+      "properties": {
+        "failures": { "type": "string" },
+        "type": { "type": "string" }
+      }
+    }
+  },
+  "properties": {
+    "tests": { "type": "integer" },
+    "failures": { "type": "integer" },
+    "disabled": { "type": "integer" },
+    "errors": { "type": "integer" },
+    "timestamp": {
+      "type": "string",
+      "format": "date-time"
+    },
+    "time": { "type": "string" },
+    "name": { "type": "string" },
+    "testsuites": {
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/TestCase"
+      }
+    }
+  }
+}
+```
+
+The report uses the format that conforms to the following Proto3 using the
+[JSON encoding](https://developers.google.com/protocol-buffers/docs/proto3#json):
+
+```proto
+syntax = "proto3";
+
+package googletest;
+
+import "google/protobuf/timestamp.proto";
+import "google/protobuf/duration.proto";
+
+message UnitTest {
+  int32 tests = 1;
+  int32 failures = 2;
+  int32 disabled = 3;
+  int32 errors = 4;
+  google.protobuf.Timestamp timestamp = 5;
+  google.protobuf.Duration time = 6;
+  string name = 7;
+  repeated TestCase testsuites = 8;
+}
+
+message TestCase {
+  string name = 1;
+  int32 tests = 2;
+  int32 failures = 3;
+  int32 disabled = 4;
+  int32 errors = 5;
+  google.protobuf.Duration time = 6;
+  repeated TestInfo testsuite = 7;
+}
+
+message TestInfo {
+  string name = 1;
+  enum Status {
+    RUN = 0;
+    NOTRUN = 1;
+  }
+  Status status = 2;
+  google.protobuf.Duration time = 3;
+  string classname = 4;
+  message Failure {
+    string failures = 1;
+    string type = 2;
+  }
+  repeated Failure failures = 5;
+}
+```
+
+For instance, the following program
+
+```c++
+TEST(MathTest, Addition) { ... }
+TEST(MathTest, Subtraction) { ... }
+TEST(LogicTest, NonContradiction) { ... }
+```
+
+could generate this report:
+
+```json
+{
+  "tests": 3,
+  "failures": 1,
+  "errors": 0,
+  "time": "0.035s",
+  "timestamp": "2011-10-31T18:52:42Z"
+  "name": "AllTests",
+  "testsuites": [
+    {
+      "name": "MathTest",
+      "tests": 2,
+      "failures": 1,
+      "errors": 0,
+      "time": "0.015s",
+      "testsuite": [
+        {
+          "name": "Addition",
+          "status": "RUN",
+          "time": "0.007s",
+          "classname": "",
+          "failures": [
+            {
+              "message": "Value of: add(1, 1)\x0A  Actual: 3\x0AExpected: 2",
+              "type": ""
+            },
+            {
+              "message": "Value of: add(1, -1)\x0A  Actual: 1\x0AExpected: 0",
+              "type": ""
+            }
+          ]
+        },
+        {
+          "name": "Subtraction",
+          "status": "RUN",
+          "time": "0.005s",
+          "classname": ""
+        }
+      ]
+    }
+    {
+      "name": "LogicTest",
+      "tests": 1,
+      "failures": 0,
+      "errors": 0,
+      "time": "0.005s",
+      "testsuite": [
+        {
+          "name": "NonContradiction",
+          "status": "RUN",
+          "time": "0.005s",
+          "classname": ""
+        }
+      ]
+    }
+  ]
+}
+```
+
+IMPORTANT: The exact format of the JSON document is subject to change.
+
+**Availability**: Linux, Windows, Mac.
+
 ## Controlling How Failures Are Reported ##
 
 ### Turning Assertion Failures into Break-Points ###

googletest/test/gtest_json_outfiles_test.py

+#!/usr/bin/env python
+# Copyright 2018, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Unit test for the gtest_json_output module."""
+
+import json
+import os
+import gtest_test_utils
+import gtest_json_test_utils
+
+
+GTEST_OUTPUT_SUBDIR = 'json_outfiles'
+GTEST_OUTPUT_1_TEST = 'gtest_xml_outfile1_test_'
+GTEST_OUTPUT_2_TEST = 'gtest_xml_outfile2_test_'
+
+EXPECTED_1 = {
+    u'tests': 1,
+    u'failures': 0,
+    u'disabled': 0,
+    u'errors': 0,
+    u'time': u'*',
+    u'timestamp': u'*',
+    u'name': u'AllTests',
+    u'testsuites': [{
+        u'name': u'PropertyOne',
+        u'tests': 1,
+        u'failures': 0,
+        u'disabled': 0,
+        u'errors': 0,
+        u'time': u'*',
+        u'testsuite': [{
+            u'name': u'TestSomeProperties',
+            u'status': u'RUN',
+            u'time': u'*',
+            u'classname': u'PropertyOne',
+            u'SetUpProp': u'1',
+            u'TestSomeProperty': u'1',
+            u'TearDownProp': u'1',
+        }],
+    }],
+}
+
+EXPECTED_2 = {
+    u'tests': 1,
+    u'failures': 0,
+    u'disabled': 0,
+    u'errors': 0,
+    u'time': u'*',
+    u'timestamp': u'*',
+    u'name': u'AllTests',
+    u'testsuites': [{
+        u'name': u'PropertyTwo',
+        u'tests': 1,
+        u'failures': 0,
+        u'disabled': 0,
+        u'errors': 0,
+        u'time': u'*',
+        u'testsuite': [{
+            u'name': u'TestSomeProperties',
+            u'status': u'RUN',
+            u'time': u'*',
+            u'classname': u'PropertyTwo',
+            u'SetUpProp': u'2',
+            u'TestSomeProperty': u'2',
+            u'TearDownProp': u'2',
+        }],
+    }],
+}
+
+
+class GTestJsonOutFilesTest(gtest_test_utils.TestCase):
+  """Unit test for Google Test's JSON output functionality."""
+
+  def setUp(self):
+    # We want the trailing '/' that the last "" provides in os.path.join, for
+    # telling Google Test to create an output directory instead of a single file
+    # for xml output.
+    self.output_dir_ = os.path.join(gtest_test_utils.GetTempDir(),
+                                    GTEST_OUTPUT_SUBDIR, '')
+    self.DeleteFilesAndDir()
+
+  def tearDown(self):
+    self.DeleteFilesAndDir()
+
+  def DeleteFilesAndDir(self):
+    try:
+      os.remove(os.path.join(self.output_dir_, GTEST_OUTPUT_1_TEST + '.json'))
+    except os.error:
+      pass
+    try:
+      os.remove(os.path.join(self.output_dir_, GTEST_OUTPUT_2_TEST + '.json'))
+    except os.error:
+      pass
+    try:
+      os.rmdir(self.output_dir_)
+    except os.error:
+      pass
+
+  def testOutfile1(self):
+    self._TestOutFile(GTEST_OUTPUT_1_TEST, EXPECTED_1)
+
+  def testOutfile2(self):
+    self._TestOutFile(GTEST_OUTPUT_2_TEST, EXPECTED_2)
+
+  def _TestOutFile(self, test_name, expected):
+    gtest_prog_path = gtest_test_utils.GetTestExecutablePath(test_name)
+    command = [gtest_prog_path, '--gtest_output=json:%s' % self.output_dir_]
+    p = gtest_test_utils.Subprocess(command,
+                                    working_dir=gtest_test_utils.GetTempDir())
+    self.assert_(p.exited)
+    self.assertEquals(0, p.exit_code)
+
+    # TODO(wan@google.com): libtool causes the built test binary to be
+    #   named lt-gtest_xml_outfiles_test_ instead of
+    #   gtest_xml_outfiles_test_.  To account for this possibility, we
+    #   allow both names in the following code.  We should remove this
+    #   hack when Chandler Carruth's libtool replacement tool is ready.
+    output_file_name1 = test_name + '.json'
+    output_file1 = os.path.join(self.output_dir_, output_file_name1)
+    output_file_name2 = 'lt-' + output_file_name1
+    output_file2 = os.path.join(self.output_dir_, output_file_name2)
+    self.assert_(os.path.isfile(output_file1) or os.path.isfile(output_file2),
+                 output_file1)
+
+    if os.path.isfile(output_file1):
+      with open(output_file1) as f:
+        actual = json.load(f)
+    else:
+      with open(output_file2) as f:
+        actual = json.load(f)
+    self.assertEqual(expected, gtest_json_test_utils.normalize(actual))
+
+
+if __name__ == '__main__':
+  os.environ['GTEST_STACK_TRACE_DEPTH'] = '0'
+  gtest_test_utils.Main()

googletest/test/gtest_json_test_utils.py

+# Copyright 2018, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Unit test utilities for gtest_json_output."""
+
+import re
+
+
+def normalize(obj):
+  """Normalize output object.
+
+  Args:
+     obj: Google Test's JSON output object to normalize.
+
+  Returns:
+     Normalized output without any references to transient information that may
+     change from run to run.
+  """
+  def _normalize(key, value):
+    if key == 'time':
+      return re.sub(r'^\d+(\.\d+)?s$', u'*', value)
+    elif key == 'timestamp':
+      return re.sub(r'^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$', '*', value)
+    elif key == 'failure':
+      value = re.sub(r'^.*[/\\](.*:)\d+\n', '\\1*\n', value)
+      return re.sub(r'Stack trace:\n(.|\n)*', 'Stack trace:\n*', value)
+    else:
+      return normalize(value)
+  if isinstance(obj, dict):
+    return {k: _normalize(k, v) for k, v in obj.items()}
+  if isinstance(obj, list):
+    return [normalize(x) for x in obj]
+  else:
+    return obj