Skip to content

J
json
Unverified Commit a8f711a2 by Niels Lohmann
Options

➕ using Google Benchmark #921

parent 6402077a
Showing with 121 additions and 370 deletions
README.md
...@@ -935,7 +935,6 @@ The library itself contains of a single header file licensed under the MIT licen ...@@ -935,7 +935,6 @@ The library itself contains of a single header file licensed under the MIT licen
- [**American fuzzy lop**](http://lcamtuf.coredump.cx/afl/) for fuzz testing - [**American fuzzy lop**](http://lcamtuf.coredump.cx/afl/) for fuzz testing
- [**AppVeyor**](https://www.appveyor.com) for [continuous integration](https://ci.appveyor.com/project/nlohmann/json) on Windows - [**AppVeyor**](https://www.appveyor.com) for [continuous integration](https://ci.appveyor.com/project/nlohmann/json) on Windows
- [**Artistic Style**](http://astyle.sourceforge.net) for automatic source code identation - [**Artistic Style**](http://astyle.sourceforge.net) for automatic source code identation
- [**benchpress**](https://github.com/sbs-ableton/benchpress) to benchmark the code
- [**Catch**](https://github.com/philsquared/Catch) for the unit tests - [**Catch**](https://github.com/philsquared/Catch) for the unit tests
- [**Clang**](http://clang.llvm.org) for compilation with code sanitizers - [**Clang**](http://clang.llvm.org) for compilation with code sanitizers
- [**Cmake**](https://cmake.org) for build automation - [**Cmake**](https://cmake.org) for build automation
...@@ -943,10 +942,10 @@ The library itself contains of a single header file licensed under the MIT licen ...@@ -943,10 +942,10 @@ The library itself contains of a single header file licensed under the MIT licen
- [**Coveralls**](https://coveralls.io) to measure [code coverage](https://coveralls.io/github/nlohmann/json) - [**Coveralls**](https://coveralls.io) to measure [code coverage](https://coveralls.io/github/nlohmann/json)
- [**Coverity Scan**](https://scan.coverity.com) for [static analysis](https://scan.coverity.com/projects/nlohmann-json) - [**Coverity Scan**](https://scan.coverity.com) for [static analysis](https://scan.coverity.com/projects/nlohmann-json)
- [**cppcheck**](http://cppcheck.sourceforge.net) for static analysis - [**cppcheck**](http://cppcheck.sourceforge.net) for static analysis
- [**cxxopts**](https://github.com/jarro2783/cxxopts) to let benchpress parse command-line parameters
- [**Doxygen**](http://www.stack.nl/~dimitri/doxygen/) to generate [documentation](https://nlohmann.github.io/json/) - [**Doxygen**](http://www.stack.nl/~dimitri/doxygen/) to generate [documentation](https://nlohmann.github.io/json/)
- [**git-update-ghpages**](https://github.com/rstacruz/git-update-ghpages) to upload the documentation to gh-pages - [**git-update-ghpages**](https://github.com/rstacruz/git-update-ghpages) to upload the documentation to gh-pages
- [**GitHub Changelog Generator**](https://github.com/skywinder/github-changelog-generator) to generate the [ChangeLog](https://github.com/nlohmann/json/blob/develop/ChangeLog.md) - [**GitHub Changelog Generator**](https://github.com/skywinder/github-changelog-generator) to generate the [ChangeLog](https://github.com/nlohmann/json/blob/develop/ChangeLog.md)
- [**Google Benchmark**]https://github.com/google/benchmark) to implement the benchmarks
- [**libFuzzer**](http://llvm.org/docs/LibFuzzer.html) to implement fuzz testing for OSS-Fuzz - [**libFuzzer**](http://llvm.org/docs/LibFuzzer.html) to implement fuzz testing for OSS-Fuzz
- [**OSS-Fuzz**](https://github.com/google/oss-fuzz) for continuous fuzz testing of the library - [**OSS-Fuzz**](https://github.com/google/oss-fuzz) for continuous fuzz testing of the library
- [**Probot**](https://probot.github.io) for automating maintainer tasks such as closing stale issues, requesting missing information, or detecting toxic comments. - [**Probot**](https://probot.github.io) for automating maintainer tasks such as closing stale issues, requesting missing information, or detecting toxic comments.
......
benchmarks/CMakeLists.txt 0 → 100644
cmake_minimum_required(VERSION 3.0)
project(JSON_Benchmarks LANGUAGES CXX)
# set compiler flags
if((CMAKE_CXX_COMPILER_ID MATCHES GNU) OR (CMAKE_CXX_COMPILER_ID MATCHES Clang))
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -flto -DNDEBUG -O3")
endif()
# configure Google Benchmarks
set(BENCHMARK_ENABLE_TESTING OFF CACHE INTERNAL "" FORCE)
add_subdirectory(thirdparty/benchmark)
# header directories
include_directories(thirdparty)
include_directories(${CMAKE_SOURCE_DIR}/src)
# copy test files to build folder
file(COPY ${CMAKE_SOURCE_DIR}/data DESTINATION .)
file(COPY ${CMAKE_SOURCE_DIR}/../test/data/regression/floats.json
${CMAKE_SOURCE_DIR}/../test/data/regression/unsigned_ints.json
${CMAKE_SOURCE_DIR}/../test/data/regression/signed_ints.json
DESTINATION data/numbers)
# benchmark binary
add_executable(json_benchmarks src/benchmarks.cpp)
target_link_libraries(json_benchmarks benchmark ${CMAKE_THREAD_LIBS_INIT})
benchmarks/Makefile deleted 100644 → 0
#
# Build/run json.hpp benchmarks, eg. CXX=g++-7 make
#
# The existing json_benchmarks did not allow optimization under some compilers
#
all: json_benchmarks json_benchmarks_simple number_jsons
bash -c 'time ./json_benchmarks'
bash -c 'time ./json_benchmarks_simple'
json_benchmarks: src/benchmarks.cpp ../src/json.hpp
$(CXX) -std=c++11 -pthread $(CXXFLAGS) -DNDEBUG -O3 -flto -I thirdparty/benchpress -I thirdparty/cxxopts -I../src src/benchmarks.cpp $(LDFLAGS) -o $@
json_benchmarks_simple: src/benchmarks_simple.cpp ../src/json.hpp
$(CXX) -std=c++11 $(CXXFLAGS) -DNDEBUG -O3 -flto -I../src $(<) $(LDFLAGS) -o $@
number_jsons:
(test -e files/numbers/floats.json -a -e files/numbers/signed_ints.json -a -e files/numbers/unsigned_ints.json) || (cd files/numbers ; python generate.py)
clean:
rm -f json_benchmarks json_benchmarks_simple files/numbers/*.json
benchmarks/README.md deleted 100644 → 0
# Bechmarks
Run `make` to compile and run a small set of benchmarks.
benchmarks/files/numbers/generate.py deleted 100755 → 0
#!/usr/bin/env python
import json
import random
import sys
random.seed(0)
# floats
result_floats = []
for x in range(0, 1000000):
result_floats.append(random.uniform(-100000000.0, 100000000.0))
json.dump(result_floats, open("floats.json", "w"), indent=2)
# unsigned integers
result_uints = []
for x in range(0, 1000000):
result_uints.append(random.randint(0, 18446744073709551615))
json.dump(result_uints, open("unsigned_ints.json", "w"), indent=2)
# signed integers
result_sints = []
for x in range(0, 1000000):
result_sints.append(random.randint(-9223372036854775808, 9223372036854775807))
json.dump(result_sints, open("signed_ints.json", "w"), indent=2)
benchmarks/src/benchmarks.cpp
#define BENCHPRESS_CONFIG_MAIN #include "benchmark/benchmark.h"
#include "json.hpp"
#include <fstream> #include <fstream>
#include <sstream>
#include <benchpress.hpp>
#include <json.hpp>
#include <pthread.h>
#include <thread>
using json = nlohmann::json; using json = nlohmann::json;
struct StartUp //////////////////////////////////////////////////////////////////////////////
// parse JSON from file
//////////////////////////////////////////////////////////////////////////////
static void ParseFile(benchmark::State& state, const char* filename)
{ {
StartUp() while (state.KeepRunning())
{ {
#ifndef __llvm__ state.PauseTiming();
// pin thread to a single CPU auto* f = new std::ifstream(filename);
cpu_set_t cpuset; auto* j = new json();
pthread_t thread; state.ResumeTiming();
thread = pthread_self();
CPU_ZERO(&cpuset); *j = json::parse(*f);
CPU_SET(std::thread::hardware_concurrency() - 1, &cpuset);
pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset); state.PauseTiming();
#endif delete f;
delete j;
state.ResumeTiming();
} }
};
StartUp startup;
enum class EMode { input, output_no_indent, output_with_indent }; std::ifstream file(filename, std::ios::binary | std::ios::ate);
state.SetBytesProcessed(state.iterations() * file.tellg());
}
BENCHMARK_CAPTURE(ParseFile, jeopardy, "data/jeopardy/jeopardy.json");
BENCHMARK_CAPTURE(ParseFile, canada, "data/nativejson-benchmark/canada.json");
BENCHMARK_CAPTURE(ParseFile, citm_catalog, "data/nativejson-benchmark/citm_catalog.json");
BENCHMARK_CAPTURE(ParseFile, twitter, "data/nativejson-benchmark/twitter.json");
BENCHMARK_CAPTURE(ParseFile, floats, "data/numbers/floats.json");
BENCHMARK_CAPTURE(ParseFile, signed_ints, "data/numbers/signed_ints.json");
BENCHMARK_CAPTURE(ParseFile, unsigned_ints, "data/numbers/unsigned_ints.json");
//////////////////////////////////////////////////////////////////////////////
// parse JSON from string
//////////////////////////////////////////////////////////////////////////////
static void bench(benchpress::context& ctx, static void ParseString(benchmark::State& state, const char* filename)
const std::string& in_path,
const EMode mode)
{ {
// using string streams for benchmarking to factor-out cold-cache disk std::ifstream f(filename);
// access. std::string str((std::istreambuf_iterator<char>(f)), std::istreambuf_iterator<char>());
#if defined( FROMFILE )
std::ifstream istr; while (state.KeepRunning())
{
istr.open( in_path, std::ifstream::in );
// read the stream once
json j;
istr >> j;
// clear flags and rewind
istr.clear();
istr.seekg(0);
}
#else
std::stringstream istr;
{ {
// read file into string stream state.PauseTiming();
std::ifstream input_file(in_path); auto* j = new json();
istr << input_file.rdbuf(); state.ResumeTiming();
input_file.close();
*j = json::parse(str);
// read the stream once
json j; state.PauseTiming();
istr >> j; delete j;
// clear flags and rewind state.ResumeTiming();
istr.clear();
istr.seekg(0);
} }
#endif
switch (mode) state.SetBytesProcessed(state.iterations() * str.size());
}
BENCHMARK_CAPTURE(ParseString, jeopardy, "data/jeopardy/jeopardy.json");
BENCHMARK_CAPTURE(ParseString, canada, "data/nativejson-benchmark/canada.json");
BENCHMARK_CAPTURE(ParseString, citm_catalog, "data/nativejson-benchmark/citm_catalog.json");
BENCHMARK_CAPTURE(ParseString, twitter, "data/nativejson-benchmark/twitter.json");
BENCHMARK_CAPTURE(ParseString, floats, "data/numbers/floats.json");
BENCHMARK_CAPTURE(ParseString, signed_ints, "data/numbers/signed_ints.json");
BENCHMARK_CAPTURE(ParseString, unsigned_ints, "data/numbers/unsigned_ints.json");
//////////////////////////////////////////////////////////////////////////////
// serialize JSON
//////////////////////////////////////////////////////////////////////////////
static void Dump(benchmark::State& state, const char* filename, int indent)
{
std::ifstream f(filename);
std::string str((std::istreambuf_iterator<char>(f)), std::istreambuf_iterator<char>());
json j = json::parse(str);
while (state.KeepRunning())
{ {
// benchmarking input j.dump(indent);
case EMode::input:
{
ctx.reset_timer();
for (size_t i = 0; i < ctx.num_iterations(); ++i)
{
// clear flags and rewind
istr.clear();
istr.seekg(0);
json j;
istr >> j;
}
break;
}
// benchmarking output
case EMode::output_no_indent:
case EMode::output_with_indent:
{
// create JSON value from input
json j;
istr >> j;
std::stringstream ostr;
ctx.reset_timer();
for (size_t i = 0; i < ctx.num_iterations(); ++i)
{
if (mode == EMode::output_no_indent)
{
ostr << j;
}
else
{
ostr << std::setw(4) << j;
}
// reset data
ostr.str(std::string());
}
break;
}
} }
}
#define BENCHMARK_I(mode, title, in_path) \ state.SetBytesProcessed(state.iterations() * j.dump(indent).size());
BENCHMARK((title), [](benchpress::context* ctx) \ }
{ \ BENCHMARK_CAPTURE(Dump, jeopardy / -, "data/jeopardy/jeopardy.json", -1);
bench(*ctx, (in_path), (mode)); \ BENCHMARK_CAPTURE(Dump, jeopardy / 4, "data/jeopardy/jeopardy.json", 4);
}) BENCHMARK_CAPTURE(Dump, canada / -, "data/nativejson-benchmark/canada.json", -1);
BENCHMARK_CAPTURE(Dump, canada / 4, "data/nativejson-benchmark/canada.json", 4);
BENCHMARK_I(EMode::input, "parse jeopardy.json", "files/jeopardy/jeopardy.json"); BENCHMARK_CAPTURE(Dump, citm_catalog / -, "data/nativejson-benchmark/citm_catalog.json", -1);
BENCHMARK_I(EMode::input, "parse canada.json", "files/nativejson-benchmark/canada.json"); BENCHMARK_CAPTURE(Dump, citm_catalog / 4, "data/nativejson-benchmark/citm_catalog.json", 4);
BENCHMARK_I(EMode::input, "parse citm_catalog.json", "files/nativejson-benchmark/citm_catalog.json"); BENCHMARK_CAPTURE(Dump, twitter / -, "data/nativejson-benchmark/twitter.json", -1);
BENCHMARK_I(EMode::input, "parse twitter.json", "files/nativejson-benchmark/twitter.json"); BENCHMARK_CAPTURE(Dump, twitter / 4, "data/nativejson-benchmark/twitter.json", 4);
BENCHMARK_I(EMode::input, "parse numbers/floats.json", "files/numbers/floats.json"); BENCHMARK_CAPTURE(Dump, floats / -, "data/numbers/floats.json", -1);
BENCHMARK_I(EMode::input, "parse numbers/signed_ints.json", "files/numbers/signed_ints.json"); BENCHMARK_CAPTURE(Dump, floats / 4, "data/numbers/floats.json", 4);
BENCHMARK_I(EMode::input, "parse numbers/unsigned_ints.json", "files/numbers/unsigned_ints.json"); BENCHMARK_CAPTURE(Dump, signed_ints / -, "data/numbers/signed_ints.json", -1);
BENCHMARK_CAPTURE(Dump, signed_ints / 4, "data/numbers/signed_ints.json", 4);
BENCHMARK_I(EMode::output_no_indent, "dump jeopardy.json", "files/jeopardy/jeopardy.json"); BENCHMARK_CAPTURE(Dump, unsigned_ints / -, "data/numbers/unsigned_ints.json", -1);
BENCHMARK_I(EMode::output_with_indent, "dump jeopardy.json with indent", "files/jeopardy/jeopardy.json"); BENCHMARK_CAPTURE(Dump, unsigned_ints / 4, "data/numbers/unsigned_ints.json", 4);
BENCHMARK_I(EMode::output_no_indent, "dump numbers/floats.json", "files/numbers/floats.json");
BENCHMARK_I(EMode::output_no_indent, "dump numbers/signed_ints.json", "files/numbers/signed_ints.json");
BENCHMARK_MAIN();
benchmarks/src/benchmarks_simple.cpp deleted 100644 → 0
//
// benchmarks_simple.cpp -- a less complex version of benchmarks.cpp, that better reflects actual performance
//
// For some reason, the complexity of benchmarks.cpp doesn't allow
// the compiler to optimize code using json.hpp effectively. The
// exact same tests, with the use of benchpress and cxxopts produces
// much faster code, at least under g++.
//
#include <fstream>
#include <iostream>
#include <chrono>
#include <list>
#include <tuple>
#include <json.hpp>
using json = nlohmann::json;
enum class EMode { input, output, indent };
static double bench(const EMode mode, size_t iters, const std::string& in_path )
{
// using string streams for benchmarking to factor-out cold-cache disk
// access. Define FROMFILE to use file I/O instead.
#if defined( FROMFILE )
std::ifstream istr;
{
istr.open( in_path, std::ifstream::in );
// read the stream once
json j;
istr >> j;
// clear flags and rewind
istr.clear();
istr.seekg(0);
}
#else
std::stringstream istr;
{
// read file into string stream
std::ifstream input_file(in_path);
istr << input_file.rdbuf();
input_file.close();
// read the stream once
json j;
istr >> j;
// clear flags and rewind
istr.clear();
istr.seekg(0);
}
#endif
double tps = 0;
switch (mode)
{
// benchmarking input
case EMode::input:
{
auto start = std::chrono::system_clock::now();
for (size_t i = 0; i < iters; ++i)
{
// clear flags and rewind
istr.clear();
istr.seekg(0);
json j;
istr >> j;
}
auto ended = std::chrono::system_clock::now();
tps = 1.0 / std::chrono::duration<double>( ended - start ).count();
break;
}
// benchmarking output
case EMode::output:
case EMode::indent:
{
// create JSON value from input
json j;
istr >> j;
std::stringstream ostr;
auto start = std::chrono::system_clock::now();
for (size_t i = 0; i < iters; ++i)
{
if (mode == EMode::indent)
{
ostr << j;
}
else
{
ostr << std::setw(4) << j;
}
// reset data
ostr.str(std::string());
}
auto ended = std::chrono::system_clock::now();
tps = 1.0 / std::chrono::duration<double>( ended - start ).count();
break;
}
}
return tps;
}
template <typename T>
struct average {
T _sum { 0 };
size_t _count { 0 };
T operator+=( const T &val_ ) { _sum += val_; +_count++; return val_; }
operator T() { return _sum / _count; }
};
// Execute each test approximately enough times to get near 1
// transaction per second, and compute the average; a single aggregate
// number that gives a performance metric representing both parsing
// and output.
int main( int, char ** )
{
std::list<std::tuple<std::string, EMode, size_t, std::string>> tests {
{ "parse jeopardy.json", EMode::input, 2, "files/jeopardy/jeopardy.json" },
{ "parse canada.json", EMode::input, 30, "files/nativejson-benchmark/canada.json" },
{ "parse citm_catalog.json", EMode::input, 120, "files/nativejson-benchmark/citm_catalog.json" },
{ "parse twitter.json", EMode::input, 225, "files/nativejson-benchmark/twitter.json" },
{ "parse floats.json", EMode::input, 5, "files/numbers/floats.json" },
{ "parse signed_ints.json", EMode::input, 6, "files/numbers/signed_ints.json" },
{ "parse unsigned_ints.json", EMode::input, 6, "files/numbers/unsigned_ints.json" },
{ "dump jeopardy.json", EMode::output, 5, "files/jeopardy/jeopardy.json" },
{ "dump jeopardy.json w/ind.", EMode::indent, 5, "files/jeopardy/jeopardy.json" },
{ "dump floats.json", EMode::output, 2, "files/numbers/floats.json" },
{ "dump signed_ints.json", EMode::output, 20, "files/numbers/signed_ints.json" },
};
average<double> avg;
for ( auto t : tests ) {
std::string name, path;
EMode mode;
size_t iters;
std::tie(name, mode, iters, path) = t;
auto tps = bench( mode, iters, path );
avg += tps;
std::cout
<< std::left
<< std::setw( 30 ) << name
<< std::right
<< " x " << std::setw( 3 ) << iters
<< std::left
<< " == " << std::setw( 10 ) << tps
<< std::right
<< " TPS, " << std::setw( 8 ) << std::round( tps * 1e6 / iters )
<< " ms/op"
<< std::endl;
}
std::cout << std::setw( 40 ) << "" << std::string( 10, '-' ) << std::endl;
std::cout << std::setw( 40 ) << "" << std::setw( 10 ) << std::left << avg << " TPS Average" << std::endl;
return 0;
}
benchmarks/thirdparty/benchpress/LICENSE deleted 100644 → 0
The MIT License (MIT)
Copyright (c) 2014 Christopher Gilbert
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
\ No newline at end of file
benchmarks/thirdparty/cxxopts/LICENSE deleted 100644 → 0
Copyright (c) 2014 Jarryd Beck
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
develop/json.hpp
...@@ -7247,7 +7247,7 @@ class basic_json ...@@ -7247,7 +7247,7 @@ class basic_json
// wrapper to get a value for an operation // wrapper to get a value for an operation
const auto get_value = [&val](const std::string & op, const auto get_value = [&val](const std::string & op,
const std::string & member, const std::string & member,
bool string_type) -> basic_json& bool string_type) -> basic_json &
{ {
// find value // find value
auto it = val.m_value.object->find(member); auto it = val.m_value.object->find(member);
......
src/json.hpp
...@@ -14535,7 +14535,7 @@ class basic_json ...@@ -14535,7 +14535,7 @@ class basic_json
// wrapper to get a value for an operation // wrapper to get a value for an operation
const auto get_value = [&val](const std::string & op, const auto get_value = [&val](const std::string & op,
const std::string & member, const std::string & member,
bool string_type) -> basic_json& bool string_type) -> basic_json &
{ {
// find value // find value
auto it = val.m_value.object->find(member); auto it = val.m_value.object->find(member);
......
test/src/unit-inspection.cpp
...@@ -316,8 +316,8 @@ TEST_CASE("object inspection") ...@@ -316,8 +316,8 @@ TEST_CASE("object inspection")
SECTION("round trips") SECTION("round trips")
{ {
for (const auto& s : for (const auto& s :
{"3.141592653589793", "1000000000000000010E5" {"3.141592653589793", "1000000000000000010E5"
}) })
{ {
json j1 = json::parse(s); json j1 = json::parse(s);
std::string s1 = j1.dump(); std::string s1 = j1.dump();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment