Skip to content

J
json
Unverified Commit daeb48b0 by Niels Lohmann Committed by GitHub
Options

Merge pull request #1411 from nickaein/develop

Improve dump_integer performance
parents 29a03f46 c9dd260a
Showing with 253 additions and 28 deletions
benchmarks/CMakeLists.txt
...@@ -19,6 +19,7 @@ file(COPY ${CMAKE_SOURCE_DIR}/data DESTINATION .) ...@@ -19,6 +19,7 @@ file(COPY ${CMAKE_SOURCE_DIR}/data DESTINATION .)
file(COPY ${CMAKE_SOURCE_DIR}/../test/data/regression/floats.json file(COPY ${CMAKE_SOURCE_DIR}/../test/data/regression/floats.json
${CMAKE_SOURCE_DIR}/../test/data/regression/unsigned_ints.json ${CMAKE_SOURCE_DIR}/../test/data/regression/unsigned_ints.json
${CMAKE_SOURCE_DIR}/../test/data/regression/signed_ints.json ${CMAKE_SOURCE_DIR}/../test/data/regression/signed_ints.json
${CMAKE_SOURCE_DIR}/../test/data/regression/small_signed_ints.json
DESTINATION data/numbers) DESTINATION data/numbers)
# benchmark binary # benchmark binary
......
benchmarks/src/benchmarks.cpp
...@@ -35,7 +35,7 @@ BENCHMARK_CAPTURE(ParseFile, twitter, "data/nativejson-benchmark/twitter.j ...@@ -35,7 +35,7 @@ BENCHMARK_CAPTURE(ParseFile, twitter, "data/nativejson-benchmark/twitter.j
BENCHMARK_CAPTURE(ParseFile, floats, "data/numbers/floats.json"); BENCHMARK_CAPTURE(ParseFile, floats, "data/numbers/floats.json");
BENCHMARK_CAPTURE(ParseFile, signed_ints, "data/numbers/signed_ints.json"); BENCHMARK_CAPTURE(ParseFile, signed_ints, "data/numbers/signed_ints.json");
BENCHMARK_CAPTURE(ParseFile, unsigned_ints, "data/numbers/unsigned_ints.json"); BENCHMARK_CAPTURE(ParseFile, unsigned_ints, "data/numbers/unsigned_ints.json");
BENCHMARK_CAPTURE(ParseFile, small_signed_ints, "data/numbers/small_signed_ints.json");
////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////
// parse JSON from string // parse JSON from string
...@@ -68,6 +68,7 @@ BENCHMARK_CAPTURE(ParseString, twitter, "data/nativejson-benchmark/twitter ...@@ -68,6 +68,7 @@ BENCHMARK_CAPTURE(ParseString, twitter, "data/nativejson-benchmark/twitter
BENCHMARK_CAPTURE(ParseString, floats, "data/numbers/floats.json"); BENCHMARK_CAPTURE(ParseString, floats, "data/numbers/floats.json");
BENCHMARK_CAPTURE(ParseString, signed_ints, "data/numbers/signed_ints.json"); BENCHMARK_CAPTURE(ParseString, signed_ints, "data/numbers/signed_ints.json");
BENCHMARK_CAPTURE(ParseString, unsigned_ints, "data/numbers/unsigned_ints.json"); BENCHMARK_CAPTURE(ParseString, unsigned_ints, "data/numbers/unsigned_ints.json");
BENCHMARK_CAPTURE(ParseString, small_signed_ints, "data/numbers/small_signed_ints.json");
////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////
...@@ -101,6 +102,8 @@ BENCHMARK_CAPTURE(Dump, signed_ints / -, "data/numbers/signed_ints.json", ...@@ -101,6 +102,8 @@ BENCHMARK_CAPTURE(Dump, signed_ints / -, "data/numbers/signed_ints.json",
BENCHMARK_CAPTURE(Dump, signed_ints / 4, "data/numbers/signed_ints.json", 4); BENCHMARK_CAPTURE(Dump, signed_ints / 4, "data/numbers/signed_ints.json", 4);
BENCHMARK_CAPTURE(Dump, unsigned_ints / -, "data/numbers/unsigned_ints.json", -1); BENCHMARK_CAPTURE(Dump, unsigned_ints / -, "data/numbers/unsigned_ints.json", -1);
BENCHMARK_CAPTURE(Dump, unsigned_ints / 4, "data/numbers/unsigned_ints.json", 4); BENCHMARK_CAPTURE(Dump, unsigned_ints / 4, "data/numbers/unsigned_ints.json", 4);
BENCHMARK_CAPTURE(Dump, small_signed_ints / -, "data/numbers/small_signed_ints.json", -1);
BENCHMARK_CAPTURE(Dump, small_signed_ints / 4, "data/numbers/small_signed_ints.json", 4);
BENCHMARK_MAIN(); BENCHMARK_MAIN();
include/nlohmann/detail/output/serializer.hpp
...@@ -528,6 +528,40 @@ class serializer ...@@ -528,6 +528,40 @@ class serializer
} }
/*! /*!
@brief count digits
Count the number of decimal (base 10) digits for an input unsigned integer.
@param[in] x unsigned integer number to count its digits
@return number of decimal digits
*/
inline unsigned int count_digits(number_unsigned_t x) noexcept
{
unsigned int n_digits = 1;
for (;;)
{
if (x < 10)
{
return n_digits;
}
if (x < 100)
{
return n_digits + 1;
}
if (x < 1000)
{
return n_digits + 2;
}
if (x < 10000)
{
return n_digits + 3;
}
x = x / 10000u;
n_digits += 4;
}
}
/*!
@brief dump an integer @brief dump an integer
Dump a given integer to output stream @a o. Works internally with Dump a given integer to output stream @a o. Works internally with
...@@ -542,6 +576,22 @@ class serializer ...@@ -542,6 +576,22 @@ class serializer
int> = 0> int> = 0>
void dump_integer(NumberType x) void dump_integer(NumberType x)
{ {
static constexpr std::array<std::array<char, 2>, 100> digits_to_99
{
{
{'0', '0'}, {'0', '1'}, {'0', '2'}, {'0', '3'}, {'0', '4'}, {'0', '5'}, {'0', '6'}, {'0', '7'}, {'0', '8'}, {'0', '9'},
{'1', '0'}, {'1', '1'}, {'1', '2'}, {'1', '3'}, {'1', '4'}, {'1', '5'}, {'1', '6'}, {'1', '7'}, {'1', '8'}, {'1', '9'},
{'2', '0'}, {'2', '1'}, {'2', '2'}, {'2', '3'}, {'2', '4'}, {'2', '5'}, {'2', '6'}, {'2', '7'}, {'2', '8'}, {'2', '9'},
{'3', '0'}, {'3', '1'}, {'3', '2'}, {'3', '3'}, {'3', '4'}, {'3', '5'}, {'3', '6'}, {'3', '7'}, {'3', '8'}, {'3', '9'},
{'4', '0'}, {'4', '1'}, {'4', '2'}, {'4', '3'}, {'4', '4'}, {'4', '5'}, {'4', '6'}, {'4', '7'}, {'4', '8'}, {'4', '9'},
{'5', '0'}, {'5', '1'}, {'5', '2'}, {'5', '3'}, {'5', '4'}, {'5', '5'}, {'5', '6'}, {'5', '7'}, {'5', '8'}, {'5', '9'},
{'6', '0'}, {'6', '1'}, {'6', '2'}, {'6', '3'}, {'6', '4'}, {'6', '5'}, {'6', '6'}, {'6', '7'}, {'6', '8'}, {'6', '9'},
{'7', '0'}, {'7', '1'}, {'7', '2'}, {'7', '3'}, {'7', '4'}, {'7', '5'}, {'7', '6'}, {'7', '7'}, {'7', '8'}, {'7', '9'},
{'8', '0'}, {'8', '1'}, {'8', '2'}, {'8', '3'}, {'8', '4'}, {'8', '5'}, {'8', '6'}, {'8', '7'}, {'8', '8'}, {'8', '9'},
{'9', '0'}, {'9', '1'}, {'9', '2'}, {'9', '3'}, {'9', '4'}, {'9', '5'}, {'9', '6'}, {'9', '7'}, {'9', '8'}, {'9', '9'},
}
};
// special case for "0" // special case for "0"
if (x == 0) if (x == 0)
{ {
...@@ -549,28 +599,58 @@ class serializer ...@@ -549,28 +599,58 @@ class serializer
return; return;
} }
const bool is_negative = std::is_same<NumberType, number_integer_t>::value and not (x >= 0); // see issue #755 // use a pointer to fill the buffer
std::size_t i = 0; auto buffer_ptr = begin(number_buffer);
const bool is_negative = std::is_same<NumberType, number_integer_t>::value and not(x >= 0); // see issue #755
number_unsigned_t abs_value;
unsigned int n_chars;
while (x != 0) if (is_negative)
{
*buffer_ptr = '-';
abs_value = static_cast<number_unsigned_t>(0 - x);
// account one more byte for the minus sign
n_chars = 1 + count_digits(abs_value);
}
else
{ {
abs_value = static_cast<number_unsigned_t>(x);
n_chars = count_digits(abs_value);
}
// spare 1 byte for '\0' // spare 1 byte for '\0'
assert(i < number_buffer.size() - 1); assert(n_chars < number_buffer.size() - 1);
const auto digit = std::labs(static_cast<long>(x % 10)); // jump to the end to generate the string from backward
number_buffer[i++] = static_cast<char>('0' + digit); // so we later avoid reversing the result
x /= 10; buffer_ptr += n_chars;
// Fast int2ascii implementation inspired by "Fastware" talk by Andrei Alexandrescu
// See: https://www.youtube.com/watch?v=o4-CwDo2zpg
const auto buffer_end = buffer_ptr;
while (abs_value >= 100)
{
const auto digits_index = static_cast<unsigned>((abs_value % 100));
abs_value /= 100;
*(--buffer_ptr) = digits_to_99[digits_index][1];
*(--buffer_ptr) = digits_to_99[digits_index][0];
} }
if (is_negative) if (abs_value >= 10)
{
const auto digits_index = static_cast<unsigned>(abs_value);
*(--buffer_ptr) = digits_to_99[digits_index][1];
*(--buffer_ptr) = digits_to_99[digits_index][0];
}
else
{ {
// make sure there is capacity for the '-' *(--buffer_ptr) = static_cast<char>('0' + abs_value);
assert(i < number_buffer.size() - 2);
number_buffer[i++] = '-';
} }
std::reverse(number_buffer.begin(), number_buffer.begin() + i); o->write_characters(number_buffer.data(), n_chars);
o->write_characters(number_buffer.data(), i);
} }
/*! /*!
......
single_include/nlohmann/json.hpp
...@@ -11411,6 +11411,40 @@ class serializer ...@@ -11411,6 +11411,40 @@ class serializer
} }
/*! /*!
@brief count digits
Count the number of decimal (base 10) digits for an input unsigned integer.
@param[in] x unsigned integer number to count its digits
@return number of decimal digits
*/
inline unsigned int count_digits(number_unsigned_t x) noexcept
{
unsigned int n_digits = 1;
for (;;)
{
if (x < 10)
{
return n_digits;
}
if (x < 100)
{
return n_digits + 1;
}
if (x < 1000)
{
return n_digits + 2;
}
if (x < 10000)
{
return n_digits + 3;
}
x = x / 10000u;
n_digits += 4;
}
}
/*!
@brief dump an integer @brief dump an integer
Dump a given integer to output stream @a o. Works internally with Dump a given integer to output stream @a o. Works internally with
...@@ -11425,6 +11459,22 @@ class serializer ...@@ -11425,6 +11459,22 @@ class serializer
int> = 0> int> = 0>
void dump_integer(NumberType x) void dump_integer(NumberType x)
{ {
static constexpr std::array<std::array<char, 2>, 100> digits_to_99
{
{
{'0', '0'}, {'0', '1'}, {'0', '2'}, {'0', '3'}, {'0', '4'}, {'0', '5'}, {'0', '6'}, {'0', '7'}, {'0', '8'}, {'0', '9'},
{'1', '0'}, {'1', '1'}, {'1', '2'}, {'1', '3'}, {'1', '4'}, {'1', '5'}, {'1', '6'}, {'1', '7'}, {'1', '8'}, {'1', '9'},
{'2', '0'}, {'2', '1'}, {'2', '2'}, {'2', '3'}, {'2', '4'}, {'2', '5'}, {'2', '6'}, {'2', '7'}, {'2', '8'}, {'2', '9'},
{'3', '0'}, {'3', '1'}, {'3', '2'}, {'3', '3'}, {'3', '4'}, {'3', '5'}, {'3', '6'}, {'3', '7'}, {'3', '8'}, {'3', '9'},
{'4', '0'}, {'4', '1'}, {'4', '2'}, {'4', '3'}, {'4', '4'}, {'4', '5'}, {'4', '6'}, {'4', '7'}, {'4', '8'}, {'4', '9'},
{'5', '0'}, {'5', '1'}, {'5', '2'}, {'5', '3'}, {'5', '4'}, {'5', '5'}, {'5', '6'}, {'5', '7'}, {'5', '8'}, {'5', '9'},
{'6', '0'}, {'6', '1'}, {'6', '2'}, {'6', '3'}, {'6', '4'}, {'6', '5'}, {'6', '6'}, {'6', '7'}, {'6', '8'}, {'6', '9'},
{'7', '0'}, {'7', '1'}, {'7', '2'}, {'7', '3'}, {'7', '4'}, {'7', '5'}, {'7', '6'}, {'7', '7'}, {'7', '8'}, {'7', '9'},
{'8', '0'}, {'8', '1'}, {'8', '2'}, {'8', '3'}, {'8', '4'}, {'8', '5'}, {'8', '6'}, {'8', '7'}, {'8', '8'}, {'8', '9'},
{'9', '0'}, {'9', '1'}, {'9', '2'}, {'9', '3'}, {'9', '4'}, {'9', '5'}, {'9', '6'}, {'9', '7'}, {'9', '8'}, {'9', '9'},
}
};
// special case for "0" // special case for "0"
if (x == 0) if (x == 0)
{ {
...@@ -11432,28 +11482,58 @@ class serializer ...@@ -11432,28 +11482,58 @@ class serializer
return; return;
} }
const bool is_negative = std::is_same<NumberType, number_integer_t>::value and not (x >= 0); // see issue #755 // use a pointer to fill the buffer
std::size_t i = 0; auto buffer_ptr = begin(number_buffer);
const bool is_negative = std::is_same<NumberType, number_integer_t>::value and not(x >= 0); // see issue #755
number_unsigned_t abs_value;
while (x != 0) unsigned int n_chars;
if (is_negative)
{ {
*buffer_ptr = '-';
abs_value = static_cast<number_unsigned_t>(0 - x);
// account one more byte for the minus sign
n_chars = 1 + count_digits(abs_value);
}
else
{
abs_value = static_cast<number_unsigned_t>(x);
n_chars = count_digits(abs_value);
}
// spare 1 byte for '\0' // spare 1 byte for '\0'
assert(i < number_buffer.size() - 1); assert(n_chars < number_buffer.size() - 1);
const auto digit = std::labs(static_cast<long>(x % 10)); // jump to the end to generate the string from backward
number_buffer[i++] = static_cast<char>('0' + digit); // so we later avoid reversing the result
x /= 10; buffer_ptr += n_chars;
// Fast int2ascii implementation inspired by "Fastware" talk by Andrei Alexandrescu
// See: https://www.youtube.com/watch?v=o4-CwDo2zpg
const auto buffer_end = buffer_ptr;
while (abs_value >= 100)
{
const auto digits_index = static_cast<unsigned>((abs_value % 100));
abs_value /= 100;
*(--buffer_ptr) = digits_to_99[digits_index][1];
*(--buffer_ptr) = digits_to_99[digits_index][0];
} }
if (is_negative) if (abs_value >= 10)
{
const auto digits_index = static_cast<unsigned>(abs_value);
*(--buffer_ptr) = digits_to_99[digits_index][1];
*(--buffer_ptr) = digits_to_99[digits_index][0];
}
else
{ {
// make sure there is capacity for the '-' *(--buffer_ptr) = static_cast<char>('0' + abs_value);
assert(i < number_buffer.size() - 2);
number_buffer[i++] = '-';
} }
std::reverse(number_buffer.begin(), number_buffer.begin() + i); o->write_characters(number_buffer.data(), n_chars);
o->write_characters(number_buffer.data(), i);
} }
/*! /*!
......
test/data/regression/small_signed_ints.json 0 → 100755
This source diff could not be displayed because it is too large. You can view the blob instead.
test/src/unit-regression.cpp
...@@ -669,7 +669,8 @@ TEST_CASE("regression tests") ...@@ -669,7 +669,8 @@ TEST_CASE("regression tests")
{ {
"test/data/regression/floats.json", "test/data/regression/floats.json",
"test/data/regression/signed_ints.json", "test/data/regression/signed_ints.json",
"test/data/regression/unsigned_ints.json" "test/data/regression/unsigned_ints.json",
"test/data/regression/small_signed_ints.json"
}) })
{ {
CAPTURE(filename) CAPTURE(filename)
......
test/src/unit-to_chars.cpp
...@@ -471,4 +471,64 @@ TEST_CASE("formatting") ...@@ -471,4 +471,64 @@ TEST_CASE("formatting")
check_double( 1.2345e+21, "1.2344999999999999e+21" ); // 1.2345e+21 1.2344999999999999e+21 1.2345e21 check_double( 1.2345e+21, "1.2344999999999999e+21" ); // 1.2345e+21 1.2344999999999999e+21 1.2345e21
check_double( 1.2345e+22, "1.2345e+22" ); // 1.2345e+22 1.2345e+22 1.2345e22 check_double( 1.2345e+22, "1.2345e+22" ); // 1.2345e+22 1.2345e+22 1.2345e22
} }
SECTION("integer")
{
auto check_integer = [](std::int64_t number, const std::string & expected)
{
nlohmann::json j = number;
CHECK(j.dump() == expected);
};
// edge cases
check_integer(INT64_MIN, "-9223372036854775808");
check_integer(INT64_MAX, "9223372036854775807");
// few random big integers
check_integer(-3456789012345678901LL, "-3456789012345678901");
check_integer(3456789012345678901LL, "3456789012345678901");
check_integer(-5678901234567890123LL, "-5678901234567890123");
check_integer(5678901234567890123LL, "5678901234567890123");
// integers with various digit counts
check_integer(-1000000000000000000LL, "-1000000000000000000");
check_integer(-100000000000000000LL, "-100000000000000000");
check_integer(-10000000000000000LL, "-10000000000000000");
check_integer(-1000000000000000LL, "-1000000000000000");
check_integer(-100000000000000LL, "-100000000000000");
check_integer(-10000000000000LL, "-10000000000000");
check_integer(-1000000000000LL, "-1000000000000");
check_integer(-100000000000LL, "-100000000000");
check_integer(-10000000000LL, "-10000000000");
check_integer(-1000000000LL, "-1000000000");
check_integer(-100000000LL, "-100000000");
check_integer(-10000000LL, "-10000000");
check_integer(-1000000LL, "-1000000");
check_integer(-100000LL, "-100000");
check_integer(-10000LL, "-10000");
check_integer(-1000LL, "-1000");
check_integer(-100LL, "-100");
check_integer(-10LL, "-10");
check_integer(-1LL, "-1");
check_integer(0, "0");
check_integer(1LL, "1");
check_integer(10LL, "10");
check_integer(100LL, "100");
check_integer(1000LL, "1000");
check_integer(10000LL, "10000");
check_integer(100000LL, "100000");
check_integer(1000000LL, "1000000");
check_integer(10000000LL, "10000000");
check_integer(100000000LL, "100000000");
check_integer(1000000000LL, "1000000000");
check_integer(10000000000LL, "10000000000");
check_integer(100000000000LL, "100000000000");
check_integer(1000000000000LL, "1000000000000");
check_integer(10000000000000LL, "10000000000000");
check_integer(100000000000000LL, "100000000000000");
check_integer(1000000000000000LL, "1000000000000000");
check_integer(10000000000000000LL, "10000000000000000");
check_integer(100000000000000000LL, "100000000000000000");
check_integer(1000000000000000000LL, "1000000000000000000");
}
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment