Skip to content

J
json
Commit 012c9665 by Michael Reilly
Options

Add binary type support to all binary file formats, as well as an internally…

Add binary type support to all binary file formats, as well as an internally represented binary type
parent 6121fc52
Showing with 860 additions and 31 deletions
include/nlohmann/detail/conversions/to_json.hpp
...@@ -68,6 +68,28 @@ struct external_constructor<value_t::string> ...@@ -68,6 +68,28 @@ struct external_constructor<value_t::string>
}; };
template<> template<>
struct external_constructor<value_t::binary>
{
template<typename BasicJsonType>
static void construct(BasicJsonType& j, const typename BasicJsonType::binary_t& b)
{
j.m_type = value_t::binary;
typename BasicJsonType::internal_binary_t value{b};
j.m_value = value;
j.assert_invariant();
}
template<typename BasicJsonType>
static void construct(BasicJsonType& j, typename BasicJsonType::binary_t&& b)
{
j.m_type = value_t::binary;
typename BasicJsonType::internal_binary_t value{std::move(b)};
j.m_value = value;
j.assert_invariant();
}
};
template<>
struct external_constructor<value_t::number_float> struct external_constructor<value_t::number_float>
{ {
template<typename BasicJsonType> template<typename BasicJsonType>
......
include/nlohmann/detail/input/json_sax.hpp
...@@ -31,6 +31,7 @@ struct json_sax ...@@ -31,6 +31,7 @@ struct json_sax
using number_float_t = typename BasicJsonType::number_float_t; using number_float_t = typename BasicJsonType::number_float_t;
/// type for strings /// type for strings
using string_t = typename BasicJsonType::string_t; using string_t = typename BasicJsonType::string_t;
using binary_t = typename BasicJsonType::binary_t;
/*! /*!
@brief a null value was read @brief a null value was read
...@@ -76,6 +77,14 @@ struct json_sax ...@@ -76,6 +77,14 @@ struct json_sax
virtual bool string(string_t& val) = 0; virtual bool string(string_t& val) = 0;
/*! /*!
@brief a binary string was read
@param[in] val binary value
@return whether parsing should proceed
@note It is safe to move the passed binary.
*/
virtual bool binary(binary_t& val) = 0;
/*!
@brief the beginning of an object was read @brief the beginning of an object was read
@param[in] elements number of object elements or -1 if unknown @param[in] elements number of object elements or -1 if unknown
@return whether parsing should proceed @return whether parsing should proceed
...@@ -149,6 +158,7 @@ class json_sax_dom_parser ...@@ -149,6 +158,7 @@ class json_sax_dom_parser
using number_unsigned_t = typename BasicJsonType::number_unsigned_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
using number_float_t = typename BasicJsonType::number_float_t; using number_float_t = typename BasicJsonType::number_float_t;
using string_t = typename BasicJsonType::string_t; using string_t = typename BasicJsonType::string_t;
using binary_t = typename BasicJsonType::binary_t;
/*! /*!
@param[in, out] r reference to a JSON value that is manipulated while @param[in, out] r reference to a JSON value that is manipulated while
...@@ -202,6 +212,12 @@ class json_sax_dom_parser ...@@ -202,6 +212,12 @@ class json_sax_dom_parser
return true; return true;
} }
bool binary(binary_t& val)
{
handle_binary(val);
return true;
}
bool start_object(std::size_t len) bool start_object(std::size_t len)
{ {
ref_stack.push_back(handle_value(BasicJsonType::value_t::object)); ref_stack.push_back(handle_value(BasicJsonType::value_t::object));
...@@ -311,6 +327,36 @@ class json_sax_dom_parser ...@@ -311,6 +327,36 @@ class json_sax_dom_parser
return object_element; return object_element;
} }
/*!
@invariant If the ref stack is empty, then the passed value will be the new
root.
@invariant If the ref stack contains a value, then it is an array or an
object to which we can add elements
*/
template<typename BinaryValue>
JSON_HEDLEY_RETURNS_NON_NULL
BasicJsonType* handle_binary(BinaryValue&& v)
{
if (ref_stack.empty())
{
root = BasicJsonType::binary_array(std::forward<BinaryValue>(v));
return &root;
}
assert(ref_stack.back()->is_array() or ref_stack.back()->is_object());
if (ref_stack.back()->is_array())
{
ref_stack.back()->m_value.array->emplace_back(BasicJsonType::binary_array(std::forward<BinaryValue>(v)));
return &(ref_stack.back()->m_value.array->back());
}
assert(ref_stack.back()->is_object());
assert(object_element);
*object_element = BasicJsonType::binary_array(std::forward<BinaryValue>(v));
return object_element;
}
/// the parsed JSON value /// the parsed JSON value
BasicJsonType& root; BasicJsonType& root;
/// stack to model hierarchy of values /// stack to model hierarchy of values
...@@ -331,6 +377,7 @@ class json_sax_dom_callback_parser ...@@ -331,6 +377,7 @@ class json_sax_dom_callback_parser
using number_unsigned_t = typename BasicJsonType::number_unsigned_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
using number_float_t = typename BasicJsonType::number_float_t; using number_float_t = typename BasicJsonType::number_float_t;
using string_t = typename BasicJsonType::string_t; using string_t = typename BasicJsonType::string_t;
using binary_t = typename BasicJsonType::binary_t;
using parser_callback_t = typename BasicJsonType::parser_callback_t; using parser_callback_t = typename BasicJsonType::parser_callback_t;
using parse_event_t = typename BasicJsonType::parse_event_t; using parse_event_t = typename BasicJsonType::parse_event_t;
...@@ -385,6 +432,12 @@ class json_sax_dom_callback_parser ...@@ -385,6 +432,12 @@ class json_sax_dom_callback_parser
return true; return true;
} }
bool binary(binary_t& val)
{
handle_value(val);
return true;
}
bool start_object(std::size_t len) bool start_object(std::size_t len)
{ {
// check callback for object start // check callback for object start
...@@ -635,6 +688,7 @@ class json_sax_acceptor ...@@ -635,6 +688,7 @@ class json_sax_acceptor
using number_unsigned_t = typename BasicJsonType::number_unsigned_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
using number_float_t = typename BasicJsonType::number_float_t; using number_float_t = typename BasicJsonType::number_float_t;
using string_t = typename BasicJsonType::string_t; using string_t = typename BasicJsonType::string_t;
using binary_t = typename BasicJsonType::binary_t;
bool null() bool null()
{ {
...@@ -666,7 +720,12 @@ class json_sax_acceptor ...@@ -666,7 +720,12 @@ class json_sax_acceptor
return true; return true;
} }
bool start_object(std::size_t /*unused*/ = std::size_t(-1)) bool binary(binary_t& /*unused*/)
{
return true;
}
bool start_object(std::size_t /*unused*/ = std::size_t(-1))
{ {
return true; return true;
} }
...@@ -681,7 +740,7 @@ class json_sax_acceptor ...@@ -681,7 +740,7 @@ class json_sax_acceptor
return true; return true;
} }
bool start_array(std::size_t /*unused*/ = std::size_t(-1)) bool start_array(std::size_t /*unused*/ = std::size_t(-1))
{ {
return true; return true;
} }
......
include/nlohmann/detail/iterators/internal_iterator.hpp
...@@ -18,6 +18,8 @@ template<typename BasicJsonType> struct internal_iterator ...@@ -18,6 +18,8 @@ template<typename BasicJsonType> struct internal_iterator
typename BasicJsonType::object_t::iterator object_iterator {}; typename BasicJsonType::object_t::iterator object_iterator {};
/// iterator for JSON arrays /// iterator for JSON arrays
typename BasicJsonType::array_t::iterator array_iterator {}; typename BasicJsonType::array_t::iterator array_iterator {};
/// iterator for JSON binary arrays
typename BasicJsonType::binary_t::iterator binary_iterator {};
/// generic iterator for all other types /// generic iterator for all other types
primitive_iterator_t primitive_iterator {}; primitive_iterator_t primitive_iterator {};
}; };
......
include/nlohmann/detail/macro_scope.hpp
...@@ -113,9 +113,10 @@ ...@@ -113,9 +113,10 @@
class StringType, class BooleanType, class NumberIntegerType, \ class StringType, class BooleanType, class NumberIntegerType, \
class NumberUnsignedType, class NumberFloatType, \ class NumberUnsignedType, class NumberFloatType, \
template<typename> class AllocatorType, \ template<typename> class AllocatorType, \
template<typename, typename = void> class JSONSerializer> template<typename, typename = void> class JSONSerializer, \
class BinaryType>
#define NLOHMANN_BASIC_JSON_TPL \ #define NLOHMANN_BASIC_JSON_TPL \
basic_json<ObjectType, ArrayType, StringType, BooleanType, \ basic_json<ObjectType, ArrayType, StringType, BooleanType, \
NumberIntegerType, NumberUnsignedType, NumberFloatType, \ NumberIntegerType, NumberUnsignedType, NumberFloatType, \
AllocatorType, JSONSerializer> AllocatorType, JSONSerializer, BinaryType>
include/nlohmann/detail/output/binary_writer.hpp
...@@ -26,6 +26,7 @@ template<typename BasicJsonType, typename CharType> ...@@ -26,6 +26,7 @@ template<typename BasicJsonType, typename CharType>
class binary_writer class binary_writer
{ {
using string_t = typename BasicJsonType::string_t; using string_t = typename BasicJsonType::string_t;
using internal_binary_t = typename BasicJsonType::internal_binary_t;
public: public:
/*! /*!
...@@ -258,6 +259,45 @@ class binary_writer ...@@ -258,6 +259,45 @@ class binary_writer
break; break;
} }
case value_t::binary:
{
// step 1: write control byte and the binary array size
const auto N = j.m_value.binary->size();
if (N <= 0x17)
{
write_number(static_cast<std::uint8_t>(0x40 + N));
}
else if (N <= (std::numeric_limits<std::uint8_t>::max)())
{
oa->write_character(to_char_type(0x58));
write_number(static_cast<std::uint8_t>(N));
}
else if (N <= (std::numeric_limits<std::uint16_t>::max)())
{
oa->write_character(to_char_type(0x59));
write_number(static_cast<std::uint16_t>(N));
}
else if (N <= (std::numeric_limits<std::uint32_t>::max)())
{
oa->write_character(to_char_type(0x5A));
write_number(static_cast<std::uint32_t>(N));
}
// LCOV_EXCL_START
else if (N <= (std::numeric_limits<std::uint64_t>::max)())
{
oa->write_character(to_char_type(0x5B));
write_number(static_cast<std::uint64_t>(N));
}
// LCOV_EXCL_STOP
// step 2: write each element
oa->write_characters(
reinterpret_cast<const CharType*>(j.m_value.binary->data()),
N);
break;
}
case value_t::object: case value_t::object:
{ {
// step 1: write control byte and the object size // step 1: write control byte and the object size
...@@ -506,6 +546,101 @@ class binary_writer ...@@ -506,6 +546,101 @@ class binary_writer
break; break;
} }
case value_t::binary:
{
// step 0: determine if the binary type has a set subtype to
// determine whether or not to use the ext or fixext types
const bool use_ext = j.m_value.binary->has_subtype;
// step 1: write control byte and the byte string length
const auto N = j.m_value.binary->size();
if (N <= (std::numeric_limits<std::uint8_t>::max)())
{
std::uint8_t output_type;
bool fixed = true;
if (use_ext)
{
switch (N)
{
case 1:
output_type = 0xD4; // fixext 1
break;
case 2:
output_type = 0xD5; // fixext 2
break;
case 4:
output_type = 0xD6; // fixext 4
break;
case 8:
output_type = 0xD7; // fixext 8
break;
case 16:
output_type = 0xD8; // fixext 16
break;
default:
output_type = 0xC7; // ext 8
fixed = false;
break;
}
}
else
{
output_type = 0xC4; // bin 8
fixed = false;
}
oa->write_character(to_char_type(output_type));
if (not fixed)
{
write_number(static_cast<std::uint8_t>(N));
}
}
else if (N <= (std::numeric_limits<std::uint16_t>::max)())
{
std::uint8_t output_type;
if (use_ext)
{
output_type = 0xC8; // ext 16
}
else
{
output_type = 0xC5; // bin 16
}
oa->write_character(to_char_type(output_type));
write_number(static_cast<std::uint16_t>(N));
}
else if (N <= (std::numeric_limits<std::uint32_t>::max)())
{
std::uint8_t output_type;
if (use_ext)
{
output_type = 0xC9; // ext 32
}
else
{
output_type = 0xC6; // bin 32
}
oa->write_character(to_char_type(output_type));
write_number(static_cast<std::uint32_t>(N));
}
// step 1.5: if this is an ext type, write the subtype
if (use_ext)
{
write_number(j.m_value.binary->subtype);
}
// step 2: write the byte string
oa->write_characters(
reinterpret_cast<const CharType*>(j.m_value.binary->data()),
N);
break;
}
case value_t::object: case value_t::object:
{ {
// step 1: write control byte and the object size // step 1: write control byte and the object size
...@@ -649,6 +784,49 @@ class binary_writer ...@@ -649,6 +784,49 @@ class binary_writer
break; break;
} }
case value_t::binary:
{
if (add_prefix)
{
oa->write_character(to_char_type('['));
}
if (use_type and not j.m_value.binary->empty())
{
assert(use_count);
oa->write_character(to_char_type('$'));
oa->write_character('U');
}
if (use_count)
{
oa->write_character(to_char_type('#'));
write_number_with_ubjson_prefix(j.m_value.binary->size(), true);
}
if (use_type)
{
oa->write_characters(
reinterpret_cast<const CharType*>(j.m_value.binary->data()),
j.m_value.binary->size());
}
else
{
for (size_t i = 0; i < j.m_value.binary->size(); ++i)
{
oa->write_character(to_char_type('U'));
oa->write_character(j.m_value.binary->data()[i]);
}
}
if (not use_count)
{
oa->write_character(to_char_type(']'));
}
break;
}
case value_t::object: case value_t::object:
{ {
if (add_prefix) if (add_prefix)
...@@ -872,6 +1050,14 @@ class binary_writer ...@@ -872,6 +1050,14 @@ class binary_writer
} }
/*! /*!
@return The size of the BSON-encoded binary array @a value
*/
static std::size_t calc_bson_binary_size(const typename BasicJsonType::internal_binary_t& value)
{
return sizeof(std::int32_t) + value.size() + 1ul;
}
/*!
@brief Writes a BSON element with key @a name and array @a value @brief Writes a BSON element with key @a name and array @a value
*/ */
void write_bson_array(const string_t& name, void write_bson_array(const string_t& name,
...@@ -891,6 +1077,27 @@ class binary_writer ...@@ -891,6 +1077,27 @@ class binary_writer
} }
/*! /*!
@brief Writes a BSON element with key @a name and binary value @a value
*/
void write_bson_binary(const string_t& name,
const internal_binary_t& value)
{
write_bson_entry_header(name, 0x05);
write_number<std::int32_t, true>(static_cast<std::int32_t>(value.size()));
std::uint8_t subtype = 0x00; // Generic Binary Subtype
if (value.has_subtype)
{
subtype = value.subtype;
}
write_number(subtype);
oa->write_characters(
reinterpret_cast<const CharType*>(value.data()),
value.size());
}
/*!
@brief Calculates the size necessary to serialize the JSON value @a j with its @a name @brief Calculates the size necessary to serialize the JSON value @a j with its @a name
@return The calculated size for the BSON document entry for @a j with the given @a name. @return The calculated size for the BSON document entry for @a j with the given @a name.
*/ */
...@@ -906,6 +1113,9 @@ class binary_writer ...@@ -906,6 +1113,9 @@ class binary_writer
case value_t::array: case value_t::array:
return header_size + calc_bson_array_size(*j.m_value.array); return header_size + calc_bson_array_size(*j.m_value.array);
case value_t::binary:
return header_size + calc_bson_binary_size(*j.m_value.binary);
case value_t::boolean: case value_t::boolean:
return header_size + 1ul; return header_size + 1ul;
...@@ -950,6 +1160,9 @@ class binary_writer ...@@ -950,6 +1160,9 @@ class binary_writer
case value_t::array: case value_t::array:
return write_bson_array(name, *j.m_value.array); return write_bson_array(name, *j.m_value.array);
case value_t::binary:
return write_bson_binary(name, *j.m_value.binary);
case value_t::boolean: case value_t::boolean:
return write_bson_boolean(name, j.m_value.boolean); return write_bson_boolean(name, j.m_value.boolean);
...@@ -1230,7 +1443,8 @@ class binary_writer ...@@ -1230,7 +1443,8 @@ class binary_writer
case value_t::string: case value_t::string:
return 'S'; return 'S';
case value_t::array: case value_t::array: // fallthrough
case value_t::binary:
return '['; return '[';
case value_t::object: case value_t::object:
......
include/nlohmann/detail/output/serializer.hpp
...@@ -45,6 +45,7 @@ class serializer ...@@ -45,6 +45,7 @@ class serializer
using number_float_t = typename BasicJsonType::number_float_t; using number_float_t = typename BasicJsonType::number_float_t;
using number_integer_t = typename BasicJsonType::number_integer_t; using number_integer_t = typename BasicJsonType::number_integer_t;
using number_unsigned_t = typename BasicJsonType::number_unsigned_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
using binary_t = typename BasicJsonType::binary_t;
static constexpr std::uint8_t UTF8_ACCEPT = 0; static constexpr std::uint8_t UTF8_ACCEPT = 0;
static constexpr std::uint8_t UTF8_REJECT = 1; static constexpr std::uint8_t UTF8_REJECT = 1;
...@@ -83,16 +84,19 @@ class serializer ...@@ -83,16 +84,19 @@ class serializer
- strings and object keys are escaped using `escape_string()` - strings and object keys are escaped using `escape_string()`
- integer numbers are converted implicitly via `operator<<` - integer numbers are converted implicitly via `operator<<`
- floating-point numbers are converted to a string using `"%g"` format - floating-point numbers are converted to a string using `"%g"` format
- if specified to, binary values are output using the syntax `b[]`, otherwise an exception is thrown
@param[in] val value to serialize @param[in] val value to serialize
@param[in] pretty_print whether the output shall be pretty-printed @param[in] pretty_print whether the output shall be pretty-printed
@param[in] indent_step the indent level @param[in] indent_step the indent level
@param[in] current_indent the current indent level (only used internally) @param[in] current_indent the current indent level (only used internally)
@param[in] serialize_binary whether the output shall include non-standard binary output
*/ */
void dump(const BasicJsonType& val, const bool pretty_print, void dump(const BasicJsonType& val, const bool pretty_print,
const bool ensure_ascii, const bool ensure_ascii,
const unsigned int indent_step, const unsigned int indent_step,
const unsigned int current_indent = 0) const unsigned int current_indent = 0,
const bool serialize_binary = false)
{ {
switch (val.m_type) switch (val.m_type)
{ {
...@@ -236,6 +240,55 @@ class serializer ...@@ -236,6 +240,55 @@ class serializer
return; return;
} }
case value_t::binary:
{
if (not serialize_binary)
{
JSON_THROW(type_error::create(317, "cannot serialize binary data to text JSON"));
}
if (val.m_value.binary->empty())
{
o->write_characters("b[]", 3);
}
else if (pretty_print)
{
o->write_characters("b[", 2);
for (auto i = val.m_value.binary->cbegin();
i != val.m_value.binary->cend() - 1; ++i)
{
dump_integer(*i);
o->write_character(',');
int index = i - val.m_value.binary->cbegin();
if (index % 16 == 0)
{
o->write_character('\n');
}
else
{
o->write_character(' ');
}
}
dump_integer(val.m_value.binary->back());
o->write_character(']');
}
else
{
o->write_characters("b[", 2);
for (auto i = val.m_value.binary->cbegin();
i != val.m_value.binary->cend() - 1; ++i)
{
dump_integer(*i);
o->write_character(',');
}
dump_integer(val.m_value.binary->back());
o->write_character(']');
}
return;
}
case value_t::boolean: case value_t::boolean:
{ {
if (val.m_value.boolean) if (val.m_value.boolean)
...@@ -592,7 +645,8 @@ class serializer ...@@ -592,7 +645,8 @@ class serializer
*/ */
template<typename NumberType, detail::enable_if_t< template<typename NumberType, detail::enable_if_t<
std::is_same<NumberType, number_unsigned_t>::value or std::is_same<NumberType, number_unsigned_t>::value or
std::is_same<NumberType, number_integer_t>::value, std::is_same<NumberType, number_integer_t>::value or
std::is_same<NumberType, typename binary_t::value_type>::value,
int> = 0> int> = 0>
void dump_integer(NumberType x) void dump_integer(NumberType x)
{ {
...@@ -630,7 +684,7 @@ class serializer ...@@ -630,7 +684,7 @@ class serializer
if (is_negative) if (is_negative)
{ {
*buffer_ptr = '-'; *buffer_ptr = '-';
abs_value = remove_sign(x); abs_value = remove_sign(static_cast<number_integer_t>(x));
// account one more byte for the minus sign // account one more byte for the minus sign
n_chars = 1 + count_digits(abs_value); n_chars = 1 + count_digits(abs_value);
......
include/nlohmann/detail/value_t.hpp
...@@ -48,6 +48,7 @@ enum class value_t : std::uint8_t ...@@ -48,6 +48,7 @@ enum class value_t : std::uint8_t
number_integer, ///< number value (signed integer) number_integer, ///< number value (signed integer)
number_unsigned, ///< number value (unsigned integer) number_unsigned, ///< number value (unsigned integer)
number_float, ///< number value (floating-point) number_float, ///< number value (floating-point)
binary, ///< binary array (ordered collection of bytes)
discarded ///< discarded by the parser callback function discarded ///< discarded by the parser callback function
}; };
...@@ -55,17 +56,21 @@ enum class value_t : std::uint8_t ...@@ -55,17 +56,21 @@ enum class value_t : std::uint8_t
@brief comparison operator for JSON types @brief comparison operator for JSON types
Returns an ordering that is similar to Python: Returns an ordering that is similar to Python:
- order: null < boolean < number < object < array < string - order: null < boolean < number < object < array < string < binary
- furthermore, each type is not smaller than itself - furthermore, each type is not smaller than itself
- discarded values are not comparable - discarded values are not comparable
- binary is represented as a b"" string in python and directly comparable to a
string; however, making a binary array directly comparable with a string would
be surprising behavior in a JSON file.
@since version 1.0.0 @since version 1.0.0
*/ */
inline bool operator<(const value_t lhs, const value_t rhs) noexcept inline bool operator<(const value_t lhs, const value_t rhs) noexcept
{ {
static constexpr std::array<std::uint8_t, 8> order = {{ static constexpr std::array<std::uint8_t, 9> order = {{
0 /* null */, 3 /* object */, 4 /* array */, 5 /* string */, 0 /* null */, 3 /* object */, 4 /* array */, 5 /* string */,
1 /* boolean */, 2 /* integer */, 2 /* unsigned */, 2 /* float */ 1 /* boolean */, 2 /* integer */, 2 /* unsigned */, 2 /* float */,
6 /* binary */
} }
}; };
......
include/nlohmann/json_fwd.hpp
...@@ -33,7 +33,8 @@ template<template<typename U, typename V, typename... Args> class ObjectType = ...@@ -33,7 +33,8 @@ template<template<typename U, typename V, typename... Args> class ObjectType =
class NumberFloatType = double, class NumberFloatType = double,
template<typename U> class AllocatorType = std::allocator, template<typename U> class AllocatorType = std::allocator,
template<typename T, typename SFINAE = void> class JSONSerializer = template<typename T, typename SFINAE = void> class JSONSerializer =
adl_serializer> adl_serializer,
class BinaryType = std::vector<std::uint8_t>>
class basic_json; class basic_json;
/*! /*!
......
test/src/unit-bson.cpp
...@@ -492,6 +492,36 @@ TEST_CASE("BSON") ...@@ -492,6 +492,36 @@ TEST_CASE("BSON")
CHECK(json::from_bson(result, true, false) == j); CHECK(json::from_bson(result, true, false) == j);
} }
SECTION("non-empty object with binary member")
{
const size_t N = 10;
const auto s = std::vector<uint8_t>(N, 'x');
json j =
{
{ "entry", json::binary_array(s) }
};
std::vector<uint8_t> expected =
{
0x1B, 0x00, 0x00, 0x00, // size (little endian)
0x05, // entry: binary
'e', 'n', 't', 'r', 'y', '\x00',
0x0A, 0x00, 0x00, 0x00, // size of binary (little endian)
0x00, // Generic binary subtype
0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78,
0x00 // end marker
};
const auto result = json::to_bson(j);
CHECK(result == expected);
// roundtrip
CHECK(json::from_bson(result) == j);
CHECK(json::from_bson(result, true, false) == j);
}
SECTION("Some more complex document") SECTION("Some more complex document")
{ {
// directly encoding uint64 is not supported in bson (only for timestamp values) // directly encoding uint64 is not supported in bson (only for timestamp values)
...@@ -646,6 +676,11 @@ class SaxCountdown ...@@ -646,6 +676,11 @@ class SaxCountdown
return events_left-- > 0; return events_left-- > 0;
} }
bool binary(std::vector<uint8_t>&)
{
return events_left-- > 0;
}
bool start_object(std::size_t) bool start_object(std::size_t)
{ {
return events_left-- > 0; return events_left-- > 0;
......
test/src/unit-cbor.cpp
...@@ -36,6 +36,7 @@ using nlohmann::json; ...@@ -36,6 +36,7 @@ using nlohmann::json;
#include <fstream> #include <fstream>
#include <sstream> #include <sstream>
#include <iomanip> #include <iomanip>
#include <iostream>
#include <set> #include <set>
namespace namespace
...@@ -76,6 +77,11 @@ class SaxCountdown ...@@ -76,6 +77,11 @@ class SaxCountdown
return events_left-- > 0; return events_left-- > 0;
} }
bool binary(std::vector<std::uint8_t>&)
{
return events_left-- > 0;
}
bool start_object(std::size_t) bool start_object(std::size_t)
{ {
return events_left-- > 0; return events_left-- > 0;
...@@ -1285,10 +1291,156 @@ TEST_CASE("CBOR") ...@@ -1285,10 +1291,156 @@ TEST_CASE("CBOR")
CHECK(json::from_cbor(result, true, false) == j); CHECK(json::from_cbor(result, true, false) == j);
} }
} }
SECTION("binary")
{
SECTION("N = 0..23")
{
for (size_t N = 0; N <= 0x17; ++N)
{
CAPTURE(N)
// create JSON value with byte array containing of N * 'x'
const auto s = std::vector<uint8_t>(N, 'x');
json j = json::binary_array(s);
// create expected byte vector
std::vector<uint8_t> expected;
expected.push_back(static_cast<uint8_t>(0x40 + N));
for (size_t i = 0; i < N; ++i)
{
expected.push_back(0x78);
}
// compare result + size
const auto result = json::to_cbor(j);
CHECK(result == expected);
CHECK(result.size() == N + 1);
// check that no null byte is appended
if (N > 0)
{
CHECK(result.back() != '\x00');
}
// roundtrip
CHECK(json::from_cbor(result) == j);
CHECK(json::from_cbor(result, true, false) == j);
}
}
SECTION("N = 24..255")
{
for (size_t N = 24; N <= 255; ++N)
{
CAPTURE(N)
// create JSON value with string containing of N * 'x'
const auto s = std::vector<uint8_t>(N, 'x');
json j = json::binary_array(s);
// create expected byte vector
std::vector<uint8_t> expected;
expected.push_back(0x58);
expected.push_back(static_cast<uint8_t>(N));
for (size_t i = 0; i < N; ++i)
{
expected.push_back('x');
}
// compare result + size
const auto result = json::to_cbor(j);
CHECK(result == expected);
CHECK(result.size() == N + 2);
// check that no null byte is appended
CHECK(result.back() != '\x00');
// roundtrip
CHECK(json::from_cbor(result) == j);
CHECK(json::from_cbor(result, true, false) == j);
}
}
SECTION("N = 256..65535")
{
for (size_t N :
{
256u, 999u, 1025u, 3333u, 2048u, 65535u
})
{
CAPTURE(N)
// create JSON value with string containing of N * 'x'
const auto s = std::vector<uint8_t>(N, 'x');
json j = json::binary_array(s);
// create expected byte vector (hack: create string first)
std::vector<uint8_t> expected(N, 'x');
// reverse order of commands, because we insert at begin()
expected.insert(expected.begin(), static_cast<uint8_t>(N & 0xff));
expected.insert(expected.begin(), static_cast<uint8_t>((N >> 8) & 0xff));
expected.insert(expected.begin(), 0x59);
// compare result + size
const auto result = json::to_cbor(j);
CHECK(result == expected);
CHECK(result.size() == N + 3);
// check that no null byte is appended
CHECK(result.back() != '\x00');
// roundtrip
CHECK(json::from_cbor(result) == j);
CHECK(json::from_cbor(result, true, false) == j);
}
}
SECTION("N = 65536..4294967295")
{
for (size_t N :
{
65536u, 77777u, 1048576u
})
{
CAPTURE(N)
// create JSON value with string containing of N * 'x'
const auto s = std::vector<uint8_t>(N, 'x');
json j = json::binary_array(s);
// create expected byte vector (hack: create string first)
std::vector<uint8_t> expected(N, 'x');
// reverse order of commands, because we insert at begin()
expected.insert(expected.begin(), static_cast<uint8_t>(N & 0xff));
expected.insert(expected.begin(), static_cast<uint8_t>((N >> 8) & 0xff));
expected.insert(expected.begin(), static_cast<uint8_t>((N >> 16) & 0xff));
expected.insert(expected.begin(), static_cast<uint8_t>((N >> 24) & 0xff));
expected.insert(expected.begin(), 0x5a);
// compare result + size
const auto result = json::to_cbor(j);
CHECK(result == expected);
CHECK(result.size() == N + 5);
// check that no null byte is appended
CHECK(result.back() != '\x00');
// roundtrip
CHECK(json::from_cbor(result) == j);
CHECK(json::from_cbor(result, true, false) == j);
}
}
}
} }
SECTION("additional deserialization") SECTION("additional deserialization")
{ {
SECTION("0x5b (byte array)")
{
std::vector<uint8_t> given = {0x5b, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x01, 0x61
};
json j = json::from_cbor(given);
CHECK(j == json::binary_array(std::vector<uint8_t> {'a'}));
}
SECTION("0x7b (string)") SECTION("0x7b (string)")
{ {
std::vector<uint8_t> given = {0x7b, 0x00, 0x00, 0x00, 0x00, std::vector<uint8_t> given = {0x7b, 0x00, 0x00, 0x00, 0x00,
...@@ -1455,14 +1607,8 @@ TEST_CASE("CBOR") ...@@ -1455,14 +1607,8 @@ TEST_CASE("CBOR")
0x1c, 0x1d, 0x1e, 0x1f, 0x1c, 0x1d, 0x1e, 0x1f,
// ? // ?
0x3c, 0x3d, 0x3e, 0x3f, 0x3c, 0x3d, 0x3e, 0x3f,
// byte strings
0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57,
// byte strings
0x58, 0x59, 0x5a, 0x5b,
// ? // ?
0x5c, 0x5d, 0x5e, 0x5c, 0x5d, 0x5e,
// byte string
0x5f,
// ? // ?
0x7c, 0x7d, 0x7e, 0x7c, 0x7d, 0x7e,
// ? // ?
...@@ -1929,12 +2075,6 @@ TEST_CASE("all CBOR first bytes") ...@@ -1929,12 +2075,6 @@ TEST_CASE("all CBOR first bytes")
{ {
//// types not supported by this library //// types not supported by this library
// byte strings
0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,
0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f,
0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57,
// byte strings
0x58, 0x59, 0x5a, 0x5b, 0x5f,
// date/time // date/time
0xc0, 0xc1, 0xc0, 0xc1,
// bignum // bignum
...@@ -2144,6 +2284,20 @@ TEST_CASE("examples from RFC 7049 Appendix A") ...@@ -2144,6 +2284,20 @@ TEST_CASE("examples from RFC 7049 Appendix A")
CHECK(json::parse("\"streaming\"") == json::from_cbor(std::vector<uint8_t>({0x7f, 0x65, 0x73, 0x74, 0x72, 0x65, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x67, 0xff}))); CHECK(json::parse("\"streaming\"") == json::from_cbor(std::vector<uint8_t>({0x7f, 0x65, 0x73, 0x74, 0x72, 0x65, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x67, 0xff})));
} }
SECTION("byte arrays")
{
std::ifstream f_cbor("test/data/binary_data/cbor_binary.cbor", std::ios::binary);
std::vector<uint8_t> packed((std::istreambuf_iterator<char>(f_cbor)),
std::istreambuf_iterator<char>());
json j;
CHECK_NOTHROW(j = json::from_cbor(packed));
std::ifstream f_bin("test/data/binary_data/cbor_binary.out", std::ios::binary);
std::vector<uint8_t> expected((std::istreambuf_iterator<char>(f_bin)),
std::istreambuf_iterator<char>());
CHECK(j == json::binary_array(expected));
}
SECTION("arrays") SECTION("arrays")
{ {
CHECK(json::to_cbor(json::parse("[]")) == std::vector<uint8_t>({0x80})); CHECK(json::to_cbor(json::parse("[]")) == std::vector<uint8_t>({0x80}));
......
test/src/unit-class_parser.cpp
...@@ -77,6 +77,21 @@ class SaxEventLogger ...@@ -77,6 +77,21 @@ class SaxEventLogger
return true; return true;
} }
bool binary(std::vector<std::uint8_t>& val)
{
std::string binary_contents = "binary(";
std::string comma_space = "";
for (auto b : val)
{
binary_contents.append(comma_space);
binary_contents.append(std::to_string(static_cast<int>(b)));
comma_space = ", ";
}
binary_contents.append(")");
events.push_back(binary_contents);
return true;
}
bool start_object(std::size_t elements) bool start_object(std::size_t elements)
{ {
if (elements == std::size_t(-1)) if (elements == std::size_t(-1))
...@@ -168,6 +183,11 @@ class SaxCountdown : public nlohmann::json::json_sax_t ...@@ -168,6 +183,11 @@ class SaxCountdown : public nlohmann::json::json_sax_t
return events_left-- > 0; return events_left-- > 0;
} }
bool binary(std::vector<std::uint8_t>&) override
{
return events_left-- > 0;
}
bool start_object(std::size_t) override bool start_object(std::size_t) override
{ {
return events_left-- > 0; return events_left-- > 0;
......
test/src/unit-deserialization.cpp
...@@ -76,6 +76,21 @@ struct SaxEventLogger : public nlohmann::json_sax<json> ...@@ -76,6 +76,21 @@ struct SaxEventLogger : public nlohmann::json_sax<json>
return true; return true;
} }
bool binary(std::vector<std::uint8_t>& val)
{
std::string binary_contents = "binary(";
std::string comma_space = "";
for (auto b : val)
{
binary_contents.append(comma_space);
binary_contents.append(std::to_string(static_cast<int>(b)));
comma_space = ", ";
}
binary_contents.append(")");
events.push_back(binary_contents);
return true;
}
bool start_object(std::size_t elements) override bool start_object(std::size_t elements) override
{ {
if (elements == std::size_t(-1)) if (elements == std::size_t(-1))
......
test/src/unit-regression.cpp
...@@ -107,7 +107,7 @@ struct foo_serializer < T, typename std::enable_if < !std::is_same<foo, T>::valu ...@@ -107,7 +107,7 @@ struct foo_serializer < T, typename std::enable_if < !std::is_same<foo, T>::valu
} }
using foo_json = nlohmann::basic_json<std::map, std::vector, std::string, bool, std::int64_t, using foo_json = nlohmann::basic_json<std::map, std::vector, std::string, bool, std::int64_t,
std::uint64_t, double, std::allocator, ns::foo_serializer>; std::uint64_t, double, std::allocator, ns::foo_serializer, std::vector<std::uint8_t>>;
///////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////
// for #805 // for #805
......
test/src/unit-ubjson.cpp
...@@ -73,6 +73,11 @@ class SaxCountdown ...@@ -73,6 +73,11 @@ class SaxCountdown
return events_left-- > 0; return events_left-- > 0;
} }
bool binary(std::vector<std::uint8_t>&)
{
return events_left-- > 0;
}
bool start_object(std::size_t) bool start_object(std::size_t)
{ {
return events_left-- > 0; return events_left-- > 0;
...@@ -905,6 +910,231 @@ TEST_CASE("UBJSON") ...@@ -905,6 +910,231 @@ TEST_CASE("UBJSON")
} }
} }
SECTION("binary")
{
SECTION("N = 0..127")
{
for (std::size_t N = 0; N <= 127; ++N)
{
CAPTURE(N)
// create JSON value with byte array containing of N * 'x'
const auto s = std::vector<std::uint8_t>(N, 'x');
json j = json::binary_array(s);
// create expected byte vector
std::vector<std::uint8_t> expected;
expected.push_back(static_cast<std::uint8_t>('['));
if (N != 0)
{
expected.push_back(static_cast<std::uint8_t>('$'));
expected.push_back(static_cast<std::uint8_t>('U'));
}
expected.push_back(static_cast<std::uint8_t>('#'));
expected.push_back(static_cast<std::uint8_t>('i'));
expected.push_back(static_cast<std::uint8_t>(N));
for (size_t i = 0; i < N; ++i)
{
expected.push_back(0x78);
}
// compare result + size
const auto result = json::to_ubjson(j, true, true);
CHECK(result == expected);
if (N == 0)
{
CHECK(result.size() == N + 4);
}
else
{
CHECK(result.size() == N + 6);
}
// check that no null byte is appended
if (N > 0)
{
CHECK(result.back() != '\x00');
}
// roundtrip only works to an array of numbers
json j_out = s;
CHECK(json::from_ubjson(result) == j_out);
CHECK(json::from_ubjson(result, true, false) == j_out);
}
}
SECTION("N = 128..255")
{
for (std::size_t N = 128; N <= 255; ++N)
{
CAPTURE(N)
// create JSON value with byte array containing of N * 'x'
const auto s = std::vector<std::uint8_t>(N, 'x');
json j = json::binary_array(s);
// create expected byte vector
std::vector<uint8_t> expected;
expected.push_back(static_cast<std::uint8_t>('['));
expected.push_back(static_cast<std::uint8_t>('$'));
expected.push_back(static_cast<std::uint8_t>('U'));
expected.push_back(static_cast<std::uint8_t>('#'));
expected.push_back(static_cast<std::uint8_t>('U'));
expected.push_back(static_cast<std::uint8_t>(N));
for (size_t i = 0; i < N; ++i)
{
expected.push_back(0x78);
}
// compare result + size
const auto result = json::to_ubjson(j, true, true);
CHECK(result == expected);
CHECK(result.size() == N + 6);
// check that no null byte is appended
CHECK(result.back() != '\x00');
// roundtrip only works to an array of numbers
json j_out = s;
CHECK(json::from_ubjson(result) == j_out);
CHECK(json::from_ubjson(result, true, false) == j_out);
}
}
SECTION("N = 256..32767")
{
for (std::size_t N :
{
256u, 999u, 1025u, 3333u, 2048u, 32767u
})
{
CAPTURE(N)
// create JSON value with byte array containing of N * 'x'
const auto s = std::vector<std::uint8_t>(N, 'x');
json j = json::binary_array(s);
// create expected byte vector
std::vector<std::uint8_t> expected(N + 7, 'x');
expected[0] = '[';
expected[1] = '$';
expected[2] = 'U';
expected[3] = '#';
expected[4] = 'I';
expected[5] = static_cast<std::uint8_t>((N >> 8) & 0xFF);
expected[6] = static_cast<std::uint8_t>(N & 0xFF);
// compare result + size
const auto result = json::to_ubjson(j, true, true);
CHECK(result == expected);
CHECK(result.size() == N + 7);
// check that no null byte is appended
CHECK(result.back() != '\x00');
// roundtrip only works to an array of numbers
json j_out = s;
CHECK(json::from_ubjson(result) == j_out);
CHECK(json::from_ubjson(result, true, false) == j_out);
}
}
SECTION("N = 32768..2147483647")
{
for (std::size_t N :
{
32768u, 77777u, 1048576u
})
{
CAPTURE(N)
// create JSON value with byte array containing of N * 'x'
const auto s = std::vector<std::uint8_t>(N, 'x');
json j = json::binary_array(s);
// create expected byte vector
std::vector<std::uint8_t> expected(N + 9, 'x');
expected[0] = '[';
expected[1] = '$';
expected[2] = 'U';
expected[3] = '#';
expected[4] = 'l';
expected[5] = static_cast<std::uint8_t>((N >> 24) & 0xFF);
expected[6] = static_cast<std::uint8_t>((N >> 16) & 0xFF);
expected[7] = static_cast<std::uint8_t>((N >> 8) & 0xFF);
expected[8] = static_cast<std::uint8_t>(N & 0xFF);
// compare result + size
const auto result = json::to_ubjson(j, true, true);
CHECK(result == expected);
CHECK(result.size() == N + 9);
// check that no null byte is appended
CHECK(result.back() != '\x00');
// roundtrip only works to an array of numbers
json j_out = s;
CHECK(json::from_ubjson(result) == j_out);
CHECK(json::from_ubjson(result, true, false) == j_out);
}
}
SECTION("Other Serializations")
{
const std::size_t N = 10;
const auto s = std::vector<std::uint8_t>(N, 'x');
json j = json::binary_array(s);
SECTION("No Count No Type")
{
std::vector<uint8_t> expected;
expected.push_back(static_cast<std::uint8_t>('['));
for (std::size_t i = 0; i < N; ++i)
{
expected.push_back(static_cast<std::uint8_t>('U'));
expected.push_back(static_cast<std::uint8_t>(0x78));
}
expected.push_back(static_cast<std::uint8_t>(']'));
// compare result + size
const auto result = json::to_ubjson(j, false, false);
CHECK(result == expected);
CHECK(result.size() == N + 12);
// check that no null byte is appended
CHECK(result.back() != '\x00');
// roundtrip only works to an array of numbers
json j_out = s;
CHECK(json::from_ubjson(result) == j_out);
CHECK(json::from_ubjson(result, true, false) == j_out);
}
SECTION("Yes Count No Type")
{
std::vector<std::uint8_t> expected;
expected.push_back(static_cast<std::uint8_t>('['));
expected.push_back(static_cast<std::uint8_t>('#'));
expected.push_back(static_cast<std::uint8_t>('i'));
expected.push_back(static_cast<std::uint8_t>(N));
for (size_t i = 0; i < N; ++i)
{
expected.push_back(static_cast<std::uint8_t>('U'));
expected.push_back(static_cast<std::uint8_t>(0x78));
}
// compare result + size
const auto result = json::to_ubjson(j, true, false);
CHECK(result == expected);
CHECK(result.size() == N + 14);
// check that no null byte is appended
CHECK(result.back() != '\x00');
// roundtrip only works to an array of numbers
json j_out = s;
CHECK(json::from_ubjson(result) == j_out);
CHECK(json::from_ubjson(result, true, false) == j_out);
}
}
}
SECTION("array") SECTION("array")
{ {
SECTION("empty") SECTION("empty")
......
test/src/unit-udt.cpp
...@@ -653,8 +653,7 @@ TEST_CASE("custom serializer for pods" * doctest::test_suite("udt")) ...@@ -653,8 +653,7 @@ TEST_CASE("custom serializer for pods" * doctest::test_suite("udt"))
{ {
using custom_json = using custom_json =
nlohmann::basic_json<std::map, std::vector, std::string, bool, nlohmann::basic_json<std::map, std::vector, std::string, bool,
std::int64_t, std::uint64_t, double, std::allocator, std::int64_t, std::uint64_t, double, std::allocator, pod_serializer>;
pod_serializer>;
auto p = udt::small_pod{42, '/', 42}; auto p = udt::small_pod{42, '/', 42};
custom_json j = p; custom_json j = p;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment