Skip to content

J
json
Commit c816fd12 by Niels
Options

cleanup

parent d609790f
Showing with 96 additions and 84 deletions
src/json.hpp.re2c
......@@ -2415,6 +2415,41 @@ class basic_json
inline lexer() = default;
inline static std::string token_type_name(token_type t)
{
switch (t)
{
case (token_type::uninitialized):
return "<uninitialized>";
case (token_type::literal_true):
return "true literal";
case (token_type::literal_false):
return "false literal";
case (token_type::literal_null):
return "null literal";
case (token_type::value_string):
return "string literal";
case (token_type::value_number):
return "number literal";
case (token_type::begin_array):
return "[";
case (token_type::begin_object):
return "{";
case (token_type::end_array):
return "]";
case (token_type::end_object):
return "}";
case (token_type::name_separator):
return ":";
case (token_type::value_separator):
return ",";
case (token_type::parse_error):
return "<parse error>";
case (token_type::end_of_input):
return "<end of input>";
}
}
/*!
This function implements a scanner for JSON. It is specified using
regular expressions that try to follow RFC 7159 and ECMA-404 as close
......@@ -2429,6 +2464,10 @@ class basic_json
*/
inline token_type scan()
{
// pointer for backtracking information
const char* m_marker = nullptr;
// remember the begin of the token
m_start = m_cursor;
/*!re2c
......@@ -2483,6 +2522,9 @@ class basic_json
// end of file
'\000' { return token_type::end_of_input; }
// anything else is an error
* { return token_type::parse_error; }
*/
}
......@@ -2542,8 +2584,6 @@ class basic_json
const char* m_cursor = nullptr;
/// pointer to the end of the buffer
const char* m_limit = nullptr;
/// pointer for backtracking information
const char* m_marker = nullptr;
};
class parser
......@@ -2699,7 +2739,7 @@ class basic_json
std::string error_msg = "parse error - unexpected \'";
error_msg += m_lexer.get_string_value();
error_msg += "\' (";
error_msg += token_type_name(last_token) + ")";
error_msg += lexer::token_type_name(last_token) + ")";
throw std::invalid_argument(error_msg);
}
}
......@@ -2713,49 +2753,14 @@ class basic_json
return last_token;
}
inline static std::string token_type_name(typename lexer::token_type t)
{
switch (t)
{
case (lexer::token_type::uninitialized):
return "<uninitialized>";
case (lexer::token_type::literal_true):
return "true literal";
case (lexer::token_type::literal_false):
return "false literal";
case (lexer::token_type::literal_null):
return "null literal";
case (lexer::token_type::value_string):
return "string literal";
case (lexer::token_type::value_number):
return "number literal";
case (lexer::token_type::begin_array):
return "[";
case (lexer::token_type::begin_object):
return "{";
case (lexer::token_type::end_array):
return "]";
case (lexer::token_type::end_object):
return "}";
case (lexer::token_type::name_separator):
return ":";
case (lexer::token_type::value_separator):
return ",";
case (lexer::token_type::parse_error):
return "<parse error>";
case (lexer::token_type::end_of_input):
return "<end of input>";
}
}
inline void expect(typename lexer::token_type t) const
{
if (t != last_token)
{
std::string error_msg = "parse error - unexpected \'";
error_msg += m_lexer.get_string_value();
error_msg += "\' (" + token_type_name(last_token);
error_msg += "); expected " + token_type_name(t);
error_msg += "\' (" + lexer::token_type_name(last_token);
error_msg += "); expected " + lexer::token_type_name(t);
throw std::invalid_argument(error_msg);
}
}
......
test/unit.cpp
......@@ -3974,50 +3974,75 @@ TEST_CASE("convenience functions")
}
}
TEST_CASE("parser class")
TEST_CASE("lexer class")
{
SECTION("get_token")
SECTION("scan")
{
SECTION("structural characters")
{
CHECK(json::parser("[").last_token == json::lexer::token_type::begin_array);
CHECK(json::parser("]").last_token == json::lexer::token_type::end_array);
CHECK(json::parser("{").last_token == json::lexer::token_type::begin_object);
CHECK(json::parser("}").last_token == json::lexer::token_type::end_object);
CHECK(json::parser(",").last_token == json::lexer::token_type::value_separator);
CHECK(json::parser(":").last_token == json::lexer::token_type::name_separator);
CHECK(json::lexer("[").scan() == json::lexer::token_type::begin_array);
CHECK(json::lexer("]").scan() == json::lexer::token_type::end_array);
CHECK(json::lexer("{").scan() == json::lexer::token_type::begin_object);
CHECK(json::lexer("}").scan() == json::lexer::token_type::end_object);
CHECK(json::lexer(",").scan() == json::lexer::token_type::value_separator);
CHECK(json::lexer(":").scan() == json::lexer::token_type::name_separator);
}
SECTION("literal names")
{
CHECK(json::parser("null").last_token == json::lexer::token_type::literal_null);
CHECK(json::parser("true").last_token == json::lexer::token_type::literal_true);
CHECK(json::parser("false").last_token == json::lexer::token_type::literal_false);
CHECK(json::lexer("null").scan() == json::lexer::token_type::literal_null);
CHECK(json::lexer("true").scan() == json::lexer::token_type::literal_true);
CHECK(json::lexer("false").scan() == json::lexer::token_type::literal_false);
}
SECTION("numbers")
{
CHECK(json::parser("0").last_token == json::lexer::token_type::value_number);
CHECK(json::parser("1").last_token == json::lexer::token_type::value_number);
CHECK(json::parser("2").last_token == json::lexer::token_type::value_number);
CHECK(json::parser("3").last_token == json::lexer::token_type::value_number);
CHECK(json::parser("4").last_token == json::lexer::token_type::value_number);
CHECK(json::parser("5").last_token == json::lexer::token_type::value_number);
CHECK(json::parser("6").last_token == json::lexer::token_type::value_number);
CHECK(json::parser("7").last_token == json::lexer::token_type::value_number);
CHECK(json::parser("8").last_token == json::lexer::token_type::value_number);
CHECK(json::parser("9").last_token == json::lexer::token_type::value_number);
CHECK(json::lexer("0").scan() == json::lexer::token_type::value_number);
CHECK(json::lexer("1").scan() == json::lexer::token_type::value_number);
CHECK(json::lexer("2").scan() == json::lexer::token_type::value_number);
CHECK(json::lexer("3").scan() == json::lexer::token_type::value_number);
CHECK(json::lexer("4").scan() == json::lexer::token_type::value_number);
CHECK(json::lexer("5").scan() == json::lexer::token_type::value_number);
CHECK(json::lexer("6").scan() == json::lexer::token_type::value_number);
CHECK(json::lexer("7").scan() == json::lexer::token_type::value_number);
CHECK(json::lexer("8").scan() == json::lexer::token_type::value_number);
CHECK(json::lexer("9").scan() == json::lexer::token_type::value_number);
}
SECTION("whitespace")
{
CHECK(json::parser(" 0").last_token == json::lexer::token_type::value_number);
CHECK(json::parser("\t0").last_token == json::lexer::token_type::value_number);
CHECK(json::parser("\n0").last_token == json::lexer::token_type::value_number);
CHECK(json::parser("\r0").last_token == json::lexer::token_type::value_number);
CHECK(json::parser(" \t\n\r\n\t 0").last_token == json::lexer::token_type::value_number);
// result is end_of_input, because not token is following
CHECK(json::lexer(" ").scan() == json::lexer::token_type::end_of_input);
CHECK(json::lexer("\t").scan() == json::lexer::token_type::end_of_input);
CHECK(json::lexer("\n").scan() == json::lexer::token_type::end_of_input);
CHECK(json::lexer("\r").scan() == json::lexer::token_type::end_of_input);
CHECK(json::lexer(" \t\n\r\n\t ").scan() == json::lexer::token_type::end_of_input);
}
}
SECTION("token_type_name")
{
CHECK(json::lexer::token_type_name(json::lexer::token_type::uninitialized) == "<uninitialized>");
CHECK(json::lexer::token_type_name(json::lexer::token_type::literal_true) == "true literal");
CHECK(json::lexer::token_type_name(json::lexer::token_type::literal_false) == "false literal");
CHECK(json::lexer::token_type_name(json::lexer::token_type::literal_null) == "null literal");
CHECK(json::lexer::token_type_name(json::lexer::token_type::value_string) == "string literal");
CHECK(json::lexer::token_type_name(json::lexer::token_type::value_number) == "number literal");
CHECK(json::lexer::token_type_name(json::lexer::token_type::begin_array) == "[");
CHECK(json::lexer::token_type_name(json::lexer::token_type::begin_object) == "{");
CHECK(json::lexer::token_type_name(json::lexer::token_type::end_array) == "]");
CHECK(json::lexer::token_type_name(json::lexer::token_type::end_object) == "}");
CHECK(json::lexer::token_type_name(json::lexer::token_type::name_separator) == ":");
CHECK(json::lexer::token_type_name(json::lexer::token_type::value_separator) == ",");
CHECK(json::lexer::token_type_name(json::lexer::token_type::parse_error) == "<parse error>");
CHECK(json::lexer::token_type_name(json::lexer::token_type::end_of_input) == "<end of input>");
}
}
TEST_CASE("parser class")
{
SECTION("get_token")
{
/*
SECTION("parse errors on first character")
{
......@@ -4090,22 +4115,4 @@ TEST_CASE("parser class")
CHECK(json::parser("false").parse() == json(false));
}
}
SECTION("token_type_name")
{
CHECK(json::parser::token_type_name(json::lexer::token_type::uninitialized) == "<uninitialized>");
CHECK(json::parser::token_type_name(json::lexer::token_type::literal_true) == "true literal");
CHECK(json::parser::token_type_name(json::lexer::token_type::literal_false) == "false literal");
CHECK(json::parser::token_type_name(json::lexer::token_type::literal_null) == "null literal");
CHECK(json::parser::token_type_name(json::lexer::token_type::value_string) == "string literal");
CHECK(json::parser::token_type_name(json::lexer::token_type::value_number) == "number literal");
CHECK(json::parser::token_type_name(json::lexer::token_type::begin_array) == "[");
CHECK(json::parser::token_type_name(json::lexer::token_type::begin_object) == "{");
CHECK(json::parser::token_type_name(json::lexer::token_type::end_array) == "]");
CHECK(json::parser::token_type_name(json::lexer::token_type::end_object) == "}");
CHECK(json::parser::token_type_name(json::lexer::token_type::name_separator) == ":");
CHECK(json::parser::token_type_name(json::lexer::token_type::value_separator) == ",");
CHECK(json::parser::token_type_name(json::lexer::token_type::parse_error) == "<parse error>");
CHECK(json::parser::token_type_name(json::lexer::token_type::end_of_input) == "<end of input>");
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment