✅ added more Unicode test cases

test/src/unit-unicode.cpp

@@ -74,8 +74,10 @@ void check_utf8string(bool success_expected, int byte1, int byte2 = -1, int byte
     }
 }
 
-TEST_CASE("RFC 3629", "[hide]")
+TEST_CASE("Unicode", "[hide]")
 {
+    SECTION("RFC 3629")
+    {
         /*
         RFC 3629 describes in Sect. 4 the syntax of UTF-8 byte sequences as
         follows:
@@ -850,20 +852,14 @@ TEST_CASE("RFC 3629", "[hide]")
                 }
             }
         }
-}
+    }
 
-TEST_CASE("Unicode", "[hide]")
-{
-    /* NOTE: to_unicode is not used any more
-    SECTION("full enumeration of Unicode code points")
+    SECTION("\\uxxxx sequences")
     {
-        // lexer to call to_unicode on
-        json::lexer dummy_lexer("", 0);
-
         // create an escaped string from a code point
         const auto codepoint_to_unicode = [](std::size_t cp)
         {
-            // copd points are represented as a six-character sequence: a
+            // code points are represented as a six-character sequence: a
             // reverse solidus, followed by the lowercase letter u, followed
             // by four hexadecimal digits that encode the character's code
             // point
@@ -872,11 +868,19 @@ TEST_CASE("Unicode", "[hide]")
             return ss.str();
         };
 
+        SECTION("correct sequences")
+        {
             // generate all UTF-8 code points; in total, 1112064 code points are
             // generated: 0x1FFFFF code points - 2048 invalid values between
             // 0xD800 and 0xDFFF.
             for (std::size_t cp = 0; cp <= 0x10FFFFu; ++cp)
             {
+                // string to store the code point as in \uxxxx format
+                std::string json_text = "\"";
+
+                // decide whether to use one or two \uxxxx sequences
+                if (cp < 0x10000u)
+                {
                     // The Unicode standard permanently reserves these code point
                     // values for UTF-16 encoding of the high and low surrogates, and
                     // they will never be assigned a character, so there should be no
@@ -889,26 +893,9 @@ TEST_CASE("Unicode", "[hide]")
                         continue;
                     }
 
-            // string to store the code point as in \uxxxx format
-            std::string escaped_string;
-            // string to store the code point as unescaped character sequence
-            std::string unescaped_string;
-
-            if (cp < 0x10000u)
-            {
                     // code points in the Basic Multilingual Plane can be
-                // represented with one \\uxxxx sequence
-                escaped_string = codepoint_to_unicode(cp);
-
-                // All Unicode characters may be placed within the quotation
-                // marks, except for the characters that must be escaped:
-                // quotation mark, reverse solidus, and the control characters
-                // (U+0000 through U+001F); we ignore these code points as
-                // they are checked with codepoint_to_unicode.
-                if (cp > 0x1f and cp != 0x22 and cp != 0x5c)
-                {
-                    unescaped_string = dummy_lexer.to_unicode(cp);
-                }
+                    // represented with one \uxxxx sequence
+                    json_text += codepoint_to_unicode(cp);
                 }
                 else
                 {
@@ -917,27 +904,65 @@ TEST_CASE("Unicode", "[hide]")
                     // 12-character sequence, encoding the UTF-16 surrogate pair
                     const auto codepoint1 = 0xd800u + (((cp - 0x10000u) >> 10) & 0x3ffu);
                     const auto codepoint2 = 0xdc00u + ((cp - 0x10000u) & 0x3ffu);
-                escaped_string = codepoint_to_unicode(codepoint1);
-                escaped_string += codepoint_to_unicode(codepoint2);
-                unescaped_string += dummy_lexer.to_unicode(codepoint1, codepoint2);
+                    json_text += codepoint_to_unicode(codepoint1) + codepoint_to_unicode(codepoint2);
                 }
 
-            // all other code points are valid and must not yield parse errors
-            CAPTURE(cp);
-            CAPTURE(escaped_string);
-            CAPTURE(unescaped_string);
+                json_text += "\"";
+                CAPTURE(json_text);
+                CHECK_NOTHROW(json::parse(json_text));
+            }
+        }
 
-            json j1, j2, j3, j4;
-            CHECK_NOTHROW(j1 = json::parse("\"" + escaped_string + "\""));
-            CHECK_NOTHROW(j2 = json::parse(j1.dump()));
-            CHECK(j1 == j2);
+        SECTION("incorrect sequences")
+        {
+            SECTION("high surrogate without low surrogate")
+            {
+                // D800..DBFF are high surrogates and must be followed by low
+                // surrogates DC00..DFFF; here, nothing follows
+                for (std::size_t cp = 0xD800u; cp <= 0xDBFFu; ++cp)
+                {
+                    std::string json_text = "\"" + codepoint_to_unicode(cp) + "\"";
+                    CAPTURE(json_text);
+                    CHECK_THROWS_AS(json::parse(json_text), json::parse_error);
+                }
+            }
+
+#if 0
+            SECTION("high surrogate with wrong low surrogate")
+            {
+                // D800..DBFF are high surrogates and must be followed by low
+                // surrogates DC00..DFFF; here a different sequence follows
+                for (std::size_t cp1 = 0xD800u; cp1 <= 0xDBFFu; ++cp1)
+                {
+                    for (std::size_t cp2 = 0x0000u; cp2 <= 0xFFFFu; ++cp2)
+                    {
+                        if (0xDC00u <= cp2 and cp2 <= 0xDFFFu)
+                        {
+                            continue;
+                        }
+
+                        std::string json_text = "\"" + codepoint_to_unicode(cp1) + codepoint_to_unicode(cp2) + "\"";
+                        CAPTURE(json_text);
+                        CHECK_THROWS_AS(json::parse(json_text), json::parse_error);
+                    }
+                }
+            }
+#endif
+
+            SECTION("low surrogate without high surrogate")
+            {
+                // low surrogates DC00..DFFF must follow high surrogates; here,
+                // they occur alone
+                for (std::size_t cp = 0xDC00u; cp <= 0xDFFFu; ++cp)
+                {
+                    std::string json_text = "\"" + codepoint_to_unicode(cp) + "\"";
+                    CAPTURE(json_text);
+                    CHECK_THROWS_AS(json::parse(json_text), json::parse_error);
+                }
+            }
 
-            CHECK_NOTHROW(j3 = json::parse("\"" + unescaped_string + "\""));
-            CHECK_NOTHROW(j4 = json::parse(j3.dump()));
-            CHECK(j3 == j4);
         }
     }
-     */
 
     SECTION("read all unicode characters")
     {