Merge pull request #1446 from scinart/develop

include/nlohmann/detail/output/serializer.hpp

@@ -454,6 +454,16 @@ class serializer
                                     string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xBF');
                                     string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xBD');
                                 }
+
+                                // write buffer and reset index; there must be 13 bytes
+                                // left, as this is the maximal number of bytes to be
+                                // written ("\uxxxx\uxxxx\0") for one code point
+                                if (string_buffer.size() - bytes < 13)
+                                {
+                                    o->write_characters(string_buffer.data(), bytes);
+                                    bytes = 0;
+                                }
+
                                 bytes_after_last_accept = bytes;
                             }
 

single_include/nlohmann/json.hpp

@@ -11344,6 +11344,16 @@ class serializer
                                     string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xBF');
                                     string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xBD');
                                 }
+
+                                // write buffer and reset index; there must be 13 bytes
+                                // left, as this is the maximal number of bytes to be
+                                // written ("\uxxxx\uxxxx\0") for one code point
+                                if (string_buffer.size() - bytes < 13)
+                                {
+                                    o->write_characters(string_buffer.data(), bytes);
+                                    bytes = 0;
+                                }
+
                                 bytes_after_last_accept = bytes;
                             }
 

test/src/unit-regression.cpp

@@ -1708,6 +1708,67 @@ TEST_CASE("regression tests")
         const auto data = j.get<decltype(expected)>();
         CHECK(expected == data);
     }
+
+    SECTION("issue #1445 - buffer overflow in dumping invalid utf-8 strings")
+    {
+        SECTION("a bunch of -1, ensure_ascii=true")
+        {
+            json dump_test;
+            std::vector<char> data(300, -1);
+            std::vector<std::string> vec_string(300, "\\ufffd");
+            std::string s{data.data(), data.size()};
+            dump_test["1"] = s;
+            std::ostringstream os;
+            os << "{\"1\":\"";
+            std::copy( vec_string.begin(), vec_string.end(), std::ostream_iterator<std::string>(os));
+            os << "\"}";
+            s = dump_test.dump(-1, ' ', true, nlohmann::json::error_handler_t::replace);
+            CHECK(s == os.str());
+        }
+        SECTION("a bunch of -2, ensure_ascii=false")
+        {
+            json dump_test;
+            std::vector<char> data(500, -2);
+            std::vector<std::string> vec_string(500, "\xEF\xBF\xBD");
+            std::string s{data.data(), data.size()};
+            dump_test["1"] = s;
+            std::ostringstream os;
+            os << "{\"1\":\"";
+            std::copy( vec_string.begin(), vec_string.end(), std::ostream_iterator<std::string>(os));
+            os << "\"}";
+            s = dump_test.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace);
+            CHECK(s == os.str());
+        }
+        SECTION("test case in issue #1445")
+        {
+            nlohmann::json dump_test;
+            const int data[] = {
+                109,  108,  103,  125,  -122, -53,  115,
+                18,   3,    0,    102,  19,   1,    15,
+                -110, 13,   -3,   -1,   -81,  32,   2,
+                0,    0,    0,    0,    0,    0,    0,
+                8,    0,    0,    0,    0,    0,    0,
+                0,    0,    0,    0,    0,    -80,  2,
+                0,    0,    96,   -118, 46,   -116, 46,
+                109,  -84,  -87,  108,  14,   109,  -24,
+                -83,  13,   -18,  -51,  -83,  -52,  -115,
+                14,   6,    32,   0,    0,    0,    0,
+                0,    0,    0,    0,    0,    0,    0,
+                64,   3,    0,    0,    0,    35,   -74,
+                -73,  55,   57,   -128, 0,    0,    0,
+                0,    0,    0,    0,    0,    0,    0,
+                0,    0,    33,   0,    0,    0,    -96,
+                -54,  -28,  -26
+            };
+            std::string s;
+            for (int i=0; i<sizeof(data)/sizeof(int); i++)
+            {
+                s += static_cast<char>(data[i]);
+            }
+            dump_test["1"] = s;
+            dump_test.dump(-1, ' ', true, nlohmann::json::error_handler_t::replace);
+        }
+    }
 }
 
 TEST_CASE("regression tests, exceptions dependent", "[!throws]")