fixed BOM handling #344

src/json.hpp.re2c

@@ -7620,6 +7620,14 @@ class basic_json
         {
             // fill buffer
             fill_line_buffer();
+
+            // skip UTF-8 byte-order mark
+            if (m_line_buffer.size() >= 3 and m_line_buffer.substr(0, 3) == "\xEF\xBB\xBF")
+            {
+                m_line_buffer[0] = ' ';
+                m_line_buffer[1] = ' ';
+                m_line_buffer[2] = ' ';
+            }
         }
 
         // switch off unwanted functions (due to pointer members)
@@ -7802,10 +7810,6 @@ class basic_json
                     ws = [ \t\n\r]+;
                     ws   { continue; }
 
-                    // ignore byte-order-mark
-                    bom = "\xEF\xBB\xBF";
-                    bom   { continue; }
-
                     // structural characters
                     "[" { last_token_type = token_type::begin_array; break; }
                     "]" { last_token_type = token_type::end_array; break; }

test/src/unit-unicode.cpp

@@ -159,7 +159,7 @@ TEST_CASE("Unicode", "[hide]")
             }
         }
     }
-/*
+
     SECTION("ignore byte-order-mark")
     {
         // read a file with a UTF-8 BOM
@@ -167,7 +167,7 @@ TEST_CASE("Unicode", "[hide]")
         json j;
         CHECK_NOTHROW(j << f);
     }
-*/
+
     SECTION("error for incomplete/wrong BOM")
     {
         CHECK_THROWS_AS(json::parse("\xef\xbb"), std::invalid_argument);