From 1a3b12fef1f1caaf20b844e9ffabe55d92dc15a8 Mon Sep 17 00:00:00 2001 From: Daniel JB Clark Date: Wed, 30 Sep 2026 09:32:51 -0400 Subject: [PATCH] Wrote valid UTF-8 in JSON strings as-is instead of escaping each byte MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A \uXXXX escape denotes a code point (RFC 8259 section 7), so "café" was read as "café" by other parsers; bytes that are not valid UTF-8 are still escaped as before. Changelog: Title Signed-off-by: Daniel JB Clark Co-Authored-By: Claude Opus 5.5 --- libutils/json.c | 77 +++++++++++++++++++++++++++++++++++++++++- tests/unit/json_test.c | 40 ++++++++++++++++++++++ 2 files changed, 116 insertions(+), 1 deletion(-) diff --git a/libutils/json.c b/libutils/json.c index e46e0716..6a77af31 100644 --- a/libutils/json.c +++ b/libutils/json.c @@ -977,6 +977,68 @@ JsonElement *JsonObjectCreate(const size_t initialCapacity) JSON_CONTAINER_TYPE_OBJECT, NULL, initialCapacity); } +/** + @brief Length of the valid multi-byte UTF-8 sequence at @c str. + @return 2 to 4, or 0 if @c str does not start with one. Overlong forms, + UTF-16 surrogates and code points above U+10FFFF are rejected. + */ +static size_t Utf8SequenceLength(const char *const str) +{ + assert(str != NULL); + + const unsigned char *const s = (const unsigned char *) str; + unsigned char second_min = 0x80; + unsigned char second_max = 0xbf; + size_t length; + + if (s[0] >= 0xc2 && s[0] <= 0xdf) + { + length = 2; + } + else if (s[0] >= 0xe0 && s[0] <= 0xef) + { + length = 3; + if (s[0] == 0xe0) + { + second_min = 0xa0; // overlong + } + else if (s[0] == 0xed) + { + second_max = 0x9f; // U+D800..U+DFFF + } + } + else if (s[0] >= 0xf0 && s[0] <= 0xf4) + { + length = 4; + if (s[0] == 0xf0) + { + second_min = 0x90; // overlong + } + else if (s[0] == 0xf4) + { + second_max = 0x8f; // above U+10FFFF + } + } + else + { + return 0; + } + + // Byte by byte, so a NUL (never a continuation byte) stops the scan + if (s[1] < second_min || s[1] > second_max) + { + return 0; + } + for (size_t i = 2; i < length; i++) + { + if (s[i] < 0x80 || s[i] > 0xbf) + { + return 0; + } + } + return length; +} + void JsonEncodeStringWriter( const char *const unescaped_string, Writer *const writer) { @@ -1018,7 +1080,20 @@ void JsonEncodeStringWriter( } else { - WriterWriteF(writer, "\\u%04x", (unsigned char) *c); + const size_t length = Utf8SequenceLength(c); + if (length > 0) + { + WriterWriteLen(writer, c, length); + c += length - 1; + } + else + { + /* Not UTF-8, so there is no code point to write. + * Escaping the byte value keeps the output valid + * JSON, and JsonDecodeString() turns it back into + * the same byte. */ + WriterWriteF(writer, "\\u%04x", (unsigned char) *c); + } } } } diff --git a/tests/unit/json_test.c b/tests/unit/json_test.c index 200478a8..5e4311e3 100644 --- a/tests/unit/json_test.c +++ b/tests/unit/json_test.c @@ -1875,6 +1875,45 @@ static void test_string_escape(void) assert_json_strings_eq(unescaped_invalid_hex, escaped_invalid_hex); } +static void test_string_escape_utf8(void) +{ + // Valid UTF-8 is written as-is (RFC 8259 section 7) + assert_json_strings_eq("caf\xc3\xa9", "caf\xc3\xa9"); + assert_json_strings_eq("\xe4\xb8\xad", "\xe4\xb8\xad"); + assert_json_strings_eq("\xf0\x9f\x98\x80", "\xf0\x9f\x98\x80"); + + // Anything else is still escaped byte by byte: invalid, truncated, + // overlong, surrogate, beyond U+10FFFF, and control characters + assert_json_strings_eq("\xe9", "\\u00e9"); + assert_json_strings_eq("\xe4\xb8" "x", "\\u00e4\\u00b8x"); + assert_json_strings_eq("\xf0\x9f\x98", "\\u00f0\\u009f\\u0098"); + assert_json_strings_eq("\xc0\xaf", "\\u00c0\\u00af"); + assert_json_strings_eq("\xed\xa0\x80", "\\u00ed\\u00a0\\u0080"); + assert_json_strings_eq( + "\xf4\x90\x80\x80", "\\u00f4\\u0090\\u0080\\u0080"); + assert_json_strings_eq("\x01\x7f", "\\u0001\\u007f"); + + const char *const inputs[] = { + "caf\xc3\xa9", + "\xe4\xb8\xad \xf0\x9f\x98\x80", + "\xe9 \xe4\xb8 \xc0\xaf \xed\xa0\x80 \x01", + }; + for (size_t i = 0; i < sizeof(inputs) / sizeof(inputs[0]); i++) + { + JsonElement *const str = JsonStringCreate(inputs[i]); + char *const written = JsonToString(str); + JsonDestroy(str); + + const char *data = written; + JsonElement *json = NULL; + assert_int_equal(JSON_PARSE_OK, JsonParse(&data, &json)); + assert_string_equal(inputs[i], JsonPrimitiveGetAsString(json)); + + JsonDestroy(json); + free(written); + } +} + #define assert_json5_data_eq(_size, unescaped, escaped) \ { \ Slice data = {.data = (void *) unescaped, .size = _size}; \ @@ -2554,6 +2593,7 @@ int main() unit_test(test_show_object_simple), unit_test(test_show_string), unit_test(test_string_escape), + unit_test(test_string_escape_utf8), unit_test(test_string_escape_json5), unit_test(test_json_null_not_null), unit_test(test_json_object_merge_deep),