From ace9f32ee7e7ae3b17dc188d323c537df49fffda Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Sun, 11 Oct 2026 03:05:16 +0000 Subject: [PATCH] fix: escape CLI span JSON strings Quote labels and span text in both CLI output paths so JSON consumers can read quotes, backslashes, and control bytes. Use explicit lengths to preserve embedded NUL bytes and span boundaries. Add a model-independent serialization test. Native validation remains pending because the maintainer host has no C++ compiler or CMake. Assisted-by: Codex:gpt-6 --- tests/CMakeLists.txt | 5 +++++ tests/test_json.cpp | 33 +++++++++++++++++++++++++++++++++ tools/json.h | 36 ++++++++++++++++++++++++++++++++++++ tools/pf-cli.cpp | 21 +++++++++++++-------- 4 files changed, 87 insertions(+), 8 deletions(-) create mode 100644 tests/test_json.cpp create mode 100644 tools/json.h diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f1efd55..c2ccc14 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -14,6 +14,11 @@ pf_add_test(test_graph_blocks) pf_add_test(test_ner) pf_add_test(test_tokenizer) +# CLI serialization needs neither model weights nor ggml. +add_executable(test_json test_json.cpp) +target_compile_options(test_json PRIVATE -Wall -Wextra) +add_test(NAME test_json COMMAND test_json) + pf_add_test(test_parity) pf_add_test(test_window_stitch) set_tests_properties(test_parity test_window_stitch test_tokenizer diff --git a/tests/test_json.cpp b/tests/test_json.cpp new file mode 100644 index 0000000..3d52977 --- /dev/null +++ b/tests/test_json.cpp @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: MIT +#include "../tools/json.h" + +#include +#include +#include + +int main() { + struct test_case { std::string input; std::string expected; }; + const test_case cases[] = { + {"", "\"\""}, + {"plain / text", "\"plain / text\""}, + {"\"André\"", "\"\\\"André\\\"\""}, + {"path\\file\tline\nnext\r\b\f", "\"path\\\\file\\tline\\nnext\\r\\b\\f\""}, + {std::string("a\0b", 3), "\"a\\u0000b\""}, + {std::string("\x01\x1f", 2), "\"\\u0001\\u001f\""}, + }; + for (const auto & c : cases) { + if (pf::cli::json_quote(c.input) != c.expected) { + std::fprintf(stderr, "JSON escaping mismatch\n"); + return 1; + } + } + // Use an explicit span length: do not consume the remainder of the input. + if (pf::cli::json_quote(std::string_view("quoted\"tail", 6)) != "\"quoted\"") return 1; + // Every ASCII control byte must have an escaped representation. + for (int i = 0; i < 32; ++i) { + const char c = static_cast(i); + const std::string quoted = pf::cli::json_quote(std::string_view(&c, 1)); + if (quoted.size() < 4 || quoted[1] != '\\') return 1; + } + return 0; +} diff --git a/tools/json.h b/tools/json.h new file mode 100644 index 0000000..e397d01 --- /dev/null +++ b/tools/json.h @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: MIT +#pragma once + +#include +#include + +namespace pf::cli { + +// Preserve UTF-8 bytes and escape JSON syntax and ASCII control characters. +inline std::string json_quote(std::string_view text) { + static constexpr char hex[] = "0123456789abcdef"; + std::string out = "\""; + for (unsigned char c : text) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\b': out += "\\b"; break; + case '\f': out += "\\f"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + default: + if (c < 0x20) { + out += "\\u00"; + out += hex[c >> 4]; + out += hex[c & 0x0f]; + } else { + out += static_cast(c); + } + } + } + out += '"'; + return out; +} + +} // namespace pf::cli diff --git a/tools/pf-cli.cpp b/tools/pf-cli.cpp index e840042..0f27114 100644 --- a/tools/pf-cli.cpp +++ b/tools/pf-cli.cpp @@ -7,6 +7,7 @@ #include "model.h" #include "ner.h" #include "tokenizer.h" +#include "json.h" #include #include @@ -147,10 +148,12 @@ int main(int argc, char ** argv) { } std::printf("["); for (size_t i = 0; i < n_ents; i++) { - std::printf("%s\n {\"entity_group\": \"%s\", \"start\": %d, \"end\": %d, " - "\"score\": %.4f, \"text\": \"%.*s\"}", - i ? "," : "", ents[i].label, ents[i].start, ents[i].end, - ents[i].score, ents[i].end - ents[i].start, text.data() + ents[i].start); + std::printf("%s\n {\"entity_group\": %s, \"start\": %d, \"end\": %d, " + "\"score\": %.4f, \"text\": %s}", + i ? "," : "", pf::cli::json_quote(ents[i].label).c_str(), + ents[i].start, ents[i].end, ents[i].score, + pf::cli::json_quote(std::string_view(text.data() + ents[i].start, + ents[i].end - ents[i].start)).c_str()); } std::printf("\n]\n"); pf_entities_free(ents, n_ents); @@ -250,10 +253,12 @@ int main(int argc, char ** argv) { while (bstart < bend && (unsigned char) text[bstart] <= ' ') bstart++; while (bend > bstart && (unsigned char) text[bend - 1] <= ' ') bend--; if (bstart >= bend) continue; - std::printf("%s\n {\"entity_group\": \"%s\", \"start\": %d, \"end\": %d, " - "\"score\": %.4f, \"text\": \"%.*s\"}", - first ? "" : ",", lt.category_name(sp.cat).c_str(), bstart, bend, - sp.score, bend - bstart, text.data() + bstart); + std::printf("%s\n {\"entity_group\": %s, \"start\": %d, \"end\": %d, " + "\"score\": %.4f, \"text\": %s}", + first ? "" : ",", pf::cli::json_quote(lt.category_name(sp.cat)).c_str(), + bstart, bend, sp.score, + pf::cli::json_quote(std::string_view(text.data() + bstart, + bend - bstart)).c_str()); first = false; } std::printf("\n]\n");