diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f1efd55..c2ccc14 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -14,6 +14,11 @@ pf_add_test(test_graph_blocks) pf_add_test(test_ner) pf_add_test(test_tokenizer) +# CLI serialization needs neither model weights nor ggml. +add_executable(test_json test_json.cpp) +target_compile_options(test_json PRIVATE -Wall -Wextra) +add_test(NAME test_json COMMAND test_json) + pf_add_test(test_parity) pf_add_test(test_window_stitch) set_tests_properties(test_parity test_window_stitch test_tokenizer diff --git a/tests/test_json.cpp b/tests/test_json.cpp new file mode 100644 index 0000000..3d52977 --- /dev/null +++ b/tests/test_json.cpp @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: MIT +#include "../tools/json.h" + +#include +#include +#include + +int main() { + struct test_case { std::string input; std::string expected; }; + const test_case cases[] = { + {"", "\"\""}, + {"plain / text", "\"plain / text\""}, + {"\"André\"", "\"\\\"André\\\"\""}, + {"path\\file\tline\nnext\r\b\f", "\"path\\\\file\\tline\\nnext\\r\\b\\f\""}, + {std::string("a\0b", 3), "\"a\\u0000b\""}, + {std::string("\x01\x1f", 2), "\"\\u0001\\u001f\""}, + }; + for (const auto & c : cases) { + if (pf::cli::json_quote(c.input) != c.expected) { + std::fprintf(stderr, "JSON escaping mismatch\n"); + return 1; + } + } + // Use an explicit span length: do not consume the remainder of the input. + if (pf::cli::json_quote(std::string_view("quoted\"tail", 6)) != "\"quoted\"") return 1; + // Every ASCII control byte must have an escaped representation. + for (int i = 0; i < 32; ++i) { + const char c = static_cast(i); + const std::string quoted = pf::cli::json_quote(std::string_view(&c, 1)); + if (quoted.size() < 4 || quoted[1] != '\\') return 1; + } + return 0; +} diff --git a/tools/json.h b/tools/json.h new file mode 100644 index 0000000..e397d01 --- /dev/null +++ b/tools/json.h @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: MIT +#pragma once + +#include +#include + +namespace pf::cli { + +// Preserve UTF-8 bytes and escape JSON syntax and ASCII control characters. +inline std::string json_quote(std::string_view text) { + static constexpr char hex[] = "0123456789abcdef"; + std::string out = "\""; + for (unsigned char c : text) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\b': out += "\\b"; break; + case '\f': out += "\\f"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + default: + if (c < 0x20) { + out += "\\u00"; + out += hex[c >> 4]; + out += hex[c & 0x0f]; + } else { + out += static_cast(c); + } + } + } + out += '"'; + return out; +} + +} // namespace pf::cli diff --git a/tools/pf-cli.cpp b/tools/pf-cli.cpp index e840042..0f27114 100644 --- a/tools/pf-cli.cpp +++ b/tools/pf-cli.cpp @@ -7,6 +7,7 @@ #include "model.h" #include "ner.h" #include "tokenizer.h" +#include "json.h" #include #include @@ -147,10 +148,12 @@ int main(int argc, char ** argv) { } std::printf("["); for (size_t i = 0; i < n_ents; i++) { - std::printf("%s\n {\"entity_group\": \"%s\", \"start\": %d, \"end\": %d, " - "\"score\": %.4f, \"text\": \"%.*s\"}", - i ? "," : "", ents[i].label, ents[i].start, ents[i].end, - ents[i].score, ents[i].end - ents[i].start, text.data() + ents[i].start); + std::printf("%s\n {\"entity_group\": %s, \"start\": %d, \"end\": %d, " + "\"score\": %.4f, \"text\": %s}", + i ? "," : "", pf::cli::json_quote(ents[i].label).c_str(), + ents[i].start, ents[i].end, ents[i].score, + pf::cli::json_quote(std::string_view(text.data() + ents[i].start, + ents[i].end - ents[i].start)).c_str()); } std::printf("\n]\n"); pf_entities_free(ents, n_ents); @@ -250,10 +253,12 @@ int main(int argc, char ** argv) { while (bstart < bend && (unsigned char) text[bstart] <= ' ') bstart++; while (bend > bstart && (unsigned char) text[bend - 1] <= ' ') bend--; if (bstart >= bend) continue; - std::printf("%s\n {\"entity_group\": \"%s\", \"start\": %d, \"end\": %d, " - "\"score\": %.4f, \"text\": \"%.*s\"}", - first ? "" : ",", lt.category_name(sp.cat).c_str(), bstart, bend, - sp.score, bend - bstart, text.data() + bstart); + std::printf("%s\n {\"entity_group\": %s, \"start\": %d, \"end\": %d, " + "\"score\": %.4f, \"text\": %s}", + first ? "" : ",", pf::cli::json_quote(lt.category_name(sp.cat)).c_str(), + bstart, bend, sp.score, + pf::cli::json_quote(std::string_view(text.data() + bstart, + bend - bstart)).c_str()); first = false; } std::printf("\n]\n");