diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 008a122d203..1b930d51d75 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,7 @@ repos: hooks: - id: check-added-large-files # Retain the exact lossless cross-platform performance evidence archives. - exclude: ^docs/benchmarks/((executor-copy-audit|bounded-read-ahead)-20261101|(directory-task-slots|packed-visit-snapshots|bounded-match-results|useful-match-workers|direct-evaluator-workers|derived-content-handoff|immutable-default-catalogs|prepared-diff-configuration|prepared-hash-operands|aggregate-comparison-totals)-20261102)\.json\.gz$ + exclude: ^docs/benchmarks/((executor-copy-audit|bounded-read-ahead)-20261101|(directory-task-slots|packed-visit-snapshots|bounded-match-results|useful-match-workers|direct-evaluator-workers|derived-content-handoff|immutable-default-catalogs|prepared-diff-configuration|prepared-hash-operands|aggregate-comparison-totals|byte-shingle-representations)-20261102)\.json\.gz$ - id: check-merge-conflict - id: check-yaml # BINARY_TYPES below: archive containers are binary but can START with text (a phar begins with diff --git a/CHANGELOG.md b/CHANGELOG.md index aaef0369b59..bcc00d1f266 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ # 0.11.0 +- Avoid repeated word-window string construction in text similarity checks while preserving exact byte-word scores. + - Aggregate ordinary comparison side totals without retaining duplicate per-path summary tables; category and effect-sensitive summaries keep their existing behavior. - Prepare explicit hash-primary algorithms and encodings once while preserving per-context defaults and conditional expected-hash templates. diff --git a/TODO.md b/TODO.md index f617ca5d61b..d7595417859 100644 --- a/TODO.md +++ b/TODO.md @@ -436,9 +436,27 @@ EP/JC/P work; candidates are not speedup claims until measured on both platforms - [ ] O16b: Add and qualify completed-directory/frontier traversal before enabling bounded paired comparison. Preserve empty/ignored completion, ordering, ignore inheritance, errors, effects, archives and deferred selections; retain explicit fallbacks and avoid repeated shallow scans. -- [ ] O17-O22: Reconcile reusable stage pools, byte-bounded overlap, archive indexing/admission, - optional bulk metadata, similarity representations and output churn with EP/JC/P acceptance. - Keep macrobenchmark, kernel and diagnostic contracts separate. +- [ ] Benchmark history: precompute a shared task order from compatible merged revisions for the + whole series. Current per-revision/per-CPU similarity ordering can visually exaggerate or + hide changes. Compare robust median adjacency distances across revisions, orient once from + series-wide performance, retain missing-task gaps, and reuse positions across the version, + CPU-pair and scale selectors. Recompute on publication; keep Alphabetical available and + verify hover/tick mapping. Judge performance from matched cells, not surface geometry. +- [ ] O17: Qualify coordinated persistent stage workers against existing pools and prior rejected + caller-inclusive/admission policies. Preserve leaf jobs, explicit CPU/role accounting and JC gates. +- [ ] O18: Qualify byte-bounded overlap with backpressure, ordered publication and cancellation; + avoid per-batch throwaway threads and nested waits on the same saturated pool. +- [ ] O19: Qualify archive seeking/index/replay and decoder admission independently. Investigate + native/memory ZIP seeking before concurrency; preserve sequential-source and shared-budget fallbacks. +- [ ] O20: Qualify optional bulk metadata by backend capability with precise fallback, while + keeping metadata-free traversal free of extra probes and extended-stat work. +- [x] O21a: Qualify staged one-word keys, borrowed normalized shingle windows and an isolated byte + equality shortcut on Linux/macOS. Preserve [rejected width-one paths and all controls](docs/benchmarks/byte-shingle-representations-20261102.md). +- [ ] O21b: Reuse immutable-source/reference representations and exact candidate indexes only after + resolving dynamic target/invalidation and retained-byte contracts; keep run-wide grouping semantics explicit. +- [ ] O22: Qualify snapshot-owned digest observers, invariant field-hash qualifiers and output + construction separately. Preserve lifetime/invalidation, direct fallback and formatting contracts. + Do not revive O05's rejected generic lookup change without new consumer-specific evidence. - [ ] Qualify executor admission after the clock correction with separate cold/warm controls and 16/32/63/64-entry matching boundaries; check Linux and macOS workload regressions before diff --git a/docs/benchmarks/byte-shingle-representations-20261102.json.gz b/docs/benchmarks/byte-shingle-representations-20261102.json.gz new file mode 100644 index 00000000000..9701348fec4 Binary files /dev/null and b/docs/benchmarks/byte-shingle-representations-20261102.json.gz differ diff --git a/docs/benchmarks/byte-shingle-representations-20261102.md b/docs/benchmarks/byte-shingle-representations-20261102.md new file mode 100644 index 00000000000..914af7b6542 --- /dev/null +++ b/docs/benchmarks/byte-shingle-representations-20261102.md @@ -0,0 +1,96 @@ +# Byte-word shingle representations (2026-11-02) + +## Decision + +Retain an exact-byte equality shortcut, staged owned keys for one-word shingles, and borrowed views over stable normalized buffers for wider windows. The one-word path keeps the existing word-vector stage and removes the unnecessary singleton join. Wider shingles avoid temporary joined strings and per-word string storage. No shared cache, worker, lock, flag, dependency, exception mechanism or approximate equality is added. + +The algorithm remains byte based: ASCII alphanumerics and bytes with the high bit set form words; only ASCII case folds. ASCII separators become internal NUL boundaries, preserving unambiguous keys. Width zero, short/empty inputs, uniqueness, Jaccard and integer rounding retain their contracts. The header explicitly distinguishes this from Unicode decoding/case folding. Views are constructed only after normalized strings reach their final local owners and die before those strings. No moved record contains self-referential views. + +O21a is per-comparison representation work only. O21b reference caching and exact candidate indexing remain open: dynamic targets, effects, immutable-source identity, bounded retention and the unresolved run-wide grouping semantics still require their own design. + +## Rejected prototypes + +A three-way initial matrix isolates baseline, equality-only and all-view implementations. All-view one-word kernels regress on macOS by about 26-33% for larger nearby/disjoint inputs, despite attractive mixed native results. Do not ship that path or hide it behind the mixed-corpus average. A direct word-to-owned-set prototype reduces the problem but does not establish a consistent macOS disjoint win against duplicate-baseline controls. The selected staged construction is separately qualified. All original sources, samples and counterexamples remain in the raw artifact. + +## Measurement contracts + +Measured parent `d528de1d81f4aeb3dbf369f9b9cb394b362b4156`, identical in tracked contents to main `6bebe940ba36328a399461a19434bf627118bf75` after #1057. Publication parent additionally contains only the isolated #1058 CI cache action/test fix; the declared tooling test was rerun on both hosts. Final runtime changes only `similarity.cc`; common header/test/benchmark support is identical across variant builds. Clean `clang_release`/ThinLTO manifests retain compiler, configuration, link actions, source and executable identities. No build, lint, sanitizer or allocation probe overlaps release timing on its host. + +Per host, final qualification has 66 native cases x three sessions (1/10/1,000/10,000 files, one/three requested workers, short/128-word/long-word inputs, pure disjoint/case-only controls, width-one and wider predicates, identity and listing), discarded warmups, nine rotating rounds/fastest seven. An independent Python byte-token/set oracle verifies exact complete stdout and zero exits. Worker requests do not imply that similarity predicates were newly parallelized. Linux uses tmpfs/affinity; macOS uses warm APFS. + +Separate whole-engine/kernel timing has 51 cases x three sessions x two variants. Whole engine excludes command parsing/fixture creation; pure kernels measure only the score function. Confirmation adds a byte-identical baseline in ten native cases x five sessions and eight memory cases x three sessions. Initial all-view matrices, equality-only counterfactuals and refinement pilots remain separate populations. + +## Native commands + +Median paired session change; negative is faster. + +| Fixture / task | CPUs | Linux | macOS | +| :-------------------------- | ---: | ------: | ------: | +| short10 / near | 1 | -0.41% | -0.06% | +| words10000 / near | 1 | -46.02% | -18.38% | +| words10000 / strict | 3 | -49.03% | -26.65% | +| words10000 / equal | 1 | -79.61% | -37.85% | +| long10000 / near | 1 | -21.72% | -20.79% | +| disjointwords10000 / single | 1 | -26.97% | -4.37% | +| disjointwords10000 / strict | 1 | -40.85% | -19.33% | +| casewords10000 / single | 3 | -30.54% | -4.80% | +| short1 / listing | 1 | +0.79% | -0.33% | +| short10000 / listing | 1 | +8.21% | -0.28% | + +## Whole-engine and pure-kernel controls + +Each unit is explicit; do not compare kernel nanoseconds with whole-engine microseconds or native startup times. Values are median session fastest-seven means. + +| Case | Unit | Linux baseline | Linux selected | Linux delta | macOS baseline | macOS selected | macOS delta | +| :----------------------------------------- | :--- | -------------: | -------------: | ----------: | -------------: | -------------: | ----------: | +| engine/production/similar-equal/10000/1 | us | 2,519.82 | 1,794.44 | -28.79% | 3,656.87 | 2,345.02 | -35.87% | +| engine/production/similar-disjoint/10000/1 | us | 2,387.61 | 2,027.65 | -15.08% | 3,565.00 | 2,887.18 | -19.01% | +| engine/production/similar-zero/10000/3 | us | 2,543.51 | 2,210.68 | -13.09% | 3,670.96 | 3,059.61 | -16.65% | +| engine/production/name/10000/3 | us | 1,134.57 | 1,133.69 | -0.08% | 1,658.76 | 1,660.70 | +0.12% | +| BmWordShinglePercent/nearby/2048/1 | ns | 106,007.25 | 71,351.44 | -32.69% | 104,337.07 | 85,083.96 | -18.45% | +| BmWordShinglePercent/disjoint/2048/1 | ns | 143,265.33 | 109,667.70 | -23.45% | 131,944.99 | 107,707.04 | -18.37% | +| BmWordShinglePercent/nearby/2048/5 | ns | 165,091.97 | 93,009.75 | -43.66% | 245,646.64 | 157,987.62 | -35.69% | +| BmWordShinglePercent/equal/2048/5 | ns | 144,571.90 | 112.39 | -99.92% | 230,074.48 | 472.33 | -99.79% | + +## Native confirmation and slower controls + +Candidate and identical-baseline repeat are independently compared with the baseline participant; no subtraction of repeat movement is applied. + +| Fixture / task | CPUs | Initial Linux | Confirm Linux | Repeat Linux | Initial macOS | Confirm macOS | Repeat macOS | +| :-------------------------- | ---: | ------------: | ------------: | -----------: | ------------: | ------------: | -----------: | +| short10 / listing | 3 | +0.90% | +1.35% | -0.90% | +1.33% | +1.18% | +0.30% | +| disjointwords10000 / single | 1 | -26.97% | -27.52% | +0.03% | -4.37% | -4.47% | +0.21% | +| words10000 / equal | 1 | -79.61% | -79.81% | -0.31% | -37.85% | -38.11% | +0.23% | +| long10 / strict | 3 | -6.46% | -3.91% | -1.03% | -2.95% | -2.50% | +0.37% | +| casewords10000 / single | 3 | -30.54% | -30.30% | -0.02% | -4.80% | -4.53% | -0.06% | +| short10 / near | 1 | -0.41% | +1.19% | -0.99% | -0.06% | +1.25% | +0.46% | +| long10000 / near | 1 | -21.72% | -21.99% | +0.04% | -20.79% | -20.58% | +0.11% | +| words10000 / near | 1 | -46.02% | -46.11% | +0.03% | -18.38% | -18.43% | -0.01% | +| short10000 / listing | 1 | +8.21% | +2.36% | +0.95% | -0.28% | +0.66% | +0.27% | +| words10000 / strict | 3 | -49.03% | -49.15% | -0.18% | -26.65% | -25.80% | -0.30% | + +## Allocation diagnostics + +ASan replaces the allocator: these are warmed calling-thread allocation/requested-byte counts, not production timings or exact production allocator behavior. The probe is a separate executable and its transient Bazel/source edits are restored. Normal release timings above come from separate uninstrumented binaries. + +| Case | Host | Baseline allocations | Selected allocations | Baseline requested bytes | Selected requested bytes | +| :----------------- | :---- | -------------------: | -------------------: | -----------------------: | -----------------------: | +| equal/2048/5 | linux | 4,211 | 0 | 657,232 | 0 | +| equal/2048/5 | macos | 4,196 | 0 | 572,976 | 0 | +| nearby/2048/1 | linux | 125 | 112 | 510,608 | 457,808 | +| nearby/2048/1 | macos | 116 | 120 | 472,688 | 459,856 | +| nearby/2048/5 | linux | 4,220 | 131 | 891,632 | 507,384 | +| nearby/2048/5 | macos | 4,220 | 126 | 940,784 | 503,520 | +| repeated/2048/4096 | linux | 28 | 20 | 217,488 | 49,104 | +| repeated/2048/4096 | macos | 28 | 20 | 217,488 | 49,104 | + +## Correctness and footprint + +Independent reference tests cover separator normalization, ambiguous boundaries, duplicate words, long words, zero/one/oversized widths, empty inputs, opaque byte values and seeded reused comparisons. Existing read-only engine tests retain conditional target reads, binary/failure handling and dynamic templates. All 323 extension-enabled release tests pass per host, with required Linux FUSE; six focused ASan suites pass per host and six Linux TSan suites pass. Scoped lint and normal enforcing hooks remain required; current pushed-head CI determines readiness. + +- linux: unstripped CLI 12,393,880 -> 12,402,544 bytes (+8,664). +- macos: unstripped CLI 4,079,152 -> 4,079,328 bytes (+176). + +## Lossless evidence + +[byte-shingle-representations-20261102.json.gz](byte-shingle-representations-20261102.json.gz): 2,751,662 bytes; SHA-256 `804df02ab92bfec0f0dde9af7cbf09d5f37151e43f1617317525091145d7e145`. diff --git a/docs/design-near-duplicates.md b/docs/design-near-duplicates.md index 892bda9688d..24522577a4d 100644 --- a/docs/design-near-duplicates.md +++ b/docs/design-near-duplicates.md @@ -4,6 +4,12 @@ run-wide grouping operation has a different scaling problem: comparing every pair is quadratic before the exact verifier has learned anything about either document. +The current exact primitive is byte based: words contain ASCII alphanumerics or high-bit bytes, +and only ASCII case folds. It does not decode Unicode or fold non-ASCII letters. Per-comparison +representation improvements preserve that contract. Reusing a peer representation across entries +requires an immutable source or an explicit mutation/invalidation contract; a dynamic target template +alone does not establish either. + ## Fixed correctness boundary Candidate generation may over-select. It must never emit a relationship. Every candidate pair passes diff --git a/docs/performance-analysis.md b/docs/performance-analysis.md index 7ae0a5875d0..9caa414031e 100644 --- a/docs/performance-analysis.md +++ b/docs/performance-analysis.md @@ -3454,3 +3454,31 @@ byte-bounded queues. Directory jobs must remain leaf work and never wait for nes own saturated pool. O08 task storage and O11 warm participation do not establish that a shared executor is faster. Global sorting/effects/deferred/archive cases need explicit fallback contracts until independently qualified. + +### O21a: byte-word shingle representations + +The exact verifier previously built a vector of owning words and another owning string for every +shingle window. Initial all-view prototypes replace those with normalized buffers and borrowed +windows. An equality-only counterfactual isolates the fast byte-equality check. Crucially, the +all-view width-one path regresses larger macOS kernels by about 26-33%, despite favorable mixed +native command results; those samples must remain visible and do not qualify that implementation. +Direct word-to-set insertion also needs a distinct decision rather than being assumed faster. + +The staged candidate retains the original word-vector construction for width one, removes the +unnecessary singleton join and retains owning word keys. Wider shingles borrow spans from final, +local normalized buffers. The score helper takes keys by reference; normalized buffers never move +or grow after views exist. ASCII folding, opaque high-bit bytes, separator boundaries, unique sets, +Jaccard rounding, width-zero behavior and conditional file observations are unchanged. This is not +Unicode decoding or a new text mode. No worker, lock, shared cache, exception path or flag is added. + +[The qualification report](benchmarks/byte-shingle-representations-20261102.md) separates the rejected +all-view/direct-insertion experiments, isolated equality counterfactual, final native and virtual +whole-engine/kernel populations, duplicate-baseline confirmation, allocation diagnostics and footprint. +Pure disjoint and case-only corpora prevent mixed-input averages from concealing width-one costs. +Allocation counts are collected separately under instrumentation; its times are never release +performance results. + +This covers O21a's per-comparison representations only. O21b reference reuse and candidate indexing +remain open: dynamic peer templates, content mutation/invalidation, immutable-source identity, +retained-byte bounds and grouping semantics require their own qualification. No source-independent +cross-file cache or approximate relationship is introduced. diff --git a/release-site.json b/release-site.json index fda6ae62d51..5cf72cb48de 100644 --- a/release-site.json +++ b/release-site.json @@ -75,7 +75,8 @@ "docs/benchmarks/immutable-default-catalogs-20261102.md": "docs/benchmarks/immutable-default-catalogs-20261102.html", "docs/benchmarks/prepared-diff-configuration-20261102.md": "docs/benchmarks/prepared-diff-configuration-20261102.html", "docs/benchmarks/prepared-hash-operands-20261102.md": "docs/benchmarks/prepared-hash-operands-20261102.html", - "docs/benchmarks/aggregate-comparison-totals-20261102.md": "docs/benchmarks/aggregate-comparison-totals-20261102.html" + "docs/benchmarks/aggregate-comparison-totals-20261102.md": "docs/benchmarks/aggregate-comparison-totals-20261102.html", + "docs/benchmarks/byte-shingle-representations-20261102.md": "docs/benchmarks/byte-shingle-representations-20261102.html" }, "generated_html": { "XFF.md": { diff --git a/xff/engine/expression_run_benchmark.cc b/xff/engine/expression_run_benchmark.cc index 45ba042c4ff..a66219a1f85 100644 --- a/xff/engine/expression_run_benchmark.cc +++ b/xff/engine/expression_run_benchmark.cc @@ -149,6 +149,18 @@ void Measure(benchmark::State& state, const Scenario& scenario, ExpressionExecut int main(int argc, char** argv) { benchmark::Initialize(&argc, argv); const auto scenarios = std::to_array({ + { + .name = "similar-equal", + .arguments = {"-type", "f", "-similar:3:100%", "{path}"}, + }, + { + .name = "similar-disjoint", + .arguments = {"-type", "f", "-similar:3:50%", "peer/diff-control"}, + }, + { + .name = "similar-zero", + .arguments = {"-type", "f", "-similar:3:0%", "peer/diff-control"}, + }, {.name = "compare-template", .arguments = {"-type", "f", "-cmp", "peer/{relpath:s/file/copy/}"}}, { .name = "diff-normalized", diff --git a/xff/matching/similarity/BUILD.bazel b/xff/matching/similarity/BUILD.bazel index 33ea3286583..afdce9bbd02 100644 --- a/xff/matching/similarity/BUILD.bazel +++ b/xff/matching/similarity/BUILD.bazel @@ -20,6 +20,8 @@ cc_library( cc_test( name = "similarity_test", size = "small", + # The allocating byte-reference matrix exceeds the small timeout under MSan origin tracking. + timeout = "moderate", srcs = ["similarity_test.cc"], deps = [ ":similarity_cc", diff --git a/xff/matching/similarity/similarity.cc b/xff/matching/similarity/similarity.cc index 0f3f31a820a..067ae0d5d9c 100644 --- a/xff/matching/similarity/similarity.cc +++ b/xff/matching/similarity/similarity.cc @@ -3,25 +3,27 @@ #include "xff/matching/similarity/similarity.h" -#include #include #include #include +#include #include #include "absl/container/flat_hash_set.h" #include "absl/strings/ascii.h" -#include "absl/strings/str_join.h" namespace xff::similarity { namespace { -std::vector Words(std::string_view text) { +bool IsByteWordChar(char value) { + return static_cast(value) >= 0x80 || absl::ascii_isalnum(value); +} + +std::vector OwnedByteWords(std::string_view text) { std::vector words; std::string word; for (const char value : text) { - const auto byte = static_cast(value); - if (byte >= 0x80 || absl::ascii_isalnum(value)) { + if (IsByteWordChar(value)) { word.push_back(absl::ascii_tolower(value)); } else if (!word.empty()) { words.push_back(std::move(word)); @@ -34,39 +36,102 @@ std::vector Words(std::string_view text) { return words; } -absl::flat_hash_set Shingles(std::string_view text, std::size_t width) { - const std::vector words = Words(text); +absl::flat_hash_set OwnedSingleByteShingles(std::string_view text) { + const auto words = OwnedByteWords(text); absl::flat_hash_set result; - if (words.empty() || width == 0) { + for (const auto& word : words) { + result.insert(word); + } + return result; +} + +struct NormalizedByteWords { + std::string bytes; + std::size_t count = 0; +}; + +NormalizedByteWords NormalizeByteWords(std::string_view text) { + NormalizedByteWords result; + bool in_word = false; + for (const char value : text) { + if (!IsByteWordChar(value)) { + in_word = false; + continue; + } + if (!in_word) { + if (result.count != 0) { + result.bytes.push_back('\0'); + } + ++result.count; + in_word = true; + } + result.bytes.push_back(absl::ascii_tolower(value)); + } + return result; +} + +// The borrowed input must remain stable until the returned views are destroyed. +absl::flat_hash_set ByteWordShingles(const NormalizedByteWords& words, std::size_t width) { + absl::flat_hash_set result; + if (words.count == 0 || width == 0) { return result; } - const std::size_t shingle_width = std::min(width, words.size()); - for (std::size_t begin = 0; begin + shingle_width <= words.size(); ++begin) { - result.insert( - absl::StrJoin( - words.begin() + static_cast(begin), - words.begin() + static_cast(begin + shingle_width), std::string_view("\0", 1))); + const std::string_view bytes = words.bytes; + if (width >= words.count) { + result.insert(bytes); + return result; + } + std::size_t end = 0; + // There are more words than the window, so each initial boundary exists. + for (std::size_t word = 0; word < width; ++word) { + end = bytes.find('\0', end); + if (word + 1 < width) { + ++end; + } + } + std::size_t begin = 0; + while (true) { + result.insert(bytes.substr(begin, end - begin)); + if (end == bytes.size()) { + break; + } + begin = bytes.find('\0', begin) + 1; + end = bytes.find('\0', end + 1); + if (end == std::string_view::npos) { + end = bytes.size(); + } } return result; } +template +int ShinglePercent(const ShingleSet& left, const ShingleSet& right) { + if (left.empty() && right.empty()) { + return 100; + } + std::size_t intersection = 0; + for (const auto& shingle : left) { + intersection += right.contains(shingle) ? 1 : 0; + } + const std::size_t union_size = left.size() + right.size() - intersection; + return static_cast(((100 * intersection) + (union_size / 2)) / union_size); +} + } // namespace int WordShinglePercent(std::string_view lhs, std::string_view rhs, std::size_t width) { if (width == 0) { return 0; } - const absl::flat_hash_set lhs_shingles = Shingles(lhs, width); - const absl::flat_hash_set rhs_shingles = Shingles(rhs, width); - if (lhs_shingles.empty() && rhs_shingles.empty()) { + if (lhs == rhs) { return 100; } - std::size_t intersection = 0; - for (const std::string& shingle : lhs_shingles) { - intersection += rhs_shingles.contains(shingle) ? 1 : 0; + if (width == 1) { + return ShinglePercent(OwnedSingleByteShingles(lhs), OwnedSingleByteShingles(rhs)); } - const std::size_t union_size = lhs_shingles.size() + rhs_shingles.size() - intersection; - return static_cast(((100 * intersection) + (union_size / 2)) / union_size); + const auto left_words = NormalizeByteWords(lhs); + const auto right_words = NormalizeByteWords(rhs); + return ShinglePercent(ByteWordShingles(left_words, width), ByteWordShingles(right_words, width)); } } // namespace xff::similarity diff --git a/xff/matching/similarity/similarity.h b/xff/matching/similarity/similarity.h index 41382e777ea..beafea56b56 100644 --- a/xff/matching/similarity/similarity.h +++ b/xff/matching/similarity/similarity.h @@ -10,10 +10,10 @@ namespace xff::similarity { // Jaccard similarity of the unique contiguous word-shingle sets, rounded to an integer percentage -// in [0, 100]. Words are maximal runs of ASCII alphanumerics or UTF-8 bytes; ASCII case is folded, -// and punctuation/whitespace are separators. A non-empty input shorter than `width` contributes one -// shingle containing all its words, so short files remain meaningfully comparable. Width zero is -// invalid and returns 0. +// in [0, 100]. Words are maximal runs of ASCII alphanumerics or bytes >= 0x80; ASCII case is folded, +// with no Unicode decoding or non-ASCII case folding. ASCII punctuation/whitespace are separators. A non-empty input +// shorter than `width` contributes one shingle containing all its words, so short files remain meaningfully comparable. +// Width zero is invalid and returns 0. [[nodiscard]] int WordShinglePercent(std::string_view lhs, std::string_view rhs, std::size_t width); } // namespace xff::similarity diff --git a/xff/matching/similarity/similarity_test.cc b/xff/matching/similarity/similarity_test.cc index 5c2ce1307f8..4a667975c83 100644 --- a/xff/matching/similarity/similarity_test.cc +++ b/xff/matching/similarity/similarity_test.cc @@ -3,6 +3,17 @@ #include "xff/matching/similarity/similarity.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -11,6 +22,58 @@ namespace { using ::testing::Eq; +// Independent, deliberately allocating reference for the existing byte-word contract. +std::set ReferenceByteShingles(std::string_view text, std::size_t width) { + std::vector words; + std::string word; + const auto flush = [&] { + if (!word.empty()) { + words.push_back(word); + word.clear(); + } + }; + for (const unsigned char byte : text) { + const bool ascii_letter = (byte >= 'a' && byte <= 'z') || (byte >= 'A' && byte <= 'Z'); + if (byte >= 128 || ascii_letter || (byte >= '0' && byte <= '9')) { + word.push_back(static_cast(byte >= 'A' && byte <= 'Z' ? byte + ('a' - 'A') : byte)); + } else { + flush(); + } + } + flush(); + std::set shingles; + if (words.empty() || width == 0) { + return shingles; + } + const std::size_t count = std::min(width, words.size()); + for (std::size_t start = 0; start <= words.size() - count; ++start) { + std::string shingle = words.at(start); + for (std::size_t offset = 1; offset < count; ++offset) { + shingle.push_back('\0'); + shingle.append(words.at(start + offset)); + } + shingles.insert(std::move(shingle)); + } + return shingles; +} + +int ReferenceBytePercent(std::string_view left, std::string_view right, std::size_t width) { + if (width == 0) { + return 0; + } + const auto lhs = ReferenceByteShingles(left, width); + const auto rhs = ReferenceByteShingles(right, width); + if (lhs.empty() && rhs.empty()) { + return 100; + } + std::size_t common = 0; + for (const auto& shingle : lhs) { + common += rhs.contains(shingle) ? 1 : 0; + } + const std::size_t total = lhs.size() + rhs.size() - common; + return static_cast(((100 * common) + (total / 2)) / total); +} + struct SimilarityTest : ::testing::Test {}; TEST_F(SimilarityTest, IdenticalWordShinglesScoreOneHundred) { @@ -47,5 +110,53 @@ TEST_F(SimilarityTest, WordBoundariesCannotCollideInTheShingleRepresentation) { EXPECT_THAT(WordShinglePercent("ab c", "a bc", 2), Eq(0)); } +TEST_F(SimilarityTest, NormalizedSeparatorsAndLongWordsRetainExactSetSemantics) { + const auto inputs = std::to_array({ + "", + "!!!...---", + " a b c d e f ", + "A\tB\nC\rD-E_F", + "a a a a a a", + "a b a b a b a", + std::string("a\0b\0c", 5), + std::string(256, 'A') + " " + std::string(512, 'b'), + std::string(256, 'a') + " " + std::string(512, 'c'), + }); + constexpr auto kWidths = std::to_array({0, 1, 2, 3, 5, 100, std::numeric_limits::max()}); + for (const auto& left : inputs) { + for (const auto& right : inputs) { + for (const auto width : kWidths) { + SCOPED_TRACE(width); + EXPECT_THAT(WordShinglePercent(left, right, width), Eq(ReferenceBytePercent(left, right, width))); + } + } + } +} + +TEST_F(SimilarityTest, SeededOpaqueByteInputsMatchTheIndependentReference) { + // This is a byte-tokenization test, not a Unicode case-folding test. All high-bit + // bytes are word constituents, including byte sequences that are not valid UTF-8. + // NOLINTNEXTLINE(bugprone-random-generator-seed,cert-msc32-c,cert-msc51-cpp): reproducible test corpus. + std::mt19937 random(20'261'102); + constexpr auto kWidths = std::to_array({0, 1, 2, 5, 20}); + for (std::size_t sample = 0; sample < 1'000; ++sample) { + std::string left; + std::string right; + const std::size_t length = random() % 256; + for (std::size_t index = 0; index < length; ++index) { + left.push_back(static_cast(random() % 256)); + right.push_back(index % 5 == 0 ? static_cast(random() % 256) : left.back()); + } + for (const auto width : kWidths) { + SCOPED_TRACE(sample); + SCOPED_TRACE(width); + const auto expected = ReferenceBytePercent(left, right, width); + EXPECT_THAT(WordShinglePercent(left, right, width), Eq(expected)); + EXPECT_THAT(WordShinglePercent(right, left, width), Eq(expected)); + EXPECT_THAT(WordShinglePercent(left, left, width), Eq(width == 0 ? 0 : 100)); + } + } +} + } // namespace } // namespace xff::similarity