diff --git a/scripts/bench-simhash-sampling.mjs b/scripts/bench-simhash-sampling.mjs new file mode 100644 index 0000000000..ff61e22d0d --- /dev/null +++ b/scripts/bench-simhash-sampling.mjs @@ -0,0 +1,26 @@ +// Run with: node --max-old-space-size=512 --expose-gc --import tsx scripts/bench-simhash-sampling.mjs [tokens] +import { performance } from "node:perf_hooks"; +import { simhashHex } from "../src/affinity-simhash.ts"; + +const count = Number(process.argv[2] ?? 1_000_000); +if (!Number.isSafeInteger(count) || count < 1) throw new Error("tokens must be a positive safe integer"); +const canonical = Array.from({ length: count }, (_, i) => `word${i % 997}`).join(" "); +const timings = []; +let sketch; +for (let i = 0; i < 4; i++) { + global.gc?.(); + const start = performance.now(); + const result = simhashHex(canonical); + const elapsed = performance.now() - start; + if (sketch !== undefined && sketch !== result) throw new Error("unstable sketch"); + sketch = result; + if (i > 0) timings.push(elapsed); +} +console.log(JSON.stringify({ + count, + bytes: Buffer.byteLength(canonical), + sketch, + timings_ms: timings, + median_ms: [...timings].sort((a, b) => a - b)[1], + max_rss_kib: process.resourceUsage().maxRSS, +})); diff --git a/src/affinity-simhash.ts b/src/affinity-simhash.ts index f1ea3c6f60..5fc9695450 100644 --- a/src/affinity-simhash.ts +++ b/src/affinity-simhash.ts @@ -54,22 +54,15 @@ function fnv1a32(text: string, basis: number): number { * so a short message still yields a stable, comparable sketch. */ export function simhashHex(canonical: string): string { const tokens = canonical.match(/[\p{L}\p{N}_]+/gu) ?? []; - const features: string[] = []; - if (tokens.length < 2) { - for (const t of tokens) features.push(t); - } else { - for (let i = 0; i + 1 < tokens.length; i++) features.push(`${tokens[i]}\u0001${tokens[i + 1]}`); - } - if (features.length === 0) { - // No word tokens at all (pure punctuation/whitespace): hash the raw - // string so even that degenerate case is deterministic and comparable. - features.push(canonical); - } - const stride = Math.max(1, Math.ceil(features.length / MAX_FEATURES)); + const featureCount = Math.max(1, tokens.length - 1); + const stride = Math.max(1, Math.ceil(featureCount / MAX_FEATURES)); const v = new Int16Array(64); - for (let f = 0; f < features.length; f += stride) { - const h1 = fnv1a32(features[f]!, 2166136261); - const h2 = fnv1a32(features[f]!, 2166136261 ^ 0x9e3779b9); + for (let f = 0; f < featureCount; f += stride) { + // Construct only sampled shingles, not an additional full-size array + // for a long message. No-token inputs still hash the raw string. + const feature = tokens.length < 2 ? (tokens[0] ?? canonical) : `${tokens[f]}\u0001${tokens[f + 1]}`; + const h1 = fnv1a32(feature, 2166136261); + const h2 = fnv1a32(feature, 2166136261 ^ 0x9e3779b9); for (let b = 0; b < 64; b++) { const bit = b < 32 ? (h1 >>> b) & 1 : (h2 >>> (b - 32)) & 1; v[b] += bit === 1 ? 1 : -1; diff --git a/tests/simhash-vote-range.test.ts b/tests/simhash-vote-range.test.ts index 932d185826..2fb28a831e 100644 --- a/tests/simhash-vote-range.test.ts +++ b/tests/simhash-vote-range.test.ts @@ -18,3 +18,29 @@ test("unrelated repeated messages keep distinct sketches at the sampling cap", ( assert.notEqual(first, second); assert.equal(hammingHex(first, second), hammingHex(simhashHex("alpha alpha"), simhashHex("omega omega"))); }); + +test("SimHash sampling preserves persisted sketches across stride boundaries", () => { + // Captured from the original full-shingle implementation. Distinct tokens + // make a shifted sample observable, unlike a repeated-shingle fixture. + const sketches: [number, string][] = [ + [2, "896b965e996b0ce5"], + [128, "015bb052e72a4ca9"], + [255, "017b4012ee3a4d91"], + [256, "037b4012ee3a4d91"], + [257, "037b4012ee3a4d91"], + [258, "00486102fe224e09"], + [511, "00880302ffbac609"], + [512, "00880102ff3ac609"], + [513, "00880102ff3ac609"], + [100_000, "009213c27f3acc29"], + ]; + for (const [count, expected] of sketches) { + const canonical = Array.from({ length: count }, (_, i) => `word${i % 997}`).join(" "); + assert.equal(simhashHex(canonical), expected, `${count} tokens`); + } + const unicode = Array.from({ length: 1025 }, (_, i) => `中${i % 257}é`).join(" "); + assert.equal(simhashHex(unicode), "568a84089d1b5303"); + assert.equal(simhashHex(""), "1f2be47c811c9dc5"); + assert.equal(simhashHex("!!! 😀"), "74c4a446e2763713"); + assert.equal(simhashHex("唯一"), "50817b0bf9fa05fa"); +});