Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions scripts/bench-simhash-sampling.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
// Run with: node --max-old-space-size=512 --expose-gc --import tsx scripts/bench-simhash-sampling.mjs [tokens]
import { performance } from "node:perf_hooks";
import { simhashHex } from "../src/affinity-simhash.ts";

const count = Number(process.argv[2] ?? 1_000_000);
if (!Number.isSafeInteger(count) || count < 1) throw new Error("tokens must be a positive safe integer");
const canonical = Array.from({ length: count }, (_, i) => `word${i % 997}`).join(" ");
const timings = [];
let sketch;
for (let i = 0; i < 4; i++) {
global.gc?.();
const start = performance.now();
const result = simhashHex(canonical);
const elapsed = performance.now() - start;
if (sketch !== undefined && sketch !== result) throw new Error("unstable sketch");
sketch = result;
if (i > 0) timings.push(elapsed);
}
console.log(JSON.stringify({
count,
bytes: Buffer.byteLength(canonical),
sketch,
timings_ms: timings,
median_ms: [...timings].sort((a, b) => a - b)[1],
max_rss_kib: process.resourceUsage().maxRSS,
}));
23 changes: 8 additions & 15 deletions src/affinity-simhash.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,22 +54,15 @@ function fnv1a32(text: string, basis: number): number {
* so a short message still yields a stable, comparable sketch. */
export function simhashHex(canonical: string): string {
const tokens = canonical.match(/[\p{L}\p{N}_]+/gu) ?? [];
const features: string[] = [];
if (tokens.length < 2) {
for (const t of tokens) features.push(t);
} else {
for (let i = 0; i + 1 < tokens.length; i++) features.push(`${tokens[i]}\u0001${tokens[i + 1]}`);
}
if (features.length === 0) {
// No word tokens at all (pure punctuation/whitespace): hash the raw
// string so even that degenerate case is deterministic and comparable.
features.push(canonical);
}
const stride = Math.max(1, Math.ceil(features.length / MAX_FEATURES));
const featureCount = Math.max(1, tokens.length - 1);
const stride = Math.max(1, Math.ceil(featureCount / MAX_FEATURES));
const v = new Int16Array(64);
for (let f = 0; f < features.length; f += stride) {
const h1 = fnv1a32(features[f]!, 2166136261);
const h2 = fnv1a32(features[f]!, 2166136261 ^ 0x9e3779b9);
for (let f = 0; f < featureCount; f += stride) {
// Construct only sampled shingles, not an additional full-size array
// for a long message. No-token inputs still hash the raw string.
const feature = tokens.length < 2 ? (tokens[0] ?? canonical) : `${tokens[f]}\u0001${tokens[f + 1]}`;
const h1 = fnv1a32(feature, 2166136261);
const h2 = fnv1a32(feature, 2166136261 ^ 0x9e3779b9);
for (let b = 0; b < 64; b++) {
const bit = b < 32 ? (h1 >>> b) & 1 : (h2 >>> (b - 32)) & 1;
v[b] += bit === 1 ? 1 : -1;
Expand Down
26 changes: 26 additions & 0 deletions tests/simhash-vote-range.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,29 @@ test("unrelated repeated messages keep distinct sketches at the sampling cap", (
assert.notEqual(first, second);
assert.equal(hammingHex(first, second), hammingHex(simhashHex("alpha alpha"), simhashHex("omega omega")));
});

test("SimHash sampling preserves persisted sketches across stride boundaries", () => {
// Captured from the original full-shingle implementation. Distinct tokens
// make a shifted sample observable, unlike a repeated-shingle fixture.
const sketches: [number, string][] = [
[2, "896b965e996b0ce5"],
[128, "015bb052e72a4ca9"],
[255, "017b4012ee3a4d91"],
[256, "037b4012ee3a4d91"],
[257, "037b4012ee3a4d91"],
[258, "00486102fe224e09"],
[511, "00880302ffbac609"],
[512, "00880102ff3ac609"],
[513, "00880102ff3ac609"],
[100_000, "009213c27f3acc29"],
];
for (const [count, expected] of sketches) {
const canonical = Array.from({ length: count }, (_, i) => `word${i % 997}`).join(" ");
assert.equal(simhashHex(canonical), expected, `${count} tokens`);
}
const unicode = Array.from({ length: 1025 }, (_, i) => `中${i % 257}é`).join(" ");
assert.equal(simhashHex(unicode), "568a84089d1b5303");
assert.equal(simhashHex(""), "1f2be47c811c9dc5");
assert.equal(simhashHex("!!! 😀"), "74c4a446e2763713");
assert.equal(simhashHex("唯一"), "50817b0bf9fa05fa");
});
Loading