diff --git a/src/affinity-simhash.ts b/src/affinity-simhash.ts index e0197751d1..f1ea3c6f60 100644 --- a/src/affinity-simhash.ts +++ b/src/affinity-simhash.ts @@ -66,7 +66,7 @@ export function simhashHex(canonical: string): string { features.push(canonical); } const stride = Math.max(1, Math.ceil(features.length / MAX_FEATURES)); - const v = new Int8Array(64); + const v = new Int16Array(64); for (let f = 0; f < features.length; f += stride) { const h1 = fnv1a32(features[f]!, 2166136261); const h2 = fnv1a32(features[f]!, 2166136261 ^ 0x9e3779b9); diff --git a/tests/golden/storage-render/anthropic.json b/tests/golden/storage-render/anthropic.json index c734ff1760..558829d15d 100644 --- a/tests/golden/storage-render/anthropic.json +++ b/tests/golden/storage-render/anthropic.json @@ -95,13 +95,13 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "8155632a1688ca680f86c7d91515308e972d4529c7aa67c4a9eb6b3c08213acd", - "stateDigestAligned": "6063c5758188d0fe5fe20c7745dcf7d785d69361816ef394d53e2d417bb3c66d", + "stateDigest": "1f528f4fee4f59026d3b005f99a0489cadc139e54698896b2e5a0dadbc95ee2b", + "stateDigestAligned": "a796f61ccd006f5fa1573e1ef33bf756dc24bf017154fab8bde4c0b369516537", "sectionDigests": { "state": "56e66f6b72c7f0dd", "blockContents": "dc7d13d250c94bb4", "stats": "e83198a55e6b2c29", - "metadata": "66953226a6298cea" + "metadata": "65894bb8839e9947" }, "stateKeyDigests": { "blocks": "3a454b9a16e4f0b9", diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index 85977c9e9f..e0f1b4aa96 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -95,13 +95,13 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "e18d9aa4b3670d965882a269e8d74b2665782a7a7f3552504d6169e4f30c284d", - "stateDigestAligned": "0555eda4101ca76adfdaeceafccc1fc012e96e09dab558b6d1b0668215122318", + "stateDigest": "97b9991ae7a398fea5d620994df0101001d9c3f246615daac95e6afb4cee14a0", + "stateDigestAligned": "fc632ea30ecb84eeb10e2e0bb4d525f06b17ebd60c6296625d71f398b569ed0b", "sectionDigests": { "state": "90857fbf82001a2b", "blockContents": "94a58f96d7c5cfc3", "stats": "3e6858671f4b9a9d", - "metadata": "9c845ae9cb99892c" + "metadata": "faa2b34b71823646" }, "stateKeyDigests": { "blocks": "d9770feee91dafc6", diff --git a/tests/golden/storage-render/google.json b/tests/golden/storage-render/google.json index 1467c5e6a2..3dde0e8cdc 100644 --- a/tests/golden/storage-render/google.json +++ b/tests/golden/storage-render/google.json @@ -95,13 +95,13 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "7a02fb1a106f9b2251d946ca621db9f699b2c1b06277b61ac23777d4678d1dd6", - "stateDigestAligned": "a64451ed89c0cf4b922b18722904b922312eca1a439a1dc6ed68113038460a3f", + "stateDigest": "f8d07462fd8881896c5b52864e84ff9a1a79b0cdb73cd2ef87e16d361b5a07d3", + "stateDigestAligned": "219c42a0f17ed7c9061174627ca56e5b6733e7413d8af98953105bf5d60a40b7", "sectionDigests": { "state": "c5243ba17364d3ac", "blockContents": "473264d9df8ba338", "stats": "a3c84817f1d0c3dd", - "metadata": "499d829c84d80acb" + "metadata": "615f125909ed135d" }, "stateKeyDigests": { "blocks": "0cfd0b9c0fc53073", diff --git a/tests/golden/storage-render/responses.json b/tests/golden/storage-render/responses.json index 82056d7785..c9aa385f1a 100644 --- a/tests/golden/storage-render/responses.json +++ b/tests/golden/storage-render/responses.json @@ -95,13 +95,13 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "38e14e96f8f3c5da000a7ff677f5a46cb7fd661d5743cd7f5069eb9614fd8042", - "stateDigestAligned": "b99e670a86ef362b0b5915267b9a4047d5c1a769381279715dc6d5f40b47515b", + "stateDigest": "aa95de9ed8ddaff40f0b0e2deb1991c85582e33a17149c030687f46cfa34a9a3", + "stateDigestAligned": "5f40c3661d4c43338d855b5a5e79b2915da934affe33d6cb07680a1da39bb996", "sectionDigests": { "state": "e686a14f31d69ce5", "blockContents": "b6701268a0bdcd8a", "stats": "099bde0a9cf64cad", - "metadata": "757ba58e6230ec75" + "metadata": "be12181116331419" }, "stateKeyDigests": { "blocks": "34b0dc17f8c24595", diff --git a/tests/simhash-vote-range.test.ts b/tests/simhash-vote-range.test.ts new file mode 100644 index 0000000000..932d185826 --- /dev/null +++ b/tests/simhash-vote-range.test.ts @@ -0,0 +1,20 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { hammingHex, simhashHex } from "../src/affinity-simhash.ts"; + +test("SimHash retains repeated-shingle votes throughout the feature budget", () => { + for (const word of ["alpha", "部署"]) { + const expected = simhashHex(`${word} ${word}`); + for (const features of [1, 127, 128, 129, 255, 256, 257, 511, 512]) { + const actual = simhashHex(Array(features + 1).fill(word).join(" ")); + assert.equal(actual, expected, `${word}: ${features} shingles`); + } + } +}); + +test("unrelated repeated messages keep distinct sketches at the sampling cap", () => { + const first = simhashHex(Array(257).fill("alpha").join(" ")); + const second = simhashHex(Array(257).fill("omega").join(" ")); + assert.notEqual(first, second); + assert.equal(hammingHex(first, second), hammingHex(simhashHex("alpha alpha"), simhashHex("omega omega"))); +});