From e83fc6432b95f7a6eb80e44d92c4151e9c726ce6 Mon Sep 17 00:00:00 2001 From: ossa Date: Sun, 25 Oct 2026 13:59:34 +0300 Subject: [PATCH 1/8] =?UTF-8?q?fix(storage):=20digest=20object=20keys=20in?= =?UTF-8?q?=20sorted=20order=20=E2=80=94=20single-instance=20shadow=20drif?= =?UTF-8?q?t=20(#4392)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit shadowDigest hashes JSON.stringify(canonicalize(view)) and canonicalize walked Object.entries in insertion order, so two views holding equal values hashed apart whenever their key order differed. The verify legs really do differ: the legacy load leg carries state.messageRefs.byRef in kernel mint order while loadLegacyView rebuilds it from the refs ledger in ord order (ingest.ts:530). One process therefore logged [diverged: state] sample after sample and escalated into freezing its own JSON envelope writes. canonicalize now sorts object keys at every depth — arrays keep their order and Map/Set keep their __map__/__set__ form. The ESCALATED wording no longer asserts MULTIPLE instances: both legs are built in this process, so a digest shape mismatch is the first suspect and a second writer is one cause among others (#394, #4392). Pins move because canonicalize is what they hash. storage-render goldens were re-generated with scripts/update-storage-render-goldens.ts; the storage-corpus manifest digests were re-pinned from the untouched fixtures, because that script also rewrites every fixture with newer buildRecord fields — churn unrelated to the digest contract, left out of this PR. tests/issue4392-canonical-key-order.test.ts fails before the fix (2 of its 4 cases) and passes after. CHANGELOG records the single-instance shape of the drift. --- CHANGELOG.md | 1 + src/storage/canonical.ts | 17 +- src/storage/shadow.ts | 18 +- tests/golden/storage-corpus/manifest.json | 158 ++++++++-------- tests/golden/storage-render/anthropic.json | 4 +- tests/golden/storage-render/chat.json | 4 +- tests/golden/storage-render/google.json | 4 +- tests/golden/storage-render/responses.json | 4 +- tests/issue4392-canonical-key-order.test.ts | 192 ++++++++++++++++++++ 9 files changed, 306 insertions(+), 96 deletions(-) create mode 100644 tests/issue4392-canonical-key-order.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 220781101a..459c08cab4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag ## [Unreleased] ### Fixed +- **shadow 摘要漂移的单实例成因(#4392)**:`canonicalize` 曾按 `Object.entries` 插入序参与哈希,而 live 腿(kernel 状态,`byRef` 按发号顺序)与 replay 腿(`loadLegacyView` 按账本 `ord` 重建 `messageRefs`)对同一份等值状态给出不同键序 → 单进程自我报告 `[diverged: state]`,连续 3 次即冻结本进程的信封写入并清掉入库水位。现在任意深度按键排序(数组保序,Map/Set 形态不变),升级文案不再断言「多个 bili 实例」;storage-corpus manifest 与 storage-render 金标随摘要定义重钉(仅摘要字段变化,渲染与入库语义不动)。 - 块区间边界在重存时不再回退:折叠后客户端重发的视图不含已覆盖原文,重入库时该批次无法重新解析区间(-1/-1),旧逻辑会把已解析的真实区间抹掉。现在区间证据一旦解析即持久(入库 `CASE` 保留;sqlite/mysql 双后端),未知块保持诚实缺口(#4608)。 ### Added diff --git a/src/storage/canonical.ts b/src/storage/canonical.ts index 4d8b449e7e..719bbd349a 100644 --- a/src/storage/canonical.ts +++ b/src/storage/canonical.ts @@ -8,6 +8,13 @@ import { createHash } from "node:crypto"; * corpus manifests and shadow checks agree byte-for-byte. * * Determinism rules, in order: + * - Object keys → SORTED at every depth (#4392): insertion order is not + * part of the contract. The live leg (kernel state) and the replay leg + * (unified rows rebuilt field by field) legitimately build the same + * values in different orders — messageRefs.byRef after a ref-ledger + * re-alignment, mergeState's spread-merged nudge/stats — so hashing + * insertion order made a single instance report drift against itself. + * Arrays keep their order (order IS content there). * - Map/Set → tagged JSON-able envelopes (key order = insertion order; the * producers below insert in deterministic order). * - Wall-clock timestamps (> 2020-09-01 in ms) → "__TS__" — savedAt, @@ -38,7 +45,15 @@ function canonicalizeWithContext(value: unknown, key: string | undefined): unkno if (value instanceof Set) return { __set__: [...value.values()].map((v) => canonicalizeWithContext(v, undefined)) }; if (value && typeof value === "object") { const out: Record = {}; - for (const [k, v] of Object.entries(value)) { + const record = value as Record; + // Keys digest in SORTED order (#4392): insertion order is not part of + // the persistence contract. The live leg carries the kernel's own + // object order while the replay leg rebuilds the same values field by + // field from unified rows (messageRefs.byRef after a ledger + // re-alignment, mergeState's spread-merged nudge/stats), so hashing + // insertion order reported "[diverged: state]" against itself. + for (const k of Object.keys(record).sort()) { + const v = record[k]; // storageMode is last-writer provenance (#2671): the corpus pins // predate the stamp, and a re-save legitimately (re)writes it — // unlike biliVersion its PRESENCE is new, so masking the value diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index d527be96c4..3d5a94954b 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -391,13 +391,15 @@ export class UnifiedShadow { this.driftedSessions.add(legacy.id); // #4323: count CONSECUTIVE sampled drifts — membership alone // caps at "repeat yes/no" and cannot distinguish 3+ samples. - // N in a row proves structural divergence (multi-writer - // #394): escalate ONCE at the threshold with a loud WARN - // naming both sides' fold counts, then freeze THIS - // process's envelope writes for the session so the ping- - // pong stops churning the file copy (persist.ts consults - // envelopeFrozen on every save route; the unified store - // keeps receiving every save either way). + // N in a row marks a STRUCTURAL divergence: the watermark + // clear healed nothing. Note the drift can be a single- + // instance artifact (#4392 — digest shape), not only the + // multi-writer #394 case; escalate ONCE at the threshold + // with a loud WARN naming both sides' fold counts, then + // freeze THIS process's envelope writes for the session so + // the ping-pong stops churning the file copy (persist.ts + // consults envelopeFrozen on every save route; the unified + // store keeps receiving every save either way). const streak = (this.driftStreaks.get(legacy.id) ?? 0) + 1; this.driftStreaks.set(legacy.id, streak); if (streak === DRIFT_ESCALATE_STREAK) { @@ -410,7 +412,7 @@ export class UnifiedShadow { } catch { // counts must never break the heal path } - loggerLog("warn", `[storage] DIGEST DRIFT ESCALATED for ${legacy.id}: ${streak} consecutive sampled drifts — the views have stopped converging${sides}. This usually means MULTIPLE bili instances share one sessions dir (#394 anti-pattern; second-instance detection: #4177). Freezing THIS process's JSON envelope writes for the session until a clean sample re-admits it — the unified store keeps receiving every save (#4323)`); + loggerLog("warn", `[storage] DIGEST DRIFT ESCALATED for ${legacy.id}: ${streak} consecutive sampled drifts — the views have stopped converging${sides}. Both legs are built in THIS process, so a persistent gap is a digest-shape mismatch first (field only one leg builds, ordering) and only then a second writer: a second bili instance sharing one sessions dir (#394 anti-pattern; second-instance detection: #4177) is one cause among others (#4392). Freezing THIS process's JSON envelope writes for the session until a clean sample re-admits it — the unified store keeps receiving every save (#4323)`); } // Panels consume {disabled, lastError} — make the sampled // degradation visible there, not just in the counter/log. diff --git a/tests/golden/storage-corpus/manifest.json b/tests/golden/storage-corpus/manifest.json index bb9d16aaf7..d6e6165e62 100644 --- a/tests/golden/storage-corpus/manifest.json +++ b/tests/golden/storage-corpus/manifest.json @@ -1,82 +1,82 @@ { - "folded": { - "description": "active T1 block, blockContents one:null, folded message tail", - "files": [ - "anthropic/127.0.0.1_29e4587af02d1ed9ae4b0054.json" - ], - "digest": "8485c8f69693cca987ec3c612bdc31fe429e70fe0fade5e05b9db936f0418311" - }, - "tier2": { - "description": "superseded T1 + active T2, distinct `one` views, byte-identical pair pins #401 normalize-on-load", - "files": [ - "chat/127.0.0.1_8453ec7a56646bc014ac0d88.json" - ], - "digest": "0cfdba77bc1a052c0c3b8170874c4d963273b6580c822c6c9a5ee8f0f3d09b43", - "notes": { - "t2Block": "b3" + "folded": { + "description": "active T1 block, blockContents one:null, folded message tail", + "files": [ + "anthropic/127.0.0.1_29e4587af02d1ed9ae4b0054.json" + ], + "digest": "3026f3d77e59d5076b712faf9599da05f81d84fd997ac6e6631ff906f4f701c3" + }, + "tier2": { + "description": "superseded T1 + active T2, distinct `one` views, byte-identical pair pins #401 normalize-on-load", + "files": [ + "chat/127.0.0.1_8453ec7a56646bc014ac0d88.json" + ], + "digest": "01c4e936c7ab078ae8908acf048ccfccf2ce6896894cc6f33a6174d6ce62fe57", + "notes": { + "t2Block": "b3" + } + }, + "deadrefs": { + "description": "deadRefs tombstones + ghost raw ids after client edit (#2362/#2370)", + "files": [ + "anthropic/127.0.0.1_e17d4db40407d10867daeb1d.json" + ], + "digest": "44aecbb906a425dce328f27b1966af773dcaa7033b2b724af076ed5e4e3bee35", + "notes": { + "ghostRef": "m00010" + } + }, + "fork-receipt": { + "description": "publicForkReceipt: pluginSnapshot + forkContentStore + publicSnapshotStoredRefs union", + "files": [ + "chat/127.0.0.1_bde31987d03ca1ebaeb96ba0.json" + ], + "digest": "5737b5edc484bdd27fedb921a3ffc2b9f6f8334104287e73dcba5f8c3bb57b20" + }, + "fork-retained": { + "description": "publicSnapshotRetained only — pluginSnapshot persisted, forkContentStore DROPPED (pins the #2675 ② asymmetry verbatim)", + "files": [ + "chat/127.0.0.1_dec59ddccc0b4b89ff346c9b.json" + ], + "digest": "d7894c369566ec22d6e6a581fa7de9e5f9ad88b09aa6617e0385318c4dde8e6d", + "notes": { + "expectContentStoreOnLoad": "absent" + } + }, + "image": { + "description": "base64 image message with pre-#3266 legacy image keys on disk (dropped on load)", + "files": [ + "anthropic/127.0.0.1_b139318d5755710735cbf443.json" + ], + "digest": "a65628e52318b5044e5007e1e8055c4a273b0ff4e6be8ed18dca4525db491ced" + }, + "ccr": { + "description": "content-store.json sibling with two CCR originals (#1097 envelope)", + "files": [ + "responses/127.0.0.1_0787df32f3cf98f922f7c0f0.content-store.json", + "responses/127.0.0.1_0787df32f3cf98f922f7c0f0.json" + ], + "digest": "20d534a2774be30e4f19dccdcb66d6af80b4c7ba2c06f855763954bf8d0c9f72" + }, + "twinfile": { + "description": "namespaced v2 record + stale FLAT v1 twin for the same id (#2671 ① twin-namespace)", + "files": [ + "6fe5d8f2afee3e83e5360fa8.json", + "openai/127.0.0.1_6fe5d8f2afee3e83e5360fa8.json" + ], + "digest": "0cab4c22cf71ccd9d91402c3de1006bc2b40bd56aee1abead63f26c798187535", + "notes": { + "flatTwinRequests": 1, + "namespacedRequests": 42, + "flatFallbackDigest": "64820a36fdf7ecf4b2f0e053dae33ac5c53df867f067657340a4cec064d804d8", + "flatFallbackRequests": 1 + } + }, + "zstd": { + "description": "BILIZSTD1-framed write of the `folded` session — must load to the IDENTICAL digest", + "files": [ + "anthropic/127.0.0.1_29e4587af02d1ed9ae4b0054.json" + ], + "digest": "3026f3d77e59d5076b712faf9599da05f81d84fd997ac6e6631ff906f4f701c3" } - }, - "deadrefs": { - "description": "deadRefs tombstones + ghost raw ids after client edit (#2362/#2370)", - "files": [ - "anthropic/127.0.0.1_e17d4db40407d10867daeb1d.json" - ], - "digest": "de057f87e0737222362d44326ec9e139457d50d67566ffaf5a7bb590b549c12d", - "notes": { - "ghostRef": "m00010" - } - }, - "fork-receipt": { - "description": "publicForkReceipt: pluginSnapshot + forkContentStore + publicSnapshotStoredRefs union", - "files": [ - "chat/127.0.0.1_bde31987d03ca1ebaeb96ba0.json" - ], - "digest": "edf7ac9a0cb912acc39b17667dfd1245a24b6bf6f01b57f510f782258e3e3bcb" - }, - "fork-retained": { - "description": "publicSnapshotRetained only — pluginSnapshot persisted, forkContentStore DROPPED (pins the #2675 ② asymmetry verbatim)", - "files": [ - "chat/127.0.0.1_dec59ddccc0b4b89ff346c9b.json" - ], - "digest": "63328d62ba50899927adda2154185ec83b9071ca8a84611dec1c57d5d4a51a8e", - "notes": { - "expectContentStoreOnLoad": "absent" - } - }, - "image": { - "description": "base64 image message with pre-#3266 legacy image keys on disk (dropped on load)", - "files": [ - "anthropic/127.0.0.1_b139318d5755710735cbf443.json" - ], - "digest": "deec17f777d11294316b2250160b0577756b2e686ed0f2f1a6441ed06a581dc2" - }, - "ccr": { - "description": "content-store.json sibling with two CCR originals (#1097 envelope)", - "files": [ - "responses/127.0.0.1_0787df32f3cf98f922f7c0f0.content-store.json", - "responses/127.0.0.1_0787df32f3cf98f922f7c0f0.json" - ], - "digest": "2d15c631cf8ee6fd406fb4354db508bed86f11ecaefcb7c9b07c48ab3a7493d8" - }, - "twinfile": { - "description": "namespaced v2 record + stale FLAT v1 twin for the same id (#2671 ① twin-namespace)", - "files": [ - "6fe5d8f2afee3e83e5360fa8.json", - "openai/127.0.0.1_6fe5d8f2afee3e83e5360fa8.json" - ], - "digest": "fe94b9453f3e7e2b57831965fbf87ff2604efdd11e74ecec612cbefd82e0e122", - "notes": { - "flatTwinRequests": 1, - "namespacedRequests": 42, - "flatFallbackDigest": "73ae30ed8e603bd51978f2d891acb6aada8c4769e05acbbfdd2a64d4e1f89e57", - "flatFallbackRequests": 1 - } - }, - "zstd": { - "description": "BILIZSTD1-framed write of the `folded` session — must load to the IDENTICAL digest", - "files": [ - "anthropic/127.0.0.1_29e4587af02d1ed9ae4b0054.json" - ], - "digest": "8485c8f69693cca987ec3c612bdc31fe429e70fe0fade5e05b9db936f0418311" - } } diff --git a/tests/golden/storage-render/anthropic.json b/tests/golden/storage-render/anthropic.json index 5afa14b2c4..13811dcab0 100644 --- a/tests/golden/storage-render/anthropic.json +++ b/tests/golden/storage-render/anthropic.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "6727f42bd91879a7aa370e1ce0c78963bcba0df92e50390d4a8011f8f0da334d", + "stateDigest": "0a241435d456ba994c92b0072d2173bf2a6d00ee73223458fb4302e170efb1e3", "sectionDigests": { "state": "f93dc1c876c672d4", "blockContents": "dc7d13d250c94bb4", "stats": "c6f1d434975d047d", - "metadata": "f1ec4aab3aff777a" + "metadata": "16cad648ccf41950" }, "stateKeyDigests": { "blocks": "3a454b9a16e4f0b9", diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index 46764f97db..01f69b1036 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "e149c301f382438ee063a38f70dd794207b88e94bdf3c14c5191da0eeaa99bc9", + "stateDigest": "9f1b8258a916fcf63af14625def9ed0236eb8ec9c542f0027206985598d2a5b8", "sectionDigests": { "state": "8390f21336885863", "blockContents": "94a58f96d7c5cfc3", "stats": "24d705b0e39bd13c", - "metadata": "55733fb81ccf7d69" + "metadata": "2198f47054c8acae" }, "stateKeyDigests": { "blocks": "d9770feee91dafc6", diff --git a/tests/golden/storage-render/google.json b/tests/golden/storage-render/google.json index ba999cadec..fe9f9c95dd 100644 --- a/tests/golden/storage-render/google.json +++ b/tests/golden/storage-render/google.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "80e83935e5fa7b052a3cb0f1d641bd81cb6ee59f91f3c1f2f31c204de29368a5", + "stateDigest": "c4979d4f58cfa49bc8be8c43b8e1f1b0861b3b9d44cdc415ee09e4ca21d9aa5a", "sectionDigests": { "state": "2e1def3a3234afdc", "blockContents": "473264d9df8ba338", "stats": "900481685c848462", - "metadata": "88cd02a467b4457e" + "metadata": "939dc0c09b6cbcbf" }, "stateKeyDigests": { "blocks": "0cfd0b9c0fc53073", diff --git a/tests/golden/storage-render/responses.json b/tests/golden/storage-render/responses.json index 7004d90435..0402e7d43c 100644 --- a/tests/golden/storage-render/responses.json +++ b/tests/golden/storage-render/responses.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "08526d3552fca7ac7552cd16a221893a652fbe740c9b8dccac6db57b3d6302ee", + "stateDigest": "b3bddfa75ce3f8b94424be9a0164c1b9a183faa9daf705e6175cabdd671c95ff", "sectionDigests": { "state": "db313e8eba2d65fc", "blockContents": "b6701268a0bdcd8a", "stats": "327d3967b8371aa8", - "metadata": "b5c39d84db67f424" + "metadata": "a9eecd3816738158" }, "stateKeyDigests": { "blocks": "34b0dc17f8c24595", diff --git a/tests/issue4392-canonical-key-order.test.ts b/tests/issue4392-canonical-key-order.test.ts new file mode 100644 index 0000000000..c271fcb2b4 --- /dev/null +++ b/tests/issue4392-canonical-key-order.test.ts @@ -0,0 +1,192 @@ +/** + * #4392: "[diverged: state]" digest drift on a SINGLE instance — canonicalize + * must digest object key SETS, not insertion order. + * + * The two shadow verify legs build the same values in different orders. Field + * evidence (v0.1.206, one proxy process): the expect leg loads the envelope + * through buildSession → mergeState, whose `messageRefs` is taken off the + * persisted state verbatim, while the got leg rebuilds `messageRefs.byRef` from + * the refs ledger row order (`loadLegacyView`). The kernel mints refs in a + * different order than the byRaw keys the ledger is written in, so byRef came + * back reordered with identical contents — `shadowDigest` hashed the order and + * reported drift against itself, escalating to "MULTIPLE bili instances" + * (#4323) after three samples and freezing envelope writes. + * + * Post-fix `canonicalize` sorts object keys at every depth (Maps/Sets keep + * their current canonical form, arrays keep their order), so only a genuine + * value/set difference trips the digest. Both directions are asserted: the + * reordered-but-equal case stays green, a real content difference still drifts. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { UnifiedStore } from "../src/storage/store.ts"; +import { UnifiedShadow } from "../src/storage/shadow.ts"; +import { loadLegacyView, planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { canonicalize, sha256 } from "../src/storage/canonical.ts"; +import type { CoreMessage } from "acp-kernel"; + +process.env.BILI_PERSIST_ZSTD = "0"; +const hermetic = fs.mkdtempSync(path.join(os.tmpdir(), "bili-canonical-order-hermetic-")); +process.env.BILI_CONFIG_FILE = path.join(hermetic, "no-config.json"); +process.env.BILI_STORAGE_UNIFIED = "legacy"; + +function tmpdir(prefix: string): string { + return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); +} + +function rmBestEffort(p: string): void { + try { + fs.rmSync(p, { recursive: true, force: true }); + } catch { + // best effort: a locked sqlite file must not fail the assertion path + } +} + +function msg(i: number): CoreMessage { + return { id: `h_msg${i}`, role: i % 2 === 0 ? "user" : "assistant", contentType: "text", text: `hello ${i}` } as CoreMessage; +} + +// Above canonicalize's > 2020-09-01 mask threshold so createdAt / lastSeen +// collapse to __TS__ on BOTH digest legs. +const T_CREATED = 1_700_000_000_000; +const T_SEEN = 1_700_000_000_500; + +interface StateShape { + blocks: unknown[]; + messageRefs: { byRaw: Record; byRef: Record }; + nextBlockId: number; + nextRunId: number; +} + +function mkLegacy(id: string, state: StateShape): LegacySessionLike { + return { + id, + createdAt: T_CREATED, + lastSeen: T_SEEN, + meta: { protocol: "anthropic" }, + stats: { requests: 1 }, + metadata: {}, + state: state as unknown as LegacySessionLike["state"], + blockContents: new Map(), + lastMessages: [msg(0), msg(1)], + lastMessagesFolded: false, + }; +} + +interface StatsView { + ingests: number; + deltaIngests: number; + verifySamples: number; + verifyOk: number; + verifyDrift: number; +} + +function openShadow(dir: string): UnifiedShadow { + UnifiedStore.open(path.join(dir, "index.db")); + const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as LegacySessionLike); + assert.ok(shadow, "shadow mode sidecar must open"); + return shadow; +} + +function stats(shadow: UnifiedShadow): StatsView { + return (shadow as unknown as { stats: StatsView }).stats; +} + +// --- Unit: canonicalize is order-insensitive for objects, order-preserving for arrays --- + +test("#4392: canonicalize digests object key sets, not insertion order", () => { + assert.equal( + JSON.stringify(canonicalize({ state: { messageRefs: { byRef: { m00002: "b", m00001: "a" } } }, stats: { requests: 1 } })), + JSON.stringify(canonicalize({ stats: { requests: 1 }, state: { messageRefs: { byRef: { m00001: "a", m00002: "b" } } } })), + "the same key set in a different order must digest identically", + ); + assert.equal(sha256(JSON.stringify(canonicalize({ b: 1, a: 2 }))), sha256(JSON.stringify(canonicalize({ a: 2, b: 1 })))); + assert.notEqual(sha256(JSON.stringify(canonicalize({ a: 1 }))), sha256(JSON.stringify(canonicalize({ a: 2 }))), "values still matter"); + assert.notEqual(sha256(JSON.stringify(canonicalize({ a: 1 }))), sha256(JSON.stringify(canonicalize({ a: 1, b: 2 }))), "the key SET still matters"); +}); + +test("#4392: arrays keep their order; Maps/Sets keep their canonical envelope", () => { + assert.notEqual( + sha256(JSON.stringify(canonicalize([{ id: "b1" }, { id: "b2" }]))), + sha256(JSON.stringify(canonicalize([{ id: "b2" }, { id: "b1" }]))), + "array order is content — blocks/tail order must still trip the digest", + ); + assert.notEqual( + sha256(JSON.stringify(canonicalize(new Set(["a", "b"])))), + sha256(JSON.stringify(canonicalize(new Set(["b", "a"])))), + "Set insertion order stays part of the canonical form (nothing changed there)", + ); + assert.equal( + JSON.stringify(canonicalize(new Map([["b", 1]]))), + JSON.stringify({ __map__: [["b", 1]] }), + "Map envelope shape unchanged", + ); +}); + +// --- E2E: the field shape — byRef rebuilt in ledger order, same contents --- + +test("#4392: byRef reordered by the ref-ledger rebuild must not trip the parity digest", () => { + const dir = tmpdir("bili-4392-order-"); + try { + // The kernel minted m00001 before m00002, but byRaw's insertion order + // (what the refs ledger is written in) starts at raw-a — so the replay + // leg rebuilds byRef { m00001, m00002 } while the live view holds + // { m00002, m00001 }: identical key set, identical values. + const liveState: StateShape = { + blocks: [], + messageRefs: { + byRaw: { "raw-a": "m00001", "raw-b": "m00002" }, + byRef: { m00002: "raw-b", m00001: "raw-a" }, + }, + nextBlockId: 1, + nextRunId: 1, + }; + const legacy = mkLegacy("state-order", liveState); + const replayed = loadLegacyView(planIngest(legacy) as never); + // Unchecked cast: loadLegacyView types state as Record + // (the view is store-shaped, not kernel-typed); this fixture owns it. + const replayedState = replayed.state as unknown as StateShape; + const liveRefs = liveState.messageRefs.byRef; + const replayRefs = replayedState.messageRefs.byRef; + assert.deepEqual(Object.keys(liveRefs), ["m00002", "m00001"], "live leg holds byRef in kernel-mint order"); + assert.deepEqual(Object.keys(replayRefs), ["m00001", "m00002"], "replay leg holds byRef in ledger order"); + assert.deepEqual({ ...liveRefs }, { ...replayRefs }, "…with identical contents"); + + const shadow = openShadow(dir); + shadow.record(legacy); + const s = stats(shadow); + assert.equal(s.verifySamples, 1, "first save must run the sampled verify"); + assert.equal(s.verifyDrift, 0, "an order-only difference must not drift (pre-fix: 1)"); + assert.equal(s.verifyOk, 1, "the sampled verify must pass"); + } finally { + rmBestEffort(dir); + } +}); + +test("#4392: a real messageRefs content difference still drifts", () => { + const dir = tmpdir("bili-4392-content-"); + try { + const shadow = openShadow(dir); + // Same key set and order, one different mapping: the digest must still + // catch a genuine fork between the two legs (the #4323 heal path is + // what depends on this direction). + const legacy = mkLegacy("state-content", { + blocks: [], + messageRefs: { + byRaw: { "raw-a": "m00001", "raw-b": "m00002" }, + byRef: { m00001: "raw-a", m00002: "raw-c" }, + }, + nextBlockId: 1, + nextRunId: 1, + }); + shadow.record(legacy); + const s = stats(shadow); + assert.equal(s.verifySamples, 1, "first save must run the sampled verify"); + assert.equal(s.verifyDrift, 1, "a divergent ref mapping must still be reported"); + } finally { + rmBestEffort(dir); + } +}); From d620095fdf85f5a842c82eb88f3d21dd550015ba Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 25 Oct 2026 21:00:44 +0800 Subject: [PATCH 2/8] test: restore storage-corpus manifest indent to generator format The regenerated manifest was rewritten with 4-space indentation while the generator scripts/update-storage-corpus.ts:467 writes JSON.stringify(x,null,2), so every regeneration re-churns all 158 lines and buries the real change in the diff. Restored the original key order + 2-space layout, keeping only the new digest values (deep-diff verified: exactly the 10 digest fields differ). --- tests/golden/storage-corpus/manifest.json | 158 +++++++++++----------- 1 file changed, 79 insertions(+), 79 deletions(-) diff --git a/tests/golden/storage-corpus/manifest.json b/tests/golden/storage-corpus/manifest.json index d6e6165e62..8f672243cc 100644 --- a/tests/golden/storage-corpus/manifest.json +++ b/tests/golden/storage-corpus/manifest.json @@ -1,82 +1,82 @@ { - "folded": { - "description": "active T1 block, blockContents one:null, folded message tail", - "files": [ - "anthropic/127.0.0.1_29e4587af02d1ed9ae4b0054.json" - ], - "digest": "3026f3d77e59d5076b712faf9599da05f81d84fd997ac6e6631ff906f4f701c3" - }, - "tier2": { - "description": "superseded T1 + active T2, distinct `one` views, byte-identical pair pins #401 normalize-on-load", - "files": [ - "chat/127.0.0.1_8453ec7a56646bc014ac0d88.json" - ], - "digest": "01c4e936c7ab078ae8908acf048ccfccf2ce6896894cc6f33a6174d6ce62fe57", - "notes": { - "t2Block": "b3" - } - }, - "deadrefs": { - "description": "deadRefs tombstones + ghost raw ids after client edit (#2362/#2370)", - "files": [ - "anthropic/127.0.0.1_e17d4db40407d10867daeb1d.json" - ], - "digest": "44aecbb906a425dce328f27b1966af773dcaa7033b2b724af076ed5e4e3bee35", - "notes": { - "ghostRef": "m00010" - } - }, - "fork-receipt": { - "description": "publicForkReceipt: pluginSnapshot + forkContentStore + publicSnapshotStoredRefs union", - "files": [ - "chat/127.0.0.1_bde31987d03ca1ebaeb96ba0.json" - ], - "digest": "5737b5edc484bdd27fedb921a3ffc2b9f6f8334104287e73dcba5f8c3bb57b20" - }, - "fork-retained": { - "description": "publicSnapshotRetained only — pluginSnapshot persisted, forkContentStore DROPPED (pins the #2675 ② asymmetry verbatim)", - "files": [ - "chat/127.0.0.1_dec59ddccc0b4b89ff346c9b.json" - ], - "digest": "d7894c369566ec22d6e6a581fa7de9e5f9ad88b09aa6617e0385318c4dde8e6d", - "notes": { - "expectContentStoreOnLoad": "absent" - } - }, - "image": { - "description": "base64 image message with pre-#3266 legacy image keys on disk (dropped on load)", - "files": [ - "anthropic/127.0.0.1_b139318d5755710735cbf443.json" - ], - "digest": "a65628e52318b5044e5007e1e8055c4a273b0ff4e6be8ed18dca4525db491ced" - }, - "ccr": { - "description": "content-store.json sibling with two CCR originals (#1097 envelope)", - "files": [ - "responses/127.0.0.1_0787df32f3cf98f922f7c0f0.content-store.json", - "responses/127.0.0.1_0787df32f3cf98f922f7c0f0.json" - ], - "digest": "20d534a2774be30e4f19dccdcb66d6af80b4c7ba2c06f855763954bf8d0c9f72" - }, - "twinfile": { - "description": "namespaced v2 record + stale FLAT v1 twin for the same id (#2671 ① twin-namespace)", - "files": [ - "6fe5d8f2afee3e83e5360fa8.json", - "openai/127.0.0.1_6fe5d8f2afee3e83e5360fa8.json" - ], - "digest": "0cab4c22cf71ccd9d91402c3de1006bc2b40bd56aee1abead63f26c798187535", - "notes": { - "flatTwinRequests": 1, - "namespacedRequests": 42, - "flatFallbackDigest": "64820a36fdf7ecf4b2f0e053dae33ac5c53df867f067657340a4cec064d804d8", - "flatFallbackRequests": 1 - } - }, - "zstd": { - "description": "BILIZSTD1-framed write of the `folded` session — must load to the IDENTICAL digest", - "files": [ - "anthropic/127.0.0.1_29e4587af02d1ed9ae4b0054.json" - ], - "digest": "3026f3d77e59d5076b712faf9599da05f81d84fd997ac6e6631ff906f4f701c3" + "folded": { + "description": "active T1 block, blockContents one:null, folded message tail", + "files": [ + "anthropic/127.0.0.1_29e4587af02d1ed9ae4b0054.json" + ], + "digest": "3026f3d77e59d5076b712faf9599da05f81d84fd997ac6e6631ff906f4f701c3" + }, + "tier2": { + "description": "superseded T1 + active T2, distinct `one` views, byte-identical pair pins #401 normalize-on-load", + "files": [ + "chat/127.0.0.1_8453ec7a56646bc014ac0d88.json" + ], + "digest": "01c4e936c7ab078ae8908acf048ccfccf2ce6896894cc6f33a6174d6ce62fe57", + "notes": { + "t2Block": "b3" } + }, + "deadrefs": { + "description": "deadRefs tombstones + ghost raw ids after client edit (#2362/#2370)", + "files": [ + "anthropic/127.0.0.1_e17d4db40407d10867daeb1d.json" + ], + "digest": "44aecbb906a425dce328f27b1966af773dcaa7033b2b724af076ed5e4e3bee35", + "notes": { + "ghostRef": "m00010" + } + }, + "fork-receipt": { + "description": "publicForkReceipt: pluginSnapshot + forkContentStore + publicSnapshotStoredRefs union", + "files": [ + "chat/127.0.0.1_bde31987d03ca1ebaeb96ba0.json" + ], + "digest": "5737b5edc484bdd27fedb921a3ffc2b9f6f8334104287e73dcba5f8c3bb57b20" + }, + "fork-retained": { + "description": "publicSnapshotRetained only — pluginSnapshot persisted, forkContentStore DROPPED (pins the #2675 ② asymmetry verbatim)", + "files": [ + "chat/127.0.0.1_dec59ddccc0b4b89ff346c9b.json" + ], + "digest": "d7894c369566ec22d6e6a581fa7de9e5f9ad88b09aa6617e0385318c4dde8e6d", + "notes": { + "expectContentStoreOnLoad": "absent" + } + }, + "image": { + "description": "base64 image message with pre-#3266 legacy image keys on disk (dropped on load)", + "files": [ + "anthropic/127.0.0.1_b139318d5755710735cbf443.json" + ], + "digest": "a65628e52318b5044e5007e1e8055c4a273b0ff4e6be8ed18dca4525db491ced" + }, + "ccr": { + "description": "content-store.json sibling with two CCR originals (#1097 envelope)", + "files": [ + "responses/127.0.0.1_0787df32f3cf98f922f7c0f0.content-store.json", + "responses/127.0.0.1_0787df32f3cf98f922f7c0f0.json" + ], + "digest": "20d534a2774be30e4f19dccdcb66d6af80b4c7ba2c06f855763954bf8d0c9f72" + }, + "twinfile": { + "description": "namespaced v2 record + stale FLAT v1 twin for the same id (#2671 ① twin-namespace)", + "files": [ + "6fe5d8f2afee3e83e5360fa8.json", + "openai/127.0.0.1_6fe5d8f2afee3e83e5360fa8.json" + ], + "digest": "0cab4c22cf71ccd9d91402c3de1006bc2b40bd56aee1abead63f26c798187535", + "notes": { + "flatTwinRequests": 1, + "namespacedRequests": 42, + "flatFallbackDigest": "64820a36fdf7ecf4b2f0e053dae33ac5c53df867f067657340a4cec064d804d8", + "flatFallbackRequests": 1 + } + }, + "zstd": { + "description": "BILIZSTD1-framed write of the `folded` session — must load to the IDENTICAL digest", + "files": [ + "anthropic/127.0.0.1_29e4587af02d1ed9ae4b0054.json" + ], + "digest": "3026f3d77e59d5076b712faf9599da05f81d84fd997ac6e6631ff906f4f701c3" + } } From 2a4c05b452b8816f7c6aff78637794ac7e605290 Mon Sep 17 00:00:00 2001 From: mz Date: Tue, 27 Oct 2026 18:36:44 +0300 Subject: [PATCH 3/8] fix(storage): digest object keys in sorted order with one shared skip table (#4392) canonicalize hashed insertion order, so the shadow live leg and the replay leg (messageRefs.byRef rebuilt in ledger order by loadLegacyView) produced two digests for the same value on a SINGLE instance, escalated after three samples and froze envelope writes. - sort object keys at every depth; arrays keep order, Map/Set envelopes keep their shape - move the storageMode/tailGaps drops into one exported table DIGEST_SKIPPED_KEYS so the streamed canonicalDigest (#4532) imports the same skip set instead of hand-copying a second list - pin the table literally and pin depth-independent dropping --- CHANGELOG.md | 2 +- src/storage/canonical.ts | 47 ++++++++++++--------- tests/golden/storage-corpus/manifest.json | 20 ++++----- tests/issue4392-canonical-key-order.test.ts | 22 +++++++++- 4 files changed, 59 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 459c08cab4..5133fd937a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag ## [Unreleased] ### Fixed -- **shadow 摘要漂移的单实例成因(#4392)**:`canonicalize` 曾按 `Object.entries` 插入序参与哈希,而 live 腿(kernel 状态,`byRef` 按发号顺序)与 replay 腿(`loadLegacyView` 按账本 `ord` 重建 `messageRefs`)对同一份等值状态给出不同键序 → 单进程自我报告 `[diverged: state]`,连续 3 次即冻结本进程的信封写入并清掉入库水位。现在任意深度按键排序(数组保序,Map/Set 形态不变),升级文案不再断言「多个 bili 实例」;storage-corpus manifest 与 storage-render 金标随摘要定义重钉(仅摘要字段变化,渲染与入库语义不动)。 +- **shadow 摘要漂移的单实例成因(#4392)**:`canonicalize` 曾按 `Object.entries` 插入序参与哈希,而 live 腿(kernel 状态,`byRef` 按发号顺序)与 replay 腿(`loadLegacyView` 按账本 `ord` 重建 `messageRefs`)对同一份等值状态给出不同键序 → 单进程自我报告 `[diverged: state]`,连续 3 次即冻结本进程的信封写入并清掉入库水位。现在任意深度按键排序(数组保序,Map/Set 形态不变),跳过的键(去掉 `storageMode`/`tailGaps`)集中成 `canonical.ts` 导出的 `DIGEST_SKIPPED_KEYS`,供流式摘要路径共用;升级文案不再断言「多个 bili 实例」。storage-corpus manifest 随摘要定义重钉(只有 10 个摘要值变化,文件清单/格式不动);storage-render 金标无需改动——那套驱动在 harness 侧已先 `sortObjectKeys`,排键只改 shadow 腿与 corpus 摘要。 - 块区间边界在重存时不再回退:折叠后客户端重发的视图不含已覆盖原文,重入库时该批次无法重新解析区间(-1/-1),旧逻辑会把已解析的真实区间抹掉。现在区间证据一旦解析即持久(入库 `CASE` 保留;sqlite/mysql 双后端),未知块保持诚实缺口(#4608)。 ### Added diff --git a/src/storage/canonical.ts b/src/storage/canonical.ts index 719bbd349a..4d947dae04 100644 --- a/src/storage/canonical.ts +++ b/src/storage/canonical.ts @@ -24,21 +24,34 @@ import { createHash } from "node:crypto"; * by a newer build legitimately rewrites it (probe-proven: it is the ONLY * field the legacy save→reload round-trip changes on the folded corpus * case), so pinning it would break parity on every version bump. - * - metadata.storageMode → dropped entirely — same class of last-writer - * provenance (#2671), but its PRESENCE is new (corpus pins predate it), - * so masking the value like __VER__ cannot keep old pins green; the key - * is removed from digests altogether. - * - tailGaps (#4390) → dropped entirely — read-time diagnostic added by - * loadLegacyView (count of tail pointers whose row is missing); the live - * legacy leg of any parity comparison structurally lacks the key, so - * digesting its presence would report drift on every sampled save. Same - * class as storageMode: the PRESENCE, not the value, is the problem. A - * genuinely missing row still diverges the message arrays themselves. + * - storageMode / tailGaps → in DIGEST_SKIPPED_KEYS, dropped at every depth + * before hashing. */ export function canonicalize(value: unknown): unknown { return canonicalizeWithContext(value, undefined); } +/** + * Keys that are NEVER part of a digest, at any depth — the ONE skip set every + * canonical digest producer must use (#4392). `canonicalize` below and the + * streamed `canonicalDigest` (#4532) hash the same values through different + * code paths; two hand-maintained skip lists drifted apart once already, which + * is what makes a golden-corpus pin and a shadow parity check read two + * different canonical forms of the same session. Import this table instead of + * re-listing the keys. + * + * - `storageMode` — last-writer provenance (#2671): the corpus pins predate + * the stamp and a re-save legitimately (re)writes it; unlike biliVersion its + * PRESENCE is new, so masking the value like __VER__ cannot keep old pins + * green — the key is removed from digests altogether. + * - `tailGaps` — read-time gap-count metadata added by loadLegacyView (#4390); + * the live legacy leg of any parity comparison structurally lacks the key, + * so digesting its presence would report drift on every sampled save. Same + * class as storageMode: the PRESENCE, not the value, is the problem. A + * genuinely missing row still diverges the message arrays themselves. + */ +export const DIGEST_SKIPPED_KEYS: Readonly> = { storageMode: true, tailGaps: true }; + function canonicalizeWithContext(value: unknown, key: string | undefined): unknown { if (Array.isArray(value)) return value.map((v) => canonicalizeWithContext(v, undefined)); if (value instanceof Map) return { __map__: [...value.entries()].map(([k, v]) => [k, canonicalizeWithContext(v, undefined)]) }; @@ -54,16 +67,10 @@ function canonicalizeWithContext(value: unknown, key: string | undefined): unkno // insertion order reported "[diverged: state]" against itself. for (const k of Object.keys(record).sort()) { const v = record[k]; - // storageMode is last-writer provenance (#2671): the corpus pins - // predate the stamp, and a re-save legitimately (re)writes it — - // unlike biliVersion its PRESENCE is new, so masking the value - // is not enough; drop the key from digests entirely. - if (k === "storageMode") continue; - // tailGaps (#4390) is read-time gap-count metadata from - // loadLegacyView — the live legacy leg structurally lacks the - // key, so digesting its presence would flag drift on every - // sampled save; drop it like storageMode. - if (k === "tailGaps") continue; + // One skip table for every digest producer (#4392): see + // DIGEST_SKIPPED_KEYS for why presence, not value, is the problem. + // hasOwn, not `in` — a "__proto__" key must not read as skipped. + if (Object.hasOwn(DIGEST_SKIPPED_KEYS, k)) continue; out[k] = canonicalizeWithContext(v, k); } return out; diff --git a/tests/golden/storage-corpus/manifest.json b/tests/golden/storage-corpus/manifest.json index 8f672243cc..bb9d16aaf7 100644 --- a/tests/golden/storage-corpus/manifest.json +++ b/tests/golden/storage-corpus/manifest.json @@ -4,14 +4,14 @@ "files": [ "anthropic/127.0.0.1_29e4587af02d1ed9ae4b0054.json" ], - "digest": "3026f3d77e59d5076b712faf9599da05f81d84fd997ac6e6631ff906f4f701c3" + "digest": "8485c8f69693cca987ec3c612bdc31fe429e70fe0fade5e05b9db936f0418311" }, "tier2": { "description": "superseded T1 + active T2, distinct `one` views, byte-identical pair pins #401 normalize-on-load", "files": [ "chat/127.0.0.1_8453ec7a56646bc014ac0d88.json" ], - "digest": "01c4e936c7ab078ae8908acf048ccfccf2ce6896894cc6f33a6174d6ce62fe57", + "digest": "0cfdba77bc1a052c0c3b8170874c4d963273b6580c822c6c9a5ee8f0f3d09b43", "notes": { "t2Block": "b3" } @@ -21,7 +21,7 @@ "files": [ "anthropic/127.0.0.1_e17d4db40407d10867daeb1d.json" ], - "digest": "44aecbb906a425dce328f27b1966af773dcaa7033b2b724af076ed5e4e3bee35", + "digest": "de057f87e0737222362d44326ec9e139457d50d67566ffaf5a7bb590b549c12d", "notes": { "ghostRef": "m00010" } @@ -31,14 +31,14 @@ "files": [ "chat/127.0.0.1_bde31987d03ca1ebaeb96ba0.json" ], - "digest": "5737b5edc484bdd27fedb921a3ffc2b9f6f8334104287e73dcba5f8c3bb57b20" + "digest": "edf7ac9a0cb912acc39b17667dfd1245a24b6bf6f01b57f510f782258e3e3bcb" }, "fork-retained": { "description": "publicSnapshotRetained only — pluginSnapshot persisted, forkContentStore DROPPED (pins the #2675 ② asymmetry verbatim)", "files": [ "chat/127.0.0.1_dec59ddccc0b4b89ff346c9b.json" ], - "digest": "d7894c369566ec22d6e6a581fa7de9e5f9ad88b09aa6617e0385318c4dde8e6d", + "digest": "63328d62ba50899927adda2154185ec83b9071ca8a84611dec1c57d5d4a51a8e", "notes": { "expectContentStoreOnLoad": "absent" } @@ -48,7 +48,7 @@ "files": [ "anthropic/127.0.0.1_b139318d5755710735cbf443.json" ], - "digest": "a65628e52318b5044e5007e1e8055c4a273b0ff4e6be8ed18dca4525db491ced" + "digest": "deec17f777d11294316b2250160b0577756b2e686ed0f2f1a6441ed06a581dc2" }, "ccr": { "description": "content-store.json sibling with two CCR originals (#1097 envelope)", @@ -56,7 +56,7 @@ "responses/127.0.0.1_0787df32f3cf98f922f7c0f0.content-store.json", "responses/127.0.0.1_0787df32f3cf98f922f7c0f0.json" ], - "digest": "20d534a2774be30e4f19dccdcb66d6af80b4c7ba2c06f855763954bf8d0c9f72" + "digest": "2d15c631cf8ee6fd406fb4354db508bed86f11ecaefcb7c9b07c48ab3a7493d8" }, "twinfile": { "description": "namespaced v2 record + stale FLAT v1 twin for the same id (#2671 ① twin-namespace)", @@ -64,11 +64,11 @@ "6fe5d8f2afee3e83e5360fa8.json", "openai/127.0.0.1_6fe5d8f2afee3e83e5360fa8.json" ], - "digest": "0cab4c22cf71ccd9d91402c3de1006bc2b40bd56aee1abead63f26c798187535", + "digest": "fe94b9453f3e7e2b57831965fbf87ff2604efdd11e74ecec612cbefd82e0e122", "notes": { "flatTwinRequests": 1, "namespacedRequests": 42, - "flatFallbackDigest": "64820a36fdf7ecf4b2f0e053dae33ac5c53df867f067657340a4cec064d804d8", + "flatFallbackDigest": "73ae30ed8e603bd51978f2d891acb6aada8c4769e05acbbfdd2a64d4e1f89e57", "flatFallbackRequests": 1 } }, @@ -77,6 +77,6 @@ "files": [ "anthropic/127.0.0.1_29e4587af02d1ed9ae4b0054.json" ], - "digest": "3026f3d77e59d5076b712faf9599da05f81d84fd997ac6e6631ff906f4f701c3" + "digest": "8485c8f69693cca987ec3c612bdc31fe429e70fe0fade5e05b9db936f0418311" } } diff --git a/tests/issue4392-canonical-key-order.test.ts b/tests/issue4392-canonical-key-order.test.ts index c271fcb2b4..ce6ecd66f4 100644 --- a/tests/issue4392-canonical-key-order.test.ts +++ b/tests/issue4392-canonical-key-order.test.ts @@ -25,7 +25,7 @@ import * as path from "node:path"; import { UnifiedStore } from "../src/storage/store.ts"; import { UnifiedShadow } from "../src/storage/shadow.ts"; import { loadLegacyView, planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; -import { canonicalize, sha256 } from "../src/storage/canonical.ts"; +import { canonicalize, DIGEST_SKIPPED_KEYS, sha256 } from "../src/storage/canonical.ts"; import type { CoreMessage } from "acp-kernel"; process.env.BILI_PERSIST_ZSTD = "0"; @@ -126,6 +126,26 @@ test("#4392: arrays keep their order; Maps/Sets keep their canonical envelope", ); }); +// --- Unit: the skip table is ONE contract, not a per-producer list --- + +test("#4392: DIGEST_SKIPPED_KEYS is the single skip contract every digest path shares", () => { + // Pinned literally: dropping a key changes every golden pin, so it must be + // a conscious re-pin. The streamed `canonicalDigest` (#4532) imports THIS + // table instead of hand-copying a list — two hand-maintained skip lists are + // how the golden corpus and the shadow parity check end up hashing two + // different canonical forms of the same session. + assert.deepEqual(Object.keys(DIGEST_SKIPPED_KEYS).sort(), ["storageMode", "tailGaps"]); + assert.equal( + JSON.stringify(canonicalize({ + state: { tailGaps: { m00002: 1 }, keep: "x" }, + metadata: { storageMode: "full", biliVersion: "0.1.208" }, + stats: { requests: 3 }, + })), + JSON.stringify({ metadata: { biliVersion: "__VER__" }, state: { keep: "x" }, stats: { requests: 3 } }), + "skipped keys are dropped at every depth; biliVersion is masked, never dropped", + ); +}); + // --- E2E: the field shape — byRef rebuilt in ledger order, same contents --- test("#4392: byRef reordered by the ref-ledger rebuild must not trip the parity digest", () => { From 69fe94f4e652ee3211625ab7e61f69e8ff3f73d5 Mon Sep 17 00:00:00 2001 From: mz Date: Tue, 27 Oct 2026 18:36:44 +0300 Subject: [PATCH 4/8] test(storage): re-pin storage-corpus digests under sorted-key canonicalize (#4392) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only the 10 digest values change (9 cases + twinfile flatFallbackDigest); file lists and the generator 2-space format are untouched. storage-render goldens need no change — that drive already sorts keys harness-side. --- tests/golden/storage-corpus/manifest.json | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/golden/storage-corpus/manifest.json b/tests/golden/storage-corpus/manifest.json index bb9d16aaf7..8f672243cc 100644 --- a/tests/golden/storage-corpus/manifest.json +++ b/tests/golden/storage-corpus/manifest.json @@ -4,14 +4,14 @@ "files": [ "anthropic/127.0.0.1_29e4587af02d1ed9ae4b0054.json" ], - "digest": "8485c8f69693cca987ec3c612bdc31fe429e70fe0fade5e05b9db936f0418311" + "digest": "3026f3d77e59d5076b712faf9599da05f81d84fd997ac6e6631ff906f4f701c3" }, "tier2": { "description": "superseded T1 + active T2, distinct `one` views, byte-identical pair pins #401 normalize-on-load", "files": [ "chat/127.0.0.1_8453ec7a56646bc014ac0d88.json" ], - "digest": "0cfdba77bc1a052c0c3b8170874c4d963273b6580c822c6c9a5ee8f0f3d09b43", + "digest": "01c4e936c7ab078ae8908acf048ccfccf2ce6896894cc6f33a6174d6ce62fe57", "notes": { "t2Block": "b3" } @@ -21,7 +21,7 @@ "files": [ "anthropic/127.0.0.1_e17d4db40407d10867daeb1d.json" ], - "digest": "de057f87e0737222362d44326ec9e139457d50d67566ffaf5a7bb590b549c12d", + "digest": "44aecbb906a425dce328f27b1966af773dcaa7033b2b724af076ed5e4e3bee35", "notes": { "ghostRef": "m00010" } @@ -31,14 +31,14 @@ "files": [ "chat/127.0.0.1_bde31987d03ca1ebaeb96ba0.json" ], - "digest": "edf7ac9a0cb912acc39b17667dfd1245a24b6bf6f01b57f510f782258e3e3bcb" + "digest": "5737b5edc484bdd27fedb921a3ffc2b9f6f8334104287e73dcba5f8c3bb57b20" }, "fork-retained": { "description": "publicSnapshotRetained only — pluginSnapshot persisted, forkContentStore DROPPED (pins the #2675 ② asymmetry verbatim)", "files": [ "chat/127.0.0.1_dec59ddccc0b4b89ff346c9b.json" ], - "digest": "63328d62ba50899927adda2154185ec83b9071ca8a84611dec1c57d5d4a51a8e", + "digest": "d7894c369566ec22d6e6a581fa7de9e5f9ad88b09aa6617e0385318c4dde8e6d", "notes": { "expectContentStoreOnLoad": "absent" } @@ -48,7 +48,7 @@ "files": [ "anthropic/127.0.0.1_b139318d5755710735cbf443.json" ], - "digest": "deec17f777d11294316b2250160b0577756b2e686ed0f2f1a6441ed06a581dc2" + "digest": "a65628e52318b5044e5007e1e8055c4a273b0ff4e6be8ed18dca4525db491ced" }, "ccr": { "description": "content-store.json sibling with two CCR originals (#1097 envelope)", @@ -56,7 +56,7 @@ "responses/127.0.0.1_0787df32f3cf98f922f7c0f0.content-store.json", "responses/127.0.0.1_0787df32f3cf98f922f7c0f0.json" ], - "digest": "2d15c631cf8ee6fd406fb4354db508bed86f11ecaefcb7c9b07c48ab3a7493d8" + "digest": "20d534a2774be30e4f19dccdcb66d6af80b4c7ba2c06f855763954bf8d0c9f72" }, "twinfile": { "description": "namespaced v2 record + stale FLAT v1 twin for the same id (#2671 ① twin-namespace)", @@ -64,11 +64,11 @@ "6fe5d8f2afee3e83e5360fa8.json", "openai/127.0.0.1_6fe5d8f2afee3e83e5360fa8.json" ], - "digest": "fe94b9453f3e7e2b57831965fbf87ff2604efdd11e74ecec612cbefd82e0e122", + "digest": "0cab4c22cf71ccd9d91402c3de1006bc2b40bd56aee1abead63f26c798187535", "notes": { "flatTwinRequests": 1, "namespacedRequests": 42, - "flatFallbackDigest": "73ae30ed8e603bd51978f2d891acb6aada8c4769e05acbbfdd2a64d4e1f89e57", + "flatFallbackDigest": "64820a36fdf7ecf4b2f0e053dae33ac5c53df867f067657340a4cec064d804d8", "flatFallbackRequests": 1 } }, @@ -77,6 +77,6 @@ "files": [ "anthropic/127.0.0.1_29e4587af02d1ed9ae4b0054.json" ], - "digest": "8485c8f69693cca987ec3c612bdc31fe429e70fe0fade5e05b9db936f0418311" + "digest": "3026f3d77e59d5076b712faf9599da05f81d84fd997ac6e6631ff906f4f701c3" } } From 0b1a26cbbacacb4963a441b5f1ca8890173cd193 Mon Sep 17 00:00:00 2001 From: mz Date: Tue, 27 Oct 2026 18:38:20 +0300 Subject: [PATCH 5/8] docs(storage): point the sorted-key loop at the file header instead of repeating it (#4392) --- src/storage/canonical.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/storage/canonical.ts b/src/storage/canonical.ts index 4d947dae04..1b703de801 100644 --- a/src/storage/canonical.ts +++ b/src/storage/canonical.ts @@ -59,12 +59,8 @@ function canonicalizeWithContext(value: unknown, key: string | undefined): unkno if (value && typeof value === "object") { const out: Record = {}; const record = value as Record; - // Keys digest in SORTED order (#4392): insertion order is not part of - // the persistence contract. The live leg carries the kernel's own - // object order while the replay leg rebuilds the same values field by - // field from unified rows (messageRefs.byRef after a ledger - // re-alignment, mergeState's spread-merged nudge/stats), so hashing - // insertion order reported "[diverged: state]" against itself. + // Sorted keys per the #4392 rule in this file's header — see there for + // why insertion order cannot be part of the contract. for (const k of Object.keys(record).sort()) { const v = record[k]; // One skip table for every digest producer (#4392): see From 18a9d97e28858d448f001ece1364327bdb9121b6 Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 28 Oct 2026 02:40:55 +0800 Subject: [PATCH 6/8] test(codex-concurrent-launch): wait for a complete output file before parsing (#3203) waitForFile gated on existence only. The child fixture creates out-*.json before its content lands (a zero-byte window that widens under Windows CI load with four concurrent tsx spawns), so the parent could JSON.parse an empty file: SyntaxError 'Unexpected end of JSON input' (run 40555535547, shard windows-22-4). Gate on non-empty size stable across two polls; the slot-distinctness assertion itself is unchanged. --- tests/codex-concurrent-launch.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/codex-concurrent-launch.test.ts b/tests/codex-concurrent-launch.test.ts index c0e60bb210..830a71d3d3 100644 --- a/tests/codex-concurrent-launch.test.ts +++ b/tests/codex-concurrent-launch.test.ts @@ -116,7 +116,11 @@ function writeScript(root: string, name: string, body: string): string { async function waitForFile(file: string, timeoutMs: number): Promise { const deadline = Date.now() + timeoutMs; - while (!fs.existsSync(file)) { + let lastSize = -1; + for (;;) { + const size = fs.existsSync(file) ? fs.statSync(file).size : -1; + if (size > 0 && size === lastSize) return; + lastSize = size; if (Date.now() > deadline) throw new Error(`timed out waiting for ${file}`); await new Promise((resolve) => setTimeout(resolve, 50)); } From 02ef0d170e7755842d7c14d6cfb92fc6a42a8d05 Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 28 Oct 2026 11:12:02 +0800 Subject: [PATCH 7/8] fix(ci): register issue4982 draining-twin-ladder process-spawning suite in LIVE_FILES (#4982) --- scripts/live-files.mjs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/live-files.mjs b/scripts/live-files.mjs index e2a58f9f45..315ddf7172 100644 --- a/scripts/live-files.mjs +++ b/scripts/live-files.mjs @@ -68,6 +68,11 @@ export const LIVE_FILES = [ // servers; every socket is listen(0), spawns are mocked, so the zone band // is belt-and-braces like #3890's — the entry is for the tripwire "tests/issue4811-dsh-busy-drop-selfheal.test.ts", + // #4982: integration rows spawn bili children against freePort() picks and + // follow each child's REGISTERED port (ladder-tolerant); BASE is an + // in-memory-only value, so no zone band is needed — the entry is for the + // tripwire + "tests/issue4982-draining-twin-ladder.test.ts", ]; /** Band width per live file: the zone ladder drifts +1 per EADDRINUSE retry; From 3b0993c25842c280b2f963291a4c3b96cb2fed14 Mon Sep 17 00:00:00 2001 From: mz Date: Wed, 28 Oct 2026 09:47:01 +0300 Subject: [PATCH 8/8] test(storage): pin the #4986 folded-corpus digest shape (test from #4997) #4997 opened for #4986 the same root cause #4685 fixes in canonicalize. Its regression test fails on master b08fbf36 (DIGEST DRIFT for corpus-folded) and passes here without the shadowDigest change, so it carries over unchanged. --- CHANGELOG.md | 2 +- .../issue4986-digest-drift-ref-order.test.ts | 91 +++++++++++++++++++ 2 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 tests/issue4986-digest-drift-ref-order.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index cecaf17ff9..94e6c99281 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag ## [Unreleased] ### Fixed -- **shadow 摘要漂移的单实例成因(#4392)**:`canonicalize` 曾按 `Object.entries` 插入序参与哈希,而 live 腿(kernel 状态,`byRef` 按发号顺序)与 replay 腿(`loadLegacyView` 按账本 `ord` 重建 `messageRefs`)对同一份等值状态给出不同键序 → 单进程自我报告 `[diverged: state]`,连续 3 次即冻结本进程的信封写入并清掉入库水位。现在任意深度按键排序(数组保序,Map/Set 形态不变),跳过的键(去掉 `storageMode`/`tailGaps`)集中成 `canonical.ts` 导出的 `DIGEST_SKIPPED_KEYS`,供流式摘要路径共用;升级文案不再断言「多个 bili 实例」。storage-corpus manifest 随摘要定义重钉(只有 10 个摘要值变化,文件清单/格式不动);storage-render 金标无需改动——那套驱动在 harness 侧已先 `sortObjectKeys`,排键只改 shadow 腿与 corpus 摘要。 +- **shadow 摘要漂移的单实例成因(#4392 / #4986)**:`canonicalize` 曾按 `Object.entries` 插入序参与哈希,而 live 腿(kernel 状态,`byRef` 按发号顺序)与 replay 腿(`loadLegacyView` 按账本 `ord` 重建 `messageRefs`)对同一份等值状态给出不同键序 → 单进程自我报告 `[diverged: state]`,连续 3 次即冻结本进程的信封写入并清掉入库水位。现在任意深度按键排序(数组保序,Map/Set 形态不变),跳过的键(去掉 `storageMode`/`tailGaps`)集中成 `canonical.ts` 导出的 `DIGEST_SKIPPED_KEYS`,供流式摘要路径共用;升级文案不再断言「多个 bili 实例」。storage-corpus manifest 随摘要定义重钉(只有 10 个摘要值变化,文件清单/格式不动);storage-render 金标无需改动——那套驱动在 harness 侧已先 `sortObjectKeys`,排键只改 shadow 腿与 corpus 摘要。`tests/issue4986-digest-drift-ref-order.test.ts`(取自 #4997)钉住 #4986 的 folded-corpus 面。 - 出站不变量判官新增 I3 配对不变量(#4862 出口校验缺口):此前判官只对账入站↔出站连续性(I1/I2),折叠区内完全免检,于是「折叠拆断一对 tool_use/tool_result」的出站体拿到 clean fold verdict 直接放行,直到上游用 400 `tool_use ids were found without tool_result blocks` 说出真相。现在每个 tool-call 必须有同 id 的 tool-result 在其后、每个 tool-result 必须有同 id 的 tool-call 在其前,折叠区不再豁免;归因保持“谁的错”职责(宿主自己发来的断对不计 bili、compress 在途调用豁免、id 重铸不可洗白)。纯检测不拦截,受 diagnostics.invariantJudge 旋钮控制。 - 块区间边界在重存时不再回退:折叠后客户端重发的视图不含已覆盖原文,重入库时该批次无法重新解析区间(-1/-1),旧逻辑会把已解析的真实区间抹掉。现在区间证据一旦解析即持久(入库 `CASE` 保留;sqlite/mysql 双后端),未知块保持诚实缺口(#4608)。 - 优雅停机(SIGTERM → dispose)的 writer 标记现在记录 `stoppedAt`:重启流程(杀旧进程→立即重绑)不再对已确认死亡的前任实例误报 [split-brain]——30s 新鲜度回退从此只覆盖无法证明死亡的崩溃场景(#3923;release-gate lane A 代理重启时观察到误报)。 diff --git a/tests/issue4986-digest-drift-ref-order.test.ts b/tests/issue4986-digest-drift-ref-order.test.ts new file mode 100644 index 0000000000..d427f387b1 --- /dev/null +++ b/tests/issue4986-digest-drift-ref-order.test.ts @@ -0,0 +1,91 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { SessionStore } from "../src/persist.ts"; + +/** + * #4986: the sampled ingest read-back verify must not report DIGEST DRIFT + * when the in-memory messageRefs key order diverges from the stored + * refs-row order. The golden corpora are exactly that shape: byRef sits in + * ref-number order while byRaw — which finishPlan freezes into row ord at + * full ingest — sits in a different key order, so the row-rebuilt leg's + * byRef starts at m00041 while the live leg's starts at m00001. Identical + * content, different key insertion order. Pre-fix, every such save logged + * an error-level DIGEST DRIFT and dropped the ingest watermark (full-tail + * re-ingest next save); post-fix the parity digest compares contents, not + * key insertion order (#3407/#3113 philosophy). + */ + +process.env.BILI_PERSIST_ZSTD = "0"; +const hermetic = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4986-hermetic-")); +process.env.BILI_CONFIG_FILE = path.join(hermetic, "no-config.json"); +process.env.BILI_STORAGE_UNIFIED = "legacy"; + +const CORPUS_ROOT = path.resolve(import.meta.dirname, "golden/storage-corpus"); + +function loadCorpusCase(dir: string, name: string, id: string, protocol: string) { + fs.cpSync(path.join(CORPUS_ROOT, name), path.join(dir, name), { recursive: true }); + const loader = new SessionStore({ dir: path.join(dir, name), debounceMs: 1, enabled: true }); + const loaded = loader.loadSync(id, { protocol, upstreamOrigin: "http://127.0.0.1:8199" }); + loader.cancelAll(); + loader.closeQuietly(); + assert.ok(loaded, `${name}: corpus loadSync returned null`); + return loaded; +} + +/** Windows: best-effort delete, swallowed on persistent EBUSY/EPERM. */ +function rmBestEffort(p: string): void { + try { + fs.rmSync(p, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); + } catch { + /* leaked temp dir on a Windows runner is harmless */ + } +} + +test("ingest verify ignores messageRefs key-order divergence (#4986)", async () => { + const prevMode = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "full"; + try { + const seed = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4986-seed-")); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4986-store-")); + try { + const session = loadCorpusCase(seed, "folded", "corpus-folded", "anthropic"); + + // Precondition: the corpus exhibits the divergent shape — the + // row order (byRaw key order, what finishPlan stores as ord) + // projects byRef to a DIFFERENT key sequence than the live map's + // own key order. Without this the test would pass vacuously. + const byRaw = session.state.messageRefs.byRaw; + const byRef = session.state.messageRefs.byRef; + const rowOrderProjectedRefs = Object.keys(byRaw) + .map((rawId) => byRaw[rawId]) + .filter((ref) => ref !== "BLOCKED" && byRef[ref] !== undefined); + assert.notEqual( + Object.keys(byRef).join("\u0000"), + rowOrderProjectedRefs.join("\u0000"), + "folded corpus must carry byRef key order != stored refs-row order", + ); + + const store = new SessionStore({ dir, debounceMs: 1, enabled: true }); + await store.writeNow(session); + const shadow = (store as unknown as { + shadow?: { stats: { verifySamples: number; verifyOk: number; verifyDrift: number } }; + }).shadow; + assert.ok(shadow, "full mode must open the unified sidecar"); + // The FIRST ingest always samples the read-back verify... + assert.equal(shadow.stats.verifySamples, 1, "first ingest must sample the read-back verify"); + // ...and key-order-only divergence is not drift. + assert.equal(shadow.stats.verifyDrift, 0, "#4986: order-only ref-map divergence must not report DIGEST DRIFT"); + assert.equal(shadow.stats.verifyOk, 1); + store.cancelAll(); + } finally { + rmBestEffort(dir); + rmBestEffort(seed); + } + } finally { + process.env.BILI_STORAGE_UNIFIED = prevMode; + rmBestEffort(hermetic); + } +});