diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a3b694488..94e6c99281 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag ## [Unreleased] ### Fixed +- **shadow 摘要漂移的单实例成因(#4392 / #4986)**:`canonicalize` 曾按 `Object.entries` 插入序参与哈希,而 live 腿(kernel 状态,`byRef` 按发号顺序)与 replay 腿(`loadLegacyView` 按账本 `ord` 重建 `messageRefs`)对同一份等值状态给出不同键序 → 单进程自我报告 `[diverged: state]`,连续 3 次即冻结本进程的信封写入并清掉入库水位。现在任意深度按键排序(数组保序,Map/Set 形态不变),跳过的键(去掉 `storageMode`/`tailGaps`)集中成 `canonical.ts` 导出的 `DIGEST_SKIPPED_KEYS`,供流式摘要路径共用;升级文案不再断言「多个 bili 实例」。storage-corpus manifest 随摘要定义重钉(只有 10 个摘要值变化,文件清单/格式不动);storage-render 金标无需改动——那套驱动在 harness 侧已先 `sortObjectKeys`,排键只改 shadow 腿与 corpus 摘要。`tests/issue4986-digest-drift-ref-order.test.ts`(取自 #4997)钉住 #4986 的 folded-corpus 面。 - 出站不变量判官新增 I3 配对不变量(#4862 出口校验缺口):此前判官只对账入站↔出站连续性(I1/I2),折叠区内完全免检,于是「折叠拆断一对 tool_use/tool_result」的出站体拿到 clean fold verdict 直接放行,直到上游用 400 `tool_use ids were found without tool_result blocks` 说出真相。现在每个 tool-call 必须有同 id 的 tool-result 在其后、每个 tool-result 必须有同 id 的 tool-call 在其前,折叠区不再豁免;归因保持“谁的错”职责(宿主自己发来的断对不计 bili、compress 在途调用豁免、id 重铸不可洗白)。纯检测不拦截,受 diagnostics.invariantJudge 旋钮控制。 - 块区间边界在重存时不再回退:折叠后客户端重发的视图不含已覆盖原文,重入库时该批次无法重新解析区间(-1/-1),旧逻辑会把已解析的真实区间抹掉。现在区间证据一旦解析即持久(入库 `CASE` 保留;sqlite/mysql 双后端),未知块保持诚实缺口(#4608)。 - 优雅停机(SIGTERM → dispose)的 writer 标记现在记录 `stoppedAt`:重启流程(杀旧进程→立即重绑)不再对已确认死亡的前任实例误报 [split-brain]——30s 新鲜度回退从此只覆盖无法证明死亡的崩溃场景(#3923;release-gate lane A 代理重启时观察到误报)。 diff --git a/scripts/live-files.mjs b/scripts/live-files.mjs index e2a58f9f45..315ddf7172 100644 --- a/scripts/live-files.mjs +++ b/scripts/live-files.mjs @@ -68,6 +68,11 @@ export const LIVE_FILES = [ // servers; every socket is listen(0), spawns are mocked, so the zone band // is belt-and-braces like #3890's — the entry is for the tripwire "tests/issue4811-dsh-busy-drop-selfheal.test.ts", + // #4982: integration rows spawn bili children against freePort() picks and + // follow each child's REGISTERED port (ladder-tolerant); BASE is an + // in-memory-only value, so no zone band is needed — the entry is for the + // tripwire + "tests/issue4982-draining-twin-ladder.test.ts", ]; /** Band width per live file: the zone ladder drifts +1 per EADDRINUSE retry; diff --git a/src/storage/canonical.ts b/src/storage/canonical.ts index 4d8b449e7e..1b703de801 100644 --- a/src/storage/canonical.ts +++ b/src/storage/canonical.ts @@ -8,6 +8,13 @@ import { createHash } from "node:crypto"; * corpus manifests and shadow checks agree byte-for-byte. * * Determinism rules, in order: + * - Object keys → SORTED at every depth (#4392): insertion order is not + * part of the contract. The live leg (kernel state) and the replay leg + * (unified rows rebuilt field by field) legitimately build the same + * values in different orders — messageRefs.byRef after a ref-ledger + * re-alignment, mergeState's spread-merged nudge/stats — so hashing + * insertion order made a single instance report drift against itself. + * Arrays keep their order (order IS content there). * - Map/Set → tagged JSON-able envelopes (key order = insertion order; the * producers below insert in deterministic order). * - Wall-clock timestamps (> 2020-09-01 in ms) → "__TS__" — savedAt, @@ -17,38 +24,49 @@ import { createHash } from "node:crypto"; * by a newer build legitimately rewrites it (probe-proven: it is the ONLY * field the legacy save→reload round-trip changes on the folded corpus * case), so pinning it would break parity on every version bump. - * - metadata.storageMode → dropped entirely — same class of last-writer - * provenance (#2671), but its PRESENCE is new (corpus pins predate it), - * so masking the value like __VER__ cannot keep old pins green; the key - * is removed from digests altogether. - * - tailGaps (#4390) → dropped entirely — read-time diagnostic added by - * loadLegacyView (count of tail pointers whose row is missing); the live - * legacy leg of any parity comparison structurally lacks the key, so - * digesting its presence would report drift on every sampled save. Same - * class as storageMode: the PRESENCE, not the value, is the problem. A - * genuinely missing row still diverges the message arrays themselves. + * - storageMode / tailGaps → in DIGEST_SKIPPED_KEYS, dropped at every depth + * before hashing. */ export function canonicalize(value: unknown): unknown { return canonicalizeWithContext(value, undefined); } +/** + * Keys that are NEVER part of a digest, at any depth — the ONE skip set every + * canonical digest producer must use (#4392). `canonicalize` below and the + * streamed `canonicalDigest` (#4532) hash the same values through different + * code paths; two hand-maintained skip lists drifted apart once already, which + * is what makes a golden-corpus pin and a shadow parity check read two + * different canonical forms of the same session. Import this table instead of + * re-listing the keys. + * + * - `storageMode` — last-writer provenance (#2671): the corpus pins predate + * the stamp and a re-save legitimately (re)writes it; unlike biliVersion its + * PRESENCE is new, so masking the value like __VER__ cannot keep old pins + * green — the key is removed from digests altogether. + * - `tailGaps` — read-time gap-count metadata added by loadLegacyView (#4390); + * the live legacy leg of any parity comparison structurally lacks the key, + * so digesting its presence would report drift on every sampled save. Same + * class as storageMode: the PRESENCE, not the value, is the problem. A + * genuinely missing row still diverges the message arrays themselves. + */ +export const DIGEST_SKIPPED_KEYS: Readonly> = { storageMode: true, tailGaps: true }; + function canonicalizeWithContext(value: unknown, key: string | undefined): unknown { if (Array.isArray(value)) return value.map((v) => canonicalizeWithContext(v, undefined)); if (value instanceof Map) return { __map__: [...value.entries()].map(([k, v]) => [k, canonicalizeWithContext(v, undefined)]) }; if (value instanceof Set) return { __set__: [...value.values()].map((v) => canonicalizeWithContext(v, undefined)) }; if (value && typeof value === "object") { const out: Record = {}; - for (const [k, v] of Object.entries(value)) { - // storageMode is last-writer provenance (#2671): the corpus pins - // predate the stamp, and a re-save legitimately (re)writes it — - // unlike biliVersion its PRESENCE is new, so masking the value - // is not enough; drop the key from digests entirely. - if (k === "storageMode") continue; - // tailGaps (#4390) is read-time gap-count metadata from - // loadLegacyView — the live legacy leg structurally lacks the - // key, so digesting its presence would flag drift on every - // sampled save; drop it like storageMode. - if (k === "tailGaps") continue; + const record = value as Record; + // Sorted keys per the #4392 rule in this file's header — see there for + // why insertion order cannot be part of the contract. + for (const k of Object.keys(record).sort()) { + const v = record[k]; + // One skip table for every digest producer (#4392): see + // DIGEST_SKIPPED_KEYS for why presence, not value, is the problem. + // hasOwn, not `in` — a "__proto__" key must not read as skipped. + if (Object.hasOwn(DIGEST_SKIPPED_KEYS, k)) continue; out[k] = canonicalizeWithContext(v, k); } return out; diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index 8a69ce5dc0..46bf0018d6 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -401,13 +401,15 @@ export class UnifiedShadow { this.driftedSessions.add(legacy.id); // #4323: count CONSECUTIVE sampled drifts — membership alone // caps at "repeat yes/no" and cannot distinguish 3+ samples. - // N in a row proves structural divergence (multi-writer - // #394): escalate ONCE at the threshold with a loud WARN - // naming both sides' fold counts, then freeze THIS - // process's envelope writes for the session so the ping- - // pong stops churning the file copy (persist.ts consults - // envelopeFrozen on every save route; the unified store - // keeps receiving every save either way). + // N in a row marks a STRUCTURAL divergence: the watermark + // clear healed nothing. Note the drift can be a single- + // instance artifact (#4392 — digest shape), not only the + // multi-writer #394 case; escalate ONCE at the threshold + // with a loud WARN naming both sides' fold counts, then + // freeze THIS process's envelope writes for the session so + // the ping-pong stops churning the file copy (persist.ts + // consults envelopeFrozen on every save route; the unified + // store keeps receiving every save either way). const streak = (this.driftStreaks.get(legacy.id) ?? 0) + 1; this.driftStreaks.set(legacy.id, streak); if (streak === DRIFT_ESCALATE_STREAK) { @@ -420,7 +422,7 @@ export class UnifiedShadow { } catch { // counts must never break the heal path } - loggerLog("warn", `[storage] DIGEST DRIFT ESCALATED for ${legacy.id}: ${streak} consecutive sampled drifts — the views have stopped converging${sides}. This usually means MULTIPLE bili instances share one sessions dir (#394 anti-pattern; second-instance detection: #4177). Freezing THIS process's JSON envelope writes for the session until a clean sample re-admits it — the unified store keeps receiving every save (#4323)`); + loggerLog("warn", `[storage] DIGEST DRIFT ESCALATED for ${legacy.id}: ${streak} consecutive sampled drifts — the views have stopped converging${sides}. Both legs are built in THIS process, so a persistent gap is a digest-shape mismatch first (field only one leg builds, ordering) and only then a second writer: a second bili instance sharing one sessions dir (#394 anti-pattern; second-instance detection: #4177) is one cause among others (#4392). Freezing THIS process's JSON envelope writes for the session until a clean sample re-admits it — the unified store keeps receiving every save (#4323)`); } // Panels consume {disabled, lastError} — make the sampled // degradation visible there, not just in the counter/log. diff --git a/tests/codex-concurrent-launch.test.ts b/tests/codex-concurrent-launch.test.ts index c0e60bb210..830a71d3d3 100644 --- a/tests/codex-concurrent-launch.test.ts +++ b/tests/codex-concurrent-launch.test.ts @@ -116,7 +116,11 @@ function writeScript(root: string, name: string, body: string): string { async function waitForFile(file: string, timeoutMs: number): Promise { const deadline = Date.now() + timeoutMs; - while (!fs.existsSync(file)) { + let lastSize = -1; + for (;;) { + const size = fs.existsSync(file) ? fs.statSync(file).size : -1; + if (size > 0 && size === lastSize) return; + lastSize = size; if (Date.now() > deadline) throw new Error(`timed out waiting for ${file}`); await new Promise((resolve) => setTimeout(resolve, 50)); } diff --git a/tests/golden/storage-corpus/manifest.json b/tests/golden/storage-corpus/manifest.json index bb9d16aaf7..8f672243cc 100644 --- a/tests/golden/storage-corpus/manifest.json +++ b/tests/golden/storage-corpus/manifest.json @@ -4,14 +4,14 @@ "files": [ "anthropic/127.0.0.1_29e4587af02d1ed9ae4b0054.json" ], - "digest": "8485c8f69693cca987ec3c612bdc31fe429e70fe0fade5e05b9db936f0418311" + "digest": "3026f3d77e59d5076b712faf9599da05f81d84fd997ac6e6631ff906f4f701c3" }, "tier2": { "description": "superseded T1 + active T2, distinct `one` views, byte-identical pair pins #401 normalize-on-load", "files": [ "chat/127.0.0.1_8453ec7a56646bc014ac0d88.json" ], - "digest": "0cfdba77bc1a052c0c3b8170874c4d963273b6580c822c6c9a5ee8f0f3d09b43", + "digest": "01c4e936c7ab078ae8908acf048ccfccf2ce6896894cc6f33a6174d6ce62fe57", "notes": { "t2Block": "b3" } @@ -21,7 +21,7 @@ "files": [ "anthropic/127.0.0.1_e17d4db40407d10867daeb1d.json" ], - "digest": "de057f87e0737222362d44326ec9e139457d50d67566ffaf5a7bb590b549c12d", + "digest": "44aecbb906a425dce328f27b1966af773dcaa7033b2b724af076ed5e4e3bee35", "notes": { "ghostRef": "m00010" } @@ -31,14 +31,14 @@ "files": [ "chat/127.0.0.1_bde31987d03ca1ebaeb96ba0.json" ], - "digest": "edf7ac9a0cb912acc39b17667dfd1245a24b6bf6f01b57f510f782258e3e3bcb" + "digest": "5737b5edc484bdd27fedb921a3ffc2b9f6f8334104287e73dcba5f8c3bb57b20" }, "fork-retained": { "description": "publicSnapshotRetained only — pluginSnapshot persisted, forkContentStore DROPPED (pins the #2675 ② asymmetry verbatim)", "files": [ "chat/127.0.0.1_dec59ddccc0b4b89ff346c9b.json" ], - "digest": "63328d62ba50899927adda2154185ec83b9071ca8a84611dec1c57d5d4a51a8e", + "digest": "d7894c369566ec22d6e6a581fa7de9e5f9ad88b09aa6617e0385318c4dde8e6d", "notes": { "expectContentStoreOnLoad": "absent" } @@ -48,7 +48,7 @@ "files": [ "anthropic/127.0.0.1_b139318d5755710735cbf443.json" ], - "digest": "deec17f777d11294316b2250160b0577756b2e686ed0f2f1a6441ed06a581dc2" + "digest": "a65628e52318b5044e5007e1e8055c4a273b0ff4e6be8ed18dca4525db491ced" }, "ccr": { "description": "content-store.json sibling with two CCR originals (#1097 envelope)", @@ -56,7 +56,7 @@ "responses/127.0.0.1_0787df32f3cf98f922f7c0f0.content-store.json", "responses/127.0.0.1_0787df32f3cf98f922f7c0f0.json" ], - "digest": "2d15c631cf8ee6fd406fb4354db508bed86f11ecaefcb7c9b07c48ab3a7493d8" + "digest": "20d534a2774be30e4f19dccdcb66d6af80b4c7ba2c06f855763954bf8d0c9f72" }, "twinfile": { "description": "namespaced v2 record + stale FLAT v1 twin for the same id (#2671 ① twin-namespace)", @@ -64,11 +64,11 @@ "6fe5d8f2afee3e83e5360fa8.json", "openai/127.0.0.1_6fe5d8f2afee3e83e5360fa8.json" ], - "digest": "fe94b9453f3e7e2b57831965fbf87ff2604efdd11e74ecec612cbefd82e0e122", + "digest": "0cab4c22cf71ccd9d91402c3de1006bc2b40bd56aee1abead63f26c798187535", "notes": { "flatTwinRequests": 1, "namespacedRequests": 42, - "flatFallbackDigest": "73ae30ed8e603bd51978f2d891acb6aada8c4769e05acbbfdd2a64d4e1f89e57", + "flatFallbackDigest": "64820a36fdf7ecf4b2f0e053dae33ac5c53df867f067657340a4cec064d804d8", "flatFallbackRequests": 1 } }, @@ -77,6 +77,6 @@ "files": [ "anthropic/127.0.0.1_29e4587af02d1ed9ae4b0054.json" ], - "digest": "8485c8f69693cca987ec3c612bdc31fe429e70fe0fade5e05b9db936f0418311" + "digest": "3026f3d77e59d5076b712faf9599da05f81d84fd997ac6e6631ff906f4f701c3" } } diff --git a/tests/issue4392-canonical-key-order.test.ts b/tests/issue4392-canonical-key-order.test.ts new file mode 100644 index 0000000000..ce6ecd66f4 --- /dev/null +++ b/tests/issue4392-canonical-key-order.test.ts @@ -0,0 +1,212 @@ +/** + * #4392: "[diverged: state]" digest drift on a SINGLE instance — canonicalize + * must digest object key SETS, not insertion order. + * + * The two shadow verify legs build the same values in different orders. Field + * evidence (v0.1.206, one proxy process): the expect leg loads the envelope + * through buildSession → mergeState, whose `messageRefs` is taken off the + * persisted state verbatim, while the got leg rebuilds `messageRefs.byRef` from + * the refs ledger row order (`loadLegacyView`). The kernel mints refs in a + * different order than the byRaw keys the ledger is written in, so byRef came + * back reordered with identical contents — `shadowDigest` hashed the order and + * reported drift against itself, escalating to "MULTIPLE bili instances" + * (#4323) after three samples and freezing envelope writes. + * + * Post-fix `canonicalize` sorts object keys at every depth (Maps/Sets keep + * their current canonical form, arrays keep their order), so only a genuine + * value/set difference trips the digest. Both directions are asserted: the + * reordered-but-equal case stays green, a real content difference still drifts. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { UnifiedStore } from "../src/storage/store.ts"; +import { UnifiedShadow } from "../src/storage/shadow.ts"; +import { loadLegacyView, planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { canonicalize, DIGEST_SKIPPED_KEYS, sha256 } from "../src/storage/canonical.ts"; +import type { CoreMessage } from "acp-kernel"; + +process.env.BILI_PERSIST_ZSTD = "0"; +const hermetic = fs.mkdtempSync(path.join(os.tmpdir(), "bili-canonical-order-hermetic-")); +process.env.BILI_CONFIG_FILE = path.join(hermetic, "no-config.json"); +process.env.BILI_STORAGE_UNIFIED = "legacy"; + +function tmpdir(prefix: string): string { + return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); +} + +function rmBestEffort(p: string): void { + try { + fs.rmSync(p, { recursive: true, force: true }); + } catch { + // best effort: a locked sqlite file must not fail the assertion path + } +} + +function msg(i: number): CoreMessage { + return { id: `h_msg${i}`, role: i % 2 === 0 ? "user" : "assistant", contentType: "text", text: `hello ${i}` } as CoreMessage; +} + +// Above canonicalize's > 2020-09-01 mask threshold so createdAt / lastSeen +// collapse to __TS__ on BOTH digest legs. +const T_CREATED = 1_700_000_000_000; +const T_SEEN = 1_700_000_000_500; + +interface StateShape { + blocks: unknown[]; + messageRefs: { byRaw: Record; byRef: Record }; + nextBlockId: number; + nextRunId: number; +} + +function mkLegacy(id: string, state: StateShape): LegacySessionLike { + return { + id, + createdAt: T_CREATED, + lastSeen: T_SEEN, + meta: { protocol: "anthropic" }, + stats: { requests: 1 }, + metadata: {}, + state: state as unknown as LegacySessionLike["state"], + blockContents: new Map(), + lastMessages: [msg(0), msg(1)], + lastMessagesFolded: false, + }; +} + +interface StatsView { + ingests: number; + deltaIngests: number; + verifySamples: number; + verifyOk: number; + verifyDrift: number; +} + +function openShadow(dir: string): UnifiedShadow { + UnifiedStore.open(path.join(dir, "index.db")); + const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as LegacySessionLike); + assert.ok(shadow, "shadow mode sidecar must open"); + return shadow; +} + +function stats(shadow: UnifiedShadow): StatsView { + return (shadow as unknown as { stats: StatsView }).stats; +} + +// --- Unit: canonicalize is order-insensitive for objects, order-preserving for arrays --- + +test("#4392: canonicalize digests object key sets, not insertion order", () => { + assert.equal( + JSON.stringify(canonicalize({ state: { messageRefs: { byRef: { m00002: "b", m00001: "a" } } }, stats: { requests: 1 } })), + JSON.stringify(canonicalize({ stats: { requests: 1 }, state: { messageRefs: { byRef: { m00001: "a", m00002: "b" } } } })), + "the same key set in a different order must digest identically", + ); + assert.equal(sha256(JSON.stringify(canonicalize({ b: 1, a: 2 }))), sha256(JSON.stringify(canonicalize({ a: 2, b: 1 })))); + assert.notEqual(sha256(JSON.stringify(canonicalize({ a: 1 }))), sha256(JSON.stringify(canonicalize({ a: 2 }))), "values still matter"); + assert.notEqual(sha256(JSON.stringify(canonicalize({ a: 1 }))), sha256(JSON.stringify(canonicalize({ a: 1, b: 2 }))), "the key SET still matters"); +}); + +test("#4392: arrays keep their order; Maps/Sets keep their canonical envelope", () => { + assert.notEqual( + sha256(JSON.stringify(canonicalize([{ id: "b1" }, { id: "b2" }]))), + sha256(JSON.stringify(canonicalize([{ id: "b2" }, { id: "b1" }]))), + "array order is content — blocks/tail order must still trip the digest", + ); + assert.notEqual( + sha256(JSON.stringify(canonicalize(new Set(["a", "b"])))), + sha256(JSON.stringify(canonicalize(new Set(["b", "a"])))), + "Set insertion order stays part of the canonical form (nothing changed there)", + ); + assert.equal( + JSON.stringify(canonicalize(new Map([["b", 1]]))), + JSON.stringify({ __map__: [["b", 1]] }), + "Map envelope shape unchanged", + ); +}); + +// --- Unit: the skip table is ONE contract, not a per-producer list --- + +test("#4392: DIGEST_SKIPPED_KEYS is the single skip contract every digest path shares", () => { + // Pinned literally: dropping a key changes every golden pin, so it must be + // a conscious re-pin. The streamed `canonicalDigest` (#4532) imports THIS + // table instead of hand-copying a list — two hand-maintained skip lists are + // how the golden corpus and the shadow parity check end up hashing two + // different canonical forms of the same session. + assert.deepEqual(Object.keys(DIGEST_SKIPPED_KEYS).sort(), ["storageMode", "tailGaps"]); + assert.equal( + JSON.stringify(canonicalize({ + state: { tailGaps: { m00002: 1 }, keep: "x" }, + metadata: { storageMode: "full", biliVersion: "0.1.208" }, + stats: { requests: 3 }, + })), + JSON.stringify({ metadata: { biliVersion: "__VER__" }, state: { keep: "x" }, stats: { requests: 3 } }), + "skipped keys are dropped at every depth; biliVersion is masked, never dropped", + ); +}); + +// --- E2E: the field shape — byRef rebuilt in ledger order, same contents --- + +test("#4392: byRef reordered by the ref-ledger rebuild must not trip the parity digest", () => { + const dir = tmpdir("bili-4392-order-"); + try { + // The kernel minted m00001 before m00002, but byRaw's insertion order + // (what the refs ledger is written in) starts at raw-a — so the replay + // leg rebuilds byRef { m00001, m00002 } while the live view holds + // { m00002, m00001 }: identical key set, identical values. + const liveState: StateShape = { + blocks: [], + messageRefs: { + byRaw: { "raw-a": "m00001", "raw-b": "m00002" }, + byRef: { m00002: "raw-b", m00001: "raw-a" }, + }, + nextBlockId: 1, + nextRunId: 1, + }; + const legacy = mkLegacy("state-order", liveState); + const replayed = loadLegacyView(planIngest(legacy) as never); + // Unchecked cast: loadLegacyView types state as Record + // (the view is store-shaped, not kernel-typed); this fixture owns it. + const replayedState = replayed.state as unknown as StateShape; + const liveRefs = liveState.messageRefs.byRef; + const replayRefs = replayedState.messageRefs.byRef; + assert.deepEqual(Object.keys(liveRefs), ["m00002", "m00001"], "live leg holds byRef in kernel-mint order"); + assert.deepEqual(Object.keys(replayRefs), ["m00001", "m00002"], "replay leg holds byRef in ledger order"); + assert.deepEqual({ ...liveRefs }, { ...replayRefs }, "…with identical contents"); + + const shadow = openShadow(dir); + shadow.record(legacy); + const s = stats(shadow); + assert.equal(s.verifySamples, 1, "first save must run the sampled verify"); + assert.equal(s.verifyDrift, 0, "an order-only difference must not drift (pre-fix: 1)"); + assert.equal(s.verifyOk, 1, "the sampled verify must pass"); + } finally { + rmBestEffort(dir); + } +}); + +test("#4392: a real messageRefs content difference still drifts", () => { + const dir = tmpdir("bili-4392-content-"); + try { + const shadow = openShadow(dir); + // Same key set and order, one different mapping: the digest must still + // catch a genuine fork between the two legs (the #4323 heal path is + // what depends on this direction). + const legacy = mkLegacy("state-content", { + blocks: [], + messageRefs: { + byRaw: { "raw-a": "m00001", "raw-b": "m00002" }, + byRef: { m00001: "raw-a", m00002: "raw-c" }, + }, + nextBlockId: 1, + nextRunId: 1, + }); + shadow.record(legacy); + const s = stats(shadow); + assert.equal(s.verifySamples, 1, "first save must run the sampled verify"); + assert.equal(s.verifyDrift, 1, "a divergent ref mapping must still be reported"); + } finally { + rmBestEffort(dir); + } +}); diff --git a/tests/issue4986-digest-drift-ref-order.test.ts b/tests/issue4986-digest-drift-ref-order.test.ts new file mode 100644 index 0000000000..d427f387b1 --- /dev/null +++ b/tests/issue4986-digest-drift-ref-order.test.ts @@ -0,0 +1,91 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { SessionStore } from "../src/persist.ts"; + +/** + * #4986: the sampled ingest read-back verify must not report DIGEST DRIFT + * when the in-memory messageRefs key order diverges from the stored + * refs-row order. The golden corpora are exactly that shape: byRef sits in + * ref-number order while byRaw — which finishPlan freezes into row ord at + * full ingest — sits in a different key order, so the row-rebuilt leg's + * byRef starts at m00041 while the live leg's starts at m00001. Identical + * content, different key insertion order. Pre-fix, every such save logged + * an error-level DIGEST DRIFT and dropped the ingest watermark (full-tail + * re-ingest next save); post-fix the parity digest compares contents, not + * key insertion order (#3407/#3113 philosophy). + */ + +process.env.BILI_PERSIST_ZSTD = "0"; +const hermetic = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4986-hermetic-")); +process.env.BILI_CONFIG_FILE = path.join(hermetic, "no-config.json"); +process.env.BILI_STORAGE_UNIFIED = "legacy"; + +const CORPUS_ROOT = path.resolve(import.meta.dirname, "golden/storage-corpus"); + +function loadCorpusCase(dir: string, name: string, id: string, protocol: string) { + fs.cpSync(path.join(CORPUS_ROOT, name), path.join(dir, name), { recursive: true }); + const loader = new SessionStore({ dir: path.join(dir, name), debounceMs: 1, enabled: true }); + const loaded = loader.loadSync(id, { protocol, upstreamOrigin: "http://127.0.0.1:8199" }); + loader.cancelAll(); + loader.closeQuietly(); + assert.ok(loaded, `${name}: corpus loadSync returned null`); + return loaded; +} + +/** Windows: best-effort delete, swallowed on persistent EBUSY/EPERM. */ +function rmBestEffort(p: string): void { + try { + fs.rmSync(p, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); + } catch { + /* leaked temp dir on a Windows runner is harmless */ + } +} + +test("ingest verify ignores messageRefs key-order divergence (#4986)", async () => { + const prevMode = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "full"; + try { + const seed = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4986-seed-")); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4986-store-")); + try { + const session = loadCorpusCase(seed, "folded", "corpus-folded", "anthropic"); + + // Precondition: the corpus exhibits the divergent shape — the + // row order (byRaw key order, what finishPlan stores as ord) + // projects byRef to a DIFFERENT key sequence than the live map's + // own key order. Without this the test would pass vacuously. + const byRaw = session.state.messageRefs.byRaw; + const byRef = session.state.messageRefs.byRef; + const rowOrderProjectedRefs = Object.keys(byRaw) + .map((rawId) => byRaw[rawId]) + .filter((ref) => ref !== "BLOCKED" && byRef[ref] !== undefined); + assert.notEqual( + Object.keys(byRef).join("\u0000"), + rowOrderProjectedRefs.join("\u0000"), + "folded corpus must carry byRef key order != stored refs-row order", + ); + + const store = new SessionStore({ dir, debounceMs: 1, enabled: true }); + await store.writeNow(session); + const shadow = (store as unknown as { + shadow?: { stats: { verifySamples: number; verifyOk: number; verifyDrift: number } }; + }).shadow; + assert.ok(shadow, "full mode must open the unified sidecar"); + // The FIRST ingest always samples the read-back verify... + assert.equal(shadow.stats.verifySamples, 1, "first ingest must sample the read-back verify"); + // ...and key-order-only divergence is not drift. + assert.equal(shadow.stats.verifyDrift, 0, "#4986: order-only ref-map divergence must not report DIGEST DRIFT"); + assert.equal(shadow.stats.verifyOk, 1); + store.cancelAll(); + } finally { + rmBestEffort(dir); + rmBestEffort(seed); + } + } finally { + process.env.BILI_STORAGE_UNIFIED = prevMode; + rmBestEffort(hermetic); + } +});