From a3fa9fd4ee5a27116760f382da8dd51c403e9dfe Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 20:36:20 +0800 Subject: [PATCH 1/3] =?UTF-8?q?fix:=20full=20mode=20=E2=80=94=20serve=20th?= =?UTF-8?q?e=20legacy=20envelope=20when=20the=20unified=20view=20drops=20a?= =?UTF-8?q?rchived=20originals=20(#3831)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/persist.ts | 78 +++++++++++++++----- tests/issue3831-orphan-blockcontents.test.ts | 71 ++++++++++++++++++ 2 files changed, 129 insertions(+), 20 deletions(-) create mode 100644 tests/issue3831-orphan-blockcontents.test.ts diff --git a/src/persist.ts b/src/persist.ts index d8e9c5bc0a..7673d27bae 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -957,7 +957,12 @@ export class SessionStore { this.shadow.importRecord(payload); } const view = this.shadow.viewFor(id); - if (view) return buildSession(viewToRecord(view)); + // #3831: the unified view drops archived originals that have no + // state.blocks slot — when the payload we just parsed still + // carries them, the legacy build below is the faithful source. + if (view && !this.originalsLostInView(payload, view)) { + return buildSession(viewToRecord(view)); + } } return buildSession(payload); } @@ -1185,38 +1190,51 @@ export class SessionStore { if (!this.enabled) return null; if (this.shadow?.readsUnified) { const unified = this.loadUnified(id); - if (unified) return this.afterLoad(id, unified); + if (unified) { + // #3831: the unified view has no standalone blockContents + // carrier — when the legacy envelope still holds archived + // originals the view dropped (no state.blocks slot), serve + // the envelope so decompress can restore them. + const envelope = this.loadEnvelope(id, meta); + if (!(envelope && this.originalsLostInView(envelope.payload, unified))) { + return this.afterLoad(id, unified); + } + this.log("warn", `[persist] ${id}: unified view dropped archived original(s) the legacy envelope still holds — serving the envelope instead (#3831)`); + return this.serveFromEnvelope(id, envelope); + } // Miss (or sidecar failure): organic import from the legacy file, // then serve from the index; fall back to the plain legacy load - // when even the import cannot run. + // when even the import cannot run — or when the imported view + // would drop originals the envelope still carries (#3831). const envelope = this.loadEnvelope(id, meta); if (!envelope) return null; const imported = this.shadow.importRecord(envelope.payload); if (imported) { const view = this.shadow.viewFor(id); - if (view) return this.afterLoad(id, view); - } - // Legacy fallback — same single-build semantics as plain legacy - // mode (see the note below on why no adapter round-trip here). - const session = buildSession(envelope.payload); - if (hasNegativePersistedTokens(envelope.payload)) { - this.scheduleSave(session); - this.log("info", `[persist] clamped negative token stats on reload for ${id} (#408)`); + if (view && !this.originalsLostInView(envelope.payload, view)) { + return this.afterLoad(id, view); + } } - this.logColdLoad(id, "legacy file", session, envelope.payload.savedAt); - return session; + return this.serveFromEnvelope(id, envelope); } const envelope = this.loadEnvelope(id, meta); if (!envelope) return null; - // Legacy path — intentionally the ORIGINAL single buildSession(payload): - // wrapping this in the toLegacyLike/viewToRecord adapter (like the - // unified branch above) would run the record through a second - // normalization round-trip and reshape persisted-state bytes (the - // storage-render goldens catch exactly that). + return this.serveFromEnvelope(id, envelope); + } + + /** Serve a session straight from its legacy envelope: the ORIGINAL single + * buildSession(payload) — wrapping it in the toLegacyLike/viewToRecord + * adapter would run the record through a second normalization round-trip + * and reshape persisted-state bytes (the storage-render goldens catch + * exactly that) — plus the #408 negative-token clamp rewrite and the + * cold-load log. The one home for legacy single-build semantics: the + * plain-legacy branch and the #3831 orphan-drop escape hatch both route + * here. */ + private serveFromEnvelope(id: string, envelope: PersistedEnvelope): Session { const session = buildSession(envelope.payload); if (hasNegativePersistedTokens(envelope.payload)) { - // #408: sync context — debounce the stale-file rewrite - // (buildSession already clamped the in-memory value). + // #408: debounce the stale-file rewrite (buildSession already + // clamped the in-memory value). this.scheduleSave(session); this.log("info", `[persist] clamped negative token stats on reload for ${id} (#408)`); } @@ -1224,6 +1242,26 @@ export class SessionStore { return session; } + /** #3831: does the persisted record carry archived originals — blockContents + * entries with NO state.blocks slot (zombie-reaped folds, #2695; compaction- + * archive prunes) — that a unified view silently drops? The unified schema + * has no standalone blockContents carrier: ingest rebuilds the map ONLY from + * block rows (ingest.ts loadLegacyView), so any recorded key without a block + * slot is lost on a unified read. The record is the authoritative source for + * blockContents, so whenever it holds keys the view lacks, callers fall back + * to the legacy build to restore the bytes. */ + private originalsLostInView( + record: { blockContents?: Record | null }, + view: LegacySessionLike, + ): boolean { + const rec = record.blockContents; + if (!rec) return false; + const recKeys = Object.keys(rec); + if (recKeys.length === 0) return false; + const viewKeys = new Set(view.blockContents.keys()); + return recKeys.some((k) => !viewKeys.has(k)); + } + /** Shared post-load fixups for the unified read path: rebuilds the * Session from a store view (viewToRecord → buildSession, the same * forward-compat pass as a legacy load) and applies the #408 clamp diff --git a/tests/issue3831-orphan-blockcontents.test.ts b/tests/issue3831-orphan-blockcontents.test.ts new file mode 100644 index 0000000000..ea25c8cfe7 --- /dev/null +++ b/tests/issue3831-orphan-blockcontents.test.ts @@ -0,0 +1,71 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import * as path from "node:path"; +import { SessionStore, _setStoreForTest, relPathFor } from "../src/persist.ts"; +import { createSession, _resetSessionsForTest } from "../src/session.ts"; + +// #3831 hermeticity: pin the config tier so a machine-level storage.unified +// cannot leak into these fixtures (same pattern as storage-unified.test.ts). +process.env.BILI_CONFIG_FILE = path.join( + mkdtempSync(path.join(tmpdir(), "bili-issue3831-")), + "billion-context.json", +); + +/** Full-mode fixture: the defect only manifests when the unified index is + * the primary read source, so every store here boots under explicit full. */ +async function fullFixture(run: (dir: string, store: SessionStore) => Promise): Promise { + const root = mkdtempSync(path.join(tmpdir(), "bili-issue3831-")); + const dir = path.join(root, "sessions"); + const modeBefore = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "full"; + const store = new SessionStore({ dir, enabled: true, debounceMs: 0 }); + _resetSessionsForTest(256, 512 * 1024 * 1024); + _setStoreForTest(store); + try { await run(dir, store); } + finally { + store.closeQuietly(); + if (modeBefore === undefined) delete process.env.BILI_STORAGE_UNIFIED; + else process.env.BILI_STORAGE_UNIFIED = modeBefore; + _resetSessionsForTest(256, 512 * 1024 * 1024); + _setStoreForTest(new SessionStore({ enabled: false })); + rmSync(root, { recursive: true, force: true }); + } +} + +test("full-mode reload restores archived originals the unified view drops (#3831)", async () => fullFixture(async (_dir, store) => { + // Zombie-reaped fold shape (#2695): state.blocks empty, blockContents + // still holding the archived original — legitimate persisted state. + const session = createSession("ses_orphan", { label: "ses_orphan" }); + session.state.messageRefs.byRaw["old-message"] = "m00001"; + session.state.messageRefs.byRef["m00001"] = "old-message"; + session.blockContents.set("b1", { one: null, full: { text: "archived original\r\n", count: 1 } }); + await store.writeNow(session); + const booted = await store.boot({ maxSessions: 8, maxBytes: 40000 }); + // Boot warm path (buildForLoadAll): the warmed session must carry the + // original bytes, not the lossy view. + assert.deepEqual(booted.get("ses_orphan")?.blockContents.get("b1"), session.blockContents.get("b1")); + // Cold path (loadSync): same guarantee after the index row exists. + const loaded = store.loadSync("ses_orphan"); + assert.equal(loaded?.id, "ses_orphan"); + assert.deepEqual(loaded?.state.messageRefs, session.state.messageRefs); + assert.deepEqual(loaded?.blockContents.get("b1"), { one: null, full: { text: "archived original\r\n", count: 1 } }); +})); + +test("full mode without orphaned originals keeps serving the unified index (#3831 negative)", async () => fullFixture(async (dir, store) => { + // Gate discipline (#3617): the #3831 escape hatch must NOT trigger for a + // session whose record carries no orphaned contents — the index view + // stays the source of truth, even once the legacy envelope is gone. + const session = createSession("ses_plain", { label: "ses_plain" }); + session.state.messageRefs.byRaw["plain-message"] = "m00001"; + session.state.messageRefs.byRef["m00001"] = "plain-message"; + session.stats.requests = 7; + await store.writeNow(session); + await store.boot({ maxSessions: 8, maxBytes: 40000 }); + rmSync(path.join(dir, relPathFor(session.id)), { force: true }); + const loaded = store.loadSync("ses_plain"); + assert.equal(loaded?.id, "ses_plain"); + assert.equal(loaded?.stats.requests, 7); + assert.deepEqual(loaded?.state.messageRefs, session.state.messageRefs); +})); From 3cc9df17e431b83c7d06d80ace5261b1da7463bc Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 22:22:56 +0800 Subject: [PATCH 2/3] test: pin the no-limits boot/export leg of the #3831 guard; correct the warm-path comment (#3835 review) --- tests/issue3831-orphan-blockcontents.test.ts | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/issue3831-orphan-blockcontents.test.ts b/tests/issue3831-orphan-blockcontents.test.ts index ea25c8cfe7..bee126d413 100644 --- a/tests/issue3831-orphan-blockcontents.test.ts +++ b/tests/issue3831-orphan-blockcontents.test.ts @@ -43,8 +43,9 @@ test("full-mode reload restores archived originals the unified view drops (#3831 session.blockContents.set("b1", { one: null, full: { text: "archived original\r\n", count: 1 } }); await store.writeNow(session); const booted = await store.boot({ maxSessions: 8, maxBytes: 40000 }); - // Boot warm path (buildForLoadAll): the warmed session must carry the - // original bytes, not the lossy view. + // Bounded-boot warm loop (loadSync per warmed id — production boot always + // passes limits): the warmed session must carry the original bytes, not + // the lossy view. assert.deepEqual(booted.get("ses_orphan")?.blockContents.get("b1"), session.blockContents.get("b1")); // Cold path (loadSync): same guarantee after the index row exists. const loaded = store.loadSync("ses_orphan"); @@ -53,6 +54,20 @@ test("full-mode reload restores archived originals the unified view drops (#3831 assert.deepEqual(loaded?.blockContents.get("b1"), { one: null, full: { text: "archived original\r\n", count: 1 } }); })); +test("full-mode unlimited boot/export (buildForLoadAll) restores archived originals (#3831)", async () => fullFixture(async (_dir, store) => { + // Third serving leg: no-limits boot (and the session export that shares + // loadAll) imports the parsed envelope and serves the view — the same + // orphan-drop exposure, pinned separately because production boot always + // passes limits (bootBounded → loadSync). + const session = createSession("ses_orphan_ub", { label: "ses_orphan_ub" }); + session.state.messageRefs.byRaw["old-message"] = "m00001"; + session.state.messageRefs.byRef["m00001"] = "old-message"; + session.blockContents.set("b1", { one: null, full: { text: "archived original\r\n", count: 1 } }); + await store.writeNow(session); + const booted = await store.boot(); + assert.deepEqual(booted.get("ses_orphan_ub")?.blockContents.get("b1"), { one: null, full: { text: "archived original\r\n", count: 1 } }); +})); + test("full mode without orphaned originals keeps serving the unified index (#3831 negative)", async () => fullFixture(async (dir, store) => { // Gate discipline (#3617): the #3831 escape hatch must NOT trigger for a // session whose record carries no orphaned contents — the index view From c7737b45ede11a10356528a6140076f0be877fac Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 22:22:56 +0800 Subject: [PATCH 3/3] fix: warn on every #3831 escape-hatch firing, not just the unified-hit path (#3835 review) --- src/persist.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/persist.ts b/src/persist.ts index 7673d27bae..2a1c7d4e8a 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -963,6 +963,7 @@ export class SessionStore { if (view && !this.originalsLostInView(payload, view)) { return buildSession(viewToRecord(view)); } + if (view) this.logOrphanFallback(id); } return buildSession(payload); } @@ -1199,7 +1200,7 @@ export class SessionStore { if (!(envelope && this.originalsLostInView(envelope.payload, unified))) { return this.afterLoad(id, unified); } - this.log("warn", `[persist] ${id}: unified view dropped archived original(s) the legacy envelope still holds — serving the envelope instead (#3831)`); + this.logOrphanFallback(id); return this.serveFromEnvelope(id, envelope); } // Miss (or sidecar failure): organic import from the legacy file, @@ -1214,6 +1215,7 @@ export class SessionStore { if (view && !this.originalsLostInView(envelope.payload, view)) { return this.afterLoad(id, view); } + if (view) this.logOrphanFallback(id); } return this.serveFromEnvelope(id, envelope); } @@ -1262,6 +1264,14 @@ export class SessionStore { return recKeys.some((k) => !viewKeys.has(k)); } + /** One warn per #3831 escape-hatch firing — all three full-mode serving + * paths (loadSync unified hit, loadSync import miss, buildForLoadAll + * boot/export warm-up) route through it so a grep for the marker finds + * every session served from the envelope for orphan-drop reasons. */ + private logOrphanFallback(id: string): void { + this.log("warn", `[persist] ${id}: unified view dropped archived original(s) the legacy envelope still holds — serving the envelope instead (#3831)`); + } + /** Shared post-load fixups for the unified read path: rebuilds the * Session from a store view (viewToRecord → buildSession, the same * forward-compat pass as a legacy load) and applies the #408 clamp