Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 68 additions & 20 deletions src/persist.ts
Original file line number Diff line number Diff line change
Expand Up @@ -957,7 +957,13 @@ export class SessionStore {
this.shadow.importRecord(payload);
}
const view = this.shadow.viewFor(id);
if (view) return buildSession(viewToRecord(view));
// #3831: the unified view drops archived originals that have no
// state.blocks slot — when the payload we just parsed still
// carries them, the legacy build below is the faithful source.
if (view && !this.originalsLostInView(payload, view)) {
return buildSession(viewToRecord(view));
}
if (view) this.logOrphanFallback(id);
}
return buildSession(payload);
}
Expand Down Expand Up @@ -1185,45 +1191,87 @@ export class SessionStore {
if (!this.enabled) return null;
if (this.shadow?.readsUnified) {
const unified = this.loadUnified(id);
if (unified) return this.afterLoad(id, unified);
if (unified) {
// #3831: the unified view has no standalone blockContents
// carrier — when the legacy envelope still holds archived
// originals the view dropped (no state.blocks slot), serve
// the envelope so decompress can restore them.
const envelope = this.loadEnvelope(id, meta);
if (!(envelope && this.originalsLostInView(envelope.payload, unified))) {
return this.afterLoad(id, unified);
}
this.logOrphanFallback(id);
return this.serveFromEnvelope(id, envelope);
}
// Miss (or sidecar failure): organic import from the legacy file,
// then serve from the index; fall back to the plain legacy load
// when even the import cannot run.
// when even the import cannot run — or when the imported view
// would drop originals the envelope still carries (#3831).
const envelope = this.loadEnvelope(id, meta);
if (!envelope) return null;
const imported = this.shadow.importRecord(envelope.payload);
if (imported) {
const view = this.shadow.viewFor(id);
if (view) return this.afterLoad(id, view);
}
// Legacy fallback — same single-build semantics as plain legacy
// mode (see the note below on why no adapter round-trip here).
const session = buildSession(envelope.payload);
if (hasNegativePersistedTokens(envelope.payload)) {
this.scheduleSave(session);
this.log("info", `[persist] clamped negative token stats on reload for ${id} (#408)`);
if (view && !this.originalsLostInView(envelope.payload, view)) {
return this.afterLoad(id, view);
}
if (view) this.logOrphanFallback(id);
}
this.logColdLoad(id, "legacy file", session, envelope.payload.savedAt);
return session;
return this.serveFromEnvelope(id, envelope);
}
const envelope = this.loadEnvelope(id, meta);
if (!envelope) return null;
// Legacy path — intentionally the ORIGINAL single buildSession(payload):
// wrapping this in the toLegacyLike/viewToRecord adapter (like the
// unified branch above) would run the record through a second
// normalization round-trip and reshape persisted-state bytes (the
// storage-render goldens catch exactly that).
return this.serveFromEnvelope(id, envelope);
}

/** Serve a session straight from its legacy envelope: the ORIGINAL single
* buildSession(payload) — wrapping it in the toLegacyLike/viewToRecord
* adapter would run the record through a second normalization round-trip
* and reshape persisted-state bytes (the storage-render goldens catch
* exactly that) — plus the #408 negative-token clamp rewrite and the
* cold-load log. The one home for legacy single-build semantics: the
* plain-legacy branch and the #3831 orphan-drop escape hatch both route
* here. */
private serveFromEnvelope(id: string, envelope: PersistedEnvelope<PersistedSession>): Session {
const session = buildSession(envelope.payload);
if (hasNegativePersistedTokens(envelope.payload)) {
// #408: sync context — debounce the stale-file rewrite
// (buildSession already clamped the in-memory value).
// #408: debounce the stale-file rewrite (buildSession already
// clamped the in-memory value).
this.scheduleSave(session);
this.log("info", `[persist] clamped negative token stats on reload for ${id} (#408)`);
}
this.logColdLoad(id, "legacy file", session, envelope.payload.savedAt);
return session;
}

/** #3831: does the persisted record carry archived originals — blockContents
* entries with NO state.blocks slot (zombie-reaped folds, #2695; compaction-
* archive prunes) — that a unified view silently drops? The unified schema
* has no standalone blockContents carrier: ingest rebuilds the map ONLY from
* block rows (ingest.ts loadLegacyView), so any recorded key without a block
* slot is lost on a unified read. The record is the authoritative source for
* blockContents, so whenever it holds keys the view lacks, callers fall back
* to the legacy build to restore the bytes. */
private originalsLostInView(
record: { blockContents?: Record<string, unknown> | null },
view: LegacySessionLike,
): boolean {
const rec = record.blockContents;
if (!rec) return false;
const recKeys = Object.keys(rec);
if (recKeys.length === 0) return false;
const viewKeys = new Set(view.blockContents.keys());
return recKeys.some((k) => !viewKeys.has(k));
}

/** One warn per #3831 escape-hatch firing — all three full-mode serving
* paths (loadSync unified hit, loadSync import miss, buildForLoadAll
* boot/export warm-up) route through it so a grep for the marker finds
* every session served from the envelope for orphan-drop reasons. */
private logOrphanFallback(id: string): void {
this.log("warn", `[persist] ${id}: unified view dropped archived original(s) the legacy envelope still holds — serving the envelope instead (#3831)`);
}

/** Shared post-load fixups for the unified read path: rebuilds the
* Session from a store view (viewToRecord → buildSession, the same
* forward-compat pass as a legacy load) and applies the #408 clamp
Expand Down
86 changes: 86 additions & 0 deletions tests/issue3831-orphan-blockcontents.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import * as path from "node:path";
import { SessionStore, _setStoreForTest, relPathFor } from "../src/persist.ts";
import { createSession, _resetSessionsForTest } from "../src/session.ts";

// #3831 hermeticity: pin the config tier so a machine-level storage.unified
// cannot leak into these fixtures (same pattern as storage-unified.test.ts).
process.env.BILI_CONFIG_FILE = path.join(
mkdtempSync(path.join(tmpdir(), "bili-issue3831-")),
"billion-context.json",
);

/** Full-mode fixture: the defect only manifests when the unified index is
* the primary read source, so every store here boots under explicit full. */
async function fullFixture(run: (dir: string, store: SessionStore) => Promise<void>): Promise<void> {
const root = mkdtempSync(path.join(tmpdir(), "bili-issue3831-"));
const dir = path.join(root, "sessions");
const modeBefore = process.env.BILI_STORAGE_UNIFIED;
process.env.BILI_STORAGE_UNIFIED = "full";
const store = new SessionStore({ dir, enabled: true, debounceMs: 0 });
_resetSessionsForTest(256, 512 * 1024 * 1024);
_setStoreForTest(store);
try { await run(dir, store); }
finally {
store.closeQuietly();
if (modeBefore === undefined) delete process.env.BILI_STORAGE_UNIFIED;
else process.env.BILI_STORAGE_UNIFIED = modeBefore;
_resetSessionsForTest(256, 512 * 1024 * 1024);
_setStoreForTest(new SessionStore({ enabled: false }));
rmSync(root, { recursive: true, force: true });
}
}

test("full-mode reload restores archived originals the unified view drops (#3831)", async () => fullFixture(async (_dir, store) => {
// Zombie-reaped fold shape (#2695): state.blocks empty, blockContents
// still holding the archived original — legitimate persisted state.
const session = createSession("ses_orphan", { label: "ses_orphan" });
session.state.messageRefs.byRaw["old-message"] = "m00001";
session.state.messageRefs.byRef["m00001"] = "old-message";
session.blockContents.set("b1", { one: null, full: { text: "archived original\r\n", count: 1 } });
await store.writeNow(session);
const booted = await store.boot({ maxSessions: 8, maxBytes: 40000 });
// Bounded-boot warm loop (loadSync per warmed id — production boot always
// passes limits): the warmed session must carry the original bytes, not
// the lossy view.
assert.deepEqual(booted.get("ses_orphan")?.blockContents.get("b1"), session.blockContents.get("b1"));
// Cold path (loadSync): same guarantee after the index row exists.
const loaded = store.loadSync("ses_orphan");
assert.equal(loaded?.id, "ses_orphan");
assert.deepEqual(loaded?.state.messageRefs, session.state.messageRefs);
assert.deepEqual(loaded?.blockContents.get("b1"), { one: null, full: { text: "archived original\r\n", count: 1 } });
}));

test("full-mode unlimited boot/export (buildForLoadAll) restores archived originals (#3831)", async () => fullFixture(async (_dir, store) => {
// Third serving leg: no-limits boot (and the session export that shares
// loadAll) imports the parsed envelope and serves the view — the same
// orphan-drop exposure, pinned separately because production boot always
// passes limits (bootBounded → loadSync).
const session = createSession("ses_orphan_ub", { label: "ses_orphan_ub" });
session.state.messageRefs.byRaw["old-message"] = "m00001";
session.state.messageRefs.byRef["m00001"] = "old-message";
session.blockContents.set("b1", { one: null, full: { text: "archived original\r\n", count: 1 } });
await store.writeNow(session);
const booted = await store.boot();
assert.deepEqual(booted.get("ses_orphan_ub")?.blockContents.get("b1"), { one: null, full: { text: "archived original\r\n", count: 1 } });
}));

test("full mode without orphaned originals keeps serving the unified index (#3831 negative)", async () => fullFixture(async (dir, store) => {
// Gate discipline (#3617): the #3831 escape hatch must NOT trigger for a
// session whose record carries no orphaned contents — the index view
// stays the source of truth, even once the legacy envelope is gone.
const session = createSession("ses_plain", { label: "ses_plain" });
session.state.messageRefs.byRaw["plain-message"] = "m00001";
session.state.messageRefs.byRef["m00001"] = "plain-message";
session.stats.requests = 7;
await store.writeNow(session);
await store.boot({ maxSessions: 8, maxBytes: 40000 });
rmSync(path.join(dir, relPathFor(session.id)), { force: true });
const loaded = store.loadSync("ses_plain");
assert.equal(loaded?.id, "ses_plain");
assert.equal(loaded?.stats.requests, 7);
assert.deepEqual(loaded?.state.messageRefs, session.state.messageRefs);
}));
Loading