From 8849b66bfd3bca7e25f2e2c352b09d8869ffae8e Mon Sep 17 00:00:00 2001 From: ework-agent Date: Thu, 12 Nov 2026 08:26:58 +0800 Subject: [PATCH 1/4] fix: persist ledger rows before the model call (durability-before-render) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A turn's refs are minted in the prepare phase but their ledger rows land in the DEBOUNCED save that runs AFTER the model call (#5155). A crash in that window (power loss / disconnect / SIGKILL) leaves the rendered refs with no durable rows; on cold-load the refs and the ledger disagree, and a later turn that cites them hits "unknown ref / above the highest allocated ref" — the #3397 orphaned-generation gap. Make the rows durable BEFORE the refs are sent to the model: - add SessionStore.ingestRowsNow(): the debounced path's row-ingest legs (record build + stale-disk guard + shadow.record + own-commit stamp) with the JSON envelope write removed — just the small SQLite rows, synchronously. - call it in all four prepare phases (anthropic/google/openai/responses) between snapshotMessages() and markDirty() — after the refs are minted and the history snapshotted, before the body is built and forwarded. - call it again after forward() in the incremental lane as a safety net (idempotent); the debounced envelope save below it is unchanged (#5155). The JSON envelope stays debounced; only the small row ingest moves ahead of the model call. Verified with an isolated crash repro: a SIGKILL while the model call is in flight no longer loses the turn's rows (durable == rendered, gap=0; before the fix gap=5). --- src/incremental-lane.ts | 13 +++++++++++ src/persist.ts | 38 +++++++++++++++++++++++++++++++++ src/server/prepare-anthropic.ts | 9 ++++++++ src/server/prepare-google.ts | 3 +++ src/server/prepare-openai.ts | 3 +++ src/server/prepare-responses.ts | 3 +++ 6 files changed, 69 insertions(+) diff --git a/src/incremental-lane.ts b/src/incremental-lane.ts index 1fd9e0343c..aadb5f3908 100644 --- a/src/incremental-lane.ts +++ b/src/incremental-lane.ts @@ -1005,6 +1005,19 @@ export async function handleIncrementalTurn( inFlight.delete(conversationId); const session = peekSession(sessionId); settled = true; + // #6150-fix: durability-before-render — the turn's ledger rows are made + // durable NOW (synchronous unified-index ingest) so a crash before the + // debounced envelope save (#5155) cannot leave the refs the model just + // saw unbacked (the #3397 orphaned-generation gap: the lane rendered the + // delta, the model could cite it, but the rows only settled in the + // debounced flush). The large JSON envelope itself stays debounced — + // only the small row ingest moves ahead of it. Best-effort: the + // debounced save below retries the full path on failure. + try { + if (session) getStore().ingestRowsNow(session); + } catch { + // best-effort, same failure posture as the debounced save + } // #5155: the debounced save coalesces consecutive turns' full-envelope // flushes (a synchronous writeNow per turn serialized + fsynced the // whole ~24MB envelope on large histories); pointer upkeep no longer diff --git a/src/persist.ts b/src/persist.ts index 718c04c60f..539b363f0e 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -3162,6 +3162,44 @@ export class SessionStore { loggerLog("error", `[persist] ${sessionId}: skipping ${route} envelope save — JSON envelope writes FROZEN since ${since} (digest-drift escalation #4323); the unified store keeps receiving the save and compression continues from the in-memory view, but the two copies are NOT converging — see the [storage] DIGEST DRIFT lines for the divergence diagnosis (#5408)`); } + /** #6150-fix: durability-before-render — make a session's unified-index + * rows durable SYNCHRONOUSLY (the small SQLite ingest) WITHOUT writing the + * JSON envelope. The incremental lane calls this the instant a turn's + * response has fully left, so the refs the model just saw are on disk before + * the debounced envelope save (#5155) settles. A crash in that window no + * longer leaves rendered refs unbacked (the #3397 orphaned-generation gap + * that made long sessions cite refs the gate rejects as "above the highest + * allocated ref"). The large JSON envelope stays debounced — only the small + * row ingest moves ahead of it. Mirrors the debounced path's ingest legs + * (stale-disk guard + rawTail + live CCR + noteOwnCommit), minus the + * envelope write and its landing stamp. */ + ingestRowsNow(session: Session): void { + if (!this.shadow) return; // legacy-only (no unified index) — nothing to back + try { + const stamp = this.liveStorageStamp; + const record = this.buildRecordForSave(session, stamp); + const disk = this.staleDiskPayload(record); + if (disk !== null) { + // #2401: disk is strictly newer — converge + ingest the disk view + // (same as the debounced path, minus the envelope rewrite). + this.convergeToDisk(session, disk); + this.shadow.record(disk); + this.noteOwnCommit(session.id, disk); + return; + } + const payload = session.decompositionLog; + if (payload !== undefined) session.decompositionLog = undefined; + const tail = session.lastMessagesFolded !== true ? session.lastMessages : undefined; + this.shadow.record(payload !== undefined ? { ...record, decompositionLog: payload } : record, + tail && tail.length > 0 + ? { lastMessages: tail, lastMessagesFolded: session.lastMessagesFolded } + : undefined, session.contentStore ?? undefined); + this.noteOwnCommit(session.id, record); + } catch (err) { + this.log("warn", `[persist] ingestRowsNow failed for ${session.id}: ${String(err)} — the debounced save retries the full path`); + } + } + scheduleSave(session: Session): void { // #5608-D: a committed fold IS the session's next legitimate // cache-breaking boundary — run the deferred #4316 citation-space diff --git a/src/server/prepare-anthropic.ts b/src/server/prepare-anthropic.ts index a61fb4833f..3c4443b296 100644 --- a/src/server/prepare-anthropic.ts +++ b/src/server/prepare-anthropic.ts @@ -17,6 +17,7 @@ import { storedMessagesFor, storedRowLookupFor } from "../fold-decomposition-loo import { nudgeSuppressed } from "../session-self-heal.js"; import { compressBreakerArmed } from "../stream.js"; import { absorbAbortedTurnGrowth, applyCompactionArchive, coveredRealHistoryIds, detectUnannouncedHistoryRewrite, foldCoverage, markCompactionBoundary, markDirty, markNativeCompactionBoundary, reconcileNativeCompactionBoundary, REWRITE_MIN_INCOMING_TOTAL, snapshotMessages, type PendingRetrieval, type Session } from "../session.js"; +import { getStore } from "../persist.js"; import { carriesDshLocalCompactionSummary, DSH_LOCAL_COMPACTION_MIN_MISSING } from "./dsh-compaction-guard.js"; import { carriesBiliSnapshotCheckpoint } from "./dsh-authored-compaction.js"; import { ABSORB_TOOL_NAME, READ_ORIGINAL_IMAGE_TOOL, RULE_TOOL, absorbToolsFor, blockOpsTools, retrieveToolsFor, withFirstSightDrain, withMarkerIntegrityNote, withSummaryBudgetNote } from "../compress-tool.js"; @@ -607,6 +608,14 @@ export async function prepareAnthropic( // this wire the system rides the top-level `system` field outside the fold. session.metadata.systemPromptTokens = countSystemAndToolsTokens(extractSystem(systemOut), toolsOut); snapshotMessages(session, originalMessages); + // #6xxx durability-before-render: make this turn's ledger rows durable NOW + // (synchronous SQLite ingest, no JSON envelope) so the refs the model is + // about to see are on disk BEFORE the response is sent. Without this, a + // crash before the debounced save (#5155) leaves rendered refs unbacked — + // the #3397 orphaned-generation gap that makes a later turn cite refs the + // gate rejects as "above the highest allocated ref". markDirty below keeps + // the JSON envelope debounced; this only moves the small row ingest ahead. + getStore().ingestRowsNow(session); markDirty(session); // [#5604] a preflight fold of the tail (soft protection relaxed to 0, or a diff --git a/src/server/prepare-google.ts b/src/server/prepare-google.ts index 501ce87846..04ab403c5e 100644 --- a/src/server/prepare-google.ts +++ b/src/server/prepare-google.ts @@ -16,6 +16,7 @@ import { storedMessagesFor, storedRowLookupFor } from "../fold-decomposition-loo import { nudgeSuppressed } from "../session-self-heal.js"; import { compressBreakerArmed } from "../stream.js"; import { absorbAbortedTurnGrowth, applyCompactionArchive, coveredRealHistoryIds, detectUnannouncedHistoryRewrite, foldCoverage, markCompactionBoundary, markDirty, markNativeCompactionBoundary, reconcileNativeCompactionBoundary, REWRITE_MIN_INCOMING_TOTAL, snapshotMessages, type Session } from "../session.js"; +import { getStore } from "../persist.js"; import { recordConflict } from "../conflict-watch.js"; import { carriesDshLocalCompactionSummary, DSH_LOCAL_COMPACTION_MIN_MISSING } from "./dsh-compaction-guard.js"; import { carriesBiliSnapshotCheckpoint } from "./dsh-authored-compaction.js"; @@ -415,6 +416,8 @@ export async function prepareGoogle( if (upstreamOrigin) session.stats.lastLocalTextEstimateOrigin = upstreamOrigin; } snapshotMessages(session, originalMessages); + // #6xxx durability-before-render (see prepare-anthropic.ts): rows durable NOW, before the model call. + getStore().ingestRowsNow(session); markDirty(session); return { body: JSON.stringify(rebuilt), parsedBody: rebuilt, session, processedMessages, originalMessages, judgeBaseline, protocol: "google", stream, compressInjected: injectTools, pluginMode, mcpDialectNote: laneDecisions.mcpDialect, nudge, nudgeOutcome, prompts, surface, google: { system: googleClientSystem, model }, systemNotes: sysNotes, renderTags: renderStrategy } as Prepared; } diff --git a/src/server/prepare-openai.ts b/src/server/prepare-openai.ts index 94acf1ed60..82aeccf9a9 100644 --- a/src/server/prepare-openai.ts +++ b/src/server/prepare-openai.ts @@ -20,6 +20,7 @@ import { storedMessagesFor, storedRowLookupFor } from "../fold-decomposition-loo import { nudgeSuppressed } from "../session-self-heal.js"; import { compressBreakerArmed } from "../stream.js"; import { absorbAbortedTurnGrowth, applyCompactionArchive, coveredRealHistoryIds, detectUnannouncedHistoryRewrite, foldCoverage, markCompactionBoundary, markDirty, markNativeCompactionBoundary, reconcileNativeCompactionBoundary, REWRITE_MIN_INCOMING_TOTAL, snapshotMessages, type PendingRetrieval, type Session } from "../session.js"; +import { getStore } from "../persist.js"; import { carriesDshLocalCompactionSummary, DSH_LOCAL_COMPACTION_MIN_MISSING } from "./dsh-compaction-guard.js"; import { carriesBiliSnapshotCheckpoint } from "./dsh-authored-compaction.js"; import { carriesCodexLocalCompactionSummary, CODEX_LOCAL_COMPACTION_MIN_MISSING, isCodexHost } from "../codex-compact.js"; @@ -638,6 +639,8 @@ export async function prepareOpenai( // stashed prefix lives OUTSIDE the Session by design; the fold machinery // applies to the pin exactly as it already does for folded sessions. snapshotMessages(session, wpHit !== undefined ? wpHit.windowCores : originalMessages); + // #6xxx durability-before-render (see prepare-anthropic.ts): rows durable NOW, before the model call. + getStore().ingestRowsNow(session); markDirty(session); let wpResponseHeaders: Record | undefined; if (wpFullText !== undefined && wpOutcome !== null && wpTurnStrip !== undefined) { diff --git a/src/server/prepare-responses.ts b/src/server/prepare-responses.ts index 03218acc57..11606e906a 100644 --- a/src/server/prepare-responses.ts +++ b/src/server/prepare-responses.ts @@ -20,6 +20,7 @@ import { storedMessagesFor, storedRowLookupFor } from "../fold-decomposition-loo import { nudgeSuppressed, pluginLaneDegraded } from "../session-self-heal.js"; import { compressBreakerArmed } from "../stream.js"; import { absorbAbortedTurnGrowth, applyCompactionArchive, coveredRealHistoryIds, foldCoverage, markDirty, markNativeCompactionBoundary, reconcileNativeCompactionBoundary, REWRITE_MIN_INCOMING_TOTAL, snapshotMessages, type Session } from "../session.js"; +import { getStore } from "../persist.js"; import { carriesDshLocalCompactionSummary, DSH_LOCAL_COMPACTION_MIN_MISSING } from "./dsh-compaction-guard.js"; import { carriesBiliSnapshotCheckpoint } from "./dsh-authored-compaction.js"; import { recordConflict } from "../conflict-watch.js"; @@ -714,6 +715,8 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr if (responsesPairOrigin) session.stats.lastLocalTextEstimateOrigin = responsesPairOrigin; } snapshotMessages(session, originalMessages); + // #6xxx durability-before-render (see prepare-anthropic.ts): rows durable NOW, before the model call. + getStore().ingestRowsNow(session); markDirty(session); // #564/#2627: ONE serialization of the outbound body. The same string is // both what we send and what the separator manifest binds to, so the claims From b34e5499501dba6128414aed8053f6d74c8c217c Mon Sep 17 00:00:00 2001 From: ework-agent Date: Thu, 12 Nov 2026 09:53:47 +0800 Subject: [PATCH 2/4] fix: keep the multi-instance stale-write guard observable in the durability path The durability-before-render pass (ingestRowsNow) converges a stale resident to the on-disk record BEFORE the debounced save does. The #405/#2401 'rejected stale snapshot' warning lived only in the debounced save's stale branch, so once this earlier pass converged the session the debounced save no longer saw a stale state and the warning (and the multi-instance test's assertion on it) went silent. Log the same throttled warning here so the guard stays observable regardless of which pass converges first. --- src/persist.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/persist.ts b/src/persist.ts index 539b363f0e..7041fb2782 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -3180,8 +3180,20 @@ export class SessionStore { const record = this.buildRecordForSave(session, stamp); const disk = this.staleDiskPayload(record); if (disk !== null) { - // #2401: disk is strictly newer — converge + ingest the disk view - // (same as the debounced path, minus the envelope rewrite). + // #2401/#405: disk is strictly newer — converge + ingest the disk view + // (same as the debounced path, minus the envelope rewrite). Log the same + // "rejected stale snapshot" line (throttled identically) so the + // multi-instance guard stays observable when this durability pass + // converges before the debounced save gets to it. + const now = Date.now(); + const last = this.staleWarnAt.get(record.id) ?? 0; + if (now - last >= STALE_WARN_THROTTLE_MS) { + this.staleWarnAt.set(record.id, now); + this.log( + "warn", + `[persist] rejected stale snapshot for session ${record.id} (durability-before-render): in-memory copy is older than the on-disk one (another bili instance holds newer state) — keeping disk state and converging in-memory state to it, so subsequent requests serve the fuller fold coverage instead of the stale view (#405/#2401)`, + ); + } this.convergeToDisk(session, disk); this.shadow.record(disk); this.noteOwnCommit(session.id, disk); From 63bde1c059907b647f15b2615662bc28f5839adf Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Thu, 12 Nov 2026 13:17:06 +0800 Subject: [PATCH 3/4] fix: point the durability-before-render comments at #3397 (PR #6326), not the closed unrelated #6150 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #6150 is the closed 'remove compressAtFoldPoint' PR — the comment refs read as pointing at a retired feature. Fill the '#6xxx' placeholders with the issue the fix actually closes. --- src/incremental-lane.ts | 2 +- src/persist.ts | 2 +- src/server/prepare-anthropic.ts | 2 +- src/server/prepare-google.ts | 2 +- src/server/prepare-openai.ts | 2 +- src/server/prepare-responses.ts | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/incremental-lane.ts b/src/incremental-lane.ts index aadb5f3908..a233e6f5b9 100644 --- a/src/incremental-lane.ts +++ b/src/incremental-lane.ts @@ -1005,7 +1005,7 @@ export async function handleIncrementalTurn( inFlight.delete(conversationId); const session = peekSession(sessionId); settled = true; - // #6150-fix: durability-before-render — the turn's ledger rows are made + // #3397 (PR #6326): durability-before-render — the turn's ledger rows are made // durable NOW (synchronous unified-index ingest) so a crash before the // debounced envelope save (#5155) cannot leave the refs the model just // saw unbacked (the #3397 orphaned-generation gap: the lane rendered the diff --git a/src/persist.ts b/src/persist.ts index 7041fb2782..6abcadff32 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -3162,7 +3162,7 @@ export class SessionStore { loggerLog("error", `[persist] ${sessionId}: skipping ${route} envelope save — JSON envelope writes FROZEN since ${since} (digest-drift escalation #4323); the unified store keeps receiving the save and compression continues from the in-memory view, but the two copies are NOT converging — see the [storage] DIGEST DRIFT lines for the divergence diagnosis (#5408)`); } - /** #6150-fix: durability-before-render — make a session's unified-index + /** #3397 (PR #6326): durability-before-render — make a session's unified-index * rows durable SYNCHRONOUSLY (the small SQLite ingest) WITHOUT writing the * JSON envelope. The incremental lane calls this the instant a turn's * response has fully left, so the refs the model just saw are on disk before diff --git a/src/server/prepare-anthropic.ts b/src/server/prepare-anthropic.ts index 3c4443b296..a7b1908856 100644 --- a/src/server/prepare-anthropic.ts +++ b/src/server/prepare-anthropic.ts @@ -608,7 +608,7 @@ export async function prepareAnthropic( // this wire the system rides the top-level `system` field outside the fold. session.metadata.systemPromptTokens = countSystemAndToolsTokens(extractSystem(systemOut), toolsOut); snapshotMessages(session, originalMessages); - // #6xxx durability-before-render: make this turn's ledger rows durable NOW + // #3397 durability-before-render: make this turn's ledger rows durable NOW // (synchronous SQLite ingest, no JSON envelope) so the refs the model is // about to see are on disk BEFORE the response is sent. Without this, a // crash before the debounced save (#5155) leaves rendered refs unbacked — diff --git a/src/server/prepare-google.ts b/src/server/prepare-google.ts index 04ab403c5e..f85b4dc1c1 100644 --- a/src/server/prepare-google.ts +++ b/src/server/prepare-google.ts @@ -416,7 +416,7 @@ export async function prepareGoogle( if (upstreamOrigin) session.stats.lastLocalTextEstimateOrigin = upstreamOrigin; } snapshotMessages(session, originalMessages); - // #6xxx durability-before-render (see prepare-anthropic.ts): rows durable NOW, before the model call. + // #3397 durability-before-render (see prepare-anthropic.ts): rows durable NOW, before the model call. getStore().ingestRowsNow(session); markDirty(session); return { body: JSON.stringify(rebuilt), parsedBody: rebuilt, session, processedMessages, originalMessages, judgeBaseline, protocol: "google", stream, compressInjected: injectTools, pluginMode, mcpDialectNote: laneDecisions.mcpDialect, nudge, nudgeOutcome, prompts, surface, google: { system: googleClientSystem, model }, systemNotes: sysNotes, renderTags: renderStrategy } as Prepared; diff --git a/src/server/prepare-openai.ts b/src/server/prepare-openai.ts index 82aeccf9a9..84a453fcee 100644 --- a/src/server/prepare-openai.ts +++ b/src/server/prepare-openai.ts @@ -639,7 +639,7 @@ export async function prepareOpenai( // stashed prefix lives OUTSIDE the Session by design; the fold machinery // applies to the pin exactly as it already does for folded sessions. snapshotMessages(session, wpHit !== undefined ? wpHit.windowCores : originalMessages); - // #6xxx durability-before-render (see prepare-anthropic.ts): rows durable NOW, before the model call. + // #3397 durability-before-render (see prepare-anthropic.ts): rows durable NOW, before the model call. getStore().ingestRowsNow(session); markDirty(session); let wpResponseHeaders: Record | undefined; diff --git a/src/server/prepare-responses.ts b/src/server/prepare-responses.ts index 11606e906a..1ae2087adb 100644 --- a/src/server/prepare-responses.ts +++ b/src/server/prepare-responses.ts @@ -715,7 +715,7 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr if (responsesPairOrigin) session.stats.lastLocalTextEstimateOrigin = responsesPairOrigin; } snapshotMessages(session, originalMessages); - // #6xxx durability-before-render (see prepare-anthropic.ts): rows durable NOW, before the model call. + // #3397 durability-before-render (see prepare-anthropic.ts): rows durable NOW, before the model call. getStore().ingestRowsNow(session); markDirty(session); // #564/#2627: ONE serialization of the outbound body. The same string is From 47a542bb30460b81b1bfa69392cb1e9fc9a69d76 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Thu, 12 Nov 2026 13:17:06 +0800 Subject: [PATCH 4/4] =?UTF-8?q?test:=20pin=20ingestRowsNow=20=E2=80=94=20r?= =?UTF-8?q?ows=20durable=20before=20the=20envelope,=20stale=20guard=20stay?= =?UTF-8?q?s=20observable=20(#3397)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two legs: 1. ingestRowsNow alone (JSON writer off) makes the session durable in the unified index; a cold restart loads it with folds + ref ledger intact — the orphaned-generation gap the fix closes. 2. the multi-instance stale-disk guard through the durability path still emits the throttled 'rejected stale snapshot' warn (tagged '(durability-before-render)') and never rolls the shared index counters back. --- ...issue3397-durability-before-render.test.ts | 180 ++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 tests/issue3397-durability-before-render.test.ts diff --git a/tests/issue3397-durability-before-render.test.ts b/tests/issue3397-durability-before-render.test.ts new file mode 100644 index 0000000000..30f3226b14 --- /dev/null +++ b/tests/issue3397-durability-before-render.test.ts @@ -0,0 +1,180 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { SessionStore } from "../src/persist.ts"; +import { UnifiedStore } from "../src/storage/store.ts"; + +/** + * #3397 (PR #6326): durability-before-render — the turn's ledger rows must be + * in the unified index BEFORE the model can cite the turn's refs, so a crash + * before the debounced envelope save (#5155) cannot leave rendered refs + * unbacked ("above the highest allocated ref" on the next compress). The new + * primitive is `SessionStore.ingestRowsNow`: the synchronous SQLite ingest leg + * of the debounced save, minus the envelope write. The prepare phases + * (server/prepare-*.ts) call it right before the model call; the incremental + * lane (incremental-lane.ts) calls it the instant a turn's response has fully + * left. + * + * Pinned here: + * 1. ingestRowsNow ALONE makes a session durable in the index — with the + * JSON envelope writer off entirely, a cold restart still resolves the + * session from the index with folds + ref ledger intact (the + * orphaned-generation gap the fix closes). + * 2. The multi-instance stale-disk guard stays observable through the + * durability path: converging a stale resident copy still emits the + * throttled "rejected stale snapshot" warn (tagged + * "(durability-before-render)") and never rolls the shared index + * counters back. + */ + +process.env.BILI_PERSIST_ZSTD = "0"; +// Same hermeticity pin as the other storage suites: pin BOTH tiers so every +// leg controls its own mode explicitly (absent now defaults to full, #3818). +const hermetic = fs.mkdtempSync(path.join(os.tmpdir(), "bili-3397-hermetic-")); +process.env.BILI_CONFIG_FILE = path.join(hermetic, "no-config.json"); +process.env.BILI_STORAGE_UNIFIED = "legacy"; + +const CORPUS_ROOT = path.resolve(import.meta.dirname, "golden/storage-corpus"); + +function loadCorpusCase(dir: string, name: string, id: string, protocol: string) { + fs.cpSync(path.join(CORPUS_ROOT, name), path.join(dir, name), { recursive: true }); + const loader = new SessionStore({ dir: path.join(dir, name), debounceMs: 1, enabled: true }); + const loaded = loader.loadSync(id, { protocol, upstreamOrigin: "http://127.0.0.1:8199" }); + loader.cancelAll(); + loader.closeQuietly(); + assert.ok(loaded, `${name}: corpus loadSync returned null`); + return loaded; +} + +/** Session envelopes live at /_.json; the probe counts + * only real session records. */ +function envelopeFiles(dir: string): string[] { + const out: string[] = []; + const walk = (p: string) => { + for (const e of fs.readdirSync(p, { withFileTypes: true })) { + const full = path.join(p, e.name); + if (e.isDirectory()) walk(full); + else if (e.name.endsWith(".json") && !e.name.endsWith(".content-store.json") && e.name !== "no-config.json") out.push(full); + } + }; + walk(dir); + return out; +} + +/** Windows: an ephemeral stray handle must not fail CI — best-effort delete + * with bounded retries, swallowed on persistent EBUSY/EPERM. */ +function rmBestEffort(p: string): void { + try { + fs.rmSync(p, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); + } catch { + /* leaked temp dir on a Windows runner is harmless */ + } +} + +test("durability-before-render: ingestRowsNow makes the rows durable without the envelope; a cold restart loads from the index", () => { + const prevMode = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "full"; + process.env.BILI_STORAGE_JSON_WRITER = "off"; // no envelope can ever appear — rows can only come from ingestRowsNow + try { + const seed = fs.mkdtempSync(path.join(os.tmpdir(), "bili-3397-seed-")); + const session = loadCorpusCase(seed, "folded", "corpus-folded", "anthropic"); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-3397-dur-")); + try { + const store = new SessionStore({ dir, debounceMs: 60_000, enabled: true }); + // Synchronous, request-path call — no await anywhere — exactly + // what the prepare phase does before the model call. + store.ingestRowsNow(session); + store.cancelAll(); + store.closeQuietly(); + + assert.equal(envelopeFiles(dir).length, 0, "the durability pass must not write the JSON envelope"); + assert.ok(fs.existsSync(path.join(dir, "index.db")), "index.db must exist after ingestRowsNow"); + const db = UnifiedStore.open(path.join(dir, "index.db")); + try { + const row = db.getSessionRow(session.id); + assert.ok(row, "the session row must exist in the unified index"); + } finally { + db.close(); + } + + // "Crash + restart": a fresh store resolves the session + // unified-first; the refs the model saw are backed by the + // ledger — the #3397 orphaned-generation gap. + const reopen = new SessionStore({ dir, debounceMs: 1, enabled: true }); + try { + const loaded = reopen.loadSync(session.id, { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }); + assert.ok(loaded, "index-only session must load after restart"); + assert.ok(loaded.state.blocks.length > 0, "folds must survive the envelope-less round-trip"); + assert.equal(loaded.state.blocks.length, session.state.blocks.length, "block count must survive the index-only round-trip"); + assert.equal(loaded.state.nextBlockId, session.state.nextBlockId, "nextBlockId cursor must be intact"); + assert.equal(loaded.state.messageRefs.byRaw.size, session.state.messageRefs.byRaw.size, "messageRefs ledger must be intact"); + reopen.cancelAll(); + } finally { + reopen.closeQuietly(); + } + } finally { + rmBestEffort(dir); + rmBestEffort(seed); + } + } finally { + delete process.env.BILI_STORAGE_JSON_WRITER; + process.env.BILI_STORAGE_UNIFIED = prevMode; + } +}); + +test("durability-before-render: the stale-disk guard stays observable (warn + converge, no counter rollback)", async () => { + const prevMode = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "full"; // envelope ON — the guard compares against it + try { + // Two resident copies of the SAME session = two instances sharing the + // dir (#394 topology): `base` belongs to instance A, `stale` to B. + const seedA = fs.mkdtempSync(path.join(os.tmpdir(), "bili-3397-a-")); + const base = loadCorpusCase(seedA, "folded", "corpus-folded", "anthropic"); + const seedB = fs.mkdtempSync(path.join(os.tmpdir(), "bili-3397-b-")); + const stale = loadCorpusCase(seedB, "folded", "corpus-folded", "anthropic"); + const r0 = base.stats.requests ?? 0; + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-3397-stale-")); + const logs: { level: string; msg: string }[] = []; + try { + const storeA = new SessionStore({ dir, debounceMs: 60_000, enabled: true }); + await storeA.writeNow(base); // disk at R0 + base.stats.requests = r0 + 1; + await storeA.writeNow(base); // disk at R0+1 — instance A advanced + storeA.cancelAll(); + storeA.closeQuietly(); + + // Instance B still holds the R0 copy. Its durability pass runs + // against the shared dir → stale-disk branch (commit b34e5499). + const storeB = new SessionStore({ dir, debounceMs: 60_000, enabled: true, log: (level, msg) => { logs.push({ level, msg }); } }); + storeB.ingestRowsNow(stale); + storeB.cancelAll(); + storeB.closeQuietly(); + + const warns = logs.filter((l) => l.msg.includes("rejected stale snapshot")); + assert.ok( + warns.some((l) => l.level === "warn" && l.msg.includes("(durability-before-render)")), + `expected a warn tagged "(durability-before-render)", got: ${logs.map((l) => `${l.level} ${l.msg}`).join(" | ")}`, + ); + assert.equal(stale.stats.requests, r0 + 1, "the stale in-memory copy must converge to the disk record"); + + // The shared index must NOT have been rolled back by the stale ingest. + const db = UnifiedStore.open(path.join(dir, "index.db")); + try { + const row = db.getSessionRow(stale.id); + assert.ok(row, "the session row must exist in the unified index"); + const stats = JSON.parse(row!.statsJson ?? "{}") as { requests?: number }; + assert.equal(stats.requests, r0 + 1, "index counters must keep the newer commit (no rollback)"); + } finally { + db.close(); + } + } finally { + rmBestEffort(dir); + rmBestEffort(seedA); + rmBestEffort(seedB); + } + } finally { + process.env.BILI_STORAGE_UNIFIED = prevMode; + } +});