diff --git a/src/incremental-lane.ts b/src/incremental-lane.ts index 1fd9e0343c..a233e6f5b9 100644 --- a/src/incremental-lane.ts +++ b/src/incremental-lane.ts @@ -1005,6 +1005,19 @@ export async function handleIncrementalTurn( inFlight.delete(conversationId); const session = peekSession(sessionId); settled = true; + // #3397 (PR #6326): durability-before-render — the turn's ledger rows are made + // durable NOW (synchronous unified-index ingest) so a crash before the + // debounced envelope save (#5155) cannot leave the refs the model just + // saw unbacked (the #3397 orphaned-generation gap: the lane rendered the + // delta, the model could cite it, but the rows only settled in the + // debounced flush). The large JSON envelope itself stays debounced — + // only the small row ingest moves ahead of it. Best-effort: the + // debounced save below retries the full path on failure. + try { + if (session) getStore().ingestRowsNow(session); + } catch { + // best-effort, same failure posture as the debounced save + } // #5155: the debounced save coalesces consecutive turns' full-envelope // flushes (a synchronous writeNow per turn serialized + fsynced the // whole ~24MB envelope on large histories); pointer upkeep no longer diff --git a/src/persist.ts b/src/persist.ts index 718c04c60f..6abcadff32 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -3162,6 +3162,56 @@ export class SessionStore { loggerLog("error", `[persist] ${sessionId}: skipping ${route} envelope save — JSON envelope writes FROZEN since ${since} (digest-drift escalation #4323); the unified store keeps receiving the save and compression continues from the in-memory view, but the two copies are NOT converging — see the [storage] DIGEST DRIFT lines for the divergence diagnosis (#5408)`); } + /** #3397 (PR #6326): durability-before-render — make a session's unified-index + * rows durable SYNCHRONOUSLY (the small SQLite ingest) WITHOUT writing the + * JSON envelope. The incremental lane calls this the instant a turn's + * response has fully left, so the refs the model just saw are on disk before + * the debounced envelope save (#5155) settles. A crash in that window no + * longer leaves rendered refs unbacked (the #3397 orphaned-generation gap + * that made long sessions cite refs the gate rejects as "above the highest + * allocated ref"). The large JSON envelope stays debounced — only the small + * row ingest moves ahead of it. Mirrors the debounced path's ingest legs + * (stale-disk guard + rawTail + live CCR + noteOwnCommit), minus the + * envelope write and its landing stamp. */ + ingestRowsNow(session: Session): void { + if (!this.shadow) return; // legacy-only (no unified index) — nothing to back + try { + const stamp = this.liveStorageStamp; + const record = this.buildRecordForSave(session, stamp); + const disk = this.staleDiskPayload(record); + if (disk !== null) { + // #2401/#405: disk is strictly newer — converge + ingest the disk view + // (same as the debounced path, minus the envelope rewrite). Log the same + // "rejected stale snapshot" line (throttled identically) so the + // multi-instance guard stays observable when this durability pass + // converges before the debounced save gets to it. + const now = Date.now(); + const last = this.staleWarnAt.get(record.id) ?? 0; + if (now - last >= STALE_WARN_THROTTLE_MS) { + this.staleWarnAt.set(record.id, now); + this.log( + "warn", + `[persist] rejected stale snapshot for session ${record.id} (durability-before-render): in-memory copy is older than the on-disk one (another bili instance holds newer state) — keeping disk state and converging in-memory state to it, so subsequent requests serve the fuller fold coverage instead of the stale view (#405/#2401)`, + ); + } + this.convergeToDisk(session, disk); + this.shadow.record(disk); + this.noteOwnCommit(session.id, disk); + return; + } + const payload = session.decompositionLog; + if (payload !== undefined) session.decompositionLog = undefined; + const tail = session.lastMessagesFolded !== true ? session.lastMessages : undefined; + this.shadow.record(payload !== undefined ? { ...record, decompositionLog: payload } : record, + tail && tail.length > 0 + ? { lastMessages: tail, lastMessagesFolded: session.lastMessagesFolded } + : undefined, session.contentStore ?? undefined); + this.noteOwnCommit(session.id, record); + } catch (err) { + this.log("warn", `[persist] ingestRowsNow failed for ${session.id}: ${String(err)} — the debounced save retries the full path`); + } + } + scheduleSave(session: Session): void { // #5608-D: a committed fold IS the session's next legitimate // cache-breaking boundary — run the deferred #4316 citation-space diff --git a/src/server/prepare-anthropic.ts b/src/server/prepare-anthropic.ts index a61fb4833f..a7b1908856 100644 --- a/src/server/prepare-anthropic.ts +++ b/src/server/prepare-anthropic.ts @@ -17,6 +17,7 @@ import { storedMessagesFor, storedRowLookupFor } from "../fold-decomposition-loo import { nudgeSuppressed } from "../session-self-heal.js"; import { compressBreakerArmed } from "../stream.js"; import { absorbAbortedTurnGrowth, applyCompactionArchive, coveredRealHistoryIds, detectUnannouncedHistoryRewrite, foldCoverage, markCompactionBoundary, markDirty, markNativeCompactionBoundary, reconcileNativeCompactionBoundary, REWRITE_MIN_INCOMING_TOTAL, snapshotMessages, type PendingRetrieval, type Session } from "../session.js"; +import { getStore } from "../persist.js"; import { carriesDshLocalCompactionSummary, DSH_LOCAL_COMPACTION_MIN_MISSING } from "./dsh-compaction-guard.js"; import { carriesBiliSnapshotCheckpoint } from "./dsh-authored-compaction.js"; import { ABSORB_TOOL_NAME, READ_ORIGINAL_IMAGE_TOOL, RULE_TOOL, absorbToolsFor, blockOpsTools, retrieveToolsFor, withFirstSightDrain, withMarkerIntegrityNote, withSummaryBudgetNote } from "../compress-tool.js"; @@ -607,6 +608,14 @@ export async function prepareAnthropic( // this wire the system rides the top-level `system` field outside the fold. session.metadata.systemPromptTokens = countSystemAndToolsTokens(extractSystem(systemOut), toolsOut); snapshotMessages(session, originalMessages); + // #3397 durability-before-render: make this turn's ledger rows durable NOW + // (synchronous SQLite ingest, no JSON envelope) so the refs the model is + // about to see are on disk BEFORE the response is sent. Without this, a + // crash before the debounced save (#5155) leaves rendered refs unbacked — + // the #3397 orphaned-generation gap that makes a later turn cite refs the + // gate rejects as "above the highest allocated ref". markDirty below keeps + // the JSON envelope debounced; this only moves the small row ingest ahead. + getStore().ingestRowsNow(session); markDirty(session); // [#5604] a preflight fold of the tail (soft protection relaxed to 0, or a diff --git a/src/server/prepare-google.ts b/src/server/prepare-google.ts index 501ce87846..f85b4dc1c1 100644 --- a/src/server/prepare-google.ts +++ b/src/server/prepare-google.ts @@ -16,6 +16,7 @@ import { storedMessagesFor, storedRowLookupFor } from "../fold-decomposition-loo import { nudgeSuppressed } from "../session-self-heal.js"; import { compressBreakerArmed } from "../stream.js"; import { absorbAbortedTurnGrowth, applyCompactionArchive, coveredRealHistoryIds, detectUnannouncedHistoryRewrite, foldCoverage, markCompactionBoundary, markDirty, markNativeCompactionBoundary, reconcileNativeCompactionBoundary, REWRITE_MIN_INCOMING_TOTAL, snapshotMessages, type Session } from "../session.js"; +import { getStore } from "../persist.js"; import { recordConflict } from "../conflict-watch.js"; import { carriesDshLocalCompactionSummary, DSH_LOCAL_COMPACTION_MIN_MISSING } from "./dsh-compaction-guard.js"; import { carriesBiliSnapshotCheckpoint } from "./dsh-authored-compaction.js"; @@ -415,6 +416,8 @@ export async function prepareGoogle( if (upstreamOrigin) session.stats.lastLocalTextEstimateOrigin = upstreamOrigin; } snapshotMessages(session, originalMessages); + // #3397 durability-before-render (see prepare-anthropic.ts): rows durable NOW, before the model call. + getStore().ingestRowsNow(session); markDirty(session); return { body: JSON.stringify(rebuilt), parsedBody: rebuilt, session, processedMessages, originalMessages, judgeBaseline, protocol: "google", stream, compressInjected: injectTools, pluginMode, mcpDialectNote: laneDecisions.mcpDialect, nudge, nudgeOutcome, prompts, surface, google: { system: googleClientSystem, model }, systemNotes: sysNotes, renderTags: renderStrategy } as Prepared; } diff --git a/src/server/prepare-openai.ts b/src/server/prepare-openai.ts index 94acf1ed60..84a453fcee 100644 --- a/src/server/prepare-openai.ts +++ b/src/server/prepare-openai.ts @@ -20,6 +20,7 @@ import { storedMessagesFor, storedRowLookupFor } from "../fold-decomposition-loo import { nudgeSuppressed } from "../session-self-heal.js"; import { compressBreakerArmed } from "../stream.js"; import { absorbAbortedTurnGrowth, applyCompactionArchive, coveredRealHistoryIds, detectUnannouncedHistoryRewrite, foldCoverage, markCompactionBoundary, markDirty, markNativeCompactionBoundary, reconcileNativeCompactionBoundary, REWRITE_MIN_INCOMING_TOTAL, snapshotMessages, type PendingRetrieval, type Session } from "../session.js"; +import { getStore } from "../persist.js"; import { carriesDshLocalCompactionSummary, DSH_LOCAL_COMPACTION_MIN_MISSING } from "./dsh-compaction-guard.js"; import { carriesBiliSnapshotCheckpoint } from "./dsh-authored-compaction.js"; import { carriesCodexLocalCompactionSummary, CODEX_LOCAL_COMPACTION_MIN_MISSING, isCodexHost } from "../codex-compact.js"; @@ -638,6 +639,8 @@ export async function prepareOpenai( // stashed prefix lives OUTSIDE the Session by design; the fold machinery // applies to the pin exactly as it already does for folded sessions. snapshotMessages(session, wpHit !== undefined ? wpHit.windowCores : originalMessages); + // #3397 durability-before-render (see prepare-anthropic.ts): rows durable NOW, before the model call. + getStore().ingestRowsNow(session); markDirty(session); let wpResponseHeaders: Record | undefined; if (wpFullText !== undefined && wpOutcome !== null && wpTurnStrip !== undefined) { diff --git a/src/server/prepare-responses.ts b/src/server/prepare-responses.ts index 03218acc57..1ae2087adb 100644 --- a/src/server/prepare-responses.ts +++ b/src/server/prepare-responses.ts @@ -20,6 +20,7 @@ import { storedMessagesFor, storedRowLookupFor } from "../fold-decomposition-loo import { nudgeSuppressed, pluginLaneDegraded } from "../session-self-heal.js"; import { compressBreakerArmed } from "../stream.js"; import { absorbAbortedTurnGrowth, applyCompactionArchive, coveredRealHistoryIds, foldCoverage, markDirty, markNativeCompactionBoundary, reconcileNativeCompactionBoundary, REWRITE_MIN_INCOMING_TOTAL, snapshotMessages, type Session } from "../session.js"; +import { getStore } from "../persist.js"; import { carriesDshLocalCompactionSummary, DSH_LOCAL_COMPACTION_MIN_MISSING } from "./dsh-compaction-guard.js"; import { carriesBiliSnapshotCheckpoint } from "./dsh-authored-compaction.js"; import { recordConflict } from "../conflict-watch.js"; @@ -714,6 +715,8 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr if (responsesPairOrigin) session.stats.lastLocalTextEstimateOrigin = responsesPairOrigin; } snapshotMessages(session, originalMessages); + // #3397 durability-before-render (see prepare-anthropic.ts): rows durable NOW, before the model call. + getStore().ingestRowsNow(session); markDirty(session); // #564/#2627: ONE serialization of the outbound body. The same string is // both what we send and what the separator manifest binds to, so the claims diff --git a/tests/issue3397-durability-before-render.test.ts b/tests/issue3397-durability-before-render.test.ts new file mode 100644 index 0000000000..30f3226b14 --- /dev/null +++ b/tests/issue3397-durability-before-render.test.ts @@ -0,0 +1,180 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { SessionStore } from "../src/persist.ts"; +import { UnifiedStore } from "../src/storage/store.ts"; + +/** + * #3397 (PR #6326): durability-before-render — the turn's ledger rows must be + * in the unified index BEFORE the model can cite the turn's refs, so a crash + * before the debounced envelope save (#5155) cannot leave rendered refs + * unbacked ("above the highest allocated ref" on the next compress). The new + * primitive is `SessionStore.ingestRowsNow`: the synchronous SQLite ingest leg + * of the debounced save, minus the envelope write. The prepare phases + * (server/prepare-*.ts) call it right before the model call; the incremental + * lane (incremental-lane.ts) calls it the instant a turn's response has fully + * left. + * + * Pinned here: + * 1. ingestRowsNow ALONE makes a session durable in the index — with the + * JSON envelope writer off entirely, a cold restart still resolves the + * session from the index with folds + ref ledger intact (the + * orphaned-generation gap the fix closes). + * 2. The multi-instance stale-disk guard stays observable through the + * durability path: converging a stale resident copy still emits the + * throttled "rejected stale snapshot" warn (tagged + * "(durability-before-render)") and never rolls the shared index + * counters back. + */ + +process.env.BILI_PERSIST_ZSTD = "0"; +// Same hermeticity pin as the other storage suites: pin BOTH tiers so every +// leg controls its own mode explicitly (absent now defaults to full, #3818). +const hermetic = fs.mkdtempSync(path.join(os.tmpdir(), "bili-3397-hermetic-")); +process.env.BILI_CONFIG_FILE = path.join(hermetic, "no-config.json"); +process.env.BILI_STORAGE_UNIFIED = "legacy"; + +const CORPUS_ROOT = path.resolve(import.meta.dirname, "golden/storage-corpus"); + +function loadCorpusCase(dir: string, name: string, id: string, protocol: string) { + fs.cpSync(path.join(CORPUS_ROOT, name), path.join(dir, name), { recursive: true }); + const loader = new SessionStore({ dir: path.join(dir, name), debounceMs: 1, enabled: true }); + const loaded = loader.loadSync(id, { protocol, upstreamOrigin: "http://127.0.0.1:8199" }); + loader.cancelAll(); + loader.closeQuietly(); + assert.ok(loaded, `${name}: corpus loadSync returned null`); + return loaded; +} + +/** Session envelopes live at /_.json; the probe counts + * only real session records. */ +function envelopeFiles(dir: string): string[] { + const out: string[] = []; + const walk = (p: string) => { + for (const e of fs.readdirSync(p, { withFileTypes: true })) { + const full = path.join(p, e.name); + if (e.isDirectory()) walk(full); + else if (e.name.endsWith(".json") && !e.name.endsWith(".content-store.json") && e.name !== "no-config.json") out.push(full); + } + }; + walk(dir); + return out; +} + +/** Windows: an ephemeral stray handle must not fail CI — best-effort delete + * with bounded retries, swallowed on persistent EBUSY/EPERM. */ +function rmBestEffort(p: string): void { + try { + fs.rmSync(p, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); + } catch { + /* leaked temp dir on a Windows runner is harmless */ + } +} + +test("durability-before-render: ingestRowsNow makes the rows durable without the envelope; a cold restart loads from the index", () => { + const prevMode = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "full"; + process.env.BILI_STORAGE_JSON_WRITER = "off"; // no envelope can ever appear — rows can only come from ingestRowsNow + try { + const seed = fs.mkdtempSync(path.join(os.tmpdir(), "bili-3397-seed-")); + const session = loadCorpusCase(seed, "folded", "corpus-folded", "anthropic"); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-3397-dur-")); + try { + const store = new SessionStore({ dir, debounceMs: 60_000, enabled: true }); + // Synchronous, request-path call — no await anywhere — exactly + // what the prepare phase does before the model call. + store.ingestRowsNow(session); + store.cancelAll(); + store.closeQuietly(); + + assert.equal(envelopeFiles(dir).length, 0, "the durability pass must not write the JSON envelope"); + assert.ok(fs.existsSync(path.join(dir, "index.db")), "index.db must exist after ingestRowsNow"); + const db = UnifiedStore.open(path.join(dir, "index.db")); + try { + const row = db.getSessionRow(session.id); + assert.ok(row, "the session row must exist in the unified index"); + } finally { + db.close(); + } + + // "Crash + restart": a fresh store resolves the session + // unified-first; the refs the model saw are backed by the + // ledger — the #3397 orphaned-generation gap. + const reopen = new SessionStore({ dir, debounceMs: 1, enabled: true }); + try { + const loaded = reopen.loadSync(session.id, { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }); + assert.ok(loaded, "index-only session must load after restart"); + assert.ok(loaded.state.blocks.length > 0, "folds must survive the envelope-less round-trip"); + assert.equal(loaded.state.blocks.length, session.state.blocks.length, "block count must survive the index-only round-trip"); + assert.equal(loaded.state.nextBlockId, session.state.nextBlockId, "nextBlockId cursor must be intact"); + assert.equal(loaded.state.messageRefs.byRaw.size, session.state.messageRefs.byRaw.size, "messageRefs ledger must be intact"); + reopen.cancelAll(); + } finally { + reopen.closeQuietly(); + } + } finally { + rmBestEffort(dir); + rmBestEffort(seed); + } + } finally { + delete process.env.BILI_STORAGE_JSON_WRITER; + process.env.BILI_STORAGE_UNIFIED = prevMode; + } +}); + +test("durability-before-render: the stale-disk guard stays observable (warn + converge, no counter rollback)", async () => { + const prevMode = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "full"; // envelope ON — the guard compares against it + try { + // Two resident copies of the SAME session = two instances sharing the + // dir (#394 topology): `base` belongs to instance A, `stale` to B. + const seedA = fs.mkdtempSync(path.join(os.tmpdir(), "bili-3397-a-")); + const base = loadCorpusCase(seedA, "folded", "corpus-folded", "anthropic"); + const seedB = fs.mkdtempSync(path.join(os.tmpdir(), "bili-3397-b-")); + const stale = loadCorpusCase(seedB, "folded", "corpus-folded", "anthropic"); + const r0 = base.stats.requests ?? 0; + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-3397-stale-")); + const logs: { level: string; msg: string }[] = []; + try { + const storeA = new SessionStore({ dir, debounceMs: 60_000, enabled: true }); + await storeA.writeNow(base); // disk at R0 + base.stats.requests = r0 + 1; + await storeA.writeNow(base); // disk at R0+1 — instance A advanced + storeA.cancelAll(); + storeA.closeQuietly(); + + // Instance B still holds the R0 copy. Its durability pass runs + // against the shared dir → stale-disk branch (commit b34e5499). + const storeB = new SessionStore({ dir, debounceMs: 60_000, enabled: true, log: (level, msg) => { logs.push({ level, msg }); } }); + storeB.ingestRowsNow(stale); + storeB.cancelAll(); + storeB.closeQuietly(); + + const warns = logs.filter((l) => l.msg.includes("rejected stale snapshot")); + assert.ok( + warns.some((l) => l.level === "warn" && l.msg.includes("(durability-before-render)")), + `expected a warn tagged "(durability-before-render)", got: ${logs.map((l) => `${l.level} ${l.msg}`).join(" | ")}`, + ); + assert.equal(stale.stats.requests, r0 + 1, "the stale in-memory copy must converge to the disk record"); + + // The shared index must NOT have been rolled back by the stale ingest. + const db = UnifiedStore.open(path.join(dir, "index.db")); + try { + const row = db.getSessionRow(stale.id); + assert.ok(row, "the session row must exist in the unified index"); + const stats = JSON.parse(row!.statsJson ?? "{}") as { requests?: number }; + assert.equal(stats.requests, r0 + 1, "index counters must keep the newer commit (no rollback)"); + } finally { + db.close(); + } + } finally { + rmBestEffort(dir); + rmBestEffort(seedA); + rmBestEffort(seedB); + } + } finally { + process.env.BILI_STORAGE_UNIFIED = prevMode; + } +});