Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions src/incremental-lane.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1005,6 +1005,19 @@ export async function handleIncrementalTurn(
inFlight.delete(conversationId);
const session = peekSession(sessionId);
settled = true;
// #3397 (PR #6326): durability-before-render — the turn's ledger rows are made
// durable NOW (synchronous unified-index ingest) so a crash before the
// debounced envelope save (#5155) cannot leave the refs the model just
// saw unbacked (the #3397 orphaned-generation gap: the lane rendered the
// delta, the model could cite it, but the rows only settled in the
// debounced flush). The large JSON envelope itself stays debounced —
// only the small row ingest moves ahead of it. Best-effort: the
// debounced save below retries the full path on failure.
try {
if (session) getStore().ingestRowsNow(session);
} catch {
// best-effort, same failure posture as the debounced save
}
// #5155: the debounced save coalesces consecutive turns' full-envelope
// flushes (a synchronous writeNow per turn serialized + fsynced the
// whole ~24MB envelope on large histories); pointer upkeep no longer
Expand Down
50 changes: 50 additions & 0 deletions src/persist.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3162,6 +3162,56 @@ export class SessionStore {
loggerLog("error", `[persist] ${sessionId}: skipping ${route} envelope save — JSON envelope writes FROZEN since ${since} (digest-drift escalation #4323); the unified store keeps receiving the save and compression continues from the in-memory view, but the two copies are NOT converging — see the [storage] DIGEST DRIFT lines for the divergence diagnosis (#5408)`);
}

/** #3397 (PR #6326): durability-before-render — make a session's unified-index
* rows durable SYNCHRONOUSLY (the small SQLite ingest) WITHOUT writing the
* JSON envelope. The incremental lane calls this the instant a turn's
* response has fully left, so the refs the model just saw are on disk before
* the debounced envelope save (#5155) settles. A crash in that window no
* longer leaves rendered refs unbacked (the #3397 orphaned-generation gap
* that made long sessions cite refs the gate rejects as "above the highest
* allocated ref"). The large JSON envelope stays debounced — only the small
* row ingest moves ahead of it. Mirrors the debounced path's ingest legs
* (stale-disk guard + rawTail + live CCR + noteOwnCommit), minus the
* envelope write and its landing stamp. */
ingestRowsNow(session: Session): void {
if (!this.shadow) return; // legacy-only (no unified index) — nothing to back
try {
const stamp = this.liveStorageStamp;
const record = this.buildRecordForSave(session, stamp);
const disk = this.staleDiskPayload(record);
if (disk !== null) {
// #2401/#405: disk is strictly newer — converge + ingest the disk view
// (same as the debounced path, minus the envelope rewrite). Log the same
// "rejected stale snapshot" line (throttled identically) so the
// multi-instance guard stays observable when this durability pass
// converges before the debounced save gets to it.
const now = Date.now();
const last = this.staleWarnAt.get(record.id) ?? 0;
if (now - last >= STALE_WARN_THROTTLE_MS) {
this.staleWarnAt.set(record.id, now);
this.log(
"warn",
`[persist] rejected stale snapshot for session ${record.id} (durability-before-render): in-memory copy is older than the on-disk one (another bili instance holds newer state) — keeping disk state and converging in-memory state to it, so subsequent requests serve the fuller fold coverage instead of the stale view (#405/#2401)`,
);
}
this.convergeToDisk(session, disk);
this.shadow.record(disk);
this.noteOwnCommit(session.id, disk);
return;
}
const payload = session.decompositionLog;
if (payload !== undefined) session.decompositionLog = undefined;
const tail = session.lastMessagesFolded !== true ? session.lastMessages : undefined;
this.shadow.record(payload !== undefined ? { ...record, decompositionLog: payload } : record,
tail && tail.length > 0
? { lastMessages: tail, lastMessagesFolded: session.lastMessagesFolded }
: undefined, session.contentStore ?? undefined);
this.noteOwnCommit(session.id, record);
} catch (err) {
this.log("warn", `[persist] ingestRowsNow failed for ${session.id}: ${String(err)} — the debounced save retries the full path`);
}
}

scheduleSave(session: Session): void {
// #5608-D: a committed fold IS the session's next legitimate
// cache-breaking boundary — run the deferred #4316 citation-space
Expand Down
9 changes: 9 additions & 0 deletions src/server/prepare-anthropic.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import { storedMessagesFor, storedRowLookupFor } from "../fold-decomposition-loo
import { nudgeSuppressed } from "../session-self-heal.js";
import { compressBreakerArmed } from "../stream.js";
import { absorbAbortedTurnGrowth, applyCompactionArchive, coveredRealHistoryIds, detectUnannouncedHistoryRewrite, foldCoverage, markCompactionBoundary, markDirty, markNativeCompactionBoundary, reconcileNativeCompactionBoundary, REWRITE_MIN_INCOMING_TOTAL, snapshotMessages, type PendingRetrieval, type Session } from "../session.js";
import { getStore } from "../persist.js";
import { carriesDshLocalCompactionSummary, DSH_LOCAL_COMPACTION_MIN_MISSING } from "./dsh-compaction-guard.js";
import { carriesBiliSnapshotCheckpoint } from "./dsh-authored-compaction.js";
import { ABSORB_TOOL_NAME, READ_ORIGINAL_IMAGE_TOOL, RULE_TOOL, absorbToolsFor, blockOpsTools, retrieveToolsFor, withFirstSightDrain, withMarkerIntegrityNote, withSummaryBudgetNote } from "../compress-tool.js";
Expand Down Expand Up @@ -607,6 +608,14 @@ export async function prepareAnthropic(
// this wire the system rides the top-level `system` field outside the fold.
session.metadata.systemPromptTokens = countSystemAndToolsTokens(extractSystem(systemOut), toolsOut);
snapshotMessages(session, originalMessages);
// #3397 durability-before-render: make this turn's ledger rows durable NOW
// (synchronous SQLite ingest, no JSON envelope) so the refs the model is
// about to see are on disk BEFORE the response is sent. Without this, a
// crash before the debounced save (#5155) leaves rendered refs unbacked —
// the #3397 orphaned-generation gap that makes a later turn cite refs the
// gate rejects as "above the highest allocated ref". markDirty below keeps
// the JSON envelope debounced; this only moves the small row ingest ahead.
getStore().ingestRowsNow(session);
markDirty(session);

// [#5604] a preflight fold of the tail (soft protection relaxed to 0, or a
Expand Down
3 changes: 3 additions & 0 deletions src/server/prepare-google.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import { storedMessagesFor, storedRowLookupFor } from "../fold-decomposition-loo
import { nudgeSuppressed } from "../session-self-heal.js";
import { compressBreakerArmed } from "../stream.js";
import { absorbAbortedTurnGrowth, applyCompactionArchive, coveredRealHistoryIds, detectUnannouncedHistoryRewrite, foldCoverage, markCompactionBoundary, markDirty, markNativeCompactionBoundary, reconcileNativeCompactionBoundary, REWRITE_MIN_INCOMING_TOTAL, snapshotMessages, type Session } from "../session.js";
import { getStore } from "../persist.js";
import { recordConflict } from "../conflict-watch.js";
import { carriesDshLocalCompactionSummary, DSH_LOCAL_COMPACTION_MIN_MISSING } from "./dsh-compaction-guard.js";
import { carriesBiliSnapshotCheckpoint } from "./dsh-authored-compaction.js";
Expand Down Expand Up @@ -415,6 +416,8 @@ export async function prepareGoogle(
if (upstreamOrigin) session.stats.lastLocalTextEstimateOrigin = upstreamOrigin;
}
snapshotMessages(session, originalMessages);
// #3397 durability-before-render (see prepare-anthropic.ts): rows durable NOW, before the model call.
getStore().ingestRowsNow(session);
markDirty(session);
return { body: JSON.stringify(rebuilt), parsedBody: rebuilt, session, processedMessages, originalMessages, judgeBaseline, protocol: "google", stream, compressInjected: injectTools, pluginMode, mcpDialectNote: laneDecisions.mcpDialect, nudge, nudgeOutcome, prompts, surface, google: { system: googleClientSystem, model }, systemNotes: sysNotes, renderTags: renderStrategy } as Prepared;
}
Expand Down
3 changes: 3 additions & 0 deletions src/server/prepare-openai.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import { storedMessagesFor, storedRowLookupFor } from "../fold-decomposition-loo
import { nudgeSuppressed } from "../session-self-heal.js";
import { compressBreakerArmed } from "../stream.js";
import { absorbAbortedTurnGrowth, applyCompactionArchive, coveredRealHistoryIds, detectUnannouncedHistoryRewrite, foldCoverage, markCompactionBoundary, markDirty, markNativeCompactionBoundary, reconcileNativeCompactionBoundary, REWRITE_MIN_INCOMING_TOTAL, snapshotMessages, type PendingRetrieval, type Session } from "../session.js";
import { getStore } from "../persist.js";
import { carriesDshLocalCompactionSummary, DSH_LOCAL_COMPACTION_MIN_MISSING } from "./dsh-compaction-guard.js";
import { carriesBiliSnapshotCheckpoint } from "./dsh-authored-compaction.js";
import { carriesCodexLocalCompactionSummary, CODEX_LOCAL_COMPACTION_MIN_MISSING, isCodexHost } from "../codex-compact.js";
Expand Down Expand Up @@ -638,6 +639,8 @@ export async function prepareOpenai(
// stashed prefix lives OUTSIDE the Session by design; the fold machinery
// applies to the pin exactly as it already does for folded sessions.
snapshotMessages(session, wpHit !== undefined ? wpHit.windowCores : originalMessages);
// #3397 durability-before-render (see prepare-anthropic.ts): rows durable NOW, before the model call.
getStore().ingestRowsNow(session);
markDirty(session);
let wpResponseHeaders: Record<string, string> | undefined;
if (wpFullText !== undefined && wpOutcome !== null && wpTurnStrip !== undefined) {
Expand Down
3 changes: 3 additions & 0 deletions src/server/prepare-responses.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import { storedMessagesFor, storedRowLookupFor } from "../fold-decomposition-loo
import { nudgeSuppressed, pluginLaneDegraded } from "../session-self-heal.js";
import { compressBreakerArmed } from "../stream.js";
import { absorbAbortedTurnGrowth, applyCompactionArchive, coveredRealHistoryIds, foldCoverage, markDirty, markNativeCompactionBoundary, reconcileNativeCompactionBoundary, REWRITE_MIN_INCOMING_TOTAL, snapshotMessages, type Session } from "../session.js";
import { getStore } from "../persist.js";
import { carriesDshLocalCompactionSummary, DSH_LOCAL_COMPACTION_MIN_MISSING } from "./dsh-compaction-guard.js";
import { carriesBiliSnapshotCheckpoint } from "./dsh-authored-compaction.js";
import { recordConflict } from "../conflict-watch.js";
Expand Down Expand Up @@ -714,6 +715,8 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr
if (responsesPairOrigin) session.stats.lastLocalTextEstimateOrigin = responsesPairOrigin;
}
snapshotMessages(session, originalMessages);
// #3397 durability-before-render (see prepare-anthropic.ts): rows durable NOW, before the model call.
getStore().ingestRowsNow(session);
markDirty(session);
// #564/#2627: ONE serialization of the outbound body. The same string is
// both what we send and what the separator manifest binds to, so the claims
Expand Down
Loading
Loading