Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 90 additions & 4 deletions src/persist.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import { toLegacyLike, type LegacySessionLike } from "./storage/ingest.js";
import { sha256Hex, type CheckpointRow } from "./storage/store.js";
import type { WireProtocol } from "./util.js";
import { currentContextObservation, CALIBRATION_CLAMP_MAX, CALIBRATION_CLAMP_MIN, CALIBRATION_SAMPLE_MAX, CALIBRATION_SAMPLE_MIN, CALIBRATION_SAMPLE_WINDOW } from "./cache-ledger.js";
import { isValidSystemSurface, SYSTEM_SURFACE_SUFFIX, type SystemSurfaceRecord } from "./system-surface.js";

/**
* On-disk persistence for proxy sessions.
Expand Down Expand Up @@ -324,6 +325,13 @@ export function contentStoreRelPathFor(id: string, protocol?: string, upstreamOr
return base.slice(0, -".json".length) + ".content-store.json";
}

/** #3201: system-surface sidecar path — same sibling discipline as the
* content store above. */
function systemSurfaceRelPathFor(id: string, protocol?: string, upstreamOrigin?: string): string {
const base = relPathFor(id, protocol, upstreamOrigin);
return base.slice(0, -".json".length) + SYSTEM_SURFACE_SUFFIX;
}

/** #1937: true when a walked FILENAME is the canonical (or legacy flat /
* .fb fallback) name for record `id` — mirrors the kernel StateStore.loadAll
* owner check so independent directory walkers agree with loadSync about
Expand Down Expand Up @@ -553,12 +561,16 @@ export class SessionStore {
legacy: (parsed) => (isValidRecord(parsed) ? { id: parsed.id, payload: parsed, version: parsed.version, savedAt: parsed.savedAt } : null),
validate: (envelope) => isValidRecord(envelope.payload),
// #2298: CCR content-store envelopes (#1097) co-reside in the SAME
// namespace by design (<proto>/<host>_<hash>.content-store.json) but
// namespace by design (<proto>/<host>_<hash>.content-store.json), as
// do #3201 system-surface sidecars (.system-surface.json), but
// are not session records — declaring them foreign keeps the boot
// scan from parsing each one and warning "skipping invalid record"
// per file (hundreds of warns per start at multi-relay scale). Same
// suffix predicate the GC (#1180) and web walkers already use.
foreignFile: (file) => path.basename(file).endsWith(".content-store.json"),
foreignFile: (file) => {
const base = path.basename(file);
return base.endsWith(".content-store.json") || base.endsWith(SYSTEM_SURFACE_SUFFIX);
},
});
}

Expand All @@ -573,6 +585,45 @@ export class SessionStore {
]);
}

/** #3201: load the session's system-surface sidecar (same probe order as
* loadContentStore). It is a DERIVED artifact, unlike the content store:
* a corrupt/unreadable file degrades to "not captured" (warned) and must
* never refuse boot or block a save. */
loadSystemSurface(session: Session): SystemSurfaceRecord | null {
return this.readSystemSurfaceRels(session.id, [
systemSurfaceRelPathFor(session.id, session.meta.protocol, session.meta.upstreamOrigin),
systemSurfaceRelPathFor(session.id),
]);
}

private readSystemSurfaceRels(id: string, rels: string[]): SystemSurfaceRecord | null {
if (!this.enabled) return null;
const source = this.sourceFiles.get(id);
const paths = [...rels.map(rel => path.join(this.sessionsDir, rel)),
...(source ? [source.replace(/\.json$/, SYSTEM_SURFACE_SUFFIX)] : []),
...this.diskFiles.filter(file => fileNameMatchesId(path.basename(file), id))
.map(file => file.replace(/\.json$/, SYSTEM_SURFACE_SUFFIX))];
for (const file of new Set(paths)) {
let buf: Buffer;
try {
buf = readFileSync(file);
} catch (err) {
if ((err as NodeJS.ErrnoException).code === "ENOENT") continue;
this.log("warn", `[persist] system surface for ${id} unreadable, ignoring: ${String(err)}`);
return null;
}
try {
const parsed: unknown = JSON.parse(this.readCodec.decode(buf));
if (isValidSystemSurface(parsed)) return parsed;
throw new Error(`system surface for ${id} is malformed`);
} catch (err) {
this.log("warn", `[persist] system surface for ${id} unreadable, ignoring: ${err instanceof Error ? err.message : String(err)}`);
return null;
}
}
return null;
}

/** Id+meta variant for replica freeze/restore (#2671 review 6): the
* checkpoint path has no live Session object, only the frozen record's
* meta — same rel-path candidates, same parse-and-degrade semantics. */
Expand Down Expand Up @@ -689,6 +740,34 @@ export class SessionStore {
}
}

/** #3201: persist the system-surface sidecar when dirty; an absent
* surface deletes the file. Same atomic tmp+rename discipline as
* saveContentStore. Unlike that one, a failure here only re-arms dirty —
* it never blocks the session state write (derived artifact, see
* readSystemSurfaceRels). */
private saveSystemSurface(session: Session): void {
if (!this.enabled || !session.systemSurfaceDirty) return;
const rel = systemSurfaceRelPathFor(session.id, session.meta.protocol, session.meta.upstreamOrigin);
const abs = path.join(this.sessionsDir, rel);
const tmp = path.join(path.dirname(abs), `.tmp-${path.basename(abs, ".json")}-${process.pid}-${this.tmpSeq++}`);
try {
if (!session.systemSurface) {
rmSync(abs, { force: true });
session.systemSurfaceDirty = false;
return;
}
mkdirSync(path.dirname(abs), { recursive: true });
const text = JSON.stringify(session.systemSurface);
writeFileSync(tmp, this.codec ? this.codec.encode(text) : text, { mode: 0o600 });
renameSync(tmp, abs);
session.systemSurfaceDirty = false;
} catch (err) {
try { rmSync(tmp, { force: true }); } catch { /* best-effort: a stuck temp is swept at next boot */ }
session.systemSurfaceDirty = true;
this.log("warn", `[persist] system-surface write failed for ${session.id}: ${err instanceof Error ? err.message : String(err)}`);
}
}

/** Bulk-load every persisted session from disk into a map keyed by the
* REAL session id (read from the file body, not the filename). Called once
* at startup before the server accepts traffic. Corrupt individual files
Expand Down Expand Up @@ -790,7 +869,8 @@ export class SessionStore {
for (const root of [this.sessionsDir, `${this.sessionsDir}-cold`]) {
const files = existsSync(root) ? await walkJsonFiles(root) : [];
for (const file of files) {
if (path.basename(file).endsWith(".content-store.json")) continue;
const base = path.basename(file);
if (base.endsWith(".content-store.json") || base.endsWith(SYSTEM_SURFACE_SUFFIX)) continue;
this.diskFiles.push(file);
filesByRoot.set(root, [...(filesByRoot.get(root) ?? []), file]);
}
Expand Down Expand Up @@ -2289,6 +2369,7 @@ export class SessionStore {
* internal debounce could not do: every per-turn save then proved its
* bytes on disk and the ownership fast path stayed live. */
scheduleSave(session: Session): void {
this.saveSystemSurface(session);
if (!this.saveContentStore(session)) return;
if (this.writerOffActive()) {
// Phase 3 (#2960): index-only persistence — the builder closure
Expand Down Expand Up @@ -2383,6 +2464,7 @@ export class SessionStore {
* check and the build), so concurrent callers there cannot interleave
* either. Do NOT regress this by eager-snapshotting the record here. */
async writeNow(session: Session): Promise<void> {
this.saveSystemSurface(session);
if (!this.saveContentStore(session)) throw new Error(`Originals were not saved for ${session.id}`);
if (this.writerOffActive()) {
// #3133: drop the armed microtask (token mismatch below) — this
Expand Down Expand Up @@ -2463,6 +2545,7 @@ export class SessionStore {
* Returns true on success, false on failure (caller must NOT evict on
* failure for a never-persisted session or it is lost permanently). */
flushSync(session: Session): boolean {
this.saveSystemSurface(session);
if (!this.saveContentStore(session)) return false;
if (this.writerOffActive()) {
// #3133: same subsume rule as writeNow.
Expand Down Expand Up @@ -2512,7 +2595,10 @@ export class SessionStore {
// a stray microtask would build against a closed store and burn a
// 3-strike disable for no reason.
this.drainWriterOffPending();
for (const session of sessions) this.saveContentStore(session);
for (const session of sessions) {
this.saveSystemSurface(session);
this.saveContentStore(session);
}
await this.flushScheduledSaves();
await this.store.flushAll();
this.shadow?.close();
Expand Down
4 changes: 4 additions & 0 deletions src/server/prepare-anthropic.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import { estimateCoreMessages, estimateCoreMessagesUpper, extractBillingAttribut
import { dshNativeCompactionWitness, recordConflict } from "../conflict-watch.js";
import { anthropicToolsCarryCacheControl, computeAnthropicMessageMarks, stampAnthropicSystemCacheControl } from "../loop/cache-control.js";
import { reconcileSystemAnchor } from "../system-anchor.js";
import { captureSystemSurface } from "../system-surface.js";
import { isStrictReasoningEcho, modelIdOf } from "../strict-echo.js";
import { logRawStripCounts, markerArtifactCount, stripRawStage } from "../marker/engine.js";
import { imageRescaleMaxDim, renderNone as knobRenderNone } from "../knobs.js";
Expand Down Expand Up @@ -513,6 +514,9 @@ export async function prepareAnthropic(
// SysPrompt row — the kernel breakdown classifies messages only, and on
// this wire the system rides the top-level `system` field outside the fold.
session.metadata.systemPromptTokens = countSystemAndToolsTokens(extractSystem(systemOut), toolsOut);
// #3201: persist the system+tools surface text (web trajectory view) at
// the same point the overhead is measured, so both stay in lockstep.
captureSystemSurface(session, extractSystem(systemOut), toolsOut);
snapshotMessages(session, originalMessages);
markDirty(session);

Expand Down
5 changes: 5 additions & 0 deletions src/server/prepare-google.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import { logRawStripCounts, markerArtifactCount, stripRawStage } from "../marker
import { imageRescaleMaxDim, renderNone as knobRenderNone } from "../knobs.js";
import { applyImageRescalePass } from "../image-rescale.js";
import { clampOutgoingOutput, countSystemAndToolsTokens, dshLedgerFloorTokens, emergencyNudge, emergencyNudgeArmed } from "./budget.js";
import { captureSystemSurface } from "../system-surface.js";
import { estimateCoreMessages } from "../preflight.js";
import { effectiveAbsorbBlock } from "./prepare-responses.js";
import { injectGoogleTool, injectTool } from "./inject.js";
Expand Down Expand Up @@ -356,6 +357,10 @@ export async function prepareGoogle(
// #532: title-gen side requests carry their own tiny system — skip them.
if (!isTitleGen && googleOutboundSystem !== undefined) {
session.metadata.systemPromptTokens = countSystemAndToolsTokens(googleOutboundSystem, toolsOut);
// #3201: persist the system+tools surface text (web trajectory view);
// same skip guard as the measurement above — title-gen side requests
// carry their own tiny system and must not clobber it.
captureSystemSurface(session, googleOutboundSystem, toolsOut);
}
// #1933 F1 + #2407: billed-caliber denominator of the k̂ learning pair.
// The Google lane was the only one of the four never recording it, so
Expand Down
5 changes: 5 additions & 0 deletions src/server/prepare-openai.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import { attachSubagentSessions } from "../subagent-sessions.js";
import { estimateCoreMessages, estimateCoreMessagesUpper } from "../preflight.js";
import { dshNativeCompactionWitness, recordConflict } from "../conflict-watch.js";
import { isStrictReasoningEcho, modelIdOf, normalizeStrictEchoReasoning } from "../strict-echo.js";
import { captureSystemSurface } from "../system-surface.js";
import { reconcileSystemAnchor } from "../system-anchor.js";
import { logRawStripCounts, markerArtifactCount, stripRawStage } from "../marker/engine.js";
import { imageRescaleMaxDim, renderNone as knobRenderNone } from "../knobs.js";
Expand Down Expand Up @@ -453,6 +454,10 @@ export async function prepareOpenai(
// clobber the conversation's measured overhead — skip them.
if (!isTitleGen && openaiOutboundSystem !== undefined) {
session.metadata.systemPromptTokens = countSystemAndToolsTokens(openaiOutboundSystem, toolsOut);
// #3201: persist the system+tools surface text (web trajectory view);
// same skip guard as the measurement above — title-gen side requests
// carry their own tiny system and must not clobber it.
captureSystemSurface(session, openaiOutboundSystem, toolsOut);
}
// #728: record this turn's outbound payload upper bound as the fallback
// token source for upstreams that never report usage (see effectiveTokenCount).
Expand Down
5 changes: 5 additions & 0 deletions src/server/prepare-responses.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import { applyImageRescaleItems } from "../image-rescale.js";
import { clampOutgoingOutput, countLoadedToolTokens, countSystemAndToolsTokens, dshLedgerFloorTokens, emergencyNudge, emergencyNudgeArmed, modelVisibleTools, responsesSysDevItemText } from "./budget.js";
import { FORCE_TEXT_PROTOCOL, injectResponsesTool, injectTool } from "./inject.js";
import { nudgeOn, resolveLaneDecisions, surfaceOffDetail } from "./lane-decisions.js";
import { captureSystemSurface } from "../system-surface.js";

export async function prepareResponses(
parsed: ResponsesRequestBody,
Expand Down Expand Up @@ -642,6 +643,10 @@ export async function prepareResponses(
const loadedToolTokens = countLoadedToolTokens(rebuilt);
if (transformOk) {
session.metadata.systemPromptTokens = countSystemAndToolsTokens(responsesDevContent ?? "", toolsOut) + loadedToolTokens + inplaceSysDevTokens;
// #3201: persist the system+tools surface text (web trajectory view);
// inside transformOk like the measurement — a failed kernel transform
// forwards unchanged and must not record a stale surface.
captureSystemSurface(session, responsesDevContent ?? "", toolsOut);
const catalogTokens = defaultCountTokens(JSON.stringify(toolsOut ?? []));
const visibleTokens = defaultCountTokens(JSON.stringify(modelVisibleTools(toolsOut)));
if (catalogTokens >= 50_000 || catalogTokens !== visibleTokens || loadedToolTokens > 0) {
Expand Down
Loading
Loading