Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions src/persist.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1143,6 +1143,30 @@ export class SessionStore {
* LRU eviction). Sync fs is acceptable here because a miss is rare and
* reads a single small file (~1ms). Returns null if missing/corrupt or the
* body id does not match what we asked for. */
/** #3573: make the lazy-load moment observable. This is where persisted
* state re-enters memory after a restart or eviction; logging what it
* actually carried turns "the session came back raw" from a post-mortem
* guess into a one-line diagnosis. refs>0 with zero blocks is the
* incident signature (#3573): identity loaded but every fold missing, so
* the full history replays and preflight must rebuild from scratch.
* savedTok (stats.tokensSaved) disambiguates that signature in one grep:
* >0 proves compression ran in this session's lifetime, so zero blocks
* means folds existed and are now missing (investigate); 0 means the
* session was likely never folded (raw replay is its normal shape). The
* warn stays broad on purpose — a loss mode that erases stats must not
* be able to go silent again. */
private logColdLoad(id: string, source: string, session: Session, savedAt?: number): void {
const refs = Object.keys(session.state.messageRefs.byRef ?? {}).length;
const blocks = session.state.blocks.length;
const active = session.state.blocks.filter((b) => b.active).length;
const savedTok = session.stats.tokensSaved;
const ageMin = savedAt ? Math.round(Math.max(0, Date.now() - savedAt) / 60000) : undefined;
this.log("info", `[persist] cold-loaded ${id} from ${source}: ${refs} ref(s), ${blocks} block(s) (${active} active), baseline ${session.stats.lastInputTokens} tok, savedTok ${savedTok}${ageMin !== undefined ? `, savedAt ~${ageMin} min ago` : ""}`);
if (refs > 0 && blocks === 0) {
this.log("warn", `[persist] ${id} loaded with ${refs} mapped ref(s) but NO persisted blocks (savedTok=${savedTok}) — the folded history will replay raw and preflight must rebuild from scratch (#3573)`);
}
}

loadSync(id: string, meta?: { protocol?: string; upstreamOrigin?: string }): Session | null {
if (!this.enabled) return null;
if (this.shadow?.readsUnified) {
Expand All @@ -1165,6 +1189,7 @@ export class SessionStore {
this.scheduleSave(session);
this.log("info", `[persist] clamped negative token stats on reload for ${id} (#408)`);
}
this.logColdLoad(id, "legacy file", session, envelope.payload.savedAt);
return session;
}
const envelope = this.loadEnvelope(id, meta);
Expand All @@ -1181,6 +1206,7 @@ export class SessionStore {
this.scheduleSave(session);
this.log("info", `[persist] clamped negative token stats on reload for ${id} (#408)`);
}
this.logColdLoad(id, "legacy file", session, envelope.payload.savedAt);
return session;
}

Expand All @@ -1195,6 +1221,7 @@ export class SessionStore {
this.scheduleSave(session);
this.log("info", `[persist] clamped negative token stats on reload for ${id} (#408)`);
}
this.logColdLoad(id, "unified store", session, record.savedAt);
return session;
}

Expand Down
153 changes: 153 additions & 0 deletions tests/issue3573-cold-load-diagnostics.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { tmpdir } from "node:os";
import { mkdtempSync } from "node:fs";
import { join } from "node:path";
import { SessionStore } from "../src/persist.ts";
import type { Session } from "../src/session.ts";
import { assignRefs, createInitialState, emptyRefMap } from "acp-kernel";
import { rmrf } from "./tmp-rm.ts";

// #3573 (diagnostic half): after the v0.1.198 upgrade incident, a folded
// session re-entered as raw history (1820 messages, usage 0%, ref=0) with NO
// fold-reconcile logs at all — reconcileFoldCoverage early-exits silently
// when the loaded state has no active blocks, and nothing logged WHAT the
// lazy cold load actually carried. Was the state ever loaded? Did it carry
// refs without blocks (identity survived, every fold missing)? Or did the
// request key onto a ghost fork id? All three were indistinguishable in the
// field. This pins the load-side answer: every on-disk cold load now reports
// its carried refs/blocks/baseline at info level, and the incident signature
// specifically (refs mapped, ZERO persisted blocks) escalates to a warning
// naming the consequence (raw replay + rebuild-from-scratch preflight), so
// the next occurrence is a one-line diagnosis instead of a post-mortem.
// Runs under the default storage mode (unified shadow double-ingest — what
// the incident machine ran); the same assertions hold on the plain-legacy
// branch, whose source label differs ("legacy file" vs "unified store").

type LogLine = { level: string; msg: string };

const META = { protocol: "openai", upstreamOrigin: "http://upstream" };

function makeSession(id: string): Session {
return {
id,
meta: { protocol: "openai", upstreamOrigin: "http://upstream" },
stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, contextTokens: 0, compressCreditTokens: 0, retrieveCalls: 0, retrieveHits: 0, retrieveMisses: 0, storedBytes: 0, storeBytesSaved: 0, rangeRestores: 0 },
metadata: {},
state: createInitialState(),
createdAt: Date.now(),
lastSeen: Date.now(),
blockContents: new Map(),
inFlight: 0,
persisted: false,
pendingRetrievals: [],
};
}

/** Seed message refs the way the stream path does after each turn (#387). */
function seedRefs(session: Session, n: number): void {
const msgs = Array.from({ length: n }, (_, i) => ({ id: `raw${i}`, role: i % 2 === 0 ? ("user" as const) : ("assistant" as const), contentType: "text" as const, text: `message ${i}` }));
session.state.messageRefs = assignRefs(msgs, { existing: emptyRefMap(), nextIndex: 0 }).map;
}

function storeIn(dir: string, logs: LogLine[]): SessionStore {
return new SessionStore({ dir, debounceMs: 0, enabled: true, log: (level, msg) => logs.push({ level, msg }) });
}

test("#3573: cold load carrying refs but NO persisted blocks warns loudly (the incident signature)", async () => {
const dir = mkdtempSync(join(tmpdir(), "bili-3573-coldload-"));
try {
// Persist a refs-only state: identity survived, every fold gone —
// the on-disk end-state the incident left behind, whatever produced it.
const writerLogs: LogLine[] = [];
const writer = storeIn(dir, writerLogs);
const s = makeSession("s-ghost");
seedRefs(s, 6);
s.stats.lastInputTokens = 119_940;
// The incident session HAD been folded — savedTok>0 is what makes the
// zero-blocks signature diagnosable (folds existed, now missing).
s.stats.tokensSaved = 42_000;
await writer.writeNow(s);

const logs: LogLine[] = [];
const reader = storeIn(dir, logs);
const loaded = reader.loadSync("s-ghost", META);
assert.ok(loaded, "a refs-only state still loads (nothing to quarantine)");
assert.equal(Object.keys(loaded.state.messageRefs.byRef ?? {}).length, 6, "refs survived the round trip");
assert.equal(loaded.state.blocks.length, 0, "blocks are genuinely absent, not dropped by the loader");

const info = logs.find((l) => l.level === "info" && l.msg.includes("cold-loaded s-ghost"));
assert.ok(info, `an info line records what the cold load carried (logs: ${JSON.stringify(logs)})`);
assert.match(info!.msg, /6 ref\(s\), 0 block\(s\) \(0 active\)/);
assert.match(info!.msg, /baseline 119940 tok, savedTok 42000/);

const warn = logs.find((l) => l.level === "warn" && l.msg.includes("s-ghost"));
assert.ok(warn, "refs-without-blocks must WARN — silence is exactly what made #3573 un-diagnosable");
assert.match(warn!.msg, /6 mapped ref\(s\) but NO persisted blocks \(savedTok=42000\)/);
assert.match(warn!.msg, /#3573/);
} finally {
rmrf(dir);
}
});

test("#3573: a healthy cold load reports its counts without warning", async () => {
const dir = mkdtempSync(join(tmpdir(), "bili-3573-coldload-"));
try {
const writerLogs: LogLine[] = [];
const writer = storeIn(dir, writerLogs);
const s = makeSession("s-healthy");
seedRefs(s, 8);
s.stats.lastInputTokens = 119_940;
s.state.blocks = [{
blockId: "blk-1",
runId: "run-1",
tier: 1,
summary: "folded segment",
directMessageIds: ["raw0", "raw1", "raw2", "raw3"],
effectiveMessageIds: ["raw0", "raw1", "raw2", "raw3"],
directBlockIds: [],
compressedTokens: 16,
createdAt: Date.now(),
survivedCount: 0,
generation: "young",
active: true,
startRef: "m00001",
endRef: "m00004",
}];
await writer.writeNow(s);

const logs: LogLine[] = [];
const reader = storeIn(dir, logs);
const loaded = reader.loadSync("s-healthy", META);
assert.ok(loaded, "healthy state loads");
assert.equal(loaded.state.blocks.length, 1, "the block survived the round trip");

const info = logs.find((l) => l.level === "info" && l.msg.includes("cold-loaded s-healthy"));
assert.ok(info, `an info line records the healthy cold load (logs: ${JSON.stringify(logs)})`);
assert.match(info!.msg, /8 ref\(s\), 1 block\(s\) \(1 active\)/);
assert.ok(!logs.some((l) => l.level === "warn" && l.msg.includes("s-healthy")), "no warning on a healthy load");
} finally {
rmrf(dir);
}
});

test("#3573: a fresh empty state logs its counts and does not false-positive the warning", async () => {
const dir = mkdtempSync(join(tmpdir(), "bili-3573-coldload-"));
try {
const writerLogs: LogLine[] = [];
const writer = storeIn(dir, writerLogs);
await writer.writeNow(makeSession("s-empty"));

const logs: LogLine[] = [];
const reader = storeIn(dir, logs);
const loaded = reader.loadSync("s-empty", META);
assert.ok(loaded, "an empty state loads");
const info = logs.find((l) => l.level === "info" && l.msg.includes("cold-loaded s-empty"));
assert.ok(info, `an info line records the empty cold load (logs: ${JSON.stringify(logs)})`);
assert.match(info!.msg, /0 ref\(s\), 0 block\(s\) \(0 active\)/);
assert.match(info!.msg, /savedTok 0/);
assert.ok(!logs.some((l) => l.level === "warn" && l.msg.includes("s-empty")), "zero refs must NOT trigger the no-blocks warning");
} finally {
rmrf(dir);
}
});
Loading