From cc12adbbebc2b33efce130dd3e8be0034aed6771 Mon Sep 17 00:00:00 2001 From: awork Date: Sat, 17 Oct 2026 22:54:55 +0800 Subject: [PATCH 1/2] =?UTF-8?q?fix:=20#3573=20loud=20cold-load=20diagnosti?= =?UTF-8?q?cs=20=E2=80=94=20refs-without-blocks=20no=20longer=20silent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The v0.1.198 incident session re-entered as raw history with zero fold-reconcile logs: reconcileFoldCoverage early-exits silently when the loaded state carries no active blocks, and nothing logged what the lazy cold load actually restored. Every on-disk load now reports its carried refs/blocks/baseline at info level, and the incident signature (refs mapped, zero persisted blocks) escalates to a warning naming the consequence, so the next occurrence is a one-line diagnosis. --- src/persist.ts | 20 +++ tests/issue3573-cold-load-diagnostics.test.ts | 149 ++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 tests/issue3573-cold-load-diagnostics.test.ts diff --git a/src/persist.ts b/src/persist.ts index f2928a4496..c04a24523a 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -1143,6 +1143,23 @@ export class SessionStore { * LRU eviction). Sync fs is acceptable here because a miss is rare and * reads a single small file (~1ms). Returns null if missing/corrupt or the * body id does not match what we asked for. */ + /** #3573: make the lazy-load moment observable. This is where persisted + * state re-enters memory after a restart or eviction; logging what it + * actually carried turns "the session came back raw" from a post-mortem + * guess into a one-line diagnosis. refs>0 with zero blocks is the + * incident signature (#3573): identity loaded but every fold missing, so + * the full history replays and preflight must rebuild from scratch. */ + private logColdLoad(id: string, source: string, session: Session, savedAt?: number): void { + const refs = Object.keys(session.state.messageRefs.byRef ?? {}).length; + const blocks = session.state.blocks.length; + const active = session.state.blocks.filter((b) => b.active).length; + const ageMin = savedAt ? Math.round(Math.max(0, Date.now() - savedAt) / 60000) : undefined; + this.log("info", `[persist] cold-loaded ${id} from ${source}: ${refs} ref(s), ${blocks} block(s) (${active} active), baseline ${session.stats.lastInputTokens} tok${ageMin !== undefined ? `, savedAt ~${ageMin} min ago` : ""}`); + if (refs > 0 && blocks === 0) { + this.log("warn", `[persist] ${id} loaded with ${refs} mapped ref(s) but NO persisted blocks — the folded history will replay raw and preflight must rebuild from scratch (#3573)`); + } + } + loadSync(id: string, meta?: { protocol?: string; upstreamOrigin?: string }): Session | null { if (!this.enabled) return null; if (this.shadow?.readsUnified) { @@ -1165,6 +1182,7 @@ export class SessionStore { this.scheduleSave(session); this.log("info", `[persist] clamped negative token stats on reload for ${id} (#408)`); } + this.logColdLoad(id, "legacy file", session, envelope.payload.savedAt); return session; } const envelope = this.loadEnvelope(id, meta); @@ -1181,6 +1199,7 @@ export class SessionStore { this.scheduleSave(session); this.log("info", `[persist] clamped negative token stats on reload for ${id} (#408)`); } + this.logColdLoad(id, "legacy file", session, envelope.payload.savedAt); return session; } @@ -1195,6 +1214,7 @@ export class SessionStore { this.scheduleSave(session); this.log("info", `[persist] clamped negative token stats on reload for ${id} (#408)`); } + this.logColdLoad(id, "unified store", session, record.savedAt); return session; } diff --git a/tests/issue3573-cold-load-diagnostics.test.ts b/tests/issue3573-cold-load-diagnostics.test.ts new file mode 100644 index 0000000000..92b404f405 --- /dev/null +++ b/tests/issue3573-cold-load-diagnostics.test.ts @@ -0,0 +1,149 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { tmpdir } from "node:os"; +import { mkdtempSync } from "node:fs"; +import { join } from "node:path"; +import { SessionStore } from "../src/persist.ts"; +import type { Session } from "../src/session.ts"; +import { assignRefs, createInitialState, emptyRefMap } from "acp-kernel"; +import { rmrf } from "./tmp-rm.ts"; + +// #3573 (diagnostic half): after the v0.1.198 upgrade incident, a folded +// session re-entered as raw history (1820 messages, usage 0%, ref=0) with NO +// fold-reconcile logs at all — reconcileFoldCoverage early-exits silently +// when the loaded state has no active blocks, and nothing logged WHAT the +// lazy cold load actually carried. Was the state ever loaded? Did it carry +// refs without blocks (identity survived, every fold missing)? Or did the +// request key onto a ghost fork id? All three were indistinguishable in the +// field. This pins the load-side answer: every on-disk cold load now reports +// its carried refs/blocks/baseline at info level, and the incident signature +// specifically (refs mapped, ZERO persisted blocks) escalates to a warning +// naming the consequence (raw replay + rebuild-from-scratch preflight), so +// the next occurrence is a one-line diagnosis instead of a post-mortem. +// Runs under the default storage mode (unified shadow double-ingest — what +// the incident machine ran); the same assertions hold on the plain-legacy +// branch, whose source label differs ("legacy file" vs "unified store"). + +type LogLine = { level: string; msg: string }; + +const META = { protocol: "openai", upstreamOrigin: "http://upstream" }; + +function makeSession(id: string): Session { + return { + id, + meta: { protocol: "openai", upstreamOrigin: "http://upstream" }, + stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, contextTokens: 0, compressCreditTokens: 0, retrieveCalls: 0, retrieveHits: 0, retrieveMisses: 0, storedBytes: 0, storeBytesSaved: 0, rangeRestores: 0 }, + metadata: {}, + state: createInitialState(), + createdAt: Date.now(), + lastSeen: Date.now(), + blockContents: new Map(), + inFlight: 0, + persisted: false, + pendingRetrievals: [], + }; +} + +/** Seed message refs the way the stream path does after each turn (#387). */ +function seedRefs(session: Session, n: number): void { + const msgs = Array.from({ length: n }, (_, i) => ({ id: `raw${i}`, role: i % 2 === 0 ? ("user" as const) : ("assistant" as const), contentType: "text" as const, text: `message ${i}` })); + session.state.messageRefs = assignRefs(msgs, { existing: emptyRefMap(), nextIndex: 0 }).map; +} + +function storeIn(dir: string, logs: LogLine[]): SessionStore { + return new SessionStore({ dir, debounceMs: 0, enabled: true, log: (level, msg) => logs.push({ level, msg }) }); +} + +test("#3573: cold load carrying refs but NO persisted blocks warns loudly (the incident signature)", async () => { + const dir = mkdtempSync(join(tmpdir(), "bili-3573-coldload-")); + try { + // Persist a refs-only state: identity survived, every fold gone — + // the on-disk end-state the incident left behind, whatever produced it. + const writerLogs: LogLine[] = []; + const writer = storeIn(dir, writerLogs); + const s = makeSession("s-ghost"); + seedRefs(s, 6); + s.stats.lastInputTokens = 119_940; + await writer.writeNow(s); + + const logs: LogLine[] = []; + const reader = storeIn(dir, logs); + const loaded = reader.loadSync("s-ghost", META); + assert.ok(loaded, "a refs-only state still loads (nothing to quarantine)"); + assert.equal(Object.keys(loaded.state.messageRefs.byRef ?? {}).length, 6, "refs survived the round trip"); + assert.equal(loaded.state.blocks.length, 0, "blocks are genuinely absent, not dropped by the loader"); + + const info = logs.find((l) => l.level === "info" && l.msg.includes("cold-loaded s-ghost")); + assert.ok(info, `an info line records what the cold load carried (logs: ${JSON.stringify(logs)})`); + assert.match(info!.msg, /6 ref\(s\), 0 block\(s\) \(0 active\)/); + assert.match(info!.msg, /baseline 119940 tok/); + + const warn = logs.find((l) => l.level === "warn" && l.msg.includes("s-ghost")); + assert.ok(warn, "refs-without-blocks must WARN — silence is exactly what made #3573 un-diagnosable"); + assert.match(warn!.msg, /6 mapped ref\(s\) but NO persisted blocks/); + assert.match(warn!.msg, /#3573/); + } finally { + rmrf(dir); + } +}); + +test("#3573: a healthy cold load reports its counts without warning", async () => { + const dir = mkdtempSync(join(tmpdir(), "bili-3573-coldload-")); + try { + const writerLogs: LogLine[] = []; + const writer = storeIn(dir, writerLogs); + const s = makeSession("s-healthy"); + seedRefs(s, 8); + s.stats.lastInputTokens = 119_940; + s.state.blocks = [{ + blockId: "blk-1", + runId: "run-1", + tier: 1, + summary: "folded segment", + directMessageIds: ["raw0", "raw1", "raw2", "raw3"], + effectiveMessageIds: ["raw0", "raw1", "raw2", "raw3"], + directBlockIds: [], + compressedTokens: 16, + createdAt: Date.now(), + survivedCount: 0, + generation: "young", + active: true, + startRef: "m00001", + endRef: "m00004", + }]; + await writer.writeNow(s); + + const logs: LogLine[] = []; + const reader = storeIn(dir, logs); + const loaded = reader.loadSync("s-healthy", META); + assert.ok(loaded, "healthy state loads"); + assert.equal(loaded.state.blocks.length, 1, "the block survived the round trip"); + + const info = logs.find((l) => l.level === "info" && l.msg.includes("cold-loaded s-healthy")); + assert.ok(info, `an info line records the healthy cold load (logs: ${JSON.stringify(logs)})`); + assert.match(info!.msg, /8 ref\(s\), 1 block\(s\) \(1 active\)/); + assert.ok(!logs.some((l) => l.level === "warn" && l.msg.includes("s-healthy")), "no warning on a healthy load"); + } finally { + rmrf(dir); + } +}); + +test("#3573: a fresh empty state logs its counts and does not false-positive the warning", async () => { + const dir = mkdtempSync(join(tmpdir(), "bili-3573-coldload-")); + try { + const writerLogs: LogLine[] = []; + const writer = storeIn(dir, writerLogs); + await writer.writeNow(makeSession("s-empty")); + + const logs: LogLine[] = []; + const reader = storeIn(dir, logs); + const loaded = reader.loadSync("s-empty", META); + assert.ok(loaded, "an empty state loads"); + const info = logs.find((l) => l.level === "info" && l.msg.includes("cold-loaded s-empty")); + assert.ok(info, `an info line records the empty cold load (logs: ${JSON.stringify(logs)})`); + assert.match(info!.msg, /0 ref\(s\), 0 block\(s\) \(0 active\)/); + assert.ok(!logs.some((l) => l.level === "warn" && l.msg.includes("s-empty")), "zero refs must NOT trigger the no-blocks warning"); + } finally { + rmrf(dir); + } +}); From 3e38248719774bcac9d2bcf40d1c19082795b5a0 Mon Sep 17 00:00:00 2001 From: awork Date: Sat, 17 Oct 2026 23:47:54 +0800 Subject: [PATCH 2/2] fix: #3573 carry savedTok in cold-load diagnostics (review supplement) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review found the refs-without-blocks warn also fires for sessions that were NEVER folded (any session with >=1 persisted turn and zero folds): after a restart, revisiting ordinary short sessions would log the same warning as a lost-fold incident, diluting the signal this PR exists to create. Both lines now carry stats.tokensSaved: >0 proves compression ran in the session's lifetime (folds existed and are now missing — investigate); 0 means likely never folded (raw replay is its normal shape). The warn condition stays deliberately broad so a loss mode that also erases stats cannot go silent again. Tests pin both classes. --- src/persist.ts | 13 ++++++++++--- tests/issue3573-cold-load-diagnostics.test.ts | 8 ++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/persist.ts b/src/persist.ts index c04a24523a..f2de7d11e6 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -1148,15 +1148,22 @@ export class SessionStore { * actually carried turns "the session came back raw" from a post-mortem * guess into a one-line diagnosis. refs>0 with zero blocks is the * incident signature (#3573): identity loaded but every fold missing, so - * the full history replays and preflight must rebuild from scratch. */ + * the full history replays and preflight must rebuild from scratch. + * savedTok (stats.tokensSaved) disambiguates that signature in one grep: + * >0 proves compression ran in this session's lifetime, so zero blocks + * means folds existed and are now missing (investigate); 0 means the + * session was likely never folded (raw replay is its normal shape). The + * warn stays broad on purpose — a loss mode that erases stats must not + * be able to go silent again. */ private logColdLoad(id: string, source: string, session: Session, savedAt?: number): void { const refs = Object.keys(session.state.messageRefs.byRef ?? {}).length; const blocks = session.state.blocks.length; const active = session.state.blocks.filter((b) => b.active).length; + const savedTok = session.stats.tokensSaved; const ageMin = savedAt ? Math.round(Math.max(0, Date.now() - savedAt) / 60000) : undefined; - this.log("info", `[persist] cold-loaded ${id} from ${source}: ${refs} ref(s), ${blocks} block(s) (${active} active), baseline ${session.stats.lastInputTokens} tok${ageMin !== undefined ? `, savedAt ~${ageMin} min ago` : ""}`); + this.log("info", `[persist] cold-loaded ${id} from ${source}: ${refs} ref(s), ${blocks} block(s) (${active} active), baseline ${session.stats.lastInputTokens} tok, savedTok ${savedTok}${ageMin !== undefined ? `, savedAt ~${ageMin} min ago` : ""}`); if (refs > 0 && blocks === 0) { - this.log("warn", `[persist] ${id} loaded with ${refs} mapped ref(s) but NO persisted blocks — the folded history will replay raw and preflight must rebuild from scratch (#3573)`); + this.log("warn", `[persist] ${id} loaded with ${refs} mapped ref(s) but NO persisted blocks (savedTok=${savedTok}) — the folded history will replay raw and preflight must rebuild from scratch (#3573)`); } } diff --git a/tests/issue3573-cold-load-diagnostics.test.ts b/tests/issue3573-cold-load-diagnostics.test.ts index 92b404f405..9707dad21f 100644 --- a/tests/issue3573-cold-load-diagnostics.test.ts +++ b/tests/issue3573-cold-load-diagnostics.test.ts @@ -64,6 +64,9 @@ test("#3573: cold load carrying refs but NO persisted blocks warns loudly (the i const s = makeSession("s-ghost"); seedRefs(s, 6); s.stats.lastInputTokens = 119_940; + // The incident session HAD been folded — savedTok>0 is what makes the + // zero-blocks signature diagnosable (folds existed, now missing). + s.stats.tokensSaved = 42_000; await writer.writeNow(s); const logs: LogLine[] = []; @@ -76,11 +79,11 @@ test("#3573: cold load carrying refs but NO persisted blocks warns loudly (the i const info = logs.find((l) => l.level === "info" && l.msg.includes("cold-loaded s-ghost")); assert.ok(info, `an info line records what the cold load carried (logs: ${JSON.stringify(logs)})`); assert.match(info!.msg, /6 ref\(s\), 0 block\(s\) \(0 active\)/); - assert.match(info!.msg, /baseline 119940 tok/); + assert.match(info!.msg, /baseline 119940 tok, savedTok 42000/); const warn = logs.find((l) => l.level === "warn" && l.msg.includes("s-ghost")); assert.ok(warn, "refs-without-blocks must WARN — silence is exactly what made #3573 un-diagnosable"); - assert.match(warn!.msg, /6 mapped ref\(s\) but NO persisted blocks/); + assert.match(warn!.msg, /6 mapped ref\(s\) but NO persisted blocks \(savedTok=42000\)/); assert.match(warn!.msg, /#3573/); } finally { rmrf(dir); @@ -142,6 +145,7 @@ test("#3573: a fresh empty state logs its counts and does not false-positive the const info = logs.find((l) => l.level === "info" && l.msg.includes("cold-loaded s-empty")); assert.ok(info, `an info line records the empty cold load (logs: ${JSON.stringify(logs)})`); assert.match(info!.msg, /0 ref\(s\), 0 block\(s\) \(0 active\)/); + assert.match(info!.msg, /savedTok 0/); assert.ok(!logs.some((l) => l.level === "warn" && l.msg.includes("s-empty")), "zero refs must NOT trigger the no-blocks warning"); } finally { rmrf(dir);