diff --git a/src/acp-status.ts b/src/acp-status.ts index 784aa6aaf3..351e1bf226 100644 --- a/src/acp-status.ts +++ b/src/acp-status.ts @@ -19,6 +19,7 @@ import { pckAblations } from "./pck-probe.js"; import { getUnrecognizedPathStats } from "./server/observability.js"; import { ccrEnabled, ccrLoopConfig, contentStoreOf } from "./store.js"; import { coveredRefSpan } from "./decompress-shared.js"; +import { formatUncoveredHistory, uncoveredHistory } from "./uncovered-history.js"; import { preCompactionArchiveOf, statusInputBaseline, type Session } from "./session.js"; import { compressBreakerDetail, compressLastFailureCause } from "./stream.js"; import { describeAdvisory, getAdvisoryState } from "./advisory.js"; @@ -275,6 +276,18 @@ export function handleAcpStatus(args: Record, ctx: AcpStatusCtx } catch { // Base-only report; never fall back to a stale snapshot. } + // #4331: the fold scanner only offers spans it can prove from the current + // view; messages that fell between two fold boundaries are invisible to it + // yet still ride the wire on every request. Audit coverage independently so + // the "no compressible ranges remain" verdict never stands unqualified. + const uncovered = uncoveredHistory(ctx.session.state, ctx.messages, { + preserveRecentMessages: ctx.config.preserveRecentMessages, + maxRuns: 3, + }); + if (uncovered.messageCount > 0) { + extra.push(""); + extra.push(...formatUncoveredHistory(uncovered)); + } // #2362: dead refs — known to the session but no longer backed by any // visible or folded message (client history rewrite / host-native // compaction). Failure receipts point the model at acp_status, so this is diff --git a/src/stream.ts b/src/stream.ts index 260029ccb3..cba8f6a8e0 100644 --- a/src/stream.ts +++ b/src/stream.ts @@ -6,6 +6,7 @@ import { type Session, cacheBlockContent, markDirty, residentPressure, scrubTurn import { COMPRESS_TOOL_NAME, parseCompressInput, ABSORB_TOOL_NAME, type ParsedRange } from "./compress-tool.js"; import { effectiveAbsorbConfig, executeAbsorb, isProxyToolFor } from "./absorb.js"; import { executeSearchContextTarget, resolveDecompress } from "./decompress-shared.js"; +import { formatUncoveredHistory, uncoveredHistory } from "./uncovered-history.js"; import { ONE_CALL_HINT } from "acp-kernel"; import { adoptContentStore, contentStoreOf, ccrEnabled, drainPendingRetrievals, executeRetrieve, retrieveToolName } from "./store.js"; import { recordThreadTail } from "./server/thread-tracking.js"; @@ -605,7 +606,25 @@ function postCompressTail(ctx: RewriteCtx, cleanSuccess: boolean, submitted: Arr // Partial failures stay silent too: the model still owes the errors an answer // before any "you are done" verdict (pi #521 gate). if (!cleanSuccess || tier !== null) return ""; - return `\n\n${NO_RANGES_REMAIN_TEXT}`; + // #4331: "no ranges remain" is scoped to the fold scanner's offer, not to + // actual coverage. Traffic between two fold boundaries is never offered, so + // the stop verdict alone would claim a completeness that does not hold. + let uncoveredNote = ""; + try { + const uncovered = uncoveredHistory(ctx.session.state, ctx.messages, { + preserveRecentMessages: ctx.config.preserveRecentMessages, + maxRuns: 2, + // #1387/pi #521: a receipt must not re-nag about leftovers the fold + // scanner deliberately declined. Only a material blind spot (the + // kind a coverage audit can prove) earns a line here; acp_status + // shows the full inventory. + minTokens: 10_000, + }); + uncoveredNote = formatUncoveredHistory(uncovered, { compact: true }).join(" "); + } catch { + uncoveredNote = ""; + } + return uncoveredNote ? `\n\n${NO_RANGES_REMAIN_TEXT}\n${uncoveredNote}` : `\n\n${NO_RANGES_REMAIN_TEXT}`; } // #1495: the kernel's lenient parser salvages complete entries from damaged diff --git a/src/uncovered-history.ts b/src/uncovered-history.ts new file mode 100644 index 0000000000..24108c51fa --- /dev/null +++ b/src/uncovered-history.ts @@ -0,0 +1,156 @@ +import { countMessageTokens, defaultCountTokens, type CompressionBlock, type CompressionState, type CoreMessage } from "acp-kernel"; + +/** A contiguous run of ref-bearing messages that no active block covers. */ +interface UncoveredRun { + startRef: string; + endRef: string; + count: number; + tokens: number; +} + +interface UncoveredHistoryReport { + /** Largest runs, capped by `maxRuns`. */ + runs: UncoveredRun[]; + /** Total uncovered runs in the report window. */ + runCount: number; + /** Total uncovered messages in the report window. */ + messageCount: number; + /** Token estimate for every uncovered message (not just the listed runs). */ + tokens: number; +} + +interface UncoveredHistoryOptions { + /** Soft-protected tail size; the last N ref-bearing messages are excluded. */ + preserveRecentMessages?: number; + /** How many runs to list (largest first). Default 3. */ + maxRuns?: number; + /** Drop runs below this token estimate. Default 0. */ + minTokens?: number; +} + +const REF_RE = /^m(\d+)$/i; +const DEFAULT_PRESERVE_RECENT = 3; + +function refNumOf(ref: string | undefined | null): number | null { + if (typeof ref !== "string") return null; + const m = REF_RE.exec(ref.trim()); + return m ? Number(m[1]) : null; +} + +function fmtRef(n: number): string { + return `m${String(n).padStart(5, "0")}`; +} + +/** + * Every ref number covered by an active block. Blocks record coverage as raw + * message ids (`effectiveMessageIds`); a tier-2/3 block additionally owns its + * children, so the walk recurses through `directBlockIds` for the cases where + * the expanded id list was not persisted. + */ +export function coveredRefNumbers(state: CompressionState): Set { + const covered = new Set(); + const byId = new Map(); + for (const b of state.blocks) byId.set(b.blockId, b); + const seen = new Set(); + const add = (block: CompressionBlock): void => { + if (seen.has(block.blockId)) return; + seen.add(block.blockId); + for (const raw of block.effectiveMessageIds ?? []) { + const num = refNumOf(state.messageRefs.byRaw[raw]); + if (num !== null) covered.add(num); + } + for (const childId of block.directBlockIds ?? []) { + const child = byId.get(childId); + if (child) add(child); + } + }; + for (const block of state.blocks) { + if (block.active) add(block); + } + return covered; +} + +/** + * Refs the fold scanner never offers: live messages older than the protected + * tail that no active block covers. They stay raw on every request, yet + * `compress` reports "no compressible ranges remain" because that verdict is + * derived from the scanner's own view, not from a coverage audit. + */ +export function uncoveredHistory( + state: CompressionState, + messages: CoreMessage[], + options: UncoveredHistoryOptions = {}, +): UncoveredHistoryReport { + const preserve = Math.max(0, options.preserveRecentMessages ?? DEFAULT_PRESERVE_RECENT); + const maxRuns = Math.max(1, options.maxRuns ?? 3); + const minTokens = Math.max(0, options.minTokens ?? 0); + const covered = coveredRefNumbers(state); + const dead = new Set(); + for (const ref of state.deadRefs ?? []) { + const num = refNumOf(ref); + if (num !== null) dead.add(num); + } + const entries: Array<{ num: number; msg: CoreMessage }> = []; + for (const msg of messages) { + const num = refNumOf(state.messageRefs.byRaw[msg.id]); + if (num === null) continue; + entries.push({ num, msg }); + } + if (entries.length === 0) return { runs: [], runCount: 0, messageCount: 0, tokens: 0 }; + // Same protection premise the fold scanner uses: the newest + // preserveRecentMessages ref-bearing messages are intentionally visible. + const cutoff = Math.max(0, entries.length - preserve); + const runs: Array<{ start: number; end: number; count: number; tokens: number }> = []; + let cur: { start: number; end: number; count: number; tokens: number } | null = null; + let messageCount = 0; + let tokens = 0; + for (let i = 0; i < cutoff; i++) { + const { num, msg } = entries[i]!; + const eligible = !covered.has(num) && !dead.has(num); + if (!eligible) { + if (cur) runs.push(cur); + cur = null; + continue; + } + const cost = countMessageTokens(msg, defaultCountTokens); + messageCount += 1; + tokens += cost; + // Duplicate refs (split cores sharing one base id) stay in the same run. + if (cur && (num === cur.end + 1 || num === cur.end)) { + cur.end = num; + cur.count += 1; + cur.tokens += cost; + } else { + if (cur) runs.push(cur); + cur = { start: num, end: num, count: 1, tokens: cost }; + } + } + if (cur) runs.push(cur); + const kept = runs.filter((r) => r.tokens >= minTokens); + const listed = kept + .slice() + .sort((a, b) => b.tokens - a.tokens) + .slice(0, maxRuns) + .map((r) => ({ startRef: fmtRef(r.start), endRef: fmtRef(r.end), count: r.count, tokens: r.tokens })); + return { runs: listed, runCount: kept.length, messageCount, tokens }; +} + +function fmtTokens(n: number): string { + if (n < 1000) return `${Math.max(1, Math.round(n))} tok`; + return `~${(n / 1000).toFixed(n < 10_000 ? 1 : 0)}K tok`; +} + +export function formatUncoveredHistory(report: UncoveredHistoryReport, opts: { compact?: boolean } = {}): string[] { + if (report.runs.length === 0 || report.messageCount === 0) return []; + const spans = report.runs + .map((r) => (r.startRef === r.endRef ? `${r.startRef} (1 msg, ${fmtTokens(r.tokens)})` : `${r.startRef}–${r.endRef} (${r.count} msgs, ${fmtTokens(r.tokens)})`)) + .join(" · "); + const head = `UNCOVERED HISTORY — ${report.messageCount} message(s) in ${report.runCount} run(s) are in no active block and still ride the wire raw (${fmtTokens(report.tokens)}). Largest: ${spans}.`; + if (opts.compact) { + return [`${head} The fold scanner does not offer these; fold a span directly ({startId,endId}) if it is worth reclaiming, otherwise continue.`]; + } + return [ + head, + " The Compressible-ranges table above comes from the fold scanner and does NOT include these refs. Fold any span directly with compress({startId,endId}); the protected recent tail is already excluded here.", + ]; +} diff --git a/tests/uncovered-history.test.ts b/tests/uncovered-history.test.ts new file mode 100644 index 0000000000..f51a84ba01 --- /dev/null +++ b/tests/uncovered-history.test.ts @@ -0,0 +1,152 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import type { CompressionBlock, CompressionState, CoreMessage, NudgeDecision } from "acp-kernel"; +import { createCore, createInitialState, assignRefs, emptyRefMap, defaultConfig } from "acp-kernel"; +import { coveredRefNumbers, formatUncoveredHistory, uncoveredHistory } from "../src/uncovered-history.ts"; +import { parseCompressInput } from "../src/compress-tool.ts"; +import { applyRanges, type RewriteCtx } from "../src/stream.ts"; +import { handleAcpStatus } from "../src/acp-status.ts"; + +const NO_RANGES_REMAIN_TEXT = "No compressible ranges remain — the context is already at its minimum; continue the task without compressing."; + +type Ctx = Omit & { log: (m: string) => void; logs: string[] }; + +function makeCtx(messages: CoreMessage[], overrides?: Record): Ctx { + const logs: string[] = []; + return { + core: createCore(), + config: defaultConfig(200000, overrides as never), + messages, + session: { + id: "uncovered-history-test", + meta: {}, + stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, compressCreditTokens: 0, contextTokens: 0, retrieveCalls: 0, retrieveHits: 0, retrieveMisses: 0, storedBytes: 0, storeBytesSaved: 0, rangeRestores: 0 }, + metadata: {}, + state: createInitialState(), + createdAt: Date.now(), + lastSeen: Date.now(), + blockContents: new Map(), + inFlight: 0, + persisted: false, + pendingRetrievals: [], + }, + log: (m: string) => { logs.push(m); }, + logs, + }; +} + +function withRefs(ctx: Ctx): Ctx { + const res = assignRefs(ctx.messages, { existing: emptyRefMap(), nextIndex: 0 }); + ctx.session.state.messageRefs = res.map; + return ctx; +} + +function textMsg(id: string, role: "user" | "assistant", text: string): CoreMessage { + return { id, role, contentType: "text", text }; +} + +function block(id: string, effective: string[], kids: string[] = [], opts: { tier?: 1 | 2 | 3; active?: boolean } = {}): CompressionBlock { + return { + blockId: id, + runId: "r1", + tier: opts.tier ?? 1, + summary: "s", + directMessageIds: effective, + effectiveMessageIds: effective, + directBlockIds: kids, + compressedTokens: 0, + createdAt: 0, + survivedCount: 0, + generation: "young", + active: opts.active ?? true, + }; +} + +function stateWithRefs(count: number): { state: CompressionState; messages: CoreMessage[] } { + const messages = Array.from({ length: count }, (_, i) => textMsg(`raw_${i + 1}`, i % 2 === 0 ? "user" : "assistant", "x".repeat(200))); + const res = assignRefs(messages, { existing: emptyRefMap(), nextIndex: 0 }); + const state = createInitialState(); + state.messageRefs = res.map; + return { state, messages }; +} + +test("#4331: coveredRefNumbers recurses into directBlockIds (tier-2 parents)", () => { + const { state } = stateWithRefs(3); + state.blocks = [ + block("b1", ["raw_1"]), + block("b2", [], ["b1"], { tier: 2 }), + block("b3", ["raw_3"], [], { active: false }), + ]; + const covered = coveredRefNumbers(state); + assert.ok(covered.has(1), "child coverage must survive an empty effectiveMessageIds parent"); + assert.ok(!covered.has(3), "inactive block coverage must not count"); +}); + +test("#4331: uncoveredHistory finds the gap between folds and respects the protected tail", () => { + const { state, messages } = stateWithRefs(8); + state.blocks = [block("b1", ["raw_1", "raw_2", "raw_3"], [], { tier: 1 })]; + const report = uncoveredHistory(state, messages, { preserveRecentMessages: 2 }); + assert.equal(report.runCount, 1); + assert.equal(report.messageCount, 3, "m00004–m00006 are the only older uncovered refs"); + assert.deepEqual(report.runs[0], { startRef: "m00004", endRef: "m00006", count: 3, tokens: report.runs[0]!.tokens }); +}); + +test("#4331: split cores sharing one ref stay in a single run", () => { + const { state, messages } = stateWithRefs(2); + state.messageRefs = { byRaw: { raw_1: "m00010", raw_2: "m00010" }, nextIndex: 11 } as never; + state.blocks = []; + const report = uncoveredHistory(state, messages, { preserveRecentMessages: 0 }); + assert.equal(report.runCount, 1); + assert.deepEqual([report.runs[0]!.startRef, report.runs[0]!.endRef], ["m00010", "m00010"]); + assert.equal(report.runs[0]!.count, 2); +}); + +test("#4331: dead refs and minTokens/maxRuns filters", () => { + const { state, messages } = stateWithRefs(6); + state.blocks = []; + state.deadRefs = ["m00002"]; + const all = uncoveredHistory(state, messages, { preserveRecentMessages: 0 }); + assert.equal(all.messageCount, 5, "the dead ref is not coverage the agent can reclaim"); + assert.equal(all.runCount, 2, "m00001 and m00003–m00006"); + const filtered = uncoveredHistory(state, messages, { preserveRecentMessages: 0, minTokens: 1000 }); + assert.equal(filtered.runCount, 0, "tiny runs are filtered by token floor"); +}); + +test("#4331: acp_status reports the coverage gap the fold scanner cannot see", () => { + const msgs = Array.from({ length: 12 }, (_, i) => textMsg(`raw_${i + 1}`, i % 2 === 0 ? "user" : "assistant", "x".repeat(5000))); + const ctx = withRefs(makeCtx(msgs)); + ctx.session.state.blocks = [ + { + ...block("b1", ["raw_1", "raw_2"], [], { tier: 1 }), + runId: "r1", + summary: "folded", + } as CompressionBlock, + ]; + const out = handleAcpStatus({}, ctx as unknown as RewriteCtx).text; + assert.match(out, /UNCOVERED HISTORY — \d+ message\(s\) in \d+ run\(s\)/); + assert.ok(out.includes("m00003"), "the first ref after the fold must be named:\n" + out); + assert.ok(out.includes("does NOT include these refs"), "the audit must say the ranges table is scoped:\n" + out); +}); + +test("#4331: the stop signal carries a material-gap note, and stays quiet for sub-threshold residue", () => { + // Material gap: 20 large messages, only the head folded, tail protected. + const big = Array.from({ length: 20 }, (_, i) => textMsg(`raw_${i + 1}`, i % 2 === 0 ? "user" : "assistant", "x".repeat(5000))); + const ctx = withRefs(makeCtx(big, { preserveRecentMessages: 3, preserveRecentTokens: 0 })); + ctx.session.stats.lastInputTokens = 80000; + const out = applyRanges(parseCompressInput({ + content: [{ startId: "m00001", endId: "m00002", summary: "TAIL-TEST-SUMMARY-PAYLOAD-LONG-ENOUGH-FOR-THE-KERNEL-MIN-LENGTH-CHECK" }], + }), ctx).text; + assert.ok(out.includes(NO_RANGES_REMAIN_TEXT), "kernel stop verdict must stay verbatim:\n" + out); + assert.ok(out.includes("UNCOVERED HISTORY"), "a material blind spot must be named:\n" + out); + assert.ok(out.includes("m00003"), "the gap must carry actionable refs:\n" + out); + + // Sub-threshold residue (the #1387 fixture): the receipt must not re-nag. + const small = Array.from({ length: 7 }, (_, i) => textMsg(`raw_${i + 1}`, i % 2 === 0 ? "user" : "assistant", "x".repeat(5000))); + const ctx2 = withRefs(makeCtx(small)); + ctx2.session.stats.lastInputTokens = 80000; + const out2 = applyRanges(parseCompressInput({ + content: [{ startId: "m00001", endId: "m00002", summary: "TAIL-TEST-SUMMARY-PAYLOAD-LONG-ENOUGH-FOR-THE-KERNEL-MIN-LENGTH-CHECK" }], + }), ctx2).text; + assert.ok(out2.includes(NO_RANGES_REMAIN_TEXT)); + assert.ok(!out2.includes("UNCOVERED HISTORY"), "declined leftovers must not be advertised:\n" + out2); +});