Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions src/acp-status.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import { pckAblations } from "./pck-probe.js";
import { getUnrecognizedPathStats } from "./server/observability.js";
import { ccrEnabled, ccrLoopConfig, contentStoreOf } from "./store.js";
import { coveredRefSpan } from "./decompress-shared.js";
import { formatUncoveredHistory, uncoveredHistory } from "./uncovered-history.js";
import { preCompactionArchiveOf, statusInputBaseline, type Session } from "./session.js";
import { compressBreakerDetail, compressLastFailureCause } from "./stream.js";
import { describeAdvisory, getAdvisoryState } from "./advisory.js";
Expand Down Expand Up @@ -275,6 +276,18 @@ export function handleAcpStatus(args: Record<string, unknown>, ctx: AcpStatusCtx
} catch {
// Base-only report; never fall back to a stale snapshot.
}
// #4331: the fold scanner only offers spans it can prove from the current
// view; messages that fell between two fold boundaries are invisible to it
// yet still ride the wire on every request. Audit coverage independently so
// the "no compressible ranges remain" verdict never stands unqualified.
const uncovered = uncoveredHistory(ctx.session.state, ctx.messages, {
preserveRecentMessages: ctx.config.preserveRecentMessages,
maxRuns: 3,
});
if (uncovered.messageCount > 0) {
extra.push("");
extra.push(...formatUncoveredHistory(uncovered));
}
// #2362: dead refs — known to the session but no longer backed by any
// visible or folded message (client history rewrite / host-native
// compaction). Failure receipts point the model at acp_status, so this is
Expand Down
21 changes: 20 additions & 1 deletion src/stream.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { type Session, cacheBlockContent, markDirty, residentPressure, scrubTurn
import { COMPRESS_TOOL_NAME, parseCompressInput, ABSORB_TOOL_NAME, type ParsedRange } from "./compress-tool.js";
import { effectiveAbsorbConfig, executeAbsorb, isProxyToolFor } from "./absorb.js";
import { executeSearchContextTarget, resolveDecompress } from "./decompress-shared.js";
import { formatUncoveredHistory, uncoveredHistory } from "./uncovered-history.js";
import { ONE_CALL_HINT } from "acp-kernel";
import { adoptContentStore, contentStoreOf, ccrEnabled, drainPendingRetrievals, executeRetrieve, retrieveToolName } from "./store.js";
import { recordThreadTail } from "./server/thread-tracking.js";
Expand Down Expand Up @@ -605,7 +606,25 @@ function postCompressTail(ctx: RewriteCtx, cleanSuccess: boolean, submitted: Arr
// Partial failures stay silent too: the model still owes the errors an answer
// before any "you are done" verdict (pi #521 gate).
if (!cleanSuccess || tier !== null) return "";
return `\n\n${NO_RANGES_REMAIN_TEXT}`;
// #4331: "no ranges remain" is scoped to the fold scanner's offer, not to
// actual coverage. Traffic between two fold boundaries is never offered, so
// the stop verdict alone would claim a completeness that does not hold.
let uncoveredNote = "";
try {
const uncovered = uncoveredHistory(ctx.session.state, ctx.messages, {
preserveRecentMessages: ctx.config.preserveRecentMessages,
maxRuns: 2,
// #1387/pi #521: a receipt must not re-nag about leftovers the fold
// scanner deliberately declined. Only a material blind spot (the
// kind a coverage audit can prove) earns a line here; acp_status
// shows the full inventory.
minTokens: 10_000,
});
uncoveredNote = formatUncoveredHistory(uncovered, { compact: true }).join(" ");
} catch {
uncoveredNote = "";
}
return uncoveredNote ? `\n\n${NO_RANGES_REMAIN_TEXT}\n${uncoveredNote}` : `\n\n${NO_RANGES_REMAIN_TEXT}`;
}

// #1495: the kernel's lenient parser salvages complete entries from damaged
Expand Down
156 changes: 156 additions & 0 deletions src/uncovered-history.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
import { countMessageTokens, defaultCountTokens, type CompressionBlock, type CompressionState, type CoreMessage } from "acp-kernel";

/** A contiguous run of ref-bearing messages that no active block covers. */
interface UncoveredRun {
startRef: string;
endRef: string;
count: number;
tokens: number;
}

interface UncoveredHistoryReport {
/** Largest runs, capped by `maxRuns`. */
runs: UncoveredRun[];
/** Total uncovered runs in the report window. */
runCount: number;
/** Total uncovered messages in the report window. */
messageCount: number;
/** Token estimate for every uncovered message (not just the listed runs). */
tokens: number;
}

interface UncoveredHistoryOptions {
/** Soft-protected tail size; the last N ref-bearing messages are excluded. */
preserveRecentMessages?: number;
/** How many runs to list (largest first). Default 3. */
maxRuns?: number;
/** Drop runs below this token estimate. Default 0. */
minTokens?: number;
}

const REF_RE = /^m(\d+)$/i;
const DEFAULT_PRESERVE_RECENT = 3;

function refNumOf(ref: string | undefined | null): number | null {
if (typeof ref !== "string") return null;
const m = REF_RE.exec(ref.trim());
return m ? Number(m[1]) : null;
}

function fmtRef(n: number): string {
return `m${String(n).padStart(5, "0")}`;
}

/**
* Every ref number covered by an active block. Blocks record coverage as raw
* message ids (`effectiveMessageIds`); a tier-2/3 block additionally owns its
* children, so the walk recurses through `directBlockIds` for the cases where
* the expanded id list was not persisted.
*/
export function coveredRefNumbers(state: CompressionState): Set<number> {
const covered = new Set<number>();
const byId = new Map<string, CompressionBlock>();
for (const b of state.blocks) byId.set(b.blockId, b);
const seen = new Set<string>();
const add = (block: CompressionBlock): void => {
if (seen.has(block.blockId)) return;
seen.add(block.blockId);
for (const raw of block.effectiveMessageIds ?? []) {
const num = refNumOf(state.messageRefs.byRaw[raw]);
if (num !== null) covered.add(num);
}
for (const childId of block.directBlockIds ?? []) {
const child = byId.get(childId);
if (child) add(child);
}
};
for (const block of state.blocks) {
if (block.active) add(block);
}
return covered;
}

/**
* Refs the fold scanner never offers: live messages older than the protected
* tail that no active block covers. They stay raw on every request, yet
* `compress` reports "no compressible ranges remain" because that verdict is
* derived from the scanner's own view, not from a coverage audit.
*/
export function uncoveredHistory(
state: CompressionState,
messages: CoreMessage[],
options: UncoveredHistoryOptions = {},
): UncoveredHistoryReport {
const preserve = Math.max(0, options.preserveRecentMessages ?? DEFAULT_PRESERVE_RECENT);
const maxRuns = Math.max(1, options.maxRuns ?? 3);
const minTokens = Math.max(0, options.minTokens ?? 0);
const covered = coveredRefNumbers(state);
const dead = new Set<number>();
for (const ref of state.deadRefs ?? []) {
const num = refNumOf(ref);
if (num !== null) dead.add(num);
}
const entries: Array<{ num: number; msg: CoreMessage }> = [];
for (const msg of messages) {
const num = refNumOf(state.messageRefs.byRaw[msg.id]);
if (num === null) continue;
entries.push({ num, msg });
}
if (entries.length === 0) return { runs: [], runCount: 0, messageCount: 0, tokens: 0 };
// Same protection premise the fold scanner uses: the newest
// preserveRecentMessages ref-bearing messages are intentionally visible.
const cutoff = Math.max(0, entries.length - preserve);
const runs: Array<{ start: number; end: number; count: number; tokens: number }> = [];
let cur: { start: number; end: number; count: number; tokens: number } | null = null;
let messageCount = 0;
let tokens = 0;
for (let i = 0; i < cutoff; i++) {
const { num, msg } = entries[i]!;
const eligible = !covered.has(num) && !dead.has(num);
if (!eligible) {
if (cur) runs.push(cur);
cur = null;
continue;
}
const cost = countMessageTokens(msg, defaultCountTokens);
messageCount += 1;
tokens += cost;
// Duplicate refs (split cores sharing one base id) stay in the same run.
if (cur && (num === cur.end + 1 || num === cur.end)) {
cur.end = num;
cur.count += 1;
cur.tokens += cost;
} else {
if (cur) runs.push(cur);
cur = { start: num, end: num, count: 1, tokens: cost };
}
}
if (cur) runs.push(cur);
const kept = runs.filter((r) => r.tokens >= minTokens);
const listed = kept
.slice()
.sort((a, b) => b.tokens - a.tokens)
.slice(0, maxRuns)
.map((r) => ({ startRef: fmtRef(r.start), endRef: fmtRef(r.end), count: r.count, tokens: r.tokens }));
return { runs: listed, runCount: kept.length, messageCount, tokens };
}

function fmtTokens(n: number): string {
if (n < 1000) return `${Math.max(1, Math.round(n))} tok`;
return `~${(n / 1000).toFixed(n < 10_000 ? 1 : 0)}K tok`;
}

export function formatUncoveredHistory(report: UncoveredHistoryReport, opts: { compact?: boolean } = {}): string[] {
if (report.runs.length === 0 || report.messageCount === 0) return [];
const spans = report.runs
.map((r) => (r.startRef === r.endRef ? `${r.startRef} (1 msg, ${fmtTokens(r.tokens)})` : `${r.startRef}–${r.endRef} (${r.count} msgs, ${fmtTokens(r.tokens)})`))
.join(" · ");
const head = `UNCOVERED HISTORY — ${report.messageCount} message(s) in ${report.runCount} run(s) are in no active block and still ride the wire raw (${fmtTokens(report.tokens)}). Largest: ${spans}.`;
if (opts.compact) {
return [`${head} The fold scanner does not offer these; fold a span directly ({startId,endId}) if it is worth reclaiming, otherwise continue.`];
}
return [
head,
" The Compressible-ranges table above comes from the fold scanner and does NOT include these refs. Fold any span directly with compress({startId,endId}); the protected recent tail is already excluded here.",
];
}
152 changes: 152 additions & 0 deletions tests/uncovered-history.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import type { CompressionBlock, CompressionState, CoreMessage, NudgeDecision } from "acp-kernel";
import { createCore, createInitialState, assignRefs, emptyRefMap, defaultConfig } from "acp-kernel";
import { coveredRefNumbers, formatUncoveredHistory, uncoveredHistory } from "../src/uncovered-history.ts";
import { parseCompressInput } from "../src/compress-tool.ts";
import { applyRanges, type RewriteCtx } from "../src/stream.ts";
import { handleAcpStatus } from "../src/acp-status.ts";

const NO_RANGES_REMAIN_TEXT = "No compressible ranges remain — the context is already at its minimum; continue the task without compressing.";

type Ctx = Omit<RewriteCtx, "log"> & { log: (m: string) => void; logs: string[] };

function makeCtx(messages: CoreMessage[], overrides?: Record<string, unknown>): Ctx {
const logs: string[] = [];
return {
core: createCore(),
config: defaultConfig(200000, overrides as never),
messages,
session: {
id: "uncovered-history-test",
meta: {},
stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, compressCreditTokens: 0, contextTokens: 0, retrieveCalls: 0, retrieveHits: 0, retrieveMisses: 0, storedBytes: 0, storeBytesSaved: 0, rangeRestores: 0 },
metadata: {},
state: createInitialState(),
createdAt: Date.now(),
lastSeen: Date.now(),
blockContents: new Map(),
inFlight: 0,
persisted: false,
pendingRetrievals: [],
},
log: (m: string) => { logs.push(m); },
logs,
};
}

function withRefs(ctx: Ctx): Ctx {
const res = assignRefs(ctx.messages, { existing: emptyRefMap(), nextIndex: 0 });
ctx.session.state.messageRefs = res.map;
return ctx;
}

function textMsg(id: string, role: "user" | "assistant", text: string): CoreMessage {
return { id, role, contentType: "text", text };
}

function block(id: string, effective: string[], kids: string[] = [], opts: { tier?: 1 | 2 | 3; active?: boolean } = {}): CompressionBlock {
return {
blockId: id,
runId: "r1",
tier: opts.tier ?? 1,
summary: "s",
directMessageIds: effective,
effectiveMessageIds: effective,
directBlockIds: kids,
compressedTokens: 0,
createdAt: 0,
survivedCount: 0,
generation: "young",
active: opts.active ?? true,
};
}

function stateWithRefs(count: number): { state: CompressionState; messages: CoreMessage[] } {
const messages = Array.from({ length: count }, (_, i) => textMsg(`raw_${i + 1}`, i % 2 === 0 ? "user" : "assistant", "x".repeat(200)));
const res = assignRefs(messages, { existing: emptyRefMap(), nextIndex: 0 });
const state = createInitialState();
state.messageRefs = res.map;
return { state, messages };
}

test("#4331: coveredRefNumbers recurses into directBlockIds (tier-2 parents)", () => {
const { state } = stateWithRefs(3);
state.blocks = [
block("b1", ["raw_1"]),
block("b2", [], ["b1"], { tier: 2 }),
block("b3", ["raw_3"], [], { active: false }),
];
const covered = coveredRefNumbers(state);
assert.ok(covered.has(1), "child coverage must survive an empty effectiveMessageIds parent");
assert.ok(!covered.has(3), "inactive block coverage must not count");
});

test("#4331: uncoveredHistory finds the gap between folds and respects the protected tail", () => {
const { state, messages } = stateWithRefs(8);
state.blocks = [block("b1", ["raw_1", "raw_2", "raw_3"], [], { tier: 1 })];
const report = uncoveredHistory(state, messages, { preserveRecentMessages: 2 });
assert.equal(report.runCount, 1);
assert.equal(report.messageCount, 3, "m00004–m00006 are the only older uncovered refs");
assert.deepEqual(report.runs[0], { startRef: "m00004", endRef: "m00006", count: 3, tokens: report.runs[0]!.tokens });
});

test("#4331: split cores sharing one ref stay in a single run", () => {
const { state, messages } = stateWithRefs(2);
state.messageRefs = { byRaw: { raw_1: "m00010", raw_2: "m00010" }, nextIndex: 11 } as never;
state.blocks = [];
const report = uncoveredHistory(state, messages, { preserveRecentMessages: 0 });
assert.equal(report.runCount, 1);
assert.deepEqual([report.runs[0]!.startRef, report.runs[0]!.endRef], ["m00010", "m00010"]);
assert.equal(report.runs[0]!.count, 2);
});

test("#4331: dead refs and minTokens/maxRuns filters", () => {
const { state, messages } = stateWithRefs(6);
state.blocks = [];
state.deadRefs = ["m00002"];
const all = uncoveredHistory(state, messages, { preserveRecentMessages: 0 });
assert.equal(all.messageCount, 5, "the dead ref is not coverage the agent can reclaim");
assert.equal(all.runCount, 2, "m00001 and m00003–m00006");
const filtered = uncoveredHistory(state, messages, { preserveRecentMessages: 0, minTokens: 1000 });
assert.equal(filtered.runCount, 0, "tiny runs are filtered by token floor");
});

test("#4331: acp_status reports the coverage gap the fold scanner cannot see", () => {
const msgs = Array.from({ length: 12 }, (_, i) => textMsg(`raw_${i + 1}`, i % 2 === 0 ? "user" : "assistant", "x".repeat(5000)));
const ctx = withRefs(makeCtx(msgs));
ctx.session.state.blocks = [
{
...block("b1", ["raw_1", "raw_2"], [], { tier: 1 }),
runId: "r1",
summary: "folded",
} as CompressionBlock,
];
const out = handleAcpStatus({}, ctx as unknown as RewriteCtx).text;
assert.match(out, /UNCOVERED HISTORY — \d+ message\(s\) in \d+ run\(s\)/);
assert.ok(out.includes("m00003"), "the first ref after the fold must be named:\n" + out);
assert.ok(out.includes("does NOT include these refs"), "the audit must say the ranges table is scoped:\n" + out);
});

test("#4331: the stop signal carries a material-gap note, and stays quiet for sub-threshold residue", () => {
// Material gap: 20 large messages, only the head folded, tail protected.
const big = Array.from({ length: 20 }, (_, i) => textMsg(`raw_${i + 1}`, i % 2 === 0 ? "user" : "assistant", "x".repeat(5000)));
const ctx = withRefs(makeCtx(big, { preserveRecentMessages: 3, preserveRecentTokens: 0 }));
ctx.session.stats.lastInputTokens = 80000;
const out = applyRanges(parseCompressInput({
content: [{ startId: "m00001", endId: "m00002", summary: "TAIL-TEST-SUMMARY-PAYLOAD-LONG-ENOUGH-FOR-THE-KERNEL-MIN-LENGTH-CHECK" }],
}), ctx).text;
assert.ok(out.includes(NO_RANGES_REMAIN_TEXT), "kernel stop verdict must stay verbatim:\n" + out);
assert.ok(out.includes("UNCOVERED HISTORY"), "a material blind spot must be named:\n" + out);
assert.ok(out.includes("m00003"), "the gap must carry actionable refs:\n" + out);

// Sub-threshold residue (the #1387 fixture): the receipt must not re-nag.
const small = Array.from({ length: 7 }, (_, i) => textMsg(`raw_${i + 1}`, i % 2 === 0 ? "user" : "assistant", "x".repeat(5000)));
const ctx2 = withRefs(makeCtx(small));
ctx2.session.stats.lastInputTokens = 80000;
const out2 = applyRanges(parseCompressInput({
content: [{ startId: "m00001", endId: "m00002", summary: "TAIL-TEST-SUMMARY-PAYLOAD-LONG-ENOUGH-FOR-THE-KERNEL-MIN-LENGTH-CHECK" }],
}), ctx2).text;
assert.ok(out2.includes(NO_RANGES_REMAIN_TEXT));
assert.ok(!out2.includes("UNCOVERED HISTORY"), "declined leftovers must not be advertised:\n" + out2);
});
Loading