diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a52768774..14a7b76577 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag ## [Unreleased] +### Changes + +- **kernel: aged compress receipts fold (#4348).** `compress` exchanges previously enjoyed a hard-coded, age-independent folding exemption (`ALWAYS_PROTECTED_TOOLS`), so every successful compression receipt stayed on the wire forever — a long session accumulated hundreds of `[Compressed mX-mY ...]` stubs (measured: 148 receipts / ~170K chars riding every request) that no range could reclaim, and ranges over them died with "Range would split N turn(s)" once the receipts interleaved with re-anchored coverage. Now only the newest `COMPRESS_EXCHANGE_KEEP_RECENT = 3` compress exchanges keep the exemption (working memory: the model can still see what it just folded); older exchanges fold like ordinary tool messages, with call+receipt+reasoning kept whole by the existing turn-integrity gate. The summaries the model wrote live on verbatim in the rendered block anchors, so decompress/search lose nothing. Host `protectedTools`/`isToolProtected` patterns still win for non-compress tools; for compress-dialect exchanges the kernel aging policy outranks host pinning (same doctrine as #4289 — without it, MCP-lane forms pinned via the host `isToolProtected` hook could never age out). `acp_rule` keeps its blanket exemption. + ### Changed (storage correctness wave 3, #4242) - **State hydration can no longer drift from `mergeState`** (#4192): `COMPRESSION_STATE_FIELDS` in the new `src/state-hydration.ts` is derived mechanically from `Object.keys(mergeState(createInitialState()))` — the ingest/restore field list is generated from the same source as hydration, so a future field can never be restored-but-never-ingested (the `sweptOrphanCallIds`/`pendingOrphanSweepMinRef` loss class). Pinned by a parity test that fails the moment `createInitialState()` grows a key the list does not know. diff --git a/kernel/package-lock.json b/kernel/package-lock.json index 21a3b75f4d..07de8f7cee 100644 --- a/kernel/package-lock.json +++ b/kernel/package-lock.json @@ -1,12 +1,12 @@ { "name": "acp-kernel", - "version": "0.0.157", + "version": "0.0.159", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "acp-kernel", - "version": "0.0.157", + "version": "0.0.159", "license": "MIT", "devDependencies": { "@types/node": "^22.0.0", diff --git a/kernel/package.json b/kernel/package.json index de4fecad7b..0d983b8180 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.157", + "version": "0.0.159", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/protected.ts b/kernel/src/protected.ts index 3ce4056c82..f79f77a16e 100644 --- a/kernel/src/protected.ts +++ b/kernel/src/protected.ts @@ -35,7 +35,7 @@ export const FAILED_COMPRESS_RESULT_PREFIXES = [ * recognizer claims (MCP lanes expose mcp__bili__compress — #2578/#2970). * Union semantics, never narrower than bare: a host without the hook * behaves exactly as before. */ -function isCompressToolName( +export function isCompressToolName( toolName: string | undefined, config?: Pick, ): boolean { @@ -77,6 +77,47 @@ export function collectFailedCompressCallIds( return ids; } +/** #4348: compress exchanges stay exempt from folding only while they are + * recent working memory. This many NEWEST compress exchanges keep the + * hard-coded ALWAYS_PROTECTED_TOOLS exemption; older ones become foldable + * like ordinary tool exchanges. Their receipts are machine exhaust + * ("[Compressed mX-mY -> N block(s), ~K tokens saved]") and the summaries + * the model wrote live on verbatim in the block anchors the pruned view + * renders, so folding an aged exchange loses nothing decompress or search + * relies on. The newest folds stay visible so the model can still see what + * it just did (which refs are spent, which blocks exist). */ +export const COMPRESS_EXCHANGE_KEEP_RECENT = 3; + +/** Tool-call ids of compress exchanges OLD enough to fold (#4348): every + * compress exchange except the newest COMPRESS_EXCHANGE_KEEP_RECENT ones. + * `keepRecent` exists for tests. Dialect-aware via the same union recognizer + * as the failed-exchange helpers (bare "compress" always counts, plus whatever + * config.isCompressTool claims — #2970 MCP "mcp__bili__compress"). O(n); ids + * are toolCallIds shared by both halves. */ +export function collectAgedCompressCallIds( + messages: CoreMessage[], + config: Pick, + keepRecent: number = COMPRESS_EXCHANGE_KEEP_RECENT, +): Set { + const recent: string[] = []; + for (const m of messages) { + if (m.contentType === "tool-call" && m.toolCallId && isCompressToolName(m.toolName, config)) { + const at = recent.indexOf(m.toolCallId); + if (at >= 0) recent.splice(at, 1); + recent.push(m.toolCallId); + if (recent.length > Math.max(0, keepRecent)) recent.shift(); + } + } + const aged = new Set(); + const keep = new Set(recent); + for (const m of messages) { + if (m.contentType !== "tool-call" || !m.toolCallId) continue; + if (!isCompressToolName(m.toolName, config)) continue; + if (!keep.has(m.toolCallId)) aged.add(m.toolCallId); + } + return aged; +} + /** Tool results that must NEVER participate in the soft-protected recent zone * (preserveRecentMessages / preserveRecentTokens / last user message). * diff --git a/kernel/src/unit-projection.ts b/kernel/src/unit-projection.ts index 0914192c24..8b228d8220 100644 --- a/kernel/src/unit-projection.ts +++ b/kernel/src/unit-projection.ts @@ -7,6 +7,7 @@ import { hasUnrecoverableMediaPayload, isMessageLatestProtected, isMessageProtected, + collectAgedCompressCallIds, } from "./protected.js"; import { computeIntegrityWithdrawals } from "./turn-integrity.js"; import { adjustBoundariesForReasoningPairs } from "./reasoning-pairs.js"; @@ -466,11 +467,22 @@ function filterProtectedToolMessages( const protectedCallIds = new Set(); const removedIds = new Set(); const latest = collectLatestProtected(messages, config); + // #4348: compress exchanges older than the newest few lose the hard-coded + // ALWAYS_PROTECTED_TOOLS exemption inside the FOLD path — their receipts + // are exhaust and the summaries live on in block anchors. Newest exchanges + // keep it (working memory). The lift overrides host protection patterns for + // recognized compress dialects too — required, since the MCP lane's + // hard-protection hook would otherwise pin mcp__bili__compress forever; + // host patterns for all other tools are unaffected. + const agedCompressCallIds = collectAgedCompressCallIds(messages, config); + const isExemptFold = (msg: CoreMessage): boolean => + !!msg.toolCallId && agedCompressCallIds.has(msg.toolCallId); for (const id of latest.callIds) protectedCallIds.add(id); for (const msg of messages) { if ( isMessageProtected(msg, config, failedCompressCallIds) && - msg.toolCallId + msg.toolCallId && + !isExemptFold(msg) ) { protectedCallIds.add(msg.toolCallId); } @@ -481,7 +493,8 @@ function filterProtectedToolMessages( if (!msg) continue; if ( hasUnrecoverableMediaPayload(msg) || - isMessageProtected(msg, config, failedCompressCallIds) || + (isMessageProtected(msg, config, failedCompressCallIds) && + !isExemptFold(msg)) || isMessageLatestProtected(msg, latest) ) { removedIds.add(id); diff --git a/kernel/tests/issue4348-receipt-lace.test.ts b/kernel/tests/issue4348-receipt-lace.test.ts new file mode 100644 index 0000000000..6f1f269904 --- /dev/null +++ b/kernel/tests/issue4348-receipt-lace.test.ts @@ -0,0 +1,193 @@ +// #4348: successful compress exchanges were permanently exempt from folding +// (ALWAYS_PROTECTED_TOOLS), so a long session accumulates "receipt lace" — +// compress-tool receipts (ref'd tool results) whose paired assistant +// tool-call messages carry NO ref tag, interleaved with re-anchored coverage +// islands. No range over the lace was reclaimable, and once it intermixed +// with covered spans any fold over it died with "Range would split N +// turn(s)". The fix: only the newest COMPRESS_EXCHANGE_KEEP_RECENT compress +// exchanges keep the hard-coded exemption (working memory); older exchanges +// fold like ordinary tool messages — both halves together, with reasoning +// runs kept whole by the existing turn-integrity gate. The summaries the +// model wrote live on verbatim in the rendered block anchors, so folding the +// exchange loses nothing decompress or search relies on. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createCore } from "../src/compress.js"; +import { createInitialState } from "../src/state.js"; +import { assignRefs } from "../src/refs.js"; +import { defaultConfig } from "../src/config.js"; +import { collectAgedCompressCallIds } from "../src/protected.js"; +import type { CoreMessage } from "../src/types.js"; + +function textMsg( + id: string, + text: string, + role: CoreMessage["role"] = "user", +): CoreMessage { + return { id, role, contentType: "text", text }; +} + +/** Production shape: an assistant turn = reasoning run + tool-call burst, + * then the tool result (receipt). The reasoning and the call have NO ref + * tag on the wire (assistant tool-call content types never carry one); + * only the tool result is ref-addressable. */ +function receiptTurn( + n: number, + opts: { refd?: boolean; body: string }, +): CoreMessage[] { + const refd = opts.refd ?? true; + return [ + { + id: `r${n}`, + role: "assistant", + contentType: "reasoning", + text: `planning fold ${n} `.repeat(30), + }, + { + id: `c${n}`, + role: "assistant", + contentType: "tool-call", + toolName: "compress", + toolCallId: `tc${n}`, + text: `{"content":[{"startId":"m${n}","endId":"m${n}"}]}`, + }, + { + id: `t${n}`, + role: "tool", + contentType: "tool-result", + toolCallId: `tc${n}`, + text: (refd ? "" : "") + opts.body.repeat(60), + }, + ]; +} + +interface Lace { + core: ReturnType; + state: ReturnType; + messages: CoreMessage[]; +} + +/** Build the lace: a re-anchored island of coverage (one active block over + * early messages) followed by uncovered receipt turns separated by plain + * filler user/assistant pairs, exactly like the incident session. */ +function buildLace(): Lace { + const core = createCore(); + const state = createInitialState(); + const messages: CoreMessage[] = []; + + messages.push(textMsg("u0", "start the session ".repeat(80), "user")); + // An island of coverage: messages the active block will cover. + for (let i = 1; i <= 4; i++) { + messages.push(textMsg(`island${i}`, `island body ${i} `.repeat(120), i % 2 ? "user" : "assistant")); + } + // The lace: three receipt turns, each separated by filler. + for (const n of [10, 11, 12]) { + for (const m of receiptTurn(n, { body: `[Compressed m${n} range(s) tokens saved receipt body ${n} ` })) messages.push(m); + messages.push(textMsg(`gap${n}`, `filler gap after ${n} `.repeat(100), "user")); + } + // Aged filler so the lace sits mid-history, outside the protected zone. + for (let i = 1; i <= 40; i++) { + messages.push(textMsg(`tail${i}`, `aged filler turn ${i} `.repeat(90), i % 2 ? "user" : "assistant")); + } + // NEWEST compress exchanges at the tail: these keep the exemption (the + // newest COMPRESS_EXCHANGE_KEEP_RECENT=3), so the lace turns above age out + // and become foldable — the production shape (folds happen all day). + for (const m of receiptTurn(20, { body: `[Compressed m20 range(s) tokens saved receipt body 20 ` })) messages.push(m); + messages.push(textMsg("post20", "question after the recent folds ".repeat(80), "user")); + for (const m of receiptTurn(21, { body: `[Compressed m21 range(s) tokens saved receipt body 21 ` })) messages.push(m); + messages.push(textMsg("post21", "another question after recent folds ".repeat(80), "user")); + for (const m of receiptTurn(22, { body: `[Compressed m22 range(s) tokens saved receipt body 22 ` })) messages.push(m); + messages.push(textMsg("post22", "final recent question ".repeat(80), "user")); + + state.messageRefs = assignRefs(messages, { existing: state.messageRefs, nextIndex: 1 }).map; + return { core, state, messages }; +} + +test("#4348: aged compress receipts fold together with their unref'd calls and reasoning", () => { + const { core, state, messages } = buildLace(); + // First fold the island (normal path) so an active block exists. + const island = core.applyCompression({ + ranges: [ + { + startRef: "m00002", + endRef: "m00005", + summary: "island summary covering the early exchange in full detail with enough characters to pass the minimum summary length gate", + topic: "island", + }, + ], + messages, + state, + config: defaultConfig(), + }); + assert.ok(island.state.blocks.some((b) => b.active)); + + // Now fold the receipt lace: range from the first receipt turn to the last. + // The receipts are the only ref-addressable members; their calls/reasoning + // are unref'd. Today this projection withdraws everything and dies with + // the split fatal; after the fix it must fold cleanly. + const refs = island.state.messageRefs.byRef; + const receiptRefs = Object.keys(refs).filter( + (ref) => ref.startsWith("m"), + ); + assert.ok(receiptRefs.length > 0, "fixture must produce refs"); + + const laceStart = island.state.messageRefs.byRaw["t10"]; + const laceEnd = island.state.messageRefs.byRaw["t12"]; + assert.ok(typeof laceStart === "string" && typeof laceEnd === "string"); + + const res = core.applyCompression( + { + ranges: [ + { + startRef: laceStart, + endRef: laceEnd, + summary: + "receipt lace recovery: folds the compression receipts together with their unref'd calls and reasoning runs into one block, turning the incident debris into a single summary", + topic: "lace", + }, + ], + messages, + state: island.state, + config: defaultConfig(), + }, + ); +const laceBlock = res.state.blocks.find((b) => b.topic === "lace"); + assert.ok(laceBlock?.active, "lace range must fold into an active block"); + // The whole turn groups fold: receipt AND its unref'd call and reasoning. + const covered = new Set(laceBlock.effectiveMessageIds); + assert.ok(covered.has("t10"), "receipt t10 covered"); + assert.ok(covered.has("c10"), "unref'd call c10 covered with it"); + assert.ok(covered.has("r10"), "unref'd reasoning r10 covered with it"); + assert.ok(covered.has("t12") && covered.has("c12")); +}); + +test("#4348 control: the newest compress exchanges keep the hard-coded exemption", () => { + const { core, state, messages } = buildLace(); + state.messageRefs = assignRefs(messages, { existing: state.messageRefs, nextIndex: 1 }).map; + const aged = collectAgedCompressCallIds(messages, defaultConfig()); + // The lace turns aged out; the three newest tail exchanges did not. + for (const n of [10, 11, 12]) assert.ok(aged.has(`tc${n}`), `tc${n} must be aged`); + for (const n of [20, 21, 22]) assert.ok(!aged.has(`tc${n}`), `tc${n} must stay protected`); + // And a fold over the newest exchanges is still refused: the blanket + // exemption holds for them (protected-tool filter + zone). + const byRaw = state.messageRefs.byRaw; + const res = core.applyCompression({ + ranges: [ + { + startRef: byRaw["t20"]!, + endRef: byRaw["t22"]!, + summary: "attempting to fold the newest compress exchanges must fail because they keep the hard-coded exemption from folding", + topic: "recent-exchanges", + }, + ], + messages, + state, + config: defaultConfig(), + }); + const folded = res.state.blocks.find((b) => b.topic === "recent-exchanges"); + assert.ok(!folded?.active, "newest exchanges must not fold"); + assert.ok( + (res.result.errors ?? []).length > 0 || (res.result.warnings ?? []).length > 0, + "engine must push back on folding the newest exchanges", + ); +}); diff --git a/package-lock.json b/package-lock.json index 83cd8930a4..f2df75e347 100644 --- a/package-lock.json +++ b/package-lock.json @@ -50,7 +50,7 @@ }, "kernel": { "name": "acp-kernel", - "version": "0.0.157", + "version": "0.0.159", "dev": true, "license": "MIT", "devDependencies": {