diff --git a/kernel/package-lock.json b/kernel/package-lock.json index e884794d82..a64c58f7bb 100644 --- a/kernel/package-lock.json +++ b/kernel/package-lock.json @@ -1,12 +1,12 @@ { "name": "acp-kernel", - "version": "0.0.168", + "version": "0.0.169", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "acp-kernel", - "version": "0.0.168", + "version": "0.0.169", "license": "MIT", "devDependencies": { "@types/node": "^22.0.0", diff --git a/kernel/package.json b/kernel/package.json index 8ba064ebc0..4c8eff8fb3 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.168", + "version": "0.0.169", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/unit-projection.ts b/kernel/src/unit-projection.ts index 8b228d8220..25892728a1 100644 --- a/kernel/src/unit-projection.ts +++ b/kernel/src/unit-projection.ts @@ -334,39 +334,106 @@ export function projectUnit(input: UnitProjectionInput): UnitProjection { // computeIntegrityWithdrawals: withdrawing a pair for the result half can // strand a turn whose reasoning still folds and whose call now stays visible. { - const { - withdrawn: withdrawIds, - splitTurnCount: turns, - splitPairCount: pairs, - } = computeIntegrityWithdrawals(input.messages, effectiveMessageIds); - withdrawnIds = withdrawIds; - splitTurnCount = turns; - splitPairCount = pairs; - if (withdrawIds.size > 0) { - for (const id of withdrawIds) effectiveMessageIds.delete(id); - const beforeWithdraw = filteredIds.length; - filteredIds = filteredIds.filter((id) => !withdrawIds.has(id)); + // #4484/#3322: on a MERGE (this unit consumes existing blocks) the integrity + // gate needs two things a fresh fold does not. (a) Judge POST-FOLD coverage — + // this unit's ids PLUS every still-active sibling block's coverage — so a tool + // half already covered by an unconsumed sibling reads as folded, not a spurious + // "visible" half (#3322 withdrew inherited coverage for exactly this reason). + // (b) Never drop an id whose ONLY covering block is consumed here: those blocks + // deactivate below, so removing their id strands it uncovered — the regression + // the host pre-commit guard then refuses (#4484). We therefore unfold only ids + // whose coverage does not depend on a consumed block, re-checking to a fixed + // point; if consistency would force unfolding such an id we fail honestly rather + // than emit a doomed regressive candidate. Fresh folds keep the legacy behavior. + const isMerge = consumedBlockIds.length > 0; + let consumedCoverage = new Set(); + let survivingCoverage = new Set(); + if (isMerge) { + const consumedSet = new Set(consumedBlockIds); + for (const block of activeBlocks(input.state)) { + const target = consumedSet.has(block.blockId) + ? consumedCoverage + : survivingCoverage; + for (const id of block.effectiveMessageIds) target.add(id); + } + } + let foldedRef = effectiveMessageIds; + if (isMerge) { + foldedRef = new Set(effectiveMessageIds); + for (const id of survivingCoverage) foldedRef.add(id); + } + const initial = computeIntegrityWithdrawals(input.messages, foldedRef); + splitTurnCount = initial.splitTurnCount; + splitPairCount = initial.splitPairCount; + if (initial.withdrawn.size > 0) { const splitDesc = [ splitTurnCount > 0 ? `${splitTurnCount} turn(s)` : null, - splitPairCount > 0 - ? `${splitPairCount} tool call/result pair(s)` - : null, + splitPairCount > 0 ? `${splitPairCount} tool call/result pair(s)` : null, ] .filter((part): part is string => part !== null) .join(" and "); - if (filteredIds.length === 0 && consumedBlockIds.length === 0) { - return finish({ - fatal: { - kind: "split", - error: new Error( - `Range would split ${splitDesc} at the protected-zone boundary: a visible tool-call must keep its reasoning run and its results (strict providers reject a rebuilt request that lost either). Shrink the range to end before the turn starts, or wait until the whole turn ages out of the protected zone.`, - ), - }, - }); + if (!isMerge) { + withdrawnIds = initial.withdrawn; + for (const id of initial.withdrawn) effectiveMessageIds.delete(id); + const beforeWithdraw = filteredIds.length; + filteredIds = filteredIds.filter((id) => !initial.withdrawn.has(id)); + if (filteredIds.length === 0 && consumedBlockIds.length === 0) { + return finish({ + fatal: { + kind: "split", + error: new Error( + `Range would split ${splitDesc} at the protected-zone boundary: a visible tool-call must keep its reasoning run and its results (strict providers reject a rebuilt request that lost either). Shrink the range to end before the turn starts, or wait until the whole turn ages out of the protected zone.`, + ), + }, + }); + } + warnings.push( + `Withdrawn ${beforeWithdraw - filteredIds.length} message(s) from compression range to keep ${splitDesc} intact (visible tool-call would lose its reasoning run or its results).`, + ); + } else { + const unfoldable = new Set( + [...effectiveMessageIds].filter((id) => !consumedCoverage.has(id)), + ); + const unfolded = new Set(); + let current = initial.withdrawn; + let guard = unfoldable.size + 2; + let stuck = false; + while (current.size > 0 && guard-- > 0) { + let progress = false; + for (const id of current) { + if (unfoldable.has(id) && !unfolded.has(id)) { + unfolded.add(id); + progress = true; + } + } + if (!progress) { + stuck = true; + break; + } + const recheck = new Set(effectiveMessageIds); + for (const id of unfolded) recheck.delete(id); + for (const id of survivingCoverage) recheck.add(id); + current = computeIntegrityWithdrawals(input.messages, recheck).withdrawn; + } + if (stuck || current.size > 0) { + const sample = [...consumedCoverage].slice(0, 5).join(", "); + return finish({ + fatal: { + kind: "split", + error: new Error( + `Merging ${consumedBlockIds.join(", ")} cannot satisfy turn integrity without unfolding already-folded message(s) owned by those blocks (${sample}) — doing so would drop their existing coverage. Align the range to whole turns (do not cut through a reasoning/tool-call/result group) or merge fewer blocks.`, + ), + }, + }); + } + withdrawnIds = unfolded; + for (const id of unfolded) effectiveMessageIds.delete(id); + const beforeWithdraw = filteredIds.length; + filteredIds = filteredIds.filter((id) => !unfolded.has(id)); + warnings.push( + `Withdrew ${beforeWithdraw - filteredIds.length} newly-folded message(s) from the merge to keep ${splitDesc} intact while preserving the existing coverage of ${consumedBlockIds.join(", ")}.`, + ); } - warnings.push( - `Withdrawn ${beforeWithdraw - filteredIds.length} message(s) from compression range to keep ${splitDesc} intact (visible tool-call would lose its reasoning run or its results).`, - ); } } diff --git a/kernel/tests/issue4484-merge-coverage.test.ts b/kernel/tests/issue4484-merge-coverage.test.ts new file mode 100644 index 0000000000..f37aa187cb --- /dev/null +++ b/kernel/tests/issue4484-merge-coverage.test.ts @@ -0,0 +1,238 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { createCore } from "../src/compress.js"; +import { createInitialState, coveredMessageIds } from "../src/state.js"; +import { defaultConfig } from "../src/config.js"; +import { computeIntegrityWithdrawals } from "../src/turn-integrity.js"; +import type { CompressionState, CoreMessage } from "../src/types.js"; + +// #4484: merging existing blocks must never drop their existing coverage. The +// turn-integrity gate used to withdraw ids carried only by consumed blocks, +// stranding them uncovered once those blocks deactivated (host then refused the +// fold via gate:coverage-regression). These fixtures mirror the two real trigger +// shapes (interior hard-protected tool; tail cut mid-tool-group); synthetic only. + +const isToolProtected = (t?: string) => + t === "mcp__bili__acp_rule" || t === "mcp__bili__compress"; +const isCompressTool = (t?: string) => + t === "mcp__bili__compress" || t === "compress"; + +const config = defaultConfig(255000, { + preserveRecentMessages: 0, + preserveRecentTokens: 0, + isToolProtected, + isCompressTool, +} as never); + +const SUMMARY = "Synthetic summary for this compression test. ".repeat(4); +const LONG = "Synthetic historical content, safe for public reproduction. ".repeat(150); + +function buildFixture(kind: "protected-rule" | "parallel-tail") { + const core = createCore(); + const messages: CoreMessage[] = [ + { id: "old-user", role: "user", contentType: "text", text: LONG }, + { id: "old-assistant", role: "assistant", contentType: "text", text: "Previous work finished." }, + { id: "reasoning", role: "assistant", contentType: "reasoning", text: "Inspect the current state." }, + { id: "read-call", role: "assistant", contentType: "tool-call", toolName: "Read", toolCallId: "read-1", text: '{"path":"/tmp/example.txt"}' }, + ]; + let seedStartId: string; + let seedEndId: string; + let toolIds: string[]; + if (kind === "protected-rule") { + messages.push( + { id: "rule-call", role: "assistant", contentType: "tool-call", toolName: "mcp__bili__acp_rule", toolCallId: "rule-1", text: '{"operation":"list"}' }, + { id: "read-result", role: "tool", contentType: "tool-result", toolName: "Read", toolCallId: "read-1", text: LONG }, + { id: "rule-result", role: "tool", contentType: "tool-result", toolName: "mcp__bili__acp_rule", toolCallId: "rule-1", text: "No rules recorded." }, + ); + seedStartId = "read-call"; + seedEndId = "read-result"; + toolIds = ["read-call", "read-result"]; + } else { + messages.push( + { id: "bash-call", role: "assistant", contentType: "tool-call", toolName: "Bash", toolCallId: "bash-1", text: '{"command":"true"}' }, + { id: "todo-call", role: "assistant", contentType: "tool-call", toolName: "TodoList", toolCallId: "todo-1", text: '{"todos":[]}' }, + { id: "read-result", role: "tool", contentType: "tool-result", toolName: "Read", toolCallId: "read-1", text: LONG }, + { id: "bash-result", role: "tool", contentType: "tool-result", toolName: "Bash", toolCallId: "bash-1", text: LONG }, + { id: "todo-result", role: "tool", contentType: "tool-result", toolName: "TodoList", toolCallId: "todo-1", text: "Updated." }, + ); + seedStartId = "read-call"; + seedEndId = "bash-result"; + toolIds = ["read-call", "bash-call", "read-result", "bash-result"]; + } + messages.push({ id: "new-user", role: "user", contentType: "text", text: "Continue." }); + + const state = createInitialState(); + let nextRef = 1; + for (const message of messages) { + const ref = message.toolName === "mcp__bili__acp_rule" ? "BLOCKED" : `m${String(nextRef++).padStart(5, "0")}`; + state.messageRefs.byRaw[message.id] = ref; + if (ref !== "BLOCKED") state.messageRefs.byRef[ref] = message.id; + } + const seeded = core.applyCompression({ + messages, + state, + config, + ranges: [{ startRef: state.messageRefs.byRaw["old-user"]!, endRef: state.messageRefs.byRaw["old-assistant"]!, summary: SUMMARY }], + }); + if (seeded.result.blocksCreated !== 1 || seeded.result.errors.length !== 0) { + throw new Error(`fixture seed failed: ${JSON.stringify(seeded.result)}`); + } + seeded.state.blocks.push({ + ...seeded.state.blocks[0]!, + blockId: "b2", + runId: "r2", + directMessageIds: [...toolIds], + effectiveMessageIds: [...toolIds], + compressedTokens: toolIds.reduce((n, id) => n + Math.ceil(messages.find((m) => m.id === id)!.text.length / 4), 0), + startRef: state.messageRefs.byRaw[seedStartId], + endRef: state.messageRefs.byRaw[seedEndId], + }); + seeded.state.nextBlockId = 3; + seeded.state.nextRunId = 3; + seeded.state.stats.compressionCount = 2; + return { core, messages, state: seeded.state, toolIds }; +} + +function lostCoverage(before: Set, state: CompressionState): string[] { + const after = coveredMessageIds(state); + return [...before].filter((id) => !after.has(id)); +} + +describe("#4484 merge preserves existing block coverage", () => { + it("protected-rule interior: merging b1..b2 keeps the Read pair folded", () => { + const { core, messages, state, toolIds } = buildFixture("protected-rule"); + const before = coveredMessageIds(state); + const res = core.applyCompression({ + messages, + state, + config, + ranges: [{ startRef: "b1", endRef: "b2", summary: SUMMARY }], + }); + assert.equal(res.result.errors.length, 0, JSON.stringify(res.result.errors)); + assert.equal(res.result.blocksCreated, 1); + const lost = lostCoverage(before, res.state); + assert.deepEqual(lost, [], `regression: lost ${lost.join(", ")}`); + const after = coveredMessageIds(res.state); + for (const id of toolIds) assert.ok(after.has(id), `${id} must stay covered`); + assert.equal(computeIntegrityWithdrawals(messages, after).withdrawn.size, 0); + }); + + it("parallel-tail cut: merging b1..b2 keeps both Read/Bash pairs folded", () => { + const { core, messages, state, toolIds } = buildFixture("parallel-tail"); + const before = coveredMessageIds(state); + const res = core.applyCompression({ + messages, + state, + config, + ranges: [{ startRef: "b1", endRef: "b2", summary: SUMMARY }], + }); + assert.equal(res.result.errors.length, 0, JSON.stringify(res.result.errors)); + assert.equal(res.result.blocksCreated, 1); + const lost = lostCoverage(before, res.state); + assert.deepEqual(lost, [], `regression: lost ${lost.join(", ")}`); + const after = coveredMessageIds(res.state); + for (const id of toolIds) assert.ok(after.has(id), `${id} must stay covered`); + assert.equal(computeIntegrityWithdrawals(messages, after).withdrawn.size, 0); + }); + + it("impossible merge: fails fast up-front without touching existing coverage", () => { + const core = createCore(); + const messages: CoreMessage[] = [ + { id: "old-user", role: "user", contentType: "text", text: LONG }, + { id: "old-assistant", role: "assistant", contentType: "text", text: "Previous work finished." }, + { id: "read-call", role: "assistant", contentType: "tool-call", toolName: "Read", toolCallId: "read-1", text: '{"path":"/tmp/example.txt"}' }, + { id: "read-result", role: "tool", contentType: "tool-result", toolName: "Read", toolCallId: "read-1", text: LONG }, + { id: "new-user", role: "user", contentType: "text", text: "Continue." }, + ]; + const state = createInitialState(); + let nextRef = 1; + for (const m of messages) { + const ref = `m${String(nextRef++).padStart(5, "0")}`; + state.messageRefs.byRaw[m.id] = ref; + state.messageRefs.byRef[ref] = m.id; + } + const seeded = core.applyCompression({ + messages, + state, + config, + ranges: [{ startRef: state.messageRefs.byRaw["old-user"]!, endRef: state.messageRefs.byRaw["old-assistant"]!, summary: SUMMARY }], + }); + if (seeded.result.blocksCreated !== 1 || seeded.result.errors.length !== 0) { + throw new Error(`fixture seed failed: ${JSON.stringify(seeded.result)}`); + } + // b2 covers ONLY the call half of the Read pair — its tail cut mid-tool-group. + // No consistent non-regressive candidate exists: folding the call splits its + // pair, unfolding it strands a message whose only covering block is consumed. + seeded.state.blocks.push({ + ...seeded.state.blocks[0]!, + blockId: "b2", + runId: "r2", + directMessageIds: ["read-call"], + effectiveMessageIds: ["read-call"], + compressedTokens: 100, + startRef: state.messageRefs.byRaw["read-call"]!, + endRef: state.messageRefs.byRaw["read-call"]!, + }); + seeded.state.nextBlockId = 3; + seeded.state.nextRunId = 3; + seeded.state.stats.compressionCount = 2; + const before = coveredMessageIds(seeded.state); + const res = core.applyCompression({ + messages, + state: seeded.state, + config, + ranges: [{ startRef: "b1", endRef: "b2", summary: SUMMARY }], + }); + assert.equal(res.result.errors.length, 1, JSON.stringify(res.result)); + assert.equal(res.result.blocksCreated, 0); + assert.match(res.result.errors[0]!, /cannot satisfy turn integrity/); + assert.deepEqual(lostCoverage(before, res.state), [], "state must stay untouched"); + }); + + it("control: a clean merge (no split, no protected tool) loses nothing", () => { + const core = createCore(); + const messages: CoreMessage[] = [ + { id: "old-user", role: "user", contentType: "text", text: LONG }, + { id: "old-assistant", role: "assistant", contentType: "text", text: "Previous work finished." }, + { id: "mid-user", role: "user", contentType: "text", text: LONG }, + { id: "mid-assistant", role: "assistant", contentType: "text", text: "More prior context." }, + { id: "new-user", role: "user", contentType: "text", text: "Continue." }, + ]; + const state = createInitialState(); + let nextRef = 1; + for (const m of messages) { + const ref = `m${String(nextRef++).padStart(5, "0")}`; + state.messageRefs.byRaw[m.id] = ref; + state.messageRefs.byRef[ref] = m.id; + } + const seeded = core.applyCompression({ + messages, + state, + config, + ranges: [{ startRef: state.messageRefs.byRaw["old-user"]!, endRef: state.messageRefs.byRaw["old-assistant"]!, summary: SUMMARY }], + }); + seeded.state.blocks.push({ + ...seeded.state.blocks[0]!, + blockId: "b2", + runId: "r2", + directMessageIds: ["mid-user", "mid-assistant"], + effectiveMessageIds: ["mid-user", "mid-assistant"], + compressedTokens: 400, + startRef: state.messageRefs.byRaw["mid-user"]!, + endRef: state.messageRefs.byRaw["mid-assistant"]!, + }); + seeded.state.nextBlockId = 3; + seeded.state.nextRunId = 3; + seeded.state.stats.compressionCount = 2; + const before = coveredMessageIds(seeded.state); + const res = core.applyCompression({ + messages, + state: seeded.state, + config, + ranges: [{ startRef: "b1", endRef: "b2", summary: SUMMARY }], + }); + assert.equal(res.result.errors.length, 0, JSON.stringify(res.result.errors)); + assert.equal(res.result.blocksCreated, 1); + assert.deepEqual(lostCoverage(before, res.state), []); + }); +}); diff --git a/package-lock.json b/package-lock.json index af5d7f8dbd..cf849e66ce 100644 --- a/package-lock.json +++ b/package-lock.json @@ -50,7 +50,7 @@ }, "kernel": { "name": "acp-kernel", - "version": "0.0.168", + "version": "0.0.169", "dev": true, "license": "MIT", "devDependencies": {