Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions kernel/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion kernel/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "acp-kernel",
"version": "0.0.168",
"version": "0.0.169",
"description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.",
"license": "MIT",
"author": "ranxianglei",
Expand Down
121 changes: 94 additions & 27 deletions kernel/src/unit-projection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -334,39 +334,106 @@ export function projectUnit(input: UnitProjectionInput): UnitProjection {
// computeIntegrityWithdrawals: withdrawing a pair for the result half can
// strand a turn whose reasoning still folds and whose call now stays visible.
{
const {
withdrawn: withdrawIds,
splitTurnCount: turns,
splitPairCount: pairs,
} = computeIntegrityWithdrawals(input.messages, effectiveMessageIds);
withdrawnIds = withdrawIds;
splitTurnCount = turns;
splitPairCount = pairs;
if (withdrawIds.size > 0) {
for (const id of withdrawIds) effectiveMessageIds.delete(id);
const beforeWithdraw = filteredIds.length;
filteredIds = filteredIds.filter((id) => !withdrawIds.has(id));
// #4484/#3322: on a MERGE (this unit consumes existing blocks) the integrity
// gate needs two things a fresh fold does not. (a) Judge POST-FOLD coverage —
// this unit's ids PLUS every still-active sibling block's coverage — so a tool
// half already covered by an unconsumed sibling reads as folded, not a spurious
// "visible" half (#3322 withdrew inherited coverage for exactly this reason).
// (b) Never drop an id whose ONLY covering block is consumed here: those blocks
// deactivate below, so removing their id strands it uncovered — the regression
// the host pre-commit guard then refuses (#4484). We therefore unfold only ids
// whose coverage does not depend on a consumed block, re-checking to a fixed
// point; if consistency would force unfolding such an id we fail honestly rather
// than emit a doomed regressive candidate. Fresh folds keep the legacy behavior.
const isMerge = consumedBlockIds.length > 0;
let consumedCoverage = new Set<string>();
let survivingCoverage = new Set<string>();
if (isMerge) {
const consumedSet = new Set(consumedBlockIds);
for (const block of activeBlocks(input.state)) {
const target = consumedSet.has(block.blockId)
? consumedCoverage
: survivingCoverage;
for (const id of block.effectiveMessageIds) target.add(id);
}
}
let foldedRef = effectiveMessageIds;
if (isMerge) {
foldedRef = new Set(effectiveMessageIds);
for (const id of survivingCoverage) foldedRef.add(id);
}
const initial = computeIntegrityWithdrawals(input.messages, foldedRef);
splitTurnCount = initial.splitTurnCount;
splitPairCount = initial.splitPairCount;
if (initial.withdrawn.size > 0) {
const splitDesc = [
splitTurnCount > 0 ? `${splitTurnCount} turn(s)` : null,
splitPairCount > 0
? `${splitPairCount} tool call/result pair(s)`
: null,
splitPairCount > 0 ? `${splitPairCount} tool call/result pair(s)` : null,
]
.filter((part): part is string => part !== null)
.join(" and ");
if (filteredIds.length === 0 && consumedBlockIds.length === 0) {
return finish({
fatal: {
kind: "split",
error: new Error(
`Range would split ${splitDesc} at the protected-zone boundary: a visible tool-call must keep its reasoning run and its results (strict providers reject a rebuilt request that lost either). Shrink the range to end before the turn starts, or wait until the whole turn ages out of the protected zone.`,
),
},
});
if (!isMerge) {
withdrawnIds = initial.withdrawn;
for (const id of initial.withdrawn) effectiveMessageIds.delete(id);
const beforeWithdraw = filteredIds.length;
filteredIds = filteredIds.filter((id) => !initial.withdrawn.has(id));
if (filteredIds.length === 0 && consumedBlockIds.length === 0) {
return finish({
fatal: {
kind: "split",
error: new Error(
`Range would split ${splitDesc} at the protected-zone boundary: a visible tool-call must keep its reasoning run and its results (strict providers reject a rebuilt request that lost either). Shrink the range to end before the turn starts, or wait until the whole turn ages out of the protected zone.`,
),
},
});
}
warnings.push(
`Withdrawn ${beforeWithdraw - filteredIds.length} message(s) from compression range to keep ${splitDesc} intact (visible tool-call would lose its reasoning run or its results).`,
);
} else {
const unfoldable = new Set<string>(
[...effectiveMessageIds].filter((id) => !consumedCoverage.has(id)),
);
const unfolded = new Set<string>();
let current = initial.withdrawn;
let guard = unfoldable.size + 2;
let stuck = false;
while (current.size > 0 && guard-- > 0) {
let progress = false;
for (const id of current) {
if (unfoldable.has(id) && !unfolded.has(id)) {
unfolded.add(id);
progress = true;
}
}
if (!progress) {
stuck = true;
break;
}
const recheck = new Set(effectiveMessageIds);
for (const id of unfolded) recheck.delete(id);
for (const id of survivingCoverage) recheck.add(id);
current = computeIntegrityWithdrawals(input.messages, recheck).withdrawn;
}
if (stuck || current.size > 0) {
const sample = [...consumedCoverage].slice(0, 5).join(", ");
return finish({
fatal: {
kind: "split",
error: new Error(
`Merging ${consumedBlockIds.join(", ")} cannot satisfy turn integrity without unfolding already-folded message(s) owned by those blocks (${sample}) — doing so would drop their existing coverage. Align the range to whole turns (do not cut through a reasoning/tool-call/result group) or merge fewer blocks.`,
),
},
});
}
withdrawnIds = unfolded;
for (const id of unfolded) effectiveMessageIds.delete(id);
const beforeWithdraw = filteredIds.length;
filteredIds = filteredIds.filter((id) => !unfolded.has(id));
warnings.push(
`Withdrew ${beforeWithdraw - filteredIds.length} newly-folded message(s) from the merge to keep ${splitDesc} intact while preserving the existing coverage of ${consumedBlockIds.join(", ")}.`,
);
}
warnings.push(
`Withdrawn ${beforeWithdraw - filteredIds.length} message(s) from compression range to keep ${splitDesc} intact (visible tool-call would lose its reasoning run or its results).`,
);
}
}

Expand Down
238 changes: 238 additions & 0 deletions kernel/tests/issue4484-merge-coverage.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,238 @@
import { describe, it } from "node:test";
import assert from "node:assert/strict";
import { createCore } from "../src/compress.js";
import { createInitialState, coveredMessageIds } from "../src/state.js";
import { defaultConfig } from "../src/config.js";
import { computeIntegrityWithdrawals } from "../src/turn-integrity.js";
import type { CompressionState, CoreMessage } from "../src/types.js";

// #4484: merging existing blocks must never drop their existing coverage. The
// turn-integrity gate used to withdraw ids carried only by consumed blocks,
// stranding them uncovered once those blocks deactivated (host then refused the
// fold via gate:coverage-regression). These fixtures mirror the two real trigger
// shapes (interior hard-protected tool; tail cut mid-tool-group); synthetic only.

const isToolProtected = (t?: string) =>
t === "mcp__bili__acp_rule" || t === "mcp__bili__compress";
const isCompressTool = (t?: string) =>
t === "mcp__bili__compress" || t === "compress";

const config = defaultConfig(255000, {
preserveRecentMessages: 0,
preserveRecentTokens: 0,
isToolProtected,
isCompressTool,
} as never);

const SUMMARY = "Synthetic summary for this compression test. ".repeat(4);
const LONG = "Synthetic historical content, safe for public reproduction. ".repeat(150);

function buildFixture(kind: "protected-rule" | "parallel-tail") {
const core = createCore();
const messages: CoreMessage[] = [
{ id: "old-user", role: "user", contentType: "text", text: LONG },
{ id: "old-assistant", role: "assistant", contentType: "text", text: "Previous work finished." },
{ id: "reasoning", role: "assistant", contentType: "reasoning", text: "Inspect the current state." },
{ id: "read-call", role: "assistant", contentType: "tool-call", toolName: "Read", toolCallId: "read-1", text: '{"path":"/tmp/example.txt"}' },
];
let seedStartId: string;
let seedEndId: string;
let toolIds: string[];
if (kind === "protected-rule") {
messages.push(
{ id: "rule-call", role: "assistant", contentType: "tool-call", toolName: "mcp__bili__acp_rule", toolCallId: "rule-1", text: '{"operation":"list"}' },
{ id: "read-result", role: "tool", contentType: "tool-result", toolName: "Read", toolCallId: "read-1", text: LONG },
{ id: "rule-result", role: "tool", contentType: "tool-result", toolName: "mcp__bili__acp_rule", toolCallId: "rule-1", text: "No rules recorded." },
);
seedStartId = "read-call";
seedEndId = "read-result";
toolIds = ["read-call", "read-result"];
} else {
messages.push(
{ id: "bash-call", role: "assistant", contentType: "tool-call", toolName: "Bash", toolCallId: "bash-1", text: '{"command":"true"}' },
{ id: "todo-call", role: "assistant", contentType: "tool-call", toolName: "TodoList", toolCallId: "todo-1", text: '{"todos":[]}' },
{ id: "read-result", role: "tool", contentType: "tool-result", toolName: "Read", toolCallId: "read-1", text: LONG },
{ id: "bash-result", role: "tool", contentType: "tool-result", toolName: "Bash", toolCallId: "bash-1", text: LONG },
{ id: "todo-result", role: "tool", contentType: "tool-result", toolName: "TodoList", toolCallId: "todo-1", text: "Updated." },
);
seedStartId = "read-call";
seedEndId = "bash-result";
toolIds = ["read-call", "bash-call", "read-result", "bash-result"];
}
messages.push({ id: "new-user", role: "user", contentType: "text", text: "Continue." });

const state = createInitialState();
let nextRef = 1;
for (const message of messages) {
const ref = message.toolName === "mcp__bili__acp_rule" ? "BLOCKED" : `m${String(nextRef++).padStart(5, "0")}`;
state.messageRefs.byRaw[message.id] = ref;
if (ref !== "BLOCKED") state.messageRefs.byRef[ref] = message.id;
}
const seeded = core.applyCompression({
messages,
state,
config,
ranges: [{ startRef: state.messageRefs.byRaw["old-user"]!, endRef: state.messageRefs.byRaw["old-assistant"]!, summary: SUMMARY }],
});
if (seeded.result.blocksCreated !== 1 || seeded.result.errors.length !== 0) {
throw new Error(`fixture seed failed: ${JSON.stringify(seeded.result)}`);
}
seeded.state.blocks.push({
...seeded.state.blocks[0]!,
blockId: "b2",
runId: "r2",
directMessageIds: [...toolIds],
effectiveMessageIds: [...toolIds],
compressedTokens: toolIds.reduce((n, id) => n + Math.ceil(messages.find((m) => m.id === id)!.text.length / 4), 0),
startRef: state.messageRefs.byRaw[seedStartId],
endRef: state.messageRefs.byRaw[seedEndId],
});
seeded.state.nextBlockId = 3;
seeded.state.nextRunId = 3;
seeded.state.stats.compressionCount = 2;
return { core, messages, state: seeded.state, toolIds };
}

function lostCoverage(before: Set<string>, state: CompressionState): string[] {
const after = coveredMessageIds(state);
return [...before].filter((id) => !after.has(id));
}

describe("#4484 merge preserves existing block coverage", () => {
it("protected-rule interior: merging b1..b2 keeps the Read pair folded", () => {
const { core, messages, state, toolIds } = buildFixture("protected-rule");
const before = coveredMessageIds(state);
const res = core.applyCompression({
messages,
state,
config,
ranges: [{ startRef: "b1", endRef: "b2", summary: SUMMARY }],
});
assert.equal(res.result.errors.length, 0, JSON.stringify(res.result.errors));
assert.equal(res.result.blocksCreated, 1);
const lost = lostCoverage(before, res.state);
assert.deepEqual(lost, [], `regression: lost ${lost.join(", ")}`);
const after = coveredMessageIds(res.state);
for (const id of toolIds) assert.ok(after.has(id), `${id} must stay covered`);
assert.equal(computeIntegrityWithdrawals(messages, after).withdrawn.size, 0);
});

it("parallel-tail cut: merging b1..b2 keeps both Read/Bash pairs folded", () => {
const { core, messages, state, toolIds } = buildFixture("parallel-tail");
const before = coveredMessageIds(state);
const res = core.applyCompression({
messages,
state,
config,
ranges: [{ startRef: "b1", endRef: "b2", summary: SUMMARY }],
});
assert.equal(res.result.errors.length, 0, JSON.stringify(res.result.errors));
assert.equal(res.result.blocksCreated, 1);
const lost = lostCoverage(before, res.state);
assert.deepEqual(lost, [], `regression: lost ${lost.join(", ")}`);
const after = coveredMessageIds(res.state);
for (const id of toolIds) assert.ok(after.has(id), `${id} must stay covered`);
assert.equal(computeIntegrityWithdrawals(messages, after).withdrawn.size, 0);
});

it("impossible merge: fails fast up-front without touching existing coverage", () => {
const core = createCore();
const messages: CoreMessage[] = [
{ id: "old-user", role: "user", contentType: "text", text: LONG },
{ id: "old-assistant", role: "assistant", contentType: "text", text: "Previous work finished." },
{ id: "read-call", role: "assistant", contentType: "tool-call", toolName: "Read", toolCallId: "read-1", text: '{"path":"/tmp/example.txt"}' },
{ id: "read-result", role: "tool", contentType: "tool-result", toolName: "Read", toolCallId: "read-1", text: LONG },
{ id: "new-user", role: "user", contentType: "text", text: "Continue." },
];
const state = createInitialState();
let nextRef = 1;
for (const m of messages) {
const ref = `m${String(nextRef++).padStart(5, "0")}`;
state.messageRefs.byRaw[m.id] = ref;
state.messageRefs.byRef[ref] = m.id;
}
const seeded = core.applyCompression({
messages,
state,
config,
ranges: [{ startRef: state.messageRefs.byRaw["old-user"]!, endRef: state.messageRefs.byRaw["old-assistant"]!, summary: SUMMARY }],
});
if (seeded.result.blocksCreated !== 1 || seeded.result.errors.length !== 0) {
throw new Error(`fixture seed failed: ${JSON.stringify(seeded.result)}`);
}
// b2 covers ONLY the call half of the Read pair — its tail cut mid-tool-group.
// No consistent non-regressive candidate exists: folding the call splits its
// pair, unfolding it strands a message whose only covering block is consumed.
seeded.state.blocks.push({
...seeded.state.blocks[0]!,
blockId: "b2",
runId: "r2",
directMessageIds: ["read-call"],
effectiveMessageIds: ["read-call"],
compressedTokens: 100,
startRef: state.messageRefs.byRaw["read-call"]!,
endRef: state.messageRefs.byRaw["read-call"]!,
});
seeded.state.nextBlockId = 3;
seeded.state.nextRunId = 3;
seeded.state.stats.compressionCount = 2;
const before = coveredMessageIds(seeded.state);
const res = core.applyCompression({
messages,
state: seeded.state,
config,
ranges: [{ startRef: "b1", endRef: "b2", summary: SUMMARY }],
});
assert.equal(res.result.errors.length, 1, JSON.stringify(res.result));
assert.equal(res.result.blocksCreated, 0);
assert.match(res.result.errors[0]!, /cannot satisfy turn integrity/);
assert.deepEqual(lostCoverage(before, res.state), [], "state must stay untouched");
});

it("control: a clean merge (no split, no protected tool) loses nothing", () => {
const core = createCore();
const messages: CoreMessage[] = [
{ id: "old-user", role: "user", contentType: "text", text: LONG },
{ id: "old-assistant", role: "assistant", contentType: "text", text: "Previous work finished." },
{ id: "mid-user", role: "user", contentType: "text", text: LONG },
{ id: "mid-assistant", role: "assistant", contentType: "text", text: "More prior context." },
{ id: "new-user", role: "user", contentType: "text", text: "Continue." },
];
const state = createInitialState();
let nextRef = 1;
for (const m of messages) {
const ref = `m${String(nextRef++).padStart(5, "0")}`;
state.messageRefs.byRaw[m.id] = ref;
state.messageRefs.byRef[ref] = m.id;
}
const seeded = core.applyCompression({
messages,
state,
config,
ranges: [{ startRef: state.messageRefs.byRaw["old-user"]!, endRef: state.messageRefs.byRaw["old-assistant"]!, summary: SUMMARY }],
});
seeded.state.blocks.push({
...seeded.state.blocks[0]!,
blockId: "b2",
runId: "r2",
directMessageIds: ["mid-user", "mid-assistant"],
effectiveMessageIds: ["mid-user", "mid-assistant"],
compressedTokens: 400,
startRef: state.messageRefs.byRaw["mid-user"]!,
endRef: state.messageRefs.byRaw["mid-assistant"]!,
});
seeded.state.nextBlockId = 3;
seeded.state.nextRunId = 3;
seeded.state.stats.compressionCount = 2;
const before = coveredMessageIds(seeded.state);
const res = core.applyCompression({
messages,
state: seeded.state,
config,
ranges: [{ startRef: "b1", endRef: "b2", summary: SUMMARY }],
});
assert.equal(res.result.errors.length, 0, JSON.stringify(res.result.errors));
assert.equal(res.result.blocksCreated, 1);
assert.deepEqual(lostCoverage(before, res.state), []);
});
});
Loading
Loading