Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions kernel/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion kernel/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "acp-kernel",
"version": "0.0.197",
"version": "0.0.198",
"description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.",
"license": "MIT",
"author": "ranxianglei",
Expand Down
54 changes: 53 additions & 1 deletion kernel/src/unit-projection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -570,10 +570,56 @@ export function projectUnit(input: UnitProjectionInput): UnitProjection {
return true;
return hasUnrecoverableMediaPayload(msg);
};
// #6709: `isNeverFoldable5919` lumps two very different cases under
// ref === "BLOCKED". One is CONTENT-FREE — the body was removed by
// native compression, so there is no live wire content to protect
// and the message can never age into foldability. Letting THAT one
// doom the merge turns the #5919 no-op into a permanent deadlock:
// every merge covering its turn re-hits the same skip and the
// session stays stuck at its context ceiling (billion-context#6709,
// session-ad47cd30, 6,115 BLOCKED). Such a partner carries nothing
// the turn-integrity gate is protecting, so it may flow to rung 2
// (folded into the merged block, closing the turn) exactly like a
// foldable partner.
// The OTHER case is LIVE — a protected tool / unrecoverable media
// whose FULL body sits in the wire (assignRefs pins those to ref
// "BLOCKED" too). That partner MUST still doom the merge: folding
// it in drops protected content from the wire, and leaving it split
// emits an orphaned tool_call / tool_result the provider rejects.
// So only a BLOCKED ref that is NOT live-protected/media is
// content-free; everything else never-foldable keeps dooming.
// COUPLING INVARIANT (do not relax one side independently of the
// other): the un-doom decision below delegates to
// filterProtectedToolMessages with the identical `shared` context
// the fresh-fold path uses (unit-projection.ts:250). A partner is
// therefore un-doomed ONLY if the fold path itself would fold it —
// it can never be un-doomed into a live-body drop or an orphaned
// tool_call/tool_result. Pinned by tests/issue5919-shredded-merge.
// test.ts ("6709-lockstep"). (Review B risk #1 / review A risk r1.)
const isContentFreeBlocked5919 = (id: string): boolean => {
const ref = refOf5919(id);
if (ref !== "BLOCKED") return false;
if (input.protectedMessageIds?.has(ref)) return false;
if (
filterProtectedToolMessages(
[id],
input.messages,
input.config,
shared,
).length === 0
)
return false;
const msg = msgById(id);
if (msg !== undefined && hasUnrecoverableMediaPayload(msg))
return false;
return true;
};
const visibleStranded = [...current].filter(
(id) => !survivingCoverage.has(id) && !consumedCoverage.has(id),
);
const doomedStranded = visibleStranded.filter(isNeverFoldable5919);
const doomedStranded = visibleStranded.filter(
(id) => isNeverFoldable5919(id) && !isContentFreeBlocked5919(id),
);
if (doomedStranded.length > 0) {
const foldablePartners = visibleStranded.filter(
(id) => !isNeverFoldable5919(id),
Expand All @@ -598,6 +644,12 @@ export function projectUnit(input: UnitProjectionInput): UnitProjection {
// advertised merge stops dead-ending (billion-context#5919).
let extended5919 = false;
const tryExtend5919 = (): boolean => {
// >16 stranded partners: bail to the actionable fatal below.
// Post-#6709 a content-free BLOCKED partner among them no longer
// no-ops the merge, so a shape that pre-fix silently no-opped now
// surfaces as the actionable fatal (fold the partners via a plain
// range compress) — the #5919 ladder's intent: an actionable
// error beats a silent no-op deadlock (review A risk r2).
if (
extended5919 ||
visibleStranded.length === 0 ||
Expand Down
262 changes: 248 additions & 14 deletions kernel/tests/issue5919-shredded-merge.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -172,36 +172,69 @@ test("5919-1: foldable stranded partner outside the span → merge extends and r
);
});

test("5919-2: protected (BLOCKED) stranded partner → no-op success, not a retryable error", () => {
test("5919-2: CONTENT-FREE (BLOCKED) stranded partner → merge proceeds (#6709)", () => {
// 'Read' is NOT a protected tool, so a ref=BLOCKED 'l' here models a
// content-free partner (body removed by native compression): nothing live
// for the integrity gate to protect. #6709: it must flow to rung 2 and be
// folded into the merged block instead of dead-locking every merge of this
// turn (the reported 6,115 BLOCKED / 67% ceiling case).
const { core, messages, state } = build("BLOCKED", "Read");
const res = core.applyCompression({
messages,
state,
config,
ranges: [{ startRef: "b2", endRef: "b3", summary: SUMMARY }],
});
// idempotent no-op success (#4521): zero errors, zero blocks
assert.deepEqual(res.result.errors, []);
assert.equal(res.result.blocksCreated, 0);
// b3 stays ACTIVE — nothing was consumed
assert.equal(res.state.blocks.find((b) => b.blockId === "b3")!.active, true);
const skip = res.result.warnings.find((w) =>
w.startsWith("Merge of b3 skipped"),
);
assert.equal(res.result.blocksCreated, 1);
// b3 is consumed (deactivated); the surviving tier-2 block stays
assert.equal(res.state.blocks.find((b) => b.blockId === "b3")!.active, false);
assert.equal(res.state.blocks.find((b) => b.blockId === "b2")!.active, true);
const merged = res.state.blocks.find((b) => b.blockId === "b4")!;
// the extension pulled the content-free BLOCKED result into coverage — the
// turn closes under the merged block
assert.ok(merged.effectiveMessageIds.includes("l"), "b4 must cover the BLOCKED result l");
assert.ok(merged.effectiveMessageIds.includes("t"));
assert.ok(merged.effectiveMessageIds.includes("k"));
// the old no-op skip warning must be gone
assert.ok(
skip,
`expected the skip warning naming the blocked merge, got: ${res.result.warnings.join(" | ")}`,
!res.result.warnings.some((w) => w.startsWith("Merge of b3 skipped")),
"the skip warning must not appear — the merge now proceeds",
);
assert.ok(skip!.includes("cannot be closed now"));
assert.ok(skip!.includes("Do not retry this merge"));
assert.ok(skip!.includes("BLOCKED"));
// the old false promise must be gone
assert.ok(
!res.result.warnings.some((w) => w.includes("age into coverage")),
"the 'age into coverage' promise must not appear anywhere",
);
});

test("5919-5: LIVE protected tool stored as ref BLOCKED → still no-op (#6709 guard)", () => {
// A protected tool exchange is pinned to ref BLOCKED by assignRefs WHILE its
// full body stays in the wire. Exempting ref==BLOCKED partners from the
// merge-deadlock must NOT weaken those: folding the live call/result in would
// drop protected content, and leaving them split emits an orphaned tool_call.
// So a live protected tool that happens to carry ref BLOCKED STILL dooms the
// merge — the honest idempotent no-op. (This is the case both #6719 and
// #6731 missed: their tests gave the protected partner a numeric ref.)
const { core, messages, state } = build("BLOCKED", "mcp__bili__acp_rule");
const res = core.applyCompression({
messages,
state,
config,
ranges: [{ startRef: "b2", endRef: "b3", summary: SUMMARY }],
});
assert.deepEqual(res.result.errors, []);
assert.equal(res.result.blocksCreated, 0);
assert.equal(res.state.blocks.find((b) => b.blockId === "b3")!.active, true);
const skip = res.result.warnings.find(
(w) =>
w.startsWith("Merge of b3 skipped") && w.includes("Do not retry"),
);
assert.ok(
skip,
`expected the no-op skip warning, got: ${res.result.warnings.join(" | ")}`,
);
});

test("5919-3: protected TOOL stranded partner (pinned latest pair) → same no-op skip", () => {
const { core, messages, state } = build("m00006", "mcp__bili__acp_rule");
const res = core.applyCompression({
Expand Down Expand Up @@ -238,3 +271,204 @@ test("5919-4: foldable partner INSIDE the span (pre-fix working case) → clean
assert.ok(merged!.effectiveMessageIds.includes("l"));
assert.equal(res.state.blocks.find((b) => b.blockId === "b3")!.active, true);
});

/**
* Build a session whose turn {r, t, k, l} carries a compress receipt 'l'
* (call k / result l, toolName mcp__bili__compress — a protected tool that
* assignRefs pins to ref BLOCKED), plus other compress exchanges, so 'l' is
* either AGED (outside the keepRecent window, #4348) or the NEWEST (inside it).
* This is the real #6709 incident shape — the 6,115 stale BLOCKED pins were aged
* compress receipts — and it exercises the #4348 exemption (isExemptFold) that
* the synthetic 5919-2 'Read' case cannot reach.
*/
function seedCompressReceipt(messages: any[], blockedId: string) {
const core = createCore();
const state = createInitialState();
let n = 1;
for (const m of messages) {
const ref =
m.id === blockedId ? "BLOCKED" : `m${String(n++).padStart(5, "0")}`;
state.messageRefs.byRaw[m.id] = ref;
if (ref !== "BLOCKED") state.messageRefs.byRef[ref] = m.id;
}
const seeded = core.applyCompression({
messages,
state,
config,
ranges: [{ startRef: "m00001", endRef: "m00002", summary: SUMMARY }],
});
const base = seeded.state.blocks[0]!;
const push = (
blockId: string,
runId: string,
ids: string[],
tier = 1,
) =>
seeded.state.blocks.push({
...base,
blockId,
runId,
tier,
directMessageIds: [...ids],
effectiveMessageIds: [...ids],
compressedTokens: ids.length * 100,
startRef: seeded.state.messageRefs.byRaw[ids[0]!]!,
endRef: seeded.state.messageRefs.byRaw[ids[ids.length - 1]!]!,
});
push("b2", "r2", ["r"], 2);
push("b3", "r3", ["t", "k"]);
seeded.state.nextBlockId = 4;
seeded.state.nextRunId = 4;
seeded.state.stats.compressionCount = 3;
return { core, messages, state: seeded.state };
}

const COMPRESS_RECEIPT =
"[Compressed m00001-m00002 -> 1 block(s), ~100 tokens saved]";
/** one compress call+result exchange (callId `cc${i}`) */
function cex(i: number) {
return [
{ id: `kc${i}`, role: "assistant", contentType: "tool-call", toolName: "mcp__bili__compress", toolCallId: `cc${i}`, text: "{}" },
{ id: `lc${i}`, role: "tool", contentType: "tool-result", toolName: "mcp__bili__compress", toolCallId: `cc${i}`, text: COMPRESS_RECEIPT },
];
}

test("5919-6: AGED compress receipt (real #6709 class) ref BLOCKED → merge proceeds", () => {
// 'k'/'l' (callId c1) is a compress receipt pinned to ref BLOCKED; three
// newer compress exchanges follow (c2,c3,c4) → l is OUTSIDE the keepRecent
// window → aged per #4348 → foldable. This is exactly the incident shape: a
// stale BLOCKED compress receipt that used to dead-lock every merge of its
// turn (the 6,115-BLOCKED / 67% ceiling case). It must now fold in.
const messages: any[] = [
{ id: "u0", role: "user", contentType: "text", text: LONG },
{ id: "a0", role: "assistant", contentType: "text", text: "Previous work finished." },
...cex(1),
{ id: "r", role: "assistant", contentType: "reasoning", text: LONG },
{ id: "t", role: "assistant", contentType: "text", text: LONG },
{ id: "k", role: "assistant", contentType: "tool-call", toolName: "mcp__bili__compress", toolCallId: "c1", text: "{}" },
{ id: "l", role: "tool", contentType: "tool-result", toolName: "mcp__bili__compress", toolCallId: "c1", text: COMPRESS_RECEIPT },
...cex(2),
...cex(3),
...cex(4),
{ id: "far", role: "assistant", contentType: "text", text: "Far tail." },
{ id: "newuser", role: "user", contentType: "text", text: "Continue." },
];
const { core, state } = seedCompressReceipt(messages, "l");
const res = core.applyCompression({
messages,
state,
config,
ranges: [{ startRef: "b2", endRef: "b3", summary: SUMMARY }],
});
assert.deepEqual(res.result.errors, []);
assert.equal(res.result.blocksCreated, 1);
assert.equal(res.state.blocks.find((b) => b.blockId === "b3")!.active, false);
assert.equal(res.state.blocks.find((b) => b.blockId === "b2")!.active, true);
const merged = res.state.blocks.find((b) => b.blockId === "b4")!;
assert.ok(merged.effectiveMessageIds.includes("l"), "b4 must cover the aged BLOCKED compress receipt l");
assert.ok(merged.effectiveMessageIds.includes("t"));
assert.ok(merged.effectiveMessageIds.includes("k"));
assert.ok(
!res.result.warnings.some((w) => w.startsWith("Merge of b3 skipped")),
"the aged receipt must not be treated as a protected no-op",
);
});

test("5919-7: NEWEST compress receipt (in keepRecent window) ref BLOCKED → still no-op", () => {
// Control for 5919-6: 'l' is the NEWEST compress exchange (zero newer), so it
// is INSIDE the keepRecent working-memory window. Aged-receipt folding must
// not bleed into the live window — a fresh BLOCKED-pinned compress receipt
// STILL dooms the merge (honest idempotent no-op, not a bypass).
const messages: any[] = [
{ id: "u0", role: "user", contentType: "text", text: LONG },
{ id: "a0", role: "assistant", contentType: "text", text: "Previous work finished." },
...cex(1),
...cex(2),
...cex(3),
{ id: "r", role: "assistant", contentType: "reasoning", text: LONG },
{ id: "t", role: "assistant", contentType: "text", text: LONG },
{ id: "k", role: "assistant", contentType: "tool-call", toolName: "mcp__bili__compress", toolCallId: "c1", text: "{}" },
{ id: "l", role: "tool", contentType: "tool-result", toolName: "mcp__bili__compress", toolCallId: "c1", text: COMPRESS_RECEIPT },
{ id: "far", role: "assistant", contentType: "text", text: "Far tail." },
{ id: "newuser", role: "user", contentType: "text", text: "Continue." },
];
const { core, state } = seedCompressReceipt(messages, "l");
const res = core.applyCompression({
messages,
state,
config,
ranges: [{ startRef: "b2", endRef: "b3", summary: SUMMARY }],
});
assert.deepEqual(res.result.errors, []);
assert.equal(res.result.blocksCreated, 0);
assert.equal(res.state.blocks.find((b) => b.blockId === "b3")!.active, true);
assert.ok(
res.result.warnings.some(
(w) => w.startsWith("Merge of b3 skipped") && w.includes("Do not retry"),
),
`expected the no-op skip warning, got: ${res.result.warnings.join(" | ")}`,
);
});

/**
* #6709 LOCKSTEP (coupling) regression — pins the invariant that the merge
* guard's content-free exemption (isContentFreeBlocked5919,
* unit-projection.ts:591) and the fresh-fold path (unit-projection.ts:250)
* make the SAME fold/protect decision for a given tool class, because BOTH
* delegate to `filterProtectedToolMessages` with the identical `shared`
* context. A partner the fold path protects is therefore NEVER un-doomed by
* the guard (no live-body drop, no orphaned tool_call/tool_result), and a
* partner the fold path folds is NEVER left dead-locked (no #6709 regression).
*
* We observe the fold path's decision on the NUMERIC-ref twin (a plain
* message-range fresh-fold, which can only address m-referenced messages) and
* the guard's decision on the BLOCKED twin (the block-range merge), then assert
* the two AGREE. The two tool classes below span both branches of the single
* delegated decision (fold vs protect); the media / paired-result /
* aged-compress branches are pinned individually by 5919-6/7 and the probe
* suite and route through the same filter. (Review B risk #1 / review A r1.)
*/
test("6709-lockstep: merge guard and fresh-fold path agree on every BLOCKED class", () => {
const cases: Array<{ label: string; toolName: string; expectFold: boolean }> = [
{ label: "ordinary (content-free) tool", toolName: "Read", expectFold: true },
{ label: "live protected tool", toolName: "mcp__bili__acp_rule", expectFold: false },
];
for (const c of cases) {
// fold path: NUMERIC-ref twin, plain message-range fresh-fold of k..l.
const fold = build("m00006", c.toolName);
const fres = fold.core.applyCompression({
messages: fold.messages,
state: fold.state,
config,
ranges: [{ startRef: "m00005", endRef: "m00006", summary: SUMMARY }],
});
const fblock = fres.state.blocks.find((b) => b.blockId === "b4");
const foldPathFolds = !!fblock && fblock.effectiveMessageIds.includes("l");

// merge guard: BLOCKED twin, block-range merge b2..b3 (strands l).
const guard = build("BLOCKED", c.toolName);
const gres = guard.core.applyCompression({
messages: guard.messages,
state: guard.state,
config,
ranges: [{ startRef: "b2", endRef: "b3", summary: SUMMARY }],
});
const guardProceeds = gres.result.blocksCreated === 1;

assert.equal(
foldPathFolds,
c.expectFold,
`fold path [${c.label}]: expected fold=${c.expectFold}, got ${foldPathFolds}`,
);
assert.equal(
guardProceeds,
c.expectFold,
`merge guard [${c.label}]: expected proceed=${c.expectFold}, got ${guardProceeds}`,
);
assert.equal(
foldPathFolds,
guardProceeds,
`COUPLING [${c.label}]: fold path (fold=${foldPathFolds}) and merge guard (proceed=${guardProceeds}) MUST agree — a partner the fold path protects must not be un-doomed, and one it folds must not be dead-locked`,
);
}
});
2 changes: 1 addition & 1 deletion package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading