Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions kernel/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion kernel/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "acp-kernel",
"version": "0.0.177",
"version": "0.0.178",
"description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.",
"license": "MIT",
"author": "ranxianglei",
Expand Down
8 changes: 8 additions & 0 deletions kernel/src/recommend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -429,9 +429,17 @@ export function buildCompressibleRanges(
// each removed message as a gap so no range spans it.
// #2663: judge foldability on the same array class apply does (see the
// `integritySource` parameter) — grouping/pairing are array-dependent.
// #5531: ids already covered by an active block are INVISIBLE, not kept —
// a covered call cannot strand its turn's reasoning. Without this, every
// #564 reverse split (call+result folded into a block, reasoning+text left
// visible) manufactures residue that no later fold may ever take: it is
// withdrawn here AND rejected at apply, so each growth fold reclaims only
// the non-residue and the session grows without bound (billion-context#5531).
const unfoldedIds = computeIntegrityWithdrawals(
integritySource ?? messages,
new Set(compressibleMsgs.map((info) => info.id)),
undefined,
covered,
).withdrawn;
if (unfoldedIds.size > 0) {
let gapPending = false;
Expand Down
16 changes: 15 additions & 1 deletion kernel/src/turn-integrity.ts
Original file line number Diff line number Diff line change
Expand Up @@ -184,11 +184,22 @@ export function buildIntegrityPre(messages: CoreMessage[]): IntegrityPre {
*
* The reverse direction stays allowed (#564): reasoning and text may stay
* visible while the call and its result fold, since that leaves a valid stream.
*
* Coverage is not "kept". An id in `invisibleIds` is already hidden by an
* active block: it neither counts as folding nor as kept-visible. A covered
* tool-call therefore cannot strand its turn's reasoning — without this, a
* #564 reverse split (call+result inside an existing block, reasoning+text
* visible) withdraws the stranded pair wholesale FOREVER: they leave no gap a
* range can span, so every later fold reclaims only everything else and the
* residue grows monotonically until nothing left is big enough to fold
* (billion-context#5531). Omit the parameter for the legacy model where
* everything not folded reads as visible.
*/
export function computeIntegrityWithdrawals(
messages: CoreMessage[],
foldedIds: ReadonlySet<string>,
pre?: IntegrityPre,
invisibleIds?: ReadonlySet<string>,
): IntegrityWithdrawals {
const remaining = new Set(foldedIds);
const withdrawn = new Set<string>();
Expand All @@ -213,7 +224,10 @@ export function computeIntegrityWithdrawals(
);
if (!foldHasReasoning) continue;
const keptHasCall = group.some(
(id) => !remaining.has(id) && callIds.has(id),
(id) =>
!remaining.has(id) &&
callIds.has(id) &&
!invisibleIds?.has(id),
);
if (!keptHasCall) continue;
handledTurns.add(g);
Expand Down
6 changes: 6 additions & 0 deletions kernel/src/unit-projection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -407,10 +407,16 @@ export function projectUnit(input: UnitProjectionInput): UnitProjection {
foldedRef = new Set(effectiveMessageIds);
for (const id of survivingCoverage) foldedRef.add(id);
}
// #5531: fresh folds judge on their OWN ids alone, so the coverage of OTHER
// active blocks must arrive as the invisible set — a covered call is not
// kept-visible and cannot strand its turn's reasoning (#564 reverse-split
// residue, billion-context#5531). Merges already model post-fold coverage
// through foldedRef ∪ survivingCoverage and stay untouched.
const initial = computeIntegrityWithdrawals(
input.messages,
foldedRef,
shared?.integrityPre,
isMerge ? undefined : input.preExistingCoverage,
);
splitTurnCount = initial.splitTurnCount;
splitPairCount = initial.splitPairCount;
Expand Down
271 changes: 271 additions & 0 deletions kernel/tests/issue5531-stranded-reasoning-residue.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,271 @@
import { describe, it } from "node:test";
import assert from "node:assert/strict";
import { createCore } from "../src/compress.js";
import { createInitialState } from "../src/state.js";
import { defaultConfig } from "../src/config.js";
import { buildCompressibleRanges } from "../src/recommend.js";
import { collectCoverage, projectUnit } from "../src/unit-projection.js";
import { computeIntegrityWithdrawals } from "../src/turn-integrity.js";
import type { Config, CoreMessage } from "../src/types.js";

/**
* billion-context#5531 — "每次增长压缩只回收一部分": strict-thinking sessions
* (DeepSeek) whose turns were folded in the #564 reverse direction (call+result
* covered by a block, reasoning+text left visible) accumulate PERMANENTLY
* un-foldable residue: the global screen withdrew the stranded pair because a
* covered call read as "kept visible", and apply rejected it the same way.
* Coverage is not "kept" — these pins lock both directions of that fix.
*/

function user(id: string): CoreMessage {
return {
id,
role: "user",
contentType: "text",
text: `user-${id} ` + "x".repeat(200),
};
}
function reasoning(id: string): CoreMessage {
return {
id,
role: "assistant",
contentType: "reasoning",
text: `think-${id} ` + "t".repeat(400),
};
}
function text(id: string): CoreMessage {
return {
id,
role: "assistant",
contentType: "text",
text: `text-${id} ` + "x".repeat(200),
};
}
function toolCall(id: string, callId: string): CoreMessage {
return {
id,
role: "assistant",
contentType: "tool-call",
toolName: "pwsh",
toolCallId: callId,
text: `call-${callId}`,
};
}
function toolResult(id: string, callId: string): CoreMessage {
return {
id,
role: "user",
contentType: "tool-result",
toolName: "pwsh",
toolCallId: callId,
text: `result-${callId} ` + "r".repeat(200),
};
}

/** One strict-thinking turn: [reasoning, text, call, result] flanked by users.
* Raw ids map to refs m00001..m00007 after processTurn. */
function buildMessages(): CoreMessage[] {
return [
user("u1"), // m00001
reasoning("r"), // m00002
text("t"), // m00003
toolCall("c", "ca"), // m00004
toolResult("res", "ca"), // m00005
user("u6"), // m00006
user("u7"), // m00007
];
}

const config = (): Config => ({
...defaultConfig(150000),
preserveRecentMessages: 0,
preserveRecentTokens: 0,
compress: {
...defaultConfig(150000).compress,
minCompressRange: 1,
minSummaryLength: 1,
},
});

const countTokens = (t: string) => Math.ceil(t.length / 4);

/** Seed the #564 reverse split: fold ONLY the call+result, leaving the turn's
* reasoning+text visible beside the new block. */
async function seedReverseSplit() {
const core = createCore();
const messages = buildMessages();
const out = core.processTurn({
messages,
state: createInitialState(),
config: config(),
tokenCount: countTokens,
});
const seed = core.applyCompression({
state: out.state,
messages,
config: config(),
ranges: [{ startRef: "m00004", endRef: "m00005", summary: "s".repeat(400) }],
});
const block = seed.state.blocks.find((b) => b.active);
assert.ok(block, "seed fold must create a block");
assert.ok(block.effectiveMessageIds.includes("c"));
assert.ok(block.effectiveMessageIds.includes("res"));
assert.ok(
!block.effectiveMessageIds.includes("r") &&
!block.effectiveMessageIds.includes("t"),
`seed must cover only call+result, got ${JSON.stringify(block.effectiveMessageIds)}`,
);
return { core, messages, state: seed.state };
}

describe("#5531 gate: a covered call cannot strand its turn's reasoning", () => {
it("legacy semantics WITHDRAW the stranded pair (the bug mechanism)", async () => {
const { messages, state } = await seedReverseSplit();
// Without the invisible set, the covered call reads as kept-visible and
// the whole turn is withdrawn — this is exactly why the residue was
// permanent. Pinning the old verdict documents what the fix overrides.
const legacy = computeIntegrityWithdrawals(
messages,
new Set(["r", "t"]),
).withdrawn;
assert.ok(legacy.has("r") && legacy.has("t"), "legacy gate strands the pair");
});

it("with coverage as invisible, the stranded pair stays foldable", async () => {
const { messages, state } = await seedReverseSplit();
const gate = computeIntegrityWithdrawals(
messages,
new Set(["r", "t"]),
undefined,
collectCoverage(state),
).withdrawn;
assert.ok(!gate.has("r") && !gate.has("t"), JSON.stringify([...gate]));
});
});

describe("#5531 recommend: stranded reasoning+text are advertised again", () => {
it("buildCompressibleRanges offers the stranded pair as compressible", async () => {
const { messages, state } = await seedReverseSplit();
const ranges = buildCompressibleRanges(messages, state, config());
const byRaw = state.messageRefs.byRaw;
const refOf = (raw: string) => byRaw[raw]!;
const inRange = (raw: string) =>
ranges.compressible.some(
(r) => r.startRef <= refOf(raw) && refOf(raw) <= r.endRef,
);
assert.ok(inRange("r"), "stranded reasoning must be advertised");
assert.ok(inRange("t"), "stranded text must be advertised");
});

it("projectUnit judges the stranded pair foldable (fatal === null)", async () => {
const { messages, state } = await seedReverseSplit();
const byRaw = state.messageRefs.byRaw;
const projection = projectUnit({
messages,
state,
config: config(),
startRef: byRaw["r"]!,
endRef: byRaw["t"]!,
preExistingCoverage: collectCoverage(state),
});
assert.equal(projection.fatal, null, projection.fatal?.error.message);
assert.ok(projection.filteredIds.includes("r"));
assert.ok(projection.filteredIds.includes("t"));
});

it("closed loop: the advertised stranded range folds without a split error", async () => {
const { core, messages, state } = await seedReverseSplit();
const byRaw = state.messageRefs.byRaw;
const ranges = buildCompressibleRanges(messages, state, config());
const target = ranges.compressible.find(
(r) => r.startRef <= byRaw["r"]! && byRaw["r"]! <= r.endRef,
);
assert.ok(target, "recommendation must contain the stranded pair");
const res = core.applyCompression({
state,
messages,
config: config(),
ranges: [
{
startRef: target!.startRef,
endRef: target!.endRef,
summary: "reclaiming the stranded turn head",
},
],
});
assert.deepEqual(res.result.errors, [], JSON.stringify(res.result.errors));
assert.ok(res.result.blocksCreated >= 1, "a recovery block must be created");
const block = res.state.blocks.find((b) => b.active && b !== state.blocks[0]);
assert.ok(block, "new block distinct from the seed block");
assert.ok(block.effectiveMessageIds.includes("r"));
assert.ok(block.effectiveMessageIds.includes("t"));
});
});

describe("#5531 negative: a VISIBLE call still guards its reasoning", () => {
// Same geometry, but nothing covers the call: protecting it (recent zone /
// protected tools) instead means folding r+t would orphan a LIVE call — the
// original #684 doctrine must still hold.
function freshSeeded() {
const core = createCore();
const messages = buildMessages();
const out = core.processTurn({
messages,
state: createInitialState(),
config: config(),
tokenCount: countTokens,
});
return { core, messages, state: out.state };
}

it("projectUnit still fatals 'split' when the call stays visible", () => {
const { messages, state } = freshSeeded();
const byRaw = state.messageRefs.byRaw;
// Protecting the call+result is what makes them KEPT (not folded) while
// the range carries the reasoning — the geometry #684 exists to refuse.
const projection = projectUnit({
messages,
state,
config: config(),
startRef: byRaw["r"]!,
endRef: byRaw["t"]!,
preExistingCoverage: new Set(),
protectedMessageIds: new Set([byRaw["c"]!, byRaw["res"]!]),
});
assert.ok(projection.fatal, "visible call must keep its reasoning");
assert.equal(projection.fatal!.kind, "split");
assert.match(projection.fatal!.error.message, /split 1 turn/);
});

it("buildCompressibleRanges does NOT advertise the pair while the call is visible", () => {
const { messages, state } = freshSeeded();
const byRaw = state.messageRefs.byRaw;
// With NO carve-out the whole turn folds atomically (legitimate), so the
// suppression case is a zone brush: the call+result sit in the protected
// zone (visible, kept) while the range would carry the reasoning.
const zoneRefs = new Set([byRaw["c"]!, byRaw["res"]!]);
const ranges = buildCompressibleRanges(messages, state, config(), zoneRefs);
const inRange = (raw: string) =>
ranges.compressible.some(
(r) => r.startRef <= byRaw[raw]! && byRaw[raw]! <= r.endRef,
);
assert.ok(!inRange("r"), "orphaned-reasoning fold must stay suppressed");
assert.ok(!inRange("t"));
});

it("applyCompression still refuses the split with the canonical error", () => {
const { core, messages, state } = freshSeeded();
const byRaw = state.messageRefs.byRaw;
const protectedRefs = new Set([byRaw["c"]!, byRaw["res"]!]);
const res = core.applyCompression({
state,
messages,
config: config(),
protectedMessageIds: protectedRefs,
ranges: [{ startRef: "m00002", endRef: "m00003", summary: "s".repeat(400) }],
});
assert.equal(res.result.blocksCreated, 0);
assert.equal(res.result.errors.length, 1);
assert.match(res.result.errors[0]!, /split 1 turn/);
});
});
2 changes: 1 addition & 1 deletion package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading