diff --git a/kernel/package-lock.json b/kernel/package-lock.json index bad3402fdd..3d36764818 100644 --- a/kernel/package-lock.json +++ b/kernel/package-lock.json @@ -1,12 +1,12 @@ { "name": "acp-kernel", - "version": "0.0.177", + "version": "0.0.178", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "acp-kernel", - "version": "0.0.177", + "version": "0.0.178", "license": "MIT", "devDependencies": { "@types/node": "^22.0.0", diff --git a/kernel/package.json b/kernel/package.json index e7b5b52f29..bb32450b06 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.177", + "version": "0.0.178", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/recommend.ts b/kernel/src/recommend.ts index 6e8639e40b..d2cd669823 100644 --- a/kernel/src/recommend.ts +++ b/kernel/src/recommend.ts @@ -429,9 +429,17 @@ export function buildCompressibleRanges( // each removed message as a gap so no range spans it. // #2663: judge foldability on the same array class apply does (see the // `integritySource` parameter) — grouping/pairing are array-dependent. + // #5531: ids already covered by an active block are INVISIBLE, not kept — + // a covered call cannot strand its turn's reasoning. Without this, every + // #564 reverse split (call+result folded into a block, reasoning+text left + // visible) manufactures residue that no later fold may ever take: it is + // withdrawn here AND rejected at apply, so each growth fold reclaims only + // the non-residue and the session grows without bound (billion-context#5531). const unfoldedIds = computeIntegrityWithdrawals( integritySource ?? messages, new Set(compressibleMsgs.map((info) => info.id)), + undefined, + covered, ).withdrawn; if (unfoldedIds.size > 0) { let gapPending = false; diff --git a/kernel/src/turn-integrity.ts b/kernel/src/turn-integrity.ts index ba839297fc..1ee828b628 100644 --- a/kernel/src/turn-integrity.ts +++ b/kernel/src/turn-integrity.ts @@ -184,11 +184,22 @@ export function buildIntegrityPre(messages: CoreMessage[]): IntegrityPre { * * The reverse direction stays allowed (#564): reasoning and text may stay * visible while the call and its result fold, since that leaves a valid stream. + * + * Coverage is not "kept". An id in `invisibleIds` is already hidden by an + * active block: it neither counts as folding nor as kept-visible. A covered + * tool-call therefore cannot strand its turn's reasoning — without this, a + * #564 reverse split (call+result inside an existing block, reasoning+text + * visible) withdraws the stranded pair wholesale FOREVER: they leave no gap a + * range can span, so every later fold reclaims only everything else and the + * residue grows monotonically until nothing left is big enough to fold + * (billion-context#5531). Omit the parameter for the legacy model where + * everything not folded reads as visible. */ export function computeIntegrityWithdrawals( messages: CoreMessage[], foldedIds: ReadonlySet, pre?: IntegrityPre, + invisibleIds?: ReadonlySet, ): IntegrityWithdrawals { const remaining = new Set(foldedIds); const withdrawn = new Set(); @@ -213,7 +224,10 @@ export function computeIntegrityWithdrawals( ); if (!foldHasReasoning) continue; const keptHasCall = group.some( - (id) => !remaining.has(id) && callIds.has(id), + (id) => + !remaining.has(id) && + callIds.has(id) && + !invisibleIds?.has(id), ); if (!keptHasCall) continue; handledTurns.add(g); diff --git a/kernel/src/unit-projection.ts b/kernel/src/unit-projection.ts index f89b772701..c51e697d3a 100644 --- a/kernel/src/unit-projection.ts +++ b/kernel/src/unit-projection.ts @@ -407,10 +407,16 @@ export function projectUnit(input: UnitProjectionInput): UnitProjection { foldedRef = new Set(effectiveMessageIds); for (const id of survivingCoverage) foldedRef.add(id); } + // #5531: fresh folds judge on their OWN ids alone, so the coverage of OTHER + // active blocks must arrive as the invisible set — a covered call is not + // kept-visible and cannot strand its turn's reasoning (#564 reverse-split + // residue, billion-context#5531). Merges already model post-fold coverage + // through foldedRef ∪ survivingCoverage and stay untouched. const initial = computeIntegrityWithdrawals( input.messages, foldedRef, shared?.integrityPre, + isMerge ? undefined : input.preExistingCoverage, ); splitTurnCount = initial.splitTurnCount; splitPairCount = initial.splitPairCount; diff --git a/kernel/tests/issue5531-stranded-reasoning-residue.test.ts b/kernel/tests/issue5531-stranded-reasoning-residue.test.ts new file mode 100644 index 0000000000..4a84ab26a9 --- /dev/null +++ b/kernel/tests/issue5531-stranded-reasoning-residue.test.ts @@ -0,0 +1,271 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { createCore } from "../src/compress.js"; +import { createInitialState } from "../src/state.js"; +import { defaultConfig } from "../src/config.js"; +import { buildCompressibleRanges } from "../src/recommend.js"; +import { collectCoverage, projectUnit } from "../src/unit-projection.js"; +import { computeIntegrityWithdrawals } from "../src/turn-integrity.js"; +import type { Config, CoreMessage } from "../src/types.js"; + +/** + * billion-context#5531 — "每次增长压缩只回收一部分": strict-thinking sessions + * (DeepSeek) whose turns were folded in the #564 reverse direction (call+result + * covered by a block, reasoning+text left visible) accumulate PERMANENTLY + * un-foldable residue: the global screen withdrew the stranded pair because a + * covered call read as "kept visible", and apply rejected it the same way. + * Coverage is not "kept" — these pins lock both directions of that fix. + */ + +function user(id: string): CoreMessage { + return { + id, + role: "user", + contentType: "text", + text: `user-${id} ` + "x".repeat(200), + }; +} +function reasoning(id: string): CoreMessage { + return { + id, + role: "assistant", + contentType: "reasoning", + text: `think-${id} ` + "t".repeat(400), + }; +} +function text(id: string): CoreMessage { + return { + id, + role: "assistant", + contentType: "text", + text: `text-${id} ` + "x".repeat(200), + }; +} +function toolCall(id: string, callId: string): CoreMessage { + return { + id, + role: "assistant", + contentType: "tool-call", + toolName: "pwsh", + toolCallId: callId, + text: `call-${callId}`, + }; +} +function toolResult(id: string, callId: string): CoreMessage { + return { + id, + role: "user", + contentType: "tool-result", + toolName: "pwsh", + toolCallId: callId, + text: `result-${callId} ` + "r".repeat(200), + }; +} + +/** One strict-thinking turn: [reasoning, text, call, result] flanked by users. + * Raw ids map to refs m00001..m00007 after processTurn. */ +function buildMessages(): CoreMessage[] { + return [ + user("u1"), // m00001 + reasoning("r"), // m00002 + text("t"), // m00003 + toolCall("c", "ca"), // m00004 + toolResult("res", "ca"), // m00005 + user("u6"), // m00006 + user("u7"), // m00007 + ]; +} + +const config = (): Config => ({ + ...defaultConfig(150000), + preserveRecentMessages: 0, + preserveRecentTokens: 0, + compress: { + ...defaultConfig(150000).compress, + minCompressRange: 1, + minSummaryLength: 1, + }, +}); + +const countTokens = (t: string) => Math.ceil(t.length / 4); + +/** Seed the #564 reverse split: fold ONLY the call+result, leaving the turn's + * reasoning+text visible beside the new block. */ +async function seedReverseSplit() { + const core = createCore(); + const messages = buildMessages(); + const out = core.processTurn({ + messages, + state: createInitialState(), + config: config(), + tokenCount: countTokens, + }); + const seed = core.applyCompression({ + state: out.state, + messages, + config: config(), + ranges: [{ startRef: "m00004", endRef: "m00005", summary: "s".repeat(400) }], + }); + const block = seed.state.blocks.find((b) => b.active); + assert.ok(block, "seed fold must create a block"); + assert.ok(block.effectiveMessageIds.includes("c")); + assert.ok(block.effectiveMessageIds.includes("res")); + assert.ok( + !block.effectiveMessageIds.includes("r") && + !block.effectiveMessageIds.includes("t"), + `seed must cover only call+result, got ${JSON.stringify(block.effectiveMessageIds)}`, + ); + return { core, messages, state: seed.state }; +} + +describe("#5531 gate: a covered call cannot strand its turn's reasoning", () => { + it("legacy semantics WITHDRAW the stranded pair (the bug mechanism)", async () => { + const { messages, state } = await seedReverseSplit(); + // Without the invisible set, the covered call reads as kept-visible and + // the whole turn is withdrawn — this is exactly why the residue was + // permanent. Pinning the old verdict documents what the fix overrides. + const legacy = computeIntegrityWithdrawals( + messages, + new Set(["r", "t"]), + ).withdrawn; + assert.ok(legacy.has("r") && legacy.has("t"), "legacy gate strands the pair"); + }); + + it("with coverage as invisible, the stranded pair stays foldable", async () => { + const { messages, state } = await seedReverseSplit(); + const gate = computeIntegrityWithdrawals( + messages, + new Set(["r", "t"]), + undefined, + collectCoverage(state), + ).withdrawn; + assert.ok(!gate.has("r") && !gate.has("t"), JSON.stringify([...gate])); + }); +}); + +describe("#5531 recommend: stranded reasoning+text are advertised again", () => { + it("buildCompressibleRanges offers the stranded pair as compressible", async () => { + const { messages, state } = await seedReverseSplit(); + const ranges = buildCompressibleRanges(messages, state, config()); + const byRaw = state.messageRefs.byRaw; + const refOf = (raw: string) => byRaw[raw]!; + const inRange = (raw: string) => + ranges.compressible.some( + (r) => r.startRef <= refOf(raw) && refOf(raw) <= r.endRef, + ); + assert.ok(inRange("r"), "stranded reasoning must be advertised"); + assert.ok(inRange("t"), "stranded text must be advertised"); + }); + + it("projectUnit judges the stranded pair foldable (fatal === null)", async () => { + const { messages, state } = await seedReverseSplit(); + const byRaw = state.messageRefs.byRaw; + const projection = projectUnit({ + messages, + state, + config: config(), + startRef: byRaw["r"]!, + endRef: byRaw["t"]!, + preExistingCoverage: collectCoverage(state), + }); + assert.equal(projection.fatal, null, projection.fatal?.error.message); + assert.ok(projection.filteredIds.includes("r")); + assert.ok(projection.filteredIds.includes("t")); + }); + + it("closed loop: the advertised stranded range folds without a split error", async () => { + const { core, messages, state } = await seedReverseSplit(); + const byRaw = state.messageRefs.byRaw; + const ranges = buildCompressibleRanges(messages, state, config()); + const target = ranges.compressible.find( + (r) => r.startRef <= byRaw["r"]! && byRaw["r"]! <= r.endRef, + ); + assert.ok(target, "recommendation must contain the stranded pair"); + const res = core.applyCompression({ + state, + messages, + config: config(), + ranges: [ + { + startRef: target!.startRef, + endRef: target!.endRef, + summary: "reclaiming the stranded turn head", + }, + ], + }); + assert.deepEqual(res.result.errors, [], JSON.stringify(res.result.errors)); + assert.ok(res.result.blocksCreated >= 1, "a recovery block must be created"); + const block = res.state.blocks.find((b) => b.active && b !== state.blocks[0]); + assert.ok(block, "new block distinct from the seed block"); + assert.ok(block.effectiveMessageIds.includes("r")); + assert.ok(block.effectiveMessageIds.includes("t")); + }); +}); + +describe("#5531 negative: a VISIBLE call still guards its reasoning", () => { + // Same geometry, but nothing covers the call: protecting it (recent zone / + // protected tools) instead means folding r+t would orphan a LIVE call — the + // original #684 doctrine must still hold. + function freshSeeded() { + const core = createCore(); + const messages = buildMessages(); + const out = core.processTurn({ + messages, + state: createInitialState(), + config: config(), + tokenCount: countTokens, + }); + return { core, messages, state: out.state }; + } + + it("projectUnit still fatals 'split' when the call stays visible", () => { + const { messages, state } = freshSeeded(); + const byRaw = state.messageRefs.byRaw; + // Protecting the call+result is what makes them KEPT (not folded) while + // the range carries the reasoning — the geometry #684 exists to refuse. + const projection = projectUnit({ + messages, + state, + config: config(), + startRef: byRaw["r"]!, + endRef: byRaw["t"]!, + preExistingCoverage: new Set(), + protectedMessageIds: new Set([byRaw["c"]!, byRaw["res"]!]), + }); + assert.ok(projection.fatal, "visible call must keep its reasoning"); + assert.equal(projection.fatal!.kind, "split"); + assert.match(projection.fatal!.error.message, /split 1 turn/); + }); + + it("buildCompressibleRanges does NOT advertise the pair while the call is visible", () => { + const { messages, state } = freshSeeded(); + const byRaw = state.messageRefs.byRaw; + // With NO carve-out the whole turn folds atomically (legitimate), so the + // suppression case is a zone brush: the call+result sit in the protected + // zone (visible, kept) while the range would carry the reasoning. + const zoneRefs = new Set([byRaw["c"]!, byRaw["res"]!]); + const ranges = buildCompressibleRanges(messages, state, config(), zoneRefs); + const inRange = (raw: string) => + ranges.compressible.some( + (r) => r.startRef <= byRaw[raw]! && byRaw[raw]! <= r.endRef, + ); + assert.ok(!inRange("r"), "orphaned-reasoning fold must stay suppressed"); + assert.ok(!inRange("t")); + }); + + it("applyCompression still refuses the split with the canonical error", () => { + const { core, messages, state } = freshSeeded(); + const byRaw = state.messageRefs.byRaw; + const protectedRefs = new Set([byRaw["c"]!, byRaw["res"]!]); + const res = core.applyCompression({ + state, + messages, + config: config(), + protectedMessageIds: protectedRefs, + ranges: [{ startRef: "m00002", endRef: "m00003", summary: "s".repeat(400) }], + }); + assert.equal(res.result.blocksCreated, 0); + assert.equal(res.result.errors.length, 1); + assert.match(res.result.errors[0]!, /split 1 turn/); + }); +}); diff --git a/package-lock.json b/package-lock.json index 561cacb77b..73e28ff4ca 100644 --- a/package-lock.json +++ b/package-lock.json @@ -50,7 +50,7 @@ }, "kernel": { "name": "acp-kernel", - "version": "0.0.177", + "version": "0.0.178", "dev": true, "license": "MIT", "devDependencies": {