From 4af1753153634287d2ba549d32e906a608c1307d Mon Sep 17 00:00:00 2001 From: awork Date: Wed, 21 Oct 2026 11:20:18 +0800 Subject: [PATCH] fix: refuse unservable compress ranges on an empty view instead of tombstoning live refs (#3859) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The no-visible-context guard (#3126) exempted any request whose range SPAN-OVERLAPPED an active block span. The kernel's dead-ref test (danglingMessageRefs) judges per BOUNDARY ref, not per span overlap: a range can overlap one block while its boundary refs sit in gaps between blocks. On an evicted/restarted session (view=0, refs mapped) such requests passed the gate, reached applyCompression with an empty message list, and the kernel recorded those still-live refs as DEAD — committed by the host even when zero blocks were created. The gate now mirrors the kernel's own test: on an empty view with mapped refs it yields to the kernel only when every requested range's message-ref boundaries are point-covered by some active block's effectiveMessageIds (block-ref boundaries carry no raw id and can never be tombstoned). That set equals exactly the shapes the kernel can serve from state alone (#403/#2748 restoredInline refold, already-compressed warnings); everything else is refused up front with zero state writes, retryable once the next model request restores the view. Tests: flipped issue3126 test (e) (block-touching request on empty view is now refused, no dead refs) and added tests/issue3859-no-visible-context-overlap.test.ts (multi-range gap-boundary shape refused whole-batch; mixed batch refused; pure b-ref distillation still yields to the kernel with its honest already-compressed warning; non-empty-view negative row). refold-wiring suite (state-alone refold landing, #2748 net-shrink, negative control) stays green. --- src/stream.ts | 54 ++++--- tests/issue3126-no-visible-context.test.ts | 16 +- ...sue3859-no-visible-context-overlap.test.ts | 140 ++++++++++++++++++ 3 files changed, 186 insertions(+), 24 deletions(-) create mode 100644 tests/issue3859-no-visible-context-overlap.test.ts diff --git a/src/stream.ts b/src/stream.ts index e2290ee6f0..df2b13753f 100644 --- a/src/stream.ts +++ b/src/stream.ts @@ -1,5 +1,5 @@ import { createHash, randomBytes } from "node:crypto"; -import { activeBlockSpans, collectBlockContent, countMessageTokens, defaultCountTokens, formatRanges, hideConsumedCompressCalls, storeCoveredOriginals, viableRanges, type CompressionCore, type Config, type CoreMessage, type CompressionState, type NudgeDecision, type CompressParseDiagnostics } from "acp-kernel"; +import { collectBlockContent, countMessageTokens, defaultCountTokens, formatRanges, hideConsumedCompressCalls, indexToRef, parseBoundary, storeCoveredOriginals, viableRanges, type CompressionCore, type Config, type CoreMessage, type CompressionState, type NudgeDecision, type CompressParseDiagnostics } from "acp-kernel"; import { excludeDeadRefRanges, handleAcpStatus, liveCompressionRanges } from "./acp-status.js"; import { handleAcpCache, recordCacheFoldsFromBlocks } from "./cache-ledger.js"; import { type Session, cacheBlockContent, markDirty, residentPressure, scrubTurnSeparatorIds, statusInputBaseline } from "./session.js"; @@ -607,7 +607,21 @@ const DEGENERATE_FOLD_COVERAGE = 0.8; * DEAD (the host commits applyCompression's cloned state even when zero blocks * were created), and the receipt blames "content-changed" — none of which is * true: the refs are intact and the view self-heals on the next model request - * (rememberPluginMessages after forward). Null = no refusal, proceed. */ + * (rememberPluginMessages after forward). + * + * #3859: WHICH empty-view calls may still be handed to the kernel is decided + * by the kernel's own dead-ref test, not by span overlap. danglingMessageRefs + * records a requested ref dead exactly when that BOUNDARY ref is known to this + * session but not visible and not inside any active block's effectiveMessageIds + * — so a range that overlaps one block while its boundary refs sit in gaps + * between blocks passed the old overlap gate and got its live refs tombstoned. + * The gate now yields only when every requested range's message-ref boundaries + * are point-covered by some active block (block-ref boundaries carry no raw id + * and can never be tombstoned). That is also exactly the set of shapes the + * kernel could ever serve from state alone (#403/#2748 restoredInline refold, + * already-compressed warnings): with a truly empty view nothing else anchors, + * so every uncovered-boundary shape can only fail or corrupt. + * Null = no refusal, proceed. */ export function noVisibleContextRefusal( ctx: RewriteCtx, ranges: ParsedRange[], @@ -616,23 +630,25 @@ export function noVisibleContextRefusal( const view = ctx.compressMessages ?? ctx.messages; const mappedRefs = Object.keys(ctx.session.state.messageRefs?.byRef ?? {}).length; if (view.length > 0 || mappedRefs === 0) return null; - // #398/#403 in-place refold works FROM state alone: after an inline - // decompress the originals leave the wire view (proxy mode) yet a - // re-compress of the restored span must still land on its block. The - // refusal therefore applies only when NO active block touches ANY - // requested ref — then nothing in the call can be served without live - // messages and every retry fails identically. Any overlap hands the - // decision back to the kernel, which keeps its own honest diagnostics - // for those shapes (refold / already-compressed / no-net-shrink). - const spans = activeBlockSpans(ctx.session.state); - const touchable = ranges.some((r) => { - const lo = refNum(r.startRef); - const hi = refNum(r.endRef); - if (lo === 0 || hi === 0) return true; - const a = Math.min(lo, hi), b = Math.max(lo, hi); - return spans.some((s) => refNum(s.startRef) <= b && refNum(s.endRef) >= a); - }); - if (touchable) return null; + // #3859: yield to the kernel only when its own dead-ref test cannot fire on + // anything here — i.e. every requested message-ref boundary is point-covered + // by an active block (mirrors danglingMessageRefs: same byRef lookup, same + // effectiveMessageIds coverage; block-ref boundaries carry no raw id and are + // skipped there too). Any uncovered boundary would be recorded DEAD on this + // empty view while still live on disk. A boundary with no known rawId is + // yielded deliberately: the kernel then only re-records ids already dead + // (#3302, idempotent) or skips unknown ones — it can never corrupt a live ref. + const byRef = ctx.session.state.messageRefs?.byRef ?? {}; + const coveredRawIds = new Set( + ctx.session.state.blocks.filter((b) => b.active).flatMap((b) => b.effectiveMessageIds), + ); + const unvouchable = ranges.some((r) => [r.startRef, r.endRef].some((ref) => { + const parsed = parseBoundary(ref); + if (!parsed || parsed.kind !== "message") return false; + const rawId = byRef[parsed.raw] ?? byRef[indexToRef(parsed.numericId)]; + return Boolean(rawId) && !coveredRawIds.has(rawId); + })); + if (!unvouchable) return null; const detail = ranges.map((r) => `${r.startRef}–${r.endRef}`).join(", "); // #3918: the "self-heals on the next model request" promise holds ONLY when // the reloaded session survives between requests. When this session's diff --git a/tests/issue3126-no-visible-context.test.ts b/tests/issue3126-no-visible-context.test.ts index eb971bab82..3c2374a4e1 100644 --- a/tests/issue3126-no-visible-context.test.ts +++ b/tests/issue3126-no-visible-context.test.ts @@ -103,17 +103,23 @@ test("#3126: three consecutive refusals arm the breaker with the honest cause", assert.equal(streak.cause, "no-visible-context", "honest cause persisted on the streak"); }); -test("#3126: a block-touching request passes the gate to the kernel (refold path intact)", () => { +test("#3859: a block-touching request on an EMPTY view is refused too — no dead refs", () => { const ctx = makeCtx({ messages: MESSAGES, refs: true }); const first = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00020", summary: "z".repeat(60) }] }), ctx); assert.equal(first.outcome, "applied", `setup fold lands: ${first.text}`); - // Proxy-shaped next round: the originals left the view; SPEC (m00010–m00040) - // partially overlaps the fresh block m00001–m00020, so the gate must yield - // and let the kernel apply its own per-range diagnostics. + // Evicted/restarted next round: the view is empty while SPEC (m00010–m00040) + // partially overlaps the fresh block m00001–m00020. The pre-#3859 span-overlap + // exemption handed this to the kernel, which could anchor neither endpoint on + // an empty view and tombstoned m00040 (not point-covered by any block) as + // DEAD — live refs killed by memory eviction. Now the gate refuses + // unconditionally and writes nothing. ctx.messages = []; ctx.compressMessages = []; const res = applyRanges(SPEC(), ctx); - assert.notEqual(res.reason, "gate:no-visible-context", `block-touching request must reach the kernel: ${res.text}`); + assert.equal(res.outcome, "refused"); + assert.equal(res.reason, "gate:no-visible-context", `block-touching request must be refused on an empty view: ${res.text}`); + assert.equal(ctx.session.state.deadRefs, undefined, "no deadRefs recorded (the #3859 corruption)"); + assert.equal(ctx.session.state.blocks.length, 1, "the setup block is untouched"); }); test("#3126: the external-summary lane hits the same gate before any network call", async () => { diff --git a/tests/issue3859-no-visible-context-overlap.test.ts b/tests/issue3859-no-visible-context-overlap.test.ts new file mode 100644 index 0000000000..b3b9577f71 --- /dev/null +++ b/tests/issue3859-no-visible-context-overlap.test.ts @@ -0,0 +1,140 @@ +// #3859: the no-visible-context gate (#3126) had a span-overlap exemption — +// any requested range touching an active block was handed to the kernel even +// when the message view was empty. The kernel cannot anchor ANY endpoint on an +// empty view (visibleBlockAnchor/activeOwnerAnchor need a visible +// representation), so every such call failed with "cannot be anchored" AND its +// dead-ref pass tombstoned each boundary ref not point-covered by an active +// block — live refs recorded DEAD while the session was merely evicted from +// the resident pool. Span overlap ≠ per-ref coverage, which is exactly how +// m00774–m00860-style ranges (overlapping some of 15 blocks, boundary refs in +// the gaps) escaped the gate. On an empty view the gate now yields to the +// kernel only when EVERY message-ref boundary of every range is point-covered +// by an active block (the #403/#2748 state-alone refolds and their +// diagnostics); anything else is refused up front, zero state writes. These +// tests pin: the reporter's multi-range shape → refused; a mixed batch with +// ONE uncovered range → refused as a whole; pure block-ref calls (never +// tombstoned) → still reach the kernel's own diagnostics; non-empty views +// never hit the gate at all. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import type { CoreMessage } from "acp-kernel"; +import { createCore, createInitialState, defaultConfig, assignRefs, emptyRefMap } from "acp-kernel"; +import { parseCompressInput } from "../src/compress-tool.ts"; +import { applyRanges, type RewriteCtx } from "../src/stream.ts"; +import { resolveDecompress } from "../src/decompress-shared.ts"; + +type Ctx = Omit & { log: (m: string) => void; logs: string[] }; + +const MESSAGES = Array.from({ length: 80 }, (_, i) => ({ + id: `raw${i}`, + role: (i % 2 ? "assistant" : "user") as "assistant" | "user", + text: "lorem ipsum dolor sit amet conseq ".repeat(8) + i, +})) as unknown as CoreMessage[]; + +function makeCtx(): Ctx { + const logs: string[] = []; + const ctx: Ctx = { + core: createCore(), + config: defaultConfig(200000, { preserveRecentMessages: 5, preserveRecentTokens: 0 }), + // View present while the setup folds run; each eviction-scenario test + // clears it afterwards (state survives, the message bytes do not). + messages: MESSAGES, + session: { + id: "issue3859-test", + meta: {}, + stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, compressCreditTokens: 0, contextTokens: 0, retrieveCalls: 0, retrieveHits: 0, retrieveMisses: 0, storedBytes: 0, storeBytesSaved: 0, rangeRestores: 0 }, + metadata: {}, + state: createInitialState(), + createdAt: Date.now(), + lastSeen: Date.now(), + blockContents: new Map(), + inFlight: 0, + persisted: false, + pendingRetrievals: [], + }, + log: (m: string) => { logs.push(m); }, + logs, + }; + ctx.session.state.messageRefs = assignRefs(MESSAGES, { existing: emptyRefMap(), nextIndex: 0 }).map; + return ctx; +} + +const fold = (ctx: Ctx, startId: string, endId: string, summary: string) => + applyRanges(parseCompressInput({ content: [{ startId, endId, summary }] }), ctx); + +test("#3859: multi-range request overlapping active blocks is refused on an empty view, nothing recorded dead", () => { + const ctx = makeCtx(); + // Three active blocks with gaps between them (the reporter's 15-block + // shape, scaled down): m00001–m00020, m00030–m00050, m00060–m00078. + assert.equal(fold(ctx, "m00001", "m00020", "z".repeat(60)).outcome, "applied"); + assert.equal(fold(ctx, "m00030", "m00050", "z".repeat(60)).outcome, "applied"); + assert.equal(fold(ctx, "m00060", "m00078", "z".repeat(60)).outcome, "applied"); + const blocksBefore = ctx.session.state.blocks.length; + // Eviction: the view drops out, refs survive. + ctx.messages = []; + ctx.compressMessages = []; + // The reporter's pattern: ranges that partially overlap blocks with their + // boundary refs in the gaps (m00028/m00058 sit in the inter-block gaps, + // m00079–m00080 past the last block — none of them is covered by ANY + // block), plus a second partial overlap. Pre-fix, the first two made the + // whole call touchable and the kernel tombstoned those four boundary refs. + const res = applyRanges(parseCompressInput({ content: [ + { startId: "m00015", endId: "m00028", summary: "y".repeat(100) }, + { startId: "m00045", endId: "m00058", summary: "y".repeat(100) }, + { startId: "m00079", endId: "m00080", summary: "y".repeat(100) }, + ] }), ctx); + assert.equal(res.outcome, "refused"); + assert.equal(res.reason, "gate:no-visible-context", `whole batch refused as one unit: ${res.text}`); + assert.ok(res.text.includes("no visible context"), `honest cause: ${res.text}`); + assert.ok(res.text.includes("Do NOT retry now"), `retry veto until next model request: ${res.text}`); + assert.equal(res.blocksCreated, 0); + assert.equal(ctx.session.state.deadRefs, undefined, "no refs recorded dead — the #3859 corruption"); + assert.equal(ctx.session.state.blocks.length, blocksBefore, "existing blocks untouched"); +}); + +test("#3859: a batch mixing a state-alone refold with an uncovered range is refused as a whole", () => { + const ctx = makeCtx(); + assert.equal(fold(ctx, "m00001", "m00020", "z".repeat(60)).outcome, "applied"); + const blockId = ctx.session.state.blocks[0]!.blockId; + resolveDecompress({ blockId }, ctx as never); // flips restoredInline while the view still exists + ctx.messages = []; + ctx.compressMessages = []; + // First range alone would be vouchable (restoredInline refold from state); + // the second has boundary refs past the last block → the WHOLE batch is + // refused instead of half-landing while tombstoning the second half. + const res = applyRanges(parseCompressInput({ content: [ + { startId: "m00001", endId: "m00020", summary: "refolded" }, + { startId: "m00079", endId: "m00080", summary: "y".repeat(100) }, + ] }), ctx); + assert.equal(res.outcome, "refused"); + assert.equal(res.reason, "gate:no-visible-context", `batch-level refusal: ${res.text}`); + assert.equal(ctx.session.state.deadRefs, undefined, "no refs recorded dead"); + assert.equal(ctx.session.state.blocks.find((b) => b.blockId === blockId)?.summary, "z".repeat(60), "the vouchable half waits for the next turn, not a partial landing"); +}); + +test("#3859: a pure block-ref distillation call on an empty view still reaches the kernel — no dead refs", () => { + const ctx = makeCtx(); + assert.equal(fold(ctx, "m00001", "m00020", "z".repeat(60)).outcome, "applied"); + assert.equal(fold(ctx, "m00030", "m00050", "z".repeat(60)).outcome, "applied"); + ctx.messages = []; + ctx.compressMessages = []; + // Block-ref boundaries are never examined by the kernel's dead-ref pass, + // so this shape stays vouchable and lands on the kernel's own + // already-compressed diagnostic (pre-fix behavior preserved) rather than + // a new DEAD record or a vague refusal. + const res = applyRanges(parseCompressInput({ content: [{ startId: "b1", endId: "b2", summary: "tier-2 ".repeat(20) }] }), ctx); + assert.notEqual(res.reason, "gate:no-visible-context", `block-ref shape must pass the gate: ${res.text}`); + assert.ok(res.text.includes("already compressed"), `kernel diagnostic preserved: ${res.text}`); + assert.equal(ctx.session.state.deadRefs, undefined); + assert.equal(ctx.session.state.blocks.length, 2, "no blocks created or destroyed"); +}); + +test("#3859 negative row: the same block-overlapping ranges on a NON-EMPTY view never hit the gate", () => { + const ctx = makeCtx(); // view stays present for the whole test + assert.equal(fold(ctx, "m00001", "m00020", "z".repeat(60)).outcome, "applied"); + assert.equal(fold(ctx, "m00030", "m00050", "z".repeat(60)).outcome, "applied"); + // View still holds the conversation (proxy-shaped round): the gate must + // stay silent and the kernel applies its own per-range diagnostics. + const res = fold(ctx, "m00015", "m00028", "y".repeat(100)); + assert.notEqual(res.reason, "gate:no-visible-context", `non-empty view must bypass the gate: ${res.text}`); +});