Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 35 additions & 19 deletions src/stream.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { createHash, randomBytes } from "node:crypto";
import { activeBlockSpans, collectBlockContent, countMessageTokens, defaultCountTokens, formatRanges, hideConsumedCompressCalls, storeCoveredOriginals, viableRanges, type CompressionCore, type Config, type CoreMessage, type CompressionState, type NudgeDecision, type CompressParseDiagnostics } from "acp-kernel";
import { collectBlockContent, countMessageTokens, defaultCountTokens, formatRanges, hideConsumedCompressCalls, indexToRef, parseBoundary, storeCoveredOriginals, viableRanges, type CompressionCore, type Config, type CoreMessage, type CompressionState, type NudgeDecision, type CompressParseDiagnostics } from "acp-kernel";
import { excludeDeadRefRanges, handleAcpStatus, liveCompressionRanges } from "./acp-status.js";
import { handleAcpCache, recordCacheFoldsFromBlocks } from "./cache-ledger.js";
import { type Session, cacheBlockContent, markDirty, residentPressure, scrubTurnSeparatorIds, statusInputBaseline } from "./session.js";
Expand Down Expand Up @@ -607,7 +607,21 @@ const DEGENERATE_FOLD_COVERAGE = 0.8;
* DEAD (the host commits applyCompression's cloned state even when zero blocks
* were created), and the receipt blames "content-changed" — none of which is
* true: the refs are intact and the view self-heals on the next model request
* (rememberPluginMessages after forward). Null = no refusal, proceed. */
* (rememberPluginMessages after forward).
*
* #3859: WHICH empty-view calls may still be handed to the kernel is decided
* by the kernel's own dead-ref test, not by span overlap. danglingMessageRefs
* records a requested ref dead exactly when that BOUNDARY ref is known to this
* session but not visible and not inside any active block's effectiveMessageIds
* — so a range that overlaps one block while its boundary refs sit in gaps
* between blocks passed the old overlap gate and got its live refs tombstoned.
* The gate now yields only when every requested range's message-ref boundaries
* are point-covered by some active block (block-ref boundaries carry no raw id
* and can never be tombstoned). That is also exactly the set of shapes the
* kernel could ever serve from state alone (#403/#2748 restoredInline refold,
* already-compressed warnings): with a truly empty view nothing else anchors,
* so every uncovered-boundary shape can only fail or corrupt.
* Null = no refusal, proceed. */
export function noVisibleContextRefusal(
ctx: RewriteCtx,
ranges: ParsedRange[],
Expand All @@ -616,23 +630,25 @@ export function noVisibleContextRefusal(
const view = ctx.compressMessages ?? ctx.messages;
const mappedRefs = Object.keys(ctx.session.state.messageRefs?.byRef ?? {}).length;
if (view.length > 0 || mappedRefs === 0) return null;
// #398/#403 in-place refold works FROM state alone: after an inline
// decompress the originals leave the wire view (proxy mode) yet a
// re-compress of the restored span must still land on its block. The
// refusal therefore applies only when NO active block touches ANY
// requested ref — then nothing in the call can be served without live
// messages and every retry fails identically. Any overlap hands the
// decision back to the kernel, which keeps its own honest diagnostics
// for those shapes (refold / already-compressed / no-net-shrink).
const spans = activeBlockSpans(ctx.session.state);
const touchable = ranges.some((r) => {
const lo = refNum(r.startRef);
const hi = refNum(r.endRef);
if (lo === 0 || hi === 0) return true;
const a = Math.min(lo, hi), b = Math.max(lo, hi);
return spans.some((s) => refNum(s.startRef) <= b && refNum(s.endRef) >= a);
});
if (touchable) return null;
// #3859: yield to the kernel only when its own dead-ref test cannot fire on
// anything here — i.e. every requested message-ref boundary is point-covered
// by an active block (mirrors danglingMessageRefs: same byRef lookup, same
// effectiveMessageIds coverage; block-ref boundaries carry no raw id and are
// skipped there too). Any uncovered boundary would be recorded DEAD on this
// empty view while still live on disk. A boundary with no known rawId is
// yielded deliberately: the kernel then only re-records ids already dead
// (#3302, idempotent) or skips unknown ones — it can never corrupt a live ref.
const byRef = ctx.session.state.messageRefs?.byRef ?? {};
const coveredRawIds = new Set(
ctx.session.state.blocks.filter((b) => b.active).flatMap((b) => b.effectiveMessageIds),
);
const unvouchable = ranges.some((r) => [r.startRef, r.endRef].some((ref) => {
const parsed = parseBoundary(ref);
if (!parsed || parsed.kind !== "message") return false;
const rawId = byRef[parsed.raw] ?? byRef[indexToRef(parsed.numericId)];
return Boolean(rawId) && !coveredRawIds.has(rawId);
}));
if (!unvouchable) return null;
const detail = ranges.map((r) => `${r.startRef}–${r.endRef}`).join(", ");
// #3918: the "self-heals on the next model request" promise holds ONLY when
// the reloaded session survives between requests. When this session's
Expand Down
16 changes: 11 additions & 5 deletions tests/issue3126-no-visible-context.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -103,17 +103,23 @@ test("#3126: three consecutive refusals arm the breaker with the honest cause",
assert.equal(streak.cause, "no-visible-context", "honest cause persisted on the streak");
});

test("#3126: a block-touching request passes the gate to the kernel (refold path intact)", () => {
test("#3859: a block-touching request on an EMPTY view is refused too — no dead refs", () => {
const ctx = makeCtx({ messages: MESSAGES, refs: true });
const first = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00020", summary: "z".repeat(60) }] }), ctx);
assert.equal(first.outcome, "applied", `setup fold lands: ${first.text}`);
// Proxy-shaped next round: the originals left the view; SPEC (m00010–m00040)
// partially overlaps the fresh block m00001–m00020, so the gate must yield
// and let the kernel apply its own per-range diagnostics.
// Evicted/restarted next round: the view is empty while SPEC (m00010–m00040)
// partially overlaps the fresh block m00001–m00020. The pre-#3859 span-overlap
// exemption handed this to the kernel, which could anchor neither endpoint on
// an empty view and tombstoned m00040 (not point-covered by any block) as
// DEAD — live refs killed by memory eviction. Now the gate refuses
// unconditionally and writes nothing.
ctx.messages = [];
ctx.compressMessages = [];
const res = applyRanges(SPEC(), ctx);
assert.notEqual(res.reason, "gate:no-visible-context", `block-touching request must reach the kernel: ${res.text}`);
assert.equal(res.outcome, "refused");
assert.equal(res.reason, "gate:no-visible-context", `block-touching request must be refused on an empty view: ${res.text}`);
assert.equal(ctx.session.state.deadRefs, undefined, "no deadRefs recorded (the #3859 corruption)");
assert.equal(ctx.session.state.blocks.length, 1, "the setup block is untouched");
});

test("#3126: the external-summary lane hits the same gate before any network call", async () => {
Expand Down
140 changes: 140 additions & 0 deletions tests/issue3859-no-visible-context-overlap.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
// #3859: the no-visible-context gate (#3126) had a span-overlap exemption —
// any requested range touching an active block was handed to the kernel even
// when the message view was empty. The kernel cannot anchor ANY endpoint on an
// empty view (visibleBlockAnchor/activeOwnerAnchor need a visible
// representation), so every such call failed with "cannot be anchored" AND its
// dead-ref pass tombstoned each boundary ref not point-covered by an active
// block — live refs recorded DEAD while the session was merely evicted from
// the resident pool. Span overlap ≠ per-ref coverage, which is exactly how
// m00774–m00860-style ranges (overlapping some of 15 blocks, boundary refs in
// the gaps) escaped the gate. On an empty view the gate now yields to the
// kernel only when EVERY message-ref boundary of every range is point-covered
// by an active block (the #403/#2748 state-alone refolds and their
// diagnostics); anything else is refused up front, zero state writes. These
// tests pin: the reporter's multi-range shape → refused; a mixed batch with
// ONE uncovered range → refused as a whole; pure block-ref calls (never
// tombstoned) → still reach the kernel's own diagnostics; non-empty views
// never hit the gate at all.
import { test } from "node:test";
import assert from "node:assert/strict";
import type { CoreMessage } from "acp-kernel";
import { createCore, createInitialState, defaultConfig, assignRefs, emptyRefMap } from "acp-kernel";
import { parseCompressInput } from "../src/compress-tool.ts";
import { applyRanges, type RewriteCtx } from "../src/stream.ts";
import { resolveDecompress } from "../src/decompress-shared.ts";

type Ctx = Omit<RewriteCtx, "log"> & { log: (m: string) => void; logs: string[] };

const MESSAGES = Array.from({ length: 80 }, (_, i) => ({
id: `raw${i}`,
role: (i % 2 ? "assistant" : "user") as "assistant" | "user",
text: "lorem ipsum dolor sit amet conseq ".repeat(8) + i,
})) as unknown as CoreMessage[];

function makeCtx(): Ctx {
const logs: string[] = [];
const ctx: Ctx = {
core: createCore(),
config: defaultConfig(200000, { preserveRecentMessages: 5, preserveRecentTokens: 0 }),
// View present while the setup folds run; each eviction-scenario test
// clears it afterwards (state survives, the message bytes do not).
messages: MESSAGES,
session: {
id: "issue3859-test",
meta: {},
stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, compressCreditTokens: 0, contextTokens: 0, retrieveCalls: 0, retrieveHits: 0, retrieveMisses: 0, storedBytes: 0, storeBytesSaved: 0, rangeRestores: 0 },
metadata: {},
state: createInitialState(),
createdAt: Date.now(),
lastSeen: Date.now(),
blockContents: new Map(),
inFlight: 0,
persisted: false,
pendingRetrievals: [],
},
log: (m: string) => { logs.push(m); },
logs,
};
ctx.session.state.messageRefs = assignRefs(MESSAGES, { existing: emptyRefMap(), nextIndex: 0 }).map;
return ctx;
}

const fold = (ctx: Ctx, startId: string, endId: string, summary: string) =>
applyRanges(parseCompressInput({ content: [{ startId, endId, summary }] }), ctx);

test("#3859: multi-range request overlapping active blocks is refused on an empty view, nothing recorded dead", () => {
const ctx = makeCtx();
// Three active blocks with gaps between them (the reporter's 15-block
// shape, scaled down): m00001–m00020, m00030–m00050, m00060–m00078.
assert.equal(fold(ctx, "m00001", "m00020", "z".repeat(60)).outcome, "applied");
assert.equal(fold(ctx, "m00030", "m00050", "z".repeat(60)).outcome, "applied");
assert.equal(fold(ctx, "m00060", "m00078", "z".repeat(60)).outcome, "applied");
const blocksBefore = ctx.session.state.blocks.length;
// Eviction: the view drops out, refs survive.
ctx.messages = [];
ctx.compressMessages = [];
// The reporter's pattern: ranges that partially overlap blocks with their
// boundary refs in the gaps (m00028/m00058 sit in the inter-block gaps,
// m00079–m00080 past the last block — none of them is covered by ANY
// block), plus a second partial overlap. Pre-fix, the first two made the
// whole call touchable and the kernel tombstoned those four boundary refs.
const res = applyRanges(parseCompressInput({ content: [
{ startId: "m00015", endId: "m00028", summary: "y".repeat(100) },
{ startId: "m00045", endId: "m00058", summary: "y".repeat(100) },
{ startId: "m00079", endId: "m00080", summary: "y".repeat(100) },
] }), ctx);
assert.equal(res.outcome, "refused");
assert.equal(res.reason, "gate:no-visible-context", `whole batch refused as one unit: ${res.text}`);
assert.ok(res.text.includes("no visible context"), `honest cause: ${res.text}`);
assert.ok(res.text.includes("Do NOT retry now"), `retry veto until next model request: ${res.text}`);
assert.equal(res.blocksCreated, 0);
assert.equal(ctx.session.state.deadRefs, undefined, "no refs recorded dead — the #3859 corruption");
assert.equal(ctx.session.state.blocks.length, blocksBefore, "existing blocks untouched");
});

test("#3859: a batch mixing a state-alone refold with an uncovered range is refused as a whole", () => {
const ctx = makeCtx();
assert.equal(fold(ctx, "m00001", "m00020", "z".repeat(60)).outcome, "applied");
const blockId = ctx.session.state.blocks[0]!.blockId;
resolveDecompress({ blockId }, ctx as never); // flips restoredInline while the view still exists
ctx.messages = [];
ctx.compressMessages = [];
// First range alone would be vouchable (restoredInline refold from state);
// the second has boundary refs past the last block → the WHOLE batch is
// refused instead of half-landing while tombstoning the second half.
const res = applyRanges(parseCompressInput({ content: [
{ startId: "m00001", endId: "m00020", summary: "refolded" },
{ startId: "m00079", endId: "m00080", summary: "y".repeat(100) },
] }), ctx);
assert.equal(res.outcome, "refused");
assert.equal(res.reason, "gate:no-visible-context", `batch-level refusal: ${res.text}`);
assert.equal(ctx.session.state.deadRefs, undefined, "no refs recorded dead");
assert.equal(ctx.session.state.blocks.find((b) => b.blockId === blockId)?.summary, "z".repeat(60), "the vouchable half waits for the next turn, not a partial landing");
});

test("#3859: a pure block-ref distillation call on an empty view still reaches the kernel — no dead refs", () => {
const ctx = makeCtx();
assert.equal(fold(ctx, "m00001", "m00020", "z".repeat(60)).outcome, "applied");
assert.equal(fold(ctx, "m00030", "m00050", "z".repeat(60)).outcome, "applied");
ctx.messages = [];
ctx.compressMessages = [];
// Block-ref boundaries are never examined by the kernel's dead-ref pass,
// so this shape stays vouchable and lands on the kernel's own
// already-compressed diagnostic (pre-fix behavior preserved) rather than
// a new DEAD record or a vague refusal.
const res = applyRanges(parseCompressInput({ content: [{ startId: "b1", endId: "b2", summary: "tier-2 ".repeat(20) }] }), ctx);
assert.notEqual(res.reason, "gate:no-visible-context", `block-ref shape must pass the gate: ${res.text}`);
assert.ok(res.text.includes("already compressed"), `kernel diagnostic preserved: ${res.text}`);
assert.equal(ctx.session.state.deadRefs, undefined);
assert.equal(ctx.session.state.blocks.length, 2, "no blocks created or destroyed");
});

test("#3859 negative row: the same block-overlapping ranges on a NON-EMPTY view never hit the gate", () => {
const ctx = makeCtx(); // view stays present for the whole test
assert.equal(fold(ctx, "m00001", "m00020", "z".repeat(60)).outcome, "applied");
assert.equal(fold(ctx, "m00030", "m00050", "z".repeat(60)).outcome, "applied");
// View still holds the conversation (proxy-shaped round): the gate must
// stay silent and the kernel applies its own per-range diagnostics.
const res = fold(ctx, "m00015", "m00028", "y".repeat(100));
assert.notEqual(res.reason, "gate:no-visible-context", `non-empty view must bypass the gate: ${res.text}`);
});
Loading