diff --git a/kernel/package-lock.json b/kernel/package-lock.json index d8b0eeff59..7166f46e2b 100644 --- a/kernel/package-lock.json +++ b/kernel/package-lock.json @@ -1,12 +1,12 @@ { "name": "acp-kernel", - "version": "0.0.203", + "version": "0.0.204", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "acp-kernel", - "version": "0.0.203", + "version": "0.0.204", "license": "MIT", "devDependencies": { "@types/node": "^22.0.0", diff --git a/kernel/package.json b/kernel/package.json index 8b2ae8252b..a3990bcb76 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.203", + "version": "0.0.204", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/report.ts b/kernel/src/report.ts index 9e50101b24..9427f4228a 100644 --- a/kernel/src/report.ts +++ b/kernel/src/report.ts @@ -511,6 +511,8 @@ function renderUncompressedRanges( return "\u26a0 not foldable: protected zone only"; if (proj.fatal?.kind === "split") return `\u26a0 not foldable: would split ${splitDescOf(proj)}`; + if (proj.fatal?.kind === "window") + return "\u26a0 not foldable: would merge block(s) beyond the requested ref window"; if (proj.filteredIds.length === 0 && proj.consumedBlockIds.length === 0) return "\u26a0 not foldable: nothing new to fold"; return null; diff --git a/kernel/src/unit-projection.ts b/kernel/src/unit-projection.ts index 4f56028e96..599fb76246 100644 --- a/kernel/src/unit-projection.ts +++ b/kernel/src/unit-projection.ts @@ -8,7 +8,9 @@ import { buildIndexByMessageId, resolveBoundaries, blockVisibleInRange, + parseBoundary, } from "./boundaries.js"; +import { indexToRef, refToIndex } from "./refs.js"; import { isSummaryMessageId } from "./prune.js"; import { collectFailedCompressCallIds, @@ -91,7 +93,7 @@ export interface UnitProjectionInput { } export interface UnitProjectionFatal { - kind: "zone" | "split"; + kind: "zone" | "split" | "window"; error: Error; } @@ -265,6 +267,95 @@ export function projectUnit(input: UnitProjectionInput): UnitProjection { } } + // #5830: the supplied summary describes exactly the requested ref window + // [startRef..endRef]. A plain message-ref range that would CONSUME an active + // block whose coverage extends outside that window replaces that block's + // wider-span summary with one that never saw part of it — silent + // active-summary coverage loss (incident: one tail-range summary left + // standing over a merged span back to the conversation start). Block- + // boundary merges (bN..bM) are exempt by contract — they exist to absorb + // the boundary blocks' coverage and the caller named the blocks. Pure + // resubmits (no new direct messages) are exempt too: they create no block, + // so there is no replacement to guard against (#4521 no-op class). + // + // #5531: pair/reasoning boundary widening is the fold's OWN defined scope — + // the widened cells ride under the same summary by design, so when both + // endpoints resolved IN PLACE (the requested message itself still occupies + // its slot), containment is judged against the WIDENED span, not the + // literal window. What must stay fatal is the #5830 producer itself: an + // endpoint that RESOLVED ELSEWHERE (stale inherited ref snapping to a + // distant inheriting-block anchor) sweeps territory the summary never + // described regardless of widening — hold the line at the requested + // window then. Coverage ids carrying NO ref entry (synthetic outbound-only + // markers, e.g. #2627 turn separators that legacy polluted views registered + // into block coverage) describe no client-side real-history content, so a + // replaced summary cannot lose anything they stand for — they ride along. + // A block whose ENTIRE coverage is unmappable still fails closed + // (containment unprovable). + if (!isBlockBoundary && consumedBlockIds.length > 0 && filteredIds.length > 0) { + const startBoundary = parseBoundary(input.startRef); + const endBoundary = parseBoundary(input.endRef); + if (startBoundary?.kind === "message" && endBoundary?.kind === "message") { + const windowLo = Math.min(startBoundary.numericId, endBoundary.numericId); + const windowHi = Math.max(startBoundary.numericId, endBoundary.numericId); + const startRaw = input.state.messageRefs.byRef[startBoundary.raw]; + const endRaw = input.state.messageRefs.byRef[endBoundary.raw]; + const resolvedEndpointIds = [ + input.messages[resolved.startIndex]?.id, + input.messages[resolved.endIndex]?.id, + ]; + const resolutionInPlace = + startRaw !== undefined && + endRaw !== undefined && + resolvedEndpointIds.includes(startRaw) && + resolvedEndpointIds.includes(endRaw); + let spanLo = windowLo; + let spanHi = windowHi; + if (resolutionInPlace) { + for (const id of rangeMessageIds) { + const ref = input.state.messageRefs.byRaw[id]; + const num = ref === undefined ? null : refToIndex(ref); + if (num === null) continue; + if (num < spanLo) spanLo = num; + if (num > spanHi) spanHi = num; + } + } + const offenders: string[] = []; + for (const consumedId of consumedBlockIds) { + const block = blockById(input.state, consumedId); + if (!block || block.effectiveMessageIds.length === 0) continue; + let minNum: number | null = null; + let maxNum: number | null = null; + let outOfSpan = false; + for (const mid of block.effectiveMessageIds) { + const ref = input.state.messageRefs.byRaw[mid]; + const num = ref === undefined ? null : refToIndex(ref); + if (num === null) continue; + if (num < spanLo || num > spanHi) outOfSpan = true; + if (minNum === null || num < minNum) minNum = num; + if (maxNum === null || num > maxNum) maxNum = num; + } + if (minNum === null || maxNum === null) { + offenders.push(`${consumedId} [unmapped coverage]`); + } else if (outOfSpan) { + offenders.push(`${consumedId} [${indexToRef(minNum)}..${indexToRef(maxNum)}]`); + } + } + if (offenders.length > 0) { + return finish({ + fatal: { + kind: "window", + error: new Error( + resolutionInPlace + ? `Range ${input.startRef}..${input.endRef} would merge ${offenders.join(", ")}, whose existing coverage lies outside the fold's span (${indexToRef(spanLo)}..${indexToRef(spanHi)}) — folding them would replace their summaries with one that does not cover their span. To merge them deliberately, reference the block ids directly (e.g. compress bN..bM); otherwise narrow the range.` + : `Range ${input.startRef}..${input.endRef} would merge ${offenders.join(", ")}, whose existing coverage extends beyond the requested ref window (${indexToRef(windowLo)}..${indexToRef(windowHi)}) — folding them would replace their summaries with one that does not cover their span. To merge them deliberately, reference the block ids directly (e.g. compress bN..bM); otherwise narrow the range so its endpoints bound the blocks' coverage.`, + ), + }, + }); + } + } + } + const mediaExcluded = directMessageIds.filter((id) => { const msg = msgById(id); return !!msg && hasUnrecoverableMediaPayload(msg); diff --git a/kernel/tests/issue5830-merge-window-guard.test.ts b/kernel/tests/issue5830-merge-window-guard.test.ts new file mode 100644 index 0000000000..bb9eb30a25 --- /dev/null +++ b/kernel/tests/issue5830-merge-window-guard.test.ts @@ -0,0 +1,174 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { createCore } from "../src/compress.js"; +import { + blockById, + coveredMessageIds, + createInitialState, +} from "../src/state.js"; +import { defaultConfig } from "../src/config.js"; +import { summaryMessageId } from "../src/prune.js"; +import type { CompressionState, CoreMessage } from "../src/types.js"; + +// #5830: a plain message-ref range must never merge an active block whose +// coverage extends beyond the requested ref window [startRef..endRef] — the +// supplied summary describes exactly that window, so such a merge replaces +// the block's wider-span summary with one that never saw part of it (the +// incident left b27 covering 1566 messages with only the last range's +// summary; prior blocks silently deactivated). +// +// Fixture = a reachable production sequence (full-resend host; the plugin +// lane judges on raw history per #3236's applyView): stage 1 builds two +// active blocks with OVERLAPPING coverage (a full-raw fold starting inside +// an existing T2's territory); stage 2 fires a range whose stale inherited +// startRef snaps to the T2's head anchor while its end is fresh, so the +// resolved span sweeps the overlapping T1 below the requested window. + +const LONG = + "Synthetic historical content, safe for public reproduction. ".repeat(150); +const sum = (label: string) => + `${label}: synthetic stage summary capturing the folded range. `.padEnd(72, "."); +const config = defaultConfig(255000, { + preserveRecentMessages: 0, + preserveRecentTokens: 0, +} as never); + +function mkMsg(id: string): CoreMessage { + return { id, role: "user", contentType: "text", text: LONG }; +} + +type Ranges = Parameters["applyCompression"]>[0]["ranges"]; + +function buildStage1() { + const core = createCore(); + const messages: CoreMessage[] = []; + const initial = createInitialState(); + for (let i = 1; i <= 14; i++) { + const id = `u${String(i).padStart(2, "0")}`; + messages.push(mkMsg(id)); + const ref = `m${String(i).padStart(5, "0")}`; + initial.messageRefs.byRaw[id] = ref; + initial.messageRefs.byRef[ref] = id; + } + let state: CompressionState = initial; + const fold = (ranges: Ranges) => { + const res = core.applyCompression({ messages, state, config, ranges }); + if (res.result.errors.length !== 0) { + throw new Error(`stage-1 fold failed: ${JSON.stringify(res.result.errors)}`); + } + state = res.state; + return res; + }; + fold([{ startRef: "m00001", endRef: "m00002", summary: sum("b1") }]); + fold([{ startRef: "m00003", endRef: "m00004", summary: sum("b2") }]); + fold([{ startRef: "b1", endRef: "b2", summary: sum("b3-distillation") }]); + const b3 = blockById(state, "b3"); + assert.equal(b3?.tier, 2, "distillation must produce a T2"); + assert.ok(b3?.active === true, "b3 must stay active"); + // Full-raw fold over the T2's territory: b3's anchor sits inside the span + // but b3 is T2 (target tier of a plain range is 1), so it survives — b4 + // ends up covering [u01..u12] and OVERLAPS b3's [u01..u04]. Both stay + // active: the stage-1 overlap product. + fold([{ startRef: "m00001", endRef: "m00012", summary: sum("b4-fullraw") }]); + const b4 = blockById(state, "b4"); + assert.equal(b4?.tier, 1, "plain fold must produce a T1"); + assert.ok( + b4?.active === true && b3.active === true, + "stage 1 must leave both blocks active with overlapping coverage", + ); + return { core, messages, state }; +} + +/** The pruned view a normal host sends after stage 1: the two rendered + * summaries at their head anchors, then the fresh uncovered tail. */ +function prunedView(messages: CoreMessage[]): CoreMessage[] { + return [ + { + id: summaryMessageId("b3"), + role: "system", + contentType: "text", + text: "[Compressed conversation section] S-b3", + }, + { + id: summaryMessageId("b4"), + role: "system", + contentType: "text", + text: "[Compressed conversation section] S-b4", + }, + messages[12]!, + messages[13]!, + ]; +} + +function lostCoverage(before: Set, state: CompressionState): string[] { + const after = coveredMessageIds(state); + return [...before].filter((id) => !after.has(id)); +} + +describe("#5830 plain-range merge window guard", () => { + it("stale-inherited start + fresh end: a sweep that swallows an out-of-window block is rejected atomically with zero commits", () => { + const { core, messages, state } = buildStage1(); + const before = coveredMessageIds(state); + const res = core.applyCompression({ + messages: prunedView(messages), + state, + config, + ranges: [{ startRef: "m00002", endRef: "m00014", summary: sum("tail-only") }], + }); + assert.equal(res.result.blocksCreated, 0, JSON.stringify(res.result)); + assert.equal(res.result.errors.length, 1, JSON.stringify(res.result.errors)); + assert.match( + res.result.errors[0]!, + /extends beyond the requested ref window/i, + ); + assert.match(res.result.errors[0]!, /b4/, "the offender block must be named"); + assert.deepEqual(lostCoverage(before, res.state), [], "no coverage may be lost"); + assert.ok(blockById(res.state, "b4")!.active, "b4 must stay active"); + assert.ok(blockById(res.state, "b3")!.active, "b3 must stay active"); + }); + + it("control: a plain-range merge whose window CONTAINS the consumed block's coverage still lands", () => { + const { core, messages, state } = buildStage1(); + const res = core.applyCompression({ + messages: prunedView(messages), + state, + config, + ranges: [{ startRef: "m00001", endRef: "m00014", summary: sum("legit-wide") }], + }); + assert.equal(res.result.errors.length, 0, JSON.stringify(res.result.errors)); + assert.equal(res.result.blocksCreated, 1, JSON.stringify(res.result)); + assert.equal( + blockById(res.state, "b4")!.active, + false, + "b4 [u01..u12] lies inside window [1..14] — legitimate absorption", + ); + }); + + it("control: a fully-covered resubmit changes nothing — no block, no coverage loss (pre-fix: refold-guidance error, NOT a silent merge)", () => { + const { core, messages, state } = buildStage1(); + const before = coveredMessageIds(state); + const res = core.applyCompression({ + messages: prunedView(messages), + state, + config, + ranges: [{ startRef: "m00002", endRef: "m00004", summary: sum("resubmit") }], + }); + assert.equal(res.result.blocksCreated, 0, JSON.stringify(res.result)); + assert.deepEqual(lostCoverage(before, res.state), []); + assert.ok( + blockById(res.state, "b4")!.active && blockById(res.state, "b3")!.active, + ); + }); + + it("control: block-boundary distillation across the same blocks is untouched by the gate", () => { + const { core, messages, state } = buildStage1(); + const res = core.applyCompression({ + messages: prunedView(messages), + state, + config, + ranges: [{ startRef: "b3", endRef: "b4", summary: sum("distill-mixed") }], + }); + assert.equal(res.result.errors.length, 0, JSON.stringify(res.result.errors)); + assert.equal(res.result.blocksCreated, 1, JSON.stringify(res.result)); + }); +}); diff --git a/kernel/tests/regression-empty-rewrap.test.ts b/kernel/tests/regression-empty-rewrap.test.ts index ff1a25dd82..55ee784f56 100644 --- a/kernel/tests/regression-empty-rewrap.test.ts +++ b/kernel/tests/regression-empty-rewrap.test.ts @@ -126,6 +126,25 @@ function withRefs( return state; } +// A production-shaped ledger: pruned messages KEEP their original refs (ids +// are never re-minted), so covered-but-pruned ids retain numbers with gaps +// relative to whatever remains visible. withRefs cannot express that gap. +function inheritedLedger( + specs: { + blockId: string; + effectiveMessageIds: string[]; + }[], + order: [string, number][], +): CompressionState { + const state = makeState(specs); + for (const [raw, n] of order) { + const ref = `m${String(n).padStart(5, "0")}`; + state.messageRefs.byRaw[raw] = ref; + state.messageRefs.byRef[ref] = raw; + } + return state; +} + test("#199: message-ref range that only re-wraps an active block is rejected, not fake-success", () => { const core = createCore(); const cfg = config(); @@ -205,14 +224,26 @@ test("#199: message-ref range that pulls in NEW messages plus a nested block is msg("raw-9", "x".repeat(6000)), ...loopedView().slice(3), ]; - const state = withRefs( - messages, - makeState([ + // b1's covered raw-2..raw-5 were pruned from the view but keep their + // original refs — inside the requested window m00001..m00009. + const state = inheritedLedger( + [ { blockId: "b1", effectiveMessageIds: ["raw-2", "raw-3", "raw-4", "raw-5"], }, - ]), + ], + [ + ["raw-1", 1], + ["tool-1", 2], + ["raw-2", 3], + ["raw-3", 4], + ["raw-4", 5], + ["raw-5", 6], + ["raw-9", 7], + ["raw-6", 8], + ["tool-2", 9], + ], ); const startRef = state.messageRefs.byRaw["raw-1"]!; const endRef = state.messageRefs.byRaw["tool-2"]!; @@ -230,3 +261,40 @@ test("#199: message-ref range that pulls in NEW messages plus a nested block is assert.deepEqual(newBlock.directMessageIds, ["raw-9"]); assert.deepEqual(newBlock.directBlockIds, ["b1"]); }); + +test("#5830 fail-closed: unmappable consumed-block coverage rejects instead of guessing containment", () => { + const core = createCore(); + const cfg = config(); + const messages = [ + ...loopedView().slice(0, 3), + msg("raw-9", "x".repeat(6000)), + ...loopedView().slice(3), + ]; + // Fresh nextIndex=1 numbering leaves b1's pruned covered ids out of byRaw + // entirely — containment is unprovable, so the fold must reject atomically + // (the positive direction is pinned by the inherited-ledger test above). + const state = withRefs( + messages, + makeState([ + { + blockId: "b1", + effectiveMessageIds: ["raw-2", "raw-3", "raw-4", "raw-5"], + }, + ]), + ); + const startRef = state.messageRefs.byRaw["raw-1"]!; + const endRef = state.messageRefs.byRaw["tool-2"]!; + + const applied = core.applyCompression({ + ranges: [{ startRef, endRef, summary: "S".repeat(80) }], + messages, + state, + config: cfg, + }); + + assert.equal(applied.result.blocksCreated, 0, "containment unprovable -> no merge"); + assert.equal(applied.result.errors.length, 1); + assert.match(applied.result.errors[0]!, /unmapped coverage/); + assert.equal(applied.state.blocks.length, 1, "atomic: nothing committed"); + assert.equal(applied.state.blocks[0]!.active, true, "b1 untouched"); +}); diff --git a/package-lock.json b/package-lock.json index b7aed42bb2..45c0b22355 100644 --- a/package-lock.json +++ b/package-lock.json @@ -51,7 +51,7 @@ }, "kernel": { "name": "acp-kernel", - "version": "0.0.203", + "version": "0.0.204", "dev": true, "license": "MIT", "devDependencies": { diff --git a/tests/fixtures/issue5152-beat-receiver.mjs b/tests/fixtures/issue5152-beat-receiver.mjs index 132fbe11aa..7176ffc7d9 100644 --- a/tests/fixtures/issue5152-beat-receiver.mjs +++ b/tests/fixtures/issue5152-beat-receiver.mjs @@ -3,7 +3,7 @@ // reaching it while the sender's own thread was pinned. An in-process receiver // would be pinned by the same block and the proof would be vacuous. import http from "node:http"; -import { appendFileSync } from "node:fs"; +import { appendFileSync, renameSync, writeFileSync } from "node:fs"; const out = process.argv[2]; if (!out) throw new Error("usage: issue5152-beat-receiver.mjs "); @@ -17,5 +17,10 @@ const server = http.createServer((req, res) => { }); }); server.listen(0, "127.0.0.1", () => { - appendFileSync(`${out}.port`, String(server.address().port)); + // Publish atomically (tmp + rename): the parent polls this file's + // existence, so a create-then-write gap lets it read an empty file and + // parse it as port 0. + const portFile = `${out}.port`; + writeFileSync(`${portFile}.tmp`, String(server.address().port)); + renameSync(`${portFile}.tmp`, portFile); }); diff --git a/tests/issue5152-owner-beat-survives-host-stall.test.ts b/tests/issue5152-owner-beat-survives-host-stall.test.ts index 900a53e120..1851c3d85a 100644 --- a/tests/issue5152-owner-beat-survives-host-stall.test.ts +++ b/tests/issue5152-owner-beat-survives-host-stall.test.ts @@ -53,21 +53,19 @@ async function startReceiver(dir: string): Promise { const portFile = `${log}.port`; const child = spawn(process.execPath, [RECEIVER, log], { stdio: "ignore", detached: true }); child.unref(); - // Poll for a VALID port, not mere file existence: the receiver creates the - // file before its write lands, so existsSync can win that race on a loaded - // runner and read back an empty file (Number("") === 0). + // Wait until the published port VALIDATES, not merely until the file + // exists: a partial/empty read must never be mistaken for a real port + // (Number("") === 0), even though the receiver now publishes tmp+rename. let port = 0; - let portText = ""; const deadline = Date.now() + 15_000; - while (port === 0 && Date.now() < deadline) { + while (!(Number.isInteger(port) && port > 0) && Date.now() < deadline) { if (existsSync(portFile)) { - portText = readFileSync(portFile, "utf8").trim(); - const parsed = Number(portText); - if (Number.isInteger(parsed) && parsed > 0) port = parsed; + port = Number(readFileSync(portFile, "utf8").trim()); } Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20); } - assert.ok(port > 0, `receiver must publish its port (file held "${portText}")`); + assert.ok(existsSync(portFile), "receiver must publish its port"); + assert.ok(Number.isInteger(port) && port > 0, `receiver port must be a number, got ${port}`); return { port, beats: () => (existsSync(log) ? readFileSync(log, "utf8").trim().split("\n").filter(Boolean).map(Number) : []),