diff --git a/kernel/package-lock.json b/kernel/package-lock.json index bb7bd588bb..8a2c4491d7 100644 --- a/kernel/package-lock.json +++ b/kernel/package-lock.json @@ -1,12 +1,12 @@ { "name": "acp-kernel", - "version": "0.0.200", + "version": "0.0.201", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "acp-kernel", - "version": "0.0.200", + "version": "0.0.201", "license": "MIT", "devDependencies": { "@types/node": "^22.0.0", diff --git a/kernel/package.json b/kernel/package.json index d20ac004ac..5ac694d897 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.200", + "version": "0.0.201", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/parse-compress-input.ts b/kernel/src/parse-compress-input.ts index 7ebc49c20a..9b67749638 100644 --- a/kernel/src/parse-compress-input.ts +++ b/kernel/src/parse-compress-input.ts @@ -51,9 +51,11 @@ export interface CompressParseDiagnostics { /** True when a non-array `content` value (single entry object or nested * `ranges` array) was recovered into the canonical array form (#470). */ contentSalvage?: boolean; - /** First 800 chars of the raw string input (string inputs only). */ + /** First 800 chars of the raw string input, or of the stringified + * content array when it was truncated (host argCorruption receipt). */ rawPrefix?: string; - /** Raw string input length (string inputs only). */ + /** Raw string input length, or the stringified content-array length when + * it was truncated (host argCorruption receipt). */ length?: number; /** Top-level keys of the parsed object — catches `content` vs `ranges` drift. */ keys?: string[]; @@ -202,6 +204,23 @@ function parseObjectValue( parsed === null || (parsed.entries.length === 0 && content.trim().length > 0) ) { + // A truncated JSON array — content opens with `[` and is structurally + // cut off (unbalanced brackets / an unterminated string, the + // mid-stream cutoff signature) — is NOT a line-form attempt. The + // line-form ladder below would misread the JSON prefix as headerless + // prose and report content-not-array / no-valid-ranges, steering the + // model away from a format that was legal all along. Classify it as + // truncated (finishSalvage already establishes "truncated input → + // kind=truncated" even with zero ranges) and record the length so the + // host's argCorruption receipt fires instead of the array-wrap hint. + // The gate fires ONLY on the truncation signature: a balanced-but- + // malformed document keeps today's line-form path (no regression). + const fenceStripped = stripFence(content.trim()); + if (fenceStripped.startsWith("[") && looksTruncated(fenceStripped)) { + diag.length = content.length; + diag.rawPrefix = clampPrefix(content, 800); + return finishSalvage(parsed?.entries ?? [], callId, diag, true); + } // Not a JSON array (or nothing salvageable): bare line-form text. // Strip JSON-wrapper residue (stringified ["…"] with broken // escaping) — the line form needs structure only in the refs @@ -255,6 +274,13 @@ function parseObjectValue( entries = parsed.entries; salvaged = parsed.salvaged; if (parsed.quoteRepaired) diag.quoteSalvage = true; + // A truncated content string that DID salvage complete entries (object + // or string) is still a truncated input; record the length so the host + // receipt reads as arg-corruption, not a generic no-valid-ranges. + if (salvaged) { + diag.length = content.length; + diag.rawPrefix = clampPrefix(content, 800); + } } else if (content !== null && typeof content === "object") { // #470: a single entry OBJECT in the content slot instead of a // one-element array — the SAME drift the top level already recovers @@ -977,9 +1003,11 @@ function looksTruncated(s: string): boolean { /** * Recover the complete entries of the `content` array from a truncated JSON * prefix. Walks the prefix with a small state machine (string / escape / - * bracket depth); every object that opens and closes at depth 1 is - * re-parsed leniently and kept only if it still parses. Partial entries are - * dropped — the parser never invents model content. + * bracket depth); every entry — an object `{…}` or a string `"…"` — that + * opens and closes at the top level of the content array (depth 0) is + * re-parsed leniently and kept only if it still parses. Partial entries + * (an unclosed trailing object or string) are dropped — the parser never + * invents model content. */ function salvageContentEntries(raw: string): unknown[] { const match = /"content"\s*:\s*\[/.exec(raw); @@ -995,11 +1023,24 @@ function salvageContentEntries(raw: string): unknown[] { if (inString) { if (escaped) escaped = false; else if (ch === "\\") escaped = true; - else if (ch === '"') inString = false; + else if (ch === '"') { + inString = false; + // A string that opened at depth 0 is a complete string entry (the + // line-form sibling of the object entries). Capture it; a partial + // trailing string never closes, so it is dropped, never guessed. + if (depth === 0 && entryStart !== -1) { + const entrySlice = raw.slice(entryStart, i + 1); + entryStart = -1; + const parsed = tryParseLenient(entrySlice); + if (parsed !== undefined) entries.push(parsed); + } + } continue; } if (ch === '"') { inString = true; + // A string opening at depth 0 is the start of a string entry. + if (depth === 0 && entryStart === -1) entryStart = i; continue; } if (ch === "{" || ch === "[") { diff --git a/kernel/tests/carrier-advertise-validate.test.ts b/kernel/tests/carrier-advertise-validate.test.ts index 9eba792dff..2402301a0f 100644 --- a/kernel/tests/carrier-advertise-validate.test.ts +++ b/kernel/tests/carrier-advertise-validate.test.ts @@ -299,10 +299,13 @@ test("archivable and ref-based media fold like text; only refless payloads stay }); // Partial-pair visibility (call visible, result covered) is UNREACHABLE -// through the pipeline today: prune's orphan stripping (stripOrphanedToolCalls -// drops any call whose result left the view) plus apply-side pair widening and -// the integrity gate keep pairs atomic end to end. The risk is structural — -// two screening sites judging two different array classes with nothing +// through the pipeline today: apply-side pair widening and the integrity gate +// keep pairs atomic end to end, and for ordinary tools prune's orphan +// stripping (stripOrphanedToolCalls) drops any call whose result left the view +// as a second line. Compress calls are EXEMPT from that strip (prune.ts, the +// isCompressTool continue), so for them the atomicity rests on the apply-side +// widening + integrity gate ALONE — the #5455 boundary. The risk is structural +// — two screening sites judging two different array classes with nothing // enforcing they agree. Pin the contract at the function level with a // hand-crafted view pair: the judgment must flip with the array class, and // the default (omitted source) must keep judging the passed array for diff --git a/kernel/tests/parse-compress-input.test.ts b/kernel/tests/parse-compress-input.test.ts index 2e548cc831..fd89753e34 100644 --- a/kernel/tests/parse-compress-input.test.ts +++ b/kernel/tests/parse-compress-input.test.ts @@ -292,6 +292,98 @@ test("parseCompressArgs does not fabricate partial entries", () => { assert.equal(ranges.length, 0); }); +// --------------------------------------------------------------------------- +// Truncated stringified content array (object input) — #6947 +// +// The vLLM-style gateway stringifies the content array, so `content` arrives +// as a JSON string of the array. When that string is cut mid-stream (the +// model's call truncated in flight), the prefix is a clean JSON-array start +// followed by EOF. Before this fix the line-form fallback misread the JSON +// prefix as headerless prose and reported content-not-array / no-valid-ranges, +// steering the model away from a format that was legal all along. The EOF +// gate classifies it as truncated; salvage recovers the complete entries +// (object or string). +// --------------------------------------------------------------------------- + +test("case A: object content string cut in the first object → truncated, not content-not-array", () => { + const content = + '[{"startRef": "m00001", "endRef": "m00002", "summary": "a long summary that is cut off mid-sentence without ever closing the string or the array'; + const { ranges, diagnostics } = parseCompressArgs({ content }); + assert.equal(diagnostics.kind, "truncated"); + assert.equal(diagnostics.ok, false); + assert.equal(ranges.length, 0); + // length recorded so the host argCorruption receipt (not the array-wrap + // hint) fires. + assert.equal(diagnostics.length, content.length); + assert.equal(diagnostics.rawPrefix, content); +}); + +test("case B: object content string cut in the second object → first range recovered, truncated", () => { + const content = + '[{"startRef": "m00001", "endRef": "m00002", "summary": "first complete"}, {"startRef": "m00003", "endRef": "m0'; + const { ranges, diagnostics } = parseCompressArgs({ content }); + assert.equal(diagnostics.kind, "truncated"); + assert.equal(diagnostics.ok, true); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "first complete"); + assert.equal(diagnostics.length, content.length); +}); + +test("case C: object content string cut in the second STRING entry → first line-form entry recovered (salvage)", () => { + const content = + '["m00001–m00002 topic one\\nsummary one", "m00003–m00004 topic two\\nsummary two that got cut'; + const { ranges, diagnostics } = parseCompressArgs({ content }); + assert.equal(diagnostics.kind, "truncated"); + assert.equal(diagnostics.ok, true); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.startRef, "m00001"); + assert.equal(ranges[0]?.endRef, "m00002"); + assert.equal(ranges[0]?.summary, "summary one"); + assert.equal(diagnostics.length, content.length); +}); + +test("case D: object content string cut in the first STRING entry → truncated, not no-valid-ranges", () => { + const content = + '["m00001–m00002 topic\\na long summary that is cut off mid-sentence without ever closing the string or the array'; + const { ranges, diagnostics } = parseCompressArgs({ content }); + assert.equal(diagnostics.kind, "truncated"); + assert.equal(diagnostics.ok, false); + assert.equal(ranges.length, 0); + assert.equal(diagnostics.length, content.length); +}); + +test("case E: object content string with complete string entries → ok (no regression)", () => { + const content = + '["m00001–m00002 topic one\\nsummary one", "m00003–m00004 topic two\\nsummary two"]'; + const { ranges, diagnostics } = parseCompressArgs({ content }); + assert.equal(diagnostics.kind, "ok"); + assert.equal(diagnostics.ok, true); + assert.equal(ranges.length, 2); + assert.equal(ranges[0]?.startRef, "m00001"); + assert.equal(ranges[1]?.startRef, "m00003"); + // Not truncated → no length recorded. + assert.equal(diagnostics.length, undefined); +}); + +test("no-regression: balanced empty array content keeps content-not-array, not truncated", () => { + const { ranges, diagnostics } = parseCompressArgs({ content: "[]" }); + assert.equal(diagnostics.kind, "content-not-array"); + assert.equal(diagnostics.ok, false); + assert.equal(ranges.length, 0); + // The EOF gate must not fire on balanced input → no length recorded. + assert.equal(diagnostics.length, undefined); +}); + +test("no-regression: non-array prose content keeps content-not-array, not truncated", () => { + const { ranges, diagnostics } = parseCompressArgs({ + content: "just some prose without any brackets", + }); + assert.equal(diagnostics.kind, "content-not-array"); + assert.equal(diagnostics.ok, false); + assert.equal(ranges.length, 0); + assert.equal(diagnostics.length, undefined); +}); + // --------------------------------------------------------------------------- // Error classification // --------------------------------------------------------------------------- diff --git a/kernel/tests/prune-orphan-mcp.test.ts b/kernel/tests/prune-orphan-mcp.test.ts index 8ea19caef7..ec11a75a6e 100644 --- a/kernel/tests/prune-orphan-mcp.test.ts +++ b/kernel/tests/prune-orphan-mcp.test.ts @@ -75,3 +75,39 @@ test("default options stay byte-identical to pre-fix behavior (bare-name only)", const bare = prune(seq("compress"), stateWithActiveCoverage()); assert.ok(bare.some((m) => m.id === "pend"), "bare-name exemption unchanged under defaults"); }); + +// #5455 boundary: an ANSWERED compress call whose result is covered still +// survives the orphan strip (the isCompressTool exemption at prune.ts:625). +// This "answered but result covered" state is currently UNREACHABLE through +// the pipeline (no producer), but the exemption means it would ship an +// unanswered compress tool_call if it ever became reachable. Compress-call +// atomicity rests on the apply-side pair widening + integrity gate ALONE — +// this test pins that boundary. Contrast: an ordinary answered call whose +// result is covered IS stripped. +test("#5455 boundary: answered compress call whose result is covered survives the orphan exemption", () => { + const messages: CoreMessage[] = [ + { id: "u0", role: "user", contentType: "text", text: "summarize" }, + { id: "call", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: "fixture-call", text: "{}" }, + { id: "result", role: "tool", contentType: "tool-result", toolCallId: "fixture-call", text: "completed" }, + { id: "u1", role: "user", contentType: "text", text: "continue" }, + ]; + const state = createInitialState(); + state.blocks.push(block({ blockId: "fixture-block", effectiveMessageIds: ["result"] })); + const out = prune(messages, state); + assert.ok(!out.some((m) => m.id === "result"), "covered result removed"); + assert.ok(out.some((m) => m.id === "call"), "answered compress call survives the orphan exemption (#5455 boundary)"); +}); + +test("#5455 contrast: an ordinary answered call whose result is covered is stripped", () => { + const messages: CoreMessage[] = [ + { id: "u0", role: "user", contentType: "text", text: "a" }, + { id: "call", role: "assistant", contentType: "tool-call", toolName: "bash", toolCallId: "fixture-call", text: "{}" }, + { id: "result", role: "tool", contentType: "tool-result", toolCallId: "fixture-call", text: "out" }, + { id: "u1", role: "user", contentType: "text", text: "b" }, + ]; + const state = createInitialState(); + state.blocks.push(block({ blockId: "fixture-block", effectiveMessageIds: ["result"] })); + const out = prune(messages, state); + assert.ok(!out.some((m) => m.id === "result"), "covered result removed"); + assert.ok(!out.some((m) => m.id === "call"), "ordinary answered call whose result is covered is stripped"); +}); diff --git a/kernel/tests/tool-pair-integrity.test.ts b/kernel/tests/tool-pair-integrity.test.ts index 6551aa77db..316f737bc6 100644 --- a/kernel/tests/tool-pair-integrity.test.ts +++ b/kernel/tests/tool-pair-integrity.test.ts @@ -340,4 +340,52 @@ describe("tool-pair integrity", () => { ); assert.deepEqual(unansweredToolCalls(view.messages), []); }); + + // #5455 C': pin the pair-atomicity invariant at the SOURCE (apply side). + // The existing tests pin the "call without result" direction (a protected + // result refuses/withdraws the call). This pins the reverse: a block must + // never contain a tool-result lacking its adjacent tool-call. This is the + // invariant that keeps the #5455 "answered compress call whose result is + // covered" state UNREACHABLE — the apply side never emits such a block, so + // prune's compress orphan exemption (prune.ts:625) never sees it. + it("#5455 C': never folds a tool-result without its adjacent tool-call (pair atomicity at source)", () => { + const core = createCore(); + const messages: CoreMessage[] = [ + user("m1"), + toolCall("m2", "c-a", "read"), + toolResult("m3", "c-a", "read"), + user("m4"), + user("m5"), + ]; + const out = core.processTurn({ + messages, + state: createInitialState(), + config: config(), + tokenCount: (t) => Math.ceil(t.length / 4), + }); + const refs = out.state.messageRefs.byRaw; + // Protect the CALL (m2), cover both call+result. The result must not fold + // without its call — that would be a "result lacking call" (#5455 state). + const res = core.applyCompression({ + state: out.state, + messages, + config: config(), + protectedMessageIds: new Set([refs["m2"]!]), + ranges: [ + { startRef: "m00002", endRef: "m00003", summary: "S".repeat(400) }, + ], + }); + // The pair is atomic: no block may contain the result without the call. + for (const b of res.state.blocks) { + const hasResult = b.effectiveMessageIds.includes("m3"); + const hasCall = b.effectiveMessageIds.includes("m2"); + assert.ok( + !(hasResult && !hasCall), + `a block must not contain a tool-result lacking its tool-call (#5455 C'): ${JSON.stringify(b.effectiveMessageIds)}`, + ); + } + // The result must not fold on its own (the call is protected). + const foldedResult = res.state.blocks.some((b) => b.effectiveMessageIds.includes("m3")); + assert.ok(!foldedResult, "a protected call's result must not fold without it"); + }); }); diff --git a/package-lock.json b/package-lock.json index dd0c2f32d2..72519f7f77 100644 --- a/package-lock.json +++ b/package-lock.json @@ -51,7 +51,7 @@ }, "kernel": { "name": "acp-kernel", - "version": "0.0.200", + "version": "0.0.201", "dev": true, "license": "MIT", "devDependencies": {