Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions kernel/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion kernel/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "acp-kernel",
"version": "0.0.200",
"version": "0.0.201",
"description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.",
"license": "MIT",
"author": "ranxianglei",
Expand Down
53 changes: 47 additions & 6 deletions kernel/src/parse-compress-input.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,11 @@ export interface CompressParseDiagnostics {
/** True when a non-array `content` value (single entry object or nested
* `ranges` array) was recovered into the canonical array form (#470). */
contentSalvage?: boolean;
/** First 800 chars of the raw string input (string inputs only). */
/** First 800 chars of the raw string input, or of the stringified
* content array when it was truncated (host argCorruption receipt). */
rawPrefix?: string;
/** Raw string input length (string inputs only). */
/** Raw string input length, or the stringified content-array length when
* it was truncated (host argCorruption receipt). */
length?: number;
/** Top-level keys of the parsed object — catches `content` vs `ranges` drift. */
keys?: string[];
Expand Down Expand Up @@ -202,6 +204,23 @@ function parseObjectValue(
parsed === null ||
(parsed.entries.length === 0 && content.trim().length > 0)
) {
// A truncated JSON array — content opens with `[` and is structurally
// cut off (unbalanced brackets / an unterminated string, the
// mid-stream cutoff signature) — is NOT a line-form attempt. The
// line-form ladder below would misread the JSON prefix as headerless
// prose and report content-not-array / no-valid-ranges, steering the
// model away from a format that was legal all along. Classify it as
// truncated (finishSalvage already establishes "truncated input →
// kind=truncated" even with zero ranges) and record the length so the
// host's argCorruption receipt fires instead of the array-wrap hint.
// The gate fires ONLY on the truncation signature: a balanced-but-
// malformed document keeps today's line-form path (no regression).
const fenceStripped = stripFence(content.trim());
if (fenceStripped.startsWith("[") && looksTruncated(fenceStripped)) {
diag.length = content.length;
diag.rawPrefix = clampPrefix(content, 800);
return finishSalvage(parsed?.entries ?? [], callId, diag, true);
}
// Not a JSON array (or nothing salvageable): bare line-form text.
// Strip JSON-wrapper residue (stringified ["…"] with broken
// escaping) — the line form needs structure only in the refs
Expand Down Expand Up @@ -255,6 +274,13 @@ function parseObjectValue(
entries = parsed.entries;
salvaged = parsed.salvaged;
if (parsed.quoteRepaired) diag.quoteSalvage = true;
// A truncated content string that DID salvage complete entries (object
// or string) is still a truncated input; record the length so the host
// receipt reads as arg-corruption, not a generic no-valid-ranges.
if (salvaged) {
diag.length = content.length;
diag.rawPrefix = clampPrefix(content, 800);
}
} else if (content !== null && typeof content === "object") {
// #470: a single entry OBJECT in the content slot instead of a
// one-element array — the SAME drift the top level already recovers
Expand Down Expand Up @@ -977,9 +1003,11 @@ function looksTruncated(s: string): boolean {
/**
* Recover the complete entries of the `content` array from a truncated JSON
* prefix. Walks the prefix with a small state machine (string / escape /
* bracket depth); every object that opens and closes at depth 1 is
* re-parsed leniently and kept only if it still parses. Partial entries are
* dropped — the parser never invents model content.
* bracket depth); every entry — an object `{…}` or a string `"…"` — that
* opens and closes at the top level of the content array (depth 0) is
* re-parsed leniently and kept only if it still parses. Partial entries
* (an unclosed trailing object or string) are dropped — the parser never
* invents model content.
*/
function salvageContentEntries(raw: string): unknown[] {
const match = /"content"\s*:\s*\[/.exec(raw);
Expand All @@ -995,11 +1023,24 @@ function salvageContentEntries(raw: string): unknown[] {
if (inString) {
if (escaped) escaped = false;
else if (ch === "\\") escaped = true;
else if (ch === '"') inString = false;
else if (ch === '"') {
inString = false;
// A string that opened at depth 0 is a complete string entry (the
// line-form sibling of the object entries). Capture it; a partial
// trailing string never closes, so it is dropped, never guessed.
if (depth === 0 && entryStart !== -1) {
const entrySlice = raw.slice(entryStart, i + 1);
entryStart = -1;
const parsed = tryParseLenient(entrySlice);
if (parsed !== undefined) entries.push(parsed);
}
}
continue;
}
if (ch === '"') {
inString = true;
// A string opening at depth 0 is the start of a string entry.
if (depth === 0 && entryStart === -1) entryStart = i;
continue;
}
if (ch === "{" || ch === "[") {
Expand Down
11 changes: 7 additions & 4 deletions kernel/tests/carrier-advertise-validate.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -299,10 +299,13 @@ test("archivable and ref-based media fold like text; only refless payloads stay
});

// Partial-pair visibility (call visible, result covered) is UNREACHABLE
// through the pipeline today: prune's orphan stripping (stripOrphanedToolCalls
// drops any call whose result left the view) plus apply-side pair widening and
// the integrity gate keep pairs atomic end to end. The risk is structural —
// two screening sites judging two different array classes with nothing
// through the pipeline today: apply-side pair widening and the integrity gate
// keep pairs atomic end to end, and for ordinary tools prune's orphan
// stripping (stripOrphanedToolCalls) drops any call whose result left the view
// as a second line. Compress calls are EXEMPT from that strip (prune.ts, the
// isCompressTool continue), so for them the atomicity rests on the apply-side
// widening + integrity gate ALONE — the #5455 boundary. The risk is structural
// — two screening sites judging two different array classes with nothing
// enforcing they agree. Pin the contract at the function level with a
// hand-crafted view pair: the judgment must flip with the array class, and
// the default (omitted source) must keep judging the passed array for
Expand Down
92 changes: 92 additions & 0 deletions kernel/tests/parse-compress-input.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,98 @@ test("parseCompressArgs does not fabricate partial entries", () => {
assert.equal(ranges.length, 0);
});

// ---------------------------------------------------------------------------
// Truncated stringified content array (object input) — #6947
//
// The vLLM-style gateway stringifies the content array, so `content` arrives
// as a JSON string of the array. When that string is cut mid-stream (the
// model's call truncated in flight), the prefix is a clean JSON-array start
// followed by EOF. Before this fix the line-form fallback misread the JSON
// prefix as headerless prose and reported content-not-array / no-valid-ranges,
// steering the model away from a format that was legal all along. The EOF
// gate classifies it as truncated; salvage recovers the complete entries
// (object or string).
// ---------------------------------------------------------------------------

test("case A: object content string cut in the first object → truncated, not content-not-array", () => {
const content =
'[{"startRef": "m00001", "endRef": "m00002", "summary": "a long summary that is cut off mid-sentence without ever closing the string or the array';
const { ranges, diagnostics } = parseCompressArgs({ content });
assert.equal(diagnostics.kind, "truncated");
assert.equal(diagnostics.ok, false);
assert.equal(ranges.length, 0);
// length recorded so the host argCorruption receipt (not the array-wrap
// hint) fires.
assert.equal(diagnostics.length, content.length);
assert.equal(diagnostics.rawPrefix, content);
});

test("case B: object content string cut in the second object → first range recovered, truncated", () => {
const content =
'[{"startRef": "m00001", "endRef": "m00002", "summary": "first complete"}, {"startRef": "m00003", "endRef": "m0';
const { ranges, diagnostics } = parseCompressArgs({ content });
assert.equal(diagnostics.kind, "truncated");
assert.equal(diagnostics.ok, true);
assert.equal(ranges.length, 1);
assert.equal(ranges[0]?.summary, "first complete");
assert.equal(diagnostics.length, content.length);
});

test("case C: object content string cut in the second STRING entry → first line-form entry recovered (salvage)", () => {
const content =
'["m00001–m00002 topic one\\nsummary one", "m00003–m00004 topic two\\nsummary two that got cut';
const { ranges, diagnostics } = parseCompressArgs({ content });
assert.equal(diagnostics.kind, "truncated");
assert.equal(diagnostics.ok, true);
assert.equal(ranges.length, 1);
assert.equal(ranges[0]?.startRef, "m00001");
assert.equal(ranges[0]?.endRef, "m00002");
assert.equal(ranges[0]?.summary, "summary one");
assert.equal(diagnostics.length, content.length);
});

test("case D: object content string cut in the first STRING entry → truncated, not no-valid-ranges", () => {
const content =
'["m00001–m00002 topic\\na long summary that is cut off mid-sentence without ever closing the string or the array';
const { ranges, diagnostics } = parseCompressArgs({ content });
assert.equal(diagnostics.kind, "truncated");
assert.equal(diagnostics.ok, false);
assert.equal(ranges.length, 0);
assert.equal(diagnostics.length, content.length);
});

test("case E: object content string with complete string entries → ok (no regression)", () => {
const content =
'["m00001–m00002 topic one\\nsummary one", "m00003–m00004 topic two\\nsummary two"]';
const { ranges, diagnostics } = parseCompressArgs({ content });
assert.equal(diagnostics.kind, "ok");
assert.equal(diagnostics.ok, true);
assert.equal(ranges.length, 2);
assert.equal(ranges[0]?.startRef, "m00001");
assert.equal(ranges[1]?.startRef, "m00003");
// Not truncated → no length recorded.
assert.equal(diagnostics.length, undefined);
});

test("no-regression: balanced empty array content keeps content-not-array, not truncated", () => {
const { ranges, diagnostics } = parseCompressArgs({ content: "[]" });
assert.equal(diagnostics.kind, "content-not-array");
assert.equal(diagnostics.ok, false);
assert.equal(ranges.length, 0);
// The EOF gate must not fire on balanced input → no length recorded.
assert.equal(diagnostics.length, undefined);
});

test("no-regression: non-array prose content keeps content-not-array, not truncated", () => {
const { ranges, diagnostics } = parseCompressArgs({
content: "just some prose without any brackets",
});
assert.equal(diagnostics.kind, "content-not-array");
assert.equal(diagnostics.ok, false);
assert.equal(ranges.length, 0);
assert.equal(diagnostics.length, undefined);
});

// ---------------------------------------------------------------------------
// Error classification
// ---------------------------------------------------------------------------
Expand Down
36 changes: 36 additions & 0 deletions kernel/tests/prune-orphan-mcp.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,3 +75,39 @@ test("default options stay byte-identical to pre-fix behavior (bare-name only)",
const bare = prune(seq("compress"), stateWithActiveCoverage());
assert.ok(bare.some((m) => m.id === "pend"), "bare-name exemption unchanged under defaults");
});

// #5455 boundary: an ANSWERED compress call whose result is covered still
// survives the orphan strip (the isCompressTool exemption at prune.ts:625).
// This "answered but result covered" state is currently UNREACHABLE through
// the pipeline (no producer), but the exemption means it would ship an
// unanswered compress tool_call if it ever became reachable. Compress-call
// atomicity rests on the apply-side pair widening + integrity gate ALONE —
// this test pins that boundary. Contrast: an ordinary answered call whose
// result is covered IS stripped.
test("#5455 boundary: answered compress call whose result is covered survives the orphan exemption", () => {
const messages: CoreMessage[] = [
{ id: "u0", role: "user", contentType: "text", text: "summarize" },
{ id: "call", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: "fixture-call", text: "{}" },
{ id: "result", role: "tool", contentType: "tool-result", toolCallId: "fixture-call", text: "completed" },
{ id: "u1", role: "user", contentType: "text", text: "continue" },
];
const state = createInitialState();
state.blocks.push(block({ blockId: "fixture-block", effectiveMessageIds: ["result"] }));
const out = prune(messages, state);
assert.ok(!out.some((m) => m.id === "result"), "covered result removed");
assert.ok(out.some((m) => m.id === "call"), "answered compress call survives the orphan exemption (#5455 boundary)");
});

test("#5455 contrast: an ordinary answered call whose result is covered is stripped", () => {
const messages: CoreMessage[] = [
{ id: "u0", role: "user", contentType: "text", text: "a" },
{ id: "call", role: "assistant", contentType: "tool-call", toolName: "bash", toolCallId: "fixture-call", text: "{}" },
{ id: "result", role: "tool", contentType: "tool-result", toolCallId: "fixture-call", text: "out" },
{ id: "u1", role: "user", contentType: "text", text: "b" },
];
const state = createInitialState();
state.blocks.push(block({ blockId: "fixture-block", effectiveMessageIds: ["result"] }));
const out = prune(messages, state);
assert.ok(!out.some((m) => m.id === "result"), "covered result removed");
assert.ok(!out.some((m) => m.id === "call"), "ordinary answered call whose result is covered is stripped");
});
48 changes: 48 additions & 0 deletions kernel/tests/tool-pair-integrity.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -340,4 +340,52 @@ describe("tool-pair integrity", () => {
);
assert.deepEqual(unansweredToolCalls(view.messages), []);
});

// #5455 C': pin the pair-atomicity invariant at the SOURCE (apply side).
// The existing tests pin the "call without result" direction (a protected
// result refuses/withdraws the call). This pins the reverse: a block must
// never contain a tool-result lacking its adjacent tool-call. This is the
// invariant that keeps the #5455 "answered compress call whose result is
// covered" state UNREACHABLE — the apply side never emits such a block, so
// prune's compress orphan exemption (prune.ts:625) never sees it.
it("#5455 C': never folds a tool-result without its adjacent tool-call (pair atomicity at source)", () => {
const core = createCore();
const messages: CoreMessage[] = [
user("m1"),
toolCall("m2", "c-a", "read"),
toolResult("m3", "c-a", "read"),
user("m4"),
user("m5"),
];
const out = core.processTurn({
messages,
state: createInitialState(),
config: config(),
tokenCount: (t) => Math.ceil(t.length / 4),
});
const refs = out.state.messageRefs.byRaw;
// Protect the CALL (m2), cover both call+result. The result must not fold
// without its call — that would be a "result lacking call" (#5455 state).
const res = core.applyCompression({
state: out.state,
messages,
config: config(),
protectedMessageIds: new Set([refs["m2"]!]),
ranges: [
{ startRef: "m00002", endRef: "m00003", summary: "S".repeat(400) },
],
});
// The pair is atomic: no block may contain the result without the call.
for (const b of res.state.blocks) {
const hasResult = b.effectiveMessageIds.includes("m3");
const hasCall = b.effectiveMessageIds.includes("m2");
assert.ok(
!(hasResult && !hasCall),
`a block must not contain a tool-result lacking its tool-call (#5455 C'): ${JSON.stringify(b.effectiveMessageIds)}`,
);
}
// The result must not fold on its own (the call is protected).
const foldedResult = res.state.blocks.some((b) => b.effectiveMessageIds.includes("m3"));
assert.ok(!foldedResult, "a protected call's result must not fold without it");
});
});
2 changes: 1 addition & 1 deletion package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading