Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions kernel/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion kernel/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "acp-kernel",
"version": "0.0.200",
"version": "0.0.201",
"description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.",
"license": "MIT",
"author": "ranxianglei",
Expand Down
53 changes: 47 additions & 6 deletions kernel/src/parse-compress-input.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,11 @@ export interface CompressParseDiagnostics {
/** True when a non-array `content` value (single entry object or nested
* `ranges` array) was recovered into the canonical array form (#470). */
contentSalvage?: boolean;
/** First 800 chars of the raw string input (string inputs only). */
/** First 800 chars of the raw string input, or of the stringified
* content array when it was truncated (host argCorruption receipt). */
rawPrefix?: string;
/** Raw string input length (string inputs only). */
/** Raw string input length, or the stringified content-array length when
* it was truncated (host argCorruption receipt). */
length?: number;
/** Top-level keys of the parsed object — catches `content` vs `ranges` drift. */
keys?: string[];
Expand Down Expand Up @@ -202,6 +204,23 @@ function parseObjectValue(
parsed === null ||
(parsed.entries.length === 0 && content.trim().length > 0)
) {
// A truncated JSON array — content opens with `[` and is structurally
// cut off (unbalanced brackets / an unterminated string, the
// mid-stream cutoff signature) — is NOT a line-form attempt. The
// line-form ladder below would misread the JSON prefix as headerless
// prose and report content-not-array / no-valid-ranges, steering the
// model away from a format that was legal all along. Classify it as
// truncated (finishSalvage already establishes "truncated input →
// kind=truncated" even with zero ranges) and record the length so the
// host's argCorruption receipt fires instead of the array-wrap hint.
// The gate fires ONLY on the truncation signature: a balanced-but-
// malformed document keeps today's line-form path (no regression).
const fenceStripped = stripFence(content.trim());
if (fenceStripped.startsWith("[") && looksTruncated(fenceStripped)) {
diag.length = content.length;
diag.rawPrefix = clampPrefix(content, 800);
return finishSalvage(parsed?.entries ?? [], callId, diag, true);
}
// Not a JSON array (or nothing salvageable): bare line-form text.
// Strip JSON-wrapper residue (stringified ["…"] with broken
// escaping) — the line form needs structure only in the refs
Expand Down Expand Up @@ -255,6 +274,13 @@ function parseObjectValue(
entries = parsed.entries;
salvaged = parsed.salvaged;
if (parsed.quoteRepaired) diag.quoteSalvage = true;
// A truncated content string that DID salvage complete entries (object
// or string) is still a truncated input; record the length so the host
// receipt reads as arg-corruption, not a generic no-valid-ranges.
if (salvaged) {
diag.length = content.length;
diag.rawPrefix = clampPrefix(content, 800);
}
} else if (content !== null && typeof content === "object") {
// #470: a single entry OBJECT in the content slot instead of a
// one-element array — the SAME drift the top level already recovers
Expand Down Expand Up @@ -977,9 +1003,11 @@ function looksTruncated(s: string): boolean {
/**
* Recover the complete entries of the `content` array from a truncated JSON
* prefix. Walks the prefix with a small state machine (string / escape /
* bracket depth); every object that opens and closes at depth 1 is
* re-parsed leniently and kept only if it still parses. Partial entries are
* dropped — the parser never invents model content.
* bracket depth); every entry — an object `{…}` or a string `"…"` — that
* opens and closes at the top level of the content array (depth 0) is
* re-parsed leniently and kept only if it still parses. Partial entries
* (an unclosed trailing object or string) are dropped — the parser never
* invents model content.
*/
function salvageContentEntries(raw: string): unknown[] {
const match = /"content"\s*:\s*\[/.exec(raw);
Expand All @@ -995,11 +1023,24 @@ function salvageContentEntries(raw: string): unknown[] {
if (inString) {
if (escaped) escaped = false;
else if (ch === "\\") escaped = true;
else if (ch === '"') inString = false;
else if (ch === '"') {
inString = false;
// A string that opened at depth 0 is a complete string entry (the
// line-form sibling of the object entries). Capture it; a partial
// trailing string never closes, so it is dropped, never guessed.
if (depth === 0 && entryStart !== -1) {
const entrySlice = raw.slice(entryStart, i + 1);
entryStart = -1;
const parsed = tryParseLenient(entrySlice);
if (parsed !== undefined) entries.push(parsed);
}
}
continue;
}
if (ch === '"') {
inString = true;
// A string opening at depth 0 is the start of a string entry.
if (depth === 0 && entryStart === -1) entryStart = i;
continue;
}
if (ch === "{" || ch === "[") {
Expand Down
92 changes: 92 additions & 0 deletions kernel/tests/parse-compress-input.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,98 @@ test("parseCompressArgs does not fabricate partial entries", () => {
assert.equal(ranges.length, 0);
});

// ---------------------------------------------------------------------------
// Truncated stringified content array (object input) — #6947
//
// The vLLM-style gateway stringifies the content array, so `content` arrives
// as a JSON string of the array. When that string is cut mid-stream (the
// model's call truncated in flight), the prefix is a clean JSON-array start
// followed by EOF. Before this fix the line-form fallback misread the JSON
// prefix as headerless prose and reported content-not-array / no-valid-ranges,
// steering the model away from a format that was legal all along. The EOF
// gate classifies it as truncated; salvage recovers the complete entries
// (object or string).
// ---------------------------------------------------------------------------

test("case A: object content string cut in the first object → truncated, not content-not-array", () => {
const content =
'[{"startRef": "m00001", "endRef": "m00002", "summary": "a long summary that is cut off mid-sentence without ever closing the string or the array';
const { ranges, diagnostics } = parseCompressArgs({ content });
assert.equal(diagnostics.kind, "truncated");
assert.equal(diagnostics.ok, false);
assert.equal(ranges.length, 0);
// length recorded so the host argCorruption receipt (not the array-wrap
// hint) fires.
assert.equal(diagnostics.length, content.length);
assert.equal(diagnostics.rawPrefix, content);
});

test("case B: object content string cut in the second object → first range recovered, truncated", () => {
const content =
'[{"startRef": "m00001", "endRef": "m00002", "summary": "first complete"}, {"startRef": "m00003", "endRef": "m0';
const { ranges, diagnostics } = parseCompressArgs({ content });
assert.equal(diagnostics.kind, "truncated");
assert.equal(diagnostics.ok, true);
assert.equal(ranges.length, 1);
assert.equal(ranges[0]?.summary, "first complete");
assert.equal(diagnostics.length, content.length);
});

test("case C: object content string cut in the second STRING entry → first line-form entry recovered (salvage)", () => {
const content =
'["m00001–m00002 topic one\\nsummary one", "m00003–m00004 topic two\\nsummary two that got cut';
const { ranges, diagnostics } = parseCompressArgs({ content });
assert.equal(diagnostics.kind, "truncated");
assert.equal(diagnostics.ok, true);
assert.equal(ranges.length, 1);
assert.equal(ranges[0]?.startRef, "m00001");
assert.equal(ranges[0]?.endRef, "m00002");
assert.equal(ranges[0]?.summary, "summary one");
assert.equal(diagnostics.length, content.length);
});

test("case D: object content string cut in the first STRING entry → truncated, not no-valid-ranges", () => {
const content =
'["m00001–m00002 topic\\na long summary that is cut off mid-sentence without ever closing the string or the array';
const { ranges, diagnostics } = parseCompressArgs({ content });
assert.equal(diagnostics.kind, "truncated");
assert.equal(diagnostics.ok, false);
assert.equal(ranges.length, 0);
assert.equal(diagnostics.length, content.length);
});

test("case E: object content string with complete string entries → ok (no regression)", () => {
const content =
'["m00001–m00002 topic one\\nsummary one", "m00003–m00004 topic two\\nsummary two"]';
const { ranges, diagnostics } = parseCompressArgs({ content });
assert.equal(diagnostics.kind, "ok");
assert.equal(diagnostics.ok, true);
assert.equal(ranges.length, 2);
assert.equal(ranges[0]?.startRef, "m00001");
assert.equal(ranges[1]?.startRef, "m00003");
// Not truncated → no length recorded.
assert.equal(diagnostics.length, undefined);
});

test("no-regression: balanced empty array content keeps content-not-array, not truncated", () => {
const { ranges, diagnostics } = parseCompressArgs({ content: "[]" });
assert.equal(diagnostics.kind, "content-not-array");
assert.equal(diagnostics.ok, false);
assert.equal(ranges.length, 0);
// The EOF gate must not fire on balanced input → no length recorded.
assert.equal(diagnostics.length, undefined);
});

test("no-regression: non-array prose content keeps content-not-array, not truncated", () => {
const { ranges, diagnostics } = parseCompressArgs({
content: "just some prose without any brackets",
});
assert.equal(diagnostics.kind, "content-not-array");
assert.equal(diagnostics.ok, false);
assert.equal(ranges.length, 0);
assert.equal(diagnostics.length, undefined);
});

// ---------------------------------------------------------------------------
// Error classification
// ---------------------------------------------------------------------------
Expand Down
2 changes: 1 addition & 1 deletion package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading