From 738f33d0bd338075c57530bf882cb605a840917c Mon Sep 17 00:00:00 2001 From: ci-check Date: Fri, 20 Nov 2026 05:43:28 +0800 Subject: [PATCH] kernel: classify tail-truncated stringified content arrays as truncated, not content-not-array (#6946) Rebased onto master (kernel 0.0.200). The salvage fix (parse-compress-input.ts + test) is byte-identical to the original; the kernel version is bumped 0.0.200 -> 0.0.201 to satisfy the version-discipline gate (kernel/src changed). This PR is superseded by #6983 (the rcagg aggregation carries the same fix as a faithful cherry-pick). Close after #6983 merges. --- kernel/package-lock.json | 4 +- kernel/package.json | 2 +- kernel/src/parse-compress-input.ts | 53 +++++++++++-- kernel/tests/parse-compress-input.test.ts | 92 +++++++++++++++++++++++ package-lock.json | 2 +- 5 files changed, 143 insertions(+), 10 deletions(-) diff --git a/kernel/package-lock.json b/kernel/package-lock.json index bb7bd588bb..8a2c4491d7 100644 --- a/kernel/package-lock.json +++ b/kernel/package-lock.json @@ -1,12 +1,12 @@ { "name": "acp-kernel", - "version": "0.0.200", + "version": "0.0.201", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "acp-kernel", - "version": "0.0.200", + "version": "0.0.201", "license": "MIT", "devDependencies": { "@types/node": "^22.0.0", diff --git a/kernel/package.json b/kernel/package.json index d20ac004ac..5ac694d897 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.200", + "version": "0.0.201", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/parse-compress-input.ts b/kernel/src/parse-compress-input.ts index 7ebc49c20a..9b67749638 100644 --- a/kernel/src/parse-compress-input.ts +++ b/kernel/src/parse-compress-input.ts @@ -51,9 +51,11 @@ export interface CompressParseDiagnostics { /** True when a non-array `content` value (single entry object or nested * `ranges` array) was recovered into the canonical array form (#470). */ contentSalvage?: boolean; - /** First 800 chars of the raw string input (string inputs only). */ + /** First 800 chars of the raw string input, or of the stringified + * content array when it was truncated (host argCorruption receipt). */ rawPrefix?: string; - /** Raw string input length (string inputs only). */ + /** Raw string input length, or the stringified content-array length when + * it was truncated (host argCorruption receipt). */ length?: number; /** Top-level keys of the parsed object — catches `content` vs `ranges` drift. */ keys?: string[]; @@ -202,6 +204,23 @@ function parseObjectValue( parsed === null || (parsed.entries.length === 0 && content.trim().length > 0) ) { + // A truncated JSON array — content opens with `[` and is structurally + // cut off (unbalanced brackets / an unterminated string, the + // mid-stream cutoff signature) — is NOT a line-form attempt. The + // line-form ladder below would misread the JSON prefix as headerless + // prose and report content-not-array / no-valid-ranges, steering the + // model away from a format that was legal all along. Classify it as + // truncated (finishSalvage already establishes "truncated input → + // kind=truncated" even with zero ranges) and record the length so the + // host's argCorruption receipt fires instead of the array-wrap hint. + // The gate fires ONLY on the truncation signature: a balanced-but- + // malformed document keeps today's line-form path (no regression). + const fenceStripped = stripFence(content.trim()); + if (fenceStripped.startsWith("[") && looksTruncated(fenceStripped)) { + diag.length = content.length; + diag.rawPrefix = clampPrefix(content, 800); + return finishSalvage(parsed?.entries ?? [], callId, diag, true); + } // Not a JSON array (or nothing salvageable): bare line-form text. // Strip JSON-wrapper residue (stringified ["…"] with broken // escaping) — the line form needs structure only in the refs @@ -255,6 +274,13 @@ function parseObjectValue( entries = parsed.entries; salvaged = parsed.salvaged; if (parsed.quoteRepaired) diag.quoteSalvage = true; + // A truncated content string that DID salvage complete entries (object + // or string) is still a truncated input; record the length so the host + // receipt reads as arg-corruption, not a generic no-valid-ranges. + if (salvaged) { + diag.length = content.length; + diag.rawPrefix = clampPrefix(content, 800); + } } else if (content !== null && typeof content === "object") { // #470: a single entry OBJECT in the content slot instead of a // one-element array — the SAME drift the top level already recovers @@ -977,9 +1003,11 @@ function looksTruncated(s: string): boolean { /** * Recover the complete entries of the `content` array from a truncated JSON * prefix. Walks the prefix with a small state machine (string / escape / - * bracket depth); every object that opens and closes at depth 1 is - * re-parsed leniently and kept only if it still parses. Partial entries are - * dropped — the parser never invents model content. + * bracket depth); every entry — an object `{…}` or a string `"…"` — that + * opens and closes at the top level of the content array (depth 0) is + * re-parsed leniently and kept only if it still parses. Partial entries + * (an unclosed trailing object or string) are dropped — the parser never + * invents model content. */ function salvageContentEntries(raw: string): unknown[] { const match = /"content"\s*:\s*\[/.exec(raw); @@ -995,11 +1023,24 @@ function salvageContentEntries(raw: string): unknown[] { if (inString) { if (escaped) escaped = false; else if (ch === "\\") escaped = true; - else if (ch === '"') inString = false; + else if (ch === '"') { + inString = false; + // A string that opened at depth 0 is a complete string entry (the + // line-form sibling of the object entries). Capture it; a partial + // trailing string never closes, so it is dropped, never guessed. + if (depth === 0 && entryStart !== -1) { + const entrySlice = raw.slice(entryStart, i + 1); + entryStart = -1; + const parsed = tryParseLenient(entrySlice); + if (parsed !== undefined) entries.push(parsed); + } + } continue; } if (ch === '"') { inString = true; + // A string opening at depth 0 is the start of a string entry. + if (depth === 0 && entryStart === -1) entryStart = i; continue; } if (ch === "{" || ch === "[") { diff --git a/kernel/tests/parse-compress-input.test.ts b/kernel/tests/parse-compress-input.test.ts index 2e548cc831..fd89753e34 100644 --- a/kernel/tests/parse-compress-input.test.ts +++ b/kernel/tests/parse-compress-input.test.ts @@ -292,6 +292,98 @@ test("parseCompressArgs does not fabricate partial entries", () => { assert.equal(ranges.length, 0); }); +// --------------------------------------------------------------------------- +// Truncated stringified content array (object input) — #6947 +// +// The vLLM-style gateway stringifies the content array, so `content` arrives +// as a JSON string of the array. When that string is cut mid-stream (the +// model's call truncated in flight), the prefix is a clean JSON-array start +// followed by EOF. Before this fix the line-form fallback misread the JSON +// prefix as headerless prose and reported content-not-array / no-valid-ranges, +// steering the model away from a format that was legal all along. The EOF +// gate classifies it as truncated; salvage recovers the complete entries +// (object or string). +// --------------------------------------------------------------------------- + +test("case A: object content string cut in the first object → truncated, not content-not-array", () => { + const content = + '[{"startRef": "m00001", "endRef": "m00002", "summary": "a long summary that is cut off mid-sentence without ever closing the string or the array'; + const { ranges, diagnostics } = parseCompressArgs({ content }); + assert.equal(diagnostics.kind, "truncated"); + assert.equal(diagnostics.ok, false); + assert.equal(ranges.length, 0); + // length recorded so the host argCorruption receipt (not the array-wrap + // hint) fires. + assert.equal(diagnostics.length, content.length); + assert.equal(diagnostics.rawPrefix, content); +}); + +test("case B: object content string cut in the second object → first range recovered, truncated", () => { + const content = + '[{"startRef": "m00001", "endRef": "m00002", "summary": "first complete"}, {"startRef": "m00003", "endRef": "m0'; + const { ranges, diagnostics } = parseCompressArgs({ content }); + assert.equal(diagnostics.kind, "truncated"); + assert.equal(diagnostics.ok, true); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "first complete"); + assert.equal(diagnostics.length, content.length); +}); + +test("case C: object content string cut in the second STRING entry → first line-form entry recovered (salvage)", () => { + const content = + '["m00001–m00002 topic one\\nsummary one", "m00003–m00004 topic two\\nsummary two that got cut'; + const { ranges, diagnostics } = parseCompressArgs({ content }); + assert.equal(diagnostics.kind, "truncated"); + assert.equal(diagnostics.ok, true); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.startRef, "m00001"); + assert.equal(ranges[0]?.endRef, "m00002"); + assert.equal(ranges[0]?.summary, "summary one"); + assert.equal(diagnostics.length, content.length); +}); + +test("case D: object content string cut in the first STRING entry → truncated, not no-valid-ranges", () => { + const content = + '["m00001–m00002 topic\\na long summary that is cut off mid-sentence without ever closing the string or the array'; + const { ranges, diagnostics } = parseCompressArgs({ content }); + assert.equal(diagnostics.kind, "truncated"); + assert.equal(diagnostics.ok, false); + assert.equal(ranges.length, 0); + assert.equal(diagnostics.length, content.length); +}); + +test("case E: object content string with complete string entries → ok (no regression)", () => { + const content = + '["m00001–m00002 topic one\\nsummary one", "m00003–m00004 topic two\\nsummary two"]'; + const { ranges, diagnostics } = parseCompressArgs({ content }); + assert.equal(diagnostics.kind, "ok"); + assert.equal(diagnostics.ok, true); + assert.equal(ranges.length, 2); + assert.equal(ranges[0]?.startRef, "m00001"); + assert.equal(ranges[1]?.startRef, "m00003"); + // Not truncated → no length recorded. + assert.equal(diagnostics.length, undefined); +}); + +test("no-regression: balanced empty array content keeps content-not-array, not truncated", () => { + const { ranges, diagnostics } = parseCompressArgs({ content: "[]" }); + assert.equal(diagnostics.kind, "content-not-array"); + assert.equal(diagnostics.ok, false); + assert.equal(ranges.length, 0); + // The EOF gate must not fire on balanced input → no length recorded. + assert.equal(diagnostics.length, undefined); +}); + +test("no-regression: non-array prose content keeps content-not-array, not truncated", () => { + const { ranges, diagnostics } = parseCompressArgs({ + content: "just some prose without any brackets", + }); + assert.equal(diagnostics.kind, "content-not-array"); + assert.equal(diagnostics.ok, false); + assert.equal(ranges.length, 0); + assert.equal(diagnostics.length, undefined); +}); + // --------------------------------------------------------------------------- // Error classification // --------------------------------------------------------------------------- diff --git a/package-lock.json b/package-lock.json index dd0c2f32d2..72519f7f77 100644 --- a/package-lock.json +++ b/package-lock.json @@ -51,7 +51,7 @@ }, "kernel": { "name": "acp-kernel", - "version": "0.0.200", + "version": "0.0.201", "dev": true, "license": "MIT", "devDependencies": {