From 68a7bb6a871930ef430d043ef73e035c6c18fb57 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Mon, 28 Sep 2026 18:17:35 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20parse=20ladder=20=E2=80=94=20three=20st?= =?UTF-8?q?ring-ladder=20rungs=20for=20weak-model=20drift=20(billion-conte?= =?UTF-8?q?xt#1559)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/parse-compress-input.ts | 135 ++++++++++++++++++++-- tests/parse-compress-input.test.ts | 176 +++++++++++++++++++++++++++++ 2 files changed, 302 insertions(+), 9 deletions(-) diff --git a/src/parse-compress-input.ts b/src/parse-compress-input.ts index 3a77446..1c2767c 100644 --- a/src/parse-compress-input.ts +++ b/src/parse-compress-input.ts @@ -12,6 +12,10 @@ // billion-context#603 / omp#121) // - a single entry OBJECT where the content array belongs (retry drift, // billion-context#1494 / acp-kernel#470) +// - line-form text with no JSON shell at all: a bare top-level string, a +// string literal decoding to text, or a single-line slot string whose +// summary sits inline after the refs (weak-model syntax drift, +// billion-context#1559) // - the stream cut off mid-arguments, leaving a truncated JSON prefix // // Hosts parse this on their own today: rebuild.ts (strict, silent skip), @@ -21,8 +25,13 @@ // // Salvage semantics: for truncated input, the complete entries of the // `content` array are recovered from the surviving prefix. A partially -// written entry is dropped, never guessed. Diagnostics are data, not logs: -// adapters decide where to emit them (log line, debug event, tool text). +// written entry is dropped, never guessed. Line-form salvage follows the +// same rule: a cut string-slot prefix keeps every complete leading block +// and drops the trailing partial one; a single line is never split on +// embedded ref pairs, because prose citations are indistinguishable from +// headers and mis-splitting corrupts a legitimate summary. Diagnostics are +// data, not logs: adapters decide where to emit them (log line, debug +// event, tool text). import { clampPrefix } from "./truncate.js"; import type { CompressRangeSpec } from "./types.js"; @@ -47,6 +56,9 @@ export interface CompressParseDiagnostics { /** True when a non-array `content` value (single entry object or nested * `ranges` array) was recovered into the canonical array form (#470). */ contentSalvage?: boolean; + /** True when line-form text outside any JSON shell was recovered: a bare + * top-level string or a string literal decoding to text (#1559). */ + stringSalvage?: boolean; /** First 800 chars of the raw string input (string inputs only). */ rawPrefix?: string; /** Raw string input length (string inputs only). */ @@ -155,15 +167,37 @@ function parseStringCore( if (value !== null && typeof value === "object" && !Array.isArray(value)) { return parseObjectValue(value as Record, callId, diag); } + if (typeof value === "string") { + // #1559 G2: a JSON string literal that decodes to line-form text + // (double-wrapped bare string, or a gateway quoting the payload). + const recovered = salvageLineFormEntries(value, callId); + if (recovered.ranges.length > 0) + return finishLineFormSalvage(recovered, diag, false); + } if (value !== undefined) { // Parsed, but not to an object: bare array, number, boolean, null. diag.kind = "not-object"; return finish([], diag); } - // Unparseable prefix: salvage the complete content-array entries. - const entries = salvageContentEntries(cleaned); - return finishSalvage(entries, callId, diag, looksTruncated(cleaned)); + // Unparseable prefix: salvage in order of specificity — complete + // content-array objects, then a string-slot line-form prefix (#1559 G3), + // then bare line-form text with no shell at all (#1559 G2). + let entries = salvageContentEntries(cleaned); + let fromStringSlot = false; + if (entries.length === 0) { + entries = salvageStringSlotEntries(cleaned); + fromStringSlot = entries.length > 0; + } + if (entries.length > 0) { + const out = finishSalvage(entries, callId, diag, true); + if (fromStringSlot && out.ranges.length > 0) diag.contentSalvage = true; + return out; + } + const recovered = salvageLineFormEntries(cleaned, callId); + if (recovered.ranges.length > 0) + return finishLineFormSalvage(recovered, diag, looksTruncated(cleaned)); + return finishSalvage([], callId, diag, looksTruncated(cleaned)); } function parseObjectValue( @@ -201,13 +235,15 @@ function parseObjectValue( // Not a JSON array (or nothing salvageable): bare line-form text. // Strip JSON-wrapper residue (stringified ["…"] with broken // escaping) — the line form needs structure only in the refs - // header line, so the summary body never has to parse. + // header line, so the summary body never has to parse. #1559 G1: + // a single-line payload takes the text after the refs as its + // inline summary (weak models and lossy gateways drop the break). const bare = stripJsonWrapperResidue(content); - const entries = splitLineEntries(bare); + const entries = inlineSingleLineEntry(splitLineEntries(bare)); const { ranges, invalid, reasons } = validateEntries(entries, callId); + diag.invalidItems = invalid; + if (reasons.length > 0) diag.invalidReasons = reasons; if (ranges.length > 0) { - diag.invalidItems = invalid; - if (reasons.length > 0) diag.invalidReasons = reasons; diag.kind = "ok"; return finish(ranges, diag); } @@ -840,6 +876,87 @@ function looksTruncated(s: string): boolean { return depth > 0 || inString; } +// #1559 G1: the line-form path received exactly ONE part and it holds no +// line break — the model or gateway collapsed the documented "header line +// + body" into one line. Re-home the text after the refs header as the +// summary body by synthesizing the missing break. A header whose remainder +// holds no letter or digit (empty, or punctuation like "…") stays +// header-only and is rejected as before. Never splits on embedded ref +// pairs: prose citations look identical to headers, and mis-splitting +// would corrupt a legitimate summary — under-recovery (first block only) +// stays visible in the receipt and retries cleanly. +function inlineSingleLineEntry(entries: unknown[]): unknown[] { + if (entries.length !== 1) return entries; + const e = entries[0]; + if (typeof e !== "string" || e.includes("\n")) return entries; + const head = e.trim(); + const m = REF_PAIR_IN_LINE.exec(head) ?? SINGLE_REF_IN_LINE.exec(head); + if (m === null) return entries; + const rest = head.slice(m.index + m[0].length).trim(); + if (!/\p{L}|\p{N}/u.test(rest)) return entries; + return [m[0] + "\n" + rest]; +} + +// #1559 G2: line-form entries recovered from a bare (non-JSON) string — +// the raw input with no JSON shell, or a JSON string literal that decoded +// to text. Zero ranges means "keep your own verdict". +function salvageLineFormEntries( + text: string, + callId: string | undefined, +): { ranges: CompressRangeSpec[]; invalid: number; reasons: string[] } { + return validateEntries( + inlineSingleLineEntry(splitLineEntries(stripJsonWrapperResidue(text))), + callId, + ); +} + +function finishLineFormSalvage( + result: { ranges: CompressRangeSpec[]; invalid: number; reasons: string[] }, + diag: CompressParseDiagnostics, + truncatedShape: boolean, +): ParsedCompressInput { + diag.stringSalvage = true; + diag.invalidItems = result.invalid; + if (result.reasons.length > 0) diag.invalidReasons = result.reasons; + diag.kind = truncatedShape ? "truncated" : "ok"; + return finish(result.ranges, diag); +} + +/** + * #1559 G3: the stream cut a STRINGIFIED call whose content slot held a + * string rather than an array. Extracts the surviving slot prefix (up to + * the closing quote, or EOF when unterminated), decodes \n escapes, and + * returns the line-form blocks. Never-guess rule: while the slot is + * unterminated the trailing block may be mid-write and is dropped; an + * unterminated single block yields nothing. + */ +function salvageStringSlotEntries(raw: string): unknown[] { + const match = /"content"\s*:\s*"/.exec(raw); + if (match === null) return []; + const start = match.index + match[0].length; + let i = start; + let terminated = false; + while (i < raw.length) { + const ch = raw.charAt(i); + if (ch === "\\") { + i += 2; + continue; + } + if (ch === '"') { + terminated = true; + break; + } + i++; + } + const body = raw.slice(start, i).replace(/\\n/g, "\n"); + const parts = splitLineEntries(body); + if (!terminated) { + if (parts.length <= 1) return []; + parts.pop(); + } + return parts; +} + /** * Recover the complete entries of the `content` array from a truncated JSON * prefix. Walks the prefix with a small state machine (string / escape / diff --git a/tests/parse-compress-input.test.ts b/tests/parse-compress-input.test.ts index c5152fe..3af412d 100644 --- a/tests/parse-compress-input.test.ts +++ b/tests/parse-compress-input.test.ts @@ -1236,3 +1236,179 @@ test("line-form split still breaks on headers after blank lines and element resi ], ); }); + +// #1559: three string-ladder rungs (billion-context#1559) — weak-model drift +// shapes the 0.0.98 ladder still rejected: single-line slot string (G1), +// top-level bare line-form string (G2), truncated string-slot prefix (G3). +// --------------------------------------------------------------------------- + +test("G1: single-line slot string takes the text after the refs as inline summary (#1559)", () => { + const { ranges, diagnostics } = parseCompressArgs({ + content: "m00150-m00220 topic: summary here", + }); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.startRef, "m00150"); + assert.equal(ranges[0]?.endRef, "m00220"); + assert.equal(ranges[0]?.summary, "topic: summary here"); + assert.equal(ranges[0]?.topic, "topic: summary here"); + assert.equal(diagnostics.contentSalvage, undefined); + assert.equal(diagnostics.stringSalvage, undefined); +}); + +test("G1: en-dash and single-ref variants of the single-line slot string (#1559)", () => { + const a = parseCompressArgs({ content: "m00150–m00220 inline body text" }); + assert.equal(a.diagnostics.kind, "ok"); + assert.equal(a.ranges.length, 1); + assert.equal(a.ranges[0]?.startRef, "m00150"); + assert.equal(a.ranges[0]?.endRef, "m00220"); + assert.equal(a.ranges[0]?.summary, "inline body text"); + const b = parseCompressArgs({ content: "m00042 lone ref with inline body" }); + assert.equal(b.ranges.length, 1); + assert.equal(b.ranges[0]?.startRef, "m00042"); + assert.equal(b.ranges[0]?.endRef, "m00042"); + assert.equal(b.ranges[0]?.summary, "lone ref with inline body"); +}); + +test("G1: single-line slot string with nothing after the refs stays rejected (#1559)", () => { + const { ranges, diagnostics } = parseCompressArgs({ content: "m1–m5 …" }); + assert.equal(ranges.length, 0); + assert.equal(diagnostics.kind, "content-not-array"); + assert.match( + String(diagnostics.invalidReasons?.[0] ?? ""), + /missing summary/, + ); +}); + +test("G1: a collapsed multi-block single line recovers the first block only, never splits (#1559)", () => { + // Embedded ref pairs inside prose are indistinguishable from headers; + // splitting would corrupt a legitimate summary, so the whole remainder is + // the first block's summary and the rest stays visible for a retry. + const { ranges, diagnostics } = parseCompressArgs({ + content: + "m00150–m00220 Auth summary one here m00300–m00350 Deploy summary two here", + }); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.startRef, "m00150"); + assert.equal(ranges[0]?.endRef, "m00220"); + assert.equal( + ranges[0]?.summary, + "Auth summary one here m00300–m00350 Deploy summary two here", + ); +}); + +test("G1: a single-line ARRAY element is still header-only, not inline (#1559)", () => { + // Array transport has sibling adoption (#1001); inline interpretation + // would invent a summary from the topic. Header-only stays rejected. + const { ranges, diagnostics } = parseCompressArgs({ + content: ["m00001–m00002 topic"], + }); + assert.equal(ranges.length, 0); + assert.equal(diagnostics.kind, "no-valid-ranges"); +}); + +test("G2: top-level bare line-form string recovers with stringSalvage (#1559)", () => { + const { ranges, diagnostics } = parseCompressArgs( + "m00150–m00220 Auth\nsummary one\nm00300–m00350 Deploy\nsummary two", + ); + assert.equal(diagnostics.kind, "ok"); + assert.equal(diagnostics.stringSalvage, true); + assert.equal(ranges.length, 2); + assert.equal(ranges[0]?.topic, "Auth"); + assert.equal(ranges[0]?.summary, "summary one"); + assert.equal(ranges[1]?.startRef, "m00300"); + assert.equal(ranges[1]?.summary, "summary two"); +}); + +test("G2: top-level bare single-line string uses the inline summary (#1559)", () => { + const { ranges, diagnostics } = parseCompressArgs( + "m00150-m00220 topic: summary here", + ); + assert.equal(diagnostics.kind, "ok"); + assert.equal(diagnostics.stringSalvage, true); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "topic: summary here"); +}); + +test("G2: a JSON string literal decoding to line-form text recovers (#1559)", () => { + const input = JSON.stringify("m00150–m00220 t\nsum"); + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "ok"); + assert.equal(diagnostics.stringSalvage, true); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "sum"); +}); + +test("G2: prose without refs stays rejected, verdicts unchanged (#1559)", () => { + const a = parseCompressArgs("please compress the old stuff"); + assert.equal(a.ranges.length, 0); + assert.equal(a.diagnostics.kind, "malformed-json"); + assert.equal(a.diagnostics.stringSalvage, undefined); + const b = parseCompressArgs(JSON.stringify("just a string")); + assert.equal(b.ranges.length, 0); + assert.equal(b.diagnostics.kind, "not-object"); +}); + +test("G3: truncated stringified args with a string slot recover complete leading blocks (#1559)", () => { + const input = + '{"content": "m00150–m00220 Auth\nfull summary one\nm00300–m00350 Dep'; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "truncated"); + assert.equal(diagnostics.ok, true); + assert.equal(diagnostics.contentSalvage, true); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.startRef, "m00150"); + assert.equal(ranges[0]?.endRef, "m00220"); + assert.equal(ranges[0]?.summary, "full summary one"); +}); + +test("G3: a single cut block in an unterminated string slot is dropped, not guessed (#1559)", () => { + const input = '{"content": "m00150–m00220 Auth\npart'; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "truncated"); + assert.equal(ranges.length, 0); + assert.equal(diagnostics.contentSalvage, undefined); +}); + +test("G3: a terminated string slot inside a truncated outer object keeps all blocks (#1559)", () => { + const input = + '{"content": "m00150–m00220 Auth\nfull summary one","topic": "x'; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "truncated"); + assert.equal(diagnostics.contentSalvage, true); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "full summary one"); +}); + +test("G3: escaped backslash-n in the surviving prefix decodes to line breaks (#1559)", () => { + const input = + '{"content": "m00150–m00220 Auth\\nfull summary one\\nm00300–m00350 Dep'; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "truncated"); + assert.equal(diagnostics.contentSalvage, true); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "full summary one"); +}); + +test("G3: single-quoted truncated string slot recovers through the quote-repair retry (#1559)", () => { + const input = `{'content': "m00150–m00220 Auth\nfull summary one\nm00300–m00350 Dep`; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "truncated"); + assert.equal(diagnostics.quoteSalvage, true); + assert.equal(diagnostics.contentSalvage, true); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "full summary one"); +}); + +test("canonical multi-line slot string stays unflagged (#1559 control)", () => { + const { ranges, diagnostics } = parseCompressArgs({ + content: "m00150–m00220 topic\nsummary line one\nmore summary", + }); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.topic, "topic"); + assert.equal(ranges[0]?.summary, "summary line one\nmore summary"); + assert.equal(diagnostics.stringSalvage, undefined); + assert.equal(diagnostics.contentSalvage, undefined); +});