Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
135 changes: 126 additions & 9 deletions src/parse-compress-input.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@
// billion-context#603 / omp#121)
// - a single entry OBJECT where the content array belongs (retry drift,
// billion-context#1494 / acp-kernel#470)
// - line-form text with no JSON shell at all: a bare top-level string, a
// string literal decoding to text, or a single-line slot string whose
// summary sits inline after the refs (weak-model syntax drift,
// billion-context#1559)
// - the stream cut off mid-arguments, leaving a truncated JSON prefix
//
// Hosts parse this on their own today: rebuild.ts (strict, silent skip),
Expand All @@ -21,8 +25,13 @@
//
// Salvage semantics: for truncated input, the complete entries of the
// `content` array are recovered from the surviving prefix. A partially
// written entry is dropped, never guessed. Diagnostics are data, not logs:
// adapters decide where to emit them (log line, debug event, tool text).
// written entry is dropped, never guessed. Line-form salvage follows the
// same rule: a cut string-slot prefix keeps every complete leading block
// and drops the trailing partial one; a single line is never split on
// embedded ref pairs, because prose citations are indistinguishable from
// headers and mis-splitting corrupts a legitimate summary. Diagnostics are
// data, not logs: adapters decide where to emit them (log line, debug
// event, tool text).

import { clampPrefix } from "./truncate.js";
import type { CompressRangeSpec } from "./types.js";
Expand All @@ -47,6 +56,9 @@ export interface CompressParseDiagnostics {
/** True when a non-array `content` value (single entry object or nested
* `ranges` array) was recovered into the canonical array form (#470). */
contentSalvage?: boolean;
/** True when line-form text outside any JSON shell was recovered: a bare
* top-level string or a string literal decoding to text (#1559). */
stringSalvage?: boolean;
/** First 800 chars of the raw string input (string inputs only). */
rawPrefix?: string;
/** Raw string input length (string inputs only). */
Expand Down Expand Up @@ -155,15 +167,37 @@ function parseStringCore(
if (value !== null && typeof value === "object" && !Array.isArray(value)) {
return parseObjectValue(value as Record<string, unknown>, callId, diag);
}
if (typeof value === "string") {
// #1559 G2: a JSON string literal that decodes to line-form text
// (double-wrapped bare string, or a gateway quoting the payload).
const recovered = salvageLineFormEntries(value, callId);
if (recovered.ranges.length > 0)
return finishLineFormSalvage(recovered, diag, false);
}
if (value !== undefined) {
// Parsed, but not to an object: bare array, number, boolean, null.
diag.kind = "not-object";
return finish([], diag);
}

// Unparseable prefix: salvage the complete content-array entries.
const entries = salvageContentEntries(cleaned);
return finishSalvage(entries, callId, diag, looksTruncated(cleaned));
// Unparseable prefix: salvage in order of specificity — complete
// content-array objects, then a string-slot line-form prefix (#1559 G3),
// then bare line-form text with no shell at all (#1559 G2).
let entries = salvageContentEntries(cleaned);
let fromStringSlot = false;
if (entries.length === 0) {
entries = salvageStringSlotEntries(cleaned);
fromStringSlot = entries.length > 0;
}
if (entries.length > 0) {
const out = finishSalvage(entries, callId, diag, true);
if (fromStringSlot && out.ranges.length > 0) diag.contentSalvage = true;
return out;
}
const recovered = salvageLineFormEntries(cleaned, callId);
if (recovered.ranges.length > 0)
return finishLineFormSalvage(recovered, diag, looksTruncated(cleaned));
return finishSalvage([], callId, diag, looksTruncated(cleaned));
}

function parseObjectValue(
Expand Down Expand Up @@ -201,13 +235,15 @@ function parseObjectValue(
// Not a JSON array (or nothing salvageable): bare line-form text.
// Strip JSON-wrapper residue (stringified ["…"] with broken
// escaping) — the line form needs structure only in the refs
// header line, so the summary body never has to parse.
// header line, so the summary body never has to parse. #1559 G1:
// a single-line payload takes the text after the refs as its
// inline summary (weak models and lossy gateways drop the break).
const bare = stripJsonWrapperResidue(content);
const entries = splitLineEntries(bare);
const entries = inlineSingleLineEntry(splitLineEntries(bare));
const { ranges, invalid, reasons } = validateEntries(entries, callId);
diag.invalidItems = invalid;
if (reasons.length > 0) diag.invalidReasons = reasons;
if (ranges.length > 0) {
diag.invalidItems = invalid;
if (reasons.length > 0) diag.invalidReasons = reasons;
diag.kind = "ok";
return finish(ranges, diag);
}
Expand Down Expand Up @@ -840,6 +876,87 @@ function looksTruncated(s: string): boolean {
return depth > 0 || inString;
}

// #1559 G1: the line-form path received exactly ONE part and it holds no
// line break — the model or gateway collapsed the documented "header line
// + body" into one line. Re-home the text after the refs header as the
// summary body by synthesizing the missing break. A header whose remainder
// holds no letter or digit (empty, or punctuation like "…") stays
// header-only and is rejected as before. Never splits on embedded ref
// pairs: prose citations look identical to headers, and mis-splitting
// would corrupt a legitimate summary — under-recovery (first block only)
// stays visible in the receipt and retries cleanly.
function inlineSingleLineEntry(entries: unknown[]): unknown[] {
if (entries.length !== 1) return entries;
const e = entries[0];
if (typeof e !== "string" || e.includes("\n")) return entries;
const head = e.trim();
const m = REF_PAIR_IN_LINE.exec(head) ?? SINGLE_REF_IN_LINE.exec(head);
if (m === null) return entries;
const rest = head.slice(m.index + m[0].length).trim();
if (!/\p{L}|\p{N}/u.test(rest)) return entries;
return [m[0] + "\n" + rest];
}

// #1559 G2: line-form entries recovered from a bare (non-JSON) string —
// the raw input with no JSON shell, or a JSON string literal that decoded
// to text. Zero ranges means "keep your own verdict".
function salvageLineFormEntries(
text: string,
callId: string | undefined,
): { ranges: CompressRangeSpec[]; invalid: number; reasons: string[] } {
return validateEntries(
inlineSingleLineEntry(splitLineEntries(stripJsonWrapperResidue(text))),
callId,
);
}

function finishLineFormSalvage(
result: { ranges: CompressRangeSpec[]; invalid: number; reasons: string[] },
diag: CompressParseDiagnostics,
truncatedShape: boolean,
): ParsedCompressInput {
diag.stringSalvage = true;
diag.invalidItems = result.invalid;
if (result.reasons.length > 0) diag.invalidReasons = result.reasons;
diag.kind = truncatedShape ? "truncated" : "ok";
return finish(result.ranges, diag);
}

/**
* #1559 G3: the stream cut a STRINGIFIED call whose content slot held a
* string rather than an array. Extracts the surviving slot prefix (up to
* the closing quote, or EOF when unterminated), decodes \n escapes, and
* returns the line-form blocks. Never-guess rule: while the slot is
* unterminated the trailing block may be mid-write and is dropped; an
* unterminated single block yields nothing.
*/
function salvageStringSlotEntries(raw: string): unknown[] {
const match = /"content"\s*:\s*"/.exec(raw);
if (match === null) return [];
const start = match.index + match[0].length;
let i = start;
let terminated = false;
while (i < raw.length) {
const ch = raw.charAt(i);
if (ch === "\\") {
i += 2;
continue;
}
if (ch === '"') {
terminated = true;
break;
}
i++;
}
const body = raw.slice(start, i).replace(/\\n/g, "\n");
const parts = splitLineEntries(body);
if (!terminated) {
if (parts.length <= 1) return [];
parts.pop();
}
return parts;
}

/**
* Recover the complete entries of the `content` array from a truncated JSON
* prefix. Walks the prefix with a small state machine (string / escape /
Expand Down
176 changes: 176 additions & 0 deletions tests/parse-compress-input.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1236,3 +1236,179 @@ test("line-form split still breaks on headers after blank lines and element resi
],
);
});

// #1559: three string-ladder rungs (billion-context#1559) — weak-model drift
// shapes the 0.0.98 ladder still rejected: single-line slot string (G1),
// top-level bare line-form string (G2), truncated string-slot prefix (G3).
// ---------------------------------------------------------------------------

test("G1: single-line slot string takes the text after the refs as inline summary (#1559)", () => {
const { ranges, diagnostics } = parseCompressArgs({
content: "m00150-m00220 topic: summary here",
});
assert.equal(diagnostics.kind, "ok");
assert.equal(ranges.length, 1);
assert.equal(ranges[0]?.startRef, "m00150");
assert.equal(ranges[0]?.endRef, "m00220");
assert.equal(ranges[0]?.summary, "topic: summary here");
assert.equal(ranges[0]?.topic, "topic: summary here");
assert.equal(diagnostics.contentSalvage, undefined);
assert.equal(diagnostics.stringSalvage, undefined);
});

test("G1: en-dash and single-ref variants of the single-line slot string (#1559)", () => {
const a = parseCompressArgs({ content: "m00150–m00220 inline body text" });
assert.equal(a.diagnostics.kind, "ok");
assert.equal(a.ranges.length, 1);
assert.equal(a.ranges[0]?.startRef, "m00150");
assert.equal(a.ranges[0]?.endRef, "m00220");
assert.equal(a.ranges[0]?.summary, "inline body text");
const b = parseCompressArgs({ content: "m00042 lone ref with inline body" });
assert.equal(b.ranges.length, 1);
assert.equal(b.ranges[0]?.startRef, "m00042");
assert.equal(b.ranges[0]?.endRef, "m00042");
assert.equal(b.ranges[0]?.summary, "lone ref with inline body");
});

test("G1: single-line slot string with nothing after the refs stays rejected (#1559)", () => {
const { ranges, diagnostics } = parseCompressArgs({ content: "m1–m5 …" });
assert.equal(ranges.length, 0);
assert.equal(diagnostics.kind, "content-not-array");
assert.match(
String(diagnostics.invalidReasons?.[0] ?? ""),
/missing summary/,
);
});

test("G1: a collapsed multi-block single line recovers the first block only, never splits (#1559)", () => {
// Embedded ref pairs inside prose are indistinguishable from headers;
// splitting would corrupt a legitimate summary, so the whole remainder is
// the first block's summary and the rest stays visible for a retry.
const { ranges, diagnostics } = parseCompressArgs({
content:
"m00150–m00220 Auth summary one here m00300–m00350 Deploy summary two here",
});
assert.equal(diagnostics.kind, "ok");
assert.equal(ranges.length, 1);
assert.equal(ranges[0]?.startRef, "m00150");
assert.equal(ranges[0]?.endRef, "m00220");
assert.equal(
ranges[0]?.summary,
"Auth summary one here m00300–m00350 Deploy summary two here",
);
});

test("G1: a single-line ARRAY element is still header-only, not inline (#1559)", () => {
// Array transport has sibling adoption (#1001); inline interpretation
// would invent a summary from the topic. Header-only stays rejected.
const { ranges, diagnostics } = parseCompressArgs({
content: ["m00001–m00002 topic"],
});
assert.equal(ranges.length, 0);
assert.equal(diagnostics.kind, "no-valid-ranges");
});

test("G2: top-level bare line-form string recovers with stringSalvage (#1559)", () => {
const { ranges, diagnostics } = parseCompressArgs(
"m00150–m00220 Auth\nsummary one\nm00300–m00350 Deploy\nsummary two",
);
assert.equal(diagnostics.kind, "ok");
assert.equal(diagnostics.stringSalvage, true);
assert.equal(ranges.length, 2);
assert.equal(ranges[0]?.topic, "Auth");
assert.equal(ranges[0]?.summary, "summary one");
assert.equal(ranges[1]?.startRef, "m00300");
assert.equal(ranges[1]?.summary, "summary two");
});

test("G2: top-level bare single-line string uses the inline summary (#1559)", () => {
const { ranges, diagnostics } = parseCompressArgs(
"m00150-m00220 topic: summary here",
);
assert.equal(diagnostics.kind, "ok");
assert.equal(diagnostics.stringSalvage, true);
assert.equal(ranges.length, 1);
assert.equal(ranges[0]?.summary, "topic: summary here");
});

test("G2: a JSON string literal decoding to line-form text recovers (#1559)", () => {
const input = JSON.stringify("m00150–m00220 t\nsum");
const { ranges, diagnostics } = parseCompressArgs(input);
assert.equal(diagnostics.kind, "ok");
assert.equal(diagnostics.stringSalvage, true);
assert.equal(ranges.length, 1);
assert.equal(ranges[0]?.summary, "sum");
});

test("G2: prose without refs stays rejected, verdicts unchanged (#1559)", () => {
const a = parseCompressArgs("please compress the old stuff");
assert.equal(a.ranges.length, 0);
assert.equal(a.diagnostics.kind, "malformed-json");
assert.equal(a.diagnostics.stringSalvage, undefined);
const b = parseCompressArgs(JSON.stringify("just a string"));
assert.equal(b.ranges.length, 0);
assert.equal(b.diagnostics.kind, "not-object");
});

test("G3: truncated stringified args with a string slot recover complete leading blocks (#1559)", () => {
const input =
'{"content": "m00150–m00220 Auth\nfull summary one\nm00300–m00350 Dep';
const { ranges, diagnostics } = parseCompressArgs(input);
assert.equal(diagnostics.kind, "truncated");
assert.equal(diagnostics.ok, true);
assert.equal(diagnostics.contentSalvage, true);
assert.equal(ranges.length, 1);
assert.equal(ranges[0]?.startRef, "m00150");
assert.equal(ranges[0]?.endRef, "m00220");
assert.equal(ranges[0]?.summary, "full summary one");
});

test("G3: a single cut block in an unterminated string slot is dropped, not guessed (#1559)", () => {
const input = '{"content": "m00150–m00220 Auth\npart';
const { ranges, diagnostics } = parseCompressArgs(input);
assert.equal(diagnostics.kind, "truncated");
assert.equal(ranges.length, 0);
assert.equal(diagnostics.contentSalvage, undefined);
});

test("G3: a terminated string slot inside a truncated outer object keeps all blocks (#1559)", () => {
const input =
'{"content": "m00150–m00220 Auth\nfull summary one","topic": "x';
const { ranges, diagnostics } = parseCompressArgs(input);
assert.equal(diagnostics.kind, "truncated");
assert.equal(diagnostics.contentSalvage, true);
assert.equal(ranges.length, 1);
assert.equal(ranges[0]?.summary, "full summary one");
});

test("G3: escaped backslash-n in the surviving prefix decodes to line breaks (#1559)", () => {
const input =
'{"content": "m00150–m00220 Auth\\nfull summary one\\nm00300–m00350 Dep';
const { ranges, diagnostics } = parseCompressArgs(input);
assert.equal(diagnostics.kind, "truncated");
assert.equal(diagnostics.contentSalvage, true);
assert.equal(ranges.length, 1);
assert.equal(ranges[0]?.summary, "full summary one");
});

test("G3: single-quoted truncated string slot recovers through the quote-repair retry (#1559)", () => {
const input = `{'content': "m00150–m00220 Auth\nfull summary one\nm00300–m00350 Dep`;
const { ranges, diagnostics } = parseCompressArgs(input);
assert.equal(diagnostics.kind, "truncated");
assert.equal(diagnostics.quoteSalvage, true);
assert.equal(diagnostics.contentSalvage, true);
assert.equal(ranges.length, 1);
assert.equal(ranges[0]?.summary, "full summary one");
});

test("canonical multi-line slot string stays unflagged (#1559 control)", () => {
const { ranges, diagnostics } = parseCompressArgs({
content: "m00150–m00220 topic\nsummary line one\nmore summary",
});
assert.equal(diagnostics.kind, "ok");
assert.equal(ranges.length, 1);
assert.equal(ranges[0]?.topic, "topic");
assert.equal(ranges[0]?.summary, "summary line one\nmore summary");
assert.equal(diagnostics.stringSalvage, undefined);
assert.equal(diagnostics.contentSalvage, undefined);
});
Loading