From 1d3942fad5aca42fa0fe822c5afed0d5a4127288 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Wed, 30 Sep 2026 13:49:54 +0800 Subject: [PATCH] fix: strip case-drifted render tags; uncapped open-side attrs in tag-echo filter (#1731) --- src/loop/tag-echo-filter.ts | 56 ++++++++++++++++++-------- tests/tag-echo.test.ts | 78 ++++++++++++++++++++++++++++++------- 2 files changed, 104 insertions(+), 30 deletions(-) diff --git a/src/loop/tag-echo-filter.ts b/src/loop/tag-echo-filter.ts index 260aac09de..c9d9018627 100644 --- a/src/loop/tag-echo-filter.ts +++ b/src/loop/tag-echo-filter.ts @@ -20,10 +20,18 @@ import { CODEX_FORGED_HANDOFF_HEADER, FORGED_SUMMARY_HEADER } from "../codex-com // including mixed correct-open + typo'd-close), so matches a bounded // mutation set instead of the exact spelling: the three core letters in any // order, plus at most ONE extra letter drawn from that set or an inserted i. -// Every match still requires the name to be followed by \s or > (attrs or -// close), so real words that merely contain the letters (acpi/acpi.h includes, -// caption, app, uppercase ACPI) never match; a false positive costs at most -// the same bounded caps as before (swallow ≤ SWALLOW_CAP, hold ≤ HOLD_LIMIT/TAG_OPEN_CAP). +// #1731: and the set folds case — models drift casing too, and uppercase +// echoes (\x3cACP …\x3e, \x3c/ACP\x3e) leaked verbatim because every name pattern was a +// lowercase literal. The folding is baked into the alternation itself as +// per-letter character classes rather than an `i` flag, so the .source-based +// reconstructions below (stripAcpTags, flush) stay behavior-identical: a +// flag would be silently dropped at every rebuild site. Every match still +// requires the name to be followed by \s or > (attrs or close), so real +// words that merely contain the letters (acpi/acpi.h includes, caption, app) +// never match; an angle-bracketed bare token like \x3cACPI\x3e is indistinguishable +// from a casing-drifted tag and is stripped, same trade as its lowercase form. +// A false positive costs at most the same bounded caps as before +// (swallow ≤ SWALLOW_CAP, hold ≤ HOLD_LIMIT/TAG_OPEN_CAP). // // ─── INVARIANT (#1039): tool-call arguments are user intent ───────────────── // Anything the host will EXECUTE or PERSIST — tool-call arguments in every @@ -41,34 +49,48 @@ import { CODEX_FORGED_HANDOFF_HEADER, FORGED_SUMMARY_HEADER } from "../codex-com // (content/reasoning_content/reasoning/thinking/text/summary fields). function buildAcplikeName(): string { const cores = ["acp", "apc", "cap", "cpa", "pac", "pca"]; - const names = new Set(cores); + const ci = (ch: string) => `[${ch}${ch.toUpperCase()}]`; + const fourLetter = new Set(); + const threeLetter = new Set(); for (const c of cores) { - for (const ch of ["a", "c", "p", "i"]) { - for (let pos = 0; pos <= c.length; pos++) names.add(c.slice(0, pos) + ch + c.slice(pos)); + threeLetter.add(ci(c.charAt(0)) + ci(c.charAt(1)) + ci(c.charAt(2))); + for (let pos = 0; pos <= c.length; pos++) { + fourLetter.add( + c.slice(0, pos).split("").map(ci).join("") + "[aAcCpPiI]" + c.slice(pos).split("").map(ci).join(""), + ); } } - return [...names].sort((a, b) => b.length - a.length).join("|"); + return [...fourLetter, ...threeLetter].join("|"); } -/** Longest-first alternation of every tolerated render-tag name (#673). */ +/** Longest-first alternation of every tolerated render-tag name (#673), case-folded via letter classes (#1731). */ export const ACP_NAME_ALT = `(?:${buildAcplikeName()})`; const NAME = ACP_NAME_ALT; -// Opening-tag attrs are bounded: a render tag opening is short (tokens + type, -// \x3c 50 chars). An unbounded \x3c …\x3e match would swallow a long prose span -// that merely starts with a tag head and contains a \x3e somewhere later. +// Opening-tag attrs carry NO length cap (#1731): [^<>] cannot cross an angle +// bracket, so these matchers stay linear for arbitrarily long attr runs, and +// a cap silently defined a bypass — a longer run escaped every open-tag +// matcher while the loose close still went, leaving orphan markup on the wire. +// Prose safety lives elsewhere: a terminated open is decided by the body's +// shape (#1720), an unterminated one by the definite-tail budget (TAG_OPEN_CAP) +// and the #644 release rules. Close-side tails keep their {0,32} bound — that +// one is load-bearing (#644: an unbounded close-side tail ate real content +// after a malformed close). // A render tag wraps exactly one bare ref: the kernel emits mNNNNN and nothing else between the tags (#1720). Content that // is not a ref is prose wearing tags — the tags go, the content stays. No g -// flag: createTagEchoFilter drives it with exec() on a sliding buffer. -const PAIRED = new RegExp("\x3c" + NAME + "\\s[^<>]{0,256}>(\\s*m\\d{4,}\\s*)\x3c\\/" + NAME + ">"); +// flag: createTagEchoFilter drives it with exec() on a sliding buffer. It is +// flag-free by construction — case folding lives inside ACP_NAME_ALT's letter +// classes (#1731) — so every .source reconstruction below preserves behavior +// verbatim; an `i` flag would be silently dropped at each rebuild site. +const PAIRED = new RegExp("\x3c" + NAME + "\\s[^<>]*>(\\s*m\\d{4,}\\s*)\x3c\\/" + NAME + ">"); const REF_LIKE = /^\s*m\d{4,}\s*$/; -const LONE_OPEN = new RegExp("\x3c" + NAME + "(?:\\s[^<>]{0,256})?>"); +const LONE_OPEN = new RegExp("\x3c" + NAME + "(?:\\s[^<>]*)?>"); const LONE_CLOSE = new RegExp("\x3c\\/" + NAME + "(?=[\\s>])[^<>]{0,32}>"); // A suffix of the buffer that could still grow into a render tag: either an // unterminated \x3c … opening (attrs so far, no \x3e yet) or a short // ambiguous prefix like \x3c, \x3ca, \x3c/ac, \x3cacip, … -const PARTIAL_TAIL = new RegExp("(\x3c" + NAME + "\\s[^<>]*|\x3c\\/" + NAME + "(?:\\s[^<>]{0,32})?|\x3c\\/?[acip]*)$"); +const PARTIAL_TAIL = new RegExp("(\x3c" + NAME + "\\s[^<>]*|\x3c\\/" + NAME + "(?:\\s[^<>]{0,32})?|\x3c\\/?[aAcCpPiI]*)$"); // An unterminated render-tag opening at the end of a string: \x3c plus // attrs, no \x3e — a truncated imitation, never prose (triggers use \x3cacp_). const TRUNC_OPEN = new RegExp("\x3c" + NAME + "\\s[^<>]*$"); @@ -85,7 +107,7 @@ const TRUNC_CLOSE = new RegExp("\x3c\\/" + NAME + "(?:\\s[^<>]{0,32})?$"); // shape: a render-tag name, whitespace, then an attribute list bounded by the // next `<`. A properly terminated opening never matches: its attribute list // ends at a `>`, and no `<` can be reached from there within the class. -const BROKEN_ATTRS = new RegExp("\x3c" + NAME + "\\s[^<>]{0,512}(?=\x3c)"); +const BROKEN_ATTRS = new RegExp("\x3c" + NAME + "\\s[^<>]*(?=\x3c)"); const DEFINITE_TAIL = new RegExp("^\x3c" + NAME + "\\s|^\x3c\\/" + NAME); const OPEN_WITH_ATTRS = new RegExp("^\x3c" + NAME + "\\s"); const CLOSE_HEAD = "\x3c/"; diff --git a/tests/tag-echo.test.ts b/tests/tag-echo.test.ts index b5eeb02a42..423ac2dc06 100644 --- a/tests/tag-echo.test.ts +++ b/tests/tag-echo.test.ts @@ -502,23 +502,75 @@ test("streaming filter still drops a ref echo whose open tag arrives alone in a assert.ok(f.dropped()); }); -test("long tag-like spans are prose, not tags (bounded open match)", () => { +// #1731 replaced the old open-side attr caps ({0,256}/{0,512}): a longer run +// escaped every open-tag matcher while the loose close still went, leaving +// orphan markup on the wire. [^<>] cannot cross a bracket, so the uncapped +// matchers stay linear; a terminated open is now decided by body shape +// (#1720), never by attr length. These tests supersede the former +// "long attr run = prose" pins, which encoded the bypass itself. +test("long-attr paired render tags are stripped, whole-text and at every split (#1731)", () => { + const attrs = "x".repeat(300); + const full = `before ${OPEN}${attrs}>m00155${CLOSE} after`; + const expected = stripAcpTags(full); + assert.equal(expected, "before after"); + for (let split = 0; split <= full.length; split++) { + const f = createTagEchoFilter(); + const out = f.push(full.slice(0, split)) + f.push(full.slice(split)) + f.flush(); + assert.equal(out, expected, `split=${split}`); + } +}); + +test("terminated long-attr open without a close is still a tag head (#1731)", () => { const span = `${OPEN}${"x".repeat(300)}>`; - assert.equal(stripAcpTags(`before ${span} after`), `before ${span} after`); + assert.equal(stripAcpTags(`before ${span} after`), "before after"); const f = createTagEchoFilter(); - assert.equal(f.push(`before ${span} after`) + f.flush(), `before ${span} after`); - const g = createTagEchoFilter(); - assert.equal(g.push(`before ${OPEN}${"x".repeat(300)}`), "before "); - assert.ok(g.pending()); - assert.equal(g.push(`> after`), `${span} after`); - assert.equal(g.flush(), ""); + assert.equal(f.push(`before ${span}`), "before "); + assert.ok(f.pending()); + assert.equal(f.flush(), ""); }); -test("prose with a tag-like opening beyond the attr bound survives intact", () => { - const prose = `${OPEN}${"word ".repeat(60)}end > kept`; - assert.equal(stripAcpTags(prose), prose); - const f = createTagEchoFilter(); - assert.equal(f.push(prose) + f.flush(), prose); +test("case-drifted render tags are stripped, whole-text and at every split (#1731)", () => { + const up = `\x3cACP tokens="1" type="text"\x3em00155\x3c/ACP\x3e`; + const mixed = `\x3cAcP tokens="1" type="text"\x3em00009\x3c/aCP\x3e`; + assert.equal(stripAcpTags(`pre ${up} post`), "pre post"); + assert.equal(stripAcpTags(mixed), ""); + assert.equal(stripAcpTags(`pre \x3cACP tokens="1" type="text"\x3e post`), "pre post"); + assert.equal(stripAcpTags(`pre \x3c/ACP\x3e post`), "pre post"); + const full = `before ${up} after`; + const expected = stripAcpTags(full); + for (let split = 0; split <= full.length; split++) { + const f = createTagEchoFilter(); + const out = f.push(full.slice(0, split)) + f.push(full.slice(split)) + f.flush(); + assert.equal(out, expected, `split=${split}`); + } +}); + +test("case-drifted tag heads engage the streaming gates (#1731)", () => { + assert.equal(containsRenderTagText(`\x3cACP tokens="1"\x3e`), true); + assert.equal(mayStartRenderTag(`chunk \x3cAC`), true); + assert.equal(mayStartRenderTag(`chunk \x3c/A`), true); +}); + +test("prose guards hold under case drift (#1731)", () => { + assert.equal(stripAcpTags(`see the \x3cCaption\x3ex\x3c/Caption\x3e here`), `see the \x3cCaption\x3ex\x3c/Caption\x3e here`); + assert.equal(stripAcpTags(`\x3capp id="1"\x3erun\x3c/app\x3e`), `\x3capp id="1"\x3erun\x3c/app\x3e`); + assert.equal(stripAcpTags(`per the ACPI spec and #include \x3cacpi/acpi.h\x3e`), `per the ACPI spec and #include \x3cacpi/acpi.h\x3e`); + assert.equal(stripAcpTags(`#include \x3cACPI/acpi.h\x3e`), `#include \x3cACPI/acpi.h\x3e`); + assert.equal(stripAcpTags(`\x3cCAPTION\x3etext\x3c/CAPTION\x3e`), `\x3cCAPTION\x3etext\x3c/CAPTION\x3e`); +}); + +test("uncapped attr runs stay linear (no ReDoS) (#1731)", () => { + const big = OPEN + "a".repeat(200_000) + ">m00155" + CLOSE; + const t0 = process.hrtime.bigint(); + const out = stripAcpTags(big); + const ms = Number(process.hrtime.bigint() - t0) / 1e6; + assert.equal(out, ""); + assert.ok(ms < 500, `single 200KB attr run took ${ms.toFixed(1)}ms`); + const heads = Array.from({ length: 500 }, () => OPEN + "b".repeat(400)).join(" prose "); + const t1 = process.hrtime.bigint(); + stripAcpTags(heads); + const ms2 = Number(process.hrtime.bigint() - t1) / 1e6; + assert.ok(ms2 < 1000, `500 unterminated heads took ${ms2.toFixed(1)}ms`); }); test("containsToolCallXmlFragment detects tool-call XML fragments, not prose", () => {