Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 39 additions & 17 deletions src/loop/tag-echo-filter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,18 @@ import { CODEX_FORGED_HANDOFF_HEADER, FORGED_SUMMARY_HEADER } from "../codex-com
// including mixed correct-open + typo'd-close), so <name> matches a bounded
// mutation set instead of the exact spelling: the three core letters in any
// order, plus at most ONE extra letter drawn from that set or an inserted i.
// Every match still requires the name to be followed by \s or > (attrs or
// close), so real words that merely contain the letters (acpi/acpi.h includes,
// caption, app, uppercase ACPI) never match; a false positive costs at most
// the same bounded caps as before (swallow ≤ SWALLOW_CAP, hold ≤ HOLD_LIMIT/TAG_OPEN_CAP).
// #1731: and the set folds case — models drift casing too, and uppercase
// echoes (\x3cACP …\x3e, \x3c/ACP\x3e) leaked verbatim because every name pattern was a
// lowercase literal. The folding is baked into the alternation itself as
// per-letter character classes rather than an `i` flag, so the .source-based
// reconstructions below (stripAcpTags, flush) stay behavior-identical: a
// flag would be silently dropped at every rebuild site. Every match still
// requires the name to be followed by \s or > (attrs or close), so real
// words that merely contain the letters (acpi/acpi.h includes, caption, app)
// never match; an angle-bracketed bare token like \x3cACPI\x3e is indistinguishable
// from a casing-drifted tag and is stripped, same trade as its lowercase form.
// A false positive costs at most the same bounded caps as before
// (swallow ≤ SWALLOW_CAP, hold ≤ HOLD_LIMIT/TAG_OPEN_CAP).
//
// ─── INVARIANT (#1039): tool-call arguments are user intent ─────────────────
// Anything the host will EXECUTE or PERSIST — tool-call arguments in every
Expand All @@ -41,34 +49,48 @@ import { CODEX_FORGED_HANDOFF_HEADER, FORGED_SUMMARY_HEADER } from "../codex-com
// (content/reasoning_content/reasoning/thinking/text/summary fields).
function buildAcplikeName(): string {
const cores = ["acp", "apc", "cap", "cpa", "pac", "pca"];
const names = new Set<string>(cores);
const ci = (ch: string) => `[${ch}${ch.toUpperCase()}]`;
const fourLetter = new Set<string>();
const threeLetter = new Set<string>();
for (const c of cores) {
for (const ch of ["a", "c", "p", "i"]) {
for (let pos = 0; pos <= c.length; pos++) names.add(c.slice(0, pos) + ch + c.slice(pos));
threeLetter.add(ci(c.charAt(0)) + ci(c.charAt(1)) + ci(c.charAt(2)));
for (let pos = 0; pos <= c.length; pos++) {
fourLetter.add(
c.slice(0, pos).split("").map(ci).join("") + "[aAcCpPiI]" + c.slice(pos).split("").map(ci).join(""),
);
}
}
return [...names].sort((a, b) => b.length - a.length).join("|");
return [...fourLetter, ...threeLetter].join("|");
}

/** Longest-first alternation of every tolerated render-tag name (#673). */
/** Longest-first alternation of every tolerated render-tag name (#673), case-folded via letter classes (#1731). */
export const ACP_NAME_ALT = `(?:${buildAcplikeName()})`;
const NAME = ACP_NAME_ALT;

// Opening-tag attrs are bounded: a render tag opening is short (tokens + type,
// \x3c 50 chars). An unbounded \x3c<name> …\x3e match would swallow a long prose span
// that merely starts with a tag head and contains a \x3e somewhere later.
// Opening-tag attrs carry NO length cap (#1731): [^<>] cannot cross an angle
// bracket, so these matchers stay linear for arbitrarily long attr runs, and
// a cap silently defined a bypass — a longer run escaped every open-tag
// matcher while the loose close still went, leaving orphan markup on the wire.
// Prose safety lives elsewhere: a terminated open is decided by the body's
// shape (#1720), an unterminated one by the definite-tail budget (TAG_OPEN_CAP)
// and the #644 release rules. Close-side tails keep their {0,32} bound — that
// one is load-bearing (#644: an unbounded close-side tail ate real content
// after a malformed close).
// A render tag wraps exactly one bare ref: the kernel emits <acp tokens="…"
// type="…">mNNNNN</acp> and nothing else between the tags (#1720). Content that
// is not a ref is prose wearing tags — the tags go, the content stays. No g
// flag: createTagEchoFilter drives it with exec() on a sliding buffer.
const PAIRED = new RegExp("\x3c" + NAME + "\\s[^<>]{0,256}>(\\s*m\\d{4,}\\s*)\x3c\\/" + NAME + ">");
// flag: createTagEchoFilter drives it with exec() on a sliding buffer. It is
// flag-free by construction — case folding lives inside ACP_NAME_ALT's letter
// classes (#1731) — so every .source reconstruction below preserves behavior
// verbatim; an `i` flag would be silently dropped at each rebuild site.
const PAIRED = new RegExp("\x3c" + NAME + "\\s[^<>]*>(\\s*m\\d{4,}\\s*)\x3c\\/" + NAME + ">");
const REF_LIKE = /^\s*m\d{4,}\s*$/;
const LONE_OPEN = new RegExp("\x3c" + NAME + "(?:\\s[^<>]{0,256})?>");
const LONE_OPEN = new RegExp("\x3c" + NAME + "(?:\\s[^<>]*)?>");
const LONE_CLOSE = new RegExp("\x3c\\/" + NAME + "(?=[\\s>])[^<>]{0,32}>");
// A suffix of the buffer that could still grow into a render tag: either an
// unterminated \x3c<name> … opening (attrs so far, no \x3e yet) or a short
// ambiguous prefix like \x3c, \x3ca, \x3c/ac, \x3cacip, …
const PARTIAL_TAIL = new RegExp("(\x3c" + NAME + "\\s[^<>]*|\x3c\\/" + NAME + "(?:\\s[^<>]{0,32})?|\x3c\\/?[acip]*)$");
const PARTIAL_TAIL = new RegExp("(\x3c" + NAME + "\\s[^<>]*|\x3c\\/" + NAME + "(?:\\s[^<>]{0,32})?|\x3c\\/?[aAcCpPiI]*)$");
// An unterminated render-tag opening at the end of a string: \x3c<name> plus
// attrs, no \x3e — a truncated imitation, never prose (triggers use \x3cacp_).
const TRUNC_OPEN = new RegExp("\x3c" + NAME + "\\s[^<>]*$");
Expand All @@ -85,7 +107,7 @@ const TRUNC_CLOSE = new RegExp("\x3c\\/" + NAME + "(?:\\s[^<>]{0,32})?$");
// shape: a render-tag name, whitespace, then an attribute list bounded by the
// next `<`. A properly terminated opening never matches: its attribute list
// ends at a `>`, and no `<` can be reached from there within the class.
const BROKEN_ATTRS = new RegExp("\x3c" + NAME + "\\s[^<>]{0,512}(?=\x3c)");
const BROKEN_ATTRS = new RegExp("\x3c" + NAME + "\\s[^<>]*(?=\x3c)");
const DEFINITE_TAIL = new RegExp("^\x3c" + NAME + "\\s|^\x3c\\/" + NAME);
const OPEN_WITH_ATTRS = new RegExp("^\x3c" + NAME + "\\s");
const CLOSE_HEAD = "\x3c/";
Expand Down
78 changes: 65 additions & 13 deletions tests/tag-echo.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -502,23 +502,75 @@ test("streaming filter still drops a ref echo whose open tag arrives alone in a
assert.ok(f.dropped());
});

test("long tag-like spans are prose, not tags (bounded open match)", () => {
// #1731 replaced the old open-side attr caps ({0,256}/{0,512}): a longer run
// escaped every open-tag matcher while the loose close still went, leaving
// orphan markup on the wire. [^<>] cannot cross a bracket, so the uncapped
// matchers stay linear; a terminated open is now decided by body shape
// (#1720), never by attr length. These tests supersede the former
// "long attr run = prose" pins, which encoded the bypass itself.
test("long-attr paired render tags are stripped, whole-text and at every split (#1731)", () => {
const attrs = "x".repeat(300);
const full = `before ${OPEN}${attrs}>m00155${CLOSE} after`;
const expected = stripAcpTags(full);
assert.equal(expected, "before after");
for (let split = 0; split <= full.length; split++) {
const f = createTagEchoFilter();
const out = f.push(full.slice(0, split)) + f.push(full.slice(split)) + f.flush();
assert.equal(out, expected, `split=${split}`);
}
});

test("terminated long-attr open without a close is still a tag head (#1731)", () => {
const span = `${OPEN}${"x".repeat(300)}>`;
assert.equal(stripAcpTags(`before ${span} after`), `before ${span} after`);
assert.equal(stripAcpTags(`before ${span} after`), "before after");
const f = createTagEchoFilter();
assert.equal(f.push(`before ${span} after`) + f.flush(), `before ${span} after`);
const g = createTagEchoFilter();
assert.equal(g.push(`before ${OPEN}${"x".repeat(300)}`), "before ");
assert.ok(g.pending());
assert.equal(g.push(`> after`), `${span} after`);
assert.equal(g.flush(), "");
assert.equal(f.push(`before ${span}`), "before ");
assert.ok(f.pending());
assert.equal(f.flush(), "");
});

test("prose with a tag-like opening beyond the attr bound survives intact", () => {
const prose = `${OPEN}${"word ".repeat(60)}end > kept`;
assert.equal(stripAcpTags(prose), prose);
const f = createTagEchoFilter();
assert.equal(f.push(prose) + f.flush(), prose);
test("case-drifted render tags are stripped, whole-text and at every split (#1731)", () => {
const up = `\x3cACP tokens="1" type="text"\x3em00155\x3c/ACP\x3e`;
const mixed = `\x3cAcP tokens="1" type="text"\x3em00009\x3c/aCP\x3e`;
assert.equal(stripAcpTags(`pre ${up} post`), "pre post");
assert.equal(stripAcpTags(mixed), "");
assert.equal(stripAcpTags(`pre \x3cACP tokens="1" type="text"\x3e post`), "pre post");
assert.equal(stripAcpTags(`pre \x3c/ACP\x3e post`), "pre post");
const full = `before ${up} after`;
const expected = stripAcpTags(full);
for (let split = 0; split <= full.length; split++) {
const f = createTagEchoFilter();
const out = f.push(full.slice(0, split)) + f.push(full.slice(split)) + f.flush();
assert.equal(out, expected, `split=${split}`);
}
});

test("case-drifted tag heads engage the streaming gates (#1731)", () => {
assert.equal(containsRenderTagText(`\x3cACP tokens="1"\x3e`), true);
assert.equal(mayStartRenderTag(`chunk \x3cAC`), true);
assert.equal(mayStartRenderTag(`chunk \x3c/A`), true);
});

test("prose guards hold under case drift (#1731)", () => {
assert.equal(stripAcpTags(`see the \x3cCaption\x3ex\x3c/Caption\x3e here`), `see the \x3cCaption\x3ex\x3c/Caption\x3e here`);
assert.equal(stripAcpTags(`\x3capp id="1"\x3erun\x3c/app\x3e`), `\x3capp id="1"\x3erun\x3c/app\x3e`);
assert.equal(stripAcpTags(`per the ACPI spec and #include \x3cacpi/acpi.h\x3e`), `per the ACPI spec and #include \x3cacpi/acpi.h\x3e`);
assert.equal(stripAcpTags(`#include \x3cACPI/acpi.h\x3e`), `#include \x3cACPI/acpi.h\x3e`);
assert.equal(stripAcpTags(`\x3cCAPTION\x3etext\x3c/CAPTION\x3e`), `\x3cCAPTION\x3etext\x3c/CAPTION\x3e`);
});

test("uncapped attr runs stay linear (no ReDoS) (#1731)", () => {
const big = OPEN + "a".repeat(200_000) + ">m00155" + CLOSE;
const t0 = process.hrtime.bigint();
const out = stripAcpTags(big);
const ms = Number(process.hrtime.bigint() - t0) / 1e6;
assert.equal(out, "");
assert.ok(ms < 500, `single 200KB attr run took ${ms.toFixed(1)}ms`);
const heads = Array.from({ length: 500 }, () => OPEN + "b".repeat(400)).join(" prose ");
const t1 = process.hrtime.bigint();
stripAcpTags(heads);
const ms2 = Number(process.hrtime.bigint() - t1) / 1e6;
assert.ok(ms2 < 1000, `500 unterminated heads took ${ms2.toFixed(1)}ms`);
});

test("containsToolCallXmlFragment detects tool-call XML fragments, not prose", () => {
Expand Down
Loading