From f349fa3182dd349e5d1495e27d5a806e916319fe Mon Sep 17 00:00:00 2001 From: awork Date: Wed, 21 Oct 2026 18:21:08 +0800 Subject: [PATCH 1/2] =?UTF-8?q?fix(#4106):=20make=20tag=20cleanup=20idempo?= =?UTF-8?q?tent=20=E2=80=94=20input-side=20gate=20attestation=20+=20fixed-?= =?UTF-8?q?point=20walk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The DEAD_REF_CLOSE gate (`out !== text`) was evaluated mid-pipeline, before the marker-line / truncate / bili-artifact channels ran, so evidence produced by those later channels never counted in the same call: a forged marker line stripped at the end of call 1 left trailing dead-ref closes standing, and a second cleanup call shrank them via the end-anchored TRUNC passes. The verdict depended on prior cleanup state, not on content — retry/rebuild paths that re-normalize already-cleaned messages shipped different bytes each pass. Two-part mechanism fix in src/loop/tag-echo-filter.ts: 1. Input-side attestation: the gate now also counts shapes produced by channels that run after it (truncated open at EOF, forged marker line, bili chain carrier / header block), computed once on the ORIGINAL text. Close-side channels (TRUNC_CLOSE / TRUNC_REF_CLOSE) deliberately do not self-attest — they are the same close-only family the gate protects (#1760/#644 content preservation, bare-ref pins unchanged). 2. Bounded fixed-point walk: stripAcpTags applies the pipeline until stable (every non-stable pass strictly shortens; cap bounds adversarial cascade depth), so one call does everything repeat calls would. --- src/loop/tag-echo-filter.ts | 57 +++++++++++- tests/issue4106-strip-idempotency.test.ts | 108 ++++++++++++++++++++++ tests/verif-tag-echo-filter.test.ts | 21 ++--- 3 files changed, 170 insertions(+), 16 deletions(-) create mode 100644 tests/issue4106-strip-idempotency.test.ts diff --git a/src/loop/tag-echo-filter.ts b/src/loop/tag-echo-filter.ts index 26591afb8d..0b7fc4d4d4 100644 --- a/src/loop/tag-echo-filter.ts +++ b/src/loop/tag-echo-filter.ts @@ -420,7 +420,30 @@ export function stripMarkerLines(text: string): string { return text.replace(MARKER_LINE, ""); } -export function stripAcpTags(text: string, dropToolCallEmission = false, requestText?: string): string { +// #4106: input-side attestation for the DEAD_REF_CLOSE gate. Evidence produced +// by channels that run AFTER the gate (a truncated open at EOF, a forged +// marker line, a bili chain carrier / header block) is computed ONCE on the +// original text here, before any pass mutates it. The old gate (`out !== +// text`, evaluated mid-pipeline) only saw what earlier channels had already +// deleted, so a marker line stripped later in the SAME call never attested — +// and a second cleanup call shrank the bytes further: the verdict depended on +// prior cleanup state, not on content. Close-side channels (TRUNC_CLOSE / +// TRUNC_REF_CLOSE) are deliberately NOT attestation: they are the same +// close-only family the gate protects (#1760/#644 — a lone truncated close +// without other echo evidence stays content-preserved, bare ref included). +const MARKER_LINE_PROBE = new RegExp(MARKER_LINE.source, "mu"); // MARKER_LINE is /gmu — .test() on it is lastIndex-stateful +function laterChannelAttested(text: string): boolean { + return TRUNC_OPEN.test(text) + || MARKER_LINE_PROBE.test(text) + || nextChainSpan(text) !== null + || headBlockCut(text) >= 0; +} + +/** One application of the full channel pipeline over one field. `attested` + * carries the input-side evidence computed once on the ORIGINAL text + * (#4106); `out !== text` adds this pass's own earlier-channel removals as + * evidence. */ +function stripPass(text: string, dropToolCallEmission: boolean, requestText: string | undefined, attested: boolean): string { // A whole-field tool-call emission first: the field IS the call, so the // span goes whole (see toolCallEmissionSpan). Gated by the caller; the // m00885 echo check keeps a span the user asked to be emitted verbatim. @@ -469,10 +492,14 @@ export function stripAcpTags(text: string, dropToolCallEmission = false, request // passes: their removals ARE the same-text attestation (an unclosed open or // a degen pair count too, not just an already-stripped span), and stripping // an open also un-glues a ref that sat right after its \x3e so the [\w>] - // lookbehind admits it. A close-only shape with no other echo evidence - // leaves out === text and stays content-preserved (#1760); such sightings - // remain observable via the residue audits. - if (out !== text) { + // lookbehind admits it. The gate is `attested || out !== text`: this pass's + // own earlier-channel removals, plus the input-side evidence computed once + // on the original text (#4106 — later channels' shapes count too, so the + // verdict rides the CONTENT, not on what an earlier cleanup already did). + // A close-only shape with no other echo evidence leaves both terms false + // and stays content-preserved (#1760); such sightings remain observable via + // the residue audits. + if (attested || out !== text) { out = out.replace(new RegExp(DEAD_REF_CLOSE.source, "g"), ""); } out = out @@ -483,6 +510,26 @@ export function stripAcpTags(text: string, dropToolCallEmission = false, request return stripBiliArtifacts(out); } +// Cascade cap for the fixed-point walk below: every non-stable pass strictly +// shortens the text, so the walk terminates anyway; the cap bounds adversarial +// cascade depth (cross-stage exposures layered more than this deep keep their +// residue rather than burn CPU on attacker-shaped fields). +const STRIP_PASS_CAP = 8; + +/** Whole-text ACP artifact strip. Returns a FIXED POINT: a repeated call never + * removes anything further (#4106 — retry/rebuild paths re-normalize already + * cleaned messages, so one call must do everything repeat calls would). */ +export function stripAcpTags(text: string, dropToolCallEmission = false, requestText?: string): string { + const attested = laterChannelAttested(text); + let out = text; + for (let i = 0; i < STRIP_PASS_CAP; i++) { + const next = stripPass(out, dropToolCallEmission, requestText, attested); + if (next === out) return out; + out = next; + } + return out; +} + /** Whole-text echo normalization for parity judges (the invariant judge, * #3938 PR-D): apply the SAME artifact set the outbound stream filters strip, * so entry-vs-exit comparisons never false-positive on shapes one side diff --git a/tests/issue4106-strip-idempotency.test.ts b/tests/issue4106-strip-idempotency.test.ts new file mode 100644 index 0000000000..746eb6c877 --- /dev/null +++ b/tests/issue4106-strip-idempotency.test.ts @@ -0,0 +1,108 @@ +/** + * #4106 — tag cleanup must be idempotent. + * + * The old DEAD_REF_CLOSE gate (`out !== text`, evaluated mid-pipeline) only + * saw deletions made EARLIER in the same call, so a marker line stripped at + * the end of call 1 never attested; call 2 then shrank the text via the + * end-anchored TRUNC passes. The verdict depended on prior cleanup state, not + * on content — retry/rebuild paths re-normalizing already-cleaned messages + * shipped different bytes each pass (#3837 family). + * + * Fixed by (a) input-side gate attestation — later channels' shapes (truncated + * open at EOF, forged marker line, bili chain carrier / header block) count as + * evidence computed once on the original text, while close-side channels still + * never self-attest (#1760/#644) — and (b) a bounded fixed-point walk, so one + * call does everything repeat calls would. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { normalizeEchoArtifacts, stripAcpTags } from "../src/loop/tag-echo-filter.ts"; + +const ISSUE_TEXT = "angle < not a tag\nm7385 { + const s1 = stripAcpTags(ISSUE_TEXT); + const s2 = stripAcpTags(s1); + assert.equal(s2, s1, "second call removed content — idempotency broken"); + assert.equal(stripAcpTags(s2), s2, "third call removed content — no fixed point"); +}); + +test("#4106: the forged marker line attests — trailing dead-ref closes die in the SAME call", () => { + const s1 = stripAcpTags(ISSUE_TEXT); + assert.ok(!s1.includes(" { + // unattested EOF: the close goes (it is a truncated imitation), the bare + // ref stays — exactly the pre-fix bytes, unchanged by the fix. + assert.equal(stripAcpTags("ref m00001 { + const t = "\x3cacp tokens=\"1\" type=\"text\"\x3em05712\x3c/acp\x3e tail m01233\x3c/acp"; + const s1 = stripAcpTags(t); + assert.equal(s1, " tail "); + assert.equal(stripAcpTags(s1), s1); +}); + +test("#4106: cascades exposed by a later channel converge within one call", () => { + // T2 strips the trailing close, exposing a truncated open at the new EOF. + const a = "a ${JSON.stringify(s1)}`); + assert.equal(normalizeEchoArtifacts(stripAcpTags(t)), s1, "normalize diverges from strip on cleaned text"); + } +}); + +// Deterministic fragment soup over the exact shape families of the incident: +// dead-ref-close heads, forged marker lines, tag fragments, refs, prose. +function mulberry32(seed: number): () => number { + let a = seed >>> 0; + return () => { + a = (a + 0x6d2b79f5) | 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +test("#4106: fixed-point property holds over generated fragment soup (seeded)", () => { + const rnd = mulberry32(0x4106c0de); + const ri = (lo: number, hi: number): number => lo + Math.floor(rnd() * (hi - lo + 1)); + const FRAGS = [ + "m7385 and", + "prose 日本語 ✓", + "m01000", + "", + "done folding m01233 ${JSON.stringify(s1)} -> ${JSON.stringify(s2)}`); + const n1 = normalizeEchoArtifacts(t); + assert.equal(normalizeEchoArtifacts(n1), n1, `normalize not idempotent: ${JSON.stringify(t)} -> ${JSON.stringify(n1)}`); + assert.ok(s1.length <= t.length, "strip lengthened the text"); + } +}); diff --git a/tests/verif-tag-echo-filter.test.ts b/tests/verif-tag-echo-filter.test.ts index 3b164014fb..303e3de835 100644 --- a/tests/verif-tag-echo-filter.test.ts +++ b/tests/verif-tag-echo-filter.test.ts @@ -166,14 +166,12 @@ test("P1b absorb-instructed emission hold is split-stable", () => { }); test("P2 stripAcpTags / normalizeEchoArtifacts are idempotent", () => { - // Domain: idempotence is a KNOWN bug while #4106 is open — the marker- - // line pass enables a later TRUNC_REF_CLOSE strip on the second call - // (pinned as control C#4106 below). Until it lands, assert idempotence - // on texts without the triggering shape (marker line AND trailing - // dead-ref-close together). + // Full domain since #4106 landed: stripAcpTags returns a fixed point + // (input-side gate attestation + bounded fixed-point walk), so a second + // call never removes anything further — marker lines and trailing + // dead-ref-closes included. for (let iter = 0; iter < 400; iter++) { const t = randomText(); - if (/\[ACP\]/.test(t) && /m\d{4,}\s*<\//.test(stripAcpTags(t))) continue; // #4106 domain exclusion const s1 = stripAcpTags(t); assert.equal(stripAcpTags(s1), s1, `strip not idempotent — input=${JSON.stringify(t)}`); const n1 = normalizeEchoArtifacts(t); @@ -313,14 +311,15 @@ test("C#2405: a pair split across pushes counts as ONE drop", () => { }); // ------------------------------------------------ pinned bugs -// #4106: stripAcpTags not idempotent — the marker-line pass (LAST) enables -// the gated DEAD_REF_CLOSE / end-anchored TRUNC_REF_CLOSE passes on the -// SECOND call. FLIP this control (assert idempotence) when #4106 lands. -test("C#4106 pinned: stripAcpTags NOT idempotent today (marker + trailing dead-ref)", () => { +// #4106 FIXED: cleanup is a fixed point — the marker line attests on the +// INPUT side, so the trailing dead-ref closes die in the SAME call and a +// second call removes nothing further. This control guards the regression. +test("C#4106 pinned FIXED: stripAcpTags idempotent (marker + trailing dead-ref)", () => { const t = "angle < not a tag\nm7385 Date: Wed, 21 Oct 2026 18:46:04 +0800 Subject: [PATCH 2/2] test(#4106): pin the pass-cap residue corner; state the bounded fixed-point guarantee in the docstring (review finding) Review of the idempotence fix found the docstring overclaimed: STRIP_PASS_CAP=8 bounds adversarial cascade depth, so an input layering more than 8 cross-stage exposures (e.g. 9 stacked $-anchored truncated closes that peel one per pass without ever attesting) keeps its residue at the cap, and a repeated call peels further. Empirically verified: K<=8 converges in one call, K=9 leaves 'A { + // Nine stacked $-anchored truncated closes peel exactly ONE per pass and + // never attest (close-only shapes deliberately do not self-attest, + // #1760/#644), so the ninth layer sits past the cap: the walk terminates + // at STRIP_PASS_CAP=8 keeping its residue instead of burning CPU on + // attacker-shaped input, and a further call peels the last layer. Pinning + // both facts keeps the docstring's bounded guarantee honest in either + // direction (cap change or peel-rate change fails this test loudly). + const t = "A" + "