Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 57 additions & 5 deletions src/loop/tag-echo-filter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -420,7 +420,30 @@ export function stripMarkerLines(text: string): string {
return text.replace(MARKER_LINE, "");
}

export function stripAcpTags(text: string, dropToolCallEmission = false, requestText?: string): string {
// #4106: input-side attestation for the DEAD_REF_CLOSE gate. Evidence produced
// by channels that run AFTER the gate (a truncated open at EOF, a forged
// marker line, a bili chain carrier / header block) is computed ONCE on the
// original text here, before any pass mutates it. The old gate (`out !==
// text`, evaluated mid-pipeline) only saw what earlier channels had already
// deleted, so a marker line stripped later in the SAME call never attested —
// and a second cleanup call shrank the bytes further: the verdict depended on
// prior cleanup state, not on content. Close-side channels (TRUNC_CLOSE /
// TRUNC_REF_CLOSE) are deliberately NOT attestation: they are the same
// close-only family the gate protects (#1760/#644 — a lone truncated close
// without other echo evidence stays content-preserved, bare ref included).
const MARKER_LINE_PROBE = new RegExp(MARKER_LINE.source, "mu"); // MARKER_LINE is /gmu — .test() on it is lastIndex-stateful
function laterChannelAttested(text: string): boolean {
return TRUNC_OPEN.test(text)
|| MARKER_LINE_PROBE.test(text)
|| nextChainSpan(text) !== null
|| headBlockCut(text) >= 0;
}

/** One application of the full channel pipeline over one field. `attested`
* carries the input-side evidence computed once on the ORIGINAL text
* (#4106); `out !== text` adds this pass's own earlier-channel removals as
* evidence. */
function stripPass(text: string, dropToolCallEmission: boolean, requestText: string | undefined, attested: boolean): string {
// A whole-field tool-call emission first: the field IS the call, so the
// span goes whole (see toolCallEmissionSpan). Gated by the caller; the
// m00885 echo check keeps a span the user asked to be emitted verbatim.
Expand Down Expand Up @@ -469,10 +492,14 @@ export function stripAcpTags(text: string, dropToolCallEmission = false, request
// passes: their removals ARE the same-text attestation (an unclosed open or
// a degen pair count too, not just an already-stripped span), and stripping
// an open also un-glues a ref that sat right after its \x3e so the [\w>]
// lookbehind admits it. A close-only shape with no other echo evidence
// leaves out === text and stays content-preserved (#1760); such sightings
// remain observable via the residue audits.
if (out !== text) {
// lookbehind admits it. The gate is `attested || out !== text`: this pass's
// own earlier-channel removals, plus the input-side evidence computed once
// on the original text (#4106 — later channels' shapes count too, so the
// verdict rides the CONTENT, not on what an earlier cleanup already did).
// A close-only shape with no other echo evidence leaves both terms false
// and stays content-preserved (#1760); such sightings remain observable via
// the residue audits.
if (attested || out !== text) {
out = out.replace(new RegExp(DEAD_REF_CLOSE.source, "g"), "");
}
out = out
Expand All @@ -483,6 +510,31 @@ export function stripAcpTags(text: string, dropToolCallEmission = false, request
return stripBiliArtifacts(out);
}

// Cascade cap for the fixed-point walk below: every non-stable pass strictly
// shortens the text, so the walk terminates anyway; the cap bounds adversarial
// cascade depth (cross-stage exposures layered more than this deep keep their
// residue rather than burn CPU on attacker-shaped fields).
const STRIP_PASS_CAP = 8;

/** Whole-text ACP artifact strip. Returns a FIXED POINT whenever the exposure
* cascade settles within STRIP_PASS_CAP passes (all attested shapes and every
* observed cascade do — one call then does everything repeat calls would,
* #4106, so retry/rebuild paths re-normalizing already-cleaned messages ship
* stable bytes). Beyond the cap — attacker-shaped fields layering more
* cross-stage exposures than this deep — the walk stops and keeps its
* residue; a repeated call may peel further (the CPU bound outranks
* completeness there, see the STRIP_PASS_CAP note above). */
export function stripAcpTags(text: string, dropToolCallEmission = false, requestText?: string): string {
const attested = laterChannelAttested(text);
let out = text;
for (let i = 0; i < STRIP_PASS_CAP; i++) {
const next = stripPass(out, dropToolCallEmission, requestText, attested);
if (next === out) return out;
out = next;
}
return out;
}

/** Whole-text echo normalization for parity judges (the invariant judge,
* #3938 PR-D): apply the SAME artifact set the outbound stream filters strip,
* so entry-vs-exit comparisons never false-positive on shapes one side
Expand Down
122 changes: 122 additions & 0 deletions tests/issue4106-strip-idempotency.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
/**
* #4106 — tag cleanup must be idempotent.
*
* The old DEAD_REF_CLOSE gate (`out !== text`, evaluated mid-pipeline) only
* saw deletions made EARLIER in the same call, so a marker line stripped at
* the end of call 1 never attested; call 2 then shrank the text via the
* end-anchored TRUNC passes. The verdict depended on prior cleanup state, not
* on content — retry/rebuild paths re-normalizing already-cleaned messages
* shipped different bytes each pass (#3837 family).
*
* Fixed by (a) input-side gate attestation — later channels' shapes (truncated
* open at EOF, forged marker line, bili chain carrier / header block) count as
* evidence computed once on the original text, while close-side channels still
* never self-attest (#1760/#644) — and (b) a bounded fixed-point walk, so one
* call does everything repeat calls would.
*/
import test from "node:test";
import assert from "node:assert/strict";
import { normalizeEchoArtifacts, stripAcpTags } from "../src/loop/tag-echo-filter.ts";

const ISSUE_TEXT = "angle < not a tag\nm7385</CpA\nm3117</cPa\n\u{1F4E6} [ACP] Compressed m00120\u2013m0300 \u2192 1 block(s)";

test("#4106: the issue repro is stable after ONE call", () => {
const s1 = stripAcpTags(ISSUE_TEXT);
const s2 = stripAcpTags(s1);
assert.equal(s2, s1, "second call removed content — idempotency broken");
assert.equal(stripAcpTags(s2), s2, "third call removed content — no fixed point");
});

test("#4106: the forged marker line attests — trailing dead-ref closes die in the SAME call", () => {
const s1 = stripAcpTags(ISSUE_TEXT);
assert.ok(!s1.includes("</CpA") && !s1.includes("</cPa"), `dead-ref closes survived an attested cleanup: ${JSON.stringify(s1)}`);
assert.ok(s1.includes("angle < not a tag"), "prose lost");
});

test("#4106: close-only shapes without other echo evidence stay content-preserved (#1760/#644)", () => {
// unattested EOF: the close goes (it is a truncated imitation), the bare
// ref stays — exactly the pre-fix bytes, unchanged by the fix.
assert.equal(stripAcpTags("ref m00001 </acp"), "ref m00001 ");
// unattested mid-text: byte-identical passthrough.
const mid = "x m00992</ap more";
assert.equal(stripAcpTags(mid), mid);
const clean = "a < b and </abcd> and see m01234 for details";
assert.equal(stripAcpTags(clean), clean);
});

test("#4106: attested EOF dead-ref unit still dies whole, not as a bare ref (#2862)", () => {
const t = "\x3cacp tokens=\"1\" type=\"text\"\x3em05712\x3c/acp\x3e tail m01233\x3c/acp";
const s1 = stripAcpTags(t);
assert.equal(s1, " tail ");
assert.equal(stripAcpTags(s1), s1);
});

test("#4106: cascades exposed by a later channel converge within one call", () => {
// T2 strips the trailing close, exposing a truncated open at the new EOF.
const a = "a <acp tok</acp";
// T2 strips the trailing close, exposing the forged marker line.
const b = "x\n\u{1F4E6} [ACP] hi</acp";
for (const t of [a, b]) {
const s1 = stripAcpTags(t);
assert.equal(stripAcpTags(s1), s1, `not a fixed point: ${JSON.stringify(t)} -> ${JSON.stringify(s1)}`);
assert.equal(normalizeEchoArtifacts(stripAcpTags(t)), s1, "normalize diverges from strip on cleaned text");
}
});

// Deterministic fragment soup over the exact shape families of the incident:
// dead-ref-close heads, forged marker lines, tag fragments, refs, prose.
function mulberry32(seed: number): () => number {
let a = seed >>> 0;
return () => {
a = (a + 0x6d2b79f5) | 0;
let t = Math.imul(a ^ (a >>> 15), 1 | a);
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
}

test("#4106: fixed-point property holds over generated fragment soup (seeded)", () => {
const rnd = mulberry32(0x4106c0de);
const ri = (lo: number, hi: number): number => lo + Math.floor(rnd() * (hi - lo + 1));
const FRAGS = [
"m7385</CpA",
"m3117</cPa",
"\u{1F4E6} [ACP] Compressed m00120\u2013m0300 \u2192 1 block(s)",
"\x3cacp tokens=\"1\" type=\"text\"\x3em00042\x3c/acp\x3e",
"</acp",
"<acp tok",
"</apc",
"see < b and </abcd> and",
"prose 日本語 ✓",
"m01000",
"<cap/>",
"done folding m01233</acp",
"",
] as const;
const JOINS = [" ", "\n", "", " ", "\n\n"] as const;
for (let i = 0; i < 300; i++) {
const n = ri(1, 6);
let t = "";
for (let j = 0; j < n; j++) t += FRAGS[ri(0, FRAGS.length - 1)]! + JOINS[ri(0, JOINS.length - 1)]!;
const s1 = stripAcpTags(t);
const s2 = stripAcpTags(s1);
assert.equal(s2, s1, `strip not idempotent: ${JSON.stringify(t)} -> ${JSON.stringify(s1)} -> ${JSON.stringify(s2)}`);
const n1 = normalizeEchoArtifacts(t);
assert.equal(normalizeEchoArtifacts(n1), n1, `normalize not idempotent: ${JSON.stringify(t)} -> ${JSON.stringify(n1)}`);
assert.ok(s1.length <= t.length, "strip lengthened the text");
}
});

test("#4106: beyond STRIP_PASS_CAP the walk keeps residue (CPU bound) — pinned corner", () => {
// Nine stacked $-anchored truncated closes peel exactly ONE per pass and
// never attest (close-only shapes deliberately do not self-attest,
// #1760/#644), so the ninth layer sits past the cap: the walk terminates
// at STRIP_PASS_CAP=8 keeping its residue instead of burning CPU on
// attacker-shaped input, and a further call peels the last layer. Pinning
// both facts keeps the docstring's bounded guarantee honest in either
// direction (cap change or peel-rate change fails this test loudly).
const t = "A" + "</acpa\n".repeat(9);
const s1 = stripAcpTags(t);
assert.equal(s1, "A</acpa\n", "cap behavior drifted");
assert.notEqual(stripAcpTags(s1), s1, "documented corner: one more call peels the final layer");
});
21 changes: 10 additions & 11 deletions tests/verif-tag-echo-filter.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -166,14 +166,12 @@ test("P1b absorb-instructed emission hold is split-stable", () => {
});

test("P2 stripAcpTags / normalizeEchoArtifacts are idempotent", () => {
// Domain: idempotence is a KNOWN bug while #4106 is open — the marker-
// line pass enables a later TRUNC_REF_CLOSE strip on the second call
// (pinned as control C#4106 below). Until it lands, assert idempotence
// on texts without the triggering shape (marker line AND trailing
// dead-ref-close together).
// Full domain since #4106 landed: stripAcpTags returns a fixed point
// (input-side gate attestation + bounded fixed-point walk), so a second
// call never removes anything further — marker lines and trailing
// dead-ref-closes included.
for (let iter = 0; iter < 400; iter++) {
const t = randomText();
if (/\[ACP\]/.test(t) && /m\d{4,}\s*<\//.test(stripAcpTags(t))) continue; // #4106 domain exclusion
const s1 = stripAcpTags(t);
assert.equal(stripAcpTags(s1), s1, `strip not idempotent — input=${JSON.stringify(t)}`);
const n1 = normalizeEchoArtifacts(t);
Expand Down Expand Up @@ -313,14 +311,15 @@ test("C#2405: a pair split across pushes counts as ONE drop", () => {
});

// ------------------------------------------------ pinned bugs
// #4106: stripAcpTags not idempotent — the marker-line pass (LAST) enables
// the gated DEAD_REF_CLOSE / end-anchored TRUNC_REF_CLOSE passes on the
// SECOND call. FLIP this control (assert idempotence) when #4106 lands.
test("C#4106 pinned: stripAcpTags NOT idempotent today (marker + trailing dead-ref)", () => {
// #4106 FIXED: cleanup is a fixed point — the marker line attests on the
// INPUT side, so the trailing dead-ref closes die in the SAME call and a
// second call removes nothing further. This control guards the regression.
test("C#4106 pinned FIXED: stripAcpTags idempotent (marker + trailing dead-ref)", () => {
const t = "angle < not a tag\nm7385</CpA\nm3117</cPa\n\u{1F4E6} [ACP] Compressed m00120\u2013m0300 \u2192 1 block(s)";
const s1 = stripAcpTags(t);
const s2 = stripAcpTags(s1);
assert.notEqual(s1, s2, "#4106 fixed? — flip this pin and re-enable the full P2 domain");
assert.equal(s1, s2, "#4106 regressed — a second cleanup call shrank the text further");
assert.ok(!s1.includes("</CpA") && !s1.includes("</cPa"), "attested dead-ref closes must die in the first call");
});
// #4107: stream output depends on chunk boundaries — the dead-ref-close
// residue is cleaned in the whole-text path but RELEASED to the client
Expand Down
Loading