diff --git a/src/loop/tag-echo-filter.ts b/src/loop/tag-echo-filter.ts index 736c312a78..260aac09de 100644 --- a/src/loop/tag-echo-filter.ts +++ b/src/loop/tag-echo-filter.ts @@ -57,7 +57,12 @@ const NAME = ACP_NAME_ALT; // Opening-tag attrs are bounded: a render tag opening is short (tokens + type, // \x3c 50 chars). An unbounded \x3c …\x3e match would swallow a long prose span // that merely starts with a tag head and contains a \x3e somewhere later. -const PAIRED = new RegExp("\x3c" + NAME + "\\s[^<>]{0,256}>([^<>]{0,64})\x3c\\/" + NAME + ">"); +// A render tag wraps exactly one bare ref: the kernel emits mNNNNN and nothing else between the tags (#1720). Content that +// is not a ref is prose wearing tags — the tags go, the content stays. No g +// flag: createTagEchoFilter drives it with exec() on a sliding buffer. +const PAIRED = new RegExp("\x3c" + NAME + "\\s[^<>]{0,256}>(\\s*m\\d{4,}\\s*)\x3c\\/" + NAME + ">"); +const REF_LIKE = /^\s*m\d{4,}\s*$/; const LONE_OPEN = new RegExp("\x3c" + NAME + "(?:\\s[^<>]{0,256})?>"); const LONE_CLOSE = new RegExp("\x3c\\/" + NAME + "(?=[\\s>])[^<>]{0,32}>"); // A suffix of the buffer that could still grow into a render tag: either an @@ -102,14 +107,18 @@ const IMITATION_SWALLOW_CAP = 4096; * tag in s, or -1. #673: the close name may be a typo variant; termination * still requires the strict \x3e right after the name — malformed closes are * LONE_CLOSE's job, not the swallow terminator's. */ -function looseCloseEnd(s: string): number { +function looseCloseSpan(s: string): { start: number; end: number } | null { let idx = s.indexOf(CLOSE_HEAD); while (idx >= 0) { const m = CLOSE_NAME_ANCHORED.exec(s.slice(idx + 2)); - if (m && s[idx + 2 + m[0].length] === ">") return idx + 2 + m[0].length + 1; + if (m && s[idx + 2 + m[0].length] === ">") return { start: idx, end: idx + 2 + m[0].length + 1 }; idx = s.indexOf(CLOSE_HEAD, idx + 1); } - return -1; + return null; +} +function looseCloseEnd(s: string): number { + const span = looseCloseSpan(s); + return span === null ? -1 : span.end; } /** The span of one wrapped-turn imitation in `s`: where it starts, and the span @@ -508,12 +517,22 @@ export function createTagEchoFilter(onDrop?: (snippet: string) => void): TagEcho for (;;) { if (swallowUntilClose) { const combined = swallowed + buf; - const end = looseCloseEnd(combined); - if (end >= 0) { - drop(combined.slice(0, end)); + const span = looseCloseSpan(combined); + if (span !== null) { + // Only a ref-shaped body is tag content (#1720): a prose + // body between paired tags is released and just the close + // goes. An attested imitation (swallowReleases=false) + // discards whatever the body is. + const inner = combined.slice(0, span.start); + if (REF_LIKE.test(inner) || !swallowReleases) { + drop(combined.slice(0, span.end)); + } else { + out += inner; + drop(combined.slice(span.start, span.end)); + } swallowed = ""; swallowUntilClose = false; - buf = combined.slice(end); + buf = combined.slice(span.end); continue; } if (combined.length > swallowLimit) { diff --git a/tests/marker-echo.test.ts b/tests/marker-echo.test.ts index 98a3d05ba3..5367a246b9 100644 --- a/tests/marker-echo.test.ts +++ b/tests/marker-echo.test.ts @@ -202,7 +202,7 @@ test("createMarkerLineFilter holds non-ASCII line-start prefixes losslessly (con test("composeStreamFilters strips render tags AND marker lines in sequence", () => { const f = composeStreamFilters(createTagEchoFilter(), createMarkerLineFilter()); - const input = `noise ${OPEN}inner${CLOSE} tail\n${FORGED}\nclean end`; + const input = `noise ${OPEN}${CLOSE} tail\n${FORGED}\nclean end`; const out = f.push(input) + f.flush(); assert.equal(out, `noise tail\nclean end`); const st = f.stats(); diff --git a/tests/plugin-passthrough-tag-strip-chat.test.ts b/tests/plugin-passthrough-tag-strip-chat.test.ts index 33a3bc17f4..6b6207e301 100644 --- a/tests/plugin-passthrough-tag-strip-chat.test.ts +++ b/tests/plugin-passthrough-tag-strip-chat.test.ts @@ -117,13 +117,13 @@ test("plugin chat passthrough drops a chunk whose delta stripped to empty", asyn const session = makeSession(); const events = [ chatChunk({ content: "real answer" }), - chatChunk({ content: `${TAG_OPEN}m1${TAG_CLOSE}${TAG_OPEN}m2${TAG_CLOSE}` }), + chatChunk({ content: `${TAG_OPEN}m00123${TAG_CLOSE}${TAG_OPEN}m00124${TAG_CLOSE}` }), chatChunk({ content: "more" }), DONE, ]; await pipePluginChatWithStrip(streamOf(events), res, "openai", session); const text = out.join(""); - assert.ok(!text.includes("m1") && !text.includes("m2"), "echo-only chunk stripped"); + assert.ok(!text.includes("m00123") && !text.includes("m00124"), "echo-only chunk stripped"); const dataLines = text.split("\n").filter((l) => l.startsWith("data:")).filter((l) => !l.includes("[DONE]")); assert.equal(dataLines.length, 2, "echo-only chunk dropped; 'real answer' and 'more' forwarded"); for (const l of dataLines) { @@ -150,12 +150,12 @@ test("plugin chat passthrough still drops a chunk where every managed field is a const res = makeRes(out); const session = makeSession(); const events = [ - chatChunk({ content: `${TAG_OPEN}m1${TAG_CLOSE}`, reasoning_content: `${TAG_OPEN}m2${TAG_CLOSE}` }), + chatChunk({ content: `${TAG_OPEN}m00123${TAG_CLOSE}`, reasoning_content: `${TAG_OPEN}m00124${TAG_CLOSE}` }), DONE, ]; await pipePluginChatWithStrip(streamOf(events), res, "openai", session); const text = out.join(""); - assert.ok(!text.includes("m1") && !text.includes("m2"), "echoes stripped"); + assert.ok(!text.includes("m00123") && !text.includes("m00124"), "echoes stripped"); const dataLines = text.split("\n").filter((l) => l.startsWith("data:")).filter((l) => !l.includes("[DONE]")); assert.equal(dataLines.length, 0, "all-echo chunk dropped"); }); diff --git a/tests/tag-echo.test.ts b/tests/tag-echo.test.ts index 57db75d5f0..b5eeb02a42 100644 --- a/tests/tag-echo.test.ts +++ b/tests/tag-echo.test.ts @@ -74,13 +74,24 @@ test("stripAcpTags removes paired render tags with their ref", () => { assert.equal(stripAcpTags(`before ${TAG("m00155")} after`), "before after"); }); +test("stripAcpTags keeps prose between paired tags and strips only the tags (#1720)", () => { + const Z = "Z".repeat(30); + assert.equal(stripAcpTags(`${OPEN}tokens="1" type="text">${Z}${CLOSE}`), Z); + assert.equal(stripAcpTags(`${OPEN}tokens="1" type="text">\n正文段落\n${CLOSE}`), "\n正文段落\n"); + assert.equal(stripAcpTags(`${OPEN}tokens="1" type="text">这是什么东西${CLOSE}`), "这是什么东西"); +}); + +test("stripAcpTags still strips whitespace-padded ref echoes (#1720)", () => { + assert.equal(stripAcpTags(`before ${OPEN}tokens="1" type="text"> m00123 ${CLOSE} after`), "before after"); +}); + test("stripAcpTags removes lone open/close tags", () => { assert.equal(stripAcpTags(`a ${OPEN}tokens="1"> tail`), "a tail"); assert.equal(stripAcpTags(`x ${CLOSE} y`), "x y"); }); test("stripAcpTags leaves underscore trigger tags intact", () => { - const s = `${LT}acp_compress${LT}/acp_compress${OPEN}x="">ref${CLOSE}`; + const s = `${LT}acp_compress${LT}/acp_compress${OPEN}x="">m00123${CLOSE}`; assert.equal(stripAcpTags(s), `${LT}acp_compress${LT}/acp_compress`); }); @@ -165,6 +176,10 @@ test("streaming filter matches stripAcpTags for every split position", () => { `mixed ${LT}acp tokens="36" type="text"\x3em00473${LT}/acip\x3e tail`, `rev ${LT}apic tokens="9" type="text"\x3em001${LT}/acp\x3e tail`, `safe #include ${LT}acpi/acpi.h\x3e and ${LT}caption\x3ex${LT}/caption\x3e ${LT}app id="1"\x3erun${LT}/app\x3e`, + `${OPEN}tokens="1" type="text">${"Z".repeat(30)}${CLOSE}`, + `${OPEN}tokens="1" type="text">\n正文段落\n${CLOSE}`, + `回答开始。${OPEN}tokens="1" type="text">这是什么东西${CLOSE}回答结束。`, + `lead ${OPEN}tokens="1" type="text"> m00123 ${CLOSE} tail`, ]; for (const full of cases) { const expected = stripAcpTags(full); @@ -467,6 +482,26 @@ test("streaming flush drops content of a tag left unclosed at end of stream", () assert.ok(f.dropped()); }); +test("streaming filter keeps prose between paired tags, dropping only the tags (#1720)", () => { + const full = `回答开始。${OPEN}tokens="1" type="text">这是什么东西${CLOSE}回答结束。`; + const f = createTagEchoFilter(); + let visible = ""; + for (let i = 0; i < full.length; i += 5) visible += f.push(full.slice(i, i + 5)); + visible += f.flush(); + assert.equal(visible, "回答开始。这是什么东西回答结束。"); + assert.ok(f.dropped(), "the tag fragments are accounted as dropped"); +}); + +test("streaming filter still drops a ref echo whose open tag arrives alone in a chunk (#1720)", () => { + const open = `${OPEN}tokens="1" type="text">`; + const f = createTagEchoFilter(); + let visible = f.push(open); + visible += f.push(`m00123${CLOSE}`); + visible += f.flush(); + assert.equal(visible, ""); + assert.ok(f.dropped()); +}); + test("long tag-like spans are prose, not tags (bounded open match)", () => { const span = `${OPEN}${"x".repeat(300)}>`; assert.equal(stripAcpTags(`before ${span} after`), `before ${span} after`); @@ -522,7 +557,7 @@ test("stripAcpTags removes typo'd acplike render tags (#673)", () => { assert.equal(stripAcpTags(`${LT}acpi tokens="36" type="text"\x3em00473${LT}/acpi\x3e`), ""); assert.equal(stripAcpTags(`before ${LT}acp tokens="2" type="text"\x3em00473${LT}/acip\x3e after`), "before after"); for (const name of ["acpi", "acip", "apic", "cap", "cpa", "pac", "pca"]) { - assert.equal(stripAcpTags(`${LT}${name} tokens="1" type="text"\x3em001${LT}/${name}\x3e`), "", name); + assert.equal(stripAcpTags(`${LT}${name} tokens="1" type="text"\x3em00473${LT}/${name}\x3e`), "", name); } }); @@ -553,7 +588,7 @@ test("legit angle-bracket text survives the loosened filter (#673)", () => { }); test("filter stats() accumulates lifetime input/output/dropped (#673)", () => { - const first = `hello ${TAG("m1")}`; + const first = `hello ${TAG("m00123")}`; const f = createTagEchoFilter(); f.push(first); f.flush();