Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 27 additions & 8 deletions src/loop/tag-echo-filter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,12 @@ const NAME = ACP_NAME_ALT;
// Opening-tag attrs are bounded: a render tag opening is short (tokens + type,
// \x3c 50 chars). An unbounded \x3c<name> …\x3e match would swallow a long prose span
// that merely starts with a tag head and contains a \x3e somewhere later.
const PAIRED = new RegExp("\x3c" + NAME + "\\s[^<>]{0,256}>([^<>]{0,64})\x3c\\/" + NAME + ">");
// A render tag wraps exactly one bare ref: the kernel emits <acp tokens="…"
// type="…">mNNNNN</acp> and nothing else between the tags (#1720). Content that
// is not a ref is prose wearing tags — the tags go, the content stays. No g
// flag: createTagEchoFilter drives it with exec() on a sliding buffer.
const PAIRED = new RegExp("\x3c" + NAME + "\\s[^<>]{0,256}>(\\s*m\\d{4,}\\s*)\x3c\\/" + NAME + ">");
const REF_LIKE = /^\s*m\d{4,}\s*$/;
const LONE_OPEN = new RegExp("\x3c" + NAME + "(?:\\s[^<>]{0,256})?>");
const LONE_CLOSE = new RegExp("\x3c\\/" + NAME + "(?=[\\s>])[^<>]{0,32}>");
// A suffix of the buffer that could still grow into a render tag: either an
Expand Down Expand Up @@ -102,14 +107,18 @@ const IMITATION_SWALLOW_CAP = 4096;
* tag in s, or -1. #673: the close name may be a typo variant; termination
* still requires the strict \x3e right after the name — malformed closes are
* LONE_CLOSE's job, not the swallow terminator's. */
function looseCloseEnd(s: string): number {
function looseCloseSpan(s: string): { start: number; end: number } | null {
let idx = s.indexOf(CLOSE_HEAD);
while (idx >= 0) {
const m = CLOSE_NAME_ANCHORED.exec(s.slice(idx + 2));
if (m && s[idx + 2 + m[0].length] === ">") return idx + 2 + m[0].length + 1;
if (m && s[idx + 2 + m[0].length] === ">") return { start: idx, end: idx + 2 + m[0].length + 1 };
idx = s.indexOf(CLOSE_HEAD, idx + 1);
}
return -1;
return null;
}
function looseCloseEnd(s: string): number {
const span = looseCloseSpan(s);
return span === null ? -1 : span.end;
}

/** The span of one wrapped-turn imitation in `s`: where it starts, and the span
Expand Down Expand Up @@ -508,12 +517,22 @@ export function createTagEchoFilter(onDrop?: (snippet: string) => void): TagEcho
for (;;) {
if (swallowUntilClose) {
const combined = swallowed + buf;
const end = looseCloseEnd(combined);
if (end >= 0) {
drop(combined.slice(0, end));
const span = looseCloseSpan(combined);
if (span !== null) {
// Only a ref-shaped body is tag content (#1720): a prose
// body between paired tags is released and just the close
// goes. An attested imitation (swallowReleases=false)
// discards whatever the body is.
const inner = combined.slice(0, span.start);
if (REF_LIKE.test(inner) || !swallowReleases) {
drop(combined.slice(0, span.end));
} else {
out += inner;
drop(combined.slice(span.start, span.end));
}
swallowed = "";
swallowUntilClose = false;
buf = combined.slice(end);
buf = combined.slice(span.end);
continue;
}
if (combined.length > swallowLimit) {
Expand Down
2 changes: 1 addition & 1 deletion tests/marker-echo.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ test("createMarkerLineFilter holds non-ASCII line-start prefixes losslessly (con

test("composeStreamFilters strips render tags AND marker lines in sequence", () => {
const f = composeStreamFilters(createTagEchoFilter(), createMarkerLineFilter());
const input = `noise ${OPEN}inner${CLOSE} tail\n${FORGED}\nclean end`;
const input = `noise ${OPEN}${CLOSE} tail\n${FORGED}\nclean end`;
const out = f.push(input) + f.flush();
assert.equal(out, `noise tail\nclean end`);
const st = f.stats();
Expand Down
8 changes: 4 additions & 4 deletions tests/plugin-passthrough-tag-strip-chat.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -117,13 +117,13 @@ test("plugin chat passthrough drops a chunk whose delta stripped to empty", asyn
const session = makeSession();
const events = [
chatChunk({ content: "real answer" }),
chatChunk({ content: `${TAG_OPEN}m1${TAG_CLOSE}${TAG_OPEN}m2${TAG_CLOSE}` }),
chatChunk({ content: `${TAG_OPEN}m00123${TAG_CLOSE}${TAG_OPEN}m00124${TAG_CLOSE}` }),
chatChunk({ content: "more" }),
DONE,
];
await pipePluginChatWithStrip(streamOf(events), res, "openai", session);
const text = out.join("");
assert.ok(!text.includes("m1") && !text.includes("m2"), "echo-only chunk stripped");
assert.ok(!text.includes("m00123") && !text.includes("m00124"), "echo-only chunk stripped");
const dataLines = text.split("\n").filter((l) => l.startsWith("data:")).filter((l) => !l.includes("[DONE]"));
assert.equal(dataLines.length, 2, "echo-only chunk dropped; 'real answer' and 'more' forwarded");
for (const l of dataLines) {
Expand All @@ -150,12 +150,12 @@ test("plugin chat passthrough still drops a chunk where every managed field is a
const res = makeRes(out);
const session = makeSession();
const events = [
chatChunk({ content: `${TAG_OPEN}m1${TAG_CLOSE}`, reasoning_content: `${TAG_OPEN}m2${TAG_CLOSE}` }),
chatChunk({ content: `${TAG_OPEN}m00123${TAG_CLOSE}`, reasoning_content: `${TAG_OPEN}m00124${TAG_CLOSE}` }),
DONE,
];
await pipePluginChatWithStrip(streamOf(events), res, "openai", session);
const text = out.join("");
assert.ok(!text.includes("m1") && !text.includes("m2"), "echoes stripped");
assert.ok(!text.includes("m00123") && !text.includes("m00124"), "echoes stripped");
const dataLines = text.split("\n").filter((l) => l.startsWith("data:")).filter((l) => !l.includes("[DONE]"));
assert.equal(dataLines.length, 0, "all-echo chunk dropped");
});
Expand Down
41 changes: 38 additions & 3 deletions tests/tag-echo.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,13 +74,24 @@ test("stripAcpTags removes paired render tags with their ref", () => {
assert.equal(stripAcpTags(`before ${TAG("m00155")} after`), "before after");
});

test("stripAcpTags keeps prose between paired tags and strips only the tags (#1720)", () => {
const Z = "Z".repeat(30);
assert.equal(stripAcpTags(`${OPEN}tokens="1" type="text">${Z}${CLOSE}`), Z);
assert.equal(stripAcpTags(`${OPEN}tokens="1" type="text">\n正文段落\n${CLOSE}`), "\n正文段落\n");
assert.equal(stripAcpTags(`${OPEN}tokens="1" type="text">这是什么东西${CLOSE}`), "这是什么东西");
});

test("stripAcpTags still strips whitespace-padded ref echoes (#1720)", () => {
assert.equal(stripAcpTags(`before ${OPEN}tokens="1" type="text"> m00123 ${CLOSE} after`), "before after");
});

test("stripAcpTags removes lone open/close tags", () => {
assert.equal(stripAcpTags(`a ${OPEN}tokens="1"> tail`), "a tail");
assert.equal(stripAcpTags(`x ${CLOSE} y`), "x y");
});

test("stripAcpTags leaves underscore trigger tags intact", () => {
const s = `${LT}acp_compress${LT}/acp_compress${OPEN}x="">ref${CLOSE}`;
const s = `${LT}acp_compress${LT}/acp_compress${OPEN}x="">m00123${CLOSE}`;
assert.equal(stripAcpTags(s), `${LT}acp_compress${LT}/acp_compress`);
});

Expand Down Expand Up @@ -165,6 +176,10 @@ test("streaming filter matches stripAcpTags for every split position", () => {
`mixed ${LT}acp tokens="36" type="text"\x3em00473${LT}/acip\x3e tail`,
`rev ${LT}apic tokens="9" type="text"\x3em001${LT}/acp\x3e tail`,
`safe #include ${LT}acpi/acpi.h\x3e and ${LT}caption\x3ex${LT}/caption\x3e ${LT}app id="1"\x3erun${LT}/app\x3e`,
`${OPEN}tokens="1" type="text">${"Z".repeat(30)}${CLOSE}`,
`${OPEN}tokens="1" type="text">\n正文段落\n${CLOSE}`,
`回答开始。${OPEN}tokens="1" type="text">这是什么东西${CLOSE}回答结束。`,
`lead ${OPEN}tokens="1" type="text"> m00123 ${CLOSE} tail`,
];
for (const full of cases) {
const expected = stripAcpTags(full);
Expand Down Expand Up @@ -467,6 +482,26 @@ test("streaming flush drops content of a tag left unclosed at end of stream", ()
assert.ok(f.dropped());
});

test("streaming filter keeps prose between paired tags, dropping only the tags (#1720)", () => {
const full = `回答开始。${OPEN}tokens="1" type="text">这是什么东西${CLOSE}回答结束。`;
const f = createTagEchoFilter();
let visible = "";
for (let i = 0; i < full.length; i += 5) visible += f.push(full.slice(i, i + 5));
visible += f.flush();
assert.equal(visible, "回答开始。这是什么东西回答结束。");
assert.ok(f.dropped(), "the tag fragments are accounted as dropped");
});

test("streaming filter still drops a ref echo whose open tag arrives alone in a chunk (#1720)", () => {
const open = `${OPEN}tokens="1" type="text">`;
const f = createTagEchoFilter();
let visible = f.push(open);
visible += f.push(`m00123${CLOSE}`);
visible += f.flush();
assert.equal(visible, "");
assert.ok(f.dropped());
});

test("long tag-like spans are prose, not tags (bounded open match)", () => {
const span = `${OPEN}${"x".repeat(300)}>`;
assert.equal(stripAcpTags(`before ${span} after`), `before ${span} after`);
Expand Down Expand Up @@ -522,7 +557,7 @@ test("stripAcpTags removes typo'd acplike render tags (#673)", () => {
assert.equal(stripAcpTags(`${LT}acpi tokens="36" type="text"\x3em00473${LT}/acpi\x3e`), "");
assert.equal(stripAcpTags(`before ${LT}acp tokens="2" type="text"\x3em00473${LT}/acip\x3e after`), "before after");
for (const name of ["acpi", "acip", "apic", "cap", "cpa", "pac", "pca"]) {
assert.equal(stripAcpTags(`${LT}${name} tokens="1" type="text"\x3em001${LT}/${name}\x3e`), "", name);
assert.equal(stripAcpTags(`${LT}${name} tokens="1" type="text"\x3em00473${LT}/${name}\x3e`), "", name);
}
});

Expand Down Expand Up @@ -553,7 +588,7 @@ test("legit angle-bracket text survives the loosened filter (#673)", () => {
});

test("filter stats() accumulates lifetime input/output/dropped (#673)", () => {
const first = `hello ${TAG("m1")}`;
const first = `hello ${TAG("m00123")}`;
const f = createTagEchoFilter();
f.push(first);
f.flush();
Expand Down
Loading