Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 25 additions & 5 deletions src/loop/tag-echo-filter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,11 @@ const NAME = ACP_NAME_ALT;
// Attrs are OPTIONAL: the kernel always emits them, but models imitate the
// bare form <name>mNNNNN</name> (#1881) — whole-span strip must cover it or
// the interior ref leaks as residue after the lone tags go.
const PAIRED = new RegExp("\x3c" + NAME + "(?:\\s[^<>]*)?>(\\s*m\\d{4,}\\s*)\x3c\\/" + NAME + ">");
// #2176: the OPEN anchors legitimacy (strict acplike spelling); the model
// degrades the CLOSE into an open set we cannot enumerate (observed </ap>,
// </a>, </p>, </aph>, </ck>). Relax only the close side to any bounded
// letter-name so a bare-ref span terminates even on a garbled close.
const PAIRED = new RegExp("\x3c" + NAME + "(?:\\s[^<>]*)?>(\\s*m\\d{4,}\\s*)\x3c\\/[A-Za-z]{1,16}>");
const REF_LIKE = /^\s*m\d{4,}\s*$/;
const LONE_OPEN = new RegExp("\x3c" + NAME + "(?:\\s[^<>]*)?>");
const LONE_CLOSE = new RegExp("\x3c\\/" + NAME + "(?=[\\s>])[^<>]{0,32}>");
Expand Down Expand Up @@ -115,6 +119,10 @@ const DEFINITE_TAIL = new RegExp("^\x3c" + NAME + "\\s|^\x3c\\/" + NAME);
const OPEN_WITH_ATTRS = new RegExp("^\x3c" + NAME + "\\s");
const CLOSE_HEAD = "\x3c/";
const CLOSE_NAME_ANCHORED = new RegExp("^" + NAME);
// #2176: a degraded close (</ap>, </p>, ...) — an open set we cannot enumerate.
// Used ONLY as a bare-ref-gated fallback in the swallow path (see process());
// never for wrapped-turn spans, so their inner markup is never cut early.
const CLOSE_ANY_ANCHORED = /^[A-Za-z]{1,16}/;
const HOLD_LIMIT = 128;
// Hold cap for a definite unterminated opening tail — far beyond any real tag
// opening; beyond this the tail is dropped instead of held or passed through.
Expand All @@ -131,11 +139,14 @@ const IMITATION_SWALLOW_CAP = 4096;
/** Exclusive end index (past the terminating \x3e) of the first loose close
* tag in s, or -1. #673: the close name may be a typo variant; termination
* still requires the strict \x3e right after the name — malformed closes are
* LONE_CLOSE's job, not the swallow terminator's. */
function looseCloseSpan(s: string): { start: number; end: number } | null {
* LONE_CLOSE's job, not the swallow terminator's. `lenient` accepts any bounded
* letter-name close (#2176 degraded closes) and is invoked ONLY as a bare-ref-
* gated fallback in the swallow path, never for wrapped-turn spans. */
function looseCloseSpan(s: string, lenient = false): { start: number; end: number } | null {
const anchored = lenient ? CLOSE_ANY_ANCHORED : CLOSE_NAME_ANCHORED;
let idx = s.indexOf(CLOSE_HEAD);
while (idx >= 0) {
const m = CLOSE_NAME_ANCHORED.exec(s.slice(idx + 2));
const m = anchored.exec(s.slice(idx + 2));
if (m && s[idx + 2 + m[0].length] === ">") return { start: idx, end: idx + 2 + m[0].length + 1 };
idx = s.indexOf(CLOSE_HEAD, idx + 1);
}
Expand Down Expand Up @@ -556,7 +567,16 @@ export function createTagEchoFilter(onDrop?: (snippet: string) => void): TagEcho
for (;;) {
if (swallowUntilClose) {
const combined = swallowed + buf;
const span = looseCloseSpan(combined);
let span = looseCloseSpan(combined);
if (span === null) {
// #2176: a degraded close (</ap>, </p>, ...) is invisible to the
// strict matcher above. Honor it only when the swallowed body is a
// bare ref — the echo shape — leaving wrapped-turn/prose bodies to
// the strict path (a non-ref body must never terminate early at an
// inner markup close such as </invoke>).
const li = looseCloseSpan(combined, true);
if (li !== null && REF_LIKE.test(combined.slice(0, li.start))) span = li;
}
if (span !== null) {
// Only a ref-shaped body is tag content (#1720): a prose
// body between paired tags is released and just the close
Expand Down
77 changes: 77 additions & 0 deletions tests/issue2176-degraded-close.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { stripAcpTags, createTagEchoFilter } from "../src/loop/tag-echo-filter.ts";

// #2176: a render-tag CLOSE name degrades into an open set the model produces
// (near-acplike short names). The OPEN stays the strict acplike legitimacy
// anchor; only PAIRED's close side relaxes to any bounded letter-name. Standalone
// closes stay strict so prose markup is never eaten (#1039). Brackets are
// escaped throughout so the source carries no literal angle-bracket pairs.
const LT = "\u003c";
const GT = "\u003e";
function op(tok: string, extra?: string): string {
return LT + 'acp tokens="' + tok + '"' + (extra ?? "") + GT;
}
const TATTR = ' type="text"';
const C_ACCP = LT + "/acp" + GT;
const C_AP = LT + "/ap" + GT;
const C_APH = LT + "/aph" + GT;
const C_P = LT + "/p" + GT;
const C_CK = LT + "/ck" + GT;

function streamWhole(input: string): string {
const f = createTagEchoFilter();
let out = "";
for (let i = 0; i < input.length; i += 3) out += f.push(input.slice(i, i + 3));
return out + f.flush();
}

const ANSWER = "这里有个我必须诚实处理的点,而且它削弱了我一条论据、同时加强了另一条。";
// Shape captured verbatim from the #2176 category-A screenshot: two well-formed
// tags then two degraded-close tags, answer after.
const BURST =
op("419", TATTR) + "m00501" + C_ACCP + " " +
op("193", TATTR) + "m00503" + C_ACCP + " " +
op("0") + "m00505" + C_P + " " +
op("291") + "m00502" + C_P + " " +
ANSWER;

test("#2176 stripAcpTags removes a single well-formed pair (regression)", () => {
assert.equal(stripAcpTags(op("5", TATTR) + "m00123" + C_ACCP + " tail"), " tail");
});

test("#2176 stripAcpTags removes a DEGRADED-close pair (whole-text)", () => {
assert.equal(stripAcpTags(op("48", TATTR) + "m00558" + C_AP), "");
assert.equal(stripAcpTags(op("137") + "m00559" + C_APH), "");
assert.equal(stripAcpTags(op("502") + "m00560" + C_P), "");
assert.equal(stripAcpTags(op("28") + "m00370" + C_CK), "");
});

test("#2176 stripAcpTags strips a multi-tag burst incl. degraded closes, keeps the answer", () => {
const out = stripAcpTags(BURST);
assert.equal(out.trim(), ANSWER);
assert.ok(!out.includes(LT), "no angle-bracket residue may reach the client");
assert.ok(!/m0\d{3}/.test(out), "no message-ref residue may reach the client");
});

test("#2176 streaming filter removes a DEGRADED-close pair", () => {
assert.equal(streamWhole(op("48", TATTR) + "m00558" + C_AP), "");
});

test("#2176 streaming filter strips a multi-tag burst incl. degraded closes, keeps the answer", () => {
const out = streamWhole(BURST);
assert.equal(out.trim(), ANSWER);
assert.ok(!out.includes(LT), "no angle-bracket residue may reach the client");
assert.ok(!/m0\d{3}/.test(out), "no message-ref residue may reach the client");
});

test("#2176 whitespace-padded ref with a degraded close drops the span wholesale", () => {
assert.equal(stripAcpTags("pre " + op("0") + " m00505 " + C_P + " post"), "pre post");
});

test("#2176 does NOT eat prose markup without an acplike open (#1039 guard)", () => {
const prose = "bold " + LT + "b" + GT + "strong" + LT + "/b" + GT + " end html " +
LT + "div" + GT + "x" + LT + "/div" + GT + " plus " + LT + "/p" + GT + " tail";
assert.equal(stripAcpTags(prose), prose);
assert.equal(streamWhole(prose), prose);
});
Loading