From 66804f671f1b50edd87e8885c464daad8a623e27 Mon Sep 17 00:00:00 2001 From: neal <130699270+possiblyneal@users.noreply.github.com> Date: Sat, 21 Nov 2026 14:14:59 -0500 Subject: [PATCH 1/2] fix(parser): recover XML list items with mismatched closing tags A model can close one list item with another tag, causing the parser to\nfuse it with the next item or drop it entirely. Keep adjacent items\nseparate without stripping markup from correctly closed items. Refs: #472 Generated-By: gpt-6.2-sol --- CLAUDE.md | 5 +++++ src/prompts/xml.ts | 26 ++++++++++++++++---------- test/xml.test.ts | 17 +++++++++++++++++ 3 files changed, 38 insertions(+), 10 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index e0198f1b5f..e8de79249d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -317,6 +317,11 @@ prompt's query is unchanged. The Injection Gate scores only the first `RERANK_QU of that combined query, so for a terse prompt it sees the prompt and the start of the reply tail. Measured in `docs/benchmarks/2026-11-08-terse-prompt-query.md`. +### Model reply lists +`getXmlChildren()` (`src/prompts/xml.ts`) keeps list items separate when a model closes one +with the wrong tag. It drops the trailing mismatched close from that item and preserves inline +markup in correctly closed items. This applies to new replies; it does not rewrite stored records. + ### Session Summaries in Injection After the gate, `renderWithSessionDecisions` (`src/functions/prompt-context.ts`) gives an Observation hit that its Session Summary covers (`summaryCovers()`, the same `coveredThrough` diff --git a/src/prompts/xml.ts b/src/prompts/xml.ts index bb0eec36ce..8c26c4824a 100644 --- a/src/prompts/xml.ts +++ b/src/prompts/xml.ts @@ -45,6 +45,11 @@ export function getXmlTag(xml: string, tag: string): string { return firstElementContent(xml, tag)?.trim() ?? ""; } +// Models sometimes close an item with the wrong tag (`x`). +// Each child runs from its open tag to its own close tag, else to the next +// open tag, dropping a wrong close tag left at its end. +const STRAY_TRAILING_CLOSE = /<\/[a-zA-Z_][a-zA-Z0-9_-]*\s*>\s*$/; + export function getXmlChildren( xml: string, parentTag: string, @@ -53,16 +58,17 @@ export function getXmlChildren( if (!VALID_TAG.test(parentTag) || !VALID_TAG.test(childTag)) return []; const parent = firstElementContent(xml, parentTag); if (parent === null) return []; - const items: string[] = []; - const re = new RegExp( - `${openTag(childTag)}([\\s\\S]*?)${closeTag(childTag)}`, - "g", - ); - let m; - while ((m = re.exec(parent)) !== null) { - items.push(m[1].trim()); - } - return items; + const close = new RegExp(closeTag(childTag)); + return parent + .split(new RegExp(openTag(childTag))) + .slice(1) + .map((piece) => { + const closed = piece.search(close); + return closed >= 0 + ? piece.slice(0, closed) + : piece.replace(STRAY_TRAILING_CLOSE, ""); + }) + .map((item) => item.trim()); } // Every `` element, inner text keyed by id. A repeated id is diff --git a/test/xml.test.ts b/test/xml.test.ts index 4f023ab52a..cdddba31e0 100644 --- a/test/xml.test.ts +++ b/test/xml.test.ts @@ -79,6 +79,23 @@ describe('getXmlChildren', () => { it('returns empty for invalid parent tag name', () => { expect(getXmlChildren('A', '.*', 'fact')).toEqual([]) }) + + // #472: models sometimes close an item with the wrong tag; the lazy match + // then fused it with the next item and stored the stray tag in the field. + it('splits an item closed with the wrong tag from the next one', () => { + const xml = 'VAAPI renderD128\n Jellyfin acceleration' + expect(getXmlChildren(xml, 'concepts', 'concept')).toEqual(['VAAPI renderD128', 'Jellyfin acceleration']) + }) + + it('keeps a last item closed with the wrong tag', () => { + const xml = 'CPALalsa.headroom\n' + expect(getXmlChildren(xml, 'concepts', 'concept')).toEqual(['CPAL', 'alsa.headroom']) + }) + + it('keeps inline markup inside a properly closed item', () => { + const xml = 'Wrap it in x' + expect(getXmlChildren(xml, 'decisions', 'decision')).toEqual(['Wrap it in x']) + }) }) describe('getXmlPayload', () => { From 9ac417b9b3b41733b4b91e3e3424bd50bb36d587 Mon Sep 17 00:00:00 2001 From: neal <130699270+possiblyneal@users.noreply.github.com> Date: Sat, 21 Nov 2026 14:20:31 -0500 Subject: [PATCH 2/2] fix(parser): preserve nested and quoted child delimiters Only real child delimiters should split items. Keep nested content\nand recover a mismatched close even before an inter-item comment. Generated-By: gpt-6.2-sol --- src/prompts/xml.ts | 45 +++++++++++++++++++++++++++++++-------------- test/xml.test.ts | 15 +++++++++++++++ 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/src/prompts/xml.ts b/src/prompts/xml.ts index 8c26c4824a..a4af56ea64 100644 --- a/src/prompts/xml.ts +++ b/src/prompts/xml.ts @@ -46,9 +46,7 @@ export function getXmlTag(xml: string, tag: string): string { } // Models sometimes close an item with the wrong tag (`x`). -// Each child runs from its open tag to its own close tag, else to the next -// open tag, dropping a wrong close tag left at its end. -const STRAY_TRAILING_CLOSE = /<\/[a-zA-Z_][a-zA-Z0-9_-]*\s*>\s*$/; +const STRAY_TRAILING_CLOSE = /<\/[a-zA-Z_][a-zA-Z0-9_-]*\s*>\s*(?:)[\s\S])*-->\s*)*$/; export function getXmlChildren( xml: string, @@ -58,17 +56,36 @@ export function getXmlChildren( if (!VALID_TAG.test(parentTag) || !VALID_TAG.test(childTag)) return []; const parent = firstElementContent(xml, parentTag); if (parent === null) return []; - const close = new RegExp(closeTag(childTag)); - return parent - .split(new RegExp(openTag(childTag))) - .slice(1) - .map((piece) => { - const closed = piece.search(close); - return closed >= 0 - ? piece.slice(0, closed) - : piece.replace(STRAY_TRAILING_CLOSE, ""); - }) - .map((item) => item.trim()); + const tokens = new RegExp( + `${String.raw`||<\?[\s\S]*?\?>`}|(${openTag(childTag)})|(${closeTag(childTag)})`, + "g", + ); + const children: { start: number; contentStart: number; close?: number }[] = []; + const open: (typeof children)[number][] = []; + for (const match of parent.matchAll(tokens)) { + if (match[1] !== undefined) { + const child = { start: match.index, contentStart: match.index + match[0].length }; + children.push(child); + open.push(child); + } else if (match[2] !== undefined) { + const child = open.pop(); + if (child) child.close = match.index; + } + } + + const items: string[] = []; + let skipBefore = 0; + for (const [i, child] of children.entries()) { + if (child.start < skipBefore) continue; + if (child.close !== undefined) { + items.push(parent.slice(child.contentStart, child.close).trim()); + skipBefore = child.close; + } else { + const end = children[i + 1]?.start ?? parent.length; + items.push(parent.slice(child.contentStart, end).replace(STRAY_TRAILING_CLOSE, "").trim()); + } + } + return items; } // Every `` element, inner text keyed by id. A repeated id is diff --git a/test/xml.test.ts b/test/xml.test.ts index cdddba31e0..3f36b0524b 100644 --- a/test/xml.test.ts +++ b/test/xml.test.ts @@ -92,6 +92,21 @@ describe('getXmlChildren', () => { expect(getXmlChildren(xml, 'concepts', 'concept')).toEqual(['CPAL', 'alsa.headroom']) }) + it('preserves nested markup with the same child tag', () => { + const xml = 'Use inner hereNext' + expect(getXmlChildren(xml, 'facts', 'fact')).toEqual(['Use inner here', 'Next']) + }) + + it('ignores child delimiters inside CDATA, comments and processing instructions', () => { + const item = ']]>?>Value' + expect(getXmlChildren(`${item}`, 'facts', 'fact')).toEqual([item]) + }) + + it('drops inter-item comments after a mismatched close', () => { + const xml = 'FirstNext' + expect(getXmlChildren(xml, 'facts', 'fact')).toEqual(['First', 'Next']) + }) + it('keeps inline markup inside a properly closed item', () => { const xml = 'Wrap it in x' expect(getXmlChildren(xml, 'decisions', 'decision')).toEqual(['Wrap it in x'])