From 278b0bcf7f59bc86726077eaa6ef1807bb3986d6 Mon Sep 17 00:00:00 2001 From: Finn Date: Sun, 25 Oct 2026 11:08:14 +0100 Subject: [PATCH 1/3] Strip corrupted DSML markers from replies (catch-all on fullwidth pipe) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The web model occasionally degenerates and emits DSML tool-call markers with a stray character between the fullwidth bars, e.g. "<｜触｜DSML｜｜ calls>", or floods bare "" closers. These variants don't match the structured DSML regexes (which expect clean ｜｜DSML), so they leak into the reply text (seen live via a chat client). Add a catch-all `_FULLWIDTH_PIPE_TAG_RE` that strips any `<...>` token containing a fullwidth pipe U+FF5C — which never appears in legitimate prose — in the prose/leftover paths (`_strip_stray_markers`, `_strip_wrapper_tags`), i.e. after tool extraction. ASCII '|' is deliberately left untouched (tables, code). The streaming holdback in `StreamToolParser.feed` also keeps an unclosed '<' tail that already contains a U+FF5C, so a split corrupted marker is never emitted as prose before the catch-all can strip it. Tests cover the prose, streaming, split-across-chunks, and ASCII-pipe-preserved cases. Co-Authored-By: Claude Opus 4.8 --- functions.py | 10 ++++++++++ tests/test_dsml_leaks.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/functions.py b/functions.py index 8b157ef..29b08a4 100644 --- a/functions.py +++ b/functions.py @@ -1703,6 +1703,13 @@ def fenced(pos): ) +# Catch-all for corrupted/degenerate DSML markers that the structured +# regexes miss (e.g. a stray char between the bars: "<\uff5c\u89e6\uff5cDSML\uff5c\uff5c calls>"): +# any <...> token containing a fullwidth pipe U+FF5C, which never appears in +# legitimate prose. ASCII '|' is deliberately NOT matched (tables, code). +_FULLWIDTH_PIPE_TAG_RE = re.compile(r"<[^<>]*\uff5c[^<>]*>") + + def _strip_stray_markers(text: str) -> str: """Prose path: drop bare DSML markers + orphan closers only. @@ -1711,6 +1718,7 @@ def _strip_stray_markers(text: str) -> str: """ if not text: return text + text = _FULLWIDTH_PIPE_TAG_RE.sub("", text) text = _BARE_DSML_MARKER_RE.sub("", text) return _ORPHAN_CLOSER_RE.sub("", text) @@ -1719,6 +1727,7 @@ def _strip_wrapper_tags(text: str) -> str: """Wrapper path: drop the whole tag family (open or close) + bare markers.""" if not text: return text + text = _FULLWIDTH_PIPE_TAG_RE.sub("", text) text = _BARE_DSML_MARKER_RE.sub("", text) return _WRAPPER_TAG_RE.sub("", text) @@ -1860,6 +1869,7 @@ def feed(self, chunk): hold = ">" not in tail and ( _is_plausible_stream_entry_prefix(tail) or _is_plausible_stream_closer_prefix(tail) + or "\uff5c" in tail ) if last_lt != -1 and hold: if last_lt > 0: diff --git a/tests/test_dsml_leaks.py b/tests/test_dsml_leaks.py index 763e0ac..ea1829f 100644 --- a/tests/test_dsml_leaks.py +++ b/tests/test_dsml_leaks.py @@ -257,3 +257,43 @@ def test_bare_invoke_without_name_is_inferred(): ] finally: request_tools.reset(token) + + +# 2026-10-25 regression: the model degenerates and emits DSML markers with a +# stray char between the fullwidth bars (e.g. "<｜触｜DSML｜｜ calls>") or floods +# bare "" closers. These evade the structured DSML regexes, so a +# catch-all strips any <...> token containing a fullwidth pipe (U+FF5C). ASCII +# '|' is deliberately NOT matched. +CORRUPT = "<｜触｜DSML｜｜ calls>" # stray CJK char between the bars + + +def test_corrupted_dsml_marker_stripped_from_prose(): + tools, clean = parse_tools(f"Hallo {CORRUPT}Welt") + assert tools == [] + assert "｜" not in clean and "DSML" not in clean + assert "Hallo" in clean and "Welt" in clean + + +def test_corrupted_dsml_marker_not_streamed_as_text(): + p = StreamToolParser() + out = p.feed(f"Probing.{CORRUPT}done") + p.flush() + text = "".join(o.get("text", "") for o in out) + assert "｜" not in text and "DSML" not in text + assert "Probing." in text and "done" in text + + +def test_corrupted_dsml_marker_split_across_chunks_not_leaked(): + s = f"ok {CORRUPT} end" + p = StreamToolParser() + out = [] + for i in range(0, len(s), 3): + out += p.feed(s[i : i + 3]) + out += p.flush() + text = "".join(o.get("text", "") for o in out) + assert "｜" not in text and "DSML" not in text and "触" not in text + assert "ok" in text and "end" in text + + +def test_ascii_pipe_preserved_in_prose(): + _, clean = parse_tools("run: grep -E 'disabled|enable' file | sort") + assert "disabled|enable" in clean and "| sort" in clean From 3b4321556af442f789dfaaf25294c0e3194d006c Mon Sep 17 00:00:00 2001 From: Finn Date: Sun, 25 Oct 2026 11:34:51 +0100 Subject: [PATCH 2/3] Also strip corrupted DSML markers from tool-call argument values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The catch-all cleaned leaked DSML markers from reply prose, but corrupted markers also land INSIDE tool-call argument values — e.g. a path parsed as "shell/core/Theme.qml" — which makes the tool call fail (file not found). Sanitize every string argument in `normalize_tool_call` (all tool calls route through it) with the same fullwidth-pipe catch-all. Only strings that actually contain U+FF5C are touched (recurses into dicts/lists), so normal arguments and ASCII '|' are untouched. Co-Authored-By: Claude Opus 4.8 --- functions.py | 17 +++++++++++++++++ tests/test_dsml_leaks.py | 16 ++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/functions.py b/functions.py index 29b08a4..2f45dc7 100644 --- a/functions.py +++ b/functions.py @@ -1143,6 +1143,7 @@ def normalize_tool_call(tool_data_or_name, args_if_name=None): if not name or not isinstance(name, str): return None + args = _clean_tool_arg(args) if isinstance(args, (dict, list)): args_str = json.dumps(args) elif isinstance(args, str): @@ -1710,6 +1711,22 @@ def fenced(pos): _FULLWIDTH_PIPE_TAG_RE = re.compile(r"<[^<>]*\uff5c[^<>]*>") +def _clean_tool_arg(value): + # Strip corrupted DSML markers that leaked INTO a tool-call argument + # value (e.g. a path parsed as "shell/core/Theme.qml"). + # Only touches strings that actually contain a fullwidth pipe U+FF5C, so + # normal arguments are untouched. Recurses into dicts/lists. + if isinstance(value, str): + if "\uff5c" in value: + value = _FULLWIDTH_PIPE_TAG_RE.sub("", value).strip() + return value + if isinstance(value, dict): + return {k: _clean_tool_arg(v) for k, v in value.items()} + if isinstance(value, list): + return [_clean_tool_arg(v) for v in value] + return value + + def _strip_stray_markers(text: str) -> str: """Prose path: drop bare DSML markers + orphan closers only. diff --git a/tests/test_dsml_leaks.py b/tests/test_dsml_leaks.py index ea1829f..ace5db9 100644 --- a/tests/test_dsml_leaks.py +++ b/tests/test_dsml_leaks.py @@ -297,3 +297,19 @@ def test_corrupted_dsml_marker_split_across_chunks_not_leaked(): def test_ascii_pipe_preserved_in_prose(): _, clean = parse_tools("run: grep -E 'disabled|enable' file | sort") assert "disabled|enable" in clean and "| sort" in clean + + +def test_corrupted_dsml_marker_stripped_from_tool_arg(): + # The corrupted closer must not survive inside a tool-call argument value + # (e.g. a file path), or the tool call fails with a bogus path. + from functions import normalize_tool_call + call = normalize_tool_call("Edit", {"file_path": "shell/core/Theme.qml"}) + args = json.loads(call["function"]["arguments"]) + assert args["file_path"] == "shell/core/Theme.qml", args + + +def test_ascii_pipe_preserved_in_tool_arg(): + from functions import normalize_tool_call + call = normalize_tool_call("Bash", {"command": "grep -E 'a|b' f | sort"}) + args = json.loads(call["function"]["arguments"]) + assert args["command"] == "grep -E 'a|b' f | sort", args From 1fc95b31e80053b254b82b342b6dc9f1d83964b7 Mon Sep 17 00:00:00 2001 From: Finn Date: Sun, 25 Oct 2026 15:23:28 +0100 Subject: [PATCH 3/3] Harden tool-use instruction: never emit a bare command/code as message text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The model occasionally emitted a raw shell command or script as its normal message content (shown to the user but never executed) instead of a tool call — e.g. a bare "cd … && python3 - <<'EOF' … EOF" delivered straight to the chat. The DSML format block alone did not rule this out. Add an explicit directive: to run a shell command, Python or any code the model MUST use a tool call, and must never write a command/script/code block as message text. Prompt-level, so it also helps the streaming path (changes what the model emits, not parsing). Co-Authored-By: Claude Opus 4.8 --- plugin_helper.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugin_helper.py b/plugin_helper.py index 6ca5496..d28c84e 100644 --- a/plugin_helper.py +++ b/plugin_helper.py @@ -184,7 +184,12 @@ def build_summary_request_prompt(messages): "Rules: close every parameter with before the next one starts; " 'string="true" for strings, string="false" for numbers, booleans, arrays and objects (valid JSON); ' "several invoke blocks may go inside one function_calls block; never put DSML tags inside a value; " - "only use tool and parameter names from the tool list." + "only use tool and parameter names from the tool list.\n" + "CRITICAL: to run a shell command, Python, or ANY code, you MUST use a tool call in the format " + "above. NEVER write a command, script, or code block as your normal message text: message text " + "is shown to the user as-is and is NOT executed. If your reply would otherwise begin with " + "something like 'cd ', 'python3 ', '#!/', a heredoc (<<'EOF'), or a fenced code block, that " + "work belongs inside a tool call, not in the message." )