diff --git a/functions.py b/functions.py index 8b157ef..2f45dc7 100644 --- a/functions.py +++ b/functions.py @@ -1143,6 +1143,7 @@ def normalize_tool_call(tool_data_or_name, args_if_name=None): if not name or not isinstance(name, str): return None + args = _clean_tool_arg(args) if isinstance(args, (dict, list)): args_str = json.dumps(args) elif isinstance(args, str): @@ -1703,6 +1704,29 @@ def fenced(pos): ) +# Catch-all for corrupted/degenerate DSML markers that the structured +# regexes miss (e.g. a stray char between the bars: "<\uff5c\u89e6\uff5cDSML\uff5c\uff5c calls>"): +# any <...> token containing a fullwidth pipe U+FF5C, which never appears in +# legitimate prose. ASCII '|' is deliberately NOT matched (tables, code). +_FULLWIDTH_PIPE_TAG_RE = re.compile(r"<[^<>]*\uff5c[^<>]*>") + + +def _clean_tool_arg(value): + # Strip corrupted DSML markers that leaked INTO a tool-call argument + # value (e.g. a path parsed as "shell/core/Theme.qml"). + # Only touches strings that actually contain a fullwidth pipe U+FF5C, so + # normal arguments are untouched. Recurses into dicts/lists. + if isinstance(value, str): + if "\uff5c" in value: + value = _FULLWIDTH_PIPE_TAG_RE.sub("", value).strip() + return value + if isinstance(value, dict): + return {k: _clean_tool_arg(v) for k, v in value.items()} + if isinstance(value, list): + return [_clean_tool_arg(v) for v in value] + return value + + def _strip_stray_markers(text: str) -> str: """Prose path: drop bare DSML markers + orphan closers only. @@ -1711,6 +1735,7 @@ def _strip_stray_markers(text: str) -> str: """ if not text: return text + text = _FULLWIDTH_PIPE_TAG_RE.sub("", text) text = _BARE_DSML_MARKER_RE.sub("", text) return _ORPHAN_CLOSER_RE.sub("", text) @@ -1719,6 +1744,7 @@ def _strip_wrapper_tags(text: str) -> str: """Wrapper path: drop the whole tag family (open or close) + bare markers.""" if not text: return text + text = _FULLWIDTH_PIPE_TAG_RE.sub("", text) text = _BARE_DSML_MARKER_RE.sub("", text) return _WRAPPER_TAG_RE.sub("", text) @@ -1860,6 +1886,7 @@ def feed(self, chunk): hold = ">" not in tail and ( _is_plausible_stream_entry_prefix(tail) or _is_plausible_stream_closer_prefix(tail) + or "\uff5c" in tail ) if last_lt != -1 and hold: if last_lt > 0: diff --git a/plugin_helper.py b/plugin_helper.py index 6ca5496..d28c84e 100644 --- a/plugin_helper.py +++ b/plugin_helper.py @@ -184,7 +184,12 @@ def build_summary_request_prompt(messages): "Rules: close every parameter with before the next one starts; " 'string="true" for strings, string="false" for numbers, booleans, arrays and objects (valid JSON); ' "several invoke blocks may go inside one function_calls block; never put DSML tags inside a value; " - "only use tool and parameter names from the tool list." + "only use tool and parameter names from the tool list.\n" + "CRITICAL: to run a shell command, Python, or ANY code, you MUST use a tool call in the format " + "above. NEVER write a command, script, or code block as your normal message text: message text " + "is shown to the user as-is and is NOT executed. If your reply would otherwise begin with " + "something like 'cd ', 'python3 ', '#!/', a heredoc (<<'EOF'), or a fenced code block, that " + "work belongs inside a tool call, not in the message." ) diff --git a/tests/test_dsml_leaks.py b/tests/test_dsml_leaks.py index 763e0ac..ace5db9 100644 --- a/tests/test_dsml_leaks.py +++ b/tests/test_dsml_leaks.py @@ -257,3 +257,59 @@ def test_bare_invoke_without_name_is_inferred(): ] finally: request_tools.reset(token) + + +# 2026-10-25 regression: the model degenerates and emits DSML markers with a +# stray char between the fullwidth bars (e.g. "<|触|DSML|| calls>") or floods +# bare "" closers. These evade the structured DSML regexes, so a +# catch-all strips any <...> token containing a fullwidth pipe (U+FF5C). ASCII +# '|' is deliberately NOT matched. +CORRUPT = "<|触|DSML|| calls>" # stray CJK char between the bars + + +def test_corrupted_dsml_marker_stripped_from_prose(): + tools, clean = parse_tools(f"Hallo {CORRUPT}Welt") + assert tools == [] + assert "|" not in clean and "DSML" not in clean + assert "Hallo" in clean and "Welt" in clean + + +def test_corrupted_dsml_marker_not_streamed_as_text(): + p = StreamToolParser() + out = p.feed(f"Probing.{CORRUPT}done") + p.flush() + text = "".join(o.get("text", "") for o in out) + assert "|" not in text and "DSML" not in text + assert "Probing." in text and "done" in text + + +def test_corrupted_dsml_marker_split_across_chunks_not_leaked(): + s = f"ok {CORRUPT} end" + p = StreamToolParser() + out = [] + for i in range(0, len(s), 3): + out += p.feed(s[i : i + 3]) + out += p.flush() + text = "".join(o.get("text", "") for o in out) + assert "|" not in text and "DSML" not in text and "触" not in text + assert "ok" in text and "end" in text + + +def test_ascii_pipe_preserved_in_prose(): + _, clean = parse_tools("run: grep -E 'disabled|enable' file | sort") + assert "disabled|enable" in clean and "| sort" in clean + + +def test_corrupted_dsml_marker_stripped_from_tool_arg(): + # The corrupted closer must not survive inside a tool-call argument value + # (e.g. a file path), or the tool call fails with a bogus path. + from functions import normalize_tool_call + call = normalize_tool_call("Edit", {"file_path": "shell/core/Theme.qml"}) + args = json.loads(call["function"]["arguments"]) + assert args["file_path"] == "shell/core/Theme.qml", args + + +def test_ascii_pipe_preserved_in_tool_arg(): + from functions import normalize_tool_call + call = normalize_tool_call("Bash", {"command": "grep -E 'a|b' f | sort"}) + args = json.loads(call["function"]["arguments"]) + assert args["command"] == "grep -E 'a|b' f | sort", args