From 86906915e6137bf4325dfc61232a3cc4cc973617 Mon Sep 17 00:00:00 2001 From: areslp <100579+areslp@users.noreply.github.com> Date: Fri, 18 Sep 2026 04:02:03 +0800 Subject: [PATCH] [Bugfix] Constrain tools + response_format with a union grammar in Qwen3XMLToolParser A chat request carrying BOTH `tools` and a `response_format` (json_schema or json_object) with `tool_choice` absent or "auto" has its tool calls silently dropped: `tool_calls` comes back empty, HTTP 200, and the model returns a fabricated schema-conforming answer instead of calling the tool. Root cause: `to_sampling_params` converts `response_format` into a grammar constraining the WHOLE output without ever consulting `tools` / `tool_choice`, while `get_json_schema_from_tools()` returns None for "auto" so no tool grammar is built ("auto" normally relies on free generation plus post-hoc text parsing). Only the answer-schema grammar is installed, and the literal `` prefix falls outside its allowed token set, so the model physically cannot start a tool call. The `required` and named paths already work because `ToolParser.adjust_request` installs the tool schema and clears `response_format`; only "auto" leaves `response_format` governing decoding. Fix: override `adjust_request` in `Qwen3XMLToolParser` so this one combination installs an xgrammar `OrFormat` union of the tool-call branch and the answer schema as `structured_outputs.structural_tag`, and clears `response_format` (otherwise `to_sampling_params` derives a second constraint and the single-constraint validation raises). Every other request shape returns `super().adjust_request(request)` unchanged. Three details that are not obvious: * The tool branch reuses `get_model_structural_tag(model="qwen_3_5", ...)` with `tool_choice="required"`, i.e. a `TagsWithSeparatorFormat`, rather than the `TriggeredTagsFormat` that "auto" would produce. Measured on a Qwen3 tokenizer (vocab 248077), the triggered shape leaves 248075 tokens allowed at position 0, which makes the union degenerate and the answer branch unreachable; the separator shape leaves 5. * A leading-whitespace prefix is allowed. With `enable_in_reasoning=False` the grammar binds at the token immediately after `` while the chat template emits `\n\n`, so without it the tool-vs-answer decision is taken at a token whose natural probability mass is masked. * The registry hardcodes `separator=""` for `qwen_3_5`, but the chat template emits `\n` between parallel calls, so the separator is corrected at the use site. The registry itself deviates from both the template and upstream xgrammar's builder here, which also affects the strict-mode `required` path; left for a separate change. The union compiles every tool's `parameters` through the qwen_xml converter, which the "auto" path never did before, so `adjust_request` self-validates with `Grammar.from_structural_tag` and, on failure, retries with tool parameters left unconstrained before falling back to the previous behaviour. That keeps a schema feature xgrammar cannot express (regex lookahead, for example) from turning into a request error via the guidance backend fallback. Verified on a 15-case matrix against a live server: 7 previously broken combinations now return real tool calls, 8 working combinations are unchanged, the answer branch remains reachable when no tool is needed, and parallel tool calls survive. Co-Authored-By: Claude Opus 5 Signed-off-by: areslp <100579+areslp@users.noreply.github.com> --- vllm/tool_parsers/qwen3xml_tool_parser.py | 165 ++++++++++++++++++++++ 1 file changed, 165 insertions(+) diff --git a/vllm/tool_parsers/qwen3xml_tool_parser.py b/vllm/tool_parsers/qwen3xml_tool_parser.py index d5b87ea074..f70ca1c0c5 100644 --- a/vllm/tool_parsers/qwen3xml_tool_parser.py +++ b/vllm/tool_parsers/qwen3xml_tool_parser.py @@ -7,6 +7,18 @@ from xml.parsers.expat import ParserCreate import regex as re +from xgrammar import Grammar +from xgrammar.structural_tag import ( + AnyTextFormat, + ConstStringFormat, + JSONSchemaFormat, + OptionalFormat, + OrFormat, + RegexFormat, + SequenceFormat, + StructuralTag, + TagFormat, +) from vllm.entrypoints.chat_utils import make_tool_call_id from vllm.entrypoints.openai.chat_completion.protocol import ( @@ -20,12 +32,22 @@ FunctionCall, ToolCall, ) +from vllm.entrypoints.openai.responses.protocol import ( + ResponsesRequest, +) from vllm.logger import init_logger +from vllm.sampling_params import ( + StructuredOutputsParams, +) from vllm.tokenizers import TokenizerLike from vllm.tool_parsers.abstract_tool_parser import ( Tool, ToolParser, ) +from vllm.tool_parsers.structural_tag_registry import ( + get_enable_structured_outputs_in_reasoning, + get_model_structural_tag, +) from vllm.tool_parsers.utils import find_tool_properties logger = init_logger(__name__) @@ -1161,6 +1183,149 @@ def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): "vLLM Successfully import tool parser %s !", self.__class__.__name__ ) + def adjust_request( + self, + request: ChatCompletionRequest | ResponsesRequest, + ) -> ChatCompletionRequest | ResponsesRequest: + # Narrowly target the one combination the base class mishandles: tools + # present, tool_choice "auto", AND a response_format set. There, + # to_sampling_params turns response_format into a grammar over the WHOLE + # output while get_json_schema_from_tools() returns None for "auto" (no + # tool grammar is built), so the answer-schema grammar leaves the literal + # "" prefix outside the allowed token set and tool calls are + # silently dropped. + if not isinstance(request, ChatCompletionRequest) or not request.tools: + return super().adjust_request(request) + + # Handle "auto" only. An ABSENT tool_choice is already normalized to + # "auto" upstream when tools are present, so nothing is lost; explicit + # None / "none" / "required" / named must keep base behaviour (serving + # treats a falsy tool_choice as "none" and would otherwise surface the + # raw XML branch as message content). + if request.tool_choice != "auto": + return super().adjust_request(request) + + # If a native structured-outputs constraint is already set, adding a + # structural_tag would stack a second constraint and trip + # StructuredOutputsParams' single-constraint check. Leave those to the + # base path (response_format wins there, as it does today). + if ( + request.structured_outputs is not None + and not request.structured_outputs.all_constraints_none() + ): + return super().adjust_request(request) + + # Derive the answer-side format from response_format the way + # to_sampling_params reads it. + answer_format = None + response_format = request.response_format + if response_format is not None: + if response_format.type == "json_object": + answer_format = JSONSchemaFormat(json_schema={"type": "object"}) + elif response_format.type == "json_schema": + json_schema = response_format.json_schema + schema = json_schema.json_schema if json_schema is not None else None + if schema is not None: + answer_format = JSONSchemaFormat(json_schema=schema) + # "text" -> no constraint; "structural_tag" -> caller composed its + # own tag. Both leave answer_format=None. + + # No answer constraint => nothing to union with; preserve today's + # free-generation "auto" tool calling (tool text is parsed post-hoc). + if answer_format is None: + return super().adjust_request(request) + + reasoning = get_enable_structured_outputs_in_reasoning() + + def build_tag_json(tools): + # Reuse the registry for the tool-call branch so the wire format + # stays in sync with vLLM. tool_choice="required" + # (TagsWithSeparatorFormat) not "auto" (TriggeredTagsFormat): on the + # production tokenizer (vocab 248077) the "auto" shape leaves ~248075 + # tokens allowed at position 0, making the union degenerate and the + # answer branch unreachable; "required" keeps the mask tight. + tools_tag = get_model_structural_tag( + model="qwen_3_5", + tools=tools, + tool_choice="required", + reasoning=False, + ) + assert tools_tag is not None + # qwen_3_5 hardcodes separator="" in the registry, but the chat + # template emits "\n" between parallel calls, so a real + # \n sequence would otherwise be rejected. Fix + # the separator at this use site (do NOT edit the shared registry). + tools_format = tools_tag.format.model_copy(update={"separator": "\n"}) + # Allow the model's natural leading whitespace: with + # enable_in_reasoning=False the grammar binds right after "" + # while the template emits "\n\n\n", so the first real token + # is "\n\n". Without this the tool-vs-answer decision would be made on + # a masked token. Whitespace alone cannot satisfy the OrFormat, so it + # cannot terminate the match. + union = SequenceFormat( + elements=[ + OptionalFormat(content=RegexFormat(pattern="[ \\n\\t]{1,8}")), + OrFormat(elements=[tools_format, answer_format]), + ] + ) + if reasoning: + prefix = SequenceFormat( + elements=[ + TagFormat(begin="", content=AnyTextFormat(), end=""), + ConstStringFormat(value="\n\n"), + ] + ) + union = SequenceFormat(elements=[prefix, union]) + return json.dumps(StructuralTag(format=union).model_dump()) + + def compiles(tag_json): + try: + Grammar.from_structural_tag(tag_json) + except Exception as exc: + return exc + return True + + # Self-validate: the union compiles every tool's parameters through the + # qwen_xml converter, which today's "auto" path never does. A schema + # feature xgrammar cannot express (e.g. regex lookahead) would raise and, + # under backend="auto", fall back to the guidance backend which crashes + # on v2 structural tags. Degrade gracefully instead of erroring. + tag_json = build_tag_json(request.tools) + outcome = compiles(tag_json) + if outcome is not True: + logger.warning( + "Qwen3XMLToolParser: tool schemas are not xgrammar-compatible " + "(%s); retrying with unconstrained tool arguments.", + outcome, + ) + # Relax every tool's parameters to unconstrained. Not lossy vs. the + # status quo: today's "auto" path does not constrain tool arguments + # at all, and the XML call structure stays constrained. + relaxed_tools = [tool.model_copy(deep=True) for tool in request.tools] + for tool in relaxed_tools: + tool.function.parameters = None + tag_json = build_tag_json(relaxed_tools) + outcome = compiles(tag_json) + if outcome is not True: + logger.warning( + "Qwen3XMLToolParser: structural tag still uncompilable (%s); " + "falling back to default tool calling.", + outcome, + ) + return super().adjust_request(request) + + # Match AbstractToolParser.adjust_request's structural-tag style. + if request.structured_outputs is None: + request.structured_outputs = StructuredOutputsParams( + structural_tag=tag_json, + ) + else: + request.structured_outputs.structural_tag = tag_json + # Mandatory: otherwise to_sampling_params derives a second constraint + # from response_format and the single-constraint validation raises. + request.response_format = None + return request + def extract_tool_calls( self, model_output: str,