From 4354e6a38f6f8f0da0a46d878c73de99b3d8a5b1 Mon Sep 17 00:00:00 2001 From: Hee Ming Shan Date: Sat, 25 Jul 2026 15:08:34 +0000 Subject: [PATCH 1/2] exclude build* files in pyproject.toml --- python/pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/pyproject.toml b/python/pyproject.toml index d388d275de02..bb2c175b5ade 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -177,6 +177,7 @@ killall_sglang = "sglang.cli.killall:main" exclude = [ "assets*", "benchmark*", + "build*", "docs*", "dist*", "playground*", @@ -188,6 +189,7 @@ exclude = [ exclude = [ "assets*", "benchmark*", + "build*", "docs*", "dist*", "playground*", From 812a83ecb52095d88a0c141d7865eea7cc9ccaf5 Mon Sep 17 00:00:00 2001 From: Hee Ming Shan Date: Sat, 25 Jul 2026 15:08:57 +0000 Subject: [PATCH 2/2] Add K2V3 fallback tracking parsers --- .../srt/entrypoints/openai/serving_chat.py | 36 ++ .../srt/function_call/function_call_parser.py | 2 + .../function_call/multi_format_detector.py | 191 ++++++++- python/sglang/srt/parser/reasoning_parser.py | 93 +++- .../function_call/test_k2v3_tool_parser.py | 402 ++++++++++++++++++ .../openai_server/basic/test_serving_chat.py | 199 +++++++++ .../unit/parser/test_k2v3_reasoning_parser.py | 139 ++++++ 7 files changed, 1040 insertions(+), 22 deletions(-) diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index dcf549aa7dfe..69e84bbf362c 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -1066,6 +1066,10 @@ def _build_chat_response( request=request, ) reasoning_text, text = parser.parse_non_stream(text) + if request.return_meta_info: + self._attach_reasoning_parser_fallback_events( + ret_item["meta_info"], parser + ) except Exception as e: logger.error(f"Reasoning parsing error: {e}") return self.create_error_response( @@ -1089,6 +1093,9 @@ def _build_chat_response( request.tool_choice, history_tool_calls_cnt, chat_template_kwargs=getattr(request, "chat_template_kwargs", None), + meta_info=( + ret_item["meta_info"] if request.return_meta_info else None + ), ) # Strip structural special tokens that leaked through because @@ -1228,6 +1235,32 @@ def _process_tool_call_id( ) return tool_call_id + @staticmethod + def _attach_reasoning_parser_fallback_events( + meta_info: Optional[Dict[str, Any]], + parser: ReasoningParser, + ) -> None: + if meta_info is None: + return + detector = getattr(parser, "detector", None) + fallback_events = getattr(detector, "fallback_events", None) + if fallback_events: + meta_info["reasoning_parser_fallback_events"] = copy.deepcopy( + fallback_events + ) + + @staticmethod + def _attach_tool_parser_fallback_events( + meta_info: Optional[Dict[str, Any]], + parser: FunctionCallParser, + ) -> None: + if meta_info is None: + return + detector = getattr(parser, "detector", None) + fallback_events = getattr(detector, "fallback_events", None) + if fallback_events: + meta_info["tool_parser_fallback_events"] = copy.deepcopy(fallback_events) + def _process_tool_calls( self, text: str, @@ -1236,6 +1269,7 @@ def _process_tool_calls( tool_choice: Optional[Union[str, ToolChoice]] = None, history_tool_calls_cnt: int = 0, chat_template_kwargs: Optional[Dict[str, Any]] = None, + meta_info: Optional[Dict[str, Any]] = None, ) -> ToolCallProcessingResult: """Process tool calls in the response""" @@ -1290,6 +1324,7 @@ def _process_tool_calls( finish_reason["matched"] = None try: text, call_info_list = parser.parse_non_stream(text) + self._attach_tool_parser_fallback_events(meta_info, parser) tool_calls = [] for call_info in call_info_list: tool_id = self._process_tool_call_id( @@ -1306,6 +1341,7 @@ def _process_tool_calls( ) return ToolCallProcessingResult(tool_calls, text, finish_reason) except Exception as e: + self._attach_tool_parser_fallback_events(meta_info, parser) logger.error(f"Tool call parsing error: {e}") # Return error but don't fail the whole request return ToolCallProcessingResult(None, text, finish_reason) diff --git a/python/sglang/srt/function_call/function_call_parser.py b/python/sglang/srt/function_call/function_call_parser.py index 2bdd4010966e..3c1484ec215e 100644 --- a/python/sglang/srt/function_call/function_call_parser.py +++ b/python/sglang/srt/function_call/function_call_parser.py @@ -28,6 +28,7 @@ from sglang.srt.function_call.mistral_detector import MistralDetector from sglang.srt.function_call.multi_format_detector import ( K2V3Detector, + K2V3DetectorTracking, MultiFormatDetector, ) from sglang.srt.function_call.pythonic_detector import PythonicDetector @@ -58,6 +59,7 @@ class FunctionCallParser: "glm47": Glm47MoeDetector, "gpt-oss": GptOssDetector, "k2_v3": K2V3Detector, + "k2_v3_tracking": K2V3DetectorTracking, "kimi_k2": KimiK2Detector, "lfm2": Lfm2Detector, "llama3": Llama32Detector, diff --git a/python/sglang/srt/function_call/multi_format_detector.py b/python/sglang/srt/function_call/multi_format_detector.py index 14a540a4706e..140bd9e3deab 100644 --- a/python/sglang/srt/function_call/multi_format_detector.py +++ b/python/sglang/srt/function_call/multi_format_detector.py @@ -180,6 +180,31 @@ def parse_streaming_increment( self._buffer += new_text return StreamingParseResult() + def _on_json_loads_failed_before_ast( + self, + value: str, + error: Exception, + *, + tool_name: Optional[str] = None, + arg_name: Optional[str] = None, + target_type: Any = None, + ) -> None: + pass + + def _on_ast_literal_eval_failed_raw_string( + self, + value: str, + error: Exception, + *, + tool_name: Optional[str] = None, + arg_name: Optional[str] = None, + target_type: Any = None, + ) -> None: + pass + + def _on_ifm_reasoning_prefix_stripped(self, prefix: str, content: str) -> None: + pass + # IFM streaming (K2-V3 format) ------------------------------------ # # vLLM's K2V3ToolParser does not stream the IFM dialects; the logic below is @@ -889,8 +914,7 @@ def _ifm_json_calls(self, block: str, tools: List[Tool]): ) yield name, args - @classmethod - def _ifm_prefix(cls, text: str, first_match_index: int) -> Optional[str]: + def _ifm_prefix(self, text: str, first_match_index: int) -> Optional[str]: """Leading content before the tool calls, with IFM reasoning stripped. vLLM cuts the prefix at the wrapper when present, @@ -898,11 +922,14 @@ def _ifm_prefix(cls, text: str, first_match_index: int) -> Optional[str]: reasoning-effort block. Whitespace-only content before the tool call is preserved. """ - group_index = text.find(cls._IFM_TOOL_CALLS_START_TOKEN) + group_index = text.find(self._IFM_TOOL_CALLS_START_TOKEN) cut = group_index if group_index != -1 else first_match_index if cut <= 0: return None - content = cls._strip_ifm_reasoning_prefix(text[:cut]) + prefix = text[:cut] + content = self._strip_ifm_reasoning_prefix(prefix) + if content != prefix: + self._on_ifm_reasoning_prefix_stripped(prefix, content) return content if content != "" else None @classmethod @@ -957,9 +984,8 @@ def _json_stringify(value: Any) -> str: return value return json.dumps(value, ensure_ascii=False) - @classmethod def _coerce_argument_value( - cls, + self, value: Any, tool_name: str, arg_name: str, @@ -968,21 +994,27 @@ def _coerce_argument_value( arg_type: Optional[str] = None, from_text: bool = False, ) -> Any: - target_type = cls._schema_arg_type(tool_name, arg_name, tools) or arg_type - if cls._arg_type_preserves_text(target_type): - if cls._arg_type_is_any(target_type): + target_type = self._schema_arg_type(tool_name, arg_name, tools) or arg_type + if self._arg_type_preserves_text(target_type): + if self._arg_type_is_any(target_type): return value - return cls._json_stringify(value) + return self._json_stringify(value) if isinstance(value, str) and (from_text or target_type is not None): - return cls._deserialize_glm_value(value) + return self._deserialize_glm_value( + value, + tool_name=tool_name, + arg_name=arg_name, + target_type=target_type, + ) return value - @classmethod def _coerce_arguments( - cls, tool_name: str, arguments: dict[str, Any], tools: List[Tool] + self, tool_name: str, arguments: dict[str, Any], tools: List[Tool] ) -> dict[str, Any]: return { - arg_name: cls._coerce_argument_value(arg_value, tool_name, arg_name, tools) + arg_name: self._coerce_argument_value( + arg_value, tool_name, arg_name, tools + ) for arg_name, arg_value in arguments.items() } @@ -1038,17 +1070,35 @@ def _json_or_string(value: str) -> Any: r"(.*?)\s*(.*?)", re.DOTALL ) - @staticmethod - def _deserialize_glm_value(value: str) -> Any: + def _deserialize_glm_value( + self, + value: str, + *, + tool_name: Optional[str] = None, + arg_name: Optional[str] = None, + target_type: Any = None, + ) -> Any: value = value.strip() try: return json.loads(value) - except Exception: - pass + except Exception as json_error: + self._on_json_loads_failed_before_ast( + value, + json_error, + tool_name=tool_name, + arg_name=arg_name, + target_type=target_type, + ) try: return ast.literal_eval(value) - except Exception: - pass + except Exception as ast_error: + self._on_ast_literal_eval_failed_raw_string( + value, + ast_error, + tool_name=tool_name, + arg_name=arg_name, + target_type=target_type, + ) return value @staticmethod @@ -1149,3 +1199,104 @@ def __init__( if tool_format is None: tool_format = chat_template_kwargs.get("tool_call_format") super().__init__(tool_format=tool_format or "xml") + + +class K2V3DetectorTracking(K2V3Detector): + """K2-V3 tool parser variant that records selected fallback events. + + This detector intentionally keeps tracking opt-in so the default ``k2_v3`` + parser remains unchanged. It records non-stream coercion fallbacks and IFM + reasoning-prefix cleanup while preserving the parsed output. + """ + + def __init__( + self, + tool_format: Optional[str] = None, + chat_template_kwargs: Optional[dict] = None, + ): + self.fallback_events: list[dict[str, Any]] = [] + self._suppress_fallback_tracking = False + super().__init__( + tool_format=tool_format, chat_template_kwargs=chat_template_kwargs + ) + + def clear_fallback_events(self) -> None: + self.fallback_events.clear() + + def _record_fallback( + self, fallback_type: str, phase: str, **details: Any + ) -> None: + if self._suppress_fallback_tracking: + return + self.fallback_events.append( + {"type": fallback_type, "phase": phase, "details": details} + ) + + @staticmethod + def _preview_value(value: Any, limit: int = 120) -> str: + preview = str(value) + if len(preview) > limit: + return preview[: limit - 3] + "..." + return preview + + def detect_and_parse( + self, text: str, tools: List[Tool] + ) -> StreamingParseResult: + self.clear_fallback_events() + return super().detect_and_parse(text, tools) + + def parse_streaming_increment( + self, new_text: str, tools: List[Tool] + ) -> StreamingParseResult: + previous = self._suppress_fallback_tracking + self._suppress_fallback_tracking = True + try: + return super().parse_streaming_increment(new_text, tools) + finally: + self._suppress_fallback_tracking = previous + + def _on_json_loads_failed_before_ast( + self, + value: str, + error: Exception, + *, + tool_name: Optional[str] = None, + arg_name: Optional[str] = None, + target_type: Any = None, + ) -> None: + self._record_fallback( + "json_loads_failed_ast_literal_eval", + "coercion", + tool_name=tool_name, + arg_name=arg_name, + target_type=target_type, + value_preview=self._preview_value(value), + error=error.__class__.__name__, + ) + + def _on_ast_literal_eval_failed_raw_string( + self, + value: str, + error: Exception, + *, + tool_name: Optional[str] = None, + arg_name: Optional[str] = None, + target_type: Any = None, + ) -> None: + self._record_fallback( + "ast_literal_eval_failed_raw_string", + "coercion", + tool_name=tool_name, + arg_name=arg_name, + target_type=target_type, + value_preview=self._preview_value(value), + error=error.__class__.__name__, + ) + + def _on_ifm_reasoning_prefix_stripped(self, prefix: str, content: str) -> None: + self._record_fallback( + "ifm_reasoning_prefix_stripped", + "non_stream", + tool_format=self.tool_format, + prefix_preview=self._preview_value(prefix), + ) diff --git a/python/sglang/srt/parser/reasoning_parser.py b/python/sglang/srt/parser/reasoning_parser.py index 5b9babefc0d7..8ad33ab383e1 100644 --- a/python/sglang/srt/parser/reasoning_parser.py +++ b/python/sglang/srt/parser/reasoning_parser.py @@ -1,4 +1,4 @@ -from typing import Dict, Optional, Tuple, Type +from typing import Any, Dict, Optional, Tuple, Type from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest from sglang.srt.parser.harmony_parser import HarmonyParser @@ -51,6 +51,15 @@ def __init__( if self.think_end_token in self.previous_content: self._in_reasoning = False + def _on_tool_start_token_fallback( + self, + *, + tool_idx: int, + reasoning_text: str, + normal_text: str, + ) -> None: + pass + def detect_and_parse(self, text: str) -> StreamingParseResult: """ One-time parsing: Detects and parses reasoning sections in the provided text. @@ -79,6 +88,11 @@ def detect_and_parse(self, text: str) -> StreamingParseResult: reasoning_text = processed_text[:tool_idx] # Preserve tool_start_token in normal text normal_text = processed_text[tool_idx:] + self._on_tool_start_token_fallback( + tool_idx=tool_idx, + reasoning_text=reasoning_text, + normal_text=normal_text, + ) return StreamingParseResult( normal_text=normal_text, reasoning_text=reasoning_text ) @@ -536,6 +550,80 @@ def __init__( previous_content=previous_content, ) +class K2V3DetectorTracking(K2V3Detector): + """K2-v3 reasoning parser variant that records selected fallback events. + + This keeps fallback instrumentation opt-in. The default ``k2_v3`` parser + remains behaviorally unchanged, while ``k2_v3_tracking`` records when the + parser recovers from a missing think-end token by splitting at the IFM tool + start token. + """ + + def __init__( + self, + stream_reasoning: bool = True, + force_reasoning: bool = True, + continue_final_message: bool = False, + previous_content: str = "", + reasoning_effort: str = "high", + ): + self.fallback_events: list[dict[str, Any]] = [] + self._suppress_fallback_tracking = False + super().__init__( + stream_reasoning=stream_reasoning, + force_reasoning=force_reasoning, + continue_final_message=continue_final_message, + previous_content=previous_content, + reasoning_effort=reasoning_effort, + ) + + def clear_fallback_events(self) -> None: + self.fallback_events.clear() + + def _record_fallback( + self, fallback_type: str, phase: str, **details: Any + ) -> None: + if self._suppress_fallback_tracking: + return + self.fallback_events.append( + {"type": fallback_type, "phase": phase, "details": details} + ) + + @staticmethod + def _preview_value(value: str, limit: int = 120) -> str: + if len(value) > limit: + return value[: limit - 3] + "..." + return value + + def detect_and_parse(self, text: str) -> StreamingParseResult: + self.clear_fallback_events() + return super().detect_and_parse(text) + + def _on_tool_start_token_fallback( + self, + *, + tool_idx: int, + reasoning_text: str, + normal_text: str, + ) -> None: + self._record_fallback( + "tool_start_token_fallback", + "non_stream", + think_end_token=self.think_end_token, + tool_start_token=self.tool_start_token, + tool_start_index=tool_idx, + reasoning_preview=self._preview_value(reasoning_text), + normal_text_preview=self._preview_value(normal_text), + ) + + def parse_streaming_increment(self, new_text: str) -> StreamingParseResult: + previous = self._suppress_fallback_tracking + self._suppress_fallback_tracking = True + try: + return super().parse_streaming_increment(new_text) + finally: + self._suppress_fallback_tracking = previous + class K2V3DetectorLegacy(K2V3Detector): """ @@ -592,6 +680,7 @@ class ReasoningParser: "nemotron_3": Nemotron3Detector, "interns1": Qwen3Detector, "k2_v3": K2V3Detector, + "k2_v3_tracking": K2V3DetectorTracking, "k2_v3_legacy": K2V3DetectorLegacy, } @@ -635,7 +724,7 @@ def __init__( # pops reasoning_effort out of chat_template_kwargs and promotes it to # request.reasoning_effort (see serving_chat.py); fall back to the # kwargs dict for callers that bypass that normalization. - if model_type.lower() in ("k2_v3", "k2_v3_legacy"): + if model_type.lower() in ("k2_v3", "k2_v3_tracking", "k2_v3_legacy"): effort = ( getattr(request, "reasoning_effort", None) or chat_template_kwargs.get("reasoning_effort") diff --git a/test/registered/function_call/test_k2v3_tool_parser.py b/test/registered/function_call/test_k2v3_tool_parser.py index f1d61a66ca79..dc00def1e1a2 100644 --- a/test/registered/function_call/test_k2v3_tool_parser.py +++ b/test/registered/function_call/test_k2v3_tool_parser.py @@ -10,6 +10,7 @@ from sglang.srt.entrypoints.openai.protocol import Function, Tool from sglang.srt.function_call.multi_format_detector import ( K2V3Detector, + K2V3DetectorTracking, MultiFormatDetector, ) from sglang.test.ci.ci_register import register_cpu_ci @@ -347,6 +348,393 @@ def _collect_stream(detector, tools, chunks): return normal_text, [by_index[i] for i in order] +def _event_types(detector): + return [event["type"] for event in detector.fallback_events] + + +class TestK2V3DetectorTracking(unittest.TestCase): + """Opt-in K2-v3 detector that records selected non-stream fallbacks.""" + + def setUp(self): + self.tools = [ + _make_tool( + "study_args", + { + "type": "object", + "properties": { + "city": {"type": "string"}, + "flag": {"type": "boolean"}, + "payload": {"type": "object"}, + "days": {"type": "integer"}, + "notes": {"type": "any"}, + "user_id": {"type": "string"}, + }, + }, + ) + ] + + @staticmethod + def _call_tuple(call): + return call.tool_index, call.name, call.parameters + + def _assert_parse_equal(self, text, *, dialect="xml", tools=None): + tools = tools or self.tools + base = K2V3Detector(tool_format=dialect).detect_and_parse(text, tools) + tracked_detector = K2V3DetectorTracking(tool_format=dialect) + tracked = tracked_detector.detect_and_parse(text, tools) + + self.assertEqual(tracked.normal_text, base.normal_text) + self.assertEqual(len(tracked.calls), len(base.calls)) + self.assertEqual( + [self._call_tuple(call) for call in tracked.calls], + [self._call_tuple(call) for call in base.calls], + ) + return tracked_detector, tracked + + def _assert_stream_equal(self, chunks, *, dialect="xml", tools=None): + tools = tools or self.tools + base = K2V3Detector(tool_format=dialect) + tracked = K2V3DetectorTracking(tool_format=dialect) + for chunk in chunks: + with self.subTest(dialect=dialect, chunk=chunk): + base_result = base.parse_streaming_increment(chunk, tools) + tracked_result = tracked.parse_streaming_increment(chunk, tools) + self.assertEqual(tracked_result.normal_text, base_result.normal_text) + self.assertEqual( + [self._call_tuple(call) for call in tracked_result.calls], + [self._call_tuple(call) for call in base_result.calls], + ) + self.assertEqual(tracked.fallback_events, []) + + def test_non_stream_output_matches_base_detector(self): + cases = [ + ( + "direct_xml", + "xml", + "study_args" + "city" + "Tokyo" + "", + ), + ("xml_no_args", "xml", "study_args"), + ("unknown_tool", "xml", "unknown"), + ( + "ordinary_prefix", + "xml", + "Sure.study_args", + ), + ( + "whitespace_prefix", + "xml", + "\nstudy_args", + ), + ( + "ifm_reasoning_no_residue", + "xml", + "need lookup" + "study_args", + ), + ( + "ifm_reasoning_newline_residue", + "xml", + "need lookup\n" + "study_args", + ), + ( + "wrapper", + "xml", + "\n" + "study_args" + "", + ), + ( + "multiple_xml_calls", + "xml", + "study_args" + "city" + "Tokyo" + "" + "study_args" + "city" + "Osaka" + "", + ), + ( + "schema_string_whitespace", + "xml", + "study_args" + "city" + "\nTokyo\n" + "", + ), + ( + "schema_integer", + "xml", + "study_args" + "days" + "3" + "", + ), + ( + "schema_boolean_true", + "xml", + "study_args" + "flag" + "True" + "", + ), + ( + "raw_string_fallback", + "xml", + "study_args" + "payload" + "abc" + "", + ), + ( + "python_literal_current_behavior", + "xml", + "study_args" + "payload" + "{1, 2}" + "", + ), + ( + "xml_typed_string", + "xml_typed", + "study_args" + "user_id" + "string" + "12345" + "", + ), + ( + "xml_typed_any_whitespace", + "xml_typed", + "study_args" + "notes" + "any" + "\nkeep me\n" + "", + ), + ( + "xml_typed_boolean", + "xml_typed", + "study_args" + "flag" + "boolean" + "True" + "", + ), + ( + "schema_wins_over_inline", + "xml_typed", + "study_args" + "days" + "string" + "3" + "", + ), + ( + "json_object_args", + "json", + "" + '{"name": "study_args", "arguments": {"city": "Tokyo"}}' + "", + ), + ( + "json_string_args", + "json", + "" + '{"name": "study_args", "arguments": "{\\"city\\": \\"Tokyo\\"}"}' + "", + ), + ( + "json_call_list", + "json", + "" + '[{"name": "study_args", "arguments": {"city": "Tokyo"}},' + ' {"name": "study_args", "arguments": {"city": "Osaka"}}]' + "", + ), + ("no_tool_call_text", "xml", "No tool call here."), + ] + + for name, dialect, text in cases: + with self.subTest(name=name, dialect=dialect): + self._assert_parse_equal(text, dialect=dialect) + + def test_tracks_json_failure_before_ast_literal_eval(self): + text = ( + "study_args" + "flag" + "True" + "" + ) + + detector, result = self._assert_parse_equal(text) + + self.assertEqual(json.loads(result.calls[0].parameters), {"flag": True}) + self.assertEqual( + _event_types(detector), ["json_loads_failed_ast_literal_eval"] + ) + self.assertEqual(detector.fallback_events[0]["phase"], "coercion") + + def test_tracks_raw_string_after_json_and_ast_fail(self): + text = ( + "study_args" + "payload" + "abc" + "" + ) + + detector, result = self._assert_parse_equal(text) + + self.assertEqual(json.loads(result.calls[0].parameters), {"payload": "abc"}) + self.assertEqual( + _event_types(detector), + [ + "json_loads_failed_ast_literal_eval", + "ast_literal_eval_failed_raw_string", + ], + ) + + def test_string_schema_does_not_track_deserialization_fallbacks(self): + text = ( + "study_args" + "user_id" + "12345" + "" + ) + + detector, result = self._assert_parse_equal(text) + + self.assertEqual( + json.loads(result.calls[0].parameters), {"user_id": "12345"} + ) + self.assertEqual(detector.fallback_events, []) + + def test_tracks_ifm_reasoning_prefix_stripped(self): + cases = [ + ( + "need lookup" + "study_args" + ), + ( + "need lookup\n" + "study_args" + ), + ] + for text in cases: + with self.subTest(text=text): + detector, _ = self._assert_parse_equal(text) + self.assertEqual( + _event_types(detector), ["ifm_reasoning_prefix_stripped"] + ) + self.assertEqual(detector.fallback_events[0]["phase"], "non_stream") + + def test_clean_parse_does_not_track_events(self): + text = ( + "study_args" + "city" + "Tokyo" + "" + ) + + detector, _ = self._assert_parse_equal(text) + + self.assertEqual(detector.fallback_events, []) + + def test_reused_detector_clears_events_on_non_stream_parse(self): + detector = K2V3DetectorTracking() + fallback_text = ( + "study_args" + "payload" + "abc" + "" + ) + clean_text = ( + "study_args" + "city" + "Tokyo" + "" + ) + + detector.detect_and_parse(fallback_text, self.tools) + self.assertTrue(detector.fallback_events) + detector.detect_and_parse(clean_text, self.tools) + + self.assertEqual(detector.fallback_events, []) + + def test_clear_fallback_events(self): + detector = K2V3DetectorTracking() + detector._record_fallback("json_loads_failed_ast_literal_eval", "coercion") + + detector.clear_fallback_events() + + self.assertEqual(detector.fallback_events, []) + + def test_streaming_output_matches_base_detector_without_events(self): + cases = [ + ( + "xml_boolean_true", + "xml", + list( + "study_args" + "flag" + "True" + "" + ), + ), + ( + "xml_raw_string_payload", + "xml", + list( + "study_args" + "payload" + "abc" + "" + ), + ), + ( + "xml_typed_any_whitespace", + "xml_typed", + list( + "study_args" + "notes" + "any" + "\nkeep me\n" + "" + ), + ), + ( + "json_streaming", + "json", + list( + "" + '{"name": "study_args", "arguments": {"city": "Tokyo"}}' + "" + ), + ), + ( + "reasoning_prefix_wrapper", + "xml", + list( + "need lookup\n" + "\n" + "study_args" + "city" + "Tokyo" + "\n" + "" + ), + ), + ] + for name, dialect, chunks in cases: + with self.subTest(name=name, dialect=dialect): + self._assert_stream_equal(chunks, dialect=dialect) + + class TestK2V3XmlStreaming(unittest.TestCase): """The IFM xml/xml_typed dialects stream incrementally (name then args).""" @@ -691,6 +1079,20 @@ def test_registry_builds_k2v3_detector(self): self.assertIsInstance(parser.detector, K2V3Detector) self.assertEqual(parser.detector.tool_format, "xml") + def test_registry_builds_k2v3_tracking_detector(self): + from sglang.srt.function_call.function_call_parser import FunctionCallParser + + tools = [_make_tool("get_weather")] + parser = FunctionCallParser( + tools=tools, + tool_call_parser="k2_v3_tracking", + chat_template_kwargs={"tool_call_format": "xml"}, + ) + + self.assertIsInstance(parser.detector, K2V3DetectorTracking) + self.assertEqual(parser.detector.tool_format, "xml") + self.assertEqual(parser.detector.fallback_events, []) + def test_registry_uses_tool_call_format_dialects_from_template(self): from sglang.srt.function_call.function_call_parser import FunctionCallParser diff --git a/test/registered/openai_server/basic/test_serving_chat.py b/test/registered/openai_server/basic/test_serving_chat.py index e495b34f180a..ca8c64915be8 100644 --- a/test/registered/openai_server/basic/test_serving_chat.py +++ b/test/registered/openai_server/basic/test_serving_chat.py @@ -213,6 +213,205 @@ def test_jinja_tool_schema_fallback_to_flat_function(self): second_tools, [tool.function.model_dump() for tool in req.tools] ) + def test_k2v3_tracking_fallback_events_are_added_to_meta_info(self): + self.chat.tool_call_parser = "k2_v3_tracking" + req = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Set flag"}], + tools=[ + { + "type": "function", + "function": { + "name": "study_args", + "description": "Study parser arguments.", + "parameters": { + "type": "object", + "properties": {"flag": {"type": "boolean"}}, + }, + }, + } + ], + return_meta_info=True, + ) + ret = [ + { + "text": ( + "study_args" + "flag" + "True" + "" + ), + "meta_info": { + "id": "chatcmpl-test", + "prompt_tokens": 5, + "completion_tokens": 9, + "reasoning_tokens": 0, + "cached_tokens": 0, + "finish_reason": {"type": "stop", "matched": None}, + "weight_version": "test", + }, + "index": 0, + } + ] + + response = self.chat._build_chat_response(req, ret, created=0) + + choice = response.choices[0] + self.assertEqual(choice.finish_reason, "tool_calls") + self.assertEqual( + json.loads(choice.message.tool_calls[0].function.arguments), + {"flag": True}, + ) + events = choice.meta_info["tool_parser_fallback_events"] + self.assertEqual(events[0]["type"], "json_loads_failed_ast_literal_eval") + self.assertEqual(events[0]["phase"], "coercion") + self.assertEqual(events[0]["details"]["tool_name"], "study_args") + + def test_k2v3_reasoning_tracking_events_are_added_to_meta_info(self): + self.chat.reasoning_parser = "k2_v3_tracking" + self.template_manager.force_reasoning = True + req = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Call a tool"}], + return_meta_info=True, + ) + ret = [ + { + "text": ( + "Need to call the tool.\n" + "study_args" + "flag" + "True" + "" + ), + "meta_info": { + "id": "chatcmpl-test", + "prompt_tokens": 5, + "completion_tokens": 9, + "reasoning_tokens": 0, + "cached_tokens": 0, + "finish_reason": {"type": "stop", "matched": None}, + "weight_version": "test", + }, + "index": 0, + } + ] + + response = self.chat._build_chat_response(req, ret, created=0) + + choice = response.choices[0] + self.assertEqual(choice.message.reasoning_content, "Need to call the tool.\n") + events = choice.meta_info["reasoning_parser_fallback_events"] + self.assertEqual(events[0]["type"], "tool_start_token_fallback") + self.assertEqual(events[0]["phase"], "non_stream") + self.assertEqual(events[0]["details"]["tool_start_token"], "") + + def test_k2v3_tracking_events_are_omitted_without_meta_info(self): + self.chat.tool_call_parser = "k2_v3_tracking" + req = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Set flag"}], + tools=[ + { + "type": "function", + "function": { + "name": "study_args", + "description": "Study parser arguments.", + "parameters": { + "type": "object", + "properties": {"flag": {"type": "boolean"}}, + }, + }, + } + ], + ) + ret = [ + { + "text": ( + "study_args" + "flag" + "True" + "" + ), + "meta_info": { + "id": "chatcmpl-test", + "prompt_tokens": 5, + "completion_tokens": 9, + "reasoning_tokens": 0, + "cached_tokens": 0, + "finish_reason": {"type": "stop", "matched": None}, + "weight_version": "test", + }, + "index": 0, + } + ] + + response = self.chat._build_chat_response(req, ret, created=0) + + self.assertIsNone(response.choices[0].meta_info) + self.assertNotIn("tool_parser_fallback_events", ret[0]["meta_info"]) + + def test_k2v3_reasoning_tracking_events_are_omitted_without_meta_info(self): + self.chat.reasoning_parser = "k2_v3_tracking" + self.template_manager.force_reasoning = True + req = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Call a tool"}], + ) + ret = [ + { + "text": ( + "Need to call the tool.\n" + "study_args" + "flag" + "True" + "" + ), + "meta_info": { + "id": "chatcmpl-test", + "prompt_tokens": 5, + "completion_tokens": 9, + "reasoning_tokens": 0, + "cached_tokens": 0, + "finish_reason": {"type": "stop", "matched": None}, + "weight_version": "test", + }, + "index": 0, + } + ] + + response = self.chat._build_chat_response(req, ret, created=0) + + self.assertIsNone(response.choices[0].meta_info) + self.assertNotIn("reasoning_parser_fallback_events", ret[0]["meta_info"]) + + def test_k2v3_fallback_event_attachment_deep_copies_events(self): + parser = Mock() + parser.detector = Mock() + parser.detector.fallback_events = [ + {"type": "event", "phase": "phase", "details": {"items": []}} + ] + meta_info = {} + + self.chat._attach_tool_parser_fallback_events(meta_info, parser) + parser.detector.fallback_events[0]["details"]["items"].append("mutated") + + self.assertEqual( + meta_info["tool_parser_fallback_events"][0]["details"]["items"], [] + ) + + parser.detector.fallback_events = [ + {"type": "event", "phase": "phase", "details": {"items": []}} + ] + meta_info = {} + + self.chat._attach_reasoning_parser_fallback_events(meta_info, parser) + parser.detector.fallback_events[0]["details"]["items"].append("mutated") + + self.assertEqual( + meta_info["reasoning_parser_fallback_events"][0]["details"]["items"], [] + ) + def test_k2v3_combined_preserves_reasoning_and_content_newlines(self): self.chat.reasoning_parser = "k2_v3" self.chat.tool_call_parser = "k2_v3" diff --git a/test/registered/unit/parser/test_k2v3_reasoning_parser.py b/test/registered/unit/parser/test_k2v3_reasoning_parser.py index d8f3b0f7be3c..258883d78a5b 100644 --- a/test/registered/unit/parser/test_k2v3_reasoning_parser.py +++ b/test/registered/unit/parser/test_k2v3_reasoning_parser.py @@ -5,6 +5,7 @@ from sglang.srt.parser.reasoning_parser import ( K2V3Detector, K2V3DetectorLegacy, + K2V3DetectorTracking, ReasoningParser, ) from sglang.test.ci.ci_register import register_cpu_ci @@ -115,6 +116,102 @@ def test_tool_call_split_streaming(self): self.assertEqual(r2.reasoning_text, "") +class TestK2V3DetectorTracking(CustomTestCase): + """The opt-in K2-v3 tracking parser records selected fallback paths.""" + + def _assert_parse_equal(self, text, *, effort="high", **kwargs): + base = K2V3Detector(reasoning_effort=effort, **kwargs) + tracked = K2V3DetectorTracking(reasoning_effort=effort, **kwargs) + base_result = base.detect_and_parse(text) + tracked_result = tracked.detect_and_parse(text) + self.assertEqual(tracked_result.reasoning_text, base_result.reasoning_text) + self.assertEqual(tracked_result.normal_text, base_result.normal_text) + return tracked, tracked_result + + def _assert_stream_equal(self, chunks, *, effort="high", **kwargs): + base = K2V3Detector(reasoning_effort=effort, **kwargs) + tracked = K2V3DetectorTracking(reasoning_effort=effort, **kwargs) + for chunk in chunks: + with self.subTest(effort=effort, chunk=chunk): + base_result = base.parse_streaming_increment(chunk) + tracked_result = tracked.parse_streaming_increment(chunk) + self.assertEqual( + tracked_result.reasoning_text, base_result.reasoning_text + ) + self.assertEqual(tracked_result.normal_text, base_result.normal_text) + self.assertEqual(tracked.fallback_events, []) + + def test_output_matches_base_detector_for_all_efforts(self): + for effort in ["high", "medium", "low"]: + base = K2V3Detector(reasoning_effort=effort) + cases = [ + "reasoning" + base.think_end_token + "final", + ( + "I'll check the file.\n" + "readfilePath" + "/tmp/f" + ), + base.think_start_token + "reasoning" + base.think_end_token + "final", + ] + for text in cases: + with self.subTest(effort=effort, text=text): + self._assert_parse_equal(text, effort=effort) + + def test_tool_start_token_fallback_event_is_recorded(self): + text = ( + "I'll check the file.\n" + "readfilePath" + "/tmp/f" + ) + + detector, result = self._assert_parse_equal(text, effort="high") + + self.assertEqual(result.reasoning_text, "I'll check the file.\n") + self.assertTrue(result.normal_text.startswith("")) + self.assertEqual(len(detector.fallback_events), 1) + event = detector.fallback_events[0] + self.assertEqual(event["type"], "tool_start_token_fallback") + self.assertEqual(event["phase"], "non_stream") + self.assertEqual(event["details"]["think_end_token"], "") + self.assertEqual(event["details"]["tool_start_token"], "") + + def test_no_event_when_think_end_token_is_present(self): + detector, _ = self._assert_parse_equal( + "reasoningfinal", effort="high" + ) + + self.assertEqual(detector.fallback_events, []) + + def test_streaming_events_are_not_tracked(self): + for effort in ["high", "medium", "low"]: + self._assert_stream_equal( + ["reasoning", "read"], + effort=effort, + ) + + def test_reused_detector_clears_events_on_non_stream_parse(self): + detector = K2V3DetectorTracking(reasoning_effort="high") + detector.detect_and_parse("reasoningread") + self.assertTrue(detector.fallback_events) + detector.detect_and_parse("reasoningfinal") + + self.assertEqual(detector.fallback_events, []) + + def test_continue_final_message_with_previous_end_token_matches_base(self): + base = K2V3Detector(reasoning_effort="high") + previous_content = "already closed" + base.think_end_token + + detector, result = self._assert_parse_equal( + "continued final", + effort="high", + continue_final_message=True, + previous_content=previous_content, + ) + + self.assertEqual(result.normal_text, "continued final") + self.assertEqual(detector.fallback_events, []) + + class TestK2V3DetectorLegacyToolCallSplit(CustomTestCase): """The legacy detector exits reasoning on the legacy boundary.""" @@ -167,6 +264,26 @@ def test_preserves_reasoning_and_post_think_newlines(self): self.assertEqual(result.reasoning_text, "\n") self.assertTrue(result.normal_text.startswith("\n")) + def test_high_effort_strips_newline_prefixed_think_start(self): + detector = K2V3Detector(reasoning_effort="high") + text = "\nreasoning herefinal answer" + result = detector.detect_and_parse(text) + self.assertEqual(result.reasoning_text, "reasoning here") + self.assertEqual(result.normal_text, "final answer") + + def test_high_effort_falls_back_to_bare_think_start(self): + detector = K2V3Detector(reasoning_effort="high") + text = "reasoning herefinal answer" + result = detector.detect_and_parse(text) + self.assertEqual(result.reasoning_text, "reasoning here") + self.assertEqual(result.normal_text, "final answer") + + def test_streaming_strips_newline_prefixed_think_start(self): + detector = K2V3Detector(reasoning_effort="high") + result = detector.parse_streaming_increment("\nreasoning") + self.assertEqual(result.reasoning_text, "reasoning") + self.assertEqual(result.normal_text, "") + def test_medium_effort_parses_end_only_output(self): detector = K2V3Detector(reasoning_effort="medium") text = "reasoning herefinal answer" @@ -270,6 +387,11 @@ def test_parser_routes_to_k2v3_detector(self): self.assertIsInstance(parser.detector, K2V3Detector) self.assertEqual(parser.detector.think_start_token, "") + def test_parser_routes_to_k2v3_tracking_detector(self): + parser = ReasoningParser(model_type="k2_v3_tracking") + self.assertIsInstance(parser.detector, K2V3DetectorTracking) + self.assertEqual(parser.detector.think_start_token, "") + def test_parser_routes_to_k2v3_legacy_detector(self): parser = ReasoningParser(model_type="k2_v3_legacy") self.assertIsInstance(parser.detector, K2V3DetectorLegacy) @@ -279,12 +401,17 @@ def test_parser_routes_to_k2v3_legacy_detector(self): def test_k2v3_and_legacy_are_both_registered_for_cli(self): keys = ReasoningParser.DetectorMap.keys() self.assertIn("k2_v3", keys) + self.assertIn("k2_v3_tracking", keys) self.assertIn("k2_v3_legacy", keys) def test_parser_rejects_force_reasoning_false(self): with self.assertRaisesRegex(ValueError, "requires force_reasoning=True"): ReasoningParser(model_type="k2_v3", force_reasoning=False) + def test_tracking_parser_rejects_force_reasoning_false(self): + with self.assertRaisesRegex(ValueError, "requires force_reasoning=True"): + ReasoningParser(model_type="k2_v3_tracking", force_reasoning=False) + def test_legacy_parser_rejects_force_reasoning_false(self): with self.assertRaisesRegex(ValueError, "requires force_reasoning=True"): ReasoningParser(model_type="k2_v3_legacy", force_reasoning=False) @@ -305,6 +432,18 @@ def test_parser_forwards_reasoning_effort_medium(self): self.assertEqual(parser.detector.think_start_token, "") self.assertEqual(parser.detector.think_end_token, "") + def test_tracking_parser_forwards_reasoning_effort_medium(self): + from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest + + req = ChatCompletionRequest( + model="k2-v3", + messages=[{"role": "user", "content": "hi"}], + chat_template_kwargs={"reasoning_effort": "medium"}, + ) + parser = ReasoningParser(model_type="k2_v3_tracking", request=req) + self.assertEqual(parser.detector.think_start_token, "") + self.assertEqual(parser.detector.think_end_token, "") + def test_parser_forwards_reasoning_effort_low(self): from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest