From e93f85b7b87e1de21d563ff48e300503d9eed4e6 Mon Sep 17 00:00:00 2001 From: Varad Pimpalkhute Date: Thu, 2 Jul 2026 08:35:41 +0000 Subject: [PATCH 1/2] Keep raw string for tool-call args that literal_eval to non-JSON types --- .../srt/function_call/multi_format_detector.py | 4 +++- .../function_call/test_multi_format_detector.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/function_call/multi_format_detector.py b/python/sglang/srt/function_call/multi_format_detector.py index bbd3347e6084..abaac2214605 100644 --- a/python/sglang/srt/function_call/multi_format_detector.py +++ b/python/sglang/srt/function_call/multi_format_detector.py @@ -1030,7 +1030,9 @@ def _deserialize_glm_value(value: str) -> Any: except Exception: pass try: - return ast.literal_eval(value) + result = ast.literal_eval(value) + json.dumps(result) + return result except Exception: pass return value diff --git a/test/registered/function_call/test_multi_format_detector.py b/test/registered/function_call/test_multi_format_detector.py index a80f49def64f..d94b1cfda03f 100644 --- a/test/registered/function_call/test_multi_format_detector.py +++ b/test/registered/function_call/test_multi_format_detector.py @@ -276,6 +276,18 @@ def test_python_tuple_literal_value(self): # ast.literal handles tuple syntax; serialized as a JSON array. self.assertEqual(args["days"], [1, 2]) + def test_python_set_literal_value_stays_string(self): + text = ( + "get_weather" + "days{1, 2}" + "" + ) + result = self.det.detect_and_parse(text, self.tools) + args = json.loads(result.calls[0].parameters) + # ast.literal_eval yields a set, which JSON cannot represent; the raw + # string is kept so the arguments dict stays serializable. + self.assertEqual(args["days"], "{1, 2}") + class TestGptOssDialect(unittest.TestCase): def setUp(self): From 4cdcd734ea83f5d4273c001a33bc5514774be084 Mon Sep 17 00:00:00 2001 From: Varad Pimpalkhute Date: Sun, 26 Jul 2026 09:38:26 +0000 Subject: [PATCH 2/2] Slice returned routed_experts at routed_experts_start_len --- python/sglang/srt/entrypoints/openai/protocol.py | 4 ++++ python/sglang/srt/entrypoints/openai/serving_chat.py | 1 + python/sglang/srt/managers/tokenizer_manager.py | 3 +++ 3 files changed, 8 insertions(+) diff --git a/python/sglang/srt/entrypoints/openai/protocol.py b/python/sglang/srt/entrypoints/openai/protocol.py index 3fd5df68353a..507ebf236777 100644 --- a/python/sglang/srt/entrypoints/openai/protocol.py +++ b/python/sglang/srt/entrypoints/openai/protocol.py @@ -588,6 +588,10 @@ class ChatCompletionRequest(BaseModel): parallel_tool_calls: bool = True return_hidden_states: bool = False return_routed_experts: bool = False + # Skip this many leading token positions in the returned routed_experts + # tensor; multi-turn clients that already hold the prefix rows receive + # only the rows for newly processed tokens. + routed_experts_start_len: int = 0 return_cached_tokens_details: bool = False return_prompt_token_ids: bool = False return_completion_token_ids: bool = False diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index dcf549aa7dfe..a7428fb96b1a 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -370,6 +370,7 @@ def _convert_to_internal_request( disagg_prefill_dp_rank=request.disagg_prefill_dp_rank, return_hidden_states=request.return_hidden_states, return_routed_experts=request.return_routed_experts, + routed_experts_start_len=request.routed_experts_start_len, rid=request.rid, extra_key=self._compute_extra_key(request), require_reasoning=self._get_reasoning_from_request(request), diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 9a3499b556ac..75fb12f4502e 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -1604,6 +1604,9 @@ def _handle_batch_output( if getattr(recv_obj, "routed_experts", None): routed_experts_tensor = recv_obj.routed_experts[i] if routed_experts_tensor is not None: + start_len = getattr(state.obj, "routed_experts_start_len", 0) or 0 + if start_len > 0: + routed_experts_tensor = routed_experts_tensor[start_len:] meta_info["routed_experts"] = pybase64.b64encode( routed_experts_tensor.numpy().tobytes() ).decode("utf-8")