From 1de3aa006b67b0ed0c571dca751dc8f0c54f9e1c Mon Sep 17 00:00:00 2001 From: guojn1 Date: Thu, 15 Oct 2026 21:43:07 +0800 Subject: [PATCH 1/9] fix(frontend): opt in GLM malformed history argument compatibility --- dingo/frontend/README.md | 32 ++++++++++++ dingo/frontend/frontend_args.py | 18 +++++++ dingo/frontend/glm_history.py | 34 +++++++++++++ dingo/frontend/main.py | 1 + dingo/frontend/sglang_prepost.py | 5 +- dingo/frontend/sglang_processor.py | 7 +++ dingo/frontend/tests/test_glm_history.py | 46 +++++++++++++++++ .../frontend/tests/test_glm_history_config.py | 31 ++++++++++++ .../tests/test_sglang_reasoning_stops_unit.py | 1 + lib/bindings/python/rust/llm/entrypoint.rs | 24 ++++++--- lib/bindings/python/src/dynamo/_core.pyi | 1 + lib/llm/src/frontend_config.rs | 26 ++++++++++ lib/llm/src/http/service/openai.rs | 28 +++++++++-- lib/llm/src/http/service/service_v2.rs | 5 ++ .../src/protocols/openai/chat_completions.rs | 50 ++++++++++++++++++- lib/llm/src/protocols/openai/validate.rs | 23 +++++++-- lib/llm/tests/common/http_harness.rs | 8 +++ .../frontend_protocol_validation_http.rs | 48 ++++++++++++++++++ 18 files changed, 371 insertions(+), 17 deletions(-) create mode 100644 dingo/frontend/glm_history.py create mode 100644 dingo/frontend/tests/test_glm_history.py create mode 100644 dingo/frontend/tests/test_glm_history_config.py diff --git a/dingo/frontend/README.md b/dingo/frontend/README.md index 878bbd730ebf..d91d7901e694 100644 --- a/dingo/frontend/README.md +++ b/dingo/frontend/README.md @@ -39,3 +39,35 @@ reasoning parser configuration. Detection uses the discovered model's `config.json`, so serving aliases do not affect it. Explicit parser settings take precedence. Unknown model types with thinking templates produce a warning requesting explicit parser configuration. + +## GLM-5.3 malformed historical tool arguments + +For a GLM-5.3 deployment using `--dyn-chat-processor sglang`, opt in with +`--glm-history-arguments-models glm-5.3` (environment default: +`DYN_GLM_HISTORY_ARGUMENTS_MODELS`). This is a comma-separated list of exact, +case-sensitive canonical served model names, after request alias resolution. +An empty list, including an explicit empty CLI value, keeps strict validation. + +The setting applies to `/v1/chat/completions` and `/v1/responses`, including +streaming. It permits malformed JSON **syntax** in historical assistant +`tool_calls[].function.arguments` / Responses `function_call.arguments`. +Valid JSON arrays, strings, numbers, booleans and null remain invalid; the +existing acceptance of empty argument strings is unchanged. Other validation +and newly generated tool-call handling are unchanged. This does not make +malformed arguments safe to execute: the client must return a tool error and +let the model retry with valid arguments. + +Rust owns admission and receives this deployment setting explicitly through +PyO3. Python preserves malformed strings and adapts the discovered GLM Jinja +in memory for the named models, including preprocessing subprocesses. Object +arguments retain the original `` / `` rendering; strings +are preserved verbatim after the tool name. No shared model files or +frontend/worker wire schemas change. Unknown template revisions fail model +initialization rather than silently applying an unverified transformation. +An explicitly supplied `--chat-template` is subject to the same check. + +Use a dedicated GLM frontend and the same immutable image on frontend/workers +for initial rollout. Verify valid/invalid history, streaming, Responses and a +tool-error/retry round trip before production rollout. Removing the option +and restarting the frontend restores strict admission and the original +template. Native `sglang serve` does not use this Dingo configuration. diff --git a/dingo/frontend/frontend_args.py b/dingo/frontend/frontend_args.py index 908c64fa93ea..d11e40eb361b 100644 --- a/dingo/frontend/frontend_args.py +++ b/dingo/frontend/frontend_args.py @@ -97,6 +97,7 @@ class FrontendConfig(RouterConfigBase, KvRouterConfigBase, AisPerfConfigBase): enable_streaming_tool_dispatch: bool enable_streaming_reasoning_dispatch: bool reasoning_field_name: str + glm_history_arguments_models: list[str] exclude_tools_when_tool_choice_none: bool preprocess_workers: int tokenizer_backend: str @@ -107,6 +108,10 @@ class FrontendConfig(RouterConfigBase, KvRouterConfigBase, AisPerfConfigBase): _VALID_TOKENIZER_BACKENDS = {"default", "fastokens", "basetenkenizer"} def validate(self) -> None: + if self.glm_history_arguments_models and self.chat_processor != "sglang": + raise ValueError( + "--glm-history-arguments-models requires --dyn-chat-processor sglang" + ) if self.load_aware: self.router_mode = "kv" self.apply_router_config() @@ -603,6 +608,19 @@ def add_arguments(self, parser) -> None: ), choices=["reasoning_content", "reasoning"], ) + add_argument( + g, + flag_name="--glm-history-arguments-models", + env_var="DYN_GLM_HISTORY_ARGUMENTS_MODELS", + default=[], + arg_type=lambda value: [ + name.strip() for name in value.split(",") if name.strip() + ], + help=( + "Comma-separated canonical GLM model names permitting malformed historical " + "tool arguments. Adapts the verified GLM Jinja in memory; default is strict." + ), + ) # NOTE: This flag also exists in DynamoRuntimeArgGroup (runtime_args.py). # Both definitions are needed: runtime_args controls the Rust-native # chat template path (oai.rs), while this one controls the Python diff --git a/dingo/frontend/glm_history.py b/dingo/frontend/glm_history.py new file mode 100644 index 000000000000..51cb05ec14ca --- /dev/null +++ b/dingo/frontend/glm_history.py @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Opt-in GLM history compatibility, preserving malformed arguments verbatim.""" + +_OBJECT_ARGS = ( + "{% set _args = tc.arguments %}" + "{% for k, v in _args.items() %}{{ k }}" + "{{ v | tojson(ensure_ascii=False) if v is not string else v }}" + "{% endfor %}" +) +_RAW_OR_OBJECT_ARGS = ( + "{% set _args = tc.arguments %}" + '{% if _args is string %}{{ "\\n" + _args }}{% else %}' + + _OBJECT_ARGS.removeprefix("{% set _args = tc.arguments %}") + + "{% endif %}" +) + + +def adapt_glm_history_template(template: str) -> str: + """Adapt the verified GLM-5.3 template; fail closed on unknown revisions. + + No model files are modified. The resulting template is shared with both + the inline tokenizer and preprocessing subprocesses. Object rendering is + unchanged; invalid JSON remains visible to the model as historical text. + """ + if not isinstance(template, str): + raise ValueError("GLM history compatibility requires a single Jinja template") + if template.count(_RAW_OR_OBJECT_ARGS) == 1: + return template + if template.count(_OBJECT_ARGS) != 1 or "" not in template: + raise ValueError( + "Unrecognized GLM history template; review its argument rendering " + "before enabling --glm-history-arguments-models" + ) + return template.replace(_OBJECT_ARGS, _RAW_OR_OBJECT_ARGS, 1) diff --git a/dingo/frontend/main.py b/dingo/frontend/main.py index 908daa783615..be297cafbb81 100644 --- a/dingo/frontend/main.py +++ b/dingo/frontend/main.py @@ -435,6 +435,7 @@ def signal_handler(): "enable_streaming_tool_dispatch": config.enable_streaming_tool_dispatch, "enable_streaming_reasoning_dispatch": config.enable_streaming_reasoning_dispatch, "reasoning_field_name": config.reasoning_field_name, + "glm_history_arguments_models": config.glm_history_arguments_models, "tokenizer_backend": config.tokenizer_backend, "tokenizer_fallback": config.tokenizer_fallback, } diff --git a/dingo/frontend/sglang_prepost.py b/dingo/frontend/sglang_prepost.py index 58d77d5110b9..7a95408bce5f 100644 --- a/dingo/frontend/sglang_prepost.py +++ b/dingo/frontend/sglang_prepost.py @@ -356,8 +356,9 @@ def _normalize_assistant_tool_call_arguments(messages: list[dict[str, Any]]) -> (``serving_chat.py``) so multi-turn conversations containing prior tool calls render correctly. - Malformed JSON is left untouched so the chat-template error remains - visible to the caller instead of being silently corrupted. + Malformed JSON is left untouched. HTTP admission rejects it by default; + explicitly enabled GLM deployments render it verbatim as historical text. + Never repair or silently replace it with an empty object. """ for msg in messages: if not isinstance(msg, dict) or msg.get("role") != "assistant": diff --git a/dingo/frontend/sglang_processor.py b/dingo/frontend/sglang_processor.py index b214c1f52886..08f903e6dbc8 100644 --- a/dingo/frontend/sglang_processor.py +++ b/dingo/frontend/sglang_processor.py @@ -1387,6 +1387,13 @@ async def chat_engine_factory( logger.info("Using custom chat template override") tokenizer.chat_template = chat_template + if mdc.name() in self.config.glm_history_arguments_models: + from .glm_history import adapt_glm_history_template + + chat_template = adapt_glm_history_template(tokenizer.chat_template) + tokenizer.chat_template = chat_template + logger.info("Enabled GLM history arguments compatibility for %s", mdc.name()) + eos_token_ids = _resolve_model_eos_token_ids( tokenizer=tokenizer, source_path=local_dir, diff --git a/dingo/frontend/tests/test_glm_history.py b/dingo/frontend/tests/test_glm_history.py new file mode 100644 index 000000000000..b37e7f07cc72 --- /dev/null +++ b/dingo/frontend/tests/test_glm_history.py @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: Apache-2.0 +"""GLM history repair must preserve valid prompts and the original invalid text.""" + +import json + +import pytest +from jinja2 import Environment, StrictUndefined + +from dingo.frontend.glm_history import adapt_glm_history_template + +pytestmark = [pytest.mark.unit, pytest.mark.gpu_0, pytest.mark.pre_merge] + +# Representative GLM-5.3 tool-call block, independent of the adapter constants. +TEMPLATE = ( + "{{ '' + tc.name }}" + "{% set _args = tc.arguments %}{% for k, v in _args.items() %}" + "{{ k }}" + "{{ v | tojson(ensure_ascii=False) if v is not string else v }}" + "{% endfor %}" +) + + +def render(template, arguments): + env = Environment(undefined=StrictUndefined, autoescape=False) + env.filters["tojson"] = lambda value, **kw: json.dumps(value, **kw) + return env.from_string(template).render(tc={"name": "lookup", "arguments": arguments}) + + +@pytest.mark.parametrize("arguments", [{}, {"city": "北京"}, {"a": [1, 2], "b": {"x": True}}]) +def test_valid_object_prompt_unchanged(arguments): + assert render(adapt_glm_history_template(TEMPLATE), arguments) == render(TEMPLATE, arguments) + + +@pytest.mark.parametrize("arguments", ['{"city":', '{"a": 1,}', 'not json', '{"x":"{{ injected }}']) +def test_invalid_history_preserved_verbatim(arguments): + assert render(adapt_glm_history_template(TEMPLATE), arguments) == ( + "lookup\n" + arguments + "" + ) + + +def test_idempotent_and_unknown_template_rejected(): + adapted = adapt_glm_history_template(TEMPLATE) + assert adapt_glm_history_template(adapted) == adapted + for unknown in (None, {}, "{{ messages }}", TEMPLATE + TEMPLATE): + with pytest.raises(ValueError): + adapt_glm_history_template(unknown) diff --git a/dingo/frontend/tests/test_glm_history_config.py b/dingo/frontend/tests/test_glm_history_config.py new file mode 100644 index 000000000000..e8e3b4ac4ce8 --- /dev/null +++ b/dingo/frontend/tests/test_glm_history_config.py @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: Apache-2.0 +import argparse + +import pytest + +from dingo.frontend.frontend_args import FrontendArgGroup, FrontendConfig + +pytestmark = [pytest.mark.unit, pytest.mark.gpu_0, pytest.mark.pre_merge] + + +def config_for(args): + parser = argparse.ArgumentParser() + FrontendArgGroup().add_arguments(parser) + return FrontendConfig.from_cli_args(parser.parse_args(args)) + + +def test_glm_history_config_default_and_backend_gate(monkeypatch): + monkeypatch.delenv("DYN_GLM_HISTORY_ARGUMENTS_MODELS", raising=False) + assert config_for([]).glm_history_arguments_models == [] + config = config_for(["--dyn-chat-processor", "dynamo", "--glm-history-arguments-models", "glm-5.3"]) + with pytest.raises(ValueError, match="requires --dyn-chat-processor sglang"): + config.validate() + + +def test_glm_history_cli_overrides_environment(monkeypatch): + monkeypatch.setenv("DYN_GLM_HISTORY_ARGUMENTS_MODELS", "env-model") + assert config_for([]).glm_history_arguments_models == ["env-model"] + config = config_for(["--dyn-chat-processor", "sglang", "--glm-history-arguments-models", "glm-5.3, second"]) + config.validate() + assert config.glm_history_arguments_models == ["glm-5.3", "second"] + assert config_for(["--glm-history-arguments-models", ""]).glm_history_arguments_models == [] diff --git a/dingo/frontend/tests/test_sglang_reasoning_stops_unit.py b/dingo/frontend/tests/test_sglang_reasoning_stops_unit.py index 212012f92e15..7b2a4e40bfe4 100644 --- a/dingo/frontend/tests/test_sglang_reasoning_stops_unit.py +++ b/dingo/frontend/tests/test_sglang_reasoning_stops_unit.py @@ -80,6 +80,7 @@ def test_factory_selects_parser_and_preserves_reasoning_before_stop( factory = processor_module.SglangEngineFactory( SimpleNamespace( trust_remote_code=False, + glm_history_arguments_models=[], preprocess_workers=0, exclude_tools_when_tool_choice_none=True, ), diff --git a/lib/bindings/python/rust/llm/entrypoint.rs b/lib/bindings/python/rust/llm/entrypoint.rs index 417d0b1c8fd0..cf93faed0f5c 100644 --- a/lib/bindings/python/rust/llm/entrypoint.rs +++ b/lib/bindings/python/rust/llm/entrypoint.rs @@ -491,7 +491,7 @@ pub(crate) struct EntrypointArgs { impl EntrypointArgs { #[allow(clippy::too_many_arguments)] #[new] - #[pyo3(signature = (engine_type, model_path=None, model_name=None, endpoint_id=None, template_file=None, router_config=None, kv_cache_block_size=None, http_host=None, http_port=None, http_metrics_port=None, tls_cert_path=None, tls_key_path=None, extra_engine_args=None, mocker_engine_args=None, runtime_config=None, namespace=None, namespace_prefix=None, is_prefill=false, is_decode=false, migration_limit=0, migration_max_seq_len=None, chat_engine_factory=None, ais_perf_config=None, *, tls_client_ca_cert_path=None, metrics_prefix=None, enable_anthropic_api=None, strip_anthropic_preamble=None, enable_streaming_tool_dispatch=None, enable_streaming_reasoning_dispatch=None, reasoning_field_name=None, tokenizer_backend=None, tokenizer_fallback=None))] + #[pyo3(signature = (engine_type, model_path=None, model_name=None, endpoint_id=None, template_file=None, router_config=None, kv_cache_block_size=None, http_host=None, http_port=None, http_metrics_port=None, tls_cert_path=None, tls_key_path=None, extra_engine_args=None, mocker_engine_args=None, runtime_config=None, namespace=None, namespace_prefix=None, is_prefill=false, is_decode=false, migration_limit=0, migration_max_seq_len=None, chat_engine_factory=None, ais_perf_config=None, *, tls_client_ca_cert_path=None, metrics_prefix=None, enable_anthropic_api=None, strip_anthropic_preamble=None, enable_streaming_tool_dispatch=None, enable_streaming_reasoning_dispatch=None, reasoning_field_name=None, glm_history_arguments_models=None, tokenizer_backend=None, tokenizer_fallback=None))] pub fn new( py: Python<'_>, engine_type: EngineType, @@ -524,6 +524,7 @@ impl EntrypointArgs { enable_streaming_tool_dispatch: Option, enable_streaming_reasoning_dispatch: Option, reasoning_field_name: Option, + glm_history_arguments_models: Option>, tokenizer_backend: Option, tokenizer_fallback: Option, ) -> PyResult { @@ -585,6 +586,19 @@ impl EntrypointArgs { } runtime_config.validate_config()?; + let mut frontend_api_config = FrontendApiConfig::from_optional_flags( + enable_anthropic_api, + strip_anthropic_preamble, + enable_streaming_tool_dispatch, + enable_streaming_reasoning_dispatch, + reasoning_field, + ); + if let Some(models) = glm_history_arguments_models { + frontend_api_config + .get_or_insert_with(FrontendApiConfig::default) + .set_glm_history_arguments_models(models); + } + Ok(EntrypointArgs { engine_type, model_path, @@ -597,13 +611,7 @@ impl EntrypointArgs { http_port: http_port.unwrap_or(DEFAULT_HTTP_PORT), http_metrics_port, metrics_config: metrics_prefix.map(|prefix| MetricsConfig::new(Some(prefix))), - frontend_api_config: FrontendApiConfig::from_optional_flags( - enable_anthropic_api, - strip_anthropic_preamble, - enable_streaming_tool_dispatch, - enable_streaming_reasoning_dispatch, - reasoning_field, - ), + frontend_api_config, tls_cert_path, tls_key_path, tls_client_ca_cert_path, diff --git a/lib/bindings/python/src/dynamo/_core.pyi b/lib/bindings/python/src/dynamo/_core.pyi index 6c39b182c343..b21eef91dd54 100644 --- a/lib/bindings/python/src/dynamo/_core.pyi +++ b/lib/bindings/python/src/dynamo/_core.pyi @@ -3055,6 +3055,7 @@ class EntrypointArgs: strip_anthropic_preamble: Optional[bool] = None, enable_streaming_tool_dispatch: Optional[bool] = None, enable_streaming_reasoning_dispatch: Optional[bool] = None, + glm_history_arguments_models: Optional[List[str]] = None, tokenizer_backend: Optional[str] = None, tokenizer_fallback: Optional[bool] = None, ) -> None: diff --git a/lib/llm/src/frontend_config.rs b/lib/llm/src/frontend_config.rs index aac63a2e431b..29d70a5f3db0 100644 --- a/lib/llm/src/frontend_config.rs +++ b/lib/llm/src/frontend_config.rs @@ -170,6 +170,7 @@ pub struct FrontendApiConfig { anthropic: AnthropicApiConfig, streaming_dispatch: StreamingDispatchConfig, reasoning_field: ReasoningField, + glm_history_arguments_models: Vec, } impl FrontendApiConfig { @@ -182,6 +183,7 @@ impl FrontendApiConfig { anthropic, streaming_dispatch, reasoning_field, + glm_history_arguments_models: Vec::new(), } } @@ -199,6 +201,7 @@ impl FrontendApiConfig { enable_streaming_reasoning_dispatch, ), reasoning_field, + glm_history_arguments_models: Vec::new(), } } @@ -250,6 +253,16 @@ impl FrontendApiConfig { self.reasoning_field } + pub fn allows_invalid_history_arguments(&self, model: &str) -> bool { + self.glm_history_arguments_models + .iter() + .any(|name| name == model) + } + + pub fn set_glm_history_arguments_models(&mut self, models: Vec) { + self.glm_history_arguments_models = models; + } + pub fn set_reasoning_field(&mut self, reasoning_field: ReasoningField) { self.reasoning_field = reasoning_field; } @@ -261,6 +274,7 @@ impl Default for FrontendApiConfig { anthropic: Default::default(), streaming_dispatch: Default::default(), reasoning_field: default_reasoning_field(), + glm_history_arguments_models: Vec::new(), } } } @@ -269,6 +283,18 @@ impl Default for FrontendApiConfig { mod tests { use super::*; + #[test] + fn history_arguments_policy_is_explicit_and_model_scoped() { + let mut config = FrontendApiConfig::default(); + assert!(!config.allows_invalid_history_arguments("glm-5.3")); + config.set_glm_history_arguments_models(vec!["glm-5.3".to_string()]); + assert!(config.allows_invalid_history_arguments("glm-5.3")); + assert!(!config.allows_invalid_history_arguments("other-glm-5.3")); + assert!(!config.allows_invalid_history_arguments("qwen")); + config.set_glm_history_arguments_models(Vec::new()); + assert!(!config.allows_invalid_history_arguments("glm-5.3")); + } + #[test] fn optional_flags_return_none_when_all_values_are_unspecified() { let config = FrontendApiConfig::from_optional_flags(None, None, None, None, None); diff --git a/lib/llm/src/http/service/openai.rs b/lib/llm/src/http/service/openai.rs index 5f1970d00100..6bfca4e81b0d 100644 --- a/lib/llm/src/http/service/openai.rs +++ b/lib/llm/src/http/service/openai.rs @@ -4244,7 +4244,10 @@ async fn chat_completions( } // Handle Rest of Validation Errors - if let Err(err_response) = validate_chat_completion_fields_generic(&request) { + if let Err(err_response) = validate_chat_completion_fields_with_history_policy( + &request, + state.allows_invalid_history_arguments(&request.inner.model), + ) { inflight_guard.mark_error(extract_error_type_from_response(&err_response)); return Err(err_response); } @@ -4684,7 +4687,14 @@ fn validate_request_fields_generic( request: &R, request_kind: &str, ) -> Result<(), ErrorResponse> { - request.validate().map_err(|e| { + validation_result_response(request.validate(), request_kind) +} + +fn validation_result_response( + result: Result<(), anyhow::Error>, + request_kind: &str, +) -> Result<(), ErrorResponse> { + result.map_err(|e| { if find_invalid_argument_in_chain(e.as_ref()).is_some() { return ErrorMessage::from_anyhow(e, &format!("Invalid {request_kind} request")); } @@ -4705,6 +4715,16 @@ pub fn validate_chat_completion_fields_generic( validate_request_fields_generic(request, "chat completion") } +fn validate_chat_completion_fields_with_history_policy( + request: &NvCreateChatCompletionRequest, + allow_invalid_history_arguments: bool, +) -> Result<(), ErrorResponse> { + validation_result_response( + request.validate_with_history_policy(allow_invalid_history_arguments), + "chat completion", + ) +} + /// Validates that stream_options is only used when stream=true for completions (NVBug 5662680) pub fn validate_completion_stream_options( request: &NvCreateCompletionRequest, @@ -4960,7 +4980,9 @@ async fn responses( inflight_guard.mark_error(extract_error_type_from_response(&err_response)); return Err(err_response); } - if let Err(error) = chat_request.validate() { + if let Err(error) = chat_request.validate_with_history_policy( + state.allows_invalid_history_arguments(&chat_request.inner.model), + ) { let err_response = ErrorMessage::from_anyhow( invalid_argument(error.to_string()).into(), "Invalid responses request", diff --git a/lib/llm/src/http/service/service_v2.rs b/lib/llm/src/http/service/service_v2.rs index c79ceb28594c..aca2c2f5d901 100644 --- a/lib/llm/src/http/service/service_v2.rs +++ b/lib/llm/src/http/service/service_v2.rs @@ -689,6 +689,11 @@ impl State { } /// Response field used for emitted OpenAI-compatible reasoning content. + pub fn allows_invalid_history_arguments(&self, model: &str) -> bool { + self.frontend_api_config + .allows_invalid_history_arguments(model) + } + pub fn reasoning_field(&self) -> ReasoningField { self.frontend_api_config.reasoning_field() } diff --git a/lib/llm/src/protocols/openai/chat_completions.rs b/lib/llm/src/protocols/openai/chat_completions.rs index 7af145cab914..182f132b3958 100644 --- a/lib/llm/src/protocols/openai/chat_completions.rs +++ b/lib/llm/src/protocols/openai/chat_completions.rs @@ -894,10 +894,23 @@ impl OpenAIOutputOptionsProvider for NvCreateChatCompletionRequest { /// allowing us to validate the data. impl ValidateRequest for NvCreateChatCompletionRequest { fn validate(&self) -> Result<(), anyhow::Error> { + self.validate_with_history_policy(false) + } +} + +impl NvCreateChatCompletionRequest { + /// Deployment-owned policy, never controllable through request JSON. + pub fn validate_with_history_policy( + &self, + allow_invalid_history_arguments: bool, + ) -> Result<(), anyhow::Error> { validate::validate_no_unsupported_fields(&self.unsupported_fields)?; validate::validate_guided_decoding(self)?; validate::validate_chat_template_args(self.chat_template_args.as_ref())?; - validate::validate_messages(&self.inner.messages)?; + validate::validate_messages_with_history_policy( + &self.inner.messages, + allow_invalid_history_arguments, + )?; validate::validate_model(&self.inner.model)?; // none for store validate::validate_reasoning_effort(&self.inner.reasoning_effort)?; @@ -952,6 +965,41 @@ impl ValidateRequest for NvCreateChatCompletionRequest { #[cfg(test)] mod tests { + + #[test] + fn history_arguments_policy_is_narrow_and_does_not_rewrite() { + use crate::engines::ValidateRequest; + for (arguments, strict_ok, tolerant_ok) in [ + (r#"{"a":1}"#, true, true), + ("{}", true, true), + ("", true, true), + (r#"{"a":1"#, false, true), + ("not json", false, true), + ("[]", false, false), + ("null", false, false), + ("1", false, false), + (r#""text""#, false, false), + ] { + let request: super::NvCreateChatCompletionRequest = + serde_json::from_value(serde_json::json!({ + "model": "glm-5.3", + "messages": [{"role": "assistant", "tool_calls": [{ + "id": "call_1", "type": "function", + "function": {"name": "lookup", "arguments": arguments} + }]}] + })) + .unwrap(); + let before = serde_json::to_value(&request).unwrap(); + assert_eq!(request.validate().is_ok(), strict_ok, "{arguments}"); + assert_eq!( + request.validate_with_history_policy(true).is_ok(), + tolerant_ok, + "{arguments}" + ); + assert_eq!(serde_json::to_value(&request).unwrap(), before); + } + } + #[test] fn legacy_cache_salt_survives_python_serialization() { use crate::protocols::common::extensions::apply_frontend_nvext_policy; diff --git a/lib/llm/src/protocols/openai/validate.rs b/lib/llm/src/protocols/openai/validate.rs index 25ef003519f6..8b143897075f 100644 --- a/lib/llm/src/protocols/openai/validate.rs +++ b/lib/llm/src/protocols/openai/validate.rs @@ -508,6 +508,15 @@ pub fn validate_stop(stop: &Option) -> Result<(), /// Validates messages array pub fn validate_messages( messages: &[dynamo_protocols::types::ChatCompletionRequestMessage], +) -> Result<(), anyhow::Error> { + validate_messages_with_history_policy(messages, false) +} + +/// Allow malformed JSON syntax only for explicitly configured historical tool calls. +/// Valid JSON with a non-object shape is still rejected. No input is rewritten. +pub fn validate_messages_with_history_policy( + messages: &[dynamo_protocols::types::ChatCompletionRequestMessage], + allow_invalid_history_arguments: bool, ) -> Result<(), anyhow::Error> { if messages.is_empty() { anyhow::bail!("Messages array cannot be empty"); @@ -522,6 +531,7 @@ pub fn validate_messages( for (tool_call_index, tool_call) in tool_calls.iter().enumerate() { validate_json_object_string( &tool_call.function.arguments, + allow_invalid_history_arguments, format!( "`messages[{message_index}].tool_calls[{tool_call_index}].function.arguments`" ), @@ -532,12 +542,19 @@ pub fn validate_messages( Ok(()) } -fn validate_json_object_string(value: &str, field: String) -> Result<(), anyhow::Error> { +fn validate_json_object_string( + value: &str, + allow_invalid_history_arguments: bool, + field: String, +) -> Result<(), anyhow::Error> { if value.trim().is_empty() { return Ok(()); } - let parsed: serde_json::Value = serde_json::from_str(value) - .map_err(|error| anyhow::anyhow!("{field} must be a valid JSON object string: {error}"))?; + let parsed: serde_json::Value = match serde_json::from_str(value) { + Ok(parsed) => parsed, + Err(_) if allow_invalid_history_arguments => return Ok(()), + Err(error) => anyhow::bail!("{field} must be a valid JSON object string: {error}"), + }; if !parsed.is_object() { anyhow::bail!("{field} must be a valid JSON object string"); } diff --git a/lib/llm/tests/common/http_harness.rs b/lib/llm/tests/common/http_harness.rs index c9253def453b..4abf2dbc5920 100644 --- a/lib/llm/tests/common/http_harness.rs +++ b/lib/llm/tests/common/http_harness.rs @@ -68,12 +68,20 @@ impl HarnessService { } pub async fn start_with_engine(engine: Arc) -> Self { + Self::start_with_frontend_config(engine, Default::default()).await + } + + pub async fn start_with_frontend_config( + engine: Arc, + config: dynamo_llm::frontend_config::FrontendApiConfig, + ) -> Self { let client = reqwest::Client::builder() .no_proxy() .build() .expect("failed to build harness HTTP client"); let (listener, port) = bind_random_port().await; let service = HttpService::builder() + .frontend_api_config(config) .port(port) .host("127.0.0.1") .enable_chat_endpoints(true) diff --git a/lib/llm/tests/frontend_protocol_validation_http.rs b/lib/llm/tests/frontend_protocol_validation_http.rs index 5beba93c542f..96dcb7ec0b7f 100644 --- a/lib/llm/tests/frontend_protocol_validation_http.rs +++ b/lib/llm/tests/frontend_protocol_validation_http.rs @@ -860,3 +860,51 @@ async fn tool_name_limit_is_shared_across_protocols() { }) .await; } + +/// Both HTTP adapters must apply the same deployment policy without changing history. +#[tokio::test] +#[serial] +async fn glm_history_policy_covers_chat_and_responses() { + temp_env::async_with_vars(BASE_ENV, async { + for allowed_models in [vec![], vec!["other-model".to_string()], vec![MODEL.to_string()]] { + let enabled = allowed_models.iter().any(|model| model == MODEL); + let mut config = dynamo_llm::frontend_config::FrontendApiConfig::default(); + config.set_glm_history_arguments_models(allowed_models); + let script = load_agent_fixture("text.sse").await.unwrap(); + let engine = Arc::new(ScriptedChatEngine::new([Ok(script.clone()), Ok(script)])); + let svc = HarnessService::start_with_frontend_config(engine, config).await; + for args in [r#"{"a":1"#, "[]"] { + for (path, body) in [ + ("/v1/chat/completions", json!({ + "model": MODEL, + "messages": [ + {"role": "assistant", "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": args}}]}, + {"role": "tool", "tool_call_id": "call_1", "content": "Invalid JSON; try again"} + ] + })), + ("/v1/responses", json!({ + "model": MODEL, + "input": [ + {"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": args}, + {"type": "function_call_output", "call_id": "call_1", "output": "Invalid JSON; try again"} + ] + })) + ] { + let response = post_json(&svc, path, body).await; + let expected = if enabled && args != "[]" { 200 } else { 400 }; + assert_eq!(response.status().as_u16(), expected, "{path} {args}"); + let _ = response.text().await.unwrap(); + let requests = svc.engine.take_requests().await; + if expected == 200 { + assert_eq!(requests.len(), 1); + let value = serde_json::to_value(&requests[0]).unwrap(); + assert_eq!(value["messages"][0]["tool_calls"][0]["function"]["arguments"], args); + } else { + assert!(requests.is_empty()); + } + } + } + svc.shutdown().await; + } + }).await; +} From e7d5a3e9be38441a7f83ebff0208dbf71a396a16 Mon Sep 17 00:00:00 2001 From: guojn1 Date: Thu, 15 Oct 2026 22:49:13 +0800 Subject: [PATCH 2/9] fix(ci): stage pinned AISimulate Rust source and repair validation tests --- .github/workflows/dingo-router-tests.yml | 3 +- container/deps/stage_aisimulate.py | 185 +++++++++++------- container/templates/wheel_builder.Dockerfile | 12 +- .../tests/test_sglang_reasoning_stops_unit.py | 2 +- .../src/protocols/openai/chat_completions.rs | 6 +- .../test_aisimulate_consistency.py | 29 +++ 6 files changed, 165 insertions(+), 72 deletions(-) diff --git a/.github/workflows/dingo-router-tests.yml b/.github/workflows/dingo-router-tests.yml index ec98249a675e..ec24b0f5cb49 100644 --- a/.github/workflows/dingo-router-tests.yml +++ b/.github/workflows/dingo-router-tests.yml @@ -57,6 +57,7 @@ jobs: # SKIP_RUNTIME_BUILD=1 跳过 binding 构建与所有 Python 测试，只跑 Rust 测试。 # 如需跑 Python 测试：先在 runner 预装 libclang（clang-devel/libclang-dev），再改为 '0'。 SKIP_RUNTIME_BUILD: '1' + TEST_RESULTS_DIR: ${{ runner.temp }}/dingo-test-results run: | cd "${GITHUB_WORKSPACE}" QUICK_MODE="${QUICK_MODE}" DOWNLOAD_MODEL="${DOWNLOAD_MODEL}" CARGO_BUILD_JOBS="${CARGO_BUILD_JOBS}" SKIP_RUNTIME_BUILD="${SKIP_RUNTIME_BUILD}" bash tests/scripts/run-all-tests-oneclick.sh @@ -67,6 +68,6 @@ jobs: with: name: test-logs-${{ inputs.quick_mode || 'smoke' }} path: | - /tmp/oneclick-*.log + ${{ runner.temp }}/dingo-test-results/ retention-days: 7 if-no-files-found: warn diff --git a/container/deps/stage_aisimulate.py b/container/deps/stage_aisimulate.py index 2e777624a2de..5fd5436e6c00 100644 --- a/container/deps/stage_aisimulate.py +++ b/container/deps/stage_aisimulate.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Stage the exact AISimulate wheel, building a pinned source if unpublished.""" +"""Stage exact AISimulate wheels or reviewed Rust source for image builds.""" import argparse import email @@ -24,6 +24,76 @@ def run(*args, cwd=None): subprocess.run(args, cwd=cwd, check=True) +def prepare_source(deps, version, work): + """Download only the reviewed commit, verify its digest, and stamp the pin.""" + rust_version = version.replace(".dev", "-dev.") + sources = json.loads((deps / "aisimulate-sources.json").read_text()) + if version not in sources: + raise SystemExit(f"No reviewed source fallback for AISimulate {version}") + source = sources[version] + commit = source["commit"] + if not re.fullmatch(r"[0-9a-f]{40}", commit): + raise SystemExit("AISimulate source must be an immutable commit") + print(f"Building AISimulate {version} from {commit}", flush=True) + archive = work / "source.tar.gz" + url = f"https://codeload.github.com/ai-dynamo/aisimulate/tar.gz/{commit}" + for attempt in range(3): + try: + with ( + urllib.request.urlopen(url, timeout=120) as response, + archive.open("wb") as dest, + ): + shutil.copyfileobj(response, dest) + break + except OSError: + if attempt == 2: + raise + time.sleep(5 * (attempt + 1)) + with archive.open("rb") as stream: + digest = hashlib.file_digest(stream, "sha256").hexdigest() + if digest != source["sha256"]: + raise SystemExit("AISimulate source archive checksum mismatch") + with tarfile.open(archive) as tar: + # Agent/editor links are unrelated to the build and some point + # outside the checkout. Preserve package/license links only. + members = [ + m + for m in tar.getmembers() + if not set(Path(m.name).parts) & {".agents", ".claude"} + ] + tar.extractall(work, members=members, filter="data") + checkout = work / f"aisimulate-{commit}" + run( + sys.executable, + str(checkout / "scripts/apply_dev_version.py"), + ".dev" + version.split(".dev", 1)[1], + str(checkout), + ) + project = checkout / "python/aisimulate" + py_version = tomllib.loads((project / "pyproject.toml").read_text())[ + "project" + ]["version"] + core_version = tomllib.loads( + (checkout / "crates/core/Cargo.toml").read_text() + )["package"]["version"] + if (py_version, core_version) != (version, rust_version): + raise SystemExit( + "Stamped source does not match the required Python/Rust versions" + ) + # Stamping changes only the local workspace package in Cargo.lock. + # Keep all transitive dependency versions locked during compilation. + lock = checkout / "Cargo.lock" + text, count = re.subn( + r'(name = "aisimulate-core"\nversion = ")[^"]+("\n)', + lambda match: match[1] + rust_version + match[2], + lock.read_text(), + ) + if count != 1: + raise SystemExit("Expected one AISimulate core package in Cargo.lock") + lock.write_text(text) + return checkout + + def stage(root, output): deps = root / "container/deps" requirements = deps / "requirements.aisimulate.txt" @@ -79,72 +149,8 @@ def stage(root, output): == 0 ) if not downloaded: - sources = json.loads((deps / "aisimulate-sources.json").read_text()) - if version not in sources: - raise SystemExit( - f"No reviewed source fallback for AISimulate {version}" - ) - source = sources[version] - commit = source["commit"] - if not re.fullmatch(r"[0-9a-f]{40}", commit): - raise SystemExit("AISimulate source must be an immutable commit") - print(f"Building AISimulate {version} from {commit}", flush=True) - archive = work / "source.tar.gz" - url = f"https://codeload.github.com/ai-dynamo/aisimulate/tar.gz/{commit}" - for attempt in range(3): - try: - with ( - urllib.request.urlopen(url, timeout=120) as response, - archive.open("wb") as dest, - ): - shutil.copyfileobj(response, dest) - break - except OSError: - if attempt == 2: - raise - time.sleep(5 * (attempt + 1)) - with archive.open("rb") as stream: - digest = hashlib.file_digest(stream, "sha256").hexdigest() - if digest != source["sha256"]: - raise SystemExit("AISimulate source archive checksum mismatch") - with tarfile.open(archive) as tar: - # Agent/editor links are unrelated to the build and some point - # outside the checkout. Preserve package/license links only. - members = [ - m - for m in tar.getmembers() - if not set(Path(m.name).parts) & {".agents", ".claude"} - ] - tar.extractall(work, members=members, filter="data") - checkout = work / f"aisimulate-{commit}" - run( - sys.executable, - str(checkout / "scripts/apply_dev_version.py"), - ".dev" + version.split(".dev", 1)[1], - str(checkout), - ) + checkout = prepare_source(deps, version, work) project = checkout / "python/aisimulate" - py_version = tomllib.loads((project / "pyproject.toml").read_text())[ - "project" - ]["version"] - core_version = tomllib.loads( - (checkout / "crates/core/Cargo.toml").read_text() - )["package"]["version"] - if (py_version, core_version) != (version, rust_version): - raise SystemExit( - "Stamped source does not match the required Python/Rust versions" - ) - # Stamping changes only the local workspace package in Cargo.lock. - # Keep all transitive dependency versions locked during compilation. - lock = checkout / "Cargo.lock" - text, count = re.subn( - r'(name = "aisimulate-core"\nversion = ")[^"]+("\n)', - lambda match: match[1] + rust_version + match[2], - lock.read_text(), - ) - if count != 1: - raise SystemExit("Expected one AISimulate core package in Cargo.lock") - lock.write_text(text) # Use the existing manylinux builder toolchain; wheels are consumed # by this image family, with the same ABI floor as Dynamo wheels. run( @@ -184,9 +190,54 @@ def stage(root, output): print(f"Staged {wheel.name}", flush=True) +def stage_rust_source(root, output): + """Use reviewed source when the exact nightly is not in the public index. + + This changes only the disposable build checkout: Cargo's manifest pins and + all transitive lockfile entries stay fixed. --locked metadata/build verifies + that the source has the same dependency requirements as the published crate. + """ + data = tomllib.loads((root / "Cargo.toml").read_text()) + pin = data["workspace"]["dependencies"]["aisimulate-core"]["version"] + if not re.fullmatch(r"=\d+\.\d+\.\d+-dev\.\d+", pin): + raise SystemExit("Rust source staging requires an exact reviewed nightly pin") + version = pin[1:].replace("-dev.", ".dev") + output.mkdir(parents=True, exist_ok=True) + checkout = prepare_source(root / "container/deps", version, output) + config = root / ".cargo/config.toml" + text = config.read_text() + if "[patch.crates-io]" in text: + raise SystemExit("Refusing to replace existing Cargo source patches") + lock = root / "Cargo.lock" + lock_text, count = re.subn( + r'(name = "aisimulate-core"\nversion = "' + re.escape(pin[1:]) + + r'"\n)source = "registry\+https://github.com/rust-lang/crates.io-index"\n' + + r'checksum = "[0-9a-f]{64}"\n', + r"\1", + lock.read_text(), + ) + if count != 1: + raise SystemExit("Expected one locked registry AISimulate core package") + config.write_text( + text + "\n[patch.crates-io]\naisimulate-core = { path = " + + json.dumps(str(checkout / "crates/core")) + " }\n" + ) + lock.write_text(lock_text) + licenses = root / "rust-licenses" / ("aisimulate-core-" + pin[1:]) + licenses.mkdir(parents=True, exist_ok=True) + for name in ("LICENSE", "NOTICE"): + if (checkout / name).is_file(): + shutil.copy2(checkout / name, licenses / name) + print(f"Staged Rust source: aisimulate-core {pin[1:]}", flush=True) + + if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--root", type=Path, default=Path("/opt/dynamo")) parser.add_argument("--output", type=Path, default=Path("/opt/dynamo/dist")) + parser.add_argument("--rust-source-only", action="store_true") args = parser.parse_args() - stage(args.root.resolve(), args.output.resolve()) + if args.rust_source_only: + stage_rust_source(args.root.resolve(), args.output.resolve()) + else: + stage(args.root.resolve(), args.output.resolve()) diff --git a/container/templates/wheel_builder.Dockerfile b/container/templates/wheel_builder.Dockerfile index 9af904217fc6..f04c168ddb5f 100644 --- a/container/templates/wheel_builder.Dockerfile +++ b/container/templates/wheel_builder.Dockerfile @@ -651,6 +651,14 @@ COPY pyproject.toml README.md LICENSE Cargo.toml Cargo.lock rust-toolchain.toml COPY lib/ /opt/dynamo/lib/ COPY dingo/ /opt/dynamo/dingo/ +# The pinned AISimulate nightly can precede public crates.io publication. +# Resolve it from the reviewed, checksum-pinned source before Cargo metadata; +# a cached registry on the CI host must not be a prerequisite for image builds. +COPY container/deps/stage_aisimulate.py container/deps/aisimulate-sources.json /opt/dynamo/container/deps/ +RUN source ${VIRTUAL_ENV}/bin/activate && \ + python /opt/dynamo/container/deps/stage_aisimulate.py \ + --rust-source-only --output /opt/build-deps/aisimulate + # Build ai-dingo (pure Python) and ai-dingo-runtime (maturin) wheels. # Runtime images supply shared libraries; audit without bundling duplicate copies. # SGLang CUDA runtime images own their NIXL stack. Use its C API dynamically: @@ -677,9 +685,9 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token uv build --wheel --out-dir /opt/dynamo/dist && \ cd /opt/dynamo/lib/bindings/python && \ if [ "$ENABLE_MEDIA_FFMPEG" = "true" ]; then \ - maturin build --release --features "full-runtime,media-ffmpeg,kv-indexer,slot-tracker,select-service,mm-routing,ais-forward-pass,request-trace-s3,custom-policy{% if framework == "sglang" and device == "cuda" and target in ("runtime", "dev", "local-dev") %},nixl-dynamic{% endif %}" --auditwheel skip --compatibility linux --out /opt/dynamo/runtime-wheel-unrepaired; \ + maturin build --release --locked --features "full-runtime,media-ffmpeg,kv-indexer,slot-tracker,select-service,mm-routing,ais-forward-pass,request-trace-s3,custom-policy{% if framework == "sglang" and device == "cuda" and target in ("runtime", "dev", "local-dev") %},nixl-dynamic{% endif %}" --auditwheel skip --compatibility linux --out /opt/dynamo/runtime-wheel-unrepaired; \ else \ - maturin build --release --features "full-runtime,kv-indexer,slot-tracker,select-service,mm-routing,ais-forward-pass,request-trace-s3,custom-policy{% if framework == "sglang" and device == "cuda" and target in ("runtime", "dev", "local-dev") %},nixl-dynamic{% endif %}" --auditwheel skip --compatibility linux --out /opt/dynamo/runtime-wheel-unrepaired; \ + maturin build --release --locked --features "full-runtime,kv-indexer,slot-tracker,select-service,mm-routing,ais-forward-pass,request-trace-s3,custom-policy{% if framework == "sglang" and device == "cuda" and target in ("runtime", "dev", "local-dev") %},nixl-dynamic{% endif %}" --auditwheel skip --compatibility linux --out /opt/dynamo/runtime-wheel-unrepaired; \ fi && \ {% if device == "cuda" -%} auditwheel repair --exclude 'lib*.so*' \ diff --git a/dingo/frontend/tests/test_sglang_reasoning_stops_unit.py b/dingo/frontend/tests/test_sglang_reasoning_stops_unit.py index 7b2a4e40bfe4..8781d958e17f 100644 --- a/dingo/frontend/tests/test_sglang_reasoning_stops_unit.py +++ b/dingo/frontend/tests/test_sglang_reasoning_stops_unit.py @@ -362,4 +362,4 @@ async def collect(): assert terminal["usage"]["completion_tokens"] == generated assert terminal["usage"]["total_tokens"] == generated + 2 assert terminal["nvext"]["stop_reason"] == "\n" - assert json.loads(items[-1]["comment"][0])["output_tokens"] == generated + assert terminal["llm_metrics"]["output_tokens"] == generated diff --git a/lib/llm/src/protocols/openai/chat_completions.rs b/lib/llm/src/protocols/openai/chat_completions.rs index 182f132b3958..8412c06f5387 100644 --- a/lib/llm/src/protocols/openai/chat_completions.rs +++ b/lib/llm/src/protocols/openai/chat_completions.rs @@ -990,7 +990,11 @@ mod tests { })) .unwrap(); let before = serde_json::to_value(&request).unwrap(); - assert_eq!(request.validate().is_ok(), strict_ok, "{arguments}"); + assert_eq!( + ValidateRequest::validate(&request).is_ok(), + strict_ok, + "{arguments}" + ); assert_eq!( request.validate_with_history_policy(true).is_ok(), tolerant_ok, diff --git a/tests/dependencies/test_aisimulate_consistency.py b/tests/dependencies/test_aisimulate_consistency.py index a066a2d9858c..d96c6decbbd1 100644 --- a/tests/dependencies/test_aisimulate_consistency.py +++ b/tests/dependencies/test_aisimulate_consistency.py @@ -150,6 +150,35 @@ def _staging_module(): return module +def test_rust_source_staging_preserves_transitive_lock(tmp_path, monkeypatch): + staging = _staging_module() + root = tmp_path / "build" + (root / ".cargo").mkdir(parents=True) + (root / "Cargo.toml").write_text((ROOT / "Cargo.toml").read_text()) + (root / "Cargo.lock").write_text((ROOT / "Cargo.lock").read_text()) + (root / ".cargo/config.toml").write_text("[build]\njobs = 2\n") + checkout = tmp_path / "source" + checkout.mkdir() + (checkout / "LICENSE").write_text("test license") + monkeypatch.setattr(staging, "prepare_source", lambda *args: checkout) + before = tomllib.loads((root / "Cargo.lock").read_text()) + staging.stage_rust_source(root, tmp_path / "deps") + after = tomllib.loads((root / "Cargo.lock").read_text()) + for package in before["package"]: + if package["name"] == "aisimulate-core": + package.pop("source") + package.pop("checksum") + assert after == before + config = tomllib.loads((root / ".cargo/config.toml").read_text()) + assert config["build"]["jobs"] == 2 + assert config["patch"]["crates-io"]["aisimulate-core"]["path"] == str( + checkout / "crates/core" + ) + assert len(list((root / "rust-licenses").glob("*/LICENSE"))) == 1 + with pytest.raises(SystemExit, match="existing Cargo source patches"): + staging.stage_rust_source(root, tmp_path / "deps") + + @pytest.mark.parametrize("matching_version", [True, False]) def test_container_stages_the_published_aisimulate_wheel( tmp_path, monkeypatch, matching_version From f48594411d2a3d481225fad02f388c44a908f59b Mon Sep 17 00:00:00 2001 From: guojn1 Date: Thu, 15 Oct 2026 23:23:58 +0800 Subject: [PATCH 3/9] fix(build): stage AISimulate in standalone binding lockfiles --- container/deps/stage_aisimulate.py | 32 ++++++++++++------- .../test_aisimulate_consistency.py | 20 +++++++----- 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/container/deps/stage_aisimulate.py b/container/deps/stage_aisimulate.py index 5fd5436e6c00..d1407c8dc7bd 100644 --- a/container/deps/stage_aisimulate.py +++ b/container/deps/stage_aisimulate.py @@ -208,21 +208,31 @@ def stage_rust_source(root, output): text = config.read_text() if "[patch.crates-io]" in text: raise SystemExit("Refusing to replace existing Cargo source patches") - lock = root / "Cargo.lock" - lock_text, count = re.subn( - r'(name = "aisimulate-core"\nversion = "' + re.escape(pin[1:]) - + r'"\n)source = "registry\+https://github.com/rust-lang/crates.io-index"\n' - + r'checksum = "[0-9a-f]{64}"\n', - r"\1", - lock.read_text(), - ) - if count != 1: - raise SystemExit("Expected one locked registry AISimulate core package") + # The runtime and KVBM bindings are independent Cargo workspaces. + # Validate every lockfile before writing any of the build-only overrides. + updated_locks = [] + for relative in ( + "Cargo.lock", + "lib/bindings/python/Cargo.lock", + "lib/bindings/kvbm/Cargo.lock", + ): + lock = root / relative + lock_text, count = re.subn( + r'(name = "aisimulate-core"\nversion = "' + re.escape(pin[1:]) + + r'"\n)source = "registry\+https://github.com/rust-lang/crates.io-index"\n' + + r'checksum = "[0-9a-f]{64}"\n', + r"\1", + lock.read_text(), + ) + if count != 1: + raise SystemExit(f"Expected one locked registry AISimulate core package: {relative}") + updated_locks.append((lock, lock_text)) config.write_text( text + "\n[patch.crates-io]\naisimulate-core = { path = " + json.dumps(str(checkout / "crates/core")) + " }\n" ) - lock.write_text(lock_text) + for lock, lock_text in updated_locks: + lock.write_text(lock_text) licenses = root / "rust-licenses" / ("aisimulate-core-" + pin[1:]) licenses.mkdir(parents=True, exist_ok=True) for name in ("LICENSE", "NOTICE"): diff --git a/tests/dependencies/test_aisimulate_consistency.py b/tests/dependencies/test_aisimulate_consistency.py index d96c6decbbd1..d95648745293 100644 --- a/tests/dependencies/test_aisimulate_consistency.py +++ b/tests/dependencies/test_aisimulate_consistency.py @@ -155,20 +155,24 @@ def test_rust_source_staging_preserves_transitive_lock(tmp_path, monkeypatch): root = tmp_path / "build" (root / ".cargo").mkdir(parents=True) (root / "Cargo.toml").write_text((ROOT / "Cargo.toml").read_text()) - (root / "Cargo.lock").write_text((ROOT / "Cargo.lock").read_text()) + for original in LOCKFILES: + destination = root / original.relative_to(ROOT) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(original.read_text()) (root / ".cargo/config.toml").write_text("[build]\njobs = 2\n") checkout = tmp_path / "source" checkout.mkdir() (checkout / "LICENSE").write_text("test license") monkeypatch.setattr(staging, "prepare_source", lambda *args: checkout) - before = tomllib.loads((root / "Cargo.lock").read_text()) + before = [tomllib.loads(path.read_text()) for path in LOCKFILES] staging.stage_rust_source(root, tmp_path / "deps") - after = tomllib.loads((root / "Cargo.lock").read_text()) - for package in before["package"]: - if package["name"] == "aisimulate-core": - package.pop("source") - package.pop("checksum") - assert after == before + for original, expected in zip(LOCKFILES, before): + after = tomllib.loads((root / original.relative_to(ROOT)).read_text()) + for package in expected["package"]: + if package["name"] == "aisimulate-core": + package.pop("source") + package.pop("checksum") + assert after == expected config = tomllib.loads((root / ".cargo/config.toml").read_text()) assert config["build"]["jobs"] == 2 assert config["patch"]["crates-io"]["aisimulate-core"]["path"] == str( From 242319ab2eb1c5ae019ef7158fd9322ebb60de87 Mon Sep 17 00:00:00 2001 From: guojn1 Date: Fri, 16 Oct 2026 12:27:17 +0800 Subject: [PATCH 4/9] fix(sglang): default GLM engine reasoner from Dynamo parser --- dingo/frontend/README.md | 17 ++++++++++++++ dingo/sglang/args.py | 14 ++++++++++++ dingo/sglang/tests/test_sglang_unit.py | 31 ++++++++++++++++++++++++++ 3 files changed, 62 insertions(+) diff --git a/dingo/frontend/README.md b/dingo/frontend/README.md index d91d7901e694..1478638bd7b8 100644 --- a/dingo/frontend/README.md +++ b/dingo/frontend/README.md @@ -40,6 +40,23 @@ Detection uses the discovered model's `config.json`, so serving aliases do not affect it. Explicit parser settings take precedence. Unknown model types with thinking templates produce a warning requesting explicit parser configuration. +For GLM-5.3 workers using SGLang tool constraints, configure +`--dyn-reasoning-parser glm45` on `python -m dingo.sglang`. When the native +reasoning parser is unset, this also defaults SGLang's `--reasoning-parser` to +`glm45`. An explicitly configured native parser, including `auto`, is preserved. +Other Dynamo reasoning parsers are not automatically mapped. +The Dynamo flag selects response reasoning parsing; +the SGLang flag enables the engine's reasoning gate before applying the tool +grammar. Frontend reasoning detection does not configure that engine gate. +Without the engine flag, a tool grammar that excludes `` can prevent +the model from ending its thinking block and cause repeated reasoning after a +tool result. The router's reasoning parser alone does not resolve this. + +When validating requests without an explicit output limit, leave room for the +8192-token default completion in the configured engine context window. An +8192-token context plus a nonempty prompt can otherwise fail SGLang's length +validation before generation starts. + ## GLM-5.3 malformed historical tool arguments For a GLM-5.3 deployment using `--dyn-chat-processor sglang`, opt in with diff --git a/dingo/sglang/args.py b/dingo/sglang/args.py index 744c99e347ad..d626e467e416 100644 --- a/dingo/sglang/args.py +++ b/dingo/sglang/args.py @@ -636,6 +636,20 @@ async def parse_args(args: list[str]) -> Config: "tool-call-parser", ) + # GLM tool grammars expect the engine to consume the thinking prefix before + # applying the tool constraint. Dynamo's response parser alone cannot enable + # that gate. Set this before ServerArgs resolution freezes the configuration; + # an explicit native parser (including "auto") always takes precedence. + if ( + dynamo_config.dyn_reasoning_parser == "glm45" + and parsed_args.reasoning_parser is None + ): + parsed_args.reasoning_parser = "glm45" + logging.info( + "Defaulting SGLang --reasoning-parser to glm45 from " + "--dyn-reasoning-parser glm45" + ) + if dynamo_config.custom_jinja_template and dynamo_config.use_sglang_tokenizer: logging.error( "Cannot use --custom-jinja-template and --use-sglang-tokenizer together. " diff --git a/dingo/sglang/tests/test_sglang_unit.py b/dingo/sglang/tests/test_sglang_unit.py index 91ff0cbe0ca5..6df1e50e7716 100644 --- a/dingo/sglang/tests/test_sglang_unit.py +++ b/dingo/sglang/tests/test_sglang_unit.py @@ -1158,6 +1158,37 @@ async def test_reasoning_parser_both_flags_are_allowed(mock_sglang_cli): assert config.server_args.reasoning_parser == "qwen3" +@pytest.mark.asyncio +@pytest.mark.parametrize( + "parser_flags, expected", + [ + (["--dyn-reasoning-parser", "glm45"], "glm45"), + ( + ["--dyn-reasoning-parser", "glm45", "--reasoning-parser", "glm45"], + "glm45", + ), + ( + ["--dyn-reasoning-parser", "glm45", "--reasoning-parser", "qwen3"], + "qwen3", + ), + ( + ["--dyn-reasoning-parser", "glm45", "--reasoning-parser", "auto"], + "auto", + ), + (["--dyn-reasoning-parser", "qwen3"], None), + ([], None), + ], +) +async def test_glm_reasoning_parser_defaults_native_gate( + mock_sglang_cli, parser_flags, expected +): + mock_sglang_cli("--model", "Qwen/Qwen3-0.6B", *parser_flags) + + config = await parse_args(sys.argv[1:]) + + assert config.server_args.reasoning_parser == expected + + @pytest.mark.asyncio async def test_namespace_flag_drives_default_endpoint_namespace(mock_sglang_cli): """CLI namespace should be used for auto-derived endpoint.""" From 914daa64c20050166b7b8d96552360dd82470bbc Mon Sep 17 00:00:00 2001 From: guojn1 Date: Fri, 16 Oct 2026 15:01:03 +0800 Subject: [PATCH 5/9] fix(dingo): preserve GLM reasoning boundaries and reject invalid tool turns Use control token IDs for GLM reasoning, report undeclared tool-only output as a generation error, validate tool result history, and run SGLang images as root. Add synthetic streaming regressions; 37 targeted Python tests passed. Rust tests deferred to CI. --- container/templates/aws.Dockerfile | 4 + container/templates/sglang_runtime.Dockerfile | 4 +- dingo/frontend/glm_reasoning.py | 76 ++++++++++++ dingo/frontend/sglang_prepost.py | 47 +++++++- .../frontend/tests/test_glm_reasoning_unit.py | 77 ++++++++++++ .../tests/test_glm_stream_boundaries.py | 113 ++++++++++++++++++ .../frontend/tests/test_sglang_tool_calls.py | 18 +-- .../src/protocols/openai/chat_completions.rs | 30 +++++ lib/llm/src/protocols/openai/validate.rs | 15 +++ 9 files changed, 368 insertions(+), 16 deletions(-) create mode 100644 dingo/frontend/glm_reasoning.py create mode 100644 dingo/frontend/tests/test_glm_reasoning_unit.py create mode 100644 dingo/frontend/tests/test_glm_stream_boundaries.py diff --git a/container/templates/aws.Dockerfile b/container/templates/aws.Dockerfile index 94919359ab9e..86d3dc479046 100644 --- a/container/templates/aws.Dockerfile +++ b/container/templates/aws.Dockerfile @@ -64,5 +64,9 @@ RUN --mount=from=wheel_builder,source=/usr/local/libfabric,target=/tmp/libfabric fi {% if target == "runtime" %} +{% if framework == "sglang" %} +USER root +{% else %} USER dynamo {% endif %} +{% endif %} diff --git a/container/templates/sglang_runtime.Dockerfile b/container/templates/sglang_runtime.Dockerfile index 02b8a1f2a705..d436595ebc55 100644 --- a/container/templates/sglang_runtime.Dockerfile +++ b/container/templates/sglang_runtime.Dockerfile @@ -347,7 +347,9 @@ RUN chmod 755 /opt/dynamo/.launch_screen && \ else echo "WARNING: no bundled nsys found under /opt/nvidia/nsight-compute"; fi {% endif %} -USER dynamo +# SGLang deployments support startup patches to the upstream engine package. +# Keep the runtime user root so those files remain writable at startup. +USER root ARG DYNAMO_COMMIT_SHA ENV DYNAMO_COMMIT_SHA=${DYNAMO_COMMIT_SHA} diff --git a/dingo/frontend/glm_reasoning.py b/dingo/frontend/glm_reasoning.py new file mode 100644 index 000000000000..42a836348d61 --- /dev/null +++ b/dingo/frontend/glm_reasoning.py @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Separate GLM reasoning using control token IDs, not literal tag text.""" + +from collections.abc import Callable +from typing import Any + + +class GlmTokenReasoningSplitter: + """Keep ordinary-token spellings of think tags in their original field. + + Once a real closing token or tool opener ends reasoning, tool arguments + pass through unchanged, including any special tokens inside those values. + """ + + def __init__(self, markers: dict[int, str], *, force_reasoning: bool): + self.markers = markers + self.in_reasoning = force_reasoning + self.ended = False + + @classmethod + def from_tokenizer(cls, tokenizer: Any, *, force_reasoning: bool): + convert = getattr(tokenizer, "convert_tokens_to_ids", None) + if not callable(convert): + return None + markers = {} + for marker in ("", "", ""): + token_id = convert(marker) + if ( + type(token_id) is not int + or token_id == getattr(tokenizer, "unk_token_id", None) + or tokenizer.decode([token_id], skip_special_tokens=False) != marker + ): + return None + markers[token_id] = marker + if len(markers) != 3: + return None + return cls(markers, force_reasoning=force_reasoning) + + def split( + self, + token_ids: list[int], + decode: Callable[..., str], + *, + finished: bool = False, + ) -> tuple[str | None, str, int]: + reasoning, content = [], [] + reasoning_tokens = 0 + start = 0 + + def emit(ids: list[int], flush: bool): + nonlocal reasoning_tokens + text = decode(ids, flush=flush) + (reasoning if self.in_reasoning else content).append(text) + if self.in_reasoning: + reasoning_tokens += len(ids) + + for index, token_id in enumerate(token_ids): + marker = self.markers.get(token_id) + if marker is None or self.ended: + continue + emit(token_ids[start:index], True) + marker_text = decode([token_id], flush=True) + if marker == "": + self.in_reasoning = True + elif marker == "": + self.in_reasoning = False + self.ended = True + else: + self.in_reasoning = False + self.ended = True + content.append(marker_text) + start = index + 1 + emit(token_ids[start:], finished) + return "".join(reasoning) or None, "".join(content), reasoning_tokens diff --git a/dingo/frontend/sglang_prepost.py b/dingo/frontend/sglang_prepost.py index 7a95408bce5f..21633fc014e8 100644 --- a/dingo/frontend/sglang_prepost.py +++ b/dingo/frontend/sglang_prepost.py @@ -1759,6 +1759,18 @@ def __init__( else None ) self._pending_guided_reasoning_token_count = 0 + self._glm_token_reasoning = None + if ( + self._reasoning_parser_name == "glm45" + and reasoning_parser is not None + and self._pending_guided_reasoning_prefix is None + ): + from .glm_reasoning import GlmTokenReasoningSplitter + + self._glm_token_reasoning = GlmTokenReasoningSplitter.from_tokenizer( + tokenizer, + force_reasoning=bool(reasoning_parser.detector._in_reasoning), + ) self._bypass_reasoning_for_bare_json = False self._eos_token_ids = set(eos_token_ids or []) self._stop_strings = {stop for stop in (stop_strings or set()) if stop} @@ -2363,9 +2375,16 @@ def process_output(self, engine_response: dict[str, Any]) -> dict[str, Any] | No if top_logprobs is not None and len(top_logprobs) == raw_token_count: top_logprobs = top_logprobs[:retained_token_count] + glm_reasoning = None + if self._glm_token_reasoning is not None: + glm_reasoning = self._glm_token_reasoning.split( + token_ids, + self._incremental_decode, + finished=finish_reason is not None, + ) delta_text = ( self._incremental_decode(token_ids, flush=finish_reason is not None) - if token_ids or finish_reason is not None + if glm_reasoning is None and (token_ids or finish_reason is not None) else "" ) openai_logprobs = None @@ -2406,7 +2425,10 @@ def process_output(self, engine_response: dict[str, Any]) -> dict[str, Any] | No # -- Reasoning parsing -- reasoning_text = None normal_text = delta_text - if self.reasoning_parser and (delta_text or finish_reason): + if glm_reasoning is not None: + reasoning_text, normal_text, reasoning_tokens = glm_reasoning + self._reasoning_token_count += reasoning_tokens + elif self.reasoning_parser and (delta_text or finish_reason): reasoning_text, normal_text, reasoning_tokens = self._parse_reasoning_delta( delta_text, finish_reason, @@ -2507,6 +2529,7 @@ def process_output(self, engine_response: dict[str, Any]) -> dict[str, Any] | No if known_names: for idx in list(self._tool_call_names): if self._tool_call_names[idx] not in known_names: + self._suppressed_tool_indices.add(idx) del self._tool_call_names[idx] self._tool_call_ids.pop(idx, None) self._tool_call_args.pop(idx, None) @@ -2597,6 +2620,11 @@ def process_output(self, engine_response: dict[str, Any]) -> dict[str, Any] | No _, final_calls = self.tool_call_parser.parse_non_stream(full_text) # Filter to known tool names (reuse set from above). if known_names: + self._suppressed_tool_indices.update( + tc.tool_index + for tc in final_calls + if tc.name not in known_names + ) final_calls = [tc for tc in final_calls if tc.name in known_names] # Re-index sequentially so repeated calls to the same # tool get distinct indices (parse_non_stream may assign @@ -2739,6 +2767,21 @@ def process_output(self, engine_response: dict[str, Any]) -> dict[str, Any] | No delta.setdefault("tool_calls", []).extend(tool_calls_out) has_content = True + # An undeclared model-generated call is not a successful empty answer. + # Keep withholding unknown tools, but surface the failure so callers + # can retry instead of accepting an empty assistant turn. The processor + # maps this exception to the standard generation error (also for SSE). + if ( + finish_reason + and self._suppressed_tool_indices + and not self._tool_call_names + and not self._saw_normal_output + ): + raise ValueError( + "Model generated a tool call not declared in request.tools; " + "no valid tool call or answer could be returned" + ) + # Rewrite finish_reason "stop" → "tool_calls" when tool calls were # detected, matching the OpenAI API spec and official SGLang behaviour. effective_finish = finish_reason diff --git a/dingo/frontend/tests/test_glm_reasoning_unit.py b/dingo/frontend/tests/test_glm_reasoning_unit.py new file mode 100644 index 000000000000..13f07a65ce80 --- /dev/null +++ b/dingo/frontend/tests/test_glm_reasoning_unit.py @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CPU regressions for GLM control-token versus literal-text boundaries.""" + +import pytest + +from dingo.frontend.glm_reasoning import GlmTokenReasoningSplitter + + +class Tokenizer: + vocab = { + 1: "", + 2: "", + 3: "", + 4: "Use `", + 5: "", + 8: "` literally.", + 9: "answer", + 10: 'submitcode"', + 11: '"', + } + unk_token_id = -1 + + def convert_tokens_to_ids(self, token): + return next((k for k, v in self.vocab.items() if v == token), -1) + + def decode(self, ids, **kwargs): + return "".join(self.vocab[i] for i in ids) + + +@pytest.mark.parametrize("batch_size", [1, 2, 4, 100]) +def test_literal_close_does_not_end_reasoning(batch_size): + tokenizer = Tokenizer() + splitter = GlmTokenReasoningSplitter.from_tokenizer(tokenizer, force_reasoning=True) + ids = [4, 5, 6, 7, 8, 2, 9] + chunks = [ + splitter.split( + ids[i : i + batch_size], + tokenizer.decode, + finished=i + batch_size >= len(ids), + ) + for i in range(0, len(ids), batch_size) + ] + assert "".join(r or "" for r, _, _ in chunks) == "Use `` literally." + assert "".join(c for _, c, _ in chunks) == "answer" + assert sum(n for _, _, n in chunks) == 5 + + +@pytest.mark.parametrize("batch_size", [1, 3, 100]) +def test_tool_opener_ends_reasoning_and_preserves_argument_markers(batch_size): + tokenizer = Tokenizer() + splitter = GlmTokenReasoningSplitter.from_tokenizer(tokenizer, force_reasoning=True) + ids = [4, 8, 3, 10, 1, 2, 11] + chunks = [ + splitter.split( + ids[i : i + batch_size], + tokenizer.decode, + finished=i + batch_size >= len(ids), + ) + for i in range(0, len(ids), batch_size) + ] + assert "".join(r or "" for r, _, _ in chunks) == "Use `` literally." + assert "".join(c for _, c, _ in chunks) == tokenizer.decode(ids[2:]) + + +def test_no_reasoning_and_unrecognized_tokenizer(): + tokenizer = Tokenizer() + splitter = GlmTokenReasoningSplitter.from_tokenizer( + tokenizer, force_reasoning=False + ) + assert splitter.split([9], tokenizer.decode, finished=True) == (None, "answer", 0) + assert ( + GlmTokenReasoningSplitter.from_tokenizer(object(), force_reasoning=True) is None + ) diff --git a/dingo/frontend/tests/test_glm_stream_boundaries.py b/dingo/frontend/tests/test_glm_stream_boundaries.py new file mode 100644 index 000000000000..2726f8ed91da --- /dev/null +++ b/dingo/frontend/tests/test_glm_stream_boundaries.py @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Exercise the real SGLang parsers with synthetic GLM token streams.""" + +import json + +import pytest + +from dingo.frontend.sglang_prepost import ( + SglangStreamingPostProcessor, + convert_tools, + create_parsers, +) + + +class ByteTokenizer: + markers = {"": 256, "": 257, "": 258} + unk_token_id = -1 + + def convert_tokens_to_ids(self, text): + return self.markers.get(text, -1) + + def decode(self, ids, **kwargs): + special = {v: k.encode() for k, v in self.markers.items()} + return b"".join(special[i] if i in special else bytes([i]) for i in ids).decode( + "utf-8", errors="replace" + ) + + +def make_post(): + tools = [ + { + "type": "function", + "function": { + "name": "lookup", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + }, + } + ] + tool, reason = create_parsers( + {"tools": tools}, + tool_call_parser_name="glm47", + reasoning_parser_name="glm45", + force_reasoning=True, + ) + return SglangStreamingPostProcessor( + tokenizer=ByteTokenizer(), + tool_call_parser=tool, + reasoning_parser=reason, + sglang_tools=convert_tools(tools), + tool_call_parser_name="glm47", + reasoning_parser_name="glm45", + ) + + +def run(post, ids, batch): + results = [] + for start in range(0, len(ids), batch): + item = post.process_output( + { + "token_ids": ids[start : start + batch], + "finish_reason": "stop" if start + batch >= len(ids) else None, + } + ) + if item: + results.append(item) + return results + + +@pytest.mark.parametrize("batch", [1, 2, 7, 10000]) +@pytest.mark.parametrize("reasoning", ["", "字面量 不结束思考。"]) +def test_literal_tags_and_utf8_before_tool_call(batch, reasoning): + argument = "中文 " + ids = ( + list(reasoning.encode()) + + [257, 258] + + list( + ( + "lookupquery" + + argument + + "" + ).encode() + ) + ) + results = run(make_post(), ids, batch) + assert ( + "".join(r["delta"].get("reasoning_content", "") for r in results) == reasoning + ) + assert not any(r["delta"].get("content") for r in results) + calls = {} + for r in results: + for call in r["delta"].get("tool_calls", []): + out = calls.setdefault(call["index"], {"name": "", "arguments": ""}) + for key in out: + out[key] += call.get("function", {}).get(key, "") + assert len(calls) == 1 + assert calls[0]["name"] == "lookup" + assert json.loads(calls[0]["arguments"]) == {"query": argument} + assert results[-1]["finish_reason"] == "tool_calls" + + +@pytest.mark.parametrize("batch", [1, 2, 7, 10000]) +def test_undeclared_call_is_an_error_not_empty_success(batch): + ids = [257, 258] + list( + b"undeclaredqueryexample" + ) + with pytest.raises(ValueError, match="not declared in request.tools"): + run(make_post(), ids, batch) diff --git a/dingo/frontend/tests/test_sglang_tool_calls.py b/dingo/frontend/tests/test_sglang_tool_calls.py index 806fff08d464..7197370ff9ab 100644 --- a/dingo/frontend/tests/test_sglang_tool_calls.py +++ b/dingo/frontend/tests/test_sglang_tool_calls.py @@ -592,16 +592,9 @@ def parse_non_stream(self, text): ) text = '\n{"name": "evil_tool", "arguments": {"x": 1}}\n' - choice = post.process_output( - { - "token_ids": [ord(c) for c in text], - "finish_reason": "stop", - } - ) - - assert choice is not None - assert choice["finish_reason"] == "stop" - assert choice.get("delta", {}).get("tool_calls", []) == [] + assert post.process_output({"token_ids": [ord(c) for c in text]}) is None + with pytest.raises(ValueError, match="not declared in request.tools"): + post.process_output({"token_ids": [], "finish_reason": "stop"}) def test_malformed_arguments_stream_optimistically(self): dummy_tokenizer = self.DummyTokenizer() @@ -1089,9 +1082,8 @@ def test_unrecoverable_name_is_not_retracted_or_given_fake_arguments(self) -> No def test_unknown_name_only_is_suppressed(self) -> None: post = self.make_post([[self.call(0, "unknown_tool", "")]], []) assert self.feed(post) is None - final = self.feed(post, text="", finish=True) - assert final["finish_reason"] == "stop" - assert not final["delta"].get("tool_calls") + with pytest.raises(ValueError, match="not declared in request.tools"): + self.feed(post, text="", finish=True) def test_glm47_union_schema_emits_identity_before_tool_closes(self) -> None: tools = [ diff --git a/lib/llm/src/protocols/openai/chat_completions.rs b/lib/llm/src/protocols/openai/chat_completions.rs index 8412c06f5387..cbc00bf14b78 100644 --- a/lib/llm/src/protocols/openai/chat_completions.rs +++ b/lib/llm/src/protocols/openai/chat_completions.rs @@ -966,6 +966,36 @@ impl NvCreateChatCompletionRequest { #[cfg(test)] mod tests { + #[test] + fn tool_results_require_an_unanswered_preceding_call() { + use crate::engines::ValidateRequest; + use serde_json::json; + let assistant = json!({"role":"assistant","tool_calls":[ + {"id":"a","type":"function","function":{"name":"lookup","arguments":"{}"}}, + {"id":"b","type":"function","function":{"name":"lookup","arguments":"{}"}} + ]}); + let a = json!({"role":"tool","tool_call_id":"a","content":"ok"}); + let b = json!({"role":"tool","tool_call_id":"b","content":"ok"}); + let user = json!({"role":"user","content":"hello"}); + for (messages, valid) in [ + (json!([user, a]), false), + (json!([a]), false), + (json!([assistant, a, b]), true), + (json!([assistant, b, a]), true), + (json!([assistant, a, a]), false), + (json!([assistant, user, a]), false), + ( + json!([assistant, {"role":"tool","tool_call_id":"unknown","content":"ok"}]), + false, + ), + (json!([assistant, a, b, user, assistant, b, a]), true), + ] { + let request: super::NvCreateChatCompletionRequest = + serde_json::from_value(json!({"model":"glm-5.3","messages":messages})).unwrap(); + assert_eq!(ValidateRequest::validate(&request).is_ok(), valid); + } + } + #[test] fn history_arguments_policy_is_narrow_and_does_not_rewrite() { use crate::engines::ValidateRequest; diff --git a/lib/llm/src/protocols/openai/validate.rs b/lib/llm/src/protocols/openai/validate.rs index 8b143897075f..30fba635a5cf 100644 --- a/lib/llm/src/protocols/openai/validate.rs +++ b/lib/llm/src/protocols/openai/validate.rs @@ -524,11 +524,26 @@ pub fn validate_messages_with_history_policy( // Prior assistant tool-call messages in the request must carry arguments // as a JSON object string; reject bad non-empty shapes before chat-template rendering. // This was caught in MiniMax-M3 multi-turn tool-call tests + let mut pending_tool_calls = std::collections::HashSet::new(); for (message_index, message) in messages.iter().enumerate() { + if let dynamo_protocols::types::ChatCompletionRequestMessage::Tool(tool) = message { + if !pending_tool_calls.remove(tool.tool_call_id.as_str()) { + anyhow::bail!( + "`messages[{message_index}].tool_call_id` must reference an unanswered tool call in the preceding assistant message" + ); + } + } else { + pending_tool_calls.clear(); + } if let dynamo_protocols::types::ChatCompletionRequestMessage::Assistant(assistant) = message && let Some(tool_calls) = &assistant.tool_calls { for (tool_call_index, tool_call) in tool_calls.iter().enumerate() { + if !pending_tool_calls.insert(tool_call.id.as_str()) { + anyhow::bail!( + "`messages[{message_index}].tool_calls[{tool_call_index}].id` must be unique within the assistant message" + ); + } validate_json_object_string( &tool_call.function.arguments, allow_invalid_history_arguments, From 2cd1ec017dd0964f4090dd3131280e974ae5db82 Mon Sep 17 00:00:00 2001 From: guojn1 Date: Fri, 16 Oct 2026 21:28:35 +0800 Subject: [PATCH 6/9] fix(dingo): align GLM-5.3 sampling and truncated responses Accept do_sample and project canonical greedy sampling options. Accept GLM-5.3 top_p=0 using the documented 0.95 default as a compatibility fallback; the hosted API's exact handling of zero is undocumented. Return empty content for reasoning-only length finishes. Validation: Rust formatting, staged diff checks, and targeted docs lint passed. Rust unit tests remain blocked by the unavailable locked aisimulate-core dependency. Full docs navigation checks have pre-existing missing targets; Fern CLI is unavailable locally. --- .../components/frontend-configuration.mdx | 18 +++ lib/llm/src/discovery/watcher.rs | 1 + lib/llm/src/entrypoint/input/text.rs | 1 + lib/llm/src/http/service/openai.rs | 10 ++ lib/llm/src/preprocessor.rs | 1 + .../src/preprocessor/speculative_prefill.rs | 1 + lib/llm/src/protocols/anthropic/types.rs | 1 + lib/llm/src/protocols/common/input_trigger.rs | 1 + .../src/protocols/openai/chat_completions.rs | 122 +++++++++++++++++- .../openai/chat_completions/aggregator.rs | 32 +++++ lib/llm/src/protocols/openai/responses/mod.rs | 1 + .../tests/parallel_tool_call_integration.rs | 1 + lib/llm/tests/preprocessor.rs | 3 + lib/llm/tests/test_common_ext.rs | 3 + lib/llm/tests/test_streaming_usage.rs | 2 + lib/llm/tests/tool_choice.rs | 1 + lib/llm/tests/tool_choice_finish_reasons.rs | 1 + 17 files changed, 197 insertions(+), 3 deletions(-) diff --git a/docs/fern/pages/reference/components/frontend-configuration.mdx b/docs/fern/pages/reference/components/frontend-configuration.mdx index 4fd7e0e514ea..35031419dd6d 100644 --- a/docs/fern/pages/reference/components/frontend-configuration.mdx +++ b/docs/fern/pages/reference/components/frontend-configuration.mdx @@ -803,6 +803,24 @@ The frontend exposes the following HTTP endpoints. | `POST` | `/v1/videos/generations/stream` | Video generation (streaming) | | `GET` | `/v1/models` | List available models | +#### GLM-5.3 chat completion compatibility + +`POST /v1/chat/completions` accepts the root-level boolean `do_sample`, following the +[Z.AI chat completion contract](https://docs.z.ai/api-reference/llm/chat-completion). +Omitting it or setting it to `true` preserves the requested sampling controls. Setting it to +`false` selects greedy decoding: the frontend supplies `temperature=0` and `top_p=1` to +both Rust and Python backend adapters, for streaming and non-streaming requests. + +For GLM-5.3 model IDs (`glm-5.3`, `glm5.3`, and equivalent case, underscore, or organization +prefix variants), the default `top_p` is `0.95`. The hosted API also accepts `top_p=0`. +Dynamo accepts this value and maps it to `0.95` before backend dispatch because SGLang rejects +zero. This is a compatibility fallback to the documented default; the hosted API's exact +sampling distribution for zero is not documented. An explicit `do_sample=false` takes precedence. + +If a non-streaming response exhausts its output budget while producing only reasoning, +`message.content` is `""` and `finish_reason` remains `"length"`. Reasoning stays in +`message.reasoning_content`; the frontend does not increase the requested token budget. + ### Anthropic (Experimental) | Method | Path | Description | diff --git a/lib/llm/src/discovery/watcher.rs b/lib/llm/src/discovery/watcher.rs index 7d369fe175db..5c30166bc084 100644 --- a/lib/llm/src/discovery/watcher.rs +++ b/lib/llm/src/discovery/watcher.rs @@ -2748,6 +2748,7 @@ request_classifier: chat_template_args: None, thinking: None, thinking_token_budget: None, + do_sample: None, media_io_kwargs: None, return_tokens_as_token_ids: None, unsupported_fields: Default::default(), diff --git a/lib/llm/src/entrypoint/input/text.rs b/lib/llm/src/entrypoint/input/text.rs index 8c078f9400cd..a17b42f0ad16 100644 --- a/lib/llm/src/entrypoint/input/text.rs +++ b/lib/llm/src/entrypoint/input/text.rs @@ -118,6 +118,7 @@ async fn main_loop( media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), }; diff --git a/lib/llm/src/http/service/openai.rs b/lib/llm/src/http/service/openai.rs index 6bfca4e81b0d..7ee2625de43d 100644 --- a/lib/llm/src/http/service/openai.rs +++ b/lib/llm/src/http/service/openai.rs @@ -4252,6 +4252,8 @@ async fn chat_completions( return Err(err_response); } + request.normalize_sampling_controls(); + // Create HTTP queue guard after template resolution so labels are correct let http_queue_guard = state.metrics_clone().create_http_queue_guard(&metric_model); @@ -9006,6 +9008,7 @@ mod tests { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), }; let result = validate_chat_completion_required_fields(&request); @@ -9041,6 +9044,7 @@ mod tests { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), }; let result = validate_chat_completion_required_fields(&request); @@ -9341,6 +9345,7 @@ mod tests { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), }; @@ -9374,6 +9379,7 @@ mod tests { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), }; let result = validate_chat_completion_fields_generic(&request); @@ -9406,6 +9412,7 @@ mod tests { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), }; let result = validate_chat_completion_fields_generic(&request); @@ -9438,6 +9445,7 @@ mod tests { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), }; let result = validate_chat_completion_fields_generic(&request); @@ -9472,6 +9480,7 @@ mod tests { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), }; let result = validate_chat_completion_fields_generic(&request); @@ -9504,6 +9513,7 @@ mod tests { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), }; let result = validate_chat_completion_fields_generic(&request); diff --git a/lib/llm/src/preprocessor.rs b/lib/llm/src/preprocessor.rs index 67b22310accc..fa203bc3cbca 100644 --- a/lib/llm/src/preprocessor.rs +++ b/lib/llm/src/preprocessor.rs @@ -9510,6 +9510,7 @@ mod tests { chat_template_args: None, thinking: None, thinking_token_budget: None, + do_sample: None, media_io_kwargs: None, return_tokens_as_token_ids: None, unsupported_fields: Default::default(), diff --git a/lib/llm/src/preprocessor/speculative_prefill.rs b/lib/llm/src/preprocessor/speculative_prefill.rs index e1c0322a5133..e660869c8302 100644 --- a/lib/llm/src/preprocessor/speculative_prefill.rs +++ b/lib/llm/src/preprocessor/speculative_prefill.rs @@ -622,6 +622,7 @@ mod tests { chat_template_args: None, thinking: None, thinking_token_budget: None, + do_sample: None, media_io_kwargs: None, return_tokens_as_token_ids: None, unsupported_fields: Default::default(), diff --git a/lib/llm/src/protocols/anthropic/types.rs b/lib/llm/src/protocols/anthropic/types.rs index 6c16f0bc4a97..d3b95337a2bf 100644 --- a/lib/llm/src/protocols/anthropic/types.rs +++ b/lib/llm/src/protocols/anthropic/types.rs @@ -183,6 +183,7 @@ impl TryFrom for NvCreateChatCompletionRequest { }, thinking: None, thinking_token_budget: None, + do_sample: None, media_io_kwargs: None, return_tokens_as_token_ids: None, unsupported_fields: Default::default(), diff --git a/lib/llm/src/protocols/common/input_trigger.rs b/lib/llm/src/protocols/common/input_trigger.rs index 51781d5a513c..e3358af6e072 100644 --- a/lib/llm/src/protocols/common/input_trigger.rs +++ b/lib/llm/src/protocols/common/input_trigger.rs @@ -160,6 +160,7 @@ mod tests { chat_template_args: None, thinking: None, thinking_token_budget: None, + do_sample: None, media_io_kwargs: None, return_tokens_as_token_ids: None, unsupported_fields: Default::default(), diff --git a/lib/llm/src/protocols/openai/chat_completions.rs b/lib/llm/src/protocols/openai/chat_completions.rs index cbc00bf14b78..872e4ed190c9 100644 --- a/lib/llm/src/protocols/openai/chat_completions.rs +++ b/lib/llm/src/protocols/openai/chat_completions.rs @@ -166,6 +166,11 @@ pub struct NvCreateChatCompletionRequest { #[serde(default, skip_serializing_if = "Option::is_none")] pub thinking_token_budget: Option, + /// GLM-compatible sampling switch. When false, temperature and top_p are + /// accepted but have no effect; generation uses greedy decoding. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub do_sample: Option, + /// Runtime media decoding parameters, forwarded verbatim to the worker when the /// worker owns decoding. When the frontend decodes, these override the MDC defaults. /// Kept opaque so options the frontend does not own pass through untouched. @@ -189,6 +194,30 @@ pub struct NvCreateChatCompletionRequest { } impl NvCreateChatCompletionRequest { + /// Apply sampling controls after validating the client's original values. + /// The resulting canonical options are shared by Rust and Python backends. + pub fn normalize_sampling_controls(&mut self) { + self.inner.temperature = self.get_temperature(); + self.inner.top_p = self.get_top_p(); + // Downstream adapters receive canonical sampling parameters, not a + // backend-specific switch that could be ignored or rejected there. + self.do_sample = None; + } + + fn compatible_top_p(&self) -> Option { + if is_glm53_model_id(&self.inner.model) + && (self.inner.top_p.is_none() || self.inner.top_p == Some(0.0)) + { + // The GLM-5.3 hosted API accepts zero and still samples. Its exact + // treatment is undocumented; use its documented default rather + // than turning zero into greedy decoding or sending it to SGLang, + // which rejects zero. Keep other models' validation unchanged. + Some(0.95) + } else { + self.inner.top_p + } + } + /// Preserve the legacy salt across serialization to Python processors. /// The unsupported-field map is deliberately not serialized; promote only /// the resolved cache namespace, leaving unrelated unknown fields private. @@ -682,12 +711,20 @@ impl AnnotationsProvider for NvCreateChatCompletionRequest { impl OpenAISamplingOptionsProvider for NvCreateChatCompletionRequest { /// Retrieves the temperature parameter for sampling, if set. fn get_temperature(&self) -> Option { - self.inner.temperature + if self.do_sample == Some(false) { + Some(0.0) + } else { + self.inner.temperature + } } /// Retrieves the top-p (nucleus sampling) parameter, if set. fn get_top_p(&self) -> Option { - self.inner.top_p + if self.do_sample == Some(false) { + Some(1.0) + } else { + self.compatible_top_p() + } } /// Retrieves the frequency penalty parameter, if set. @@ -941,7 +978,7 @@ impl NvCreateChatCompletionRequest { // none for stream // none for stream_options validate::validate_temperature(self.inner.temperature)?; - validate::validate_top_p(self.inner.top_p)?; + validate::validate_top_p(self.compatible_top_p())?; let effective_tools = validate::validated_effective_tools(&self.inner)?; validate::validate_tool_choice(&self.inner.tool_choice, Some(effective_tools.as_ref()))?; // none for parallel_tool_calls @@ -1106,6 +1143,85 @@ mod tests { serde_json::from_value(body).expect("Failed to deserialize request") } + #[test] + fn test_do_sample_false_normalizes_sampling_after_validation() { + let mut request = chat_request_with(&json!({ + "do_sample": false, + "temperature": 1.0, + "top_p": 0.95 + })); + assert!(request.unsupported_fields.is_empty()); + assert_eq!(serde_json::to_value(&request).unwrap()["do_sample"], false); + ValidateRequest::validate(&request).expect("do_sample is accepted"); + request.normalize_sampling_controls(); + let sampling = request.extract_sampling_options().unwrap(); + assert_eq!(sampling.temperature, Some(0.0)); + assert_eq!(sampling.top_p, Some(1.0)); + let python_request = serde_json::to_value(&request).unwrap(); + assert!(python_request.get("do_sample").is_none()); + assert_eq!(python_request["temperature"], 0.0); + assert_eq!(python_request["top_p"], 1.0); + + let mut request = chat_request_with(&json!({"do_sample": true, "temperature": 0.7})); + ValidateRequest::validate(&request).expect("do_sample=true is accepted"); + request.normalize_sampling_controls(); + assert_eq!( + request.extract_sampling_options().unwrap().temperature, + Some(0.7) + ); + + let invalid = chat_request_with(&json!({"do_sample": false, "top_p": 0})); + assert!(ValidateRequest::validate(&invalid).is_err()); + } + + #[test] + fn test_do_sample_rejects_non_boolean_values() { + for value in [json!("true"), json!(0), json!({})] { + let request = serde_json::from_value::(json!({ + "model": "glm-5.3", + "messages": [{"role": "user", "content": "hi"}], + "do_sample": value + })); + assert!(request.is_err()); + } + } + + #[test] + fn test_glm53_zero_top_p_is_accepted_and_normalized_for_backends() { + for model in ["glm-5.3", "glm5.3", "zai-org/GLM-5.3"] { + for do_sample in [None, Some(true), Some(false)] { + let mut request = chat_request_with(&json!({ + "model": model, "top_p": 0, "temperature": 1.0 + })); + request.do_sample = do_sample; + ValidateRequest::validate(&request).expect("GLM-5.3 accepts top_p=0"); + let expected = if do_sample == Some(false) { 1.0 } else { 0.95 }; + assert_eq!( + request.extract_sampling_options().unwrap().top_p, + Some(expected) + ); + request.normalize_sampling_controls(); + let python_request = serde_json::to_value(&request).unwrap(); + assert_eq!(python_request["top_p"], expected); + assert_eq!( + python_request["temperature"], + if do_sample == Some(false) { 0.0 } else { 1.0 } + ); + } + } + + let mut request = chat_request_with(&json!({"model": "glm-5.3", "top_p": 0.7})); + request.normalize_sampling_controls(); + assert_eq!(request.inner.top_p, Some(0.7)); + let mut default = chat_request_with(&json!({"model": "glm-5.3"})); + default.normalize_sampling_controls(); + assert_eq!(default.inner.top_p, Some(0.95)); + for model in ["test-model", "glm-5.2", "glm-5.3-flash"] { + let request = chat_request_with(&json!({"model": model, "top_p": 0})); + assert!(ValidateRequest::validate(&request).is_err()); + } + } + /// Extracts sampling options for `extra` and returns the guided-decoding options it /// produced, failing if extraction rejected the request or engaged nothing. fn guided_for(extra: &serde_json::Value) -> GuidedDecodingOptions { diff --git a/lib/llm/src/protocols/openai/chat_completions/aggregator.rs b/lib/llm/src/protocols/openai/chat_completions/aggregator.rs index eda80c947f52..3132da54cfd0 100644 --- a/lib/llm/src/protocols/openai/chat_completions/aggregator.rs +++ b/lib/llm/src/protocols/openai/chat_completions/aggregator.rs @@ -937,6 +937,13 @@ impl From for dynamo_protocols::types::ChatChoice { } else if !delta.text.is_empty() { // Text-only response (backward compatible) Some(ChatCompletionMessageContent::Text(delta.text)) + } else if finish_reason == Some(dynamo_protocols::types::FinishReason::Length) + && delta.reasoning_content.is_some() + && !has_tool_calls + { + // A reasoning-only turn cut off by max_tokens has an empty answer. + // GLM's chat-completion response represents that answer as "". + Some(ChatCompletionMessageContent::Text(String::new())) } else { None }; @@ -3560,6 +3567,31 @@ mod tests { assert!(json.get("reasoning_content").is_some()); } + #[tokio::test] + async fn test_reasoning_only_length_response_has_empty_content() { + let mut delta = create_reasoning_delta(0, "", "Still reasoning"); + delta.data.as_mut().unwrap().inner.choices[0].finish_reason = + Some(dynamo_protocols::types::FinishReason::Length); + let response = DeltaAggregator::apply( + Box::pin(stream::iter(vec![delta])), + ParsingOptions::default(), + ) + .await + .unwrap(); + let choice = &response.inner.choices[0]; + assert_eq!( + choice.message.content, + Some(ChatCompletionMessageContent::Text(String::new())) + ); + assert_eq!( + choice.finish_reason, + Some(dynamo_protocols::types::FinishReason::Length) + ); + let json = serde_json::to_value(&choice.message).unwrap(); + assert_eq!(json["content"], ""); + assert_eq!(json["reasoning_content"], "Still reasoning"); + } + // --- glm47 truncation recovery tests --- // // Reproduces the drop confirmed on dynamo-parsers 5.1.3: diff --git a/lib/llm/src/protocols/openai/responses/mod.rs b/lib/llm/src/protocols/openai/responses/mod.rs index cf686ae39224..bca0d4dba6b9 100644 --- a/lib/llm/src/protocols/openai/responses/mod.rs +++ b/lib/llm/src/protocols/openai/responses/mod.rs @@ -1033,6 +1033,7 @@ impl TryFrom for NvCreateChatCompletionRequest { chat_template_args: resp.chat_template_args, thinking: None, thinking_token_budget: resp.thinking_token_budget, + do_sample: None, media_io_kwargs: None, return_tokens_as_token_ids: None, unsupported_fields: Default::default(), diff --git a/lib/llm/tests/parallel_tool_call_integration.rs b/lib/llm/tests/parallel_tool_call_integration.rs index dfd0310492a8..e5b4614b3086 100644 --- a/lib/llm/tests/parallel_tool_call_integration.rs +++ b/lib/llm/tests/parallel_tool_call_integration.rs @@ -96,6 +96,7 @@ fn create_mock_chat_completion_request() -> NvCreateChatCompletionRequest { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), } } diff --git a/lib/llm/tests/preprocessor.rs b/lib/llm/tests/preprocessor.rs index 29faf27d913d..733567b4899d 100644 --- a/lib/llm/tests/preprocessor.rs +++ b/lib/llm/tests/preprocessor.rs @@ -265,6 +265,7 @@ impl Request { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), } } @@ -954,6 +955,7 @@ mod context_length_validation { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), } } @@ -1335,6 +1337,7 @@ mod embedding_without_chat_template { return_tokens_as_token_ids: None, unsupported_fields: Default::default(), thinking_token_budget: None, + do_sample: None, } } diff --git a/lib/llm/tests/test_common_ext.rs b/lib/llm/tests/test_common_ext.rs index 8b0c5ddc6968..44ae59d1383c 100644 --- a/lib/llm/tests/test_common_ext.rs +++ b/lib/llm/tests/test_common_ext.rs @@ -72,6 +72,7 @@ fn test_sampling_parameters_include_stop_str_in_output_extraction() { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), }; @@ -343,6 +344,7 @@ fn test_serialization_preserves_structure() { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), }; @@ -398,6 +400,7 @@ fn test_sampling_parameters_extraction() { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), }; diff --git a/lib/llm/tests/test_streaming_usage.rs b/lib/llm/tests/test_streaming_usage.rs index 0b7c95b3ae18..58197a4786f3 100644 --- a/lib/llm/tests/test_streaming_usage.rs +++ b/lib/llm/tests/test_streaming_usage.rs @@ -213,6 +213,7 @@ fn create_chat_request( media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), } } @@ -807,6 +808,7 @@ fn create_nonstreaming_chat_request() -> NvCreateChatCompletionRequest { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), } } diff --git a/lib/llm/tests/tool_choice.rs b/lib/llm/tests/tool_choice.rs index b4225fe26706..3db33bc23ee0 100644 --- a/lib/llm/tests/tool_choice.rs +++ b/lib/llm/tests/tool_choice.rs @@ -73,6 +73,7 @@ fn create_test_request() -> NvCreateChatCompletionRequest { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), } } diff --git a/lib/llm/tests/tool_choice_finish_reasons.rs b/lib/llm/tests/tool_choice_finish_reasons.rs index f6dc184dda66..f3510e569db5 100644 --- a/lib/llm/tests/tool_choice_finish_reasons.rs +++ b/lib/llm/tests/tool_choice_finish_reasons.rs @@ -36,6 +36,7 @@ fn create_test_request() -> NvCreateChatCompletionRequest { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), } } From 403ee2c57b17e0200bfa57d6e707a4d3692d0f8f Mon Sep 17 00:00:00 2001 From: guojn1 Date: Sat, 17 Oct 2026 16:22:34 +0800 Subject: [PATCH 7/9] Revert "fix(dingo): align GLM-5.3 sampling and truncated responses" This reverts commit 2cd1ec017dd0964f4090dd3131280e974ae5db82. --- .../components/frontend-configuration.mdx | 18 --- lib/llm/src/discovery/watcher.rs | 1 - lib/llm/src/entrypoint/input/text.rs | 1 - lib/llm/src/http/service/openai.rs | 10 -- lib/llm/src/preprocessor.rs | 1 - .../src/preprocessor/speculative_prefill.rs | 1 - lib/llm/src/protocols/anthropic/types.rs | 1 - lib/llm/src/protocols/common/input_trigger.rs | 1 - .../src/protocols/openai/chat_completions.rs | 122 +----------------- .../openai/chat_completions/aggregator.rs | 32 ----- lib/llm/src/protocols/openai/responses/mod.rs | 1 - .../tests/parallel_tool_call_integration.rs | 1 - lib/llm/tests/preprocessor.rs | 3 - lib/llm/tests/test_common_ext.rs | 3 - lib/llm/tests/test_streaming_usage.rs | 2 - lib/llm/tests/tool_choice.rs | 1 - lib/llm/tests/tool_choice_finish_reasons.rs | 1 - 17 files changed, 3 insertions(+), 197 deletions(-) diff --git a/docs/fern/pages/reference/components/frontend-configuration.mdx b/docs/fern/pages/reference/components/frontend-configuration.mdx index 35031419dd6d..4fd7e0e514ea 100644 --- a/docs/fern/pages/reference/components/frontend-configuration.mdx +++ b/docs/fern/pages/reference/components/frontend-configuration.mdx @@ -803,24 +803,6 @@ The frontend exposes the following HTTP endpoints. | `POST` | `/v1/videos/generations/stream` | Video generation (streaming) | | `GET` | `/v1/models` | List available models | -#### GLM-5.3 chat completion compatibility - -`POST /v1/chat/completions` accepts the root-level boolean `do_sample`, following the -[Z.AI chat completion contract](https://docs.z.ai/api-reference/llm/chat-completion). -Omitting it or setting it to `true` preserves the requested sampling controls. Setting it to -`false` selects greedy decoding: the frontend supplies `temperature=0` and `top_p=1` to -both Rust and Python backend adapters, for streaming and non-streaming requests. - -For GLM-5.3 model IDs (`glm-5.3`, `glm5.3`, and equivalent case, underscore, or organization -prefix variants), the default `top_p` is `0.95`. The hosted API also accepts `top_p=0`. -Dynamo accepts this value and maps it to `0.95` before backend dispatch because SGLang rejects -zero. This is a compatibility fallback to the documented default; the hosted API's exact -sampling distribution for zero is not documented. An explicit `do_sample=false` takes precedence. - -If a non-streaming response exhausts its output budget while producing only reasoning, -`message.content` is `""` and `finish_reason` remains `"length"`. Reasoning stays in -`message.reasoning_content`; the frontend does not increase the requested token budget. - ### Anthropic (Experimental) | Method | Path | Description | diff --git a/lib/llm/src/discovery/watcher.rs b/lib/llm/src/discovery/watcher.rs index 5c30166bc084..7d369fe175db 100644 --- a/lib/llm/src/discovery/watcher.rs +++ b/lib/llm/src/discovery/watcher.rs @@ -2748,7 +2748,6 @@ request_classifier: chat_template_args: None, thinking: None, thinking_token_budget: None, - do_sample: None, media_io_kwargs: None, return_tokens_as_token_ids: None, unsupported_fields: Default::default(), diff --git a/lib/llm/src/entrypoint/input/text.rs b/lib/llm/src/entrypoint/input/text.rs index a17b42f0ad16..8c078f9400cd 100644 --- a/lib/llm/src/entrypoint/input/text.rs +++ b/lib/llm/src/entrypoint/input/text.rs @@ -118,7 +118,6 @@ async fn main_loop( media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, - do_sample: None, unsupported_fields: Default::default(), }; diff --git a/lib/llm/src/http/service/openai.rs b/lib/llm/src/http/service/openai.rs index 7ee2625de43d..6bfca4e81b0d 100644 --- a/lib/llm/src/http/service/openai.rs +++ b/lib/llm/src/http/service/openai.rs @@ -4252,8 +4252,6 @@ async fn chat_completions( return Err(err_response); } - request.normalize_sampling_controls(); - // Create HTTP queue guard after template resolution so labels are correct let http_queue_guard = state.metrics_clone().create_http_queue_guard(&metric_model); @@ -9008,7 +9006,6 @@ mod tests { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, - do_sample: None, unsupported_fields: Default::default(), }; let result = validate_chat_completion_required_fields(&request); @@ -9044,7 +9041,6 @@ mod tests { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, - do_sample: None, unsupported_fields: Default::default(), }; let result = validate_chat_completion_required_fields(&request); @@ -9345,7 +9341,6 @@ mod tests { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, - do_sample: None, unsupported_fields: Default::default(), }; @@ -9379,7 +9374,6 @@ mod tests { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, - do_sample: None, unsupported_fields: Default::default(), }; let result = validate_chat_completion_fields_generic(&request); @@ -9412,7 +9406,6 @@ mod tests { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, - do_sample: None, unsupported_fields: Default::default(), }; let result = validate_chat_completion_fields_generic(&request); @@ -9445,7 +9438,6 @@ mod tests { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, - do_sample: None, unsupported_fields: Default::default(), }; let result = validate_chat_completion_fields_generic(&request); @@ -9480,7 +9472,6 @@ mod tests { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, - do_sample: None, unsupported_fields: Default::default(), }; let result = validate_chat_completion_fields_generic(&request); @@ -9513,7 +9504,6 @@ mod tests { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, - do_sample: None, unsupported_fields: Default::default(), }; let result = validate_chat_completion_fields_generic(&request); diff --git a/lib/llm/src/preprocessor.rs b/lib/llm/src/preprocessor.rs index fa203bc3cbca..67b22310accc 100644 --- a/lib/llm/src/preprocessor.rs +++ b/lib/llm/src/preprocessor.rs @@ -9510,7 +9510,6 @@ mod tests { chat_template_args: None, thinking: None, thinking_token_budget: None, - do_sample: None, media_io_kwargs: None, return_tokens_as_token_ids: None, unsupported_fields: Default::default(), diff --git a/lib/llm/src/preprocessor/speculative_prefill.rs b/lib/llm/src/preprocessor/speculative_prefill.rs index e660869c8302..e1c0322a5133 100644 --- a/lib/llm/src/preprocessor/speculative_prefill.rs +++ b/lib/llm/src/preprocessor/speculative_prefill.rs @@ -622,7 +622,6 @@ mod tests { chat_template_args: None, thinking: None, thinking_token_budget: None, - do_sample: None, media_io_kwargs: None, return_tokens_as_token_ids: None, unsupported_fields: Default::default(), diff --git a/lib/llm/src/protocols/anthropic/types.rs b/lib/llm/src/protocols/anthropic/types.rs index d3b95337a2bf..6c16f0bc4a97 100644 --- a/lib/llm/src/protocols/anthropic/types.rs +++ b/lib/llm/src/protocols/anthropic/types.rs @@ -183,7 +183,6 @@ impl TryFrom for NvCreateChatCompletionRequest { }, thinking: None, thinking_token_budget: None, - do_sample: None, media_io_kwargs: None, return_tokens_as_token_ids: None, unsupported_fields: Default::default(), diff --git a/lib/llm/src/protocols/common/input_trigger.rs b/lib/llm/src/protocols/common/input_trigger.rs index e3358af6e072..51781d5a513c 100644 --- a/lib/llm/src/protocols/common/input_trigger.rs +++ b/lib/llm/src/protocols/common/input_trigger.rs @@ -160,7 +160,6 @@ mod tests { chat_template_args: None, thinking: None, thinking_token_budget: None, - do_sample: None, media_io_kwargs: None, return_tokens_as_token_ids: None, unsupported_fields: Default::default(), diff --git a/lib/llm/src/protocols/openai/chat_completions.rs b/lib/llm/src/protocols/openai/chat_completions.rs index 872e4ed190c9..cbc00bf14b78 100644 --- a/lib/llm/src/protocols/openai/chat_completions.rs +++ b/lib/llm/src/protocols/openai/chat_completions.rs @@ -166,11 +166,6 @@ pub struct NvCreateChatCompletionRequest { #[serde(default, skip_serializing_if = "Option::is_none")] pub thinking_token_budget: Option, - /// GLM-compatible sampling switch. When false, temperature and top_p are - /// accepted but have no effect; generation uses greedy decoding. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub do_sample: Option, - /// Runtime media decoding parameters, forwarded verbatim to the worker when the /// worker owns decoding. When the frontend decodes, these override the MDC defaults. /// Kept opaque so options the frontend does not own pass through untouched. @@ -194,30 +189,6 @@ pub struct NvCreateChatCompletionRequest { } impl NvCreateChatCompletionRequest { - /// Apply sampling controls after validating the client's original values. - /// The resulting canonical options are shared by Rust and Python backends. - pub fn normalize_sampling_controls(&mut self) { - self.inner.temperature = self.get_temperature(); - self.inner.top_p = self.get_top_p(); - // Downstream adapters receive canonical sampling parameters, not a - // backend-specific switch that could be ignored or rejected there. - self.do_sample = None; - } - - fn compatible_top_p(&self) -> Option { - if is_glm53_model_id(&self.inner.model) - && (self.inner.top_p.is_none() || self.inner.top_p == Some(0.0)) - { - // The GLM-5.3 hosted API accepts zero and still samples. Its exact - // treatment is undocumented; use its documented default rather - // than turning zero into greedy decoding or sending it to SGLang, - // which rejects zero. Keep other models' validation unchanged. - Some(0.95) - } else { - self.inner.top_p - } - } - /// Preserve the legacy salt across serialization to Python processors. /// The unsupported-field map is deliberately not serialized; promote only /// the resolved cache namespace, leaving unrelated unknown fields private. @@ -711,20 +682,12 @@ impl AnnotationsProvider for NvCreateChatCompletionRequest { impl OpenAISamplingOptionsProvider for NvCreateChatCompletionRequest { /// Retrieves the temperature parameter for sampling, if set. fn get_temperature(&self) -> Option { - if self.do_sample == Some(false) { - Some(0.0) - } else { - self.inner.temperature - } + self.inner.temperature } /// Retrieves the top-p (nucleus sampling) parameter, if set. fn get_top_p(&self) -> Option { - if self.do_sample == Some(false) { - Some(1.0) - } else { - self.compatible_top_p() - } + self.inner.top_p } /// Retrieves the frequency penalty parameter, if set. @@ -978,7 +941,7 @@ impl NvCreateChatCompletionRequest { // none for stream // none for stream_options validate::validate_temperature(self.inner.temperature)?; - validate::validate_top_p(self.compatible_top_p())?; + validate::validate_top_p(self.inner.top_p)?; let effective_tools = validate::validated_effective_tools(&self.inner)?; validate::validate_tool_choice(&self.inner.tool_choice, Some(effective_tools.as_ref()))?; // none for parallel_tool_calls @@ -1143,85 +1106,6 @@ mod tests { serde_json::from_value(body).expect("Failed to deserialize request") } - #[test] - fn test_do_sample_false_normalizes_sampling_after_validation() { - let mut request = chat_request_with(&json!({ - "do_sample": false, - "temperature": 1.0, - "top_p": 0.95 - })); - assert!(request.unsupported_fields.is_empty()); - assert_eq!(serde_json::to_value(&request).unwrap()["do_sample"], false); - ValidateRequest::validate(&request).expect("do_sample is accepted"); - request.normalize_sampling_controls(); - let sampling = request.extract_sampling_options().unwrap(); - assert_eq!(sampling.temperature, Some(0.0)); - assert_eq!(sampling.top_p, Some(1.0)); - let python_request = serde_json::to_value(&request).unwrap(); - assert!(python_request.get("do_sample").is_none()); - assert_eq!(python_request["temperature"], 0.0); - assert_eq!(python_request["top_p"], 1.0); - - let mut request = chat_request_with(&json!({"do_sample": true, "temperature": 0.7})); - ValidateRequest::validate(&request).expect("do_sample=true is accepted"); - request.normalize_sampling_controls(); - assert_eq!( - request.extract_sampling_options().unwrap().temperature, - Some(0.7) - ); - - let invalid = chat_request_with(&json!({"do_sample": false, "top_p": 0})); - assert!(ValidateRequest::validate(&invalid).is_err()); - } - - #[test] - fn test_do_sample_rejects_non_boolean_values() { - for value in [json!("true"), json!(0), json!({})] { - let request = serde_json::from_value::(json!({ - "model": "glm-5.3", - "messages": [{"role": "user", "content": "hi"}], - "do_sample": value - })); - assert!(request.is_err()); - } - } - - #[test] - fn test_glm53_zero_top_p_is_accepted_and_normalized_for_backends() { - for model in ["glm-5.3", "glm5.3", "zai-org/GLM-5.3"] { - for do_sample in [None, Some(true), Some(false)] { - let mut request = chat_request_with(&json!({ - "model": model, "top_p": 0, "temperature": 1.0 - })); - request.do_sample = do_sample; - ValidateRequest::validate(&request).expect("GLM-5.3 accepts top_p=0"); - let expected = if do_sample == Some(false) { 1.0 } else { 0.95 }; - assert_eq!( - request.extract_sampling_options().unwrap().top_p, - Some(expected) - ); - request.normalize_sampling_controls(); - let python_request = serde_json::to_value(&request).unwrap(); - assert_eq!(python_request["top_p"], expected); - assert_eq!( - python_request["temperature"], - if do_sample == Some(false) { 0.0 } else { 1.0 } - ); - } - } - - let mut request = chat_request_with(&json!({"model": "glm-5.3", "top_p": 0.7})); - request.normalize_sampling_controls(); - assert_eq!(request.inner.top_p, Some(0.7)); - let mut default = chat_request_with(&json!({"model": "glm-5.3"})); - default.normalize_sampling_controls(); - assert_eq!(default.inner.top_p, Some(0.95)); - for model in ["test-model", "glm-5.2", "glm-5.3-flash"] { - let request = chat_request_with(&json!({"model": model, "top_p": 0})); - assert!(ValidateRequest::validate(&request).is_err()); - } - } - /// Extracts sampling options for `extra` and returns the guided-decoding options it /// produced, failing if extraction rejected the request or engaged nothing. fn guided_for(extra: &serde_json::Value) -> GuidedDecodingOptions { diff --git a/lib/llm/src/protocols/openai/chat_completions/aggregator.rs b/lib/llm/src/protocols/openai/chat_completions/aggregator.rs index 3132da54cfd0..eda80c947f52 100644 --- a/lib/llm/src/protocols/openai/chat_completions/aggregator.rs +++ b/lib/llm/src/protocols/openai/chat_completions/aggregator.rs @@ -937,13 +937,6 @@ impl From for dynamo_protocols::types::ChatChoice { } else if !delta.text.is_empty() { // Text-only response (backward compatible) Some(ChatCompletionMessageContent::Text(delta.text)) - } else if finish_reason == Some(dynamo_protocols::types::FinishReason::Length) - && delta.reasoning_content.is_some() - && !has_tool_calls - { - // A reasoning-only turn cut off by max_tokens has an empty answer. - // GLM's chat-completion response represents that answer as "". - Some(ChatCompletionMessageContent::Text(String::new())) } else { None }; @@ -3567,31 +3560,6 @@ mod tests { assert!(json.get("reasoning_content").is_some()); } - #[tokio::test] - async fn test_reasoning_only_length_response_has_empty_content() { - let mut delta = create_reasoning_delta(0, "", "Still reasoning"); - delta.data.as_mut().unwrap().inner.choices[0].finish_reason = - Some(dynamo_protocols::types::FinishReason::Length); - let response = DeltaAggregator::apply( - Box::pin(stream::iter(vec![delta])), - ParsingOptions::default(), - ) - .await - .unwrap(); - let choice = &response.inner.choices[0]; - assert_eq!( - choice.message.content, - Some(ChatCompletionMessageContent::Text(String::new())) - ); - assert_eq!( - choice.finish_reason, - Some(dynamo_protocols::types::FinishReason::Length) - ); - let json = serde_json::to_value(&choice.message).unwrap(); - assert_eq!(json["content"], ""); - assert_eq!(json["reasoning_content"], "Still reasoning"); - } - // --- glm47 truncation recovery tests --- // // Reproduces the drop confirmed on dynamo-parsers 5.1.3: diff --git a/lib/llm/src/protocols/openai/responses/mod.rs b/lib/llm/src/protocols/openai/responses/mod.rs index bca0d4dba6b9..cf686ae39224 100644 --- a/lib/llm/src/protocols/openai/responses/mod.rs +++ b/lib/llm/src/protocols/openai/responses/mod.rs @@ -1033,7 +1033,6 @@ impl TryFrom for NvCreateChatCompletionRequest { chat_template_args: resp.chat_template_args, thinking: None, thinking_token_budget: resp.thinking_token_budget, - do_sample: None, media_io_kwargs: None, return_tokens_as_token_ids: None, unsupported_fields: Default::default(), diff --git a/lib/llm/tests/parallel_tool_call_integration.rs b/lib/llm/tests/parallel_tool_call_integration.rs index e5b4614b3086..dfd0310492a8 100644 --- a/lib/llm/tests/parallel_tool_call_integration.rs +++ b/lib/llm/tests/parallel_tool_call_integration.rs @@ -96,7 +96,6 @@ fn create_mock_chat_completion_request() -> NvCreateChatCompletionRequest { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, - do_sample: None, unsupported_fields: Default::default(), } } diff --git a/lib/llm/tests/preprocessor.rs b/lib/llm/tests/preprocessor.rs index 733567b4899d..29faf27d913d 100644 --- a/lib/llm/tests/preprocessor.rs +++ b/lib/llm/tests/preprocessor.rs @@ -265,7 +265,6 @@ impl Request { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, - do_sample: None, unsupported_fields: Default::default(), } } @@ -955,7 +954,6 @@ mod context_length_validation { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, - do_sample: None, unsupported_fields: Default::default(), } } @@ -1337,7 +1335,6 @@ mod embedding_without_chat_template { return_tokens_as_token_ids: None, unsupported_fields: Default::default(), thinking_token_budget: None, - do_sample: None, } } diff --git a/lib/llm/tests/test_common_ext.rs b/lib/llm/tests/test_common_ext.rs index 44ae59d1383c..8b0c5ddc6968 100644 --- a/lib/llm/tests/test_common_ext.rs +++ b/lib/llm/tests/test_common_ext.rs @@ -72,7 +72,6 @@ fn test_sampling_parameters_include_stop_str_in_output_extraction() { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, - do_sample: None, unsupported_fields: Default::default(), }; @@ -344,7 +343,6 @@ fn test_serialization_preserves_structure() { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, - do_sample: None, unsupported_fields: Default::default(), }; @@ -400,7 +398,6 @@ fn test_sampling_parameters_extraction() { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, - do_sample: None, unsupported_fields: Default::default(), }; diff --git a/lib/llm/tests/test_streaming_usage.rs b/lib/llm/tests/test_streaming_usage.rs index 58197a4786f3..0b7c95b3ae18 100644 --- a/lib/llm/tests/test_streaming_usage.rs +++ b/lib/llm/tests/test_streaming_usage.rs @@ -213,7 +213,6 @@ fn create_chat_request( media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, - do_sample: None, unsupported_fields: Default::default(), } } @@ -808,7 +807,6 @@ fn create_nonstreaming_chat_request() -> NvCreateChatCompletionRequest { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, - do_sample: None, unsupported_fields: Default::default(), } } diff --git a/lib/llm/tests/tool_choice.rs b/lib/llm/tests/tool_choice.rs index 3db33bc23ee0..b4225fe26706 100644 --- a/lib/llm/tests/tool_choice.rs +++ b/lib/llm/tests/tool_choice.rs @@ -73,7 +73,6 @@ fn create_test_request() -> NvCreateChatCompletionRequest { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, - do_sample: None, unsupported_fields: Default::default(), } } diff --git a/lib/llm/tests/tool_choice_finish_reasons.rs b/lib/llm/tests/tool_choice_finish_reasons.rs index f3510e569db5..f6dc184dda66 100644 --- a/lib/llm/tests/tool_choice_finish_reasons.rs +++ b/lib/llm/tests/tool_choice_finish_reasons.rs @@ -36,7 +36,6 @@ fn create_test_request() -> NvCreateChatCompletionRequest { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, - do_sample: None, unsupported_fields: Default::default(), } } From df86fccc519b7e7955a1e4f1e7bbe9d655685ece Mon Sep 17 00:00:00 2001 From: guojn1 Date: Sat, 17 Oct 2026 16:26:44 +0800 Subject: [PATCH 8/9] fix(dingo): normalize nonpositive GLM-5.3 top_p to 0.01 Replace reverted commit 2cd1ec017 while retaining its do_sample support and empty content for reasoning-only length finishes. Normalize explicit finite GLM-5.3 top_p values <=0 to 0.01 using the official synchronous SDK rule. Preserve model defaults, positive-value validation, and greedy sampling precedence. Validation: 7 isolated checks of the actual normalization source passed, along with Rust formatting, staged diff checks, and targeted docs lint. Full crate tests remain blocked by the unavailable locked aisimulate-core dependency. --- .../components/frontend-configuration.mdx | 20 +++ lib/llm/src/discovery/watcher.rs | 1 + lib/llm/src/entrypoint/input/text.rs | 1 + lib/llm/src/http/service/openai.rs | 10 ++ lib/llm/src/preprocessor.rs | 1 + .../src/preprocessor/speculative_prefill.rs | 1 + lib/llm/src/protocols/anthropic/types.rs | 1 + lib/llm/src/protocols/common/input_trigger.rs | 1 + .../src/protocols/openai/chat_completions.rs | 137 +++++++++++++++++- .../openai/chat_completions/aggregator.rs | 32 ++++ lib/llm/src/protocols/openai/responses/mod.rs | 1 + .../tests/parallel_tool_call_integration.rs | 1 + lib/llm/tests/preprocessor.rs | 3 + lib/llm/tests/test_common_ext.rs | 3 + lib/llm/tests/test_streaming_usage.rs | 2 + lib/llm/tests/tool_choice.rs | 1 + lib/llm/tests/tool_choice_finish_reasons.rs | 1 + 17 files changed, 214 insertions(+), 3 deletions(-) diff --git a/docs/fern/pages/reference/components/frontend-configuration.mdx b/docs/fern/pages/reference/components/frontend-configuration.mdx index 4fd7e0e514ea..4be0e59bafcc 100644 --- a/docs/fern/pages/reference/components/frontend-configuration.mdx +++ b/docs/fern/pages/reference/components/frontend-configuration.mdx @@ -803,6 +803,26 @@ The frontend exposes the following HTTP endpoints. | `POST` | `/v1/videos/generations/stream` | Video generation (streaming) | | `GET` | `/v1/models` | List available models | +#### GLM-5.3 chat completion compatibility + +`POST /v1/chat/completions` accepts the root-level boolean `do_sample`, following the +[Z.AI chat completion contract](https://docs.z.ai/api-reference/llm/chat-completion). +Omitting it or setting it to `true` preserves the requested sampling controls. Setting it to +`false` selects greedy decoding: the frontend supplies `temperature=0` and `top_p=1` to +both Rust and Python backend adapters, for streaming and non-streaming requests. + +For GLM-5.3 model IDs (`glm-5.3`, `glm5.3`, and equivalent case, underscore, or organization +prefix variants), the default `top_p` is `0.95`. An explicitly supplied finite `top_p<=0` +is normalized to `0.01` before backend dispatch, matching the synchronous chat-completion +client in the [official Z.AI Python SDK 0.2.3](https://pypi.org/project/zai-sdk/0.2.3/). +This follows the SDK's client-side rule; the hosted API's internal handling of zero is not +documented. Positive values retain their existing validation. An explicit `do_sample=false` +takes precedence and selects greedy decoding. + +If a non-streaming response exhausts its output budget while producing only reasoning, +`message.content` is `""` and `finish_reason` remains `"length"`. Reasoning stays in +`message.reasoning_content`; the frontend does not increase the requested token budget. + ### Anthropic (Experimental) | Method | Path | Description | diff --git a/lib/llm/src/discovery/watcher.rs b/lib/llm/src/discovery/watcher.rs index 7d369fe175db..5c30166bc084 100644 --- a/lib/llm/src/discovery/watcher.rs +++ b/lib/llm/src/discovery/watcher.rs @@ -2748,6 +2748,7 @@ request_classifier: chat_template_args: None, thinking: None, thinking_token_budget: None, + do_sample: None, media_io_kwargs: None, return_tokens_as_token_ids: None, unsupported_fields: Default::default(), diff --git a/lib/llm/src/entrypoint/input/text.rs b/lib/llm/src/entrypoint/input/text.rs index 8c078f9400cd..a17b42f0ad16 100644 --- a/lib/llm/src/entrypoint/input/text.rs +++ b/lib/llm/src/entrypoint/input/text.rs @@ -118,6 +118,7 @@ async fn main_loop( media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), }; diff --git a/lib/llm/src/http/service/openai.rs b/lib/llm/src/http/service/openai.rs index 6bfca4e81b0d..7ee2625de43d 100644 --- a/lib/llm/src/http/service/openai.rs +++ b/lib/llm/src/http/service/openai.rs @@ -4252,6 +4252,8 @@ async fn chat_completions( return Err(err_response); } + request.normalize_sampling_controls(); + // Create HTTP queue guard after template resolution so labels are correct let http_queue_guard = state.metrics_clone().create_http_queue_guard(&metric_model); @@ -9006,6 +9008,7 @@ mod tests { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), }; let result = validate_chat_completion_required_fields(&request); @@ -9041,6 +9044,7 @@ mod tests { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), }; let result = validate_chat_completion_required_fields(&request); @@ -9341,6 +9345,7 @@ mod tests { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), }; @@ -9374,6 +9379,7 @@ mod tests { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), }; let result = validate_chat_completion_fields_generic(&request); @@ -9406,6 +9412,7 @@ mod tests { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), }; let result = validate_chat_completion_fields_generic(&request); @@ -9438,6 +9445,7 @@ mod tests { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), }; let result = validate_chat_completion_fields_generic(&request); @@ -9472,6 +9480,7 @@ mod tests { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), }; let result = validate_chat_completion_fields_generic(&request); @@ -9504,6 +9513,7 @@ mod tests { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), }; let result = validate_chat_completion_fields_generic(&request); diff --git a/lib/llm/src/preprocessor.rs b/lib/llm/src/preprocessor.rs index 67b22310accc..fa203bc3cbca 100644 --- a/lib/llm/src/preprocessor.rs +++ b/lib/llm/src/preprocessor.rs @@ -9510,6 +9510,7 @@ mod tests { chat_template_args: None, thinking: None, thinking_token_budget: None, + do_sample: None, media_io_kwargs: None, return_tokens_as_token_ids: None, unsupported_fields: Default::default(), diff --git a/lib/llm/src/preprocessor/speculative_prefill.rs b/lib/llm/src/preprocessor/speculative_prefill.rs index e1c0322a5133..e660869c8302 100644 --- a/lib/llm/src/preprocessor/speculative_prefill.rs +++ b/lib/llm/src/preprocessor/speculative_prefill.rs @@ -622,6 +622,7 @@ mod tests { chat_template_args: None, thinking: None, thinking_token_budget: None, + do_sample: None, media_io_kwargs: None, return_tokens_as_token_ids: None, unsupported_fields: Default::default(), diff --git a/lib/llm/src/protocols/anthropic/types.rs b/lib/llm/src/protocols/anthropic/types.rs index 6c16f0bc4a97..d3b95337a2bf 100644 --- a/lib/llm/src/protocols/anthropic/types.rs +++ b/lib/llm/src/protocols/anthropic/types.rs @@ -183,6 +183,7 @@ impl TryFrom for NvCreateChatCompletionRequest { }, thinking: None, thinking_token_budget: None, + do_sample: None, media_io_kwargs: None, return_tokens_as_token_ids: None, unsupported_fields: Default::default(), diff --git a/lib/llm/src/protocols/common/input_trigger.rs b/lib/llm/src/protocols/common/input_trigger.rs index 51781d5a513c..e3358af6e072 100644 --- a/lib/llm/src/protocols/common/input_trigger.rs +++ b/lib/llm/src/protocols/common/input_trigger.rs @@ -160,6 +160,7 @@ mod tests { chat_template_args: None, thinking: None, thinking_token_budget: None, + do_sample: None, media_io_kwargs: None, return_tokens_as_token_ids: None, unsupported_fields: Default::default(), diff --git a/lib/llm/src/protocols/openai/chat_completions.rs b/lib/llm/src/protocols/openai/chat_completions.rs index cbc00bf14b78..8736264f36fe 100644 --- a/lib/llm/src/protocols/openai/chat_completions.rs +++ b/lib/llm/src/protocols/openai/chat_completions.rs @@ -166,6 +166,11 @@ pub struct NvCreateChatCompletionRequest { #[serde(default, skip_serializing_if = "Option::is_none")] pub thinking_token_budget: Option, + /// GLM-compatible sampling switch. When false, temperature and top_p are + /// accepted but have no effect; generation uses greedy decoding. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub do_sample: Option, + /// Runtime media decoding parameters, forwarded verbatim to the worker when the /// worker owns decoding. When the frontend decodes, these override the MDC defaults. /// Kept opaque so options the frontend does not own pass through untouched. @@ -189,6 +194,30 @@ pub struct NvCreateChatCompletionRequest { } impl NvCreateChatCompletionRequest { + /// Apply sampling controls after validating the client's original values. + /// The resulting canonical options are shared by Rust and Python backends. + pub fn normalize_sampling_controls(&mut self) { + self.inner.temperature = self.get_temperature(); + self.inner.top_p = self.get_top_p(); + // Downstream adapters receive canonical sampling parameters, not a + // backend-specific switch that could be ignored or rejected there. + self.do_sample = None; + } + + fn compatible_top_p(&self) -> Option { + if !is_glm53_model_id(&self.inner.model) { + return self.inner.top_p; + } + match self.inner.top_p { + // Match zai-sdk 0.2.3's synchronous chat-completion client: + // top_p <= 0 becomes 0.01, not the model's default of 0.95. + // Non-finite values still fail validation; they are not JSON numbers. + Some(top_p) if top_p.is_finite() && top_p <= 0.0 => Some(0.01), + None => Some(0.95), + other => other, + } + } + /// Preserve the legacy salt across serialization to Python processors. /// The unsupported-field map is deliberately not serialized; promote only /// the resolved cache namespace, leaving unrelated unknown fields private. @@ -682,12 +711,20 @@ impl AnnotationsProvider for NvCreateChatCompletionRequest { impl OpenAISamplingOptionsProvider for NvCreateChatCompletionRequest { /// Retrieves the temperature parameter for sampling, if set. fn get_temperature(&self) -> Option { - self.inner.temperature + if self.do_sample == Some(false) { + Some(0.0) + } else { + self.inner.temperature + } } /// Retrieves the top-p (nucleus sampling) parameter, if set. fn get_top_p(&self) -> Option { - self.inner.top_p + if self.do_sample == Some(false) { + Some(1.0) + } else { + self.compatible_top_p() + } } /// Retrieves the frequency penalty parameter, if set. @@ -941,7 +978,7 @@ impl NvCreateChatCompletionRequest { // none for stream // none for stream_options validate::validate_temperature(self.inner.temperature)?; - validate::validate_top_p(self.inner.top_p)?; + validate::validate_top_p(self.compatible_top_p())?; let effective_tools = validate::validated_effective_tools(&self.inner)?; validate::validate_tool_choice(&self.inner.tool_choice, Some(effective_tools.as_ref()))?; // none for parallel_tool_calls @@ -1106,6 +1143,100 @@ mod tests { serde_json::from_value(body).expect("Failed to deserialize request") } + #[test] + fn test_do_sample_false_normalizes_sampling_after_validation() { + let mut request = chat_request_with(&json!({ + "do_sample": false, + "temperature": 1.0, + "top_p": 0.95 + })); + assert!(request.unsupported_fields.is_empty()); + assert_eq!(serde_json::to_value(&request).unwrap()["do_sample"], false); + ValidateRequest::validate(&request).expect("do_sample is accepted"); + request.normalize_sampling_controls(); + let sampling = request.extract_sampling_options().unwrap(); + assert_eq!(sampling.temperature, Some(0.0)); + assert_eq!(sampling.top_p, Some(1.0)); + let python_request = serde_json::to_value(&request).unwrap(); + assert!(python_request.get("do_sample").is_none()); + assert_eq!(python_request["temperature"], 0.0); + assert_eq!(python_request["top_p"], 1.0); + + let mut request = chat_request_with(&json!({"do_sample": true, "temperature": 0.7})); + ValidateRequest::validate(&request).expect("do_sample=true is accepted"); + request.normalize_sampling_controls(); + assert_eq!( + request.extract_sampling_options().unwrap().temperature, + Some(0.7) + ); + + let invalid = chat_request_with(&json!({"do_sample": false, "top_p": 0})); + assert!(ValidateRequest::validate(&invalid).is_err()); + } + + #[test] + fn test_do_sample_rejects_non_boolean_values() { + for value in [json!("true"), json!(0), json!({})] { + let request = serde_json::from_value::(json!({ + "model": "glm-5.3", + "messages": [{"role": "user", "content": "hi"}], + "do_sample": value + })); + assert!(request.is_err()); + } + } + + #[test] + fn test_glm53_nonpositive_top_p_is_accepted_and_normalized_for_backends() { + for model in ["glm-5.3", "glm5.3", "zai-org/GLM-5.3"] { + for top_p in [-1.0, -0.01, -0.0, 0.0] { + for do_sample in [None, Some(true), Some(false)] { + let mut request = chat_request_with(&json!({ + "model": model, "top_p": top_p, "temperature": 1.0 + })); + request.do_sample = do_sample; + ValidateRequest::validate(&request).expect("GLM-5.3 accepts top_p<=0"); + let expected = if do_sample == Some(false) { 1.0 } else { 0.01 }; + assert_eq!( + request.extract_sampling_options().unwrap().top_p, + Some(expected) + ); + request.normalize_sampling_controls(); + let python_request = serde_json::to_value(&request).unwrap(); + assert_eq!(python_request["top_p"], expected); + assert_eq!( + python_request["temperature"], + if do_sample == Some(false) { 0.0 } else { 1.0 } + ); + } + } + } + + for top_p in [0.0001, 0.7, 1.0] { + let mut request = chat_request_with(&json!({"model": "glm-5.3", "top_p": top_p})); + ValidateRequest::validate(&request).unwrap(); + request.normalize_sampling_controls(); + assert_eq!(request.inner.top_p, Some(top_p as f32)); + } + let mut default = chat_request_with(&json!({"model": "glm-5.3"})); + default.normalize_sampling_controls(); + assert_eq!(default.inner.top_p, Some(0.95)); + for model in ["test-model", "glm-5.2", "glm-5.3-flash"] { + let request = chat_request_with(&json!({"model": model, "top_p": 0})); + assert!(ValidateRequest::validate(&request).is_err()); + } + } + + #[test] + fn test_glm53_invalid_top_p_is_not_normalized_to_a_valid_value() { + for top_p in [f32::NAN, f32::NEG_INFINITY, f32::INFINITY, 1.1] { + let mut request = chat_request_with(&json!({"model": "glm-5.3"})); + request.inner.top_p = Some(top_p); + assert!(ValidateRequest::validate(&request).is_err()); + assert!(request.extract_sampling_options().is_err()); + } + } + /// Extracts sampling options for `extra` and returns the guided-decoding options it /// produced, failing if extraction rejected the request or engaged nothing. fn guided_for(extra: &serde_json::Value) -> GuidedDecodingOptions { diff --git a/lib/llm/src/protocols/openai/chat_completions/aggregator.rs b/lib/llm/src/protocols/openai/chat_completions/aggregator.rs index eda80c947f52..3132da54cfd0 100644 --- a/lib/llm/src/protocols/openai/chat_completions/aggregator.rs +++ b/lib/llm/src/protocols/openai/chat_completions/aggregator.rs @@ -937,6 +937,13 @@ impl From for dynamo_protocols::types::ChatChoice { } else if !delta.text.is_empty() { // Text-only response (backward compatible) Some(ChatCompletionMessageContent::Text(delta.text)) + } else if finish_reason == Some(dynamo_protocols::types::FinishReason::Length) + && delta.reasoning_content.is_some() + && !has_tool_calls + { + // A reasoning-only turn cut off by max_tokens has an empty answer. + // GLM's chat-completion response represents that answer as "". + Some(ChatCompletionMessageContent::Text(String::new())) } else { None }; @@ -3560,6 +3567,31 @@ mod tests { assert!(json.get("reasoning_content").is_some()); } + #[tokio::test] + async fn test_reasoning_only_length_response_has_empty_content() { + let mut delta = create_reasoning_delta(0, "", "Still reasoning"); + delta.data.as_mut().unwrap().inner.choices[0].finish_reason = + Some(dynamo_protocols::types::FinishReason::Length); + let response = DeltaAggregator::apply( + Box::pin(stream::iter(vec![delta])), + ParsingOptions::default(), + ) + .await + .unwrap(); + let choice = &response.inner.choices[0]; + assert_eq!( + choice.message.content, + Some(ChatCompletionMessageContent::Text(String::new())) + ); + assert_eq!( + choice.finish_reason, + Some(dynamo_protocols::types::FinishReason::Length) + ); + let json = serde_json::to_value(&choice.message).unwrap(); + assert_eq!(json["content"], ""); + assert_eq!(json["reasoning_content"], "Still reasoning"); + } + // --- glm47 truncation recovery tests --- // // Reproduces the drop confirmed on dynamo-parsers 5.1.3: diff --git a/lib/llm/src/protocols/openai/responses/mod.rs b/lib/llm/src/protocols/openai/responses/mod.rs index cf686ae39224..bca0d4dba6b9 100644 --- a/lib/llm/src/protocols/openai/responses/mod.rs +++ b/lib/llm/src/protocols/openai/responses/mod.rs @@ -1033,6 +1033,7 @@ impl TryFrom for NvCreateChatCompletionRequest { chat_template_args: resp.chat_template_args, thinking: None, thinking_token_budget: resp.thinking_token_budget, + do_sample: None, media_io_kwargs: None, return_tokens_as_token_ids: None, unsupported_fields: Default::default(), diff --git a/lib/llm/tests/parallel_tool_call_integration.rs b/lib/llm/tests/parallel_tool_call_integration.rs index dfd0310492a8..e5b4614b3086 100644 --- a/lib/llm/tests/parallel_tool_call_integration.rs +++ b/lib/llm/tests/parallel_tool_call_integration.rs @@ -96,6 +96,7 @@ fn create_mock_chat_completion_request() -> NvCreateChatCompletionRequest { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), } } diff --git a/lib/llm/tests/preprocessor.rs b/lib/llm/tests/preprocessor.rs index 29faf27d913d..733567b4899d 100644 --- a/lib/llm/tests/preprocessor.rs +++ b/lib/llm/tests/preprocessor.rs @@ -265,6 +265,7 @@ impl Request { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), } } @@ -954,6 +955,7 @@ mod context_length_validation { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), } } @@ -1335,6 +1337,7 @@ mod embedding_without_chat_template { return_tokens_as_token_ids: None, unsupported_fields: Default::default(), thinking_token_budget: None, + do_sample: None, } } diff --git a/lib/llm/tests/test_common_ext.rs b/lib/llm/tests/test_common_ext.rs index 8b0c5ddc6968..44ae59d1383c 100644 --- a/lib/llm/tests/test_common_ext.rs +++ b/lib/llm/tests/test_common_ext.rs @@ -72,6 +72,7 @@ fn test_sampling_parameters_include_stop_str_in_output_extraction() { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), }; @@ -343,6 +344,7 @@ fn test_serialization_preserves_structure() { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), }; @@ -398,6 +400,7 @@ fn test_sampling_parameters_extraction() { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), }; diff --git a/lib/llm/tests/test_streaming_usage.rs b/lib/llm/tests/test_streaming_usage.rs index 0b7c95b3ae18..58197a4786f3 100644 --- a/lib/llm/tests/test_streaming_usage.rs +++ b/lib/llm/tests/test_streaming_usage.rs @@ -213,6 +213,7 @@ fn create_chat_request( media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), } } @@ -807,6 +808,7 @@ fn create_nonstreaming_chat_request() -> NvCreateChatCompletionRequest { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), } } diff --git a/lib/llm/tests/tool_choice.rs b/lib/llm/tests/tool_choice.rs index b4225fe26706..3db33bc23ee0 100644 --- a/lib/llm/tests/tool_choice.rs +++ b/lib/llm/tests/tool_choice.rs @@ -73,6 +73,7 @@ fn create_test_request() -> NvCreateChatCompletionRequest { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), } } diff --git a/lib/llm/tests/tool_choice_finish_reasons.rs b/lib/llm/tests/tool_choice_finish_reasons.rs index f6dc184dda66..f3510e569db5 100644 --- a/lib/llm/tests/tool_choice_finish_reasons.rs +++ b/lib/llm/tests/tool_choice_finish_reasons.rs @@ -36,6 +36,7 @@ fn create_test_request() -> NvCreateChatCompletionRequest { media_io_kwargs: None, return_tokens_as_token_ids: None, thinking_token_budget: None, + do_sample: None, unsupported_fields: Default::default(), } } From 3d6f61974db3e95b80e5ab472d12bd9e6d25ae90 Mon Sep 17 00:00:00 2001 From: guojn1 Date: Sat, 17 Oct 2026 22:01:12 +0800 Subject: [PATCH 9/9] fix(dingo): preserve GLM reasoning tokens with tool guidance --- dingo/frontend/sglang_prepost.py | 50 ++++++++++--- .../tests/test_glm_stream_boundaries.py | 71 +++++++++++++++++-- 2 files changed, 109 insertions(+), 12 deletions(-) diff --git a/dingo/frontend/sglang_prepost.py b/dingo/frontend/sglang_prepost.py index 21633fc014e8..8f7b9668572c 100644 --- a/dingo/frontend/sglang_prepost.py +++ b/dingo/frontend/sglang_prepost.py @@ -510,7 +510,7 @@ def _guided_json_start_chars( starts.add("[") return frozenset(starts) - if tool_guided_active: + if tool_guided_active and "json" in guided_decoding: # SGLang's required/named tool constraints use a JSON array root. return frozenset({"["}) @@ -1759,12 +1759,9 @@ def __init__( else None ) self._pending_guided_reasoning_token_count = 0 + self._pending_glm_reasoning_ids: list[int] = [] self._glm_token_reasoning = None - if ( - self._reasoning_parser_name == "glm45" - and reasoning_parser is not None - and self._pending_guided_reasoning_prefix is None - ): + if self._reasoning_parser_name == "glm45" and reasoning_parser is not None: from .glm_reasoning import GlmTokenReasoningSplitter self._glm_token_reasoning = GlmTokenReasoningSplitter.from_tokenizer( @@ -2344,6 +2341,44 @@ def _parse_reasoning_delta( ) return reasoning_text, normal_text, reasoning_tokens + def _split_glm_reasoning_tokens( + self, token_ids: list[int], *, finished: bool + ) -> tuple[str | None, str, int]: + """Resolve bare JSON without losing the token identities of reasoning. + + Structural tags do not enter this prefix probe. JSON constraints may + start with bare JSON or with reasoning; buffer IDs until that choice is + known, then keep using control IDs for the entire reasoning stream. + """ + splitter = self._glm_token_reasoning + assert splitter is not None + if self._pending_guided_reasoning_prefix is not None: + self._pending_glm_reasoning_ids.extend(token_ids) + token_ids = self._pending_glm_reasoning_ids + prefix = self.tokenizer.decode(token_ids, skip_special_tokens=False) + stripped = prefix.lstrip() + array_tail = ( + stripped[1:].lstrip() + if self._guided_tool_array and stripped.startswith("[") + else None + ) + if ( + not finished + and (not stripped or array_tail == "" or prefix.endswith("\ufffd")) + and len(prefix) < self.GUIDED_REASONING_PREFIX_LIMIT + ): + return None, "", 0 + self._pending_guided_reasoning_prefix = None + self._pending_glm_reasoning_ids = [] + if ( + stripped + and stripped[0] in self._guided_json_start_chars + and (array_tail is None or array_tail.startswith(("{", "]"))) + ): + splitter.in_reasoning = False + splitter.ended = True + return splitter.split(token_ids, self._incremental_decode, finished=finished) + def process_output(self, engine_response: dict[str, Any]) -> dict[str, Any] | None: """Process a single engine response chunk into an OpenAI SSE choice dict. @@ -2377,9 +2412,8 @@ def process_output(self, engine_response: dict[str, Any]) -> dict[str, Any] | No glm_reasoning = None if self._glm_token_reasoning is not None: - glm_reasoning = self._glm_token_reasoning.split( + glm_reasoning = self._split_glm_reasoning_tokens( token_ids, - self._incremental_decode, finished=finish_reason is not None, ) delta_text = ( diff --git a/dingo/frontend/tests/test_glm_stream_boundaries.py b/dingo/frontend/tests/test_glm_stream_boundaries.py index 2726f8ed91da..d56855f2d39d 100644 --- a/dingo/frontend/tests/test_glm_stream_boundaries.py +++ b/dingo/frontend/tests/test_glm_stream_boundaries.py @@ -28,7 +28,7 @@ def decode(self, ids, **kwargs): ) -def make_post(): +def make_post(*, guided_decoding=None, tool_choice="auto"): tools = [ { "type": "function", @@ -43,10 +43,11 @@ def make_post(): } ] tool, reason = create_parsers( - {"tools": tools}, + {"tools": tools, "tool_choice": tool_choice}, tool_call_parser_name="glm47", reasoning_parser_name="glm45", force_reasoning=True, + guided_decoding=guided_decoding, ) return SglangStreamingPostProcessor( tokenizer=ByteTokenizer(), @@ -55,6 +56,8 @@ def make_post(): sglang_tools=convert_tools(tools), tool_call_parser_name="glm47", reasoning_parser_name="glm45", + guided_decoding=guided_decoding, + tool_guided_active=guided_decoding is not None, ) @@ -74,7 +77,9 @@ def run(post, ids, batch): @pytest.mark.parametrize("batch", [1, 2, 7, 10000]) @pytest.mark.parametrize("reasoning", ["", "字面量 不结束思考。"]) -def test_literal_tags_and_utf8_before_tool_call(batch, reasoning): +@pytest.mark.parametrize("tool_choice", ["auto", "required"]) +@pytest.mark.parametrize("guided", [False, True]) +def test_literal_tags_and_utf8_before_tool_call(batch, reasoning, tool_choice, guided): argument = "中文 " ids = ( list(reasoning.encode()) @@ -87,7 +92,12 @@ def test_literal_tags_and_utf8_before_tool_call(batch, reasoning): ).encode() ) ) - results = run(make_post(), ids, batch) + # Structural-tag payloads still use GLM tool markers, including for required. + guidance = {"structural_tag": {}} if guided else None + post = make_post( + guided_decoding=guidance, tool_choice=tool_choice if guided else "auto" + ) + results = run(post, ids, batch) assert ( "".join(r["delta"].get("reasoning_content", "") for r in results) == reasoning ) @@ -104,6 +114,59 @@ def test_literal_tags_and_utf8_before_tool_call(batch, reasoning): assert results[-1]["finish_reason"] == "tool_calls" +@pytest.mark.parametrize("batch", [1, 2, 7, 10000]) +@pytest.mark.parametrize( + "reasoning", [None, "", "[check] 引用 和 后继续思考。"] +) +def test_glm_guided_json_keeps_token_boundaries_and_bare_json(batch, reasoning): + post = make_post( + guided_decoding={"json": {"type": "array"}}, tool_choice="required" + ) + argument = "literal " + payload = json.dumps([{"name": "lookup", "parameters": {"query": argument}}]) + # None means bare JSON; empty reasoning still has a real closing token. + ids = [] if reasoning is None else list(reasoning.encode()) + [257] + ids += list((" \n" + payload).encode()) + results = run(post, ids, batch) + assert "".join(r["delta"].get("reasoning_content", "") for r in results) == ( + reasoning or "" + ) + # The tool parser may pass through whitespace before the JSON array. + assert not "".join(r["delta"].get("content", "") for r in results).strip() + calls = [c for r in results for c in r["delta"].get("tool_calls", [])] + assert {c["index"] for c in calls} == {0} + assert sum("id" in c for c in calls) == 1 + assert "".join(c.get("function", {}).get("name", "") for c in calls) == "lookup" + assert json.loads( + "".join(c.get("function", {}).get("arguments", "") for c in calls) + ) == {"query": argument} + assert results[-1]["finish_reason"] == "tool_calls" + + +@pytest.mark.parametrize("batch", [1, 2, 7, 10000]) +@pytest.mark.parametrize("bare", [False, True]) +def test_glm_response_json_preserves_literal_tags(batch, bare): + from sglang.srt.parser.reasoning_parser import ReasoningParser + + post = SglangStreamingPostProcessor( + tokenizer=ByteTokenizer(), + tool_call_parser=None, + reasoning_parser=ReasoningParser(model_type="glm45", force_reasoning=True), + reasoning_parser_name="glm45", + guided_decoding={"json": {"type": "object"}}, + response_format_guided_active=True, + ) + thought = 'The user wants "\\n\\n\\n\\n42". Wait, let me re-read.' + answer = ' {"text":""}' + ids = ([] if bare else list(thought.encode()) + [257]) + list(answer.encode()) + results = run(post, ids, batch) + assert "".join(r["delta"].get("reasoning_content", "") for r in results) == ( + "" if bare else thought + ) + assert "".join(r["delta"].get("content", "") for r in results) == answer + assert results[-1]["finish_reason"] == "stop" + + @pytest.mark.parametrize("batch", [1, 2, 7, 10000]) def test_undeclared_call_is_an_error_not_empty_success(batch): ids = [257, 258] + list(