fix(frontend): opt in GLM malformed history argument compatibility

.github/workflows/dingo-router-tests.yml

@@ -57,6 +57,7 @@ jobs:
           # SKIP_RUNTIME_BUILD=1 跳过 binding 构建与所有 Python 测试，只跑 Rust 测试。
           # 如需跑 Python 测试：先在 runner 预装 libclang（clang-devel/libclang-dev），再改为 '0'。
           SKIP_RUNTIME_BUILD: '1'
+          TEST_RESULTS_DIR: ${{ runner.temp }}/dingo-test-results
         run: |
           cd "${GITHUB_WORKSPACE}"
           QUICK_MODE="${QUICK_MODE}" DOWNLOAD_MODEL="${DOWNLOAD_MODEL}" CARGO_BUILD_JOBS="${CARGO_BUILD_JOBS}" SKIP_RUNTIME_BUILD="${SKIP_RUNTIME_BUILD}" bash tests/scripts/run-all-tests-oneclick.sh
@@ -67,6 +68,6 @@ jobs:
         with:
           name: test-logs-${{ inputs.quick_mode || 'smoke' }}
           path: |
-            /tmp/oneclick-*.log
+            ${{ runner.temp }}/dingo-test-results/
           retention-days: 7
           if-no-files-found: warn

container/deps/stage_aisimulate.py

@@ -1,6 +1,6 @@
 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
-"""Stage the exact AISimulate wheel, building a pinned source if unpublished."""
+"""Stage exact AISimulate wheels or reviewed Rust source for image builds."""
 
 import argparse
 import email
@@ -24,6 +24,76 @@ def run(*args, cwd=None):
     subprocess.run(args, cwd=cwd, check=True)
 
 
+def prepare_source(deps, version, work):
+    """Download only the reviewed commit, verify its digest, and stamp the pin."""
+    rust_version = version.replace(".dev", "-dev.")
+    sources = json.loads((deps / "aisimulate-sources.json").read_text())
+    if version not in sources:
+        raise SystemExit(f"No reviewed source fallback for AISimulate {version}")
+    source = sources[version]
+    commit = source["commit"]
+    if not re.fullmatch(r"[0-9a-f]{40}", commit):
+        raise SystemExit("AISimulate source must be an immutable commit")
+    print(f"Building AISimulate {version} from {commit}", flush=True)
+    archive = work / "source.tar.gz"
+    url = f"https://codeload.github.com/ai-dynamo/aisimulate/tar.gz/{commit}"
+    for attempt in range(3):
+        try:
+            with (
+                urllib.request.urlopen(url, timeout=120) as response,
+                archive.open("wb") as dest,
+            ):
+                shutil.copyfileobj(response, dest)
+            break
+        except OSError:
+            if attempt == 2:
+                raise
+            time.sleep(5 * (attempt + 1))
+    with archive.open("rb") as stream:
+        digest = hashlib.file_digest(stream, "sha256").hexdigest()
+    if digest != source["sha256"]:
+        raise SystemExit("AISimulate source archive checksum mismatch")
+    with tarfile.open(archive) as tar:
+        # Agent/editor links are unrelated to the build and some point
+        # outside the checkout. Preserve package/license links only.
+        members = [
+            m
+            for m in tar.getmembers()
+            if not set(Path(m.name).parts) & {".agents", ".claude"}
+        ]
+        tar.extractall(work, members=members, filter="data")
+    checkout = work / f"aisimulate-{commit}"
+    run(
+        sys.executable,
+        str(checkout / "scripts/apply_dev_version.py"),
+        ".dev" + version.split(".dev", 1)[1],
+        str(checkout),
+    )
+    project = checkout / "python/aisimulate"
+    py_version = tomllib.loads((project / "pyproject.toml").read_text())[
+        "project"
+    ]["version"]
+    core_version = tomllib.loads(
+        (checkout / "crates/core/Cargo.toml").read_text()
+    )["package"]["version"]
+    if (py_version, core_version) != (version, rust_version):
+        raise SystemExit(
+            "Stamped source does not match the required Python/Rust versions"
+        )
+    # Stamping changes only the local workspace package in Cargo.lock.
+    # Keep all transitive dependency versions locked during compilation.
+    lock = checkout / "Cargo.lock"
+    text, count = re.subn(
+        r'(name = "aisimulate-core"\nversion = ")[^"]+("\n)',
+        lambda match: match[1] + rust_version + match[2],
+        lock.read_text(),
+    )
+    if count != 1:
+        raise SystemExit("Expected one AISimulate core package in Cargo.lock")
+    lock.write_text(text)
+    return checkout
+
+
 def stage(root, output):
     deps = root / "container/deps"
     requirements = deps / "requirements.aisimulate.txt"
@@ -79,72 +149,8 @@ def stage(root, output):
             == 0
         )
         if not downloaded:
-            sources = json.loads((deps / "aisimulate-sources.json").read_text())
-            if version not in sources:
-                raise SystemExit(
-                    f"No reviewed source fallback for AISimulate {version}"
-                )
-            source = sources[version]
-            commit = source["commit"]
-            if not re.fullmatch(r"[0-9a-f]{40}", commit):
-                raise SystemExit("AISimulate source must be an immutable commit")
-            print(f"Building AISimulate {version} from {commit}", flush=True)
-            archive = work / "source.tar.gz"
-            url = f"https://codeload.github.com/ai-dynamo/aisimulate/tar.gz/{commit}"
-            for attempt in range(3):
-                try:
-                    with (
-                        urllib.request.urlopen(url, timeout=120) as response,
-                        archive.open("wb") as dest,
-                    ):
-                        shutil.copyfileobj(response, dest)
-                    break
-                except OSError:
-                    if attempt == 2:
-                        raise
-                    time.sleep(5 * (attempt + 1))
-            with archive.open("rb") as stream:
-                digest = hashlib.file_digest(stream, "sha256").hexdigest()
-            if digest != source["sha256"]:
-                raise SystemExit("AISimulate source archive checksum mismatch")
-            with tarfile.open(archive) as tar:
-                # Agent/editor links are unrelated to the build and some point
-                # outside the checkout. Preserve package/license links only.
-                members = [
-                    m
-                    for m in tar.getmembers()
-                    if not set(Path(m.name).parts) & {".agents", ".claude"}
-                ]
-                tar.extractall(work, members=members, filter="data")
-            checkout = work / f"aisimulate-{commit}"
-            run(
-                sys.executable,
-                str(checkout / "scripts/apply_dev_version.py"),
-                ".dev" + version.split(".dev", 1)[1],
-                str(checkout),
-            )
+            checkout = prepare_source(deps, version, work)
             project = checkout / "python/aisimulate"
-            py_version = tomllib.loads((project / "pyproject.toml").read_text())[
-                "project"
-            ]["version"]
-            core_version = tomllib.loads(
-                (checkout / "crates/core/Cargo.toml").read_text()
-            )["package"]["version"]
-            if (py_version, core_version) != (version, rust_version):
-                raise SystemExit(
-                    "Stamped source does not match the required Python/Rust versions"
-                )
-            # Stamping changes only the local workspace package in Cargo.lock.
-            # Keep all transitive dependency versions locked during compilation.
-            lock = checkout / "Cargo.lock"
-            text, count = re.subn(
-                r'(name = "aisimulate-core"\nversion = ")[^"]+("\n)',
-                lambda match: match[1] + rust_version + match[2],
-                lock.read_text(),
-            )
-            if count != 1:
-                raise SystemExit("Expected one AISimulate core package in Cargo.lock")
-            lock.write_text(text)
             # Use the existing manylinux builder toolchain; wheels are consumed
             # by this image family, with the same ABI floor as Dynamo wheels.
             run(
@@ -184,9 +190,64 @@ def stage(root, output):
         print(f"Staged {wheel.name}", flush=True)
 
 
+def stage_rust_source(root, output):
+    """Use reviewed source when the exact nightly is not in the public index.
+
+    This changes only the disposable build checkout: Cargo's manifest pins and
+    all transitive lockfile entries stay fixed. --locked metadata/build verifies
+    that the source has the same dependency requirements as the published crate.
+    """
+    data = tomllib.loads((root / "Cargo.toml").read_text())
+    pin = data["workspace"]["dependencies"]["aisimulate-core"]["version"]
+    if not re.fullmatch(r"=\d+\.\d+\.\d+-dev\.\d+", pin):
+        raise SystemExit("Rust source staging requires an exact reviewed nightly pin")
+    version = pin[1:].replace("-dev.", ".dev")
+    output.mkdir(parents=True, exist_ok=True)
+    checkout = prepare_source(root / "container/deps", version, output)
+    config = root / ".cargo/config.toml"
+    text = config.read_text()
+    if "[patch.crates-io]" in text:
+        raise SystemExit("Refusing to replace existing Cargo source patches")
+    # The runtime and KVBM bindings are independent Cargo workspaces.
+    # Validate every lockfile before writing any of the build-only overrides.
+    updated_locks = []
+    for relative in (
+        "Cargo.lock",
+        "lib/bindings/python/Cargo.lock",
+        "lib/bindings/kvbm/Cargo.lock",
+    ):
+        lock = root / relative
+        lock_text, count = re.subn(
+            r'(name = "aisimulate-core"\nversion = "' + re.escape(pin[1:])
+            + r'"\n)source = "registry\+https://github.com/rust-lang/crates.io-index"\n'
+            + r'checksum = "[0-9a-f]{64}"\n',
+            r"\1",
+            lock.read_text(),
+        )
+        if count != 1:
+            raise SystemExit(f"Expected one locked registry AISimulate core package: {relative}")
+        updated_locks.append((lock, lock_text))
+    config.write_text(
+        text + "\n[patch.crates-io]\naisimulate-core = { path = "
+        + json.dumps(str(checkout / "crates/core")) + " }\n"
+    )
+    for lock, lock_text in updated_locks:
+        lock.write_text(lock_text)
+    licenses = root / "rust-licenses" / ("aisimulate-core-" + pin[1:])
+    licenses.mkdir(parents=True, exist_ok=True)
+    for name in ("LICENSE", "NOTICE"):
+        if (checkout / name).is_file():
+            shutil.copy2(checkout / name, licenses / name)
+    print(f"Staged Rust source: aisimulate-core {pin[1:]}", flush=True)
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--root", type=Path, default=Path("/opt/dynamo"))
     parser.add_argument("--output", type=Path, default=Path("/opt/dynamo/dist"))
+    parser.add_argument("--rust-source-only", action="store_true")
     args = parser.parse_args()
-    stage(args.root.resolve(), args.output.resolve())
+    if args.rust_source_only:
+        stage_rust_source(args.root.resolve(), args.output.resolve())
+    else:
+        stage(args.root.resolve(), args.output.resolve())

container/templates/aws.Dockerfile

@@ -64,5 +64,9 @@ RUN --mount=from=wheel_builder,source=/usr/local/libfabric,target=/tmp/libfabric
     fi
 
 {% if target == "runtime" %}
+{% if framework == "sglang" %}
+USER root
+{% else %}
 USER dynamo
 {% endif %}
+{% endif %}

container/templates/sglang_runtime.Dockerfile

@@ -347,7 +347,9 @@ RUN chmod 755 /opt/dynamo/.launch_screen && \
     else echo "WARNING: no bundled nsys found under /opt/nvidia/nsight-compute"; fi
 {% endif %}
 
-USER dynamo
+# SGLang deployments support startup patches to the upstream engine package.
+# Keep the runtime user root so those files remain writable at startup.
+USER root
 ARG DYNAMO_COMMIT_SHA
 ENV DYNAMO_COMMIT_SHA=${DYNAMO_COMMIT_SHA}
 

container/templates/wheel_builder.Dockerfile

@@ -651,6 +651,14 @@ COPY pyproject.toml README.md LICENSE Cargo.toml Cargo.lock rust-toolchain.toml
 COPY lib/ /opt/dynamo/lib/
 COPY dingo/ /opt/dynamo/dingo/
 
+# The pinned AISimulate nightly can precede public crates.io publication.
+# Resolve it from the reviewed, checksum-pinned source before Cargo metadata;
+# a cached registry on the CI host must not be a prerequisite for image builds.
+COPY container/deps/stage_aisimulate.py container/deps/aisimulate-sources.json /opt/dynamo/container/deps/
+RUN source ${VIRTUAL_ENV}/bin/activate && \
+    python /opt/dynamo/container/deps/stage_aisimulate.py \
+        --rust-source-only --output /opt/build-deps/aisimulate
+
 # Build ai-dingo (pure Python) and ai-dingo-runtime (maturin) wheels.
 # Runtime images supply shared libraries; audit without bundling duplicate copies.
 # SGLang CUDA runtime images own their NIXL stack. Use its C API dynamically:
@@ -677,9 +685,9 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token
     uv build --wheel --out-dir /opt/dynamo/dist && \
     cd /opt/dynamo/lib/bindings/python && \
     if [ "$ENABLE_MEDIA_FFMPEG" = "true" ]; then \
-        maturin build --release --features "full-runtime,media-ffmpeg,kv-indexer,slot-tracker,select-service,mm-routing,ais-forward-pass,request-trace-s3,custom-policy{% if framework == "sglang" and device == "cuda" and target in ("runtime", "dev", "local-dev") %},nixl-dynamic{% endif %}" --auditwheel skip --compatibility linux --out /opt/dynamo/runtime-wheel-unrepaired; \
+        maturin build --release --locked --features "full-runtime,media-ffmpeg,kv-indexer,slot-tracker,select-service,mm-routing,ais-forward-pass,request-trace-s3,custom-policy{% if framework == "sglang" and device == "cuda" and target in ("runtime", "dev", "local-dev") %},nixl-dynamic{% endif %}" --auditwheel skip --compatibility linux --out /opt/dynamo/runtime-wheel-unrepaired; \
     else \
-        maturin build --release --features "full-runtime,kv-indexer,slot-tracker,select-service,mm-routing,ais-forward-pass,request-trace-s3,custom-policy{% if framework == "sglang" and device == "cuda" and target in ("runtime", "dev", "local-dev") %},nixl-dynamic{% endif %}" --auditwheel skip --compatibility linux --out /opt/dynamo/runtime-wheel-unrepaired; \
+        maturin build --release --locked --features "full-runtime,kv-indexer,slot-tracker,select-service,mm-routing,ais-forward-pass,request-trace-s3,custom-policy{% if framework == "sglang" and device == "cuda" and target in ("runtime", "dev", "local-dev") %},nixl-dynamic{% endif %}" --auditwheel skip --compatibility linux --out /opt/dynamo/runtime-wheel-unrepaired; \
     fi && \
     {% if device == "cuda" -%}
     auditwheel repair --exclude 'lib*.so*' \

dingo/frontend/README.md

@@ -39,3 +39,52 @@ reasoning parser configuration.
 Detection uses the discovered model's `config.json`, so serving aliases do not
 affect it. Explicit parser settings take precedence. Unknown model types with
 thinking templates produce a warning requesting explicit parser configuration.
+
+For GLM-5.3 workers using SGLang tool constraints, configure
+`--dyn-reasoning-parser glm45` on `python -m dingo.sglang`. When the native
+reasoning parser is unset, this also defaults SGLang's `--reasoning-parser` to
+`glm45`. An explicitly configured native parser, including `auto`, is preserved.
+Other Dynamo reasoning parsers are not automatically mapped.
+The Dynamo flag selects response reasoning parsing;
+the SGLang flag enables the engine's reasoning gate before applying the tool
+grammar. Frontend reasoning detection does not configure that engine gate.
+Without the engine flag, a tool grammar that excludes `</think>` can prevent
+the model from ending its thinking block and cause repeated reasoning after a
+tool result. The router's reasoning parser alone does not resolve this.
+
+When validating requests without an explicit output limit, leave room for the
+8192-token default completion in the configured engine context window. An
+8192-token context plus a nonempty prompt can otherwise fail SGLang's length
+validation before generation starts.
+
+## GLM-5.3 malformed historical tool arguments
+
+For a GLM-5.3 deployment using `--dyn-chat-processor sglang`, opt in with
+`--glm-history-arguments-models glm-5.3` (environment default:
+`DYN_GLM_HISTORY_ARGUMENTS_MODELS`). This is a comma-separated list of exact,
+case-sensitive canonical served model names, after request alias resolution.
+An empty list, including an explicit empty CLI value, keeps strict validation.
+
+The setting applies to `/v1/chat/completions` and `/v1/responses`, including
+streaming. It permits malformed JSON **syntax** in historical assistant
+`tool_calls[].function.arguments` / Responses `function_call.arguments`.
+Valid JSON arrays, strings, numbers, booleans and null remain invalid; the
+existing acceptance of empty argument strings is unchanged. Other validation
+and newly generated tool-call handling are unchanged. This does not make
+malformed arguments safe to execute: the client must return a tool error and
+let the model retry with valid arguments.
+
+Rust owns admission and receives this deployment setting explicitly through
+PyO3. Python preserves malformed strings and adapts the discovered GLM Jinja
+in memory for the named models, including preprocessing subprocesses. Object
+arguments retain the original `<arg_key>` / `<arg_value>` rendering; strings
+are preserved verbatim after the tool name. No shared model files or
+frontend/worker wire schemas change. Unknown template revisions fail model
+initialization rather than silently applying an unverified transformation.
+An explicitly supplied `--chat-template` is subject to the same check.
+
+Use a dedicated GLM frontend and the same immutable image on frontend/workers
+for initial rollout. Verify valid/invalid history, streaming, Responses and a
+tool-error/retry round trip before production rollout. Removing the option
+and restarting the frontend restores strict admission and the original
+template. Native `sglang serve` does not use this Dingo configuration.

dingo/frontend/frontend_args.py

@@ -97,6 +97,7 @@ class FrontendConfig(RouterConfigBase, KvRouterConfigBase, AisPerfConfigBase):
     enable_streaming_tool_dispatch: bool
     enable_streaming_reasoning_dispatch: bool
     reasoning_field_name: str
+    glm_history_arguments_models: list[str]
     exclude_tools_when_tool_choice_none: bool
     preprocess_workers: int
     tokenizer_backend: str
@@ -107,6 +108,10 @@ class FrontendConfig(RouterConfigBase, KvRouterConfigBase, AisPerfConfigBase):
     _VALID_TOKENIZER_BACKENDS = {"default", "fastokens", "basetenkenizer"}
 
     def validate(self) -> None:
+        if self.glm_history_arguments_models and self.chat_processor != "sglang":
+            raise ValueError(
+                "--glm-history-arguments-models requires --dyn-chat-processor sglang"
+            )
         if self.load_aware:
             self.router_mode = "kv"
         self.apply_router_config()
@@ -603,6 +608,19 @@ def add_arguments(self, parser) -> None:
             ),
             choices=["reasoning_content", "reasoning"],
         )
+        add_argument(
+            g,
+            flag_name="--glm-history-arguments-models",
+            env_var="DYN_GLM_HISTORY_ARGUMENTS_MODELS",
+            default=[],
+            arg_type=lambda value: [
+                name.strip() for name in value.split(",") if name.strip()
+            ],
+            help=(
+                "Comma-separated canonical GLM model names permitting malformed historical "
+                "tool arguments. Adapts the verified GLM Jinja in memory; default is strict."
+            ),
+        )
         # NOTE: This flag also exists in DynamoRuntimeArgGroup (runtime_args.py).
         # Both definitions are needed: runtime_args controls the Rust-native
         # chat template path (oai.rs), while this one controls the Python