From 73c2b637872b8e2091aeed6c5abd433258b3de1b Mon Sep 17 00:00:00 2001 From: luis manrique <166242911+lluisinthedesert@users.noreply.github.com> Date: Tue, 4 Aug 2026 00:27:42 -0700 Subject: [PATCH 01/18] feat: add faithful Tinker serving compatibility gate --- docs/adapter-serving-compatibility.md | 24 +++ scripts/adapter-serving-compat.py | 111 ++++++++++++++ scripts/adapter_serving_compat_test.py | 48 ++++++ scripts/tinker-openai-shim.py | 194 +++++++++++++++++++++++++ scripts/tinker_openai_compat.py | 104 +++++++++++++ scripts/tinker_openai_compat_test.py | 113 ++++++++++++++ 6 files changed, 594 insertions(+) create mode 100644 docs/adapter-serving-compatibility.md create mode 100644 scripts/adapter-serving-compat.py create mode 100644 scripts/adapter_serving_compat_test.py create mode 100644 scripts/tinker-openai-shim.py create mode 100644 scripts/tinker_openai_compat.py create mode 100644 scripts/tinker_openai_compat_test.py diff --git a/docs/adapter-serving-compatibility.md b/docs/adapter-serving-compatibility.md new file mode 100644 index 00000000..3b4ea9bc --- /dev/null +++ b/docs/adapter-serving-compatibility.md @@ -0,0 +1,24 @@ +# Adapter serving compatibility is a pre-training gate + +An exported LoRA file is not proof that a serving runtime can reproduce the +trained model. Training and evaluation plans must name the intended serving +runtime and run `scripts/adapter-serving-compat.py` before paid training and +again against the frozen adapter receipt before evaluation. + +For Nemotron-H, a Tinker adapter trained with `target_modules: "all-linear"` +is not faithfully portable to vLLM. It contains separate Mamba projections and +routed-MoE factors that the vLLM Nemotron-H LoRA surface cannot represent. +Dropping or remapping those weights may test plumbing, but the result is a +different model and must never support a quality claim. + +Use one of two truthful paths: + +1. Serve an existing all-linear checkpoint through Tinker's native sampling + client and put the authenticated OpenAI-compatible shim behind Understudy + Gateway. +2. If vLLM deployment is required, constrain training targets up front to the + supported projection set and hash-bind that choice into the training + manifest. + +The preflight emits a JSON receipt and exits non-zero for incompatible or +unknown combinations. Unknown is deliberately not equivalent to compatible. diff --git a/scripts/adapter-serving-compat.py b/scripts/adapter-serving-compat.py new file mode 100644 index 00000000..aa0fcf43 --- /dev/null +++ b/scripts/adapter-serving-compat.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Fail-closed LoRA training/serving compatibility preflight.""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +NEMOTRON_H_MARKERS = ("NVIDIA-Nemotron-3-Nano", "Nemotron-3-Nano") +VLLM_NEMOTRON_H_TARGETS = frozenset({ + "q_proj", "k_proj", "v_proj", "o_proj", "out_proj", + "up_proj", "down_proj", "lm_head", +}) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _normalize_targets(value: object) -> tuple[list[str], str | None]: + if isinstance(value, str): + return [value], value if value in {"all-linear", "all_linear"} else None + if isinstance(value, list) and all(isinstance(item, str) for item in value): + targets = sorted(set(value)) + wildcard = next((item for item in targets if item in {"all-linear", "all_linear"}), None) + return targets, wildcard + return [], "missing_or_invalid" + + +def assess(config_path: Path, runtime: str, base_model: str) -> dict: + config = json.loads(config_path.read_text(encoding="utf-8")) + targets, wildcard = _normalize_targets(config.get("target_modules")) + receipt = { + "schema_version": 1, + "adapter_config": str(config_path), + "adapter_config_sha256": _sha256(config_path), + "runtime": runtime, + "base_model": base_model, + "training_target_modules": targets, + "compatibility": "unknown", + "faithful": False, + "unsupported_target_modules": [], + "reason": "unsupported runtime or model; no faithful compatibility claim is available", + } + if runtime == "tinker-sampling": + receipt.update( + compatibility="faithful", + faithful=True, + reason="Tinker sampling serves the checkpoint through its native trained-weight path", + ) + return receipt + + is_nemotron_h = any(marker in base_model for marker in NEMOTRON_H_MARKERS) + if runtime != "vllm-nemotron-h" or not is_nemotron_h: + return receipt + if wildcard: + receipt.update( + compatibility="incompatible", + reason=( + "wildcard all-linear training includes Nemotron-H Mamba and routed-MoE " + "targets that vLLM cannot faithfully represent" + ), + unsupported_target_modules=[wildcard], + ) + return receipt + if not targets: + receipt.update( + compatibility="incompatible", + reason="adapter target_modules is missing or invalid; compatibility must fail closed", + unsupported_target_modules=["missing_or_invalid"], + ) + return receipt + unsupported = sorted(set(targets) - VLLM_NEMOTRON_H_TARGETS) + if unsupported: + receipt.update( + compatibility="incompatible", + reason="one or more trained targets are outside the faithful vLLM Nemotron-H surface", + unsupported_target_modules=unsupported, + ) + return receipt + receipt.update( + compatibility="faithful", + faithful=True, + reason="all declared training targets are within the supported vLLM Nemotron-H surface", + ) + return receipt + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--adapter-config", type=Path, required=True) + parser.add_argument("--runtime", choices=("tinker-sampling", "vllm-nemotron-h"), required=True) + parser.add_argument("--base-model", required=True) + parser.add_argument("--receipt", type=Path) + args = parser.parse_args() + result = assess(args.adapter_config, args.runtime, args.base_model) + encoded = json.dumps(result, indent=2, sort_keys=True) + "\n" + if args.receipt: + args.receipt.parent.mkdir(parents=True, exist_ok=True) + args.receipt.write_text(encoded, encoding="utf-8") + print(encoded, end="") + return 0 if result["faithful"] else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/adapter_serving_compat_test.py b/scripts/adapter_serving_compat_test.py new file mode 100644 index 00000000..637e2052 --- /dev/null +++ b/scripts/adapter_serving_compat_test.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Provider-free regression tests for adapter-serving-compat.py.""" +from __future__ import annotations + +import importlib.util +import json +import tempfile +from pathlib import Path + +SCRIPT = Path(__file__).with_name("adapter-serving-compat.py") +SPEC = importlib.util.spec_from_file_location("adapter_serving_compat", SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +assert SPEC.loader +SPEC.loader.exec_module(MODULE) +MODEL = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" + + +def config(root: Path, targets: object) -> Path: + path = root / "adapter_config.json" + path.write_text(json.dumps({"target_modules": targets}) + "\n", encoding="utf-8") + return path + + +def main() -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + all_linear = MODULE.assess(config(root, "all-linear"), "vllm-nemotron-h", MODEL) + assert all_linear["compatibility"] == "incompatible" + assert all_linear["faithful"] is False + supported = MODULE.assess( + config(root, ["q_proj", "k_proj", "v_proj", "o_proj", "up_proj", "down_proj"]), + "vllm-nemotron-h", MODEL, + ) + assert supported["compatibility"] == "faithful" + unsupported = MODULE.assess( + config(root, ["q_proj", "gate_proj", "experts.w1"]), "vllm-nemotron-h", MODEL + ) + assert unsupported["unsupported_target_modules"] == ["experts.w1", "gate_proj"] + native = MODULE.assess(config(root, "all-linear"), "tinker-sampling", MODEL) + assert native["compatibility"] == "faithful" + unknown = MODULE.assess(config(root, ["q_proj"]), "vllm-nemotron-h", "some/other-model") + assert unknown["compatibility"] == "unknown" + assert unknown["faithful"] is False + print("ALL ADAPTER SERVING COMPAT TESTS PASSED") + + +if __name__ == "__main__": + main() diff --git a/scripts/tinker-openai-shim.py b/scripts/tinker-openai-shim.py new file mode 100644 index 00000000..fb9bbcd3 --- /dev/null +++ b/scripts/tinker-openai-shim.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Minimal OpenAI-compatible /v1/chat/completions shim in front of a Tinker +sampling client, so the Node AutomationBench runner can score Tinker base models +and Tinker-trained checkpoints without a dedicated deployment. + +Tinker's `tools=` path raises NotImplementedError, so tool calls are driven +through plain sampling with the model's own renderer, exactly as the RL arms do. + + TINKER_API_KEY=... python scripts/tinker-openai-shim.py \ + --base-model nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 --renderer nemotron3 --port 8099 + +Pass `--model-path tinker://...` to serve a trained checkpoint (a LoRA adapter +over the same base) instead of the base weights; everything else is unchanged, +so base and tuned runs are scored through one identical sampling path. +""" +from __future__ import annotations + +import argparse +import json +import os +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import TimeoutError as FutureTimeoutError +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +# pyqwest (the Rust HTTP backend tinker prefers) carries its own root store and +# rejects otherwise-valid certificates on some Linux hosts. Opt in to httpx's +# system trust store when that happens; the wire protocol is identical. +if os.environ.get("TINKER_DISABLE_PYQWEST") == "1": + import httpx + import tinker._base_client as _tinker_base_client + + _tinker_base_client._default_pyqwest_transport = lambda: httpx.AsyncHTTPTransport(retries=2) + +import tinker +from tinker_cookbook.renderers import get_renderer +from tinker_cookbook.tokenizer_utils import get_tokenizer + +from tinker_openai_compat import ( + bearer_authorized, + build_chat_completion, + normalize_finish_reason, +) + +parser = argparse.ArgumentParser() +model_group = parser.add_mutually_exclusive_group(required=True) +model_group.add_argument("--base-model") +model_group.add_argument("--model-path", help="Tinker sampler-state/checkpoint path returned by save_weights_for_sampler().") +parser.add_argument("--tokenizer-model", help="Base model used for tokenization/rendering when --model-path is selected.") +parser.add_argument("--renderer", required=True) +parser.add_argument("--host", default="127.0.0.1") +parser.add_argument("--port", type=int, default=8099) +parser.add_argument("--max-tokens", type=int, default=512) +parser.add_argument("--max-workers", type=int, default=16, help="in-flight samples; raise it for rollout mining") +args = parser.parse_args() +service_token = os.environ.get("TINKER_SHIM_BEARER_TOKEN") +if args.host not in {"127.0.0.1", "::1", "localhost"} and not service_token: + raise SystemExit("TINKER_SHIM_BEARER_TOKEN is required for non-loopback binds") +request_timeout = 300 +log_path = os.environ.get("TINKER_SHIM_LOG", "/tmp/tinker-openai-shim.log") +log_lock = threading.Lock() +active_lock = threading.Lock() +active_requests = 0 + + +def log_event(event, **fields): + record = {"ts": time.time(), "event": event, **fields} + with log_lock: + with open(log_path, "a", encoding="utf-8") as stream: + stream.write(json.dumps(record) + "\n") + + +service = tinker.ServiceClient(_client_config={"use_pyqwest_transport": False}) +sampler = ( + service.create_sampling_client(model_path=args.model_path) + if args.model_path + else service.create_sampling_client(base_model=args.base_model) +) +tokenizer_model = args.tokenizer_model or args.base_model +if not tokenizer_model: + raise SystemExit("--tokenizer-model is required with --model-path") +renderer = get_renderer(args.renderer, get_tokenizer(tokenizer_model)) +pool = ThreadPoolExecutor(max_workers=args.max_workers) +served_model = args.model_path or args.base_model + + +def sample(messages, temperature, max_tokens): + prompt = renderer.build_generation_prompt([{"role": m["role"], "content": m["content"]} for m in messages]) + params = tinker.types.SamplingParams( + max_tokens=max_tokens, + temperature=temperature, + stop=renderer.get_stop_sequences(), + ) + result = sampler.sample(prompt=prompt, sampling_params=params, num_samples=1).result() + sequence = result.sequences[0] + tokens = sequence.tokens + message, termination = renderer.parse_response(tokens) + content = message.get("content") if isinstance(message, dict) else getattr(message, "content", "") + if isinstance(content, list): + content = "".join(part.get("text", "") for part in content if isinstance(part, dict)) + finish_reason = normalize_finish_reason( + stop_reason=sequence.stop_reason, + termination=getattr(termination, "value", termination), + completion_tokens=len(tokens), + max_tokens=max_tokens, + ) + return content or "", prompt.length, len(tokens), finish_reason + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *_args): # keep the console readable + return + + def _send_json(self, status, payload): + encoded = json.dumps(payload).encode() + self.send_response(status) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(encoded))) + self.end_headers() + try: + self.wfile.write(encoded) + except BrokenPipeError: + pass + + def _authorized(self): + # Loopback remains usable for local provider-parity tests. Any + # network-facing bind is already required to configure a token. + if not service_token: + return True + return bearer_authorized(self.headers.get("authorization"), service_token) + + def do_GET(self): # noqa: N802 - required by BaseHTTPRequestHandler + if not self._authorized(): + self._send_json(401, {"error": {"message": "unauthorized", "type": "authentication_error"}}) + return + if self.path == "/health": + self._send_json(200, {"status": "ok", "model": served_model}) + return + if self.path == "/v1/models": + self._send_json(200, {"object": "list", "data": [{"id": served_model, "object": "model"}]}) + return + self._send_json(404, {"error": {"message": "not found", "type": "invalid_request_error"}}) + + def do_POST(self): # noqa: N802 - required by BaseHTTPRequestHandler + global active_requests + if not self._authorized(): + self._send_json(401, {"error": {"message": "unauthorized", "type": "authentication_error"}}) + return + if self.path != "/v1/chat/completions": + self._send_json(404, {"error": {"message": "not found", "type": "invalid_request_error"}}) + return + request_id = self.headers.get("x-request-id") or str(uuid.uuid4()) + started = time.monotonic() + with active_lock: + active_requests += 1 + in_flight = active_requests + log_event("start", request_id=request_id, in_flight=in_flight) + body = json.loads(self.rfile.read(int(self.headers["content-length"]))) + try: + for attempt in range(2): + try: + content, prompt_tokens, completion_tokens, finish_reason = pool.submit( + sample, + body["messages"], + float(body.get("temperature", 0.0)), + int(body.get("max_tokens", args.max_tokens)), + ).result(timeout=request_timeout) + break + except FutureTimeoutError: + log_event("timeout", request_id=request_id, attempt=attempt + 1, seconds=request_timeout) + if attempt == 1: + raise TimeoutError(f"sampling exceeded {request_timeout}s twice") + log_event("retry", request_id=request_id, attempt=attempt + 2) + payload = build_chat_completion(content, prompt_tokens, completion_tokens, finish_reason) + status = 200 + except Exception as error: # surface upstream failures as HTTP errors + log_event("error", request_id=request_id, error=type(error).__name__, detail=str(error)[:240]) + payload = {"error": f"{type(error).__name__}: {error}"} + status = 500 + finally: + elapsed = time.monotonic() - started + with active_lock: + active_requests -= 1 + in_flight = active_requests + log_event("done", request_id=request_id, elapsed_seconds=round(elapsed, 3), in_flight=in_flight) + self._send_json(status, payload) + + +print(f"tinker shim on {args.host}:{args.port} for {served_model} ({args.renderer})", flush=True) +ThreadingHTTPServer((args.host, args.port), Handler).serve_forever() diff --git a/scripts/tinker_openai_compat.py b/scripts/tinker_openai_compat.py new file mode 100644 index 00000000..b3524e31 --- /dev/null +++ b/scripts/tinker_openai_compat.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Side-effect-free OpenAI-compat helpers for the Tinker sampling shim. + +Importing this module must NOT touch argparse, the network, or a Tinker +ServiceClient, so provider-free unit tests can import it directly. The shim +(``scripts/tinker-openai-shim.py``) imports these two helpers and does the live +sampling itself. + +Finish-reason contract (see tinker.types.StopReason == Literal['length','stop'] +and tinker_cookbook ParseTermination == {'stop_sequence','eos','malformed'}): + + * Prefer the upstream sampler stop_reason when present ('length'->'length', + 'stop'->'stop'). + * Else use the renderer termination: a clean stop ('stop_sequence'/'eos') + -> 'stop'; a 'malformed' (truncated) termination is treated as reason-absent + for the stop/length distinction and falls through to cap inference. + * Only when no upstream reason is present do we infer 'length' from + completion_tokens >= max_tokens; otherwise fall back to 'stop'. + +We never invent OpenAI reasons the upstream cannot justify (e.g. no +'content_filter' unless the upstream actually reports one). +""" +from __future__ import annotations + +import hmac +from typing import Optional + +# Values tinker.types.StopReason may take. +_SAMPLER_STOP = "stop" +_SAMPLER_LENGTH = "length" +# tinker_cookbook renderers.base.ParseTermination values that mean a clean end. +_CLEAN_TERMINATIONS = ("stop_sequence", "eos") + + +def bearer_authorized(header: Optional[str], expected_token: Optional[str]) -> bool: + """Constant-time Bearer-token verification. + + A configured service token is mandatory for non-loopback deployments. The + caller decides whether an unset token is acceptable for its bind address. + """ + if not expected_token or not header or not header.startswith("Bearer "): + return False + supplied = header.removeprefix("Bearer ").strip() + return bool(supplied) and hmac.compare_digest(supplied, expected_token) + + +def normalize_finish_reason( + stop_reason: Optional[str] = None, + termination: Optional[str] = None, + completion_tokens: Optional[int] = None, + max_tokens: Optional[int] = None, +) -> str: + """Return an OpenAI-compatible finish_reason ('stop' or 'length'). + + ``stop_reason`` : upstream sampler StopReason ('stop'/'length') or None. + ``termination`` : renderer ParseTermination value or None. + ``completion_tokens`` / ``max_tokens`` : used only for cap inference when no + upstream reason is available. + """ + # Coerce both signals to plain strings so an enum/StrEnum (or anything with a + # str form) compares correctly; None stays None. + stop = None if stop_reason is None else str(stop_reason) + term = None if termination is None else str(termination) + if stop == _SAMPLER_LENGTH: + return "length" + if stop == _SAMPLER_STOP: + return "stop" + # No authoritative sampler reason: consult a clean renderer termination. + if term in _CLEAN_TERMINATIONS: + return "stop" + # Reason absent (or 'malformed'/truncated): infer length only at the cap. + if ( + completion_tokens is not None + and max_tokens is not None + and completion_tokens >= max_tokens + ): + return "length" + return "stop" + + +def build_chat_completion( + content: str, + prompt_tokens: int, + completion_tokens: int, + finish_reason: str, +) -> dict: + """Build the /v1/chat/completions response body. + + finish_reason is a required positional argument: constructing the choice + object without a defined finish_reason is impossible here, which is exactly + the undefined-variable (NameError) failure this module exists to prevent. + """ + return { + "choices": [ + { + "message": {"role": "assistant", "content": content}, + "finish_reason": finish_reason, + } + ], + "usage": { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + }, + } diff --git a/scripts/tinker_openai_compat_test.py b/scripts/tinker_openai_compat_test.py new file mode 100644 index 00000000..5880511c --- /dev/null +++ b/scripts/tinker_openai_compat_test.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Provider-free regression for the Tinker shim's OpenAI-compat helpers. + +Runs with plain system python3 (no tinker, no network): + python3 scripts/tinker_openai_compat_test.py + +Proves the finish-reason contract and that building the response body cannot +recur the original undefined-`finish_reason` NameError. +""" +from enum import Enum + +from tinker_openai_compat import ( + bearer_authorized, + build_chat_completion, + normalize_finish_reason, +) + + +class _ParseTermination(str, Enum): + """Shape-compatible stand-in for tinker_cookbook ParseTermination.""" + STOP_SEQUENCE = "stop_sequence" + EOS = "eos" + MALFORMED = "malformed" + + +def check(name, cond): + if not cond: + raise AssertionError(f"FAIL: {name}") + print(f" ok: {name}") + + +def test_upstream_stop(): + check("upstream stop -> stop", + normalize_finish_reason(stop_reason="stop", completion_tokens=384, max_tokens=384) == "stop") + + +def test_upstream_length(): + # upstream length wins even below the cap + check("upstream length -> length", + normalize_finish_reason(stop_reason="length", completion_tokens=10, max_tokens=384) == "length") + + +def test_absent_at_cap_infers_length(): + check("absent + at cap -> length", + normalize_finish_reason(stop_reason=None, termination=None, completion_tokens=384, max_tokens=384) == "length") + + +def test_absent_under_cap_falls_back_stop(): + check("absent + under cap -> stop", + normalize_finish_reason(stop_reason=None, termination=None, completion_tokens=12, max_tokens=384) == "stop") + + +def test_enum_shaped_terminations_map_to_stop(): + # exactly how the shim passes it: getattr(term, "value", term) + for term in (_ParseTermination.EOS, _ParseTermination.STOP_SEQUENCE): + passed = getattr(term, "value", term) + check(f"enum termination {term.value!r} -> stop", + normalize_finish_reason(termination=passed, completion_tokens=5, max_tokens=384) == "stop") + + +def test_unknown_non_clean_termination_falls_through_to_cap(): + # 'malformed' (or any non-clean) is NOT treated as a stop/length signal; + # it falls through to cap inference. + passed = getattr(_ParseTermination.MALFORMED, "value", _ParseTermination.MALFORMED) + check("malformed + at cap -> length (cap inference)", + normalize_finish_reason(termination=passed, completion_tokens=384, max_tokens=384) == "length") + check("malformed + under cap -> stop (fallback)", + normalize_finish_reason(termination=passed, completion_tokens=7, max_tokens=384) == "stop") + + +def test_payload_requires_defined_finish_reason(): + # The exact code path that once raised NameError: build the choice object + # with a normalized finish_reason. It is a required positional arg, so an + # undefined variable cannot silently slip through. + fr = normalize_finish_reason(stop_reason="stop", completion_tokens=3, max_tokens=384) + payload = build_chat_completion("hello", 100, 3, fr) + check("payload choices[0].finish_reason present", payload["choices"][0]["finish_reason"] == "stop") + check("payload content propagated", payload["choices"][0]["message"]["content"] == "hello") + check("payload usage propagated", + payload["usage"] == {"prompt_tokens": 100, "completion_tokens": 3}) + + fr_len = normalize_finish_reason(stop_reason="length", completion_tokens=384, max_tokens=384) + payload_len = build_chat_completion("", 100, 384, fr_len) + check("length propagates into payload", payload_len["choices"][0]["finish_reason"] == "length") + + +def test_bearer_auth_is_fail_closed(): + check("missing expected token rejects", bearer_authorized("Bearer x", None) is False) + check("missing header rejects", bearer_authorized(None, "secret") is False) + check("wrong scheme rejects", bearer_authorized("Basic secret", "secret") is False) + check("wrong token rejects", bearer_authorized("Bearer wrong", "secret") is False) + check("matching token authorizes", bearer_authorized("Bearer secret", "secret") is True) + + +def main(): + tests = [ + test_upstream_stop, + test_upstream_length, + test_absent_at_cap_infers_length, + test_absent_under_cap_falls_back_stop, + test_enum_shaped_terminations_map_to_stop, + test_unknown_non_clean_termination_falls_through_to_cap, + test_payload_requires_defined_finish_reason, + test_bearer_auth_is_fail_closed, + ] + for t in tests: + print(t.__name__) + t() + print(f"\nALL {len(tests)} SHIM COMPAT TESTS PASSED") + + +if __name__ == "__main__": + main() From f8d07a95dc502a4ebde516208333b0064f787bdf Mon Sep 17 00:00:00 2001 From: luis manrique <166242911+lluisinthedesert@users.noreply.github.com> Date: Tue, 4 Aug 2026 00:31:40 -0700 Subject: [PATCH 02/18] feat: serve multiple Tinker checkpoints privately on Modal --- scripts/modal-tinker-openai-shim.py | 72 +++++++++++++++++++++++++++++ scripts/tinker-openai-shim.py | 51 +++++++++++++++----- 2 files changed, 111 insertions(+), 12 deletions(-) create mode 100644 scripts/modal-tinker-openai-shim.py diff --git a/scripts/modal-tinker-openai-shim.py b/scripts/modal-tinker-openai-shim.py new file mode 100644 index 00000000..af9be71f --- /dev/null +++ b/scripts/modal-tinker-openai-shim.py @@ -0,0 +1,72 @@ +"""Private multi-checkpoint Tinker sampling bridge for Understudy Gateway. + +Deploy with a Modal secret named understudy-tinker-serving containing +TINKER_API_KEY, TINKER_SHIM_BEARER_TOKEN, and TINKER_MODEL_REGISTRY_JSON. +""" +from __future__ import annotations + +import json +import os +import subprocess +from pathlib import Path + +import modal + +APP_NAME = "understudy-tinker-checkpoint-serving" +PORT = 8099 +SECRET_NAME = "understudy-tinker-serving" +BASE_MODEL = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" +TINKER_COMMIT = "3eb9e87d52efacede992931b1bb51d000b0c70ed" +COOKBOOK_COMMIT = "0b5c01eaee49bdb0d476f4f383e1c0fb9aced590" + +app = modal.App(APP_NAME) +image = ( + modal.Image.debian_slim(python_version="3.11") + .apt_install("git") + .pip_install( + f"tinker @ git+https://github.com/thinking-machines-lab/tinker.git@{TINKER_COMMIT}", + f"tinker-cookbook @ git+https://github.com/thinking-machines-lab/tinker-cookbook.git@{COOKBOOK_COMMIT}", + "httpx>=0.27,<1", + ) + .add_local_file("scripts/tinker-openai-shim.py", "/opt/understudy/tinker-openai-shim.py") + .add_local_file("scripts/tinker_openai_compat.py", "/opt/understudy/tinker_openai_compat.py") +) + + +@app.function( + image=image, + secrets=[modal.Secret.from_name(SECRET_NAME)], + timeout=60 * 60, + scaledown_window=300, + max_containers=4, +) +@modal.concurrent(max_inputs=64) +@modal.web_server(PORT, startup_timeout=10 * 60) +def serve() -> None: + registry = json.loads(os.environ["TINKER_MODEL_REGISTRY_JSON"]) + if not isinstance(registry, dict) or not registry: + raise RuntimeError("TINKER_MODEL_REGISTRY_JSON must be a non-empty object") + registry_path = Path("/tmp/tinker-model-registry.json") + registry_path.write_text(json.dumps(registry, sort_keys=True), encoding="utf-8") + registry_path.chmod(0o600) + subprocess.Popen( + [ + "python", + "/opt/understudy/tinker-openai-shim.py", + "--model-registry-file", + str(registry_path), + "--tokenizer-model", + BASE_MODEL, + "--renderer", + "nemotron3", + "--host", + "0.0.0.0", + "--port", + str(PORT), + "--max-workers", + "64", + "--max-tokens", + "2048", + ], + env=os.environ.copy(), + ) diff --git a/scripts/tinker-openai-shim.py b/scripts/tinker-openai-shim.py index fb9bbcd3..90508dc7 100644 --- a/scripts/tinker-openai-shim.py +++ b/scripts/tinker-openai-shim.py @@ -48,6 +48,10 @@ model_group = parser.add_mutually_exclusive_group(required=True) model_group.add_argument("--base-model") model_group.add_argument("--model-path", help="Tinker sampler-state/checkpoint path returned by save_weights_for_sampler().") +model_group.add_argument( + "--model-registry-file", + help="JSON object mapping public model aliases to Tinker checkpoint paths.", +) parser.add_argument("--tokenizer-model", help="Base model used for tokenization/rendering when --model-path is selected.") parser.add_argument("--renderer", required=True) parser.add_argument("--host", default="127.0.0.1") @@ -73,27 +77,37 @@ def log_event(event, **fields): service = tinker.ServiceClient(_client_config={"use_pyqwest_transport": False}) -sampler = ( - service.create_sampling_client(model_path=args.model_path) - if args.model_path - else service.create_sampling_client(base_model=args.base_model) -) +if args.model_registry_file: + registry = json.loads(open(args.model_registry_file, encoding="utf-8").read()) + if not isinstance(registry, dict) or not registry or not all( + isinstance(alias, str) and alias and isinstance(path, str) and path + for alias, path in registry.items() + ): + raise SystemExit("--model-registry-file must contain a non-empty string-to-string JSON object") + samplers = { + alias: service.create_sampling_client(model_path=path) + for alias, path in registry.items() + } +elif args.model_path: + samplers = {args.model_path: service.create_sampling_client(model_path=args.model_path)} +else: + samplers = {args.base_model: service.create_sampling_client(base_model=args.base_model)} tokenizer_model = args.tokenizer_model or args.base_model if not tokenizer_model: - raise SystemExit("--tokenizer-model is required with --model-path") + raise SystemExit("--tokenizer-model is required with checkpoint paths") renderer = get_renderer(args.renderer, get_tokenizer(tokenizer_model)) pool = ThreadPoolExecutor(max_workers=args.max_workers) -served_model = args.model_path or args.base_model +served_models = sorted(samplers) -def sample(messages, temperature, max_tokens): +def sample(model, messages, temperature, max_tokens): prompt = renderer.build_generation_prompt([{"role": m["role"], "content": m["content"]} for m in messages]) params = tinker.types.SamplingParams( max_tokens=max_tokens, temperature=temperature, stop=renderer.get_stop_sequences(), ) - result = sampler.sample(prompt=prompt, sampling_params=params, num_samples=1).result() + result = samplers[model].sample(prompt=prompt, sampling_params=params, num_samples=1).result() sequence = result.sequences[0] tokens = sequence.tokens message, termination = renderer.parse_response(tokens) @@ -138,10 +152,13 @@ def do_GET(self): # noqa: N802 - required by BaseHTTPRequestHandler self._send_json(401, {"error": {"message": "unauthorized", "type": "authentication_error"}}) return if self.path == "/health": - self._send_json(200, {"status": "ok", "model": served_model}) + self._send_json(200, {"status": "ok", "models": served_models}) return if self.path == "/v1/models": - self._send_json(200, {"object": "list", "data": [{"id": served_model, "object": "model"}]}) + self._send_json( + 200, + {"object": "list", "data": [{"id": model, "object": "model"} for model in served_models]}, + ) return self._send_json(404, {"error": {"message": "not found", "type": "invalid_request_error"}}) @@ -161,10 +178,20 @@ def do_POST(self): # noqa: N802 - required by BaseHTTPRequestHandler log_event("start", request_id=request_id, in_flight=in_flight) body = json.loads(self.rfile.read(int(self.headers["content-length"]))) try: + requested_model = body.get("model") + if requested_model is None and len(served_models) == 1: + requested_model = served_models[0] + if requested_model not in samplers: + self._send_json( + 400, + {"error": {"message": "unknown model", "type": "invalid_request_error"}}, + ) + return for attempt in range(2): try: content, prompt_tokens, completion_tokens, finish_reason = pool.submit( sample, + requested_model, body["messages"], float(body.get("temperature", 0.0)), int(body.get("max_tokens", args.max_tokens)), @@ -190,5 +217,5 @@ def do_POST(self): # noqa: N802 - required by BaseHTTPRequestHandler self._send_json(status, payload) -print(f"tinker shim on {args.host}:{args.port} for {served_model} ({args.renderer})", flush=True) +print(f"tinker shim on {args.host}:{args.port} for {served_models} ({args.renderer})", flush=True) ThreadingHTTPServer((args.host, args.port), Handler).serve_forever() From 618a894e97f55745573ce9640ef3dc803d4b7770 Mon Sep 17 00:00:00 2001 From: luis manrique <166242911+lluisinthedesert@users.noreply.github.com> Date: Tue, 4 Aug 2026 00:40:43 -0700 Subject: [PATCH 03/18] fix: enforce Modal proxy auth on Tinker serving --- scripts/modal-tinker-openai-shim.py | 6 ++++-- scripts/tinker-openai-shim.py | 15 +++++++++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/scripts/modal-tinker-openai-shim.py b/scripts/modal-tinker-openai-shim.py index af9be71f..d9343f2c 100644 --- a/scripts/modal-tinker-openai-shim.py +++ b/scripts/modal-tinker-openai-shim.py @@ -1,7 +1,8 @@ """Private multi-checkpoint Tinker sampling bridge for Understudy Gateway. Deploy with a Modal secret named understudy-tinker-serving containing -TINKER_API_KEY, TINKER_SHIM_BEARER_TOKEN, and TINKER_MODEL_REGISTRY_JSON. +TINKER_API_KEY and TINKER_MODEL_REGISTRY_JSON. Modal proxy authentication is +required before requests reach the shim. """ from __future__ import annotations @@ -41,7 +42,7 @@ max_containers=4, ) @modal.concurrent(max_inputs=64) -@modal.web_server(PORT, startup_timeout=10 * 60) +@modal.web_server(PORT, startup_timeout=10 * 60, requires_proxy_auth=True) def serve() -> None: registry = json.loads(os.environ["TINKER_MODEL_REGISTRY_JSON"]) if not isinstance(registry, dict) or not registry: @@ -61,6 +62,7 @@ def serve() -> None: "nemotron3", "--host", "0.0.0.0", + "--trusted-proxy-auth", "--port", str(PORT), "--max-workers", diff --git a/scripts/tinker-openai-shim.py b/scripts/tinker-openai-shim.py index 90508dc7..8a7776b9 100644 --- a/scripts/tinker-openai-shim.py +++ b/scripts/tinker-openai-shim.py @@ -55,13 +55,24 @@ parser.add_argument("--tokenizer-model", help="Base model used for tokenization/rendering when --model-path is selected.") parser.add_argument("--renderer", required=True) parser.add_argument("--host", default="127.0.0.1") +parser.add_argument( + "--trusted-proxy-auth", + action="store_true", + help="Allow a non-loopback bind without app bearer auth only behind an authenticated reverse proxy.", +) parser.add_argument("--port", type=int, default=8099) parser.add_argument("--max-tokens", type=int, default=512) parser.add_argument("--max-workers", type=int, default=16, help="in-flight samples; raise it for rollout mining") args = parser.parse_args() service_token = os.environ.get("TINKER_SHIM_BEARER_TOKEN") -if args.host not in {"127.0.0.1", "::1", "localhost"} and not service_token: - raise SystemExit("TINKER_SHIM_BEARER_TOKEN is required for non-loopback binds") +if ( + args.host not in {"127.0.0.1", "::1", "localhost"} + and not service_token + and not args.trusted_proxy_auth +): + raise SystemExit( + "TINKER_SHIM_BEARER_TOKEN or --trusted-proxy-auth is required for non-loopback binds" + ) request_timeout = 300 log_path = os.environ.get("TINKER_SHIM_LOG", "/tmp/tinker-openai-shim.log") log_lock = threading.Lock() From d58ca88c16e9ba872c2bfbcf62099f314b0e2bb0 Mon Sep 17 00:00:00 2001 From: luis manrique <166242911+lluisinthedesert@users.noreply.github.com> Date: Tue, 4 Aug 2026 00:49:36 -0700 Subject: [PATCH 04/18] fix: pin Tinker serving to training versions --- scripts/modal-tinker-openai-shim.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/modal-tinker-openai-shim.py b/scripts/modal-tinker-openai-shim.py index d9343f2c..1f38f1fb 100644 --- a/scripts/modal-tinker-openai-shim.py +++ b/scripts/modal-tinker-openai-shim.py @@ -17,16 +17,16 @@ PORT = 8099 SECRET_NAME = "understudy-tinker-serving" BASE_MODEL = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" -TINKER_COMMIT = "3eb9e87d52efacede992931b1bb51d000b0c70ed" -COOKBOOK_COMMIT = "0b5c01eaee49bdb0d476f4f383e1c0fb9aced590" +TINKER_VERSION = "0.24.0" +COOKBOOK_VERSION = "0.5.3" app = modal.App(APP_NAME) image = ( modal.Image.debian_slim(python_version="3.11") .apt_install("git") .pip_install( - f"tinker @ git+https://github.com/thinking-machines-lab/tinker.git@{TINKER_COMMIT}", - f"tinker-cookbook @ git+https://github.com/thinking-machines-lab/tinker-cookbook.git@{COOKBOOK_COMMIT}", + f"tinker=={TINKER_VERSION}", + f"tinker-cookbook=={COOKBOOK_VERSION}", "httpx>=0.27,<1", ) .add_local_file("scripts/tinker-openai-shim.py", "/opt/understudy/tinker-openai-shim.py") From 1fea71a0315fd6d99d18b387292f51fe78b9a018 Mon Sep 17 00:00:00 2001 From: luis manrique <166242911+lluisinthedesert@users.noreply.github.com> Date: Tue, 4 Aug 2026 01:17:08 -0700 Subject: [PATCH 05/18] fix: preserve Tinker tool-call roundtrips --- scripts/modal-tinker-openai-shim.py | 3 +- scripts/tinker-openai-shim.py | 24 ++++++--- scripts/tinker_openai_compat.py | 40 ++++++++++++-- scripts/tinker_openai_compat_test.py | 41 ++++++++++++++- scripts/tinker_renderer_compat.py | 72 ++++++++++++++++++++++++++ scripts/tinker_renderer_compat_test.py | 70 +++++++++++++++++++++++++ tests/tinker_openai_compat_test.py | 42 +++++++++++++++ 7 files changed, 278 insertions(+), 14 deletions(-) create mode 100644 scripts/tinker_renderer_compat.py create mode 100644 scripts/tinker_renderer_compat_test.py create mode 100644 tests/tinker_openai_compat_test.py diff --git a/scripts/modal-tinker-openai-shim.py b/scripts/modal-tinker-openai-shim.py index 1f38f1fb..bed14e0a 100644 --- a/scripts/modal-tinker-openai-shim.py +++ b/scripts/modal-tinker-openai-shim.py @@ -31,6 +31,7 @@ ) .add_local_file("scripts/tinker-openai-shim.py", "/opt/understudy/tinker-openai-shim.py") .add_local_file("scripts/tinker_openai_compat.py", "/opt/understudy/tinker_openai_compat.py") + .add_local_file("scripts/tinker_renderer_compat.py", "/opt/understudy/tinker_renderer_compat.py") ) @@ -59,7 +60,7 @@ def serve() -> None: "--tokenizer-model", BASE_MODEL, "--renderer", - "nemotron3", + "nemotron3_disable_thinking", "--host", "0.0.0.0", "--trusted-proxy-auth", diff --git a/scripts/tinker-openai-shim.py b/scripts/tinker-openai-shim.py index 8a7776b9..d4b70631 100644 --- a/scripts/tinker-openai-shim.py +++ b/scripts/tinker-openai-shim.py @@ -41,8 +41,10 @@ from tinker_openai_compat import ( bearer_authorized, build_chat_completion, + normalize_assistant_message, normalize_finish_reason, ) +from tinker_renderer_compat import renderer_messages, renderer_tools parser = argparse.ArgumentParser() model_group = parser.add_mutually_exclusive_group(required=True) @@ -111,8 +113,14 @@ def log_event(event, **fields): served_models = sorted(samplers) -def sample(model, messages, temperature, max_tokens): - prompt = renderer.build_generation_prompt([{"role": m["role"], "content": m["content"]} for m in messages]) +def sample(model, messages, tools, temperature, max_tokens): + system_prompt, conversation = renderer_messages(messages) + tool_specs = renderer_tools(tools) + prefix = renderer.create_conversation_prefix_with_tools( + tool_specs, + system_prompt=system_prompt, + ) + prompt = renderer.build_generation_prompt([*prefix, *conversation]) params = tinker.types.SamplingParams( max_tokens=max_tokens, temperature=temperature, @@ -122,16 +130,14 @@ def sample(model, messages, temperature, max_tokens): sequence = result.sequences[0] tokens = sequence.tokens message, termination = renderer.parse_response(tokens) - content = message.get("content") if isinstance(message, dict) else getattr(message, "content", "") - if isinstance(content, list): - content = "".join(part.get("text", "") for part in content if isinstance(part, dict)) + openai_message = renderer.to_openai_message(message) finish_reason = normalize_finish_reason( stop_reason=sequence.stop_reason, termination=getattr(termination, "value", termination), completion_tokens=len(tokens), max_tokens=max_tokens, ) - return content or "", prompt.length, len(tokens), finish_reason + return openai_message, prompt.length, len(tokens), finish_reason class Handler(BaseHTTPRequestHandler): @@ -200,10 +206,11 @@ def do_POST(self): # noqa: N802 - required by BaseHTTPRequestHandler return for attempt in range(2): try: - content, prompt_tokens, completion_tokens, finish_reason = pool.submit( + message, prompt_tokens, completion_tokens, finish_reason = pool.submit( sample, requested_model, body["messages"], + body.get("tools") or [], float(body.get("temperature", 0.0)), int(body.get("max_tokens", args.max_tokens)), ).result(timeout=request_timeout) @@ -213,7 +220,8 @@ def do_POST(self): # noqa: N802 - required by BaseHTTPRequestHandler if attempt == 1: raise TimeoutError(f"sampling exceeded {request_timeout}s twice") log_event("retry", request_id=request_id, attempt=attempt + 2) - payload = build_chat_completion(content, prompt_tokens, completion_tokens, finish_reason) + message = normalize_assistant_message(message, request_id) + payload = build_chat_completion(message, prompt_tokens, completion_tokens, finish_reason) status = 200 except Exception as error: # surface upstream failures as HTTP errors log_event("error", request_id=request_id, error=type(error).__name__, detail=str(error)[:240]) diff --git a/scripts/tinker_openai_compat.py b/scripts/tinker_openai_compat.py index b3524e31..353a5353 100644 --- a/scripts/tinker_openai_compat.py +++ b/scripts/tinker_openai_compat.py @@ -23,7 +23,9 @@ from __future__ import annotations import hmac -from typing import Optional +import json +import uuid +from typing import Any, Optional # Values tinker.types.StopReason may take. _SAMPLER_STOP = "stop" @@ -32,6 +34,38 @@ _CLEAN_TERMINATIONS = ("stop_sequence", "eos") +def normalize_assistant_message(message: dict[str, Any], request_id: str) -> dict[str, Any]: + """Return strict OpenAI assistant shape with stable tool-call IDs/arguments.""" + normalized = {"role": "assistant", "content": message.get("content") or ""} + raw_calls = message.get("tool_calls") or [] + if raw_calls: + calls = [] + for index, raw in enumerate(raw_calls): + function = raw.get("function") or {} + arguments = function.get("arguments", "{}") + if isinstance(arguments, dict): + arguments = json.dumps(arguments, sort_keys=True) + if not isinstance(arguments, str): + raise ValueError("parsed tool arguments must be a JSON string or object") + json.loads(arguments) + call_id = raw.get("id") or ( + "call_" + + uuid.uuid5( + uuid.NAMESPACE_URL, + f"understudy:tinker:{request_id}:{index}:{function.get('name')}", + ).hex + ) + calls.append( + { + "type": "function", + "id": call_id, + "function": {"name": function["name"], "arguments": arguments}, + } + ) + normalized["tool_calls"] = calls + return normalized + + def bearer_authorized(header: Optional[str], expected_token: Optional[str]) -> bool: """Constant-time Bearer-token verification. @@ -79,7 +113,7 @@ def normalize_finish_reason( def build_chat_completion( - content: str, + message: dict[str, Any], prompt_tokens: int, completion_tokens: int, finish_reason: str, @@ -93,7 +127,7 @@ def build_chat_completion( return { "choices": [ { - "message": {"role": "assistant", "content": content}, + "message": message, "finish_reason": finish_reason, } ], diff --git a/scripts/tinker_openai_compat_test.py b/scripts/tinker_openai_compat_test.py index 5880511c..b024b9d9 100644 --- a/scripts/tinker_openai_compat_test.py +++ b/scripts/tinker_openai_compat_test.py @@ -12,6 +12,7 @@ from tinker_openai_compat import ( bearer_authorized, build_chat_completion, + normalize_assistant_message, normalize_finish_reason, ) @@ -73,16 +74,31 @@ def test_payload_requires_defined_finish_reason(): # with a normalized finish_reason. It is a required positional arg, so an # undefined variable cannot silently slip through. fr = normalize_finish_reason(stop_reason="stop", completion_tokens=3, max_tokens=384) - payload = build_chat_completion("hello", 100, 3, fr) + payload = build_chat_completion({"role": "assistant", "content": "hello"}, 100, 3, fr) check("payload choices[0].finish_reason present", payload["choices"][0]["finish_reason"] == "stop") check("payload content propagated", payload["choices"][0]["message"]["content"] == "hello") check("payload usage propagated", payload["usage"] == {"prompt_tokens": 100, "completion_tokens": 3}) fr_len = normalize_finish_reason(stop_reason="length", completion_tokens=384, max_tokens=384) - payload_len = build_chat_completion("", 100, 384, fr_len) + payload_len = build_chat_completion({"role": "assistant", "content": ""}, 100, 384, fr_len) check("length propagates into payload", payload_len["choices"][0]["finish_reason"] == "length") + tool_message = { + "role": "assistant", + "content": "", + "tool_calls": [{ + "type": "function", + "id": "call_1", + "function": {"name": "create-task", "arguments": '{"title":"A"}'}, + }], + } + payload_tool = build_chat_completion(tool_message, 100, 20, "stop") + check( + "tool calls propagate without text flattening", + payload_tool["choices"][0]["message"] == tool_message, + ) + def test_bearer_auth_is_fail_closed(): check("missing expected token rejects", bearer_authorized("Bearer x", None) is False) @@ -92,6 +108,26 @@ def test_bearer_auth_is_fail_closed(): check("matching token authorizes", bearer_authorized("Bearer secret", "secret") is True) +def test_parsed_tool_call_gets_stable_openai_shape(): + parsed = { + "role": "assistant", + "content": "", + "tool_calls": [{ + "type": "function", + "id": None, + "function": {"name": "create-task", "arguments": {"title": "A"}}, + }], + } + first = normalize_assistant_message(parsed, "request-1") + second = normalize_assistant_message(parsed, "request-1") + check("missing tool id synthesized", first["tool_calls"][0]["id"].startswith("call_")) + check("synthesized tool id stable", first == second) + check( + "tool arguments serialized as JSON string", + first["tool_calls"][0]["function"]["arguments"] == '{"title": "A"}', + ) + + def main(): tests = [ test_upstream_stop, @@ -102,6 +138,7 @@ def main(): test_unknown_non_clean_termination_falls_through_to_cap, test_payload_requires_defined_finish_reason, test_bearer_auth_is_fail_closed, + test_parsed_tool_call_gets_stable_openai_shape, ] for t in tests: print(t.__name__) diff --git a/scripts/tinker_renderer_compat.py b/scripts/tinker_renderer_compat.py new file mode 100644 index 00000000..47f220cc --- /dev/null +++ b/scripts/tinker_renderer_compat.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""OpenAI-to-Tinker renderer conversion for faithful multi-turn tool use.""" +from __future__ import annotations + +import json + +from tinker_cookbook.renderers.base import Message, ToolCall, ToolSpec + + +def renderer_messages(messages: list[dict]) -> tuple[str, list[Message]]: + """Losslessly convert OpenAI history into the cookbook renderer contract.""" + system_parts: list[str] = [] + converted: list[Message] = [] + for raw in messages: + role = raw.get("role") + content = raw.get("content") + if content is None: + content = "" + if not isinstance(content, str): + raise ValueError("only string/null message content is supported") + if role == "system": + system_parts.append(content) + continue + if role not in {"user", "assistant", "tool"}: + raise ValueError(f"unsupported message role: {role!r}") + message: Message = {"role": role, "content": content} + if role == "assistant" and raw.get("tool_calls"): + calls: list[ToolCall] = [] + for raw_call in raw["tool_calls"]: + function = raw_call.get("function") or {} + arguments = function.get("arguments", "{}") + if isinstance(arguments, dict): + arguments = json.dumps(arguments, sort_keys=True) + if not isinstance(arguments, str): + raise ValueError("tool call arguments must be a JSON string or object") + json.loads(arguments) + calls.append( + ToolCall( + id=raw_call.get("id"), + function=ToolCall.FunctionBody( + name=function["name"], + arguments=arguments, + ), + ) + ) + message["tool_calls"] = calls + if role == "tool": + if not raw.get("tool_call_id"): + raise ValueError("tool result requires tool_call_id") + message["tool_call_id"] = raw["tool_call_id"] + if raw.get("name"): + message["name"] = raw["name"] + converted.append(message) + return "\n\n".join(system_parts), converted + + +def renderer_tools(tools: list[dict]) -> list[ToolSpec]: + """Convert OpenAI function-tool declarations to cookbook ToolSpec values.""" + converted: list[ToolSpec] = [] + for raw in tools: + if raw.get("type") != "function" or not isinstance(raw.get("function"), dict): + raise ValueError("only OpenAI function tools are supported") + function = raw["function"] + converted.append( + { + "name": function["name"], + "description": function.get("description") or "", + "parameters": function.get("parameters") + or {"type": "object", "properties": {}}, + } + ) + return converted diff --git a/scripts/tinker_renderer_compat_test.py b/scripts/tinker_renderer_compat_test.py new file mode 100644 index 00000000..6bad8893 --- /dev/null +++ b/scripts/tinker_renderer_compat_test.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Provider-free exact Nemotron renderer roundtrip for tool-capable serving.""" +from __future__ import annotations + +from tinker_cookbook.renderers import get_renderer +from tinker_cookbook.tokenizer_utils import get_tokenizer + +from tinker_openai_compat import normalize_assistant_message +from tinker_renderer_compat import renderer_messages, renderer_tools + +MODEL = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" + + +def main() -> None: + tokenizer = get_tokenizer(MODEL) + renderer = get_renderer("nemotron3_disable_thinking", tokenizer) + tools = [{ + "type": "function", + "function": { + "name": "create-task", + "description": "Create a task", + "parameters": { + "type": "object", + "properties": {"title": {"type": "string"}}, + "required": ["title"], + }, + }, + }] + messages = [ + {"role": "system", "content": "Operate Cedar faithfully."}, + {"role": "user", "content": "Create task A."}, + { + "role": "assistant", + "content": "", + "tool_calls": [{ + "type": "function", + "id": "call_prior", + "function": {"name": "create-task", "arguments": '{"title":"A"}'}, + }], + }, + {"role": "tool", "tool_call_id": "call_prior", "content": '{"success":true}'}, + {"role": "user", "content": "Continue."}, + ] + system, history = renderer_messages(messages) + prefix = renderer.create_conversation_prefix_with_tools(renderer_tools(tools), system) + prompt = renderer.build_generation_prompt([*prefix, *history]) + assert prompt.length > 0 + decoded = tokenizer.decode(prompt.to_ints()) + assert "create-task" in decoded + assert "Create task A." in decoded + assert "" in decoded + assert '{"success":true}' in decoded + + sampled = tokenizer.encode( + "\n\n\nA\n" + "\n\n\n<|im_end|>", + add_special_tokens=False, + ) + parsed, termination = renderer.parse_response(sampled) + assert str(termination) == "stop_sequence" + openai = normalize_assistant_message(renderer.to_openai_message(parsed), "probe-request") + call = openai["tool_calls"][0] + assert call["id"].startswith("call_") + assert call["function"]["name"] == "create-task" + assert call["function"]["arguments"] == '{"title": "A"}' + print("EXACT NEMOTRON TOOL ROUNDTRIP PASSED") + + +if __name__ == "__main__": + main() diff --git a/tests/tinker_openai_compat_test.py b/tests/tinker_openai_compat_test.py new file mode 100644 index 00000000..81d205c2 --- /dev/null +++ b/tests/tinker_openai_compat_test.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +MODULE_PATH = Path(__file__).parents[1] / "scripts" / "tinker_openai_compat.py" +SPEC = importlib.util.spec_from_file_location("tinker_openai_compat", MODULE_PATH) +assert SPEC and SPEC.loader +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +def test_chat_completion_preserves_tool_calls() -> None: + message = { + "role": "assistant", + "content": "", + "tool_calls": [{ + "type": "function", + "id": "call-1", + "function": {"name": "create-task", "arguments": '{"title":"A"}'}, + }], + } + payload = MODULE.build_chat_completion(message, 10, 4, "stop") + assert payload["choices"][0]["message"] == message + + +def test_parsed_tool_call_gets_stable_id_and_json_arguments() -> None: + parsed = { + "role": "assistant", + "content": "", + "tool_calls": [{ + "type": "function", + "id": None, + "function": {"name": "create-task", "arguments": {"title": "A"}}, + }], + } + first = MODULE.normalize_assistant_message(parsed, "request-1") + second = MODULE.normalize_assistant_message(parsed, "request-1") + assert first == second + assert first["tool_calls"][0]["id"].startswith("call_") + assert first["tool_calls"][0]["function"]["arguments"] == '{"title": "A"}' From e5a69aec54d39a4a15654f165bbf591d1643a3fb Mon Sep 17 00:00:00 2001 From: luis manrique <166242911+lluisinthedesert@users.noreply.github.com> Date: Tue, 4 Aug 2026 01:33:31 -0700 Subject: [PATCH 06/18] fix: preserve deterministic sampler error semantics --- scripts/tinker-openai-shim.py | 4 +-- scripts/tinker_openai_compat.py | 38 ++++++++++++++++++++++++++++ scripts/tinker_openai_compat_test.py | 16 ++++++++++++ tests/tinker_openai_compat_test.py | 18 +++++++++++++ 4 files changed, 74 insertions(+), 2 deletions(-) diff --git a/scripts/tinker-openai-shim.py b/scripts/tinker-openai-shim.py index d4b70631..66382ac7 100644 --- a/scripts/tinker-openai-shim.py +++ b/scripts/tinker-openai-shim.py @@ -43,6 +43,7 @@ build_chat_completion, normalize_assistant_message, normalize_finish_reason, + openai_error_response, ) from tinker_renderer_compat import renderer_messages, renderer_tools @@ -225,8 +226,7 @@ def do_POST(self): # noqa: N802 - required by BaseHTTPRequestHandler status = 200 except Exception as error: # surface upstream failures as HTTP errors log_event("error", request_id=request_id, error=type(error).__name__, detail=str(error)[:240]) - payload = {"error": f"{type(error).__name__}: {error}"} - status = 500 + status, payload = openai_error_response(error) finally: elapsed = time.monotonic() - started with active_lock: diff --git a/scripts/tinker_openai_compat.py b/scripts/tinker_openai_compat.py index 353a5353..72e756a5 100644 --- a/scripts/tinker_openai_compat.py +++ b/scripts/tinker_openai_compat.py @@ -34,6 +34,44 @@ _CLEAN_TERMINATIONS = ("stop_sequence", "eos") +def openai_error_response(error: Exception) -> tuple[int, dict[str, Any]]: + """Map sampler failures without disguising deterministic client errors. + + Tinker raises ``BadRequestError`` for prompt-plus-generation context-window + overflow. Returning 500 for that condition made the Gateway and evaluator + treat deterministic, model-incompatible requests as retryable provider + pressure. Keep the response bounded and OpenAI-shaped while preserving the + actionable error class. + """ + detail = str(error) + lowered = detail.lower() + type_name = type(error).__name__ + if type_name == "BadRequestError" or "exceeds the model's context window" in lowered: + code = "context_length_exceeded" if "context window" in lowered else "invalid_request" + return 400, { + "error": { + "message": detail[:500], + "type": "invalid_request_error", + "code": code, + } + } + if isinstance(error, TimeoutError): + return 504, { + "error": { + "message": detail[:500], + "type": "server_error", + "code": "upstream_timeout", + } + } + return 500, { + "error": { + "message": f"{type_name}: {detail}"[:500], + "type": "server_error", + "code": "upstream_error", + } + } + + def normalize_assistant_message(message: dict[str, Any], request_id: str) -> dict[str, Any]: """Return strict OpenAI assistant shape with stable tool-call IDs/arguments.""" normalized = {"role": "assistant", "content": message.get("content") or ""} diff --git a/scripts/tinker_openai_compat_test.py b/scripts/tinker_openai_compat_test.py index b024b9d9..dda00c6a 100644 --- a/scripts/tinker_openai_compat_test.py +++ b/scripts/tinker_openai_compat_test.py @@ -14,6 +14,7 @@ build_chat_completion, normalize_assistant_message, normalize_finish_reason, + openai_error_response, ) @@ -128,6 +129,20 @@ def test_parsed_tool_call_gets_stable_openai_shape(): ) +def test_error_mapping_preserves_context_overflow_semantics(): + class BadRequestError(Exception): + pass + + status, payload = openai_error_response( + BadRequestError("Prompt length plus max_tokens exceeds the model's context window") + ) + check("context overflow is HTTP 400", status == 400) + check("context overflow has stable code", payload["error"]["code"] == "context_length_exceeded") + status, payload = openai_error_response(TimeoutError("sampling timed out")) + check("timeout is HTTP 504", status == 504) + check("timeout is retry-class server error", payload["error"]["type"] == "server_error") + + def main(): tests = [ test_upstream_stop, @@ -139,6 +154,7 @@ def main(): test_payload_requires_defined_finish_reason, test_bearer_auth_is_fail_closed, test_parsed_tool_call_gets_stable_openai_shape, + test_error_mapping_preserves_context_overflow_semantics, ] for t in tests: print(t.__name__) diff --git a/tests/tinker_openai_compat_test.py b/tests/tinker_openai_compat_test.py index 81d205c2..51d222f2 100644 --- a/tests/tinker_openai_compat_test.py +++ b/tests/tinker_openai_compat_test.py @@ -40,3 +40,21 @@ def test_parsed_tool_call_gets_stable_id_and_json_arguments() -> None: assert first == second assert first["tool_calls"][0]["id"].startswith("call_") assert first["tool_calls"][0]["function"]["arguments"] == '{"title": "A"}' + + +def test_context_overflow_is_not_mislabeled_as_retryable_provider_failure() -> None: + class BadRequestError(Exception): + pass + + status, payload = MODULE.openai_error_response( + BadRequestError("Prompt length plus max_tokens exceeds the model's context window") + ) + assert status == 400 + assert payload["error"]["type"] == "invalid_request_error" + assert payload["error"]["code"] == "context_length_exceeded" + + +def test_timeout_remains_a_retry_class_server_error() -> None: + status, payload = MODULE.openai_error_response(TimeoutError("sampling timed out")) + assert status == 504 + assert payload["error"]["code"] == "upstream_timeout" From 53021200b9631f14c3a7ac173bc955a1cfbdac51 Mon Sep 17 00:00:00 2001 From: luis manrique <166242911+lluisinthedesert@users.noreply.github.com> Date: Tue, 4 Aug 2026 15:49:46 -0700 Subject: [PATCH 07/18] ops: isolate Cedar seed37 checkpoint serving --- scripts/modal-tinker-openai-shim.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/modal-tinker-openai-shim.py b/scripts/modal-tinker-openai-shim.py index bed14e0a..64e8551e 100644 --- a/scripts/modal-tinker-openai-shim.py +++ b/scripts/modal-tinker-openai-shim.py @@ -1,6 +1,6 @@ """Private multi-checkpoint Tinker sampling bridge for Understudy Gateway. -Deploy with a Modal secret named understudy-tinker-serving containing +Deploy with a Modal secret named understudy-tinker-serving-seed37 containing TINKER_API_KEY and TINKER_MODEL_REGISTRY_JSON. Modal proxy authentication is required before requests reach the shim. """ @@ -13,9 +13,9 @@ import modal -APP_NAME = "understudy-tinker-checkpoint-serving" +APP_NAME = "understudy-tinker-cedar-seed37-serving" PORT = 8099 -SECRET_NAME = "understudy-tinker-serving" +SECRET_NAME = "understudy-tinker-serving-seed37" BASE_MODEL = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" TINKER_VERSION = "0.24.0" COOKBOOK_VERSION = "0.5.3" From 92d878593f51498ee4259ae345c6c7e662090635 Mon Sep 17 00:00:00 2001 From: luis manrique <166242911+lluisinthedesert@users.noreply.github.com> Date: Tue, 4 Aug 2026 16:05:52 -0700 Subject: [PATCH 08/18] ops: surface bounded Tinker shim diagnostics --- scripts/tinker-openai-shim.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/tinker-openai-shim.py b/scripts/tinker-openai-shim.py index 66382ac7..a80a124b 100644 --- a/scripts/tinker-openai-shim.py +++ b/scripts/tinker-openai-shim.py @@ -88,6 +88,10 @@ def log_event(event, **fields): with log_lock: with open(log_path, "a", encoding="utf-8") as stream: stream.write(json.dumps(record) + "\n") + # Modal's durable app logs are the only operator-visible surface for this + # web-server process. Records contain request ids and bounded error types, + # never prompts, tool arguments, credentials, or response text. + print(json.dumps(record, sort_keys=True), flush=True) service = tinker.ServiceClient(_client_config={"use_pyqwest_transport": False}) From 0f9a80d510e12a8468bceef8089d5d563a041f3c Mon Sep 17 00:00:00 2001 From: luis manrique <166242911+lluisinthedesert@users.noreply.github.com> Date: Tue, 4 Aug 2026 16:13:12 -0700 Subject: [PATCH 09/18] fix: accept OpenAI text content arrays in Tinker shim --- scripts/tinker_renderer_compat.py | 26 +++++++++++++++++++++----- scripts/tinker_renderer_compat_test.py | 18 +++++++++++++++++- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/scripts/tinker_renderer_compat.py b/scripts/tinker_renderer_compat.py index 47f220cc..b521018f 100644 --- a/scripts/tinker_renderer_compat.py +++ b/scripts/tinker_renderer_compat.py @@ -7,17 +7,33 @@ from tinker_cookbook.renderers.base import Message, ToolCall, ToolSpec +def _message_text(content: object) -> str: + """Normalize OpenAI text content without silently dropping non-text parts.""" + if content is None: + return "" + if isinstance(content, str): + return content + if not isinstance(content, list): + raise ValueError("message content must be string, null, or a text-part array") + + parts: list[str] = [] + for part in content: + if not isinstance(part, dict) or part.get("type") not in {"text", "input_text"}: + raise ValueError("only text/input_text message content parts are supported") + text = part.get("text") + if not isinstance(text, str): + raise ValueError("message text content part requires a string text field") + parts.append(text) + return "".join(parts) + + def renderer_messages(messages: list[dict]) -> tuple[str, list[Message]]: """Losslessly convert OpenAI history into the cookbook renderer contract.""" system_parts: list[str] = [] converted: list[Message] = [] for raw in messages: role = raw.get("role") - content = raw.get("content") - if content is None: - content = "" - if not isinstance(content, str): - raise ValueError("only string/null message content is supported") + content = _message_text(raw.get("content")) if role == "system": system_parts.append(content) continue diff --git a/scripts/tinker_renderer_compat_test.py b/scripts/tinker_renderer_compat_test.py index 6bad8893..19f4f027 100644 --- a/scripts/tinker_renderer_compat_test.py +++ b/scripts/tinker_renderer_compat_test.py @@ -28,7 +28,13 @@ def main() -> None: }] messages = [ {"role": "system", "content": "Operate Cedar faithfully."}, - {"role": "user", "content": "Create task A."}, + { + "role": "user", + "content": [ + {"type": "text", "text": "Create task "}, + {"type": "input_text", "text": "A."}, + ], + }, { "role": "assistant", "content": "", @@ -51,6 +57,16 @@ def main() -> None: assert "" in decoded assert '{"success":true}' in decoded + try: + renderer_messages([{ + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": "private"}}], + }]) + except ValueError as error: + assert "only text/input_text" in str(error) + else: + raise AssertionError("non-text content must fail closed") + sampled = tokenizer.encode( "\n\n\nA\n" "\n\n\n<|im_end|>", From 8fe682a2af4ba461e118b3e6c32551825cb3bb65 Mon Sep 17 00:00:00 2001 From: luis manrique <166242911+lluisinthedesert@users.noreply.github.com> Date: Tue, 4 Aug 2026 16:15:44 -0700 Subject: [PATCH 10/18] fix: emit complete OpenAI chat completion shape --- scripts/tinker-openai-shim.py | 9 ++++++++- scripts/tinker_openai_compat.py | 12 ++++++++++++ tests/tinker_openai_compat_test.py | 16 +++++++++++++++- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/scripts/tinker-openai-shim.py b/scripts/tinker-openai-shim.py index a80a124b..4ddcdc74 100644 --- a/scripts/tinker-openai-shim.py +++ b/scripts/tinker-openai-shim.py @@ -226,7 +226,14 @@ def do_POST(self): # noqa: N802 - required by BaseHTTPRequestHandler raise TimeoutError(f"sampling exceeded {request_timeout}s twice") log_event("retry", request_id=request_id, attempt=attempt + 2) message = normalize_assistant_message(message, request_id) - payload = build_chat_completion(message, prompt_tokens, completion_tokens, finish_reason) + payload = build_chat_completion( + message, + prompt_tokens, + completion_tokens, + finish_reason, + model=requested_model, + request_id=request_id, + ) status = 200 except Exception as error: # surface upstream failures as HTTP errors log_event("error", request_id=request_id, error=type(error).__name__, detail=str(error)[:240]) diff --git a/scripts/tinker_openai_compat.py b/scripts/tinker_openai_compat.py index 72e756a5..7466d871 100644 --- a/scripts/tinker_openai_compat.py +++ b/scripts/tinker_openai_compat.py @@ -24,6 +24,7 @@ import hmac import json +import time import uuid from typing import Any, Optional @@ -155,6 +156,10 @@ def build_chat_completion( prompt_tokens: int, completion_tokens: int, finish_reason: str, + *, + model: str = "unknown", + request_id: Optional[str] = None, + created: Optional[int] = None, ) -> dict: """Build the /v1/chat/completions response body. @@ -162,9 +167,15 @@ def build_chat_completion( object without a defined finish_reason is impossible here, which is exactly the undefined-variable (NameError) failure this module exists to prevent. """ + completion_id = request_id or str(uuid.uuid4()) return { + "id": f"chatcmpl-{completion_id}", + "object": "chat.completion", + "created": int(time.time()) if created is None else created, + "model": model, "choices": [ { + "index": 0, "message": message, "finish_reason": finish_reason, } @@ -172,5 +183,6 @@ def build_chat_completion( "usage": { "prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens, }, } diff --git a/tests/tinker_openai_compat_test.py b/tests/tinker_openai_compat_test.py index 51d222f2..45a9f6c8 100644 --- a/tests/tinker_openai_compat_test.py +++ b/tests/tinker_openai_compat_test.py @@ -21,8 +21,22 @@ def test_chat_completion_preserves_tool_calls() -> None: "function": {"name": "create-task", "arguments": '{"title":"A"}'}, }], } - payload = MODULE.build_chat_completion(message, 10, 4, "stop") + payload = MODULE.build_chat_completion( + message, + 10, + 4, + "stop", + model="cedar-test", + request_id="request-1", + created=123, + ) assert payload["choices"][0]["message"] == message + assert payload["id"] == "chatcmpl-request-1" + assert payload["object"] == "chat.completion" + assert payload["created"] == 123 + assert payload["model"] == "cedar-test" + assert payload["choices"][0]["index"] == 0 + assert payload["usage"]["total_tokens"] == 14 def test_parsed_tool_call_gets_stable_id_and_json_arguments() -> None: From 2d85442849885e626d9457554fc1210a2df511c9 Mon Sep 17 00:00:00 2001 From: luis manrique <166242911+lluisinthedesert@users.noreply.github.com> Date: Tue, 4 Aug 2026 16:16:00 -0700 Subject: [PATCH 11/18] test: require complete token usage shape --- scripts/tinker_openai_compat_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/tinker_openai_compat_test.py b/scripts/tinker_openai_compat_test.py index dda00c6a..6cfe5ff1 100644 --- a/scripts/tinker_openai_compat_test.py +++ b/scripts/tinker_openai_compat_test.py @@ -79,7 +79,11 @@ def test_payload_requires_defined_finish_reason(): check("payload choices[0].finish_reason present", payload["choices"][0]["finish_reason"] == "stop") check("payload content propagated", payload["choices"][0]["message"]["content"] == "hello") check("payload usage propagated", - payload["usage"] == {"prompt_tokens": 100, "completion_tokens": 3}) + payload["usage"] == { + "prompt_tokens": 100, + "completion_tokens": 3, + "total_tokens": 103, + }) fr_len = normalize_finish_reason(stop_reason="length", completion_tokens=384, max_tokens=384) payload_len = build_chat_completion({"role": "assistant", "content": ""}, 100, 384, fr_len) From 4249551ea1250e65c248a2209e984e010009914a Mon Sep 17 00:00:00 2001 From: luis manrique <166242911+lluisinthedesert@users.noreply.github.com> Date: Tue, 4 Aug 2026 16:29:27 -0700 Subject: [PATCH 12/18] Fail closed on malformed shim requests --- scripts/tinker-openai-shim.py | 13 ++++++++++++- scripts/tinker_openai_compat.py | 25 +++++++++++++++++++++++++ tests/tinker_openai_compat_test.py | 22 ++++++++++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/scripts/tinker-openai-shim.py b/scripts/tinker-openai-shim.py index 4ddcdc74..9eb09b8d 100644 --- a/scripts/tinker-openai-shim.py +++ b/scripts/tinker-openai-shim.py @@ -39,11 +39,13 @@ from tinker_cookbook.tokenizer_utils import get_tokenizer from tinker_openai_compat import ( + InvalidRequestError, bearer_authorized, build_chat_completion, normalize_assistant_message, normalize_finish_reason, openai_error_response, + parse_chat_request, ) from tinker_renderer_compat import renderer_messages, renderer_tools @@ -198,8 +200,17 @@ def do_POST(self): # noqa: N802 - required by BaseHTTPRequestHandler active_requests += 1 in_flight = active_requests log_event("start", request_id=request_id, in_flight=in_flight) - body = json.loads(self.rfile.read(int(self.headers["content-length"]))) try: + raw_content_length = self.headers.get("content-length") + if raw_content_length is None: + raise InvalidRequestError("content-length header is required") + try: + content_length = int(raw_content_length) + except ValueError as error: + raise InvalidRequestError("content-length header must be an integer") from error + if content_length < 0: + raise InvalidRequestError("content-length header must be non-negative") + body = parse_chat_request(self.rfile.read(content_length)) requested_model = body.get("model") if requested_model is None and len(served_models) == 1: requested_model = served_models[0] diff --git a/scripts/tinker_openai_compat.py b/scripts/tinker_openai_compat.py index 7466d871..9c9958a0 100644 --- a/scripts/tinker_openai_compat.py +++ b/scripts/tinker_openai_compat.py @@ -35,6 +35,23 @@ _CLEAN_TERMINATIONS = ("stop_sequence", "eos") +class InvalidRequestError(ValueError): + """A bounded client request error that must be returned as HTTP 400.""" + + +def parse_chat_request(raw: bytes) -> dict[str, Any]: + """Decode and minimally validate an OpenAI chat-completions request.""" + try: + body = json.loads(raw) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise InvalidRequestError("request body must be valid UTF-8 JSON") from error + if not isinstance(body, dict): + raise InvalidRequestError("request body must be a JSON object") + if not isinstance(body.get("messages"), list): + raise InvalidRequestError("messages must be a JSON array") + return body + + def openai_error_response(error: Exception) -> tuple[int, dict[str, Any]]: """Map sampler failures without disguising deterministic client errors. @@ -47,6 +64,14 @@ def openai_error_response(error: Exception) -> tuple[int, dict[str, Any]]: detail = str(error) lowered = detail.lower() type_name = type(error).__name__ + if isinstance(error, InvalidRequestError): + return 400, { + "error": { + "message": detail[:500], + "type": "invalid_request_error", + "code": "invalid_request", + } + } if type_name == "BadRequestError" or "exceeds the model's context window" in lowered: code = "context_length_exceeded" if "context window" in lowered else "invalid_request" return 400, { diff --git a/tests/tinker_openai_compat_test.py b/tests/tinker_openai_compat_test.py index 45a9f6c8..64677df5 100644 --- a/tests/tinker_openai_compat_test.py +++ b/tests/tinker_openai_compat_test.py @@ -72,3 +72,25 @@ def test_timeout_remains_a_retry_class_server_error() -> None: status, payload = MODULE.openai_error_response(TimeoutError("sampling timed out")) assert status == 504 assert payload["error"]["code"] == "upstream_timeout" + + +def test_malformed_chat_requests_fail_closed() -> None: + cases = ( + (b"{", "valid UTF-8 JSON"), + (b"[]", "JSON object"), + (b'{"model":"cedar-test"}', "messages"), + (b'{"messages":{}}', "messages"), + ) + for raw, expected in cases: + try: + MODULE.parse_chat_request(raw) + except MODULE.InvalidRequestError as error: + assert expected in str(error) + status, payload = MODULE.openai_error_response(error) + assert status == 400 + assert payload["error"]["type"] == "invalid_request_error" + assert payload["error"]["code"] == "invalid_request" + else: + raise AssertionError(f"invalid request unexpectedly accepted: {raw!r}") + + assert MODULE.parse_chat_request(b'{"messages":[]}') == {"messages": []} From 917e7e161894c5b5d8146f3490c7e53b354ceb1b Mon Sep 17 00:00:00 2001 From: luis manrique <166242911+lluisinthedesert@users.noreply.github.com> Date: Tue, 4 Aug 2026 16:36:47 -0700 Subject: [PATCH 13/18] Harden Tinker serving contract --- scripts/adapter-serving-compat.py | 2 +- scripts/adapter_serving_compat_test.py | 4 ++++ scripts/modal-tinker-openai-shim.py | 2 +- scripts/tinker-openai-shim.py | 13 ++++++++++--- scripts/tinker_openai_compat.py | 10 +++++++--- tests/tinker_openai_compat_test.py | 22 ++++++++++++++++++++++ 6 files changed, 45 insertions(+), 8 deletions(-) diff --git a/scripts/adapter-serving-compat.py b/scripts/adapter-serving-compat.py index aa0fcf43..19013779 100644 --- a/scripts/adapter-serving-compat.py +++ b/scripts/adapter-serving-compat.py @@ -29,7 +29,7 @@ def _normalize_targets(value: object) -> tuple[list[str], str | None]: targets = sorted(set(value)) wildcard = next((item for item in targets if item in {"all-linear", "all_linear"}), None) return targets, wildcard - return [], "missing_or_invalid" + return [], None def assess(config_path: Path, runtime: str, base_model: str) -> dict: diff --git a/scripts/adapter_serving_compat_test.py b/scripts/adapter_serving_compat_test.py index 637e2052..47e4444c 100644 --- a/scripts/adapter_serving_compat_test.py +++ b/scripts/adapter_serving_compat_test.py @@ -41,6 +41,10 @@ def main() -> None: unknown = MODULE.assess(config(root, ["q_proj"]), "vllm-nemotron-h", "some/other-model") assert unknown["compatibility"] == "unknown" assert unknown["faithful"] is False + missing = MODULE.assess(config(root, None), "vllm-nemotron-h", MODEL) + assert missing["compatibility"] == "incompatible" + assert missing["unsupported_target_modules"] == ["missing_or_invalid"] + assert "missing or invalid" in missing["reason"] print("ALL ADAPTER SERVING COMPAT TESTS PASSED") diff --git a/scripts/modal-tinker-openai-shim.py b/scripts/modal-tinker-openai-shim.py index 64e8551e..7bc784ef 100644 --- a/scripts/modal-tinker-openai-shim.py +++ b/scripts/modal-tinker-openai-shim.py @@ -71,5 +71,5 @@ def serve() -> None: "--max-tokens", "2048", ], - env=os.environ.copy(), + env={**os.environ, "TINKER_TRUSTED_PROXY_AUTH": "modal"}, ) diff --git a/scripts/tinker-openai-shim.py b/scripts/tinker-openai-shim.py index 9eb09b8d..292053e6 100644 --- a/scripts/tinker-openai-shim.py +++ b/scripts/tinker-openai-shim.py @@ -70,13 +70,18 @@ parser.add_argument("--max-workers", type=int, default=16, help="in-flight samples; raise it for rollout mining") args = parser.parse_args() service_token = os.environ.get("TINKER_SHIM_BEARER_TOKEN") +trusted_modal_proxy = ( + args.trusted_proxy_auth + and os.environ.get("TINKER_TRUSTED_PROXY_AUTH") == "modal" + and bool(os.environ.get("MODAL_TASK_ID")) +) if ( args.host not in {"127.0.0.1", "::1", "localhost"} and not service_token - and not args.trusted_proxy_auth + and not trusted_modal_proxy ): raise SystemExit( - "TINKER_SHIM_BEARER_TOKEN or --trusted-proxy-auth is required for non-loopback binds" + "TINKER_SHIM_BEARER_TOKEN or an attested Modal proxy runtime is required for non-loopback binds" ) request_timeout = 300 log_path = os.environ.get("TINKER_SHIM_LOG", "/tmp/tinker-openai-shim.log") @@ -237,6 +242,8 @@ def do_POST(self): # noqa: N802 - required by BaseHTTPRequestHandler raise TimeoutError(f"sampling exceeded {request_timeout}s twice") log_event("retry", request_id=request_id, attempt=attempt + 2) message = normalize_assistant_message(message, request_id) + if message.get("tool_calls"): + finish_reason = "tool_calls" payload = build_chat_completion( message, prompt_tokens, @@ -247,7 +254,7 @@ def do_POST(self): # noqa: N802 - required by BaseHTTPRequestHandler ) status = 200 except Exception as error: # surface upstream failures as HTTP errors - log_event("error", request_id=request_id, error=type(error).__name__, detail=str(error)[:240]) + log_event("error", request_id=request_id, error=type(error).__name__) status, payload = openai_error_response(error) finally: elapsed = time.monotonic() - started diff --git a/scripts/tinker_openai_compat.py b/scripts/tinker_openai_compat.py index 9c9958a0..591db6da 100644 --- a/scripts/tinker_openai_compat.py +++ b/scripts/tinker_openai_compat.py @@ -76,7 +76,11 @@ def openai_error_response(error: Exception) -> tuple[int, dict[str, Any]]: code = "context_length_exceeded" if "context window" in lowered else "invalid_request" return 400, { "error": { - "message": detail[:500], + "message": ( + "request exceeds the model context window" + if code == "context_length_exceeded" + else "upstream rejected the request" + ), "type": "invalid_request_error", "code": code, } @@ -84,14 +88,14 @@ def openai_error_response(error: Exception) -> tuple[int, dict[str, Any]]: if isinstance(error, TimeoutError): return 504, { "error": { - "message": detail[:500], + "message": "upstream sampling timed out", "type": "server_error", "code": "upstream_timeout", } } return 500, { "error": { - "message": f"{type_name}: {detail}"[:500], + "message": "upstream sampling failed", "type": "server_error", "code": "upstream_error", } diff --git a/tests/tinker_openai_compat_test.py b/tests/tinker_openai_compat_test.py index 64677df5..a9b9e553 100644 --- a/tests/tinker_openai_compat_test.py +++ b/tests/tinker_openai_compat_test.py @@ -39,6 +39,16 @@ def test_chat_completion_preserves_tool_calls() -> None: assert payload["usage"]["total_tokens"] == 14 +def test_tool_call_finish_reason_is_supported() -> None: + payload = MODULE.build_chat_completion( + {"role": "assistant", "content": "", "tool_calls": []}, + 1, + 1, + "tool_calls", + ) + assert payload["choices"][0]["finish_reason"] == "tool_calls" + + def test_parsed_tool_call_gets_stable_id_and_json_arguments() -> None: parsed = { "role": "assistant", @@ -66,6 +76,7 @@ class BadRequestError(Exception): assert status == 400 assert payload["error"]["type"] == "invalid_request_error" assert payload["error"]["code"] == "context_length_exceeded" + assert "Prompt length" not in payload["error"]["message"] def test_timeout_remains_a_retry_class_server_error() -> None: @@ -74,6 +85,17 @@ def test_timeout_remains_a_retry_class_server_error() -> None: assert payload["error"]["code"] == "upstream_timeout" +def test_upstream_error_details_are_redacted() -> None: + status, payload = MODULE.openai_error_response( + RuntimeError("https://private.invalid tinker://private secret-header") + ) + assert status == 500 + rendered = str(payload) + assert "private.invalid" not in rendered + assert "tinker://" not in rendered + assert "secret-header" not in rendered + + def test_malformed_chat_requests_fail_closed() -> None: cases = ( (b"{", "valid UTF-8 JSON"), From c111e3aa9e41debc7c40d0abf49651f7afaa606f Mon Sep 17 00:00:00 2001 From: luis manrique <166242911+lluisinthedesert@users.noreply.github.com> Date: Tue, 4 Aug 2026 16:49:22 -0700 Subject: [PATCH 14/18] Allow isolated Tinker serving deployments --- scripts/modal-tinker-openai-shim.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/modal-tinker-openai-shim.py b/scripts/modal-tinker-openai-shim.py index 7bc784ef..f0d70a68 100644 --- a/scripts/modal-tinker-openai-shim.py +++ b/scripts/modal-tinker-openai-shim.py @@ -1,8 +1,9 @@ """Private multi-checkpoint Tinker sampling bridge for Understudy Gateway. -Deploy with a Modal secret named understudy-tinker-serving-seed37 containing -TINKER_API_KEY and TINKER_MODEL_REGISTRY_JSON. Modal proxy authentication is -required before requests reach the shim. +Deploy with a Modal secret containing TINKER_API_KEY and +TINKER_MODEL_REGISTRY_JSON. Set TINKER_SERVING_APP_NAME and +TINKER_SERVING_SECRET_NAME at deploy time to isolate checkpoint lineages. +Modal proxy authentication is required before requests reach the shim. """ from __future__ import annotations @@ -13,9 +14,13 @@ import modal -APP_NAME = "understudy-tinker-cedar-seed37-serving" +APP_NAME = os.environ.get( + "TINKER_SERVING_APP_NAME", "understudy-tinker-cedar-seed37-serving" +) PORT = 8099 -SECRET_NAME = "understudy-tinker-serving-seed37" +SECRET_NAME = os.environ.get( + "TINKER_SERVING_SECRET_NAME", "understudy-tinker-serving-seed37" +) BASE_MODEL = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" TINKER_VERSION = "0.24.0" COOKBOOK_VERSION = "0.5.3" From 73680ae367f0922406e681e3224c2ddeccc98d74 Mon Sep 17 00:00:00 2001 From: luis manrique <166242911+lluisinthedesert@users.noreply.github.com> Date: Tue, 4 Aug 2026 17:57:04 -0700 Subject: [PATCH 15/18] test: lock private Tinker shim safety invariants --- tests/tinker_shim_static_test.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 tests/tinker_shim_static_test.py diff --git a/tests/tinker_shim_static_test.py b/tests/tinker_shim_static_test.py new file mode 100644 index 00000000..f46bdb1f --- /dev/null +++ b/tests/tinker_shim_static_test.py @@ -0,0 +1,31 @@ +"""Provider-free static safety checks for the private Tinker serving entrypoint.""" + +from pathlib import Path + + +SHIM = (Path(__file__).parents[1] / "scripts" / "tinker-openai-shim.py").read_text() + + +def test_request_accounting_wraps_request_parsing() -> None: + increment = SHIM.index("active_requests += 1") + protected = SHIM.index("try:", increment) + parse = SHIM.index("body = parse_chat_request", protected) + cleanup = SHIM.index("active_requests -= 1", parse) + assert increment < protected < parse < cleanup + + +def test_network_auth_requires_attested_modal_runtime_or_token() -> None: + assert 'os.environ.get("TINKER_TRUSTED_PROXY_AUTH") == "modal"' in SHIM + assert 'bool(os.environ.get("MODAL_TASK_ID"))' in SHIM + assert "and not trusted_modal_proxy" in SHIM + + +def test_tool_calls_override_finish_reason() -> None: + assert 'if message.get("tool_calls"):' in SHIM + assert 'finish_reason = "tool_calls"' in SHIM + + +def test_logs_expose_only_error_class() -> None: + error_log = 'log_event("error", request_id=request_id, error=type(error).__name__)' + assert error_log in SHIM + assert "detail=str(error)" not in SHIM From df42aadbb600b762ab7c7fd000fe6c4d3a13e054 Mon Sep 17 00:00:00 2001 From: luis manrique <166242911+lluisinthedesert@users.noreply.github.com> Date: Tue, 4 Aug 2026 18:02:23 -0700 Subject: [PATCH 16/18] feat: isolate private checkpoint registry secrets --- scripts/modal-tinker-openai-shim.py | 20 ++++++++++++++++++-- tests/tinker_shim_static_test.py | 10 ++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/scripts/modal-tinker-openai-shim.py b/scripts/modal-tinker-openai-shim.py index f0d70a68..6684e3b1 100644 --- a/scripts/modal-tinker-openai-shim.py +++ b/scripts/modal-tinker-openai-shim.py @@ -21,6 +21,19 @@ SECRET_NAME = os.environ.get( "TINKER_SERVING_SECRET_NAME", "understudy-tinker-serving-seed37" ) +API_SECRET_NAME = os.environ.get("TINKER_SERVING_API_SECRET_NAME", SECRET_NAME) +registry_override = os.environ.get("TINKER_SERVING_REGISTRY_JSON") +runtime_secrets = [modal.Secret.from_name(API_SECRET_NAME)] +if registry_override: + # The checkpoint registry is supplied only to Modal's encrypted secret + # plane at deploy time. It never enters Git, the image, or app logs, and it + # can reuse an existing TINKER_API_KEY secret without copying that key back + # to the operator workstation. + runtime_secrets.append( + modal.Secret.from_dict( + {"TINKER_MODEL_REGISTRY_JSON_OVERRIDE": registry_override} + ) + ) BASE_MODEL = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" TINKER_VERSION = "0.24.0" COOKBOOK_VERSION = "0.5.3" @@ -42,7 +55,7 @@ @app.function( image=image, - secrets=[modal.Secret.from_name(SECRET_NAME)], + secrets=runtime_secrets, timeout=60 * 60, scaledown_window=300, max_containers=4, @@ -50,7 +63,10 @@ @modal.concurrent(max_inputs=64) @modal.web_server(PORT, startup_timeout=10 * 60, requires_proxy_auth=True) def serve() -> None: - registry = json.loads(os.environ["TINKER_MODEL_REGISTRY_JSON"]) + registry = json.loads( + os.environ.get("TINKER_MODEL_REGISTRY_JSON_OVERRIDE") + or os.environ["TINKER_MODEL_REGISTRY_JSON"] + ) if not isinstance(registry, dict) or not registry: raise RuntimeError("TINKER_MODEL_REGISTRY_JSON must be a non-empty object") registry_path = Path("/tmp/tinker-model-registry.json") diff --git a/tests/tinker_shim_static_test.py b/tests/tinker_shim_static_test.py index f46bdb1f..bec67057 100644 --- a/tests/tinker_shim_static_test.py +++ b/tests/tinker_shim_static_test.py @@ -4,6 +4,9 @@ SHIM = (Path(__file__).parents[1] / "scripts" / "tinker-openai-shim.py").read_text() +MODAL_SHIM = ( + Path(__file__).parents[1] / "scripts" / "modal-tinker-openai-shim.py" +).read_text() def test_request_accounting_wraps_request_parsing() -> None: @@ -29,3 +32,10 @@ def test_logs_expose_only_error_class() -> None: error_log = 'log_event("error", request_id=request_id, error=type(error).__name__)' assert error_log in SHIM assert "detail=str(error)" not in SHIM + + +def test_modal_registry_override_uses_secret_plane() -> None: + assert 'modal.Secret.from_dict(' in MODAL_SHIM + assert '"TINKER_MODEL_REGISTRY_JSON_OVERRIDE": registry_override' in MODAL_SHIM + assert 'os.environ.get("TINKER_MODEL_REGISTRY_JSON_OVERRIDE")' in MODAL_SHIM + assert '.env({"TINKER_MODEL_REGISTRY_JSON"' not in MODAL_SHIM From c424d4b9a054d0220b756f7ca7edf373c08c8b4c Mon Sep 17 00:00:00 2001 From: luis manrique <166242911+lluisinthedesert@users.noreply.github.com> Date: Tue, 4 Aug 2026 18:04:48 -0700 Subject: [PATCH 17/18] docs: explain private registry secret deployment --- scripts/modal-tinker-openai-shim.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/modal-tinker-openai-shim.py b/scripts/modal-tinker-openai-shim.py index 6684e3b1..95f82044 100644 --- a/scripts/modal-tinker-openai-shim.py +++ b/scripts/modal-tinker-openai-shim.py @@ -3,7 +3,12 @@ Deploy with a Modal secret containing TINKER_API_KEY and TINKER_MODEL_REGISTRY_JSON. Set TINKER_SERVING_APP_NAME and TINKER_SERVING_SECRET_NAME at deploy time to isolate checkpoint lineages. -Modal proxy authentication is required before requests reach the shim. + +For a new private checkpoint, reuse an existing API-key secret without reading +it back by setting TINKER_SERVING_API_SECRET_NAME, and pass the checkpoint-only +registry in TINKER_SERVING_REGISTRY_JSON. The latter is converted to a Modal +Secret at deploy time and must never be committed or printed. Modal proxy +authentication is required before requests reach the shim. """ from __future__ import annotations From a4576ce496583098ac932b704640876af61c113e Mon Sep 17 00:00:00 2001 From: luis manrique <166242911+lluisinthedesert@users.noreply.github.com> Date: Tue, 4 Aug 2026 18:37:40 -0700 Subject: [PATCH 18/18] fix: make Modal secret dependencies deterministic --- scripts/modal-tinker-openai-shim.py | 31 ++++++++++++++--------------- tests/tinker_shim_static_test.py | 7 ++++--- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/scripts/modal-tinker-openai-shim.py b/scripts/modal-tinker-openai-shim.py index 95f82044..ac33f8f8 100644 --- a/scripts/modal-tinker-openai-shim.py +++ b/scripts/modal-tinker-openai-shim.py @@ -5,10 +5,12 @@ TINKER_SERVING_SECRET_NAME at deploy time to isolate checkpoint lineages. For a new private checkpoint, reuse an existing API-key secret without reading -it back by setting TINKER_SERVING_API_SECRET_NAME, and pass the checkpoint-only -registry in TINKER_SERVING_REGISTRY_JSON. The latter is converted to a Modal -Secret at deploy time and must never be committed or printed. Modal proxy -authentication is required before requests reach the shim. +it back by setting TINKER_SERVING_API_SECRET_NAME, and place the checkpoint-only +registry in a second named secret selected with +TINKER_SERVING_REGISTRY_SECRET_NAME. Both dependencies are declared +unconditionally because Modal imports this module again in the remote runtime; +conditional Secret objects produce a local/remote dependency-count mismatch. +Modal proxy authentication is required before requests reach the shim. """ from __future__ import annotations @@ -27,18 +29,15 @@ "TINKER_SERVING_SECRET_NAME", "understudy-tinker-serving-seed37" ) API_SECRET_NAME = os.environ.get("TINKER_SERVING_API_SECRET_NAME", SECRET_NAME) -registry_override = os.environ.get("TINKER_SERVING_REGISTRY_JSON") -runtime_secrets = [modal.Secret.from_name(API_SECRET_NAME)] -if registry_override: - # The checkpoint registry is supplied only to Modal's encrypted secret - # plane at deploy time. It never enters Git, the image, or app logs, and it - # can reuse an existing TINKER_API_KEY secret without copying that key back - # to the operator workstation. - runtime_secrets.append( - modal.Secret.from_dict( - {"TINKER_MODEL_REGISTRY_JSON_OVERRIDE": registry_override} - ) - ) +REGISTRY_SECRET_NAME = os.environ.get( + "TINKER_SERVING_REGISTRY_SECRET_NAME", SECRET_NAME +) +# Keep this list structurally identical in the deploy process and the remote +# container import. Values stay in Modal's encrypted secret plane. +runtime_secrets = [ + modal.Secret.from_name(API_SECRET_NAME), + modal.Secret.from_name(REGISTRY_SECRET_NAME), +] BASE_MODEL = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" TINKER_VERSION = "0.24.0" COOKBOOK_VERSION = "0.5.3" diff --git a/tests/tinker_shim_static_test.py b/tests/tinker_shim_static_test.py index bec67057..d39916c3 100644 --- a/tests/tinker_shim_static_test.py +++ b/tests/tinker_shim_static_test.py @@ -34,8 +34,9 @@ def test_logs_expose_only_error_class() -> None: assert "detail=str(error)" not in SHIM -def test_modal_registry_override_uses_secret_plane() -> None: - assert 'modal.Secret.from_dict(' in MODAL_SHIM - assert '"TINKER_MODEL_REGISTRY_JSON_OVERRIDE": registry_override' in MODAL_SHIM +def test_modal_registry_uses_static_named_secret_dependency() -> None: + assert '"TINKER_SERVING_REGISTRY_SECRET_NAME", SECRET_NAME' in MODAL_SHIM + assert "modal.Secret.from_name(REGISTRY_SECRET_NAME)" in MODAL_SHIM + assert "modal.Secret.from_dict(" not in MODAL_SHIM assert 'os.environ.get("TINKER_MODEL_REGISTRY_JSON_OVERRIDE")' in MODAL_SHIM assert '.env({"TINKER_MODEL_REGISTRY_JSON"' not in MODAL_SHIM