diff --git a/docs/adapter-serving-compatibility.md b/docs/adapter-serving-compatibility.md new file mode 100644 index 00000000..3b4ea9bc --- /dev/null +++ b/docs/adapter-serving-compatibility.md @@ -0,0 +1,24 @@ +# Adapter serving compatibility is a pre-training gate + +An exported LoRA file is not proof that a serving runtime can reproduce the +trained model. Training and evaluation plans must name the intended serving +runtime and run `scripts/adapter-serving-compat.py` before paid training and +again against the frozen adapter receipt before evaluation. + +For Nemotron-H, a Tinker adapter trained with `target_modules: "all-linear"` +is not faithfully portable to vLLM. It contains separate Mamba projections and +routed-MoE factors that the vLLM Nemotron-H LoRA surface cannot represent. +Dropping or remapping those weights may test plumbing, but the result is a +different model and must never support a quality claim. + +Use one of two truthful paths: + +1. Serve an existing all-linear checkpoint through Tinker's native sampling + client and put the authenticated OpenAI-compatible shim behind Understudy + Gateway. +2. If vLLM deployment is required, constrain training targets up front to the + supported projection set and hash-bind that choice into the training + manifest. + +The preflight emits a JSON receipt and exits non-zero for incompatible or +unknown combinations. Unknown is deliberately not equivalent to compatible. diff --git a/scripts/adapter-serving-compat.py b/scripts/adapter-serving-compat.py new file mode 100644 index 00000000..19013779 --- /dev/null +++ b/scripts/adapter-serving-compat.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Fail-closed LoRA training/serving compatibility preflight.""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +NEMOTRON_H_MARKERS = ("NVIDIA-Nemotron-3-Nano", "Nemotron-3-Nano") +VLLM_NEMOTRON_H_TARGETS = frozenset({ + "q_proj", "k_proj", "v_proj", "o_proj", "out_proj", + "up_proj", "down_proj", "lm_head", +}) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _normalize_targets(value: object) -> tuple[list[str], str | None]: + if isinstance(value, str): + return [value], value if value in {"all-linear", "all_linear"} else None + if isinstance(value, list) and all(isinstance(item, str) for item in value): + targets = sorted(set(value)) + wildcard = next((item for item in targets if item in {"all-linear", "all_linear"}), None) + return targets, wildcard + return [], None + + +def assess(config_path: Path, runtime: str, base_model: str) -> dict: + config = json.loads(config_path.read_text(encoding="utf-8")) + targets, wildcard = _normalize_targets(config.get("target_modules")) + receipt = { + "schema_version": 1, + "adapter_config": str(config_path), + "adapter_config_sha256": _sha256(config_path), + "runtime": runtime, + "base_model": base_model, + "training_target_modules": targets, + "compatibility": "unknown", + "faithful": False, + "unsupported_target_modules": [], + "reason": "unsupported runtime or model; no faithful compatibility claim is available", + } + if runtime == "tinker-sampling": + receipt.update( + compatibility="faithful", + faithful=True, + reason="Tinker sampling serves the checkpoint through its native trained-weight path", + ) + return receipt + + is_nemotron_h = any(marker in base_model for marker in NEMOTRON_H_MARKERS) + if runtime != "vllm-nemotron-h" or not is_nemotron_h: + return receipt + if wildcard: + receipt.update( + compatibility="incompatible", + reason=( + "wildcard all-linear training includes Nemotron-H Mamba and routed-MoE " + "targets that vLLM cannot faithfully represent" + ), + unsupported_target_modules=[wildcard], + ) + return receipt + if not targets: + receipt.update( + compatibility="incompatible", + reason="adapter target_modules is missing or invalid; compatibility must fail closed", + unsupported_target_modules=["missing_or_invalid"], + ) + return receipt + unsupported = sorted(set(targets) - VLLM_NEMOTRON_H_TARGETS) + if unsupported: + receipt.update( + compatibility="incompatible", + reason="one or more trained targets are outside the faithful vLLM Nemotron-H surface", + unsupported_target_modules=unsupported, + ) + return receipt + receipt.update( + compatibility="faithful", + faithful=True, + reason="all declared training targets are within the supported vLLM Nemotron-H surface", + ) + return receipt + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--adapter-config", type=Path, required=True) + parser.add_argument("--runtime", choices=("tinker-sampling", "vllm-nemotron-h"), required=True) + parser.add_argument("--base-model", required=True) + parser.add_argument("--receipt", type=Path) + args = parser.parse_args() + result = assess(args.adapter_config, args.runtime, args.base_model) + encoded = json.dumps(result, indent=2, sort_keys=True) + "\n" + if args.receipt: + args.receipt.parent.mkdir(parents=True, exist_ok=True) + args.receipt.write_text(encoded, encoding="utf-8") + print(encoded, end="") + return 0 if result["faithful"] else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/adapter_serving_compat_test.py b/scripts/adapter_serving_compat_test.py new file mode 100644 index 00000000..47e4444c --- /dev/null +++ b/scripts/adapter_serving_compat_test.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Provider-free regression tests for adapter-serving-compat.py.""" +from __future__ import annotations + +import importlib.util +import json +import tempfile +from pathlib import Path + +SCRIPT = Path(__file__).with_name("adapter-serving-compat.py") +SPEC = importlib.util.spec_from_file_location("adapter_serving_compat", SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +assert SPEC.loader +SPEC.loader.exec_module(MODULE) +MODEL = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" + + +def config(root: Path, targets: object) -> Path: + path = root / "adapter_config.json" + path.write_text(json.dumps({"target_modules": targets}) + "\n", encoding="utf-8") + return path + + +def main() -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + all_linear = MODULE.assess(config(root, "all-linear"), "vllm-nemotron-h", MODEL) + assert all_linear["compatibility"] == "incompatible" + assert all_linear["faithful"] is False + supported = MODULE.assess( + config(root, ["q_proj", "k_proj", "v_proj", "o_proj", "up_proj", "down_proj"]), + "vllm-nemotron-h", MODEL, + ) + assert supported["compatibility"] == "faithful" + unsupported = MODULE.assess( + config(root, ["q_proj", "gate_proj", "experts.w1"]), "vllm-nemotron-h", MODEL + ) + assert unsupported["unsupported_target_modules"] == ["experts.w1", "gate_proj"] + native = MODULE.assess(config(root, "all-linear"), "tinker-sampling", MODEL) + assert native["compatibility"] == "faithful" + unknown = MODULE.assess(config(root, ["q_proj"]), "vllm-nemotron-h", "some/other-model") + assert unknown["compatibility"] == "unknown" + assert unknown["faithful"] is False + missing = MODULE.assess(config(root, None), "vllm-nemotron-h", MODEL) + assert missing["compatibility"] == "incompatible" + assert missing["unsupported_target_modules"] == ["missing_or_invalid"] + assert "missing or invalid" in missing["reason"] + print("ALL ADAPTER SERVING COMPAT TESTS PASSED") + + +if __name__ == "__main__": + main() diff --git a/scripts/modal-tinker-openai-shim.py b/scripts/modal-tinker-openai-shim.py new file mode 100644 index 00000000..ac33f8f8 --- /dev/null +++ b/scripts/modal-tinker-openai-shim.py @@ -0,0 +1,100 @@ +"""Private multi-checkpoint Tinker sampling bridge for Understudy Gateway. + +Deploy with a Modal secret containing TINKER_API_KEY and +TINKER_MODEL_REGISTRY_JSON. Set TINKER_SERVING_APP_NAME and +TINKER_SERVING_SECRET_NAME at deploy time to isolate checkpoint lineages. + +For a new private checkpoint, reuse an existing API-key secret without reading +it back by setting TINKER_SERVING_API_SECRET_NAME, and place the checkpoint-only +registry in a second named secret selected with +TINKER_SERVING_REGISTRY_SECRET_NAME. Both dependencies are declared +unconditionally because Modal imports this module again in the remote runtime; +conditional Secret objects produce a local/remote dependency-count mismatch. +Modal proxy authentication is required before requests reach the shim. +""" +from __future__ import annotations + +import json +import os +import subprocess +from pathlib import Path + +import modal + +APP_NAME = os.environ.get( + "TINKER_SERVING_APP_NAME", "understudy-tinker-cedar-seed37-serving" +) +PORT = 8099 +SECRET_NAME = os.environ.get( + "TINKER_SERVING_SECRET_NAME", "understudy-tinker-serving-seed37" +) +API_SECRET_NAME = os.environ.get("TINKER_SERVING_API_SECRET_NAME", SECRET_NAME) +REGISTRY_SECRET_NAME = os.environ.get( + "TINKER_SERVING_REGISTRY_SECRET_NAME", SECRET_NAME +) +# Keep this list structurally identical in the deploy process and the remote +# container import. Values stay in Modal's encrypted secret plane. +runtime_secrets = [ + modal.Secret.from_name(API_SECRET_NAME), + modal.Secret.from_name(REGISTRY_SECRET_NAME), +] +BASE_MODEL = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" +TINKER_VERSION = "0.24.0" +COOKBOOK_VERSION = "0.5.3" + +app = modal.App(APP_NAME) +image = ( + modal.Image.debian_slim(python_version="3.11") + .apt_install("git") + .pip_install( + f"tinker=={TINKER_VERSION}", + f"tinker-cookbook=={COOKBOOK_VERSION}", + "httpx>=0.27,<1", + ) + .add_local_file("scripts/tinker-openai-shim.py", "/opt/understudy/tinker-openai-shim.py") + .add_local_file("scripts/tinker_openai_compat.py", "/opt/understudy/tinker_openai_compat.py") + .add_local_file("scripts/tinker_renderer_compat.py", "/opt/understudy/tinker_renderer_compat.py") +) + + +@app.function( + image=image, + secrets=runtime_secrets, + timeout=60 * 60, + scaledown_window=300, + max_containers=4, +) +@modal.concurrent(max_inputs=64) +@modal.web_server(PORT, startup_timeout=10 * 60, requires_proxy_auth=True) +def serve() -> None: + registry = json.loads( + os.environ.get("TINKER_MODEL_REGISTRY_JSON_OVERRIDE") + or os.environ["TINKER_MODEL_REGISTRY_JSON"] + ) + if not isinstance(registry, dict) or not registry: + raise RuntimeError("TINKER_MODEL_REGISTRY_JSON must be a non-empty object") + registry_path = Path("/tmp/tinker-model-registry.json") + registry_path.write_text(json.dumps(registry, sort_keys=True), encoding="utf-8") + registry_path.chmod(0o600) + subprocess.Popen( + [ + "python", + "/opt/understudy/tinker-openai-shim.py", + "--model-registry-file", + str(registry_path), + "--tokenizer-model", + BASE_MODEL, + "--renderer", + "nemotron3_disable_thinking", + "--host", + "0.0.0.0", + "--trusted-proxy-auth", + "--port", + str(PORT), + "--max-workers", + "64", + "--max-tokens", + "2048", + ], + env={**os.environ, "TINKER_TRUSTED_PROXY_AUTH": "modal"}, + ) diff --git a/scripts/tinker-openai-shim.py b/scripts/tinker-openai-shim.py new file mode 100644 index 00000000..292053e6 --- /dev/null +++ b/scripts/tinker-openai-shim.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +"""Minimal OpenAI-compatible /v1/chat/completions shim in front of a Tinker +sampling client, so the Node AutomationBench runner can score Tinker base models +and Tinker-trained checkpoints without a dedicated deployment. + +Tinker's `tools=` path raises NotImplementedError, so tool calls are driven +through plain sampling with the model's own renderer, exactly as the RL arms do. + + TINKER_API_KEY=... python scripts/tinker-openai-shim.py \ + --base-model nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 --renderer nemotron3 --port 8099 + +Pass `--model-path tinker://...` to serve a trained checkpoint (a LoRA adapter +over the same base) instead of the base weights; everything else is unchanged, +so base and tuned runs are scored through one identical sampling path. +""" +from __future__ import annotations + +import argparse +import json +import os +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import TimeoutError as FutureTimeoutError +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +# pyqwest (the Rust HTTP backend tinker prefers) carries its own root store and +# rejects otherwise-valid certificates on some Linux hosts. Opt in to httpx's +# system trust store when that happens; the wire protocol is identical. +if os.environ.get("TINKER_DISABLE_PYQWEST") == "1": + import httpx + import tinker._base_client as _tinker_base_client + + _tinker_base_client._default_pyqwest_transport = lambda: httpx.AsyncHTTPTransport(retries=2) + +import tinker +from tinker_cookbook.renderers import get_renderer +from tinker_cookbook.tokenizer_utils import get_tokenizer + +from tinker_openai_compat import ( + InvalidRequestError, + bearer_authorized, + build_chat_completion, + normalize_assistant_message, + normalize_finish_reason, + openai_error_response, + parse_chat_request, +) +from tinker_renderer_compat import renderer_messages, renderer_tools + +parser = argparse.ArgumentParser() +model_group = parser.add_mutually_exclusive_group(required=True) +model_group.add_argument("--base-model") +model_group.add_argument("--model-path", help="Tinker sampler-state/checkpoint path returned by save_weights_for_sampler().") +model_group.add_argument( + "--model-registry-file", + help="JSON object mapping public model aliases to Tinker checkpoint paths.", +) +parser.add_argument("--tokenizer-model", help="Base model used for tokenization/rendering when --model-path is selected.") +parser.add_argument("--renderer", required=True) +parser.add_argument("--host", default="127.0.0.1") +parser.add_argument( + "--trusted-proxy-auth", + action="store_true", + help="Allow a non-loopback bind without app bearer auth only behind an authenticated reverse proxy.", +) +parser.add_argument("--port", type=int, default=8099) +parser.add_argument("--max-tokens", type=int, default=512) +parser.add_argument("--max-workers", type=int, default=16, help="in-flight samples; raise it for rollout mining") +args = parser.parse_args() +service_token = os.environ.get("TINKER_SHIM_BEARER_TOKEN") +trusted_modal_proxy = ( + args.trusted_proxy_auth + and os.environ.get("TINKER_TRUSTED_PROXY_AUTH") == "modal" + and bool(os.environ.get("MODAL_TASK_ID")) +) +if ( + args.host not in {"127.0.0.1", "::1", "localhost"} + and not service_token + and not trusted_modal_proxy +): + raise SystemExit( + "TINKER_SHIM_BEARER_TOKEN or an attested Modal proxy runtime is required for non-loopback binds" + ) +request_timeout = 300 +log_path = os.environ.get("TINKER_SHIM_LOG", "/tmp/tinker-openai-shim.log") +log_lock = threading.Lock() +active_lock = threading.Lock() +active_requests = 0 + + +def log_event(event, **fields): + record = {"ts": time.time(), "event": event, **fields} + with log_lock: + with open(log_path, "a", encoding="utf-8") as stream: + stream.write(json.dumps(record) + "\n") + # Modal's durable app logs are the only operator-visible surface for this + # web-server process. Records contain request ids and bounded error types, + # never prompts, tool arguments, credentials, or response text. + print(json.dumps(record, sort_keys=True), flush=True) + + +service = tinker.ServiceClient(_client_config={"use_pyqwest_transport": False}) +if args.model_registry_file: + registry = json.loads(open(args.model_registry_file, encoding="utf-8").read()) + if not isinstance(registry, dict) or not registry or not all( + isinstance(alias, str) and alias and isinstance(path, str) and path + for alias, path in registry.items() + ): + raise SystemExit("--model-registry-file must contain a non-empty string-to-string JSON object") + samplers = { + alias: service.create_sampling_client(model_path=path) + for alias, path in registry.items() + } +elif args.model_path: + samplers = {args.model_path: service.create_sampling_client(model_path=args.model_path)} +else: + samplers = {args.base_model: service.create_sampling_client(base_model=args.base_model)} +tokenizer_model = args.tokenizer_model or args.base_model +if not tokenizer_model: + raise SystemExit("--tokenizer-model is required with checkpoint paths") +renderer = get_renderer(args.renderer, get_tokenizer(tokenizer_model)) +pool = ThreadPoolExecutor(max_workers=args.max_workers) +served_models = sorted(samplers) + + +def sample(model, messages, tools, temperature, max_tokens): + system_prompt, conversation = renderer_messages(messages) + tool_specs = renderer_tools(tools) + prefix = renderer.create_conversation_prefix_with_tools( + tool_specs, + system_prompt=system_prompt, + ) + prompt = renderer.build_generation_prompt([*prefix, *conversation]) + params = tinker.types.SamplingParams( + max_tokens=max_tokens, + temperature=temperature, + stop=renderer.get_stop_sequences(), + ) + result = samplers[model].sample(prompt=prompt, sampling_params=params, num_samples=1).result() + sequence = result.sequences[0] + tokens = sequence.tokens + message, termination = renderer.parse_response(tokens) + openai_message = renderer.to_openai_message(message) + finish_reason = normalize_finish_reason( + stop_reason=sequence.stop_reason, + termination=getattr(termination, "value", termination), + completion_tokens=len(tokens), + max_tokens=max_tokens, + ) + return openai_message, prompt.length, len(tokens), finish_reason + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *_args): # keep the console readable + return + + def _send_json(self, status, payload): + encoded = json.dumps(payload).encode() + self.send_response(status) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(encoded))) + self.end_headers() + try: + self.wfile.write(encoded) + except BrokenPipeError: + pass + + def _authorized(self): + # Loopback remains usable for local provider-parity tests. Any + # network-facing bind is already required to configure a token. + if not service_token: + return True + return bearer_authorized(self.headers.get("authorization"), service_token) + + def do_GET(self): # noqa: N802 - required by BaseHTTPRequestHandler + if not self._authorized(): + self._send_json(401, {"error": {"message": "unauthorized", "type": "authentication_error"}}) + return + if self.path == "/health": + self._send_json(200, {"status": "ok", "models": served_models}) + return + if self.path == "/v1/models": + self._send_json( + 200, + {"object": "list", "data": [{"id": model, "object": "model"} for model in served_models]}, + ) + return + self._send_json(404, {"error": {"message": "not found", "type": "invalid_request_error"}}) + + def do_POST(self): # noqa: N802 - required by BaseHTTPRequestHandler + global active_requests + if not self._authorized(): + self._send_json(401, {"error": {"message": "unauthorized", "type": "authentication_error"}}) + return + if self.path != "/v1/chat/completions": + self._send_json(404, {"error": {"message": "not found", "type": "invalid_request_error"}}) + return + request_id = self.headers.get("x-request-id") or str(uuid.uuid4()) + started = time.monotonic() + with active_lock: + active_requests += 1 + in_flight = active_requests + log_event("start", request_id=request_id, in_flight=in_flight) + try: + raw_content_length = self.headers.get("content-length") + if raw_content_length is None: + raise InvalidRequestError("content-length header is required") + try: + content_length = int(raw_content_length) + except ValueError as error: + raise InvalidRequestError("content-length header must be an integer") from error + if content_length < 0: + raise InvalidRequestError("content-length header must be non-negative") + body = parse_chat_request(self.rfile.read(content_length)) + requested_model = body.get("model") + if requested_model is None and len(served_models) == 1: + requested_model = served_models[0] + if requested_model not in samplers: + self._send_json( + 400, + {"error": {"message": "unknown model", "type": "invalid_request_error"}}, + ) + return + for attempt in range(2): + try: + message, prompt_tokens, completion_tokens, finish_reason = pool.submit( + sample, + requested_model, + body["messages"], + body.get("tools") or [], + float(body.get("temperature", 0.0)), + int(body.get("max_tokens", args.max_tokens)), + ).result(timeout=request_timeout) + break + except FutureTimeoutError: + log_event("timeout", request_id=request_id, attempt=attempt + 1, seconds=request_timeout) + if attempt == 1: + raise TimeoutError(f"sampling exceeded {request_timeout}s twice") + log_event("retry", request_id=request_id, attempt=attempt + 2) + message = normalize_assistant_message(message, request_id) + if message.get("tool_calls"): + finish_reason = "tool_calls" + payload = build_chat_completion( + message, + prompt_tokens, + completion_tokens, + finish_reason, + model=requested_model, + request_id=request_id, + ) + status = 200 + except Exception as error: # surface upstream failures as HTTP errors + log_event("error", request_id=request_id, error=type(error).__name__) + status, payload = openai_error_response(error) + finally: + elapsed = time.monotonic() - started + with active_lock: + active_requests -= 1 + in_flight = active_requests + log_event("done", request_id=request_id, elapsed_seconds=round(elapsed, 3), in_flight=in_flight) + self._send_json(status, payload) + + +print(f"tinker shim on {args.host}:{args.port} for {served_models} ({args.renderer})", flush=True) +ThreadingHTTPServer((args.host, args.port), Handler).serve_forever() diff --git a/scripts/tinker_openai_compat.py b/scripts/tinker_openai_compat.py new file mode 100644 index 00000000..591db6da --- /dev/null +++ b/scripts/tinker_openai_compat.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +"""Side-effect-free OpenAI-compat helpers for the Tinker sampling shim. + +Importing this module must NOT touch argparse, the network, or a Tinker +ServiceClient, so provider-free unit tests can import it directly. The shim +(``scripts/tinker-openai-shim.py``) imports these two helpers and does the live +sampling itself. + +Finish-reason contract (see tinker.types.StopReason == Literal['length','stop'] +and tinker_cookbook ParseTermination == {'stop_sequence','eos','malformed'}): + + * Prefer the upstream sampler stop_reason when present ('length'->'length', + 'stop'->'stop'). + * Else use the renderer termination: a clean stop ('stop_sequence'/'eos') + -> 'stop'; a 'malformed' (truncated) termination is treated as reason-absent + for the stop/length distinction and falls through to cap inference. + * Only when no upstream reason is present do we infer 'length' from + completion_tokens >= max_tokens; otherwise fall back to 'stop'. + +We never invent OpenAI reasons the upstream cannot justify (e.g. no +'content_filter' unless the upstream actually reports one). +""" +from __future__ import annotations + +import hmac +import json +import time +import uuid +from typing import Any, Optional + +# Values tinker.types.StopReason may take. +_SAMPLER_STOP = "stop" +_SAMPLER_LENGTH = "length" +# tinker_cookbook renderers.base.ParseTermination values that mean a clean end. +_CLEAN_TERMINATIONS = ("stop_sequence", "eos") + + +class InvalidRequestError(ValueError): + """A bounded client request error that must be returned as HTTP 400.""" + + +def parse_chat_request(raw: bytes) -> dict[str, Any]: + """Decode and minimally validate an OpenAI chat-completions request.""" + try: + body = json.loads(raw) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise InvalidRequestError("request body must be valid UTF-8 JSON") from error + if not isinstance(body, dict): + raise InvalidRequestError("request body must be a JSON object") + if not isinstance(body.get("messages"), list): + raise InvalidRequestError("messages must be a JSON array") + return body + + +def openai_error_response(error: Exception) -> tuple[int, dict[str, Any]]: + """Map sampler failures without disguising deterministic client errors. + + Tinker raises ``BadRequestError`` for prompt-plus-generation context-window + overflow. Returning 500 for that condition made the Gateway and evaluator + treat deterministic, model-incompatible requests as retryable provider + pressure. Keep the response bounded and OpenAI-shaped while preserving the + actionable error class. + """ + detail = str(error) + lowered = detail.lower() + type_name = type(error).__name__ + if isinstance(error, InvalidRequestError): + return 400, { + "error": { + "message": detail[:500], + "type": "invalid_request_error", + "code": "invalid_request", + } + } + if type_name == "BadRequestError" or "exceeds the model's context window" in lowered: + code = "context_length_exceeded" if "context window" in lowered else "invalid_request" + return 400, { + "error": { + "message": ( + "request exceeds the model context window" + if code == "context_length_exceeded" + else "upstream rejected the request" + ), + "type": "invalid_request_error", + "code": code, + } + } + if isinstance(error, TimeoutError): + return 504, { + "error": { + "message": "upstream sampling timed out", + "type": "server_error", + "code": "upstream_timeout", + } + } + return 500, { + "error": { + "message": "upstream sampling failed", + "type": "server_error", + "code": "upstream_error", + } + } + + +def normalize_assistant_message(message: dict[str, Any], request_id: str) -> dict[str, Any]: + """Return strict OpenAI assistant shape with stable tool-call IDs/arguments.""" + normalized = {"role": "assistant", "content": message.get("content") or ""} + raw_calls = message.get("tool_calls") or [] + if raw_calls: + calls = [] + for index, raw in enumerate(raw_calls): + function = raw.get("function") or {} + arguments = function.get("arguments", "{}") + if isinstance(arguments, dict): + arguments = json.dumps(arguments, sort_keys=True) + if not isinstance(arguments, str): + raise ValueError("parsed tool arguments must be a JSON string or object") + json.loads(arguments) + call_id = raw.get("id") or ( + "call_" + + uuid.uuid5( + uuid.NAMESPACE_URL, + f"understudy:tinker:{request_id}:{index}:{function.get('name')}", + ).hex + ) + calls.append( + { + "type": "function", + "id": call_id, + "function": {"name": function["name"], "arguments": arguments}, + } + ) + normalized["tool_calls"] = calls + return normalized + + +def bearer_authorized(header: Optional[str], expected_token: Optional[str]) -> bool: + """Constant-time Bearer-token verification. + + A configured service token is mandatory for non-loopback deployments. The + caller decides whether an unset token is acceptable for its bind address. + """ + if not expected_token or not header or not header.startswith("Bearer "): + return False + supplied = header.removeprefix("Bearer ").strip() + return bool(supplied) and hmac.compare_digest(supplied, expected_token) + + +def normalize_finish_reason( + stop_reason: Optional[str] = None, + termination: Optional[str] = None, + completion_tokens: Optional[int] = None, + max_tokens: Optional[int] = None, +) -> str: + """Return an OpenAI-compatible finish_reason ('stop' or 'length'). + + ``stop_reason`` : upstream sampler StopReason ('stop'/'length') or None. + ``termination`` : renderer ParseTermination value or None. + ``completion_tokens`` / ``max_tokens`` : used only for cap inference when no + upstream reason is available. + """ + # Coerce both signals to plain strings so an enum/StrEnum (or anything with a + # str form) compares correctly; None stays None. + stop = None if stop_reason is None else str(stop_reason) + term = None if termination is None else str(termination) + if stop == _SAMPLER_LENGTH: + return "length" + if stop == _SAMPLER_STOP: + return "stop" + # No authoritative sampler reason: consult a clean renderer termination. + if term in _CLEAN_TERMINATIONS: + return "stop" + # Reason absent (or 'malformed'/truncated): infer length only at the cap. + if ( + completion_tokens is not None + and max_tokens is not None + and completion_tokens >= max_tokens + ): + return "length" + return "stop" + + +def build_chat_completion( + message: dict[str, Any], + prompt_tokens: int, + completion_tokens: int, + finish_reason: str, + *, + model: str = "unknown", + request_id: Optional[str] = None, + created: Optional[int] = None, +) -> dict: + """Build the /v1/chat/completions response body. + + finish_reason is a required positional argument: constructing the choice + object without a defined finish_reason is impossible here, which is exactly + the undefined-variable (NameError) failure this module exists to prevent. + """ + completion_id = request_id or str(uuid.uuid4()) + return { + "id": f"chatcmpl-{completion_id}", + "object": "chat.completion", + "created": int(time.time()) if created is None else created, + "model": model, + "choices": [ + { + "index": 0, + "message": message, + "finish_reason": finish_reason, + } + ], + "usage": { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens, + }, + } diff --git a/scripts/tinker_openai_compat_test.py b/scripts/tinker_openai_compat_test.py new file mode 100644 index 00000000..6cfe5ff1 --- /dev/null +++ b/scripts/tinker_openai_compat_test.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""Provider-free regression for the Tinker shim's OpenAI-compat helpers. + +Runs with plain system python3 (no tinker, no network): + python3 scripts/tinker_openai_compat_test.py + +Proves the finish-reason contract and that building the response body cannot +recur the original undefined-`finish_reason` NameError. +""" +from enum import Enum + +from tinker_openai_compat import ( + bearer_authorized, + build_chat_completion, + normalize_assistant_message, + normalize_finish_reason, + openai_error_response, +) + + +class _ParseTermination(str, Enum): + """Shape-compatible stand-in for tinker_cookbook ParseTermination.""" + STOP_SEQUENCE = "stop_sequence" + EOS = "eos" + MALFORMED = "malformed" + + +def check(name, cond): + if not cond: + raise AssertionError(f"FAIL: {name}") + print(f" ok: {name}") + + +def test_upstream_stop(): + check("upstream stop -> stop", + normalize_finish_reason(stop_reason="stop", completion_tokens=384, max_tokens=384) == "stop") + + +def test_upstream_length(): + # upstream length wins even below the cap + check("upstream length -> length", + normalize_finish_reason(stop_reason="length", completion_tokens=10, max_tokens=384) == "length") + + +def test_absent_at_cap_infers_length(): + check("absent + at cap -> length", + normalize_finish_reason(stop_reason=None, termination=None, completion_tokens=384, max_tokens=384) == "length") + + +def test_absent_under_cap_falls_back_stop(): + check("absent + under cap -> stop", + normalize_finish_reason(stop_reason=None, termination=None, completion_tokens=12, max_tokens=384) == "stop") + + +def test_enum_shaped_terminations_map_to_stop(): + # exactly how the shim passes it: getattr(term, "value", term) + for term in (_ParseTermination.EOS, _ParseTermination.STOP_SEQUENCE): + passed = getattr(term, "value", term) + check(f"enum termination {term.value!r} -> stop", + normalize_finish_reason(termination=passed, completion_tokens=5, max_tokens=384) == "stop") + + +def test_unknown_non_clean_termination_falls_through_to_cap(): + # 'malformed' (or any non-clean) is NOT treated as a stop/length signal; + # it falls through to cap inference. + passed = getattr(_ParseTermination.MALFORMED, "value", _ParseTermination.MALFORMED) + check("malformed + at cap -> length (cap inference)", + normalize_finish_reason(termination=passed, completion_tokens=384, max_tokens=384) == "length") + check("malformed + under cap -> stop (fallback)", + normalize_finish_reason(termination=passed, completion_tokens=7, max_tokens=384) == "stop") + + +def test_payload_requires_defined_finish_reason(): + # The exact code path that once raised NameError: build the choice object + # with a normalized finish_reason. It is a required positional arg, so an + # undefined variable cannot silently slip through. + fr = normalize_finish_reason(stop_reason="stop", completion_tokens=3, max_tokens=384) + payload = build_chat_completion({"role": "assistant", "content": "hello"}, 100, 3, fr) + check("payload choices[0].finish_reason present", payload["choices"][0]["finish_reason"] == "stop") + check("payload content propagated", payload["choices"][0]["message"]["content"] == "hello") + check("payload usage propagated", + payload["usage"] == { + "prompt_tokens": 100, + "completion_tokens": 3, + "total_tokens": 103, + }) + + fr_len = normalize_finish_reason(stop_reason="length", completion_tokens=384, max_tokens=384) + payload_len = build_chat_completion({"role": "assistant", "content": ""}, 100, 384, fr_len) + check("length propagates into payload", payload_len["choices"][0]["finish_reason"] == "length") + + tool_message = { + "role": "assistant", + "content": "", + "tool_calls": [{ + "type": "function", + "id": "call_1", + "function": {"name": "create-task", "arguments": '{"title":"A"}'}, + }], + } + payload_tool = build_chat_completion(tool_message, 100, 20, "stop") + check( + "tool calls propagate without text flattening", + payload_tool["choices"][0]["message"] == tool_message, + ) + + +def test_bearer_auth_is_fail_closed(): + check("missing expected token rejects", bearer_authorized("Bearer x", None) is False) + check("missing header rejects", bearer_authorized(None, "secret") is False) + check("wrong scheme rejects", bearer_authorized("Basic secret", "secret") is False) + check("wrong token rejects", bearer_authorized("Bearer wrong", "secret") is False) + check("matching token authorizes", bearer_authorized("Bearer secret", "secret") is True) + + +def test_parsed_tool_call_gets_stable_openai_shape(): + parsed = { + "role": "assistant", + "content": "", + "tool_calls": [{ + "type": "function", + "id": None, + "function": {"name": "create-task", "arguments": {"title": "A"}}, + }], + } + first = normalize_assistant_message(parsed, "request-1") + second = normalize_assistant_message(parsed, "request-1") + check("missing tool id synthesized", first["tool_calls"][0]["id"].startswith("call_")) + check("synthesized tool id stable", first == second) + check( + "tool arguments serialized as JSON string", + first["tool_calls"][0]["function"]["arguments"] == '{"title": "A"}', + ) + + +def test_error_mapping_preserves_context_overflow_semantics(): + class BadRequestError(Exception): + pass + + status, payload = openai_error_response( + BadRequestError("Prompt length plus max_tokens exceeds the model's context window") + ) + check("context overflow is HTTP 400", status == 400) + check("context overflow has stable code", payload["error"]["code"] == "context_length_exceeded") + status, payload = openai_error_response(TimeoutError("sampling timed out")) + check("timeout is HTTP 504", status == 504) + check("timeout is retry-class server error", payload["error"]["type"] == "server_error") + + +def main(): + tests = [ + test_upstream_stop, + test_upstream_length, + test_absent_at_cap_infers_length, + test_absent_under_cap_falls_back_stop, + test_enum_shaped_terminations_map_to_stop, + test_unknown_non_clean_termination_falls_through_to_cap, + test_payload_requires_defined_finish_reason, + test_bearer_auth_is_fail_closed, + test_parsed_tool_call_gets_stable_openai_shape, + test_error_mapping_preserves_context_overflow_semantics, + ] + for t in tests: + print(t.__name__) + t() + print(f"\nALL {len(tests)} SHIM COMPAT TESTS PASSED") + + +if __name__ == "__main__": + main() diff --git a/scripts/tinker_renderer_compat.py b/scripts/tinker_renderer_compat.py new file mode 100644 index 00000000..b521018f --- /dev/null +++ b/scripts/tinker_renderer_compat.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""OpenAI-to-Tinker renderer conversion for faithful multi-turn tool use.""" +from __future__ import annotations + +import json + +from tinker_cookbook.renderers.base import Message, ToolCall, ToolSpec + + +def _message_text(content: object) -> str: + """Normalize OpenAI text content without silently dropping non-text parts.""" + if content is None: + return "" + if isinstance(content, str): + return content + if not isinstance(content, list): + raise ValueError("message content must be string, null, or a text-part array") + + parts: list[str] = [] + for part in content: + if not isinstance(part, dict) or part.get("type") not in {"text", "input_text"}: + raise ValueError("only text/input_text message content parts are supported") + text = part.get("text") + if not isinstance(text, str): + raise ValueError("message text content part requires a string text field") + parts.append(text) + return "".join(parts) + + +def renderer_messages(messages: list[dict]) -> tuple[str, list[Message]]: + """Losslessly convert OpenAI history into the cookbook renderer contract.""" + system_parts: list[str] = [] + converted: list[Message] = [] + for raw in messages: + role = raw.get("role") + content = _message_text(raw.get("content")) + if role == "system": + system_parts.append(content) + continue + if role not in {"user", "assistant", "tool"}: + raise ValueError(f"unsupported message role: {role!r}") + message: Message = {"role": role, "content": content} + if role == "assistant" and raw.get("tool_calls"): + calls: list[ToolCall] = [] + for raw_call in raw["tool_calls"]: + function = raw_call.get("function") or {} + arguments = function.get("arguments", "{}") + if isinstance(arguments, dict): + arguments = json.dumps(arguments, sort_keys=True) + if not isinstance(arguments, str): + raise ValueError("tool call arguments must be a JSON string or object") + json.loads(arguments) + calls.append( + ToolCall( + id=raw_call.get("id"), + function=ToolCall.FunctionBody( + name=function["name"], + arguments=arguments, + ), + ) + ) + message["tool_calls"] = calls + if role == "tool": + if not raw.get("tool_call_id"): + raise ValueError("tool result requires tool_call_id") + message["tool_call_id"] = raw["tool_call_id"] + if raw.get("name"): + message["name"] = raw["name"] + converted.append(message) + return "\n\n".join(system_parts), converted + + +def renderer_tools(tools: list[dict]) -> list[ToolSpec]: + """Convert OpenAI function-tool declarations to cookbook ToolSpec values.""" + converted: list[ToolSpec] = [] + for raw in tools: + if raw.get("type") != "function" or not isinstance(raw.get("function"), dict): + raise ValueError("only OpenAI function tools are supported") + function = raw["function"] + converted.append( + { + "name": function["name"], + "description": function.get("description") or "", + "parameters": function.get("parameters") + or {"type": "object", "properties": {}}, + } + ) + return converted diff --git a/scripts/tinker_renderer_compat_test.py b/scripts/tinker_renderer_compat_test.py new file mode 100644 index 00000000..19f4f027 --- /dev/null +++ b/scripts/tinker_renderer_compat_test.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Provider-free exact Nemotron renderer roundtrip for tool-capable serving.""" +from __future__ import annotations + +from tinker_cookbook.renderers import get_renderer +from tinker_cookbook.tokenizer_utils import get_tokenizer + +from tinker_openai_compat import normalize_assistant_message +from tinker_renderer_compat import renderer_messages, renderer_tools + +MODEL = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" + + +def main() -> None: + tokenizer = get_tokenizer(MODEL) + renderer = get_renderer("nemotron3_disable_thinking", tokenizer) + tools = [{ + "type": "function", + "function": { + "name": "create-task", + "description": "Create a task", + "parameters": { + "type": "object", + "properties": {"title": {"type": "string"}}, + "required": ["title"], + }, + }, + }] + messages = [ + {"role": "system", "content": "Operate Cedar faithfully."}, + { + "role": "user", + "content": [ + {"type": "text", "text": "Create task "}, + {"type": "input_text", "text": "A."}, + ], + }, + { + "role": "assistant", + "content": "", + "tool_calls": [{ + "type": "function", + "id": "call_prior", + "function": {"name": "create-task", "arguments": '{"title":"A"}'}, + }], + }, + {"role": "tool", "tool_call_id": "call_prior", "content": '{"success":true}'}, + {"role": "user", "content": "Continue."}, + ] + system, history = renderer_messages(messages) + prefix = renderer.create_conversation_prefix_with_tools(renderer_tools(tools), system) + prompt = renderer.build_generation_prompt([*prefix, *history]) + assert prompt.length > 0 + decoded = tokenizer.decode(prompt.to_ints()) + assert "create-task" in decoded + assert "Create task A." in decoded + assert "" in decoded + assert '{"success":true}' in decoded + + try: + renderer_messages([{ + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": "private"}}], + }]) + except ValueError as error: + assert "only text/input_text" in str(error) + else: + raise AssertionError("non-text content must fail closed") + + sampled = tokenizer.encode( + "\n\n\nA\n" + "\n\n\n<|im_end|>", + add_special_tokens=False, + ) + parsed, termination = renderer.parse_response(sampled) + assert str(termination) == "stop_sequence" + openai = normalize_assistant_message(renderer.to_openai_message(parsed), "probe-request") + call = openai["tool_calls"][0] + assert call["id"].startswith("call_") + assert call["function"]["name"] == "create-task" + assert call["function"]["arguments"] == '{"title": "A"}' + print("EXACT NEMOTRON TOOL ROUNDTRIP PASSED") + + +if __name__ == "__main__": + main() diff --git a/tests/tinker_openai_compat_test.py b/tests/tinker_openai_compat_test.py new file mode 100644 index 00000000..a9b9e553 --- /dev/null +++ b/tests/tinker_openai_compat_test.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +MODULE_PATH = Path(__file__).parents[1] / "scripts" / "tinker_openai_compat.py" +SPEC = importlib.util.spec_from_file_location("tinker_openai_compat", MODULE_PATH) +assert SPEC and SPEC.loader +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +def test_chat_completion_preserves_tool_calls() -> None: + message = { + "role": "assistant", + "content": "", + "tool_calls": [{ + "type": "function", + "id": "call-1", + "function": {"name": "create-task", "arguments": '{"title":"A"}'}, + }], + } + payload = MODULE.build_chat_completion( + message, + 10, + 4, + "stop", + model="cedar-test", + request_id="request-1", + created=123, + ) + assert payload["choices"][0]["message"] == message + assert payload["id"] == "chatcmpl-request-1" + assert payload["object"] == "chat.completion" + assert payload["created"] == 123 + assert payload["model"] == "cedar-test" + assert payload["choices"][0]["index"] == 0 + assert payload["usage"]["total_tokens"] == 14 + + +def test_tool_call_finish_reason_is_supported() -> None: + payload = MODULE.build_chat_completion( + {"role": "assistant", "content": "", "tool_calls": []}, + 1, + 1, + "tool_calls", + ) + assert payload["choices"][0]["finish_reason"] == "tool_calls" + + +def test_parsed_tool_call_gets_stable_id_and_json_arguments() -> None: + parsed = { + "role": "assistant", + "content": "", + "tool_calls": [{ + "type": "function", + "id": None, + "function": {"name": "create-task", "arguments": {"title": "A"}}, + }], + } + first = MODULE.normalize_assistant_message(parsed, "request-1") + second = MODULE.normalize_assistant_message(parsed, "request-1") + assert first == second + assert first["tool_calls"][0]["id"].startswith("call_") + assert first["tool_calls"][0]["function"]["arguments"] == '{"title": "A"}' + + +def test_context_overflow_is_not_mislabeled_as_retryable_provider_failure() -> None: + class BadRequestError(Exception): + pass + + status, payload = MODULE.openai_error_response( + BadRequestError("Prompt length plus max_tokens exceeds the model's context window") + ) + assert status == 400 + assert payload["error"]["type"] == "invalid_request_error" + assert payload["error"]["code"] == "context_length_exceeded" + assert "Prompt length" not in payload["error"]["message"] + + +def test_timeout_remains_a_retry_class_server_error() -> None: + status, payload = MODULE.openai_error_response(TimeoutError("sampling timed out")) + assert status == 504 + assert payload["error"]["code"] == "upstream_timeout" + + +def test_upstream_error_details_are_redacted() -> None: + status, payload = MODULE.openai_error_response( + RuntimeError("https://private.invalid tinker://private secret-header") + ) + assert status == 500 + rendered = str(payload) + assert "private.invalid" not in rendered + assert "tinker://" not in rendered + assert "secret-header" not in rendered + + +def test_malformed_chat_requests_fail_closed() -> None: + cases = ( + (b"{", "valid UTF-8 JSON"), + (b"[]", "JSON object"), + (b'{"model":"cedar-test"}', "messages"), + (b'{"messages":{}}', "messages"), + ) + for raw, expected in cases: + try: + MODULE.parse_chat_request(raw) + except MODULE.InvalidRequestError as error: + assert expected in str(error) + status, payload = MODULE.openai_error_response(error) + assert status == 400 + assert payload["error"]["type"] == "invalid_request_error" + assert payload["error"]["code"] == "invalid_request" + else: + raise AssertionError(f"invalid request unexpectedly accepted: {raw!r}") + + assert MODULE.parse_chat_request(b'{"messages":[]}') == {"messages": []} diff --git a/tests/tinker_shim_static_test.py b/tests/tinker_shim_static_test.py new file mode 100644 index 00000000..d39916c3 --- /dev/null +++ b/tests/tinker_shim_static_test.py @@ -0,0 +1,42 @@ +"""Provider-free static safety checks for the private Tinker serving entrypoint.""" + +from pathlib import Path + + +SHIM = (Path(__file__).parents[1] / "scripts" / "tinker-openai-shim.py").read_text() +MODAL_SHIM = ( + Path(__file__).parents[1] / "scripts" / "modal-tinker-openai-shim.py" +).read_text() + + +def test_request_accounting_wraps_request_parsing() -> None: + increment = SHIM.index("active_requests += 1") + protected = SHIM.index("try:", increment) + parse = SHIM.index("body = parse_chat_request", protected) + cleanup = SHIM.index("active_requests -= 1", parse) + assert increment < protected < parse < cleanup + + +def test_network_auth_requires_attested_modal_runtime_or_token() -> None: + assert 'os.environ.get("TINKER_TRUSTED_PROXY_AUTH") == "modal"' in SHIM + assert 'bool(os.environ.get("MODAL_TASK_ID"))' in SHIM + assert "and not trusted_modal_proxy" in SHIM + + +def test_tool_calls_override_finish_reason() -> None: + assert 'if message.get("tool_calls"):' in SHIM + assert 'finish_reason = "tool_calls"' in SHIM + + +def test_logs_expose_only_error_class() -> None: + error_log = 'log_event("error", request_id=request_id, error=type(error).__name__)' + assert error_log in SHIM + assert "detail=str(error)" not in SHIM + + +def test_modal_registry_uses_static_named_secret_dependency() -> None: + assert '"TINKER_SERVING_REGISTRY_SECRET_NAME", SECRET_NAME' in MODAL_SHIM + assert "modal.Secret.from_name(REGISTRY_SECRET_NAME)" in MODAL_SHIM + assert "modal.Secret.from_dict(" not in MODAL_SHIM + assert 'os.environ.get("TINKER_MODEL_REGISTRY_JSON_OVERRIDE")' in MODAL_SHIM + assert '.env({"TINKER_MODEL_REGISTRY_JSON"' not in MODAL_SHIM