feat(deploy): add container image, Helm chart and Envoy AI Gateway, litellm examples

.dockerignore

@@ -1,5 +1,6 @@
 .git
 .mypy_cache
+target
 .pytest_cache
 .ruff_cache
 .venv

Dockerfile

@@ -0,0 +1,92 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# syntax=docker/dockerfile:1.7
+
+# Production image for the standalone `switchyard-server` proxy.
+#
+# The Dockerfiles under `benchmark/` build the Python launcher and an
+# unoptimised server for benchmark harnesses. This one builds only the release
+# proxy and ships it on a slim runtime with no toolchain attached.
+#
+#   docker build -t switchyard-server:0.2.0 .
+#   docker run --rm -p 4000:4000 \
+#     -v "$PWD/routes.toml:/etc/switchyard/routes.toml:ro" \
+#     -e OPENROUTER_API_KEY \
+#     switchyard-server:0.2.0 --config /etc/switchyard/routes.toml
+#
+# CPU baseline: `.cargo/config.toml` compiles x86_64 with `-C
+# target-cpu=x86-64-v3`, so the resulting binary needs an AVX2-class CPU
+# (Haswell 2013+), and aarch64 with `-C target-cpu=neoverse-n1`. This matches
+# the published wheels documented in INSTALLATION.md.
+
+ARG RUST_VERSION=1.96.1
+ARG DEBIAN_RELEASE=bookworm
+
+########################################
+# Build stage
+########################################
+FROM rust:${RUST_VERSION}-${DEBIAN_RELEASE} AS builder
+
+# `aws-lc-rs`, pulled in by rustls, builds native code and needs cmake plus a
+# libclang for its bindgen step. Everything else in the dependency graph is
+# pure Rust: reqwest is configured for rustls, so no OpenSSL headers.
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y \
+        cmake \
+        clang \
+        libclang-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /src
+
+# Copy only what the server's dependency graph needs to resolve. Cargo parses
+# every workspace manifest even for `-p switchyard-server`, so all of `crates`
+# comes along; the Python package and test corpus do not.
+COPY Cargo.toml Cargo.lock rust-toolchain.toml ./
+COPY .cargo ./.cargo
+COPY crates ./crates
+
+# The cache mounts make incremental rebuilds cheap. The binary is copied out of
+# the mounted target directory in the same layer, because cache mounts are not
+# present in the resulting image.
+RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \
+    --mount=type=cache,target=/src/target,sharing=locked \
+    cargo build --locked --release -p switchyard-server \
+    && install -Dm0755 target/release/switchyard-server /out/switchyard-server
+
+########################################
+# Runtime stage
+########################################
+FROM debian:${DEBIAN_RELEASE}-slim AS runtime
+
+ARG SWITCHYARD_VERSION=0.2.0
+
+LABEL org.opencontainers.image.title="switchyard-server" \
+      org.opencontainers.image.description="Rust proxy for LLM traffic: routing, translation and metrics" \
+      org.opencontainers.image.version="${SWITCHYARD_VERSION}" \
+      org.opencontainers.image.source="https://github.com/NVIDIA-NeMo/Switchyard" \
+      org.opencontainers.image.licenses="Apache-2.0" \
+      org.opencontainers.image.vendor="NVIDIA Corporation"
+
+# ca-certificates is required to reach HTTPS upstreams through rustls.
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y ca-certificates \
+    && rm -rf /var/lib/apt/lists/* \
+    && groupadd --system --gid 65532 switchyard \
+    && useradd --system --uid 65532 --gid switchyard --no-create-home switchyard
+
+COPY --from=builder /out/switchyard-server /usr/local/bin/switchyard-server
+
+# A read-only root filesystem is the intended deployment posture, so keep the
+# only writable expectation on /tmp.
+ENV HOME=/tmp \
+    RUST_LOG=switchyard_server=info,libsy=info
+
+USER 65532:65532
+EXPOSE 4000
+
+# The server traps SIGTERM and drains in-flight requests for --shutdown-timeout
+# (30s default), which lines up with the Kubernetes termination grace period.
+ENTRYPOINT ["switchyard-server"]
+CMD ["--config", "/etc/switchyard/routes.toml"]

deploy/helm/switchyard/.helmignore

@@ -0,0 +1,7 @@
+.DS_Store
+.git/
+.gitignore
+*.tmproj
+.idea/
+.vscode/
+ci/

deploy/helm/switchyard/Chart.yaml

@@ -0,0 +1,32 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: v2
+name: switchyard
+description: Switchyard -- a Rust proxy for LLM traffic that routes across providers, translates between OpenAI and Anthropic APIs, and exports Prometheus metrics
+type: application
+
+# `version` is the chart version; `appVersion` tracks the switchyard-server
+# release the default image tag points at.
+version: 0.1.0
+appVersion: "0.2.0"
+
+home: https://github.com/NVIDIA-NeMo/Switchyard
+sources:
+  - https://github.com/NVIDIA-NeMo/Switchyard
+icon: https://raw.githubusercontent.com/NVIDIA-NeMo/Switchyard/main/assets/logo.png
+
+keywords:
+  - llm
+  - proxy
+  - routing
+  - openai
+  - anthropic
+  - gateway
+
+maintainers:
+  - name: NVIDIA Corporation
+    url: https://github.com/NVIDIA-NeMo/Switchyard
+
+annotations:
+  artifacthub.io/license: Apache-2.0

deploy/helm/switchyard/README.md

@@ -0,0 +1,162 @@
+# Switchyard Helm chart
+
+Deploys the standalone `switchyard-server` proxy on Kubernetes.
+
+The chart renders the deployment TOML into a ConfigMap, supplies upstream API
+keys from a Secret, and exposes a single ClusterIP port that serves the LLM
+endpoints, `/health` and `/metrics` alike.
+
+## Prerequisites
+
+- Kubernetes 1.27 or newer
+- Helm 3.8 or newer
+- A container image built from the repository root `Dockerfile`
+- Nodes with an AVX2-class x86_64 CPU or a Neoverse-N1-class arm64 CPU, because
+  `.cargo/config.toml` compiles with `-C target-cpu=x86-64-v3` and
+  `-C target-cpu=neoverse-n1` respectively
+
+## Install
+
+Build and publish the image:
+
+```bash
+docker build -t ghcr.io/nvidia-nemo/switchyard/switchyard-server:0.2.0 .
+docker push ghcr.io/nvidia-nemo/switchyard/switchyard-server:0.2.0
+```
+
+Put the upstream key in a Secret, then install:
+
+```bash
+kubectl create namespace switchyard
+
+kubectl -n switchyard create secret generic switchyard-keys \
+  --from-literal=OPENROUTER_API_KEY="$OPENROUTER_API_KEY"
+
+helm install switchyard deploy/helm/switchyard \
+  --namespace switchyard \
+  --set apiKeySecret.name=switchyard-keys
+```
+
+The Secret's keys become environment variables, so each name must match an
+`api_key_env` in the deployment TOML.
+
+## Configuration
+
+`config.routes` holds the deployment TOML documented in
+[`crates/switchyard-server/README.md`](../../../crates/switchyard-server/README.md).
+Validate it before rolling it out — the server exits non-zero on an invalid
+deployment, and a bad ConfigMap otherwise surfaces as a crash-looping pod:
+
+```bash
+switchyard-server --config routes.toml --dry-run
+```
+
+A multi-target routing deployment, supplied as a values file:
+
+```yaml
+# values.routing.yaml
+image:
+  repository: ghcr.io/nvidia-nemo/switchyard/switchyard-server
+  tag: "0.2.0"
+
+apiKeySecret:
+  name: switchyard-keys
+
+config:
+  routes: |
+    schema_version = 1
+
+    [llm_clients.openrouter]
+    format = "openai_chat"
+    base_url = "https://openrouter.ai/api/v1"
+    api_key_env = "OPENROUTER_API_KEY"
+    max_retries = 2
+
+    [targets.strong]
+    id = "anthropic/claude-sonnet-4.5"
+    llm_client = "openrouter"
+
+    [targets.weak]
+    id = "openai/gpt-4o-mini"
+    llm_client = "openrouter"
+
+    [routes.classified]
+    id = "switchyard/classified"
+    type = "llm_classifier"
+    mode = "capability"
+    classifier_target = "weak"
+    strong_target = "strong"
+    weak_target = "weak"
+    base_threshold = 0.5
+```
+
+```bash
+helm upgrade --install switchyard deploy/helm/switchyard \
+  --namespace switchyard -f values.routing.yaml
+```
+
+Pods carry a `checksum/config` annotation, so editing `config.routes` rolls the
+Deployment automatically.
+
+To manage the TOML outside Helm, set `config.create=false` and
+`config.existingConfigMap` to a ConfigMap whose `config.key` entry holds the
+document.
+
+## Values
+
+| Key | Default | Description |
+|---|---|---|
+| `replicaCount` | `1` | Replicas, ignored when `autoscaling.enabled` |
+| `image.repository` | `ghcr.io/nvidia-nemo/switchyard/switchyard-server` | Image repository |
+| `image.tag` | `""` | Image tag; defaults to `.Chart.AppVersion` |
+| `config.create` | `true` | Render `config.routes` into a ConfigMap |
+| `config.existingConfigMap` | `""` | ConfigMap to use when `config.create` is false |
+| `config.key` | `routes.toml` | ConfigMap key holding the TOML |
+| `config.mountPath` | `/etc/switchyard` | Mount point for the TOML |
+| `config.routes` | passthrough example | Deployment TOML |
+| `apiKeySecret.create` | `false` | Create a Secret from `apiKeySecret.data` |
+| `apiKeySecret.name` | `""` | Existing Secret loaded with `envFrom` |
+| `apiKeySecret.data` | `{}` | Key/value pairs, read only when `create` is true |
+| `env` / `envFrom` | `[]` | Additional environment |
+| `extraArgs` | `[]` | Extra `switchyard-server` flags |
+| `service.type` / `service.port` | `ClusterIP` / `4000` | Service exposure |
+| `containerPort` | `4000` | Port the server binds |
+| `resources` | 200m/128Mi → 2/1Gi | Requests and limits |
+| `terminationGracePeriodSeconds` | `60` | Must exceed `--shutdown-timeout` |
+| `routingLog.enabled` | `false` | Enable `--routing-log-file` and session stats |
+| `tls.enabled` | `false` | Terminate TLS at Switchyard |
+| `metrics.podAnnotations` | `true` | Prometheus scrape annotations |
+| `metrics.serviceMonitor.enabled` | `false` | Create a ServiceMonitor |
+| `podDisruptionBudget.enabled` | `false` | Create a PDB |
+| `autoscaling.enabled` | `false` | Create an HPA |
+| `extraObjects` | `[]` | Extra manifests, templated with `tpl` |
+
+See [`values.yaml`](values.yaml) for the full set.
+
+## Operational notes
+
+**Graceful shutdown.** The server drains in-flight requests for
+`--shutdown-timeout` (30s by default) on SIGTERM.
+`terminationGracePeriodSeconds` defaults to 60 so streaming completions finish
+rather than being cut off. Raise both together if your workload streams for
+longer.
+
+**Health semantics.** `/health` reports that the process is serving. It does
+not check upstream reachability, so it stays healthy during a provider outage —
+watch `switchyard_errors_total` and `switchyard_upstream_attempts_total` for
+that.
+
+**Session affinity.** `llm_classifier` routes with `session_affinity = true`
+keep decisions in process memory, so a given session must reach the same
+replica to benefit. With more than one replica, either front the Service with
+session-aware routing or accept that affinity is per-replica.
+
+**Read-only root.** The container runs as UID 65532 with a read-only root
+filesystem; `/tmp` is an emptyDir because the image sets `HOME=/tmp`. Enabling
+`routingLog` adds a writable volume at the log's parent directory.
+
+## Envoy AI Gateway
+
+To front Switchyard with Envoy AI Gateway, or to route Switchyard's upstream
+traffic through it, see
+[`examples/kubernetes/`](../../../examples/kubernetes/README.md).

deploy/helm/switchyard/templates/NOTES.txt

@@ -0,0 +1,39 @@
+Switchyard {{ .Chart.AppVersion }} is installed as release {{ .Release.Name }} in namespace {{ .Release.Namespace }}.
+
+Service: {{ include "switchyard.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.port }}
+
+Routes served by this deployment:
+
+  kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "switchyard.fullname" . }} {{ .Values.service.port }}:{{ .Values.service.port }}
+  curl -s localhost:{{ .Values.service.port }}/v1/models | jq
+
+Send a completion, naming a route id from your deployment TOML as the model:
+
+  curl -s localhost:{{ .Values.service.port }}/v1/chat/completions \
+    -H 'content-type: application/json' \
+    -d '{"model":"<route-id>","messages":[{"role":"user","content":"hello"}]}' | jq
+
+Liveness and Prometheus metrics:
+
+  curl -s localhost:{{ .Values.service.port }}/health
+  curl -s localhost:{{ .Values.service.port }}/metrics
+
+{{ if not (include "switchyard.apiKeySecretName" .) -}}
+WARNING: no API-key Secret is configured. Every `api_key_env` named in your
+deployment TOML must resolve to an environment variable, or upstream calls will
+be sent unauthenticated. Set `apiKeySecret.name` to an existing Secret, or
+`envFrom`, and reinstall.
+{{- end }}
+{{- if .Values.apiKeySecret.create }}
+
+NOTE: apiKeySecret.create is true, so provider keys are stored in values. That
+is fine for a test cluster; for production point `apiKeySecret.name` at a Secret
+managed by your secret store instead.
+{{- end }}
+{{- if .Values.config.create }}
+
+The deployment TOML is held in ConfigMap {{ include "switchyard.configMapName" . }}.
+Validate changes before rolling them out:
+
+  switchyard-server --config routes.toml --dry-run
+{{- end }}