From 2151a4eb79efb14afb0b41156f196722825fe4f3 Mon Sep 17 00:00:00 2001 From: ansjindal Date: Thu, 6 Aug 2026 09:34:57 +0200 Subject: [PATCH 1/6] feat(deploy): add container image, Helm chart and Envoy AI Gateway examples Signed-off-by: ansjindal --- .dockerignore | 1 + Dockerfile | 92 ++++++++ deploy/helm/switchyard/.helmignore | 7 + deploy/helm/switchyard/Chart.yaml | 32 +++ deploy/helm/switchyard/README.md | 162 +++++++++++++ deploy/helm/switchyard/templates/NOTES.txt | 39 ++++ deploy/helm/switchyard/templates/_helpers.tpl | 79 +++++++ .../helm/switchyard/templates/configmap.yaml | 15 ++ .../helm/switchyard/templates/deployment.yaml | 183 +++++++++++++++ .../switchyard/templates/extra-objects.yaml | 8 + deploy/helm/switchyard/templates/hpa.yaml | 36 +++ deploy/helm/switchyard/templates/pdb.yaml | 21 ++ deploy/helm/switchyard/templates/secret.yaml | 17 ++ deploy/helm/switchyard/templates/service.yaml | 23 ++ .../switchyard/templates/serviceaccount.yaml | 18 ++ .../switchyard/templates/servicemonitor.yaml | 35 +++ deploy/helm/switchyard/values.yaml | 221 ++++++++++++++++++ examples/kubernetes/README.md | 210 +++++++++++++++++ .../envoy-ai-gateway-in-front/01-gateway.yaml | 82 +++++++ .../02-switchyard-backend.yaml | 103 ++++++++ .../03-forwarded-host.yaml | 39 ++++ .../04-client-auth.yaml | 101 ++++++++ .../values.switchyard.yaml | 86 +++++++ .../01-gateway.yaml | 85 +++++++ .../02-provider-backend.yaml | 98 ++++++++ .../03-forwarded-host.yaml | 42 ++++ .../04-restrict-access.yaml | 56 +++++ .../values.switchyard.yaml | 55 +++++ 28 files changed, 1946 insertions(+) create mode 100644 Dockerfile create mode 100644 deploy/helm/switchyard/.helmignore create mode 100644 deploy/helm/switchyard/Chart.yaml create mode 100644 deploy/helm/switchyard/README.md create mode 100644 deploy/helm/switchyard/templates/NOTES.txt create mode 100644 deploy/helm/switchyard/templates/_helpers.tpl create mode 100644 deploy/helm/switchyard/templates/configmap.yaml create mode 100644 deploy/helm/switchyard/templates/deployment.yaml create mode 100644 deploy/helm/switchyard/templates/extra-objects.yaml create mode 100644 deploy/helm/switchyard/templates/hpa.yaml create mode 100644 deploy/helm/switchyard/templates/pdb.yaml create mode 100644 deploy/helm/switchyard/templates/secret.yaml create mode 100644 deploy/helm/switchyard/templates/service.yaml create mode 100644 deploy/helm/switchyard/templates/serviceaccount.yaml create mode 100644 deploy/helm/switchyard/templates/servicemonitor.yaml create mode 100644 deploy/helm/switchyard/values.yaml create mode 100644 examples/kubernetes/README.md create mode 100644 examples/kubernetes/envoy-ai-gateway-in-front/01-gateway.yaml create mode 100644 examples/kubernetes/envoy-ai-gateway-in-front/02-switchyard-backend.yaml create mode 100644 examples/kubernetes/envoy-ai-gateway-in-front/03-forwarded-host.yaml create mode 100644 examples/kubernetes/envoy-ai-gateway-in-front/04-client-auth.yaml create mode 100644 examples/kubernetes/envoy-ai-gateway-in-front/values.switchyard.yaml create mode 100644 examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/01-gateway.yaml create mode 100644 examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/02-provider-backend.yaml create mode 100644 examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/03-forwarded-host.yaml create mode 100644 examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/04-restrict-access.yaml create mode 100644 examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/values.switchyard.yaml diff --git a/.dockerignore b/.dockerignore index 0941d4626..b4a663375 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ .git .mypy_cache +target .pytest_cache .ruff_cache .venv diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..c6532da0e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,92 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# syntax=docker/dockerfile:1.7 + +# Production image for the standalone `switchyard-server` proxy. +# +# The Dockerfiles under `benchmark/` build the Python launcher and an +# unoptimised server for benchmark harnesses. This one builds only the release +# proxy and ships it on a slim runtime with no toolchain attached. +# +# docker build -t switchyard-server:0.2.0 . +# docker run --rm -p 4000:4000 \ +# -v "$PWD/routes.toml:/etc/switchyard/routes.toml:ro" \ +# -e OPENROUTER_API_KEY \ +# switchyard-server:0.2.0 --config /etc/switchyard/routes.toml +# +# CPU baseline: `.cargo/config.toml` compiles x86_64 with `-C +# target-cpu=x86-64-v3`, so the resulting binary needs an AVX2-class CPU +# (Haswell 2013+), and aarch64 with `-C target-cpu=neoverse-n1`. This matches +# the published wheels documented in INSTALLATION.md. + +ARG RUST_VERSION=1.96.1 +ARG DEBIAN_RELEASE=bookworm + +######################################## +# Build stage +######################################## +FROM rust:${RUST_VERSION}-${DEBIAN_RELEASE} AS builder + +# `aws-lc-rs`, pulled in by rustls, builds native code and needs cmake plus a +# libclang for its bindgen step. Everything else in the dependency graph is +# pure Rust: reqwest is configured for rustls, so no OpenSSL headers. +RUN apt-get update \ + && apt-get install --no-install-recommends -y \ + cmake \ + clang \ + libclang-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /src + +# Copy only what the server's dependency graph needs to resolve. Cargo parses +# every workspace manifest even for `-p switchyard-server`, so all of `crates` +# comes along; the Python package and test corpus do not. +COPY Cargo.toml Cargo.lock rust-toolchain.toml ./ +COPY .cargo ./.cargo +COPY crates ./crates + +# The cache mounts make incremental rebuilds cheap. The binary is copied out of +# the mounted target directory in the same layer, because cache mounts are not +# present in the resulting image. +RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,target=/src/target,sharing=locked \ + cargo build --locked --release -p switchyard-server \ + && install -Dm0755 target/release/switchyard-server /out/switchyard-server + +######################################## +# Runtime stage +######################################## +FROM debian:${DEBIAN_RELEASE}-slim AS runtime + +ARG SWITCHYARD_VERSION=0.2.0 + +LABEL org.opencontainers.image.title="switchyard-server" \ + org.opencontainers.image.description="Rust proxy for LLM traffic: routing, translation and metrics" \ + org.opencontainers.image.version="${SWITCHYARD_VERSION}" \ + org.opencontainers.image.source="https://github.com/NVIDIA-NeMo/Switchyard" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.vendor="NVIDIA Corporation" + +# ca-certificates is required to reach HTTPS upstreams through rustls. +RUN apt-get update \ + && apt-get install --no-install-recommends -y ca-certificates \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd --system --gid 65532 switchyard \ + && useradd --system --uid 65532 --gid switchyard --no-create-home switchyard + +COPY --from=builder /out/switchyard-server /usr/local/bin/switchyard-server + +# A read-only root filesystem is the intended deployment posture, so keep the +# only writable expectation on /tmp. +ENV HOME=/tmp \ + RUST_LOG=switchyard_server=info,libsy=info + +USER 65532:65532 +EXPOSE 4000 + +# The server traps SIGTERM and drains in-flight requests for --shutdown-timeout +# (30s default), which lines up with the Kubernetes termination grace period. +ENTRYPOINT ["switchyard-server"] +CMD ["--config", "/etc/switchyard/routes.toml"] diff --git a/deploy/helm/switchyard/.helmignore b/deploy/helm/switchyard/.helmignore new file mode 100644 index 000000000..1369a801a --- /dev/null +++ b/deploy/helm/switchyard/.helmignore @@ -0,0 +1,7 @@ +.DS_Store +.git/ +.gitignore +*.tmproj +.idea/ +.vscode/ +ci/ diff --git a/deploy/helm/switchyard/Chart.yaml b/deploy/helm/switchyard/Chart.yaml new file mode 100644 index 000000000..e2d1cd10e --- /dev/null +++ b/deploy/helm/switchyard/Chart.yaml @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: v2 +name: switchyard +description: Switchyard -- a Rust proxy for LLM traffic that routes across providers, translates between OpenAI and Anthropic APIs, and exports Prometheus metrics +type: application + +# `version` is the chart version; `appVersion` tracks the switchyard-server +# release the default image tag points at. +version: 0.1.0 +appVersion: "0.2.0" + +home: https://github.com/NVIDIA-NeMo/Switchyard +sources: + - https://github.com/NVIDIA-NeMo/Switchyard +icon: https://raw.githubusercontent.com/NVIDIA-NeMo/Switchyard/main/assets/logo.png + +keywords: + - llm + - proxy + - routing + - openai + - anthropic + - gateway + +maintainers: + - name: NVIDIA Corporation + url: https://github.com/NVIDIA-NeMo/Switchyard + +annotations: + artifacthub.io/license: Apache-2.0 diff --git a/deploy/helm/switchyard/README.md b/deploy/helm/switchyard/README.md new file mode 100644 index 000000000..8e0838c04 --- /dev/null +++ b/deploy/helm/switchyard/README.md @@ -0,0 +1,162 @@ +# Switchyard Helm chart + +Deploys the standalone `switchyard-server` proxy on Kubernetes. + +The chart renders the deployment TOML into a ConfigMap, supplies upstream API +keys from a Secret, and exposes a single ClusterIP port that serves the LLM +endpoints, `/health` and `/metrics` alike. + +## Prerequisites + +- Kubernetes 1.27 or newer +- Helm 3.8 or newer +- A container image built from the repository root `Dockerfile` +- Nodes with an AVX2-class x86_64 CPU or a Neoverse-N1-class arm64 CPU, because + `.cargo/config.toml` compiles with `-C target-cpu=x86-64-v3` and + `-C target-cpu=neoverse-n1` respectively + +## Install + +Build and publish the image: + +```bash +docker build -t ghcr.io/nvidia-nemo/switchyard/switchyard-server:0.2.0 . +docker push ghcr.io/nvidia-nemo/switchyard/switchyard-server:0.2.0 +``` + +Put the upstream key in a Secret, then install: + +```bash +kubectl create namespace switchyard + +kubectl -n switchyard create secret generic switchyard-keys \ + --from-literal=OPENROUTER_API_KEY="$OPENROUTER_API_KEY" + +helm install switchyard deploy/helm/switchyard \ + --namespace switchyard \ + --set apiKeySecret.name=switchyard-keys +``` + +The Secret's keys become environment variables, so each name must match an +`api_key_env` in the deployment TOML. + +## Configuration + +`config.routes` holds the deployment TOML documented in +[`crates/switchyard-server/README.md`](../../../crates/switchyard-server/README.md). +Validate it before rolling it out — the server exits non-zero on an invalid +deployment, and a bad ConfigMap otherwise surfaces as a crash-looping pod: + +```bash +switchyard-server --config routes.toml --dry-run +``` + +A multi-target routing deployment, supplied as a values file: + +```yaml +# values.routing.yaml +image: + repository: ghcr.io/nvidia-nemo/switchyard/switchyard-server + tag: "0.2.0" + +apiKeySecret: + name: switchyard-keys + +config: + routes: | + schema_version = 1 + + [llm_clients.openrouter] + format = "openai_chat" + base_url = "https://openrouter.ai/api/v1" + api_key_env = "OPENROUTER_API_KEY" + max_retries = 2 + + [targets.strong] + id = "anthropic/claude-sonnet-4.5" + llm_client = "openrouter" + + [targets.weak] + id = "openai/gpt-4o-mini" + llm_client = "openrouter" + + [routes.classified] + id = "switchyard/classified" + type = "llm_classifier" + mode = "capability" + classifier_target = "weak" + strong_target = "strong" + weak_target = "weak" + base_threshold = 0.5 +``` + +```bash +helm upgrade --install switchyard deploy/helm/switchyard \ + --namespace switchyard -f values.routing.yaml +``` + +Pods carry a `checksum/config` annotation, so editing `config.routes` rolls the +Deployment automatically. + +To manage the TOML outside Helm, set `config.create=false` and +`config.existingConfigMap` to a ConfigMap whose `config.key` entry holds the +document. + +## Values + +| Key | Default | Description | +|---|---|---| +| `replicaCount` | `1` | Replicas, ignored when `autoscaling.enabled` | +| `image.repository` | `ghcr.io/nvidia-nemo/switchyard/switchyard-server` | Image repository | +| `image.tag` | `""` | Image tag; defaults to `.Chart.AppVersion` | +| `config.create` | `true` | Render `config.routes` into a ConfigMap | +| `config.existingConfigMap` | `""` | ConfigMap to use when `config.create` is false | +| `config.key` | `routes.toml` | ConfigMap key holding the TOML | +| `config.mountPath` | `/etc/switchyard` | Mount point for the TOML | +| `config.routes` | passthrough example | Deployment TOML | +| `apiKeySecret.create` | `false` | Create a Secret from `apiKeySecret.data` | +| `apiKeySecret.name` | `""` | Existing Secret loaded with `envFrom` | +| `apiKeySecret.data` | `{}` | Key/value pairs, read only when `create` is true | +| `env` / `envFrom` | `[]` | Additional environment | +| `extraArgs` | `[]` | Extra `switchyard-server` flags | +| `service.type` / `service.port` | `ClusterIP` / `4000` | Service exposure | +| `containerPort` | `4000` | Port the server binds | +| `resources` | 200m/128Mi → 2/1Gi | Requests and limits | +| `terminationGracePeriodSeconds` | `60` | Must exceed `--shutdown-timeout` | +| `routingLog.enabled` | `false` | Enable `--routing-log-file` and session stats | +| `tls.enabled` | `false` | Terminate TLS at Switchyard | +| `metrics.podAnnotations` | `true` | Prometheus scrape annotations | +| `metrics.serviceMonitor.enabled` | `false` | Create a ServiceMonitor | +| `podDisruptionBudget.enabled` | `false` | Create a PDB | +| `autoscaling.enabled` | `false` | Create an HPA | +| `extraObjects` | `[]` | Extra manifests, templated with `tpl` | + +See [`values.yaml`](values.yaml) for the full set. + +## Operational notes + +**Graceful shutdown.** The server drains in-flight requests for +`--shutdown-timeout` (30s by default) on SIGTERM. +`terminationGracePeriodSeconds` defaults to 60 so streaming completions finish +rather than being cut off. Raise both together if your workload streams for +longer. + +**Health semantics.** `/health` reports that the process is serving. It does +not check upstream reachability, so it stays healthy during a provider outage — +watch `switchyard_errors_total` and `switchyard_upstream_attempts_total` for +that. + +**Session affinity.** `llm_classifier` routes with `session_affinity = true` +keep decisions in process memory, so a given session must reach the same +replica to benefit. With more than one replica, either front the Service with +session-aware routing or accept that affinity is per-replica. + +**Read-only root.** The container runs as UID 65532 with a read-only root +filesystem; `/tmp` is an emptyDir because the image sets `HOME=/tmp`. Enabling +`routingLog` adds a writable volume at the log's parent directory. + +## Envoy AI Gateway + +To front Switchyard with Envoy AI Gateway, or to route Switchyard's upstream +traffic through it, see +[`examples/kubernetes/`](../../../examples/kubernetes/README.md). diff --git a/deploy/helm/switchyard/templates/NOTES.txt b/deploy/helm/switchyard/templates/NOTES.txt new file mode 100644 index 000000000..87eb90d48 --- /dev/null +++ b/deploy/helm/switchyard/templates/NOTES.txt @@ -0,0 +1,39 @@ +Switchyard {{ .Chart.AppVersion }} is installed as release {{ .Release.Name }} in namespace {{ .Release.Namespace }}. + +Service: {{ include "switchyard.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.port }} + +Routes served by this deployment: + + kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "switchyard.fullname" . }} {{ .Values.service.port }}:{{ .Values.service.port }} + curl -s localhost:{{ .Values.service.port }}/v1/models | jq + +Send a completion, naming a route id from your deployment TOML as the model: + + curl -s localhost:{{ .Values.service.port }}/v1/chat/completions \ + -H 'content-type: application/json' \ + -d '{"model":"","messages":[{"role":"user","content":"hello"}]}' | jq + +Liveness and Prometheus metrics: + + curl -s localhost:{{ .Values.service.port }}/health + curl -s localhost:{{ .Values.service.port }}/metrics + +{{ if not (include "switchyard.apiKeySecretName" .) -}} +WARNING: no API-key Secret is configured. Every `api_key_env` named in your +deployment TOML must resolve to an environment variable, or upstream calls will +be sent unauthenticated. Set `apiKeySecret.name` to an existing Secret, or +`envFrom`, and reinstall. +{{- end }} +{{- if .Values.apiKeySecret.create }} + +NOTE: apiKeySecret.create is true, so provider keys are stored in values. That +is fine for a test cluster; for production point `apiKeySecret.name` at a Secret +managed by your secret store instead. +{{- end }} +{{- if .Values.config.create }} + +The deployment TOML is held in ConfigMap {{ include "switchyard.configMapName" . }}. +Validate changes before rolling them out: + + switchyard-server --config routes.toml --dry-run +{{- end }} diff --git a/deploy/helm/switchyard/templates/_helpers.tpl b/deploy/helm/switchyard/templates/_helpers.tpl new file mode 100644 index 000000000..22536e98c --- /dev/null +++ b/deploy/helm/switchyard/templates/_helpers.tpl @@ -0,0 +1,79 @@ +{{/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/}} + +{{- define "switchyard.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "switchyard.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{- define "switchyard.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "switchyard.labels" -}} +helm.sh/chart: {{ include "switchyard.chart" . }} +{{ include "switchyard.selectorLabels" . }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/part-of: switchyard +{{- end }} + +{{- define "switchyard.selectorLabels" -}} +app.kubernetes.io/name: {{ include "switchyard.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{- define "switchyard.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "switchyard.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Name of the ConfigMap holding the deployment TOML. +*/}} +{{- define "switchyard.configMapName" -}} +{{- if .Values.config.create }} +{{- include "switchyard.fullname" . }} +{{- else }} +{{- required "config.existingConfigMap is required when config.create is false" .Values.config.existingConfigMap }} +{{- end }} +{{- end }} + +{{/* +Name of the Secret providing upstream API keys, or "" when none is configured. +*/}} +{{- define "switchyard.apiKeySecretName" -}} +{{- if .Values.apiKeySecret.create }} +{{- default (include "switchyard.fullname" .) .Values.apiKeySecret.name }} +{{- else }} +{{- .Values.apiKeySecret.name }} +{{- end }} +{{- end }} + +{{- define "switchyard.image" -}} +{{- printf "%s:%s" .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }} +{{- end }} + +{{/* +Absolute path to the deployment TOML inside the container. +*/}} +{{- define "switchyard.configPath" -}} +{{- printf "%s/%s" (trimSuffix "/" .Values.config.mountPath) .Values.config.key }} +{{- end }} diff --git a/deploy/helm/switchyard/templates/configmap.yaml b/deploy/helm/switchyard/templates/configmap.yaml new file mode 100644 index 000000000..bc030e6ab --- /dev/null +++ b/deploy/helm/switchyard/templates/configmap.yaml @@ -0,0 +1,15 @@ +{{- /* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ -}} +{{- if .Values.config.create }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "switchyard.fullname" . }} + labels: + {{- include "switchyard.labels" . | nindent 4 }} +data: + {{ .Values.config.key }}: | + {{- required "config.routes must define the deployment TOML" .Values.config.routes | nindent 4 }} +{{- end }} diff --git a/deploy/helm/switchyard/templates/deployment.yaml b/deploy/helm/switchyard/templates/deployment.yaml new file mode 100644 index 000000000..5c0a31b23 --- /dev/null +++ b/deploy/helm/switchyard/templates/deployment.yaml @@ -0,0 +1,183 @@ +{{- /* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "switchyard.fullname" . }} + labels: + {{- include "switchyard.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + strategy: + {{- toYaml .Values.updateStrategy | nindent 4 }} + selector: + matchLabels: + {{- include "switchyard.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- if .Values.config.create }} + # Roll the pods whenever the deployment TOML changes. + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- end }} + {{- if .Values.apiKeySecret.create }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- end }} + {{- if .Values.metrics.podAnnotations }} + prometheus.io/scrape: "true" + prometheus.io/port: {{ .Values.containerPort | quote }} + prometheus.io/path: /metrics + {{- end }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "switchyard.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "switchyard.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.dnsPolicy }} + dnsPolicy: {{ . }} + {{- end }} + {{- with .Values.dnsConfig }} + dnsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} + containers: + - name: switchyard + image: {{ include "switchyard.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + args: + - --config + - {{ include "switchyard.configPath" . }} + - --host + - 0.0.0.0 + - --port + - {{ .Values.containerPort | quote }} + {{- if .Values.routingLog.enabled }} + - --routing-log-file + - {{ .Values.routingLog.path }} + {{- end }} + {{- if .Values.tls.enabled }} + - --tls-cert + - {{ printf "%s/tls.crt" (trimSuffix "/" .Values.tls.mountPath) }} + - --tls-key + - {{ printf "%s/tls.key" (trimSuffix "/" .Values.tls.mountPath) }} + {{- end }} + {{- with .Values.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.containerPort }} + protocol: TCP + {{- $apiKeySecret := include "switchyard.apiKeySecretName" . }} + {{- if or $apiKeySecret .Values.envFrom }} + envFrom: + {{- if $apiKeySecret }} + - secretRef: + name: {{ $apiKeySecret }} + {{- end }} + {{- with .Values.envFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.startupProbe }} + startupProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: {{ .Values.config.mountPath }} + readOnly: true + # readOnlyRootFilesystem is on and the image sets HOME=/tmp. + - name: tmp + mountPath: /tmp + {{- if .Values.routingLog.enabled }} + - name: routing-log + mountPath: {{ dir .Values.routingLog.path }} + {{- end }} + {{- if .Values.tls.enabled }} + - name: tls + mountPath: {{ .Values.tls.mountPath }} + readOnly: true + {{- end }} + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: config + configMap: + name: {{ include "switchyard.configMapName" . }} + items: + - key: {{ .Values.config.key }} + path: {{ .Values.config.key }} + - name: tmp + emptyDir: {} + {{- if .Values.routingLog.enabled }} + - name: routing-log + {{- if .Values.routingLog.existingClaim }} + persistentVolumeClaim: + claimName: {{ .Values.routingLog.existingClaim }} + {{- else }} + emptyDir: + sizeLimit: {{ .Values.routingLog.sizeLimit }} + {{- end }} + {{- end }} + {{- if .Values.tls.enabled }} + - name: tls + secret: + secretName: {{ required "tls.secretName is required when tls.enabled is true" .Values.tls.secretName }} + {{- end }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/deploy/helm/switchyard/templates/extra-objects.yaml b/deploy/helm/switchyard/templates/extra-objects.yaml new file mode 100644 index 000000000..ef8bf0ecd --- /dev/null +++ b/deploy/helm/switchyard/templates/extra-objects.yaml @@ -0,0 +1,8 @@ +{{- /* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ -}} +{{- range .Values.extraObjects }} +--- +{{ tpl (toYaml .) $ }} +{{- end }} diff --git a/deploy/helm/switchyard/templates/hpa.yaml b/deploy/helm/switchyard/templates/hpa.yaml new file mode 100644 index 000000000..9c65b7087 --- /dev/null +++ b/deploy/helm/switchyard/templates/hpa.yaml @@ -0,0 +1,36 @@ +{{- /* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ -}} +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "switchyard.fullname" . }} + labels: + {{- include "switchyard.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "switchyard.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/deploy/helm/switchyard/templates/pdb.yaml b/deploy/helm/switchyard/templates/pdb.yaml new file mode 100644 index 000000000..0ed7cd26d --- /dev/null +++ b/deploy/helm/switchyard/templates/pdb.yaml @@ -0,0 +1,21 @@ +{{- /* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ -}} +{{- if .Values.podDisruptionBudget.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "switchyard.fullname" . }} + labels: + {{- include "switchyard.labels" . | nindent 4 }} +spec: + {{- if .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} + {{- else }} + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} + {{- end }} + selector: + matchLabels: + {{- include "switchyard.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/deploy/helm/switchyard/templates/secret.yaml b/deploy/helm/switchyard/templates/secret.yaml new file mode 100644 index 000000000..f91b796f9 --- /dev/null +++ b/deploy/helm/switchyard/templates/secret.yaml @@ -0,0 +1,17 @@ +{{- /* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ -}} +{{- if .Values.apiKeySecret.create }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "switchyard.apiKeySecretName" . }} + labels: + {{- include "switchyard.labels" . | nindent 4 }} +type: Opaque +stringData: + {{- range $name, $value := .Values.apiKeySecret.data }} + {{ $name }}: {{ $value | quote }} + {{- end }} +{{- end }} diff --git a/deploy/helm/switchyard/templates/service.yaml b/deploy/helm/switchyard/templates/service.yaml new file mode 100644 index 000000000..620643e81 --- /dev/null +++ b/deploy/helm/switchyard/templates/service.yaml @@ -0,0 +1,23 @@ +{{- /* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "switchyard.fullname" . }} + labels: + {{- include "switchyard.labels" . | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - name: http + port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + selector: + {{- include "switchyard.selectorLabels" . | nindent 4 }} diff --git a/deploy/helm/switchyard/templates/serviceaccount.yaml b/deploy/helm/switchyard/templates/serviceaccount.yaml new file mode 100644 index 000000000..6fc01f06d --- /dev/null +++ b/deploy/helm/switchyard/templates/serviceaccount.yaml @@ -0,0 +1,18 @@ +{{- /* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ -}} +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "switchyard.serviceAccountName" . }} + labels: + {{- include "switchyard.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +# Switchyard never calls the Kubernetes API. +automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} +{{- end }} diff --git a/deploy/helm/switchyard/templates/servicemonitor.yaml b/deploy/helm/switchyard/templates/servicemonitor.yaml new file mode 100644 index 000000000..8c05be985 --- /dev/null +++ b/deploy/helm/switchyard/templates/servicemonitor.yaml @@ -0,0 +1,35 @@ +{{- /* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ -}} +{{- if .Values.metrics.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "switchyard.fullname" . }} + labels: + {{- include "switchyard.labels" . | nindent 4 }} + {{- with .Values.metrics.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "switchyard.selectorLabels" . | nindent 6 }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + endpoints: + - port: http + path: /metrics + interval: {{ .Values.metrics.serviceMonitor.interval }} + scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} + {{- with .Values.metrics.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.metrics.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/deploy/helm/switchyard/values.yaml b/deploy/helm/switchyard/values.yaml new file mode 100644 index 000000000..e1fb8f64b --- /dev/null +++ b/deploy/helm/switchyard/values.yaml @@ -0,0 +1,221 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Default values for the switchyard chart. + +replicaCount: 1 + +image: + repository: ghcr.io/nvidia-nemo/switchyard/switchyard-server + # Defaults to .Chart.AppVersion when empty. + tag: "" + pullPolicy: IfNotPresent + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +# -- Deployment TOML defining llm_clients, targets and routes. +# +# The schema is documented in crates/switchyard-server/README.md. Secrets never +# appear here: `api_key_env` names an environment variable, which this chart +# supplies from `apiKeySecret` or `envFrom`. +config: + # Render `config.routes` into a ConfigMap owned by this release. Set to false + # and populate `config.existingConfigMap` to manage the TOML out of band. + create: true + existingConfigMap: "" + # Key within the ConfigMap holding the TOML document. + key: routes.toml + # Directory the ConfigMap is mounted at inside the container. + mountPath: /etc/switchyard + routes: | + schema_version = 1 + + [llm_clients.upstream] + format = "openai_chat" + base_url = "https://openrouter.ai/api/v1" + api_key_env = "OPENROUTER_API_KEY" + max_retries = 2 + + [targets.primary] + id = "openai/gpt-4o-mini" + llm_client = "upstream" + + [routes.default] + id = "switchyard/default" + type = "passthrough" + target = "primary" + +# -- Upstream credentials. +# +# Every `api_key_env` named in the deployment TOML must resolve to an +# environment variable in the container. +apiKeySecret: + # Create a Secret from `apiKeySecret.data`. Convenient for development; in + # production leave this false and point `apiKeySecret.name` at a Secret + # managed by your secret store (External Secrets, Vault, SOPS, ...). + create: false + # Name of the Secret to load with `envFrom`. Defaults to the release + # fullname when `create` is true. + name: "" + # Plain-text key/value pairs, only read when `create` is true. + # e.g. OPENROUTER_API_KEY: sk-or-... + data: {} + +# Additional environment variables. +env: [] +# - name: RUST_LOG +# value: switchyard_server=debug,libsy=debug + +# Additional envFrom sources (ConfigMaps or Secrets). +envFrom: [] +# - secretRef: +# name: my-provider-keys + +# Extra flags appended to the switchyard-server command line. +# See docs/cli_reference.md for the full list. +extraArgs: [] +# - --shutdown-timeout=45s + +service: + type: ClusterIP + port: 4000 + annotations: {} + +# The server listens on a single port that serves the LLM API, /health and +# /metrics alike. +containerPort: 4000 + +serviceAccount: + create: true + name: "" + annotations: {} + automountServiceAccountToken: false + +# Switchyard holds no state on disk unless routingLog is enabled, so it runs +# unprivileged on a read-only root filesystem. +podSecurityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + +securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + +# The binary is compiled with `-C target-cpu=x86-64-v3`, so it needs an +# AVX2-class x86_64 CPU or a Neoverse-N1-class arm64 CPU. Pin nodeSelector +# accordingly on heterogeneous clusters. +resources: + requests: + cpu: 200m + memory: 128Mi + limits: + cpu: "2" + memory: 1Gi + +# /health is a liveness endpoint: it reports that the process is serving, not +# that upstreams are reachable. +livenessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 5 + periodSeconds: 20 + timeoutSeconds: 3 + failureThreshold: 3 + +readinessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 2 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + +startupProbe: + httpGet: + path: /health + port: http + periodSeconds: 3 + failureThreshold: 20 + +# Must exceed --shutdown-timeout (30s by default) so in-flight LLM requests +# drain instead of being cut off. Streaming completions can be long-lived. +terminationGracePeriodSeconds: 60 + +# -- Durable per-request routing records (--routing-log-file). +# +# Enabling this also registers GET /v1/routing/session-stats. +routingLog: + enabled: false + path: /var/log/switchyard/routing.jsonl + # emptyDir is scoped to the pod lifetime. Supply `existingClaim` for + # records that must outlive a restart. + existingClaim: "" + sizeLimit: 1Gi + +# -- Terminate TLS at Switchyard itself (--tls-cert / --tls-key). +# +# Leave disabled when a gateway or service mesh terminates TLS in front. +tls: + enabled: false + # Secret of type kubernetes.io/tls. + secretName: "" + mountPath: /etc/switchyard/tls + +metrics: + # Prometheus scrape annotations on the pod. + podAnnotations: true + # Requires the Prometheus Operator CRDs. + serviceMonitor: + enabled: false + interval: 30s + scrapeTimeout: 10s + labels: {} + relabelings: [] + metricRelabelings: [] + +podDisruptionBudget: + enabled: false + minAvailable: 1 + maxUnavailable: "" + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 80 + targetMemoryUtilizationPercentage: "" + +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + +podAnnotations: {} +podLabels: {} +nodeSelector: {} +tolerations: [] +affinity: {} +topologySpreadConstraints: [] +priorityClassName: "" +dnsPolicy: "" +dnsConfig: {} + +# Extra volumes and mounts, e.g. a CA bundle for a private upstream. +extraVolumes: [] +extraVolumeMounts: [] + +# Extra objects rendered verbatim, e.g. an Envoy AI Gateway AIServiceBackend. +# See examples/kubernetes/ for a complete integration. +extraObjects: [] diff --git a/examples/kubernetes/README.md b/examples/kubernetes/README.md new file mode 100644 index 000000000..5b3d9c558 --- /dev/null +++ b/examples/kubernetes/README.md @@ -0,0 +1,210 @@ + + +# Switchyard with Envoy AI Gateway + +Two ways to combine Switchyard with [Envoy AI Gateway](https://aigateway.envoyproxy.io/), +depending on which component you want to own ingress and which you want to own +provider credentials. + +Both use the [Switchyard Helm chart](../../deploy/helm/switchyard) for the +Switchyard half and plain manifests for the Envoy half. + +## Which topology + +| | [`envoy-ai-gateway-in-front/`](envoy-ai-gateway-in-front) | [`switchyard-in-front-of-envoy-ai-gateway/`](switchyard-in-front-of-envoy-ai-gateway) | +|---|---|---| +| Chain | client → Envoy AI Gateway → Switchyard → provider | client → Switchyard → Envoy AI Gateway → provider | +| Ingress owner | Envoy Gateway | nothing, by default — see [Client authentication](#client-authentication) | +| Provider credentials | Switchyard pod env | `BackendSecurityPolicy`, injected by Envoy | +| Envoy matches on | Switchyard route ids | provider model ids | +| Token rate limiting applies to | client traffic | Switchyard's upstream traffic | + +Pick `envoy-ai-gateway-in-front` when Envoy should be the front door: client +authentication, per-client token budgets, and a single Kubernetes-native +ingress point, with Switchyard as the routing brain behind it. + +Pick `switchyard-in-front-of-envoy-ai-gateway` when Switchyard is already the +client's endpoint and you want provider keys and upstream TLS out of the +application pod. No provider credential is mounted into Switchyard at all. + +The two are not exclusive. Running both, as the manifests here do, gives Envoy +at the edge and Envoy at the egress with Switchyard in the middle. + +## Client authentication + +**Switchyard authenticates no one.** `switchyard-server` serves every request +that reaches its port — there is no API-key check, no JWT validation, no mTLS. +Whatever sits in front of it owns client identity. + +That makes the two topologies differ in an important way: + +- **Envoy AI Gateway in front** — solved by the Gateway. + [`04-client-auth.yaml`](envoy-ai-gateway-in-front/04-client-auth.yaml) shows a + `SecurityPolicy` with API-key auth, plus `jwt` and `oidc` alternatives, and + the `BackendTrafficPolicy` that turns the resulting client identity into a + per-client token budget. + +- **Switchyard in front** — *not* solved. The Gateway is downstream, so it + authenticates Switchyard to the provider, not the client to Switchyard. Read + [`04-restrict-access.yaml`](switchyard-in-front-of-envoy-ai-gateway/04-restrict-access.yaml) + before exposing anything: either keep Switchyard cluster-internal behind a + NetworkPolicy, or put a Gateway in front of it too, making a sandwich — + Gateway (client auth) → Switchyard (routing) → Gateway (provider auth). + +Note that the provider credential is not the whole risk. Even when the key +lives safely in the Gateway, an unauthenticated caller can still spend it. + +For machine clients prefer `jwt` over `oidc`: the OIDC flow needs a browser +redirect an SDK client cannot complete. + +## Prerequisites + +Kubernetes 1.32 or newer, which Envoy AI Gateway v1.0.0 requires. + +```bash +helm upgrade -i aieg-crd oci://docker.io/envoyproxy/ai-gateway-crds-helm \ + --version v1.0.0 --namespace envoy-ai-gateway-system --create-namespace + +helm upgrade -i eg oci://docker.io/envoyproxy/gateway-helm \ + --version v1.8.3 --namespace envoy-gateway-system --create-namespace \ + -f https://raw.githubusercontent.com/envoyproxy/ai-gateway/v1.0.0/manifests/envoy-gateway-values.yaml + +helm upgrade -i aieg oci://docker.io/envoyproxy/ai-gateway-helm \ + --version v1.0.0 --namespace envoy-ai-gateway-system --create-namespace + +# Envoy Gateway registers the AI Gateway extension hooks at startup, so it has +# to be restarted once the AI Gateway controller Service exists. +kubectl -n envoy-gateway-system rollout restart deployment/envoy-gateway +``` + +Build and publish the Switchyard image from the repository root: + +```bash +docker build -t ghcr.io/nvidia-nemo/switchyard/switchyard-server:0.2.0 . +docker push ghcr.io/nvidia-nemo/switchyard/switchyard-server:0.2.0 +``` + +## Envoy AI Gateway in front of Switchyard + +```bash +kubectl create namespace switchyard + +kubectl -n switchyard create secret generic switchyard-keys \ + --from-literal=NVIDIA_API_KEY="$NVIDIA_API_KEY" + +helm upgrade --install switchyard deploy/helm/switchyard \ + --namespace switchyard \ + -f examples/kubernetes/envoy-ai-gateway-in-front/values.switchyard.yaml + +kubectl apply -f examples/kubernetes/envoy-ai-gateway-in-front/01-gateway.yaml +kubectl apply -f examples/kubernetes/envoy-ai-gateway-in-front/02-switchyard-backend.yaml +kubectl apply -f examples/kubernetes/envoy-ai-gateway-in-front/03-forwarded-host.yaml + +kubectl -n switchyard wait --for=condition=Programmed gateway/switchyard-ai-gateway --timeout=5m +``` + +Send a request naming a Switchyard route id as the model: + +```bash +GW=$(kubectl -n switchyard get gateway switchyard-ai-gateway \ + -o jsonpath='{.status.addresses[0].value}') + +curl -s "http://$GW/v1/chat/completions" \ + -H 'content-type: application/json' \ + -d '{"model":"switchyard/general","messages":[{"role":"user","content":"hello"}],"max_tokens":600}' +``` + +Envoy extracts `model` from the body into the `x-ai-eg-model` header, matches +it against the `AIGatewayRoute` rules, and forwards to the `AIServiceBackend` +that points at the Switchyard Service. Switchyard then runs the named algorithm +and calls the provider it selects. + +Every route id you want reachable needs a matcher in +[`02-switchyard-backend.yaml`](envoy-ai-gateway-in-front/02-switchyard-backend.yaml). +A model with no matching rule gets a 404 from Envoy, never reaching Switchyard. + +Apply [`04-client-auth.yaml`](envoy-ai-gateway-in-front/04-client-auth.yaml) to +require a client credential; requests then need `-H "x-api-key: ..."`. + +## Switchyard in front of Envoy AI Gateway + +```bash +kubectl create namespace switchyard + +# BackendSecurityPolicy requires the Secret key to be literally `apiKey`. +kubectl -n switchyard create secret generic nvidia-apikey \ + --from-literal=apiKey="$NVIDIA_API_KEY" + +helm upgrade --install switchyard-egress deploy/helm/switchyard \ + --namespace switchyard \ + -f examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/values.switchyard.yaml + +kubectl apply -f examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/01-gateway.yaml +kubectl apply -f examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/02-provider-backend.yaml +kubectl apply -f examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/03-forwarded-host.yaml + +kubectl -n switchyard wait --for=condition=Programmed gateway/switchyard-upstream --timeout=5m +``` + +```bash +kubectl -n switchyard port-forward svc/switchyard-egress 4000:4000 & + +curl -s localhost:4000/v1/chat/completions \ + -H 'content-type: application/json' \ + -d '{"model":"switchyard/general","messages":[{"role":"user","content":"hello"}],"max_tokens":600}' +``` + +Here the deployment TOML's `base_url` points at the Gateway's ClusterIP +Service, and no `api_key_env` is set, so Switchyard sends no `Authorization` +header. Envoy adds one from the `BackendSecurityPolicy` on the way out. + +The Service name is pinned through +`EnvoyProxy.spec.provider.kubernetes.envoyService.name`, because Envoy Gateway +otherwise generates a hashed name that would change if the Gateway were +recreated — and `base_url` has to be stable. + +Envoy matches on **provider** model ids here, so the values in +[`02-provider-backend.yaml`](switchyard-in-front-of-envoy-ai-gateway/02-provider-backend.yaml) +must match the `id` field of each `[targets.*]` table, not the route ids. + +## Notes + +**`x-forwarded-host` breaks some providers.** Envoy preserves the client's +original Host in `x-forwarded-host` when it rewrites Host for the backend, and +Switchyard forwards inbound request headers to the provider it selects — so the +header travels all the way upstream. A provider that routes on it resolves the +wrong virtual host and rejects the call; the NVIDIA inference endpoint answers +with a model-group 404 even when the header holds its own hostname. Both +examples ship an `EnvoyPatchPolicy` (`03-forwarded-host.yaml`) that removes it. +`AIServiceBackend.headerMutation` is not sufficient on its own, because the AI +Gateway's ext-proc mutation runs before Envoy sets the header. + +**Buffer limits.** Envoy Gateway defaults client connection buffers to 32KiB, +which truncates realistic chat payloads. Both examples set a +`ClientTrafficPolicy` with `bufferLimit: 50Mi`. + +**Timeouts.** Both examples set a `BackendTrafficPolicy` request timeout of +300s. LLM calls are slow, and an `llm_classifier` route adds a classifier call +in front of the served call, so the Envoy default would cut requests off. + +**Reasoning models.** Models that emit `reasoning_content` spend the token +budget on reasoning before producing `content`. Too small a `max_tokens` yields +`finish_reason: "length"` with `content: null` — allow a few hundred tokens. + +**Path prefixes.** `AIServiceBackend.spec.schema.prefix` sets the upstream +path. The examples use `/v1`; OpenRouter needs `/api/v1`. + +**Startup validation.** `switchyard-server` refuses to start when an +`api_key_env` named in the TOML is missing from the environment. A missing or +misnamed Secret key therefore surfaces as a crash-looping pod, not as +unauthenticated upstream calls. Validate a deployment before rolling it out +with `switchyard-server --config routes.toml --dry-run`. + +**Metrics.** Switchyard exports its own Prometheus metrics on the same port at +`/metrics`, covering routing overhead, per-model tokens, and classifier +fail-open counts. Those complement, rather than duplicate, Envoy's token +metrics; see the +[metrics table](../../crates/switchyard-server/README.md#metrics). diff --git a/examples/kubernetes/envoy-ai-gateway-in-front/01-gateway.yaml b/examples/kubernetes/envoy-ai-gateway-in-front/01-gateway.yaml new file mode 100644 index 000000000..67d0cf3d3 --- /dev/null +++ b/examples/kubernetes/envoy-ai-gateway-in-front/01-gateway.yaml @@ -0,0 +1,82 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Envoy AI Gateway in front of Switchyard: the Gateway itself. +# +# Clients reach this Gateway, which applies gateway-level concerns (routing by +# model, token rate limiting, upstream authentication) and forwards to +# Switchyard, which then translates and routes to model backends. +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: switchyard-ai-gateway +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: switchyard-ai-gateway + namespace: switchyard +spec: + gatewayClassName: switchyard-ai-gateway + listeners: + - name: http + protocol: HTTP + port: 80 + infrastructure: + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: switchyard-ai-gateway +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: switchyard-ai-gateway + namespace: switchyard +spec: + provider: + type: Kubernetes + kubernetes: + envoyDeployment: + container: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi +--- +# Envoy Gateway defaults the client buffer limit to 32KiB, which truncates +# large chat payloads. AI workloads routinely exceed it. +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: ClientTrafficPolicy +metadata: + name: switchyard-buffer-limit + namespace: switchyard +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: switchyard-ai-gateway + connection: + bufferLimit: 50Mi +--- +# LLM calls are slow, and Switchyard's llm_classifier routes add a classifier +# call in front of the served call. The Envoy Gateway default route timeout +# would cut those off. +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: BackendTrafficPolicy +metadata: + name: switchyard-timeouts + namespace: switchyard +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: switchyard-ai-gateway + timeout: + http: + requestTimeout: 300s diff --git a/examples/kubernetes/envoy-ai-gateway-in-front/02-switchyard-backend.yaml b/examples/kubernetes/envoy-ai-gateway-in-front/02-switchyard-backend.yaml new file mode 100644 index 000000000..b2d5d2a44 --- /dev/null +++ b/examples/kubernetes/envoy-ai-gateway-in-front/02-switchyard-backend.yaml @@ -0,0 +1,103 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Registers Switchyard as an AI service backend and routes model names to it. +# +# The model names matched here are Switchyard *route ids* -- the `id` field of +# each `[routes.*]` table in the deployment TOML. A client asking for +# "switchyard/general" reaches this Gateway, which forwards to Switchyard, +# which runs the `general` algorithm and picks a target. +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: Backend +metadata: + name: switchyard + namespace: switchyard +spec: + endpoints: + # Service and port created by the switchyard Helm chart. + - fqdn: + hostname: switchyard.switchyard.svc.cluster.local + port: 4000 +--- +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: AIServiceBackend +metadata: + name: switchyard + namespace: switchyard +spec: + # Switchyard accepts OpenAI Chat Completions, so the gateway needs no + # translation on this hop. Switchyard does its own translation downstream if + # a target speaks Anthropic Messages or OpenAI Responses. + schema: + name: OpenAI + backendRef: + name: switchyard + kind: Backend + group: gateway.envoyproxy.io + # Switchyard forwards inbound request headers to the provider it selects, so + # the proxy headers Envoy adds on this hop would travel all the way upstream. + # Providers that route on `x-forwarded-host` then resolve the wrong virtual + # host and reject the call. Drop it here rather than at the provider, since + # the header carries no meaning past this hop. + headerMutation: + remove: + - x-forwarded-host +--- +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: AIGatewayRoute +metadata: + name: switchyard + namespace: switchyard +spec: + parentRefs: + - name: switchyard-ai-gateway + kind: Gateway + group: gateway.networking.k8s.io + rules: + # x-ai-eg-model is set by the AI Gateway from the request body's `model` + # field. One matcher per Switchyard route id you expose; a model with no + # matching rule gets a 404 here and never reaches Switchyard. + - matches: + - headers: + - type: Exact + name: x-ai-eg-model + value: switchyard/classified + backendRefs: + - name: switchyard + - matches: + - headers: + - type: Exact + name: x-ai-eg-model + value: switchyard/general + backendRefs: + - name: switchyard + - matches: + - headers: + - type: Exact + name: x-ai-eg-model + value: switchyard/nano + backendRefs: + - name: switchyard + - matches: + - headers: + - type: Exact + name: x-ai-eg-model + value: switchyard/super + backendRefs: + - name: switchyard + - matches: + - headers: + - type: Exact + name: x-ai-eg-model + value: switchyard/ultra + backendRefs: + - name: switchyard + # Token accounting for gateway-level rate limiting. Switchyard reports usage + # in the OpenAI-standard `usage` object, so the gateway can meter it. + llmRequestCosts: + - metadataKey: llm_input_token + type: InputToken + - metadataKey: llm_output_token + type: OutputToken + - metadataKey: llm_total_token + type: TotalToken diff --git a/examples/kubernetes/envoy-ai-gateway-in-front/03-forwarded-host.yaml b/examples/kubernetes/envoy-ai-gateway-in-front/03-forwarded-host.yaml new file mode 100644 index 000000000..2ebb5d112 --- /dev/null +++ b/examples/kubernetes/envoy-ai-gateway-in-front/03-forwarded-host.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Stops `x-forwarded-host` from reaching the model provider. +# +# Envoy preserves the client's original Host in `x-forwarded-host` when it +# rewrites Host for the backend. Switchyard forwards inbound request headers to +# the provider it selects, so a header added on this hop travels all the way +# upstream. Providers that route on `x-forwarded-host` then resolve the wrong +# virtual host and reject the call -- the NVIDIA inference endpoint answers +# with a model-group 404 even when the header holds its own hostname. +# +# `AIServiceBackend.headerMutation` is not sufficient on its own: the AI +# Gateway's ext-proc mutation runs before Envoy sets the header, so it is +# re-added afterwards. Removing it on the RouteConfiguration happens during +# routing, after ext-proc. +# +# Requires `extensionApis.enableEnvoyPatchPolicy: true` in the Envoy Gateway +# Helm values, which the AI Gateway base values already set. +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyPatchPolicy +metadata: + name: switchyard-strip-forwarded-host + namespace: switchyard +spec: + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: switchyard-ai-gateway + type: JSONPatch + jsonPatches: + # Name is "//". + - type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration" + name: switchyard/switchyard-ai-gateway/http + operation: + op: add + path: /request_headers_to_remove + value: + - x-forwarded-host diff --git a/examples/kubernetes/envoy-ai-gateway-in-front/04-client-auth.yaml b/examples/kubernetes/envoy-ai-gateway-in-front/04-client-auth.yaml new file mode 100644 index 000000000..53706694a --- /dev/null +++ b/examples/kubernetes/envoy-ai-gateway-in-front/04-client-auth.yaml @@ -0,0 +1,101 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Client authentication at the Gateway. +# +# Switchyard has no authentication of its own: `switchyard-server` serves every +# request that reaches its port. In this topology the Gateway is the front +# door, so client identity is established here, before anything reaches +# Switchyard. +# +# API-key auth is shown because it is the least infrastructure. Swap in the +# `jwt` or `oidc` blocks below for an identity provider. +# +# kubectl -n switchyard create secret generic switchyard-client-keys \ +# --from-literal=team-a="$(openssl rand -hex 32)" \ +# --from-literal=team-b="$(openssl rand -hex 32)" +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: SecurityPolicy +metadata: + name: switchyard-client-auth + namespace: switchyard +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: switchyard-ai-gateway + apiKeyAuth: + credentialRefs: + - name: switchyard-client-keys + extractFrom: + - headers: + - x-api-key + # Strip the client key so it never reaches Switchyard or the provider. + sanitize: true + # Surfaces the matched credential name downstream, which is what the token + # rate-limit policy buckets on. + forwardClientIDHeader: x-client-id + +# --- OIDC alternative ------------------------------------------------------- +# Replace the apiKeyAuth block above with this to authenticate humans through +# an identity provider. Machine clients should use `jwt` instead: OIDC runs a +# browser redirect flow, which an SDK client cannot complete. +# +# oidc: +# provider: +# issuer: "https://accounts.example.com" +# clientID: "switchyard-gateway" +# clientSecret: +# name: switchyard-oidc-client-secret +# scopes: ["openid", "email"] +# redirectURL: "http://switchyard.example.com/oauth2/callback" +# logoutPath: "/logout" +# +# --- JWT alternative, for machine clients ----------------------------------- +# +# jwt: +# providers: +# - name: corp-idp +# issuer: "https://accounts.example.com" +# remoteJWKS: +# uri: "https://accounts.example.com/.well-known/jwks.json" +# claimToHeaders: +# - claim: sub +# header: x-client-id +--- +# Per-client token budgets, keyed on the identity established above. +# +# The AIGatewayRoute records token usage into `llm_input_token` and friends; +# this turns those counters into an enforced quota. Without a client identity +# the only available bucket is the source IP, which is why authentication and +# rate limiting are configured together. +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: BackendTrafficPolicy +metadata: + name: switchyard-token-limit + namespace: switchyard +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: switchyard-ai-gateway + rateLimit: + type: Global + global: + rules: + - clientSelectors: + - headers: + - name: x-client-id + type: Distinct + limit: + requests: 100000 + unit: Hour + cost: + request: + from: Number + number: 0 + response: + from: Metadata + metadata: + namespace: io.envoy.ai_gateway + key: llm_total_token diff --git a/examples/kubernetes/envoy-ai-gateway-in-front/values.switchyard.yaml b/examples/kubernetes/envoy-ai-gateway-in-front/values.switchyard.yaml new file mode 100644 index 000000000..770f84ff1 --- /dev/null +++ b/examples/kubernetes/envoy-ai-gateway-in-front/values.switchyard.yaml @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Helm values for Switchyard when Envoy AI Gateway sits in front. +# +# helm upgrade --install switchyard deploy/helm/switchyard \ +# --namespace switchyard --create-namespace \ +# -f examples/kubernetes/envoy-ai-gateway-in-front/values.switchyard.yaml + +image: + repository: ghcr.io/nvidia-nemo/switchyard/switchyard-server + tag: "0.2.0" + +# Secret holding NVIDIA_API_KEY. Create it before installing: +# kubectl -n switchyard create secret generic switchyard-keys \ +# --from-literal=NVIDIA_API_KEY="$NVIDIA_API_KEY" +# +# switchyard-server exits non-zero when an `api_key_env` named below is missing +# from the environment, so a misnamed key surfaces as a crash-looping pod +# rather than as unauthenticated upstream calls. +apiKeySecret: + name: switchyard-keys + +# Switchyard owns provider credentials in this topology and reaches the +# provider directly. The Gateway in front handles ingress concerns only. +config: + routes: | + schema_version = 1 + + [llm_clients.nvidia] + format = "openai_chat" + base_url = "https://inference-api.nvidia.com/v1" + api_key_env = "NVIDIA_API_KEY" + max_retries = 2 + + # Three tiers of the same model family. The point of the classifier route + # below is to spend `ultra` tokens only on the requests that need them. + [targets.nano] + id = "nvidia/nvidia/nemotron-3-nano-30b-a3b" + llm_client = "nvidia" + + [targets.super] + id = "nvidia/nvidia/nemotron-3-super-v3" + llm_client = "nvidia" + + [targets.ultra] + id = "nvidia/nvidia/nemotron-3-ultra" + llm_client = "nvidia" + + # Single-tier routes, useful as cost and accuracy baselines. + [routes.nano] + id = "switchyard/nano" + type = "passthrough" + target = "nano" + + [routes.super] + id = "switchyard/super" + type = "passthrough" + target = "super" + + [routes.ultra] + id = "switchyard/ultra" + type = "passthrough" + target = "ultra" + + # Sends each task to `classifier_target` for a capability verdict, then + # serves it from `weak_target` or `strong_target`. Using the cheap tier as + # its own judge keeps the routing decision inexpensive. + # + # Raise base_threshold to send less traffic to the weak tier. Anything the + # judge cannot decide goes to strong_target. + [routes.classified] + id = "switchyard/classified" + type = "llm_classifier" + mode = "capability" + classifier_target = "nano" + weak_target = "nano" + strong_target = "ultra" + base_threshold = 0.5 + + # Even split across tiers, for A/B comparison rather than cost control. + [routes.general] + id = "switchyard/general" + type = "random" + targets = ["nano", "super", "ultra"] + weights = [1, 1, 1] diff --git a/examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/01-gateway.yaml b/examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/01-gateway.yaml new file mode 100644 index 000000000..1962004a7 --- /dev/null +++ b/examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/01-gateway.yaml @@ -0,0 +1,85 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Switchyard in front of Envoy AI Gateway: the Gateway itself. +# +# Here the Gateway is an internal egress hop. Clients talk to Switchyard, which +# runs its algorithm, picks a target, and sends the call to this Gateway. The +# Gateway owns provider credentials and upstream TLS, so no provider key ever +# reaches the Switchyard pod. +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: switchyard-upstream-gateway +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: switchyard-upstream + namespace: switchyard +spec: + gatewayClassName: switchyard-upstream-gateway + listeners: + - name: http + protocol: HTTP + port: 80 + infrastructure: + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: switchyard-upstream +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: switchyard-upstream + namespace: switchyard +spec: + provider: + type: Kubernetes + kubernetes: + envoyService: + # Pinning the name gives Switchyard a stable base_url. Without this, + # Envoy Gateway generates a hashed service name that changes if the + # Gateway is recreated. + name: switchyard-upstream-gateway + # This Gateway is reachable only from inside the cluster. + type: ClusterIP + envoyDeployment: + container: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: ClientTrafficPolicy +metadata: + name: switchyard-upstream-buffer-limit + namespace: switchyard +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: switchyard-upstream + connection: + bufferLimit: 50Mi +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: BackendTrafficPolicy +metadata: + name: switchyard-upstream-timeouts + namespace: switchyard +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: switchyard-upstream + timeout: + http: + requestTimeout: 300s diff --git a/examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/02-provider-backend.yaml b/examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/02-provider-backend.yaml new file mode 100644 index 000000000..08fd47894 --- /dev/null +++ b/examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/02-provider-backend.yaml @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# The upstream provider, fronted by Envoy AI Gateway. +# +# Switchyard sends OpenAI Chat Completions to the Gateway with the *provider's* +# model id. The Gateway matches on that id, terminates upstream TLS, and +# injects the API key from a Secret -- so the key stays in the gateway's trust +# boundary rather than in the Switchyard pod's environment. +# +# Swap the hostname and model ids below for your own provider. +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: Backend +metadata: + name: nvidia + namespace: switchyard +spec: + endpoints: + - fqdn: + hostname: inference-api.nvidia.com + port: 443 +--- +apiVersion: gateway.networking.k8s.io/v1alpha3 +kind: BackendTLSPolicy +metadata: + name: nvidia-tls + namespace: switchyard +spec: + targetRefs: + - group: gateway.envoyproxy.io + kind: Backend + name: nvidia + validation: + wellKnownCACertificates: System + hostname: inference-api.nvidia.com +--- +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: AIServiceBackend +metadata: + name: nvidia + namespace: switchyard +spec: + schema: + name: OpenAI + # This endpoint serves the OpenAI API at /v1. Set `/api/v1` for OpenRouter, + # or drop the field entirely when the provider uses the default. + prefix: /v1 + backendRef: + name: nvidia + kind: Backend + group: gateway.envoyproxy.io +--- +# Injects `Authorization: Bearer ` on the way to the provider. +# The Secret's key must literally be `apiKey`: +# kubectl -n switchyard create secret generic nvidia-apikey \ +# --from-literal=apiKey="$NVIDIA_API_KEY" +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: BackendSecurityPolicy +metadata: + name: nvidia-apikey + namespace: switchyard +spec: + type: APIKey + apiKey: + secretRef: + name: nvidia-apikey + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: nvidia +--- +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: AIGatewayRoute +metadata: + name: nvidia + namespace: switchyard +spec: + parentRefs: + - name: switchyard-upstream + kind: Gateway + group: gateway.networking.k8s.io + rules: + # One rule per provider model id. These are the `id` fields of the + # `[targets.*]` tables in the deployment TOML, not Switchyard route ids. + - matches: + - headers: + - type: Exact + name: x-ai-eg-model + value: nvidia/nvidia/nemotron-3-super-v3 + backendRefs: + - name: nvidia + llmRequestCosts: + - metadataKey: llm_input_token + type: InputToken + - metadataKey: llm_output_token + type: OutputToken + - metadataKey: llm_total_token + type: TotalToken diff --git a/examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/03-forwarded-host.yaml b/examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/03-forwarded-host.yaml new file mode 100644 index 000000000..58e5f704e --- /dev/null +++ b/examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/03-forwarded-host.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Stops `x-forwarded-host` from reaching the model provider. +# +# Same underlying issue as the ingress case, but this Gateway is the last hop +# before the provider, so `request_headers_to_remove` alone does not help: +# Envoy sets `x-forwarded-host` in the router filter, which runs after header +# removal. `append_x_forwarded_host` is the switch that puts it there at all, +# and Envoy Gateway turns it on for Backend-backed routes. +# +# Both patches are applied: the flag stops the header being added, and the +# removal covers any route the flag patch does not reach. +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyPatchPolicy +metadata: + name: switchyard-upstream-strip-forwarded-host + namespace: switchyard +spec: + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: switchyard-upstream + type: JSONPatch + jsonPatches: + # The route index is positional. Re-check it after changing the rules in + # the AIGatewayRoute, via the Envoy admin endpoint on an envoy pod: + # kubectl -n envoy-gateway-system port-forward pod/ 19000:19000 + # curl -s 'localhost:19000/config_dump?resource=dynamic_route_configs' + - type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration" + name: switchyard/switchyard-upstream/http + operation: + op: replace + path: /virtual_hosts/0/routes/0/route/append_x_forwarded_host + value: false + - type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration" + name: switchyard/switchyard-upstream/http + operation: + op: add + path: /request_headers_to_remove + value: + - x-forwarded-host diff --git a/examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/04-restrict-access.yaml b/examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/04-restrict-access.yaml new file mode 100644 index 000000000..9adb8cbb0 --- /dev/null +++ b/examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/04-restrict-access.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Who authenticates the client in this topology? +# +# Nothing in this directory does, and that is the point to be deliberate about. +# `switchyard-server` serves every request that reaches its port -- it has no +# API-key check, no JWT validation, no mTLS. In this topology the Gateway sits +# *behind* Switchyard, so it authenticates Switchyard to the provider, not the +# client to Switchyard. +# +# That leaves three honest options: +# +# 1. Keep Switchyard cluster-internal. The Service is ClusterIP and callers +# are trusted in-cluster workloads. The NetworkPolicy below makes that a +# property of the cluster rather than an assumption. +# +# 2. Put an Envoy Gateway in front of Switchyard as well, giving a sandwich: +# Gateway (client auth) -> Switchyard (routing) -> Gateway (provider auth). +# Take 01-gateway.yaml and 04-client-auth.yaml from +# ../envoy-ai-gateway-in-front/ and point them at this release. This is the +# right answer whenever Switchyard is reachable from outside the cluster. +# +# 3. Terminate auth in a sidecar in front of the Switchyard container. +# Equivalent to 2 with more moving parts and no gateway features; only +# worth it if a gateway is not an option. +# +# Do not expose this Service through an Ingress or a LoadBalancer without one +# of the above. The provider credential is safe -- it lives in the Gateway -- +# but an unauthenticated caller can still spend it. +# +# Note: NetworkPolicy is only enforced if the cluster's CNI implements it. +# k3s does so through its bundled kube-router policy controller; plain flannel +# does not. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: switchyard-egress-ingress-restriction + namespace: switchyard +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: switchyard + app.kubernetes.io/instance: switchyard-egress + policyTypes: + - Ingress + ingress: + # Only workloads explicitly labelled as Switchyard clients may call it. + # Label them with `switchyard.nvidia.com/client: "true"`. + - from: + - podSelector: + matchLabels: + switchyard.nvidia.com/client: "true" + ports: + - protocol: TCP + port: 4000 diff --git a/examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/values.switchyard.yaml b/examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/values.switchyard.yaml new file mode 100644 index 000000000..760c4c3ec --- /dev/null +++ b/examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/values.switchyard.yaml @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Helm values for Switchyard when it sits in front of Envoy AI Gateway. +# +# helm upgrade --install switchyard-egress deploy/helm/switchyard \ +# --namespace switchyard --create-namespace \ +# -f examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/values.switchyard.yaml + +image: + repository: ghcr.io/nvidia-nemo/switchyard/switchyard-server + tag: "0.2.0" + +# No apiKeySecret: the Gateway injects provider credentials downstream, so the +# Switchyard pod holds none. `api_key_env` is therefore omitted from the +# llm_client below, which makes Switchyard send no Authorization header. + +# Switchyard authenticates no one. Keep this ClusterIP and read +# 04-restrict-access.yaml before exposing it. +service: + type: ClusterIP + +config: + routes: | + schema_version = 1 + + [llm_clients.gateway] + format = "openai_chat" + # The ClusterIP Service pinned by EnvoyProxy.provider.kubernetes.envoyService + # in 01-gateway.yaml. Plain HTTP: the hop is in-cluster and the Gateway + # terminates TLS to the provider. + base_url = "http://switchyard-upstream-gateway.envoy-gateway-system.svc.cluster.local/v1" + max_retries = 2 + + # Target ids are provider model ids. The Gateway matches on them via + # x-ai-eg-model and picks the provider backend. + [targets.nemotron] + id = "nvidia/nvidia/nemotron-3-super-v3" + llm_client = "gateway" + + [routes.general] + id = "switchyard/general" + type = "random" + targets = ["nemotron"] + + # See the note in the sibling example: two distinct tiers are needed for a + # classifier route to change any outcome. + [routes.classified] + id = "switchyard/classified" + type = "llm_classifier" + mode = "capability" + classifier_target = "nemotron" + strong_target = "nemotron" + weak_target = "nemotron" + base_threshold = 0.5 From 20711f4e501ca76dbae09f4c7a0b42a2887641dd Mon Sep 17 00:00:00 2001 From: ansjindal Date: Thu, 6 Aug 2026 11:34:48 +0200 Subject: [PATCH 2/6] fix(deploy): honour TLS probe scheme, zero maxUnavailable, and restrict gateway access Signed-off-by: ansjindal --- deploy/helm/switchyard/templates/_helpers.tpl | 17 +++++++ .../helm/switchyard/templates/deployment.yaml | 6 +-- deploy/helm/switchyard/templates/pdb.yaml | 5 +- examples/kubernetes/README.md | 14 +++++- .../04-restrict-access.yaml | 47 +++++++++++++++++++ 5 files changed, 84 insertions(+), 5 deletions(-) diff --git a/deploy/helm/switchyard/templates/_helpers.tpl b/deploy/helm/switchyard/templates/_helpers.tpl index 22536e98c..c5754457d 100644 --- a/deploy/helm/switchyard/templates/_helpers.tpl +++ b/deploy/helm/switchyard/templates/_helpers.tpl @@ -71,6 +71,23 @@ Name of the Secret providing upstream API keys, or "" when none is configured. {{- printf "%s:%s" .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }} {{- end }} +{{/* +Render a probe, defaulting its scheme to HTTPS when Switchyard terminates TLS. + +kubelet probes default to HTTP. Against a TLS listener that fails, so the pod +would never pass its probes when tls.enabled is set. An explicitly configured +scheme always wins. +*/}} +{{- define "switchyard.probe" -}} +{{- $probe := deepCopy .probe -}} +{{- if and .root.Values.tls.enabled (hasKey $probe "httpGet") -}} +{{- if not (hasKey $probe.httpGet "scheme") -}} +{{- $_ := set $probe.httpGet "scheme" "HTTPS" -}} +{{- end -}} +{{- end -}} +{{- toYaml $probe -}} +{{- end }} + {{/* Absolute path to the deployment TOML inside the container. */}} diff --git a/deploy/helm/switchyard/templates/deployment.yaml b/deploy/helm/switchyard/templates/deployment.yaml index 5c0a31b23..58babe273 100644 --- a/deploy/helm/switchyard/templates/deployment.yaml +++ b/deploy/helm/switchyard/templates/deployment.yaml @@ -107,15 +107,15 @@ spec: {{- end }} {{- with .Values.livenessProbe }} livenessProbe: - {{- toYaml . | nindent 12 }} + {{- include "switchyard.probe" (dict "probe" . "root" $) | nindent 12 }} {{- end }} {{- with .Values.readinessProbe }} readinessProbe: - {{- toYaml . | nindent 12 }} + {{- include "switchyard.probe" (dict "probe" . "root" $) | nindent 12 }} {{- end }} {{- with .Values.startupProbe }} startupProbe: - {{- toYaml . | nindent 12 }} + {{- include "switchyard.probe" (dict "probe" . "root" $) | nindent 12 }} {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} diff --git a/deploy/helm/switchyard/templates/pdb.yaml b/deploy/helm/switchyard/templates/pdb.yaml index 0ed7cd26d..157f5ef4b 100644 --- a/deploy/helm/switchyard/templates/pdb.yaml +++ b/deploy/helm/switchyard/templates/pdb.yaml @@ -10,7 +10,10 @@ metadata: labels: {{- include "switchyard.labels" . | nindent 4 }} spec: - {{- if .Values.podDisruptionBudget.maxUnavailable }} + {{- /* Compare against the unset default rather than truthiness, so an + explicit maxUnavailable of 0 is honoured instead of falling through + to minAvailable. */}} + {{- if ne (toString .Values.podDisruptionBudget.maxUnavailable) "" }} maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} {{- else }} minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} diff --git a/examples/kubernetes/README.md b/examples/kubernetes/README.md index 5b3d9c558..ab119e31b 100644 --- a/examples/kubernetes/README.md +++ b/examples/kubernetes/README.md @@ -92,9 +92,15 @@ docker push ghcr.io/nvidia-nemo/switchyard/switchyard-server:0.2.0 ```bash kubectl create namespace switchyard +# Provider credential, read by Switchyard. kubectl -n switchyard create secret generic switchyard-keys \ --from-literal=NVIDIA_API_KEY="$NVIDIA_API_KEY" +# Client credential, checked by the Gateway. Without this the Gateway accepts +# anonymous traffic and anyone who can reach it can spend the provider key. +kubectl -n switchyard create secret generic switchyard-client-keys \ + --from-literal=team-a="$(openssl rand -hex 32)" + helm upgrade --install switchyard deploy/helm/switchyard \ --namespace switchyard \ -f examples/kubernetes/envoy-ai-gateway-in-front/values.switchyard.yaml @@ -102,21 +108,27 @@ helm upgrade --install switchyard deploy/helm/switchyard \ kubectl apply -f examples/kubernetes/envoy-ai-gateway-in-front/01-gateway.yaml kubectl apply -f examples/kubernetes/envoy-ai-gateway-in-front/02-switchyard-backend.yaml kubectl apply -f examples/kubernetes/envoy-ai-gateway-in-front/03-forwarded-host.yaml +kubectl apply -f examples/kubernetes/envoy-ai-gateway-in-front/04-client-auth.yaml kubectl -n switchyard wait --for=condition=Programmed gateway/switchyard-ai-gateway --timeout=5m ``` -Send a request naming a Switchyard route id as the model: +Send a request naming a Switchyard route id as the model, with the client key: ```bash GW=$(kubectl -n switchyard get gateway switchyard-ai-gateway \ -o jsonpath='{.status.addresses[0].value}') +API_KEY=$(kubectl -n switchyard get secret switchyard-client-keys \ + -o jsonpath='{.data.team-a}' | base64 -d) curl -s "http://$GW/v1/chat/completions" \ + -H "x-api-key: $API_KEY" \ -H 'content-type: application/json' \ -d '{"model":"switchyard/general","messages":[{"role":"user","content":"hello"}],"max_tokens":600}' ``` +Requests without a valid `x-api-key` are rejected with 401 at the Gateway. + Envoy extracts `model` from the body into the `x-ai-eg-model` header, matches it against the `AIGatewayRoute` rules, and forwards to the `AIServiceBackend` that points at the Switchyard Service. Switchyard then runs the named algorithm diff --git a/examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/04-restrict-access.yaml b/examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/04-restrict-access.yaml index 9adb8cbb0..166ac1b14 100644 --- a/examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/04-restrict-access.yaml +++ b/examples/kubernetes/switchyard-in-front-of-envoy-ai-gateway/04-restrict-access.yaml @@ -54,3 +54,50 @@ spec: ports: - protocol: TCP port: 4000 +--- +# The Gateway is the other half of the problem, and the more sensitive half. +# +# It is a ClusterIP Service and the BackendSecurityPolicy attaches the provider +# credential to anything it forwards, so any pod in the cluster that can reach +# it can spend that credential -- without ever touching Switchyard. Restricting +# Switchyard alone leaves that path open. +# +# Envoy Gateway labels the pods it generates with the owning Gateway's name and +# namespace; both are matched so this cannot select another Gateway's proxies. +# Confirm they match your cluster before relying on it: +# kubectl -n envoy-gateway-system get pods \ +# -l gateway.envoyproxy.io/owning-gateway-name=switchyard-upstream --show-labels +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: switchyard-upstream-gateway-restriction + namespace: envoy-gateway-system +spec: + podSelector: + matchLabels: + gateway.envoyproxy.io/owning-gateway-name: switchyard-upstream + gateway.envoyproxy.io/owning-gateway-namespace: switchyard + policyTypes: + - Ingress + ingress: + # Switchyard is the only legitimate client of this Gateway. namespaceSelector + # and podSelector sit in one `from` entry, so both must match. + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: switchyard + podSelector: + matchLabels: + app.kubernetes.io/name: switchyard + ports: + - protocol: TCP + # Envoy listens on listener port + 10000; the Service maps 80 -> 10080. + port: 10080 + # Readiness probes and metrics scraping arrive from the kubelet and the + # monitoring stack, not from Switchyard. Omitting these would take the + # Gateway out of service rather than secure it. + - ports: + - protocol: TCP + port: 19001 + - protocol: TCP + port: 19003 From d3f30a461185f71b85f035f76759759658171f7a Mon Sep 17 00:00:00 2001 From: ansjindal Date: Thu, 6 Aug 2026 13:01:00 +0200 Subject: [PATCH 3/6] docs(examples): add topology diagrams and component inventory to kubernetes README Signed-off-by: ansjindal --- examples/kubernetes/README.md | 138 ++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) diff --git a/examples/kubernetes/README.md b/examples/kubernetes/README.md index ab119e31b..bb5d0b3e3 100644 --- a/examples/kubernetes/README.md +++ b/examples/kubernetes/README.md @@ -14,6 +14,29 @@ Switchyard half and plain manifests for the Envoy half. ## Which topology +```mermaid +flowchart LR + client["Client"] + aeg["Envoy proxy
client auth, token budgets"] + asy["Switchyard
holds provider key"] + bsy["Switchyard
no credential"] + beg["Envoy proxy
injects provider key"] + prov["Model provider"] + + client --> aeg --> asy --> prov + client --> bsy --> beg --> prov + + classDef cred fill:#fdf0d5,stroke:#b8860b,color:#5c4400 + classDef envoy fill:#e8e6ff,stroke:#6b5fd6,color:#2d2483 + classDef plain fill:#eef2f7,stroke:#8899aa,color:#22303c + class asy,beg cred + class aeg,bsy envoy + class client,prov plain +``` + +The top path is Envoy in front, the bottom is Switchyard in front. Amber marks +where the provider credential sits, which is the whole decision. + | | [`envoy-ai-gateway-in-front/`](envoy-ai-gateway-in-front) | [`switchyard-in-front-of-envoy-ai-gateway/`](switchyard-in-front-of-envoy-ai-gateway) | |---|---|---| | Chain | client → Envoy AI Gateway → Switchyard → provider | client → Switchyard → Envoy AI Gateway → provider | @@ -33,6 +56,23 @@ application pod. No provider credential is mounted into Switchyard at all. The two are not exclusive. Running both, as the manifests here do, gives Envoy at the edge and Envoy at the egress with Switchyard in the middle. +### The namespace split + +Applying either example creates objects in two namespaces: + +| namespace | what lands there | +|---|---| +| `switchyard` | every Gateway API and AI Gateway object below — all *declarations* — plus the Switchyard Deployment and Service | +| `envoy-gateway-system` | the Envoy proxy Deployment and Service that Envoy Gateway *generates*, and which traffic actually flows through | + +A `Gateway` declared in `switchyard` is served by a proxy pod running in +`envoy-gateway-system`. That is why +[`04-restrict-access.yaml`](switchyard-in-front-of-envoy-ai-gateway/04-restrict-access.yaml) +puts its NetworkPolicy in `envoy-gateway-system` — one in `switchyard` would +not touch the pod carrying the provider credential — and why `01-gateway.yaml` +pins `EnvoyProxy.provider.kubernetes.envoyService.name`, since the generated +name is otherwise hashed and changes when the Gateway is recreated. + ## Client authentication **Switchyard authenticates no one.** `switchyard-server` serves every request @@ -89,6 +129,54 @@ docker push ghcr.io/nvidia-nemo/switchyard/switchyard-server:0.2.0 ## Envoy AI Gateway in front of Switchyard +```mermaid +flowchart LR + client["Client"] + + subgraph egns["ns: envoy-gateway-system"] + eg["Envoy proxy"] + end + + subgraph ns["ns: switchyard"] + cfg["Gateway, routes and policies"] + sy["Switchyard :4000"] + sec[("Secret")] + end + + prov["Model provider"] + + client -->|"switchyard/general
x-api-key"| eg + cfg -.-> eg + eg -->|"matched on x-ai-eg-model"| sy + sec -.-> sy + sy -->|"nemotron-3-super-v3
Bearer key"| prov + + classDef cred fill:#fdf0d5,stroke:#b8860b,color:#5c4400 + classDef envoy fill:#e8e6ff,stroke:#6b5fd6,color:#2d2483 + classDef cfgc fill:#f2f0ff,stroke:#9a8fe0,color:#3b3183 + classDef plain fill:#eef2f7,stroke:#8899aa,color:#22303c + class sy,sec cred + class eg envoy + class cfg cfgc + class client,prov plain +``` + +Dashed arrows are configuration rather than traffic. The objects the manifests +create: + +| object | file | why | +|---|---|---| +| `GatewayClass`, `Gateway` `switchyard-ai-gateway` | `01-gateway.yaml` | the listener clients reach | +| `EnvoyProxy` `switchyard-ai-gateway` | `01-gateway.yaml` | sizes the generated proxy | +| `ClientTrafficPolicy` `switchyard-buffer-limit` | `01-gateway.yaml` | 50Mi buffer; the 32KiB default truncates AI payloads | +| `BackendTrafficPolicy` `switchyard-timeouts` | `01-gateway.yaml` | 300s timeout; the default cuts LLM calls off | +| `Backend` `switchyard` | `02-switchyard-backend.yaml` | points at `switchyard.switchyard.svc:4000` | +| `AIServiceBackend` `switchyard` | `02-switchyard-backend.yaml` | declares OpenAI schema; strips `x-forwarded-host` | +| `AIGatewayRoute` `switchyard` | `02-switchyard-backend.yaml` | one matcher per exposed route id; records token costs | +| `EnvoyPatchPolicy` `switchyard-strip-forwarded-host` | `03-forwarded-host.yaml` | removes `x-forwarded-host` during routing, after ext-proc | +| `SecurityPolicy` `switchyard-client-auth` | `04-client-auth.yaml` | requires `x-api-key`; `jwt` and `oidc` shown as alternatives | +| `BackendTrafficPolicy` `switchyard-token-limit` | `04-client-auth.yaml` | per-client token budget, keyed on the authenticated identity | + ```bash kubectl create namespace switchyard @@ -143,6 +231,56 @@ require a client credential; requests then need `-H "x-api-key: ..."`. ## Switchyard in front of Envoy AI Gateway +```mermaid +flowchart LR + client["Client"] + + subgraph ns["ns: switchyard"] + sy["Switchyard :4000
no client auth"] + cfg["Gateway, routes and policies"] + sec[("Secret")] + end + + subgraph egns["ns: envoy-gateway-system"] + eg["Envoy proxy
svc: switchyard-upstream-gateway"] + end + + prov["Model provider"] + + client -->|"NetworkPolicy"| sy + sy -->|"nemotron-3-super-v3
no Authorization"| eg + cfg -.-> eg + sec -.-> eg + eg -->|"Authorization injected
TLS to provider"| prov + + classDef cred fill:#fdf0d5,stroke:#b8860b,color:#5c4400 + classDef envoy fill:#e8e6ff,stroke:#6b5fd6,color:#2d2483 + classDef cfgc fill:#f2f0ff,stroke:#9a8fe0,color:#3b3183 + classDef plain fill:#eef2f7,stroke:#8899aa,color:#22303c + class eg,sec cred + class sy envoy + class cfg cfgc + class client,prov plain +``` + +The credential reaches the proxy, never the Switchyard pod. The objects the +manifests create: + +| object | file | why | +|---|---|---| +| `GatewayClass`, `Gateway` `switchyard-upstream` | `01-gateway.yaml` | internal egress listener, not client-facing | +| `EnvoyProxy` `switchyard-upstream` | `01-gateway.yaml` | pins the Service name, sets it ClusterIP | +| `ClientTrafficPolicy` `switchyard-upstream-buffer-limit` | `01-gateway.yaml` | 50Mi buffer | +| `BackendTrafficPolicy` `switchyard-upstream-timeouts` | `01-gateway.yaml` | 300s timeout | +| `Backend` `nvidia` | `02-provider-backend.yaml` | provider hostname, port 443 | +| `BackendTLSPolicy` `nvidia-tls` | `02-provider-backend.yaml` | validates upstream TLS against system roots | +| `AIServiceBackend` `nvidia` | `02-provider-backend.yaml` | OpenAI schema, `/v1` prefix | +| `BackendSecurityPolicy` `nvidia-apikey` | `02-provider-backend.yaml` | injects `Authorization`, keeping the key out of the pod | +| `AIGatewayRoute` `nvidia` | `02-provider-backend.yaml` | one matcher per provider model id | +| `EnvoyPatchPolicy` `switchyard-upstream-strip-forwarded-host` | `03-forwarded-host.yaml` | `append_x_forwarded_host: false`; removal alone is too late on egress | +| `NetworkPolicy` `switchyard-egress-ingress-restriction` | `04-restrict-access.yaml` | limits who may call Switchyard | +| `NetworkPolicy` `switchyard-upstream-gateway-restriction` | `04-restrict-access.yaml` | limits who may reach the credential-bearing proxy | + ```bash kubectl create namespace switchyard From e0e33a6950da4e6e32e7ef700a90b3c524823f57 Mon Sep 17 00:00:00 2001 From: ansjindal Date: Thu, 6 Aug 2026 15:23:08 +0200 Subject: [PATCH 4/6] fix(examples): give the classifier a judge that does not fail open Signed-off-by: ansjindal --- .../values.switchyard.yaml | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/examples/kubernetes/envoy-ai-gateway-in-front/values.switchyard.yaml b/examples/kubernetes/envoy-ai-gateway-in-front/values.switchyard.yaml index 770f84ff1..223d036fc 100644 --- a/examples/kubernetes/envoy-ai-gateway-in-front/values.switchyard.yaml +++ b/examples/kubernetes/envoy-ai-gateway-in-front/values.switchyard.yaml @@ -47,6 +47,22 @@ config: id = "nvidia/nvidia/nemotron-3-ultra" llm_client = "nvidia" + # The classifier judge must return a JSON verdict. A reasoning model spends + # its output budget on reasoning_content first and the JSON is truncated, + # which Switchyard records as + # `switchyard_classifier_fail_open_total{reason="parse_error"}` and routes + # to strong_target -- so every judge failure silently spends the expensive + # tier. Measured on this deployment before suppressing reasoning: 21 of 25 + # escalations were fail-opens rather than decisions; after: zero. + # + # `extra_body` is shallow-merged into the upstream request, so this applies + # to the judge only and the serving targets keep full reasoning. Use a + # genuinely non-reasoning model here where one is available. + [targets.judge] + id = "nvidia/nvidia/nemotron-3-nano-30b-a3b" + llm_client = "nvidia" + extra_body = { chat_template_kwargs = { thinking = false } } + # Single-tier routes, useful as cost and accuracy baselines. [routes.nano] id = "switchyard/nano" @@ -64,19 +80,24 @@ config: target = "ultra" # Sends each task to `classifier_target` for a capability verdict, then - # serves it from `weak_target` or `strong_target`. Using the cheap tier as - # its own judge keeps the routing decision inexpensive. + # serves it from `weak_target` or `strong_target`. # # Raise base_threshold to send less traffic to the weak tier. Anything the - # judge cannot decide goes to strong_target. + # judge cannot decide goes to strong_target -- which is why the judge is a + # dedicated target with reasoning suppressed; see [targets.judge] above. [routes.classified] id = "switchyard/classified" type = "llm_classifier" mode = "capability" - classifier_target = "nano" + classifier_target = "judge" weak_target = "nano" strong_target = "ultra" base_threshold = 0.5 + # The judge otherwise runs on every turn, which is most of the routing + # cost. Both flags match the reference config in + # benchmark/server-configs/tb-lite-llm-classifier-opus-kimi-gemini.toml. + session_affinity = true + message_hash_fallback = true # Even split across tiers, for A/B comparison rather than cost control. [routes.general] From b64bbc28336c0f9879230e56796bf9600dbdefb4 Mon Sep 17 00:00:00 2001 From: ansjindal Date: Thu, 6 Aug 2026 18:44:01 +0200 Subject: [PATCH 5/6] feat(examples): add LiteLLM gateway topology for kubernetes Signed-off-by: ansjindal --- examples/kubernetes/README.md | 130 ++++++++++++++-- .../01-litellm.yaml | 145 ++++++++++++++++++ .../values.switchyard.yaml | 66 ++++++++ 3 files changed, 332 insertions(+), 9 deletions(-) create mode 100644 examples/kubernetes/switchyard-in-front-of-litellm/01-litellm.yaml create mode 100644 examples/kubernetes/switchyard-in-front-of-litellm/values.switchyard.yaml diff --git a/examples/kubernetes/README.md b/examples/kubernetes/README.md index bb5d0b3e3..639244a4e 100644 --- a/examples/kubernetes/README.md +++ b/examples/kubernetes/README.md @@ -3,16 +3,31 @@ SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All SPDX-License-Identifier: Apache-2.0 --> -# Switchyard with Envoy AI Gateway +# Switchyard on Kubernetes -Two ways to combine Switchyard with [Envoy AI Gateway](https://aigateway.envoyproxy.io/), -depending on which component you want to own ingress and which you want to own -provider credentials. +Three worked deployments, differing in which component owns ingress and which +owns provider credentials: -Both use the [Switchyard Helm chart](../../deploy/helm/switchyard) for the -Switchyard half and plain manifests for the Envoy half. +| directory | chain | +|---|---| +| [`envoy-ai-gateway-in-front/`](envoy-ai-gateway-in-front) | client → Envoy AI Gateway → Switchyard → provider | +| [`switchyard-in-front-of-envoy-ai-gateway/`](switchyard-in-front-of-envoy-ai-gateway) | client → Switchyard → Envoy AI Gateway → provider | +| [`switchyard-in-front-of-litellm/`](switchyard-in-front-of-litellm) | client → Switchyard → LiteLLM → provider | + +All three use the [Switchyard Helm chart](../../deploy/helm/switchyard) for the +Switchyard half and plain manifests for the gateway half. + +The first two are covered together below, since they are the same components in +opposite order. The LiteLLM variant is the Kubernetes form of +[`examples/experimental/litellm`](../experimental/litellm), where the gateway +owns model aliases rather than Gateway API routing. + +## Which Envoy topology -## Which topology +Both Envoy examples use the same components in opposite order. The choice is +about where the provider credential ends up. (For the LiteLLM variant, see +[its section](#switchyard-in-front-of-litellm) — there the gateway is always +downstream and always holds the credential.) ```mermaid flowchart LR @@ -79,7 +94,7 @@ name is otherwise hashed and changes when the Gateway is recreated. that reaches its port — there is no API-key check, no JWT validation, no mTLS. Whatever sits in front of it owns client identity. -That makes the two topologies differ in an important way: +That makes the topologies differ in an important way: - **Envoy AI Gateway in front** — solved by the Gateway. [`04-client-auth.yaml`](envoy-ai-gateway-in-front/04-client-auth.yaml) shows a @@ -94,8 +109,15 @@ That makes the two topologies differ in an important way: NetworkPolicy, or put a Gateway in front of it too, making a sandwich — Gateway (client auth) → Switchyard (routing) → Gateway (provider auth). +- **Switchyard in front of LiteLLM** — also *not* solved, and for the same + reason. LiteLLM holds the provider key but sits downstream, so it does not + see the client. `01-litellm.yaml` restricts who may reach LiteLLM; nothing + there restricts who may reach Switchyard. Set a LiteLLM master key if you + want the gateway to authenticate its callers, and front Switchyard itself if + it is reachable from outside the cluster. + Note that the provider credential is not the whole risk. Even when the key -lives safely in the Gateway, an unauthenticated caller can still spend it. +lives safely in the gateway, an unauthenticated caller can still spend it. For machine clients prefer `jwt` over `oidc`: the OIDC flow needs a browser redirect an SDK client cannot complete. @@ -320,6 +342,96 @@ Envoy matches on **provider** model ids here, so the values in [`02-provider-backend.yaml`](switchyard-in-front-of-envoy-ai-gateway/02-provider-backend.yaml) must match the `id` field of each `[targets.*]` table, not the route ids. +## Switchyard in front of LiteLLM + +The Kubernetes form of [`examples/experimental/litellm`](../experimental/litellm). +LiteLLM owns provider access, credentials and model aliases; Switchyard owns the +routing policy and addresses aliases rather than provider model ids. + +```mermaid +flowchart LR + client["Client"] + + subgraph ns["ns: switchyard"] + sy["Switchyard :4000
stage router"] + np{{"NetworkPolicy
only Switchyard may connect"}} + ll["LiteLLM :4000
aliases: fast, strong"] + cfg["ConfigMap
model_list"] + sec[("Secret
NVIDIA_API_KEY")] + end + + prov["Model provider"] + + client --> sy + sy -->|"model: fast or strong"| np + np --> ll + cfg -.-> ll + sec -.-> ll + ll -->|"resolved model
Authorization added"| prov + + classDef cred fill:#fdf0d5,stroke:#b8860b,color:#5c4400 + classDef envoy fill:#e8e6ff,stroke:#6b5fd6,color:#2d2483 + classDef cfgc fill:#f2f0ff,stroke:#9a8fe0,color:#3b3183 + classDef plain fill:#eef2f7,stroke:#8899aa,color:#22303c + classDef warn fill:#ffe8e8,stroke:#c04a4a,color:#7a1f1f + class ll,sec cred + class sy envoy + class cfg cfgc + class np warn + class client,prov plain +``` + +**Aliases are the contract between the two.** Switchyard's `[targets.*].id` +values match `model_name` in the LiteLLM `model_list`, so repointing an alias at +a different provider or model is a change to the ConfigMap alone — the +Switchyard deployment TOML never mentions a provider model id. + +| object | file | why | +|---|---|---| +| `ConfigMap` `litellm-config` | `01-litellm.yaml` | the `model_list`; aliases `fast` and `strong` | +| `Deployment` `litellm` | `01-litellm.yaml` | the gateway; probes `/health/liveliness` and `/health/readiness` | +| `Service` `litellm` | `01-litellm.yaml` | ClusterIP on 4000, what Switchyard's `base_url` points at | +| `NetworkPolicy` `litellm-ingress-restriction` | `01-litellm.yaml` | limits who may reach the credential-bearing gateway | +| `Secret` `litellm-provider-keys` | created below | resolved by `os.environ/NVIDIA_API_KEY` in the ConfigMap | + +```bash +kubectl create namespace switchyard + +# LiteLLM holds the provider credential; Switchyard gets none. +kubectl -n switchyard create secret generic litellm-provider-keys \ + --from-literal=NVIDIA_API_KEY="$NVIDIA_API_KEY" + +kubectl apply -f examples/kubernetes/switchyard-in-front-of-litellm/01-litellm.yaml +kubectl -n switchyard rollout status deploy/litellm --timeout=15m + +helm upgrade --install switchyard deploy/helm/switchyard \ + --namespace switchyard \ + -f examples/kubernetes/switchyard-in-front-of-litellm/values.switchyard.yaml +``` + +```bash +kubectl -n switchyard port-forward svc/switchyard 4000:4000 & + +curl -s localhost:4000/v1/chat/completions \ + -H 'content-type: application/json' \ + -d '{"model":"switchyard/stage","messages":[{"role":"user","content":"hello"}],"max_tokens":600}' +``` + +Two things that cost time on a first run: + +- **The image is 350MB and took 7m33s to pull** on a cold node, so the first + `rollout status` can look like a hang. The 15m timeout above is deliberate. +- **LiteLLM needs real memory.** At a 1Gi limit it was `OOMKilled` nine seconds + into startup, crash-looping with no log output at all — the manifest looks + fine and the pod simply dies. The example sets 3Gi. + +`switchyard/stage` uses the same stage router as the Compose example: an initial +or ambiguous turn falls open to `fast`, and error-recovery signals move a turn to +`strong`. Those signals come from recent tool results, so on single-turn traffic +every turn takes the picker default — see the note on `stage_router` below. +`switchyard/fast` and `switchyard/strong` are passthrough routes to each alias, +useful as smoke tests and as cost baselines. + ## Notes **`x-forwarded-host` breaks some providers.** Envoy preserves the client's diff --git a/examples/kubernetes/switchyard-in-front-of-litellm/01-litellm.yaml b/examples/kubernetes/switchyard-in-front-of-litellm/01-litellm.yaml new file mode 100644 index 000000000..c9d634538 --- /dev/null +++ b/examples/kubernetes/switchyard-in-front-of-litellm/01-litellm.yaml @@ -0,0 +1,145 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# LiteLLM as the model gateway behind Switchyard. +# +# The Kubernetes equivalent of examples/experimental/litellm/compose.yaml: +# LiteLLM owns provider access, credentials and model aliases; Switchyard owns +# the routing policy and addresses the aliases rather than provider model ids. +# +# Aliases are the contract between the two. Switchyard's `[targets.*].id` +# values must match `model_name` here, so a provider swap is a change to this +# ConfigMap alone and the Switchyard deployment TOML stays untouched. +apiVersion: v1 +kind: ConfigMap +metadata: + name: litellm-config + namespace: switchyard +data: + config.yaml: | + model_list: + - model_name: fast + litellm_params: + model: openai/nvidia/nvidia/nemotron-3-nano-30b-a3b + api_base: https://inference-api.nvidia.com/v1 + api_key: os.environ/NVIDIA_API_KEY + - model_name: strong + litellm_params: + model: openai/nvidia/nvidia/nemotron-3-ultra + api_base: https://inference-api.nvidia.com/v1 + api_key: os.environ/NVIDIA_API_KEY + + litellm_settings: + # Switchyard forwards the client's parameters as-is; dropping unsupported + # ones here keeps a provider from rejecting the whole request. + drop_params: true +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: litellm + namespace: switchyard + labels: + app.kubernetes.io/name: litellm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: litellm + template: + metadata: + labels: + app.kubernetes.io/name: litellm + annotations: + # Roll the pod when the alias list changes. + checksum/config: "replace-with-a-hash-or-let-your-tooling-manage-it" + spec: + containers: + - name: litellm + image: ghcr.io/berriai/litellm:v1.92.0 + args: ["--config", "/app/config.yaml", "--port", "4000"] + ports: + - name: http + containerPort: 4000 + env: + # Provider credential. LiteLLM resolves `os.environ/NVIDIA_API_KEY` + # from this, so the key lives here rather than in the ConfigMap. + - name: NVIDIA_API_KEY + valueFrom: + secretKeyRef: + name: litellm-provider-keys + key: NVIDIA_API_KEY + volumeMounts: + - name: config + mountPath: /app/config.yaml + subPath: config.yaml + readOnly: true + # LiteLLM separates liveliness from readiness: /health/readiness also + # reports on the configured providers. + livenessProbe: + httpGet: + path: /health/liveliness + port: http + initialDelaySeconds: 10 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /health/readiness + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + # LiteLLM's proxy imports a large dependency tree at boot and was + # OOMKilled (exit 137) about nine seconds in at a 1Gi limit, before + # writing any log line. Give it real headroom; this is a Python + # gateway, not a Rust proxy like Switchyard next door. + resources: + requests: + cpu: 200m + memory: 1Gi + limits: + cpu: "2" + memory: 3Gi + volumes: + - name: config + configMap: + name: litellm-config +--- +apiVersion: v1 +kind: Service +metadata: + name: litellm + namespace: switchyard + labels: + app.kubernetes.io/name: litellm +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: litellm + ports: + - name: http + port: 4000 + targetPort: http +--- +# LiteLLM holds the provider credential, so restrict who may reach it. Without +# this any pod in the cluster can spend the key by calling the gateway. +# +# NetworkPolicy is only enforced if the cluster's CNI implements it. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: litellm-ingress-restriction + namespace: switchyard +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: litellm + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: switchyard + ports: + - protocol: TCP + port: 4000 diff --git a/examples/kubernetes/switchyard-in-front-of-litellm/values.switchyard.yaml b/examples/kubernetes/switchyard-in-front-of-litellm/values.switchyard.yaml new file mode 100644 index 000000000..d58cbe4c7 --- /dev/null +++ b/examples/kubernetes/switchyard-in-front-of-litellm/values.switchyard.yaml @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Helm values for Switchyard routing across LiteLLM aliases. +# +# helm upgrade --install switchyard deploy/helm/switchyard \ +# --namespace switchyard --create-namespace \ +# -f examples/kubernetes/switchyard-in-front-of-litellm/values.switchyard.yaml + +image: + repository: ghcr.io/nvidia-nemo/switchyard/switchyard-server + tag: "0.2.0" + +# No provider credential here: LiteLLM holds it. `api_key_env` is omitted from +# the llm_client below, so Switchyard sends no Authorization header on the +# in-cluster hop. Set a LiteLLM master key and add it here if you want the +# gateway to authenticate its callers. +service: + type: ClusterIP + +config: + routes: | + schema_version = 1 + + [llm_clients.litellm] + format = "openai_chat" + base_url = "http://litellm.switchyard.svc.cluster.local:4000/v1" + max_retries = 2 + + # Target ids are LiteLLM *aliases*, not provider model ids. Repointing an + # alias in the LiteLLM ConfigMap swaps the underlying model without + # touching this file. + [targets.fast] + id = "fast" + llm_client = "litellm" + + [targets.strong] + id = "strong" + llm_client = "litellm" + + # Mirrors the stage router in examples/experimental/litellm: an initial or + # ambiguous turn falls open to the efficient tier, and decisive + # error-recovery signals move a turn to the capable one. + # + # This router reads tool-result and agent-progress signals from recent + # turns, so it is aimed at coding agents. On single-turn traffic there are + # no such signals and every turn takes the picker default. + [routes.stage] + id = "switchyard/stage" + type = "stage_router" + capable_target = "strong" + efficient_target = "fast" + picker = "efficient_first" + confidence_threshold = 0.5 + + # Straight passthrough to each alias, useful for smoke tests and as + # cost/accuracy baselines. + [routes.fast] + id = "switchyard/fast" + type = "passthrough" + target = "fast" + + [routes.strong] + id = "switchyard/strong" + type = "passthrough" + target = "strong" From 72b209514042dc221246c38edc8a55755f0554bd Mon Sep 17 00:00:00 2001 From: ansjindal Date: Thu, 6 Aug 2026 20:07:16 +0200 Subject: [PATCH 6/6] fix(examples): stop the judge target colliding with the weak tier Signed-off-by: ansjindal --- .../values.switchyard.yaml | 42 +++++++++++++++---- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/examples/kubernetes/envoy-ai-gateway-in-front/values.switchyard.yaml b/examples/kubernetes/envoy-ai-gateway-in-front/values.switchyard.yaml index 223d036fc..7e86daa7b 100644 --- a/examples/kubernetes/envoy-ai-gateway-in-front/values.switchyard.yaml +++ b/examples/kubernetes/envoy-ai-gateway-in-front/values.switchyard.yaml @@ -33,6 +33,21 @@ config: api_key_env = "NVIDIA_API_KEY" max_retries = 2 + # A second client onto the same endpoint, so the judge target has a + # distinct (model id, client) key. + # + # Switchyard deduplicates targets by that pair and drops one of any + # collision, warning "only one target per id is kept and the other is + # dropped". The judge and the weak tier are the same model here, so without + # this they collapse into one target and whichever definition loses is + # discarded -- silently changing what you deployed. Check the startup log + # for that warning whenever two targets share a model. + [llm_clients.nvidia_judge] + format = "openai_chat" + base_url = "https://inference-api.nvidia.com/v1" + api_key_env = "NVIDIA_API_KEY" + max_retries = 2 + # Three tiers of the same model family. The point of the classifier route # below is to spend `ultra` tokens only on the requests that need them. [targets.nano] @@ -47,20 +62,29 @@ config: id = "nvidia/nvidia/nemotron-3-ultra" llm_client = "nvidia" - # The classifier judge must return a JSON verdict. A reasoning model spends - # its output budget on reasoning_content first and the JSON is truncated, - # which Switchyard records as + # The classifier judge must return a JSON verdict. A reasoning model can + # spend its output budget on reasoning_content and leave the JSON + # truncated, which Switchyard records as # `switchyard_classifier_fail_open_total{reason="parse_error"}` and routes # to strong_target -- so every judge failure silently spends the expensive - # tier. Measured on this deployment before suppressing reasoning: 21 of 25 - # escalations were fail-opens rather than decisions; after: zero. + # tier. On this deployment that metric was non-zero throughout, and most + # escalations were fail-opens rather than decisions. + # + # Prefer a genuinely non-reasoning judge where one is available; upstream's + # reference config uses google/gemini-3.5-flash for exactly this role. # - # `extra_body` is shallow-merged into the upstream request, so this applies - # to the judge only and the serving targets keep full reasoning. Use a - # genuinely non-reasoning model here where one is available. + # `extra_body` is shallow-merged into the upstream request, so the hint + # below applies to the judge only and the serving targets keep full + # reasoning. Treat it as a hint, not a guarantee: whether it suppresses + # reasoning is model- and prompt-dependent, and it did nothing on some + # prompts here. `session_affinity` on the route is the more reliable lever, + # because it runs the judge once per session instead of once per turn. + # + # Whatever you configure, alert on classifier_fail_open_total rather than + # assuming the judge is deciding. [targets.judge] id = "nvidia/nvidia/nemotron-3-nano-30b-a3b" - llm_client = "nvidia" + llm_client = "nvidia_judge" extra_body = { chat_template_kwargs = { thinking = false } } # Single-tier routes, useful as cost and accuracy baselines.