From a404d9afc1a89a10dc092243996e0ba5b2ff5de4 Mon Sep 17 00:00:00 2001 From: Max Xing Date: Wed, 19 Aug 2026 17:51:10 -0700 Subject: [PATCH] feat(vanity-gateway): add the LLM Gateway endpoint chart knob The chart had no way to set LLM_GATEWAY_ENDPOINT, which the gateway needs whenever a host is declared under mappingConfig.v2config.llmGateway. The deployment sources its environment from a generated ConfigMap and has no passthrough for extra variables, so the value could not be supplied at all. Adds vanityGateway.config.llmGatewayEndpoint, defaulting to the in-cluster LLM Gateway service, and emits it into the env ConfigMap. The value is inert until a host declares it, so enabling the chart knob alone changes nothing. values.schema.json gains the key, which is required because the config block is additionalProperties false, plus an llmGatewayRoutes definition so the new mapping section is validated rather than passing only because v2config allows additional properties. The openai and vanity sections are no longer required under v2config: a deployment that declares only llmGateway hosts is valid, and the stack passes mappingConfig through wholesale, so requiring them forced empty placeholder sections. Signed-off-by: Max Xing --- deploy/helm/vanity-gateway/README.md | 18 ++++++--- .../templates/configmap.yaml | 1 + .../values.schema.json | 40 +++++++++++++++++-- .../helm-nvcf-vanity-gateway/values.yaml | 2 + 4 files changed, 52 insertions(+), 9 deletions(-) diff --git a/deploy/helm/vanity-gateway/README.md b/deploy/helm/vanity-gateway/README.md index a6d406c06..d575003be 100644 --- a/deploy/helm/vanity-gateway/README.md +++ b/deploy/helm/vanity-gateway/README.md @@ -81,9 +81,12 @@ Important settings to review before deployment: - `vanityGateway.replicaCount`, resource requests, and limits for your environment - `vanityGateway.config.nvcfApiEndpoint` for the invocation endpoint +- `vanityGateway.config.llmGatewayEndpoint` for the LLM Gateway endpoint, used + only by hosts declared under `mappingConfig.v2config.llmGateway` - `vanityGateway.config.otelExporterOtlpEndpoint` for trace export, empty by default -- `vanityGateway.mappingConfig.v2config` for the OpenAI and vanity route tables +- `vanityGateway.mappingConfig.v2config` for the OpenAI, vanity, and LLM Gateway + route tables - `vanityGateway.serviceMonitor.enabled` for Prometheus Operator scraping ### Ports @@ -103,7 +106,7 @@ or the pod is killed mid-drain. ## Route mapping -`vanityGateway.mappingConfig.v2config` has two sections: +`vanityGateway.mappingConfig.v2config` has three sections: - `openai`: per-endpoint model routes, keyed by endpoint (`chatCompletions`, `completions`, `embeddings`, `responses`, and the image endpoints). Each route @@ -112,9 +115,14 @@ or the pod is killed mid-drain. `shadowCancelOnClientDisconnect`. - `vanity`: host-based routes, each requiring a `host` and a `paths` map. Each path requires `path` and `functionID`. - -Both sections are empty by default. `vanityGateway.config.shadowMaxConcurrent` -bounds concurrent shadow requests across all routes. +- `llmGateway`: hosts that proxy the LLM Gateway's OpenAI-compatible routes, + each requiring only a `host`. Requests are forwarded unchanged, so an entry + carries no function or model selection. Declaring one makes + `vanityGateway.config.llmGatewayEndpoint` required at startup. + +All three sections are empty by default. +`vanityGateway.config.shadowMaxConcurrent` bounds concurrent shadow requests +across all routes. ## Notes diff --git a/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/templates/configmap.yaml b/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/templates/configmap.yaml index e14195a72..7ee60f3b8 100644 --- a/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/templates/configmap.yaml +++ b/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/templates/configmap.yaml @@ -22,6 +22,7 @@ metadata: data: MAPPING_PATH: {{ .Values.vanityGateway.config.mappingPath | quote }} NVCF_API_ENDPOINT: {{ .Values.vanityGateway.config.nvcfApiEndpoint | quote }} + LLM_GATEWAY_ENDPOINT: {{ .Values.vanityGateway.config.llmGatewayEndpoint | quote }} OTEL_EXPORTER_OTLP_ENDPOINT: {{ .Values.vanityGateway.config.otelExporterOtlpEndpoint | quote }} SECRETS_PATH: {{ .Values.vanityGateway.config.secretsPath | quote }} PRIVATE_MODEL_NAME_REGEX_PATTERN: {{ .Values.vanityGateway.config.privateModelNameRegexPattern | quote }} diff --git a/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.schema.json b/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.schema.json index 90340b28c..edfdeb70b 100644 --- a/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.schema.json +++ b/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.schema.json @@ -110,6 +110,9 @@ "nvcfApiEndpoint": { "$ref": "#/definitions/nonEmptyString" }, + "llmGatewayEndpoint": { + "type": "string" + }, "otelExporterOtlpEndpoint": { "type": "string" }, @@ -140,12 +143,11 @@ }, "vanity": { "$ref": "#/definitions/vanityRoutes" + }, + "llmGateway": { + "$ref": "#/definitions/llmGatewayRoutes" } }, - "required": [ - "openai", - "vanity" - ], "additionalProperties": true } }, @@ -341,6 +343,36 @@ "functionID" ], "additionalProperties": true + }, + "llmGatewayRoutes": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/llmGatewayRoute" + } + }, + "llmGatewayRoute": { + "type": "object", + "properties": { + "host": { + "$ref": "#/definitions/nonEmptyString" + }, + "customHeaders": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "eol": { + "type": "string" + }, + "offlineMessage": { + "type": "string" + } + }, + "required": [ + "host" + ], + "additionalProperties": true } }, "additionalProperties": true diff --git a/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.yaml b/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.yaml index d9af9863a..4ba9d0d78 100644 --- a/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.yaml +++ b/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.yaml @@ -48,6 +48,8 @@ vanityGateway: config: mappingPath: /etc/vanity-gateway/config/config.yaml nvcfApiEndpoint: http://invocation.nvcf.svc.cluster.local:8080 + # Used only by hosts declared under mappingConfig.v2config.llmGateway. + llmGatewayEndpoint: http://llm-api-gateway.nvcf.svc.cluster.local:8080 otelExporterOtlpEndpoint: "" secretsPath: "" privateModelNameRegexPattern: ""