diff --git a/deploy/helm/vanity-gateway/README.md b/deploy/helm/vanity-gateway/README.md index a6d406c06..d575003be 100644 --- a/deploy/helm/vanity-gateway/README.md +++ b/deploy/helm/vanity-gateway/README.md @@ -81,9 +81,12 @@ Important settings to review before deployment: - `vanityGateway.replicaCount`, resource requests, and limits for your environment - `vanityGateway.config.nvcfApiEndpoint` for the invocation endpoint +- `vanityGateway.config.llmGatewayEndpoint` for the LLM Gateway endpoint, used + only by hosts declared under `mappingConfig.v2config.llmGateway` - `vanityGateway.config.otelExporterOtlpEndpoint` for trace export, empty by default -- `vanityGateway.mappingConfig.v2config` for the OpenAI and vanity route tables +- `vanityGateway.mappingConfig.v2config` for the OpenAI, vanity, and LLM Gateway + route tables - `vanityGateway.serviceMonitor.enabled` for Prometheus Operator scraping ### Ports @@ -103,7 +106,7 @@ or the pod is killed mid-drain. ## Route mapping -`vanityGateway.mappingConfig.v2config` has two sections: +`vanityGateway.mappingConfig.v2config` has three sections: - `openai`: per-endpoint model routes, keyed by endpoint (`chatCompletions`, `completions`, `embeddings`, `responses`, and the image endpoints). Each route @@ -112,9 +115,14 @@ or the pod is killed mid-drain. `shadowCancelOnClientDisconnect`. - `vanity`: host-based routes, each requiring a `host` and a `paths` map. Each path requires `path` and `functionID`. - -Both sections are empty by default. `vanityGateway.config.shadowMaxConcurrent` -bounds concurrent shadow requests across all routes. +- `llmGateway`: hosts that proxy the LLM Gateway's OpenAI-compatible routes, + each requiring only a `host`. Requests are forwarded unchanged, so an entry + carries no function or model selection. Declaring one makes + `vanityGateway.config.llmGatewayEndpoint` required at startup. + +All three sections are empty by default. +`vanityGateway.config.shadowMaxConcurrent` bounds concurrent shadow requests +across all routes. ## Notes diff --git a/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/templates/configmap.yaml b/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/templates/configmap.yaml index e14195a72..7ee60f3b8 100644 --- a/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/templates/configmap.yaml +++ b/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/templates/configmap.yaml @@ -22,6 +22,7 @@ metadata: data: MAPPING_PATH: {{ .Values.vanityGateway.config.mappingPath | quote }} NVCF_API_ENDPOINT: {{ .Values.vanityGateway.config.nvcfApiEndpoint | quote }} + LLM_GATEWAY_ENDPOINT: {{ .Values.vanityGateway.config.llmGatewayEndpoint | quote }} OTEL_EXPORTER_OTLP_ENDPOINT: {{ .Values.vanityGateway.config.otelExporterOtlpEndpoint | quote }} SECRETS_PATH: {{ .Values.vanityGateway.config.secretsPath | quote }} PRIVATE_MODEL_NAME_REGEX_PATTERN: {{ .Values.vanityGateway.config.privateModelNameRegexPattern | quote }} diff --git a/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.schema.json b/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.schema.json index 90340b28c..edfdeb70b 100644 --- a/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.schema.json +++ b/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.schema.json @@ -110,6 +110,9 @@ "nvcfApiEndpoint": { "$ref": "#/definitions/nonEmptyString" }, + "llmGatewayEndpoint": { + "type": "string" + }, "otelExporterOtlpEndpoint": { "type": "string" }, @@ -140,12 +143,11 @@ }, "vanity": { "$ref": "#/definitions/vanityRoutes" + }, + "llmGateway": { + "$ref": "#/definitions/llmGatewayRoutes" } }, - "required": [ - "openai", - "vanity" - ], "additionalProperties": true } }, @@ -341,6 +343,36 @@ "functionID" ], "additionalProperties": true + }, + "llmGatewayRoutes": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/llmGatewayRoute" + } + }, + "llmGatewayRoute": { + "type": "object", + "properties": { + "host": { + "$ref": "#/definitions/nonEmptyString" + }, + "customHeaders": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "eol": { + "type": "string" + }, + "offlineMessage": { + "type": "string" + } + }, + "required": [ + "host" + ], + "additionalProperties": true } }, "additionalProperties": true diff --git a/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.yaml b/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.yaml index d9af9863a..4ba9d0d78 100644 --- a/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.yaml +++ b/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.yaml @@ -48,6 +48,8 @@ vanityGateway: config: mappingPath: /etc/vanity-gateway/config/config.yaml nvcfApiEndpoint: http://invocation.nvcf.svc.cluster.local:8080 + # Used only by hosts declared under mappingConfig.v2config.llmGateway. + llmGatewayEndpoint: http://llm-api-gateway.nvcf.svc.cluster.local:8080 otelExporterOtlpEndpoint: "" secretsPath: "" privateModelNameRegexPattern: ""