From e2139b1662e011cb3a37bc8805410e78ac7cd69e Mon Sep 17 00:00:00 2001 From: Max Xing Date: Wed, 19 Aug 2026 16:05:20 -0700 Subject: [PATCH] feat(vanity-gateway): serve LLM Gateway routes on configured hosts The gateway could only proxy to the invocation service, because the upstream came from a single process-wide NVCF_API_ENDPOINT. Operators who run the LLM Gateway had no way to serve its OpenAI-compatible endpoints from a gateway hostname. A new v2config.llmGateway section maps hosts to the LLM Gateway named by the new LLM_GATEWAY_ENDPOINT variable. Each host serves the routes the LLM Gateway registers: POST /v1/chat/completions, /v1/responses, and /v1/embeddings. The proxy is pass-through: the request body is never read or rewritten, and function-id, function-version-id, and NVCF-POLL-SECONDS are not set. The LLM Gateway already resolves the target function from the model field the client sends, so an entry carries no function or model selection. Entries accept eol, offlineMessage, and customHeaders, matching vanity routes. An X-Priority custom header is rejected because the LLM Gateway answers 400 on header presence alone. Validation also rejects a host claimed by more than one section, since routing is keyed by host and a collision would silently drop one section's routes. When the section declares a host, /health gains a check against the LLM Gateway on /healthz, which it serves instead of /health. Signed-off-by: Max Xing --- .../vanity-gateway/README.md | 70 +++ .../vanity-gateway/gateway/BUILD.bazel | 2 + .../vanity-gateway/gateway/gateway.go | 1 + .../vanity-gateway/gateway/h2.go | 61 ++- .../vanity-gateway/gateway/health.go | 36 +- .../gateway/llm_gateway_director.go | 99 ++++ .../gateway/llm_gateway_director_test.go | 427 ++++++++++++++++++ .../vanity-gateway/gateway/tracing_attrs.go | 3 +- .../vanity-gateway/gateway/vanity_director.go | 8 +- .../gateway_config/gateway_config.go | 78 +++- .../gateway_config/gateway_config_test.go | 146 ++++++ 11 files changed, 918 insertions(+), 13 deletions(-) create mode 100644 src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director.go create mode 100644 src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director_test.go diff --git a/src/invocation-plane-services/vanity-gateway/README.md b/src/invocation-plane-services/vanity-gateway/README.md index fb45f42a1..aeba32f32 100644 --- a/src/invocation-plane-services/vanity-gateway/README.md +++ b/src/invocation-plane-services/vanity-gateway/README.md @@ -44,6 +44,7 @@ Configuration is passed through environment variables. | --- | --- | --- | --- | | `MAPPING_PATH` | Yes | None | Path to the rendered mapping config file. | | `NVCF_API_ENDPOINT` | No | Service default | Upstream invocation service endpoint. In cluster deployments, this usually points to the in-cluster invocation service. | +| `LLM_GATEWAY_ENDPOINT` | No | Empty | Upstream LLM Gateway endpoint. Required only when `v2config.llmGateway` declares a host. | | `OTEL_EXPORTER_OTLP_ENDPOINT` | No | Empty | OTLP endpoint for tracing. Empty disables OTLP export. | | `TRACING_ACCESS_TOKEN` | No | Empty | Access token for OTLP tracing. Also configurable in the secrets file under `$.tracingAccessToken`. | | `SECRETS_PATH` | No | `vault/secrets.json` | File used to read `tracingAccessToken` when `TRACING_ACCESS_TOKEN` is not set. | @@ -164,6 +165,9 @@ v2config: functionID: 00000000-0000-0000-0000-000000000004 functionVersionID: 11111111-1111-1111-1111-111111111114 usePexec: true + llmGateway: + example_llm: + host: llm.api.example.com ``` ### OpenAI Mapping Fields @@ -243,6 +247,67 @@ response completes. `customHeaders` only affects outbound proxied requests; it does not add response headers. Header names must be valid HTTP field names and cannot include reserved routing, auth, protocol, proxy, or NVCF-managed names such as `Authorization`, `Host`, `function-id`, `function-version-id`, `Content-Length`, `Connection`, or any `NVCF-*` header. +### LLM Gateway Mapping Fields + +Hosts under `v2config.llmGateway` serve the LLM Gateway's OpenAI-compatible +routes instead of invoking a function. Each host serves exactly +`POST /v1/chat/completions`, `POST /v1/responses`, and `POST /v1/embeddings`, +the routes the LLM Gateway registers, plus `GET /health` and `GET /info`. + +The gateway proxies these routes to `LLM_GATEWAY_ENDPOINT` unchanged. It does +not read or rewrite the request body, and it does not set `function-id`, +`function-version-id`, or `NVCF-POLL-SECONDS`. Clients send the same body they +would send to the LLM Gateway directly, including the `functionID/model-name` +form of `model`, and the LLM Gateway resolves the function from it. An entry +therefore has no `functionID` and no model list. + +```yaml +v2config: + llmGateway: + example_llm: + host: llm.api.example.com +``` + +`host` is a hostname this gateway serves, not the LLM Gateway's own hostname. +Requests have to reach this gateway first, so the hostname must resolve to it +and its ingress must route it here. Pointing `host` at a name that already +routes directly to the LLM Gateway registers a route table that never receives +a request. + +| Field | Required | Description | +| --- | --- | --- | +| llmGateway map key | Yes | YAML map key for a host entry. It cannot contain periods. | +| `host` | Yes | Host header served by this gateway and proxied to the LLM Gateway. Must not be used by the `openai` or `vanity` sections. | +| `customHeaders` | No | Map of static request headers to set on the upstream request. Same rules as vanity routes, and `X-Priority` is additionally rejected because the LLM Gateway answers `400 Bad Request` for any request carrying it. | +| `eol` | No | RFC3339 timestamp. Future dates add a `Deprecation` header; past dates return `410 Gone`. | +| `offlineMessage` | No | Non-empty value returns `503 Service Unavailable` with this message. | + +`LLM_GATEWAY_ENDPOINT` is required whenever this section declares a host. It is +the outbound address of the LLM Gateway, so it is a full URL with a scheme, and +it is not the same value as `host`. `host` is the inbound name clients dial. + +Startup rejects a `host` that matches the `LLM_GATEWAY_ENDPOINT` hostname. The +proxy clears the inbound `Host`, so the outbound request would come back to this +gateway, match the same entry, and loop rather than fail. + +Config validation also rejects a host claimed by more than one section, because +routing is keyed by host and a collision would silently drop one section's +routes. + +The optional fields apply to every route on the host. `offlineMessage` takes +priority over `eol`, and both are answered by the gateway without contacting the +LLM Gateway: + +```yaml +v2config: + llmGateway: + retiring_llm: + host: old.llm.api.example.com + eol: "2026-12-31T23:59:59Z" + customHeaders: + X-Provider-Feature: enabled +``` + ## Invoking OpenAI-compatible Endpoints ### Chat Completions @@ -369,6 +434,11 @@ curl -v -H 'Host: api.example.com' localhost:10081/health curl -v localhost:10083/metrics ``` +`/health` reports one check per configured upstream. The `nvcf api` check probes +`/health` on `NVCF_API_ENDPOINT`. When `v2config.llmGateway` declares a host, an +`llm api gateway` check probes `/healthz` on `LLM_GATEWAY_ENDPOINT`, since the +LLM Gateway does not serve `/health`. + `nvcf_ai_api_gateway_shadow_requests_dropped_total` counts shadow dispatches dropped before replay. The `openai_model_name` label identifies the shadow target. The `reason` label is one of `body_read_error`, `body_rewrite_error`, diff --git a/src/invocation-plane-services/vanity-gateway/gateway/BUILD.bazel b/src/invocation-plane-services/vanity-gateway/gateway/BUILD.bazel index a492b7316..aa994aa8f 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/BUILD.bazel +++ b/src/invocation-plane-services/vanity-gateway/gateway/BUILD.bazel @@ -6,6 +6,7 @@ go_library( "gateway.go", "h2.go", "health.go", + "llm_gateway_director.go", "openai_director.go", "shadow.go", "shadow_metrics.go", @@ -61,6 +62,7 @@ go_test( "gateway_test.go", "h2_test.go", "info_test.go", + "llm_gateway_director_test.go", "openai_director_test.go", "shadow_metrics_test.go", "shadow_test.go", diff --git a/src/invocation-plane-services/vanity-gateway/gateway/gateway.go b/src/invocation-plane-services/vanity-gateway/gateway/gateway.go index a9315f296..90a4ea8b5 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/gateway.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/gateway.go @@ -43,6 +43,7 @@ type Config struct { SecretsPath string `mapstructure:"SECRETS_PATH"` MappingPath string `mapstructure:"MAPPING_PATH"` NvcfApiEndpoint string `mapstructure:"NVCF_API_ENDPOINT"` + LLMGatewayEndpoint string `mapstructure:"LLM_GATEWAY_ENDPOINT"` PrivateModelNameRegexPattern string `mapstructure:"PRIVATE_MODEL_NAME_REGEX_PATTERN"` PodIP string `mapstructure:"POD_IP"` AWSRegion string `mapstructure:"AWS_REGION"` diff --git a/src/invocation-plane-services/vanity-gateway/gateway/h2.go b/src/invocation-plane-services/vanity-gateway/gateway/h2.go index 85675569f..e7eaa52f4 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/h2.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/h2.go @@ -71,7 +71,23 @@ func buildChiMux(mappings *config.GatewayConfig, serverConfig Config) (*chi.Mux, MaxIdleConnsPerHost: 64, DialContext: (&net.Dialer{Timeout: 5 * time.Second}).DialContext, }) - healthManager, err := healthManager(serverConfig.NvcfApiEndpoint, transport) + llmGatewayEndpoint := "" + var llmGatewayDirector *LLMGatewayDirector + if mappings.HasLLMGatewayRoute() { + if serverConfig.LLMGatewayEndpoint == "" { + return nil, fmt.Errorf("LLM_GATEWAY_ENDPOINT is required when v2config.llmGateway declares a host") + } + llmGatewayEndpoint = serverConfig.LLMGatewayEndpoint + llmGatewayDirector, err = NewLLMGatewayDirector(llmGatewayEndpoint, transport) + if err != nil { + return nil, err + } + if err := rejectSelfProxyingHosts(mappings, llmGatewayDirector.UpstreamHostname()); err != nil { + return nil, err + } + } + + healthManager, err := healthManager(serverConfig.NvcfApiEndpoint, llmGatewayEndpoint, transport) if err != nil { return nil, fmt.Errorf("failed to create health manager: %w", err) } @@ -109,6 +125,7 @@ func buildChiMux(mappings *config.GatewayConfig, serverConfig Config) (*chi.Mux, registerOpenAI(hostRouter, mappings, openAIDirector, healthManager, serverTelemetry) registerVanity(hostRouter, mappings, vanityDirector, healthManager, serverTelemetry) + registerLLMGateway(hostRouter, mappings, llmGatewayDirector, healthManager, serverTelemetry) r.Use(hostRouter.Handler) r.With(serverTelemetry).Get(healthPath, healthManager.HandlerFunc) @@ -152,6 +169,48 @@ func registerVanity(hostRouter *middleware.HostRouter, mappings *config.GatewayC } } +// rejectSelfProxyingHosts fails the build when a configured host resolves to the +// LLM Gateway endpoint itself. The proxy clears request.Host, so the outbound +// Host header becomes the endpoint host, and a match would loop indefinitely +// rather than fail cleanly. +func rejectSelfProxyingHosts(mappings *config.GatewayConfig, upstreamHostname string) error { + for entryKey, entry := range mappings.LLMGateway { + if hostWithoutPort(entry.Host) == upstreamHostname { + return fmt.Errorf("llmGateway.%s: host %q is the LLM Gateway endpoint; the gateway would proxy to itself", entryKey, entry.Host) + } + } + return nil +} + +// llmGatewaySupportedPaths are the OpenAI-compatible routes the LLM Gateway +// registers. Hosts in the llmGateway section serve exactly these. +var llmGatewaySupportedPaths = []string{ + "/v1/chat/completions", + "/v1/embeddings", + "/v1/responses", +} + +// domains that proxy the LLM Gateway's OpenAI-compatible routes +func registerLLMGateway(hostRouter *middleware.HostRouter, mappings *config.GatewayConfig, llmGatewayDirector *LLMGatewayDirector, healthManager *health.Health, serverTelemetry func(http.Handler) http.Handler) { + for _, entry := range mappings.LLMGateway { + target := LLMGatewayRequest{ + CustomHeaders: entry.CustomHeaders, + EOL: entry.EOL, + OfflineMessage: entry.OfflineMessage, + } + r := chi.NewRouter() + r.Use(serverTelemetry) + for _, path := range llmGatewaySupportedPaths { + r.Method(http.MethodPost, path, chimiddleware.RequestSize(maxRequestSize)(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + llmGatewayDirector.ServeProxy(target, writer, request) + }))) + } + r.Get(healthPath, healthManager.HandlerFunc) + r.Get("/info", golibversion.Handler().ServeHTTP) + hostRouter.Register(entry.Host, chimiddleware.New(r)) + } +} + // openai specific domain func registerOpenAI(hostRouter *middleware.HostRouter, mappings *config.GatewayConfig, openAIDirector *OpenAIDirector, healthManager *health.Health, serverTelemetry func(http.Handler) http.Handler) { r := chi.NewRouter() diff --git a/src/invocation-plane-services/vanity-gateway/gateway/health.go b/src/invocation-plane-services/vanity-gateway/gateway/health.go index 3ddc18dbb..0a0ab7df0 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/health.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/health.go @@ -26,16 +26,38 @@ import ( "time" ) -func healthManager(nvcfApiHost string, transport http.RoundTripper) (*health.Health, error) { +// healthManager probes the NVCF API, plus the LLM Gateway when any vanity route +// targets it. The LLM Gateway serves /healthz rather than /health. +func healthManager(nvcfApiHost string, llmGatewayEndpoint string, transport http.RoundTripper) (*health.Health, error) { client := http.Client{Timeout: 5 * time.Second, Transport: transport} - healthUrl, err := url.JoinPath(nvcfApiHost, "/health") + + nvcfCheck, err := upstreamHealthCheck(client, "nvcf api", nvcfApiHost, "/health") if err != nil { return nil, err } - return health.New(health.WithComponent(health.Component{ + options := []health.Option{health.WithChecks(nvcfCheck)} + + if llmGatewayEndpoint != "" { + llmCheck, err := upstreamHealthCheck(client, "llm api gateway", llmGatewayEndpoint, "/healthz") + if err != nil { + return nil, err + } + options = append(options, health.WithChecks(llmCheck)) + } + + options = append(options, health.WithComponent(health.Component{ Name: "vanity gateway", - }), health.WithChecks(health.Config{ - Name: "nvcf api", + })) + return health.New(options...) +} + +func upstreamHealthCheck(client http.Client, name string, endpoint string, path string) (health.Config, error) { + healthUrl, err := url.JoinPath(endpoint, path) + if err != nil { + return health.Config{}, err + } + return health.Config{ + Name: name, Timeout: 5 * time.Second, Check: func(ctx context.Context) error { request, err := http.NewRequestWithContext(ctx, http.MethodGet, healthUrl, nil) @@ -50,7 +72,7 @@ func healthManager(nvcfApiHost string, transport http.RoundTripper) (*health.Hea if resp.StatusCode == 200 { return nil } - return fmt.Errorf("invalid nvcf api health response %d", resp.StatusCode) + return fmt.Errorf("invalid %s health response %d", name, resp.StatusCode) }, - })) + }, nil } diff --git a/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director.go b/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director.go new file mode 100644 index 000000000..de531e645 --- /dev/null +++ b/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director.go @@ -0,0 +1,99 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gateway + +import ( + "fmt" + "net" + "net/http" + "net/http/httputil" + "net/url" + "time" + + config "ai-api-gateway-service/gateway_config" + + "go.opentelemetry.io/otel/trace" +) + +// LLMGatewayDirector proxies vanity routes to the LLM Gateway without altering +// the request body. The LLM Gateway resolves the target function from the model +// field the client already supplies, so the gateway forwards the request as-is +// rather than stamping function-id headers or rewriting the path. +type LLMGatewayDirector struct { + rp *httputil.ReverseProxy + host string + scheme string +} + +type LLMGatewayRequest struct { + CustomHeaders config.CustomHeaders + EOL time.Time + OfflineMessage string +} + +func NewLLMGatewayDirector(endpoint string, transport http.RoundTripper) (*LLMGatewayDirector, error) { + endpointUrl, err := url.Parse(endpoint) + if err != nil || endpointUrl.Scheme == "" || endpointUrl.Host == "" { + return nil, fmt.Errorf("invalid LLM Gateway endpoint: %s", endpoint) + } + return &LLMGatewayDirector{ + rp: newGatewayReverseProxy(transport), + host: endpointUrl.Host, + scheme: endpointUrl.Scheme, + }, nil +} + +// UpstreamHostname is the LLM Gateway host without its port, used to reject a +// configured host that would make the gateway proxy to itself. +func (d *LLMGatewayDirector) UpstreamHostname() string { + return hostWithoutPort(d.host) +} + +func hostWithoutPort(host string) string { + if hostname, _, err := net.SplitHostPort(host); err == nil { + return hostname + } + return host +} + +func (d *LLMGatewayDirector) ServeProxy(target LLMGatewayRequest, writer http.ResponseWriter, request *http.Request) error { + span := trace.SpanFromContext(request.Context()) + span.SetAttributes(traceAttrEndpointType.String(traceAttrValueEndpointLLMGateway)) + + if writeFunctionStatusError(writer, target.OfflineMessage, target.EOL, "") { + return nil + } + + request.URL.Host = d.host + request.URL.Scheme = d.scheme + request.Host = "" + applyCustomHeaders(request, target.CustomHeaders) + + if !target.EOL.IsZero() { + writer.Header().Set("Deprecation", target.EOL.Format(time.RFC3339)) + } + + var proxyErr error + rp := *d.rp + rp.ErrorHandler = func(writer http.ResponseWriter, request *http.Request, err error) { + proxyErr = err + writeProxyError(writer, request, err) + } + rp.ServeHTTP(writer, request) + return proxyErr +} diff --git a/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director_test.go b/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director_test.go new file mode 100644 index 000000000..088a1b7a7 --- /dev/null +++ b/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director_test.go @@ -0,0 +1,427 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gateway + +import ( + config "ai-api-gateway-service/gateway_config" + "bufio" + "bytes" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const llmHost = "llm.test" + +type capturedRequest struct { + path string + headers http.Header + body string +} + +func captureServer(t *testing.T, requests chan<- capturedRequest, handler http.HandlerFunc) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + require.NoError(t, err) + requests <- capturedRequest{path: r.URL.Path, headers: r.Header.Clone(), body: string(body)} + if handler != nil { + handler(w, r) + return + } + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + return server +} + +func llmGatewayMappings(entry config.LLMGatewayEntry) *config.GatewayConfig { + mappings := &config.GatewayConfig{} + mappings.LLMGateway = map[string]config.LLMGatewayEntry{"llm_example": entry} + return mappings +} + +func llmGatewayMux(t *testing.T, mappings *config.GatewayConfig, llmEndpoint string) http.Handler { + t.Helper() + mux, err := buildChiMux(mappings, Config{ + NvcfApiEndpoint: "http://nvcf.invalid", + LLMGatewayEndpoint: llmEndpoint, + PrivateModelNameRegexPattern: "^$", + }) + require.NoError(t, err) + return mux +} + +func llmGatewayRequest(t *testing.T, path string, body string) *http.Request { + t.Helper() + req := httptest.NewRequest(http.MethodPost, path, bytes.NewBufferString(body)) + req.Host = llmHost + return req +} + +func awaitRequest(t *testing.T, requests <-chan capturedRequest) capturedRequest { + t.Helper() + select { + case received := <-requests: + return received + case <-time.After(time.Second): + t.Fatal("timed out waiting for upstream request") + return capturedRequest{} + } +} + +func TestNewLLMGatewayDirectorRejectsInvalidEndpoint(t *testing.T) { + for _, endpoint := range []string{"", "llm-api-gateway:8080", "://bad", "/relative"} { + t.Run(endpoint, func(t *testing.T) { + director, err := NewLLMGatewayDirector(endpoint, http.DefaultTransport) + require.Error(t, err) + assert.Nil(t, director) + assert.ErrorContains(t, err, "invalid LLM Gateway endpoint") + }) + } +} + +func TestBuildChiMux_LLMGatewayServesSupportedRoutes(t *testing.T) { + requests := make(chan capturedRequest, 1) + backend := captureServer(t, requests, nil) + mux := llmGatewayMux(t, llmGatewayMappings(config.LLMGatewayEntry{Host: llmHost}), backend.URL) + + for _, path := range llmGatewaySupportedPaths { + t.Run(path, func(t *testing.T) { + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, llmGatewayRequest(t, path, `{"model":"func-id/meta/llama"}`)) + + require.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, path, awaitRequest(t, requests).path) + }) + } +} + +func TestBuildChiMux_LLMGatewayDoesNotServeUnsupportedRoutes(t *testing.T) { + requests := make(chan capturedRequest, 1) + backend := captureServer(t, requests, nil) + mux := llmGatewayMux(t, llmGatewayMappings(config.LLMGatewayEntry{Host: llmHost}), backend.URL) + + for _, path := range []string{"/v1/completions", "/v1/models", "/v1/images/generations"} { + t.Run(path, func(t *testing.T) { + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, llmGatewayRequest(t, path, `{}`)) + + assert.Equal(t, http.StatusNotFound, rec.Code) + assert.Empty(t, requests, "unsupported routes must not reach the LLM Gateway") + }) + } +} + +func TestBuildChiMux_LLMGatewayAndVanityCoexist(t *testing.T) { + nvcfRequests := make(chan capturedRequest, 1) + nvcfBackend := captureServer(t, nvcfRequests, nil) + llmRequests := make(chan capturedRequest, 1) + llmBackend := captureServer(t, llmRequests, nil) + + mappings := llmGatewayMappings(config.LLMGatewayEntry{Host: llmHost}) + mappings.Vanity = map[string]config.VanityEntry{ + "example": { + Host: "vanity.test", + Paths: map[string]config.PathFunctionDetails{ + "infer": {Path: "/v1/example/infer", FunctionID: "vanity-func"}, + }, + }, + } + + mux, err := buildChiMux(mappings, Config{ + NvcfApiEndpoint: nvcfBackend.URL, + LLMGatewayEndpoint: llmBackend.URL, + PrivateModelNameRegexPattern: "^$", + }) + require.NoError(t, err) + + chatRec := httptest.NewRecorder() + mux.ServeHTTP(chatRec, llmGatewayRequest(t, "/v1/chat/completions", `{"model":"func-id/meta/llama"}`)) + require.Equal(t, http.StatusOK, chatRec.Code) + + llmReceived := awaitRequest(t, llmRequests) + assert.Equal(t, "/v1/chat/completions", llmReceived.path) + assert.Empty(t, llmReceived.headers.Get("function-id")) + + inferReq := httptest.NewRequest(http.MethodPost, "/v1/example/infer", bytes.NewBufferString(`{}`)) + inferReq.Host = "vanity.test" + inferRec := httptest.NewRecorder() + mux.ServeHTTP(inferRec, inferReq) + require.Equal(t, http.StatusOK, inferRec.Code) + + nvcfReceived := awaitRequest(t, nvcfRequests) + assert.Equal(t, "/v1/example/infer", nvcfReceived.path) + assert.Equal(t, "vanity-func", nvcfReceived.headers.Get("function-id")) + assert.Empty(t, llmRequests) +} + +func TestBuildChiMux_LLMGatewayForwardsRequestUnchanged(t *testing.T) { + requests := make(chan capturedRequest, 1) + backend := captureServer(t, requests, nil) + mux := llmGatewayMux(t, llmGatewayMappings(config.LLMGatewayEntry{Host: llmHost}), backend.URL) + + body := `{"model":"func-id/meta/llama-3.3-70b","messages":[{"role":"user","content":"hi"}]}` + req := llmGatewayRequest(t, "/v1/chat/completions", body) + req.Header.Set("Authorization", "Bearer caller-token") + req.Header.Set("traceparent", "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01") + rec := httptest.NewRecorder() + + mux.ServeHTTP(rec, req) + require.Equal(t, http.StatusOK, rec.Code) + + received := awaitRequest(t, requests) + assert.Equal(t, body, received.body, "request body must reach the LLM Gateway unmodified") + assert.Equal(t, "Bearer caller-token", received.headers.Get("Authorization")) + assert.Equal(t, "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", received.headers.Get("traceparent")) + assert.Empty(t, received.headers.Get("function-id")) + assert.Empty(t, received.headers.Get("function-version-id")) + assert.Empty(t, received.headers.Get("NVCF-POLL-SECONDS")) +} + +func TestBuildChiMux_LLMGatewayAppliesCustomHeaders(t *testing.T) { + requests := make(chan capturedRequest, 1) + backend := captureServer(t, requests, nil) + mappings := llmGatewayMappings(config.LLMGatewayEntry{ + Host: llmHost, + CustomHeaders: config.CustomHeaders{"X-Provider-Feature": "enabled"}, + }) + mux := llmGatewayMux(t, mappings, backend.URL) + + req := llmGatewayRequest(t, "/v1/embeddings", `{}`) + req.Header.Set("X-Provider-Feature", "caller-value") + rec := httptest.NewRecorder() + + mux.ServeHTTP(rec, req) + require.Equal(t, http.StatusOK, rec.Code) + + assert.Equal(t, "enabled", awaitRequest(t, requests).headers.Get("X-Provider-Feature")) +} + +func TestBuildChiMux_LLMGatewayRequiresEndpoint(t *testing.T) { + _, err := buildChiMux(llmGatewayMappings(config.LLMGatewayEntry{Host: llmHost}), Config{ + NvcfApiEndpoint: "http://nvcf.invalid", + PrivateModelNameRegexPattern: "^$", + }) + require.Error(t, err) + assert.ErrorContains(t, err, "LLM_GATEWAY_ENDPOINT is required") +} + +func TestBuildChiMux_LLMGatewayEndpointNotRequiredWithoutSection(t *testing.T) { + requests := make(chan capturedRequest, 1) + backend := captureServer(t, requests, nil) + + mappings := &config.GatewayConfig{} + mappings.Vanity = map[string]config.VanityEntry{ + "example": { + Host: "vanity.test", + Paths: map[string]config.PathFunctionDetails{"infer": {Path: "/v1/example/infer", FunctionID: "vanity-func"}}, + }, + } + + mux, err := buildChiMux(mappings, Config{ + NvcfApiEndpoint: backend.URL, + PrivateModelNameRegexPattern: "^$", + }) + require.NoError(t, err) + + req := httptest.NewRequest(http.MethodPost, "/v1/example/infer", bytes.NewBufferString(`{}`)) + req.Host = "vanity.test" + rec := httptest.NewRecorder() + + mux.ServeHTTP(rec, req) + assert.Equal(t, http.StatusOK, rec.Code) +} + +func TestLLMGatewayDirectorOfflineMessageReturns503(t *testing.T) { + requests := make(chan capturedRequest, 1) + backend := captureServer(t, requests, nil) + mappings := llmGatewayMappings(config.LLMGatewayEntry{ + Host: llmHost, + OfflineMessage: "temporarily offline", + }) + mux := llmGatewayMux(t, mappings, backend.URL) + + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, llmGatewayRequest(t, "/v1/chat/completions", `{}`)) + + assert.Equal(t, http.StatusServiceUnavailable, rec.Code) + assert.Contains(t, rec.Body.String(), "temporarily offline") + assert.Empty(t, requests, "offline hosts must not reach the LLM Gateway") +} + +func TestLLMGatewayDirectorEOLHandling(t *testing.T) { + tests := []struct { + name string + eol time.Time + wantStatus int + wantDeprecated bool + }{ + {name: "future EOL adds Deprecation header", eol: time.Now().Add(24 * time.Hour), wantStatus: http.StatusOK, wantDeprecated: true}, + {name: "expired EOL returns 410", eol: time.Now().Add(-24 * time.Hour), wantStatus: http.StatusGone}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + requests := make(chan capturedRequest, 1) + backend := captureServer(t, requests, nil) + mux := llmGatewayMux(t, llmGatewayMappings(config.LLMGatewayEntry{Host: llmHost, EOL: tc.eol}), backend.URL) + + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, llmGatewayRequest(t, "/v1/chat/completions", `{}`)) + + assert.Equal(t, tc.wantStatus, rec.Code) + if tc.wantDeprecated { + assert.Equal(t, tc.eol.Format(time.RFC3339), rec.Header().Get("Deprecation")) + } else { + assert.Empty(t, rec.Header().Get("Deprecation")) + } + }) + } +} + +func TestLLMGatewayDirectorUpstreamFailureReturns502(t *testing.T) { + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + endpoint := backend.URL + backend.Close() + + mux := llmGatewayMux(t, llmGatewayMappings(config.LLMGatewayEntry{Host: llmHost}), endpoint) + + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, llmGatewayRequest(t, "/v1/chat/completions", `{}`)) + + assert.Equal(t, http.StatusBadGateway, rec.Code) + assert.Equal(t, "application/problem+json", rec.Header().Get("Content-Type")) +} + +func TestLLMGatewayDirectorStreamsResponseIncrementally(t *testing.T) { + release := make(chan struct{}) + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(http.StatusOK) + flusher, ok := w.(http.Flusher) + require.True(t, ok) + _, _ = w.Write([]byte("data: first\n\n")) + flusher.Flush() + <-release + _, _ = w.Write([]byte("data: [DONE]\n\n")) + flusher.Flush() + })) + t.Cleanup(backend.Close) + + mux := llmGatewayMux(t, llmGatewayMappings(config.LLMGatewayEntry{Host: llmHost}), backend.URL) + proxy := httptest.NewServer(mux) + t.Cleanup(proxy.Close) + + req, err := http.NewRequest(http.MethodPost, proxy.URL+"/v1/chat/completions", bytes.NewBufferString(`{"stream":true}`)) + require.NoError(t, err) + req.Host = llmHost + + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + t.Cleanup(func() { _ = resp.Body.Close() }) + require.Equal(t, http.StatusOK, resp.StatusCode) + + reader := bufio.NewReader(resp.Body) + line, err := reader.ReadString('\n') + require.NoError(t, err) + assert.Equal(t, "data: first\n", line, "first chunk must arrive before the upstream finishes") + + close(release) + rest, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Contains(t, string(rest), "data: [DONE]") +} + +// The openai section and the llmGateway section both serve +// /v1/chat/completions. They never compete, because routing selects a host +// first and each host owns a separate route table. +func TestBuildChiMux_SamePathOnOpenAIAndLLMGatewayHosts(t *testing.T) { + nvcfRequests := make(chan capturedRequest, 1) + nvcfBackend := captureServer(t, nvcfRequests, nil) + llmRequests := make(chan capturedRequest, 1) + llmBackend := captureServer(t, llmRequests, nil) + + mappings := llmGatewayMappings(config.LLMGatewayEntry{Host: llmHost}) + mappings.OpenAI.Host = "openai.test" + mappings.OpenAI.ChatCompletions = map[string]config.ModelFunctionDetails{ + "llama": {ModelName: "meta/llama-3.3-70b", FunctionID: "openai-func"}, + } + + mux, err := buildChiMux(mappings, Config{ + NvcfApiEndpoint: nvcfBackend.URL, + LLMGatewayEndpoint: llmBackend.URL, + PrivateModelNameRegexPattern: "^$", + }) + require.NoError(t, err) + + openAIReq := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewBufferString(`{"model":"meta/llama-3.3-70b"}`)) + openAIReq.Host = "openai.test" + openAIRec := httptest.NewRecorder() + mux.ServeHTTP(openAIRec, openAIReq) + require.Equal(t, http.StatusOK, openAIRec.Code) + + openAIReceived := awaitRequest(t, nvcfRequests) + assert.Equal(t, "openai-func", openAIReceived.headers.Get("function-id")) + assert.Empty(t, llmRequests, "the openai host must not reach the LLM Gateway") + + llmRec := httptest.NewRecorder() + mux.ServeHTTP(llmRec, llmGatewayRequest(t, "/v1/chat/completions", `{"model":"func-id/meta/llama-3.3-70b"}`)) + require.Equal(t, http.StatusOK, llmRec.Code) + + llmReceived := awaitRequest(t, llmRequests) + assert.Equal(t, `{"model":"func-id/meta/llama-3.3-70b"}`, llmReceived.body) + assert.Empty(t, llmReceived.headers.Get("function-id")) + assert.Empty(t, nvcfRequests, "the llmGateway host must not reach the invocation service") +} + +func TestBuildChiMux_LLMGatewayRejectsSelfProxyingHost(t *testing.T) { + tests := []struct { + name string + host string + endpoint string + wantErr bool + }{ + {name: "host matches endpoint with port", host: "llm.test", endpoint: "http://llm.test:8080", wantErr: true}, + {name: "host matches endpoint without port", host: "llm.test", endpoint: "http://llm.test", wantErr: true}, + {name: "host carries a port too", host: "llm.test:443", endpoint: "http://llm.test:8080", wantErr: true}, + {name: "distinct host and endpoint", host: "llm.test", endpoint: "http://llm-api-gateway.nvcf.svc.cluster.local:8080", wantErr: false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := buildChiMux(llmGatewayMappings(config.LLMGatewayEntry{Host: tc.host}), Config{ + NvcfApiEndpoint: "http://nvcf.invalid", + LLMGatewayEndpoint: tc.endpoint, + PrivateModelNameRegexPattern: "^$", + }) + + if !tc.wantErr { + require.NoError(t, err) + return + } + require.Error(t, err) + assert.ErrorContains(t, err, "the gateway would proxy to itself") + }) + } +} diff --git a/src/invocation-plane-services/vanity-gateway/gateway/tracing_attrs.go b/src/invocation-plane-services/vanity-gateway/gateway/tracing_attrs.go index f48e4f903..83a4a3228 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/tracing_attrs.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/tracing_attrs.go @@ -37,7 +37,8 @@ const ( traceAttrShadowTargetModel attribute.Key = "shadow.target_model" traceAttrShadowTargetModels attribute.Key = "shadow.target_models" - traceAttrValueEndpointOpenAI = "openai" + traceAttrValueEndpointOpenAI = "openai" + traceAttrValueEndpointLLMGateway = "llm_gateway" shadowDroppedReasonBodyReadError = "body_read_error" shadowDroppedReasonBodyRewriteError = "body_rewrite_error" diff --git a/src/invocation-plane-services/vanity-gateway/gateway/vanity_director.go b/src/invocation-plane-services/vanity-gateway/gateway/vanity_director.go index 56fae7661..a7c416048 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/vanity_director.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/vanity_director.go @@ -167,8 +167,8 @@ type ProblemDetails struct { Detail string `json:"detail"` } -func NewVanityDirector(nvcfApiHost string, transport http.RoundTripper) (*VanityDirector, error) { - rp := &httputil.ReverseProxy{ +func newGatewayReverseProxy(transport http.RoundTripper) *httputil.ReverseProxy { + return &httputil.ReverseProxy{ Director: func(request *http.Request) { // already directed, needed to be able to error }, @@ -178,6 +178,10 @@ func NewVanityDirector(nvcfApiHost string, transport http.RoundTripper) (*Vanity ModifyResponse: modifyTooManyRequestsResponse, ErrorHandler: writeProxyError, } +} + +func NewVanityDirector(nvcfApiHost string, transport http.RoundTripper) (*VanityDirector, error) { + rp := newGatewayReverseProxy(transport) nvcfApiUrl, err := url.Parse(nvcfApiHost) if err != nil || nvcfApiUrl.Scheme == "" || nvcfApiUrl.Host == "" { return nil, fmt.Errorf("invalid NVCF API host: %s", nvcfApiHost) diff --git a/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config.go b/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config.go index 998fb1046..759eb3bbf 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config.go +++ b/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config.go @@ -131,6 +131,16 @@ type VanityEntry struct { Paths map[string]PathFunctionDetails `json:"paths"` } +// LLMGatewayEntry serves the LLM Gateway's OpenAI-compatible routes on Host. +// The gateway proxies them verbatim, so an entry carries no function or model +// selection: the LLM Gateway resolves the function from the request model. +type LLMGatewayEntry struct { + Host string `json:"host"` + CustomHeaders CustomHeaders `json:"customHeaders,omitempty"` + EOL time.Time `json:"eol,omitempty"` + OfflineMessage string `json:"offlineMessage,omitempty"` +} + type V2Config struct { OpenAI struct { Host string `json:"host"` @@ -142,7 +152,8 @@ type V2Config struct { ImageEdits map[string]ModelFunctionDetails `json:"imageEdits"` ImageVariations map[string]ModelFunctionDetails `json:"imageVariations"` } `json:"openai"` - Vanity map[string]VanityEntry `json:"vanity"` + Vanity map[string]VanityEntry `json:"vanity"` + LLMGateway map[string]LLMGatewayEntry `json:"llmGateway"` } // sharedNotifications is a package-level channel shared across all config instances. @@ -240,7 +251,11 @@ func (c *GatewayConfig) Validate() error { } } - return c.validateVanityConfig() + if err := c.validateVanityConfig(); err != nil { + return err + } + + return c.validateLLMGatewayConfig() } func (c *GatewayConfig) openAISections() map[string]map[string]ModelFunctionDetails { @@ -348,6 +363,12 @@ var reservedCustomHeaderNames = map[string]struct{}{ "x-forwarded-proto": {}, } +// The LLM Gateway rejects any request carrying X-Priority, on header presence +// rather than value, so a configured value would fail every request. +var llmGatewayReservedCustomHeaderNames = map[string]struct{}{ + "x-priority": {}, +} + func validateCustomHeaders(location string, headers CustomHeaders) error { seenNames := make(map[string]string, len(headers)) for name := range headers { @@ -425,6 +446,59 @@ func (c *GatewayConfig) validateVanityConfig() error { return nil } +// validateLLMGatewayConfig checks the hosts that proxy to the LLM Gateway. The +// gateway registers a fixed route set on each of them, so the only things an +// entry can get wrong are the host itself and the static headers. +func (c *GatewayConfig) validateLLMGatewayConfig() error { + for entryKey, entry := range c.LLMGateway { + location := "llmGateway." + entryKey + if entry.Host == "" { + return fmt.Errorf("%s: host is required", location) + } + if err := validateCustomHeaders(location, entry.CustomHeaders); err != nil { + return err + } + for name := range entry.CustomHeaders { + if _, ok := llmGatewayReservedCustomHeaderNames[strings.ToLower(name)]; ok { + return fmt.Errorf("%s: customHeaders cannot set %q; the LLM Gateway rejects requests carrying it", location, name) + } + } + } + + return c.validateHostUniqueness() +} + +// validateHostUniqueness rejects a host claimed by more than one section. +// Registration is a plain map assignment keyed by host, so a collision would +// silently drop one section's routes. +func (c *GatewayConfig) validateHostUniqueness() error { + owners := make(map[string]string, len(c.LLMGateway)+len(c.Vanity)+1) + if c.OpenAI.Host != "" { + owners[c.OpenAI.Host] = "openai" + } + for vanityName, vanity := range c.Vanity { + if vanity.Host == "" { + continue + } + if owner, ok := owners[vanity.Host]; ok { + return fmt.Errorf("vanity.%s: host %q is already served by %s", vanityName, vanity.Host, owner) + } + owners[vanity.Host] = "vanity." + vanityName + } + for entryKey, entry := range c.LLMGateway { + if owner, ok := owners[entry.Host]; ok { + return fmt.Errorf("llmGateway.%s: host %q is already served by %s", entryKey, entry.Host, owner) + } + owners[entry.Host] = "llmGateway." + entryKey + } + return nil +} + +// HasLLMGatewayRoute reports whether any host proxies to the LLM Gateway. +func (c *GatewayConfig) HasLLMGatewayRoute() bool { + return len(c.LLMGateway) > 0 +} + func SetupConfigWithConfigPath(path string) (rc.ReloadableConfig[GatewayConfig], error) { config, err := rc.SetupConfig[GatewayConfig](path, rc.WithValidateFunc(func(c *GatewayConfig) error { diff --git a/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config_test.go b/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config_test.go index 4a59f6bc0..35a1a43fa 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config_test.go +++ b/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config_test.go @@ -867,3 +867,149 @@ func TestGatewayConfigValidateRejectsVanityShadowSamplingMethod(t *testing.T) { require.Error(t, err) assert.ErrorContains(t, err, "shadow config is unsupported for vanity routes") } + +func llmGatewayConfig(entry LLMGatewayEntry) *GatewayConfig { + cfg := &GatewayConfig{} + cfg.LLMGateway = map[string]LLMGatewayEntry{"llm_example": entry} + return cfg +} + +func TestGatewayConfigValidateAcceptsLLMGatewayEntry(t *testing.T) { + cfg := llmGatewayConfig(LLMGatewayEntry{ + Host: "llm.example.com", + CustomHeaders: CustomHeaders{"X-Provider-Feature": "enabled"}, + OfflineMessage: "", + }) + + require.NoError(t, cfg.Validate()) + assert.True(t, cfg.HasLLMGatewayRoute()) +} + +func TestGatewayConfigHasLLMGatewayRouteFalseWhenSectionEmpty(t *testing.T) { + cfg := &GatewayConfig{} + cfg.Vanity = map[string]VanityEntry{ + "example": { + Host: "ai.example.com", + Paths: map[string]PathFunctionDetails{"sample": {Path: "/v1/example/infer", FunctionID: "func-id"}}, + }, + } + + require.NoError(t, cfg.Validate()) + assert.False(t, cfg.HasLLMGatewayRoute()) +} + +func TestGatewayConfigValidateRejectsLLMGatewayEntryWithoutHost(t *testing.T) { + err := llmGatewayConfig(LLMGatewayEntry{}).Validate() + require.Error(t, err) + assert.ErrorContains(t, err, "llmGateway.llm_example: host is required") +} + +func TestGatewayConfigValidateRejectsPriorityHeaderOnLLMGatewayEntry(t *testing.T) { + for _, name := range []string{"X-Priority", "x-priority"} { + t.Run(name, func(t *testing.T) { + cfg := llmGatewayConfig(LLMGatewayEntry{ + Host: "llm.example.com", + CustomHeaders: CustomHeaders{name: "high"}, + }) + + err := cfg.Validate() + require.Error(t, err) + assert.ErrorContains(t, err, "the LLM Gateway rejects requests carrying it") + }) + } +} + +func TestGatewayConfigValidateRejectsReservedHeaderOnLLMGatewayEntry(t *testing.T) { + cfg := llmGatewayConfig(LLMGatewayEntry{ + Host: "llm.example.com", + CustomHeaders: CustomHeaders{"Authorization": "Bearer nope"}, + }) + + err := cfg.Validate() + require.Error(t, err) + assert.ErrorContains(t, err, "cannot set reserved header") +} + +func TestGatewayConfigValidateRejectsDuplicateHostsAcrossSections(t *testing.T) { + tests := []struct { + name string + mutate func(cfg *GatewayConfig) + wantMessage string + }{ + { + name: "llmGateway collides with openai", + mutate: func(cfg *GatewayConfig) { cfg.OpenAI.Host = "shared.example.com" }, + wantMessage: `host "shared.example.com" is already served by openai`, + }, + { + name: "llmGateway collides with vanity", + mutate: func(cfg *GatewayConfig) { + cfg.Vanity = map[string]VanityEntry{ + "example": { + Host: "shared.example.com", + Paths: map[string]PathFunctionDetails{"sample": {Path: "/v1/example/infer", FunctionID: "func-id"}}, + }, + } + }, + wantMessage: `host "shared.example.com" is already served by vanity.example`, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cfg := llmGatewayConfig(LLMGatewayEntry{Host: "shared.example.com"}) + tc.mutate(cfg) + + err := cfg.Validate() + require.Error(t, err) + assert.ErrorContains(t, err, tc.wantMessage) + }) + } +} + +func TestGatewayConfigValidateRejectsDuplicateHostAcrossTwoLLMGatewayEntries(t *testing.T) { + cfg := &GatewayConfig{} + cfg.LLMGateway = map[string]LLMGatewayEntry{ + "first": {Host: "llm.example.com"}, + "second": {Host: "llm.example.com"}, + } + + err := cfg.Validate() + require.Error(t, err) + assert.ErrorContains(t, err, `host "llm.example.com" is already served by llmGateway.`) +} + +func TestGatewayConfigValidateAllowsEmptyOpenAIHostAlongsideLLMGateway(t *testing.T) { + cfg := llmGatewayConfig(LLMGatewayEntry{Host: "llm.example.com"}) + + require.NoError(t, cfg.Validate()) +} + +func TestGatewayConfigLoadAcceptsLLMGatewaySection(t *testing.T) { + configPath := filepath.Join(t.TempDir(), "config.yaml") + err := os.WriteFile(configPath, []byte(` +v2config: + vanity: + example: + host: ai.example.com + paths: + infer: + path: /v1/example/infer + functionID: func-id + llmGateway: + llm_example: + host: llm.example.com + offlineMessage: "" + customHeaders: + X-Provider-Feature: enabled +`), 0600) + require.NoError(t, err) + + loaded, err := SetupConfigWithConfigPath(configPath) + require.NoError(t, err) + + cfg := loaded.Get() + require.True(t, cfg.HasLLMGatewayRoute()) + assert.Equal(t, "llm.example.com", cfg.LLMGateway["llm_example"].Host) + assert.Equal(t, "enabled", cfg.LLMGateway["llm_example"].CustomHeaders["X-Provider-Feature"]) +}