diff --git a/src/invocation-plane-services/vanity-gateway/README.md b/src/invocation-plane-services/vanity-gateway/README.md index fb45f42a1..aeba32f32 100644 --- a/src/invocation-plane-services/vanity-gateway/README.md +++ b/src/invocation-plane-services/vanity-gateway/README.md @@ -44,6 +44,7 @@ Configuration is passed through environment variables. | --- | --- | --- | --- | | `MAPPING_PATH` | Yes | None | Path to the rendered mapping config file. | | `NVCF_API_ENDPOINT` | No | Service default | Upstream invocation service endpoint. In cluster deployments, this usually points to the in-cluster invocation service. | +| `LLM_GATEWAY_ENDPOINT` | No | Empty | Upstream LLM Gateway endpoint. Required only when `v2config.llmGateway` declares a host. | | `OTEL_EXPORTER_OTLP_ENDPOINT` | No | Empty | OTLP endpoint for tracing. Empty disables OTLP export. | | `TRACING_ACCESS_TOKEN` | No | Empty | Access token for OTLP tracing. Also configurable in the secrets file under `$.tracingAccessToken`. | | `SECRETS_PATH` | No | `vault/secrets.json` | File used to read `tracingAccessToken` when `TRACING_ACCESS_TOKEN` is not set. | @@ -164,6 +165,9 @@ v2config: functionID: 00000000-0000-0000-0000-000000000004 functionVersionID: 11111111-1111-1111-1111-111111111114 usePexec: true + llmGateway: + example_llm: + host: llm.api.example.com ``` ### OpenAI Mapping Fields @@ -243,6 +247,67 @@ response completes. `customHeaders` only affects outbound proxied requests; it does not add response headers. Header names must be valid HTTP field names and cannot include reserved routing, auth, protocol, proxy, or NVCF-managed names such as `Authorization`, `Host`, `function-id`, `function-version-id`, `Content-Length`, `Connection`, or any `NVCF-*` header. +### LLM Gateway Mapping Fields + +Hosts under `v2config.llmGateway` serve the LLM Gateway's OpenAI-compatible +routes instead of invoking a function. Each host serves exactly +`POST /v1/chat/completions`, `POST /v1/responses`, and `POST /v1/embeddings`, +the routes the LLM Gateway registers, plus `GET /health` and `GET /info`. + +The gateway proxies these routes to `LLM_GATEWAY_ENDPOINT` unchanged. It does +not read or rewrite the request body, and it does not set `function-id`, +`function-version-id`, or `NVCF-POLL-SECONDS`. Clients send the same body they +would send to the LLM Gateway directly, including the `functionID/model-name` +form of `model`, and the LLM Gateway resolves the function from it. An entry +therefore has no `functionID` and no model list. + +```yaml +v2config: + llmGateway: + example_llm: + host: llm.api.example.com +``` + +`host` is a hostname this gateway serves, not the LLM Gateway's own hostname. +Requests have to reach this gateway first, so the hostname must resolve to it +and its ingress must route it here. Pointing `host` at a name that already +routes directly to the LLM Gateway registers a route table that never receives +a request. + +| Field | Required | Description | +| --- | --- | --- | +| llmGateway map key | Yes | YAML map key for a host entry. It cannot contain periods. | +| `host` | Yes | Host header served by this gateway and proxied to the LLM Gateway. Must not be used by the `openai` or `vanity` sections. | +| `customHeaders` | No | Map of static request headers to set on the upstream request. Same rules as vanity routes, and `X-Priority` is additionally rejected because the LLM Gateway answers `400 Bad Request` for any request carrying it. | +| `eol` | No | RFC3339 timestamp. Future dates add a `Deprecation` header; past dates return `410 Gone`. | +| `offlineMessage` | No | Non-empty value returns `503 Service Unavailable` with this message. | + +`LLM_GATEWAY_ENDPOINT` is required whenever this section declares a host. It is +the outbound address of the LLM Gateway, so it is a full URL with a scheme, and +it is not the same value as `host`. `host` is the inbound name clients dial. + +Startup rejects a `host` that matches the `LLM_GATEWAY_ENDPOINT` hostname. The +proxy clears the inbound `Host`, so the outbound request would come back to this +gateway, match the same entry, and loop rather than fail. + +Config validation also rejects a host claimed by more than one section, because +routing is keyed by host and a collision would silently drop one section's +routes. + +The optional fields apply to every route on the host. `offlineMessage` takes +priority over `eol`, and both are answered by the gateway without contacting the +LLM Gateway: + +```yaml +v2config: + llmGateway: + retiring_llm: + host: old.llm.api.example.com + eol: "2026-12-31T23:59:59Z" + customHeaders: + X-Provider-Feature: enabled +``` + ## Invoking OpenAI-compatible Endpoints ### Chat Completions @@ -369,6 +434,11 @@ curl -v -H 'Host: api.example.com' localhost:10081/health curl -v localhost:10083/metrics ``` +`/health` reports one check per configured upstream. The `nvcf api` check probes +`/health` on `NVCF_API_ENDPOINT`. When `v2config.llmGateway` declares a host, an +`llm api gateway` check probes `/healthz` on `LLM_GATEWAY_ENDPOINT`, since the +LLM Gateway does not serve `/health`. + `nvcf_ai_api_gateway_shadow_requests_dropped_total` counts shadow dispatches dropped before replay. The `openai_model_name` label identifies the shadow target. The `reason` label is one of `body_read_error`, `body_rewrite_error`, diff --git a/src/invocation-plane-services/vanity-gateway/gateway/BUILD.bazel b/src/invocation-plane-services/vanity-gateway/gateway/BUILD.bazel index a492b7316..aa994aa8f 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/BUILD.bazel +++ b/src/invocation-plane-services/vanity-gateway/gateway/BUILD.bazel @@ -6,6 +6,7 @@ go_library( "gateway.go", "h2.go", "health.go", + "llm_gateway_director.go", "openai_director.go", "shadow.go", "shadow_metrics.go", @@ -61,6 +62,7 @@ go_test( "gateway_test.go", "h2_test.go", "info_test.go", + "llm_gateway_director_test.go", "openai_director_test.go", "shadow_metrics_test.go", "shadow_test.go", diff --git a/src/invocation-plane-services/vanity-gateway/gateway/gateway.go b/src/invocation-plane-services/vanity-gateway/gateway/gateway.go index a9315f296..90a4ea8b5 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/gateway.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/gateway.go @@ -43,6 +43,7 @@ type Config struct { SecretsPath string `mapstructure:"SECRETS_PATH"` MappingPath string `mapstructure:"MAPPING_PATH"` NvcfApiEndpoint string `mapstructure:"NVCF_API_ENDPOINT"` + LLMGatewayEndpoint string `mapstructure:"LLM_GATEWAY_ENDPOINT"` PrivateModelNameRegexPattern string `mapstructure:"PRIVATE_MODEL_NAME_REGEX_PATTERN"` PodIP string `mapstructure:"POD_IP"` AWSRegion string `mapstructure:"AWS_REGION"` diff --git a/src/invocation-plane-services/vanity-gateway/gateway/h2.go b/src/invocation-plane-services/vanity-gateway/gateway/h2.go index 85675569f..e7eaa52f4 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/h2.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/h2.go @@ -71,7 +71,23 @@ func buildChiMux(mappings *config.GatewayConfig, serverConfig Config) (*chi.Mux, MaxIdleConnsPerHost: 64, DialContext: (&net.Dialer{Timeout: 5 * time.Second}).DialContext, }) - healthManager, err := healthManager(serverConfig.NvcfApiEndpoint, transport) + llmGatewayEndpoint := "" + var llmGatewayDirector *LLMGatewayDirector + if mappings.HasLLMGatewayRoute() { + if serverConfig.LLMGatewayEndpoint == "" { + return nil, fmt.Errorf("LLM_GATEWAY_ENDPOINT is required when v2config.llmGateway declares a host") + } + llmGatewayEndpoint = serverConfig.LLMGatewayEndpoint + llmGatewayDirector, err = NewLLMGatewayDirector(llmGatewayEndpoint, transport) + if err != nil { + return nil, err + } + if err := rejectSelfProxyingHosts(mappings, llmGatewayDirector.UpstreamHostname()); err != nil { + return nil, err + } + } + + healthManager, err := healthManager(serverConfig.NvcfApiEndpoint, llmGatewayEndpoint, transport) if err != nil { return nil, fmt.Errorf("failed to create health manager: %w", err) } @@ -109,6 +125,7 @@ func buildChiMux(mappings *config.GatewayConfig, serverConfig Config) (*chi.Mux, registerOpenAI(hostRouter, mappings, openAIDirector, healthManager, serverTelemetry) registerVanity(hostRouter, mappings, vanityDirector, healthManager, serverTelemetry) + registerLLMGateway(hostRouter, mappings, llmGatewayDirector, healthManager, serverTelemetry) r.Use(hostRouter.Handler) r.With(serverTelemetry).Get(healthPath, healthManager.HandlerFunc) @@ -152,6 +169,48 @@ func registerVanity(hostRouter *middleware.HostRouter, mappings *config.GatewayC } } +// rejectSelfProxyingHosts fails the build when a configured host resolves to the +// LLM Gateway endpoint itself. The proxy clears request.Host, so the outbound +// Host header becomes the endpoint host, and a match would loop indefinitely +// rather than fail cleanly. +func rejectSelfProxyingHosts(mappings *config.GatewayConfig, upstreamHostname string) error { + for entryKey, entry := range mappings.LLMGateway { + if hostWithoutPort(entry.Host) == upstreamHostname { + return fmt.Errorf("llmGateway.%s: host %q is the LLM Gateway endpoint; the gateway would proxy to itself", entryKey, entry.Host) + } + } + return nil +} + +// llmGatewaySupportedPaths are the OpenAI-compatible routes the LLM Gateway +// registers. Hosts in the llmGateway section serve exactly these. +var llmGatewaySupportedPaths = []string{ + "/v1/chat/completions", + "/v1/embeddings", + "/v1/responses", +} + +// domains that proxy the LLM Gateway's OpenAI-compatible routes +func registerLLMGateway(hostRouter *middleware.HostRouter, mappings *config.GatewayConfig, llmGatewayDirector *LLMGatewayDirector, healthManager *health.Health, serverTelemetry func(http.Handler) http.Handler) { + for _, entry := range mappings.LLMGateway { + target := LLMGatewayRequest{ + CustomHeaders: entry.CustomHeaders, + EOL: entry.EOL, + OfflineMessage: entry.OfflineMessage, + } + r := chi.NewRouter() + r.Use(serverTelemetry) + for _, path := range llmGatewaySupportedPaths { + r.Method(http.MethodPost, path, chimiddleware.RequestSize(maxRequestSize)(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + llmGatewayDirector.ServeProxy(target, writer, request) + }))) + } + r.Get(healthPath, healthManager.HandlerFunc) + r.Get("/info", golibversion.Handler().ServeHTTP) + hostRouter.Register(entry.Host, chimiddleware.New(r)) + } +} + // openai specific domain func registerOpenAI(hostRouter *middleware.HostRouter, mappings *config.GatewayConfig, openAIDirector *OpenAIDirector, healthManager *health.Health, serverTelemetry func(http.Handler) http.Handler) { r := chi.NewRouter() diff --git a/src/invocation-plane-services/vanity-gateway/gateway/health.go b/src/invocation-plane-services/vanity-gateway/gateway/health.go index 3ddc18dbb..0a0ab7df0 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/health.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/health.go @@ -26,16 +26,38 @@ import ( "time" ) -func healthManager(nvcfApiHost string, transport http.RoundTripper) (*health.Health, error) { +// healthManager probes the NVCF API, plus the LLM Gateway when any vanity route +// targets it. The LLM Gateway serves /healthz rather than /health. +func healthManager(nvcfApiHost string, llmGatewayEndpoint string, transport http.RoundTripper) (*health.Health, error) { client := http.Client{Timeout: 5 * time.Second, Transport: transport} - healthUrl, err := url.JoinPath(nvcfApiHost, "/health") + + nvcfCheck, err := upstreamHealthCheck(client, "nvcf api", nvcfApiHost, "/health") if err != nil { return nil, err } - return health.New(health.WithComponent(health.Component{ + options := []health.Option{health.WithChecks(nvcfCheck)} + + if llmGatewayEndpoint != "" { + llmCheck, err := upstreamHealthCheck(client, "llm api gateway", llmGatewayEndpoint, "/healthz") + if err != nil { + return nil, err + } + options = append(options, health.WithChecks(llmCheck)) + } + + options = append(options, health.WithComponent(health.Component{ Name: "vanity gateway", - }), health.WithChecks(health.Config{ - Name: "nvcf api", + })) + return health.New(options...) +} + +func upstreamHealthCheck(client http.Client, name string, endpoint string, path string) (health.Config, error) { + healthUrl, err := url.JoinPath(endpoint, path) + if err != nil { + return health.Config{}, err + } + return health.Config{ + Name: name, Timeout: 5 * time.Second, Check: func(ctx context.Context) error { request, err := http.NewRequestWithContext(ctx, http.MethodGet, healthUrl, nil) @@ -50,7 +72,7 @@ func healthManager(nvcfApiHost string, transport http.RoundTripper) (*health.Hea if resp.StatusCode == 200 { return nil } - return fmt.Errorf("invalid nvcf api health response %d", resp.StatusCode) + return fmt.Errorf("invalid %s health response %d", name, resp.StatusCode) }, - })) + }, nil } diff --git a/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director.go b/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director.go new file mode 100644 index 000000000..de531e645 --- /dev/null +++ b/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director.go @@ -0,0 +1,99 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gateway + +import ( + "fmt" + "net" + "net/http" + "net/http/httputil" + "net/url" + "time" + + config "ai-api-gateway-service/gateway_config" + + "go.opentelemetry.io/otel/trace" +) + +// LLMGatewayDirector proxies vanity routes to the LLM Gateway without altering +// the request body. The LLM Gateway resolves the target function from the model +// field the client already supplies, so the gateway forwards the request as-is +// rather than stamping function-id headers or rewriting the path. +type LLMGatewayDirector struct { + rp *httputil.ReverseProxy + host string + scheme string +} + +type LLMGatewayRequest struct { + CustomHeaders config.CustomHeaders + EOL time.Time + OfflineMessage string +} + +func NewLLMGatewayDirector(endpoint string, transport http.RoundTripper) (*LLMGatewayDirector, error) { + endpointUrl, err := url.Parse(endpoint) + if err != nil || endpointUrl.Scheme == "" || endpointUrl.Host == "" { + return nil, fmt.Errorf("invalid LLM Gateway endpoint: %s", endpoint) + } + return &LLMGatewayDirector{ + rp: newGatewayReverseProxy(transport), + host: endpointUrl.Host, + scheme: endpointUrl.Scheme, + }, nil +} + +// UpstreamHostname is the LLM Gateway host without its port, used to reject a +// configured host that would make the gateway proxy to itself. +func (d *LLMGatewayDirector) UpstreamHostname() string { + return hostWithoutPort(d.host) +} + +func hostWithoutPort(host string) string { + if hostname, _, err := net.SplitHostPort(host); err == nil { + return hostname + } + return host +} + +func (d *LLMGatewayDirector) ServeProxy(target LLMGatewayRequest, writer http.ResponseWriter, request *http.Request) error { + span := trace.SpanFromContext(request.Context()) + span.SetAttributes(traceAttrEndpointType.String(traceAttrValueEndpointLLMGateway)) + + if writeFunctionStatusError(writer, target.OfflineMessage, target.EOL, "") { + return nil + } + + request.URL.Host = d.host + request.URL.Scheme = d.scheme + request.Host = "" + applyCustomHeaders(request, target.CustomHeaders) + + if !target.EOL.IsZero() { + writer.Header().Set("Deprecation", target.EOL.Format(time.RFC3339)) + } + + var proxyErr error + rp := *d.rp + rp.ErrorHandler = func(writer http.ResponseWriter, request *http.Request, err error) { + proxyErr = err + writeProxyError(writer, request, err) + } + rp.ServeHTTP(writer, request) + return proxyErr +} diff --git a/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director_test.go b/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director_test.go new file mode 100644 index 000000000..088a1b7a7 --- /dev/null +++ b/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director_test.go @@ -0,0 +1,427 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gateway + +import ( + config "ai-api-gateway-service/gateway_config" + "bufio" + "bytes" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const llmHost = "llm.test" + +type capturedRequest struct { + path string + headers http.Header + body string +} + +func captureServer(t *testing.T, requests chan<- capturedRequest, handler http.HandlerFunc) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + require.NoError(t, err) + requests <- capturedRequest{path: r.URL.Path, headers: r.Header.Clone(), body: string(body)} + if handler != nil { + handler(w, r) + return + } + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + return server +} + +func llmGatewayMappings(entry config.LLMGatewayEntry) *config.GatewayConfig { + mappings := &config.GatewayConfig{} + mappings.LLMGateway = map[string]config.LLMGatewayEntry{"llm_example": entry} + return mappings +} + +func llmGatewayMux(t *testing.T, mappings *config.GatewayConfig, llmEndpoint string) http.Handler { + t.Helper() + mux, err := buildChiMux(mappings, Config{ + NvcfApiEndpoint: "http://nvcf.invalid", + LLMGatewayEndpoint: llmEndpoint, + PrivateModelNameRegexPattern: "^$", + }) + require.NoError(t, err) + return mux +} + +func llmGatewayRequest(t *testing.T, path string, body string) *http.Request { + t.Helper() + req := httptest.NewRequest(http.MethodPost, path, bytes.NewBufferString(body)) + req.Host = llmHost + return req +} + +func awaitRequest(t *testing.T, requests <-chan capturedRequest) capturedRequest { + t.Helper() + select { + case received := <-requests: + return received + case <-time.After(time.Second): + t.Fatal("timed out waiting for upstream request") + return capturedRequest{} + } +} + +func TestNewLLMGatewayDirectorRejectsInvalidEndpoint(t *testing.T) { + for _, endpoint := range []string{"", "llm-api-gateway:8080", "://bad", "/relative"} { + t.Run(endpoint, func(t *testing.T) { + director, err := NewLLMGatewayDirector(endpoint, http.DefaultTransport) + require.Error(t, err) + assert.Nil(t, director) + assert.ErrorContains(t, err, "invalid LLM Gateway endpoint") + }) + } +} + +func TestBuildChiMux_LLMGatewayServesSupportedRoutes(t *testing.T) { + requests := make(chan capturedRequest, 1) + backend := captureServer(t, requests, nil) + mux := llmGatewayMux(t, llmGatewayMappings(config.LLMGatewayEntry{Host: llmHost}), backend.URL) + + for _, path := range llmGatewaySupportedPaths { + t.Run(path, func(t *testing.T) { + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, llmGatewayRequest(t, path, `{"model":"func-id/meta/llama"}`)) + + require.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, path, awaitRequest(t, requests).path) + }) + } +} + +func TestBuildChiMux_LLMGatewayDoesNotServeUnsupportedRoutes(t *testing.T) { + requests := make(chan capturedRequest, 1) + backend := captureServer(t, requests, nil) + mux := llmGatewayMux(t, llmGatewayMappings(config.LLMGatewayEntry{Host: llmHost}), backend.URL) + + for _, path := range []string{"/v1/completions", "/v1/models", "/v1/images/generations"} { + t.Run(path, func(t *testing.T) { + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, llmGatewayRequest(t, path, `{}`)) + + assert.Equal(t, http.StatusNotFound, rec.Code) + assert.Empty(t, requests, "unsupported routes must not reach the LLM Gateway") + }) + } +} + +func TestBuildChiMux_LLMGatewayAndVanityCoexist(t *testing.T) { + nvcfRequests := make(chan capturedRequest, 1) + nvcfBackend := captureServer(t, nvcfRequests, nil) + llmRequests := make(chan capturedRequest, 1) + llmBackend := captureServer(t, llmRequests, nil) + + mappings := llmGatewayMappings(config.LLMGatewayEntry{Host: llmHost}) + mappings.Vanity = map[string]config.VanityEntry{ + "example": { + Host: "vanity.test", + Paths: map[string]config.PathFunctionDetails{ + "infer": {Path: "/v1/example/infer", FunctionID: "vanity-func"}, + }, + }, + } + + mux, err := buildChiMux(mappings, Config{ + NvcfApiEndpoint: nvcfBackend.URL, + LLMGatewayEndpoint: llmBackend.URL, + PrivateModelNameRegexPattern: "^$", + }) + require.NoError(t, err) + + chatRec := httptest.NewRecorder() + mux.ServeHTTP(chatRec, llmGatewayRequest(t, "/v1/chat/completions", `{"model":"func-id/meta/llama"}`)) + require.Equal(t, http.StatusOK, chatRec.Code) + + llmReceived := awaitRequest(t, llmRequests) + assert.Equal(t, "/v1/chat/completions", llmReceived.path) + assert.Empty(t, llmReceived.headers.Get("function-id")) + + inferReq := httptest.NewRequest(http.MethodPost, "/v1/example/infer", bytes.NewBufferString(`{}`)) + inferReq.Host = "vanity.test" + inferRec := httptest.NewRecorder() + mux.ServeHTTP(inferRec, inferReq) + require.Equal(t, http.StatusOK, inferRec.Code) + + nvcfReceived := awaitRequest(t, nvcfRequests) + assert.Equal(t, "/v1/example/infer", nvcfReceived.path) + assert.Equal(t, "vanity-func", nvcfReceived.headers.Get("function-id")) + assert.Empty(t, llmRequests) +} + +func TestBuildChiMux_LLMGatewayForwardsRequestUnchanged(t *testing.T) { + requests := make(chan capturedRequest, 1) + backend := captureServer(t, requests, nil) + mux := llmGatewayMux(t, llmGatewayMappings(config.LLMGatewayEntry{Host: llmHost}), backend.URL) + + body := `{"model":"func-id/meta/llama-3.3-70b","messages":[{"role":"user","content":"hi"}]}` + req := llmGatewayRequest(t, "/v1/chat/completions", body) + req.Header.Set("Authorization", "Bearer caller-token") + req.Header.Set("traceparent", "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01") + rec := httptest.NewRecorder() + + mux.ServeHTTP(rec, req) + require.Equal(t, http.StatusOK, rec.Code) + + received := awaitRequest(t, requests) + assert.Equal(t, body, received.body, "request body must reach the LLM Gateway unmodified") + assert.Equal(t, "Bearer caller-token", received.headers.Get("Authorization")) + assert.Equal(t, "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", received.headers.Get("traceparent")) + assert.Empty(t, received.headers.Get("function-id")) + assert.Empty(t, received.headers.Get("function-version-id")) + assert.Empty(t, received.headers.Get("NVCF-POLL-SECONDS")) +} + +func TestBuildChiMux_LLMGatewayAppliesCustomHeaders(t *testing.T) { + requests := make(chan capturedRequest, 1) + backend := captureServer(t, requests, nil) + mappings := llmGatewayMappings(config.LLMGatewayEntry{ + Host: llmHost, + CustomHeaders: config.CustomHeaders{"X-Provider-Feature": "enabled"}, + }) + mux := llmGatewayMux(t, mappings, backend.URL) + + req := llmGatewayRequest(t, "/v1/embeddings", `{}`) + req.Header.Set("X-Provider-Feature", "caller-value") + rec := httptest.NewRecorder() + + mux.ServeHTTP(rec, req) + require.Equal(t, http.StatusOK, rec.Code) + + assert.Equal(t, "enabled", awaitRequest(t, requests).headers.Get("X-Provider-Feature")) +} + +func TestBuildChiMux_LLMGatewayRequiresEndpoint(t *testing.T) { + _, err := buildChiMux(llmGatewayMappings(config.LLMGatewayEntry{Host: llmHost}), Config{ + NvcfApiEndpoint: "http://nvcf.invalid", + PrivateModelNameRegexPattern: "^$", + }) + require.Error(t, err) + assert.ErrorContains(t, err, "LLM_GATEWAY_ENDPOINT is required") +} + +func TestBuildChiMux_LLMGatewayEndpointNotRequiredWithoutSection(t *testing.T) { + requests := make(chan capturedRequest, 1) + backend := captureServer(t, requests, nil) + + mappings := &config.GatewayConfig{} + mappings.Vanity = map[string]config.VanityEntry{ + "example": { + Host: "vanity.test", + Paths: map[string]config.PathFunctionDetails{"infer": {Path: "/v1/example/infer", FunctionID: "vanity-func"}}, + }, + } + + mux, err := buildChiMux(mappings, Config{ + NvcfApiEndpoint: backend.URL, + PrivateModelNameRegexPattern: "^$", + }) + require.NoError(t, err) + + req := httptest.NewRequest(http.MethodPost, "/v1/example/infer", bytes.NewBufferString(`{}`)) + req.Host = "vanity.test" + rec := httptest.NewRecorder() + + mux.ServeHTTP(rec, req) + assert.Equal(t, http.StatusOK, rec.Code) +} + +func TestLLMGatewayDirectorOfflineMessageReturns503(t *testing.T) { + requests := make(chan capturedRequest, 1) + backend := captureServer(t, requests, nil) + mappings := llmGatewayMappings(config.LLMGatewayEntry{ + Host: llmHost, + OfflineMessage: "temporarily offline", + }) + mux := llmGatewayMux(t, mappings, backend.URL) + + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, llmGatewayRequest(t, "/v1/chat/completions", `{}`)) + + assert.Equal(t, http.StatusServiceUnavailable, rec.Code) + assert.Contains(t, rec.Body.String(), "temporarily offline") + assert.Empty(t, requests, "offline hosts must not reach the LLM Gateway") +} + +func TestLLMGatewayDirectorEOLHandling(t *testing.T) { + tests := []struct { + name string + eol time.Time + wantStatus int + wantDeprecated bool + }{ + {name: "future EOL adds Deprecation header", eol: time.Now().Add(24 * time.Hour), wantStatus: http.StatusOK, wantDeprecated: true}, + {name: "expired EOL returns 410", eol: time.Now().Add(-24 * time.Hour), wantStatus: http.StatusGone}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + requests := make(chan capturedRequest, 1) + backend := captureServer(t, requests, nil) + mux := llmGatewayMux(t, llmGatewayMappings(config.LLMGatewayEntry{Host: llmHost, EOL: tc.eol}), backend.URL) + + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, llmGatewayRequest(t, "/v1/chat/completions", `{}`)) + + assert.Equal(t, tc.wantStatus, rec.Code) + if tc.wantDeprecated { + assert.Equal(t, tc.eol.Format(time.RFC3339), rec.Header().Get("Deprecation")) + } else { + assert.Empty(t, rec.Header().Get("Deprecation")) + } + }) + } +} + +func TestLLMGatewayDirectorUpstreamFailureReturns502(t *testing.T) { + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + endpoint := backend.URL + backend.Close() + + mux := llmGatewayMux(t, llmGatewayMappings(config.LLMGatewayEntry{Host: llmHost}), endpoint) + + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, llmGatewayRequest(t, "/v1/chat/completions", `{}`)) + + assert.Equal(t, http.StatusBadGateway, rec.Code) + assert.Equal(t, "application/problem+json", rec.Header().Get("Content-Type")) +} + +func TestLLMGatewayDirectorStreamsResponseIncrementally(t *testing.T) { + release := make(chan struct{}) + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(http.StatusOK) + flusher, ok := w.(http.Flusher) + require.True(t, ok) + _, _ = w.Write([]byte("data: first\n\n")) + flusher.Flush() + <-release + _, _ = w.Write([]byte("data: [DONE]\n\n")) + flusher.Flush() + })) + t.Cleanup(backend.Close) + + mux := llmGatewayMux(t, llmGatewayMappings(config.LLMGatewayEntry{Host: llmHost}), backend.URL) + proxy := httptest.NewServer(mux) + t.Cleanup(proxy.Close) + + req, err := http.NewRequest(http.MethodPost, proxy.URL+"/v1/chat/completions", bytes.NewBufferString(`{"stream":true}`)) + require.NoError(t, err) + req.Host = llmHost + + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + t.Cleanup(func() { _ = resp.Body.Close() }) + require.Equal(t, http.StatusOK, resp.StatusCode) + + reader := bufio.NewReader(resp.Body) + line, err := reader.ReadString('\n') + require.NoError(t, err) + assert.Equal(t, "data: first\n", line, "first chunk must arrive before the upstream finishes") + + close(release) + rest, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Contains(t, string(rest), "data: [DONE]") +} + +// The openai section and the llmGateway section both serve +// /v1/chat/completions. They never compete, because routing selects a host +// first and each host owns a separate route table. +func TestBuildChiMux_SamePathOnOpenAIAndLLMGatewayHosts(t *testing.T) { + nvcfRequests := make(chan capturedRequest, 1) + nvcfBackend := captureServer(t, nvcfRequests, nil) + llmRequests := make(chan capturedRequest, 1) + llmBackend := captureServer(t, llmRequests, nil) + + mappings := llmGatewayMappings(config.LLMGatewayEntry{Host: llmHost}) + mappings.OpenAI.Host = "openai.test" + mappings.OpenAI.ChatCompletions = map[string]config.ModelFunctionDetails{ + "llama": {ModelName: "meta/llama-3.3-70b", FunctionID: "openai-func"}, + } + + mux, err := buildChiMux(mappings, Config{ + NvcfApiEndpoint: nvcfBackend.URL, + LLMGatewayEndpoint: llmBackend.URL, + PrivateModelNameRegexPattern: "^$", + }) + require.NoError(t, err) + + openAIReq := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewBufferString(`{"model":"meta/llama-3.3-70b"}`)) + openAIReq.Host = "openai.test" + openAIRec := httptest.NewRecorder() + mux.ServeHTTP(openAIRec, openAIReq) + require.Equal(t, http.StatusOK, openAIRec.Code) + + openAIReceived := awaitRequest(t, nvcfRequests) + assert.Equal(t, "openai-func", openAIReceived.headers.Get("function-id")) + assert.Empty(t, llmRequests, "the openai host must not reach the LLM Gateway") + + llmRec := httptest.NewRecorder() + mux.ServeHTTP(llmRec, llmGatewayRequest(t, "/v1/chat/completions", `{"model":"func-id/meta/llama-3.3-70b"}`)) + require.Equal(t, http.StatusOK, llmRec.Code) + + llmReceived := awaitRequest(t, llmRequests) + assert.Equal(t, `{"model":"func-id/meta/llama-3.3-70b"}`, llmReceived.body) + assert.Empty(t, llmReceived.headers.Get("function-id")) + assert.Empty(t, nvcfRequests, "the llmGateway host must not reach the invocation service") +} + +func TestBuildChiMux_LLMGatewayRejectsSelfProxyingHost(t *testing.T) { + tests := []struct { + name string + host string + endpoint string + wantErr bool + }{ + {name: "host matches endpoint with port", host: "llm.test", endpoint: "http://llm.test:8080", wantErr: true}, + {name: "host matches endpoint without port", host: "llm.test", endpoint: "http://llm.test", wantErr: true}, + {name: "host carries a port too", host: "llm.test:443", endpoint: "http://llm.test:8080", wantErr: true}, + {name: "distinct host and endpoint", host: "llm.test", endpoint: "http://llm-api-gateway.nvcf.svc.cluster.local:8080", wantErr: false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := buildChiMux(llmGatewayMappings(config.LLMGatewayEntry{Host: tc.host}), Config{ + NvcfApiEndpoint: "http://nvcf.invalid", + LLMGatewayEndpoint: tc.endpoint, + PrivateModelNameRegexPattern: "^$", + }) + + if !tc.wantErr { + require.NoError(t, err) + return + } + require.Error(t, err) + assert.ErrorContains(t, err, "the gateway would proxy to itself") + }) + } +} diff --git a/src/invocation-plane-services/vanity-gateway/gateway/tracing_attrs.go b/src/invocation-plane-services/vanity-gateway/gateway/tracing_attrs.go index f48e4f903..83a4a3228 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/tracing_attrs.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/tracing_attrs.go @@ -37,7 +37,8 @@ const ( traceAttrShadowTargetModel attribute.Key = "shadow.target_model" traceAttrShadowTargetModels attribute.Key = "shadow.target_models" - traceAttrValueEndpointOpenAI = "openai" + traceAttrValueEndpointOpenAI = "openai" + traceAttrValueEndpointLLMGateway = "llm_gateway" shadowDroppedReasonBodyReadError = "body_read_error" shadowDroppedReasonBodyRewriteError = "body_rewrite_error" diff --git a/src/invocation-plane-services/vanity-gateway/gateway/vanity_director.go b/src/invocation-plane-services/vanity-gateway/gateway/vanity_director.go index 56fae7661..a7c416048 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/vanity_director.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/vanity_director.go @@ -167,8 +167,8 @@ type ProblemDetails struct { Detail string `json:"detail"` } -func NewVanityDirector(nvcfApiHost string, transport http.RoundTripper) (*VanityDirector, error) { - rp := &httputil.ReverseProxy{ +func newGatewayReverseProxy(transport http.RoundTripper) *httputil.ReverseProxy { + return &httputil.ReverseProxy{ Director: func(request *http.Request) { // already directed, needed to be able to error }, @@ -178,6 +178,10 @@ func NewVanityDirector(nvcfApiHost string, transport http.RoundTripper) (*Vanity ModifyResponse: modifyTooManyRequestsResponse, ErrorHandler: writeProxyError, } +} + +func NewVanityDirector(nvcfApiHost string, transport http.RoundTripper) (*VanityDirector, error) { + rp := newGatewayReverseProxy(transport) nvcfApiUrl, err := url.Parse(nvcfApiHost) if err != nil || nvcfApiUrl.Scheme == "" || nvcfApiUrl.Host == "" { return nil, fmt.Errorf("invalid NVCF API host: %s", nvcfApiHost) diff --git a/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config.go b/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config.go index 998fb1046..759eb3bbf 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config.go +++ b/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config.go @@ -131,6 +131,16 @@ type VanityEntry struct { Paths map[string]PathFunctionDetails `json:"paths"` } +// LLMGatewayEntry serves the LLM Gateway's OpenAI-compatible routes on Host. +// The gateway proxies them verbatim, so an entry carries no function or model +// selection: the LLM Gateway resolves the function from the request model. +type LLMGatewayEntry struct { + Host string `json:"host"` + CustomHeaders CustomHeaders `json:"customHeaders,omitempty"` + EOL time.Time `json:"eol,omitempty"` + OfflineMessage string `json:"offlineMessage,omitempty"` +} + type V2Config struct { OpenAI struct { Host string `json:"host"` @@ -142,7 +152,8 @@ type V2Config struct { ImageEdits map[string]ModelFunctionDetails `json:"imageEdits"` ImageVariations map[string]ModelFunctionDetails `json:"imageVariations"` } `json:"openai"` - Vanity map[string]VanityEntry `json:"vanity"` + Vanity map[string]VanityEntry `json:"vanity"` + LLMGateway map[string]LLMGatewayEntry `json:"llmGateway"` } // sharedNotifications is a package-level channel shared across all config instances. @@ -240,7 +251,11 @@ func (c *GatewayConfig) Validate() error { } } - return c.validateVanityConfig() + if err := c.validateVanityConfig(); err != nil { + return err + } + + return c.validateLLMGatewayConfig() } func (c *GatewayConfig) openAISections() map[string]map[string]ModelFunctionDetails { @@ -348,6 +363,12 @@ var reservedCustomHeaderNames = map[string]struct{}{ "x-forwarded-proto": {}, } +// The LLM Gateway rejects any request carrying X-Priority, on header presence +// rather than value, so a configured value would fail every request. +var llmGatewayReservedCustomHeaderNames = map[string]struct{}{ + "x-priority": {}, +} + func validateCustomHeaders(location string, headers CustomHeaders) error { seenNames := make(map[string]string, len(headers)) for name := range headers { @@ -425,6 +446,59 @@ func (c *GatewayConfig) validateVanityConfig() error { return nil } +// validateLLMGatewayConfig checks the hosts that proxy to the LLM Gateway. The +// gateway registers a fixed route set on each of them, so the only things an +// entry can get wrong are the host itself and the static headers. +func (c *GatewayConfig) validateLLMGatewayConfig() error { + for entryKey, entry := range c.LLMGateway { + location := "llmGateway." + entryKey + if entry.Host == "" { + return fmt.Errorf("%s: host is required", location) + } + if err := validateCustomHeaders(location, entry.CustomHeaders); err != nil { + return err + } + for name := range entry.CustomHeaders { + if _, ok := llmGatewayReservedCustomHeaderNames[strings.ToLower(name)]; ok { + return fmt.Errorf("%s: customHeaders cannot set %q; the LLM Gateway rejects requests carrying it", location, name) + } + } + } + + return c.validateHostUniqueness() +} + +// validateHostUniqueness rejects a host claimed by more than one section. +// Registration is a plain map assignment keyed by host, so a collision would +// silently drop one section's routes. +func (c *GatewayConfig) validateHostUniqueness() error { + owners := make(map[string]string, len(c.LLMGateway)+len(c.Vanity)+1) + if c.OpenAI.Host != "" { + owners[c.OpenAI.Host] = "openai" + } + for vanityName, vanity := range c.Vanity { + if vanity.Host == "" { + continue + } + if owner, ok := owners[vanity.Host]; ok { + return fmt.Errorf("vanity.%s: host %q is already served by %s", vanityName, vanity.Host, owner) + } + owners[vanity.Host] = "vanity." + vanityName + } + for entryKey, entry := range c.LLMGateway { + if owner, ok := owners[entry.Host]; ok { + return fmt.Errorf("llmGateway.%s: host %q is already served by %s", entryKey, entry.Host, owner) + } + owners[entry.Host] = "llmGateway." + entryKey + } + return nil +} + +// HasLLMGatewayRoute reports whether any host proxies to the LLM Gateway. +func (c *GatewayConfig) HasLLMGatewayRoute() bool { + return len(c.LLMGateway) > 0 +} + func SetupConfigWithConfigPath(path string) (rc.ReloadableConfig[GatewayConfig], error) { config, err := rc.SetupConfig[GatewayConfig](path, rc.WithValidateFunc(func(c *GatewayConfig) error { diff --git a/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config_test.go b/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config_test.go index 4a59f6bc0..35a1a43fa 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config_test.go +++ b/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config_test.go @@ -867,3 +867,149 @@ func TestGatewayConfigValidateRejectsVanityShadowSamplingMethod(t *testing.T) { require.Error(t, err) assert.ErrorContains(t, err, "shadow config is unsupported for vanity routes") } + +func llmGatewayConfig(entry LLMGatewayEntry) *GatewayConfig { + cfg := &GatewayConfig{} + cfg.LLMGateway = map[string]LLMGatewayEntry{"llm_example": entry} + return cfg +} + +func TestGatewayConfigValidateAcceptsLLMGatewayEntry(t *testing.T) { + cfg := llmGatewayConfig(LLMGatewayEntry{ + Host: "llm.example.com", + CustomHeaders: CustomHeaders{"X-Provider-Feature": "enabled"}, + OfflineMessage: "", + }) + + require.NoError(t, cfg.Validate()) + assert.True(t, cfg.HasLLMGatewayRoute()) +} + +func TestGatewayConfigHasLLMGatewayRouteFalseWhenSectionEmpty(t *testing.T) { + cfg := &GatewayConfig{} + cfg.Vanity = map[string]VanityEntry{ + "example": { + Host: "ai.example.com", + Paths: map[string]PathFunctionDetails{"sample": {Path: "/v1/example/infer", FunctionID: "func-id"}}, + }, + } + + require.NoError(t, cfg.Validate()) + assert.False(t, cfg.HasLLMGatewayRoute()) +} + +func TestGatewayConfigValidateRejectsLLMGatewayEntryWithoutHost(t *testing.T) { + err := llmGatewayConfig(LLMGatewayEntry{}).Validate() + require.Error(t, err) + assert.ErrorContains(t, err, "llmGateway.llm_example: host is required") +} + +func TestGatewayConfigValidateRejectsPriorityHeaderOnLLMGatewayEntry(t *testing.T) { + for _, name := range []string{"X-Priority", "x-priority"} { + t.Run(name, func(t *testing.T) { + cfg := llmGatewayConfig(LLMGatewayEntry{ + Host: "llm.example.com", + CustomHeaders: CustomHeaders{name: "high"}, + }) + + err := cfg.Validate() + require.Error(t, err) + assert.ErrorContains(t, err, "the LLM Gateway rejects requests carrying it") + }) + } +} + +func TestGatewayConfigValidateRejectsReservedHeaderOnLLMGatewayEntry(t *testing.T) { + cfg := llmGatewayConfig(LLMGatewayEntry{ + Host: "llm.example.com", + CustomHeaders: CustomHeaders{"Authorization": "Bearer nope"}, + }) + + err := cfg.Validate() + require.Error(t, err) + assert.ErrorContains(t, err, "cannot set reserved header") +} + +func TestGatewayConfigValidateRejectsDuplicateHostsAcrossSections(t *testing.T) { + tests := []struct { + name string + mutate func(cfg *GatewayConfig) + wantMessage string + }{ + { + name: "llmGateway collides with openai", + mutate: func(cfg *GatewayConfig) { cfg.OpenAI.Host = "shared.example.com" }, + wantMessage: `host "shared.example.com" is already served by openai`, + }, + { + name: "llmGateway collides with vanity", + mutate: func(cfg *GatewayConfig) { + cfg.Vanity = map[string]VanityEntry{ + "example": { + Host: "shared.example.com", + Paths: map[string]PathFunctionDetails{"sample": {Path: "/v1/example/infer", FunctionID: "func-id"}}, + }, + } + }, + wantMessage: `host "shared.example.com" is already served by vanity.example`, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cfg := llmGatewayConfig(LLMGatewayEntry{Host: "shared.example.com"}) + tc.mutate(cfg) + + err := cfg.Validate() + require.Error(t, err) + assert.ErrorContains(t, err, tc.wantMessage) + }) + } +} + +func TestGatewayConfigValidateRejectsDuplicateHostAcrossTwoLLMGatewayEntries(t *testing.T) { + cfg := &GatewayConfig{} + cfg.LLMGateway = map[string]LLMGatewayEntry{ + "first": {Host: "llm.example.com"}, + "second": {Host: "llm.example.com"}, + } + + err := cfg.Validate() + require.Error(t, err) + assert.ErrorContains(t, err, `host "llm.example.com" is already served by llmGateway.`) +} + +func TestGatewayConfigValidateAllowsEmptyOpenAIHostAlongsideLLMGateway(t *testing.T) { + cfg := llmGatewayConfig(LLMGatewayEntry{Host: "llm.example.com"}) + + require.NoError(t, cfg.Validate()) +} + +func TestGatewayConfigLoadAcceptsLLMGatewaySection(t *testing.T) { + configPath := filepath.Join(t.TempDir(), "config.yaml") + err := os.WriteFile(configPath, []byte(` +v2config: + vanity: + example: + host: ai.example.com + paths: + infer: + path: /v1/example/infer + functionID: func-id + llmGateway: + llm_example: + host: llm.example.com + offlineMessage: "" + customHeaders: + X-Provider-Feature: enabled +`), 0600) + require.NoError(t, err) + + loaded, err := SetupConfigWithConfigPath(configPath) + require.NoError(t, err) + + cfg := loaded.Get() + require.True(t, cfg.HasLLMGatewayRoute()) + assert.Equal(t, "llm.example.com", cfg.LLMGateway["llm_example"].Host) + assert.Equal(t, "enabled", cfg.LLMGateway["llm_example"].CustomHeaders["X-Provider-Feature"]) +}