Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-18532-18533-litellm-key-resolution.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- The agent inference proxy now picks up a rotated LiteLLM or gateway key file without a restart, re-resolves the key and retries once on an upstream 401, and logs `key_source` and `key_changed_since_start` on auth failures; an `api_key_file` outside `/secrets` or `/data/secrets` is now rejected at dashboard/API save with the allowed dirs listed, warned about at boot and once at runtime with the source actually used, and the effective key source is shown in the governor config API and the LiteLLM tab (#18532, #18533).
2 changes: 1 addition & 1 deletion docs/inference-backends.md
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ Endpoint environment overrides used by v2 include:
- `HIVE_VLLM_ENDPOINT` for `vllm`
- `HIVE_LLMD_ENDPOINT` for `llm-d`
- `HIVE_LITELLM_ENDPOINT` for LiteLLM
- `HIVE_LITELLM_API_KEY` or `api_key_file` for LiteLLM bearer auth
- `HIVE_LITELLM_API_KEY` or `api_key_file` for LiteLLM bearer auth. `api_key_file` must be under `/secrets` or `/data/secrets`; any other path is refused at save, warned about at boot and at runtime, and never read. Mount the Secret under one of those dirs. A rotated key file is picked up without a restart. See [Where a gateway key file may live](../src/docs/agent-configuration.md#where-a-gateway-key-file-may-live).
- `HIVE_LITELLM_MODELS` as a comma-separated fallback model list when discovery fails

Never put the key value itself in YAML.
Expand Down
35 changes: 35 additions & 0 deletions src/cmd/hive/inference_key_resolver.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package main

import (
"strings"

"github.com/hivecommons/hive/crux/proxy"
"github.com/hivecommons/hive/pkg/config"
)

// litellmInferenceKeyResolver returns the resolver the inference proxy uses to
// re-read a litellm route's key at request time (hive#18532). It runs the same
// resolution as the route's initial key (ResolveLiteLLMInferenceKeyWithSource)
// against the LIVE config, so a rotated Secret, a re-saved key file or a config
// reload all reach the next request without a restart.
func litellmInferenceKeyResolver(cfg *config.Config, backend string) *proxy.InferenceKeyResolver {
return proxy.NewInferenceKeyResolver(backend, func() (string, string) {
return cfg.Governor.ResolveLiteLLMInferenceKeyWithSource(backend)
})
}

// gatewayInferenceKeyResolver is the named-gateway twin of
// litellmInferenceKeyResolver. It returns nil for watsonx: that route carries a
// short-lived IAM bearer minted from the key, not the key, so it keeps its
// existing minting path.
func gatewayInferenceKeyResolver(cfg *config.Config, gw *config.GatewayConfig, backend string) *proxy.InferenceKeyResolver {
if gw == nil || strings.EqualFold(gw.Kind, config.GatewayKindWatsonx) {
return nil
}
return proxy.NewInferenceKeyResolver(backend, func() (string, string) {
if cur := cfg.Governor.ResolveGateway(backend); cur != nil {
return cur.ResolveAPIKeyWithSource()
}
return "", ""
})
}
93 changes: 93 additions & 0 deletions src/cmd/hive/inference_key_resolver_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
package main

import (
"os"
"path/filepath"
"testing"

"github.com/hivecommons/hive/pkg/config"
)

// TestInferenceKeyResolvers_ReadTheLiveConfig: the resolvers handed to the
// proxy (hive#18532) re-resolve from the live config each time, so a changed
// key file or env var reaches the next request without a restart.
func TestInferenceKeyResolvers_ReadTheLiveConfig(t *testing.T) {
root := filepath.Join(t.TempDir(), "secrets")
if err := os.MkdirAll(root, 0o700); err != nil {
t.Fatal(err)
}
t.Cleanup(config.SetSecretFileRootsForTest(root))
keyFile := filepath.Join(root, "gw_key")
if err := os.WriteFile(keyFile, []byte("gw-one"), 0o600); err != nil {
t.Fatal(err)
}

cfg := bootProxyConfig()
cfg.Governor.Gateways = []config.GatewayConfig{{Name: "corp", Kind: config.GatewayKindLiteLLM, APIKeyFile: keyFile}}

gw := gatewayInferenceKeyResolver(cfg, &cfg.Governor.Gateways[0], "corp")
if gw == nil {
t.Fatal("a litellm-kind gateway must get a resolver")
}
if key, src, _ := gw.Refresh(); key != "gw-one" || src != "file:"+keyFile {
t.Fatalf("gateway resolver = (%q, %q)", key, src)
}
if err := os.WriteFile(keyFile, []byte("gw-two"), 0o600); err != nil {
t.Fatal(err)
}
if key, _, changed := gw.Refresh(); key != "gw-two" || !changed {
t.Fatalf("gateway resolver did not pick up the rewritten file: %q changed=%v", key, changed)
}
cfg.Governor.Gateways = nil
if key, src, _ := gw.Refresh(); key != "" || src != "" {
t.Fatalf("a gateway removed from the live config must resolve nothing, got (%q, %q)", key, src)
}

if gatewayInferenceKeyResolver(cfg, nil, "x") != nil {
t.Fatal("a nil gateway gets no resolver")
}
if gatewayInferenceKeyResolver(cfg, &config.GatewayConfig{Name: "ibm", Kind: config.GatewayKindWatsonx}, "ibm") != nil {
t.Fatal("watsonx carries a minted IAM token and must keep a static key")
}

t.Setenv(config.DefaultLiteLLMAPIKeyEnv, "legacy-one")
ll := litellmInferenceKeyResolver(cfg, "litellm")
if key, src, _ := ll.Refresh(); key != "legacy-one" || src != "env:"+config.DefaultLiteLLMAPIKeyEnv {
t.Fatalf("litellm resolver = (%q, %q)", key, src)
}
t.Setenv(config.DefaultLiteLLMAPIKeyEnv, "legacy-two")
if key, _, changed := ll.Refresh(); key != "legacy-two" || !changed {
t.Fatalf("litellm resolver did not follow the env: %q changed=%v", key, changed)
}
}

// TestBootProxyWith_RoutesCarryKeyResolvers: the named-gateway and litellm
// branches of the route callback attach a resolver; vllm does not.
func TestBootProxyWith_RoutesCarryKeyResolvers(t *testing.T) {
cfg := bootProxyConfig()
cfg.Governor.LiteLLM.Endpoint = "http://127.0.0.1:1"
cfg.Governor.Gateways = []config.GatewayConfig{
{Name: "openrouter", Kind: "openrouter", Endpoint: "https://openrouter.ai/api/v1", DefaultModel: "or/default"},
}
f := newBootProxyFake()
b := newBootProxyBoot(t, f, cfg)
b.bootProxyWith(f.deps)

f.setRoute("scanner", "openrouter", "")
if r := f.routes["scanner"]; r == nil || r.KeyResolver == nil {
t.Fatalf("gateway route = %+v, want a key resolver", r)
}
}

func TestBootProxyWith_LiteLLMRouteCarriesKeyResolver(t *testing.T) {
cfg := bootProxyConfig()
cfg.Governor.LiteLLM.Endpoint = "http://127.0.0.1:1"
f := newBootProxyFake()
b := newBootProxyBoot(t, f, cfg)
b.bootProxyWith(f.deps)

f.setRoute("triage", "litellm", "m")
if r := f.routes["triage"]; r == nil || r.KeyResolver == nil {
t.Fatalf("litellm route = %+v, want a key resolver", r)
}
}
2 changes: 2 additions & 0 deletions src/cmd/hive/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -4338,6 +4338,7 @@ func (b *boot) bootProxyWith(deps bootProxyDeps) {
APIKey: apiKey,
CABundle: gw.CABundle,
ExtraHeaders: extraHeaders,
KeyResolver: gatewayInferenceKeyResolver(b.cfg, gw, backend),
})
return
}
Expand Down Expand Up @@ -4395,6 +4396,7 @@ func (b *boot) bootProxyWith(deps bootProxyDeps) {
DefaultModel: defaultModel,
APIKey: apiKey,
CABundle: caBundle,
KeyResolver: litellmInferenceKeyResolver(b.cfg, backend),
})
return
}
Expand Down
77 changes: 0 additions & 77 deletions src/crux/config/agent_runtime.go
Original file line number Diff line number Diff line change
Expand Up @@ -777,27 +777,6 @@ func APIKeySHA256(key string) string {
return hex.EncodeToString(sum[:])
}

func (gw GatewayConfig) ResolveAPIKey() string {
if gw.APIKeyEnv != "" {
if v := os.Getenv(gw.APIKeyEnv); v != "" {
return v
}
}
if gw.APIKeyFile != "" {
// SECURITY (audit N8): confine the read to the managed secrets dirs. This
// gate lives HERE, not only in the HTTP handler, so every caller is
// covered — including a path that reached hive.yaml some other way (a
// hand-edited config, an older build, a restored backup).
if !SecretFilePathAllowed(gw.APIKeyFile) {
return ""
}
if b, err := readSecretFile(gw.APIKeyFile); err == nil {
return strings.TrimSpace(string(b))
}
}
return ""
}

// IsEnabled reports whether the trajectory-review lane is on. Default is ON:
// a nil Enabled (key omitted) counts as enabled, only an explicit false
// disables it. The lane itself no-ops when no reviewer endpoint resolves, so
Expand Down Expand Up @@ -1024,62 +1003,6 @@ const (
BobStateDirName = ".bob"
)

// ResolveAPIKey returns the LiteLLM API key. Key FILES are consulted in
// priority order — the configured api_key_file, then the k8s Secret mount
// (DefaultLiteLLMAPIKeyFile), then the dashboard-written PVC file
// (WritableLiteLLMAPIKeyFile) — followed by the env var named by
// api_key_env and finally DefaultLiteLLMAPIKeyEnv. Returns "" when no key
// is configured. The key value itself is never stored in hive.yaml.
//
// Consulting all three file locations means a key saved via the dashboard
// keeps working even if hive.yaml is reset (e.g. re-seeded from a
// ConfigMap) and the api_key_file pointer is lost, and an admin-managed
// Secret key keeps working if the PVC copy is wiped.
func (c *LiteLLMConfig) ResolveAPIKey() string {
key, _ := c.resolveAPIKeyWithSource()
return key
}

// ResolveAPIKeySource reports where ResolveAPIKey found the key without
// exposing the value: "file:<path>", "env:<NAME>", or "" when no key is
// configured. Safe to return from APIs (the dashboard shows it as the
// "Key detected" store).
func (c *LiteLLMConfig) ResolveAPIKeySource() string {
_, source := c.resolveAPIKeyWithSource()
return source
}

func (c *LiteLLMConfig) resolveAPIKeyWithSource() (string, string) {
files := []string{c.APIKeyFile, DefaultLiteLLMAPIKeyFile, WritableLiteLLMAPIKeyFile}
seen := map[string]bool{"": true}
for _, f := range files {
if seen[f] {
continue
}
seen[f] = true
// SECURITY (audit N8): only the managed secrets dirs. The two defaults
// appended above already satisfy this; the gate is for c.APIKeyFile,
// which is operator/API-supplied.
if !SecretFilePathAllowed(f) {
continue
}
if data, err := readSecretFile(f); err == nil {
if key := strings.TrimSpace(string(data)); key != "" {
return key, "file:" + f
}
}
}
if c.APIKeyEnv != "" {
if key := os.Getenv(c.APIKeyEnv); key != "" {
return key, "env:" + c.APIKeyEnv
}
}
if key := os.Getenv(DefaultLiteLLMAPIKeyEnv); key != "" {
return key, "env:" + DefaultLiteLLMAPIKeyEnv
}
return "", ""
}

// ResolveEndpoint returns the effective LiteLLM base URL: the
// HIVE_LITELLM_ENDPOINT env var when set, otherwise the YAML endpoint.
func (c *LiteLLMConfig) ResolveEndpoint() string {
Expand Down
Loading
Loading