diff --git a/changelog.d/fixed-18532-18533-litellm-key-resolution.md b/changelog.d/fixed-18532-18533-litellm-key-resolution.md new file mode 100644 index 0000000000..d066c7582a --- /dev/null +++ b/changelog.d/fixed-18532-18533-litellm-key-resolution.md @@ -0,0 +1 @@ +- The agent inference proxy now picks up a rotated LiteLLM or gateway key file without a restart, re-resolves the key and retries once on an upstream 401, and logs `key_source` and `key_changed_since_start` on auth failures; an `api_key_file` outside `/secrets` or `/data/secrets` is now rejected at dashboard/API save with the allowed dirs listed, warned about at boot and once at runtime with the source actually used, and the effective key source is shown in the governor config API and the LiteLLM tab (#18532, #18533). diff --git a/docs/inference-backends.md b/docs/inference-backends.md index 655d6fcc5f..6fd6901363 100644 --- a/docs/inference-backends.md +++ b/docs/inference-backends.md @@ -113,7 +113,7 @@ Endpoint environment overrides used by v2 include: - `HIVE_VLLM_ENDPOINT` for `vllm` - `HIVE_LLMD_ENDPOINT` for `llm-d` - `HIVE_LITELLM_ENDPOINT` for LiteLLM -- `HIVE_LITELLM_API_KEY` or `api_key_file` for LiteLLM bearer auth +- `HIVE_LITELLM_API_KEY` or `api_key_file` for LiteLLM bearer auth. `api_key_file` must be under `/secrets` or `/data/secrets`; any other path is refused at save, warned about at boot and at runtime, and never read. Mount the Secret under one of those dirs. A rotated key file is picked up without a restart. See [Where a gateway key file may live](../src/docs/agent-configuration.md#where-a-gateway-key-file-may-live). - `HIVE_LITELLM_MODELS` as a comma-separated fallback model list when discovery fails Never put the key value itself in YAML. diff --git a/src/cmd/hive/inference_key_resolver.go b/src/cmd/hive/inference_key_resolver.go new file mode 100644 index 0000000000..a59ed275ea --- /dev/null +++ b/src/cmd/hive/inference_key_resolver.go @@ -0,0 +1,35 @@ +package main + +import ( + "strings" + + "github.com/hivecommons/hive/crux/proxy" + "github.com/hivecommons/hive/pkg/config" +) + +// litellmInferenceKeyResolver returns the resolver the inference proxy uses to +// re-read a litellm route's key at request time (hive#18532). It runs the same +// resolution as the route's initial key (ResolveLiteLLMInferenceKeyWithSource) +// against the LIVE config, so a rotated Secret, a re-saved key file or a config +// reload all reach the next request without a restart. +func litellmInferenceKeyResolver(cfg *config.Config, backend string) *proxy.InferenceKeyResolver { + return proxy.NewInferenceKeyResolver(backend, func() (string, string) { + return cfg.Governor.ResolveLiteLLMInferenceKeyWithSource(backend) + }) +} + +// gatewayInferenceKeyResolver is the named-gateway twin of +// litellmInferenceKeyResolver. It returns nil for watsonx: that route carries a +// short-lived IAM bearer minted from the key, not the key, so it keeps its +// existing minting path. +func gatewayInferenceKeyResolver(cfg *config.Config, gw *config.GatewayConfig, backend string) *proxy.InferenceKeyResolver { + if gw == nil || strings.EqualFold(gw.Kind, config.GatewayKindWatsonx) { + return nil + } + return proxy.NewInferenceKeyResolver(backend, func() (string, string) { + if cur := cfg.Governor.ResolveGateway(backend); cur != nil { + return cur.ResolveAPIKeyWithSource() + } + return "", "" + }) +} diff --git a/src/cmd/hive/inference_key_resolver_test.go b/src/cmd/hive/inference_key_resolver_test.go new file mode 100644 index 0000000000..7e31bbf0c1 --- /dev/null +++ b/src/cmd/hive/inference_key_resolver_test.go @@ -0,0 +1,93 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/hivecommons/hive/pkg/config" +) + +// TestInferenceKeyResolvers_ReadTheLiveConfig: the resolvers handed to the +// proxy (hive#18532) re-resolve from the live config each time, so a changed +// key file or env var reaches the next request without a restart. +func TestInferenceKeyResolvers_ReadTheLiveConfig(t *testing.T) { + root := filepath.Join(t.TempDir(), "secrets") + if err := os.MkdirAll(root, 0o700); err != nil { + t.Fatal(err) + } + t.Cleanup(config.SetSecretFileRootsForTest(root)) + keyFile := filepath.Join(root, "gw_key") + if err := os.WriteFile(keyFile, []byte("gw-one"), 0o600); err != nil { + t.Fatal(err) + } + + cfg := bootProxyConfig() + cfg.Governor.Gateways = []config.GatewayConfig{{Name: "corp", Kind: config.GatewayKindLiteLLM, APIKeyFile: keyFile}} + + gw := gatewayInferenceKeyResolver(cfg, &cfg.Governor.Gateways[0], "corp") + if gw == nil { + t.Fatal("a litellm-kind gateway must get a resolver") + } + if key, src, _ := gw.Refresh(); key != "gw-one" || src != "file:"+keyFile { + t.Fatalf("gateway resolver = (%q, %q)", key, src) + } + if err := os.WriteFile(keyFile, []byte("gw-two"), 0o600); err != nil { + t.Fatal(err) + } + if key, _, changed := gw.Refresh(); key != "gw-two" || !changed { + t.Fatalf("gateway resolver did not pick up the rewritten file: %q changed=%v", key, changed) + } + cfg.Governor.Gateways = nil + if key, src, _ := gw.Refresh(); key != "" || src != "" { + t.Fatalf("a gateway removed from the live config must resolve nothing, got (%q, %q)", key, src) + } + + if gatewayInferenceKeyResolver(cfg, nil, "x") != nil { + t.Fatal("a nil gateway gets no resolver") + } + if gatewayInferenceKeyResolver(cfg, &config.GatewayConfig{Name: "ibm", Kind: config.GatewayKindWatsonx}, "ibm") != nil { + t.Fatal("watsonx carries a minted IAM token and must keep a static key") + } + + t.Setenv(config.DefaultLiteLLMAPIKeyEnv, "legacy-one") + ll := litellmInferenceKeyResolver(cfg, "litellm") + if key, src, _ := ll.Refresh(); key != "legacy-one" || src != "env:"+config.DefaultLiteLLMAPIKeyEnv { + t.Fatalf("litellm resolver = (%q, %q)", key, src) + } + t.Setenv(config.DefaultLiteLLMAPIKeyEnv, "legacy-two") + if key, _, changed := ll.Refresh(); key != "legacy-two" || !changed { + t.Fatalf("litellm resolver did not follow the env: %q changed=%v", key, changed) + } +} + +// TestBootProxyWith_RoutesCarryKeyResolvers: the named-gateway and litellm +// branches of the route callback attach a resolver; vllm does not. +func TestBootProxyWith_RoutesCarryKeyResolvers(t *testing.T) { + cfg := bootProxyConfig() + cfg.Governor.LiteLLM.Endpoint = "http://127.0.0.1:1" + cfg.Governor.Gateways = []config.GatewayConfig{ + {Name: "openrouter", Kind: "openrouter", Endpoint: "https://openrouter.ai/api/v1", DefaultModel: "or/default"}, + } + f := newBootProxyFake() + b := newBootProxyBoot(t, f, cfg) + b.bootProxyWith(f.deps) + + f.setRoute("scanner", "openrouter", "") + if r := f.routes["scanner"]; r == nil || r.KeyResolver == nil { + t.Fatalf("gateway route = %+v, want a key resolver", r) + } +} + +func TestBootProxyWith_LiteLLMRouteCarriesKeyResolver(t *testing.T) { + cfg := bootProxyConfig() + cfg.Governor.LiteLLM.Endpoint = "http://127.0.0.1:1" + f := newBootProxyFake() + b := newBootProxyBoot(t, f, cfg) + b.bootProxyWith(f.deps) + + f.setRoute("triage", "litellm", "m") + if r := f.routes["triage"]; r == nil || r.KeyResolver == nil { + t.Fatalf("litellm route = %+v, want a key resolver", r) + } +} diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index af8950203b..802d7a4394 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -4338,6 +4338,7 @@ func (b *boot) bootProxyWith(deps bootProxyDeps) { APIKey: apiKey, CABundle: gw.CABundle, ExtraHeaders: extraHeaders, + KeyResolver: gatewayInferenceKeyResolver(b.cfg, gw, backend), }) return } @@ -4395,6 +4396,7 @@ func (b *boot) bootProxyWith(deps bootProxyDeps) { DefaultModel: defaultModel, APIKey: apiKey, CABundle: caBundle, + KeyResolver: litellmInferenceKeyResolver(b.cfg, backend), }) return } diff --git a/src/crux/config/agent_runtime.go b/src/crux/config/agent_runtime.go index b054f73131..c1f672eda0 100644 --- a/src/crux/config/agent_runtime.go +++ b/src/crux/config/agent_runtime.go @@ -777,27 +777,6 @@ func APIKeySHA256(key string) string { return hex.EncodeToString(sum[:]) } -func (gw GatewayConfig) ResolveAPIKey() string { - if gw.APIKeyEnv != "" { - if v := os.Getenv(gw.APIKeyEnv); v != "" { - return v - } - } - if gw.APIKeyFile != "" { - // SECURITY (audit N8): confine the read to the managed secrets dirs. This - // gate lives HERE, not only in the HTTP handler, so every caller is - // covered — including a path that reached hive.yaml some other way (a - // hand-edited config, an older build, a restored backup). - if !SecretFilePathAllowed(gw.APIKeyFile) { - return "" - } - if b, err := readSecretFile(gw.APIKeyFile); err == nil { - return strings.TrimSpace(string(b)) - } - } - return "" -} - // IsEnabled reports whether the trajectory-review lane is on. Default is ON: // a nil Enabled (key omitted) counts as enabled, only an explicit false // disables it. The lane itself no-ops when no reviewer endpoint resolves, so @@ -1024,62 +1003,6 @@ const ( BobStateDirName = ".bob" ) -// ResolveAPIKey returns the LiteLLM API key. Key FILES are consulted in -// priority order — the configured api_key_file, then the k8s Secret mount -// (DefaultLiteLLMAPIKeyFile), then the dashboard-written PVC file -// (WritableLiteLLMAPIKeyFile) — followed by the env var named by -// api_key_env and finally DefaultLiteLLMAPIKeyEnv. Returns "" when no key -// is configured. The key value itself is never stored in hive.yaml. -// -// Consulting all three file locations means a key saved via the dashboard -// keeps working even if hive.yaml is reset (e.g. re-seeded from a -// ConfigMap) and the api_key_file pointer is lost, and an admin-managed -// Secret key keeps working if the PVC copy is wiped. -func (c *LiteLLMConfig) ResolveAPIKey() string { - key, _ := c.resolveAPIKeyWithSource() - return key -} - -// ResolveAPIKeySource reports where ResolveAPIKey found the key without -// exposing the value: "file:", "env:", or "" when no key is -// configured. Safe to return from APIs (the dashboard shows it as the -// "Key detected" store). -func (c *LiteLLMConfig) ResolveAPIKeySource() string { - _, source := c.resolveAPIKeyWithSource() - return source -} - -func (c *LiteLLMConfig) resolveAPIKeyWithSource() (string, string) { - files := []string{c.APIKeyFile, DefaultLiteLLMAPIKeyFile, WritableLiteLLMAPIKeyFile} - seen := map[string]bool{"": true} - for _, f := range files { - if seen[f] { - continue - } - seen[f] = true - // SECURITY (audit N8): only the managed secrets dirs. The two defaults - // appended above already satisfy this; the gate is for c.APIKeyFile, - // which is operator/API-supplied. - if !SecretFilePathAllowed(f) { - continue - } - if data, err := readSecretFile(f); err == nil { - if key := strings.TrimSpace(string(data)); key != "" { - return key, "file:" + f - } - } - } - if c.APIKeyEnv != "" { - if key := os.Getenv(c.APIKeyEnv); key != "" { - return key, "env:" + c.APIKeyEnv - } - } - if key := os.Getenv(DefaultLiteLLMAPIKeyEnv); key != "" { - return key, "env:" + DefaultLiteLLMAPIKeyEnv - } - return "", "" -} - // ResolveEndpoint returns the effective LiteLLM base URL: the // HIVE_LITELLM_ENDPOINT env var when set, otherwise the YAML endpoint. func (c *LiteLLMConfig) ResolveEndpoint() string { diff --git a/src/crux/config/api_key_resolution.go b/src/crux/config/api_key_resolution.go new file mode 100644 index 0000000000..1a27c15703 --- /dev/null +++ b/src/crux/config/api_key_resolution.go @@ -0,0 +1,153 @@ +package config + +import ( + "os" + "strings" +) + +// API key resolution for gateways and the legacy LiteLLM block, split out of +// agent_runtime.go. Every reader goes through SecretFilePathAllowed (audit N8) +// and reports a log-safe source alongside the key (hive#18532, #18533). + +// ResolveAPIKey returns this gateway's key value, preferring the env var when +// set and falling back to the file. Returns "" when neither yields a value +// (a keyless endpoint). +func (gw GatewayConfig) ResolveAPIKey() string { + key, _ := gw.ResolveAPIKeyWithSource() + return key +} + +// ResolveAPIKeyWithSource is ResolveAPIKey plus WHERE the key came from, as a +// log-safe "env:" or "file:" ("" when no key resolved). When the +// configured api_key_file is outside the allowed secret dirs it is skipped +// (audit N8) and a WARN names the skipped path and the source actually used, +// once per process (hive#18533). +func (gw GatewayConfig) ResolveAPIKeyWithSource() (string, string) { + key, source := gw.resolveAPIKeyFromSources() + if keyFileSkipped(gw.APIKeyFile) { + warnSkippedKeyFileOnce(GatewayAPIKeyFileField(gw.Name), gw.APIKeyFile, source) + } + return key, source +} + +// APIKeyFileProblem returns the operator-facing rejection for an api_key_file +// outside the allowed secret dirs, or "" when it is empty or allowed. +func (gw GatewayConfig) APIKeyFileProblem() string { + return SecretFilePathRejection(GatewayAPIKeyFileField(gw.Name), gw.APIKeyFile) +} + +func (gw GatewayConfig) resolveAPIKeyFromSources() (string, string) { + if gw.APIKeyEnv != "" { + if v := os.Getenv(gw.APIKeyEnv); v != "" { + return v, KeySourceEnvPrefix + gw.APIKeyEnv + } + } + if gw.APIKeyFile != "" { + // SECURITY (audit N8): confine the read to the managed secrets dirs. This + // gate lives HERE, not only in the HTTP handler, so every caller is + // covered — including a path that reached hive.yaml some other way (a + // hand-edited config, an older build, a restored backup). + if !SecretFilePathAllowed(gw.APIKeyFile) { + return "", "" + } + if b, err := readSecretFile(gw.APIKeyFile); err == nil { + if key := strings.TrimSpace(string(b)); key != "" { + return key, KeySourceFilePrefix + gw.APIKeyFile + } + } + } + return "", "" +} + +// ResolveAPIKey returns the LiteLLM API key. Key FILES are consulted in +// priority order — the configured api_key_file, then the k8s Secret mount +// (DefaultLiteLLMAPIKeyFile), then the dashboard-written PVC file +// (WritableLiteLLMAPIKeyFile) — followed by the env var named by +// api_key_env and finally DefaultLiteLLMAPIKeyEnv. Returns "" when no key +// is configured. The key value itself is never stored in hive.yaml. +// +// Consulting all three file locations means a key saved via the dashboard +// keeps working even if hive.yaml is reset (e.g. re-seeded from a +// ConfigMap) and the api_key_file pointer is lost, and an admin-managed +// Secret key keeps working if the PVC copy is wiped. +func (c *LiteLLMConfig) ResolveAPIKey() string { + key, _ := c.resolveAPIKeyWithSource() + return key +} + +// ResolveAPIKeySource reports where ResolveAPIKey found the key without +// exposing the value: "file:", "env:", or "" when no key is +// configured. Safe to return from APIs (the dashboard shows it as the +// "Key detected" store). +func (c *LiteLLMConfig) ResolveAPIKeySource() string { + _, source := c.resolveAPIKeyWithSource() + return source +} + +// ResolveAPIKeyWithSource returns the key and its log-safe source in one +// resolution, so a caller needing both reads the key file once. +func (c *LiteLLMConfig) ResolveAPIKeyWithSource() (string, string) { + return c.resolveAPIKeyWithSource() +} + +func (c *LiteLLMConfig) resolveAPIKeyWithSource() (string, string) { + key, source := c.resolveAPIKeyFromSources() + // hive#18533: a configured api_key_file outside the allowed dirs is skipped + // (audit N8) and resolution falls through to the defaults or the env var. + // Say so, once, naming the path skipped and the source used instead, so an + // operator who mounted a new Secret somewhere else can see why the old key + // is still in use. + if c.ConfiguredKeyFileSkipped() { + warnSkippedKeyFileOnce(LiteLLMAPIKeyFileField, c.APIKeyFile, source) + } + return key, source +} + +// ConfiguredKeyFileSkipped reports whether api_key_file is set but outside the +// allowed secret dirs, so ResolveAPIKey ignores it. +func (c *LiteLLMConfig) ConfiguredKeyFileSkipped() bool { + return keyFileSkipped(c.APIKeyFile) +} + +// APIKeyFileProblem returns the operator-facing rejection for an api_key_file +// outside the allowed secret dirs, or "" when it is empty or allowed. +func (c *LiteLLMConfig) APIKeyFileProblem() string { + return SecretFilePathRejection(LiteLLMAPIKeyFileField, c.APIKeyFile) +} + +// KeySourceDescription is the effective key source in display form, noting a +// skipped configured path. Never contains key material. +func (c *LiteLLMConfig) KeySourceDescription() string { + return DescribeKeySource(c.ResolveAPIKeySource(), c.APIKeyFile) +} + +func (c *LiteLLMConfig) resolveAPIKeyFromSources() (string, string) { + files := []string{c.APIKeyFile, DefaultLiteLLMAPIKeyFile, WritableLiteLLMAPIKeyFile} + seen := map[string]bool{"": true} + for _, f := range files { + if seen[f] { + continue + } + seen[f] = true + // SECURITY (audit N8): only the managed secrets dirs. The two defaults + // appended above already satisfy this; the gate is for c.APIKeyFile, + // which is operator/API-supplied. + if !SecretFilePathAllowed(f) { + continue + } + if data, err := readSecretFile(f); err == nil { + if key := strings.TrimSpace(string(data)); key != "" { + return key, KeySourceFilePrefix + f + } + } + } + if c.APIKeyEnv != "" { + if key := os.Getenv(c.APIKeyEnv); key != "" { + return key, KeySourceEnvPrefix + c.APIKeyEnv + } + } + if key := os.Getenv(DefaultLiteLLMAPIKeyEnv); key != "" { + return key, KeySourceEnvPrefix + DefaultLiteLLMAPIKeyEnv + } + return "", "" +} diff --git a/src/crux/config/secret_file_reporting.go b/src/crux/config/secret_file_reporting.go new file mode 100644 index 0000000000..f2890c809f --- /dev/null +++ b/src/crux/config/secret_file_reporting.go @@ -0,0 +1,115 @@ +package config + +import ( + "fmt" + "log/slog" + "strings" + "sync" +) + +// Key-source prefixes. Every resolver in this package reports where it found a +// key as "" so callers (dashboard, proxy logs) can show the +// source without ever touching the value. +const ( + KeySourceFilePrefix = "file:" + KeySourceEnvPrefix = "env:" +) + +// LiteLLMAPIKeyFileField and gatewayAPIKeyFileFieldFormat name the config +// fields an operator sets, so validation and runtime warnings point at the +// exact hive.yaml key to fix. +const ( + LiteLLMAPIKeyFileField = "governor.litellm.api_key_file" + gatewayAPIKeyFileFieldFormat = "governor.gateways[%s].api_key_file" +) + +// SecretFileRoots returns a copy of the directories an api_key_file may live +// under (see SecretFilePathAllowed). Safe to show to operators: these are +// paths, never key material. +func SecretFileRoots() []string { + return append([]string(nil), secretFileRoots...) +} + +// keyFileSkipped reports whether p is a configured key file the resolvers will +// refuse to read (set, but outside the allowed dirs). +func keyFileSkipped(p string) bool { + p = strings.TrimSpace(p) + return p != "" && !SecretFilePathAllowed(p) +} + +// SecretFilePathRejection returns the operator-facing message for an +// api_key_file outside the allowed secret dirs (hive#18533), or "" when path +// is empty or allowed. It names the field and path, lists the allowed dirs, +// and says how to fix it: mount the Secret under one of them. The audit N8 +// gate itself (SecretFilePathAllowed) is unchanged; this only explains it. +func SecretFilePathRejection(field, path string) string { + path = strings.TrimSpace(path) + if !keyFileSkipped(path) { + return "" + } + return fmt.Sprintf("%s %q is outside the allowed secret dirs (%s), so hive will never read it; "+ + "mount the Kubernetes Secret under one of those dirs (for example a volumeMount at /secrets/) "+ + "and point %s at the file there", + field, path, strings.Join(SecretFileRoots(), ", "), field) +} + +// GatewayAPIKeyFileField names a gateway's api_key_file field for messages. +func GatewayAPIKeyFileField(name string) string { + return fmt.Sprintf(gatewayAPIKeyFileFieldFormat, name) +} + +// DescribeKeySource renders an effective key source for display, noting a +// configured api_key_file that was skipped, for example +// "key from /secrets/litellm_api_key (configured path /secrets-cb/k not allowed)". +// It takes the log-safe source string and never contains key material. +func DescribeKeySource(source, configuredFile string) string { + var desc string + switch { + case source == "": + desc = "no key found" + case strings.HasPrefix(source, KeySourceFilePrefix): + desc = "key from " + strings.TrimPrefix(source, KeySourceFilePrefix) + case strings.HasPrefix(source, KeySourceEnvPrefix): + desc = "key from env " + strings.TrimPrefix(source, KeySourceEnvPrefix) + default: + desc = "key from " + source + } + if keyFileSkipped(configuredFile) { + desc += fmt.Sprintf(" (configured path %s not allowed)", strings.TrimSpace(configuredFile)) + } + return desc +} + +// skippedKeyFileWarned remembers which (field, path) pairs have already been +// warned about, so a skipped api_key_file logs ONE warning per process instead +// of one per resolution (the dashboard and proxy resolve keys constantly). +var skippedKeyFileWarned sync.Map + +// warnSkippedKeyFileOnce logs, once per field and path, that a configured +// api_key_file was skipped and which source was used instead. Returns true +// when this call logged. Never logs key material. +func warnSkippedKeyFileOnce(field, path, effectiveSource string) bool { + path = strings.TrimSpace(path) + if _, loaded := skippedKeyFileWarned.LoadOrStore(field+"\x00"+path, struct{}{}); loaded { + return false + } + effective := effectiveSource + if effective == "" { + effective = "none" + } + slog.Warn("configured api_key_file is outside the allowed secret dirs and was skipped", + "field", field, + "skipped_path", path, + "effective_key_source", effective, + "allowed_dirs", strings.Join(SecretFileRoots(), ","), + "remedy", "mount the Secret under an allowed dir and point "+field+" at the file there") + return true +} + +// resetSkippedKeyFileWarnings clears the warn-once memory (tests only). +func resetSkippedKeyFileWarnings() { + skippedKeyFileWarned.Range(func(k, _ any) bool { + skippedKeyFileWarned.Delete(k) + return true + }) +} diff --git a/src/crux/config/secret_file_reporting_test.go b/src/crux/config/secret_file_reporting_test.go new file mode 100644 index 0000000000..2a3290c034 --- /dev/null +++ b/src/crux/config/secret_file_reporting_test.go @@ -0,0 +1,198 @@ +package config + +import ( + "bytes" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" +) + +// captureSlog points the default slog logger at a buffer for the test. +func captureSlog(t *testing.T) *bytes.Buffer { + t.Helper() + var buf bytes.Buffer + prev := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil))) + t.Cleanup(func() { slog.SetDefault(prev) }) + return &buf +} + +// allowedRoot makes a temp secrets root, registers it as the only allowed +// root, and resets the warn-once memory. +func allowedRoot(t *testing.T) string { + t.Helper() + root := filepath.Join(t.TempDir(), "secrets") + if err := os.MkdirAll(root, 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + t.Cleanup(SetSecretFileRootsForTest(root)) + resetSkippedKeyFileWarnings() + t.Cleanup(resetSkippedKeyFileWarnings) + return root +} + +func writeKey(t *testing.T, path, key string) { + t.Helper() + if err := os.WriteFile(path, []byte(key+"\n"), 0o600); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +func TestSecretFilePathRejection(t *testing.T) { + root := allowedRoot(t) + if got := SecretFilePathRejection("api_key_file", ""); got != "" { + t.Fatalf("empty path rejected: %q", got) + } + if got := SecretFilePathRejection("api_key_file", filepath.Join(root, "k")); got != "" { + t.Fatalf("allowed path rejected: %q", got) + } + outside := "/secrets-cb/litellm_api_key" + msg := SecretFilePathRejection("governor.litellm.api_key_file", outside) + for _, want := range []string{"governor.litellm.api_key_file", outside, root, "mount the Kubernetes Secret"} { + if !strings.Contains(msg, want) { + t.Fatalf("rejection %q does not mention %q", msg, want) + } + } +} + +func TestSecretFileRootsIsACopy(t *testing.T) { + root := allowedRoot(t) + roots := SecretFileRoots() + if len(roots) != 1 || roots[0] != root { + t.Fatalf("SecretFileRoots() = %v, want [%s]", roots, root) + } + roots[0] = "/etc" + if SecretFilePathAllowed("/etc/passwd") { + t.Fatal("mutating the returned slice widened the allowed roots") + } +} + +func TestDescribeKeySource(t *testing.T) { + allowedRoot(t) + cases := []struct{ source, configured, want string }{ + {"", "", "no key found"}, + {"file:/secrets/litellm_api_key", "", "key from /secrets/litellm_api_key"}, + {"env:HIVE_LITELLM_API_KEY", "", "key from env HIVE_LITELLM_API_KEY"}, + {"other", "", "key from other"}, + {"file:/secrets/litellm_api_key", "/secrets-cb/k", "key from /secrets/litellm_api_key (configured path /secrets-cb/k not allowed)"}, + } + for _, c := range cases { + if got := DescribeKeySource(c.source, c.configured); got != c.want { + t.Errorf("DescribeKeySource(%q, %q) = %q, want %q", c.source, c.configured, got, c.want) + } + } +} + +// TestLiteLLMSkippedKeyFileWarnsOnce covers hive#18533 at runtime: the +// configured path outside the allowed dirs is skipped, the fallback source is +// used, and exactly one WARN names both. +func TestLiteLLMSkippedKeyFileWarnsOnce(t *testing.T) { + allowedRoot(t) + logs := captureSlog(t) + outside := filepath.Join(t.TempDir(), "litellm_api_key") + writeKey(t, outside, "new-key-never-read") + t.Setenv(DefaultLiteLLMAPIKeyEnv, "env-fallback-key") + + c := &LiteLLMConfig{APIKeyFile: outside} + if !c.ConfiguredKeyFileSkipped() { + t.Fatal("outside path not reported as skipped") + } + for i := 0; i < 3; i++ { + key, source := c.ResolveAPIKeyWithSource() + if key != "env-fallback-key" || source != KeySourceEnvPrefix+DefaultLiteLLMAPIKeyEnv { + t.Fatalf("resolve = (%q, %q), want the env fallback", key, source) + } + } + out := logs.String() + if n := strings.Count(out, "configured api_key_file is outside the allowed secret dirs"); n != 1 { + t.Fatalf("warning logged %d times, want once:\n%s", n, out) + } + for _, want := range []string{"skipped_path=" + outside, "effective_key_source=env:" + DefaultLiteLLMAPIKeyEnv, LiteLLMAPIKeyFileField} { + if !strings.Contains(out, want) { + t.Fatalf("warning missing %q:\n%s", want, out) + } + } + if strings.Contains(out, "env-fallback-key") || strings.Contains(out, "new-key-never-read") { + t.Fatalf("warning leaked key material:\n%s", out) + } + if got := c.KeySourceDescription(); !strings.Contains(got, "not allowed") { + t.Fatalf("KeySourceDescription() = %q, want the skipped-path note", got) + } + if c.APIKeyFileProblem() == "" { + t.Fatal("APIKeyFileProblem() empty for an outside path") + } +} + +func TestLiteLLMAllowedKeyFileDoesNotWarn(t *testing.T) { + root := allowedRoot(t) + logs := captureSlog(t) + path := filepath.Join(root, "litellm_api_key") + writeKey(t, path, "allowed-key") + c := &LiteLLMConfig{APIKeyFile: path} + key, source := c.ResolveAPIKeyWithSource() + if key != "allowed-key" || source != KeySourceFilePrefix+path { + t.Fatalf("resolve = (%q, %q)", key, source) + } + if c.ConfiguredKeyFileSkipped() || c.APIKeyFileProblem() != "" { + t.Fatal("allowed path reported as a problem") + } + if strings.Contains(logs.String(), "outside the allowed secret dirs") { + t.Fatalf("allowed path warned:\n%s", logs.String()) + } +} + +func TestGatewayResolveAPIKeyWithSource(t *testing.T) { + root := allowedRoot(t) + logs := captureSlog(t) + + path := filepath.Join(root, "gw_key") + writeKey(t, path, "gw-file-key") + if key, source := (GatewayConfig{Name: "gw", APIKeyFile: path}).ResolveAPIKeyWithSource(); key != "gw-file-key" || source != KeySourceFilePrefix+path { + t.Fatalf("file resolve = (%q, %q)", key, source) + } + + t.Setenv("HIVE_TEST_GW_KEY", "gw-env-key") + if key, source := (GatewayConfig{Name: "gw", APIKeyEnv: "HIVE_TEST_GW_KEY", APIKeyFile: path}).ResolveAPIKeyWithSource(); key != "gw-env-key" || source != "env:HIVE_TEST_GW_KEY" { + t.Fatalf("env resolve = (%q, %q)", key, source) + } + + empty := filepath.Join(root, "empty") + writeKey(t, empty, "") + if key, source := (GatewayConfig{Name: "gw", APIKeyFile: empty}).ResolveAPIKeyWithSource(); key != "" || source != "" { + t.Fatalf("empty file resolve = (%q, %q), want nothing", key, source) + } + + outside := filepath.Join(t.TempDir(), "gw_key") + writeKey(t, outside, "outside") + gw := GatewayConfig{Name: "corp", APIKeyFile: outside} + for i := 0; i < 2; i++ { + if key := gw.ResolveAPIKey(); key != "" { + t.Fatalf("outside gateway key read: %q", key) + } + } + if n := strings.Count(logs.String(), "skipped_path="+outside); n != 1 { + t.Fatalf("gateway warning logged %d times, want once:\n%s", n, logs.String()) + } + if !strings.Contains(logs.String(), "effective_key_source=none") || !strings.Contains(logs.String(), GatewayAPIKeyFileField("corp")) { + t.Fatalf("gateway warning missing fields:\n%s", logs.String()) + } + if !strings.Contains(gw.APIKeyFileProblem(), "governor.gateways[corp].api_key_file") { + t.Fatalf("APIKeyFileProblem() = %q", gw.APIKeyFileProblem()) + } +} + +func TestWarnSkippedKeyFileOnceReturnsWhetherItLogged(t *testing.T) { + allowedRoot(t) + captureSlog(t) + if !warnSkippedKeyFileOnce("f", "/x", "") { + t.Fatal("first call did not log") + } + if warnSkippedKeyFileOnce("f", "/x", "") { + t.Fatal("second call logged again") + } + if !warnSkippedKeyFileOnce("f", "/y", "") { + t.Fatal("a different path did not log") + } +} diff --git a/src/crux/proxy/github_proxy_inference.go b/src/crux/proxy/github_proxy_inference.go index 45fd160ea6..1d2f52883f 100644 --- a/src/crux/proxy/github_proxy_inference.go +++ b/src/crux/proxy/github_proxy_inference.go @@ -215,8 +215,10 @@ func (p *GitHubProxy) recordInferenceError(route *InferenceRoute, agentName stri if isInferenceAuthStatus(status) && p.inferenceAuth != nil { msg := inferenceAuthErrorMessage(route.Backend, status, truncateBytes(body, 200)) p.inferenceAuth.recordFailure(msg, time.Now()) + keySource, keyChanged := inferenceKeyLogFields(route) p.logger.Warn("inference backend auth failure", - "agent", agentName, "backend", route.Backend, "endpoint", route.Endpoint, "status", status) + "agent", agentName, "backend", route.Backend, "endpoint", route.Endpoint, "status", status, + "key_source", keySource, "key_changed_since_start", keyChanged) } // PROVIDER SPEND REBUFF (#4294). Checked before the 403/entitlement branch @@ -433,7 +435,7 @@ func (p *GitHubProxy) inferenceTranslatorHandler() http.Handler { http.Error(w, `{"type":"error","error":{"type":"api_error","message":"no inference route for agent"}}`, http.StatusBadGateway) return } - route = p.routeWithEntitledModel(route, agentName) + route = p.routeWithEntitledModel(routeWithCurrentKey(route), agentName) body, err := io.ReadAll(r.Body) if r.Body != nil { @@ -499,6 +501,9 @@ func (p *GitHubProxy) inferenceTranslatorHandler() http.Handler { return } resp, err := client.Do(upstreamReq) + if err == nil { + resp, route = p.retryInferenceOnAuthFailure(r.Context(), client, route, agentName, upstreamURL, openaiBody, resp) + } if err != nil { if errors.Is(err, context.Canceled) && errors.Is(r.Context().Err(), context.Canceled) { p.logger.Debug("inference request canceled", "agent", agentName) @@ -661,7 +666,7 @@ func (p *GitHubProxy) handleAnthropicReroute(conn net.Conn, r *http.Request, hos // handleInferenceRequest translates a single Anthropic API request and // forwards it to the inference backend. func (p *GitHubProxy) handleInferenceRequest(conn net.Conn, req *http.Request, agentName string, route *InferenceRoute) { - route = p.routeWithEntitledModel(route, agentName) + route = p.routeWithEntitledModel(routeWithCurrentKey(route), agentName) _ = conn.SetReadDeadline(time.Now().Add(httpReadTimeout)) body, err := io.ReadAll(req.Body) _ = conn.SetReadDeadline(time.Time{}) @@ -710,6 +715,9 @@ func (p *GitHubProxy) handleInferenceRequest(conn net.Conn, req *http.Request, a return } resp, err := client.Do(upstreamReq) + if err == nil { + resp, route = p.retryInferenceOnAuthFailure(req.Context(), client, route, agentName, upstreamURL, openaiBody, resp) + } if err != nil { p.logger.Error("inference upstream failed", "agent", agentName, "url", upstreamURL, "error", err) if p.gatewayHealth != nil { diff --git a/src/crux/proxy/inference_key_resolver.go b/src/crux/proxy/inference_key_resolver.go new file mode 100644 index 0000000000..8bca467579 --- /dev/null +++ b/src/crux/proxy/inference_key_resolver.go @@ -0,0 +1,242 @@ +package proxy + +import ( + "bytes" + "context" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/crux/config" +) + +// Upstream key refresh (hive#18532). A route used to carry the key resolved +// when the route was installed, so a key rotated in its Secret kept 401ing +// until the hive restarted. A route may now carry an InferenceKeyResolver that +// re-reads the key while the proxy runs. +const ( + // inferenceKeyRecheckTTL bounds how often a cached key's source file is + // re-stat'ed. Requests inside the window reuse the cached key with no + // filesystem access at all. + inferenceKeyRecheckTTL = 3 * time.Second + // inferenceKeyFullRefreshInterval forces a full re-resolve even when the + // source file looks unchanged, so a NEW higher-priority source (an + // api_key_file created after boot, a source switch) is still noticed. + inferenceKeyFullRefreshInterval = 30 * time.Second + // inferenceKeySourceStatic is the key_source logged for a route whose key + // was fixed when the route was set (no resolver, e.g. a minted watsonx + // IAM token). + inferenceKeySourceStatic = "static" + // inferenceKeySourceNone is logged when the resolver found no key. + inferenceKeySourceNone = "none" +) + +// InferenceKeyResolveFunc returns the current upstream key and a log-safe +// description of where it came from ("file:", "env:"). The source +// must never contain key material. +type InferenceKeyResolveFunc func() (key, source string) + +// keyFileStamp identifies one version of a key file. Kubernetes Secret volumes +// rotate by swapping the ..data symlink, which changes the resolved target +// path; a file rewritten in place changes its mtime or size. +type keyFileStamp struct { + target string + modTime time.Time + size int64 + ok bool +} + +// equal reports whether two stamps describe the same file version. +func (s keyFileStamp) equal(o keyFileStamp) bool { + return s.ok == o.ok && s.target == o.target && s.size == o.size && s.modTime.Equal(o.modTime) +} + +func statKeyFile(source string) keyFileStamp { + if !strings.HasPrefix(source, config.KeySourceFilePrefix) { + return keyFileStamp{} + } + path := strings.TrimPrefix(source, config.KeySourceFilePrefix) + target, err := filepath.EvalSymlinks(path) + if err != nil { + return keyFileStamp{} + } + fi, err := os.Stat(target) + if err != nil { + return keyFileStamp{} + } + return keyFileStamp{target: target, modTime: fi.ModTime(), size: fi.Size(), ok: true} +} + +// processFirstKeySHA records, per resolver name (the route's backend), the +// SHA-256 of the first key this process resolved, so the auth-failure log can +// say whether the key changed since start without keeping the key itself. +var processFirstKeySHA sync.Map + +// InferenceKeyResolver caches an upstream key and refreshes it when its source +// changes. Safe for concurrent use; one resolver is shared by every copy of a +// route. +type InferenceKeyResolver struct { + name string + resolve InferenceKeyResolveFunc + now func() time.Time + + mu sync.Mutex + loaded bool + key string + source string + stamp keyFileStamp + checkedAt time.Time + resolvedAt time.Time +} + +// NewInferenceKeyResolver returns a resolver for the route named name (its +// backend), or nil when resolve is nil. +func NewInferenceKeyResolver(name string, resolve InferenceKeyResolveFunc) *InferenceKeyResolver { + if resolve == nil { + return nil + } + return &InferenceKeyResolver{name: name, resolve: resolve, now: time.Now} +} + +// load re-runs the resolve func. Caller holds r.mu. Reports whether the key +// changed. +func (r *InferenceKeyResolver) load(now time.Time) bool { + prev, hadPrev := r.key, r.loaded + r.key, r.source = r.resolve() + r.stamp = statKeyFile(r.source) + r.loaded = true + r.checkedAt, r.resolvedAt = now, now + if sha := config.APIKeySHA256(r.key); sha != "" { + processFirstKeySHA.LoadOrStore(r.name, sha) + } + return hadPrev && prev != r.key +} + +// Current returns the key to send now. It re-stats the key file at most once +// per inferenceKeyRecheckTTL and re-resolves when the file's symlink target, +// mtime or size changed, or when inferenceKeyFullRefreshInterval has passed. +func (r *InferenceKeyResolver) Current() (key, source string) { + r.mu.Lock() + defer r.mu.Unlock() + now := r.now() + if !r.loaded { + r.load(now) + return r.key, r.source + } + if now.Sub(r.checkedAt) < inferenceKeyRecheckTTL { + return r.key, r.source + } + r.checkedAt = now + if now.Sub(r.resolvedAt) >= inferenceKeyFullRefreshInterval || !statKeyFile(r.source).equal(r.stamp) { + r.load(now) + } + return r.key, r.source +} + +// Refresh forces a re-resolve (used after an upstream 401) and reports whether +// the key differs from the one cached before. +func (r *InferenceKeyResolver) Refresh() (key, source string, changed bool) { + r.mu.Lock() + defer r.mu.Unlock() + if !r.loaded { + r.load(r.now()) + return r.key, r.source, false + } + changed = r.load(r.now()) + return r.key, r.source, changed +} + +// Source is the log-safe source of the cached key ("none" when no key). +func (r *InferenceKeyResolver) Source() string { + r.mu.Lock() + defer r.mu.Unlock() + if r.source == "" { + return inferenceKeySourceNone + } + return r.source +} + +// ChangedSinceStart reports whether the cached key differs from the first key +// this process resolved for the same route name. +func (r *InferenceKeyResolver) ChangedSinceStart() bool { + r.mu.Lock() + sha := config.APIKeySHA256(r.key) + r.mu.Unlock() + first, ok := processFirstKeySHA.Load(r.name) + if !ok { + return false + } + firstSHA, _ := first.(string) + return sha != firstSHA +} + +// inferenceKeyLogFields returns the key_source and key_changed_since_start +// values for a route. Never key material. +func inferenceKeyLogFields(route *InferenceRoute) (source string, changedSinceStart bool) { + if route == nil || route.KeyResolver == nil { + return inferenceKeySourceStatic, false + } + return route.KeyResolver.Source(), route.KeyResolver.ChangedSinceStart() +} + +// routeWithCurrentKey returns the route carrying the resolver's current key: a +// COPY when the key differs from the one stored, the route itself otherwise +// (routes are shared and must not be mutated in place). +func routeWithCurrentKey(route *InferenceRoute) *InferenceRoute { + if route == nil || route.KeyResolver == nil { + return route + } + key, _ := route.KeyResolver.Current() + if key == route.APIKey { + return route + } + updated := *route + updated.APIKey = key + return &updated +} + +// inferenceRetryDrainLimit caps how much of a discarded 401 body is read so +// the connection can be reused. +const inferenceRetryDrainLimit = 64 << 10 + +// retryInferenceOnAuthFailure handles an upstream 401 on a route whose key +// comes from a resolver: it forces one re-resolve and, when that yields a +// DIFFERENT key, retries the request once with it (hive#18532). It returns the +// response to use and the route that produced it, so a failure is counted +// (recordInferenceError) only after the retry, against the key actually sent. +// +// An unchanged key is not retried: the same request with the same key would +// 401 again and only double the gateway's failed-auth traffic. If the retry +// cannot be sent, the original 401 is returned untouched. +func (p *GitHubProxy) retryInferenceOnAuthFailure(ctx context.Context, client *http.Client, route *InferenceRoute, agentName, upstreamURL string, body []byte, resp *http.Response) (*http.Response, *InferenceRoute) { + if resp == nil || resp.StatusCode != http.StatusUnauthorized || route == nil || route.KeyResolver == nil { + return resp, route + } + key, source, changed := route.KeyResolver.Refresh() + if !changed { + return resp, route + } + retryRoute := *route + retryRoute.APIKey = key + req, err := http.NewRequestWithContext(ctx, http.MethodPost, upstreamURL, bytes.NewReader(body)) + if err != nil { + return resp, route + } + req.Header.Set("Content-Type", "application/json") + applyInferenceAuth(req, &retryRoute) + p.logger.Info("inference upstream 401; retrying once with the re-resolved key", + "agent", agentName, "backend", route.Backend, "key_source", source) + retryResp, err := client.Do(req) + if err != nil { + p.logger.Warn("inference auth retry failed to send; keeping the original 401", + "agent", agentName, "backend", route.Backend, "error", err) + return resp, route + } + _, _ = io.CopyN(io.Discard, resp.Body, inferenceRetryDrainLimit) + _ = resp.Body.Close() + return retryResp, &retryRoute +} diff --git a/src/crux/proxy/inference_key_resolver_test.go b/src/crux/proxy/inference_key_resolver_test.go new file mode 100644 index 0000000000..b70a68008f --- /dev/null +++ b/src/crux/proxy/inference_key_resolver_test.go @@ -0,0 +1,497 @@ +package proxy + +import ( + "bufio" + "bytes" + "context" + "errors" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +const keyResolverTestBody = `{"model":"claude-cli-alias","max_tokens":8,"messages":[{"role":"user","content":"hi"}]}` + +const keyResolverOKResponse = `{"id":"t","object":"chat.completion","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}]}` + +// fakeClock is a manually advanced clock for the resolver's TTL logic. +type fakeClock struct { + mu sync.Mutex + now time.Time +} + +func (c *fakeClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.now +} + +func (c *fakeClock) Advance(d time.Duration) { + c.mu.Lock() + c.now = c.now.Add(d) + c.mu.Unlock() +} + +// secretVolume mimics a Kubernetes Secret volume: /key is a symlink to +// ..data/key, and ..data is a symlink to a timestamped directory that a +// rotation replaces atomically. +type secretVolume struct { + t *testing.T + dir string + gen int +} + +func newSecretVolume(t *testing.T, key string) *secretVolume { + t.Helper() + v := &secretVolume{t: t, dir: t.TempDir()} + v.rotate(key) + if err := os.Symlink(filepath.Join("..data", "key"), v.path()); err != nil { + t.Fatalf("symlink key: %v", err) + } + return v +} + +func (v *secretVolume) path() string { return filepath.Join(v.dir, "key") } + +func (v *secretVolume) rotate(key string) { + v.t.Helper() + v.gen++ + gen := filepath.Join(v.dir, "..gen"+strings.Repeat("x", v.gen)) + if err := os.MkdirAll(gen, 0o700); err != nil { + v.t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(filepath.Join(gen, "key"), []byte(key), 0o600); err != nil { + v.t.Fatalf("write: %v", err) + } + tmp := filepath.Join(v.dir, "..data_tmp") + _ = os.Remove(tmp) + if err := os.Symlink(filepath.Base(gen), tmp); err != nil { + v.t.Fatalf("symlink: %v", err) + } + if err := os.Rename(tmp, filepath.Join(v.dir, "..data")); err != nil { + v.t.Fatalf("swap ..data: %v", err) + } +} + +// fileResolver resolves a key from a file, counting resolutions. +func fileResolver(t *testing.T, name, path string, clock *fakeClock, calls *atomic.Int32) *InferenceKeyResolver { + t.Helper() + r := NewInferenceKeyResolver(name, func() (string, string) { + calls.Add(1) + b, err := os.ReadFile(path) + if err != nil { + return "", "" + } + return strings.TrimSpace(string(b)), "file:" + path + }) + r.now = clock.Now + return r +} + +func TestNewInferenceKeyResolver_NilFunc(t *testing.T) { + if NewInferenceKeyResolver("x", nil) != nil { + t.Fatal("nil resolve func must yield a nil resolver") + } +} + +// TestInferenceKeyResolver_PicksUpSecretSwap is the hive#18532 core: a Secret +// rotation (..data symlink swap) is picked up once the recheck TTL passes, +// without a restart, and requests inside the TTL touch nothing. +func TestInferenceKeyResolver_PicksUpSecretSwap(t *testing.T) { + vol := newSecretVolume(t, "key-one") + clock := &fakeClock{now: time.Unix(1_000_000, 0)} + var calls atomic.Int32 + r := fileResolver(t, t.Name(), vol.path(), clock, &calls) + + if key, src := r.Current(); key != "key-one" || src != "file:"+vol.path() { + t.Fatalf("Current() = (%q, %q)", key, src) + } + vol.rotate("key-two") + + clock.Advance(inferenceKeyRecheckTTL / 2) + if key, _ := r.Current(); key != "key-one" { + t.Fatalf("inside the TTL the cached key must be reused, got %q", key) + } + clock.Advance(inferenceKeyRecheckTTL) + if key, _ := r.Current(); key != "key-two" { + t.Fatalf("after the TTL the swapped key must be picked up, got %q", key) + } + if !r.ChangedSinceStart() { + t.Fatal("key changed since start but ChangedSinceStart() is false") + } + + // Unchanged file past the TTL: stat only, no re-resolve. + before := calls.Load() + clock.Advance(inferenceKeyRecheckTTL) + r.Current() + if calls.Load() != before { + t.Fatal("an unchanged file was re-resolved before the full refresh interval") + } + // Full refresh interval: re-resolve even though nothing changed. + clock.Advance(inferenceKeyFullRefreshInterval) + r.Current() + if calls.Load() != before+1 { + t.Fatal("the full refresh interval did not force a re-resolve") + } +} + +func TestInferenceKeyResolver_RefreshAndSource(t *testing.T) { + clock := &fakeClock{now: time.Unix(2_000_000, 0)} + var key atomic.Value + key.Store("") + r := NewInferenceKeyResolver(t.Name(), func() (string, string) { + k, _ := key.Load().(string) + if k == "" { + return "", "" + } + return k, "env:HIVE_TEST" + }) + r.now = clock.Now + + if _, _, changed := r.Refresh(); changed { + t.Fatal("the first Refresh has nothing to compare against") + } + if r.Source() != inferenceKeySourceNone { + t.Fatalf("Source() = %q, want %q", r.Source(), inferenceKeySourceNone) + } + if r.ChangedSinceStart() { + t.Fatal("no key ever resolved, so nothing changed") + } + key.Store("a") + if k, src, changed := r.Refresh(); k != "a" || src != "env:HIVE_TEST" || !changed { + t.Fatalf("Refresh() = (%q, %q, %v)", k, src, changed) + } + if r.ChangedSinceStart() { + t.Fatal("the first key seen IS the start key") + } + if _, _, changed := r.Refresh(); changed { + t.Fatal("an unchanged key reported as changed") + } + key.Store("b") + if _, _, changed := r.Refresh(); !changed || !r.ChangedSinceStart() { + t.Fatal("a new key must report changed and changed-since-start") + } +} + +func TestStatKeyFile(t *testing.T) { + if (statKeyFile("env:X") != keyFileStamp{}) { + t.Fatal("a non-file source has no stamp") + } + if statKeyFile("file:" + filepath.Join(t.TempDir(), "missing")).ok { + t.Fatal("a missing file has no stamp") + } + vol := newSecretVolume(t, "k") + a := statKeyFile("file:" + vol.path()) + if !a.ok || !a.equal(statKeyFile("file:"+vol.path())) { + t.Fatal("the same file version must stamp equal") + } + vol.rotate("k") + if a.equal(statKeyFile("file:" + vol.path())) { + t.Fatal("a ..data swap must change the stamp even when the content is the same") + } +} + +func TestInferenceKeyLogFieldsAndRouteWithCurrentKey(t *testing.T) { + if src, changed := inferenceKeyLogFields(nil); src != inferenceKeySourceStatic || changed { + t.Fatal("a nil route logs as static") + } + static := &InferenceRoute{APIKey: "k"} + if routeWithCurrentKey(static) != static || routeWithCurrentKey(nil) != nil { + t.Fatal("a route without a resolver is returned as is") + } + r := NewInferenceKeyResolver(t.Name(), func() (string, string) { return "fresh", "env:X" }) + route := &InferenceRoute{APIKey: "stale", KeyResolver: r} + got := routeWithCurrentKey(route) + if got == route || got.APIKey != "fresh" || route.APIKey != "stale" { + t.Fatal("a changed key must come back on a COPY, leaving the shared route untouched") + } + if routeWithCurrentKey(got) != got { + t.Fatal("an up-to-date route must not be copied") + } + if src, _ := inferenceKeyLogFields(route); src != "env:X" { + t.Fatalf("key_source = %q", src) + } +} + +// keyedUpstream is a gateway that accepts only the bearer keys in accept. +type keyedUpstream struct { + srv *httptest.Server + hits atomic.Int32 + mu sync.Mutex + seen []string + accept map[string]bool +} + +func newKeyedUpstream(t *testing.T, accept ...string) *keyedUpstream { + t.Helper() + u := &keyedUpstream{accept: map[string]bool{}} + for _, k := range accept { + u.accept["Bearer "+k] = true + } + u.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.hits.Add(1) + auth := r.Header.Get("Authorization") + u.mu.Lock() + u.seen = append(u.seen, auth) + u.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + if !u.accept[auth] { + w.WriteHeader(http.StatusUnauthorized) + _, _ = io.WriteString(w, `{"error":{"message":"Authentication Error, Invalid proxy server token passed"}}`) + return + } + _, _ = io.WriteString(w, keyResolverOKResponse) + })) + t.Cleanup(u.srv.Close) + return u +} + +func (u *keyedUpstream) lastAuth() string { + u.mu.Lock() + defer u.mu.Unlock() + if len(u.seen) == 0 { + return "" + } + return u.seen[len(u.seen)-1] +} + +func newKeyResolverTestProxy(logs *bytes.Buffer) *GitHubProxy { + p := newInferenceTestProxy() + p.inferenceLastErrors = newInferenceLastErrorState() + p.inferenceAuth = &inferenceAuthState{} + p.logger = slog.New(slog.NewTextHandler(logs, nil)) + return p +} + +func (p *GitHubProxy) authFailures() int { + p.inferenceAuth.mu.Lock() + defer p.inferenceAuth.mu.Unlock() + return p.inferenceAuth.consecutive +} + +func serveTranslator(t *testing.T, p *GitHubProxy, route *InferenceRoute) int { + t.Helper() + p.inference.Set("scanner", route) + req := httptest.NewRequest(http.MethodPost, "/v1/messages", strings.NewReader(keyResolverTestBody)) + req.Header.Set("x-api-key", "sk-hive-scanner") + w := httptest.NewRecorder() + p.inferenceTranslatorHandler().ServeHTTP(w, req) + return w.Code +} + +func serveMITM(t *testing.T, p *GitHubProxy, route *InferenceRoute) int { + t.Helper() + client, server := net.Pipe() + done := make(chan struct{}) + go func() { + defer close(done) + defer server.Close() + req, _ := http.NewRequest(http.MethodPost, "https://api.anthropic.com/v1/messages", strings.NewReader(keyResolverTestBody)) + p.handleInferenceRequest(server, req, "scanner", route) + }() + resp, err := http.ReadResponse(bufio.NewReader(client), nil) + if err != nil { + t.Fatal(err) + } + _, _ = io.ReadAll(resp.Body) + _ = resp.Body.Close() + _ = client.Close() + <-done + return resp.StatusCode +} + +// TestInferenceProxy_KeySwappedAtRuntime: the next request after a Secret +// rotation carries the new key, with no restart and no route re-install. +func TestInferenceProxy_KeySwappedAtRuntime(t *testing.T) { + up := newKeyedUpstream(t, "key-one", "key-two") + var logs bytes.Buffer + p := newKeyResolverTestProxy(&logs) + vol := newSecretVolume(t, "key-one") + clock := &fakeClock{now: time.Unix(3_000_000, 0)} + var calls atomic.Int32 + r := fileResolver(t, t.Name(), vol.path(), clock, &calls) + route := &InferenceRoute{Backend: "litellm", Endpoint: up.srv.URL, Model: "m", APIKey: "key-one", KeyResolver: r} + + if code := serveTranslator(t, p, route); code != http.StatusOK || up.lastAuth() != "Bearer key-one" { + t.Fatalf("first request: status %d auth %q", code, up.lastAuth()) + } + vol.rotate("key-two") + clock.Advance(inferenceKeyRecheckTTL) + if code := serveTranslator(t, p, route); code != http.StatusOK || up.lastAuth() != "Bearer key-two" { + t.Fatalf("after the swap: status %d auth %q, want the new key", code, up.lastAuth()) + } + if code := serveMITM(t, p, route); code != http.StatusOK || up.lastAuth() != "Bearer key-two" { + t.Fatalf("MITM path: status %d auth %q, want the new key", code, up.lastAuth()) + } + if route.APIKey != "key-one" { + t.Fatal("the shared route was mutated in place") + } +} + +// TestInferenceProxy_401ReResolvesAndRetries: a key rotated inside the recheck +// TTL 401s once, is re-resolved, and the single retry succeeds with no auth +// failure counted. Both entry points. +func TestInferenceProxy_401ReResolvesAndRetries(t *testing.T) { + for _, entry := range []string{"translator", "mitm"} { + t.Run(entry, func(t *testing.T) { + up := newKeyedUpstream(t, "new-key") + var logs bytes.Buffer + p := newKeyResolverTestProxy(&logs) + var key atomic.Value + key.Store("old-key") + r := NewInferenceKeyResolver(t.Name(), func() (string, string) { + k, _ := key.Load().(string) + return k, "file:/secrets/litellm_api_key" + }) + r.Current() // cache the old key + key.Store("new-key") + route := &InferenceRoute{Backend: "litellm", Endpoint: up.srv.URL, Model: "m", APIKey: "old-key", KeyResolver: r} + + serve := serveTranslator + if entry == "mitm" { + serve = serveMITM + } + if code := serve(t, p, route); code != http.StatusOK { + t.Fatalf("status %d, want the retry's 200\n%s", code, logs.String()) + } + if up.hits.Load() != 2 || up.lastAuth() != "Bearer new-key" { + t.Fatalf("hits %d last auth %q, want one retry with the new key", up.hits.Load(), up.lastAuth()) + } + if p.authFailures() != 0 { + t.Fatal("a 401 that the retry recovered must not count as an auth failure") + } + if strings.Contains(logs.String(), "new-key") || strings.Contains(logs.String(), "old-key") { + t.Fatalf("logs leaked key material:\n%s", logs.String()) + } + }) + } +} + +// TestInferenceProxy_Second401CountsFailure: when the re-resolved key is ALSO +// rejected, exactly one auth failure is counted and the WARN carries +// key_source and key_changed_since_start. +func TestInferenceProxy_Second401CountsFailure(t *testing.T) { + up := newKeyedUpstream(t) // accepts nothing + var logs bytes.Buffer + p := newKeyResolverTestProxy(&logs) + var key atomic.Value + key.Store("first-key") + r := NewInferenceKeyResolver(t.Name(), func() (string, string) { + k, _ := key.Load().(string) + return k, "file:/secrets/litellm_api_key" + }) + r.Current() + key.Store("second-key") + route := &InferenceRoute{Backend: "litellm", Endpoint: up.srv.URL, Model: "m", APIKey: "first-key", KeyResolver: r} + + if code := serveTranslator(t, p, route); code != http.StatusUnauthorized { + t.Fatalf("status %d, want 401", code) + } + if up.hits.Load() != 2 { + t.Fatalf("hits %d, want the original plus one retry", up.hits.Load()) + } + if p.authFailures() != 1 { + t.Fatalf("auth failures %d, want exactly 1", p.authFailures()) + } + out := logs.String() + for _, want := range []string{"inference backend auth failure", "key_source=file:/secrets/litellm_api_key", "key_changed_since_start=true"} { + if !strings.Contains(out, want) { + t.Fatalf("log missing %q:\n%s", want, out) + } + } + if strings.Contains(out, "first-key") || strings.Contains(out, "second-key") { + t.Fatalf("logs leaked key material:\n%s", out) + } +} + +// TestInferenceProxy_401UnchangedKeyIsNotRetried: re-resolving to the SAME +// key does not resend the request; the failure counts once. +func TestInferenceProxy_401UnchangedKeyIsNotRetried(t *testing.T) { + up := newKeyedUpstream(t) + var logs bytes.Buffer + p := newKeyResolverTestProxy(&logs) + r := NewInferenceKeyResolver(t.Name(), func() (string, string) { return "same-key", "env:HIVE_LITELLM_API_KEY" }) + route := &InferenceRoute{Backend: "litellm", Endpoint: up.srv.URL, Model: "m", APIKey: "same-key", KeyResolver: r} + + if code := serveMITM(t, p, route); code != http.StatusUnauthorized { + t.Fatalf("status %d, want 401", code) + } + if up.hits.Load() != 1 || p.authFailures() != 1 { + t.Fatalf("hits %d failures %d, want 1 and 1", up.hits.Load(), p.authFailures()) + } + if !strings.Contains(logs.String(), "key_changed_since_start=false") { + t.Fatalf("log missing key_changed_since_start=false:\n%s", logs.String()) + } +} + +// TestInferenceProxy_StaticRouteLogsStaticSource: a route with no resolver +// keeps today's behaviour and logs key_source=static. +func TestInferenceProxy_StaticRouteLogsStaticSource(t *testing.T) { + up := newKeyedUpstream(t) + var logs bytes.Buffer + p := newKeyResolverTestProxy(&logs) + route := &InferenceRoute{Backend: "litellm", Endpoint: up.srv.URL, Model: "m", APIKey: "k"} + if code := serveTranslator(t, p, route); code != http.StatusUnauthorized || up.hits.Load() != 1 { + t.Fatalf("status %d hits %d", code, up.hits.Load()) + } + if !strings.Contains(logs.String(), "key_source=static") { + t.Fatalf("log missing key_source=static:\n%s", logs.String()) + } +} + +type failingRoundTripper struct{} + +func (failingRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, errors.New("dial refused") +} + +// TestRetryInferenceOnAuthFailure_KeepsOriginalWhenRetryFails: if the retry +// cannot be sent, the original 401 is returned so it is counted normally. +func TestRetryInferenceOnAuthFailure_KeepsOriginalWhenRetryFails(t *testing.T) { + var logs bytes.Buffer + p := newKeyResolverTestProxy(&logs) + var key atomic.Value + key.Store("a") + r := NewInferenceKeyResolver(t.Name(), func() (string, string) { + k, _ := key.Load().(string) + return k, "env:X" + }) + r.Current() + key.Store("b") + route := &InferenceRoute{Backend: "litellm", APIKey: "a", KeyResolver: r} + orig := &http.Response{StatusCode: http.StatusUnauthorized, Body: io.NopCloser(strings.NewReader("no"))} + client := &http.Client{Transport: failingRoundTripper{}} + + resp, used := p.retryInferenceOnAuthFailure(context.Background(), client, route, "scanner", "http://gw.invalid/v1/chat/completions", nil, orig) + if resp != orig || used != route { + t.Fatal("a failed retry must hand back the original response and route") + } + if !strings.Contains(logs.String(), "inference auth retry failed to send") { + t.Fatalf("failed retry not logged:\n%s", logs.String()) + } + + // A bad URL cannot even build the retry request. + key.Store("c") + if resp, _ := p.retryInferenceOnAuthFailure(context.Background(), client, route, "scanner", "://bad", nil, orig); resp != orig { + t.Fatal("an unbuildable retry must hand back the original response") + } + + // Non-401s and resolver-less routes pass straight through. + ok := &http.Response{StatusCode: http.StatusOK} + if resp, _ := p.retryInferenceOnAuthFailure(context.Background(), client, route, "scanner", "", nil, ok); resp != ok { + t.Fatal("a 200 must pass through") + } + static := &InferenceRoute{APIKey: "a"} + if resp, used := p.retryInferenceOnAuthFailure(context.Background(), client, static, "scanner", "", nil, orig); resp != orig || used != static { + t.Fatal("a resolver-less route must pass through") + } +} diff --git a/src/crux/proxy/inference_route.go b/src/crux/proxy/inference_route.go index 654d49cf2d..57afc861c3 100644 --- a/src/crux/proxy/inference_route.go +++ b/src/crux/proxy/inference_route.go @@ -37,6 +37,12 @@ type InferenceRoute struct { // client would not send it). Empty for all other backends. Header VALUES // here are identifiers, not secrets (the bearer stays in APIKey). ExtraHeaders map[string]string + // KeyResolver, when set, supplies the CURRENT upstream key at request time + // and replaces APIKey on each request, so a key rotated in its Secret is + // picked up without a restart; an upstream 401 forces one re-resolve and + // retry (hive#18532). nil keeps APIKey fixed (vllm/llm-d, watsonx tokens). + // Shared by every copy of the route. + KeyResolver *InferenceKeyResolver } // DefaultInferencePreamble is injected at the top of the system prompt when diff --git a/src/docs/agent-configuration.md b/src/docs/agent-configuration.md index e730be44ed..66357b4ca4 100644 --- a/src/docs/agent-configuration.md +++ b/src/docs/agent-configuration.md @@ -698,6 +698,21 @@ Set `auto_memory: true` on a single agent to let it keep a private notebook whil Every Model Gateway (and the bob backend) also accepts an optional `key_name` — a human-chosen LABEL for the configured key, e.g. `key_name: openrouter-prod-key`. It is safe-to-show metadata, not a secret: the dashboard's gateway row displays it as "Using key: ``", or "(unnamed)" when no label is set, so operators can tell keys apart without ever seeing the value. For gateway keys, the row and settings/status APIs also expose `keySHA256`, the lowercase SHA-256 of the exact effective key string Hive will inject through the inference proxy; this lets operators compare Hive with LiteLLM/OpenRouter key-hash displays without revealing the secret. Saving a replacement key refreshes live inference proxy routes for running agents, so the next kick uses the new key without an agent restart. See [`inference-backends.md`](../../docs/inference-backends.md) for a full YAML example. +#### Where a gateway key file may live + +`api_key_file` for `governor.litellm` and for every entry under `governor.gateways` is only ever read from the allowed secret dirs: `/secrets` (the Kubernetes Secret projection) and `/data/secrets` (the PVC dir the dashboard writes UI-entered keys to). Any other path is never read, a security gate that stops a config value from turning into an arbitrary file read. To use a Secret, mount it under one of those dirs, for example a `volumeMount` at `/secrets/litellm_api_key`, and point `api_key_file` at the file there. + +- **Save.** The dashboard and the config API reject an `api_key_file` outside those dirs with a `400` that lists the allowed dirs and says where to mount the Secret ([#18533](https://github.com/hivecommons/hive/issues/18533)). +- **Boot.** An existing `hive.yaml` with such a path still boots, because the path was already being ignored and the hive has been running on its fallback key. Boot logs a `WARNING:` line for each one naming the field, the path, the allowed dirs and the fix. +- **Runtime.** When a configured path is skipped, the hive logs one `WARN` per path, `configured api_key_file is outside the allowed secret dirs and was skipped`, with `skipped_path` and `effective_key_source` (the source actually used: the Secret mount, the PVC copy, or an env var). +- **Dashboard and API.** The LiteLLM section of `GET /api/config/governor` and each gateway row carry `keySource` (`file:` or `env:`), `keySourceDescription` (for example `key from /secrets/litellm_api_key (configured path /secrets-cb/litellm_api_key not allowed)`) and `apiKeyFileAllowed`; the LiteLLM section also lists `allowedKeyFileDirs`. None of them carries key material. + +#### Key rotation without a restart + +The agent inference proxy (`127.0.0.1:18444`) no longer keeps the key it resolved when an agent's route was set ([#18532](https://github.com/hivecommons/hive/issues/18532)). For `litellm` and named-gateway routes it re-checks the key file every few seconds, and re-reads it when the file's symlink target, mtime or size changed (a Kubernetes Secret update swaps the `..data` symlink). It also re-resolves fully every 30 seconds, so a newly created key file or a changed env var is noticed too. A rotated key therefore reaches the next request without a hive restart. + +When the gateway answers `401`, the proxy re-resolves the key once. If that yields a different key, it retries the request once with it, and counts an auth failure only when the retry is also rejected. An unchanged key is not retried. The `inference backend auth failure` WARN carries `key_source` (`file:`, `env:`, `static` for a route with a fixed credential such as a minted watsonx IAM token, or `none`) and `key_changed_since_start` (whether the key differs from the first key this process resolved for that backend). It never logs the key. + ### Claude Managed Agents hosted execution Claude Managed Agents is an external hosted engine, not a `backend:` value. diff --git a/src/pkg/config/api_key_file_validation.go b/src/pkg/config/api_key_file_validation.go new file mode 100644 index 0000000000..bf660ae07b --- /dev/null +++ b/src/pkg/config/api_key_file_validation.go @@ -0,0 +1,25 @@ +package config + +// APIKeyFileWarnings returns one operator-facing message for every litellm or +// gateway api_key_file outside the allowed secret dirs (hive#18533). Such a +// path is never read (audit N8), and key resolution silently falls back to the +// Secret mount, the PVC copy or an env var, so the operator gets a key other +// than the one they pointed at. +// +// Boot WARNS rather than failing: an existing hive.yaml with such a path has +// been running on its fallback key, and refusing to boot it would take a +// working spoke down over a setting that was already being ignored. The +// dashboard and API save paths REJECT a new out-of-root path with the same +// message (see pkg/dashboard). +func (g GovernorConfig) APIKeyFileWarnings() []string { + var out []string + if msg := g.LiteLLM.APIKeyFileProblem(); msg != "" { + out = append(out, msg) + } + for _, gw := range g.Gateways { + if msg := gw.APIKeyFileProblem(); msg != "" { + out = append(out, msg) + } + } + return out +} diff --git a/src/pkg/config/api_key_file_validation_test.go b/src/pkg/config/api_key_file_validation_test.go new file mode 100644 index 0000000000..341d92b2e7 --- /dev/null +++ b/src/pkg/config/api_key_file_validation_test.go @@ -0,0 +1,89 @@ +package config + +import ( + "bytes" + "log" + "os" + "path/filepath" + "strings" + "testing" +) + +func keyFileTestRoot(t *testing.T) string { + t.Helper() + root := filepath.Join(t.TempDir(), "secrets") + if err := os.MkdirAll(root, 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + t.Cleanup(SetSecretFileRootsForTest(root)) + return root +} + +// TestAPIKeyFileWarnings covers hive#18533 boot validation: every litellm and +// gateway api_key_file outside the allowed dirs yields one message naming the +// field, the path, the allowed dirs and the mount remedy. +func TestAPIKeyFileWarnings(t *testing.T) { + root := keyFileTestRoot(t) + g := GovernorConfig{} + g.LiteLLM.APIKeyFile = "/secrets-cb/litellm_api_key" + g.Gateways = []GatewayConfig{ + {Name: "ok", APIKeyFile: filepath.Join(root, "gw")}, + {Name: "corp", APIKeyFile: "/etc/corp/key"}, + {Name: "envonly", APIKeyEnv: "X"}, + } + warnings := g.APIKeyFileWarnings() + if len(warnings) != 2 { + t.Fatalf("warnings = %q, want 2", warnings) + } + if !strings.Contains(warnings[0], "governor.litellm.api_key_file") || !strings.Contains(warnings[0], "/secrets-cb/litellm_api_key") { + t.Fatalf("litellm warning = %q", warnings[0]) + } + if !strings.Contains(warnings[1], "governor.gateways[corp].api_key_file") || !strings.Contains(warnings[1], root) { + t.Fatalf("gateway warning = %q", warnings[1]) + } + if got := (GovernorConfig{}).APIKeyFileWarnings(); len(got) != 0 { + t.Fatalf("empty config warned: %q", got) + } +} + +// TestValidateWarnsOnOutsideAPIKeyFile: boot validation WARNS, it does not +// fail, on an outside path (see APIKeyFileWarnings for why). +func TestValidateWarnsOnOutsideAPIKeyFile(t *testing.T) { + keyFileTestRoot(t) + var buf bytes.Buffer + prev := log.Writer() + log.SetOutput(&buf) + t.Cleanup(func() { log.SetOutput(prev) }) + + cfg := validatableConfig(map[string]AgentConfig{}) + cfg.Governor.LiteLLM.APIKeyFile = "/secrets-cb/litellm_api_key" + if err := cfg.ValidateWithOptions(ValidateOptions{}); err != nil { + t.Fatalf("an outside api_key_file must warn, not fail boot: %v", err) + } + if !strings.Contains(buf.String(), "WARNING: governor.litellm.api_key_file \"/secrets-cb/litellm_api_key\" is outside the allowed secret dirs") { + t.Fatalf("boot warning missing:\n%s", buf.String()) + } +} + +func TestResolveLiteLLMInferenceKeyWithSource(t *testing.T) { + root := keyFileTestRoot(t) + gwKey := filepath.Join(root, "gw_key") + if err := os.WriteFile(gwKey, []byte("gw-key"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv(DefaultLiteLLMAPIKeyEnv, "legacy-env-key") + + legacy := GovernorConfig{} + if key, src := legacy.ResolveLiteLLMInferenceKeyWithSource("litellm"); key != "legacy-env-key" || src != "env:"+DefaultLiteLLMAPIKeyEnv { + t.Fatalf("legacy = (%q, %q)", key, src) + } + if legacy.ResolveLiteLLMInferenceKey("litellm") != "legacy-env-key" { + t.Fatal("ResolveLiteLLMInferenceKey disagrees with its WithSource twin") + } + + withGateway := GovernorConfig{} + withGateway.Gateways = []GatewayConfig{{Name: "litellm", Kind: GatewayKindLiteLLM, APIKeyFile: gwKey}} + if key, src := withGateway.ResolveLiteLLMInferenceKeyWithSource("litellm"); key != "gw-key" || src != "file:"+gwKey { + t.Fatalf("gateway = (%q, %q)", key, src) + } +} diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index e570d0e703..172da59fe9 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -2858,12 +2858,21 @@ func (g *GovernorConfig) AttributionTrailerEnabled() bool { // backend is the agent's backend name (typically "litellm"); it selects which // explicit gateway to consult. func (g GovernorConfig) ResolveLiteLLMInferenceKey(backend string) string { + key, _ := g.ResolveLiteLLMInferenceKeyWithSource(backend) + return key +} + +// ResolveLiteLLMInferenceKeyWithSource is ResolveLiteLLMInferenceKey plus the +// log-safe source ("file:" / "env:") the key came from. The +// inference proxy re-runs it at request time so a rotated key is picked up +// without a restart (hive#18532). +func (g GovernorConfig) ResolveLiteLLMInferenceKeyWithSource(backend string) (string, string) { if len(g.Gateways) > 0 { if gw := g.ResolveGateway(backend); gw != nil { - return gw.ResolveAPIKey() + return gw.ResolveAPIKeyWithSource() } } - return g.LiteLLM.ResolveAPIKey() + return g.LiteLLM.ResolveAPIKeyWithSource() } // ReplanConfig governs the Phase 3 stall-replan lane: a periodic check that diff --git a/src/pkg/config/crux_aliases.go b/src/pkg/config/crux_aliases.go index c735259c6f..96b1271e6a 100644 --- a/src/pkg/config/crux_aliases.go +++ b/src/pkg/config/crux_aliases.go @@ -181,6 +181,9 @@ var ( QualifyRepo = cruxconfig.QualifyRepo ReviewHeadKey = cruxconfig.ReviewHeadKey SecretFilePathAllowed = cruxconfig.SecretFilePathAllowed + SecretFilePathRejection = cruxconfig.SecretFilePathRejection + SecretFileRoots = cruxconfig.SecretFileRoots + DescribeKeySource = cruxconfig.DescribeKeySource SetSecretFileRootsForTest = cruxconfig.SetSecretFileRootsForTest SupportedBackends = cruxconfig.SupportedBackends ValidEffort = cruxconfig.ValidEffort diff --git a/src/pkg/config/validate.go b/src/pkg/config/validate.go index 6ecaceddf6..20fe80b1e8 100644 --- a/src/pkg/config/validate.go +++ b/src/pkg/config/validate.go @@ -84,6 +84,11 @@ func (c *Config) ValidateWithOptions(opts ValidateOptions) error { if err := c.Governor.LiteLLM.Validate(); err != nil { return err } + // hive#18533: warn loudly (not fail) on an api_key_file that will never be + // read. See GovernorConfig.APIKeyFileWarnings for why boot does not fail. + for _, w := range c.Governor.APIKeyFileWarnings() { + log.Printf("WARNING: %s", w) + } if err := c.GitHub.Attention.Validate(); err != nil { return err } diff --git a/src/pkg/dashboard/api_governor.go b/src/pkg/dashboard/api_governor.go index f2bf29dff2..71d79a79d1 100644 --- a/src/pkg/dashboard/api_governor.go +++ b/src/pkg/dashboard/api_governor.go @@ -2187,7 +2187,7 @@ func litellmSectionResponse(lc *config.LiteLLMConfig) map[string]interface{} { if apiKeyFileLooksLikeKey { apiKeyFile = maskSecretHint(apiKeyFile) } - key := lc.ResolveAPIKey() + key, keySource := lc.ResolveAPIKeyWithSource() keySHA256 := "" if key != "" { keySHA256 = config.APIKeySHA256(key) @@ -2205,7 +2205,13 @@ func litellmSectionResponse(lc *config.LiteLLMConfig) map[string]interface{} { "keySHA256": keySHA256, // redactSecret guards the pathological case of a key-like env var // NAME appearing in the source string. - "keySource": redactSecret(lc.ResolveAPIKeySource(), key), + "keySource": redactSecret(keySource, key), + // hive#18533: the effective source in display form, noting a configured + // api_key_file that is outside the allowed dirs and therefore ignored, + // plus the allowed dirs so the tab can say where to mount the Secret. + "keySourceDescription": redactSecret(config.DescribeKeySource(keySource, lc.APIKeyFile), key), + "apiKeyFileAllowed": !lc.ConfiguredKeyFileSkipped(), + "allowedKeyFileDirs": config.SecretFileRoots(), } } @@ -2380,9 +2386,8 @@ func (s *Server) handleGovernorLiteLLM(w http.ResponseWriter, r *http.Request) { // upsert applies. This is the LiteLLM twin of that handler and carries // the identical defect — the guardrail above short-circuits for ANY // absolute path, so an arbitrary file could be stored and later read. - if keyFile != "" && !config.SecretFilePathAllowed(keyFile) { - jsonError(w, "api_key_file must be under /secrets or "+config.WritableSecretsDir, - http.StatusBadRequest) + if msg := config.SecretFilePathRejection("api_key_file", keyFile); msg != "" { + jsonError(w, msg, http.StatusBadRequest) return } lc.APIKeyFile = keyFile diff --git a/src/pkg/dashboard/api_governor_inference_auth.go b/src/pkg/dashboard/api_governor_inference_auth.go index 245eec55a8..35c3b0c510 100644 --- a/src/pkg/dashboard/api_governor_inference_auth.go +++ b/src/pkg/dashboard/api_governor_inference_auth.go @@ -68,8 +68,8 @@ func applyInferenceAuthBody(cur config.InferenceAuthConfig, body *inferenceAuthB } // SECURITY (audit N8, CWE-200/918): same confinement as the gateway // and LiteLLM handlers — reject an out-of-root path at the door. - if keyFile != "" && !config.SecretFilePathAllowed(keyFile) { - return cur, "api_key_file must be under /secrets or " + config.WritableSecretsDir + if msg := config.SecretFilePathRejection("api_key_file", keyFile); msg != "" { + return cur, msg } cur.APIKeyFile = keyFile } diff --git a/src/pkg/dashboard/api_key_file_source_test.go b/src/pkg/dashboard/api_key_file_source_test.go new file mode 100644 index 0000000000..ef7c0d9f3b --- /dev/null +++ b/src/pkg/dashboard/api_key_file_source_test.go @@ -0,0 +1,141 @@ +package dashboard + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/config" +) + +// hive#18533: every save path that takes an api_key_file rejects one outside +// the allowed secret dirs with a message that lists the dirs and says to mount +// the Secret there. +func assertKeyFileRejection(t *testing.T, code int, body string) { + t.Helper() + if code != http.StatusBadRequest { + t.Fatalf("code = %d, want 400 (body: %s)", code, body) + } + for _, want := range []string{"outside the allowed secret dirs", "mount the Kubernetes Secret"} { + if !strings.Contains(body, want) { + t.Fatalf("rejection %q does not contain %q", body, want) + } + } + for _, root := range config.SecretFileRoots() { + if !strings.Contains(body, root) { + t.Fatalf("rejection %q does not list allowed dir %q", body, root) + } + } +} + +func TestHandleGovernorLiteLLM_RejectsOutsideKeyFile(t *testing.T) { + srv := newFullServer(t) + body := `{"endpoint": "https://litellm.example.com", "apiKeyFile": "/secrets-cb/litellm_api_key"}` + req := httptest.NewRequest("PUT", "/api/config/governor/litellm", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + markOwnerRequest(req) + srv.handleGovernorLiteLLM(w, req) + assertKeyFileRejection(t, w.Code, w.Body.String()) + if srv.deps.Config.Governor.LiteLLM.APIKeyFile != "" { + t.Fatal("rejected path was stored") + } +} + +func TestGatewaysUpsert_RejectsOutsideKeyFile(t *testing.T) { + srv := newFullServer(t) + body := `{"name":"corp","kind":"litellm","endpoint":"https://openrouter.ai/api/v1","api_key_file":"/secrets-cb/gw_key"}` + req := httptest.NewRequest("PUT", "/api/config/governor/gateways", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + markOwnerRequest(req) + srv.handleGovernorGatewaysUpsert(w, req) + assertKeyFileRejection(t, w.Code, w.Body.String()) + if len(srv.deps.Config.Governor.Gateways) != 0 { + t.Fatal("config mutated on a rejected path") + } +} + +func TestInferenceAuth_RejectsOutsideKeyFileWithRemedy(t *testing.T) { + s := covApiServer(t) + rec := doPut(s, "/api/config/governor/inference-auth", map[string]any{ + "vllm": map[string]any{"api_key_file": "/secrets-cb/vllm_key"}, + }) + assertKeyFileRejection(t, rec.Code, rec.Body.String()) +} + +// TestLiteLLMSectionResponse_EffectiveKeySource: the governor config API shows +// the source actually in use and flags the ignored configured path. Never the +// key itself. +func TestLiteLLMSectionResponse_EffectiveKeySource(t *testing.T) { + root := filepath.Join(t.TempDir(), "secrets") + if err := os.MkdirAll(root, 0o700); err != nil { + t.Fatal(err) + } + t.Cleanup(config.SetSecretFileRootsForTest(root)) + t.Setenv(config.DefaultLiteLLMAPIKeyEnv, "sk-effective-env-key") + + lc := &config.LiteLLMConfig{APIKeyFile: "/secrets-cb/litellm_api_key"} + resp := litellmSectionResponse(lc) + if resp["keySource"] != "env:"+config.DefaultLiteLLMAPIKeyEnv { + t.Fatalf("keySource = %v", resp["keySource"]) + } + desc, _ := resp["keySourceDescription"].(string) + if desc != "key from env "+config.DefaultLiteLLMAPIKeyEnv+" (configured path /secrets-cb/litellm_api_key not allowed)" { + t.Fatalf("keySourceDescription = %q", desc) + } + if resp["apiKeyFileAllowed"] != false { + t.Fatalf("apiKeyFileAllowed = %v, want false", resp["apiKeyFileAllowed"]) + } + dirs, _ := resp["allowedKeyFileDirs"].([]string) + if len(dirs) != 1 || dirs[0] != root { + t.Fatalf("allowedKeyFileDirs = %v", resp["allowedKeyFileDirs"]) + } + for k, v := range resp { + if s, ok := v.(string); ok && strings.Contains(s, "sk-effective-env-key") { + t.Fatalf("field %q leaked the key", k) + } + } + + good := filepath.Join(root, "litellm_api_key") + if err := os.WriteFile(good, []byte("sk-file-key"), 0o600); err != nil { + t.Fatal(err) + } + resp = litellmSectionResponse(&config.LiteLLMConfig{APIKeyFile: good}) + if resp["keySource"] != "file:"+good || resp["apiKeyFileAllowed"] != true || resp["keySourceDescription"] != "key from "+good { + t.Fatalf("allowed file response = %v", resp) + } +} + +func TestGatewaySectionResponse_EffectiveKeySource(t *testing.T) { + root := filepath.Join(t.TempDir(), "secrets") + if err := os.MkdirAll(root, 0o700); err != nil { + t.Fatal(err) + } + t.Cleanup(config.SetSecretFileRootsForTest(root)) + good := filepath.Join(root, "gw_key") + if err := os.WriteFile(good, []byte("sk-gw-key"), 0o600); err != nil { + t.Fatal(err) + } + + resp := gatewaySectionResponse(config.GatewayConfig{Name: "ok", APIKeyFile: good}) + if resp["keySource"] != "file:"+good || resp["apiKeyFileAllowed"] != true { + t.Fatalf("allowed gateway response = %v", resp) + } + + resp = gatewaySectionResponse(config.GatewayConfig{Name: "corp", APIKeyFile: "/secrets-cb/gw_key"}) + if resp["keySource"] != "" || resp["apiKeyFileAllowed"] != false { + t.Fatalf("outside gateway response = %v", resp) + } + if resp["keySourceDescription"] != "no key found (configured path /secrets-cb/gw_key not allowed)" { + t.Fatalf("keySourceDescription = %v", resp["keySourceDescription"]) + } + + resp = gatewaySectionResponse(config.GatewayConfig{Name: "envonly"}) + if resp["apiKeyFileAllowed"] != true { + t.Fatal("a gateway with no api_key_file is not a problem") + } +} diff --git a/src/pkg/dashboard/gateways.go b/src/pkg/dashboard/gateways.go index 55285fc32c..f04ad90337 100644 --- a/src/pkg/dashboard/gateways.go +++ b/src/pkg/dashboard/gateways.go @@ -55,7 +55,7 @@ var validGatewayKinds = map[string]bool{ // struct (both are references, safe to expose), plus a hasKey flag and a masked // tail hint resolved from those references. func gatewaySectionResponse(gw config.GatewayConfig) map[string]interface{} { - key := gw.ResolveAPIKey() + key, keySource := gw.ResolveAPIKeyWithSource() keySHA256 := "" if key != "" { keySHA256 = config.APIKeySHA256(key) @@ -80,6 +80,11 @@ func gatewaySectionResponse(gw config.GatewayConfig) map[string]interface{} { // a name (the dashboard renders that as "(unnamed)"), so no // backwards-compat break. "keyName": gw.KeyName, + // hive#18533: where the key actually came from (never the value), and + // whether the configured api_key_file is one hive will read. + "keySource": redactSecret(keySource, key), + "keySourceDescription": redactSecret(config.DescribeKeySource(keySource, gw.APIKeyFile), key), + "apiKeyFileAllowed": strings.TrimSpace(gw.APIKeyFile) == "" || config.SecretFilePathAllowed(gw.APIKeyFile), } } @@ -211,8 +216,8 @@ func (s *Server) handleGovernorGatewaysUpsert(w http.ResponseWriter, r *http.Req // path, which is precisely why any absolute path used to be accepted and // stored verbatim. ResolveAPIKey enforces the same confinement at read // time for paths that reach hive.yaml by other routes. - if keyFile != "" && !config.SecretFilePathAllowed(keyFile) { - jsonError(w, "api_key_file must be under /secrets or "+config.WritableSecretsDir, http.StatusBadRequest) + if msg := config.SecretFilePathRejection("api_key_file", keyFile); msg != "" { + jsonError(w, msg, http.StatusBadRequest) return } } diff --git a/src/pkg/dashboard/static-src/js/config-tabs.js b/src/pkg/dashboard/static-src/js/config-tabs.js index c51d344fed..adaa5b2159 100644 --- a/src/pkg/dashboard/static-src/js/config-tabs.js +++ b/src/pkg/dashboard/static-src/js/config-tabs.js @@ -2900,9 +2900,13 @@ const keyEnvWarning = litellm.apiKeyEnvLooksLikeKey ? `
The saved value here (${esc(litellm.apiKeyEnv || '')}) looks like a key VALUE, not an environment variable name — move it to the API Key field above and Save; storing a key clears this field automatically.
` : ''; + // hive#18533: a configured path outside the allowed secret dirs is never + // read; say so and name the key source actually in use. const keyFileWarning = litellm.apiKeyFileLooksLikeKey ? `
The saved value here (${esc(litellm.apiKeyFile || '')}) looks like a key VALUE, not a file path — move it to the API Key field above and Save.
` - : ''; + : (litellm.apiKeyFileAllowed === false + ? `
This path is outside the allowed secret dirs (${esc((litellm.allowedKeyFileDirs || []).join(', '))}) and is ignored: ${esc(litellm.keySourceDescription || '')}. Mount the Secret under an allowed dir and point this field at the file there.
` + : ''); const keyEnvValue = litellm.apiKeyEnvLooksLikeKey ? '' : (litellm.apiKeyEnv || ''); const keyFileValue = litellm.apiKeyFileLooksLikeKey ? '' : (litellm.apiKeyFile || ''); return ` diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 33ba10bdba..97fa706040 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -64201,9 +64201,13 @@

Configure LiteLLM

const keyEnvWarning = litellm.apiKeyEnvLooksLikeKey ? `
The saved value here (${esc(litellm.apiKeyEnv || '')}) looks like a key VALUE, not an environment variable name — move it to the API Key field above and Save; storing a key clears this field automatically.
` : ''; + // hive#18533: a configured path outside the allowed secret dirs is never + // read; say so and name the key source actually in use. const keyFileWarning = litellm.apiKeyFileLooksLikeKey ? `
The saved value here (${esc(litellm.apiKeyFile || '')}) looks like a key VALUE, not a file path — move it to the API Key field above and Save.
` - : ''; + : (litellm.apiKeyFileAllowed === false + ? `
This path is outside the allowed secret dirs (${esc((litellm.allowedKeyFileDirs || []).join(', '))}) and is ignored: ${esc(litellm.keySourceDescription || '')}. Mount the Secret under an allowed dir and point this field at the file there.
` + : ''); const keyEnvValue = litellm.apiKeyEnvLooksLikeKey ? '' : (litellm.apiKeyEnv || ''); const keyFileValue = litellm.apiKeyFileLooksLikeKey ? '' : (litellm.apiKeyFile || ''); return `