diff --git a/changelog.d/added-19057-translator-timeouts-retry.md b/changelog.d/added-19057-translator-timeouts-retry.md new file mode 100644 index 0000000000..4f00be6b63 --- /dev/null +++ b/changelog.d/added-19057-translator-timeouts-retry.md @@ -0,0 +1 @@ +- The inference translator can now cut a stalled upstream and retry a rate limit or server error, both opt-in and off by default. New `governor.inference_resilience` (or `HIVE_INFERENCE_*` environment variables) sets a connect timeout, a response-header timeout and an idle-stream timeout (no bytes for N seconds, so a slow but active stream is never cut), and a bounded retry with exponential backoff, jitter and `Retry-After` for `429` and `5xx` that only runs while no response byte has reached the agent, with a small attempt cap. Each retry and timeout logs the route and status (never content or headers) and increments `crux.proxy.inference.retries` or `crux.proxy.inference.timeouts`. A stream that dies mid-response is still logged and stopped, and key resolution and the single `401` retry are unchanged. Nothing changes while the settings are at their defaults ([#19057](https://github.com/hivecommons/hive/issues/19057)). diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 37029483f7..def018f610 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -4253,6 +4253,18 @@ func (b *boot) bootProxyWith(deps bootProxyDeps) { // leak handler. canaryLeakHandler := b.handleCanaryLeak + // Opt-in translator timeouts and bounded 429/5xx retry (#19057). The + // all-off default leaves the translator unchanged; settings are read once + // here, so a change needs a restart. + inferenceResilience, ignoredResilienceEnv := b.cfg.Governor.InferenceResilience.Resolve(os.Getenv) + proxy.SetInferenceResilience(inferenceResilience) + if len(ignoredResilienceEnv) > 0 { + b.logger.Warn("inference resilience env ignored (invalid value)", "vars", ignoredResilienceEnv) + } + if inferenceResilience.Enabled() { + b.logger.Info("inference resilience enabled", "settings", inferenceResilience.Summary()) + } + b.githubProxy, err = deps.newGitHubProxy(b.logger, b.cfg.Project.Org, b.cfg.Project.Repos) if err != nil { b.logger.Error("failed to create github proxy", "error", err) diff --git a/src/crux/config/agent_config.go b/src/crux/config/agent_config.go index 44f54c03ce..66d3a7cab3 100644 --- a/src/crux/config/agent_config.go +++ b/src/crux/config/agent_config.go @@ -194,6 +194,11 @@ type AgentRuntimeGovernorConfig struct { // (#4294) — the gateway declining to spend more money, as distinct from the // hive's own token Budget above. See ProviderBudgetConfig. ProviderBudget ProviderBudgetConfig `yaml:"provider_budget,omitempty" json:"provider_budget,omitempty"` + // InferenceResilience holds the opt-in timeouts and bounded 429/5xx retry + // of the inference translator (hivecommons/hive#19057). A nil block (the + // default) leaves the translator unchanged; it is a pointer so an absent + // block is omitted from yaml and json output. + InferenceResilience *InferenceResilienceConfig `yaml:"inference_resilience,omitempty" json:"inference_resilience,omitempty"` } type AgentConfig struct { diff --git a/src/crux/config/inference_resilience.go b/src/crux/config/inference_resilience.go new file mode 100644 index 0000000000..9f7b5ac91a --- /dev/null +++ b/src/crux/config/inference_resilience.go @@ -0,0 +1,182 @@ +package config + +import ( + "fmt" + "strconv" + "strings" + "time" +) + +// Inference translator resilience (hivecommons/hive#19057). +// +// The translator that carries every agent's model calls to a self-hosted +// gateway has no overall timeout (a stream can run for minutes) and does not +// retry. This block adds three opt-in timeouts and an opt-in bounded retry. +// Every default below is "off": an absent or empty `governor.inference_resilience` +// block leaves the translator byte-for-byte as it was. +const ( + // DefaultInferenceConnectTimeoutS is the default TCP connect timeout in + // seconds. 0 = off (the transport keeps its own dial timeout). + DefaultInferenceConnectTimeoutS = 0 + // DefaultInferenceResponseHeaderTimeoutS is the default time in seconds the + // upstream may take to start answering. 0 = off. + DefaultInferenceResponseHeaderTimeoutS = 0 + // DefaultInferenceIdleStreamTimeoutS is the default number of seconds a + // response body may go without a single byte before it is cut. 0 = off. + DefaultInferenceIdleStreamTimeoutS = 0 + // DefaultInferenceRetryEnabled is the default for the 429/5xx retry. Off. + DefaultInferenceRetryEnabled = false + // DefaultInferenceRetryMaxRetries is the number of retries after the first + // attempt when retry is enabled and no value is set. + DefaultInferenceRetryMaxRetries = 2 + // MaxInferenceRetryMaxRetries is the hard ceiling on retries; a larger + // configured value is clamped to it. + MaxInferenceRetryMaxRetries = 5 + // DefaultInferenceRetryBaseDelayMS is the first backoff step in + // milliseconds; each further retry doubles it. + DefaultInferenceRetryBaseDelayMS = 500 + // DefaultInferenceRetryMaxDelayS caps a single wait in seconds, and is the + // longest Retry-After the translator will honor. A larger Retry-After skips + // the retry rather than waiting less than the upstream asked for. + DefaultInferenceRetryMaxDelayS = 20 + // MaxInferenceTimeoutS bounds any configured timeout or delay in seconds. + MaxInferenceTimeoutS = 3600 +) + +// Environment overrides. An env var that holds a valid value wins over the +// yaml key; an unset or invalid one is ignored (and reported by Resolve). +const ( + EnvInferenceConnectTimeoutS = "HIVE_INFERENCE_CONNECT_TIMEOUT_S" + EnvInferenceResponseHeaderTimeoutS = "HIVE_INFERENCE_RESPONSE_HEADER_TIMEOUT_S" + EnvInferenceIdleStreamTimeoutS = "HIVE_INFERENCE_IDLE_STREAM_TIMEOUT_S" + EnvInferenceRetryEnabled = "HIVE_INFERENCE_RETRY_ENABLED" + EnvInferenceRetryMaxRetries = "HIVE_INFERENCE_RETRY_MAX_RETRIES" + EnvInferenceRetryBaseDelayMS = "HIVE_INFERENCE_RETRY_BASE_DELAY_MS" + EnvInferenceRetryMaxDelayS = "HIVE_INFERENCE_RETRY_MAX_DELAY_S" +) + +// InferenceResilienceConfig is `governor.inference_resilience:`. All fields are +// optional; zero means "default", and every default is off. +type InferenceResilienceConfig struct { + // ConnectTimeoutS bounds the TCP connect to the gateway, in seconds. + ConnectTimeoutS int `yaml:"connect_timeout_s,omitempty" json:"connect_timeout_s,omitempty"` + // ResponseHeaderTimeoutS bounds the wait for the upstream's response + // headers, in seconds. A non-streaming call only gets headers once the model + // has finished, so set it above the longest healthy generation. + ResponseHeaderTimeoutS int `yaml:"response_header_timeout_s,omitempty" json:"response_header_timeout_s,omitempty"` + // IdleStreamTimeoutS cuts a response body that delivers no bytes for this + // many seconds. A slow stream that keeps producing bytes is never cut. + IdleStreamTimeoutS int `yaml:"idle_stream_timeout_s,omitempty" json:"idle_stream_timeout_s,omitempty"` + // RetryEnabled turns on the bounded retry for 429 and 5xx answers, which + // only ever happens before any response byte reaches the agent. + RetryEnabled bool `yaml:"retry_enabled,omitempty" json:"retry_enabled,omitempty"` + // RetryMaxRetries is how many retries follow the first attempt (default 2, + // at most 5). + RetryMaxRetries int `yaml:"retry_max_retries,omitempty" json:"retry_max_retries,omitempty"` + // RetryBaseDelayMS is the first backoff step in milliseconds (default 500). + RetryBaseDelayMS int `yaml:"retry_base_delay_ms,omitempty" json:"retry_base_delay_ms,omitempty"` + // RetryMaxDelayS caps one wait and the honored Retry-After, in seconds + // (default 20). + RetryMaxDelayS int `yaml:"retry_max_delay_s,omitempty" json:"retry_max_delay_s,omitempty"` +} + +// InferenceResilience is the resolved, ready-to-use form of the config: plain +// durations, with the defaults and the env overrides already applied. +type InferenceResilience struct { + ConnectTimeout time.Duration + ResponseHeaderTimeout time.Duration + IdleStreamTimeout time.Duration + RetryEnabled bool + MaxRetries int + BaseDelay time.Duration + MaxDelay time.Duration +} + +// TimeoutsEnabled reports whether any of the three timeouts is on. +func (r InferenceResilience) TimeoutsEnabled() bool { + return r.ConnectTimeout > 0 || r.ResponseHeaderTimeout > 0 || r.IdleStreamTimeout > 0 +} + +// Enabled reports whether anything differs from the unchanged translator. +func (r InferenceResilience) Enabled() bool { + return r.TimeoutsEnabled() || r.RetryEnabled +} + +// Resolve applies the defaults and the env overrides to c. A nil receiver is +// the all-off default. getenv is os.Getenv in production. The second result +// lists env vars that were set but ignored (invalid or out of range); it never +// carries their values. +func (c *InferenceResilienceConfig) Resolve(getenv func(string) string) (InferenceResilience, []string) { + var cfg InferenceResilienceConfig + if c != nil { + cfg = *c + } + var ignored []string + pickInt := func(env string, yamlVal int) int { + raw := strings.TrimSpace(getenv(env)) + if raw == "" { + return yamlVal + } + v, err := strconv.Atoi(raw) + if err != nil || v < 0 { + ignored = append(ignored, env) + return yamlVal + } + return v + } + connect := clampInferenceSeconds(pickInt(EnvInferenceConnectTimeoutS, cfg.ConnectTimeoutS)) + header := clampInferenceSeconds(pickInt(EnvInferenceResponseHeaderTimeoutS, cfg.ResponseHeaderTimeoutS)) + idle := clampInferenceSeconds(pickInt(EnvInferenceIdleStreamTimeoutS, cfg.IdleStreamTimeoutS)) + retries := pickInt(EnvInferenceRetryMaxRetries, cfg.RetryMaxRetries) + baseMS := pickInt(EnvInferenceRetryBaseDelayMS, cfg.RetryBaseDelayMS) + maxDelayS := clampInferenceSeconds(pickInt(EnvInferenceRetryMaxDelayS, cfg.RetryMaxDelayS)) + + enabled := cfg.RetryEnabled + if raw := strings.TrimSpace(getenv(EnvInferenceRetryEnabled)); raw != "" { + v, err := strconv.ParseBool(raw) + if err != nil { + ignored = append(ignored, EnvInferenceRetryEnabled) + } else { + enabled = v + } + } + + if retries == 0 { + retries = DefaultInferenceRetryMaxRetries + } + if retries > MaxInferenceRetryMaxRetries { + retries = MaxInferenceRetryMaxRetries + } + if baseMS == 0 { + baseMS = DefaultInferenceRetryBaseDelayMS + } + if maxBaseMS := MaxInferenceTimeoutS * int(time.Second/time.Millisecond); baseMS > maxBaseMS { + baseMS = maxBaseMS + } + if maxDelayS == 0 { + maxDelayS = DefaultInferenceRetryMaxDelayS + } + return InferenceResilience{ + ConnectTimeout: time.Duration(connect) * time.Second, + ResponseHeaderTimeout: time.Duration(header) * time.Second, + IdleStreamTimeout: time.Duration(idle) * time.Second, + RetryEnabled: enabled, + MaxRetries: retries, + BaseDelay: time.Duration(baseMS) * time.Millisecond, + MaxDelay: time.Duration(maxDelayS) * time.Second, + }, ignored +} + +func clampInferenceSeconds(v int) int { + if v > MaxInferenceTimeoutS { + return MaxInferenceTimeoutS + } + return v +} + +// Summary is a log-safe one-line description (numbers only). +func (r InferenceResilience) Summary() string { + return fmt.Sprintf("connect=%s header=%s idle=%s retry=%t max_retries=%d base=%s max_delay=%s", + r.ConnectTimeout, r.ResponseHeaderTimeout, r.IdleStreamTimeout, + r.RetryEnabled, r.MaxRetries, r.BaseDelay, r.MaxDelay) +} diff --git a/src/crux/config/inference_resilience_test.go b/src/crux/config/inference_resilience_test.go new file mode 100644 index 0000000000..d522664aa2 --- /dev/null +++ b/src/crux/config/inference_resilience_test.go @@ -0,0 +1,72 @@ +package config + +import ( + "testing" + "time" +) + +func noEnv(string) string { return "" } + +func TestInferenceResilienceResolve_DefaultsAreOff(t *testing.T) { + for _, c := range []*InferenceResilienceConfig{nil, {}} { + got, ignored := c.Resolve(noEnv) + if got.Enabled() || got.TimeoutsEnabled() || got.RetryEnabled || len(ignored) != 0 { + t.Fatalf("default must be off, got %+v ignored %v", got, ignored) + } + if got.MaxRetries != DefaultInferenceRetryMaxRetries || + got.BaseDelay != DefaultInferenceRetryBaseDelayMS*time.Millisecond || + got.MaxDelay != DefaultInferenceRetryMaxDelayS*time.Second { + t.Fatalf("retry defaults not applied: %+v", got) + } + } + if DefaultInferenceConnectTimeoutS != 0 || DefaultInferenceResponseHeaderTimeoutS != 0 || + DefaultInferenceIdleStreamTimeoutS != 0 || DefaultInferenceRetryEnabled { + t.Fatal("every default must stay off") + } +} + +func TestInferenceResilienceResolve_YAMLEnvAndClamps(t *testing.T) { + c := &InferenceResilienceConfig{ + ConnectTimeoutS: 3, ResponseHeaderTimeoutS: 30, IdleStreamTimeoutS: 60, + RetryEnabled: true, RetryMaxRetries: 99, RetryBaseDelayMS: 250, RetryMaxDelayS: 10, + } + got, _ := c.Resolve(noEnv) + if got.ConnectTimeout != 3*time.Second || got.ResponseHeaderTimeout != 30*time.Second || + got.IdleStreamTimeout != 60*time.Second || !got.RetryEnabled || + got.MaxRetries != MaxInferenceRetryMaxRetries || + got.BaseDelay != 250*time.Millisecond || got.MaxDelay != 10*time.Second { + t.Fatalf("yaml not resolved or not clamped: %+v", got) + } + + env := map[string]string{ + EnvInferenceConnectTimeoutS: "5", + EnvInferenceResponseHeaderTimeoutS: "bad", + EnvInferenceIdleStreamTimeoutS: "-1", + EnvInferenceRetryEnabled: "false", + EnvInferenceRetryMaxRetries: "1", + EnvInferenceRetryBaseDelayMS: "100", + EnvInferenceRetryMaxDelayS: "999999", + } + got, ignored := c.Resolve(func(k string) string { return env[k] }) + if got.ConnectTimeout != 5*time.Second || got.ResponseHeaderTimeout != 30*time.Second || + got.IdleStreamTimeout != 60*time.Second || got.RetryEnabled || got.MaxRetries != 1 || + got.BaseDelay != 100*time.Millisecond || got.MaxDelay != MaxInferenceTimeoutS*time.Second { + t.Fatalf("env override wrong: %+v", got) + } + if len(ignored) != 2 { + t.Fatalf("want the two invalid vars reported, got %v", ignored) + } + + onlyEnv, _ := (*InferenceResilienceConfig)(nil).Resolve(func(k string) string { + if k == EnvInferenceRetryEnabled { + return "true" + } + return "" + }) + if !onlyEnv.RetryEnabled || !onlyEnv.Enabled() { + t.Fatal("env alone must be able to enable retry") + } + if onlyEnv.Summary() == "" { + t.Fatal("summary must not be empty") + } +} diff --git a/src/crux/proxy/github_proxy_inference.go b/src/crux/proxy/github_proxy_inference.go index 1d2f52883f..11e8c860be 100644 --- a/src/crux/proxy/github_proxy_inference.go +++ b/src/crux/proxy/github_proxy_inference.go @@ -247,91 +247,6 @@ func (p *GitHubProxy) recordInferenceError(route *InferenceRoute, agentName stri "agent", agentName, "endpoint", route.Endpoint, "count", len(entitled), "model", route.Model) } -// routeWithEntitledModel returns the route to actually forward with: when the -// gateway's entitled model set is known (key-info probe or a prior team-scope -// 403) and the route's stored model differs from an entitled id only by -// provider-prefix / dot-hyphen spelling drift, a COPY of the route carrying -// the exact entitled id is returned. LiteLLM matches the request model string -// verbatim, so without this a stored copilot-spelled id (e.g. -// "claude-sonnet-4.6" vs entitled "aws/claude-sonnet-4-6") 403s on every call -// forever (#4400) — while the dashboard's tolerant preselect matcher shows the -// agent on a perfectly valid model. Self-healing: the first 403 teaches the -// entitled set (recordInferenceError), so the next request resolves. -// -// When the stored model is NOT entitled at all — verbatim or by drift — and -// the backend's DefaultModel IS, the request goes out for the default instead -// (hivecommons/hive#9945). The FMA hive's agents carried a `model: -// claude-sonnet-4-6` written for their original copilot backend into a -// switch to the litellm gateway whose default_model was the only model the -// team could use; the per-agent value won, and every call 403'd forever with -// "team not allowed to access model" while the operator stared at a gateway -// card showing the right default. The gateway's own entitlement signal (the -// key-info probe or that very 403) is what authorises the substitution, so a -// model the team can't use is never sent when one it can use is configured. -// The stored route is still untouched: a key/team change that re-entitles the -// per-agent model takes effect on the next request. -// -// The stored route is never mutated: the rewrite is per-request, so a -// key/team change that re-entitles the verbatim id takes effect immediately, -// and concurrent requests race on nothing. Unknown entitlements, verbatim -// matches, and ambiguous candidates all pass the route through unchanged. -func (p *GitHubProxy) routeWithEntitledModel(route *InferenceRoute, agentName string) *InferenceRoute { - // A blank per-agent selection inherits the configured backend default, - // even before an entitlement probe has completed. Keep the stored snapshot - // untouched, as with entitlement rewrites below. - if route != nil && strings.TrimSpace(route.Model) == "" && strings.TrimSpace(route.DefaultModel) != "" { - rewritten := *route - rewritten.Model = route.DefaultModel - route = &rewritten - } - if route == nil || route.Backend != "litellm" || p.entitlements == nil { - return route - } - entitled, _, known := p.EntitledModels(route.Endpoint) - if !known { - return route - } - if entitledModelListed(entitled, route.Model) { - return route - } - if resolved, ok := resolveEntitledModelID(entitled, route.Model); ok { - p.logger.Info("inference model resolved to entitled gateway id", - "agent", agentName, "endpoint", route.Endpoint, - "configured", route.Model, "resolved", resolved) - rewritten := *route - rewritten.Model = resolved - return &rewritten - } - fallback := strings.TrimSpace(route.DefaultModel) - if fallback == "" || fallback == route.Model { - return route - } - if !entitledModelListed(entitled, fallback) { - resolved, ok := resolveEntitledModelID(entitled, fallback) - if !ok { - return route - } - fallback = resolved - } - p.logger.Warn("inference model not entitled on gateway; sending the backend default model instead", - "agent", agentName, "backend", route.Backend, "endpoint", route.Endpoint, - "configured", route.Model, "default", fallback, - "remedy", "clear the agent's model (or pick one of the gateway's entitled models) so the agent uses the gateway default") - rewritten := *route - rewritten.Model = fallback - return &rewritten -} - -// entitledModelListed reports whether model appears verbatim in entitled. -func entitledModelListed(entitled []string, model string) bool { - for _, id := range entitled { - if id == model { - return true - } - } - return false -} - // recordInferenceSuccess records a successful inference call so a previously // latched auth-failure signal clears — the self-heal for the inference-auth // health signal. Called from every inference forward path on a 2xx response. @@ -502,6 +417,7 @@ func (p *GitHubProxy) inferenceTranslatorHandler() http.Handler { } resp, err := client.Do(upstreamReq) if err == nil { + resp = retryInferenceOnOverload(r.Context(), p.logger, client, route, agentName, upstreamURL, openaiBody, resp) resp, route = p.retryInferenceOnAuthFailure(r.Context(), client, route, agentName, upstreamURL, openaiBody, resp) } if err != nil { @@ -509,6 +425,7 @@ func (p *GitHubProxy) inferenceTranslatorHandler() http.Handler { p.logger.Debug("inference request canceled", "agent", agentName) return } + noteInferenceSendError(p.logger, agentName, route, err) p.logger.Error("inference upstream failed", "agent", agentName, "url", upstreamURL, "error", err) if p.gatewayHealth != nil { p.gatewayHealth.RecordEndpointError(route.Backend, route.Endpoint, err, time.Now()) @@ -518,6 +435,7 @@ func (p *GitHubProxy) inferenceTranslatorHandler() http.Handler { http.Error(w, fmt.Sprintf(`{"type":"error","error":{"type":"api_error","message":"%s"}}`, jsonEscape(unreachableMsg)), http.StatusBadGateway) return } + applyInferenceIdleTimeout(p.logger, resp, agentName, route) defer func() { _ = resp.Body.Close() }() p.logger.Info("inference upstream response", @@ -716,9 +634,11 @@ func (p *GitHubProxy) handleInferenceRequest(conn net.Conn, req *http.Request, a } resp, err := client.Do(upstreamReq) if err == nil { + resp = retryInferenceOnOverload(req.Context(), p.logger, client, route, agentName, upstreamURL, openaiBody, resp) resp, route = p.retryInferenceOnAuthFailure(req.Context(), client, route, agentName, upstreamURL, openaiBody, resp) } if err != nil { + noteInferenceSendError(p.logger, agentName, route, err) p.logger.Error("inference upstream failed", "agent", agentName, "url", upstreamURL, "error", err) if p.gatewayHealth != nil { p.gatewayHealth.RecordEndpointError(route.Backend, route.Endpoint, err, time.Now()) @@ -728,6 +648,7 @@ func (p *GitHubProxy) handleInferenceRequest(conn net.Conn, req *http.Request, a p.writeHTTPError(conn, http.StatusBadGateway, unreachableMsg) return } + applyInferenceIdleTimeout(p.logger, resp, agentName, route) defer func() { _ = resp.Body.Close() }() if resp.StatusCode < 200 || resp.StatusCode >= 300 { diff --git a/src/crux/proxy/inference_model_route.go b/src/crux/proxy/inference_model_route.go new file mode 100644 index 0000000000..976c07d668 --- /dev/null +++ b/src/crux/proxy/inference_model_route.go @@ -0,0 +1,88 @@ +package proxy + +import "strings" + +// routeWithEntitledModel returns the route to actually forward with: when the +// gateway's entitled model set is known (key-info probe or a prior team-scope +// 403) and the route's stored model differs from an entitled id only by +// provider-prefix / dot-hyphen spelling drift, a COPY of the route carrying +// the exact entitled id is returned. LiteLLM matches the request model string +// verbatim, so without this a stored copilot-spelled id (e.g. +// "claude-sonnet-4.6" vs entitled "aws/claude-sonnet-4-6") 403s on every call +// forever (#4400) — while the dashboard's tolerant preselect matcher shows the +// agent on a perfectly valid model. Self-healing: the first 403 teaches the +// entitled set (recordInferenceError), so the next request resolves. +// +// When the stored model is NOT entitled at all — verbatim or by drift — and +// the backend's DefaultModel IS, the request goes out for the default instead +// (hivecommons/hive#9945). The FMA hive's agents carried a `model: +// claude-sonnet-4-6` written for their original copilot backend into a +// switch to the litellm gateway whose default_model was the only model the +// team could use; the per-agent value won, and every call 403'd forever with +// "team not allowed to access model" while the operator stared at a gateway +// card showing the right default. The gateway's own entitlement signal (the +// key-info probe or that very 403) is what authorises the substitution, so a +// model the team can't use is never sent when one it can use is configured. +// The stored route is still untouched: a key/team change that re-entitles the +// per-agent model takes effect on the next request. +// +// The stored route is never mutated: the rewrite is per-request, so a +// key/team change that re-entitles the verbatim id takes effect immediately, +// and concurrent requests race on nothing. Unknown entitlements, verbatim +// matches, and ambiguous candidates all pass the route through unchanged. +func (p *GitHubProxy) routeWithEntitledModel(route *InferenceRoute, agentName string) *InferenceRoute { + // A blank per-agent selection inherits the configured backend default, + // even before an entitlement probe has completed. Keep the stored snapshot + // untouched, as with entitlement rewrites below. + if route != nil && strings.TrimSpace(route.Model) == "" && strings.TrimSpace(route.DefaultModel) != "" { + rewritten := *route + rewritten.Model = route.DefaultModel + route = &rewritten + } + if route == nil || route.Backend != "litellm" || p.entitlements == nil { + return route + } + entitled, _, known := p.EntitledModels(route.Endpoint) + if !known { + return route + } + if entitledModelListed(entitled, route.Model) { + return route + } + if resolved, ok := resolveEntitledModelID(entitled, route.Model); ok { + p.logger.Info("inference model resolved to entitled gateway id", + "agent", agentName, "endpoint", route.Endpoint, + "configured", route.Model, "resolved", resolved) + rewritten := *route + rewritten.Model = resolved + return &rewritten + } + fallback := strings.TrimSpace(route.DefaultModel) + if fallback == "" || fallback == route.Model { + return route + } + if !entitledModelListed(entitled, fallback) { + resolved, ok := resolveEntitledModelID(entitled, fallback) + if !ok { + return route + } + fallback = resolved + } + p.logger.Warn("inference model not entitled on gateway; sending the backend default model instead", + "agent", agentName, "backend", route.Backend, "endpoint", route.Endpoint, + "configured", route.Model, "default", fallback, + "remedy", "clear the agent's model (or pick one of the gateway's entitled models) so the agent uses the gateway default") + rewritten := *route + rewritten.Model = fallback + return &rewritten +} + +// entitledModelListed reports whether model appears verbatim in entitled. +func entitledModelListed(entitled []string, model string) bool { + for _, id := range entitled { + if id == model { + return true + } + } + return false +} diff --git a/src/crux/proxy/inference_resilience.go b/src/crux/proxy/inference_resilience.go new file mode 100644 index 0000000000..605cc3d6af --- /dev/null +++ b/src/crux/proxy/inference_resilience.go @@ -0,0 +1,378 @@ +package proxy + +import ( + "bytes" + "context" + "errors" + "io" + "log/slog" + "math/rand/v2" + "net" + "net/http" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + + "github.com/hivecommons/hive/crux/config" +) + +// Opt-in timeouts and bounded retry for the inference translator +// (hivecommons/hive#19057). Everything here is inert while the resolved +// settings are the all-off default: no transport is cloned, no body is wrapped, +// no retry runs, and nothing extra is logged. +// +// Logs and metrics name the route (backend) and the status, never a request +// or response body and never a header value (a Retry-After is logged only as +// the delay the translator derived from it). + +const ( + inferenceRetryCountMetric = "crux.proxy.inference.retries" + inferenceTimeoutCountMetric = "crux.proxy.inference.timeouts" + + // inferenceDialKeepAlive matches the keep-alive of http.DefaultTransport's + // dialer, so enabling a connect timeout changes nothing else about dialing. + inferenceDialKeepAlive = 30 * time.Second + + // inferenceRetryAfterBase is the base used to parse a Retry-After delay. + inferenceRetryAfterBase = 10 + + // inferenceStatusNotImplemented is the one 5xx never retried: the upstream + // says it will not do this, so a retry cannot help. + inferenceStatusNotImplemented = http.StatusNotImplemented +) + +// Timeout kinds: a fixed enum so the metric attribute stays bounded. +const ( + inferenceTimeoutConnect = "connect" + inferenceTimeoutResponseHeader = "response_header" + inferenceTimeoutIdle = "idle" +) + +var ( + attrRoute = attribute.Key("route") + attrStatus = attribute.Key("status") + attrTimeoutKind = attribute.Key("kind") +) + +// inferenceResilienceSettings holds the resolved settings. nil = all off. +var inferenceResilienceSettings atomic.Pointer[config.InferenceResilience] + +// SetInferenceResilience installs the resolved translator timeouts and retry +// policy. The zero value (every default) leaves the translator unchanged. +func SetInferenceResilience(r config.InferenceResilience) { + inferenceResilienceSettings.Store(&r) +} + +func currentInferenceResilience() config.InferenceResilience { + if r := inferenceResilienceSettings.Load(); r != nil { + return *r + } + return config.InferenceResilience{} +} + +type inferenceResilienceMetrics struct { + retries metric.Int64Counter + timeouts metric.Int64Counter +} + +var globalInferenceResilienceMetrics = sync.OnceValue(func() inferenceResilienceMetrics { + meter := otel.Meter(instrumentationName) + // Constructors only fail on an invalid name or unit; both are constants. + retries, _ := meter.Int64Counter(inferenceRetryCountMetric, + metric.WithDescription("Inference upstream retries after a 429 or 5xx, by route and status.")) + timeouts, _ := meter.Int64Counter(inferenceTimeoutCountMetric, + metric.WithDescription("Inference upstream timeouts, by route and kind (connect, response_header, idle).")) + return inferenceResilienceMetrics{retries: retries, timeouts: timeouts} +}) + +// ---- transport (connect and response-header timeouts) ---- + +type tunedTransportKey struct { + caBundle string + connect time.Duration + header time.Duration +} + +var ( + tunedTransportMu sync.Mutex + tunedTransportCache = map[tunedTransportKey]*http.Transport{} +) + +// tunedInferenceTransport returns a clone of base with the connect and +// response-header timeouts applied, cached per (CA bundle, settings). It +// returns nil when base is not an *http.Transport, in which case the caller +// keeps base untouched. +func tunedInferenceTransport(caBundle string, base http.RoundTripper, res config.InferenceResilience) *http.Transport { + baseT, ok := base.(*http.Transport) + if !ok { + return nil + } + key := tunedTransportKey{caBundle: caBundle, connect: res.ConnectTimeout, header: res.ResponseHeaderTimeout} + tunedTransportMu.Lock() + defer tunedTransportMu.Unlock() + if t, ok := tunedTransportCache[key]; ok { + return t + } + t := baseT.Clone() + if res.ConnectTimeout > 0 { + dialer := &net.Dialer{Timeout: res.ConnectTimeout, KeepAlive: inferenceDialKeepAlive} + t.DialContext = dialer.DialContext + } + if res.ResponseHeaderTimeout > 0 { + t.ResponseHeaderTimeout = res.ResponseHeaderTimeout + } + tunedTransportCache[key] = t + return t +} + +// noteInferenceSendError counts and logs a connect or response-header timeout +// when the matching timeout is configured. It changes nothing about how the +// caller handles err. +func noteInferenceSendError(logger *slog.Logger, agentName string, route *InferenceRoute, err error) { + res := currentInferenceResilience() + if err == nil || route == nil || (res.ConnectTimeout <= 0 && res.ResponseHeaderTimeout <= 0) { + return + } + var netErr net.Error + if !errors.As(err, &netErr) || !netErr.Timeout() { + return + } + kind := "" + var opErr *net.OpError + switch { + case res.ResponseHeaderTimeout > 0 && strings.Contains(err.Error(), "awaiting response headers"): + kind = inferenceTimeoutResponseHeader + case res.ConnectTimeout > 0 && errors.As(err, &opErr) && opErr.Op == "dial": + kind = inferenceTimeoutConnect + } + if kind == "" { + return + } + recordInferenceTimeout(logger, agentName, route.Backend, kind, 0) +} + +func recordInferenceTimeout(logger *slog.Logger, agentName, backend, kind string, status int) { + globalInferenceResilienceMetrics().timeouts.Add(context.Background(), 1, + metric.WithAttributes(attrRoute.String(backend), attrTimeoutKind.String(kind))) + if logger == nil { + return + } + logger.Warn("inference upstream timeout", "agent", agentName, "backend", backend, "kind", kind, "status", status) +} + +// ---- idle-stream timeout ---- + +// inferenceIdleTimeoutError is the read error of a body cut for being idle. +type inferenceIdleTimeoutError struct{ idle time.Duration } + +func (e inferenceIdleTimeoutError) Error() string { + return "inference upstream sent no bytes for " + e.idle.String() +} +func (inferenceIdleTimeoutError) Timeout() bool { return true } +func (inferenceIdleTimeoutError) Temporary() bool { return false } + +// idleTimeoutBody cuts a response body that delivers no bytes for idle. The +// timer runs only while a Read is blocked on the upstream, so a slow consumer +// never trips it and a slow but active stream (bytes keep arriving) is never +// cut. +type idleTimeoutBody struct { + rc io.ReadCloser + idle time.Duration + timer *time.Timer + tripped atomic.Bool + onTrip func() +} + +func newIdleTimeoutBody(rc io.ReadCloser, idle time.Duration, onTrip func()) *idleTimeoutBody { + b := &idleTimeoutBody{rc: rc, idle: idle, onTrip: onTrip} + b.timer = time.AfterFunc(idle, b.trip) + b.timer.Stop() + return b +} + +func (b *idleTimeoutBody) trip() { + if !b.tripped.CompareAndSwap(false, true) { + return + } + if b.onTrip != nil { + b.onTrip() + } + _ = b.rc.Close() +} + +func (b *idleTimeoutBody) Read(p []byte) (int, error) { + b.timer.Reset(b.idle) + n, err := b.rc.Read(p) + b.timer.Stop() + if err != nil && !errors.Is(err, io.EOF) && b.tripped.Load() { + err = inferenceIdleTimeoutError{idle: b.idle} + } + return n, err +} + +func (b *idleTimeoutBody) Close() error { + b.timer.Stop() + return b.rc.Close() +} + +// applyInferenceIdleTimeout wraps resp.Body with the idle-stream timeout when +// one is configured. With the default (off) it does nothing. +func applyInferenceIdleTimeout(logger *slog.Logger, resp *http.Response, agentName string, route *InferenceRoute) { + idle := currentInferenceResilience().IdleStreamTimeout + if idle <= 0 || resp == nil || resp.Body == nil || route == nil { + return + } + status, backend := resp.StatusCode, route.Backend + resp.Body = newIdleTimeoutBody(resp.Body, idle, func() { + recordInferenceTimeout(logger, agentName, backend, inferenceTimeoutIdle, status) + }) +} + +// ---- bounded retry for 429 and 5xx ---- + +// inferenceJitter returns a uniform duration in [0, n]. A variable so tests can +// pin it. +var inferenceJitter = func(n time.Duration) time.Duration { + if n <= 0 { + return 0 + } + return time.Duration(rand.Int64N(int64(n) + 1)) +} + +// inferenceSleep waits d or until ctx ends. A variable so tests need not wait. +var inferenceSleep = func(ctx context.Context, d time.Duration) error { + if d <= 0 { + return ctx.Err() + } + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + return nil + } +} + +// inferenceRetryableStatus is true for 429 and for 5xx other than 501. +func inferenceRetryableStatus(status int) bool { + if status == http.StatusTooManyRequests { + return true + } + return status >= http.StatusInternalServerError && status < 600 && status != inferenceStatusNotImplemented +} + +// parseInferenceRetryAfter reads a Retry-After value (delta-seconds or an HTTP +// date). ok is false when absent or malformed. +func parseInferenceRetryAfter(v string, now time.Time) (time.Duration, bool) { + v = strings.TrimSpace(v) + if v == "" { + return 0, false + } + if secs, err := strconv.ParseInt(v, inferenceRetryAfterBase, 64); err == nil { + if secs < 0 { + return 0, false + } + if secs > config.MaxInferenceTimeoutS { + secs = config.MaxInferenceTimeoutS + } + return time.Duration(secs) * time.Second, true + } + if t, err := http.ParseTime(v); err == nil { + d := t.Sub(now) + if d < 0 { + d = 0 + } + if d > config.MaxInferenceTimeoutS*time.Second { + d = config.MaxInferenceTimeoutS * time.Second + } + return d, true + } + return 0, false +} + +// inferenceRetryDelay is how long to wait before retry number attempt (1-based). +// With a usable Retry-After the wait is that delay plus jitter in [0, base]; a +// Retry-After above the max delay returns ok=false (retrying sooner than the +// upstream asked would only fail again). Without one it is exponential +// backoff, base*2^(attempt-1) capped at the max delay, with jitter in +// [delay/2, delay]. +func inferenceRetryDelay(res config.InferenceResilience, attempt int, retryAfter string, now time.Time) (time.Duration, bool) { + if ra, ok := parseInferenceRetryAfter(retryAfter, now); ok { + if ra > res.MaxDelay { + return 0, false + } + return ra + inferenceJitter(res.BaseDelay), true + } + d := res.BaseDelay + for i := 1; i < attempt && d < res.MaxDelay; i++ { + d *= 2 + } + if d > res.MaxDelay { + d = res.MaxDelay + } + half := d / 2 + return half + inferenceJitter(d-half), true +} + +// retryInferenceOnOverload resends the request after a 429 or 5xx while +// retries remain, honoring Retry-After. It runs before the response is handed +// to the handler, so no byte has reached the agent when it retries: a stream +// that dies after its first byte is never retried. The request body is the +// translated payload the handler already holds in memory, so replaying it +// needs no extra buffer and no size cap. It resends the same headers the first +// attempt carried (content type and the route's current auth) and leaves key +// resolution and the 401 retry to retryInferenceOnAuthFailure, which runs +// after it. With retry off (the default) it returns resp untouched. +func retryInferenceOnOverload(ctx context.Context, logger *slog.Logger, client *http.Client, route *InferenceRoute, agentName, upstreamURL string, body []byte, resp *http.Response) *http.Response { + res := currentInferenceResilience() + if !res.RetryEnabled || route == nil || resp == nil { + return resp + } + for attempt := 1; attempt <= res.MaxRetries && inferenceRetryableStatus(resp.StatusCode); attempt++ { + delay, ok := inferenceRetryDelay(res, attempt, resp.Header.Get("Retry-After"), time.Now()) + if !ok { + if logger != nil { + logger.Warn("inference upstream retry skipped: requested wait exceeds the configured maximum", + "agent", agentName, "backend", route.Backend, "status", resp.StatusCode, + "max_delay_ms", res.MaxDelay.Milliseconds()) + } + return resp + } + if logger != nil { + logger.Warn("inference upstream retrying", + "agent", agentName, "backend", route.Backend, "status", resp.StatusCode, + "attempt", attempt, "max_retries", res.MaxRetries, "delay_ms", delay.Milliseconds()) + } + globalInferenceResilienceMetrics().retries.Add(ctx, 1, + metric.WithAttributes(attrRoute.String(route.Backend), attrStatus.Int(resp.StatusCode))) + if err := inferenceSleep(ctx, delay); err != nil { + return resp + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, upstreamURL, bytes.NewReader(body)) + if err != nil { + return resp + } + req.Header.Set("Content-Type", "application/json") + applyInferenceAuth(req, route) + next, err := client.Do(req) + if err != nil { + if logger != nil { + logger.Warn("inference upstream retry failed to send; keeping the previous response", + "agent", agentName, "backend", route.Backend, "attempt", attempt, "error", err) + } + noteInferenceSendError(logger, agentName, route, err) + return resp + } + _, _ = io.CopyN(io.Discard, resp.Body, inferenceRetryDrainLimit) + _ = resp.Body.Close() + resp = next + } + return resp +} diff --git a/src/crux/proxy/inference_resilience_test.go b/src/crux/proxy/inference_resilience_test.go new file mode 100644 index 0000000000..38d25f02ee --- /dev/null +++ b/src/crux/proxy/inference_resilience_test.go @@ -0,0 +1,511 @@ +package proxy + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/hivecommons/hive/crux/config" +) + +const ( + resilienceTestBase = 10 * time.Millisecond + resilienceTestMaxDelay = 5 * time.Second + // resilienceStallCeiling is how long a deliberately stalled upstream waits + // if nothing cuts it; every cut test must finish well inside it. + resilienceStallCeiling = 5 * time.Second + resilienceCutBudget = 3 * time.Second + resilienceSSETail = "TAILMARKER" +) + +// setInferenceResilienceForTest installs res and restores every global the +// retry code exposes for tests. +func setInferenceResilienceForTest(t *testing.T, res config.InferenceResilience) *sleepRecorder { + t.Helper() + prevSleep, prevJitter := inferenceSleep, inferenceJitter + rec := &sleepRecorder{} + inferenceSleep = rec.sleep + SetInferenceResilience(res) + t.Cleanup(func() { + inferenceResilienceSettings.Store(nil) + inferenceSleep, inferenceJitter = prevSleep, prevJitter + }) + return rec +} + +type sleepRecorder struct { + mu sync.Mutex + d []time.Duration +} + +func (s *sleepRecorder) sleep(_ context.Context, d time.Duration) error { + s.mu.Lock() + defer s.mu.Unlock() + s.d = append(s.d, d) + return nil +} + +func (s *sleepRecorder) durations() []time.Duration { + s.mu.Lock() + defer s.mu.Unlock() + return append([]time.Duration(nil), s.d...) +} + +func retryResilience(maxRetries int) config.InferenceResilience { + return config.InferenceResilience{ + RetryEnabled: true, + MaxRetries: maxRetries, + BaseDelay: resilienceTestBase, + MaxDelay: resilienceTestMaxDelay, + } +} + +// serveTranslatorFull is serveTranslator that also returns the response body. +func serveTranslatorFull(t *testing.T, p *GitHubProxy, route *InferenceRoute) (int, string) { + t.Helper() + p.inference.Set("scanner", route) + req := httptest.NewRequest(http.MethodPost, "/v1/messages", strings.NewReader(keyResolverTestBody)) + req.Header.Set("x-api-key", "sk-hive-scanner") + w := httptest.NewRecorder() + p.inferenceTranslatorHandler().ServeHTTP(w, req) + return w.Code, w.Body.String() +} + +// scriptedUpstream answers request n (1-based) with script(n, w, r). +type scriptedUpstream struct { + srv *httptest.Server + hits atomic.Int32 +} + +func newScriptedUpstream(t *testing.T, script func(n int, w http.ResponseWriter, r *http.Request)) *scriptedUpstream { + t.Helper() + u := &scriptedUpstream{} + u.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + script(int(u.hits.Add(1)), w, r) + })) + t.Cleanup(u.srv.Close) + return u +} + +func writeOK(w http.ResponseWriter) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, keyResolverOKResponse) +} + +func sseChunk(text string) string { + return fmt.Sprintf("data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":%q}}]}\n\n", text) +} + +func resilienceRoute(u *scriptedUpstream) *InferenceRoute { + return &InferenceRoute{Backend: "litellm", Endpoint: u.srv.URL, Model: "m", APIKey: "k"} +} + +func waitOrDone(r *http.Request, d time.Duration) { + select { + case <-r.Context().Done(): + case <-time.After(d): + } +} + +// Defaults: nothing changes. + +func TestInferenceResilience_DefaultsUnchanged(t *testing.T) { + var zero config.InferenceResilience + if zero.Enabled() { + t.Fatal("the zero value must be all off") + } + res, ignored := (*config.InferenceResilienceConfig)(nil).Resolve(func(string) string { return "" }) + if res.Enabled() || len(ignored) != 0 { + t.Fatalf("absent block must resolve to all off, got %+v ignored %v", res, ignored) + } + + // The client is the very same default client as before. + c, err := inferenceHTTPClient(&InferenceRoute{}) + if err != nil || c != http.DefaultClient { + t.Fatalf("default client changed: %v %v", c, err) + } + + // A 429 is passed straight through with one upstream call, and a slow + // upstream is not cut, with no extra log line. + slow := newScriptedUpstream(t, func(n int, w http.ResponseWriter, r *http.Request) { + waitOrDone(r, 300*time.Millisecond) + writeOK(w) + }) + var logs bytes.Buffer + p := newKeyResolverTestProxy(&logs) + if code, _ := serveTranslatorFull(t, p, resilienceRoute(slow)); code != http.StatusOK { + t.Fatalf("slow upstream status %d, want 200", code) + } + limited := newScriptedUpstream(t, func(n int, w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + }) + if code, _ := serveTranslatorFull(t, p, resilienceRoute(limited)); code != http.StatusTooManyRequests || limited.hits.Load() != 1 { + t.Fatalf("429 status %d hits %d, want passthrough with one call", code, limited.hits.Load()) + } + for _, banned := range []string{"inference upstream retrying", "inference upstream timeout"} { + if strings.Contains(logs.String(), banned) { + t.Fatalf("defaults logged %q:\n%s", banned, logs.String()) + } + } +} + +// Timeouts. + +func TestInferenceResilience_HeaderTimeoutCutsStalledUpstream(t *testing.T) { + for _, entry := range []string{"translator", "mitm"} { + t.Run(entry, func(t *testing.T) { + setInferenceResilienceForTest(t, config.InferenceResilience{ResponseHeaderTimeout: 150 * time.Millisecond}) + up := newScriptedUpstream(t, func(n int, w http.ResponseWriter, r *http.Request) { + waitOrDone(r, resilienceStallCeiling) + }) + var logs bytes.Buffer + p := newKeyResolverTestProxy(&logs) + serve := serveTranslator + if entry == "mitm" { + serve = serveMITM + } + start := time.Now() + code := serve(t, p, resilienceRoute(up)) + if code != http.StatusBadGateway { + t.Fatalf("status %d, want 502", code) + } + if time.Since(start) > resilienceCutBudget { + t.Fatalf("took %s, the stalled upstream was not cut at the header timeout", time.Since(start)) + } + out := logs.String() + if !strings.Contains(out, "inference upstream timeout") || !strings.Contains(out, "kind=response_header") || !strings.Contains(out, "backend=litellm") { + t.Fatalf("missing timeout log naming the route and kind:\n%s", out) + } + }) + } +} + +func TestInferenceResilience_TunedTransport(t *testing.T) { + res := config.InferenceResilience{ConnectTimeout: 3 * time.Second, ResponseHeaderTimeout: 7 * time.Second} + tr := tunedInferenceTransport("", http.DefaultTransport, res) + if tr == nil || tr.ResponseHeaderTimeout != res.ResponseHeaderTimeout || tr.DialContext == nil { + t.Fatalf("tuned transport not applied: %+v", tr) + } + if again := tunedInferenceTransport("", http.DefaultTransport, res); again != tr { + t.Fatal("tuned transport must be cached per settings") + } + if base, _ := http.DefaultTransport.(*http.Transport); base.ResponseHeaderTimeout != 0 { + t.Fatal("the shared default transport was mutated") + } + if tunedInferenceTransport("", failingRoundTripper{}, res) != nil { + t.Fatal("a non-http.Transport base must be left alone") + } +} + +func sseUpstream(chunks int, gap time.Duration, stallAfter int) func(int, http.ResponseWriter, *http.Request) { + return func(n int, w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + fl, _ := w.(http.Flusher) + for i := 1; i <= chunks; i++ { + text := "x" + if i == chunks { + text = resilienceSSETail + } + _, _ = io.WriteString(w, sseChunk(text)) + fl.Flush() + if stallAfter > 0 && i == stallAfter { + waitOrDone(r, resilienceStallCeiling) + return + } + waitOrDone(r, gap) + } + _, _ = io.WriteString(w, "data: [DONE]\n\n") + fl.Flush() + } +} + +func TestInferenceResilience_IdleTimeoutCutsStalledStream(t *testing.T) { + setInferenceResilienceForTest(t, config.InferenceResilience{IdleStreamTimeout: 200 * time.Millisecond}) + up := newScriptedUpstream(t, sseUpstream(5, 0, 1)) // one chunk, then silence + var logs bytes.Buffer + p := newKeyResolverTestProxy(&logs) + start := time.Now() + code, body := serveTranslatorFull(t, p, resilienceRoute(up)) + if time.Since(start) > resilienceCutBudget { + t.Fatalf("took %s, the idle stream was not cut", time.Since(start)) + } + if code != http.StatusOK || strings.Contains(body, resilienceSSETail) { + t.Fatalf("status %d; the stalled stream must end without its tail:\n%s", code, body) + } + out := logs.String() + if !strings.Contains(out, "kind=idle") || !strings.Contains(out, "backend=litellm") || !strings.Contains(out, "inference SSE translation failed") { + t.Fatalf("missing idle timeout log or the existing stream-death log:\n%s", out) + } + if strings.Contains(out, "x-api-key") || strings.Contains(out, "Bearer") { + t.Fatalf("log leaked a header:\n%s", out) + } +} + +func TestInferenceResilience_IdleTimeoutSparesSlowActiveStream(t *testing.T) { + // 8 chunks, 80ms apart: the stream lasts ~640ms, well past the 300ms idle + // timeout, but never goes 300ms without a byte. + setInferenceResilienceForTest(t, config.InferenceResilience{IdleStreamTimeout: 300 * time.Millisecond}) + up := newScriptedUpstream(t, sseUpstream(8, 80*time.Millisecond, 0)) + var logs bytes.Buffer + p := newKeyResolverTestProxy(&logs) + code, body := serveTranslatorFull(t, p, resilienceRoute(up)) + if code != http.StatusOK || !strings.Contains(body, resilienceSSETail) { + t.Fatalf("status %d; the slow but active stream must complete:\n%s", code, body) + } + if strings.Contains(logs.String(), "inference upstream timeout") { + t.Fatalf("an active stream was reported as timed out:\n%s", logs.String()) + } +} + +// Retry. + +func TestInferenceResilience_RetriesWithinCap(t *testing.T) { + cases := []struct { + name string + status int + retryAfter string + failures int + maxRetries int + wantHits int32 + wantCode int + }{ + {"429 with Retry-After recovers", http.StatusTooManyRequests, "1", 1, 2, 2, http.StatusOK}, + {"500 recovers", http.StatusInternalServerError, "", 1, 2, 2, http.StatusOK}, + {"502 recovers", http.StatusBadGateway, "", 2, 2, 3, http.StatusOK}, + {"503 recovers", http.StatusServiceUnavailable, "", 2, 2, 3, http.StatusOK}, + {"504 recovers", http.StatusGatewayTimeout, "", 1, 1, 2, http.StatusOK}, + {"cap respected on 503", http.StatusServiceUnavailable, "", 100, 2, 3, http.StatusServiceUnavailable}, + {"cap respected on 429", http.StatusTooManyRequests, "", 100, 1, 2, http.StatusTooManyRequests}, + {"501 is not retried", http.StatusNotImplemented, "", 100, 3, 1, http.StatusNotImplemented}, + {"400 is not retried", http.StatusBadRequest, "", 100, 3, 1, http.StatusBadRequest}, + {"404 is not retried", http.StatusNotFound, "", 100, 3, 1, http.StatusNotFound}, + {"403 is not retried", http.StatusForbidden, "", 100, 3, 1, http.StatusForbidden}, + } + for _, tc := range cases { + for _, entry := range []string{"translator", "mitm"} { + t.Run(tc.name+"/"+entry, func(t *testing.T) { + rec := setInferenceResilienceForTest(t, retryResilience(tc.maxRetries)) + inferenceJitter = func(time.Duration) time.Duration { return 0 } + up := newScriptedUpstream(t, func(n int, w http.ResponseWriter, r *http.Request) { + if n <= tc.failures { + if tc.retryAfter != "" { + w.Header().Set("Retry-After", tc.retryAfter) + } + w.WriteHeader(tc.status) + _, _ = io.WriteString(w, `{"error":{"message":"busy"}}`) + return + } + writeOK(w) + }) + var logs bytes.Buffer + p := newKeyResolverTestProxy(&logs) + serve := serveTranslator + if entry == "mitm" { + serve = serveMITM + } + code := serve(t, p, resilienceRoute(up)) + if code != tc.wantCode || up.hits.Load() != tc.wantHits { + t.Fatalf("status %d hits %d, want status %d hits %d\n%s", code, up.hits.Load(), tc.wantCode, tc.wantHits, logs.String()) + } + if want := int(tc.wantHits) - 1; len(rec.durations()) != want { + t.Fatalf("slept %d times, want %d", len(rec.durations()), want) + } + if tc.retryAfter == "1" { + if d := rec.durations(); len(d) != 1 || d[0] != time.Second { + t.Fatalf("Retry-After: 1 slept %v, want exactly 1s with zero jitter", d) + } + } + if int(tc.wantHits) > 1 { + out := logs.String() + if !strings.Contains(out, "inference upstream retrying") || !strings.Contains(out, fmt.Sprintf("status=%d", tc.status)) || !strings.Contains(out, "backend=litellm") { + t.Fatalf("missing retry log naming route and status:\n%s", out) + } + if strings.Contains(out, "Retry-After") || strings.Contains(out, "Bearer") { + t.Fatalf("retry log carried a header:\n%s", out) + } + } + }) + } + } +} + +func TestInferenceResilience_RetryReplaysSameRequest(t *testing.T) { + setInferenceResilienceForTest(t, retryResilience(2)) + var mu sync.Mutex + var bodies, auths []string + up := newScriptedUpstream(t, func(n int, w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + mu.Lock() + bodies = append(bodies, string(b)) + auths = append(auths, r.Header.Get("Authorization")+"|"+r.Header.Get("Content-Type")) + mu.Unlock() + if n == 1 { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + writeOK(w) + }) + p := newKeyResolverTestProxy(&bytes.Buffer{}) + if code, _ := serveTranslatorFull(t, p, resilienceRoute(up)); code != http.StatusOK { + t.Fatalf("status %d", code) + } + mu.Lock() + defer mu.Unlock() + if len(bodies) != 2 || bodies[0] == "" || bodies[0] != bodies[1] || auths[0] != auths[1] { + t.Fatalf("the retry must replay the same body and headers: %q %q", bodies, auths) + } +} + +func TestInferenceResilience_RetryAfterAboveMaxIsNotRetried(t *testing.T) { + res := retryResilience(3) + res.MaxDelay = 2 * time.Second + rec := setInferenceResilienceForTest(t, res) + up := newScriptedUpstream(t, func(n int, w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "30") + w.WriteHeader(http.StatusTooManyRequests) + }) + var logs bytes.Buffer + p := newKeyResolverTestProxy(&logs) + if code, _ := serveTranslatorFull(t, p, resilienceRoute(up)); code != http.StatusTooManyRequests || up.hits.Load() != 1 || len(rec.durations()) != 0 { + t.Fatalf("status %d hits %d: a Retry-After above the cap must not be retried", code, up.hits.Load()) + } +} + +func TestInferenceResilience_NoRetryAfterBytesReachedAgent(t *testing.T) { + for _, entry := range []string{"translator", "mitm"} { + t.Run(entry, func(t *testing.T) { + rec := setInferenceResilienceForTest(t, retryResilience(3)) + // 200 with one chunk, then the connection dies mid-stream. + up := newScriptedUpstream(t, func(n int, w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = io.WriteString(w, sseChunk("partial")) + w.(http.Flusher).Flush() + panic(http.ErrAbortHandler) + }) + var logs bytes.Buffer + p := newKeyResolverTestProxy(&logs) + serve := serveTranslator + if entry == "mitm" { + serve = serveMITM + } + if code := serve(t, p, resilienceRoute(up)); code != http.StatusOK { + t.Fatalf("status %d, the stream had already started", code) + } + if up.hits.Load() != 1 || len(rec.durations()) != 0 { + t.Fatalf("hits %d sleeps %d: nothing may be retried once bytes were sent", up.hits.Load(), len(rec.durations())) + } + }) + } +} + +func TestInferenceResilience_RetryStopsWhenContextEnds(t *testing.T) { + setInferenceResilienceForTest(t, retryResilience(3)) + ctx, cancel := context.WithCancel(context.Background()) + inferenceSleep = func(ctx context.Context, d time.Duration) error { cancel(); return ctx.Err() } + up := newScriptedUpstream(t, func(n int, w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + }) + route := resilienceRoute(up) + client, err := inferenceHTTPClient(route) + if err != nil { + t.Fatal(err) + } + req, _ := http.NewRequestWithContext(ctx, http.MethodPost, up.srv.URL, strings.NewReader("{}")) + first, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + got := retryInferenceOnOverload(ctx, nil, client, route, "scanner", up.srv.URL, []byte("{}"), first) + _ = got.Body.Close() + if got != first || up.hits.Load() != 1 { + t.Fatalf("a canceled context must return the original response without resending (hits %d)", up.hits.Load()) + } +} + +// Jitter and Retry-After parsing. + +func TestInferenceRetryDelay_JitterBounds(t *testing.T) { + res := config.InferenceResilience{RetryEnabled: true, MaxRetries: 5, BaseDelay: 100 * time.Millisecond, MaxDelay: time.Second} + now := time.Now() + wantCeil := []time.Duration{100 * time.Millisecond, 200 * time.Millisecond, 400 * time.Millisecond, 800 * time.Millisecond, time.Second} + for i, ceil := range wantCeil { + for k := 0; k < 300; k++ { + d, ok := inferenceRetryDelay(res, i+1, "", now) + if !ok || d < ceil/2 || d > ceil { + t.Fatalf("attempt %d: delay %s outside [%s, %s]", i+1, d, ceil/2, ceil) + } + } + } + for k := 0; k < 300; k++ { + d, ok := inferenceRetryDelay(res, 1, "1", now) + if !ok || d < time.Second || d > time.Second+res.BaseDelay { + t.Fatalf("Retry-After delay %s outside [1s, 1s+base]", d) + } + } + if _, ok := inferenceRetryDelay(res, 1, "2", now); ok { + t.Fatal("a Retry-After above the max delay must not be retried") + } +} + +func TestParseInferenceRetryAfter(t *testing.T) { + now := time.Date(2026, 11, 18, 12, 0, 0, 0, time.UTC) + cases := []struct { + in string + want time.Duration + wantOK bool + }{ + {"", 0, false}, + {"abc", 0, false}, + {"-5", 0, false}, + {"0", 0, true}, + {"7", 7 * time.Second, true}, + {" 7 ", 7 * time.Second, true}, + {now.Add(9 * time.Second).UTC().Format(http.TimeFormat), 9 * time.Second, true}, + {now.Add(-time.Hour).UTC().Format(http.TimeFormat), 0, true}, + {"99999999999", config.MaxInferenceTimeoutS * time.Second, true}, + } + for _, tc := range cases { + got, ok := parseInferenceRetryAfter(tc.in, now) + if ok != tc.wantOK || got != tc.want { + t.Errorf("parse(%q) = %s,%t want %s,%t", tc.in, got, ok, tc.want, tc.wantOK) + } + } +} + +// 401 path untouched. + +func TestInferenceResilience_401PathUntouched(t *testing.T) { + setInferenceResilienceForTest(t, retryResilience(3)) + + // No resolver: a 401 is not retried. + none := newKeyedUpstream(t) + p := newKeyResolverTestProxy(&bytes.Buffer{}) + route := &InferenceRoute{Backend: "litellm", Endpoint: none.srv.URL, Model: "m", APIKey: "k"} + if code := serveTranslator(t, p, route); code != http.StatusUnauthorized || none.hits.Load() != 1 { + t.Fatalf("status %d hits %d, a 401 with no resolver must not be retried", code, none.hits.Load()) + } + + // Rotated key: still exactly one retry with the new key. + up := newKeyedUpstream(t, "new-key") + var key atomic.Value + key.Store("old-key") + r := NewInferenceKeyResolver(t.Name(), func() (string, string) { + k, _ := key.Load().(string) + return k, "file:/secrets/litellm_api_key" + }) + r.Current() + key.Store("new-key") + rotated := &InferenceRoute{Backend: "litellm", Endpoint: up.srv.URL, Model: "m", APIKey: "old-key", KeyResolver: r} + if code := serveTranslator(t, p, rotated); code != http.StatusOK || up.hits.Load() != 2 || up.lastAuth() != "Bearer new-key" { + t.Fatalf("status %d hits %d auth %q, want the single 401 retry with the new key", code, up.hits.Load(), up.lastAuth()) + } +} diff --git a/src/crux/proxy/inference_route.go b/src/crux/proxy/inference_route.go index 57afc861c3..c229013d5c 100644 --- a/src/crux/proxy/inference_route.go +++ b/src/crux/proxy/inference_route.go @@ -303,11 +303,22 @@ func inferenceTransport(caBundle string) (http.RoundTripper, error) { // inferenceHTTPClient returns the client for forwarding translated requests // to the route's endpoint. Like http.DefaultClient it has no overall // timeout — streaming completions can legitimately run for minutes. +// The connect and response-header timeouts are opt-in +// (governor.inference_resilience) and the idle-stream timeout wraps the +// response body, not the client. func inferenceHTTPClient(route *InferenceRoute) (*http.Client, error) { rt, err := inferenceTransport(route.CABundle) if err != nil { return nil, err } + // Opt-in connect and response-header timeouts (hivecommons/hive#19057). + // Off by default: the branch below is skipped and the client is exactly + // the one returned before. + if res := currentInferenceResilience(); res.ConnectTimeout > 0 || res.ResponseHeaderTimeout > 0 { + if tuned := tunedInferenceTransport(route.CABundle, rt, res); tuned != nil { + return &http.Client{Transport: tuned}, nil + } + } if rt == http.DefaultTransport { return http.DefaultClient, nil } diff --git a/src/docs/agent-configuration.md b/src/docs/agent-configuration.md index eda0952baf..bf5f28bb83 100644 --- a/src/docs/agent-configuration.md +++ b/src/docs/agent-configuration.md @@ -722,6 +722,39 @@ The agent inference proxy (`127.0.0.1:18444`) no longer keeps the key it resolve When the gateway answers `401`, the proxy re-resolves the key once. If that yields a different key, it retries the request once with it, and counts an auth failure only when the retry is also rejected. An unchanged key is not retried. The `inference backend auth failure` WARN carries `key_source` (`file:`, `env:`, `static` for a route with a fixed credential such as a minted watsonx IAM token, or `none`) and `key_changed_since_start` (whether the key differs from the first key this process resolved for that backend). It never logs the key. +#### Translator timeouts and bounded retry + +The inference translator that carries every agent's model calls to a gateway has no timeout and no retry for a rate limit or server error unless you opt in ([#19057](https://github.com/hivecommons/hive/issues/19057)). Everything below is **off by default**: with the block absent, or every value at its default, the translator behaves exactly as before. Settings are read once at start, so a change needs a restart. + +```yaml +governor: + inference_resilience: + connect_timeout_s: 0 # TCP connect to the gateway; 0 = off + response_header_timeout_s: 0 # wait for the upstream to start answering; 0 = off + idle_stream_timeout_s: 0 # cut a response that sends no bytes for this long; 0 = off + retry_enabled: false # bounded retry for 429 and 5xx + retry_max_retries: 2 # retries after the first attempt (at most 5) + retry_base_delay_ms: 500 # first backoff step; doubles each retry + retry_max_delay_s: 20 # cap on one wait and on an honored Retry-After +``` + +| Key | Environment variable | Default | +| --- | --- | --- | +| `connect_timeout_s` | `HIVE_INFERENCE_CONNECT_TIMEOUT_S` | `0` (off) | +| `response_header_timeout_s` | `HIVE_INFERENCE_RESPONSE_HEADER_TIMEOUT_S` | `0` (off) | +| `idle_stream_timeout_s` | `HIVE_INFERENCE_IDLE_STREAM_TIMEOUT_S` | `0` (off) | +| `retry_enabled` | `HIVE_INFERENCE_RETRY_ENABLED` | `false` | +| `retry_max_retries` | `HIVE_INFERENCE_RETRY_MAX_RETRIES` | `2` (maximum `5`) | +| `retry_base_delay_ms` | `HIVE_INFERENCE_RETRY_BASE_DELAY_MS` | `500` | +| `retry_max_delay_s` | `HIVE_INFERENCE_RETRY_MAX_DELAY_S` | `20` | + +A valid environment variable wins over the yaml key; an invalid one is ignored and named in a start-up warning. Values are whole numbers, and seconds are capped at 3600. + +- **Timeouts.** The connect and response-header timeouts apply to the request to the gateway. A non-streaming call only gets headers once the model has finished, so keep `response_header_timeout_s` above your longest healthy generation. The idle-stream timeout counts only time spent waiting for the upstream, from one byte to the next, so a stream that is slow but keeps producing bytes is never cut. A timeout is not retried: the agent gets the same `502` (before any byte) or the same stopped stream (after) that a dead upstream always gave. +- **Retry.** With `retry_enabled`, a `429` or a `5xx` other than `501` is resent up to `retry_max_retries` times, with exponential backoff (`base * 2^(n-1)`, capped at `retry_max_delay_s`) and jitter in the upper half of each step. A `Retry-After` (seconds or an HTTP date) is honored, plus jitter of up to one base step; one longer than `retry_max_delay_s` is not waited for, and the response is passed to the agent instead. A retry happens only while no response byte has reached the agent, so a stream that has started is never replayed. The retry resends the translated request that is already held in memory with the same headers and the route's current key, so there is no buffer to size and no body-size cap. The existing single resend after a `401` is separate and unchanged, and key resolution is untouched. +- **Mid-stream death.** A stream that dies after its first byte is logged (`inference SSE translation failed`) and stopped, as before. +- **Observability.** Each retry logs `inference upstream retrying` and each timeout logs `inference upstream timeout`, with `backend`, `status` and `agent` (and `attempt`, `delay_ms` or `kind`), never a request or response body and never a header value. Counters `crux.proxy.inference.retries` (attributes `route`, `status`) and `crux.proxy.inference.timeouts` (attributes `route`, `kind` of `connect`, `response_header` or `idle`) are exported through the OpenTelemetry meter. + ### Claude Managed Agents hosted execution Claude Managed Agents is an external hosted engine, not a `backend:` value. diff --git a/src/docs/ai-gateway-evaluation.md b/src/docs/ai-gateway-evaluation.md index 3493082528..21b431442f 100644 --- a/src/docs/ai-gateway-evaluation.md +++ b/src/docs/ai-gateway-evaluation.md @@ -59,12 +59,12 @@ Consequences that matter for a gateway: ### Errors, retries and recovery in the translator -- **Transport failure** (gateway down, refused, timeout from the OS) becomes `502` with `inference backend unreachable: ...` (`src/crux/proxy/github_proxy_inference.go:516-520`, `src/crux/proxy/inference_error_message.go:22-25`). +- **Transport failure** (gateway down, refused, timeout from the OS) becomes `502` with `inference backend unreachable: ...` (`src/crux/proxy/github_proxy_inference.go:518-523`, `src/crux/proxy/inference_error_message.go:22-25`). - **Non-2xx** becomes an Anthropic-shaped error with the upstream status and the text `inference backend returned : ` plus the redacted upstream URL and `Server` header (`src/crux/proxy/inference_error_message.go:9-20`, `src/crux/proxy/github_proxy_inference.go:535-557`). That literal prefix is parsed by the agent pane classifier and the dashboard alerts. -- **There is no timeout and no retry for 429 or 5xx in the translator.** The client has no overall timeout because streams can run for minutes (`src/crux/proxy/inference_route.go:303-315`). The only retry is one resend after a `401` when the key resolver yields a different key (`retryInferenceOnAuthFailure` at `src/crux/proxy/github_proxy_inference.go:215`; see also [Key rotation without a restart](agent-configuration.md#key-rotation-without-a-restart)). Whatever retrying an agent does on a `429` or `5xx` is the Claude CLI's own behaviour and has to be observed in the spike. +- **Timeouts and retry for 429 or 5xx are off by default, configurable.** The client has no overall timeout because streams can run for minutes (`inferenceHTTPClient` in `src/crux/proxy/inference_route.go`), and an upstream `429` or `5xx` is passed to the agent as it always was. Under `governor.inference_resilience` (or the `HIVE_INFERENCE_*` environment variables) an operator can opt in to a connect timeout, a response-header timeout, an idle-stream timeout (no bytes for N seconds, so a slow but active stream is never cut) and a bounded retry with exponential backoff, jitter and `Retry-After` for `429` and `5xx`, applied only while no response byte has reached the agent (`src/crux/proxy/inference_resilience.go`; see [Translator timeouts and bounded retry](agent-configuration.md#translator-timeouts-and-bounded-retry)). The only other retry is one resend after a `401` when the key resolver yields a different key (`retryInferenceOnAuthFailure` in `src/crux/proxy/inference_key_resolver.go`; see also [Key rotation without a restart](agent-configuration.md#key-rotation-without-a-restart)); it is unchanged. Whatever retrying an agent does on a `429` or `5xx` beyond that is the Claude CLI's own behaviour and has to be observed in the spike. - **Spend refusals are told apart from rate limits by body text.** A `429` or `400` whose body contains one of a fixed list of phrases (`budget has been exceeded`, `exceeded max budget`, `insufficient_quota`, `exceeded your current quota`, `billing_hard_limit_reached`, `credit balance is too low` and a few more) latches the spend-limit signal and suppresses agent kicks; any other `429` stays an ordinary transient error (`src/crux/proxy/inference_budget.go:66-119`). A generic gateway whose quota error uses different wording will be treated as an ordinary rate limit, which is a spike item. - **Recovery.** The next 2xx clears both the auth and spend latches (`src/crux/proxy/github_proxy_inference.go:340-358`). -- **Streams that die mid-response.** Headers and a 200 have already been sent, so the translator can only log the translation error and stop (`src/crux/proxy/github_proxy_inference.go:567-571`). What the CLI then does is a spike item. +- **Streams that die mid-response.** Headers and a 200 have already been sent, so the translator can only log the translation error and stop (`src/crux/proxy/github_proxy_inference.go:570-574`). That stays as it is, with or without the opt-in settings: an idle-stream timeout that cuts such a stream ends it the same way, after one `inference upstream timeout` line. What the CLI then does is a spike item. ### Keys, files and rotation @@ -169,7 +169,7 @@ Run every arm on the same hardware, models and traffic. Arms: **direct** (transl | `RUN_REPEATS` | 5 | Repeats per arm and level; report the median of per-run percentiles and the spread. | | `GATEWAY_DOWN_DETECT_SECONDS` | 30 | Time from gateway outage to a clear refusal in the agent pane and the dashboard. | | `GATEWAY_RECOVERY_SECONDS` | 60 | Time from the gateway returning to the next successful agent request, with no restart. | -| `SLOW_GATEWAY_STALL_SECONDS` | 120 | A stalled upstream must surface a clear error within this time or be recorded as a miss (the translator has no timeout, so this is expected to fail until measured otherwise). | +| `SLOW_GATEWAY_STALL_SECONDS` | 120 | A stalled upstream must surface a clear error within this time or be recorded as a miss (the translator's timeouts are off by default, so this is expected to fail until `response_header_timeout_s` or `idle_stream_timeout_s` is set). | | `METERING_TOLERANCE_PCT` | 2.0 | Maximum difference of gateway-reported tokens versus Hive's recorded usage, per model, over the replayed workload. | | `METERING_COST_TOLERANCE_PCT` | 3.0 | Maximum difference in cost per model versus the gateway's own cost totals. | | `PER_AGENT_QUOTA_REQUIRED` | decided before measuring | Whether per-agent attribution is required or per-spoke metering is enough (open question in the issue). | diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index 7e32c2f547..e6f6aafbfd 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -301,6 +301,13 @@ governor.ci.auto_approve_workflow_actors[] # trusted workflow-approval actors (# governor.ci.auto_approve_workflows # gated workflow auto-approval flag (#16603); yaml-only governor.ci_failure_kick_cooldown_s # webhook CI-failure kick cooldown (#15291); yaml-only governor.ci_failure_webhook_kick # webhook CI-failure kick flag (#15291); yaml-only +governor.inference_resilience.connect_timeout_s # inference translator timeouts and retry (#19057); yaml and HIVE_INFERENCE_* env only +governor.inference_resilience.idle_stream_timeout_s # inference translator timeouts and retry (#19057); yaml and HIVE_INFERENCE_* env only +governor.inference_resilience.response_header_timeout_s # inference translator timeouts and retry (#19057); yaml and HIVE_INFERENCE_* env only +governor.inference_resilience.retry_base_delay_ms # inference translator timeouts and retry (#19057); yaml and HIVE_INFERENCE_* env only +governor.inference_resilience.retry_enabled # inference translator timeouts and retry (#19057); yaml and HIVE_INFERENCE_* env only +governor.inference_resilience.retry_max_delay_s # inference translator timeouts and retry (#19057); yaml and HIVE_INFERENCE_* env only +governor.inference_resilience.retry_max_retries # inference translator timeouts and retry (#19057); yaml and HIVE_INFERENCE_* env only governor.release_feeds.agents..cooldown_s # release-feed watcher (#18494); yaml-only governor.release_feeds.agents..sources[].auth_var # release-feed watcher (#18494); yaml-only governor.release_feeds.agents..sources[].kind # release-feed watcher (#18494); yaml-only