diff --git a/changelog.d/fixed-15333-inference-caller-class.md b/changelog.d/fixed-15333-inference-caller-class.md new file mode 100644 index 0000000000..d7cf2a397d --- /dev/null +++ b/changelog.d/fixed-15333-inference-caller-class.md @@ -0,0 +1 @@ +- Repeated inference translator caller-identification refusals now report as `inference_caller_unidentified` under the inference gateway, with medium severity, instead of `unclassified_error_repeated`; the HTTP 401 identity guard remains enforced (#15333). diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index 333db16919..e40c44888a 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -154,6 +154,7 @@ title is `[selfreport] : `. | `proxy_tls_handshake_failed` | the hive proxy's TLS handshake with an upstream host failed (`proxy client tls handshake failed`, `transparent proxy tls handshake failed`, `copilot sniff: client tls handshake failed`, `crux/proxy/github_proxy_connect.go`, `crux/proxy/github_proxy_copilot.go`) — typically a certificate validation failure or encryption mismatch on an agent-requested host, repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `review_evidence_write_failed` | the github-client failed to write a review evidence bundle to persistent storage (`review evidence: could not write bundle`, `pkg/github`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `selfreport_cleared_comment_failed` | the fleet report publisher failed to post the "condition cleared" comment on a self-report issue (`fleet report: cleared comment failed`, `pkg/github`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | +| `inference_caller_unidentified` | the inference translator refused a caller whose identity could not be established (`inference request from unidentified caller refused`, `crux/proxy/github_proxy_inference.go`). The request remains rejected with HTTP 401; this is distinct from backend credential failures. Check the per-agent UID map and socket attribution for the calling process; the log carries the remote address, method and path. Log-fed: ≥5 refusals in 10 min, clears after 20 quiet minutes | medium | | `inference_upstream_error` | a failed or non-2xx response from the configured inference backend (LiteLLM, vLLM/llm-d) on the Anthropic-to-OpenAI translator or MITM reroute path (`inference upstream error` / `inference upstream failed`, `crux/proxy/github_proxy_inference.go`), repeated ≥5 times in 10 min — covers transport failures and statuses other than the dedicated 401/403 auth signal. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `sha_poll_commit_message_unavailable` | the SHA poller's commit-message fetch got a non-200 from GitHub (typically a 403 rate limit or token scope; `SHA poll: commit message fetch non-200`, `pkg/hub/saas_sha_poller.go`). The poller backs off per SHA and keeps serving the SHA without a message. The WARN itself is deduped to once per hour per (SHA, status) (`warnGitHubFetchOnce`), so one stuck commit logs one line and never trips this class; it is reported only when ≥5 distinct SHA/status pairs fail within 10 min (a broad outage such as an exhausted rate limit or a revoked token). Its main job is keeping the residual WARN out of `unclassified_error_repeated`. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | | `tmux_session_check_failed` | the agent's tmux session `has-session` check failed (`tmux has-session failed`, `crux/agent/manager.go`) — typically the session was lost or the tmux server is unhealthy, repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | diff --git a/src/pkg/fleetreport/runtime.go b/src/pkg/fleetreport/runtime.go index 380e1eb39a..321b71ff44 100644 --- a/src/pkg/fleetreport/runtime.go +++ b/src/pkg/fleetreport/runtime.go @@ -78,6 +78,7 @@ const ( ClassGitHubAppRepoNotCovered = "github_app_repo_not_covered" ClassProxyConnectDialFailed = "proxy_connect_dial_failed" ClassInferenceUpstreamError = "inference_upstream_error" + ClassInferenceCallerUnknown = "inference_caller_unidentified" ClassSHAPollCommitMsgUnavail = "sha_poll_commit_message_unavailable" ClassAgentLoginRequired = "agent_login_required" ClassPRRequestHeadUnpushed = "pr_request_head_unpushed" @@ -189,6 +190,7 @@ var Classes = map[string]ClassSpec{ ClassSelfReportClearedFailed: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"fleet report: cleared comment failed"}}, ClassGitHubAppRepoNotCovered: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"installation does not cover every configured repo", "state=repo-not-covered"}}, ClassProxyConnectDialFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: connect dial failed"}}, + ClassInferenceCallerUnknown: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference request from unidentified caller refused"}}, ClassInferenceUpstreamError: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference upstream error", "inference upstream failed"}}, ClassSHAPollCommitMsgUnavail: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"sha poll: commit message fetch non-200"}}, ClassAgentLoginRequired: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"login required detected"}}, diff --git a/src/pkg/fleetreport/runtime_test.go b/src/pkg/fleetreport/runtime_test.go index ea7a84b081..a9e67c6a59 100644 --- a/src/pkg/fleetreport/runtime_test.go +++ b/src/pkg/fleetreport/runtime_test.go @@ -357,6 +357,7 @@ func TestEveryRuntimeClassTriggerAndNonTriggerThresholds(t *testing.T) { {class: ClassGitHubAppRepoNotCovered, message: "github app recheck: installation does not cover every configured repo"}, {class: ClassGitHubAppNotInstalled, message: "github app installation id is unset — per-agent scoped token NOT delivered"}, {class: ClassProxyConnectDialFailed, message: "proxy: CONNECT dial failed"}, + {class: ClassInferenceCallerUnknown, message: "inference request from unidentified caller refused"}, {class: ClassInferenceUpstreamError, message: "inference upstream error"}, {class: ClassSHAPollCommitMsgUnavail, message: "SHA poll: commit message fetch non-200"}, {class: ClassAgentLoginRequired, message: "login required detected"}, diff --git a/src/pkg/fleetreport/slogfeed/slogfeed_test.go b/src/pkg/fleetreport/slogfeed/slogfeed_test.go index a69a48a7cd..c8a806f157 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed_test.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed_test.go @@ -564,3 +564,51 @@ func TestGitHubRateLimitStormLogsWithoutFleetReport(t *testing.T) { t.Fatalf("non-rate-limit failures must remain reportable: %+v", *seen) } } + +// #15333: keep repeated translator identity refusals visible under their own +// gateway class, without treating them as backend credential failures. +func TestInferenceCallerUnidentifiedFeedThresholdAndClear(t *testing.T) { + now := time.Date(2026, 10, 29, 6, 26, 0, 0, time.UTC) + tr := fleetreport.NewRuntimeTracker(func() time.Time { return now }) + var out bytes.Buffer + logger := slog.New(NewWithObserver(slog.NewJSONHandler(&out, nil), tr.Observe)) + const msg = "inference request from unidentified caller refused" + warn := func() { + logger.Warn(msg, "remote", "127.0.0.1:47724", "method", "POST", "path", "/v1/messages") + } + for i := 0; i < 4; i++ { + warn() + now = now.Add(time.Minute) + if ev := tr.Evidence(); len(ev) != 0 { + t.Fatalf("below threshold produced evidence: %+v", ev) + } + } + warn() + ev := tr.Evidence() + if len(ev) != 1 || ev[0].Class != fleetreport.ClassInferenceCallerUnknown || ev[0].Count != 5 || ev[0].Severity != "medium" || ev[0].Component != "inference-gateway" { + t.Fatalf("repeated identity refusals = %+v", ev) + } + for _, want := range []string{"remote=127.0.0.1:47724", "method=POST", "path=/v1/messages"} { + if !strings.Contains(ev[0].Diagnostics, want) { + t.Errorf("diagnostics %q missing %q", ev[0].Diagnostics, want) + } + } + if n := strings.Count(out.String(), "\n"); n != 5 { + t.Fatalf("wrapped handler saw %d records, want 5", n) + } + now = now.Add(21 * time.Minute) + if ev := tr.Evidence(); len(ev) != 0 { + t.Fatalf("idle identity refusals still reported: %+v", ev) + } + cleared := tr.TakeCleared() + if len(cleared) != 1 || cleared[0].Class != fleetreport.ClassInferenceCallerUnknown { + t.Fatalf("cleared = %+v", cleared) + } + // Other translator failures must still reach their existing classes. + logger2, seen, _ := newFeed(t) + logger2.Warn("inference upstream failed") + logger2.Warn("inference no route", "agent", "scanner") + if len(*seen) != 2 || (*seen)[0].class != fleetreport.ClassInferenceUpstreamError || (*seen)[1].class != fleetreport.ClassUnclassifiedRepeated { + t.Fatalf("unrelated inference failures = %+v", *seen) + } +}