Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-15333-inference-caller-class.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Repeated inference translator caller-identification refusals now report as `inference_caller_unidentified` under the inference gateway, with medium severity, instead of `unclassified_error_repeated`; the HTTP 401 identity guard remains enforced (#15333).
1 change: 1 addition & 0 deletions src/docs/fleet-report.md
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,7 @@ title is `[selfreport] <class>: <short message>`.
| `proxy_tls_handshake_failed` | the hive proxy's TLS handshake with an upstream host failed (`proxy client tls handshake failed`, `transparent proxy tls handshake failed`, `copilot sniff: client tls handshake failed`, `crux/proxy/github_proxy_connect.go`, `crux/proxy/github_proxy_copilot.go`) — typically a certificate validation failure or encryption mismatch on an agent-requested host, repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium |
| `review_evidence_write_failed` | the github-client failed to write a review evidence bundle to persistent storage (`review evidence: could not write bundle`, `pkg/github`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium |
| `selfreport_cleared_comment_failed` | the fleet report publisher failed to post the "condition cleared" comment on a self-report issue (`fleet report: cleared comment failed`, `pkg/github`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium |
| `inference_caller_unidentified` | the inference translator refused a caller whose identity could not be established (`inference request from unidentified caller refused`, `crux/proxy/github_proxy_inference.go`). The request remains rejected with HTTP 401; this is distinct from backend credential failures. Check the per-agent UID map and socket attribution for the calling process; the log carries the remote address, method and path. Log-fed: ≥5 refusals in 10 min, clears after 20 quiet minutes | medium |
| `inference_upstream_error` | a failed or non-2xx response from the configured inference backend (LiteLLM, vLLM/llm-d) on the Anthropic-to-OpenAI translator or MITM reroute path (`inference upstream error` / `inference upstream failed`, `crux/proxy/github_proxy_inference.go`), repeated ≥5 times in 10 min — covers transport failures and statuses other than the dedicated 401/403 auth signal. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium |
| `sha_poll_commit_message_unavailable` | the SHA poller's commit-message fetch got a non-200 from GitHub (typically a 403 rate limit or token scope; `SHA poll: commit message fetch non-200`, `pkg/hub/saas_sha_poller.go`). The poller backs off per SHA and keeps serving the SHA without a message. The WARN itself is deduped to once per hour per (SHA, status) (`warnGitHubFetchOnce`), so one stuck commit logs one line and never trips this class; it is reported only when ≥5 distinct SHA/status pairs fail within 10 min (a broad outage such as an exhausted rate limit or a revoked token). Its main job is keeping the residual WARN out of `unclassified_error_repeated`. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low |
| `tmux_session_check_failed` | the agent's tmux session `has-session` check failed (`tmux has-session failed`, `crux/agent/manager.go`) — typically the session was lost or the tmux server is unhealthy, repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium |
Expand Down
2 changes: 2 additions & 0 deletions src/pkg/fleetreport/runtime.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ const (
ClassGitHubAppRepoNotCovered = "github_app_repo_not_covered"
ClassProxyConnectDialFailed = "proxy_connect_dial_failed"
ClassInferenceUpstreamError = "inference_upstream_error"
ClassInferenceCallerUnknown = "inference_caller_unidentified"
ClassSHAPollCommitMsgUnavail = "sha_poll_commit_message_unavailable"
ClassAgentLoginRequired = "agent_login_required"
ClassPRRequestHeadUnpushed = "pr_request_head_unpushed"
Expand Down Expand Up @@ -189,6 +190,7 @@ var Classes = map[string]ClassSpec{
ClassSelfReportClearedFailed: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"fleet report: cleared comment failed"}},
ClassGitHubAppRepoNotCovered: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"installation does not cover every configured repo", "state=repo-not-covered"}},
ClassProxyConnectDialFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: connect dial failed"}},
ClassInferenceCallerUnknown: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference request from unidentified caller refused"}},
ClassInferenceUpstreamError: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference upstream error", "inference upstream failed"}},
ClassSHAPollCommitMsgUnavail: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"sha poll: commit message fetch non-200"}},
ClassAgentLoginRequired: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"login required detected"}},
Expand Down
1 change: 1 addition & 0 deletions src/pkg/fleetreport/runtime_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -357,6 +357,7 @@ func TestEveryRuntimeClassTriggerAndNonTriggerThresholds(t *testing.T) {
{class: ClassGitHubAppRepoNotCovered, message: "github app recheck: installation does not cover every configured repo"},
{class: ClassGitHubAppNotInstalled, message: "github app installation id is unset — per-agent scoped token NOT delivered"},
{class: ClassProxyConnectDialFailed, message: "proxy: CONNECT dial failed"},
{class: ClassInferenceCallerUnknown, message: "inference request from unidentified caller refused"},
{class: ClassInferenceUpstreamError, message: "inference upstream error"},
{class: ClassSHAPollCommitMsgUnavail, message: "SHA poll: commit message fetch non-200"},
{class: ClassAgentLoginRequired, message: "login required detected"},
Expand Down
48 changes: 48 additions & 0 deletions src/pkg/fleetreport/slogfeed/slogfeed_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -564,3 +564,51 @@ func TestGitHubRateLimitStormLogsWithoutFleetReport(t *testing.T) {
t.Fatalf("non-rate-limit failures must remain reportable: %+v", *seen)
}
}

// #15333: keep repeated translator identity refusals visible under their own
// gateway class, without treating them as backend credential failures.
func TestInferenceCallerUnidentifiedFeedThresholdAndClear(t *testing.T) {
now := time.Date(2026, 10, 29, 6, 26, 0, 0, time.UTC)
tr := fleetreport.NewRuntimeTracker(func() time.Time { return now })
var out bytes.Buffer
logger := slog.New(NewWithObserver(slog.NewJSONHandler(&out, nil), tr.Observe))
const msg = "inference request from unidentified caller refused"
warn := func() {
logger.Warn(msg, "remote", "127.0.0.1:47724", "method", "POST", "path", "/v1/messages")
}
for i := 0; i < 4; i++ {
warn()
now = now.Add(time.Minute)
if ev := tr.Evidence(); len(ev) != 0 {
t.Fatalf("below threshold produced evidence: %+v", ev)
}
}
warn()
ev := tr.Evidence()
if len(ev) != 1 || ev[0].Class != fleetreport.ClassInferenceCallerUnknown || ev[0].Count != 5 || ev[0].Severity != "medium" || ev[0].Component != "inference-gateway" {
t.Fatalf("repeated identity refusals = %+v", ev)
}
for _, want := range []string{"remote=127.0.0.1:47724", "method=POST", "path=/v1/messages"} {
if !strings.Contains(ev[0].Diagnostics, want) {
t.Errorf("diagnostics %q missing %q", ev[0].Diagnostics, want)
}
}
if n := strings.Count(out.String(), "\n"); n != 5 {
t.Fatalf("wrapped handler saw %d records, want 5", n)
}
now = now.Add(21 * time.Minute)
if ev := tr.Evidence(); len(ev) != 0 {
t.Fatalf("idle identity refusals still reported: %+v", ev)
}
cleared := tr.TakeCleared()
if len(cleared) != 1 || cleared[0].Class != fleetreport.ClassInferenceCallerUnknown {
t.Fatalf("cleared = %+v", cleared)
}
// Other translator failures must still reach their existing classes.
logger2, seen, _ := newFeed(t)
logger2.Warn("inference upstream failed")
logger2.Warn("inference no route", "agent", "scanner")
if len(*seen) != 2 || (*seen)[0].class != fleetreport.ClassInferenceUpstreamError || (*seen)[1].class != fleetreport.ClassUnclassifiedRepeated {
t.Fatalf("unrelated inference failures = %+v", *seen)
}
}
Loading