diff --git a/changelog.d/fixed-15391.md b/changelog.d/fixed-15391.md new file mode 100644 index 0000000000..b435cd3119 --- /dev/null +++ b/changelog.d/fixed-15391.md @@ -0,0 +1 @@ +- The selfreport classifier now recognises the proxy's `plain HTTP forward refused — destination is not globally routable` WARN as its own log-fed class, `proxy_plain_forward_refused`, instead of letting repeats fall into the generic `unclassified_error_repeated` issue. The refusal is an expected policy decision (a client ignored its proxy bypass for a private address) and is still counted as a violation. (#15391) diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index e76b825dda..ebe7aa8a80 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -156,6 +156,7 @@ title is `[selfreport] : `. | `selfreport_cleared_comment_failed` | the fleet report publisher failed to post the "condition cleared" comment on a self-report issue (`fleet report: cleared comment failed`, `pkg/github`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `inference_caller_unidentified` | the inference translator refused a caller whose identity could not be established (`inference request from unidentified caller refused`, `crux/proxy/github_proxy_inference.go`). The request remains rejected with HTTP 401; this is distinct from backend credential failures. Check the per-agent UID map and socket attribution for the calling process; the log carries the remote address, method and path. Log-fed: ≥5 refusals in 10 min, clears after 20 quiet minutes | medium | | `inference_upstream_error` | a failed or non-2xx response from the configured inference backend (LiteLLM, vLLM/llm-d) on the Anthropic-to-OpenAI translator or MITM reroute path (`inference upstream error` / `inference upstream failed`, `crux/proxy/github_proxy_inference.go`), repeated ≥5 times in 10 min — covers transport failures and statuses other than the dedicated 401/403 auth signal. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | +| `proxy_plain_forward_refused` | the hive proxy refused a plain-HTTP forward to a non-globally-routable destination (`proxy: plain HTTP forward refused`, `crux/proxy/github_proxy_connect.go`) — a client ignored its proxy bypass for a private address (e.g. the Docker bridge gateway). Log-fed: same ≥5-in-10-min repeat threshold and 20-minute idle clear as `agent_pane_timeout`. Operator diagnostics: fix the client's `NO_PROXY` or target address; the refusal is counted as a proxy violation | low | | `sha_poll_commit_message_unavailable` | the SHA poller's commit-message fetch got a non-200 from GitHub (typically a 403 rate limit or token scope; `SHA poll: commit message fetch non-200`, `pkg/hub/saas_sha_poller.go`). The poller backs off per SHA and keeps serving the SHA without a message. The WARN itself is deduped to once per hour per (SHA, status) (`warnGitHubFetchOnce`), so one stuck commit logs one line and never trips this class; it is reported only when ≥5 distinct SHA/status pairs fail within 10 min (a broad outage such as an exhausted rate limit or a revoked token). Its main job is keeping the residual WARN out of `unclassified_error_repeated`. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | | `tmux_session_check_failed` | the agent's tmux session `has-session` check failed (`tmux has-session failed`, `crux/agent/manager.go`) — typically the session was lost or the tmux server is unhealthy, repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `transcript_scan_slow` | a Copilot transcript activity scan took longer than its interval (`transcript activity scan outlived its interval`, `crux/agent/busy_visibility.go`) — agent liveness reads may lag. Log-fed: same ≥5-in-10-min repeat threshold and 20-minute idle clear as `agent_pane_timeout`. Operator diagnostics: check the agent's transcript/session volume; it is bounded by concurrency, so repeated hits indicate slow disk or very large session directories | low | diff --git a/src/pkg/fleetreport/runtime.go b/src/pkg/fleetreport/runtime.go index 553f1d3e0e..ae819ce52a 100644 --- a/src/pkg/fleetreport/runtime.go +++ b/src/pkg/fleetreport/runtime.go @@ -77,6 +77,7 @@ const ( ClassSelfReportClearedFailed = "selfreport_cleared_comment_failed" ClassGitHubAppRepoNotCovered = "github_app_repo_not_covered" ClassProxyConnectDialFailed = "proxy_connect_dial_failed" + ClassProxyPlainForwardRefused = "proxy_plain_forward_refused" ClassInferenceUpstreamError = "inference_upstream_error" ClassInferenceCallerUnknown = "inference_caller_unidentified" ClassSHAPollCommitMsgUnavail = "sha_poll_commit_message_unavailable" @@ -191,6 +192,7 @@ var Classes = map[string]ClassSpec{ ClassSelfReportClearedFailed: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"fleet report: cleared comment failed"}}, ClassGitHubAppRepoNotCovered: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"installation does not cover every configured repo", "state=repo-not-covered"}}, ClassProxyConnectDialFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: connect dial failed"}}, + ClassProxyPlainForwardRefused: {Component: "github-proxy", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: plain http forward refused"}}, ClassInferenceCallerUnknown: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference request from unidentified caller refused"}}, ClassInferenceUpstreamError: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference upstream error", "inference upstream failed"}}, ClassSHAPollCommitMsgUnavail: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"sha poll: commit message fetch non-200"}}, diff --git a/src/pkg/fleetreport/runtime_test.go b/src/pkg/fleetreport/runtime_test.go index cfd85aae66..06d81e0dd4 100644 --- a/src/pkg/fleetreport/runtime_test.go +++ b/src/pkg/fleetreport/runtime_test.go @@ -358,6 +358,7 @@ func TestEveryRuntimeClassTriggerAndNonTriggerThresholds(t *testing.T) { {class: ClassGitHubAppRepoNotCovered, message: "github app recheck: installation does not cover every configured repo"}, {class: ClassGitHubAppNotInstalled, message: "github app installation id is unset — per-agent scoped token NOT delivered"}, {class: ClassProxyConnectDialFailed, message: "proxy: CONNECT dial failed"}, + {class: ClassProxyPlainForwardRefused, message: "proxy: plain HTTP forward refused — destination is not globally routable"}, {class: ClassInferenceCallerUnknown, message: "inference request from unidentified caller refused"}, {class: ClassInferenceUpstreamError, message: "inference upstream error"}, {class: ClassSHAPollCommitMsgUnavail, message: "SHA poll: commit message fetch non-200"}, @@ -1040,6 +1041,7 @@ func TestRecurringRuntimeMessagesAreClassified(t *testing.T) { "fleet report: cleared comment failed": ClassSelfReportClearedFailed, "copilot sniff: client TLS handshake failed": ClassProxyTLSHandshakeFailed, "github app recheck: installation does not cover every configured repo": ClassGitHubAppRepoNotCovered, + "proxy: plain HTTP forward refused — destination is not globally routable": ClassProxyPlainForwardRefused, "proxy: CONNECT dial failed": ClassProxyConnectDialFailed, "inference upstream error": ClassInferenceUpstreamError, "copilot transcript activity scan outlived its interval; agent liveness reads may lag": ClassTranscriptScanSlow,