From 309e14369912719d16a1eab41046400701693472 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 05:51:21 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20fix:=20stop=20reporting=20GitHub?= =?UTF-8?q?=20502/503/504=20fetch=20failures=20as=20repeated=20runtime=20e?= =?UTF-8?q?rrors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A GitHub-side 503 on fetch-pr is transient and retried on the next sweep tick; the slog feed counted it as unclassified_error_repeated like the rate-limit storm in #15106. Apply the same exclusion for gateway-class statuses; 500 stays reportable. Fixes #15365 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...d-15365-selfreport-upstream-unavailable.md | 1 + src/docs/fleet-report.md | 2 +- src/pkg/fleetreport/fleetreport.go | 28 +++++++++++++++++ src/pkg/fleetreport/runtime_test.go | 21 +++++++++++++ src/pkg/fleetreport/slogfeed/slogfeed.go | 19 +++++++++++- src/pkg/fleetreport/slogfeed/slogfeed_test.go | 31 +++++++++++++++++++ 6 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 changelog.d/fixed-15365-selfreport-upstream-unavailable.md diff --git a/changelog.d/fixed-15365-selfreport-upstream-unavailable.md b/changelog.d/fixed-15365-selfreport-upstream-unavailable.md new file mode 100644 index 0000000000..31bfa91d41 --- /dev/null +++ b/changelog.d/fixed-15365-selfreport-upstream-unavailable.md @@ -0,0 +1 @@ +- Stop filing `unclassified_error_repeated` self-reports for GitHub 502/503/504 transient failures (e.g. the self-authored automerge sweep's `fetch-pr` skip); the caller retries on its next tick and the WARN stays in local logs ([#15365](https://github.com/hivecommons/hive/issues/15365)). diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index eb66ba318e..abe4109fe1 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -159,7 +159,7 @@ title is `[selfreport] : `. | `sha_poll_commit_message_unavailable` | the SHA poller's commit-message fetch got a non-200 from GitHub (typically a 403 rate limit or token scope; `SHA poll: commit message fetch non-200`, `pkg/hub/saas_sha_poller.go`). The poller backs off per SHA and keeps serving the SHA without a message. The WARN itself is deduped to once per hour per (SHA, status) (`warnGitHubFetchOnce`), so one stuck commit logs one line and never trips this class; it is reported only when ≥5 distinct SHA/status pairs fail within 10 min (a broad outage such as an exhausted rate limit or a revoked token). Its main job is keeping the residual WARN out of `unclassified_error_repeated`. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | | `tmux_session_check_failed` | the agent's tmux session `has-session` check failed (`tmux has-session failed`, `crux/agent/manager.go`) — typically the session was lost or the tmux server is unhealthy, repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `transcript_scan_slow` | a Copilot transcript activity scan took longer than its interval (`transcript activity scan outlived its interval`, `crux/agent/busy_visibility.go`) — agent liveness reads may lag. Log-fed: same ≥5-in-10-min repeat threshold and 20-minute idle clear as `agent_pane_timeout`. Operator diagnostics: check the agent's transcript/session volume; it is bounded by concurrency, so repeated hits indicate slow disk or very large session directories | low | -| `unclassified_error_repeated` | **generic catch-all**: the same WARN/ERROR log line (numbers, hashes, paths and quoted strings collapsed) repeated ≥5 times in 10 min. Fed by a `slog.Handler` tee (`pkg/fleetreport/slogfeed`) on the hive's root logger, so a repeating defect is noticed even with no hook site. Skips benign lines (read/keepalive i/o timeouts, context cancellation, the permissions watcher's `fixed …` self-healing lines) and any message that already matches another class's keywords. Clears automatically after 20 quiet minutes (2× the window), so a "cleared" comment follows once the symptom stops | low | +| `unclassified_error_repeated` | **generic catch-all**: the same WARN/ERROR log line (numbers, hashes, paths and quoted strings collapsed) repeated ≥5 times in 10 min. Fed by a `slog.Handler` tee (`pkg/fleetreport/slogfeed`) on the hive's root logger, so a repeating defect is noticed even with no hook site. Skips benign lines (read/keepalive i/o timeouts, WARNs whose `error` is a GitHub rate-limit refusal or a 502/503/504 gateway failure — the caller retries next tick, context cancellation, the permissions watcher's `fixed …` self-healing lines) and any message that already matches another class's keywords. Clears automatically after 20 quiet minutes (2× the window), so a "cleared" comment follows once the symptom stops | low | The generic feed also skips the exact `review pending expired` message: it records the review planner's bounded retry, and a burst across unrelated PRs diff --git a/src/pkg/fleetreport/fleetreport.go b/src/pkg/fleetreport/fleetreport.go index 8ff66035cc..4158bd87ee 100644 --- a/src/pkg/fleetreport/fleetreport.go +++ b/src/pkg/fleetreport/fleetreport.go @@ -822,6 +822,34 @@ func IsGitHubRateLimited(s string) bool { return false } +// upstreamUnavailableSubstrings are lower-case fragments that mark a GitHub +// gateway-class transient failure (see IsGitHubUpstreamUnavailable). The +// status codes are anchored to go-github's " : 503 ..." shape so +// a PR number or page=503 cannot match; 500 is deliberately absent. +var upstreamUnavailableSubstrings = []string{ + ": 502 ", + ": 503 ", + ": 504 ", + "bad gateway", + "service unavailable", + "gateway timeout", + "gateway time-out", +} + +// IsGitHubUpstreamUnavailable reports whether s (a log message or an error +// string) is a GitHub 502/503/504 gateway-class failure. The caller retries +// on its next tick, so the generic repeated-error feed treats such lines as +// benign (hivecommons/hive#15365); the WARN stays in local logs. +func IsGitHubUpstreamUnavailable(s string) bool { + m := strings.ToLower(s) + " " + for _, sub := range upstreamUnavailableSubstrings { + if strings.Contains(m, sub) { + return true + } + } + return false +} + // IsBenignMessage reports whether a raw log message is noise the generic // repeated-error feed must not report: the read/keepalive i/o timeout rule // above applied to the whole line, GitHub rate-limit and pacing refusals diff --git a/src/pkg/fleetreport/runtime_test.go b/src/pkg/fleetreport/runtime_test.go index 9bc341ff96..b46a572bac 100644 --- a/src/pkg/fleetreport/runtime_test.go +++ b/src/pkg/fleetreport/runtime_test.go @@ -795,6 +795,27 @@ func TestIsGitHubRateLimited(t *testing.T) { } } +func TestIsGitHubUpstreamUnavailable(t *testing.T) { + for _, tc := range []struct { + in string + want bool + }{ + {"GET https://api.github.com/repos/o/r/pulls/1: 502 Bad Gateway []", true}, + {"GET https://api.github.com/repos/o/r/pulls/1336: 503 []", true}, + {"GET https://api.github.com/repos/o/r/pulls/1: 504 Gateway Timeout []", true}, + {"upstream: Service Unavailable", true}, + {"GET https://api.github.com/repos/o/r: 500 Internal Server Error", false}, + {"GET https://api.github.com/repos/o/r: 404 Not Found", false}, + {"GET https://api.github.com/repos/o/r/pulls/503: 404 Not Found", false}, + {"GET https://api.github.com/repos/o/r/pulls?page=503", false}, + {"", false}, + } { + if got := IsGitHubUpstreamUnavailable(tc.in); got != tc.want { + t.Errorf("IsGitHubUpstreamUnavailable(%q) = %v, want %v", tc.in, got, tc.want) + } + } +} + func TestIsBenignMessageReviewPendingExpiry(t *testing.T) { for _, msg := range []string{"review pending expired", " REVIEW PENDING EXPIRED "} { if !IsBenignMessage(msg) { diff --git a/src/pkg/fleetreport/slogfeed/slogfeed.go b/src/pkg/fleetreport/slogfeed/slogfeed.go index 6f2152a444..fe5459a271 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed.go @@ -101,7 +101,7 @@ func Key(msg string) string { } func (h *Handler) feed(r slog.Record) { - if expectedKickHold(r) || rateLimitedError(r) { + if expectedKickHold(r) || rateLimitedError(r) || upstreamUnavailableError(r) { return } class := fleetreport.ClassUnclassifiedRepeated @@ -160,6 +160,23 @@ func rateLimitedError(r slog.Record) bool { return limited } +// upstreamUnavailableError reports whether the record's error attribute is a +// GitHub 502/503/504 (fleetreport.IsGitHubUpstreamUnavailable). Callers such +// as the self-authored automerge sweep ("fetch-pr") log a per-site message and +// retry on their next tick, so a GitHub-side blip is not a hive defect +// (hivecommons/hive#15365). The WARN stays in local logs. +func upstreamUnavailableError(r slog.Record) bool { + unavailable := false + r.Attrs(func(a slog.Attr) bool { + if a.Key == "error" && fleetreport.IsGitHubUpstreamUnavailable(a.Value.String()) { + unavailable = true + return false + } + return true + }) + return unavailable +} + // Classifier errors stay in local logs only: transport errors can expose the // private endpoint in both the URL and the nested DNS/dial error. func redactClassifierError(a slog.Attr) slog.Attr { diff --git a/src/pkg/fleetreport/slogfeed/slogfeed_test.go b/src/pkg/fleetreport/slogfeed/slogfeed_test.go index c8a806f157..9b0e9bee16 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed_test.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed_test.go @@ -565,6 +565,37 @@ func TestGitHubRateLimitStormLogsWithoutFleetReport(t *testing.T) { } } +// #15365: a GitHub 502/503/504 on a fetch is transient and retried next tick. +func TestGitHubUpstreamUnavailableLogsWithoutFleetReport(t *testing.T) { + cases := []struct { + msg string + args []any + }{ + {"self-authored automerge sweep skipped PR", []any{"repo", "kubestellar-mcp", "pr", 1336, "reason", "fetch-pr", "error", "GET https://api.github.com/repos/o/r/pulls/1336: 503 []"}}, + {"failed to fetch check runs", []any{"error", "GET https://api.github.com/repos/o/r/commits/abc/check-runs: 502 Bad Gateway []"}}, + {"failed to fetch check runs", []any{"error", "GET https://api.github.com/repos/o/r/commits/abc/check-runs: 504 Gateway Timeout []"}}, + } + for _, tc := range cases { + logger, seen, out := newFeed(t) + for i := 0; i < 8; i++ { + logger.Warn(tc.msg, tc.args...) + } + if len(*seen) != 0 { + t.Errorf("%q: upstream-unavailable WARN produced observations: %+v", tc.msg, *seen) + } + if n := strings.Count(out.String(), "\n"); n != 8 { + t.Errorf("%q: wrapped handler saw %d records, want 8", tc.msg, n) + } + } + + logger, seen, _ := newFeed(t) + logger.Warn("self-authored automerge sweep skipped PR", "error", "GET https://api.github.com/repos/o/r/pulls/1336: 500 Internal Server Error []") + logger.Warn("self-authored automerge sweep skipped PR", "error", "GET https://api.github.com/repos/o/r/pulls/1336: 403 Resource not accessible by integration") + if len(*seen) != 2 { + t.Fatalf("non-gateway failures must remain reportable: %+v", *seen) + } +} + // #15333: keep repeated translator identity refusals visible under their own // gateway class, without treating them as backend credential failures. func TestInferenceCallerUnidentifiedFeedThresholdAndClear(t *testing.T) {