Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-15365-selfreport-upstream-unavailable.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Stop filing `unclassified_error_repeated` self-reports for GitHub 502/503/504 transient failures (e.g. the self-authored automerge sweep's `fetch-pr` skip); the caller retries on its next tick and the WARN stays in local logs ([#15365](https://github.com/hivecommons/hive/issues/15365)).
2 changes: 1 addition & 1 deletion src/docs/fleet-report.md
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ title is `[selfreport] <class>: <short message>`.
| `sha_poll_commit_message_unavailable` | the SHA poller's commit-message fetch got a non-200 from GitHub (typically a 403 rate limit or token scope; `SHA poll: commit message fetch non-200`, `pkg/hub/saas_sha_poller.go`). The poller backs off per SHA and keeps serving the SHA without a message. The WARN itself is deduped to once per hour per (SHA, status) (`warnGitHubFetchOnce`), so one stuck commit logs one line and never trips this class; it is reported only when ≥5 distinct SHA/status pairs fail within 10 min (a broad outage such as an exhausted rate limit or a revoked token). Its main job is keeping the residual WARN out of `unclassified_error_repeated`. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low |
| `tmux_session_check_failed` | the agent's tmux session `has-session` check failed (`tmux has-session failed`, `crux/agent/manager.go`) — typically the session was lost or the tmux server is unhealthy, repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium |
| `transcript_scan_slow` | a Copilot transcript activity scan took longer than its interval (`transcript activity scan outlived its interval`, `crux/agent/busy_visibility.go`) — agent liveness reads may lag. Log-fed: same ≥5-in-10-min repeat threshold and 20-minute idle clear as `agent_pane_timeout`. Operator diagnostics: check the agent's transcript/session volume; it is bounded by concurrency, so repeated hits indicate slow disk or very large session directories | low |
| `unclassified_error_repeated` | **generic catch-all**: the same WARN/ERROR log line (numbers, hashes, paths and quoted strings collapsed) repeated ≥5 times in 10 min. Fed by a `slog.Handler` tee (`pkg/fleetreport/slogfeed`) on the hive's root logger, so a repeating defect is noticed even with no hook site. Skips benign lines (read/keepalive i/o timeouts, context cancellation, the permissions watcher's `fixed …` self-healing lines) and any message that already matches another class's keywords. Clears automatically after 20 quiet minutes (2× the window), so a "cleared" comment follows once the symptom stops | low |
| `unclassified_error_repeated` | **generic catch-all**: the same WARN/ERROR log line (numbers, hashes, paths and quoted strings collapsed) repeated ≥5 times in 10 min. Fed by a `slog.Handler` tee (`pkg/fleetreport/slogfeed`) on the hive's root logger, so a repeating defect is noticed even with no hook site. Skips benign lines (read/keepalive i/o timeouts, WARNs whose `error` is a GitHub rate-limit refusal or a 502/503/504 gateway failure — the caller retries next tick, context cancellation, the permissions watcher's `fixed …` self-healing lines) and any message that already matches another class's keywords. Clears automatically after 20 quiet minutes (2× the window), so a "cleared" comment follows once the symptom stops | low |

The generic feed also skips the exact `review pending expired` message: it
records the review planner's bounded retry, and a burst across unrelated PRs
Expand Down
28 changes: 28 additions & 0 deletions src/pkg/fleetreport/fleetreport.go
Original file line number Diff line number Diff line change
Expand Up @@ -822,6 +822,34 @@ func IsGitHubRateLimited(s string) bool {
return false
}

// upstreamUnavailableSubstrings are lower-case fragments that mark a GitHub
// gateway-class transient failure (see IsGitHubUpstreamUnavailable). The
// status codes are anchored to go-github's "<METHOD> <url>: 503 ..." shape so
// a PR number or page=503 cannot match; 500 is deliberately absent.
var upstreamUnavailableSubstrings = []string{
": 502 ",
": 503 ",
": 504 ",
"bad gateway",
"service unavailable",
"gateway timeout",
"gateway time-out",
}

// IsGitHubUpstreamUnavailable reports whether s (a log message or an error
// string) is a GitHub 502/503/504 gateway-class failure. The caller retries
// on its next tick, so the generic repeated-error feed treats such lines as
// benign (hivecommons/hive#15365); the WARN stays in local logs.
func IsGitHubUpstreamUnavailable(s string) bool {
m := strings.ToLower(s) + " "
for _, sub := range upstreamUnavailableSubstrings {
if strings.Contains(m, sub) {
return true
}
}
return false
}

// IsBenignMessage reports whether a raw log message is noise the generic
// repeated-error feed must not report: the read/keepalive i/o timeout rule
// above applied to the whole line, GitHub rate-limit and pacing refusals
Expand Down
21 changes: 21 additions & 0 deletions src/pkg/fleetreport/runtime_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -795,6 +795,27 @@ func TestIsGitHubRateLimited(t *testing.T) {
}
}

func TestIsGitHubUpstreamUnavailable(t *testing.T) {
for _, tc := range []struct {
in string
want bool
}{
{"GET https://api.github.com/repos/o/r/pulls/1: 502 Bad Gateway []", true},
{"GET https://api.github.com/repos/o/r/pulls/1336: 503 []", true},
{"GET https://api.github.com/repos/o/r/pulls/1: 504 Gateway Timeout []", true},
{"upstream: Service Unavailable", true},
{"GET https://api.github.com/repos/o/r: 500 Internal Server Error", false},
{"GET https://api.github.com/repos/o/r: 404 Not Found", false},
{"GET https://api.github.com/repos/o/r/pulls/503: 404 Not Found", false},
{"GET https://api.github.com/repos/o/r/pulls?page=503", false},
{"", false},
} {
if got := IsGitHubUpstreamUnavailable(tc.in); got != tc.want {
t.Errorf("IsGitHubUpstreamUnavailable(%q) = %v, want %v", tc.in, got, tc.want)
}
}
}

func TestIsBenignMessageReviewPendingExpiry(t *testing.T) {
for _, msg := range []string{"review pending expired", " REVIEW PENDING EXPIRED "} {
if !IsBenignMessage(msg) {
Expand Down
19 changes: 18 additions & 1 deletion src/pkg/fleetreport/slogfeed/slogfeed.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ func Key(msg string) string {
}

func (h *Handler) feed(r slog.Record) {
if expectedKickHold(r) || rateLimitedError(r) {
if expectedKickHold(r) || rateLimitedError(r) || upstreamUnavailableError(r) {
return
}
class := fleetreport.ClassUnclassifiedRepeated
Expand Down Expand Up @@ -160,6 +160,23 @@ func rateLimitedError(r slog.Record) bool {
return limited
}

// upstreamUnavailableError reports whether the record's error attribute is a
// GitHub 502/503/504 (fleetreport.IsGitHubUpstreamUnavailable). Callers such
// as the self-authored automerge sweep ("fetch-pr") log a per-site message and
// retry on their next tick, so a GitHub-side blip is not a hive defect
// (hivecommons/hive#15365). The WARN stays in local logs.
func upstreamUnavailableError(r slog.Record) bool {
unavailable := false
r.Attrs(func(a slog.Attr) bool {
if a.Key == "error" && fleetreport.IsGitHubUpstreamUnavailable(a.Value.String()) {
unavailable = true
return false
}
return true
})
return unavailable
}

// Classifier errors stay in local logs only: transport errors can expose the
// private endpoint in both the URL and the nested DNS/dial error.
func redactClassifierError(a slog.Attr) slog.Attr {
Expand Down
31 changes: 31 additions & 0 deletions src/pkg/fleetreport/slogfeed/slogfeed_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -565,6 +565,37 @@ func TestGitHubRateLimitStormLogsWithoutFleetReport(t *testing.T) {
}
}

// #15365: a GitHub 502/503/504 on a fetch is transient and retried next tick.
func TestGitHubUpstreamUnavailableLogsWithoutFleetReport(t *testing.T) {
cases := []struct {
msg string
args []any
}{
{"self-authored automerge sweep skipped PR", []any{"repo", "kubestellar-mcp", "pr", 1336, "reason", "fetch-pr", "error", "GET https://api.github.com/repos/o/r/pulls/1336: 503 []"}},
{"failed to fetch check runs", []any{"error", "GET https://api.github.com/repos/o/r/commits/abc/check-runs: 502 Bad Gateway []"}},
{"failed to fetch check runs", []any{"error", "GET https://api.github.com/repos/o/r/commits/abc/check-runs: 504 Gateway Timeout []"}},
}
for _, tc := range cases {
logger, seen, out := newFeed(t)
for i := 0; i < 8; i++ {
logger.Warn(tc.msg, tc.args...)
}
if len(*seen) != 0 {
t.Errorf("%q: upstream-unavailable WARN produced observations: %+v", tc.msg, *seen)
}
if n := strings.Count(out.String(), "\n"); n != 8 {
t.Errorf("%q: wrapped handler saw %d records, want 8", tc.msg, n)
}
}

logger, seen, _ := newFeed(t)
logger.Warn("self-authored automerge sweep skipped PR", "error", "GET https://api.github.com/repos/o/r/pulls/1336: 500 Internal Server Error []")
logger.Warn("self-authored automerge sweep skipped PR", "error", "GET https://api.github.com/repos/o/r/pulls/1336: 403 Resource not accessible by integration")
if len(*seen) != 2 {
t.Fatalf("non-gateway failures must remain reportable: %+v", *seen)
}
}

// #15333: keep repeated translator identity refusals visible under their own
// gateway class, without treating them as backend credential failures.
func TestInferenceCallerUnidentifiedFeedThresholdAndClear(t *testing.T) {
Expand Down
Loading