diff --git a/changelog.d/fixed-selfreport-rate-limit-noise.md b/changelog.d/fixed-selfreport-rate-limit-noise.md new file mode 100644 index 0000000000..94a07620ea --- /dev/null +++ b/changelog.d/fixed-selfreport-rate-limit-noise.md @@ -0,0 +1 @@ +- Fleet selfreports no longer file an unclassified-runtime-error issue per caller when GitHub rate limiting or the hive's own request pacing makes many callers log the same transient failure; the WARNs stay in local logs (#15106). diff --git a/src/pkg/fleetreport/fleetreport.go b/src/pkg/fleetreport/fleetreport.go index 8dd2fe62ae..b28da234a0 100644 --- a/src/pkg/fleetreport/fleetreport.go +++ b/src/pkg/fleetreport/fleetreport.go @@ -779,9 +779,42 @@ var benignSubstrings = []string{ "forward-merge guard: refusing to squash", } +// rateLimitSubstrings are lower-case fragments that mark a GitHub rate-limit +// refusal or the hive's own pacing shedding a request while the caution +// window is engaged (see IsGitHubRateLimited). One rate-limit episode makes +// every caller log its own WARN — check runs, statuses, permissions, repo +// enumeration, advisories — and each distinct message used to cross the +// repeat threshold as its own unclassified_error_repeated issue +// (hivecommons/hive#15106). The condition is self-limiting: the client +// backs off and the callers retry on their next tick. +var rateLimitSubstrings = []string{ + "api secondary rate limit exceeded", + "exceeded a secondary rate limit", + "api rate limit exceeded", + "api rate limit of", + "not making remote request", + "429 too many requests", + "github pacing: request shed", + "rate-limit caution window is engaged", +} + +// IsGitHubRateLimited reports whether s (a log message or an error string) +// is a GitHub rate-limit refusal or a pacing shed. The generic repeated-error +// feed treats such lines as benign; the WARN itself stays in local logs. +func IsGitHubRateLimited(s string) bool { + m := strings.ToLower(s) + for _, sub := range rateLimitSubstrings { + if strings.Contains(m, sub) { + return true + } + } + return false +} + // IsBenignMessage reports whether a raw log message is noise the generic // repeated-error feed must not report: the read/keepalive i/o timeout rule -// above applied to the whole line, every benignSubstrings entry, and the +// above applied to the whole line, GitHub rate-limit and pacing refusals +// (IsGitHubRateLimited), every benignSubstrings entry, and the // exact-message exclusions matched in the function body below. func IsBenignMessage(msg string) bool { m := strings.ToLower(msg) @@ -814,7 +847,7 @@ func IsBenignMessage(msg string) bool { "[contribute-ws] refusing task: daily rate limit reached": return true } - if benignIOTimeout(m, m) { + if benignIOTimeout(m, m) || IsGitHubRateLimited(m) { return true } for _, s := range benignSubstrings { diff --git a/src/pkg/fleetreport/runtime_test.go b/src/pkg/fleetreport/runtime_test.go index c8ec623fa2..3cb02fbe28 100644 --- a/src/pkg/fleetreport/runtime_test.go +++ b/src/pkg/fleetreport/runtime_test.go @@ -763,6 +763,35 @@ func TestIsBenignMessage(t *testing.T) { } } +func TestIsGitHubRateLimited(t *testing.T) { + for _, m := range []string{ + "GET https://api.github.com/repos/o/r/pulls/798: 403 API secondary rate limit exceeded until 2026-10-28 17:42:28 -0400 EDT, not making remote request.", + "GET https://api.github.com/repos/o/r: 403 API rate limit of 5000 still exceeded until 2026-10-28 17:42:28 -0400 EDT, not making remote request.", + "GET https://api.github.com/repos/o/r: 403 API rate limit exceeded for installation ID 1.", + "POST https://api.github.com/repos/o/r/issues: 403 You have exceeded a secondary rate limit. Please wait a few minutes before you try again.", + "GET https://api.github.com/repos/o/r: 429 Too Many Requests", + `Get "https://api.github.com/repos/o/r": github pacing: request shed while rate-limit caution window is engaged (class low, queue 8.209s ahead, limit 5s)`, + "github pacing: low-priority work shed while rate-limit caution window is engaged", + } { + if !IsGitHubRateLimited(m) { + t.Errorf("%q not recognised as rate limited", m) + } + if !IsBenignMessage(m) { + t.Errorf("%q not benign", m) + } + } + for _, m := range []string{ + "GET https://api.github.com/repos/o/r: 403 Resource not accessible by integration", + "GET https://api.github.com/repos/o/r: 404 Not Found", + "failed to fetch check runs", + "github pacing: queue accounting failed", + } { + if IsGitHubRateLimited(m) { + t.Errorf("%q wrongly recognised as rate limited", m) + } + } +} + func TestIsBenignMessageReviewPendingExpiry(t *testing.T) { for _, msg := range []string{"review pending expired", " REVIEW PENDING EXPIRED "} { if !IsBenignMessage(msg) { diff --git a/src/pkg/fleetreport/slogfeed/slogfeed.go b/src/pkg/fleetreport/slogfeed/slogfeed.go index d261261cff..6f2152a444 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed.go @@ -101,7 +101,7 @@ func Key(msg string) string { } func (h *Handler) feed(r slog.Record) { - if expectedKickHold(r) { + if expectedKickHold(r) || rateLimitedError(r) { return } class := fleetreport.ClassUnclassifiedRepeated @@ -141,6 +141,25 @@ func expectedKickHold(r slog.Record) bool { return held } +// rateLimitedError reports whether the record's error attribute is a GitHub +// rate-limit refusal or a pacing shed (fleetreport.IsGitHubRateLimited). The +// message names the caller ("failed to fetch check runs", "stage labels: +// check digest failed", ...) and differs per call site, so one rate-limit +// episode would otherwise file one unclassified issue per caller +// (hivecommons/hive#15106). The client backs off and callers retry on their +// next tick; the WARN stays in local logs. +func rateLimitedError(r slog.Record) bool { + limited := false + r.Attrs(func(a slog.Attr) bool { + if a.Key == "error" && fleetreport.IsGitHubRateLimited(a.Value.String()) { + limited = true + return false + } + return true + }) + return limited +} + // Classifier errors stay in local logs only: transport errors can expose the // private endpoint in both the URL and the nested DNS/dial error. func redactClassifierError(a slog.Attr) slog.Attr { diff --git a/src/pkg/fleetreport/slogfeed/slogfeed_test.go b/src/pkg/fleetreport/slogfeed/slogfeed_test.go index 6ba2adc1b6..a69a48a7cd 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed_test.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed_test.go @@ -517,3 +517,50 @@ func TestClassifierErrorsStayInLocalLogs(t *testing.T) { } } } + +// One GitHub rate-limit episode made every caller log its own WARN, and each +// distinct message was filed as its own unclassified_error_repeated issue +// (hivecommons/hive#15106 and siblings). The WARNs stay in local logs; none +// is observed by the generic feed. +func TestGitHubRateLimitStormLogsWithoutFleetReport(t *testing.T) { + const secondary = ": 403 API secondary rate limit exceeded until 2026-10-28 17:42:28 -0400 EDT, not making remote request." + const shed = ": github pacing: request shed while rate-limit caution window is engaged (class low, queue 8.209s ahead, limit 5s)" + cases := []struct { + issue int + msg string + args []any + }{ + {15110, "stage labels: check digest failed", []any{"repo", "utah-packages", "pr", 797, "error", "GET https://api.github.com/repos/o/r/commits/abc/status?per_page=100" + secondary}}, + {15111, "GitHub refused repo enumeration transiently; serving last-good snapshot for this repo (marked stale)", []any{"repo", "website", "side", "issues", "error", "listing issues for o/website: GET https://api.github.com/repos/o/website/issues?page=1" + secondary}}, + {15112, "failed to fetch check runs", []any{"repo", "testsuite", "pr", 1037, "error", "GET https://api.github.com/repos/o/r/commits/abc/check-runs?per_page=100" + secondary}}, + {15113, "could not check watched repo has_issues", []any{"repo", "o/website", "error", `Get "https://api.github.com/repos/o/website"` + shed}}, + {15114, "advisory: issue state lookup failed", []any{"ref", "o/bluefin#961", "error", `reading o/bluefin#961: Get "https://api.github.com/repos/o/bluefin/issues/961"` + shed}}, + {15115, "stage labels: could not read the author's repository permission; adoption deferred", []any{"repo", "utah-packages", "pr", 798, "error", "GET https://api.github.com/repos/o/r/collaborators/u/permission" + secondary}}, + {15107, "github pacing: low-priority work shed while rate-limit caution window is engaged", []any{"caller", "review-request-watcher", "queue_eta", "6.882s", "budget", "5s"}}, + {15106, "stage labels: could not read maintainer_can_modify; keeping in-place adoption this sweep", []any{"repo", "utah-packages", "pr", 798, "error", "GET https://api.github.com/repos/o/r/pulls/798" + secondary}}, + {15109, "advisory: path existence check failed", []any{"path", "a/b.sh", "repo", "common", "error", `checking o/common/a/b.sh@abc: Get "https://api.github.com/repos/o/common/contents/a/b.sh?ref=abc"` + shed}}, + {0, "failed to fetch check runs", []any{"error", "GET https://api.github.com/repos/o/r/commits/abc/check-runs: 429 Too Many Requests"}}, + {0, "failed to fetch check runs", []any{"error", "GET https://api.github.com/repos/o/r/commits/abc/check-runs: 403 API rate limit of 5000 still exceeded until 2026-10-28 17:42:28 -0400 EDT, not making remote request."}}, + } + for _, tc := range cases { + logger, seen, out := newFeed(t) + for i := 0; i < 8; i++ { + logger.Warn(tc.msg, tc.args...) + } + if len(*seen) != 0 { + t.Errorf("#%d %q: rate-limited WARN produced observations: %+v", tc.issue, tc.msg, *seen) + } + if n := strings.Count(out.String(), "\n"); n != 8 { + t.Errorf("#%d %q: wrapped handler saw %d records, want 8", tc.issue, tc.msg, n) + } + } + + // The same callers failing for any other reason must remain reportable, + // including a plain 403 permission refusal. + logger, seen, _ := newFeed(t) + logger.Warn("failed to fetch check runs", "error", "GET https://api.github.com/repos/o/r/commits/abc/check-runs: 403 Resource not accessible by integration") + logger.Warn("stage labels: check digest failed", "error", "GET https://api.github.com/repos/o/r/commits/abc/status: 500 Internal Server Error") + if len(*seen) != 2 { + t.Fatalf("non-rate-limit failures must remain reportable: %+v", *seen) + } +}