From b9dc278ecb035eae1ca5656bdc5f6e322934182f Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:26:34 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20fix:=20classify=20rate-limit=20a?= =?UTF-8?q?nd=20pacing=20WARN=20storms=20as=20one=20benign=20selfreport=20?= =?UTF-8?q?class?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When GitHub engages a secondary rate limit, or the hive's own pacing sheds low-priority requests during the caution window, every caller (check runs, commit statuses, permission reads, repo enumeration, advisory lookups, has_issues probes) logs its own WARN. The slog feed keyed each distinct message separately, so a single self-limiting episode crossed the repeat threshold once per caller and filed ~10 unclassified_error_repeated "defect" issues (#15106 and siblings). The rate-limit marker lives in the record's error attribute, not the message, so IsBenignMessage alone never saw it. Add IsGitHubRateLimited next to the existing benign substrings, fold it into IsBenignMessage (covers the pacing shed notice itself), and have the feed skip records whose error attribute matches it, mirroring the existing expectedKickHold attribute check. The WARNs stay in local logs; other failures from the same callers (plain 403 permission refusals, 5xx) remain reportable. Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-selfreport-rate-limit-noise.md | 1 + src/pkg/fleetreport/fleetreport.go | 37 ++++++++++++++- src/pkg/fleetreport/runtime_test.go | 29 ++++++++++++ src/pkg/fleetreport/slogfeed/slogfeed.go | 21 ++++++++- src/pkg/fleetreport/slogfeed/slogfeed_test.go | 47 +++++++++++++++++++ 5 files changed, 132 insertions(+), 3 deletions(-) create mode 100644 changelog.d/fixed-selfreport-rate-limit-noise.md diff --git a/changelog.d/fixed-selfreport-rate-limit-noise.md b/changelog.d/fixed-selfreport-rate-limit-noise.md new file mode 100644 index 0000000000..94a07620ea --- /dev/null +++ b/changelog.d/fixed-selfreport-rate-limit-noise.md @@ -0,0 +1 @@ +- Fleet selfreports no longer file an unclassified-runtime-error issue per caller when GitHub rate limiting or the hive's own request pacing makes many callers log the same transient failure; the WARNs stay in local logs (#15106). diff --git a/src/pkg/fleetreport/fleetreport.go b/src/pkg/fleetreport/fleetreport.go index 8dd2fe62ae..b28da234a0 100644 --- a/src/pkg/fleetreport/fleetreport.go +++ b/src/pkg/fleetreport/fleetreport.go @@ -779,9 +779,42 @@ var benignSubstrings = []string{ "forward-merge guard: refusing to squash", } +// rateLimitSubstrings are lower-case fragments that mark a GitHub rate-limit +// refusal or the hive's own pacing shedding a request while the caution +// window is engaged (see IsGitHubRateLimited). One rate-limit episode makes +// every caller log its own WARN — check runs, statuses, permissions, repo +// enumeration, advisories — and each distinct message used to cross the +// repeat threshold as its own unclassified_error_repeated issue +// (hivecommons/hive#15106). The condition is self-limiting: the client +// backs off and the callers retry on their next tick. +var rateLimitSubstrings = []string{ + "api secondary rate limit exceeded", + "exceeded a secondary rate limit", + "api rate limit exceeded", + "api rate limit of", + "not making remote request", + "429 too many requests", + "github pacing: request shed", + "rate-limit caution window is engaged", +} + +// IsGitHubRateLimited reports whether s (a log message or an error string) +// is a GitHub rate-limit refusal or a pacing shed. The generic repeated-error +// feed treats such lines as benign; the WARN itself stays in local logs. +func IsGitHubRateLimited(s string) bool { + m := strings.ToLower(s) + for _, sub := range rateLimitSubstrings { + if strings.Contains(m, sub) { + return true + } + } + return false +} + // IsBenignMessage reports whether a raw log message is noise the generic // repeated-error feed must not report: the read/keepalive i/o timeout rule -// above applied to the whole line, every benignSubstrings entry, and the +// above applied to the whole line, GitHub rate-limit and pacing refusals +// (IsGitHubRateLimited), every benignSubstrings entry, and the // exact-message exclusions matched in the function body below. func IsBenignMessage(msg string) bool { m := strings.ToLower(msg) @@ -814,7 +847,7 @@ func IsBenignMessage(msg string) bool { "[contribute-ws] refusing task: daily rate limit reached": return true } - if benignIOTimeout(m, m) { + if benignIOTimeout(m, m) || IsGitHubRateLimited(m) { return true } for _, s := range benignSubstrings { diff --git a/src/pkg/fleetreport/runtime_test.go b/src/pkg/fleetreport/runtime_test.go index c8ec623fa2..3cb02fbe28 100644 --- a/src/pkg/fleetreport/runtime_test.go +++ b/src/pkg/fleetreport/runtime_test.go @@ -763,6 +763,35 @@ func TestIsBenignMessage(t *testing.T) { } } +func TestIsGitHubRateLimited(t *testing.T) { + for _, m := range []string{ + "GET https://api.github.com/repos/o/r/pulls/798: 403 API secondary rate limit exceeded until 2026-10-28 17:42:28 -0400 EDT, not making remote request.", + "GET https://api.github.com/repos/o/r: 403 API rate limit of 5000 still exceeded until 2026-10-28 17:42:28 -0400 EDT, not making remote request.", + "GET https://api.github.com/repos/o/r: 403 API rate limit exceeded for installation ID 1.", + "POST https://api.github.com/repos/o/r/issues: 403 You have exceeded a secondary rate limit. Please wait a few minutes before you try again.", + "GET https://api.github.com/repos/o/r: 429 Too Many Requests", + `Get "https://api.github.com/repos/o/r": github pacing: request shed while rate-limit caution window is engaged (class low, queue 8.209s ahead, limit 5s)`, + "github pacing: low-priority work shed while rate-limit caution window is engaged", + } { + if !IsGitHubRateLimited(m) { + t.Errorf("%q not recognised as rate limited", m) + } + if !IsBenignMessage(m) { + t.Errorf("%q not benign", m) + } + } + for _, m := range []string{ + "GET https://api.github.com/repos/o/r: 403 Resource not accessible by integration", + "GET https://api.github.com/repos/o/r: 404 Not Found", + "failed to fetch check runs", + "github pacing: queue accounting failed", + } { + if IsGitHubRateLimited(m) { + t.Errorf("%q wrongly recognised as rate limited", m) + } + } +} + func TestIsBenignMessageReviewPendingExpiry(t *testing.T) { for _, msg := range []string{"review pending expired", " REVIEW PENDING EXPIRED "} { if !IsBenignMessage(msg) { diff --git a/src/pkg/fleetreport/slogfeed/slogfeed.go b/src/pkg/fleetreport/slogfeed/slogfeed.go index d261261cff..6f2152a444 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed.go @@ -101,7 +101,7 @@ func Key(msg string) string { } func (h *Handler) feed(r slog.Record) { - if expectedKickHold(r) { + if expectedKickHold(r) || rateLimitedError(r) { return } class := fleetreport.ClassUnclassifiedRepeated @@ -141,6 +141,25 @@ func expectedKickHold(r slog.Record) bool { return held } +// rateLimitedError reports whether the record's error attribute is a GitHub +// rate-limit refusal or a pacing shed (fleetreport.IsGitHubRateLimited). The +// message names the caller ("failed to fetch check runs", "stage labels: +// check digest failed", ...) and differs per call site, so one rate-limit +// episode would otherwise file one unclassified issue per caller +// (hivecommons/hive#15106). The client backs off and callers retry on their +// next tick; the WARN stays in local logs. +func rateLimitedError(r slog.Record) bool { + limited := false + r.Attrs(func(a slog.Attr) bool { + if a.Key == "error" && fleetreport.IsGitHubRateLimited(a.Value.String()) { + limited = true + return false + } + return true + }) + return limited +} + // Classifier errors stay in local logs only: transport errors can expose the // private endpoint in both the URL and the nested DNS/dial error. func redactClassifierError(a slog.Attr) slog.Attr { diff --git a/src/pkg/fleetreport/slogfeed/slogfeed_test.go b/src/pkg/fleetreport/slogfeed/slogfeed_test.go index 6ba2adc1b6..a69a48a7cd 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed_test.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed_test.go @@ -517,3 +517,50 @@ func TestClassifierErrorsStayInLocalLogs(t *testing.T) { } } } + +// One GitHub rate-limit episode made every caller log its own WARN, and each +// distinct message was filed as its own unclassified_error_repeated issue +// (hivecommons/hive#15106 and siblings). The WARNs stay in local logs; none +// is observed by the generic feed. +func TestGitHubRateLimitStormLogsWithoutFleetReport(t *testing.T) { + const secondary = ": 403 API secondary rate limit exceeded until 2026-10-28 17:42:28 -0400 EDT, not making remote request." + const shed = ": github pacing: request shed while rate-limit caution window is engaged (class low, queue 8.209s ahead, limit 5s)" + cases := []struct { + issue int + msg string + args []any + }{ + {15110, "stage labels: check digest failed", []any{"repo", "utah-packages", "pr", 797, "error", "GET https://api.github.com/repos/o/r/commits/abc/status?per_page=100" + secondary}}, + {15111, "GitHub refused repo enumeration transiently; serving last-good snapshot for this repo (marked stale)", []any{"repo", "website", "side", "issues", "error", "listing issues for o/website: GET https://api.github.com/repos/o/website/issues?page=1" + secondary}}, + {15112, "failed to fetch check runs", []any{"repo", "testsuite", "pr", 1037, "error", "GET https://api.github.com/repos/o/r/commits/abc/check-runs?per_page=100" + secondary}}, + {15113, "could not check watched repo has_issues", []any{"repo", "o/website", "error", `Get "https://api.github.com/repos/o/website"` + shed}}, + {15114, "advisory: issue state lookup failed", []any{"ref", "o/bluefin#961", "error", `reading o/bluefin#961: Get "https://api.github.com/repos/o/bluefin/issues/961"` + shed}}, + {15115, "stage labels: could not read the author's repository permission; adoption deferred", []any{"repo", "utah-packages", "pr", 798, "error", "GET https://api.github.com/repos/o/r/collaborators/u/permission" + secondary}}, + {15107, "github pacing: low-priority work shed while rate-limit caution window is engaged", []any{"caller", "review-request-watcher", "queue_eta", "6.882s", "budget", "5s"}}, + {15106, "stage labels: could not read maintainer_can_modify; keeping in-place adoption this sweep", []any{"repo", "utah-packages", "pr", 798, "error", "GET https://api.github.com/repos/o/r/pulls/798" + secondary}}, + {15109, "advisory: path existence check failed", []any{"path", "a/b.sh", "repo", "common", "error", `checking o/common/a/b.sh@abc: Get "https://api.github.com/repos/o/common/contents/a/b.sh?ref=abc"` + shed}}, + {0, "failed to fetch check runs", []any{"error", "GET https://api.github.com/repos/o/r/commits/abc/check-runs: 429 Too Many Requests"}}, + {0, "failed to fetch check runs", []any{"error", "GET https://api.github.com/repos/o/r/commits/abc/check-runs: 403 API rate limit of 5000 still exceeded until 2026-10-28 17:42:28 -0400 EDT, not making remote request."}}, + } + for _, tc := range cases { + logger, seen, out := newFeed(t) + for i := 0; i < 8; i++ { + logger.Warn(tc.msg, tc.args...) + } + if len(*seen) != 0 { + t.Errorf("#%d %q: rate-limited WARN produced observations: %+v", tc.issue, tc.msg, *seen) + } + if n := strings.Count(out.String(), "\n"); n != 8 { + t.Errorf("#%d %q: wrapped handler saw %d records, want 8", tc.issue, tc.msg, n) + } + } + + // The same callers failing for any other reason must remain reportable, + // including a plain 403 permission refusal. + logger, seen, _ := newFeed(t) + logger.Warn("failed to fetch check runs", "error", "GET https://api.github.com/repos/o/r/commits/abc/check-runs: 403 Resource not accessible by integration") + logger.Warn("stage labels: check digest failed", "error", "GET https://api.github.com/repos/o/r/commits/abc/status: 500 Internal Server Error") + if len(*seen) != 2 { + t.Fatalf("non-rate-limit failures must remain reportable: %+v", *seen) + } +}