diff --git a/changelog.d/fixed-sentinel-pacing-selfreport.md b/changelog.d/fixed-sentinel-pacing-selfreport.md new file mode 100644 index 0000000000..1f3a9c2a04 --- /dev/null +++ b/changelog.d/fixed-sentinel-pacing-selfreport.md @@ -0,0 +1 @@ +- Avoid filing repeated-error fleet reports for sentinel skips caused by GitHub pacing, while retaining the local warnings and reporting other failures. diff --git a/src/pkg/fleetreport/fleetreport.go b/src/pkg/fleetreport/fleetreport.go index 5306d87104..f7f2c69641 100644 --- a/src/pkg/fleetreport/fleetreport.go +++ b/src/pkg/fleetreport/fleetreport.go @@ -897,7 +897,7 @@ func IsBenignMessage(msg string) bool { } // A sentinel finding flags a PR for a human: the sweep working as designed, // not a runtime defect (#16112). Match only the finding notice; "sentinel - // sweep failed" and "sentinel sweep: PR skipped" stay reportable. + // sweep failed" and non-pacing "sentinel sweep: PR skipped" remain reportable. if strings.TrimSpace(m) == "sentinel alert" { return true } diff --git a/src/pkg/fleetreport/slogfeed/slogfeed.go b/src/pkg/fleetreport/slogfeed/slogfeed.go index 594dc8fdc8..46af39966e 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed.go @@ -114,7 +114,7 @@ func Key(msg string) string { } func (h *Handler) feed(r slog.Record) { - if expectedKickHold(r) || rateLimitedError(r) || upstreamUnavailableError(r) || failClosedStatus500(r) || hubHeartbeatUnavailable(r) || hubHeartbeatCachedFallback(r) || reviewCommentCapped(r) || inferenceBlockedStall(r) || automergeMergeGuardSkip(r) { + if expectedKickHold(r) || rateLimitedError(r) || sentinelPacingSkip(r) || upstreamUnavailableError(r) || failClosedStatus500(r) || hubHeartbeatUnavailable(r) || hubHeartbeatCachedFallback(r) || reviewCommentCapped(r) || inferenceBlockedStall(r) || automergeMergeGuardSkip(r) { return } class := fleetreport.ClassUnclassifiedRepeated @@ -173,6 +173,23 @@ func rateLimitedError(r slog.Record) bool { return limited } +// sentinelPacingSkip handles the sentinel's per-item errors, which are logged +// as detail rather than error. Other skipped PRs remain reportable. +func sentinelPacingSkip(r slog.Record) bool { + if r.Message != "sentinel sweep: PR skipped" { + return false + } + limited := false + r.Attrs(func(a slog.Attr) bool { + if a.Key == "detail" && fleetreport.IsGitHubRateLimited(a.Value.String()) { + limited = true + return false + } + return true + }) + return limited +} + // upstreamUnavailableError reports whether the record's error attribute is a // GitHub 502/503/504 (fleetreport.IsGitHubUpstreamUnavailable). Callers such // as the self-authored automerge sweep ("fetch-pr") log a per-site message and diff --git a/src/pkg/fleetreport/slogfeed/slogfeed_test.go b/src/pkg/fleetreport/slogfeed/slogfeed_test.go index 71679f87c6..005ff77439 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed_test.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed_test.go @@ -870,6 +870,25 @@ func TestSentinelAlertLogsWithoutFleetReport(t *testing.T) { } } +func TestSentinelPacingSkipLogsWithoutFleetReport(t *testing.T) { + logger, seen, out := newFeed(t) + const detail = `website: list PRs: listing open PRs for supersession sweep in projectbluefin/website: Get "https://api.github.com/repos/projectbluefin/website/pulls?per_page=100&state=open": github pacing: slot would miss request deadline: context deadline exceeded` + for range 16 { + logger.Warn("sentinel sweep: PR skipped", "detail", detail) + } + if len(*seen) != 0 { + t.Fatalf("pacing skips produced observations: %+v", *seen) + } + if n := strings.Count(out.String(), "\n"); n != 16 { + t.Fatalf("wrapped handler saw %d records, want 16", n) + } + logger.Warn("sentinel sweep: PR skipped", "detail", "website: permission denied") + logger.Warn("another warning", "detail", detail) + if len(*seen) != 2 { + t.Fatalf("other failures must remain reportable: %+v", *seen) + } +} + // #15906: the per-head review cap suppressing a duplicate comment is expected. func TestReviewCommentCappedLogsWithoutFleetReport(t *testing.T) { logger, seen, out := newFeed(t)