Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-sentinel-pacing-selfreport.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Avoid filing repeated-error fleet reports for sentinel skips caused by GitHub pacing, while retaining the local warnings and reporting other failures.
2 changes: 1 addition & 1 deletion src/pkg/fleetreport/fleetreport.go
Original file line number Diff line number Diff line change
Expand Up @@ -897,7 +897,7 @@ func IsBenignMessage(msg string) bool {
}
// A sentinel finding flags a PR for a human: the sweep working as designed,
// not a runtime defect (#16112). Match only the finding notice; "sentinel
// sweep failed" and "sentinel sweep: PR skipped" stay reportable.
// sweep failed" and non-pacing "sentinel sweep: PR skipped" remain reportable.
if strings.TrimSpace(m) == "sentinel alert" {
return true
}
Expand Down
19 changes: 18 additions & 1 deletion src/pkg/fleetreport/slogfeed/slogfeed.go
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ func Key(msg string) string {
}

func (h *Handler) feed(r slog.Record) {
if expectedKickHold(r) || rateLimitedError(r) || upstreamUnavailableError(r) || failClosedStatus500(r) || hubHeartbeatUnavailable(r) || hubHeartbeatCachedFallback(r) || reviewCommentCapped(r) || inferenceBlockedStall(r) || automergeMergeGuardSkip(r) {
if expectedKickHold(r) || rateLimitedError(r) || sentinelPacingSkip(r) || upstreamUnavailableError(r) || failClosedStatus500(r) || hubHeartbeatUnavailable(r) || hubHeartbeatCachedFallback(r) || reviewCommentCapped(r) || inferenceBlockedStall(r) || automergeMergeGuardSkip(r) {
return
}
class := fleetreport.ClassUnclassifiedRepeated
Expand Down Expand Up @@ -173,6 +173,23 @@ func rateLimitedError(r slog.Record) bool {
return limited
}

// sentinelPacingSkip handles the sentinel's per-item errors, which are logged
// as detail rather than error. Other skipped PRs remain reportable.
func sentinelPacingSkip(r slog.Record) bool {
if r.Message != "sentinel sweep: PR skipped" {
return false
}
limited := false
r.Attrs(func(a slog.Attr) bool {
if a.Key == "detail" && fleetreport.IsGitHubRateLimited(a.Value.String()) {
limited = true
return false
}
return true
})
return limited
}

// upstreamUnavailableError reports whether the record's error attribute is a
// GitHub 502/503/504 (fleetreport.IsGitHubUpstreamUnavailable). Callers such
// as the self-authored automerge sweep ("fetch-pr") log a per-site message and
Expand Down
19 changes: 19 additions & 0 deletions src/pkg/fleetreport/slogfeed/slogfeed_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -870,6 +870,25 @@ func TestSentinelAlertLogsWithoutFleetReport(t *testing.T) {
}
}

func TestSentinelPacingSkipLogsWithoutFleetReport(t *testing.T) {
logger, seen, out := newFeed(t)
const detail = `website: list PRs: listing open PRs for supersession sweep in projectbluefin/website: Get "https://api.github.com/repos/projectbluefin/website/pulls?per_page=100&state=open": github pacing: slot would miss request deadline: context deadline exceeded`
for range 16 {
logger.Warn("sentinel sweep: PR skipped", "detail", detail)
}
if len(*seen) != 0 {
t.Fatalf("pacing skips produced observations: %+v", *seen)
}
if n := strings.Count(out.String(), "\n"); n != 16 {
t.Fatalf("wrapped handler saw %d records, want 16", n)
}
logger.Warn("sentinel sweep: PR skipped", "detail", "website: permission denied")
logger.Warn("another warning", "detail", detail)
if len(*seen) != 2 {
t.Fatalf("other failures must remain reportable: %+v", *seen)
}
}

// #15906: the per-head review cap suppressing a duplicate comment is expected.
func TestReviewCommentCappedLogsWithoutFleetReport(t *testing.T) {
logger, seen, out := newFeed(t)
Expand Down
Loading