diff --git a/changelog.d/fixed-selfreport-lane-gate-transient-classified.md b/changelog.d/fixed-selfreport-lane-gate-transient-classified.md new file mode 100644 index 0000000000..ff1a340201 --- /dev/null +++ b/changelog.d/fixed-selfreport-lane-gate-transient-classified.md @@ -0,0 +1 @@ +- Repeated lane-gate and trusted-author sweep warnings caused by a transient GitHub 500 on the status read are no longer filed as selfreport issues (#15785, #15786); a hub heartbeat rejected with a restarting-hub 502/503/504 is likewise classified as transient (#15789) diff --git a/src/pkg/fleetreport/slogfeed/slogfeed.go b/src/pkg/fleetreport/slogfeed/slogfeed.go index 5f70efe64e..097dcb5bbd 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed.go @@ -114,7 +114,7 @@ func Key(msg string) string { } func (h *Handler) feed(r slog.Record) { - if expectedKickHold(r) || rateLimitedError(r) || upstreamUnavailableError(r) { + if expectedKickHold(r) || rateLimitedError(r) || upstreamUnavailableError(r) || failClosedStatus500(r) || hubHeartbeatUnavailable(r) { return } class := fleetreport.ClassUnclassifiedRepeated @@ -190,6 +190,52 @@ func upstreamUnavailableError(r slog.Record) bool { return unavailable } +// failClosedStatus500 reports whether r is one of the two gates that fail +// closed when GitHub answers the combined-status read with a 500: the +// merge-eligible lane gate and the trusted-author automerge sweep skip. The PR +// is retried next tick, so the transient 5xx is expected behaviour +// (hivecommons/hive#15785, #15786). A 500 elsewhere stays reportable. +func failClosedStatus500(r slog.Record) bool { + switch strings.TrimSpace(r.Message) { + case "merge-eligible lane gate could not evaluate PR; leaving it listed", + "trusted-author automerge sweep skipped PR": + default: + return false + } + match := false + r.Attrs(func(a slog.Attr) bool { + if a.Key == "error" { + e := a.Value.String() + " " + match = strings.Contains(e, "/status") && strings.Contains(e, ": 500 ") + return false + } + return true + }) + return match +} + +// hubHeartbeatUnavailable reports whether r is a spoke heartbeat the hub +// answered with 502/503/504 (restarting or temporarily unreachable). The +// heartbeat retries on the next beat, so this is expected transient behaviour +// (hivecommons/hive#15789). 401/403 (bad HIVE_HUB_SECRET) stay reportable. +func hubHeartbeatUnavailable(r slog.Record) bool { + if strings.TrimSpace(r.Message) != "hub heartbeat rejected" { + return false + } + match := false + r.Attrs(func(a slog.Attr) bool { + if a.Key == "status" { + switch a.Value.String() { + case "502", "503", "504": + match = true + } + return false + } + return true + }) + return match +} + // Classifier errors stay in local logs only: transport errors can expose the // private endpoint in both the URL and the nested DNS/dial error. func redactClassifierError(a slog.Attr) slog.Attr { diff --git a/src/pkg/fleetreport/slogfeed/slogfeed_test.go b/src/pkg/fleetreport/slogfeed/slogfeed_test.go index d6805796fc..5a0969c58b 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed_test.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed_test.go @@ -643,6 +643,64 @@ func TestGitHubUpstreamUnavailableLogsWithoutFleetReport(t *testing.T) { } } +// #15785, #15786: fail-closed gates hitting a GitHub 500 on the status read +// are expected and retried, so they are not reported as unclassified. +func TestFailClosedStatus500LogsWithoutFleetReport(t *testing.T) { + const statusErr = "GET https://api.github.com/repos/o/r/commits/abc/status?per_page=100: 500 []" + cases := []struct { + msg string + args []any + }{ + {"merge-eligible lane gate could not evaluate PR; leaving it listed", []any{"repo", "o/r", "number", 3929, "error", "o/r#3929 lane evidence unavailable (status-check; fail closed): " + statusErr}}, + {"trusted-author automerge sweep skipped PR", []any{"repo", "hive", "pr", 15763, "reason", "release-hold-check", "error", statusErr}}, + } + for _, tc := range cases { + logger, seen, out := newFeed(t) + for i := 0; i < 8; i++ { + logger.Warn(tc.msg, tc.args...) + } + if len(*seen) != 0 { + t.Errorf("%q: produced observations: %+v", tc.msg, *seen) + } + if n := strings.Count(out.String(), "\n"); n != 8 { + t.Errorf("%q: wrapped handler saw %d records, want 8", tc.msg, n) + } + } + + logger, seen, _ := newFeed(t) + logger.Warn("trusted-author automerge sweep skipped PR", "error", "GET https://api.github.com/repos/o/r/pulls/1: 404 Not Found") + logger.Warn("merge-eligible lane gate could not evaluate PR; leaving it listed", "error", "boom") + if len(*seen) != 2 { + t.Fatalf("other failures must remain reportable: %+v", *seen) + } +} + +// #15789: a hub that is restarting answers the heartbeat 5xx and the spoke +// retries on the next beat; auth rejections stay reportable. +func TestHubHeartbeatUnavailableLogsWithoutFleetReport(t *testing.T) { + body := `{"ok":false,"error":"hive unavailable"}` + for _, status := range []int{502, 503, 504} { + logger, seen, out := newFeed(t) + for i := 0; i < 8; i++ { + logger.Warn("hub heartbeat rejected", "status", status, "body", body) + } + if len(*seen) != 0 { + t.Errorf("status %d: produced observations: %+v", status, *seen) + } + if n := strings.Count(out.String(), "\n"); n != 8 { + t.Errorf("status %d: wrapped handler saw %d records, want 8", status, n) + } + } + + for _, status := range []int{401, 403} { + logger, seen, _ := newFeed(t) + logger.Warn("hub heartbeat rejected", "status", status, "body", "unauthorized") + if len(*seen) != 1 { + t.Errorf("status %d must remain reportable: %+v", status, *seen) + } + } +} + // #15333: keep repeated translator identity refusals visible under their own // gateway class, without treating them as backend credential failures. func TestInferenceCallerUnidentifiedFeedThresholdAndClear(t *testing.T) {