Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Repeated lane-gate and trusted-author sweep warnings caused by a transient GitHub 500 on the status read are no longer filed as selfreport issues (#15785, #15786); a hub heartbeat rejected with a restarting-hub 502/503/504 is likewise classified as transient (#15789)
48 changes: 47 additions & 1 deletion src/pkg/fleetreport/slogfeed/slogfeed.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
// Package slogfeed tees WARN and ERROR log records into the fleet-report
// runtime tracker as fleetreport.ClassUnclassifiedRepeated observations, so a
// log line that keeps repeating becomes a self-report even when no hook site
Expand Down Expand Up @@ -114,7 +114,7 @@
}

func (h *Handler) feed(r slog.Record) {
if expectedKickHold(r) || rateLimitedError(r) || upstreamUnavailableError(r) {
if expectedKickHold(r) || rateLimitedError(r) || upstreamUnavailableError(r) || failClosedStatus500(r) || hubHeartbeatUnavailable(r) {
return
}
class := fleetreport.ClassUnclassifiedRepeated
Expand Down Expand Up @@ -190,6 +190,52 @@
return unavailable
}

// failClosedStatus500 reports whether r is one of the two gates that fail
// closed when GitHub answers the combined-status read with a 500: the
// merge-eligible lane gate and the trusted-author automerge sweep skip. The PR
// is retried next tick, so the transient 5xx is expected behaviour
// (hivecommons/hive#15785, #15786). A 500 elsewhere stays reportable.
func failClosedStatus500(r slog.Record) bool {
switch strings.TrimSpace(r.Message) {
case "merge-eligible lane gate could not evaluate PR; leaving it listed",
"trusted-author automerge sweep skipped PR":
default:
return false
}
match := false
r.Attrs(func(a slog.Attr) bool {
if a.Key == "error" {
e := a.Value.String() + " "
match = strings.Contains(e, "/status") && strings.Contains(e, ": 500 ")
return false
}
return true
})
return match
}

// hubHeartbeatUnavailable reports whether r is a spoke heartbeat the hub
// answered with 502/503/504 (restarting or temporarily unreachable). The
// heartbeat retries on the next beat, so this is expected transient behaviour
// (hivecommons/hive#15789). 401/403 (bad HIVE_HUB_SECRET) stay reportable.
func hubHeartbeatUnavailable(r slog.Record) bool {
if strings.TrimSpace(r.Message) != "hub heartbeat rejected" {
return false
}
match := false
r.Attrs(func(a slog.Attr) bool {
if a.Key == "status" {
switch a.Value.String() {
case "502", "503", "504":
match = true
}
return false
}
return true
})
return match
}

// Classifier errors stay in local logs only: transport errors can expose the
// private endpoint in both the URL and the nested DNS/dial error.
func redactClassifierError(a slog.Attr) slog.Attr {
Expand Down
58 changes: 58 additions & 0 deletions src/pkg/fleetreport/slogfeed/slogfeed_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -643,6 +643,64 @@ func TestGitHubUpstreamUnavailableLogsWithoutFleetReport(t *testing.T) {
}
}

// #15785, #15786: fail-closed gates hitting a GitHub 500 on the status read
// are expected and retried, so they are not reported as unclassified.
func TestFailClosedStatus500LogsWithoutFleetReport(t *testing.T) {
const statusErr = "GET https://api.github.com/repos/o/r/commits/abc/status?per_page=100: 500 []"
cases := []struct {
msg string
args []any
}{
{"merge-eligible lane gate could not evaluate PR; leaving it listed", []any{"repo", "o/r", "number", 3929, "error", "o/r#3929 lane evidence unavailable (status-check; fail closed): " + statusErr}},
{"trusted-author automerge sweep skipped PR", []any{"repo", "hive", "pr", 15763, "reason", "release-hold-check", "error", statusErr}},
}
for _, tc := range cases {
logger, seen, out := newFeed(t)
for i := 0; i < 8; i++ {
logger.Warn(tc.msg, tc.args...)
}
if len(*seen) != 0 {
t.Errorf("%q: produced observations: %+v", tc.msg, *seen)
}
if n := strings.Count(out.String(), "\n"); n != 8 {
t.Errorf("%q: wrapped handler saw %d records, want 8", tc.msg, n)
}
}

logger, seen, _ := newFeed(t)
logger.Warn("trusted-author automerge sweep skipped PR", "error", "GET https://api.github.com/repos/o/r/pulls/1: 404 Not Found")
logger.Warn("merge-eligible lane gate could not evaluate PR; leaving it listed", "error", "boom")
if len(*seen) != 2 {
t.Fatalf("other failures must remain reportable: %+v", *seen)
}
}

// #15789: a hub that is restarting answers the heartbeat 5xx and the spoke
// retries on the next beat; auth rejections stay reportable.
func TestHubHeartbeatUnavailableLogsWithoutFleetReport(t *testing.T) {
body := `{"ok":false,"error":"hive unavailable"}`
for _, status := range []int{502, 503, 504} {
logger, seen, out := newFeed(t)
for i := 0; i < 8; i++ {
logger.Warn("hub heartbeat rejected", "status", status, "body", body)
}
if len(*seen) != 0 {
t.Errorf("status %d: produced observations: %+v", status, *seen)
}
if n := strings.Count(out.String(), "\n"); n != 8 {
t.Errorf("status %d: wrapped handler saw %d records, want 8", status, n)
}
}

for _, status := range []int{401, 403} {
logger, seen, _ := newFeed(t)
logger.Warn("hub heartbeat rejected", "status", status, "body", "unauthorized")
if len(*seen) != 1 {
t.Errorf("status %d must remain reportable: %+v", status, *seen)
}
}
}

// #15333: keep repeated translator identity refusals visible under their own
// gateway class, without treating them as backend credential failures.
func TestInferenceCallerUnidentifiedFeedThresholdAndClear(t *testing.T) {
Expand Down
Loading