diff --git a/changelog.d/fixed-18589-open-pr-guard-label.md b/changelog.d/fixed-18589-open-pr-guard-label.md new file mode 100644 index 0000000000..c101e74d42 --- /dev/null +++ b/changelog.d/fixed-18589-open-pr-guard-label.md @@ -0,0 +1 @@ +- The open-PR guard now writes the `has-open-pr` label on its own time budget, remembers a failed write per issue instead of retrying it on every claim pass, logs refusals (no write access, issue gone or locked) at INFO, and reports repeated transient failures under a new `open_pr_guard_label_failed` fleet-report class instead of `unclassified_error_repeated` (#18589). diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index 4547cdbf2a..a8eb083044 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -170,6 +170,7 @@ title is `[selfreport] : `. | `tmux_session_check_failed` | the agent's tmux session `has-session` check failed (`tmux has-session failed`, `crux/agent/manager.go`) — typically the session was lost or the tmux server is unhealthy, repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `transcript_scan_slow` | a Copilot transcript activity scan took longer than its interval (`transcript activity scan outlived its interval`, `crux/agent/busy_visibility.go`) — agent liveness reads may lag. The WARN itself is debounced at the source (`transcriptScanPace`, `crux/agent/transcript_scan_pace.go`): it fires once per slow episode, only after 3 consecutive passes outlive the interval, and re-arms only after a pass finishes under 80% of the interval, so one sustained overrun logs one line and never trips this class; it is reported only when ≥5 distinct slow→fast→slow episodes occur within 10 min. Log-fed: same ≥5-in-10-min repeat threshold and 20-minute idle clear as `agent_pane_timeout`. Operator diagnostics: check the agent's transcript/session volume; it is bounded by concurrency, so repeated episodes indicate slow disk or very large session directories | low | | `kick_freshness_withheld` | a kick was withheld because every issue/PR ref failed the pre-kick freshness check (`kick withheld: every ref failed the freshness check`, `cmd/hive/kick_freshness.go`). The kick is retried on the next cycle; reported only when the same withholding repeats ≥5 times in 10 min (refs persistently unreadable). Diagnostics count stale snapshots, closed items, deferred reads and other read errors for the last withheld kick, without including refs or raw errors. See local per-ref logs for details. Deferred API-budget refusals, refs in a stale repo and refs the re-read found closed or merged log at INFO, so a kick withheld only by those never trips it; a stale repo is reported where it is produced (`forgestate: repo cache is stale; serving last-good items (marked stale)`, `GitHub refused repo enumeration transiently; serving last-good snapshot for this repo (marked stale)`, both WARN in `pkg/github/client.go`). Its main job is keeping the residual WARN out of `unclassified_error_repeated`. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | +| `open_pr_guard_label_failed` | the pre-dispatch open-PR guard refused a claim but could not put `has-open-pr` on the issue (`open-PR guard: could not label issue` / `could not ensure label`, `pkg/github/open_pr_links.go`). The label is a tracker signal only: the claim is refused either way. Each issue is retried at most once per 15 minutes after a transient failure (timeout, rate limit, 5xx); a refusal (403/404/410/422: no write access, issue gone or locked) is remembered for 6 hours and logs at INFO, so it never feeds this class. Log-fed: ≥5 WARNs in 10 minutes (hivecommons/hive#18589) | low | | `models_endpoint_empty` | the dashboard's inference model discovery (`/api/inference/{backend}/models`) found no models from any configured endpoint (`no models found from any endpoint`, `pkg/dashboard/api_models.go`) and fell back to the static aliases, flagged unverified in the UI. The source logs it at WARN once per backend per hour (DEBUG on repeats, re-armed when discovery succeeds), so this fires only when the condition flaps ≥5 times in 10 min (gateway key rejected, endpoint down). Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | | `unclassified_error_repeated` | **generic catch-all**: the same WARN/ERROR log line (numbers, hashes, paths and quoted strings collapsed) repeated ≥5 times in 10 min. Fed by a `slog.Handler` tee (`pkg/fleetreport/slogfeed`) on the hive's root logger, so a repeating defect is noticed even with no hook site. Skips benign lines (read/keepalive i/o timeouts, WARNs whose `error` is a GitHub rate-limit refusal or a 502/503/504 gateway failure — the caller retries next tick, context cancellation, the permissions watcher's `fixed …` self-healing lines) and any message that already matches another class's keywords. Clears automatically after 20 quiet minutes (2× the window), so a "cleared" comment follows once the symptom stops | low | diff --git a/src/pkg/fleetreport/runtime.go b/src/pkg/fleetreport/runtime.go index e0db4f2f06..3aad09475a 100644 --- a/src/pkg/fleetreport/runtime.go +++ b/src/pkg/fleetreport/runtime.go @@ -90,6 +90,7 @@ const ( ClassInferenceBackendAuthFailure = "inference_backend_auth_failure" ClassSHAPollCommitMsgUnavail = "sha_poll_commit_message_unavailable" ClassKickFreshnessWithheld = "kick_freshness_withheld" + ClassOpenPRGuardLabelFailed = "open_pr_guard_label_failed" ClassModelsEndpointEmpty = "models_endpoint_empty" ClassAgentLoginRequired = "agent_login_required" ClassPRRequestHeadUnpushed = "pr_request_head_unpushed" @@ -217,6 +218,7 @@ var Classes = map[string]ClassSpec{ ClassInferenceUpstreamError: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference upstream error", "inference upstream failed", "inference translate request failed"}}, ClassSHAPollCommitMsgUnavail: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"sha poll: commit message fetch non-200"}}, ClassKickFreshnessWithheld: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"kick withheld: every ref failed the freshness check"}}, + ClassOpenPRGuardLabelFailed: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"open-pr guard: could not label issue", "open-pr guard: could not ensure label"}}, ClassModelsEndpointEmpty: {Component: "dashboard", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"no models found from any endpoint"}}, ClassAgentLoginRequired: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"login required detected", "watchdog: agent credential failure"}}, ClassPRRequestHeadUnpushed: {Component: "github-relay", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"head branch is not on the remote"}}, diff --git a/src/pkg/fleetreport/runtime_test.go b/src/pkg/fleetreport/runtime_test.go index 3e7b49b7c4..9ffc7aa9c8 100644 --- a/src/pkg/fleetreport/runtime_test.go +++ b/src/pkg/fleetreport/runtime_test.go @@ -467,6 +467,7 @@ func TestEveryRuntimeClassTriggerAndNonTriggerThresholds(t *testing.T) { {class: ClassInferenceUpstreamError, message: "inference upstream error"}, {class: ClassSHAPollCommitMsgUnavail, message: "SHA poll: commit message fetch non-200"}, {class: ClassKickFreshnessWithheld, message: "kick withheld: every ref failed the freshness check"}, + {class: ClassOpenPRGuardLabelFailed, message: "open-PR guard: could not label issue"}, {class: ClassModelsEndpointEmpty, message: "no models found from any endpoint"}, {class: ClassAgentLoginRequired, message: "login required detected"}, {class: ClassPRRequestHeadUnpushed, message: "pr-request watcher: head branch is not on the remote"}, @@ -1267,6 +1268,8 @@ func TestRecurringRuntimeMessagesAreClassified(t *testing.T) { "issue-claims: GitHub side effect failed": ClassIssueClaimEffectFailed, "SHA poll: commit message fetch non-200": ClassSHAPollCommitMsgUnavail, "kick withheld: every ref failed the freshness check": ClassKickFreshnessWithheld, + "open-PR guard: could not label issue": ClassOpenPRGuardLabelFailed, + "open-PR guard: could not ensure label": ClassOpenPRGuardLabelFailed, "no models found from any endpoint": ClassModelsEndpointEmpty, "login required detected": ClassAgentLoginRequired, "pr-request watcher: head branch is not on the remote": ClassPRRequestHeadUnpushed, diff --git a/src/pkg/github/open_pr_links.go b/src/pkg/github/open_pr_links.go index bc4009fc13..7e77215931 100644 --- a/src/pkg/github/open_pr_links.go +++ b/src/pkg/github/open_pr_links.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "log/slog" + "net/http" "sort" "strings" "sync" @@ -49,6 +50,22 @@ const ( // for its repo: a surge of claims fails open at once instead of each one // re-running a listing that just timed out and re-logging the warning. openPRLinkFailureTTL = time.Minute + + // openPRLabelTimeout bounds the has-open-pr label write on its own + // budget. It used to share the caller's deadline (15s from the + // contribute lane) with the open-PR listing ahead of it, so a slow + // listing of a large repo left the label write an already-spent + // context and every guard pass failed with `context deadline exceeded` + // (hivecommons/hive#18589). + openPRLabelTimeout = 10 * time.Second + // openPRLabelRetryBackoff is how long a transient label failure + // (timeout, rate limit, 5xx) keeps the guard from re-trying that issue: + // the label is a tracker signal, and a claim surge on the same issue + // must not turn one slow write into a WARN per pass. + openPRLabelRetryBackoff = 15 * time.Minute + // openPRLabelRefusedBackoff is the same for a refusal that will not fix + // itself (no write access, issue gone or locked). + openPRLabelRefusedBackoff = 6 * time.Hour ) // errOpenPRListingBackedOff marks an error replayed from the failure cache. @@ -93,6 +110,9 @@ type openPRLinkCache struct { repos map[string]*openPRLinkIndex // failures holds each repo's last listing failure for openPRLinkFailureTTL. failures map[string]openPRLinkFailure + // labelBackoff holds, per claimKey, when the guard may next try to + // label that issue after a failed has-open-pr write. + labelBackoff map[string]time.Time // disabled is scanner.skip_issues_with_open_pr: false. disabled bool now func() time.Time @@ -100,7 +120,7 @@ type openPRLinkCache struct { func (c *Client) openPRLinks() *openPRLinkCache { c.openPRLinkOnce.Do(func() { - c.openPRLinkState = &openPRLinkCache{repos: map[string]*openPRLinkIndex{}, failures: map[string]openPRLinkFailure{}, now: time.Now} + c.openPRLinkState = &openPRLinkCache{repos: map[string]*openPRLinkIndex{}, failures: map[string]openPRLinkFailure{}, labelBackoff: map[string]time.Time{}, now: time.Now} }) return c.openPRLinkState } @@ -297,18 +317,90 @@ func (c *Client) openPRLinkBlockReason(ctx context.Context, repo string, number // markIssueHasOpenPR applies HasOpenPRLabel once. Idempotent on the labels // the caller just read; a label failure is logged, never fatal — the claim -// is refused either way. +// is refused either way. The write runs on its own budget, and a failed +// write is remembered per issue so later passes skip it quietly until the +// backoff lapses (hivecommons/hive#18589). A refusal that cannot fix itself +// logs at INFO; a transient failure logs one WARN per backoff window. func (c *Client) markIssueHasOpenPR(ctx context.Context, repo string, number int, labels []string, link OpenPRLink) { for _, l := range labels { if strings.EqualFold(strings.TrimSpace(l), HasOpenPRLabel) { return } } - if err := c.EnsureIssueLabel(ctx, repo, HasOpenPRLabel, hasOpenPRLabelColor, hasOpenPRLabelDescription); err != nil && c.logger != nil { - c.logger.Warn("open-PR guard: could not ensure label", slog.String("label", HasOpenPRLabel), slog.String("repo", repo), slog.String("error", err.Error())) + key := claimKey(c.canonicalRepo(repo), number) + cache := c.openPRLinks() + cache.mu.Lock() + until, backedOff := cache.labelBackoff[key] + backedOff = backedOff && cache.now().Before(until) + cache.mu.Unlock() + if backedOff { + return + } + + labelCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), openPRLabelTimeout) + defer cancel() + step := "could not ensure label" + err := c.EnsureIssueLabel(labelCtx, repo, HasOpenPRLabel, hasOpenPRLabelColor, hasOpenPRLabelDescription) + if err == nil || !errors.Is(err, context.DeadlineExceeded) { + // A failed lookup or create does not mean the label is missing: + // the add below still lands when it exists (or the forge creates + // it), so only a spent budget skips it. + if addErr := c.AddLabels(labelCtx, repo, number, []string{HasOpenPRLabel}); addErr != nil { + step, err = "could not label issue", addErr + } else { + err = nil + } + } + + cache.mu.Lock() + now := cache.now() + for k, t := range cache.labelBackoff { + if !now.Before(t) { + delete(cache.labelBackoff, k) + } + } + if err == nil { + delete(cache.labelBackoff, key) + cache.mu.Unlock() + return } - if err := c.AddLabels(ctx, repo, number, []string{HasOpenPRLabel}); err != nil && c.logger != nil { - c.logger.Warn("open-PR guard: could not label issue", slog.String("label", HasOpenPRLabel), - slog.String("repo", repo), slog.Int("issue", number), slog.Int("pr", link.PRNumber), slog.String("error", err.Error())) + reason, refused := openPRLabelFailureReason(err) + backoff := openPRLabelRetryBackoff + if refused { + backoff = openPRLabelRefusedBackoff + } + cache.labelBackoff[key] = now.Add(backoff) + cache.mu.Unlock() + + if c.logger == nil { + return + } + attrs := []any{slog.String("label", HasOpenPRLabel), slog.String("repo", repo), slog.Int("issue", number), + slog.Int("pr", link.PRNumber), slog.String("reason", reason), slog.Duration("retry_after", backoff), slog.String("error", err.Error())} + if refused { + c.logger.Info("open-PR guard: "+step+" (refused; not retrying this issue for a while)", attrs...) + return + } + c.logger.Warn("open-PR guard: "+step, attrs...) +} + +// openPRLabelFailureReason names a has-open-pr write failure for the log and +// reports whether it is a refusal that will not fix itself on retry: no +// write access (403), the repo or issue is gone (404/410), or the forge +// rejected the write — a locked issue answers 422. +func openPRLabelFailureReason(err error) (reason string, refused bool) { + switch { + case isRateLimitErr(err): + return "rate-limited", false + case errors.Is(err, context.DeadlineExceeded): + return "timeout", false + case githubStatusError(err, http.StatusForbidden): + return "forbidden", true + case githubStatusError(err, http.StatusNotFound), githubStatusError(err, http.StatusGone): + return "not-found", true + case githubStatusError(err, http.StatusUnprocessableEntity): + return "unprocessable", true + default: + return "error", false } } diff --git a/src/pkg/github/open_pr_links_test.go b/src/pkg/github/open_pr_links_test.go index ae97b0676d..fa79d30884 100644 --- a/src/pkg/github/open_pr_links_test.go +++ b/src/pkg/github/open_pr_links_test.go @@ -5,12 +5,15 @@ import ( "encoding/json" "fmt" "io" + "log/slog" "net/http" "net/http/httptest" "strings" "sync" "testing" "time" + + gh "github.com/google/go-github/v72/github" ) // openPRGuardServer serves one repo with the given open PRs and issue #42 @@ -21,6 +24,9 @@ type openPRGuardServer struct { addedLabels []string createdLbl []string failPulls bool + labelCalls int + // failLabels, when non-zero, answers every label add with that status. + failLabels int } func newOpenPRGuardServer(t *testing.T, prs []map[string]any, issueLabels []string) (*httptest.Server, *openPRGuardServer) { @@ -49,8 +55,17 @@ func newOpenPRGuardServer(t *testing.T, prs []map[string]any, issueLabels []stri var names []string _ = json.NewDecoder(r.Body).Decode(&names) obs.mu.Lock() - obs.addedLabels = append(obs.addedLabels, names...) + obs.labelCalls++ + fail := obs.failLabels + if fail == 0 { + obs.addedLabels = append(obs.addedLabels, names...) + } obs.mu.Unlock() + if fail != 0 { + w.WriteHeader(fail) + _, _ = io.WriteString(w, `{"message":"refused"}`) + return + } _, _ = io.WriteString(w, `[]`) }) mux.HandleFunc("/repos/o/r/labels/", func(w http.ResponseWriter, r *http.Request) { @@ -267,3 +282,153 @@ func TestOpenPRGuard_InvalidateForcesRelist(t *testing.T) { t.Fatalf("prefix %q drifted from the log line", ClaimBlockReasonOpenPRPrefix) } } + +// guardLogBuffer captures the client's log so a test can assert levels. +type guardLogBuffer struct { + mu sync.Mutex + buf strings.Builder +} + +func (b *guardLogBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *guardLogBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +func captureGuardLog(c *Client) *guardLogBuffer { + b := &guardLogBuffer{} + c.logger = slog.New(slog.NewTextHandler(b, &slog.HandlerOptions{Level: slog.LevelDebug})) + return b +} + +func linkedIssueFixture() []map[string]any { + return []map[string]any{openPRFixture(18587, "Fixes #42", "someone", time.Date(2026, 11, 17, 1, 0, 0, 0, time.UTC))} +} + +func (o *openPRGuardServer) labelAttempts() int { + o.mu.Lock() + defer o.mu.Unlock() + return o.labelCalls +} + +// hivecommons/hive#18589: a refused label write (no write access) is +// remembered per issue and logged at INFO, so the claim surge that follows +// neither re-tries it nor files an unclassified self-report. +func TestOpenPRGuard_RefusedLabelIsRememberedAndLoggedAtInfo(t *testing.T) { + srv, obs := newOpenPRGuardServer(t, linkedIssueFixture(), nil) + defer srv.Close() + obs.failLabels = http.StatusForbidden + c := newTestClient(t, srv, "o", []string{"r"}) + logs := captureGuardLog(c) + + for i := 0; i < 5; i++ { + if _, ok := c.OpenPRLinkGuard(context.Background(), "o/r", 42, nil); !ok { + t.Fatalf("pass %d: guard admitted; want the claim refused even when labelling fails", i) + } + } + if got := obs.labelAttempts(); got != 1 { + t.Fatalf("label attempts = %d, want 1: a refusal must not be retried every pass", got) + } + out := logs.String() + if strings.Contains(out, "level=WARN") { + t.Fatalf("refused label logged at WARN; want INFO only:\n%s", out) + } + if !strings.Contains(out, "level=INFO") || !strings.Contains(out, "reason=forbidden") { + t.Fatalf("log = %s; want one INFO line with reason=forbidden", out) + } +} + +// A transient failure WARNs once, backs off, and is retried once the +// backoff lapses; a success then clears the memory. +func TestOpenPRGuard_TransientLabelFailureBacksOffThenRetries(t *testing.T) { + srv, obs := newOpenPRGuardServer(t, linkedIssueFixture(), nil) + defer srv.Close() + obs.failLabels = http.StatusInternalServerError + c := newTestClient(t, srv, "o", []string{"r"}) + logs := captureGuardLog(c) + cache := c.openPRLinks() + now := time.Date(2026, 11, 17, 1, 30, 0, 0, time.UTC) + cache.mu.Lock() + cache.now = func() time.Time { return now } + cache.mu.Unlock() + + c.OpenPRLinkGuard(context.Background(), "o/r", 42, nil) + first := obs.labelAttempts() + if first == 0 { + t.Fatal("first pass did not try to label the issue") + } + for i := 0; i < 4; i++ { + c.OpenPRLinkGuard(context.Background(), "o/r", 42, nil) + } + if got := obs.labelAttempts(); got != first { + t.Fatalf("label attempts = %d, want %d inside the backoff window", got, first) + } + if n := strings.Count(logs.String(), "level=WARN msg=\"open-PR guard: could not label issue\""); n != 1 { + t.Fatalf("WARN lines = %d, want 1 per backoff window:\n%s", n, logs.String()) + } + + obs.mu.Lock() + obs.failLabels = 0 + obs.mu.Unlock() + cache.mu.Lock() + now = now.Add(openPRLabelRetryBackoff + time.Second) + cache.repos = map[string]*openPRLinkIndex{} + cache.mu.Unlock() + c.OpenPRLinkGuard(context.Background(), "o/r", 42, nil) + if len(obs.addedLabels) != 1 || obs.addedLabels[0] != HasOpenPRLabel { + t.Fatalf("labels added = %v, want the label applied after the backoff", obs.addedLabels) + } + cache.mu.Lock() + _, remembered := cache.labelBackoff[claimKey(c.canonicalRepo("o/r"), 42)] + cache.mu.Unlock() + if remembered { + t.Fatal("a successful label write must clear the issue's backoff") + } +} + +// The label write no longer inherits the caller's deadline: the listing +// ahead of it may spend the whole budget, which is how every pass on a +// large repo failed with `context deadline exceeded`. +func TestOpenPRGuard_LabelWriteSurvivesSpentCallerDeadline(t *testing.T) { + srv, obs := newOpenPRGuardServer(t, nil, nil) + defer srv.Close() + c := newTestClient(t, srv, "o", []string{"r"}) + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second)) + defer cancel() + c.markIssueHasOpenPR(ctx, "o/r", 42, nil, OpenPRLink{PRNumber: 18587}) + if len(obs.addedLabels) != 1 || obs.addedLabels[0] != HasOpenPRLabel { + t.Fatalf("labels added = %v, want the label written on its own budget", obs.addedLabels) + } +} + +func TestOpenPRLabelFailureReason(t *testing.T) { + status := func(code int) error { + return &gh.ErrorResponse{Response: &http.Response{StatusCode: code}} + } + cases := []struct { + name string + err error + reason string + refused bool + }{ + {"forbidden", status(http.StatusForbidden), "forbidden", true}, + {"missing", status(http.StatusNotFound), "not-found", true}, + {"gone", status(http.StatusGone), "not-found", true}, + {"locked", status(http.StatusUnprocessableEntity), "unprocessable", true}, + {"rate limit", &gh.RateLimitError{Response: &http.Response{StatusCode: http.StatusForbidden}}, "rate-limited", false}, + {"timeout", fmt.Errorf("add labels: %w", context.DeadlineExceeded), "timeout", false}, + {"server", status(http.StatusBadGateway), "error", false}, + } + for _, tc := range cases { + reason, refused := openPRLabelFailureReason(tc.err) + if reason != tc.reason || refused != tc.refused { + t.Errorf("%s: got (%q, %v), want (%q, %v)", tc.name, reason, refused, tc.reason, tc.refused) + } + } +}