From 308800131da64831235719917540721dc1f9374d Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:41:28 -0400 Subject: [PATCH 1/2] fix(contribute): treat an instantly failed relay fix-pr task as undelivered and retry A relay that failed a stage-072 fix-pr task 116 ms after dispatch was still reported as reached (mode=spawn, "your relay session received this as a fix-pr task"), and handleTaskFailed settled the dispatch exactly like task_complete, so the PR silently left the fix queue and was never retried. Separately, a conflicting contributor PR kept its hive:unauthorized mark because the lane returned "conflicting" before the step that clears it, so the row named "approve runs" as the blocker when only a rebase can move the PR. - Nudge waits briefly for the relay to refuse the dispatch; a refusal is reported as queued with the relay's reason, and the comment says so. - task_failed on a fix requeues it with backoff (served at the relay's next task boundary); after three failures the PR leaves the queue for the stage engine's re-nudge / adoption path. - The contributor lane clears the approve-workflow-runs unauthorized mark when the head conflicts; the next head re-marks if its runs are held. Closes #14668 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> --- changelog.d/fixed-14668.md | 1 + src/docs/contributor-relay.md | 2 + src/docs/labels-and-control-signals.md | 8 +- src/pkg/dashboard/contribute_fixqueue.go | 150 ++++++++++++++++++ src/pkg/dashboard/contribute_fixqueue_test.go | 61 +++++++ src/pkg/dashboard/contribute_ws.go | 15 +- src/pkg/fixqueue/fixqueue.go | 51 +++++- src/pkg/fixqueue/fixqueue_test.go | 40 +++++ .../automerge_contributor_held_runs_test.go | 39 +++++ .../automerge/automerge_contributor_lane.go | 5 + .../automerge/automerge_contributor_nudge.go | 15 +- .../automerge_contributor_nudge_test.go | 17 ++ 12 files changed, 394 insertions(+), 10 deletions(-) create mode 100644 changelog.d/fixed-14668.md diff --git a/changelog.d/fixed-14668.md b/changelog.d/fixed-14668.md new file mode 100644 index 0000000000..55e8c8155d --- /dev/null +++ b/changelog.d/fixed-14668.md @@ -0,0 +1 @@ +- A relay `fix-pr` task that fails right after dispatch is no longer reported as delivered: the stage-072 nudge comment says the relay could not take the task (with its reason), the fix is requeued with backoff and retried at the relay's next task boundary (three attempts before the re-nudge / adoption path takes over), and a conflicting contributor PR drops its stale `hive:unauthorized` mark so the conflict is the row's blocker ([#14668](https://github.com/hivecommons/hive/issues/14668)). diff --git a/src/docs/contributor-relay.md b/src/docs/contributor-relay.md index d0dd238c5d..13a9a9452d 100644 --- a/src/docs/contributor-relay.md +++ b/src/docs/contributor-relay.md @@ -1221,6 +1221,8 @@ The one place a declaration DOES shape dispatch is the stage-072 contributor nud The hub records the mode it used in the nudge comment's marker (`mode=spawn|preempt|queued|comment`) and exposes queue depth / in-flight as `fix_queue` on the contributor profile API. +**When the relay fails the task.** `task_failed` for a fix's `task_id` does not settle it as done: its PRs go back in the hub's queue (same place by age) and are not interrupted back in for 5 minutes, though the relay's next `ready` serves them at once. After three failed dispatches a PR leaves the fix queue and the stage engine's re-nudge / adoption path takes over. A dispatch the relay fails within one second of the nudge is not reported as delivered: the nudge comment reads `mode=queued`, says the relay could not take the task (quoting the relay's `message`/`reason`), and the stage log carries `relay_failed=true` (hivecommons/hive#14668). + #### What the operator sees (Operations → Connected contributor agents) Each row of the "Connected contributor agents (ClankeR)" list on `/contribute/operations` reads the same declaration back (hivecommons/hive#12502), so an operator can check whether the rework the stage labels promise is actually reaching the agent. The row is fed by `/api/contribute/fleet`; each `clankers[]` entry carries: diff --git a/src/docs/labels-and-control-signals.md b/src/docs/labels-and-control-signals.md index 500a1f1629..cff5237536 100644 --- a/src/docs/labels-and-control-signals.md +++ b/src/docs/labels-and-control-signals.md @@ -1132,9 +1132,11 @@ When a clanker-authored contributor PR goes red the hive does not just relabel i focus-window rule; this ordering applies at task-boundary time. 7. Capacity ≥ 2: fixes are dispatched immediately as parallel sub-sessions, up to the advertised count so the primary session keeps going and is - never preempted; further failures queue, and each finished fix session - (`task_complete`/`task_failed` for the fix's `task_id`) frees a slot that - is refilled automatically with the next queued `fix-pr`. + never preempted; further failures queue, and each completed fix session + (`task_complete` for the fix's `task_id`) frees a slot that is refilled + automatically with the next queued `fix-pr`. A `task_failed` requeues the + fix with backoff instead (three attempts, then the re-nudge / adoption + path; see [contributor-relay.md § fix-pr tasks](contributor-relay.md#fix-pr-tasks)). All seven steps ship with the stage engine and the contribute hub (hivecommons/hive#12436). diff --git a/src/pkg/dashboard/contribute_fixqueue.go b/src/pkg/dashboard/contribute_fixqueue.go index c40c3e2921..3128d7fa33 100644 --- a/src/pkg/dashboard/contribute_fixqueue.go +++ b/src/pkg/dashboard/contribute_fixqueue.go @@ -43,6 +43,18 @@ const ( // taskUnavailableFixInFlight is the task_unavailable reason when a // contributor's queued fixes exist but every slot is busy: new work waits. taskUnavailableFixInFlight = "fix_pr_in_flight" + + // fixTaskAckGrace is how long Nudge waits after an interrupt dispatch for + // the relay to refuse it before reporting the nudge as delivered; relays + // that cannot take the task fail it within ~100ms (hivecommons/hive#14668). + fixTaskAckGrace = time.Second + // fixTaskRetryBackoff keeps a failed fix from being interrupted straight + // back into a relay that may be mid-turn; a task boundary still serves it. + fixTaskRetryBackoff = 5 * time.Minute + // fixTaskMaxAttempts is how many failed dispatches a PR gets before it + // leaves the fix queue and the stage engine's re-nudge / adoption path + // takes over. + fixTaskMaxAttempts = 3 ) // WSFixPR is one red PR inside a fix-pr task_assign. @@ -257,8 +269,15 @@ func (h *ContributeWSHub) Nudge(_ context.Context, n automerge.ContributorNudge) h.rememberFixOwner(author, item.Key()) out.Mode = "queued" if online { + failed := h.beginFixWait() + defer h.endFixWait() if msg := h.dispatchFixes(author, conns, false); msg != nil { out.Mode = msg.Mode + if reason, refused := h.awaitFixTaskRefusal(msg.TaskID, failed); refused { + out.Mode = "queued" + out.RelayFailed = true + out.RelayError = reason + } } } st := sched.Status(author, fixqueue.Capacity{Max: 1}) @@ -464,6 +483,137 @@ func (h *ContributeWSHub) completeFixTask(c *ContributorConnection, taskID strin return true } +// failFixTask settles a fix-pr task the relay reported failed: its PRs go +// back in the queue with backoff, and a PR that has failed +// fixTaskMaxAttempts times leaves the queue for the stage engine's re-nudge +// and adoption path. The freed slot is not refilled here — that would hand +// the same fix straight back. It returns false when msg.TaskID is not a fix +// task on this connection. +func (h *ContributeWSHub) failFixTask(c *ContributorConnection, msg WSMessage) bool { + c.mu.Lock() + _, ok := c.fixTasks[msg.TaskID] + if ok { + delete(c.fixTasks, msg.TaskID) + delete(c.fixTasksAt, msg.TaskID) + } + c.mu.Unlock() + if !ok { + return false + } + reason := relayFailureReason(msg) + if c.profile == nil { + h.logger.Info("[contribute-ws] fix-pr task failed", "username", identityOf(c), "task", msg.TaskID, "reason", reason) + return true + } + user := c.profile.GitHubUsername + sched := h.fixQueue() + requeued := sched.Requeue(user, msg.TaskID, fixTaskRetryBackoff) + if len(requeued) == 0 { + h.logger.Info("[contribute-ws] fix-pr task failed", "username", user, "task", msg.TaskID, "reason", reason) + } + for _, it := range requeued { + if it.Attempts >= fixTaskMaxAttempts { + sched.Remove(user, it.Repo, it.Number) + h.forgetFixOwner(user, it.Key()) + h.logger.Warn("[contribute-ws] fix-pr task failed; giving up on relay delivery", "username", user, "task", msg.TaskID, "pr", it.Key(), "attempts", it.Attempts, "reason", reason) + continue + } + h.logger.Info("[contribute-ws] fix-pr task failed; requeued", "username", user, "task", msg.TaskID, "pr", it.Key(), "attempts", it.Attempts, "retry_after", it.NotBefore, "reason", reason) + } + h.pulseFixFailed(msg.TaskID, reason) + return true +} + +// relayFailureReason is the relay's own words for a task_failed. +func relayFailureReason(msg WSMessage) string { + for _, r := range []string{msg.Message, msg.Reason, msg.FailureKind} { + if r = strings.TrimSpace(r); r != "" { + return r + } + } + return "" +} + +// beginFixWait registers a Nudge waiting to learn whether its dispatch is +// refused and returns the channel closed at the next fix-pr failure. Call it +// BEFORE dispatching so a failure racing the dispatch is not missed; pair it +// with endFixWait. Failures are only remembered while someone waits. +func (h *ContributeWSHub) beginFixWait() <-chan struct{} { + h.fixQueueMu.Lock() + defer h.fixQueueMu.Unlock() + h.fixWaiters++ + return h.fixFailedSignalLocked() +} + +func (h *ContributeWSHub) endFixWait() { + h.fixQueueMu.Lock() + defer h.fixQueueMu.Unlock() + if h.fixWaiters > 0 { + h.fixWaiters-- + } + if h.fixWaiters == 0 { + h.fixFailures = nil + } +} + +func (h *ContributeWSHub) fixFailedSignalLocked() chan struct{} { + if h.fixFailedSig == nil { + h.fixFailedSig = make(chan struct{}) + } + return h.fixFailedSig +} + +// pulseFixFailed records why taskID failed (while a Nudge waits) and wakes +// every waiter. +func (h *ContributeWSHub) pulseFixFailed(taskID, reason string) { + h.fixQueueMu.Lock() + defer h.fixQueueMu.Unlock() + if h.fixWaiters > 0 { + if h.fixFailures == nil { + h.fixFailures = map[string]string{} + } + h.fixFailures[taskID] = reason + } + if h.fixFailedSig != nil { + close(h.fixFailedSig) + h.fixFailedSig = nil + } +} + +// takeFixFailure reports (and forgets) whether taskID failed, and why. +func (h *ContributeWSHub) takeFixFailure(taskID string) (string, bool) { + h.fixQueueMu.Lock() + defer h.fixQueueMu.Unlock() + reason, ok := h.fixFailures[taskID] + delete(h.fixFailures, taskID) + return reason, ok +} + +// awaitFixTaskRefusal waits up to the ack grace for the relay to fail the +// fix-pr task it was just handed. refused=true means it did, so the nudge +// was not delivered as a task. +func (h *ContributeWSHub) awaitFixTaskRefusal(taskID string, failed <-chan struct{}) (reason string, refused bool) { + grace := h.fixAckGrace + if grace <= 0 { + grace = fixTaskAckGrace + } + timer := time.NewTimer(grace) + defer timer.Stop() + for { + if r, ok := h.takeFixFailure(taskID); ok { + return r, true + } + select { + case <-failed: + h.fixQueueMu.Lock() + failed = h.fixFailedSignalLocked() + h.fixQueueMu.Unlock() + case <-timer.C: + return h.takeFixFailure(taskID) + } + } +} + // FixQueueStatus is the contributor's fix-queue view for the profile API. func (h *ContributeWSHub) FixQueueStatus(username string) fixqueue.Status { conns, _ := h.liveConnectionsFor(username, time.Now()) diff --git a/src/pkg/dashboard/contribute_fixqueue_test.go b/src/pkg/dashboard/contribute_fixqueue_test.go index acf4600d5d..1926312fce 100644 --- a/src/pkg/dashboard/contribute_fixqueue_test.go +++ b/src/pkg/dashboard/contribute_fixqueue_test.go @@ -275,3 +275,64 @@ func TestFixPRInstructionCoalesced(t *testing.T) { } } } + +// hivecommons/hive#14668: a relay that fails the fix-pr task right after +// dispatch must not be reported as reached. The nudge reads "queued" with the +// relay's reason, the PR waits in the queue (no immediate re-interrupt), the +// next task boundary retries it, and after fixTaskMaxAttempts failures it +// leaves the queue for the stage engine's re-nudge / adoption path. +func TestFixQueueInstantRelayFailureIsNotDelivered(t *testing.T) { + hub, _ := covK2Hub(t) + hub.fixAckGrace = time.Minute + c := fixConn(hub, "alice", fixCaps(capFixPRInterrupt), time.Now()) + s := &wsSession{h: hub, contributor: c} + var got []WSMessage + hub.fixDispatched = func(_ *ContributorConnection, m WSMessage) { + got = append(got, m) + if len(got) == 1 { + go s.handleTaskFailed(WSMessage{Type: "task_failed", TaskID: m.TaskID, Message: "codex: session busy"}) + } + } + + out := hub.Nudge(context.Background(), fixNudge("r", 1, "build")) + if !out.Online || out.Mode != "queued" || !out.RelayFailed || out.RelayError != "codex: session busy" || out.QueueDepth != 1 { + t.Fatalf("instantly failed dispatch reported as %+v, want queued + relay failure", out) + } + if len(got) != 1 { + t.Fatalf("dispatches = %d, want the failed fix not interrupted straight back", len(got)) + } + if st := hub.FixQueueStatus("alice"); st.Queued != 1 || st.InFlight != 0 { + t.Fatalf("status after failure = %+v, want requeued", st) + } + if q := hub.FixQueueItems("alice"); len(q) != 1 || q[0].Attempts != 1 { + t.Fatalf("queued = %+v, want one item with 1 failed attempt", q) + } + c.mu.Lock() + tracked := len(c.fixTasks) + c.mu.Unlock() + if tracked != 0 { + t.Fatalf("failed fix task still tracked on the connection (%d)", tracked) + } + + // The relay's next task boundary retries it; two more failures exhaust it. + for attempt := 2; attempt <= fixTaskMaxAttempts; attempt++ { + msg := hub.selectFixTask(c) + if msg == nil || msg.Kind != taskKindFixPR || msg.Priority != taskPriorityQueued { + t.Fatalf("attempt %d: boundary should serve the queued fix, got %+v", attempt, msg) + } + s.handleTaskFailed(WSMessage{Type: "task_failed", TaskID: msg.TaskID}) + } + if st := hub.FixQueueStatus("alice"); st.Queued != 0 || st.InFlight != 0 { + t.Fatalf("status after %d failures = %+v, want the PR out of the queue", fixTaskMaxAttempts, st) + } + if msg := hub.selectFixTask(c); msg != nil { + t.Fatalf("exhausted fix served again: %+v", msg) + } + hub.fixQueueMu.Lock() + owned := len(hub.fixOwners["alice"]) + waiters, failures := hub.fixWaiters, len(hub.fixFailures) + hub.fixQueueMu.Unlock() + if owned != 0 || waiters != 0 || failures != 0 { + t.Fatalf("leftover state: owners=%d waiters=%d failures=%d", owned, waiters, failures) + } +} diff --git a/src/pkg/dashboard/contribute_ws.go b/src/pkg/dashboard/contribute_ws.go index 5690de07ae..c6804da8ff 100644 --- a/src/pkg/dashboard/contribute_ws.go +++ b/src/pkg/dashboard/contribute_ws.go @@ -903,6 +903,16 @@ type ContributeWSHub struct { fixSched *fixqueue.Scheduler fixOwners map[string]map[string]bool fixDispatched func(*ContributorConnection, WSMessage) + // fixFailedSig is closed (and replaced) whenever a relay reports a + // fix-pr task failed, waking a Nudge that is waiting to learn whether its + // dispatch was refused (hivecommons/hive#14668). fixAckGrace overrides + // that wait (zero = fixTaskAckGrace); a test hook. + // fixWaiters counts those Nudges; fixFailures (task id → relay reason) + // is kept only while one waits. + fixFailedSig chan struct{} + fixWaiters int + fixFailures map[string]string + fixAckGrace time.Duration // stageStore is the stage-label memory the fleet view reads "PRs returned // to this contributor" from (hivecommons/hive#12502). Nil means the // process-wide stagelabels.Shared(); tests inject an in-memory store. @@ -4156,8 +4166,9 @@ func (s *wsSession) handleTaskFailed(msg WSMessage) { h := s.h // hivecommons/hive#12436: a fix-pr task settles in the contributor's fix // queue (freeing its slot and refilling it) and never touches currentTask. - if s.contributor != nil && msg.TaskID != "" && h.completeFixTask(s.contributor, msg.TaskID) { - h.logger.Info("[contribute-ws] fix-pr task failed", "username", identityOf(s.contributor), "task", msg.TaskID) + // hivecommons/hive#14668: a failed fix is requeued with backoff (retried, + // then dropped back to the stage engine), not settled as done. + if s.contributor != nil && msg.TaskID != "" && h.failFixTask(s.contributor, msg) { return } if s.contributor != nil { diff --git a/src/pkg/fixqueue/fixqueue.go b/src/pkg/fixqueue/fixqueue.go index 732667e0d2..50ec91abac 100644 --- a/src/pkg/fixqueue/fixqueue.go +++ b/src/pkg/fixqueue/fixqueue.go @@ -71,6 +71,10 @@ type Item struct { // EnqueuedAt is when the item entered the queue (set by Enqueue; read by // Overflow). EnqueuedAt time.Time + // Attempts counts the dispatches of this item the relay reported failed + // (set by Requeue); NotBefore is when an interrupt may retry it. + Attempts int + NotBefore time.Time } // Key identifies an item. @@ -225,14 +229,14 @@ func (s *Scheduler) Enqueue(user string, it Item) bool { key := it.Key() for _, d := range c.inflight { if old, ok := d.items[key]; ok { - it.EnqueuedAt = old.EnqueuedAt + it.EnqueuedAt, it.Attempts, it.NotBefore = old.EnqueuedAt, old.Attempts, old.NotBefore d.items[key] = it return false } } for i, q := range c.queue { if q.Key() == key { - it.EnqueuedAt = q.EnqueuedAt + it.EnqueuedAt, it.Attempts, it.NotBefore = q.EnqueuedAt, q.Attempts, q.NotBefore c.queue[i] = it sortQueue(c.queue) return false @@ -338,10 +342,46 @@ func (s *Scheduler) CompleteDispatch(user, id string) bool { return false } +// Requeue settles a dispatch the relay reported failed and puts its items +// back in the queue (keeping their place by age) with Attempts incremented +// and NotBefore = now+backoff, so an interrupt does not hand the same fix +// straight back to a relay that just refused it. It returns the requeued +// items; nil when id is not in flight. +func (s *Scheduler) Requeue(user, id string, backoff time.Duration) []Item { + s.mu.Lock() + defer s.mu.Unlock() + c, ok := s.by[norm(user)] + if !ok { + return nil + } + for i, d := range c.inflight { + if d.id != id { + continue + } + c.inflight = append(c.inflight[:i], c.inflight[i+1:]...) + if d.mode == ModePreempt { + // The primary was never parked for long: no focus window. + c.preempted = false + } + out := d.public().Items + notBefore := s.opts.Now().Add(backoff) + for j := range out { + out[j].Attempts++ + out[j].NotBefore = notBefore + c.queue = append(c.queue, out[j]) + } + sortQueue(c.queue) + s.log("requeue", user, Dispatch{ID: d.id, Mode: d.mode, Items: out}, len(c.queue)) + return out + } + return nil +} + // Dispatch tries to hand user the next fix while a primary task is running. // It returns nil when nothing may be dispatched now: empty queue, all slots -// busy, a preemption still in flight, or the resumed primary's focus window -// has not elapsed. The returned Dispatch's Mode is spawn or preempt. +// busy, a preemption still in flight, the resumed primary's focus window +// has not elapsed, or the queue head is backing off after a failed dispatch. +// The returned Dispatch's Mode is spawn or preempt. func (s *Scheduler) Dispatch(user string, cap Capacity) *Dispatch { s.mu.Lock() defer s.mu.Unlock() @@ -349,6 +389,9 @@ func (s *Scheduler) Dispatch(user string, cap Capacity) *Dispatch { if !ok || len(c.queue) == 0 || len(c.inflight) >= cap.Slots() { return nil } + if s.opts.Now().Before(c.queue[0].NotBefore) { + return nil + } mode := cap.Mode() if mode == ModePreempt && (c.preempted || s.opts.Now().Before(c.focusUntil)) { return nil diff --git a/src/pkg/fixqueue/fixqueue_test.go b/src/pkg/fixqueue/fixqueue_test.go index 34e2e7178f..6bd34cb1ef 100644 --- a/src/pkg/fixqueue/fixqueue_test.go +++ b/src/pkg/fixqueue/fixqueue_test.go @@ -276,6 +276,46 @@ func TestEnqueueIdempotentAndRemove(t *testing.T) { } } +// A dispatch the relay failed goes back in the queue with its attempt +// counted, frees the preemption, and is not interrupted back in until the +// backoff elapses; a task boundary may serve it at once (hivecommons/hive#14668). +func TestRequeueFailedDispatchBacksOff(t *testing.T) { + c := newClock() + s := newSched(c) + s.Enqueue("ida", red(1, time.Hour, c)) + d := s.Dispatch("ida", Capacity{Max: 1}) + if d == nil || d.Mode != ModePreempt { + t.Fatalf("dispatch = %+v", d) + } + if s.Requeue("ida", "nope", time.Minute) != nil || s.Requeue("zed", d.ID, time.Minute) != nil { + t.Fatal("unknown dispatch requeued") + } + got := s.Requeue("ida", d.ID, time.Minute) + if len(got) != 1 || got[0].Attempts != 1 || !got[0].NotBefore.Equal(c.at.Add(time.Minute)) { + t.Fatalf("requeued = %+v", got) + } + if st := s.Status("ida", Capacity{Max: 1}); st.Queued != 1 || st.InFlight != 0 || st.FocusUntil != nil { + t.Fatalf("status after requeue = %+v", st) + } + if s.Dispatch("ida", Capacity{Max: 1}) != nil { + t.Fatal("interrupt re-dispatched inside the backoff") + } + // A refreshed nudge keeps the attempt count and the backoff. + s.Enqueue("ida", red(1, time.Hour, c)) + if q := s.Queued("ida"); len(q) != 1 || q[0].Attempts != 1 || q[0].NotBefore.IsZero() { + t.Fatalf("refresh lost the retry state: %+v", q) + } + c.advance(time.Minute) + d2 := s.Dispatch("ida", Capacity{Max: 1}) + if d2 == nil || d2.ID == d.ID || d2.Items[0].Attempts != 1 { + t.Fatalf("retry after backoff = %+v", d2) + } + s.Requeue("ida", d2.ID, time.Hour) + if b := s.DispatchAtBoundary("ida", Capacity{Max: 1}); b == nil || b.Items[0].Attempts != 2 { + t.Fatalf("boundary should serve a backing-off fix: %+v", b) + } +} + func TestCompleteDispatchAndPartialRemove(t *testing.T) { c := newClock() s := newSched(c) diff --git a/src/pkg/github/automerge/automerge_contributor_held_runs_test.go b/src/pkg/github/automerge/automerge_contributor_held_runs_test.go index 967918da3e..b31434424e 100644 --- a/src/pkg/github/automerge/automerge_contributor_held_runs_test.go +++ b/src/pkg/github/automerge/automerge_contributor_held_runs_test.go @@ -367,6 +367,45 @@ func TestContributorLaneUnauthorizedWhenApproveForbidden(t *testing.T) { } } +// hivecommons/hive#14668: once the head conflicts, the held runs are no +// longer the blocker (the rebase push brings new ones), so the lane reports +// `conflicting` and takes the stale hive:unauthorized mark off in that tick. +func TestContributorLaneConflictClearsUnauthorized(t *testing.T) { + fx := validContributorFixture() + fx.baseRef = "main" + fx.body = "Part of #42\n\nSome change." + fx.issueComments = nil + fx.requiredContexts = []string{"ci/build", contributorHeldRunsTestContext} + fx.workflowRuns = []map[string]any{heldRun(11, "acme/widget")} + fx.approveStatus = http.StatusForbidden + api := newContributorLaneAPI(t, &fx) + defer api.Close() + policy := defaultContributorLanePolicy() + policy.RegistryTiersSufficient = true + policy.ApproveHeldRunsForRegistryTrusted = true + clock := &contributorLaneClock{at: time.Date(2026, 10, 27, 13, 0, 0, 0, time.UTC)} + engine := newContributorLaneEngine(api.URL, policy, clock, Options{ContributorRegistryTier: registryTiers(map[string]string{contributorLaneTestAuthor: "trusted"}), ContributorSoak: soakedTracker(clock)}) + + if _, reason, err := engine.trySweepContributorLanePR(context.Background(), "widget", "acme", "widget", 7, policy); err != nil || !strings.HasPrefix(reason, unauthorizedReasonPrefix) { + t.Fatalf("mark tick reason = %q err = %v, want %s…", reason, err, unauthorizedReasonPrefix) + } + if !slices.Contains(fx.labels, unauthorizedLabel) { + t.Fatalf("labels = %v, want %q after the forbidden approval", fx.labels, unauthorizedLabel) + } + + fx.mergeableState = "dirty" + _, reason, err := engine.trySweepContributorLanePR(context.Background(), "widget", "acme", "widget", 7, policy) + if err != nil || reason != "conflicting" { + t.Fatalf("conflict tick reason = %q err = %v, want conflicting", reason, err) + } + if len(fx.labelsRemoved) != 1 || fx.labelsRemoved[0] != unauthorizedLabel || slices.Contains(fx.labels, unauthorizedLabel) { + t.Fatalf("labels removed = %v labels = %v, want %q removed once the head conflicts", fx.labelsRemoved, fx.labels, unauthorizedLabel) + } + if step := engine.unauthorizedStep("acme", "widget", 7, "sha7"); step != "" { + t.Fatalf("unauthorizedStep after conflict = %q, want empty", step) + } +} + func TestIsResourceNotAccessible(t *testing.T) { mk := func(status int, msg string) error { return &gh.ErrorResponse{Response: &http.Response{StatusCode: status, Request: &http.Request{Method: "POST", URL: &url.URL{Path: "/x"}}}, Message: msg} diff --git a/src/pkg/github/automerge/automerge_contributor_lane.go b/src/pkg/github/automerge/automerge_contributor_lane.go index d5bebc782c..8a3c01cc09 100644 --- a/src/pkg/github/automerge/automerge_contributor_lane.go +++ b/src/pkg/github/automerge/automerge_contributor_lane.go @@ -1241,6 +1241,11 @@ func (c *Engine) evaluateContributorLanePR(ctx context.Context, displayRepo, own return eval, reason, err } if mergeableState == "dirty" || mergeableState == "conflicting" { + // A conflicted head is the contributor's to fix, and the rebase push + // brings new held runs: approving this head's runs is wasted, so the + // unauthorized mark must not stay the row's blocker. The next head + // re-marks if its runs are held too (hivecommons/hive#14668). + c.clearUnauthorized(ctx, displayRepo, owner, repo, pr, number, unauthorizedStepApproveWorkflowRuns) return eval, "conflicting", nil } var methodReason string diff --git a/src/pkg/github/automerge/automerge_contributor_nudge.go b/src/pkg/github/automerge/automerge_contributor_nudge.go index aaea39004a..e4b2dd0f4d 100644 --- a/src/pkg/github/automerge/automerge_contributor_nudge.go +++ b/src/pkg/github/automerge/automerge_contributor_nudge.go @@ -67,6 +67,11 @@ type ContributorNudgeOutcome struct { Online bool // QueueDepth is the author's fix-queue depth after this nudge. QueueDepth int + // RelayFailed is true when the relay reported the fix-pr task failed + // right after dispatch: the nudge was not delivered as a task and the + // fix waits in the queue for a retry; RelayError is the relay's reason. + RelayFailed bool + RelayError string } // ContributorNudgeSink receives stage-072 nudges. The dashboard hub implements @@ -204,6 +209,14 @@ func ContributorNudgeComment(n ContributorNudge, out ContributorNudgeOutcome) st case "preempt": b.WriteString("\n_Your relay session received this as a `fix-pr` task; your current task is parked (claim kept alive) until this PR is green._\n") case "queued": + if out.RelayFailed { + b.WriteString("\n_Your relay could not take this as a `fix-pr` task") + if e := strings.TrimSpace(out.RelayError); e != "" { + fmt.Fprintf(&b, " (%s)", e) + } + b.WriteString("; the hive will retry it from your relay fix queue._\n") + break + } fmt.Fprintf(&b, "\n_Queued in your relay fix queue (%d waiting): fixes are handed out before new work as soon as you have capacity._\n", out.QueueDepth) } if n.Stalled { @@ -270,7 +283,7 @@ func (c *Engine) nudgeContributor(ctx context.Context, store *stagelabels.Store, return 0, err } store.MarkNudged(key, obs.HeadSHA, ordinal, out.Online || out.Mode == "spawn" || out.Mode == "preempt" || out.Mode == "queued", now) - c.info("stage labels: contributor nudged", "repo", obs.Display, "pr", obs.Number, "author", obs.Author, "head", obs.HeadSHA, "nudge", ordinal, "mode", nonEmpty(out.Mode, "comment"), "online", out.Online, "queue_depth", out.QueueDepth, "reason", obs.Reason) + c.info("stage labels: contributor nudged", "repo", obs.Display, "pr", obs.Number, "author", obs.Author, "head", obs.HeadSHA, "nudge", ordinal, "mode", nonEmpty(out.Mode, "comment"), "online", out.Online, "queue_depth", out.QueueDepth, "relay_failed", out.RelayFailed, "reason", obs.Reason) return 1, nil } diff --git a/src/pkg/github/automerge/automerge_contributor_nudge_test.go b/src/pkg/github/automerge/automerge_contributor_nudge_test.go index d0e704c7ae..2657e20828 100644 --- a/src/pkg/github/automerge/automerge_contributor_nudge_test.go +++ b/src/pkg/github/automerge/automerge_contributor_nudge_test.go @@ -403,3 +403,20 @@ func TestTrustedHuman071CommentReasons(t *testing.T) { }) } } + +// hivecommons/hive#14668: a fix-pr task the relay failed right after +// dispatch is not reported as received; the comment says the relay could not +// take it, names the relay's reason and promises the retry. +func TestContributorNudgeCommentRelayFailed(t *testing.T) { + n := ContributorNudge{Author: "alice", HeadSHA: "abc", Nudge: 1} + body := ContributorNudgeComment(n, ContributorNudgeOutcome{Mode: "queued", Online: true, RelayFailed: true, RelayError: "codex: session busy"}) + if !strings.Contains(body, "could not take this as a `fix-pr` task (codex: session busy)") || !strings.Contains(body, "retry") { + t.Fatalf("relay-failed comment = %q", body) + } + if strings.Contains(body, "received this as a `fix-pr` task") || strings.Contains(body, "Queued in your relay fix queue") { + t.Fatalf("relay-failed comment must not claim delivery or plain queueing: %q", body) + } + if body := ContributorNudgeComment(n, ContributorNudgeOutcome{Mode: "queued", QueueDepth: 2}); !strings.Contains(body, "Queued in your relay fix queue (2 waiting)") { + t.Fatalf("plain queued comment = %q", body) + } +} From c0f861c48aaede0765a2ff63e4963e841a4b1f51 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:36:56 -0400 Subject: [PATCH 2/2] fix(contribute): clear hive:unauthorized at the early conflict return contributorLaneConflictReason returns "conflicting" before the later dirty/conflicting branch, so the clearUnauthorized added there never ran and the label stayed on a conflicting head. Clear it at the early return instead. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../github/automerge/automerge_contributor_lane.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/pkg/github/automerge/automerge_contributor_lane.go b/src/pkg/github/automerge/automerge_contributor_lane.go index 8a3c01cc09..80b5799a76 100644 --- a/src/pkg/github/automerge/automerge_contributor_lane.go +++ b/src/pkg/github/automerge/automerge_contributor_lane.go @@ -1177,6 +1177,13 @@ func (c *Engine) evaluateContributorLanePR(ctx context.Context, displayRepo, own return contributorLaneEval{pr: pr}, reason, nil } if reason := contributorLaneConflictReason(pr); reason != "" { + if reason == "conflicting" { + // A conflicted head is the contributor's to fix, and the rebase push + // brings new held runs: approving this head's runs is wasted, so the + // unauthorized mark must not stay the row's blocker. The next head + // re-marks if its runs are held too (hivecommons/hive#14668). + c.clearUnauthorized(ctx, displayRepo, owner, repo, pr, number, unauthorizedStepApproveWorkflowRuns) + } return contributorLaneEval{pr: pr}, reason, nil } author := hgithub.SafeGetLogin(pr.GetUser()) @@ -1241,11 +1248,6 @@ func (c *Engine) evaluateContributorLanePR(ctx context.Context, displayRepo, own return eval, reason, err } if mergeableState == "dirty" || mergeableState == "conflicting" { - // A conflicted head is the contributor's to fix, and the rebase push - // brings new held runs: approving this head's runs is wasted, so the - // unauthorized mark must not stay the row's blocker. The next head - // re-marks if its runs are held too (hivecommons/hive#14668). - c.clearUnauthorized(ctx, displayRepo, owner, repo, pr, number, unauthorizedStepApproveWorkflowRuns) return eval, "conflicting", nil } var methodReason string