diff --git a/bin/contributor-relay.js b/bin/contributor-relay.js index 01d000550d..17b694b0ac 100755 --- a/bin/contributor-relay.js +++ b/bin/contributor-relay.js @@ -9995,7 +9995,16 @@ function handleMessage(data, hub) { // A standby assignment is pushed by the hub that acknowledged this relay's // standby; the relay is not polling, so that hub is never the active slot. const standbyPush = !!msg.standby_lane && hub.standbyAcked === true; - if (!currentTask && !standbyPush && hub !== hubs[activeHubIndex]) { + // A fix-pr task is rework on this contributor's own PR, and it is + // pushed by the hive that owns the PR whichever hive the relay was + // polling. Refusing it from a non-active hub threw away a fix the relay + // had already accepted and deferred (projectbluefin/utah-packages#972, + // 2026-11-09: "Hub is not the active polling slot" the moment the + // headless primary ended). It is taken from any subscribed hub; the + // ordinary path below makes that hub the active slot and tells every + // other hive the relay is busy elsewhere. + const fixPush = msg.kind === FIX_PR_KIND; + if (!currentTask && !standbyPush && !fixPush && hub !== hubs[activeHubIndex]) { console.log(`Rejecting task ${taskDisplay(msg)} from ${hub.url} — hub is not the active polling slot`); sendTo(hub, { type: 'task_failed', seq: nextSeq(), task_id: msg.task_id, reason: 'Hub is not the active polling slot' }); break; diff --git a/bin/contributor-relay.test.js b/bin/contributor-relay.test.js index e5ce79755a..c0fd7b226b 100644 --- a/bin/contributor-relay.test.js +++ b/bin/contributor-relay.test.js @@ -17417,6 +17417,75 @@ test('busy_elsewhere_v1: a hive that does not advertise the token never sees the } finally { console.log = log; console.warn = warn; relay.setCurrentTask(null); teardown(relay); } }); +// --------------------------------------------------------------------------- +// projectbluefin/utah-packages#972 (2026-11-09): the projectbluefin hive sent +// Karibhou's relay a fix-pr preempt for its own PR at 14:42:58Z while the +// relay was busy. The relay accepted it and deferred it behind the headless +// primary, then refused it at 14:45:39Z with "Hub is not the active polling +// slot" the moment the primary ended, because the polling slot was on another +// hive. A fix-pr is taken from any subscribed hub. +// --------------------------------------------------------------------------- + +test('#972 a fix-pr pushed by a non-active hub to an idle relay is taken, and the other hive hears busy_elsewhere', () => { + const relay = loadRelay({ env: { ...MULTI_HUB_ENV, HIVE_CONTRIBUTOR_QUOTA_GUARD: 'off' } }); + const log = console.log; const warn = console.warn; + console.log = () => {}; console.warn = () => {}; + try { + const { hubs, sentA, sentB } = attachHubSinks(relay); + busyAuth(relay, hubs[0]); busyAuth(relay, hubs[1]); + relay.setCurrentTask(null); + relay.handleMessage(JSON.stringify(FIX_PR_TASK('fix-karibhou-1', { mode: 'preempt' })), hubs[1]); + assert.ok(!sentB.some(m => m.type === 'task_failed' && m.task_id === 'fix-karibhou-1'), + `the fix must not be refused for the polling slot: ${JSON.stringify(sentB.filter(m => m.type === 'task_failed'))}`); + const task = relay.getCurrentTask(); + assert.strictEqual(task && task.task_id, 'fix-karibhou-1', 'the fix becomes the current task'); + assert.strictEqual(task._hub, hubs[1], 'the fix belongs to the hive that sent it'); + assert.deepStrictEqual(busyElsewhereFrames(sentA).map(m => m.state), ['busy'], + 'the hive the relay was polling must hear that it is busy elsewhere'); + assert.deepStrictEqual(busyElsewhereFrames(sentB), [], 'the hive that owns the fix is never told it is busy elsewhere'); + + // Finishing the fix clears the report on the other hive. + relay.finishCurrentTask({ completionSignal: 'verdict', summary: 'done', tmuxLines: [], prURL: null, noWork: null }); + assert.deepStrictEqual(busyElsewhereFrames(sentA).map(m => m.state), ['busy', 'clear']); + } finally { console.log = log; console.warn = warn; relay.setCurrentTask(null); teardown(relay); } +}); + +test('#972 a fix-pr deferred behind a headless primary runs when the primary ends, even with the polling slot on another hive', () => { + const relay = loadRelay({ backend: 'claude', mode: 'headless', env: { ...MULTI_HUB_ENV, HIVE_MAX_PARALLEL_FIXES: '0', HIVE_CONTRIBUTOR_QUOTA_GUARD: 'off' }, execFileResult: { defer: true } }); + const log = console.log; const warn = console.warn; + console.log = () => {}; console.warn = () => {}; + try { + const { hubs, sentA, sentB } = attachHubSinks(relay); + busyAuth(relay, hubs[0]); busyAuth(relay, hubs[1]); + relay.setCurrentTask({ task_id: 'pr-review-1794235343201', task_gen: 0, kind: 'pr-review', repo: 'org/a', number: 0, title: 'local review' }); + relay.handleMessage(JSON.stringify(FIX_PR_TASK('fix-karibhou-1', { mode: 'preempt' })), hubs[1]); + assert.deepStrictEqual(relay.fixPRState().deferred, ['fix-karibhou-1'], 'precondition: the fix waits behind the headless primary'); + assert.ok(sentB.some(m => m.type === 'task_accepted' && m.task_id === 'fix-karibhou-1'), 'precondition: the relay accepted the fix'); + + // The primary ends and the relay asks hive A (its polling slot) for work. + relay.setCurrentTask(null); + relay.requestWork(hubs[0], 'headless_task_complete'); + assert.ok(!sentB.some(m => m.type === 'task_failed' && m.task_id === 'fix-karibhou-1'), + `an accepted fix must not be refused when it starts: ${JSON.stringify(sentB.filter(m => m.type === 'task_failed'))}`); + const task = relay.getCurrentTask(); + assert.strictEqual(task && task.task_id, 'fix-karibhou-1', 'the deferred fix runs'); + assert.strictEqual(task._hub, hubs[1]); + assert.deepStrictEqual(relay.fixPRState().deferred, []); + assert.deepStrictEqual(busyElsewhereFrames(sentA).map(m => m.state), ['busy'], + 'the hive the relay was polling must hear that it is busy elsewhere'); + } finally { console.log = log; console.warn = warn; relay.setCurrentTask(null); teardown(relay); } +}); + +test('#972 the polling-slot refusal still applies to ordinary work from a non-active hub', () => { + const relay = loadRelay({ env: MULTI_HUB_ENV }); + try { + const { hubs, sentB } = attachHubSinks(relay); + relay.handleMessage(JSON.stringify({ type: 'task_assign', task_id: 'issue-1', kind: 'issue', repo: 'foo/bar', number: 2, title: 'y' }), hubs[1]); + assert.strictEqual(relay.getCurrentTask(), null); + assert.ok(sentB.some(m => m.type === 'task_failed' && m.task_id === 'issue-1' && m.reason === 'Hub is not the active polling slot')); + } finally { teardown(relay); } +}); + // --------------------------------------------------------------------------- // quota_hold_v1: a held relay tells the hub, so it stops counting as // contributor-first cover. Observed on 2026-11-08: a relay acting as scanner diff --git a/changelog.d/fixed-fix-pr-any-hub.md b/changelog.d/fixed-fix-pr-any-hub.md new file mode 100644 index 0000000000..c15eaf73c9 --- /dev/null +++ b/changelog.d/fixed-fix-pr-any-hub.md @@ -0,0 +1 @@ +- A relay subscribed to several hives no longer refuses a `fix-pr` task for its contributor's own PR because it was polling another hive. On projectbluefin/utah-packages#972 (2026-11-09) the relay accepted a fix at 14:42:58Z, deferred it behind a headless task, and refused it with `Hub is not the active polling slot` the moment that task ended at 14:45:39Z; the hub booked a failed attempt and a five-minute retry. The relay now takes a `fix-pr` from any subscribed hub, makes that hub its active one and tells the other hives it is busy elsewhere, and the hub puts a fix refused for the polling slot back in its queue without counting an attempt. Ordinary work from a hive the relay is not polling is still refused. diff --git a/src/docs/contributor-relay.md b/src/docs/contributor-relay.md index ce93ff4263..9021c5e6e0 100644 --- a/src/docs/contributor-relay.md +++ b/src/docs/contributor-relay.md @@ -959,6 +959,8 @@ The lists are positional: the first token belongs to the first hub, the second t The relay keeps a WebSocket and heartbeat for each subscribed hub, but shares one CLI/tmux session and works on only one task at a time. It rotates to another hub when the active hub has no assignable work. A task that is blocked on human action stays with its owning hub; the relay does not mix task state across hubs. +Only the hub the relay is polling may hand it ordinary work; an assignment from any other hub is refused with `Hub is not the active polling slot`. A `fix-pr` task is the exception. It is rework on one of your own PRs, pushed by the hive that owns the PR, so the relay takes it from any subscribed hub, makes that hub its active one, and tells the other hives it is busy elsewhere (`busy_elsewhere_v1`). This also covers a fix the relay accepted and deferred behind a headless task: it starts when that task ends, wherever the polling slot is (projectbluefin/utah-packages#972). A hub that gets `Hub is not the active polling slot` for a fix from an older relay puts the fix back in the queue without counting a failed attempt, so such refusals never hand the PR to adoption. + ### Named profiles instead of hand-edited lists (`hivectl hives`) Positional lists have no names, and one hand-edit that drops a field transposes every hub/token pair after it. `hivectl hives` ([#8097](https://github.com/hivecommons/hive/issues/8097)) saves the same set as named profiles in `~/.config/hive/profiles.yml` (mode 0600) and **generates** `contributor.env` from them, so the lists are aligned by construction and the relay keeps reading exactly the variables documented above: diff --git a/src/pkg/dashboard/contribute_fixqueue.go b/src/pkg/dashboard/contribute_fixqueue.go index 673aca1627..8e812bca1b 100644 --- a/src/pkg/dashboard/contribute_fixqueue.go +++ b/src/pkg/dashboard/contribute_fixqueue.go @@ -539,6 +539,20 @@ func (h *ContributeWSHub) failFixTask(c *ContributorConnection, msg WSMessage) b } user := c.profile.GitHubUsername sched := h.fixQueue() + if relayDidNotTakeFix(msg) { + // The relay never started the fix (it was polling another hive): + // nothing about the PR failed, so no attempt is booked toward + // fixTaskMaxAttempts. + returned := sched.Return(user, msg.TaskID, fixTaskRetryBackoff) + for _, it := range returned { + h.logger.Info("[contribute-ws] fix-pr task not taken by the relay; requeued without counting an attempt", "username", user, "task", msg.TaskID, "pr", it.Key(), "attempts", it.Attempts, "retry_after", it.NotBefore, "reason", reason) + } + if len(returned) == 0 { + h.logger.Info("[contribute-ws] fix-pr task failed", "username", user, "task", msg.TaskID, "reason", reason) + } + h.pulseFixFailed(msg.TaskID, reason) + return true + } requeued := sched.Requeue(user, msg.TaskID, fixTaskRetryBackoff) if len(requeued) == 0 { h.logger.Info("[contribute-ws] fix-pr task failed", "username", user, "task", msg.TaskID, "reason", reason) @@ -556,6 +570,20 @@ func (h *ContributeWSHub) failFixTask(c *ContributorConnection, msg WSMessage) b return true } +// relayRefusedNotActiveSlot is the task_failed reason a relay sends for an +// assignment from a hive it is not polling (bin/contributor-relay.js, the +// task_assign handler). Relays from before projectbluefin/utah-packages#972 +// send it for a fix-pr too. +const relayRefusedNotActiveSlot = "Hub is not the active polling slot" + +// relayDidNotTakeFix reports a fix-pr task_failed that says the relay never +// started the task because it was polling another hive, not that the fix +// was tried. On projectbluefin/utah-packages#972 one such refusal booked +// attempts=1, a third of the way to handing the PR to adoption. +func relayDidNotTakeFix(msg WSMessage) bool { + return strings.EqualFold(strings.TrimSpace(msg.Reason), relayRefusedNotActiveSlot) +} + // relayFailureReason is the relay's own words for a task_failed. func relayFailureReason(msg WSMessage) string { for _, r := range []string{msg.Message, msg.Reason, msg.FailureKind} { diff --git a/src/pkg/dashboard/contribute_fixqueue_test.go b/src/pkg/dashboard/contribute_fixqueue_test.go index 1926312fce..e644b5a9c9 100644 --- a/src/pkg/dashboard/contribute_fixqueue_test.go +++ b/src/pkg/dashboard/contribute_fixqueue_test.go @@ -336,3 +336,46 @@ func TestFixQueueInstantRelayFailureIsNotDelivered(t *testing.T) { t.Fatalf("leftover state: owners=%d waiters=%d failures=%d", owned, waiters, failures) } } + +// projectbluefin/utah-packages#972 (2026-11-09 14:45:39Z): a relay polling +// another hive refused the fix-pr with "Hub is not the active polling slot" +// and the hub booked attempts=1 toward fixTaskMaxAttempts. The relay never +// started the fix, so the refusal is requeued without counting: any number +// of such refusals never hands the PR to adoption, and a real failure after +// them still counts from zero. +func TestFixQueuePollingSlotRefusalIsNotAnAttempt(t *testing.T) { + hub, _ := covK2Hub(t) + hub.fixAckGrace = time.Millisecond + c := fixConn(hub, "alice", fixCaps(capFixPRInterrupt), time.Now()) + s := &wsSession{h: hub, contributor: c} + got := captureFixDispatch(hub) + hub.Nudge(context.Background(), fixNudge("r", 1, "build")) + if len(*got) != 1 { + t.Fatalf("dispatches = %d, want the nudge's interrupt", len(*got)) + } + + // The interrupt is refused, then every task-boundary retry is too. + taskID := (*got)[0].TaskID + for i := 0; i <= fixTaskMaxAttempts; i++ { + if i > 0 { + msg := hub.selectFixTask(c) + if msg == nil || msg.Kind != taskKindFixPR { + t.Fatalf("refusal %d: the queued fix must still be served, got %+v", i, msg) + } + taskID = msg.TaskID + } + s.handleTaskFailed(WSMessage{Type: "task_failed", TaskID: taskID, Reason: "Hub is not the active polling slot"}) + if q := hub.FixQueueItems("alice"); len(q) != 1 || q[0].Attempts != 0 { + t.Fatalf("after refusal %d: queued = %+v, want the PR back with 0 attempts", i, q) + } + } + + msg := hub.selectFixTask(c) + if msg == nil || msg.Kind != taskKindFixPR { + t.Fatalf("the fix left the queue after refusals only: %+v", msg) + } + s.handleTaskFailed(WSMessage{Type: "task_failed", TaskID: msg.TaskID, Message: "codex: session busy"}) + if q := hub.FixQueueItems("alice"); len(q) != 1 || q[0].Attempts != 1 { + t.Fatalf("a real failure after refusals: queued = %+v, want 1 attempt", q) + } +} diff --git a/src/pkg/fixqueue/fixqueue.go b/src/pkg/fixqueue/fixqueue.go index d0078def6d..bfcd546021 100644 --- a/src/pkg/fixqueue/fixqueue.go +++ b/src/pkg/fixqueue/fixqueue.go @@ -362,6 +362,18 @@ func (s *Scheduler) CompleteDispatch(user, id string) bool { // straight back to a relay that just refused it. It returns the requeued // items; nil when id is not in flight. func (s *Scheduler) Requeue(user, id string, backoff time.Duration) []Item { + return s.requeue(user, id, backoff, true) +} + +// Return is Requeue for a dispatch the relay never started because it was +// not free to take it (busy, or polling another hive): the items go back +// with the same backoff but Attempts unchanged, so a refusal that says +// nothing about the fix never moves a PR toward the relay hand-off limit. +func (s *Scheduler) Return(user, id string, backoff time.Duration) []Item { + return s.requeue(user, id, backoff, false) +} + +func (s *Scheduler) requeue(user, id string, backoff time.Duration, attempted bool) []Item { s.mu.Lock() defer s.mu.Unlock() c, ok := s.by[norm(user)] @@ -380,7 +392,9 @@ func (s *Scheduler) Requeue(user, id string, backoff time.Duration) []Item { out := d.public().Items notBefore := s.opts.Now().Add(backoff) for j := range out { - out[j].Attempts++ + if attempted { + out[j].Attempts++ + } out[j].NotBefore = notBefore c.queue = append(c.queue, out[j]) } diff --git a/src/pkg/fixqueue/fixqueue_test.go b/src/pkg/fixqueue/fixqueue_test.go index 94c3c5f7b6..0a042f6b7f 100644 --- a/src/pkg/fixqueue/fixqueue_test.go +++ b/src/pkg/fixqueue/fixqueue_test.go @@ -316,6 +316,39 @@ func TestRequeueFailedDispatchBacksOff(t *testing.T) { } } +// A dispatch the relay never started because it was not free (busy, or +// polling another hive) goes back like a failed one but keeps its attempt +// count (projectbluefin/utah-packages#972: one such refusal booked +// attempts=1 toward the relay hand-off). +func TestReturnKeepsAttempts(t *testing.T) { + c := newClock() + s := newSched(c) + s.Enqueue("ida", red(1, time.Hour, c)) + d := s.Dispatch("ida", Capacity{Max: 1}) + if d == nil { + t.Fatal("no dispatch") + } + if s.Return("ida", "nope", time.Minute) != nil || s.Return("zed", d.ID, time.Minute) != nil { + t.Fatal("unknown dispatch returned") + } + got := s.Return("ida", d.ID, time.Minute) + if len(got) != 1 || got[0].Attempts != 0 || !got[0].NotBefore.Equal(c.at.Add(time.Minute)) { + t.Fatalf("returned = %+v, want Attempts 0 and the backoff", got) + } + if st := s.Status("ida", Capacity{Max: 1}); st.Queued != 1 || st.InFlight != 0 { + t.Fatalf("status after return = %+v", st) + } + // A real failure after the refusal is still counted from zero. + c.advance(time.Minute) + d2 := s.Dispatch("ida", Capacity{Max: 1}) + if d2 == nil { + t.Fatal("no retry after the backoff") + } + if got := s.Requeue("ida", d2.ID, time.Minute); len(got) != 1 || got[0].Attempts != 1 { + t.Fatalf("requeue after a return = %+v, want Attempts 1", got) + } +} + func TestCompleteDispatchAndPartialRemove(t *testing.T) { c := newClock() s := newSched(c)