From de041344dab214e5557a278f65dfa6c5d34ed64f Mon Sep 17 00:00:00 2001 From: Danathar Date: Mon, 9 Nov 2026 09:50:57 -0500 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=90=9B=20[v6]=20contribute:=20never?= =?UTF-8?q?=20dispatch=20rework=20for=20a=20merged=20or=20closed=20PR?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hivecommons/hive#17385 merged at 13:13:58Z; at ~14:25Z the hub still handed Danathar's relay "fix-pr hive#17385", and the relay ran it twice. Hub: - Queue a nudged PR under owner/name, not the nudge's Display (the bare configured name on a hive that lists repos without the org). Resolve keys on owner/name, so the bare entry was never removed when the PR left stage 072, and the relay could not match the task's repo "hive" to an owner/name. - Before dispatching a queued fix, check its PR is still open (state cache first, else one PR read tagged hive:contribute_fix_dispatch). Merged or closed PRs leave the queue with a log line; an unreadable state holds the dispatch for this tick. fixqueue.Next previews what a dispatch would take so only those PRs are checked. - Send a boundary fix task once: the ready handler ships the returned message, so sendFixTask no longer pushes it as well. - Progress for an in-flight fix task (a boundary fix runs as the relay's primary task) is not a resume claim and is no longer revoked. Relay: - A fix-pr task_id the relay already runs (primary, parallel, preempting or deferred) is ignored on a second task_assign, with a log line naming where it runs; a fix that becomes the primary task says so. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BQYnPGFJx81c8JXR3o1sg1 Signed-off-by: Danathar --- bin/contributor-relay.js | 25 +- bin/contributor-relay.test.js | 52 ++++ ...ixed-contribute-no-rework-for-merged-pr.md | 1 + src/docs/contributor-relay.md | 3 + .../dashboard/api_repo_item_takeover_test.go | 4 + src/pkg/dashboard/contribute_coverage_test.go | 2 +- src/pkg/dashboard/contribute_fixqueue.go | 158 ++++++++++- .../contribute_fixqueue_closed_pr_test.go | 259 ++++++++++++++++++ src/pkg/dashboard/contribute_ws.go | 11 + src/pkg/fixqueue/fixqueue.go | 54 +++- src/pkg/fixqueue/fixqueue_test.go | 39 +++ 11 files changed, 585 insertions(+), 23 deletions(-) create mode 100644 changelog.d/fixed-contribute-no-rework-for-merged-pr.md create mode 100644 src/pkg/dashboard/contribute_fixqueue_closed_pr_test.go diff --git a/bin/contributor-relay.js b/bin/contributor-relay.js index 80fc208d33..fe3484c7ea 100755 --- a/bin/contributor-relay.js +++ b/bin/contributor-relay.js @@ -4967,12 +4967,25 @@ function escapeRegExp(s) { return String(s).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); } +// fixPRRunningWhere says where this relay is already running fix-pr task +// taskId, or null. One fix-pr task runs in exactly one place: a second +// task_assign with the same task_id (a hub that delivered it twice) must not +// start it again as the primary task or in a parallel session. +function fixPRRunningWhere(taskId) { + if (!taskId) return null; + if (currentTask && currentTask.task_id === taskId) return 'as the primary task'; + if (parallelFixes.has(taskId)) return 'in a parallel fix session'; + if (preemptingFix && preemptingFix.task_id === taskId) return 'as the fix preempting the primary task'; + if (deferredFixes.some(d => d.task.task_id === taskId)) return 'deferred until the current task ends'; + return null; +} + // handleFixPRInterrupt decides spawn / preempt / defer for a fix-pr task that // arrived while currentTask is running. function handleFixPRInterrupt(msg, hub) { - if (parallelFixes.has(msg.task_id) || deferredFixes.some(d => d.task.task_id === msg.task_id) - || (preemptingFix && preemptingFix.task_id === msg.task_id)) { - console.log(`Ignoring duplicate fix-pr ${sanitizeHubText(msg.task_id)} — already handling it`); + const where = fixPRRunningWhere(msg.task_id); + if (where) { + console.log(`Ignoring duplicate fix-pr ${sanitizeHubText(msg.task_id)} — already running it ${where}`); return; } noteTaskAttemptStart(hub, msg); @@ -9963,6 +9976,10 @@ function handleMessage(data, hub) { handleFixPRInterrupt(msg, hub); break; } + if (msg.kind === FIX_PR_KIND && fixPRRunningWhere(msg.task_id)) { + console.log(`Ignoring duplicate fix-pr ${sanitizeHubText(msg.task_id)} — already running it ${fixPRRunningWhere(msg.task_id)}`); + break; + } if (currentTask) { console.log(`Rejecting task ${taskDisplay(msg)} from ${hub.url} — already working on ${taskDisplay(currentTask)}`); sendTo(hub, { type: 'task_failed', seq: nextSeq(), task_id: msg.task_id, reason: 'Already has active task' }); @@ -10055,6 +10072,7 @@ function handleMessage(data, hub) { writeRelayStatus(); activeHubIndex = hubs.indexOf(hub); console.log(`Task assigned: ${sanitizeHubText(msg.kind)} ${sanitizeHubText(taskDisplay(msg))} — ${sanitizeHubText(msg.title)} (from ${hub.url})`); + if (msg.kind === FIX_PR_KIND) console.log(`fix-pr ${fixPRLabel(msg)}: running as the primary task (no task was running)`); if (msg.github_token) { injectGhToken(msg.github_token); tokenExpiresAt = msg.token_expires_at ? new Date(msg.token_expires_at).getTime() : null; @@ -10818,6 +10836,7 @@ if (process.env.HIVE_RELAY_TEST_MODE === '1') { FIX_PR_DONE_MARKER, FIX_PR_FAILED_MARKER, fixPRState, + fixPRRunningWhere, scanPaneForFixMarkers, relaySupportsParallelFixes, blockingPromptKey, diff --git a/bin/contributor-relay.test.js b/bin/contributor-relay.test.js index 02a6903518..f4008f087f 100644 --- a/bin/contributor-relay.test.js +++ b/bin/contributor-relay.test.js @@ -15600,6 +15600,58 @@ test('#12436 headless primary: a preempt fix is deferred and runs before the nex } finally { console.log = log; teardown(relay); } }); +// hivecommons/hive#17385 (2026-11-09): the hub delivered one boundary fix-pr +// task twice. The relay started the first copy as its primary task and the +// second in a parallel fix session, so the same fix ran in two places. +test('#17385 a fix-pr task_id delivered twice runs once, as the primary task', () => { + const relay = loadRelay({ backend: 'claude', env: { HIVE_MAX_PARALLEL_FIXES: '1', HIVE_CONTRIBUTOR_QUOTA_GUARD: 'off' }, execFileResult: { defer: true } }); + const log = console.log; + const lines = []; + console.log = (...a) => lines.push(a.join(' ')); + try { + const hub = relay.getHubs()[0]; + const sent = []; + hub.ws = { readyState: 1, send: p => sent.push(JSON.parse(p)) }; + relay.setCurrentTask(null); + const boundary = FIX_PR_TASK('fix-danathar-1', { priority: 'queued', mode: 'boundary' }); + relay.handleMessage(JSON.stringify(boundary), hub); + assert.strictEqual(relay.getCurrentTask() && relay.getCurrentTask().task_id, 'fix-danathar-1', 'the first copy becomes the primary task'); + const spawnsBefore = relay.__execFileCalls.length; + relay.handleMessage(JSON.stringify(boundary), hub); + assert.deepStrictEqual(relay.fixPRState().parallel, [], 'the second copy must not start a parallel fix session'); + assert.strictEqual(relay.__execFileCalls.length, spawnsBefore, 'no parallel child spawned for the duplicate'); + assert.strictEqual(relay.getCurrentTask().task_id, 'fix-danathar-1'); + assert.strictEqual(sent.filter(m => m.type === 'task_accepted' && m.task_id === 'fix-danathar-1').length, 1, `accepted once: ${JSON.stringify(sent)}`); + assert.ok(!sent.some(m => m.type === 'task_failed' && m.task_id === 'fix-danathar-1'), 'the duplicate is ignored, not failed back'); + assert.ok(lines.some(l => /running as the primary task/.test(l)), `log names the path taken: ${lines.join('\n')}`); + assert.ok(lines.some(l => /Ignoring duplicate fix-pr fix-danathar-1 — already running it as the primary task/.test(l)), lines.join('\n')); + } finally { console.log = log; teardown(relay); } +}); + +test('#17385 a fix-pr task_id already in a parallel session is not also started as the primary task', () => { + const relay = loadRelay({ backend: 'claude', env: { HIVE_MAX_PARALLEL_FIXES: '1', HIVE_CONTRIBUTOR_QUOTA_GUARD: 'off' }, execFileResult: { defer: true } }); + const log = console.log; + const lines = []; + console.log = (...a) => lines.push(a.join(' ')); + try { + const hub = relay.getHubs()[0]; + const sent = []; + hub.ws = { readyState: 1, send: p => sent.push(JSON.parse(p)) }; + relay.setCurrentTask({ task_id: 't-primary', task_gen: 1, kind: 'issue', repo: 'foo/bar', number: 1, title: 'primary' }); + relay.handleMessage(JSON.stringify(FIX_PR_TASK('fix-r')), hub); + assert.deepStrictEqual(relay.fixPRState().parallel, ['fix-r']); + // The primary ends while the fix still runs; the same task arrives again. + relay.setCurrentTask(null); + relay.handleMessage(JSON.stringify(FIX_PR_TASK('fix-r')), hub); + assert.strictEqual(relay.getCurrentTask(), null, 'the duplicate must not become the primary task'); + assert.deepStrictEqual(relay.fixPRState().parallel, ['fix-r']); + assert.strictEqual(sent.filter(m => m.type === 'task_accepted' && m.task_id === 'fix-r').length, 1, JSON.stringify(sent)); + assert.ok(lines.some(l => /Ignoring duplicate fix-pr fix-r — already running it in a parallel fix session/.test(l)), lines.join('\n')); + assert.strictEqual(relay.fixPRRunningWhere('fix-r'), 'in a parallel fix session'); + assert.strictEqual(relay.fixPRRunningWhere('other'), null); + } finally { console.log = log; teardown(relay); } +}); + (async () => { for (const [name, fn] of only ? tests.filter(([n]) => n.includes(only)) : tests) { try { diff --git a/changelog.d/fixed-contribute-no-rework-for-merged-pr.md b/changelog.d/fixed-contribute-no-rework-for-merged-pr.md new file mode 100644 index 0000000000..ae3559f0db --- /dev/null +++ b/changelog.d/fixed-contribute-no-rework-for-merged-pr.md @@ -0,0 +1 @@ +- A contributor's relay is no longer sent to rework a PR that has already merged or closed, and one `fix-pr` task now runs once. On hivecommons/hive#17385 a stage-072 nudge queued a fix at 12:32Z, the PR merged at 13:13:58Z, and at about 14:25Z the hub still handed the relay `fix-pr hive#17385`, which the relay then ran twice: about 4.5 minutes lost, nothing pushed. Each link in that chain is fixed. The fix queue took the nudge's display name (`hive` on a hive whose repos are listed without the org), so the "left stage 072" resolve, which looks up `owner/name`, never found the entry, and the relay got a repo it could not match to its authorized list; queued fixes are now keyed and sent as `owner/name`. Right before a queued fix is handed out, the hub now checks the PR is still open (from the state cache when it holds the repo, else one PR read tagged `hive:contribute_fix_dispatch`); a merged or closed PR leaves the queue with a `fix queue: dropped a PR that is no longer open` log line, and if the state cannot be read nothing is dispatched that tick. A fix handed out at a task boundary was pushed to the relay twice (once by the dispatcher, once by the `ready` handler); it is now sent once, and the relay ignores a second `task_assign` for a `fix-pr` task it is already running (as its primary task, in a parallel session, preempting, or deferred) and logs where it runs. Progress reports from a boundary fix running as the relay's primary task are no longer answered with `task_revoke: no active lease for this task`, which killed the fix mid-run. diff --git a/src/docs/contributor-relay.md b/src/docs/contributor-relay.md index de062c3cbd..994f8f78cd 100644 --- a/src/docs/contributor-relay.md +++ b/src/docs/contributor-relay.md @@ -1755,9 +1755,12 @@ Set `HIVE_MAX_PARALLEL_FIXES=0` in `~/.config/hive/contributor.env` and restart - `mode=preempt`, interactive pane: `task_accepted`, then the pane receives an `INTERRUPT (hive fix-pr, preempt mode)` prompt telling the agent to park the current issue (commit/stash WIP; the claim stays alive), fix the PR, print exactly `HIVE_FIX_PR_DONE ` (or `HIVE_FIX_PR_FAILED `), then resume. The progress tick reads the marker off the pane and reports `task_complete`/`task_failed` for the fix's `task_id`. Only one fix preempts at a time; more are queued locally. - `mode=preempt`, headless primary: there is no pane to interrupt, so the fix is deferred (`task_accepted` immediately) and runs before the relay's next `ready` — "fix what you broke before taking new work" holds on the client too. - With no task running, a `fix-pr` is just the next task. +- One `fix-pr` task runs in one place. A second `task_assign` with a `task_id` the relay is already running — as its primary task, in a parallel session, as the preempting fix, or deferred — is ignored, and the log says where it is already running. The hub records the mode it used in the nudge comment's marker (`mode=spawn|preempt|queued|comment`) and exposes queue depth / in-flight as `fix_queue` on the contributor profile API. +**Only open PRs are dispatched.** Right before a queued fix is handed out, the hub checks that its PR is still open: from the state cache when it holds the repo (free while the cached copy is inside its freshness bound, otherwise one re-read), else with one PR read tagged `hive:contribute_fix_dispatch`. A PR that has merged or closed leaves the queue with a `fix queue: dropped a PR that is no longer open` log line, and no relay is sent to rework it. When the state cannot be read, nothing is dispatched that tick and the next one asks again. Queued fixes carry the repo as `owner/name` even on a hive that lists its repos without the org, so the fix task's `repo` and the PR leaving stage 072 both match the queued entry. + **When the relay fails the task.** `task_failed` for a fix's `task_id` does not settle it as done: its PRs go back in the hub's queue (same place by age) and are not interrupted back in for 5 minutes, though the relay's next `ready` serves them at once. After three failed dispatches a PR leaves the fix queue and the stage engine's re-nudge / adoption path takes over. A dispatch the relay fails within one second of the nudge is not reported as delivered: the nudge comment reads `mode=queued`, says the relay could not take the task (quoting the relay's `message`/`reason`), and the stage log carries `relay_failed=true` (hivecommons/hive#14668). #### What the operator sees (Operations → Connected contributor agents) diff --git a/src/pkg/dashboard/api_repo_item_takeover_test.go b/src/pkg/dashboard/api_repo_item_takeover_test.go index 614a11ca78..5cb873a61e 100644 --- a/src/pkg/dashboard/api_repo_item_takeover_test.go +++ b/src/pkg/dashboard/api_repo_item_takeover_test.go @@ -1,6 +1,7 @@ package dashboard import ( + "context" "encoding/json" "net/http" "slices" @@ -236,6 +237,9 @@ func TestRepoItemReroutePickupLapseFallsBackToAdoptNow(t *testing.T) { srv, store := takeoverServer(t, mock) hub := srv.contributeHub hub.fixDispatched = func(_ *ContributorConnection, _ WSMessage) {} + // org/other#1 is a fixture the GitHub mock does not serve: report it + // open so the dispatch check keeps it. + hub.fixPRStateHook = func(context.Context, string, int) (fixPRState, string, error) { return fixPRStateOpen, "", nil } // bee2 is online but already holds a task: the reroute queues behind it. c := fixConn(hub, "bee2", fixCaps(capFixPRInterrupt), time.Now()) c.mu.Lock() diff --git a/src/pkg/dashboard/contribute_coverage_test.go b/src/pkg/dashboard/contribute_coverage_test.go index cc5b53ea64..55baedf499 100644 --- a/src/pkg/dashboard/contribute_coverage_test.go +++ b/src/pkg/dashboard/contribute_coverage_test.go @@ -603,7 +603,7 @@ func TestRoleCoverage_FixTaskDispatchEndsNoWorkRun(t *testing.T) { relay.mu.Unlock() hub.sendFixTask(relay, "alice", &fixqueue.Dispatch{ID: "fix-1", Mode: fixqueue.ModeBoundary, Items: []fixqueue.Item{ {Repo: "myorg/repo1", Number: 7, URL: "https://github.com/myorg/repo1/pull/7", Title: "red PR"}, - }}) + }}, true) if got := coverageAt(hub, relay, later); !reflect.DeepEqual(got, scannerAlice) { t.Fatalf("relay working a fix task: RoleCoverage = %+v, want %+v", got, scannerAlice) } diff --git a/src/pkg/dashboard/contribute_fixqueue.go b/src/pkg/dashboard/contribute_fixqueue.go index 673aca1627..9ceddadf3a 100644 --- a/src/pkg/dashboard/contribute_fixqueue.go +++ b/src/pkg/dashboard/contribute_fixqueue.go @@ -2,11 +2,15 @@ package dashboard import ( "context" + "errors" "fmt" + "net/http" "strings" "time" "github.com/hivecommons/hive/pkg/fixqueue" + "github.com/hivecommons/hive/pkg/forgestate" + "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/github/automerge" ) @@ -55,6 +59,25 @@ const ( // leaves the fix queue and the stage engine's re-nudge / adoption path // takes over. fixTaskMaxAttempts = 3 + + // fixPRStateRESTCaller tags the one PR read a fix dispatch may make + // when the state cache cannot answer. + fixPRStateRESTCaller = "hive:contribute_fix_dispatch" + // fixPRStateTimeout bounds that read. + fixPRStateTimeout = 10 * time.Second +) + +// fixPRState is what the hub knows about a queued PR right before it is +// handed to a relay. +type fixPRState int + +const ( + // fixPRStateUnchecked: no GitHub client or no owner/name to read (a + // hub built without one, as in unit tests); dispatch as before. + fixPRStateUnchecked fixPRState = iota + fixPRStateOpen + // fixPRStateGone: merged, closed or deleted; the fix is dropped. + fixPRStateGone ) // WSFixPR is one red PR inside a fix-pr task_assign. @@ -285,14 +308,11 @@ func (h *ContributeWSHub) Nudge(_ context.Context, n automerge.ContributorNudge) return out } item := fixqueue.Item{ - Repo: n.Display, Number: n.Number, HeadSHA: n.HeadSHA, Title: n.Title, URL: n.URL, + Repo: nudgeFixRepo(n), Number: n.Number, HeadSHA: n.HeadSHA, Title: n.Title, URL: n.URL, OpenedAt: n.OpenedAt, HumanChangesRequested: n.HumanChangesRequested, Cause: fixqueue.Cause{Check: firstOf(n.FailingChecks), Signature: n.Reason}, Summary: strings.Join(append(append([]string(nil), n.FailingChecks...), n.ReviewAsks...), "; "), } - if item.Repo == "" { - item.Repo = n.Owner + "/" + n.Repo - } sched := h.fixQueue() sched.Enqueue(author, item) h.rememberFixOwner(author, item.Key()) @@ -309,11 +329,31 @@ func (h *ContributeWSHub) Nudge(_ context.Context, n automerge.ContributorNudge) } } } + if _, held := h.FixWorkFor(author, item.Repo, item.Number); !held { + // The dispatch check found the PR merged or closed and dropped it. + out.Mode = string(fixqueue.ModeComment) + } st := sched.Status(author, fixqueue.Capacity{Max: 1}) out.QueueDepth = st.Queued return out } +// nudgeFixRepo is the owner/name a nudged PR is queued under. Resolve keys +// on owner/name, and the relay records the fix task's repo as the one it +// is authorized for, so the queue never takes the bare configured name the +// nudge's Display carries on a hive whose repos are listed without the org +// (that spelling left a merged PR in the queue: Resolve could not find it). +func nudgeFixRepo(n automerge.ContributorNudge) string { + owner, repo := strings.TrimSpace(n.Owner), strings.TrimSpace(n.Repo) + if owner != "" && repo != "" { + return owner + "/" + repo + } + if display := strings.TrimSpace(n.Display); display != "" { + return display + } + return owner + "/" + repo +} + // Resolve implements automerge.ContributorNudgeSink: the PR left stage 072. func (h *ContributeWSHub) Resolve(_ context.Context, owner, repo string, number int) { full := owner + "/" + repo @@ -395,6 +435,18 @@ func (h *ContributeWSHub) dispatchFixes(user string, conns []*ContributorConnect cap := h.fixCapacity(target) var first *WSMessage for { + next := sched.Next(user, cap, boundary) + if len(next) == 0 { + break + } + dropped, hold := h.dropClosedFixes(user, next) + if hold { + break + } + if dropped { + // The queue changed under the preview: look again. + continue + } var d *fixqueue.Dispatch if boundary { d = sched.DispatchAtBoundary(user, cap) @@ -404,7 +456,11 @@ func (h *ContributeWSHub) dispatchFixes(user string, conns []*ContributorConnect if d == nil { break } - msg := h.sendFixTask(target, user, d) + // At a boundary the `ready` handler ships the returned message + // itself; pushing it here as well delivered the same task twice, + // and the relay ran it both as its primary task and in a parallel + // fix session. + msg := h.sendFixTask(target, user, d, !boundary) if first == nil { first = &msg } @@ -416,8 +472,87 @@ func (h *ContributeWSHub) dispatchFixes(user string, conns []*ContributorConnect return first } -// sendFixTask builds and pushes the fix-pr task_assign for d. -func (h *ContributeWSHub) sendFixTask(c *ContributorConnection, user string, d *fixqueue.Dispatch) WSMessage { +// dropClosedFixes checks the PRs about to be dispatched to user. A PR that +// merged or closed since it was queued leaves the queue (logged), so no +// relay is sent to rework it. hold is true when a PR's state could not be +// read: nothing is dispatched this tick, and the next one asks again. +func (h *ContributeWSHub) dropClosedFixes(user string, items []fixqueue.Item) (dropped, hold bool) { + sched := h.fixQueue() + for _, it := range items { + state, reason, err := h.fixPRStateOf(context.Background(), it.Repo, it.Number) + if err != nil { + h.logger.Warn("[contribute-ws] fix queue: could not read the PR's state; holding the dispatch", "username", user, "pr", it.Key(), "error", err) + return dropped, true + } + if state != fixPRStateGone { + continue + } + sched.Remove(user, it.Repo, it.Number) + h.forgetFixOwner(user, it.Key()) + h.logger.Info("[contribute-ws] fix queue: dropped a PR that is no longer open", "username", user, "pr", it.Key(), "state", reason, "head", shortSHA(it.HeadSHA)) + dropped = true + } + return dropped, false +} + +// fixPRStateOf answers whether repo#number is still open, from the state +// cache when it holds the repo (a cached copy inside its freshness bound +// costs nothing; an older one is re-read once), else with one PR read. +// reason says "merged" or "closed" for a gone PR. A read that fails is an +// error, never "open". +func (h *ContributeWSHub) fixPRStateOf(ctx context.Context, repo string, number int) (fixPRState, string, error) { + if h.fixPRStateHook != nil { + return h.fixPRStateHook(ctx, repo, number) + } + if h.server == nil || h.server.deps == nil || h.server.deps.GHClient == nil { + return fixPRStateUnchecked, "", nil + } + owner, name, ok := strings.Cut(strings.TrimSpace(repo), "/") + if !ok || owner == "" || name == "" || number <= 0 { + return fixPRStateUnchecked, "", nil + } + client := h.server.deps.GHClient + ctx, cancel := context.WithTimeout(github.WithRESTCaller(ctx, fixPRStateRESTCaller), fixPRStateTimeout) + defer cancel() + if view := client.ForgeStateView(); view != nil { + if _, held := view.Snapshot(repo); held { + _, open, err := view.ReadThrough(ctx, repo, forgestate.KindPR, number, 0) + switch { + case err == nil && open: + return fixPRStateOpen, "", nil + case err == nil: + // The cache holds open PRs only: absent after a fresh read + // means merged, closed or deleted. + return fixPRStateGone, "merged or closed", nil + case !errors.Is(err, forgestate.ErrStale) && !errors.Is(err, forgestate.ErrNotReady) && !errors.Is(err, forgestate.ErrUnknownRepo): + return fixPRStateUnchecked, "", err + } + // A stale or not-yet-ready cache made no call: read the PR below. + } + } + gh := client.GoGitHub() + if gh == nil { + return fixPRStateUnchecked, "", nil + } + pr, resp, err := gh.PullRequests.Get(ctx, owner, name, number) + if err != nil { + if resp != nil && resp.Response != nil && (resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone) { + return fixPRStateGone, "not found", nil + } + return fixPRStateUnchecked, "", err + } + switch { + case pr.GetMerged() || !pr.GetMergedAt().Time.IsZero(): + return fixPRStateGone, "merged", nil + case !strings.EqualFold(pr.GetState(), "open"): + return fixPRStateGone, "closed", nil + } + return fixPRStateOpen, "", nil +} + +// sendFixTask builds the fix-pr task_assign for d and, when push is true, +// sends it on c (a boundary dispatch is sent by the `ready` handler). +func (h *ContributeWSHub) sendFixTask(c *ContributorConnection, user string, d *fixqueue.Dispatch, push bool) WSMessage { prs := make([]WSFixPR, 0, len(d.Items)) for _, it := range d.Items { pr := WSFixPR{Repo: it.Repo, Number: it.Number, URL: it.URL, HeadSHA: it.HeadSHA, Title: it.Title, Reason: it.Cause.Signature, @@ -461,7 +596,7 @@ func (h *ContributeWSHub) sendFixTask(c *ContributorConnection, user string, d * if h.fixDispatched != nil { h.fixDispatched(c, msg) } - if c.ws != nil { + if push && c.ws != nil { if err := c.send(msg); err != nil { h.logger.Warn("[contribute-ws] fix-pr task send failed", "username", user, "task", d.ID, "error", err) } @@ -484,7 +619,12 @@ func (h *ContributeWSHub) selectFixTask(c *ContributorConnection) *WSMessage { if msg := h.dispatchFixes(user, []*ContributorConnection{c}, true); msg != nil { return msg } - // Fixes are queued but every slot is busy: no new work until one finishes. + if !sched.Pending(user) { + // Every queued fix was for a PR that has since merged or closed. + return nil + } + // Fixes are queued but every slot is busy (or a PR's state could not be + // read this tick): no new work until one finishes. return h.taskUnavailable(taskUnavailableFixInFlight) } diff --git a/src/pkg/dashboard/contribute_fixqueue_closed_pr_test.go b/src/pkg/dashboard/contribute_fixqueue_closed_pr_test.go new file mode 100644 index 0000000000..014b20c4e7 --- /dev/null +++ b/src/pkg/dashboard/contribute_fixqueue_closed_pr_test.go @@ -0,0 +1,259 @@ +package dashboard + +import ( + "context" + "encoding/json" + "errors" + "strconv" + "testing" + "time" + + "github.com/gorilla/websocket" + + "github.com/hivecommons/hive/pkg/github/automerge" +) + +// A merged PR must never come back to its author as rework. Observed +// 2026-11-09 on hivecommons/hive#17385: the 12:32Z stage-072 nudge queued a +// fix, the PR left 072 and merged at 13:13:58Z, and at ~14:25Z the hub +// still handed the relay "fix-pr hive#17385", which then ran twice. + +// fixPRStates answers the dispatch check from a map keyed by fixqueue key; +// a PR not in the map is open. +func fixPRStates(hub *ContributeWSHub, states map[string]fixPRState) *[]string { + var asked []string + hub.fixPRStateHook = func(_ context.Context, repo string, number int) (fixPRState, string, error) { + key := repo + "#" + strconv.Itoa(number) + asked = append(asked, key) + if st, ok := states[key]; ok { + if st == fixPRStateGone { + return st, "merged", nil + } + return st, "", nil + } + return fixPRStateOpen, "", nil + } + return &asked +} + +// bareNudge is a nudge from a hive whose repos are configured without the +// org: the stage engine's Display is the bare name. +func bareNudge(n int) automerge.ContributorNudge { + return automerge.ContributorNudge{ + Owner: "hivecommons", Repo: "hive", Display: "hive", Number: n, HeadSHA: "6be425a93fe8", + Author: "alice", Title: "feat", URL: "https://github.com/hivecommons/hive/pull/" + strconv.Itoa(n), + OpenedAt: time.Now().Add(-time.Duration(n) * time.Minute), Reason: "conflicting", + ReviewAsks: []string{"the branch conflicts with the base and needs a rebase"}, + } +} + +// The fix task names the repo as owner/name, and the PR leaving stage 072 +// (Resolve, which keys on owner/name) finds the queued entry. +func TestFixQueueBareConfiguredRepoQueuesOwnerName(t *testing.T) { + hub, _ := covK2Hub(t) + got := captureFixDispatch(hub) + fixConn(hub, "alice", fixCaps(capFixPRInterrupt), time.Now()) + + hub.Nudge(context.Background(), bareNudge(1)) + if len(*got) != 1 { + t.Fatalf("dispatches = %d, want 1", len(*got)) + } + m := (*got)[0] + if m.Repo != "hivecommons/hive" || len(m.FixPRs) != 1 || m.FixPRs[0].Repo != "hivecommons/hive" { + t.Fatalf("fix task repo = %q / %+v, want owner/name hivecommons/hive", m.Repo, m.FixPRs) + } + + // A second PR waits behind the preemption; it leaves 072 before its turn. + hub.Nudge(context.Background(), bareNudge(17385)) + if q := hub.FixQueueItems("alice"); len(q) != 1 || q[0].Repo != "hivecommons/hive" { + t.Fatalf("queued = %+v, want hivecommons/hive#17385", q) + } + hub.Resolve(context.Background(), "hivecommons", "hive", 17385) + if q := hub.FixQueueItems("alice"); len(q) != 0 { + t.Fatalf("Resolve left %+v in the queue", q) + } +} + +// A PR that merged while it waited is dropped at the task boundary: the relay +// gets no fix task, falls through to issue selection, and the queue is empty. +func TestFixQueueBoundaryDropsPRThatMergedWhileQueued(t *testing.T) { + hub, _ := covK2Hub(t) + if err := saveContributorProfile(&ContributorProfile{GitHubUsername: "alice", TrustTier: "contributor"}); err != nil { + t.Fatal(err) + } + got := captureFixDispatch(hub) + // Queued while the relay was away. + if out := hub.Nudge(context.Background(), bareNudge(17385)); out.Mode != "queued" || out.QueueDepth != 1 { + t.Fatalf("offline nudge = %+v, want queued", out) + } + // It merges; the relay comes back and asks for work. + asked := fixPRStates(hub, map[string]fixPRState{"hivecommons/hive#17385": fixPRStateGone}) + c := fixConn(hub, "alice", fixCaps(capFixPRInterrupt), time.Now()) + if msg := hub.selectFixTask(c); msg != nil { + t.Fatalf("selectFixTask = %+v, want nil (no fix for a merged PR, go select an issue)", msg) + } + if len(*got) != 0 { + t.Fatalf("dispatched %+v for a merged PR", *got) + } + if len(*asked) != 1 { + t.Fatalf("state reads = %v, want one", *asked) + } + if st := hub.FixQueueStatus("alice"); st.Queued != 0 || st.InFlight != 0 { + t.Fatalf("status = %+v, want the merged PR out of the queue", st) + } + hub.fixQueueMu.Lock() + owned := len(hub.fixOwners["alice"]) + hub.fixQueueMu.Unlock() + if owned != 0 { + t.Fatalf("owner index still holds %d key(s)", owned) + } +} + +// Only the merged one is dropped: an open PR behind it is still dispatched. +func TestFixQueueDispatchSkipsClosedPRAndServesTheNextOpenOne(t *testing.T) { + hub, _ := covK2Hub(t) + if err := saveContributorProfile(&ContributorProfile{GitHubUsername: "alice", TrustTier: "contributor"}); err != nil { + t.Fatal(err) + } + got := captureFixDispatch(hub) + old, open := bareNudge(30), bareNudge(20) + old.Reason, open.Reason = "conflicting-a", "conflicting-b" + hub.Nudge(context.Background(), old) + hub.Nudge(context.Background(), open) + fixPRStates(hub, map[string]fixPRState{"hivecommons/hive#30": fixPRStateGone}) + c := fixConn(hub, "alice", fixCaps(capFixPRInterrupt), time.Now()) + msg := hub.selectFixTask(c) + if msg == nil || msg.Kind != taskKindFixPR || msg.Number != 20 { + t.Fatalf("selectFixTask = %+v, want the open PR #20", msg) + } + if len(*got) != 1 || (*got)[0].Number != 20 { + t.Fatalf("dispatches = %+v", *got) + } +} + +// A nudge for a PR that is already merged by the time it reaches the hub +// queues nothing and is reported as a comment-only nudge. +func TestFixQueueNudgeForMergedPRDispatchesNothing(t *testing.T) { + hub, _ := covK2Hub(t) + got := captureFixDispatch(hub) + fixConn(hub, "alice", fixCaps(capFixPRInterrupt), time.Now()) + fixPRStates(hub, map[string]fixPRState{"hivecommons/hive#17385": fixPRStateGone}) + out := hub.Nudge(context.Background(), bareNudge(17385)) + if out.Mode != "comment" || out.QueueDepth != 0 { + t.Fatalf("nudge = %+v, want comment with nothing queued", out) + } + if len(*got) != 0 { + t.Fatalf("dispatched %+v for a merged PR", *got) + } +} + +// When the PR's state cannot be read nothing is dispatched (and no new work +// is offered in its place); the next boundary asks again. +func TestFixQueueHoldsDispatchWhenPRStateUnreadable(t *testing.T) { + hub, _ := covK2Hub(t) + if err := saveContributorProfile(&ContributorProfile{GitHubUsername: "alice", TrustTier: "contributor"}); err != nil { + t.Fatal(err) + } + got := captureFixDispatch(hub) + hub.Nudge(context.Background(), bareNudge(5)) + hub.fixPRStateHook = func(context.Context, string, int) (fixPRState, string, error) { + return fixPRStateUnchecked, "", errors.New("GET pulls/5: 502") + } + c := fixConn(hub, "alice", fixCaps(capFixPRInterrupt), time.Now()) + msg := hub.selectFixTask(c) + if msg == nil || msg.Type != "task_unavailable" || msg.Reason != taskUnavailableFixInFlight { + t.Fatalf("selectFixTask = %+v, want task_unavailable/%s while the state is unreadable", msg, taskUnavailableFixInFlight) + } + if len(*got) != 0 || hub.FixQueueStatus("alice").Queued != 1 { + t.Fatalf("dispatches=%d status=%+v, want nothing dispatched and the fix still queued", len(*got), hub.FixQueueStatus("alice")) + } + fixPRStates(hub, nil) + if msg := hub.selectFixTask(c); msg == nil || msg.Kind != taskKindFixPR { + t.Fatalf("readable again: selectFixTask = %+v, want the fix", msg) + } +} + +// readFrames collects every frame the relay receives within d. +func readFrames(t *testing.T, client *websocket.Conn, d time.Duration) []map[string]any { + t.Helper() + var out []map[string]any + deadline := time.Now().Add(d) + for { + _ = client.SetReadDeadline(deadline) + _, raw, err := client.ReadMessage() + if err != nil { + return out + } + var m map[string]any + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatalf("unmarshal: %v", err) + } + out = append(out, m) + } +} + +// At a task boundary the fix task reaches the relay once. It used to be +// pushed by sendFixTask and then again by the ready handler, so the relay +// started it as its primary task and, on the second copy, in a parallel fix +// session as well. +func TestFixQueueBoundaryTaskIsSentOnce(t *testing.T) { + hub, s := covK2Hub(t) + s.statusMu.Lock() + s.status = &StatusPayload{} + s.statusMu.Unlock() + if err := saveContributorProfile(&ContributorProfile{GitHubUsername: "alice", TrustTier: "contributor"}); err != nil { + t.Fatal(err) + } + hub.Nudge(context.Background(), bareNudge(17385)) + + server, client := wsPipe(t) + relay := coverageConn("alice", "scanner", time.Now()) + relay.capabilities = fixCaps(capFixPRInterrupt, capParallelSessions) + relay.ws = server + hub.mu.Lock() + hub.connections["alice"] = relay + hub.mu.Unlock() + sess := &wsSession{h: hub, conn: server, contributor: relay} + if stop := sess.handleReady(WSMessage{Type: "ready"}); stop { + t.Fatal("ready must not stop the session") + } + assigns := 0 + for _, f := range readFrames(t, client, 500*time.Millisecond) { + if f["type"] == "task_assign" { + assigns++ + if f["kind"] != taskKindFixPR || f["repo"] != "hivecommons/hive" { + t.Fatalf("task_assign = %v, want fix-pr for hivecommons/hive", f) + } + } + } + if assigns != 1 { + t.Fatalf("relay received %d task_assign frames for one boundary fix, want 1", assigns) + } +} + +// A boundary fix runs as the relay's primary task and reports progress like +// one. It holds no lease, so the progress must not be read as a resume +// claim and answered with task_revoke (which killed the fix mid-run). +func TestFixTaskProgressIsNotRevoked(t *testing.T) { + hub, _ := covK2Hub(t) + server, client := wsPipe(t) + relay := fixConn(hub, "alice", fixCaps(capFixPRInterrupt), time.Now()) + relay.ws = server + relay.fixTasks = map[string]*WSTaskAssign{"fix-alice-1": {TaskID: "fix-alice-1", Kind: taskKindFixPR, Repo: "hivecommons/hive", Number: 17385}} + sess := &wsSession{h: hub, conn: server, contributor: relay} + sess.handleTaskProgress(WSMessage{Type: "task_progress", TaskID: "fix-alice-1", Repo: "hivecommons/hive", Number: 17385}) + // A task the hub never issued is still a resume claim and is still + // revoked; its frame doubles as the marker that the first progress was + // handled (frames arrive in order). + sess.handleTaskProgress(WSMessage{Type: "task_progress", TaskID: "not-mine", Repo: "hivecommons/hive", Number: 1}) + var revoked []string + for _, f := range readFrames(t, client, time.Second) { + if f["type"] == "task_revoke" { + id, _ := f["task_id"].(string) + revoked = append(revoked, id) + } + } + if len(revoked) != 1 || revoked[0] != "not-mine" { + t.Fatalf("task_revoke frames for %v, want only [not-mine] (never the in-flight fix task)", revoked) + } +} diff --git a/src/pkg/dashboard/contribute_ws.go b/src/pkg/dashboard/contribute_ws.go index 658f544127..73abadc6cc 100644 --- a/src/pkg/dashboard/contribute_ws.go +++ b/src/pkg/dashboard/contribute_ws.go @@ -993,6 +993,9 @@ type ContributeWSHub struct { fixWaiters int fixFailures map[string]string fixAckGrace time.Duration + // fixPRStateHook replaces fixPRStateOf's GitHub read of a queued PR's + // state (open, merged, closed); a test hook. + fixPRStateHook func(ctx context.Context, repo string, number int) (fixPRState, string, error) // stageStore is the stage-label memory the fleet view reads "PRs returned // to this contributor" from (hivecommons/hive#12502). Nil means the // process-wide stagelabels.Shared(); tests inject an in-memory store. @@ -4088,6 +4091,14 @@ func (s *wsSession) handleTaskProgress(msg WSMessage) { s.contributor.mu.Unlock() return } + // A fix-pr task handed out at a task boundary runs as the + // relay's primary task and reports progress like one, but it + // is tracked in fixTasks and holds no lease. It is not a resume + // claim: revoking it killed the fix mid-run. + if _, fix := s.contributor.fixTasks[msg.TaskID]; fix { + s.contributor.mu.Unlock() + return + } identity := identityOf(s.contributor) canonRepo := h.canonicalRepoKey(msg.Repo) s.contributor.mu.Unlock() diff --git a/src/pkg/fixqueue/fixqueue.go b/src/pkg/fixqueue/fixqueue.go index d0078def6d..f8a803f12e 100644 --- a/src/pkg/fixqueue/fixqueue.go +++ b/src/pkg/fixqueue/fixqueue.go @@ -400,17 +400,53 @@ func (s *Scheduler) Dispatch(user string, cap Capacity) *Dispatch { s.mu.Lock() defer s.mu.Unlock() c, ok := s.by[norm(user)] - if !ok || len(c.queue) == 0 || len(c.inflight) >= cap.Slots() { + if !ok || !s.dispatchableLocked(c, cap, false) { return nil } - if s.opts.Now().Before(c.queue[0].NotBefore) { - return nil + return s.pop(user, c, cap.Mode()) +} + +// dispatchableLocked holds the guards of Dispatch (boundary=false) and +// DispatchAtBoundary (boundary=true). +func (s *Scheduler) dispatchableLocked(c *contributor, cap Capacity, boundary bool) bool { + if len(c.queue) == 0 { + return false + } + if boundary { + return len(c.inflight) < cap.BoundarySlots() } - mode := cap.Mode() - if mode == ModePreempt && (c.preempted || s.opts.Now().Before(c.focusUntil)) { + if len(c.inflight) >= cap.Slots() || s.opts.Now().Before(c.queue[0].NotBefore) { + return false + } + return cap.Mode() != ModePreempt || (!c.preempted && !s.opts.Now().Before(c.focusUntil)) +} + +// Next returns the items the next Dispatch (boundary=false) or +// DispatchAtBoundary (boundary=true) would hand out, without taking them: +// the caller can check them before they leave the queue. nil when that call +// would return nil. +func (s *Scheduler) Next(user string, cap Capacity, boundary bool) []Item { + s.mu.Lock() + defer s.mu.Unlock() + c, ok := s.by[norm(user)] + if !ok || !s.dispatchableLocked(c, cap, boundary) { return nil } - return s.pop(user, c, mode) + head := c.queue[0] + out := []Item{head} + for _, q := range c.queue[1:] { + if coalesces(head, q) { + out = append(out, q) + } + } + return out +} + +// coalesces reports whether queued item q joins head's fix task. A rerouted +// PR (another author's) never shares a task with the owner's own PRs: its +// prompt differs. +func coalesces(head, q Item) bool { + return head.Cause.coalescable() && q.Cause == head.Cause && !head.Rerouted() && !q.Rerouted() } // DispatchAtBoundary hands user the next fix when their relay asked for its @@ -421,7 +457,7 @@ func (s *Scheduler) DispatchAtBoundary(user string, cap Capacity) *Dispatch { s.mu.Lock() defer s.mu.Unlock() c, ok := s.by[norm(user)] - if !ok || len(c.queue) == 0 || len(c.inflight) >= cap.BoundarySlots() { + if !ok || !s.dispatchableLocked(c, cap, true) { return nil } return s.pop(user, c, ModeBoundary) @@ -433,9 +469,7 @@ func (s *Scheduler) pop(user string, c *contributor, mode Mode) *Dispatch { d := &dispatch{mode: mode, items: map[string]Item{head.Key(): head}, at: s.opts.Now()} rest := c.queue[:0:0] for _, q := range c.queue[1:] { - // A rerouted PR (another author's) never shares a task with the - // owner's own PRs: its prompt differs. - if head.Cause.coalescable() && q.Cause == head.Cause && !head.Rerouted() && !q.Rerouted() { + if coalesces(head, q) { d.items[q.Key()] = q continue } diff --git a/src/pkg/fixqueue/fixqueue_test.go b/src/pkg/fixqueue/fixqueue_test.go index 94c3c5f7b6..ccdfff2131 100644 --- a/src/pkg/fixqueue/fixqueue_test.go +++ b/src/pkg/fixqueue/fixqueue_test.go @@ -418,3 +418,42 @@ func TestDispatchAt(t *testing.T) { t.Fatalf("inflight=%+v", got) } } + +// Next previews exactly what Dispatch / DispatchAtBoundary would hand out, +// coalesced siblings included, and takes nothing; it is nil whenever they +// would return nil. +func TestNextPreviewsDispatchWithoutTakingIt(t *testing.T) { + c := newClock() + s := newSched(c) + cap := Capacity{Max: 1} + if got := s.Next("alice", cap, false); got != nil { + t.Fatalf("empty queue: Next = %v, want nil", keys(got)) + } + shared := Cause{Check: "lint", Signature: "gofmt"} + a, b, other := red(1, 3*time.Hour, c), red(2, 2*time.Hour, c), red(3, time.Hour, c) + a.Cause, b.Cause = shared, shared + for _, it := range []Item{a, b, other} { + s.Enqueue("alice", it) + } + next := s.Next("alice", cap, false) + if !eq(keys(next), []int{1, 2}) { + t.Fatalf("Next = %v, want the head and its coalesced sibling [1 2]", keys(next)) + } + if st := s.Status("alice", cap); st.Queued != 3 || st.InFlight != 0 { + t.Fatalf("Next must not take anything: %+v", st) + } + d := s.Dispatch("alice", cap) + if d == nil || !eq(keys(d.Items), keys(next)) { + t.Fatalf("Dispatch = %+v, want what Next previewed %v", d, keys(next)) + } + // A preemption is in flight and capacity 1 is used: both previews are nil. + if got := s.Next("alice", cap, false); got != nil { + t.Fatalf("preemption in flight: Next = %v, want nil", keys(got)) + } + if got := s.Next("alice", cap, true); got != nil { + t.Fatalf("boundary with the only slot busy: Next = %v, want nil", keys(got)) + } + if got := s.Next("alice", Capacity{Max: 2}, true); !eq(keys(got), []int{3}) { + t.Fatalf("boundary with a free slot: Next = %v, want [3]", keys(got)) + } +} From 4d0afcc5086be0ebc553975bc869b19fa2550fc6 Mon Sep 17 00:00:00 2001 From: Danathar Date: Mon, 9 Nov 2026 10:12:31 -0500 Subject: [PATCH 2/3] test(contribute): a retried boundary fix's progress is not revoked (utah-packages#972) projectbluefin/utah-packages#972, 2026-11-09: the relay refused fix-karibhou-1, the hub requeued it and served it at the next ready as fix-karibhou-2 on the same connection, and the relay's progress for the retry (repo empty, client_gen 1870) was answered with "no matching server-issued lease" and task_revoke at 14:50:21Z and 14:54:36Z. The handleTaskProgress change in this PR already covers it; this test drives the real fail -> requeue -> boundary dispatch path and sends progress without a repo, as the relay did, so the retried task id must be tracked on the connection and never revoked. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BQYnPGFJx81c8JXR3o1sg1 Signed-off-by: Danathar --- .../contribute_fixqueue_closed_pr_test.go | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/pkg/dashboard/contribute_fixqueue_closed_pr_test.go b/src/pkg/dashboard/contribute_fixqueue_closed_pr_test.go index 014b20c4e7..f98518bb20 100644 --- a/src/pkg/dashboard/contribute_fixqueue_closed_pr_test.go +++ b/src/pkg/dashboard/contribute_fixqueue_closed_pr_test.go @@ -257,3 +257,48 @@ func TestFixTaskProgressIsNotRevoked(t *testing.T) { t.Fatalf("task_revoke frames for %v, want only [not-mine] (never the in-flight fix task)", revoked) } } + +// projectbluefin/utah-packages#972 (2026-11-09): the relay refused +// fix-karibhou-1, the hub requeued it and served it again at the relay's +// next ready as fix-karibhou-2 (a boundary fix on the same connection), and +// the relay's progress for that retried task (14:50:21Z and 14:54:36Z, repo +// empty, client_gen 1870) was answered with "task_progress resume rejected: +// no matching server-issued lease" and task_revoke. Here the retry goes +// through the real fail → requeue → boundary dispatch path, and its progress +// carries no repo, as the relay's did: it must not be revoked. The first +// task, which the relay failed, is no longer the relay's and still is. +func TestRetriedBoundaryFixProgressIsNotRevoked(t *testing.T) { + hub, _ := covK2Hub(t) + hub.fixAckGrace = time.Millisecond + fixPRStates(hub, nil) + server, client := wsPipe(t) + relay := fixConn(hub, "alice", fixCaps(capFixPRInterrupt), time.Now()) + relay.ws = server + sess := &wsSession{h: hub, conn: server, contributor: relay} + got := captureFixDispatch(hub) + + hub.Nudge(context.Background(), fixNudge("utah-packages", 972, "Canary")) + if len(*got) != 1 { + t.Fatalf("dispatches = %d, want the nudge's interrupt", len(*got)) + } + first := (*got)[0] + sess.handleTaskFailed(WSMessage{Type: "task_failed", TaskID: first.TaskID, Reason: "Hub is not the active polling slot"}) + retry := hub.selectFixTask(relay) + if retry == nil || retry.Kind != taskKindFixPR || retry.TaskID == first.TaskID { + t.Fatalf("the relay's next ready must get the fix again under a new task id, got %+v", retry) + } + + sess.handleTaskProgress(WSMessage{Type: "task_progress", TaskID: retry.TaskID, TaskGen: retry.TaskGen}) + sess.handleTaskProgress(WSMessage{Type: "task_progress", TaskID: retry.TaskID, TaskGen: retry.TaskGen}) + sess.handleTaskProgress(WSMessage{Type: "task_progress", TaskID: first.TaskID, TaskGen: first.TaskGen}) + var revoked []string + for _, f := range readFrames(t, client, time.Second) { + if f["type"] == "task_revoke" { + id, _ := f["task_id"].(string) + revoked = append(revoked, id) + } + } + if len(revoked) != 1 || revoked[0] != first.TaskID { + t.Fatalf("task_revoke frames for %v, want only the failed first task %q (never the retried fix %q)", revoked, first.TaskID, retry.TaskID) + } +} From ddaf8594fc4ef23046eef0ec9077df5cfc1dcf1c Mon Sep 17 00:00:00 2001 From: Douglas Baggett Date: Mon, 9 Nov 2026 10:13:59 -0500 Subject: [PATCH 3/3] contribute: an unreadable PR state no longer keeps the relay idle When a queued fix's PR state could not be read, selectFixTask answered task_unavailable/fix_pr_in_flight, so the relay got no new issue either. That happens when GitHub is struggling or the API budget is low, the time the hive needs its contributors most. The fix now stays queued and is retried on the next ready, and the relay takes ordinary work meanwhile. Busy fix slots still answer task_unavailable as before. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BQYnPGFJx81c8JXR3o1sg1 Signed-off-by: Douglas Baggett --- ...ixed-contribute-no-rework-for-merged-pr.md | 2 +- src/docs/contributor-relay.md | 2 +- src/pkg/dashboard/contribute_fixqueue.go | 30 +++++++++++++++---- .../contribute_fixqueue_closed_pr_test.go | 10 +++---- 4 files changed, 31 insertions(+), 13 deletions(-) diff --git a/changelog.d/fixed-contribute-no-rework-for-merged-pr.md b/changelog.d/fixed-contribute-no-rework-for-merged-pr.md index ae3559f0db..8f1bcdfb18 100644 --- a/changelog.d/fixed-contribute-no-rework-for-merged-pr.md +++ b/changelog.d/fixed-contribute-no-rework-for-merged-pr.md @@ -1 +1 @@ -- A contributor's relay is no longer sent to rework a PR that has already merged or closed, and one `fix-pr` task now runs once. On hivecommons/hive#17385 a stage-072 nudge queued a fix at 12:32Z, the PR merged at 13:13:58Z, and at about 14:25Z the hub still handed the relay `fix-pr hive#17385`, which the relay then ran twice: about 4.5 minutes lost, nothing pushed. Each link in that chain is fixed. The fix queue took the nudge's display name (`hive` on a hive whose repos are listed without the org), so the "left stage 072" resolve, which looks up `owner/name`, never found the entry, and the relay got a repo it could not match to its authorized list; queued fixes are now keyed and sent as `owner/name`. Right before a queued fix is handed out, the hub now checks the PR is still open (from the state cache when it holds the repo, else one PR read tagged `hive:contribute_fix_dispatch`); a merged or closed PR leaves the queue with a `fix queue: dropped a PR that is no longer open` log line, and if the state cannot be read nothing is dispatched that tick. A fix handed out at a task boundary was pushed to the relay twice (once by the dispatcher, once by the `ready` handler); it is now sent once, and the relay ignores a second `task_assign` for a `fix-pr` task it is already running (as its primary task, in a parallel session, preempting, or deferred) and logs where it runs. Progress reports from a boundary fix running as the relay's primary task are no longer answered with `task_revoke: no active lease for this task`, which killed the fix mid-run. +- A contributor's relay is no longer sent to rework a PR that has already merged or closed, and one `fix-pr` task now runs once. On hivecommons/hive#17385 a stage-072 nudge queued a fix at 12:32Z, the PR merged at 13:13:58Z, and at about 14:25Z the hub still handed the relay `fix-pr hive#17385`, which the relay then ran twice: about 4.5 minutes lost, nothing pushed. Each link in that chain is fixed. The fix queue took the nudge's display name (`hive` on a hive whose repos are listed without the org), so the "left stage 072" resolve, which looks up `owner/name`, never found the entry, and the relay got a repo it could not match to its authorized list; queued fixes are now keyed and sent as `owner/name`. Right before a queued fix is handed out, the hub now checks the PR is still open (from the state cache when it holds the repo, else one PR read tagged `hive:contribute_fix_dispatch`); a merged or closed PR leaves the queue with a `fix queue: dropped a PR that is no longer open` log line, and if the state cannot be read the fix waits for the next tick while the relay takes ordinary work. A fix handed out at a task boundary was pushed to the relay twice (once by the dispatcher, once by the `ready` handler); it is now sent once, and the relay ignores a second `task_assign` for a `fix-pr` task it is already running (as its primary task, in a parallel session, preempting, or deferred) and logs where it runs. Progress reports from a boundary fix running as the relay's primary task are no longer answered with `task_revoke: no active lease for this task`, which killed the fix mid-run. diff --git a/src/docs/contributor-relay.md b/src/docs/contributor-relay.md index 994f8f78cd..ec654f617e 100644 --- a/src/docs/contributor-relay.md +++ b/src/docs/contributor-relay.md @@ -1759,7 +1759,7 @@ Set `HIVE_MAX_PARALLEL_FIXES=0` in `~/.config/hive/contributor.env` and restart The hub records the mode it used in the nudge comment's marker (`mode=spawn|preempt|queued|comment`) and exposes queue depth / in-flight as `fix_queue` on the contributor profile API. -**Only open PRs are dispatched.** Right before a queued fix is handed out, the hub checks that its PR is still open: from the state cache when it holds the repo (free while the cached copy is inside its freshness bound, otherwise one re-read), else with one PR read tagged `hive:contribute_fix_dispatch`. A PR that has merged or closed leaves the queue with a `fix queue: dropped a PR that is no longer open` log line, and no relay is sent to rework it. When the state cannot be read, nothing is dispatched that tick and the next one asks again. Queued fixes carry the repo as `owner/name` even on a hive that lists its repos without the org, so the fix task's `repo` and the PR leaving stage 072 both match the queued entry. +**Only open PRs are dispatched.** Right before a queued fix is handed out, the hub checks that its PR is still open: from the state cache when it holds the repo (free while the cached copy is inside its freshness bound, otherwise one re-read), else with one PR read tagged `hive:contribute_fix_dispatch`. A PR that has merged or closed leaves the queue with a `fix queue: dropped a PR that is no longer open` log line, and no relay is sent to rework it. When the state cannot be read, the fix is not dispatched that tick: it stays queued and the next one asks again, and a relay asking for its next task gets ordinary work meanwhile instead of waiting idle. Queued fixes carry the repo as `owner/name` even on a hive that lists its repos without the org, so the fix task's `repo` and the PR leaving stage 072 both match the queued entry. **When the relay fails the task.** `task_failed` for a fix's `task_id` does not settle it as done: its PRs go back in the hub's queue (same place by age) and are not interrupted back in for 5 minutes, though the relay's next `ready` serves them at once. After three failed dispatches a PR leaves the fix queue and the stage engine's re-nudge / adoption path takes over. A dispatch the relay fails within one second of the nudge is not reported as delivered: the nudge comment reads `mode=queued`, says the relay could not take the task (quoting the relay's `message`/`reason`), and the stage log carries `relay_failed=true` (hivecommons/hive#14668). diff --git a/src/pkg/dashboard/contribute_fixqueue.go b/src/pkg/dashboard/contribute_fixqueue.go index 9ceddadf3a..d0e1a1a887 100644 --- a/src/pkg/dashboard/contribute_fixqueue.go +++ b/src/pkg/dashboard/contribute_fixqueue.go @@ -415,6 +415,13 @@ func (h *ContributeWSHub) forgetFixOwner(user, key string) { // asked for its next task (nothing running); the first dispatched message is // returned so the caller can reply with it instead of an issue. func (h *ContributeWSHub) dispatchFixes(user string, conns []*ContributorConnection, boundary bool) *WSMessage { + msg, _ := h.dispatchFixesHeld(user, conns, boundary) + return msg +} + +// dispatchFixesHeld is dispatchFixes that also reports whether dispatch +// stopped because a queued PR's state could not be read (dropClosedFixes). +func (h *ContributeWSHub) dispatchFixesHeld(user string, conns []*ContributorConnection, boundary bool) (*WSMessage, bool) { var target *ContributorConnection var targetPong time.Time for _, c := range conns { @@ -429,11 +436,12 @@ func (h *ContributeWSHub) dispatchFixes(user string, conns []*ContributorConnect } } if target == nil { - return nil + return nil, false } sched := h.fixQueue() cap := h.fixCapacity(target) var first *WSMessage + held := false for { next := sched.Next(user, cap, boundary) if len(next) == 0 { @@ -441,6 +449,7 @@ func (h *ContributeWSHub) dispatchFixes(user string, conns []*ContributorConnect } dropped, hold := h.dropClosedFixes(user, next) if hold { + held = true break } if dropped { @@ -469,13 +478,14 @@ func (h *ContributeWSHub) dispatchFixes(user string, conns []*ContributorConnect break } } - return first + return first, held } // dropClosedFixes checks the PRs about to be dispatched to user. A PR that // merged or closed since it was queued leaves the queue (logged), so no // relay is sent to rework it. hold is true when a PR's state could not be -// read: nothing is dispatched this tick, and the next one asks again. +// read: no fix is dispatched this tick (the relay gets ordinary work at a +// boundary, see selectFixTask), and the next one asks again. func (h *ContributeWSHub) dropClosedFixes(user string, items []fixqueue.Item) (dropped, hold bool) { sched := h.fixQueue() for _, it := range items { @@ -616,15 +626,23 @@ func (h *ContributeWSHub) selectFixTask(c *ContributorConnection) *WSMessage { if !sched.Pending(user) { return nil } - if msg := h.dispatchFixes(user, []*ContributorConnection{c}, true); msg != nil { + msg, held := h.dispatchFixesHeld(user, []*ContributorConnection{c}, true) + if msg != nil { return msg } if !sched.Pending(user) { // Every queued fix was for a PR that has since merged or closed. return nil } - // Fixes are queued but every slot is busy (or a PR's state could not be - // read this tick): no new work until one finishes. + if held { + // A queued PR's state could not be read this tick (GitHub trouble or + // a low API budget). The fix stays queued and is tried again on the + // next ready; meanwhile the relay takes ordinary work rather than + // sitting idle. The worst case is a fix that starts a little late. + return nil + } + // Fixes are queued but every slot is busy: no new work until one + // finishes. return h.taskUnavailable(taskUnavailableFixInFlight) } diff --git a/src/pkg/dashboard/contribute_fixqueue_closed_pr_test.go b/src/pkg/dashboard/contribute_fixqueue_closed_pr_test.go index f98518bb20..c6581bc496 100644 --- a/src/pkg/dashboard/contribute_fixqueue_closed_pr_test.go +++ b/src/pkg/dashboard/contribute_fixqueue_closed_pr_test.go @@ -147,8 +147,9 @@ func TestFixQueueNudgeForMergedPRDispatchesNothing(t *testing.T) { } } -// When the PR's state cannot be read nothing is dispatched (and no new work -// is offered in its place); the next boundary asks again. +// When the PR's state cannot be read the fix is not dispatched and stays +// queued, and the relay is not held idle: selectFixTask returns nothing, so +// the ready handler offers ordinary work. The next boundary asks again. func TestFixQueueHoldsDispatchWhenPRStateUnreadable(t *testing.T) { hub, _ := covK2Hub(t) if err := saveContributorProfile(&ContributorProfile{GitHubUsername: "alice", TrustTier: "contributor"}); err != nil { @@ -160,9 +161,8 @@ func TestFixQueueHoldsDispatchWhenPRStateUnreadable(t *testing.T) { return fixPRStateUnchecked, "", errors.New("GET pulls/5: 502") } c := fixConn(hub, "alice", fixCaps(capFixPRInterrupt), time.Now()) - msg := hub.selectFixTask(c) - if msg == nil || msg.Type != "task_unavailable" || msg.Reason != taskUnavailableFixInFlight { - t.Fatalf("selectFixTask = %+v, want task_unavailable/%s while the state is unreadable", msg, taskUnavailableFixInFlight) + if msg := hub.selectFixTask(c); msg != nil { + t.Fatalf("selectFixTask = %+v, want nil (ordinary work offered) while the state is unreadable", msg) } if len(*got) != 0 || hub.FixQueueStatus("alice").Queued != 1 { t.Fatalf("dispatches=%d status=%+v, want nothing dispatched and the fix still queued", len(*got), hub.FixQueueStatus("alice"))