diff --git a/bin/contributor-relay.js b/bin/contributor-relay.js index 17b694b0ac..0bf53d3922 100755 --- a/bin/contributor-relay.js +++ b/bin/contributor-relay.js @@ -5007,12 +5007,25 @@ function escapeRegExp(s) { return String(s).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); } +// fixPRRunningWhere says where this relay is already running fix-pr task +// taskId, or null. One fix-pr task runs in exactly one place: a second +// task_assign with the same task_id (a hub that delivered it twice) must not +// start it again as the primary task or in a parallel session. +function fixPRRunningWhere(taskId) { + if (!taskId) return null; + if (currentTask && currentTask.task_id === taskId) return 'as the primary task'; + if (parallelFixes.has(taskId)) return 'in a parallel fix session'; + if (preemptingFix && preemptingFix.task_id === taskId) return 'as the fix preempting the primary task'; + if (deferredFixes.some(d => d.task.task_id === taskId)) return 'deferred until the current task ends'; + return null; +} + // handleFixPRInterrupt decides spawn / preempt / defer for a fix-pr task that // arrived while currentTask is running. function handleFixPRInterrupt(msg, hub) { - if (parallelFixes.has(msg.task_id) || deferredFixes.some(d => d.task.task_id === msg.task_id) - || (preemptingFix && preemptingFix.task_id === msg.task_id)) { - console.log(`Ignoring duplicate fix-pr ${sanitizeHubText(msg.task_id)} — already handling it`); + const where = fixPRRunningWhere(msg.task_id); + if (where) { + console.log(`Ignoring duplicate fix-pr ${sanitizeHubText(msg.task_id)} — already running it ${where}`); return; } noteTaskAttemptStart(hub, msg); @@ -10016,6 +10029,10 @@ function handleMessage(data, hub) { handleFixPRInterrupt(msg, hub); break; } + if (msg.kind === FIX_PR_KIND && fixPRRunningWhere(msg.task_id)) { + console.log(`Ignoring duplicate fix-pr ${sanitizeHubText(msg.task_id)} — already running it ${fixPRRunningWhere(msg.task_id)}`); + break; + } if (currentTask) { console.log(`Rejecting task ${taskDisplay(msg)} from ${hub.url} — already working on ${taskDisplay(currentTask)}`); sendTo(hub, { type: 'task_failed', seq: nextSeq(), task_id: msg.task_id, reason: 'Already has active task' }); @@ -10108,6 +10125,7 @@ function handleMessage(data, hub) { writeRelayStatus(); activeHubIndex = hubs.indexOf(hub); console.log(`Task assigned: ${sanitizeHubText(msg.kind)} ${sanitizeHubText(taskDisplay(msg))} — ${sanitizeHubText(msg.title)} (from ${hub.url})`); + if (msg.kind === FIX_PR_KIND) console.log(`fix-pr ${fixPRLabel(msg)}: running as the primary task (no task was running)`); if (msg.github_token) { injectGhToken(msg.github_token); tokenExpiresAt = msg.token_expires_at ? new Date(msg.token_expires_at).getTime() : null; @@ -10871,6 +10889,7 @@ if (process.env.HIVE_RELAY_TEST_MODE === '1') { FIX_PR_DONE_MARKER, FIX_PR_FAILED_MARKER, fixPRState, + fixPRRunningWhere, scanPaneForFixMarkers, relaySupportsParallelFixes, blockingPromptKey, diff --git a/bin/contributor-relay.test.js b/bin/contributor-relay.test.js index c0fd7b226b..20b712e5bf 100644 --- a/bin/contributor-relay.test.js +++ b/bin/contributor-relay.test.js @@ -15600,6 +15600,58 @@ test('#12436 headless primary: a preempt fix is deferred and runs before the nex } finally { console.log = log; teardown(relay); } }); +// hivecommons/hive#17385 (2026-11-09): the hub delivered one boundary fix-pr +// task twice. The relay started the first copy as its primary task and the +// second in a parallel fix session, so the same fix ran in two places. +test('#17385 a fix-pr task_id delivered twice runs once, as the primary task', () => { + const relay = loadRelay({ backend: 'claude', env: { HIVE_MAX_PARALLEL_FIXES: '1', HIVE_CONTRIBUTOR_QUOTA_GUARD: 'off' }, execFileResult: { defer: true } }); + const log = console.log; + const lines = []; + console.log = (...a) => lines.push(a.join(' ')); + try { + const hub = relay.getHubs()[0]; + const sent = []; + hub.ws = { readyState: 1, send: p => sent.push(JSON.parse(p)) }; + relay.setCurrentTask(null); + const boundary = FIX_PR_TASK('fix-danathar-1', { priority: 'queued', mode: 'boundary' }); + relay.handleMessage(JSON.stringify(boundary), hub); + assert.strictEqual(relay.getCurrentTask() && relay.getCurrentTask().task_id, 'fix-danathar-1', 'the first copy becomes the primary task'); + const spawnsBefore = relay.__execFileCalls.length; + relay.handleMessage(JSON.stringify(boundary), hub); + assert.deepStrictEqual(relay.fixPRState().parallel, [], 'the second copy must not start a parallel fix session'); + assert.strictEqual(relay.__execFileCalls.length, spawnsBefore, 'no parallel child spawned for the duplicate'); + assert.strictEqual(relay.getCurrentTask().task_id, 'fix-danathar-1'); + assert.strictEqual(sent.filter(m => m.type === 'task_accepted' && m.task_id === 'fix-danathar-1').length, 1, `accepted once: ${JSON.stringify(sent)}`); + assert.ok(!sent.some(m => m.type === 'task_failed' && m.task_id === 'fix-danathar-1'), 'the duplicate is ignored, not failed back'); + assert.ok(lines.some(l => /running as the primary task/.test(l)), `log names the path taken: ${lines.join('\n')}`); + assert.ok(lines.some(l => /Ignoring duplicate fix-pr fix-danathar-1 — already running it as the primary task/.test(l)), lines.join('\n')); + } finally { console.log = log; teardown(relay); } +}); + +test('#17385 a fix-pr task_id already in a parallel session is not also started as the primary task', () => { + const relay = loadRelay({ backend: 'claude', env: { HIVE_MAX_PARALLEL_FIXES: '1', HIVE_CONTRIBUTOR_QUOTA_GUARD: 'off' }, execFileResult: { defer: true } }); + const log = console.log; + const lines = []; + console.log = (...a) => lines.push(a.join(' ')); + try { + const hub = relay.getHubs()[0]; + const sent = []; + hub.ws = { readyState: 1, send: p => sent.push(JSON.parse(p)) }; + relay.setCurrentTask({ task_id: 't-primary', task_gen: 1, kind: 'issue', repo: 'foo/bar', number: 1, title: 'primary' }); + relay.handleMessage(JSON.stringify(FIX_PR_TASK('fix-r')), hub); + assert.deepStrictEqual(relay.fixPRState().parallel, ['fix-r']); + // The primary ends while the fix still runs; the same task arrives again. + relay.setCurrentTask(null); + relay.handleMessage(JSON.stringify(FIX_PR_TASK('fix-r')), hub); + assert.strictEqual(relay.getCurrentTask(), null, 'the duplicate must not become the primary task'); + assert.deepStrictEqual(relay.fixPRState().parallel, ['fix-r']); + assert.strictEqual(sent.filter(m => m.type === 'task_accepted' && m.task_id === 'fix-r').length, 1, JSON.stringify(sent)); + assert.ok(lines.some(l => /Ignoring duplicate fix-pr fix-r — already running it in a parallel fix session/.test(l)), lines.join('\n')); + assert.strictEqual(relay.fixPRRunningWhere('fix-r'), 'in a parallel fix session'); + assert.strictEqual(relay.fixPRRunningWhere('other'), null); + } finally { console.log = log; teardown(relay); } +}); + (async () => { for (const [name, fn] of only ? tests.filter(([n]) => n.includes(only)) : tests) { try { diff --git a/changelog.d/fixed-contribute-no-rework-for-merged-pr.md b/changelog.d/fixed-contribute-no-rework-for-merged-pr.md new file mode 100644 index 0000000000..8f1bcdfb18 --- /dev/null +++ b/changelog.d/fixed-contribute-no-rework-for-merged-pr.md @@ -0,0 +1 @@ +- A contributor's relay is no longer sent to rework a PR that has already merged or closed, and one `fix-pr` task now runs once. On hivecommons/hive#17385 a stage-072 nudge queued a fix at 12:32Z, the PR merged at 13:13:58Z, and at about 14:25Z the hub still handed the relay `fix-pr hive#17385`, which the relay then ran twice: about 4.5 minutes lost, nothing pushed. Each link in that chain is fixed. The fix queue took the nudge's display name (`hive` on a hive whose repos are listed without the org), so the "left stage 072" resolve, which looks up `owner/name`, never found the entry, and the relay got a repo it could not match to its authorized list; queued fixes are now keyed and sent as `owner/name`. Right before a queued fix is handed out, the hub now checks the PR is still open (from the state cache when it holds the repo, else one PR read tagged `hive:contribute_fix_dispatch`); a merged or closed PR leaves the queue with a `fix queue: dropped a PR that is no longer open` log line, and if the state cannot be read the fix waits for the next tick while the relay takes ordinary work. A fix handed out at a task boundary was pushed to the relay twice (once by the dispatcher, once by the `ready` handler); it is now sent once, and the relay ignores a second `task_assign` for a `fix-pr` task it is already running (as its primary task, in a parallel session, preempting, or deferred) and logs where it runs. Progress reports from a boundary fix running as the relay's primary task are no longer answered with `task_revoke: no active lease for this task`, which killed the fix mid-run. diff --git a/src/docs/contributor-relay.md b/src/docs/contributor-relay.md index 9021c5e6e0..f6ed1fa2a2 100644 --- a/src/docs/contributor-relay.md +++ b/src/docs/contributor-relay.md @@ -1757,9 +1757,12 @@ Set `HIVE_MAX_PARALLEL_FIXES=0` in `~/.config/hive/contributor.env` and restart - `mode=preempt`, interactive pane: `task_accepted`, then the pane receives an `INTERRUPT (hive fix-pr, preempt mode)` prompt telling the agent to park the current issue (commit/stash WIP; the claim stays alive), fix the PR, print exactly `HIVE_FIX_PR_DONE ` (or `HIVE_FIX_PR_FAILED `), then resume. The progress tick reads the marker off the pane and reports `task_complete`/`task_failed` for the fix's `task_id`. Only one fix preempts at a time; more are queued locally. - `mode=preempt`, headless primary: there is no pane to interrupt, so the fix is deferred (`task_accepted` immediately) and runs before the relay's next `ready` — "fix what you broke before taking new work" holds on the client too. - With no task running, a `fix-pr` is just the next task. +- One `fix-pr` task runs in one place. A second `task_assign` with a `task_id` the relay is already running — as its primary task, in a parallel session, as the preempting fix, or deferred — is ignored, and the log says where it is already running. The hub records the mode it used in the nudge comment's marker (`mode=spawn|preempt|queued|comment`) and exposes queue depth / in-flight as `fix_queue` on the contributor profile API. +**Only open PRs are dispatched.** Right before a queued fix is handed out, the hub checks that its PR is still open: from the state cache when it holds the repo (free while the cached copy is inside its freshness bound, otherwise one re-read), else with one PR read tagged `hive:contribute_fix_dispatch`. A PR that has merged or closed leaves the queue with a `fix queue: dropped a PR that is no longer open` log line, and no relay is sent to rework it. When the state cannot be read, the fix is not dispatched that tick: it stays queued and the next one asks again, and a relay asking for its next task gets ordinary work meanwhile instead of waiting idle. Queued fixes carry the repo as `owner/name` even on a hive that lists its repos without the org, so the fix task's `repo` and the PR leaving stage 072 both match the queued entry. + **When the relay fails the task.** `task_failed` for a fix's `task_id` does not settle it as done: its PRs go back in the hub's queue (same place by age) and are not interrupted back in for 5 minutes, though the relay's next `ready` serves them at once. After three failed dispatches a PR leaves the fix queue and the stage engine's re-nudge / adoption path takes over. A dispatch the relay fails within one second of the nudge is not reported as delivered: the nudge comment reads `mode=queued`, says the relay could not take the task (quoting the relay's `message`/`reason`), and the stage log carries `relay_failed=true` (hivecommons/hive#14668). #### What the operator sees (Operations → Connected contributor agents) diff --git a/src/pkg/dashboard/api_repo_item_takeover_test.go b/src/pkg/dashboard/api_repo_item_takeover_test.go index 614a11ca78..5cb873a61e 100644 --- a/src/pkg/dashboard/api_repo_item_takeover_test.go +++ b/src/pkg/dashboard/api_repo_item_takeover_test.go @@ -1,6 +1,7 @@ package dashboard import ( + "context" "encoding/json" "net/http" "slices" @@ -236,6 +237,9 @@ func TestRepoItemReroutePickupLapseFallsBackToAdoptNow(t *testing.T) { srv, store := takeoverServer(t, mock) hub := srv.contributeHub hub.fixDispatched = func(_ *ContributorConnection, _ WSMessage) {} + // org/other#1 is a fixture the GitHub mock does not serve: report it + // open so the dispatch check keeps it. + hub.fixPRStateHook = func(context.Context, string, int) (fixPRState, string, error) { return fixPRStateOpen, "", nil } // bee2 is online but already holds a task: the reroute queues behind it. c := fixConn(hub, "bee2", fixCaps(capFixPRInterrupt), time.Now()) c.mu.Lock() diff --git a/src/pkg/dashboard/contribute_coverage_test.go b/src/pkg/dashboard/contribute_coverage_test.go index cc7102b652..c1ae4dc329 100644 --- a/src/pkg/dashboard/contribute_coverage_test.go +++ b/src/pkg/dashboard/contribute_coverage_test.go @@ -603,7 +603,7 @@ func TestRoleCoverage_FixTaskDispatchEndsNoWorkRun(t *testing.T) { relay.mu.Unlock() hub.sendFixTask(relay, "alice", &fixqueue.Dispatch{ID: "fix-1", Mode: fixqueue.ModeBoundary, Items: []fixqueue.Item{ {Repo: "myorg/repo1", Number: 7, URL: "https://github.com/myorg/repo1/pull/7", Title: "red PR"}, - }}) + }}, true) if got := coverageAt(hub, relay, later); !reflect.DeepEqual(got, scannerAlice) { t.Fatalf("relay working a fix task: RoleCoverage = %+v, want %+v", got, scannerAlice) } diff --git a/src/pkg/dashboard/contribute_fixqueue.go b/src/pkg/dashboard/contribute_fixqueue.go index 8e812bca1b..65b1f36049 100644 --- a/src/pkg/dashboard/contribute_fixqueue.go +++ b/src/pkg/dashboard/contribute_fixqueue.go @@ -2,11 +2,15 @@ package dashboard import ( "context" + "errors" "fmt" + "net/http" "strings" "time" "github.com/hivecommons/hive/pkg/fixqueue" + "github.com/hivecommons/hive/pkg/forgestate" + "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/github/automerge" ) @@ -55,6 +59,25 @@ const ( // leaves the fix queue and the stage engine's re-nudge / adoption path // takes over. fixTaskMaxAttempts = 3 + + // fixPRStateRESTCaller tags the one PR read a fix dispatch may make + // when the state cache cannot answer. + fixPRStateRESTCaller = "hive:contribute_fix_dispatch" + // fixPRStateTimeout bounds that read. + fixPRStateTimeout = 10 * time.Second +) + +// fixPRState is what the hub knows about a queued PR right before it is +// handed to a relay. +type fixPRState int + +const ( + // fixPRStateUnchecked: no GitHub client or no owner/name to read (a + // hub built without one, as in unit tests); dispatch as before. + fixPRStateUnchecked fixPRState = iota + fixPRStateOpen + // fixPRStateGone: merged, closed or deleted; the fix is dropped. + fixPRStateGone ) // WSFixPR is one red PR inside a fix-pr task_assign. @@ -285,14 +308,11 @@ func (h *ContributeWSHub) Nudge(_ context.Context, n automerge.ContributorNudge) return out } item := fixqueue.Item{ - Repo: n.Display, Number: n.Number, HeadSHA: n.HeadSHA, Title: n.Title, URL: n.URL, + Repo: nudgeFixRepo(n), Number: n.Number, HeadSHA: n.HeadSHA, Title: n.Title, URL: n.URL, OpenedAt: n.OpenedAt, HumanChangesRequested: n.HumanChangesRequested, Cause: fixqueue.Cause{Check: firstOf(n.FailingChecks), Signature: n.Reason}, Summary: strings.Join(append(append([]string(nil), n.FailingChecks...), n.ReviewAsks...), "; "), } - if item.Repo == "" { - item.Repo = n.Owner + "/" + n.Repo - } sched := h.fixQueue() sched.Enqueue(author, item) h.rememberFixOwner(author, item.Key()) @@ -309,11 +329,31 @@ func (h *ContributeWSHub) Nudge(_ context.Context, n automerge.ContributorNudge) } } } + if _, held := h.FixWorkFor(author, item.Repo, item.Number); !held { + // The dispatch check found the PR merged or closed and dropped it. + out.Mode = string(fixqueue.ModeComment) + } st := sched.Status(author, fixqueue.Capacity{Max: 1}) out.QueueDepth = st.Queued return out } +// nudgeFixRepo is the owner/name a nudged PR is queued under. Resolve keys +// on owner/name, and the relay records the fix task's repo as the one it +// is authorized for, so the queue never takes the bare configured name the +// nudge's Display carries on a hive whose repos are listed without the org +// (that spelling left a merged PR in the queue: Resolve could not find it). +func nudgeFixRepo(n automerge.ContributorNudge) string { + owner, repo := strings.TrimSpace(n.Owner), strings.TrimSpace(n.Repo) + if owner != "" && repo != "" { + return owner + "/" + repo + } + if display := strings.TrimSpace(n.Display); display != "" { + return display + } + return owner + "/" + repo +} + // Resolve implements automerge.ContributorNudgeSink: the PR left stage 072. func (h *ContributeWSHub) Resolve(_ context.Context, owner, repo string, number int) { full := owner + "/" + repo @@ -375,6 +415,13 @@ func (h *ContributeWSHub) forgetFixOwner(user, key string) { // asked for its next task (nothing running); the first dispatched message is // returned so the caller can reply with it instead of an issue. func (h *ContributeWSHub) dispatchFixes(user string, conns []*ContributorConnection, boundary bool) *WSMessage { + msg, _ := h.dispatchFixesHeld(user, conns, boundary) + return msg +} + +// dispatchFixesHeld is dispatchFixes that also reports whether dispatch +// stopped because a queued PR's state could not be read (dropClosedFixes). +func (h *ContributeWSHub) dispatchFixesHeld(user string, conns []*ContributorConnection, boundary bool) (*WSMessage, bool) { var target *ContributorConnection var targetPong time.Time for _, c := range conns { @@ -389,12 +436,26 @@ func (h *ContributeWSHub) dispatchFixes(user string, conns []*ContributorConnect } } if target == nil { - return nil + return nil, false } sched := h.fixQueue() cap := h.fixCapacity(target) var first *WSMessage + held := false for { + next := sched.Next(user, cap, boundary) + if len(next) == 0 { + break + } + dropped, hold := h.dropClosedFixes(user, next) + if hold { + held = true + break + } + if dropped { + // The queue changed under the preview: look again. + continue + } var d *fixqueue.Dispatch if boundary { d = sched.DispatchAtBoundary(user, cap) @@ -404,7 +465,11 @@ func (h *ContributeWSHub) dispatchFixes(user string, conns []*ContributorConnect if d == nil { break } - msg := h.sendFixTask(target, user, d) + // At a boundary the `ready` handler ships the returned message + // itself; pushing it here as well delivered the same task twice, + // and the relay ran it both as its primary task and in a parallel + // fix session. + msg := h.sendFixTask(target, user, d, !boundary) if first == nil { first = &msg } @@ -413,11 +478,91 @@ func (h *ContributeWSHub) dispatchFixes(user string, conns []*ContributorConnect break } } - return first + return first, held } -// sendFixTask builds and pushes the fix-pr task_assign for d. -func (h *ContributeWSHub) sendFixTask(c *ContributorConnection, user string, d *fixqueue.Dispatch) WSMessage { +// dropClosedFixes checks the PRs about to be dispatched to user. A PR that +// merged or closed since it was queued leaves the queue (logged), so no +// relay is sent to rework it. hold is true when a PR's state could not be +// read: no fix is dispatched this tick (the relay gets ordinary work at a +// boundary, see selectFixTask), and the next one asks again. +func (h *ContributeWSHub) dropClosedFixes(user string, items []fixqueue.Item) (dropped, hold bool) { + sched := h.fixQueue() + for _, it := range items { + state, reason, err := h.fixPRStateOf(context.Background(), it.Repo, it.Number) + if err != nil { + h.logger.Warn("[contribute-ws] fix queue: could not read the PR's state; holding the dispatch", "username", user, "pr", it.Key(), "error", err) + return dropped, true + } + if state != fixPRStateGone { + continue + } + sched.Remove(user, it.Repo, it.Number) + h.forgetFixOwner(user, it.Key()) + h.logger.Info("[contribute-ws] fix queue: dropped a PR that is no longer open", "username", user, "pr", it.Key(), "state", reason, "head", shortSHA(it.HeadSHA)) + dropped = true + } + return dropped, false +} + +// fixPRStateOf answers whether repo#number is still open, from the state +// cache when it holds the repo (a cached copy inside its freshness bound +// costs nothing; an older one is re-read once), else with one PR read. +// reason says "merged" or "closed" for a gone PR. A read that fails is an +// error, never "open". +func (h *ContributeWSHub) fixPRStateOf(ctx context.Context, repo string, number int) (fixPRState, string, error) { + if h.fixPRStateHook != nil { + return h.fixPRStateHook(ctx, repo, number) + } + if h.server == nil || h.server.deps == nil || h.server.deps.GHClient == nil { + return fixPRStateUnchecked, "", nil + } + owner, name, ok := strings.Cut(strings.TrimSpace(repo), "/") + if !ok || owner == "" || name == "" || number <= 0 { + return fixPRStateUnchecked, "", nil + } + client := h.server.deps.GHClient + ctx, cancel := context.WithTimeout(github.WithRESTCaller(ctx, fixPRStateRESTCaller), fixPRStateTimeout) + defer cancel() + if view := client.ForgeStateView(); view != nil { + if _, held := view.Snapshot(repo); held { + _, open, err := view.ReadThrough(ctx, repo, forgestate.KindPR, number, 0) + switch { + case err == nil && open: + return fixPRStateOpen, "", nil + case err == nil: + // The cache holds open PRs only: absent after a fresh read + // means merged, closed or deleted. + return fixPRStateGone, "merged or closed", nil + case !errors.Is(err, forgestate.ErrStale) && !errors.Is(err, forgestate.ErrNotReady) && !errors.Is(err, forgestate.ErrUnknownRepo): + return fixPRStateUnchecked, "", err + } + // A stale or not-yet-ready cache made no call: read the PR below. + } + } + gh := client.GoGitHub() + if gh == nil { + return fixPRStateUnchecked, "", nil + } + pr, resp, err := gh.PullRequests.Get(ctx, owner, name, number) + if err != nil { + if resp != nil && resp.Response != nil && (resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone) { + return fixPRStateGone, "not found", nil + } + return fixPRStateUnchecked, "", err + } + switch { + case pr.GetMerged() || !pr.GetMergedAt().Time.IsZero(): + return fixPRStateGone, "merged", nil + case !strings.EqualFold(pr.GetState(), "open"): + return fixPRStateGone, "closed", nil + } + return fixPRStateOpen, "", nil +} + +// sendFixTask builds the fix-pr task_assign for d and, when push is true, +// sends it on c (a boundary dispatch is sent by the `ready` handler). +func (h *ContributeWSHub) sendFixTask(c *ContributorConnection, user string, d *fixqueue.Dispatch, push bool) WSMessage { prs := make([]WSFixPR, 0, len(d.Items)) for _, it := range d.Items { pr := WSFixPR{Repo: it.Repo, Number: it.Number, URL: it.URL, HeadSHA: it.HeadSHA, Title: it.Title, Reason: it.Cause.Signature, @@ -461,7 +606,7 @@ func (h *ContributeWSHub) sendFixTask(c *ContributorConnection, user string, d * if h.fixDispatched != nil { h.fixDispatched(c, msg) } - if c.ws != nil { + if push && c.ws != nil { if err := c.send(msg); err != nil { h.logger.Warn("[contribute-ws] fix-pr task send failed", "username", user, "task", d.ID, "error", err) } @@ -481,10 +626,23 @@ func (h *ContributeWSHub) selectFixTask(c *ContributorConnection) *WSMessage { if !sched.Pending(user) { return nil } - if msg := h.dispatchFixes(user, []*ContributorConnection{c}, true); msg != nil { + msg, held := h.dispatchFixesHeld(user, []*ContributorConnection{c}, true) + if msg != nil { return msg } - // Fixes are queued but every slot is busy: no new work until one finishes. + if !sched.Pending(user) { + // Every queued fix was for a PR that has since merged or closed. + return nil + } + if held { + // A queued PR's state could not be read this tick (GitHub trouble or + // a low API budget). The fix stays queued and is tried again on the + // next ready; meanwhile the relay takes ordinary work rather than + // sitting idle. The worst case is a fix that starts a little late. + return nil + } + // Fixes are queued but every slot is busy: no new work until one + // finishes. return h.taskUnavailable(taskUnavailableFixInFlight) } diff --git a/src/pkg/dashboard/contribute_fixqueue_closed_pr_test.go b/src/pkg/dashboard/contribute_fixqueue_closed_pr_test.go new file mode 100644 index 0000000000..c6581bc496 --- /dev/null +++ b/src/pkg/dashboard/contribute_fixqueue_closed_pr_test.go @@ -0,0 +1,304 @@ +package dashboard + +import ( + "context" + "encoding/json" + "errors" + "strconv" + "testing" + "time" + + "github.com/gorilla/websocket" + + "github.com/hivecommons/hive/pkg/github/automerge" +) + +// A merged PR must never come back to its author as rework. Observed +// 2026-11-09 on hivecommons/hive#17385: the 12:32Z stage-072 nudge queued a +// fix, the PR left 072 and merged at 13:13:58Z, and at ~14:25Z the hub +// still handed the relay "fix-pr hive#17385", which then ran twice. + +// fixPRStates answers the dispatch check from a map keyed by fixqueue key; +// a PR not in the map is open. +func fixPRStates(hub *ContributeWSHub, states map[string]fixPRState) *[]string { + var asked []string + hub.fixPRStateHook = func(_ context.Context, repo string, number int) (fixPRState, string, error) { + key := repo + "#" + strconv.Itoa(number) + asked = append(asked, key) + if st, ok := states[key]; ok { + if st == fixPRStateGone { + return st, "merged", nil + } + return st, "", nil + } + return fixPRStateOpen, "", nil + } + return &asked +} + +// bareNudge is a nudge from a hive whose repos are configured without the +// org: the stage engine's Display is the bare name. +func bareNudge(n int) automerge.ContributorNudge { + return automerge.ContributorNudge{ + Owner: "hivecommons", Repo: "hive", Display: "hive", Number: n, HeadSHA: "6be425a93fe8", + Author: "alice", Title: "feat", URL: "https://github.com/hivecommons/hive/pull/" + strconv.Itoa(n), + OpenedAt: time.Now().Add(-time.Duration(n) * time.Minute), Reason: "conflicting", + ReviewAsks: []string{"the branch conflicts with the base and needs a rebase"}, + } +} + +// The fix task names the repo as owner/name, and the PR leaving stage 072 +// (Resolve, which keys on owner/name) finds the queued entry. +func TestFixQueueBareConfiguredRepoQueuesOwnerName(t *testing.T) { + hub, _ := covK2Hub(t) + got := captureFixDispatch(hub) + fixConn(hub, "alice", fixCaps(capFixPRInterrupt), time.Now()) + + hub.Nudge(context.Background(), bareNudge(1)) + if len(*got) != 1 { + t.Fatalf("dispatches = %d, want 1", len(*got)) + } + m := (*got)[0] + if m.Repo != "hivecommons/hive" || len(m.FixPRs) != 1 || m.FixPRs[0].Repo != "hivecommons/hive" { + t.Fatalf("fix task repo = %q / %+v, want owner/name hivecommons/hive", m.Repo, m.FixPRs) + } + + // A second PR waits behind the preemption; it leaves 072 before its turn. + hub.Nudge(context.Background(), bareNudge(17385)) + if q := hub.FixQueueItems("alice"); len(q) != 1 || q[0].Repo != "hivecommons/hive" { + t.Fatalf("queued = %+v, want hivecommons/hive#17385", q) + } + hub.Resolve(context.Background(), "hivecommons", "hive", 17385) + if q := hub.FixQueueItems("alice"); len(q) != 0 { + t.Fatalf("Resolve left %+v in the queue", q) + } +} + +// A PR that merged while it waited is dropped at the task boundary: the relay +// gets no fix task, falls through to issue selection, and the queue is empty. +func TestFixQueueBoundaryDropsPRThatMergedWhileQueued(t *testing.T) { + hub, _ := covK2Hub(t) + if err := saveContributorProfile(&ContributorProfile{GitHubUsername: "alice", TrustTier: "contributor"}); err != nil { + t.Fatal(err) + } + got := captureFixDispatch(hub) + // Queued while the relay was away. + if out := hub.Nudge(context.Background(), bareNudge(17385)); out.Mode != "queued" || out.QueueDepth != 1 { + t.Fatalf("offline nudge = %+v, want queued", out) + } + // It merges; the relay comes back and asks for work. + asked := fixPRStates(hub, map[string]fixPRState{"hivecommons/hive#17385": fixPRStateGone}) + c := fixConn(hub, "alice", fixCaps(capFixPRInterrupt), time.Now()) + if msg := hub.selectFixTask(c); msg != nil { + t.Fatalf("selectFixTask = %+v, want nil (no fix for a merged PR, go select an issue)", msg) + } + if len(*got) != 0 { + t.Fatalf("dispatched %+v for a merged PR", *got) + } + if len(*asked) != 1 { + t.Fatalf("state reads = %v, want one", *asked) + } + if st := hub.FixQueueStatus("alice"); st.Queued != 0 || st.InFlight != 0 { + t.Fatalf("status = %+v, want the merged PR out of the queue", st) + } + hub.fixQueueMu.Lock() + owned := len(hub.fixOwners["alice"]) + hub.fixQueueMu.Unlock() + if owned != 0 { + t.Fatalf("owner index still holds %d key(s)", owned) + } +} + +// Only the merged one is dropped: an open PR behind it is still dispatched. +func TestFixQueueDispatchSkipsClosedPRAndServesTheNextOpenOne(t *testing.T) { + hub, _ := covK2Hub(t) + if err := saveContributorProfile(&ContributorProfile{GitHubUsername: "alice", TrustTier: "contributor"}); err != nil { + t.Fatal(err) + } + got := captureFixDispatch(hub) + old, open := bareNudge(30), bareNudge(20) + old.Reason, open.Reason = "conflicting-a", "conflicting-b" + hub.Nudge(context.Background(), old) + hub.Nudge(context.Background(), open) + fixPRStates(hub, map[string]fixPRState{"hivecommons/hive#30": fixPRStateGone}) + c := fixConn(hub, "alice", fixCaps(capFixPRInterrupt), time.Now()) + msg := hub.selectFixTask(c) + if msg == nil || msg.Kind != taskKindFixPR || msg.Number != 20 { + t.Fatalf("selectFixTask = %+v, want the open PR #20", msg) + } + if len(*got) != 1 || (*got)[0].Number != 20 { + t.Fatalf("dispatches = %+v", *got) + } +} + +// A nudge for a PR that is already merged by the time it reaches the hub +// queues nothing and is reported as a comment-only nudge. +func TestFixQueueNudgeForMergedPRDispatchesNothing(t *testing.T) { + hub, _ := covK2Hub(t) + got := captureFixDispatch(hub) + fixConn(hub, "alice", fixCaps(capFixPRInterrupt), time.Now()) + fixPRStates(hub, map[string]fixPRState{"hivecommons/hive#17385": fixPRStateGone}) + out := hub.Nudge(context.Background(), bareNudge(17385)) + if out.Mode != "comment" || out.QueueDepth != 0 { + t.Fatalf("nudge = %+v, want comment with nothing queued", out) + } + if len(*got) != 0 { + t.Fatalf("dispatched %+v for a merged PR", *got) + } +} + +// When the PR's state cannot be read the fix is not dispatched and stays +// queued, and the relay is not held idle: selectFixTask returns nothing, so +// the ready handler offers ordinary work. The next boundary asks again. +func TestFixQueueHoldsDispatchWhenPRStateUnreadable(t *testing.T) { + hub, _ := covK2Hub(t) + if err := saveContributorProfile(&ContributorProfile{GitHubUsername: "alice", TrustTier: "contributor"}); err != nil { + t.Fatal(err) + } + got := captureFixDispatch(hub) + hub.Nudge(context.Background(), bareNudge(5)) + hub.fixPRStateHook = func(context.Context, string, int) (fixPRState, string, error) { + return fixPRStateUnchecked, "", errors.New("GET pulls/5: 502") + } + c := fixConn(hub, "alice", fixCaps(capFixPRInterrupt), time.Now()) + if msg := hub.selectFixTask(c); msg != nil { + t.Fatalf("selectFixTask = %+v, want nil (ordinary work offered) while the state is unreadable", msg) + } + if len(*got) != 0 || hub.FixQueueStatus("alice").Queued != 1 { + t.Fatalf("dispatches=%d status=%+v, want nothing dispatched and the fix still queued", len(*got), hub.FixQueueStatus("alice")) + } + fixPRStates(hub, nil) + if msg := hub.selectFixTask(c); msg == nil || msg.Kind != taskKindFixPR { + t.Fatalf("readable again: selectFixTask = %+v, want the fix", msg) + } +} + +// readFrames collects every frame the relay receives within d. +func readFrames(t *testing.T, client *websocket.Conn, d time.Duration) []map[string]any { + t.Helper() + var out []map[string]any + deadline := time.Now().Add(d) + for { + _ = client.SetReadDeadline(deadline) + _, raw, err := client.ReadMessage() + if err != nil { + return out + } + var m map[string]any + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatalf("unmarshal: %v", err) + } + out = append(out, m) + } +} + +// At a task boundary the fix task reaches the relay once. It used to be +// pushed by sendFixTask and then again by the ready handler, so the relay +// started it as its primary task and, on the second copy, in a parallel fix +// session as well. +func TestFixQueueBoundaryTaskIsSentOnce(t *testing.T) { + hub, s := covK2Hub(t) + s.statusMu.Lock() + s.status = &StatusPayload{} + s.statusMu.Unlock() + if err := saveContributorProfile(&ContributorProfile{GitHubUsername: "alice", TrustTier: "contributor"}); err != nil { + t.Fatal(err) + } + hub.Nudge(context.Background(), bareNudge(17385)) + + server, client := wsPipe(t) + relay := coverageConn("alice", "scanner", time.Now()) + relay.capabilities = fixCaps(capFixPRInterrupt, capParallelSessions) + relay.ws = server + hub.mu.Lock() + hub.connections["alice"] = relay + hub.mu.Unlock() + sess := &wsSession{h: hub, conn: server, contributor: relay} + if stop := sess.handleReady(WSMessage{Type: "ready"}); stop { + t.Fatal("ready must not stop the session") + } + assigns := 0 + for _, f := range readFrames(t, client, 500*time.Millisecond) { + if f["type"] == "task_assign" { + assigns++ + if f["kind"] != taskKindFixPR || f["repo"] != "hivecommons/hive" { + t.Fatalf("task_assign = %v, want fix-pr for hivecommons/hive", f) + } + } + } + if assigns != 1 { + t.Fatalf("relay received %d task_assign frames for one boundary fix, want 1", assigns) + } +} + +// A boundary fix runs as the relay's primary task and reports progress like +// one. It holds no lease, so the progress must not be read as a resume +// claim and answered with task_revoke (which killed the fix mid-run). +func TestFixTaskProgressIsNotRevoked(t *testing.T) { + hub, _ := covK2Hub(t) + server, client := wsPipe(t) + relay := fixConn(hub, "alice", fixCaps(capFixPRInterrupt), time.Now()) + relay.ws = server + relay.fixTasks = map[string]*WSTaskAssign{"fix-alice-1": {TaskID: "fix-alice-1", Kind: taskKindFixPR, Repo: "hivecommons/hive", Number: 17385}} + sess := &wsSession{h: hub, conn: server, contributor: relay} + sess.handleTaskProgress(WSMessage{Type: "task_progress", TaskID: "fix-alice-1", Repo: "hivecommons/hive", Number: 17385}) + // A task the hub never issued is still a resume claim and is still + // revoked; its frame doubles as the marker that the first progress was + // handled (frames arrive in order). + sess.handleTaskProgress(WSMessage{Type: "task_progress", TaskID: "not-mine", Repo: "hivecommons/hive", Number: 1}) + var revoked []string + for _, f := range readFrames(t, client, time.Second) { + if f["type"] == "task_revoke" { + id, _ := f["task_id"].(string) + revoked = append(revoked, id) + } + } + if len(revoked) != 1 || revoked[0] != "not-mine" { + t.Fatalf("task_revoke frames for %v, want only [not-mine] (never the in-flight fix task)", revoked) + } +} + +// projectbluefin/utah-packages#972 (2026-11-09): the relay refused +// fix-karibhou-1, the hub requeued it and served it again at the relay's +// next ready as fix-karibhou-2 (a boundary fix on the same connection), and +// the relay's progress for that retried task (14:50:21Z and 14:54:36Z, repo +// empty, client_gen 1870) was answered with "task_progress resume rejected: +// no matching server-issued lease" and task_revoke. Here the retry goes +// through the real fail → requeue → boundary dispatch path, and its progress +// carries no repo, as the relay's did: it must not be revoked. The first +// task, which the relay failed, is no longer the relay's and still is. +func TestRetriedBoundaryFixProgressIsNotRevoked(t *testing.T) { + hub, _ := covK2Hub(t) + hub.fixAckGrace = time.Millisecond + fixPRStates(hub, nil) + server, client := wsPipe(t) + relay := fixConn(hub, "alice", fixCaps(capFixPRInterrupt), time.Now()) + relay.ws = server + sess := &wsSession{h: hub, conn: server, contributor: relay} + got := captureFixDispatch(hub) + + hub.Nudge(context.Background(), fixNudge("utah-packages", 972, "Canary")) + if len(*got) != 1 { + t.Fatalf("dispatches = %d, want the nudge's interrupt", len(*got)) + } + first := (*got)[0] + sess.handleTaskFailed(WSMessage{Type: "task_failed", TaskID: first.TaskID, Reason: "Hub is not the active polling slot"}) + retry := hub.selectFixTask(relay) + if retry == nil || retry.Kind != taskKindFixPR || retry.TaskID == first.TaskID { + t.Fatalf("the relay's next ready must get the fix again under a new task id, got %+v", retry) + } + + sess.handleTaskProgress(WSMessage{Type: "task_progress", TaskID: retry.TaskID, TaskGen: retry.TaskGen}) + sess.handleTaskProgress(WSMessage{Type: "task_progress", TaskID: retry.TaskID, TaskGen: retry.TaskGen}) + sess.handleTaskProgress(WSMessage{Type: "task_progress", TaskID: first.TaskID, TaskGen: first.TaskGen}) + var revoked []string + for _, f := range readFrames(t, client, time.Second) { + if f["type"] == "task_revoke" { + id, _ := f["task_id"].(string) + revoked = append(revoked, id) + } + } + if len(revoked) != 1 || revoked[0] != first.TaskID { + t.Fatalf("task_revoke frames for %v, want only the failed first task %q (never the retried fix %q)", revoked, first.TaskID, retry.TaskID) + } +} diff --git a/src/pkg/dashboard/contribute_ws.go b/src/pkg/dashboard/contribute_ws.go index 34311169f6..0065bbe9f2 100644 --- a/src/pkg/dashboard/contribute_ws.go +++ b/src/pkg/dashboard/contribute_ws.go @@ -1003,6 +1003,9 @@ type ContributeWSHub struct { fixWaiters int fixFailures map[string]string fixAckGrace time.Duration + // fixPRStateHook replaces fixPRStateOf's GitHub read of a queued PR's + // state (open, merged, closed); a test hook. + fixPRStateHook func(ctx context.Context, repo string, number int) (fixPRState, string, error) // stageStore is the stage-label memory the fleet view reads "PRs returned // to this contributor" from (hivecommons/hive#12502). Nil means the // process-wide stagelabels.Shared(); tests inject an in-memory store. @@ -4146,6 +4149,14 @@ func (s *wsSession) handleTaskProgress(msg WSMessage) { s.contributor.mu.Unlock() return } + // A fix-pr task handed out at a task boundary runs as the + // relay's primary task and reports progress like one, but it + // is tracked in fixTasks and holds no lease. It is not a resume + // claim: revoking it killed the fix mid-run. + if _, fix := s.contributor.fixTasks[msg.TaskID]; fix { + s.contributor.mu.Unlock() + return + } identity := identityOf(s.contributor) canonRepo := h.canonicalRepoKey(msg.Repo) s.contributor.mu.Unlock() diff --git a/src/pkg/fixqueue/fixqueue.go b/src/pkg/fixqueue/fixqueue.go index bfcd546021..fad66ce801 100644 --- a/src/pkg/fixqueue/fixqueue.go +++ b/src/pkg/fixqueue/fixqueue.go @@ -414,17 +414,53 @@ func (s *Scheduler) Dispatch(user string, cap Capacity) *Dispatch { s.mu.Lock() defer s.mu.Unlock() c, ok := s.by[norm(user)] - if !ok || len(c.queue) == 0 || len(c.inflight) >= cap.Slots() { + if !ok || !s.dispatchableLocked(c, cap, false) { return nil } - if s.opts.Now().Before(c.queue[0].NotBefore) { - return nil + return s.pop(user, c, cap.Mode()) +} + +// dispatchableLocked holds the guards of Dispatch (boundary=false) and +// DispatchAtBoundary (boundary=true). +func (s *Scheduler) dispatchableLocked(c *contributor, cap Capacity, boundary bool) bool { + if len(c.queue) == 0 { + return false + } + if boundary { + return len(c.inflight) < cap.BoundarySlots() } - mode := cap.Mode() - if mode == ModePreempt && (c.preempted || s.opts.Now().Before(c.focusUntil)) { + if len(c.inflight) >= cap.Slots() || s.opts.Now().Before(c.queue[0].NotBefore) { + return false + } + return cap.Mode() != ModePreempt || (!c.preempted && !s.opts.Now().Before(c.focusUntil)) +} + +// Next returns the items the next Dispatch (boundary=false) or +// DispatchAtBoundary (boundary=true) would hand out, without taking them: +// the caller can check them before they leave the queue. nil when that call +// would return nil. +func (s *Scheduler) Next(user string, cap Capacity, boundary bool) []Item { + s.mu.Lock() + defer s.mu.Unlock() + c, ok := s.by[norm(user)] + if !ok || !s.dispatchableLocked(c, cap, boundary) { return nil } - return s.pop(user, c, mode) + head := c.queue[0] + out := []Item{head} + for _, q := range c.queue[1:] { + if coalesces(head, q) { + out = append(out, q) + } + } + return out +} + +// coalesces reports whether queued item q joins head's fix task. A rerouted +// PR (another author's) never shares a task with the owner's own PRs: its +// prompt differs. +func coalesces(head, q Item) bool { + return head.Cause.coalescable() && q.Cause == head.Cause && !head.Rerouted() && !q.Rerouted() } // DispatchAtBoundary hands user the next fix when their relay asked for its @@ -435,7 +471,7 @@ func (s *Scheduler) DispatchAtBoundary(user string, cap Capacity) *Dispatch { s.mu.Lock() defer s.mu.Unlock() c, ok := s.by[norm(user)] - if !ok || len(c.queue) == 0 || len(c.inflight) >= cap.BoundarySlots() { + if !ok || !s.dispatchableLocked(c, cap, true) { return nil } return s.pop(user, c, ModeBoundary) @@ -447,9 +483,7 @@ func (s *Scheduler) pop(user string, c *contributor, mode Mode) *Dispatch { d := &dispatch{mode: mode, items: map[string]Item{head.Key(): head}, at: s.opts.Now()} rest := c.queue[:0:0] for _, q := range c.queue[1:] { - // A rerouted PR (another author's) never shares a task with the - // owner's own PRs: its prompt differs. - if head.Cause.coalescable() && q.Cause == head.Cause && !head.Rerouted() && !q.Rerouted() { + if coalesces(head, q) { d.items[q.Key()] = q continue } diff --git a/src/pkg/fixqueue/fixqueue_test.go b/src/pkg/fixqueue/fixqueue_test.go index 0a042f6b7f..6746a2e326 100644 --- a/src/pkg/fixqueue/fixqueue_test.go +++ b/src/pkg/fixqueue/fixqueue_test.go @@ -451,3 +451,42 @@ func TestDispatchAt(t *testing.T) { t.Fatalf("inflight=%+v", got) } } + +// Next previews exactly what Dispatch / DispatchAtBoundary would hand out, +// coalesced siblings included, and takes nothing; it is nil whenever they +// would return nil. +func TestNextPreviewsDispatchWithoutTakingIt(t *testing.T) { + c := newClock() + s := newSched(c) + cap := Capacity{Max: 1} + if got := s.Next("alice", cap, false); got != nil { + t.Fatalf("empty queue: Next = %v, want nil", keys(got)) + } + shared := Cause{Check: "lint", Signature: "gofmt"} + a, b, other := red(1, 3*time.Hour, c), red(2, 2*time.Hour, c), red(3, time.Hour, c) + a.Cause, b.Cause = shared, shared + for _, it := range []Item{a, b, other} { + s.Enqueue("alice", it) + } + next := s.Next("alice", cap, false) + if !eq(keys(next), []int{1, 2}) { + t.Fatalf("Next = %v, want the head and its coalesced sibling [1 2]", keys(next)) + } + if st := s.Status("alice", cap); st.Queued != 3 || st.InFlight != 0 { + t.Fatalf("Next must not take anything: %+v", st) + } + d := s.Dispatch("alice", cap) + if d == nil || !eq(keys(d.Items), keys(next)) { + t.Fatalf("Dispatch = %+v, want what Next previewed %v", d, keys(next)) + } + // A preemption is in flight and capacity 1 is used: both previews are nil. + if got := s.Next("alice", cap, false); got != nil { + t.Fatalf("preemption in flight: Next = %v, want nil", keys(got)) + } + if got := s.Next("alice", cap, true); got != nil { + t.Fatalf("boundary with the only slot busy: Next = %v, want nil", keys(got)) + } + if got := s.Next("alice", Capacity{Max: 2}, true); !eq(keys(got), []int{3}) { + t.Fatalf("boundary with a free slot: Next = %v, want [3]", keys(got)) + } +}