diff --git a/changelog.d/fixed-army-rework-queue.md b/changelog.d/fixed-army-rework-queue.md new file mode 100644 index 0000000000..93eaad0192 --- /dev/null +++ b/changelog.d/fixed-army-rework-queue.md @@ -0,0 +1 @@ +- Contribute "Your army" card: a remote agent's "PRs awaiting rework" now lists only its open, not-yet-dispatched PRs — merged, closed and hive-adopted PRs are dropped (and a queued fix for a PR that is gone leaves the fix queue instead of reaching the relay); the line says why a fix is not dispatched (offline, no fix-pr capability, scope, orphaned dispatch, backoff, focus window, waiting for `ready`) instead of "not yet dispatched"; a fix dispatched to a socket that dropped is released at the relay's next `ready` and a completion reported on a reconnected socket settles it; links collapse to three inline plus a "show N more" disclosure. diff --git a/src/docs/contributor-relay.md b/src/docs/contributor-relay.md index e42998d277..fa2bd52863 100644 --- a/src/docs/contributor-relay.md +++ b/src/docs/contributor-relay.md @@ -1775,9 +1775,10 @@ Each row of the "Connected contributor agents (ClankeR)" list on `/contribute/op | `fix_mode` | `spawn` or `preempt`, the interrupt mode that capacity implies. | | `in_flight[]` | One entry per work item on the connection — the primary task plus every `fix-pr` sub-session — each with `kind` (`new` or `rework`), `task_kind`, `repo`, `number`, `title`, `url`, `stage` (the item's `stage/NNN-*` label as the hive last observed it), `mode` and `started_at`. The row shows one line per item with a **NEW** / **REWORK** badge. A relay running only a boundary-dispatched fix therefore shows as *working*, not *idle*. | | `fix_queue_depth` | Red PRs waiting in the contributor's server-side fix queue (not dispatched). | -| `pending_rework`, `pending_rework_prs[]` | The contributor's open PRs in `stage/072-returned-to-contributor` that have **not** been dispatched to them. The row prints a red **N PRs awaiting rework (not yet dispatched)** line, and an idle relay reads **idle · N PRs awaiting rework** instead of `idle: no matching work`. | +| `pending_rework`, `pending_rework_prs[]` | The contributor's **open** PRs in `stage/072-returned-to-contributor` that have **not** been dispatched to them. Merged and closed PRs, PRs the hive adopted (074) and stage-store records no stage pass has seen within the `072` dwell are never listed or counted. The row prints a red **N PRs awaiting rework (…)** line with at most three PRs linked inline and the rest behind a *show N more* disclosure, and an idle relay reads **idle · N PRs awaiting rework** instead of `idle: no matching work`. | +| `pending_rework_blocked` | Why the pending rework is not on its way to the relay right now, in the dispatcher's own order: relay offline, no `fix_pr_interrupt_v1` capability, repository scope not acknowledged, not in this hub's fix queue, an earlier fix dispatch still booked on a dropped connection, retry backoff, focus window, slots busy, or simply waiting for the relay's next `ready`. Printed in place of the old *not yet dispatched*. Empty when nothing pends. | -`kind: rework` means the item repairs one of the contributor's own PRs: any `fix-pr` task, or a task whose target the stage store has in a repair stage (`070-fixing`, `071-needs-author`, `072-returned-to-contributor`, `074-adopted-by-hive`). Everything else is `new`. `pending_rework` is derived without a GitHub call: the stage-label store (which remembers each PR's author) ∪ the fix queue, minus the fix tasks in flight on any of the contributor's connections, minus any PR the status snapshot shows as merged or closed. A stage record can outlive its PR when the stage sweep never saw it close, so a PR is dropped when its repo's open-PR list is complete (enumerated, not paused, not incomplete, stale or carrying unlisted PRs) and does not include it. Until that listing is complete (for example in the first minutes after a hub restart) a stage record counts only if a stage pass saw its PR within the last 24 hours (the `072` stalled dwell). The stage pass sees every open PR on each sweep, so a freshly returned PR still shows, while a PR a person closed or merged days ago does not. A record the store marks superseded (the hive's copy merged, or the original was closed in favour of its copy) never counts. A non-zero `pending_rework` on an idle row is the signal to look at — typically a relay that did not advertise `fix_pr_interrupt_v1`, a preempt-mode relay inside its `contribute.min_focus_window`, or a contributor who is offline. +`kind: rework` means the item repairs one of the contributor's own PRs: any `fix-pr` task, or a task whose target the stage store has in a repair stage (`070-fixing`, `071-needs-author`, `072-returned-to-contributor`, `074-adopted-by-hive`). Everything else is `new`. `pending_rework` is derived without a GitHub call: the stage-label store (which remembers each PR's author) ∪ the fix queue, minus the fix tasks in flight on any of the contributor's connections, minus any PR the status snapshot shows as merged or closed. A stage record can outlive its PR when the stage sweep never saw it close, so a PR is dropped when its repo's open-PR list is complete (enumerated, not paused, not incomplete, stale or carrying unlisted PRs) and does not include it. Until that listing is complete (for example in the first minutes after a hub restart) a stage record counts only if a stage pass saw its PR within the last 24 hours (the `072` stalled dwell). The stage pass sees every open PR on each sweep, so a freshly returned PR still shows, while a PR a person closed or merged days ago does not. A record the store marks superseded (the hive's copy merged, or the original was closed in favour of its copy) never counts. Where the snapshot cannot vouch for a repo, the forge cache the hive already holds decides when it knows the PR. A PR in `074-adopted-by-hive` is the hive's and never counts. A non-zero `pending_rework` on an idle row is the signal to look at — `pending_rework_blocked` names the cause (typically a relay that did not advertise `fix_pr_interrupt_v1`, a preempt-mode relay inside its `contribute.min_focus_window`, a fix dispatched to a socket that dropped before the relay reported on it, or a contributor who is offline). A queued fix whose PR turns out merged or closed leaves the fix queue instead of being handed to the relay, and a fix booked in flight that no live connection holds is released at the relay's next `ready`. ## Reconnecting without losing in-flight work diff --git a/src/docs/labels-and-control-signals/stage-labels.md b/src/docs/labels-and-control-signals/stage-labels.md index 8348eaf079..fb6ee9b246 100644 --- a/src/docs/labels-and-control-signals/stage-labels.md +++ b/src/docs/labels-and-control-signals/stage-labels.md @@ -1388,7 +1388,8 @@ When a contributor-agent PR goes red the hive does not just relabel it (a human- in-place adoption their push makes the hive yield (above). The contribute page's "PRs awaiting rework" list for a relay shows `adopting (contributor offline)` on a nudged PR whose relay connection is - stale, and `adopted by hive` once the PR is in 074. + stale; once the PR is in 074 it is the hive's and leaves that relay's + list. 5. Backpressure — the per-contributor fix budget. Capacity = the relay's advertised `max_parallel_fixes` (fix sub-sessions beside the primary), clamped by `contribute.max_parallel_fixes` (default 2); an unadvertised diff --git a/src/pkg/dashboard/contribute_fixqueue.go b/src/pkg/dashboard/contribute_fixqueue.go index 65b1f36049..3baa54a3cd 100644 --- a/src/pkg/dashboard/contribute_fixqueue.go +++ b/src/pkg/dashboard/contribute_fixqueue.go @@ -623,6 +623,12 @@ func (h *ContributeWSHub) selectFixTask(c *ContributorConnection) *WSMessage { } user := c.profile.GitHubUsername sched := h.fixQueue() + // A `ready` says nothing is running on this relay. A fix dispatch the + // scheduler still books for this contributor that no live connection + // holds (its socket dropped before the relay reported on it) would + // otherwise occupy the only slot forever: nothing dispatched, new work + // refused with fix_pr_in_flight. Put it back at the head of the queue. + h.releaseOrphanedFixes(user, sched) if !sched.Pending(user) { return nil } @@ -656,13 +662,34 @@ func (h *ContributeWSHub) completeFixTask(c *ContributorConnection, taskID strin delete(c.fixTasksAt, taskID) } c.mu.Unlock() - if !ok || c.profile == nil { + if c.profile == nil { return ok } user := c.profile.GitHubUsername sched := h.fixQueue() + var prs []WSFixPR + if ok { + prs = assign.FixPRs + } else { + // The relay finished a fix it was handed on an earlier socket (the + // 1006 flap): this connection never saw the task_assign, but the + // scheduler still books the dispatch under this contributor. Settle + // it there rather than leaving the slot occupied for good. + for _, d := range sched.InFlight(user) { + if d.ID != taskID { + continue + } + ok = true + for _, it := range d.Items { + prs = append(prs, WSFixPR{Repo: it.Repo, Number: it.Number}) + } + } + if !ok { + return false + } + } if sched.CompleteDispatch(user, taskID) { - for _, pr := range assign.FixPRs { + for _, pr := range prs { h.forgetFixOwner(user, fixqueue.Key(pr.Repo, pr.Number)) } // Refill the freed slot with the next queued fix. @@ -673,6 +700,42 @@ func (h *ContributeWSHub) completeFixTask(c *ContributorConnection, taskID strin return true } +// heldFixTaskIDs returns the fix-pr task IDs every live connection of user +// currently holds. +func (h *ContributeWSHub) heldFixTaskIDs(user string) map[string]bool { + held := map[string]bool{} + h.mu.RLock() + defer h.mu.RUnlock() + for _, c := range h.connections { + c.mu.Lock() + if c.profile != nil && strings.EqualFold(c.profile.GitHubUsername, user) { + for id := range c.fixTasks { + held[id] = true + } + } + c.mu.Unlock() + } + return held +} + +// releaseOrphanedFixes returns to user's queue every dispatch the scheduler +// books in flight that no live connection holds. Called at a task boundary, +// when the relay has just said it is running nothing. +func (h *ContributeWSHub) releaseOrphanedFixes(user string, sched *fixqueue.Scheduler) { + var held map[string]bool + for _, d := range sched.InFlight(user) { + if held == nil { + held = h.heldFixTaskIDs(user) + } + if held[d.ID] { + continue + } + if items := sched.Release(user, d.ID); len(items) > 0 { + h.logger.Warn("[contribute-ws] fix-pr dispatch orphaned by a dropped connection; requeued", "username", user, "task", d.ID, "prs", len(items)) + } + } +} + // failFixTask settles a fix-pr task the relay reported failed: its PRs go // back in the queue with backoff, and a PR that has failed // fixTaskMaxAttempts times leaves the queue for the stage engine's re-nudge diff --git a/src/pkg/dashboard/contribute_fleet.go b/src/pkg/dashboard/contribute_fleet.go index a13f7733ef..88c27c6439 100644 --- a/src/pkg/dashboard/contribute_fleet.go +++ b/src/pkg/dashboard/contribute_fleet.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "sort" + "strings" "time" ) @@ -260,6 +261,23 @@ func (h *ContributeWSHub) FleetSnapshot() FleetSnapshot { Clankers: make([]FleetClanker, 0, len(h.connections)), Work: make([]FleetWorkItem, 0), } + // Fix-pr task IDs held per contributor across all their live sockets, + // gathered before any row lock is taken so no two connections are ever + // locked at once (fleetReworkFor names a dispatch nobody holds). + heldFix := map[string]map[string]bool{} + for _, c := range h.connections { + c.mu.Lock() + if c.profile != nil && len(c.fixTasks) > 0 { + user := strings.ToLower(c.profile.GitHubUsername) + if heldFix[user] == nil { + heldFix[user] = map[string]bool{} + } + for id := range c.fixTasks { + heldFix[user][id] = true + } + } + c.mu.Unlock() + } for _, c := range h.connections { c.mu.Lock() fc := FleetClanker{ @@ -353,7 +371,11 @@ func (h *ContributeWSHub) FleetSnapshot() FleetSnapshot { if task == nil { fc.IdleReason = c.lastIdleReason } - fc.FleetRework = h.fleetReworkFor(c, fc.Stale, openPRs) + var held map[string]bool + if c.profile != nil { + held = heldFix[strings.ToLower(c.profile.GitHubUsername)] + } + fc.FleetRework = h.fleetReworkFor(c, fc.Stale, openPRs, held) c.mu.Unlock() fc.CurrentTask = task fc.PromptPreview = promptPreview diff --git a/src/pkg/dashboard/contribute_fleet_rework.go b/src/pkg/dashboard/contribute_fleet_rework.go index 961edd52de..53a33280c8 100644 --- a/src/pkg/dashboard/contribute_fleet_rework.go +++ b/src/pkg/dashboard/contribute_fleet_rework.go @@ -1,6 +1,7 @@ package dashboard import ( + "fmt" "sort" "strings" "time" @@ -51,8 +52,8 @@ type FleetInFlightItem struct { StartedAt string `json:"started_at,omitempty"` } -// FleetReworkPR is one of the contributor's PRs waiting in stage 072 that has -// not been dispatched to them. +// FleetReworkPR is one of the contributor's OPEN PRs waiting in stage 072 +// that has not been dispatched to them. type FleetReworkPR struct { Repo string `json:"repo"` Number int `json:"number"` @@ -63,16 +64,32 @@ type FleetReworkPR struct { Since string `json:"since,omitempty"` // Cause is the failing check / review ask, when the fix queue knows it. Cause string `json:"cause,omitempty"` - // State explains what the hive is doing about a PR the contributor is - // not working: "adopting (contributor offline)" while the connection is - // stale and the nudge went unanswered; "adopted by hive" once stage 074 - // took it over (hivecommons/hive#12436). + // State is "adopting (contributor offline)" while the connection is + // stale and the nudge went unanswered (hivecommons/hive#12436). A PR + // stage 074 already took over is not the contributor's any more and is + // not listed at all. State string `json:"state,omitempty"` } const ( FleetReworkStateAdopting = "adopting (contributor offline)" - FleetReworkStateAdopted = "adopted by hive" + // FleetReworkStateAdopted names the hand-off in logs and tests; such a + // PR no longer appears on the contributor's card. + FleetReworkStateAdopted = "adopted by hive" +) + +// Why a queued fix is not on its way to the relay (FleetRework.PendingReworkBlocked). +const ( + FleetReworkBlockedOffline = "relay offline: no heartbeat within the presence window; the hive adopts the PR if the nudge stays unanswered" + FleetReworkBlockedNoFixPR = "relay does not accept fix-pr tasks (no fix_pr_interrupt_v1 capability); the PR only gets a GitHub comment" + FleetReworkBlockedScope = "relay's repository scope is not acknowledged (%s); the task boundary serves nothing until it is" + FleetReworkBlockedOrphan = "earlier fix dispatch %s is still booked in flight on a dropped connection; released at the relay's next ready" + FleetReworkBlockedBackoff = "retry backoff until %s after %d failed dispatch(es)" + FleetReworkBlockedFocus = "primary task protected by the focus window until %s; served at the next task boundary" + FleetReworkBlockedSlotsBusy = "all %d fix slot(s) busy" + FleetReworkBlockedAwaitReady = "waiting for the relay's next ready (fixes are served before new work)" + FleetReworkBlockedNotQueued = "not in this hub's fix queue; queued when the stage engine re-nudges" + fleetReworkBlockedIdleReasonFm = "%s; relay last reported idle: %s" ) // FleetRework is the per-contributor rework view spliced onto FleetClanker. @@ -88,14 +105,19 @@ type FleetRework struct { // InFlight lists every item this connection is working: the primary task // plus any fix-pr tasks running beside / instead of it. InFlight []FleetInFlightItem `json:"in_flight,omitempty"` - // FixQueueDepth is how many red PRs wait in the contributor's fix queue. + // FixQueueDepth is how many open red PRs wait in the contributor's fix queue. FixQueueDepth int `json:"fix_queue_depth"` - // PendingRework counts the contributor's PRs in + // PendingRework counts the contributor's OPEN PRs in // stage/072-returned-to-contributor that have NOT been dispatched to them - // (fix queue ∪ stage store, minus in-flight). Non-zero while the relay is - // idle is the operator's tell that rework is not reaching the agent. + // (fix queue ∪ stage store, minus in-flight, minus anything merged, + // closed, adopted by the hive or no longer observed by the stage lane). + // Non-zero while the relay is idle is the operator's tell that rework is + // not reaching the agent. PendingRework int `json:"pending_rework"` PendingReworkPRs []FleetReworkPR `json:"pending_rework_prs,omitempty"` + // PendingReworkBlocked says why the pending rework is not on its way to + // the relay right now (FleetReworkBlocked*); empty when nothing pends. + PendingReworkBlocked string `json:"pending_rework_blocked,omitempty"` } // stageStoreOrShared returns the injected stage store or the process-wide one. @@ -106,6 +128,48 @@ func (h *ContributeWSHub) stageStoreOrShared() *stagelabels.Store { return stagelabels.Shared() } +// prOpenState answers whether repo#number is still open from the forge +// cache the hive already holds (never a GitHub call): known=false when the +// cache has no entry. prOpenFn is the test seam. +func (h *ContributeWSHub) prOpenState(repo string, number int) (open, known bool) { + if h.prOpenFn != nil { + return h.prOpenFn(repo, number) + } + if h.server == nil || h.server.deps == nil || h.server.deps.GHClient == nil { + return false, false + } + fs := h.server.deps.GHClient.ForgeState() + if fs == nil { + return false, false + } + item, ok := fs.PR(repo, number) + if !ok { + return false, false + } + return item.Open(), true +} + +// reworkCandidateOpen reports whether a rework candidate is still an open +// PR of the contributor's. A record stage 074 took over is the hive's now, +// not the contributor's. Where the open-PR index vouches for the repo it +// decides; otherwise the forge cache decides when it knows the PR; otherwise +// owedRework's unseen bound on the stage record. +func (h *ContributeWSHub) reworkCandidateOpen(open openPRIndex, repo string, number int, rec stagelabels.Record, hasRec bool, now time.Time) bool { + if hasRec && rec.Stage == stagelabels.AdoptedByHive { + return false + } + if open.vouches(repo) { + return !open.closed(repo, number) + } + if isOpen, known := h.prOpenState(repo, number); known { + return isOpen + } + if !hasRec { + return true + } + return owedRework(open, repo, number, rec, now) +} + // isReworkStage reports whether a stage means "this PR is being repaired". func isReworkStage(st stagelabels.Stage) bool { switch st { @@ -229,11 +293,14 @@ func owedRework(open openPRIndex, repo string, number int, rec stagelabels.Recor // held; it takes only the fix scheduler's and stage store's own locks. open // drops PRs that have merged or closed: stage records outlive a PR whose // close the stage sweep never observed (hivecommons/hive#14708). Where open -// cannot vouch for a repo, owedRework drops stage records no pass has seen -// for pendingReworkUnseenBound instead. -func (h *ContributeWSHub) fleetReworkFor(c *ContributorConnection, stale bool, open openPRIndex) FleetRework { +// cannot vouch for a repo the forge cache decides, then owedRework drops +// stage records no pass has seen for pendingReworkUnseenBound. +// heldFixIDs are the fix-pr task IDs every live connection of this +// contributor holds (FleetSnapshot gathers them before locking c). +func (h *ContributeWSHub) fleetReworkFor(c *ContributorConnection, stale bool, open openPRIndex, heldFixIDs map[string]bool) FleetRework { var out FleetRework store := h.stageStoreOrShared() + now := time.Now() cap := h.fixCapacity(c) switch { case cap.Parallel: @@ -293,18 +360,24 @@ func (h *ContributeWSHub) fleetReworkFor(c *ContributorConnection, stale bool, o return out } sched := h.fixQueue() - queued := sched.Queued(user) - out.FixQueueDepth = len(queued) // Dispatched on ANY of this contributor's connections counts as in flight. for _, d := range sched.InFlight(user) { for _, it := range d.Items { inFlightKeys[it.Key()] = true } } - now := time.Now() pending := map[string]FleetReworkPR{} - for _, it := range queued { - if inFlightKeys[it.Key()] || open.closed(it.Repo, it.Number) { + for _, it := range sched.Queued(user) { + rec, hasRec := store.Get(stagelabels.Key(it.Repo, it.Number)) + if !h.reworkCandidateOpen(open, it.Repo, it.Number, rec, hasRec, now) { + // Merged, closed or adopted while queued: nothing left for the + // relay to fix, and handing it out would waste the agent's turn. + sched.Remove(user, it.Repo, it.Number) + h.forgetFixOwner(user, it.Key()) + continue + } + out.FixQueueDepth++ + if inFlightKeys[it.Key()] { continue } cause := it.Cause.Check @@ -325,7 +398,7 @@ func (h *ContributeWSHub) fleetReworkFor(c *ContributorConnection, stale bool, o continue } repo, number := splitStageKey(ai.Key) - if repo == "" || number == 0 || !owedRework(open, repo, number, ai.Record, now) { + if repo == "" || number == 0 || !h.reworkCandidateOpen(open, repo, number, ai.Record, true, now) { continue } pr := FleetReworkPR{Repo: repo, Number: number, Stage: ai.Record.Stage.Label()} @@ -337,20 +410,6 @@ func (h *ContributeWSHub) fleetReworkFor(c *ContributorConnection, stale bool, o } pending[ai.Key] = pr } - for _, ai := range store.ByAuthor(user, stagelabels.AdoptedByHive) { - if _, dup := pending[ai.Key]; dup { - continue - } - repo, number := splitStageKey(ai.Key) - if repo == "" || number == 0 || !owedRework(open, repo, number, ai.Record, now) { - continue - } - pr := FleetReworkPR{Repo: repo, Number: number, Stage: ai.Record.Stage.Label(), State: FleetReworkStateAdopted} - if !ai.Record.EnteredAt.IsZero() { - pr.Since = ai.Record.EnteredAt.UTC().Format(time.RFC3339) - } - pending[ai.Key] = pr - } keys := make([]string, 0, len(pending)) for k := range pending { keys = append(keys, k) @@ -360,9 +419,59 @@ func (h *ContributeWSHub) fleetReworkFor(c *ContributorConnection, stale bool, o out.PendingReworkPRs = append(out.PendingReworkPRs, pending[k]) } out.PendingRework = len(out.PendingReworkPRs) + if out.PendingRework > 0 { + out.PendingReworkBlocked = h.fleetReworkBlockReason(c, stale, user, cap, sched, heldFixIDs, now) + } return out } +// fleetReworkBlockReason explains why user's pending rework is not on its +// way to the relay, in the order the dispatcher itself refuses. Call with +// c.mu held. +func (h *ContributeWSHub) fleetReworkBlockReason(c *ContributorConnection, stale bool, user string, cap fixqueue.Capacity, sched *fixqueue.Scheduler, heldFixIDs map[string]bool, now time.Time) string { + reason := func(r string) string { + if c.currentTask == nil && c.lastIdleReason != "" { + return fmt.Sprintf(fleetReworkBlockedIdleReasonFm, r, c.lastIdleReason) + } + return r + } + if stale { + return FleetReworkBlockedOffline + } + if !relaySupportsFixPR(c) { + return FleetReworkBlockedNoFixPR + } + if gate := repoScopeGateReasonLocked(c); gate != "" { + return reason(fmt.Sprintf(FleetReworkBlockedScope, gate)) + } + queued := sched.Queued(user) + if len(queued) == 0 { + return reason(FleetReworkBlockedNotQueued) + } + inflight := sched.InFlight(user) + for _, d := range inflight { + if !heldFixIDs[d.ID] { + return reason(fmt.Sprintf(FleetReworkBlockedOrphan, d.ID)) + } + } + if head := queued[0]; now.Before(head.NotBefore) { + return reason(fmt.Sprintf(FleetReworkBlockedBackoff, head.NotBefore.UTC().Format(time.RFC3339), head.Attempts)) + } + st := sched.Status(user, cap) + if c.currentTask != nil { + if st.FocusUntil != nil && now.Before(*st.FocusUntil) { + return fmt.Sprintf(FleetReworkBlockedFocus, st.FocusUntil.UTC().Format(time.RFC3339)) + } + if len(inflight) >= cap.Slots() { + return fmt.Sprintf(FleetReworkBlockedSlotsBusy, cap.Slots()) + } + } + if len(inflight) >= cap.BoundarySlots() { + return reason(fmt.Sprintf(FleetReworkBlockedSlotsBusy, cap.BoundarySlots())) + } + return reason(FleetReworkBlockedAwaitReady) +} + // splitStageKey inverts stagelabels.Key ("owner/repo#N"). func splitStageKey(key string) (repo string, number int) { i := strings.LastIndex(key, "#") diff --git a/src/pkg/dashboard/contribute_fleet_rework_14708_test.go b/src/pkg/dashboard/contribute_fleet_rework_14708_test.go index e60107d7f3..7290679506 100644 --- a/src/pkg/dashboard/contribute_fleet_rework_14708_test.go +++ b/src/pkg/dashboard/contribute_fleet_rework_14708_test.go @@ -48,22 +48,23 @@ func TestFleetReworkExcludesMergedAndClosedPRs(t *testing.T) { } } -func TestFleetReworkKeepsOpenDraftAndAdoptedPRs(t *testing.T) { +func TestFleetReworkKeepsOpenDraftDropsAdoptedPRs(t *testing.T) { hub, srv, store := reworkHub(t) fixConn(hub, "alice", fixCaps(capFixPRInterrupt), time.Now()) noteRework(store, "org/r", 4, stagelabels.ReturnedToContributor, "alice") noteRework(store, "org/r", 5, stagelabels.AdoptedByHive, "alice") // A draft is not in OpenPrs; the enumeration carries it as a sidelined - // row, which is still the canonical open set. + // row, which is still the canonical open set. #5 is open too, but the + // hive owns a 074 PR: it is no longer this contributor's rework. setReworkStatus(srv, FrontendRepo{Name: "r", Full: "org/r", OpenPrs: []any{FrontendPR{PullRequest: github.PullRequest{Repo: "org/r", Number: 5}}}, NonActionablePrs: []any{github.RepoWorkPR{Repo: "org/r", Number: 4, Draft: true, Bucket: "draft"}}, }) fc := snapshotFor(t, hub, "alice") - if got := pendingNumbers(fc); fc.PendingRework != 2 || len(got) != 2 || got[0] != 4 || got[1] != 5 { - t.Fatalf("pending_rework=%d prs=%v, want open #4 (draft) and #5 (074)", fc.PendingRework, got) + if got := pendingNumbers(fc); fc.PendingRework != 1 || len(got) != 1 || got[0] != 4 { + t.Fatalf("pending_rework=%d prs=%v, want only the open draft #4; #5 (074) is the hive's", fc.PendingRework, got) } } diff --git a/src/pkg/dashboard/contribute_fleet_rework_restart_test.go b/src/pkg/dashboard/contribute_fleet_rework_restart_test.go index 3f4e5282bf..b0a4fe9f58 100644 --- a/src/pkg/dashboard/contribute_fleet_rework_restart_test.go +++ b/src/pkg/dashboard/contribute_fleet_rework_restart_test.go @@ -76,7 +76,7 @@ func TestFleetReworkAfterRestartKeepsFreshlyReturnedPR(t *testing.T) { noteRework(s, "org/r", 5, stagelabels.ReturnedToContributor, "alice") // Seen just inside the bound: an outage shorter than a day keeps it. at(now.Add(-pendingReworkUnseenBound + time.Hour)) - noteRework(s, "org/r", 6, stagelabels.AdoptedByHive, "alice") + noteRework(s, "org/r", 6, stagelabels.ReturnedToContributor, "alice") }) hub, srv, _ := reworkHub(t) hub.stageStore = store @@ -85,7 +85,7 @@ func TestFleetReworkAfterRestartKeepsFreshlyReturnedPR(t *testing.T) { fc := snapshotFor(t, hub, "alice") if got := pendingNumbers(fc); fc.PendingRework != 2 || len(got) != 2 || got[0] != 5 || got[1] != 6 { - t.Fatalf("pending_rework=%d prs=%v, want #5 (072) and #6 (074): both seen within %s", fc.PendingRework, got, pendingReworkUnseenBound) + t.Fatalf("pending_rework=%d prs=%v, want #5 and #6 (072): both seen within %s", fc.PendingRework, got, pendingReworkUnseenBound) } if fc.PendingReworkPRs[0].Stage != stagelabels.ReturnedToContributor.Label() { t.Fatalf("stage = %q, want 072", fc.PendingReworkPRs[0].Stage) @@ -132,13 +132,17 @@ func TestFleetReworkSkipsRecordsTheStoreMarksFinished(t *testing.T) { // A lane reported the PR closed or merged: the stage pass forgets it. noteRework(store, "org/r", 3, stagelabels.ReturnedToContributor, "alice") store.Forget(stagelabels.Key("org/r", 3)) - // A reopen for a failed copy clears the supersession: owed again. + // A reopen for a failed copy clears the supersession and the next stage + // pass returns the PR to its author: owed again. noteRework(store, "org/r", 4, stagelabels.AdoptedByHive, "alice") store.MarkSuperseded(stagelabels.Key("org/r", 4), 14, false, now) store.MarkAdoptionReopened(stagelabels.Key("org/r", 4), 14, now) + noteRework(store, "org/r", 4, stagelabels.ReturnedToContributor, "alice") - if got := store.ByAuthor("alice", stagelabels.ReturnedToContributor); len(got) != 0 { - t.Fatalf("ByAuthor returned a forgotten record: %+v", got) + for _, ai := range store.ByAuthor("alice", stagelabels.ReturnedToContributor) { + if _, n := splitStageKey(ai.Key); n == 3 { + t.Fatalf("ByAuthor returned a forgotten record: %+v", ai) + } } fc := snapshotFor(t, hub, "alice") if got := pendingNumbers(fc); fc.PendingRework != 1 || len(got) != 1 || got[0] != 4 { diff --git a/src/pkg/dashboard/contribute_fleet_rework_test.go b/src/pkg/dashboard/contribute_fleet_rework_test.go index 5ccf9572f7..bf7c24a7b4 100644 --- a/src/pkg/dashboard/contribute_fleet_rework_test.go +++ b/src/pkg/dashboard/contribute_fleet_rework_test.go @@ -3,6 +3,7 @@ package dashboard import ( "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "os/exec" @@ -266,6 +267,7 @@ func TestOpsClankerRowRendersCapacityAndRework(t *testing.T) { ".work-kind.new{", ".work-kind.rework{", ".clanker-idle-rework,.clanker-rework-pending{color:var(--status-error)", + ".clanker-rework-more>summary{", } { if !strings.Contains(body, want) { t.Errorf("operations page missing %q", want) @@ -273,7 +275,7 @@ func TestOpsClankerRowRendersCapacityAndRework(t *testing.T) { } script := ` const assert = require('node:assert/strict'); -var ccDeferredClankers = null, ccKnownClankers = {}, adminEnabled = false; +var ccDeferredClankers = null, ccKnownClankers = {}, adminEnabled = false, ccOpenPanes = {}, CLANKER_REWORK_INLINE_MAX = 3; const fleet = { querySelector() { return null; }, set innerHTML(v) { this.html = v; } }; const document = { getElementById(id) { return id === 'clanker-list' ? fleet : {}; } }; function ccUpdateArmy() {} @@ -330,6 +332,29 @@ assert.doesNotMatch(html, /idle: no matching work/, 'idle reason replaced while assert.match(html, /class="clanker-sub clanker-rework-pending" title="org\/r#2 \(build\), org\/r#3">2 PRs awaiting rework \(not yet dispatched\) · fix queue 1: org\/r#2<\/a>, org\/r#3<\/a>earlier fix dispatch fix-erin-1 is still booked[^<]*<\/span>\) · fix queue 1: org\/r#9<\/a>/, 'block reason'); +assert.doesNotMatch(html, /not yet dispatched/, 'generic wording gone when the hub explains'); + +// Five pending: three inline, the other two behind a closed disclosure. +var five = [1,2,3,4,5].map(function(n){return {repo: 'org/r', number: n, url: 'https://x/' + n};}); +renderClankers([{github_username: 'frank', fix_queue_depth: 5, pending_rework: 5, pending_rework_prs: five}]); +html = fleet.html; +assert.equal((html.match(/class="cc-issue-link work-ref"/g) || []).length, 5, 'every PR is still linked'); +var moreAt = html.indexOf('
show 2 more'); +assert.ok(moreAt > 0, 'disclosure present and closed'); +assert.ok(html.indexOf('org/r#3') < moreAt && html.indexOf('org/r#4') > moreAt, 'first three inline, rest behind the summary'); +ccOpenPanes['frank:rework'] = true; +renderClankers([{github_username: 'frank', fix_queue_depth: 5, pending_rework: 5, pending_rework_prs: five}]); +assert.match(fleet.html, /clanker-rework-more" data-pane="frank:rework" open>/, 'disclosure stays open across re-renders'); +delete ccOpenPanes['frank:rework']; +renderClankers([{github_username: 'gina', fix_queue_depth: 3, pending_rework: 3, pending_rework_prs: five.slice(0,3)}]); +assert.doesNotMatch(fleet.html, /clanker-rework-more/, 'three or fewer: no disclosure'); + // Idle, nothing pending: the pre-#12502 idle reason line is unchanged. renderClankers([{github_username: 'carol', idle_reason: 'no_matching_work', pending_rework: 0}]); html = fleet.html; @@ -362,14 +387,15 @@ func TestFleetReworkAdoptionState(t *testing.T) { store.NoteAuthor(stagelabels.Key("org/r", 3), "alice") fc := snapshotFor(t, hub, "alice") - if fc.Stale || len(fc.PendingReworkPRs) != 3 { + // The 074 PR is the hive's now: not listed, not counted. + if fc.Stale || fc.PendingRework != 2 || len(fc.PendingReworkPRs) != 2 { t.Fatalf("live: %+v", fc.FleetRework) } byNumber := map[int]FleetReworkPR{} for _, pr := range fc.PendingReworkPRs { byNumber[pr.Number] = pr } - if byNumber[1].State != "" || byNumber[2].State != "" || byNumber[3].State != FleetReworkStateAdopted || byNumber[3].Stage != stagelabels.AdoptedByHive.Label() { + if _, adopted := byNumber[3]; adopted || byNumber[1].State != "" || byNumber[2].State != "" { t.Fatalf("live states: %+v", fc.PendingReworkPRs) } @@ -384,10 +410,13 @@ func TestFleetReworkAdoptionState(t *testing.T) { for _, pr := range fc.PendingReworkPRs { byNumber[pr.Number] = pr } - // Nudged and unanswered → adopting; never nudged → plain 072; 074 → adopted. - if byNumber[1].State != FleetReworkStateAdopting || byNumber[2].State != "" || byNumber[3].State != FleetReworkStateAdopted { + // Nudged and unanswered → adopting; never nudged → plain 072; 074 → gone. + if _, adopted := byNumber[3]; adopted || byNumber[1].State != FleetReworkStateAdopting || byNumber[2].State != "" { t.Fatalf("stale states: %+v", fc.PendingReworkPRs) } + if fc.PendingReworkBlocked != FleetReworkBlockedOffline { + t.Fatalf("offline relay must say so: %q", fc.PendingReworkBlocked) + } raw, _ := json.Marshal(fc.PendingReworkPRs) if !strings.Contains(string(raw), `"state":"adopting (contributor offline)"`) { t.Fatalf("json: %s", raw) @@ -419,3 +448,154 @@ func TestContributeHubPresence(t *testing.T) { t.Fatalf("offline relay: %+v", p) } } + +// Only the contributor's OPEN PRs are their rework: a PR the forge cache +// reports merged or closed, or a stage-store record the lane stopped +// observing, is dropped from the count and (when queued) from the fix +// queue, so the relay is never handed a fix for a PR that is gone. +func TestFleetReworkDropsMergedClosedAndUnobserved(t *testing.T) { + hub, _, store := reworkHub(t) + now := time.Now() + fixConn(hub, "alice", fixCaps(capFixPRInterrupt), now) + + closed := map[string]bool{"org/r#2": true, "org/r#3": true} + hub.prOpenFn = func(repo string, number int) (bool, bool) { + k := fixqueue.Key(repo, number) + if closed[k] { + return false, true + } + if k == "org/r#1" { + return true, true + } + return false, false + } + for _, n := range []int{1, 2, 3, 4} { + store.Observe(stagelabels.Key("org/r", n), stagelabels.ReturnedToContributor, "check-failing:build") + store.NoteAuthor(stagelabels.Key("org/r", n), "alice") + } + // Queue #2 (closed) and #4 (unknown to the cache, fresh in the store). + hub.fixQueue().Enqueue("alice", fixqueue.Item{Repo: "org/r", Number: 2, Title: "closed"}) + hub.fixQueue().Enqueue("alice", fixqueue.Item{Repo: "org/r", Number: 4, Title: "open"}) + hub.rememberFixOwner("alice", "org/r#2") + hub.rememberFixOwner("alice", "org/r#4") + + fc := snapshotFor(t, hub, "alice") + got := map[int]bool{} + for _, pr := range fc.PendingReworkPRs { + got[pr.Number] = true + } + if fc.PendingRework != 2 || !got[1] || !got[4] || got[2] || got[3] { + t.Fatalf("pending: %+v", fc.FleetRework) + } + if fc.FixQueueDepth != 1 { + t.Fatalf("closed PR must leave the fix queue: %+v", fc.FleetRework) + } + if q := hub.fixQueue().Queued("alice"); len(q) != 1 || q[0].Number != 4 { + t.Fatalf("scheduler still holds the closed PR: %+v", q) + } + if fc.PendingReworkBlocked != FleetReworkBlockedAwaitReady { + t.Fatalf("idle relay with a queued fix waits for ready: %q", fc.PendingReworkBlocked) + } +} + +// A stage-store record the lane has not re-observed within +// pendingReworkUnseenBound describes a PR that left the open listing. +func TestFleetReworkDropsRecordTheLaneStoppedObserving(t *testing.T) { + hub, _, _ := reworkHub(t) + now := time.Now() + fixConn(hub, "alice", fixCaps(capFixPRInterrupt), now) + clock := now.Add(-2 * pendingReworkUnseenBound) + store := stagelabels.NewStore("", func() time.Time { return clock }) + hub.stageStore = store + store.Observe(stagelabels.Key("org/r", 1), stagelabels.ReturnedToContributor, "check-failing:build") + store.NoteAuthor(stagelabels.Key("org/r", 1), "alice") + clock = now + store.Observe(stagelabels.Key("org/r", 2), stagelabels.ReturnedToContributor, "check-failing:build") + store.NoteAuthor(stagelabels.Key("org/r", 2), "alice") + + fc := snapshotFor(t, hub, "alice") + if fc.PendingRework != 1 || fc.PendingReworkPRs[0].Number != 2 { + t.Fatalf("unobserved #1 must be dropped: %+v", fc.FleetRework) + } + // With no fix queue entry the hub can only wait for the engine's re-nudge. + if fc.PendingReworkBlocked != FleetReworkBlockedNotQueued { + t.Fatalf("block reason: %q", fc.PendingReworkBlocked) + } +} + +// A fix dispatched to a socket that then dropped stays booked in the +// scheduler with nobody holding it: capacity 1 is "busy" for good, so the +// relay is served nothing — not the fix, not new work. The snapshot names +// the orphan, and the relay's next `ready` releases and re-dispatches it. +func TestFleetReworkOrphanedDispatchIsNamedAndReleasedAtReady(t *testing.T) { + hub, _, store := reworkHub(t) + got := captureFixDispatch(hub) + now := time.Now() + c := fixConn(hub, "alice", fixCaps(capFixPRInterrupt), now) + store.Observe(stagelabels.Key("org/r", 1), stagelabels.ReturnedToContributor, "check-failing:build") + store.NoteAuthor(stagelabels.Key("org/r", 1), "alice") + hub.Nudge(context.Background(), fixNudge("r", 1, "build")) + if len(*got) != 1 { + t.Fatalf("preempt dispatch expected, got %d", len(*got)) + } + orphanID := (*got)[0].TaskID + + // The socket that took the fix dies; the relay comes back on a new one. + hub.mu.Lock() + delete(hub.connections, c.connID) + hub.mu.Unlock() + c2 := fixConn(hub, "alice", fixCaps(capFixPRInterrupt), now.Add(time.Second)) + c2.lastIdleReason = taskUnavailableFixInFlight + + // Another red PR queues behind the orphan. + hub.Nudge(context.Background(), fixNudge("r", 2, "lint")) + fc := snapshotFor(t, hub, "alice") + if fc.PendingRework != 1 || fc.PendingReworkPRs[0].Number != 2 || len(fc.InFlight) != 0 { + t.Fatalf("pending: %+v", fc.FleetRework) + } + want := fmt.Sprintf(FleetReworkBlockedOrphan, orphanID) + "; relay last reported idle: " + taskUnavailableFixInFlight + if fc.PendingReworkBlocked != want { + t.Fatalf("block reason:\n got %q\nwant %q", fc.PendingReworkBlocked, want) + } + + // `ready` on the new socket: the orphan is released (no attempt counted) + // and the queue head goes out as a boundary fix. + msg := hub.selectFixTask(c2) + if msg == nil || msg.Kind != taskKindFixPR || msg.Mode != "boundary" { + t.Fatalf("boundary fix expected after release: %+v", msg) + } + if n := len(hub.fixQueue().InFlight("alice")); n != 1 { + t.Fatalf("exactly the new dispatch in flight, got %d", n) + } + for _, it := range hub.fixQueue().Queued("alice") { + if it.Attempts != 0 { + t.Fatalf("release must not count an attempt: %+v", it) + } + } +} + +// A relay that finished a fix handed out on an earlier socket reports +// task_complete on the new one; the scheduler settles it all the same. +func TestFleetReworkCompleteFixOnReconnectedSocket(t *testing.T) { + hub, _, _ := reworkHub(t) + got := captureFixDispatch(hub) + now := time.Now() + c := fixConn(hub, "alice", fixCaps(capFixPRInterrupt), now) + hub.Nudge(context.Background(), fixNudge("r", 1, "build")) + if len(*got) != 1 { + t.Fatalf("preempt dispatch expected, got %d", len(*got)) + } + hub.mu.Lock() + delete(hub.connections, c.connID) + hub.mu.Unlock() + c2 := fixConn(hub, "alice", fixCaps(capFixPRInterrupt), now.Add(time.Second)) + if !hub.completeFixTask(c2, (*got)[0].TaskID) { + t.Fatal("completion on the new socket must settle the scheduler's dispatch") + } + if st := hub.FixQueueStatus("alice"); st.InFlight != 0 || st.Queued != 0 { + t.Fatalf("dispatch should be settled: %+v", st) + } + if hub.completeFixTask(c2, "fix-alice-999") { + t.Fatal("an unknown task id is not a fix task") + } +} diff --git a/src/pkg/dashboard/contribute_landing.go b/src/pkg/dashboard/contribute_landing.go index 16132dde52..5bde30c5f5 100644 --- a/src/pkg/dashboard/contribute_landing.go +++ b/src/pkg/dashboard/contribute_landing.go @@ -472,6 +472,11 @@ code{background:var(--surface-0);padding:var(--sp-1) var(--sp-4);border-radius:v .work-kind.rework{background:color-mix(in srgb,var(--status-attention) var(--component-tint),transparent);color:var(--status-attention);border-color:color-mix(in srgb,var(--status-attention) var(--component-border),transparent)} .clanker-idle-rework,.clanker-rework-pending{color:var(--status-error);font-weight:600} .clanker-rework-pending .cc-issue-link{color:var(--status-error)} +.clanker-rework-pending .clanker-rework-why{font-weight:400;color:var(--text-muted)} +.clanker-rework-more{display:inline;font-weight:400} +.clanker-rework-more>summary{display:inline;cursor:pointer;color:var(--text-faint);user-select:none;list-style:none} +.clanker-rework-more>summary::-webkit-details-marker{display:none} +.clanker-rework-more[open]>summary{display:none} .clanker-pane{margin-top:var(--sp-3);font-size:.72rem} .clanker-pane>summary{cursor:pointer;color:var(--text-faint);font-family:ui-monospace,SFMono-Regular,Menlo,monospace;user-select:none;list-style:none} .clanker-pane>summary::before{content:"\25B8";display:inline-block;width:1em;color:var(--text-muted)} @@ -5538,14 +5543,29 @@ function clankerIdleLine(c){ if(pend>0)return '
idle · '+pend+' PR'+(pend===1?'':'s')+' awaiting rework
'; return c.idle_reason?('
idle: '+esc(idleReasonLabel(c.idle_reason))+'
'):''; } +// At most this many pending PRs are linked inline; the rest sit behind a +// "show N more" disclosure (the card is a status line, not a backlog). +var CLANKER_REWORK_INLINE_MAX=3; function clankerPendingReworkLine(c){ var pend=c.pending_rework|0; if(pend<=0)return ''; - var prs=(c.pending_rework_prs||[]).map(function(p){return (p.repo||'')+'#'+(p.number||'')+(p.cause?(' ('+p.cause+')'):'')+(p.state?(' — '+p.state):'');}).join(', '); - var links=(c.pending_rework_prs||[]).map(function(p){return clankerWorkRef(p)+(p.state?(' '+esc(p.state)+''):'');}).join(', '); + var items=c.pending_rework_prs||[]; + var prs=items.map(function(p){return (p.repo||'')+'#'+(p.number||'')+(p.cause?(' ('+p.cause+')'):'')+(p.state?(' — '+p.state):'');}).join(', '); + var refs=items.map(function(p){return clankerWorkRef(p)+(p.state?(' '+esc(p.state)+''):'');}); + var links=refs.slice(0,CLANKER_REWORK_INLINE_MAX).join(', '); + var rest=refs.slice(CLANKER_REWORK_INLINE_MAX); + if(rest.length){ + var key=(c.github_username||c.contributor_id||'').toLowerCase()+':rework'; + var open=ccOpenPanes[key]?' open':''; + links+=',
show '+rest.length+' more'+rest.join(', ')+'
'; + } var depth=c.fix_queue_depth|0; var q=depth>0?(' · fix queue '+depth):''; - return '
'+pend+' PR'+(pend===1?'':'s')+' awaiting rework (not yet dispatched)'+q+(links?(': '+links):'')+'
'; + // The server's own account of why the dispatcher is holding the fix + // (offline, no fix-pr capability, orphaned dispatch, backoff, focus + // window, waiting for the relay's next ready); older hubs send none. + var why=c.pending_rework_blocked?('not dispatched: '+esc(c.pending_rework_blocked)+''):'not yet dispatched'; + return '
'+pend+' PR'+(pend===1?'':'s')+' awaiting rework ('+why+')'+q+(links?(': '+links):'')+'
'; } // #7317 item 3 helpers. Both return '' when there is nothing to show so a // healthy row is byte-for-byte what it was before. @@ -5566,7 +5586,7 @@ function clankerPaneBlock(lines,key){ return '
agent pane · '+lines.length+' line'+(lines.length===1?'':'s')+'
'+esc(lines.join('\n'))+'
'; } onEl('clanker-list','toggle',function(e){ - var d=e.target;if(!d||!d.classList||!d.classList.contains('clanker-pane'))return; + var d=e.target;if(!d||!d.classList||!(d.classList.contains('clanker-pane')||d.classList.contains('clanker-rework-more')))return; var k=d.getAttribute('data-pane');if(!k)return; if(d.open)ccOpenPanes[k]=true;else delete ccOpenPanes[k]; },true); diff --git a/src/pkg/dashboard/contribute_ws.go b/src/pkg/dashboard/contribute_ws.go index afd56e729e..a6a2252be1 100644 --- a/src/pkg/dashboard/contribute_ws.go +++ b/src/pkg/dashboard/contribute_ws.go @@ -1004,6 +1004,9 @@ type ContributeWSHub struct { fixSched *fixqueue.Scheduler fixOwners map[string]map[string]bool fixDispatched func(*ContributorConnection, WSMessage) + // prOpenFn overrides the forge-cache lookup the fleet snapshot uses to + // drop merged/closed PRs from a contributor's rework; a test hook. + prOpenFn func(repo string, number int) (open, known bool) // fixFailedSig is closed (and replaced) whenever a relay reports a // fix-pr task failed, waking a Nudge that is waiting to learn whether its // dispatch was refused (hivecommons/hive#14668). fixAckGrace overrides @@ -4132,6 +4135,12 @@ func repoScopeGateReason(c *ContributorConnection) string { } c.mu.Lock() defer c.mu.Unlock() + return repoScopeGateReasonLocked(c) +} + +// repoScopeGateReasonLocked is repoScopeGateReason for a caller that already +// holds c.mu (the fleet snapshot explains an undispatched fix with it). +func repoScopeGateReasonLocked(c *ContributorConnection) string { if c.repoScopeInvalid { return taskUnavailableRepoScopeInvalid } diff --git a/src/pkg/fixqueue/fixqueue.go b/src/pkg/fixqueue/fixqueue.go index fad66ce801..8b6c14df4a 100644 --- a/src/pkg/fixqueue/fixqueue.go +++ b/src/pkg/fixqueue/fixqueue.go @@ -362,7 +362,7 @@ func (s *Scheduler) CompleteDispatch(user, id string) bool { // straight back to a relay that just refused it. It returns the requeued // items; nil when id is not in flight. func (s *Scheduler) Requeue(user, id string, backoff time.Duration) []Item { - return s.requeue(user, id, backoff, true) + return s.unwind(user, id, backoff, true, true, "requeue") } // Return is Requeue for a dispatch the relay never started because it was @@ -370,10 +370,19 @@ func (s *Scheduler) Requeue(user, id string, backoff time.Duration) []Item { // with the same backoff but Attempts unchanged, so a refusal that says // nothing about the fix never moves a PR toward the relay hand-off limit. func (s *Scheduler) Return(user, id string, backoff time.Duration) []Item { - return s.requeue(user, id, backoff, false) + return s.unwind(user, id, backoff, false, true, "return") } -func (s *Scheduler) requeue(user, id string, backoff time.Duration, attempted bool) []Item { +// Release puts an in-flight dispatch back at the head of user's queue +// without counting an attempt or booking a backoff: the relay never reported +// on it (its socket dropped, or it asked for new work while the dispatch was +// still booked), so nothing about the PR itself failed. nil when id is not +// in flight. +func (s *Scheduler) Release(user, id string) []Item { + return s.unwind(user, id, 0, false, false, "release") +} + +func (s *Scheduler) unwind(user, id string, backoff time.Duration, countAttempt, backOff bool, event string) []Item { s.mu.Lock() defer s.mu.Unlock() c, ok := s.by[norm(user)] @@ -392,14 +401,16 @@ func (s *Scheduler) requeue(user, id string, backoff time.Duration, attempted bo out := d.public().Items notBefore := s.opts.Now().Add(backoff) for j := range out { - if attempted { + if countAttempt { out[j].Attempts++ } - out[j].NotBefore = notBefore + if backOff { + out[j].NotBefore = notBefore + } c.queue = append(c.queue, out[j]) } sortQueue(c.queue) - s.log("requeue", user, Dispatch{ID: d.id, Mode: d.mode, Items: out}, len(c.queue)) + s.log(event, user, Dispatch{ID: d.id, Mode: d.mode, Items: out}, len(c.queue)) return out } return nil diff --git a/src/pkg/fixqueue/fixqueue_test.go b/src/pkg/fixqueue/fixqueue_test.go index 6746a2e326..313e121006 100644 --- a/src/pkg/fixqueue/fixqueue_test.go +++ b/src/pkg/fixqueue/fixqueue_test.go @@ -490,3 +490,28 @@ func TestNextPreviewsDispatchWithoutTakingIt(t *testing.T) { t.Fatalf("boundary with a free slot: Next = %v, want [3]", keys(got)) } } + +// Release puts an in-flight dispatch back without an attempt or a backoff: +// the relay never reported on it, so the PR is immediately dispatchable. +func TestReleaseReturnsDispatchWithoutAttempt(t *testing.T) { + c := newClock() + s := newSched(c) + s.Enqueue("ida", red(1, time.Hour, c)) + d := s.Dispatch("ida", Capacity{Max: 1}) + if d == nil { + t.Fatal("dispatch expected") + } + if s.Release("ida", "nope") != nil || s.Release("zed", d.ID) != nil { + t.Fatal("unknown dispatch released") + } + got := s.Release("ida", d.ID) + if len(got) != 1 || got[0].Attempts != 0 || !got[0].NotBefore.IsZero() { + t.Fatalf("released = %+v", got) + } + if st := s.Status("ida", Capacity{Max: 1}); st.Queued != 1 || st.InFlight != 0 || st.FocusUntil != nil { + t.Fatalf("status after release = %+v", st) + } + if s.Dispatch("ida", Capacity{Max: 1}) == nil { + t.Fatal("released item must be dispatchable at once") + } +}