diff --git a/changelog.d/fixed-fleet-rework-closed-after-restart.md b/changelog.d/fixed-fleet-rework-closed-after-restart.md new file mode 100644 index 0000000000..49e2e3fb95 --- /dev/null +++ b/changelog.d/fixed-fleet-rework-closed-after-restart.md @@ -0,0 +1 @@ +- The contributor card on Operations no longer lists closed or merged PRs as "N PRs awaiting rework (not yet dispatched)" right after a hub restart. Stage records stay in the store for 14 days after a PR leaves the listings, and until a repo's open-PR listing was complete again the card counted every one of them as owed rework: on projectbluefin, after the 2026-11-09 upgrade, one contributor's card went from 1 to 7 PRs, all closed or merged by a maintainer 9-13 days earlier, and another's showed 17 with 14 closed. While a repo's listing cannot vouch, the card now counts a 072/074 record only if a stage pass saw the PR within the 072 stalled dwell (24h), so a freshly returned PR still shows; a record the store marks superseded never counts, and a complete listing still decides as before (hivecommons/hive#14708). Display only: nothing was dispatched. diff --git a/src/docs/contributor-relay.md b/src/docs/contributor-relay.md index f6ed1fa2a2..f1f87c3806 100644 --- a/src/docs/contributor-relay.md +++ b/src/docs/contributor-relay.md @@ -1777,7 +1777,7 @@ Each row of the "Connected contributor agents (ClankeR)" list on `/contribute/op | `fix_queue_depth` | Red PRs waiting in the contributor's server-side fix queue (not dispatched). | | `pending_rework`, `pending_rework_prs[]` | The contributor's open PRs in `stage/072-returned-to-contributor` that have **not** been dispatched to them. The row prints a red **N PRs awaiting rework (not yet dispatched)** line, and an idle relay reads **idle · N PRs awaiting rework** instead of `idle: no matching work`. | -`kind: rework` means the item repairs one of the contributor's own PRs: any `fix-pr` task, or a task whose target the stage store has in a repair stage (`070-fixing`, `071-needs-author`, `072-returned-to-contributor`, `074-adopted-by-hive`). Everything else is `new`. `pending_rework` is derived without a GitHub call: the stage-label store (which remembers each PR's author) ∪ the fix queue, minus the fix tasks in flight on any of the contributor's connections, minus any PR the status snapshot shows as merged or closed. A stage record can outlive its PR when the stage sweep never saw it close, so a PR is dropped when its repo's open-PR list is complete (enumerated, not paused, not incomplete, stale or carrying unlisted PRs) and does not include it; with no complete listing the record is kept. A non-zero `pending_rework` on an idle row is the signal to look at — typically a relay that did not advertise `fix_pr_interrupt_v1`, a preempt-mode relay inside its `contribute.min_focus_window`, or a contributor who is offline. +`kind: rework` means the item repairs one of the contributor's own PRs: any `fix-pr` task, or a task whose target the stage store has in a repair stage (`070-fixing`, `071-needs-author`, `072-returned-to-contributor`, `074-adopted-by-hive`). Everything else is `new`. `pending_rework` is derived without a GitHub call: the stage-label store (which remembers each PR's author) ∪ the fix queue, minus the fix tasks in flight on any of the contributor's connections, minus any PR the status snapshot shows as merged or closed. A stage record can outlive its PR when the stage sweep never saw it close, so a PR is dropped when its repo's open-PR list is complete (enumerated, not paused, not incomplete, stale or carrying unlisted PRs) and does not include it. Until that listing is complete (for example in the first minutes after a hub restart) a stage record counts only if a stage pass saw its PR within the last 24 hours (the `072` stalled dwell). The stage pass sees every open PR on each sweep, so a freshly returned PR still shows, while a PR a person closed or merged days ago does not. A record the store marks superseded (the hive's copy merged, or the original was closed in favour of its copy) never counts. A non-zero `pending_rework` on an idle row is the signal to look at — typically a relay that did not advertise `fix_pr_interrupt_v1`, a preempt-mode relay inside its `contribute.min_focus_window`, or a contributor who is offline. ## Reconnecting without losing in-flight work diff --git a/src/pkg/dashboard/contribute_fleet_rework.go b/src/pkg/dashboard/contribute_fleet_rework.go index aed0851250..961edd52de 100644 --- a/src/pkg/dashboard/contribute_fleet_rework.go +++ b/src/pkg/dashboard/contribute_fleet_rework.go @@ -179,20 +179,58 @@ func (h *ContributeWSHub) openPRIndex() openPRIndex { return idx } +// vouches reports whether the snapshot's PR list for repo is complete, so +// that a PR missing from it has left (closed answers for the repo). +func (idx openPRIndex) vouches(repo string) bool { + return idx.repos[strings.ToLower(strings.TrimSpace(repo))] +} + // closed reports whether the snapshot shows repo#number as no longer open: // the repo's PR list is complete and does not carry it // (hivecommons/hive#14708). func (idx openPRIndex) closed(repo string, number int) bool { - if !idx.repos[strings.ToLower(strings.TrimSpace(repo))] { + if !idx.vouches(repo) { return false } return !idx.prs[stagelabels.Key(repo, number)] } +// pendingReworkUnseenBound is how long a 072/074 stage record may go +// without a stage pass observing its PR and still count as owed rework, +// for a repo the open-PR index cannot vouch for (right after a hub +// restart, before the repo listings complete). The stage pass re-observes +// every listed open PR each sweep and persists LastSeen at least every +// stagelabels.LastSeenPersistEvery, so an open PR's record is minutes old +// even after a restart. A PR a person closed or merged leaves the listing +// and its record stops moving, but Prune keeps it for 14 days; without +// this bound every such record read as owed rework until the listings +// completed. The bound is stage 072's own stalled dwell (24h): a record +// no pass has touched for a whole 072 dwell is not evidence of an open PR, +// and an outage shorter than a day keeps an open PR on the card. +var pendingReworkUnseenBound = stagelabels.ReturnedToContributor.Dwell() + +// owedRework reports whether a 072/074 stage record for repo#number still +// counts as rework the contributor owes. A record the store marks +// superseded is finished work (the stage tracker drops it the same way, +// hivecommons/hive#14471). Where the open-PR index vouches for the repo it +// decides (hivecommons/hive#14708); elsewhere a record no stage pass has +// seen for pendingReworkUnseenBound is left out. +func owedRework(open openPRIndex, repo string, number int, rec stagelabels.Record, now time.Time) bool { + if rec.Superseded() { + return false + } + if open.vouches(repo) { + return !open.closed(repo, number) + } + return rec.LastSeen.IsZero() || now.Sub(rec.LastSeen) <= pendingReworkUnseenBound +} + // fleetReworkFor builds the rework view for one connection. Call with c.mu // held; it takes only the fix scheduler's and stage store's own locks. open // drops PRs that have merged or closed: stage records outlive a PR whose -// close the stage sweep never observed (hivecommons/hive#14708). +// close the stage sweep never observed (hivecommons/hive#14708). Where open +// cannot vouch for a repo, owedRework drops stage records no pass has seen +// for pendingReworkUnseenBound instead. func (h *ContributeWSHub) fleetReworkFor(c *ContributorConnection, stale bool, open openPRIndex) FleetRework { var out FleetRework store := h.stageStoreOrShared() @@ -263,6 +301,7 @@ func (h *ContributeWSHub) fleetReworkFor(c *ContributorConnection, stale bool, o inFlightKeys[it.Key()] = true } } + now := time.Now() pending := map[string]FleetReworkPR{} for _, it := range queued { if inFlightKeys[it.Key()] || open.closed(it.Repo, it.Number) { @@ -286,7 +325,7 @@ func (h *ContributeWSHub) fleetReworkFor(c *ContributorConnection, stale bool, o continue } repo, number := splitStageKey(ai.Key) - if repo == "" || number == 0 || open.closed(repo, number) { + if repo == "" || number == 0 || !owedRework(open, repo, number, ai.Record, now) { continue } pr := FleetReworkPR{Repo: repo, Number: number, Stage: ai.Record.Stage.Label()} @@ -303,7 +342,7 @@ func (h *ContributeWSHub) fleetReworkFor(c *ContributorConnection, stale bool, o continue } repo, number := splitStageKey(ai.Key) - if repo == "" || number == 0 || open.closed(repo, number) { + if repo == "" || number == 0 || !owedRework(open, repo, number, ai.Record, now) { continue } pr := FleetReworkPR{Repo: repo, Number: number, Stage: ai.Record.Stage.Label(), State: FleetReworkStateAdopted} diff --git a/src/pkg/dashboard/contribute_fleet_rework_restart_test.go b/src/pkg/dashboard/contribute_fleet_rework_restart_test.go new file mode 100644 index 0000000000..3f4e5282bf --- /dev/null +++ b/src/pkg/dashboard/contribute_fleet_rework_restart_test.go @@ -0,0 +1,167 @@ +package dashboard + +import ( + "path/filepath" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/stagelabels" +) + +// After a hub restart the status snapshot cannot vouch for any repo until +// its listing completes, so #14708's open-PR check kept every stage record. +// On projectbluefin (2026-11-09) a contributor card then listed seven +// utah-packages PRs as awaiting rework; six had been closed and one merged +// by a maintainer 9-13 days earlier, and no stage pass had seen them since. + +// restartedReworkStore writes the records through a store whose clock is +// set per record, saves it, and loads it into a fresh store the way the hub +// does on start-up. +func restartedReworkStore(t *testing.T, write func(store *stagelabels.Store, at func(time.Time))) *stagelabels.Store { + t.Helper() + path := filepath.Join(t.TempDir(), stagelabels.StateFile) + clock := time.Now() + before := stagelabels.NewStore(path, func() time.Time { return clock }) + write(before, func(at time.Time) { clock = at }) + if err := before.Save(); err != nil { + t.Fatalf("save: %v", err) + } + after := stagelabels.NewStore(path, nil) + if err := after.Load(); err != nil { + t.Fatalf("load: %v", err) + } + return after +} + +func TestFleetReworkAfterRestartDropsLongUnseenRecords(t *testing.T) { + now := time.Now() + store := restartedReworkStore(t, func(s *stagelabels.Store, at func(time.Time)) { + at(now.Add(-13 * 24 * time.Hour)) // closed by a maintainer, unseen since + noteRework(s, "org/r", 702, stagelabels.AdoptedByHive, "alice") + noteRework(s, "org/r", 769, stagelabels.ReturnedToContributor, "alice") + at(now.Add(-9 * 24 * time.Hour)) // merged by a maintainer, unseen since + noteRework(s, "org/r", 801, stagelabels.AdoptedByHive, "alice") + at(now.Add(-5 * time.Minute)) // open, seen by the last pass before the restart + noteRework(s, "org/r", 972, stagelabels.ReturnedToContributor, "alice") + }) + cases := map[string]*FrontendRepo{ + "no status yet": nil, + "listing incomplete": {Name: "r", Full: "org/r", CountsIncomplete: true}, + "never enumerated": {Name: "r", Full: "org/r", ownershipDataMissing: true}, + } + for name, repo := range cases { + t.Run(name, func(t *testing.T) { + hub, srv, _ := reworkHub(t) + hub.stageStore = store + fixConn(hub, "alice", fixCaps(capFixPRInterrupt), time.Now()) + if repo != nil { + setReworkStatus(srv, *repo) + } + + fc := snapshotFor(t, hub, "alice") + if got := pendingNumbers(fc); fc.PendingRework != 1 || len(got) != 1 || got[0] != 972 { + t.Fatalf("pending_rework=%d prs=%v, want only #972 (seen by the last pass); #702/#769/#801 have not been listed for days", fc.PendingRework, got) + } + }) + } +} + +func TestFleetReworkAfterRestartKeepsFreshlyReturnedPR(t *testing.T) { + now := time.Now() + store := restartedReworkStore(t, func(s *stagelabels.Store, at func(time.Time)) { + // Returned to the contributor just before the restart; the last + // pass saw it a minute before the hub went down. + at(now.Add(-time.Minute)) + noteRework(s, "org/r", 5, stagelabels.ReturnedToContributor, "alice") + // Seen just inside the bound: an outage shorter than a day keeps it. + at(now.Add(-pendingReworkUnseenBound + time.Hour)) + noteRework(s, "org/r", 6, stagelabels.AdoptedByHive, "alice") + }) + hub, srv, _ := reworkHub(t) + hub.stageStore = store + fixConn(hub, "alice", fixCaps(capFixPRInterrupt), time.Now()) + setReworkStatus(srv, FrontendRepo{Name: "r", Full: "org/r", CountsStale: true}) + + fc := snapshotFor(t, hub, "alice") + if got := pendingNumbers(fc); fc.PendingRework != 2 || len(got) != 2 || got[0] != 5 || got[1] != 6 { + t.Fatalf("pending_rework=%d prs=%v, want #5 (072) and #6 (074): both seen within %s", fc.PendingRework, got, pendingReworkUnseenBound) + } + if fc.PendingReworkPRs[0].Stage != stagelabels.ReturnedToContributor.Label() { + t.Fatalf("stage = %q, want 072", fc.PendingReworkPRs[0].Stage) + } +} + +func TestFleetReworkVouchedListingDecidesOverRecordAge(t *testing.T) { + // #14708 behaviour stands where the listing is complete: a PR it lists + // open is owed rework however long ago a stage pass last touched it. + now := time.Now() + store := restartedReworkStore(t, func(s *stagelabels.Store, at func(time.Time)) { + at(now.Add(-3 * 24 * time.Hour)) + noteRework(s, "org/r", 1, stagelabels.ReturnedToContributor, "alice") // listed open + noteRework(s, "org/r", 2, stagelabels.ReturnedToContributor, "alice") // not listed + at(now) + noteRework(s, "org/r", 3, stagelabels.ReturnedToContributor, "alice") // fresh, not listed + }) + hub, srv, _ := reworkHub(t) + hub.stageStore = store + fixConn(hub, "alice", fixCaps(capFixPRInterrupt), time.Now()) + setReworkStatus(srv, FrontendRepo{Name: "r", Full: "org/r", OpenPrs: []any{ + FrontendPR{PullRequest: github.PullRequest{Repo: "org/r", Number: 1, Author: "alice"}}, + }}) + + fc := snapshotFor(t, hub, "alice") + if got := pendingNumbers(fc); fc.PendingRework != 1 || len(got) != 1 || got[0] != 1 { + t.Fatalf("pending_rework=%d prs=%v, want only #1: a complete listing decides", fc.PendingRework, got) + } +} + +func TestFleetReworkSkipsRecordsTheStoreMarksFinished(t *testing.T) { + hub, _, store := reworkHub(t) + fixConn(hub, "alice", fixCaps(capFixPRInterrupt), time.Now()) + now := time.Now() + + // Superseded: the hive's copy merged and the original was closed + // (or told, when the close was refused). The stage tracker drops it. + noteRework(store, "org/r", 1, stagelabels.AdoptedByHive, "alice") + store.MarkSuperseded(stagelabels.Key("org/r", 1), 11, false, now) + // Close-first supersession: the original closed as soon as the copy + // existed, while the copy is still in flight. + noteRework(store, "org/r", 2, stagelabels.AdoptedByHive, "alice") + store.MarkSupersededClosedFirst(stagelabels.Key("org/r", 2), 12, false, now) + // A lane reported the PR closed or merged: the stage pass forgets it. + noteRework(store, "org/r", 3, stagelabels.ReturnedToContributor, "alice") + store.Forget(stagelabels.Key("org/r", 3)) + // A reopen for a failed copy clears the supersession: owed again. + noteRework(store, "org/r", 4, stagelabels.AdoptedByHive, "alice") + store.MarkSuperseded(stagelabels.Key("org/r", 4), 14, false, now) + store.MarkAdoptionReopened(stagelabels.Key("org/r", 4), 14, now) + + if got := store.ByAuthor("alice", stagelabels.ReturnedToContributor); len(got) != 0 { + t.Fatalf("ByAuthor returned a forgotten record: %+v", got) + } + fc := snapshotFor(t, hub, "alice") + if got := pendingNumbers(fc); fc.PendingRework != 1 || len(got) != 1 || got[0] != 4 { + t.Fatalf("pending_rework=%d prs=%v, want only the reopened #4", fc.PendingRework, got) + } +} + +func TestOwedReworkBound(t *testing.T) { + now := time.Now() + var unvouched openPRIndex + rec := func(seen time.Time) stagelabels.Record { + return stagelabels.Record{Stage: stagelabels.ReturnedToContributor, LastSeen: seen} + } + if !owedRework(unvouched, "org/r", 1, rec(now.Add(-pendingReworkUnseenBound)), now) { + t.Fatal("a record seen exactly one bound ago is still owed") + } + if owedRework(unvouched, "org/r", 1, rec(now.Add(-pendingReworkUnseenBound-time.Minute)), now) { + t.Fatal("a record unseen past the bound is not owed") + } + if !owedRework(unvouched, "org/r", 1, rec(time.Time{}), now) { + t.Fatal("a record with no LastSeen is not evidence of a close") + } + if pendingReworkUnseenBound != stagelabels.ReturnedToContributor.Dwell() || pendingReworkUnseenBound <= stagelabels.LastSeenPersistEvery { + t.Fatalf("bound %s: want 072's dwell, well above the LastSeen persist interval", pendingReworkUnseenBound) + } +}