From a473b368e18533bb9ebbbd511af75d50a3a5751b Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Fri, 30 Oct 2026 10:29:07 -0400 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=90=9B=20Run=20the=20reconcile=20pass?= =?UTF-8?q?=20when=20wakes=20outpace=20the=20eval=20interval;=20age=20unbl?= =?UTF-8?q?ocked=20rows=20from=20the=20lift?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wake-driven cycles (review, verdict, webhook, continuous re-kick) plan from the cached snapshot; only the cadence ticker enumerates, and on a busy spoke its tick lands during long ticks and then competes with always-ready wake channels, so the snapshot aged 50m-1h30m with no reconcile pass. The loop now tracks the last reconcile start: a wake cycle past the interval enumerates instead, and the loop top runs an owed reconcile pass whichever channel woke it. The blocked sweep records its lift on the stage store; an unblocked issue's tracker row ages from the lift, not from issue creation. Closes #15779 Signed-off-by: Andrew Anderson --- .../fixed-15779-reconcile-starvation.md | 1 + src/cmd/hive/main.go | 58 ++++++++++-- src/cmd/hive/reconcile_starvation_test.go | 92 +++++++++++++++++++ src/cmd/hive/webhook_wake_test.go | 10 +- src/pkg/dashboard/stage_tracker_build.go | 15 ++- .../stage_tracker_unblocked_since_test.go | 52 +++++++++++ src/pkg/github/item_refresh_hook.go | 5 + src/pkg/stagelabels/store.go | 33 +++++++ src/pkg/stagelabels/unblocked_test.go | 43 +++++++++ 9 files changed, 299 insertions(+), 10 deletions(-) create mode 100644 changelog.d/fixed-15779-reconcile-starvation.md create mode 100644 src/cmd/hive/reconcile_starvation_test.go create mode 100644 src/pkg/dashboard/stage_tracker_unblocked_since_test.go create mode 100644 src/pkg/stagelabels/unblocked_test.go diff --git a/changelog.d/fixed-15779-reconcile-starvation.md b/changelog.d/fixed-15779-reconcile-starvation.md new file mode 100644 index 0000000000..8a3e81610e --- /dev/null +++ b/changelog.d/fixed-15779-reconcile-starvation.md @@ -0,0 +1 @@ +- Fixed the reconcile (enumerate) pass being starved by wake-driven cached cycles: a wake whose cached snapshot is older than the effective eval interval now enumerates, and the loop runs an owed reconcile pass regardless of which channel woke it, so the snapshot can no longer age for an hour while the ticker tick loses to webhook, verdict and re-kick wakes. The stage tracker's "since" for an issue the blocked sweep just unblocked now restarts at the lift instead of counting from issue creation ([#15779](https://github.com/hivecommons/hive/issues/15779)). diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index d5ad531f8e..f51a1ffaca 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -5878,8 +5878,22 @@ func (b *boot) runLoopWith(deps runLoopDeps) { if err := waitForgeStateReady(b.ctx, b.forgeState.Load(), b.logger, forgeStateReadyLogInterval); err != nil { return } + loopNow := deps.now + if loopNow == nil { + loopNow = time.Now + } b.evalCycles++ deps.runEval(b, nil, evalSourceEnumerate) + // lastEnumerateAt is when the last reconcile pass started. Wake-driven + // cycles plan from the cached snapshot, and a busy spoke can be woken + // every few minutes for hours; the reconcile cadence must not depend on + // the ticker tick winning the select against those wakes + // (hivecommons/hive#15779). reconcileOverdue is the one check both the + // wake paths and the loop top consult. + lastEnumerateAt := loopNow() + reconcileOverdue := func(now time.Time) bool { + return lastEvalInterval > 0 && now.Sub(lastEnumerateAt) >= lastEvalInterval + } deps.runRotation(b) if b.wd != nil { b.wd.Tick(b.ctx) @@ -5894,6 +5908,15 @@ func (b *boot) runLoopWith(deps runLoopDeps) { // gates, planning from the stored snapshot — already patched by the // delivery's targeted re-read — so a wake costs no listing. tick := func(source evalCycleSource) { + if now := loopNow(); source == evalSourceCached && reconcileOverdue(now) { + b.logger.Info("cached snapshot older than eval interval; enumerating", + "snapshot_age", now.Sub(lastEnumerateAt).Round(time.Second).String(), + "eval_interval_s", int(lastEvalInterval.Seconds())) + source = evalSourceEnumerate + } + if source == evalSourceEnumerate { + lastEnumerateAt = loopNow() + } b.cfgReloadMu.Lock() restarted := deps.restartCrashed(b.ctx, b.agentMgr) for _, name := range restarted { @@ -6095,12 +6118,18 @@ func (b *boot) runLoopWith(deps runLoopDeps) { "agent", who, "next_kick", next.UTC().Format(time.RFC3339), "idle_for", idleFor.String(), "was_armed_for", wakeArmedFor.UTC().Format(time.RFC3339)) lastWake = now - // Keep this cycle lean: planning and delivery without sweeps. - b.cfgReloadMu.Lock() - b.evalCycles++ - deps.runEval(b, nil, evalSourceCached) - deps.persist(b) - b.cfgReloadMu.Unlock() + if reconcileOverdue(now) { + // The reconcile pass is owed; run the full tick instead of + // the lean cached cycle so it is not starved by re-kicks. + tick(evalSourceCached) + } else { + // Keep this cycle lean: planning and delivery without sweeps. + b.cfgReloadMu.Lock() + b.evalCycles++ + deps.runEval(b, nil, evalSourceCached) + deps.persist(b) + b.cfgReloadMu.Unlock() + } after, afterWho := b.gov.NextContinuousKickAt() if backoff, stalled := wakeHold.observe(next, after, who, afterWho, wakeNow()); stalled { b.logger.Info("continuous re-kick wake: eval cycle left the due re-kick unchanged (agent gated outside the governor); backing off", @@ -6112,6 +6141,23 @@ func (b *boot) runLoopWith(deps runLoopDeps) { for { // Serve due kicks between cycles even when other channels stay ready. serveContinuousWake() + // A reconcile pass owed for a full interval runs here regardless of + // which channel woke the loop, so a ticker tick that was consumed, + // dropped or outraced by wake channels cannot starve enumeration. + if now := loopNow(); b.ctx.Err() == nil && reconcileOverdue(now) { + b.logger.Info("eval interval elapsed without a reconcile pass; enumerating", + "since_last_enumerate", now.Sub(lastEnumerateAt).Round(time.Second).String(), + "eval_interval_s", int(lastEvalInterval.Seconds())) + tick(evalSourceEnumerate) + armContinuousWake() + armWebhookHealthRecheck() + // The cadence tick for this interval, if it is already pending, + // would only repeat the pass that just ran. + select { + case <-ticker.Chan(): + default: + } + } select { case <-b.ctx.Done(): b.logger.Info("shutting down, persisting state") diff --git a/src/cmd/hive/reconcile_starvation_test.go b/src/cmd/hive/reconcile_starvation_test.go new file mode 100644 index 0000000000..d04ae854c0 --- /dev/null +++ b/src/cmd/hive/reconcile_starvation_test.go @@ -0,0 +1,92 @@ +package main + +import ( + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/governor" +) + +// Wake-driven cached cycles arriving more often than the eval interval +// must not starve the reconcile pass: once the cached snapshot is older +// than the interval, the next wake enumerates even if the ticker never +// fires (hivecommons/hive#15779). +func TestRunLoopWakeEnumeratesWhenSnapshotOlderThanInterval(t *testing.T) { + b, log, cancel := newLoopBoot(t, 900, 0) + defer cancel() + b.reviewWake = make(chan struct{}, 1) + clock := time.Now() + gov := newFakeTicker() + h := &loopHarness{startupOK: true, pending: []*fakeTicker{gov}, persisted: make(chan struct{}, 16)} + deps := h.deps() + deps.now = func() time.Time { return clock } + done := runLoopAsync(b, deps) + <-h.persisted // initial reconcile pass + + // Four wakes 5 minutes apart against a 900s interval: the first two + // plan from the cache, the third lands 15m after the last reconcile + // and must enumerate; the fourth is cached again. + for i := 0; i < 4; i++ { + clock = clock.Add(5 * time.Minute) + b.reviewWake <- struct{}{} + <-h.persisted + } + cancel() + waitDone(t, done) + + want := []evalCycleSource{evalSourceEnumerate, evalSourceCached, evalSourceCached, evalSourceEnumerate, evalSourceCached} + if len(h.sources) != len(want) { + t.Fatalf("sources = %v, want %v", h.sources, want) + } + for i := range want { + if h.sources[i] != want[i] { + t.Fatalf("sources = %v, want %v", h.sources, want) + } + } + if !strings.Contains(log.String(), "cached snapshot older than eval interval; enumerating") { + t.Fatalf("missing upgrade log line:\n%s", log.String()) + } +} + +// A ticker tick that is lost (never delivered) must not stop the reconcile +// pass: any loop iteration that finds a full interval elapsed since the +// last reconcile runs one, here on a not-yet-due continuous wake timer. +func TestRunLoopTopRunsOverdueReconcileWithoutTickerTick(t *testing.T) { + b, log, cancel := newLoopBoot(t, 900, 0) + defer cancel() + b.cfg.Governor.Modes = map[string]config.ModeConfig{"idle": {Cadences: map[string]config.Cadence{"alpha": "continuous"}}} + b.gov = governor.New(b.cfg.Governor, b.cfg.EnabledAgents(), b.logger) + b.gov.Evaluate(0, 0, 0, 0) + clock := time.Now() + b.gov.SeedContinuousNextKickForTest("alpha", clock.Add(24*time.Hour)) + gov := newFakeTicker() + wake := newFakeTicker() + h := &loopHarness{startupOK: true, pending: []*fakeTicker{gov}, persisted: make(chan struct{}, 16)} + deps := h.deps() + deps.now = func() time.Time { return clock } + deps.newWakeTimer = func(time.Duration) loopTicker { return wake } + done := runLoopAsync(b, deps) + <-h.persisted + + // Poll wake inside the interval: nothing runs, only a re-arm. + clock = clock.Add(10 * time.Minute) + wake.fire() + // Poll wake after the interval elapsed with no ticker tick delivered. + clock = clock.Add(6 * time.Minute) + wake.fire() + <-h.persisted + cancel() + waitDone(t, done) + + if len(h.sources) != 2 || h.sources[1] != evalSourceEnumerate { + t.Fatalf("sources = %v, want initial + loop-top reconcile pass", h.sources) + } + if h.sweeps != 2 { + t.Fatalf("sweeps = %d, want the owed reconcile to run as a full tick", h.sweeps) + } + if !strings.Contains(log.String(), "eval interval elapsed without a reconcile pass; enumerating") { + t.Fatalf("missing loop-top log line:\n%s", log.String()) + } +} diff --git a/src/cmd/hive/webhook_wake_test.go b/src/cmd/hive/webhook_wake_test.go index 945b774d4d..74db9e1e38 100644 --- a/src/cmd/hive/webhook_wake_test.go +++ b/src/cmd/hive/webhook_wake_test.go @@ -324,7 +324,13 @@ func TestRunLoopHealthRecheckResetsTickerWhenWebhooksLapse(t *testing.T) { if !strings.Contains(log.String(), "webhooks degraded; polling cadence restored") { t.Fatalf("log = %s", log.String()) } - if len(h.evals) != 1 { - t.Fatalf("health re-check must not run an eval cycle, evals = %d", len(h.evals)) + // The re-check itself runs no eval cycle; the one extra cycle is the + // loop top's owed reconcile pass, because 121s have passed since the + // last one against the restored 60s cadence (hivecommons/hive#15779). + if len(h.sources) != 2 || h.sources[1] != evalSourceEnumerate { + t.Fatalf("sources = %v, want first cycle + one overdue reconcile pass", h.sources) + } + if !strings.Contains(log.String(), "eval interval elapsed without a reconcile pass; enumerating") { + t.Fatalf("missing overdue reconcile log: %s", log.String()) } } diff --git a/src/pkg/dashboard/stage_tracker_build.go b/src/pkg/dashboard/stage_tracker_build.go index e35072aa5c..354d572dec 100644 --- a/src/pkg/dashboard/stage_tracker_build.go +++ b/src/pkg/dashboard/stage_tracker_build.go @@ -80,7 +80,14 @@ func (s *Server) buildStageTracker(status *StatusPayload, store *stagelabels.Sto if store == nil { return stagelabels.Record{}, false } - return store.Get(stagelabels.Key(repo, number)) + rec, ok := store.Get(stagelabels.Key(repo, number)) + if ok && rec.Stage == stagelabels.None { + // A stage-less record only remembers the blocked sweep's lift + // (hivecommons/hive#15779): the row is still unstaged, but its + // since-time starts at the lift rather than at creation. + return stagelabels.Record{UnblockedAt: rec.UnblockedAt}, false + } + return rec, ok } presence, maintainerPolicy := s.stageTrackerPresence(), s.stageTrackerMaintainerPolicy() fixWork := s.stageTrackerFixWorkLookup() @@ -111,7 +118,7 @@ func (s *Server) buildStageTracker(status *StatusPayload, store *stagelabels.Sto } if hasRec && rec.Stage != stagelabels.None { stage = rec.Stage - t := rec.EnteredAt + t := rec.StageSinceAt() it.StageSince = &t it.Reason = rec.Reason stageTrackerRegression(it, rec) @@ -129,6 +136,10 @@ func (s *Server) buildStageTracker(status *StatusPayload, store *stagelabels.Sto it.DCOTotal, it.DCOUnsigned = total, unsigned } } + if !hasRec && !rec.UnblockedAt.IsZero() { + t := rec.UnblockedAt + it.StageSince = &t + } it.Hold = stageTrackerHoldFrom(rec, hasRec, labels) var attempt github.MergeAttempt var hasAttempt bool diff --git a/src/pkg/dashboard/stage_tracker_unblocked_since_test.go b/src/pkg/dashboard/stage_tracker_unblocked_since_test.go new file mode 100644 index 0000000000..e0a34284ce --- /dev/null +++ b/src/pkg/dashboard/stage_tracker_unblocked_since_test.go @@ -0,0 +1,52 @@ +package dashboard + +import ( + "testing" + "time" + + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/stagelabels" +) + +// An issue the blocked sweep just unblocked has no stage record yet; its +// row must age from the lift, not from issue creation +// (hivecommons/hive#15779). +func TestStageTrackerNewRowAgesFromUnblock(t *testing.T) { + now := time.Date(2026, 10, 30, 12, 0, 0, 0, time.UTC) + s, _ := stageTrackerFixture(t, now) + clock := now + store := stagelabels.NewStore("", func() time.Time { return clock }) + s.stageStore = store + created := now.Add(-17 * time.Hour) + lifted := now.Add(-20 * time.Minute) + s.statusMu.Lock() + s.status.Repos[0].ActionableIssues = append(s.status.Repos[0].ActionableIssues, + github.Issue{Number: 15517, Title: "Was blocked", Author: "pat", AuthorIsHuman: true, CreatedAt: created, UpdatedAt: created}) + s.statusMu.Unlock() + + resp := s.buildStageTracker(s.status, store, now) + it := trackerItem(t, resp, "issue", 15517) + if it.StageSlug != StageTrackerNewSlug || it.StageSince == nil || !it.StageSince.Equal(created) { + t.Fatalf("before the lift: slug %q since %v, want 000-new since creation %v", it.StageSlug, it.StageSince, created) + } + + store.NoteUnblocked(stagelabels.Key("acme/console", 15517), lifted) + resp = s.buildStageTracker(s.status, store, now) + it = trackerItem(t, resp, "issue", 15517) + if it.StageSlug != StageTrackerNewSlug || it.Stage != 0 || it.New == nil { + t.Fatalf("after the lift the row must still be 000-new: stage %d slug %q new %+v", it.Stage, it.StageSlug, it.New) + } + if it.StageSince == nil || !it.StageSince.Equal(lifted) { + t.Fatalf("after the lift since = %v, want the lift %v", it.StageSince, lifted) + } + + // The next stage-label pass stages it: since is the stage entry. + staged := now.Add(-5 * time.Minute) + clock = staged + store.Observe(stagelabels.Key("acme/console", 15517), stagelabels.Actionable, "actionable") + resp = s.buildStageTracker(s.status, store, now) + it = trackerItem(t, resp, "issue", 15517) + if it.Stage != stagelabels.Actionable.Number() || it.StageSince == nil || !it.StageSince.Equal(staged) { + t.Fatalf("after staging: stage %d since %v, want 010 since %v", it.Stage, it.StageSince, staged) + } +} diff --git a/src/pkg/github/item_refresh_hook.go b/src/pkg/github/item_refresh_hook.go index fd828c819c..cb4da9f828 100644 --- a/src/pkg/github/item_refresh_hook.go +++ b/src/pkg/github/item_refresh_hook.go @@ -3,6 +3,9 @@ package github import ( "context" "sync" + "time" + + "github.com/hivecommons/hive/pkg/stagelabels" ) // ItemRefreshHook is told, with the configured repo name and the item number, @@ -47,6 +50,8 @@ func (c *Client) unblockIssue(ctx context.Context, repo string, number int) erro if err := c.RemoveLabel(ctx, repo, number, BlockedLabel); err != nil { return err } + // The row's since-time restarts at the lift (hivecommons/hive#15779). + stagelabels.Shared().NoteUnblocked(stagelabels.Key(repo, number), time.Now()) notifyItemRefresh(repo, number, false) return nil } diff --git a/src/pkg/stagelabels/store.go b/src/pkg/stagelabels/store.go index 9b8db788d9..eecf29cd3c 100644 --- a/src/pkg/stagelabels/store.go +++ b/src/pkg/stagelabels/store.go @@ -58,6 +58,12 @@ type Record struct { // read as a fresh note (hivecommons/hive#15729). Zero on records // written before it existed; NoteChangedAt falls back to EnteredAt. ReasonChangedAt time.Time `json:"reason_changed_at,omitempty"` + // UnblockedAt is when the blocked sweep last lifted the item's block + // (hivecommons/hive#15779). A blocked issue has no stage, so the lift + // leaves a stage-less record carrying only this time until the next + // stage-label pass stages the issue; the dashboard ages the row from + // StageSinceAt so "since" restarts at the lift, not at issue creation. + UnblockedAt time.Time `json:"unblocked_at,omitempty"` // Author is the item's author login, lower-cased, as the lane saw it // (hivecommons/hive#12502). It lets the dashboard answer "which of this // contributor's PRs are returned to them" from the store alone, with no @@ -746,6 +752,33 @@ func (r Record) NoteChangedAt() time.Time { return r.EnteredAt } +// StageSinceAt is when the row's since-time starts: stage entry, or the +// later block lift when the sweep unblocked the item after it entered the +// stage (or before any stage was recorded). +func (r Record) StageSinceAt() time.Time { + if r.UnblockedAt.After(r.EnteredAt) { + return r.UnblockedAt + } + return r.EnteredAt +} + +// NoteUnblocked records that the blocked sweep lifted key's block at now. +// A blocked issue carries no stage record (its stage is None), so a +// stage-less record is created to hold the time; the next ObserveProgress +// keeps UnblockedAt while it stages the issue, and a None observation +// drops the record as usual. +func (s *Store) NoteUnblocked(key string, now time.Time) { + s.mu.Lock() + defer s.mu.Unlock() + r := s.items[key] + r.UnblockedAt = now + if r.LastSeen.IsZero() { + r.LastSeen = now + } + s.items[key] = r + s.dirty = true +} + // NoteActivity folds this cycle's activity inputs into key's record and // reports whether anything a person could have done changed since the last // observation. The first observation of a record seeds LastActivityAt diff --git a/src/pkg/stagelabels/unblocked_test.go b/src/pkg/stagelabels/unblocked_test.go new file mode 100644 index 0000000000..1398fb9b95 --- /dev/null +++ b/src/pkg/stagelabels/unblocked_test.go @@ -0,0 +1,43 @@ +package stagelabels + +import ( + "testing" + "time" +) + +// NoteUnblocked keeps the lift on a stage-less record, StageSinceAt reads +// it as the since-time, and the next staging observation keeps it while +// stage entry (later) takes over (hivecommons/hive#15779). +func TestNoteUnblockedAgesRecordFromLift(t *testing.T) { + clock := time.Date(2026, 10, 30, 12, 0, 0, 0, time.UTC) + s := NewStore("", func() time.Time { return clock }) + key := Key("o/r", 15517) + lifted := clock.Add(-20 * time.Minute) + + s.NoteUnblocked(key, lifted) + rec, ok := s.Get(key) + if !ok || rec.Stage != None || !rec.UnblockedAt.Equal(lifted) || !rec.StageSinceAt().Equal(lifted) { + t.Fatalf("after lift: ok=%v rec=%+v since=%v, want stage-less record since the lift", ok, rec, rec.StageSinceAt()) + } + + tr := s.Observe(key, Actionable, "actionable") + if !tr.Changed || tr.Prev != None { + t.Fatalf("staging after lift transition = %+v, want a change from None", tr) + } + rec, _ = s.Get(key) + if !rec.UnblockedAt.Equal(lifted) || !rec.StageSinceAt().Equal(clock) { + t.Fatalf("after staging: rec=%+v since=%v, want stage entry %v", rec, rec.StageSinceAt(), clock) + } + + // A lift after stage entry moves the since-time forward again. + later := clock.Add(time.Hour) + s.NoteUnblocked(key, later) + rec, _ = s.Get(key) + if !rec.StageSinceAt().Equal(later) { + t.Fatalf("since after a later lift = %v, want %v", rec.StageSinceAt(), later) + } + s.Observe(key, None, "") + if _, ok := s.Get(key); ok { + t.Fatal("a None observation must drop the record, lift included") + } +} From cfeacca7c1ea3a1a74135659ace465c4e7d5d9a7 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Fri, 30 Oct 2026 13:19:38 -0400 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=90=9B=20Keep=20the=20unblock=20lift?= =?UTF-8?q?=20across=20None=20stage=20decisions;=20race-free=20loop=20test?= =?UTF-8?q?=20clock?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review: the issue stage-label pass Forget()s every None decision, which dropped the lift record right after the blocked sweep wrote it. Open issues deciding None now go through ForgetStage, which keeps a lift-only record; Forget (closed/merged) still drops everything. The tracker only ages from the lift when the row is no longer blocked. The new loop tests advanced a plain closure clock from the test goroutine while the loop read it; a mutex-guarded clock fixes the race, and the health re-check test now waits for the overdue pass it asserts. Signed-off-by: Andrew Anderson --- src/cmd/hive/reconcile_starvation_test.go | 36 ++++++++++++++----- src/cmd/hive/webhook_wake_test.go | 1 + src/pkg/dashboard/stage_tracker_build.go | 16 ++++++--- .../stage_tracker_unblocked_since_test.go | 9 +++++ src/pkg/github/issue_stage_labels.go | 2 +- src/pkg/stagelabels/store.go | 28 +++++++++++++-- src/pkg/stagelabels/unblocked_test.go | 20 ++++++++++- 7 files changed, 95 insertions(+), 17 deletions(-) diff --git a/src/cmd/hive/reconcile_starvation_test.go b/src/cmd/hive/reconcile_starvation_test.go index d04ae854c0..cb43358a8c 100644 --- a/src/cmd/hive/reconcile_starvation_test.go +++ b/src/cmd/hive/reconcile_starvation_test.go @@ -2,6 +2,7 @@ package main import ( "strings" + "sync" "testing" "time" @@ -9,6 +10,25 @@ import ( "github.com/hivecommons/hive/pkg/governor" ) +// loopClock is a fake clock the test advances from its own goroutine while +// the loop reads it through deps.now. +type loopClock struct { + mu sync.Mutex + t time.Time +} + +func (c *loopClock) now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.t +} + +func (c *loopClock) advance(d time.Duration) { + c.mu.Lock() + c.t = c.t.Add(d) + c.mu.Unlock() +} + // Wake-driven cached cycles arriving more often than the eval interval // must not starve the reconcile pass: once the cached snapshot is older // than the interval, the next wake enumerates even if the ticker never @@ -17,11 +37,11 @@ func TestRunLoopWakeEnumeratesWhenSnapshotOlderThanInterval(t *testing.T) { b, log, cancel := newLoopBoot(t, 900, 0) defer cancel() b.reviewWake = make(chan struct{}, 1) - clock := time.Now() + clock := &loopClock{t: time.Now()} gov := newFakeTicker() h := &loopHarness{startupOK: true, pending: []*fakeTicker{gov}, persisted: make(chan struct{}, 16)} deps := h.deps() - deps.now = func() time.Time { return clock } + deps.now = clock.now done := runLoopAsync(b, deps) <-h.persisted // initial reconcile pass @@ -29,7 +49,7 @@ func TestRunLoopWakeEnumeratesWhenSnapshotOlderThanInterval(t *testing.T) { // plan from the cache, the third lands 15m after the last reconcile // and must enumerate; the fourth is cached again. for i := 0; i < 4; i++ { - clock = clock.Add(5 * time.Minute) + clock.advance(5 * time.Minute) b.reviewWake <- struct{}{} <-h.persisted } @@ -59,22 +79,22 @@ func TestRunLoopTopRunsOverdueReconcileWithoutTickerTick(t *testing.T) { b.cfg.Governor.Modes = map[string]config.ModeConfig{"idle": {Cadences: map[string]config.Cadence{"alpha": "continuous"}}} b.gov = governor.New(b.cfg.Governor, b.cfg.EnabledAgents(), b.logger) b.gov.Evaluate(0, 0, 0, 0) - clock := time.Now() - b.gov.SeedContinuousNextKickForTest("alpha", clock.Add(24*time.Hour)) + clock := &loopClock{t: time.Now()} + b.gov.SeedContinuousNextKickForTest("alpha", clock.now().Add(24*time.Hour)) gov := newFakeTicker() wake := newFakeTicker() h := &loopHarness{startupOK: true, pending: []*fakeTicker{gov}, persisted: make(chan struct{}, 16)} deps := h.deps() - deps.now = func() time.Time { return clock } + deps.now = clock.now deps.newWakeTimer = func(time.Duration) loopTicker { return wake } done := runLoopAsync(b, deps) <-h.persisted // Poll wake inside the interval: nothing runs, only a re-arm. - clock = clock.Add(10 * time.Minute) + clock.advance(10 * time.Minute) wake.fire() // Poll wake after the interval elapsed with no ticker tick delivered. - clock = clock.Add(6 * time.Minute) + clock.advance(6 * time.Minute) wake.fire() <-h.persisted cancel() diff --git a/src/cmd/hive/webhook_wake_test.go b/src/cmd/hive/webhook_wake_test.go index 74db9e1e38..ee9be04979 100644 --- a/src/cmd/hive/webhook_wake_test.go +++ b/src/cmd/hive/webhook_wake_test.go @@ -312,6 +312,7 @@ func TestRunLoopHealthRecheckResetsTickerWhenWebhooksLapse(t *testing.T) { clock.Advance(121 * time.Second) github.RecordGitHubActivity(clock.Now()) // a poll saw a change webhooks missed health.fire() + <-h.persisted // the loop top's overdue reconcile pass (asserted below) cancel() waitDone(t, done) diff --git a/src/pkg/dashboard/stage_tracker_build.go b/src/pkg/dashboard/stage_tracker_build.go index 354d572dec..9997c7c76a 100644 --- a/src/pkg/dashboard/stage_tracker_build.go +++ b/src/pkg/dashboard/stage_tracker_build.go @@ -136,10 +136,6 @@ func (s *Server) buildStageTracker(status *StatusPayload, store *stagelabels.Sto it.DCOTotal, it.DCOUnsigned = total, unsigned } } - if !hasRec && !rec.UnblockedAt.IsZero() { - t := rec.UnblockedAt - it.StageSince = &t - } it.Hold = stageTrackerHoldFrom(rec, hasRec, labels) var attempt github.MergeAttempt var hasAttempt bool @@ -322,6 +318,7 @@ func (s *Server) buildStageTracker(status *StatusPayload, store *stagelabels.Sto finish(&it, issue.Labels, stage, rec, ok, false) stageTrackerBlockedBy(&it, issue.OpenBlockers()) stageTrackerBlockedOnPR(&it, issue.Body, bucketReason, status.GitHubBaseURL, prState) + stageTrackerSinceUnblock(&it, rec, ok) advisory := stageTrackerAdvisoryFor(issue.Labels, issue.Body, advisoryHint) if advisory == nil { applyStageTrackerNew(&it, stageTrackerNewState(it, ok, issueFilter, status.Governor, now)) @@ -943,3 +940,14 @@ func stageTrackerParkedOnDecision(it StageTrackerItem) bool { lower := strings.ToLower(strings.TrimSpace(base)) return lower == stageengine.StageReasonVerdictRequiresHuman || lower == stageengine.StageReasonDecisionRequested } + +// stageTrackerSinceUnblock ages an unstaged, no-longer-blocked issue from +// the blocked sweep's lift rather than from creation +// (hivecommons/hive#15779). A row blocked again keeps its own age. +func stageTrackerSinceUnblock(it *StageTrackerItem, rec stagelabels.Record, hasRec bool) { + if hasRec || rec.UnblockedAt.IsZero() || it.BlockedOnPR != nil || len(it.BlockedOn) > 0 || slices.Contains(it.Modifiers, modBlocked) { + return + } + t := rec.UnblockedAt + it.StageSince = &t +} diff --git a/src/pkg/dashboard/stage_tracker_unblocked_since_test.go b/src/pkg/dashboard/stage_tracker_unblocked_since_test.go index e0a34284ce..9048dabcfd 100644 --- a/src/pkg/dashboard/stage_tracker_unblocked_since_test.go +++ b/src/pkg/dashboard/stage_tracker_unblocked_since_test.go @@ -40,6 +40,15 @@ func TestStageTrackerNewRowAgesFromUnblock(t *testing.T) { t.Fatalf("after the lift since = %v, want the lift %v", it.StageSince, lifted) } + // The stage-label pass that follows may still decide None for the + // issue (SyncIssueStageLabels → ForgetStage); the lift survives it. + store.ForgetStage(stagelabels.Key("acme/console", 15517)) + resp = s.buildStageTracker(s.status, store, now) + it = trackerItem(t, resp, "issue", 15517) + if it.StageSlug != StageTrackerNewSlug || it.StageSince == nil || !it.StageSince.Equal(lifted) { + t.Fatalf("after a None pass: slug %q since %v, want 000-new since the lift %v", it.StageSlug, it.StageSince, lifted) + } + // The next stage-label pass stages it: since is the stage entry. staged := now.Add(-5 * time.Minute) clock = staged diff --git a/src/pkg/github/issue_stage_labels.go b/src/pkg/github/issue_stage_labels.go index 49e86aa014..677ed29b6c 100644 --- a/src/pkg/github/issue_stage_labels.go +++ b/src/pkg/github/issue_stage_labels.go @@ -477,7 +477,7 @@ func SyncIssueStageLabelsWith(ctx context.Context, client *Client, result *Actio var want stagelabels.Desired var tr stagelabels.Transition if stage == stagelabels.None { - store.Forget(key) + store.ForgetStage(key) } else { tr = store.ObserveProgress(key, stage, reason, d.Progress) noteIssueCoveringPR(client, store, key, in, stage) diff --git a/src/pkg/stagelabels/store.go b/src/pkg/stagelabels/store.go index eecf29cd3c..049502e366 100644 --- a/src/pkg/stagelabels/store.go +++ b/src/pkg/stagelabels/store.go @@ -655,8 +655,7 @@ func (s *Store) ObserveProgress(key string, stage Stage, reason string, progress tr := Transition{Prev: prev.Stage, Next: stage} if stage == None { if had { - delete(s.items, key) - s.dirty = true + s.forgetStageLocked(key, prev) tr.Changed = prev.Stage != None } return tr @@ -1982,7 +1981,7 @@ func (s *Store) SetCoveringPR(key, prKey string) { s.dirty = true } -// Forget drops key (closed/merged item). +// Forget drops key (closed/merged item), lift record included. func (s *Store) Forget(key string) { s.mu.Lock() defer s.mu.Unlock() @@ -1992,6 +1991,29 @@ func (s *Store) Forget(key string) { } } +// ForgetStage drops key's stage for an item that is still open but has no +// stage (a blocked issue, or one outside the hive's buckets). Unlike +// Forget it keeps the blocked sweep's lift, so the stage-label pass that +// runs right after an unblock does not reset the row's since-time back to +// creation (hivecommons/hive#15779). +func (s *Store) ForgetStage(key string) { + s.mu.Lock() + defer s.mu.Unlock() + if r, ok := s.items[key]; ok { + s.forgetStageLocked(key, r) + } +} + +// forgetStageLocked is ForgetStage with s.mu held and the record in hand. +func (s *Store) forgetStageLocked(key string, r Record) { + if r.UnblockedAt.IsZero() { + delete(s.items, key) + } else { + s.items[key] = Record{UnblockedAt: r.UnblockedAt, LastSeen: s.now()} + } + s.dirty = true +} + // Prune drops records not seen for maxAge (closed items the sweep stopped // listing). Returns how many were dropped. Author-wide trust-promotion // records (TrustPromotionKey) are per contributor, not per item, and diff --git a/src/pkg/stagelabels/unblocked_test.go b/src/pkg/stagelabels/unblocked_test.go index 1398fb9b95..6e9351e1da 100644 --- a/src/pkg/stagelabels/unblocked_test.go +++ b/src/pkg/stagelabels/unblocked_test.go @@ -36,8 +36,26 @@ func TestNoteUnblockedAgesRecordFromLift(t *testing.T) { if !rec.StageSinceAt().Equal(later) { t.Fatalf("since after a later lift = %v, want %v", rec.StageSinceAt(), later) } + // The stage-label pass deciding None for a still-open issue (the + // row the symptom describes) keeps the lift; only Forget — the + // closed/merged path — drops it. s.Observe(key, None, "") + rec, ok = s.Get(key) + if !ok || rec.Stage != None || !rec.UnblockedAt.Equal(later) || rec.Reason != "" { + t.Fatalf("after a None observation: ok=%v rec=%+v, want a lift-only record", ok, rec) + } + s.ForgetStage(key) + if rec, ok = s.Get(key); !ok || !rec.UnblockedAt.Equal(later) { + t.Fatalf("after ForgetStage: ok=%v rec=%+v, want the lift kept", ok, rec) + } + s.Forget(key) if _, ok := s.Get(key); ok { - t.Fatal("a None observation must drop the record, lift included") + t.Fatal("Forget must drop the record, lift included") + } + s.ForgetStage(Key("o/r", 1)) + s.Observe(Key("o/r", 2), Actionable, "actionable") + s.ForgetStage(Key("o/r", 2)) + if _, ok := s.Get(Key("o/r", 2)); ok { + t.Fatal("ForgetStage without a lift must drop the record") } }