Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-15779-reconcile-starvation.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Fixed the reconcile (enumerate) pass being starved by wake-driven cached cycles: a wake whose cached snapshot is older than the effective eval interval now enumerates, and the loop runs an owed reconcile pass regardless of which channel woke it, so the snapshot can no longer age for an hour while the ticker tick loses to webhook, verdict and re-kick wakes. The stage tracker's "since" for an issue the blocked sweep just unblocked now restarts at the lift instead of counting from issue creation ([#15779](https://github.com/hivecommons/hive/issues/15779)).
58 changes: 52 additions & 6 deletions src/cmd/hive/main.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package main

import (
Expand Down Expand Up @@ -5878,8 +5878,22 @@
if err := waitForgeStateReady(b.ctx, b.forgeState.Load(), b.logger, forgeStateReadyLogInterval); err != nil {
return
}
loopNow := deps.now
if loopNow == nil {
loopNow = time.Now
}
b.evalCycles++
deps.runEval(b, nil, evalSourceEnumerate)
// lastEnumerateAt is when the last reconcile pass started. Wake-driven
// cycles plan from the cached snapshot, and a busy spoke can be woken
// every few minutes for hours; the reconcile cadence must not depend on
// the ticker tick winning the select against those wakes
// (hivecommons/hive#15779). reconcileOverdue is the one check both the
// wake paths and the loop top consult.
lastEnumerateAt := loopNow()
reconcileOverdue := func(now time.Time) bool {
return lastEvalInterval > 0 && now.Sub(lastEnumerateAt) >= lastEvalInterval
}
deps.runRotation(b)
if b.wd != nil {
b.wd.Tick(b.ctx)
Expand All @@ -5894,6 +5908,15 @@
// gates, planning from the stored snapshot β€” already patched by the
// delivery's targeted re-read β€” so a wake costs no listing.
tick := func(source evalCycleSource) {
if now := loopNow(); source == evalSourceCached && reconcileOverdue(now) {
b.logger.Info("cached snapshot older than eval interval; enumerating",
"snapshot_age", now.Sub(lastEnumerateAt).Round(time.Second).String(),
"eval_interval_s", int(lastEvalInterval.Seconds()))
source = evalSourceEnumerate
}
if source == evalSourceEnumerate {
lastEnumerateAt = loopNow()
}
b.cfgReloadMu.Lock()
restarted := deps.restartCrashed(b.ctx, b.agentMgr)
for _, name := range restarted {
Expand Down Expand Up @@ -6095,12 +6118,18 @@
"agent", who, "next_kick", next.UTC().Format(time.RFC3339), "idle_for", idleFor.String(),
"was_armed_for", wakeArmedFor.UTC().Format(time.RFC3339))
lastWake = now
// Keep this cycle lean: planning and delivery without sweeps.
b.cfgReloadMu.Lock()
b.evalCycles++
deps.runEval(b, nil, evalSourceCached)
deps.persist(b)
b.cfgReloadMu.Unlock()
if reconcileOverdue(now) {
// The reconcile pass is owed; run the full tick instead of
// the lean cached cycle so it is not starved by re-kicks.
tick(evalSourceCached)
} else {
// Keep this cycle lean: planning and delivery without sweeps.
b.cfgReloadMu.Lock()
b.evalCycles++
deps.runEval(b, nil, evalSourceCached)
deps.persist(b)
b.cfgReloadMu.Unlock()
}
after, afterWho := b.gov.NextContinuousKickAt()
if backoff, stalled := wakeHold.observe(next, after, who, afterWho, wakeNow()); stalled {
b.logger.Info("continuous re-kick wake: eval cycle left the due re-kick unchanged (agent gated outside the governor); backing off",
Expand All @@ -6112,6 +6141,23 @@
for {
// Serve due kicks between cycles even when other channels stay ready.
serveContinuousWake()
// A reconcile pass owed for a full interval runs here regardless of
// which channel woke the loop, so a ticker tick that was consumed,
// dropped or outraced by wake channels cannot starve enumeration.
if now := loopNow(); b.ctx.Err() == nil && reconcileOverdue(now) {
b.logger.Info("eval interval elapsed without a reconcile pass; enumerating",
"since_last_enumerate", now.Sub(lastEnumerateAt).Round(time.Second).String(),
"eval_interval_s", int(lastEvalInterval.Seconds()))
tick(evalSourceEnumerate)
armContinuousWake()
armWebhookHealthRecheck()
// The cadence tick for this interval, if it is already pending,
// would only repeat the pass that just ran.
select {
case <-ticker.Chan():
default:
}
}
select {
case <-b.ctx.Done():
b.logger.Info("shutting down, persisting state")
Expand Down
112 changes: 112 additions & 0 deletions src/cmd/hive/reconcile_starvation_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
package main

import (
"strings"
"sync"
"testing"
"time"

"github.com/hivecommons/hive/pkg/config"
"github.com/hivecommons/hive/pkg/governor"
)

// loopClock is a fake clock the test advances from its own goroutine while
// the loop reads it through deps.now.
type loopClock struct {
mu sync.Mutex
t time.Time
}

func (c *loopClock) now() time.Time {
c.mu.Lock()
defer c.mu.Unlock()
return c.t
}

func (c *loopClock) advance(d time.Duration) {
c.mu.Lock()
c.t = c.t.Add(d)
c.mu.Unlock()
}

// Wake-driven cached cycles arriving more often than the eval interval
// must not starve the reconcile pass: once the cached snapshot is older
// than the interval, the next wake enumerates even if the ticker never
// fires (hivecommons/hive#15779).
func TestRunLoopWakeEnumeratesWhenSnapshotOlderThanInterval(t *testing.T) {
b, log, cancel := newLoopBoot(t, 900, 0)
defer cancel()
b.reviewWake = make(chan struct{}, 1)
clock := &loopClock{t: time.Now()}
gov := newFakeTicker()
h := &loopHarness{startupOK: true, pending: []*fakeTicker{gov}, persisted: make(chan struct{}, 16)}
deps := h.deps()
deps.now = clock.now
done := runLoopAsync(b, deps)
<-h.persisted // initial reconcile pass

// Four wakes 5 minutes apart against a 900s interval: the first two
// plan from the cache, the third lands 15m after the last reconcile
// and must enumerate; the fourth is cached again.
for i := 0; i < 4; i++ {
clock.advance(5 * time.Minute)
b.reviewWake <- struct{}{}
<-h.persisted
}
cancel()
waitDone(t, done)

want := []evalCycleSource{evalSourceEnumerate, evalSourceCached, evalSourceCached, evalSourceEnumerate, evalSourceCached}
if len(h.sources) != len(want) {
t.Fatalf("sources = %v, want %v", h.sources, want)
}
for i := range want {
if h.sources[i] != want[i] {
t.Fatalf("sources = %v, want %v", h.sources, want)
}
}
if !strings.Contains(log.String(), "cached snapshot older than eval interval; enumerating") {
t.Fatalf("missing upgrade log line:\n%s", log.String())
}
}

// A ticker tick that is lost (never delivered) must not stop the reconcile
// pass: any loop iteration that finds a full interval elapsed since the
// last reconcile runs one, here on a not-yet-due continuous wake timer.
func TestRunLoopTopRunsOverdueReconcileWithoutTickerTick(t *testing.T) {
b, log, cancel := newLoopBoot(t, 900, 0)
defer cancel()
b.cfg.Governor.Modes = map[string]config.ModeConfig{"idle": {Cadences: map[string]config.Cadence{"alpha": "continuous"}}}
b.gov = governor.New(b.cfg.Governor, b.cfg.EnabledAgents(), b.logger)
b.gov.Evaluate(0, 0, 0, 0)
clock := &loopClock{t: time.Now()}
b.gov.SeedContinuousNextKickForTest("alpha", clock.now().Add(24*time.Hour))
gov := newFakeTicker()
wake := newFakeTicker()
h := &loopHarness{startupOK: true, pending: []*fakeTicker{gov}, persisted: make(chan struct{}, 16)}
deps := h.deps()
deps.now = clock.now
deps.newWakeTimer = func(time.Duration) loopTicker { return wake }
done := runLoopAsync(b, deps)
<-h.persisted

// Poll wake inside the interval: nothing runs, only a re-arm.
clock.advance(10 * time.Minute)
wake.fire()
// Poll wake after the interval elapsed with no ticker tick delivered.
clock.advance(6 * time.Minute)
wake.fire()
<-h.persisted
cancel()
waitDone(t, done)

if len(h.sources) != 2 || h.sources[1] != evalSourceEnumerate {
t.Fatalf("sources = %v, want initial + loop-top reconcile pass", h.sources)
}
if h.sweeps != 2 {
t.Fatalf("sweeps = %d, want the owed reconcile to run as a full tick", h.sweeps)
}
if !strings.Contains(log.String(), "eval interval elapsed without a reconcile pass; enumerating") {
t.Fatalf("missing loop-top log line:\n%s", log.String())
}
}
11 changes: 9 additions & 2 deletions src/cmd/hive/webhook_wake_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,7 @@ func TestRunLoopHealthRecheckResetsTickerWhenWebhooksLapse(t *testing.T) {
clock.Advance(121 * time.Second)
github.RecordGitHubActivity(clock.Now()) // a poll saw a change webhooks missed
health.fire()
<-h.persisted // the loop top's overdue reconcile pass (asserted below)
cancel()
waitDone(t, done)

Expand All @@ -324,7 +325,13 @@ func TestRunLoopHealthRecheckResetsTickerWhenWebhooksLapse(t *testing.T) {
if !strings.Contains(log.String(), "webhooks degraded; polling cadence restored") {
t.Fatalf("log = %s", log.String())
}
if len(h.evals) != 1 {
t.Fatalf("health re-check must not run an eval cycle, evals = %d", len(h.evals))
// The re-check itself runs no eval cycle; the one extra cycle is the
// loop top's owed reconcile pass, because 121s have passed since the
// last one against the restored 60s cadence (hivecommons/hive#15779).
if len(h.sources) != 2 || h.sources[1] != evalSourceEnumerate {
t.Fatalf("sources = %v, want first cycle + one overdue reconcile pass", h.sources)
}
if !strings.Contains(log.String(), "eval interval elapsed without a reconcile pass; enumerating") {
t.Fatalf("missing overdue reconcile log: %s", log.String())
}
}
23 changes: 21 additions & 2 deletions src/pkg/dashboard/stage_tracker_build.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,14 @@ func (s *Server) buildStageTracker(status *StatusPayload, store *stagelabels.Sto
if store == nil {
return stagelabels.Record{}, false
}
return store.Get(stagelabels.Key(repo, number))
rec, ok := store.Get(stagelabels.Key(repo, number))
if ok && rec.Stage == stagelabels.None {
// A stage-less record only remembers the blocked sweep's lift
// (hivecommons/hive#15779): the row is still unstaged, but its
// since-time starts at the lift rather than at creation.
return stagelabels.Record{UnblockedAt: rec.UnblockedAt}, false
}
return rec, ok
}
presence, maintainerPolicy := s.stageTrackerPresence(), s.stageTrackerMaintainerPolicy()
fixWork := s.stageTrackerFixWorkLookup()
Expand Down Expand Up @@ -113,7 +120,7 @@ func (s *Server) buildStageTracker(status *StatusPayload, store *stagelabels.Sto
}
if hasRec && rec.Stage != stagelabels.None {
stage = rec.Stage
t := rec.EnteredAt
t := rec.StageSinceAt()
it.StageSince = &t
it.Reason = rec.Reason
stageTrackerRegression(it, rec)
Expand Down Expand Up @@ -316,6 +323,7 @@ func (s *Server) buildStageTracker(status *StatusPayload, store *stagelabels.Sto
finish(&it, issue.Labels, stage, rec, ok, false)
stageTrackerBlockedBy(&it, issue.OpenBlockers())
stageTrackerBlockedOnPR(&it, issue.Body, bucketReason, status.GitHubBaseURL, prState)
stageTrackerSinceUnblock(&it, rec, ok)
advisory := stageTrackerAdvisoryFor(issue.Labels, issue.Body, advisoryHint)
if advisory == nil {
applyStageTrackerNew(&it, stageTrackerNewState(it, ok, issueFilter, status.Governor, now))
Expand Down Expand Up @@ -938,3 +946,14 @@ func stageTrackerParkedOnDecision(it StageTrackerItem) bool {
lower := strings.ToLower(strings.TrimSpace(base))
return lower == stageengine.StageReasonVerdictRequiresHuman || lower == stageengine.StageReasonDecisionRequested
}

// stageTrackerSinceUnblock ages an unstaged, no-longer-blocked issue from
// the blocked sweep's lift rather than from creation
// (hivecommons/hive#15779). A row blocked again keeps its own age.
func stageTrackerSinceUnblock(it *StageTrackerItem, rec stagelabels.Record, hasRec bool) {
if hasRec || rec.UnblockedAt.IsZero() || it.BlockedOnPR != nil || len(it.BlockedOn) > 0 || slices.Contains(it.Modifiers, modBlocked) {
return
}
t := rec.UnblockedAt
it.StageSince = &t
}
61 changes: 61 additions & 0 deletions src/pkg/dashboard/stage_tracker_unblocked_since_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package dashboard

import (
"testing"
"time"

"github.com/hivecommons/hive/pkg/github"
"github.com/hivecommons/hive/pkg/stagelabels"
)

// An issue the blocked sweep just unblocked has no stage record yet; its
// row must age from the lift, not from issue creation
// (hivecommons/hive#15779).
func TestStageTrackerNewRowAgesFromUnblock(t *testing.T) {
now := time.Date(2026, 10, 30, 12, 0, 0, 0, time.UTC)
s, _ := stageTrackerFixture(t, now)
clock := now
store := stagelabels.NewStore("", func() time.Time { return clock })
s.stageStore = store
created := now.Add(-17 * time.Hour)
lifted := now.Add(-20 * time.Minute)
s.statusMu.Lock()
s.status.Repos[0].ActionableIssues = append(s.status.Repos[0].ActionableIssues,
github.Issue{Number: 15517, Title: "Was blocked", Author: "pat", AuthorIsHuman: true, CreatedAt: created, UpdatedAt: created})
s.statusMu.Unlock()

resp := s.buildStageTracker(s.status, store, now)
it := trackerItem(t, resp, "issue", 15517)
if it.StageSlug != StageTrackerNewSlug || it.StageSince == nil || !it.StageSince.Equal(created) {
t.Fatalf("before the lift: slug %q since %v, want 000-new since creation %v", it.StageSlug, it.StageSince, created)
}

store.NoteUnblocked(stagelabels.Key("acme/console", 15517), lifted)
resp = s.buildStageTracker(s.status, store, now)
it = trackerItem(t, resp, "issue", 15517)
if it.StageSlug != StageTrackerNewSlug || it.Stage != 0 || it.New == nil {
t.Fatalf("after the lift the row must still be 000-new: stage %d slug %q new %+v", it.Stage, it.StageSlug, it.New)
}
if it.StageSince == nil || !it.StageSince.Equal(lifted) {
t.Fatalf("after the lift since = %v, want the lift %v", it.StageSince, lifted)
}

// The stage-label pass that follows may still decide None for the
// issue (SyncIssueStageLabels β†’ ForgetStage); the lift survives it.
store.ForgetStage(stagelabels.Key("acme/console", 15517))
resp = s.buildStageTracker(s.status, store, now)
it = trackerItem(t, resp, "issue", 15517)
if it.StageSlug != StageTrackerNewSlug || it.StageSince == nil || !it.StageSince.Equal(lifted) {
t.Fatalf("after a None pass: slug %q since %v, want 000-new since the lift %v", it.StageSlug, it.StageSince, lifted)
}

// The next stage-label pass stages it: since is the stage entry.
staged := now.Add(-5 * time.Minute)
clock = staged
store.Observe(stagelabels.Key("acme/console", 15517), stagelabels.Actionable, "actionable")
resp = s.buildStageTracker(s.status, store, now)
it = trackerItem(t, resp, "issue", 15517)
if it.Stage != stagelabels.Actionable.Number() || it.StageSince == nil || !it.StageSince.Equal(staged) {
t.Fatalf("after staging: stage %d since %v, want 010 since %v", it.Stage, it.StageSince, staged)
}
}
2 changes: 1 addition & 1 deletion src/pkg/github/issue_stage_labels.go
Original file line number Diff line number Diff line change
Expand Up @@ -477,7 +477,7 @@ func SyncIssueStageLabelsWith(ctx context.Context, client *Client, result *Actio
var want stagelabels.Desired
var tr stagelabels.Transition
if stage == stagelabels.None {
store.Forget(key)
store.ForgetStage(key)
} else {
tr = store.ObserveProgress(key, stage, reason, d.Progress)
noteIssueCoveringPR(client, store, key, in, stage)
Expand Down
5 changes: 5 additions & 0 deletions src/pkg/github/item_refresh_hook.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@ package github
import (
"context"
"sync"
"time"

"github.com/hivecommons/hive/pkg/stagelabels"
)

// ItemRefreshHook is told, with the configured repo name and the item number,
Expand Down Expand Up @@ -47,6 +50,8 @@ func (c *Client) unblockIssue(ctx context.Context, repo string, number int) erro
if err := c.RemoveLabel(ctx, repo, number, BlockedLabel); err != nil {
return err
}
// The row's since-time restarts at the lift (hivecommons/hive#15779).
stagelabels.Shared().NoteUnblocked(stagelabels.Key(repo, number), time.Now())
notifyItemRefresh(repo, number, false)
return nil
}
Loading
Loading