Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-14952.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- The Activity Monitor's `blocked` pill now names and links the PR an issue is waiting on ("blocked on #N"), and an issue the hive unblocks no longer reverts to `blocked` when a reconcile pass that started before the unblock finishes after it.
99 changes: 79 additions & 20 deletions src/cmd/hive/eval_cycle_source.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package main

import (
Expand Down Expand Up @@ -86,37 +86,96 @@

// apply upserts each snapshot into the stored result and returns how many
// changed it. No stored result (before the first enumeration) is a no-op:
// the first cycle lists everything anyway.
// the first cycle lists everything anyway. Each applied read is remembered
// on the published snapshot (WebhookReads) so a reconcile pass already in
// flight can replay it (publishReconciledSnapshot). The swap is a
// compare-and-swap: a reconcile result published meanwhile is patched
// rather than overwritten.
func (p *snapshotPatcher) apply(snaps []github.WebhookItemSnapshot) int {
if p == nil || p.target == nil || len(snaps) == 0 {
return 0
}
p.mu.Lock()
defer p.mu.Unlock()
cur := p.target.Load()
if cur == nil {
return 0
}
next := cur.Clone()
applied := 0
for _, snap := range snaps {
if next.ApplyWebhookItem(snap) {
applied++
for {
cur := p.target.Load()
if cur == nil {
return 0
}
next := cur.Clone()
now := time.Now()
applied := 0
for _, snap := range snaps {
if next.ApplyWebhookItem(snap) {
applied++
if snap.FetchedAt.IsZero() {
snap.FetchedAt = now
}
next.WebhookReads = append(next.WebhookReads, snap)
}
}
if applied == 0 {
return 0
}
next.WebhookReads = pruneWebhookReads(next.WebhookReads, now.Add(-webhookReadReplayWindow))
if !p.target.CompareAndSwap(cur, next) {
continue
}
if p.logger != nil {
p.logger.Info("webhook re-read patched the cached actionable snapshot",
"items", applied, "issues", next.Issues.Count, "prs", next.PRs.Count, "hold", next.Hold.Total,
"snapshot_age", time.Since(next.GeneratedAt).Round(time.Second).String())
}
if p.onPatch != nil {
p.onPatch(applied)
}
return applied
}
if applied == 0 {
return 0
}

// webhookReadReplayWindow bounds how long an applied webhook re-read is
// remembered for replay: far longer than any reconcile pass runs.
const webhookReadReplayWindow = 2 * time.Hour

// pruneWebhookReads drops reads fetched before cutoff, into a fresh slice.
func pruneWebhookReads(reads []github.WebhookItemSnapshot, cutoff time.Time) []github.WebhookItemSnapshot {
var out []github.WebhookItemSnapshot
for _, r := range reads {
if !r.FetchedAt.Before(cutoff) {
out = append(out, r)
}
}
p.target.Store(next)
if p.logger != nil {
p.logger.Info("webhook re-read patched the cached actionable snapshot",
"items", applied, "issues", next.Issues.Count, "prs", next.PRs.Count, "hold", next.Hold.Total,
"snapshot_age", time.Since(next.GeneratedAt).Round(time.Second).String())
return out
}

// publishReconciledSnapshot stores a reconcile pass's result. Webhook
// re-reads fetched after the pass started (passStart) describe a newer
// GitHub state than the pass may have listed — the hive's own `blocked`
// removal at 12:41 under a pass that listed at 12:38 — so they are replayed
// onto the result first (ApplyWebhookItem drops any the pass already saw a
// newer copy of) and kept on it for the next pass. Compare-and-swap, so a
// patch landing meanwhile is replayed too rather than lost
// (hivecommons/hive#14952).
func publishReconciledSnapshot(last *atomic.Pointer[github.ActionableResult], actionable *github.ActionableResult, passStart time.Time) {
if last == nil || actionable == nil {
return
}
if p.onPatch != nil {
p.onPatch(applied)
for {
prev := last.Load()
var reads []github.WebhookItemSnapshot
if prev != nil {
for _, r := range prev.WebhookReads {
if r.FetchedAt.After(passStart) {
actionable.ApplyWebhookItem(r)
reads = append(reads, r)
}
}
}
actionable.WebhookReads = reads
if last.CompareAndSwap(prev, actionable) {
return
}
}
return applied
}

// applyWebhookItemsFunc is the dashboard.Dependencies.ApplyWebhookItems hook.
Expand Down
5 changes: 4 additions & 1 deletion src/cmd/hive/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -7077,6 +7077,9 @@ func runEvalCycle(
return
}

// A reconcile pass replays webhook re-reads fetched after this instant
// onto its result before publishing it (publishReconciledSnapshot).
cycleStart := time.Now()
budgetMode, _ := ghClient.APIBudgetMode()
budgetDecision := decideEvalBudgetWork(budgetMode, cfg.Governor.OptionalSweepEveryNCycles, evalAPIBudgetCycle.Add(1))

Expand Down Expand Up @@ -7211,7 +7214,7 @@ func runEvalCycle(
// on a copy, and storing it back would overwrite a webhook patch that
// landed while the cycle ran.
if cachedSnapshot == nil {
lastActionable.Store(actionable)
publishReconciledSnapshot(lastActionable, actionable, cycleStart)
if data, err := json.Marshal(actionable); err == nil {
atomicWrite(lastActionablePath, data)
}
Expand Down
119 changes: 119 additions & 0 deletions src/cmd/hive/reconcile_webhook_replay_14952_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
package main

import (
"sync/atomic"
"testing"
"time"

"github.com/hivecommons/hive/pkg/github"
)

// blockedIssueSnapshot is a reconcile pass's copy of org/repo#1 still
// parked as blocked, listed at listedAt.
func blockedIssueSnapshot(listedAt time.Time) *github.ActionableResult {
return &github.ActionableResult{
GeneratedAt: listedAt,
TotalByRepo: map[string]github.RepoCounts{"org/repo": {Issues: 1}},
WorkDetailsByRepo: map[string]github.RepoWorkDetails{"org/repo": {Issues: []github.RepoWorkIssue{{
Repo: "org/repo", Number: 1, Title: "finding", Labels: []string{github.BlockedLabel},
Bucket: "filtered", Reason: "blocked on org/repo#9", UpdatedAt: listedAt.Add(-6 * time.Hour),
}}}},
}
}

// unblockRead is the webhook re-read after the hive removed `blocked`.
func unblockRead(updatedAt, fetchedAt time.Time) github.WebhookItemSnapshot {
return github.WebhookItemSnapshot{
Repo: "org/repo", Number: 1, State: "open", UpdatedAt: updatedAt, FetchedAt: fetchedAt,
Patch: github.WebhookItemPatch{Bucket: github.WebhookBucketActionable, Issue: &github.Issue{Repo: "org/repo", Number: 1, Title: "finding", UpdatedAt: updatedAt}},
}
}

func issueOneListedActionable(r *github.ActionableResult) bool {
for _, it := range r.Issues.Items {
if it.Repo == "org/repo" && it.Number == 1 {
return true
}
}
return false
}

func issueOneSidelinedBlocked(r *github.ActionableResult) bool {
for _, it := range r.WorkDetailsByRepo["org/repo"].Issues {
if it.Number == 1 {
return true
}
}
return false
}

// hivecommons/hive#14952: a reconcile pass that listed the issue before the
// hive's own unblock and finished after the webhook re-read patched the
// snapshot must not publish its pre-unblock copy over the patch.
func TestReconcilePassReplaysWebhookReadsFetchedAfterItStarted(t *testing.T) {
now := time.Now()
passStart := now.Add(-4 * time.Minute)

var last atomic.Pointer[github.ActionableResult]
last.Store(blockedIssueSnapshot(now.Add(-30 * time.Minute)))
p := &snapshotPatcher{target: &last}
if p.apply([]github.WebhookItemSnapshot{unblockRead(now.Add(-time.Minute), now.Add(-time.Minute))}) != 1 {
t.Fatal("unblock re-read not applied")
}
if !issueOneListedActionable(last.Load()) || len(last.Load().WebhookReads) != 1 {
t.Fatalf("patched snapshot does not carry the unblock: %+v", last.Load().Issues)
}

// The pass that started before the unblock now publishes its stale copy.
stale := blockedIssueSnapshot(passStart)
publishReconciledSnapshot(&last, stale, passStart)

got := last.Load()
if !issueOneListedActionable(got) || issueOneSidelinedBlocked(got) {
t.Fatalf("reconcile pass overwrote the webhook unblock: issues=%+v work=%+v", got.Issues, got.WorkDetailsByRepo)
}
if len(got.WebhookReads) != 1 {
t.Fatalf("replayed read not kept for the next pass: %d", len(got.WebhookReads))
}
}

// A read fetched before the pass started is already covered by the pass's
// own listing, which stays authoritative.
func TestReconcilePassDoesNotReplayReadsOlderThanItsStart(t *testing.T) {
now := time.Now()
var last atomic.Pointer[github.ActionableResult]
last.Store(blockedIssueSnapshot(now.Add(-time.Hour)))
p := &snapshotPatcher{target: &last}
p.apply([]github.WebhookItemSnapshot{unblockRead(now.Add(-20*time.Minute), now.Add(-20*time.Minute))})

passStart := now.Add(-10 * time.Minute)
fresh := blockedIssueSnapshot(passStart)
publishReconciledSnapshot(&last, fresh, passStart)

got := last.Load()
if got != fresh || !issueOneSidelinedBlocked(got) || issueOneListedActionable(got) {
t.Fatalf("pre-pass read replayed onto the reconcile result: %+v", got.Issues)
}
if len(got.WebhookReads) != 0 {
t.Fatalf("pre-pass reads must be dropped: %d", len(got.WebhookReads))
}
}

// A read whose updated_at is older than the pass's own copy is dropped by
// the monotonic guard even when fetched after the pass started.
func TestReconcilePassReplayKeepsNewerListedCopy(t *testing.T) {
now := time.Now()
passStart := now.Add(-4 * time.Minute)
var last atomic.Pointer[github.ActionableResult]
last.Store(&github.ActionableResult{GeneratedAt: now.Add(-time.Hour), TotalByRepo: map[string]github.RepoCounts{"org/repo": {}}})
p := &snapshotPatcher{target: &last}
if p.apply([]github.WebhookItemSnapshot{unblockRead(now.Add(-8*time.Hour), now.Add(-time.Minute))}) != 1 {
t.Fatal("read not applied to a snapshot that never listed the item")
}

fresh := blockedIssueSnapshot(passStart)
publishReconciledSnapshot(&last, fresh, passStart)
if got := last.Load(); !issueOneSidelinedBlocked(got) || issueOneListedActionable(got) {
t.Fatalf("older read replaced the pass's newer copy: %+v", got.Issues)
}
}
4 changes: 3 additions & 1 deletion src/cmd/hive/repos_rescan.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"errors"
"log/slog"
"sync/atomic"
"time"

"github.com/hivecommons/hive/pkg/config"
"github.com/hivecommons/hive/pkg/github"
Expand Down Expand Up @@ -53,6 +54,7 @@ func rescanRepos(
return nil, errNoForgeCredentials
}

passStart := time.Now()
actionable, err := ghClient.EnumerateActionable(ctx)
if err != nil {
// Same tolerance as the eval cycle: on a hive whose issues come from a
Expand All @@ -77,7 +79,7 @@ func rescanRepos(
ghClient.EnrichLinkedIssueTitles(ctx, actionable.WorkDetailsByRepo, actionable.PRs.Items, actionable.PRs.Held, actionable.PRs.StaleDrafts)
applyDuplicatePRGuard(ctx, cfg, ghClient, actionable, logger)

lastActionable.Store(actionable)
publishReconciledSnapshot(lastActionable, actionable, passStart)
publishStageSharedCIIncidents(actionable)
if data, marshalErr := json.Marshal(actionable); marshalErr == nil {
atomicWrite(lastActionablePath, data)
Expand Down
2 changes: 1 addition & 1 deletion src/docs/scale-envelope.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ ones to:
2. **Truncation must preserve the prioritization signal, not cut an arbitrary slice.**
This holds today and is easy to assume backwards:
- Issues are sorted by actionable rank tier, then **oldest first** within each tier before any cap is applied
(`RankActionableIssues`, `src/pkg/github/client.go:4759-4769`, descending `AgeMinutes`).
(`RankActionableIssues`, `src/pkg/github/client.go:4766-4776`, descending `AgeMinutes`).
- PRs are sorted by **review class then oldest-first within class**
(`SortPullRequestsForReview`, `src/pkg/github/review_priority.go:334`;
fixes → refactors/docs → tests).
Expand Down
15 changes: 11 additions & 4 deletions src/pkg/dashboard/stage_tracker.go
Original file line number Diff line number Diff line change
Expand Up @@ -340,10 +340,15 @@ type StageTrackerItem struct {
// (scheduler.partitionBlockedIssues). Without it a sequenced sub-issue
// read as plain Actionable nobody was picking up (hivecommons/hive#14765);
// the row wears the `blocked` modifier and the caption names the wait.
BlockedOn []string `json:"blocked_on,omitempty"`
NudgeCount int `json:"nudge_count,omitempty"`
LastNudgeAt *time.Time `json:"last_nudge_at,omitempty"`
AdoptedBy int `json:"adopted_by,omitempty"`
BlockedOn []string `json:"blocked_on,omitempty"`
// BlockedOnPR names the pull request a `blocked` issue waits on, read
// from the review-backlog marker `<!-- hive:blocked-on pr=owner/repo#N -->`
// (or the enumeration's "blocked on owner/repo#N" reason), so the pill
// and caption say what lifts the block (hivecommons/hive#14952).
BlockedOnPR *StageTrackerBlockedOnPR `json:"blocked_on_pr,omitempty"`
NudgeCount int `json:"nudge_count,omitempty"`
LastNudgeAt *time.Time `json:"last_nudge_at,omitempty"`
AdoptedBy int `json:"adopted_by,omitempty"`
// ClaimantKind is the author kind of whoever holds the live claim on a
// claimed issue (human / clanker / hive), from the worker-claim ledger.
ClaimantKind string `json:"claimant_kind,omitempty"`
Expand Down Expand Up @@ -1462,6 +1467,7 @@ func (s *Server) buildStageTracker(status *StatusPayload, store *stagelabels.Sto
}
finish(&it, issue.Labels, stage, rec, ok, false)
stageTrackerBlockedBy(&it, issue.OpenBlockers())
stageTrackerBlockedOnPR(&it, issue.Body, bucketReason, status.GitHubBaseURL)
if it.Stage == trackerNumberClaimed {
it.ClaimantKind = s.stageTrackerClaimantKind(repoName, issue.Number)
}
Expand Down Expand Up @@ -1536,6 +1542,7 @@ func (s *Server) buildStageTracker(status *StatusPayload, store *stagelabels.Sto
}
}
resp.OpenIssues, resp.OpenPRs = stageTrackerOpenCounts(resp.Items)
stageTrackerResolveBlockedOnPRState(resp.Items, resp.RecentlyMerged)
stageTrackerResolveLinkedPRs(resp.Items)
sort.SliceStable(resp.Items, stageTrackerLess(resp.Items))
return resp
Expand Down
66 changes: 66 additions & 0 deletions src/pkg/dashboard/stage_tracker_blocked_on_pr.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
package dashboard

import (
"slices"
"strconv"
"strings"

"github.com/hivecommons/hive/pkg/github"
)

// StageTrackerBlockedOnPR is the pull request a `blocked` issue waits on
// (hivecommons/hive#14952). State is "open" or "merged" when the hive's own
// caches know it (the status snapshot's open PRs, which the forgestate
// overlay feeds, and the recent-merge ring); "" when they do not. No GitHub
// call is ever made to fill it.
type StageTrackerBlockedOnPR struct {
Repo string `json:"repo"`
Number int `json:"number"`
URL string `json:"url,omitempty"`
State string `json:"state,omitempty"`
}

// stageTrackerBlockedOnPR names the PR a `blocked` issue waits on, from the
// review-backlog marker in its body when the row carries one, else from the
// enumeration's filtered reason or the stage reason ("blocked on
// owner/repo#N"). The caption then says what lifts the block instead of
// "No stage recorded yet".
func stageTrackerBlockedOnPR(it *StageTrackerItem, body, bucketReason, githubBaseURL string) {
if it == nil || it.Kind != "issue" || !slices.ContainsFunc(it.Labels, func(l string) bool { return strings.EqualFold(strings.TrimSpace(l), github.BlockedLabel) }) {
return
}
repo, number, ok := github.ParseBlockedOnMarker(body)
if !ok {
repo, number, ok = github.ParseBlockedFilterReason(bucketReason)
}
if !ok {
repo, number, ok = github.ParseBlockedFilterReason(it.Reason)
}
if !ok {
return
}
it.BlockedOnPR = &StageTrackerBlockedOnPR{Repo: repo, Number: number, URL: overviewItemURL("", repo, number, "pr", githubBaseURL)}
it.Caption = "Blocked on PR " + stageTrackerBlockerList(it.Repo, []string{repo + "#" + strconv.Itoa(number)}) + " — lifts when it merges"
it.Actor = "its blocking PR"
}

// stageTrackerResolveBlockedOnPRState fills each BlockedOnPR's state from
// what the response already holds: a PR listed as a row is open, one in the
// recent-merge ring merged.
func stageTrackerResolveBlockedOnPRState(items []StageTrackerItem, merged []StageTrackerMerged) {
key := func(repo string, number int) string { return strings.ToLower(repo) + "#" + strconv.Itoa(number) }
state := map[string]string{}
for _, it := range items {
if it.Kind == "pr" {
state[key(it.Repo, it.Number)] = "open"
}
}
for _, m := range merged {
state[key(m.Repo, m.Number)] = "merged"
}
for i := range items {
if b := items[i].BlockedOnPR; b != nil {
b.State = state[key(b.Repo, b.Number)]
}
}
}
Loading
Loading