Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-16537-fixer-queue.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Deferred PR fix kicks now retain their queue position until delivery or closure; the governor API exposes the queue and the dashboard review queue shows “fixer queued, N ahead” ([#16537](https://github.com/hivecommons/hive/issues/16537)).
6 changes: 6 additions & 0 deletions src/cmd/hive/eval_cycle_seams.go
Original file line number Diff line number Diff line change
Expand Up @@ -482,6 +482,9 @@ type kickDispatchDeps struct {
// continuous-mode scheduler uses this to exponential-backoff instead of
// returning the same due agent on every eval while the CLI is stalled.
onFailed func(msg scheduler.KickMessage, err error)
// onDeferred optionally records kicks that wait for a busy agent's lane.
// It runs in message order, like the delivered and failed effects.
onDeferred func(msg scheduler.KickMessage)
// markProbeReleased stamps the provider-budget probe. Called at most once
// per dispatch, and only after a kick actually goes out.
markProbeReleased func(at time.Time)
Expand Down Expand Up @@ -609,6 +612,9 @@ func dispatchAgentKicks(msgs []scheduler.KickMessage, releaseProbe bool, deps ki
if out.deferred || errors.Is(out.sendErr, crux.ErrAgentBusy) {
endSpan(nil)
logger.Info("agent busy: deferring kick", "agent", msg.Agent)
if deps.onDeferred != nil {
deps.onDeferred(msg)
}
continue
}
if out.startErr != nil {
Expand Down
90 changes: 90 additions & 0 deletions src/cmd/hive/fixer_queue.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
package main

import (
"strconv"
"strings"
"time"

"github.com/hivecommons/hive/pkg/github"
"github.com/hivecommons/hive/pkg/governor"
"github.com/hivecommons/hive/pkg/review"
"github.com/hivecommons/hive/pkg/scheduler"
)

// reviewFixKickQueue maps delivery outcomes back to PRs. Review and routine
// governor kicks must not create or clear a PR's deferred-fixer hint.
type reviewFixKickQueue struct {
governor *governor.Governor
org string
kicks map[string]review.DispatchKick
}

func (q reviewFixKickQueue) Deferred(msg scheduler.KickMessage) {
if k, ok := q.kicks[reviewKickKey(msg)]; ok && k.Kind == "fix" {
q.governor.RecordFixKickDeferred(fixerQueueKey(q.org, k), time.Now())
}
}

func (q reviewFixKickQueue) Delivered(msg scheduler.KickMessage) {
if k, ok := q.kicks[reviewKickKey(msg)]; ok && k.Kind == "fix" {
q.governor.RecordFixKickDelivered(fixerQueueKey(q.org, k))
}
}

// fixerQueueKey is the governor's identity for a fix kick's PR. Dispatch plans
// carry pr.Repo as configured (often the bare "repo"), while the enumeration
// reconcile and /api/review/queue key by github.ReviewQueueKey("org/repo", N).
// Every writer and reader must agree on the qualified form, or a bare-repo
// entry is evicted on the next reconcile and the dashboard never finds it.
func fixerQueueKey(org string, k review.DispatchKick) string {
repo, number := k.Repo, k.Number
if repo == "" || number == 0 {
// Older callers (and tests) only populate PRRef.
ref := strings.TrimSpace(k.PRRef)
if i := strings.LastIndex(ref, "#"); i > 0 {
repo = ref[:i]
if n, err := strconv.Atoi(ref[i+1:]); err == nil {
number = n
}
}
}
repo = strings.TrimSpace(repo)
if org != "" && repo != "" && !strings.Contains(repo, "/") {
repo = org + "/" + repo
}
return github.ReviewQueueKey(repo, number)
}

func reconcileDeferredFixQueue(gov *governor.Governor, org string, actionable *github.ActionableResult) {
if actionable == nil {
return // no new enumeration is not evidence that a PR closed
}
open := map[string]bool{}
for _, prs := range [][]github.PullRequest{actionable.PRs.Items, actionable.PRs.Held, actionable.PRs.StaleDrafts, actionable.PRs.Attributed} {
for _, pr := range prs {
if strings.EqualFold(pr.State, "closed") || !pr.ClosedAt.IsZero() || !pr.MergedAt.IsZero() {
continue
}
repo := pr.Repo
if org != "" && !strings.Contains(repo, "/") {
repo = org + "/" + repo
}
open[github.ReviewQueueKey(repo, pr.Number)] = true
}
}
gov.ReconcileFixerQueue(open)
}

// reconcilePlannedFixQueue drops deferred fixes the current plan no longer
// kicks (new head, review passed, lane reassigned away) for PRs that stay
// open; otherwise they would sit in the queue and inflate every "ahead".
func reconcilePlannedFixQueue(gov *governor.Governor, org string, actionable *github.ActionableResult, fixKicks []review.DispatchKick) {
if actionable == nil {
return // no plan was built, so nothing says the fix is no longer needed
}
planned := map[string]bool{}
for _, k := range fixKicks {
planned[fixerQueueKey(org, k)] = true
}
gov.ReconcileFixerQueue(planned)
}
185 changes: 185 additions & 0 deletions src/cmd/hive/fixer_queue_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,185 @@
package main

import (
"errors"
"fmt"
"testing"
"time"

"github.com/hivecommons/hive/crux"
"github.com/hivecommons/hive/pkg/github"
"github.com/hivecommons/hive/pkg/governor"
"github.com/hivecommons/hive/pkg/review"
"github.com/hivecommons/hive/pkg/scheduler"
)

func TestDispatchDeferredFixQueueLifecycle(t *testing.T) {
for _, busy := range []bool{false, true} {
t.Run(fmt.Sprintf("busy=%v", busy), func(t *testing.T) {
gov := &governor.Governor{}
msgs := []scheduler.KickMessage{
{Agent: "scanner", Message: "fix first"},
{Agent: "scanner", Message: "fix second"},
{Agent: "scanner", Message: "review third"},
{Agent: "scanner", Message: "routine governor"},
}
q := reviewFixKickQueue{governor: gov, kicks: map[string]review.DispatchKick{
reviewKickKey(msgs[0]): {Kind: "fix", PRRef: "org/repo#1"},
reviewKickKey(msgs[1]): {Kind: "fix", PRRef: "org/repo#2"},
reviewKickKey(msgs[2]): {Kind: "review", PRRef: "org/repo#3"},
}}
r := newKickDispatchRecorder()
if busy {
r.sendErr["scanner"] = fmt.Errorf("prompt wait: %w", crux.ErrAgentBusy)
}
deps := r.deps()
deps.onDeferred = q.Deferred
deps.onReviewDelivered = q.Delivered
dispatchAgentKicks(msgs, false, deps, discardLogger())
queue := gov.FixerQueue()
if busy {
if len(queue) != 2 || queue["org/repo#1"].FixerQueuePosition != 1 || queue["org/repo#2"].FixerQueuePosition != 2 {
t.Fatalf("busy lane queue = %+v", queue)
}
// Reassign the first fix to a fallback and actually deliver it.
fallback := scheduler.KickMessage{Agent: "fallback", Message: "new fix prompt"}
q.kicks[reviewKickKey(fallback)] = review.DispatchKick{Kind: "fix", PRRef: "org/repo#1"}
dispatchAgentKicks([]scheduler.KickMessage{fallback}, false, deps, discardLogger())
} else if len(queue) != 1 {
t.Fatalf("kick behind delivered first fix must be queued: %+v", queue)
}
// Delivery from another source must not clear a queued fix.
q.kicks[reviewKickKey(msgs[2])] = review.DispatchKick{Kind: "review", PRRef: "org/repo#2"}
q.Delivered(msgs[2])
q.Delivered(msgs[3])
queue = gov.FixerQueue()
second := queue["org/repo#2"]
if len(queue) != 1 || second.FixerQueuePosition != 1 || second.FixerQueueAhead == nil || *second.FixerQueueAhead != 0 {
t.Fatalf("after first delivery: %+v", queue)
}
delete(r.sendErr, "scanner")
dispatchAgentKicks(msgs[1:2], false, deps, discardLogger())
if len(gov.FixerQueue()) != 0 {
t.Fatal("delivered second fix still queued")
}
})
}
}

func TestFixQueueDoesNotRecordWithheldOrFailedKicks(t *testing.T) {
for _, withheld := range []bool{false, true} {
gov := &governor.Governor{}
msg := scheduler.KickMessage{Agent: "scanner", Message: "fix"}
q := reviewFixKickQueue{governor: gov, kicks: map[string]review.DispatchKick{
reviewKickKey(msg): {Kind: "fix", PRRef: "org/repo#1"},
}}
r := newKickDispatchRecorder()
r.backoff["scanner"] = withheld
r.sendErr["scanner"] = errors.New("send failed")
deps := r.deps()
deps.onDeferred = q.Deferred
deps.onReviewDelivered = q.Delivered
dispatchAgentKicks([]scheduler.KickMessage{msg}, false, deps, discardLogger())
if len(gov.FixerQueue()) != 0 {
t.Fatal("non-busy failure or provider backoff recorded as a deferred fix")
}
}
}

func TestReconcileDeferredFixQueueOpenHeldAndClosedPRs(t *testing.T) {
gov := &governor.Governor{}
for i := 1; i <= 6; i++ {
gov.RecordFixKickDeferred(fmt.Sprintf("org/repo#%d", i), time.Now())
}
reconcileDeferredFixQueue(gov, "org", nil)
if len(gov.FixerQueue()) != 6 {
t.Fatal("missing enumeration cleared queue")
}
reconcileDeferredFixQueue(gov, "org", &github.ActionableResult{PRs: github.PRResult{
Items: []github.PullRequest{{Repo: "repo", Number: 1}},
Held: []github.PullRequest{{Repo: "org/repo", Number: 2}},
StaleDrafts: []github.PullRequest{{Repo: "repo", Number: 3}},
Attributed: []github.PullRequest{
{Repo: "org/repo", Number: 4, State: "open"},
{Repo: "org/repo", Number: 5, State: "closed"},
{Repo: "org/repo", Number: 6, MergedAt: time.Now()},
},
}})
queue := gov.FixerQueue()
if len(queue) != 4 || queue["org/repo#4"].FixerQueuePosition != 4 {
t.Fatalf("reconciled queue = %+v", queue)
}
reconcileDeferredFixQueue(gov, "org", &github.ActionableResult{})
if len(gov.FixerQueue()) != 0 {
t.Fatal("closed or removed PRs still queued")
}
}

func TestReconcilePlannedFixQueueDropsPRsNoLongerPlanningAFix(t *testing.T) {
gov := &governor.Governor{}
gov.RecordFixKickDeferred("org/repo#1", time.Now())
gov.RecordFixKickDeferred("org/repo#2", time.Now().Add(time.Minute))
reconcilePlannedFixQueue(gov, "org", nil, nil)
if len(gov.FixerQueue()) != 2 {
t.Fatal("missing enumeration cleared queue")
}
// #1 stays open but its fix is no longer planned; #2 still needs one.
reconcilePlannedFixQueue(gov, "org", &github.ActionableResult{}, []review.DispatchKick{{Kind: "fix", PRRef: "Org/Repo#2"}})
queue := gov.FixerQueue()
second := queue["org/repo#2"]
if len(queue) != 1 || second.FixerQueuePosition != 1 || second.FixerQueueAhead == nil || *second.FixerQueueAhead != 0 {
t.Fatalf("planned queue = %+v", queue)
}
}

// Regression for hivecommons/hive#16537: dispatch plans carry the bare repo
// name from config, so a deferred fix recorded as "repo#N" used to be evicted
// by the next enumeration reconcile (keyed "org/repo#N") and never matched the
// dashboard's ReviewQueueKey lookup. Every path must agree on the qualified key.
func TestFixerQueueBareRepoKicksSurviveReconcileAndMatchDashboardKey(t *testing.T) {
gov := &governor.Governor{}
msg := scheduler.KickMessage{Agent: "scanner", Message: "fix bare"}
kick := review.DispatchKick{Kind: "fix", PRRef: "Repo#7", Repo: "Repo", Number: 7}
q := reviewFixKickQueue{governor: gov, org: "Org", kicks: map[string]review.DispatchKick{reviewKickKey(msg): kick}}
r := newKickDispatchRecorder()
r.sendErr["scanner"] = fmt.Errorf("prompt wait: %w", crux.ErrAgentBusy)
deps := r.deps()
deps.onDeferred = q.Deferred
deps.onReviewDelivered = q.Delivered
dispatchAgentKicks([]scheduler.KickMessage{msg}, false, deps, discardLogger())

want := github.ReviewQueueKey("org/repo", 7)
if got := fixerQueueKey("Org", kick); got != want {
t.Fatalf("fixerQueueKey = %q, want %q", got, want)
}
if got := fixerQueueKey("Org", review.DispatchKick{Kind: "fix", PRRef: "Repo#7"}); got != want {
t.Fatalf("fixerQueueKey from PRRef only = %q, want %q", got, want)
}
if got := fixerQueueKey("Org", review.DispatchKick{Kind: "fix", PRRef: "other/repo#7"}); got != "other/repo#7" {
t.Fatalf("already-qualified repo must not be re-prefixed: %q", got)
}
if queue := gov.FixerQueue(); queue[want].FixerQueuePosition != 1 {
t.Fatalf("deferred bare-repo fix not recorded under %q: %+v", want, queue)
}

// Several enumeration + plan cycles while the fixer stays busy must keep
// the entry (and its first-deferral position) rather than evicting it.
for i := 0; i < 3; i++ {
reconcileDeferredFixQueue(gov, "Org", &github.ActionableResult{PRs: github.PRResult{
Items: []github.PullRequest{{Repo: "Repo", Number: 7, State: "open"}},
}})
reconcilePlannedFixQueue(gov, "Org", &github.ActionableResult{}, []review.DispatchKick{kick})
dispatchAgentKicks([]scheduler.KickMessage{msg}, false, deps, discardLogger())
}
queue := gov.FixerQueue()
if len(queue) != 1 || queue[want].FixerQueuePosition != 1 {
t.Fatalf("bare-repo deferred fix evicted across cycles: %+v", queue)
}

// Delivery clears it under the same key.
delete(r.sendErr, "scanner")
dispatchAgentKicks([]scheduler.KickMessage{msg}, false, deps, discardLogger())
if len(gov.FixerQueue()) != 0 {
t.Fatalf("delivered bare-repo fix still queued: %+v", gov.FixerQueue())
}
}
5 changes: 5 additions & 0 deletions src/cmd/hive/main_eval_cycle.go
Original file line number Diff line number Diff line change
Expand Up @@ -574,6 +574,7 @@ func runEvalCycle(
actionable.Issues = workSourceIssuesForConfiguredCycle(ctx, cfg, ghClient, actionable.Issues, logger)
}
}
reconcileDeferredFixQueue(gov, cfg.Project.Org, actionable)
if swarmRepo := dashSrv.ActiveSwarmRepo(); swarmRepo != "" {
github.BoostActionableRepoPriority(actionable.Issues.Items, swarmRepo)
logger.Info("swarm priority boost applied", "repo", swarmRepo)
Expand Down Expand Up @@ -1003,6 +1004,7 @@ func runEvalCycle(
"agents_skipped", agentsSkipped,
)

reconcilePlannedFixQueue(gov, cfg.Project.Org, actionable, reviewPlan.FixKicks)
reviewKickByMessage := map[string]review.DispatchKick{}
for _, k := range append(reviewPlan.ReviewKicks, reviewPlan.FixKicks...) {
if !gov.AgentEligibleForCELKick(k.Agent) || agentMgr.IsPaused(k.Agent) {
Expand Down Expand Up @@ -1140,6 +1142,7 @@ func runEvalCycle(
// previously unreachable without a tmux session and a live governor.
var deliveredReviewKicks []review.DispatchKick
var deliveredKicks []scheduler.KickMessage
fixQueue := reviewFixKickQueue{governor: gov, org: cfg.Project.Org, kicks: reviewKickByMessage}
if len(messages) > 0 {
// governor.kick wraps the whole dispatch pass; each delivered kick
// still gets its own agent.kick child via startKickSpan below.
Expand Down Expand Up @@ -1182,11 +1185,13 @@ func runEvalCycle(
}
},
onReviewDelivered: func(msg scheduler.KickMessage) {
fixQueue.Delivered(msg)
if k, ok := reviewKickByMessage[msg.Agent+"\x00"+msg.Message]; ok {
deliveredReviewKicks = append(deliveredReviewKicks, k)
persistReviewDispatchStateInProgress(reviewPlan, deliveredReviewKicks, logger)
}
},
onDeferred: fixQueue.Deferred,
onFailed: func(msg scheduler.KickMessage, err error) {
gov.RecordKickFailure(msg.Agent, err, time.Now())
if k, ok := reviewKickByMessage[reviewKickKey(msg)]; ok {
Expand Down
15 changes: 9 additions & 6 deletions src/pkg/dashboard/api_governor_pr_models.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"time"

ghpkg "github.com/hivecommons/hive/pkg/github"
"github.com/hivecommons/hive/pkg/governor"
"github.com/hivecommons/hive/pkg/modelfamily"
)

Expand All @@ -21,12 +22,13 @@ const (
)

type governorPRModelsResponse struct {
Window string `json:"window"`
MinMergedPRs int `json:"min_merged_prs"`
Total int `json:"total"`
Buckets []governorPRModelBucket `json:"buckets"`
Unknown int `json:"unknown"`
MostReworked []governorPRMostReworkedItem `json:"most_reworked,omitempty"`
Window string `json:"window"`
MinMergedPRs int `json:"min_merged_prs"`
Total int `json:"total"`
Buckets []governorPRModelBucket `json:"buckets"`
Unknown int `json:"unknown"`
MostReworked []governorPRMostReworkedItem `json:"most_reworked,omitempty"`
FixerQueue map[string]governor.FixerQueueStatus `json:"fixer_queue,omitempty"`
}

// governorPRModelBucket is one canonical model (modelfamily.Canonical folds
Expand Down Expand Up @@ -111,6 +113,7 @@ func (s *Server) handleGovernorPRModels(w http.ResponseWriter, r *http.Request)
minMergedPRs := effectiveModelsMinMergedPRs()
actionable := s.lastActionableForPRModels()
resp := aggregateGovernorPRModels(actionable.PRs.Attributed, window, time.Now())
resp.FixerQueue = s.governorFixerQueue()
runs, _, _ := readEffectiveModelRuns(s.effectiveModelTaskRunLogPath(), governorPRModelsWindowDuration(window))
effective := aggregateEffectiveModelsFromPRAggregation(resp, aggregateEffectiveRunStats(runs, effectiveModelsFilterAll), effectiveModelsFilterAll, minMergedPRs)
applyGovernorPRModelEffectiveness(&resp, effective, minMergedPRs)
Expand Down
Loading
Loading