Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-16537-fixer-queue.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Deferred PR fix kicks now retain their queue position until delivery or closure; the governor API exposes the queue and the dashboard review queue shows “fixer queued, N ahead” ([#16537](https://github.com/hivecommons/hive/issues/16537)).
6 changes: 6 additions & 0 deletions src/cmd/hive/eval_cycle_seams.go
Original file line number Diff line number Diff line change
Expand Up @@ -482,6 +482,9 @@ type kickDispatchDeps struct {
// continuous-mode scheduler uses this to exponential-backoff instead of
// returning the same due agent on every eval while the CLI is stalled.
onFailed func(msg scheduler.KickMessage, err error)
// onDeferred optionally records kicks that wait for a busy agent's lane.
// It runs in message order, like the delivered and failed effects.
onDeferred func(msg scheduler.KickMessage)
// markProbeReleased stamps the provider-budget probe. Called at most once
// per dispatch, and only after a kick actually goes out.
markProbeReleased func(at time.Time)
Expand Down Expand Up @@ -609,6 +612,9 @@ func dispatchAgentKicks(msgs []scheduler.KickMessage, releaseProbe bool, deps ki
if out.deferred || errors.Is(out.sendErr, crux.ErrAgentBusy) {
endSpan(nil)
logger.Info("agent busy: deferring kick", "agent", msg.Agent)
if deps.onDeferred != nil {
deps.onDeferred(msg)
}
continue
}
if out.startErr != nil {
Expand Down
64 changes: 64 additions & 0 deletions src/cmd/hive/fixer_queue.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package main

import (
"strings"
"time"

"github.com/hivecommons/hive/pkg/github"
"github.com/hivecommons/hive/pkg/governor"
"github.com/hivecommons/hive/pkg/review"
"github.com/hivecommons/hive/pkg/scheduler"
)

// reviewFixKickQueue maps delivery outcomes back to PRs. Review and routine
// governor kicks must not create or clear a PR's deferred-fixer hint.
type reviewFixKickQueue struct {
governor *governor.Governor
kicks map[string]review.DispatchKick
}

func (q reviewFixKickQueue) Deferred(msg scheduler.KickMessage) {
if k, ok := q.kicks[reviewKickKey(msg)]; ok && k.Kind == "fix" {
q.governor.RecordFixKickDeferred(k.PRRef, time.Now())
}
}

func (q reviewFixKickQueue) Delivered(msg scheduler.KickMessage) {
if k, ok := q.kicks[reviewKickKey(msg)]; ok && k.Kind == "fix" {
q.governor.RecordFixKickDelivered(k.PRRef)
}
}

func reconcileDeferredFixQueue(gov *governor.Governor, org string, actionable *github.ActionableResult) {
if actionable == nil {
return // no new enumeration is not evidence that a PR closed
}
open := map[string]bool{}
for _, prs := range [][]github.PullRequest{actionable.PRs.Items, actionable.PRs.Held, actionable.PRs.StaleDrafts, actionable.PRs.Attributed} {
for _, pr := range prs {
if strings.EqualFold(pr.State, "closed") || !pr.ClosedAt.IsZero() || !pr.MergedAt.IsZero() {
continue
}
repo := pr.Repo
if org != "" && !strings.Contains(repo, "/") {
repo = org + "/" + repo
}
open[github.ReviewQueueKey(repo, pr.Number)] = true
}
}
gov.ReconcileFixerQueue(open)
}

// reconcilePlannedFixQueue drops deferred fixes the current plan no longer
// kicks (new head, review passed, lane reassigned away) for PRs that stay
// open; otherwise they would sit in the queue and inflate every "ahead".
func reconcilePlannedFixQueue(gov *governor.Governor, actionable *github.ActionableResult, fixKicks []review.DispatchKick) {
if actionable == nil {
return // no plan was built, so nothing says the fix is no longer needed
}
planned := map[string]bool{}
for _, k := range fixKicks {
planned[strings.ToLower(strings.TrimSpace(k.PRRef))] = true
}
gov.ReconcileFixerQueue(planned)
}
133 changes: 133 additions & 0 deletions src/cmd/hive/fixer_queue_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
package main

import (
"errors"
"fmt"
"testing"
"time"

"github.com/hivecommons/hive/crux"
"github.com/hivecommons/hive/pkg/github"
"github.com/hivecommons/hive/pkg/governor"
"github.com/hivecommons/hive/pkg/review"
"github.com/hivecommons/hive/pkg/scheduler"
)

func TestDispatchDeferredFixQueueLifecycle(t *testing.T) {
for _, busy := range []bool{false, true} {
t.Run(fmt.Sprintf("busy=%v", busy), func(t *testing.T) {
gov := &governor.Governor{}
msgs := []scheduler.KickMessage{
{Agent: "scanner", Message: "fix first"},
{Agent: "scanner", Message: "fix second"},
{Agent: "scanner", Message: "review third"},
{Agent: "scanner", Message: "routine governor"},
}
q := reviewFixKickQueue{governor: gov, kicks: map[string]review.DispatchKick{
reviewKickKey(msgs[0]): {Kind: "fix", PRRef: "org/repo#1"},
reviewKickKey(msgs[1]): {Kind: "fix", PRRef: "org/repo#2"},
reviewKickKey(msgs[2]): {Kind: "review", PRRef: "org/repo#3"},
}}
r := newKickDispatchRecorder()
if busy {
r.sendErr["scanner"] = fmt.Errorf("prompt wait: %w", crux.ErrAgentBusy)
}
deps := r.deps()
deps.onDeferred = q.Deferred
deps.onReviewDelivered = q.Delivered
dispatchAgentKicks(msgs, false, deps, discardLogger())
queue := gov.FixerQueue()
if busy {
if len(queue) != 2 || queue["org/repo#1"].FixerQueuePosition != 1 || queue["org/repo#2"].FixerQueuePosition != 2 {
t.Fatalf("busy lane queue = %+v", queue)
}
// Reassign the first fix to a fallback and actually deliver it.
fallback := scheduler.KickMessage{Agent: "fallback", Message: "new fix prompt"}
q.kicks[reviewKickKey(fallback)] = review.DispatchKick{Kind: "fix", PRRef: "org/repo#1"}
dispatchAgentKicks([]scheduler.KickMessage{fallback}, false, deps, discardLogger())
} else if len(queue) != 1 {
t.Fatalf("kick behind delivered first fix must be queued: %+v", queue)
}
// Delivery from another source must not clear a queued fix.
q.kicks[reviewKickKey(msgs[2])] = review.DispatchKick{Kind: "review", PRRef: "org/repo#2"}
q.Delivered(msgs[2])
q.Delivered(msgs[3])
queue = gov.FixerQueue()
second := queue["org/repo#2"]
if len(queue) != 1 || second.FixerQueuePosition != 1 || second.FixerQueueAhead == nil || *second.FixerQueueAhead != 0 {
t.Fatalf("after first delivery: %+v", queue)
}
delete(r.sendErr, "scanner")
dispatchAgentKicks(msgs[1:2], false, deps, discardLogger())
if len(gov.FixerQueue()) != 0 {
t.Fatal("delivered second fix still queued")
}
})
}
}

func TestFixQueueDoesNotRecordWithheldOrFailedKicks(t *testing.T) {
for _, withheld := range []bool{false, true} {
gov := &governor.Governor{}
msg := scheduler.KickMessage{Agent: "scanner", Message: "fix"}
q := reviewFixKickQueue{governor: gov, kicks: map[string]review.DispatchKick{
reviewKickKey(msg): {Kind: "fix", PRRef: "org/repo#1"},
}}
r := newKickDispatchRecorder()
r.backoff["scanner"] = withheld
r.sendErr["scanner"] = errors.New("send failed")
deps := r.deps()
deps.onDeferred = q.Deferred
deps.onReviewDelivered = q.Delivered
dispatchAgentKicks([]scheduler.KickMessage{msg}, false, deps, discardLogger())
if len(gov.FixerQueue()) != 0 {
t.Fatal("non-busy failure or provider backoff recorded as a deferred fix")
}
}
}

func TestReconcileDeferredFixQueueOpenHeldAndClosedPRs(t *testing.T) {
gov := &governor.Governor{}
for i := 1; i <= 6; i++ {
gov.RecordFixKickDeferred(fmt.Sprintf("org/repo#%d", i), time.Now())
}
reconcileDeferredFixQueue(gov, "org", nil)
if len(gov.FixerQueue()) != 6 {
t.Fatal("missing enumeration cleared queue")
}
reconcileDeferredFixQueue(gov, "org", &github.ActionableResult{PRs: github.PRResult{
Items: []github.PullRequest{{Repo: "repo", Number: 1}},
Held: []github.PullRequest{{Repo: "org/repo", Number: 2}},
StaleDrafts: []github.PullRequest{{Repo: "repo", Number: 3}},
Attributed: []github.PullRequest{
{Repo: "org/repo", Number: 4, State: "open"},
{Repo: "org/repo", Number: 5, State: "closed"},
{Repo: "org/repo", Number: 6, MergedAt: time.Now()},
},
}})
queue := gov.FixerQueue()
if len(queue) != 4 || queue["org/repo#4"].FixerQueuePosition != 4 {
t.Fatalf("reconciled queue = %+v", queue)
}
reconcileDeferredFixQueue(gov, "org", &github.ActionableResult{})
if len(gov.FixerQueue()) != 0 {
t.Fatal("closed or removed PRs still queued")
}
}

func TestReconcilePlannedFixQueueDropsPRsNoLongerPlanningAFix(t *testing.T) {
gov := &governor.Governor{}
gov.RecordFixKickDeferred("org/repo#1", time.Now())
gov.RecordFixKickDeferred("org/repo#2", time.Now().Add(time.Minute))
reconcilePlannedFixQueue(gov, nil, nil)
if len(gov.FixerQueue()) != 2 {
t.Fatal("missing enumeration cleared queue")
}
// #1 stays open but its fix is no longer planned; #2 still needs one.
reconcilePlannedFixQueue(gov, &github.ActionableResult{}, []review.DispatchKick{{Kind: "fix", PRRef: "Org/Repo#2"}})
queue := gov.FixerQueue()
second := queue["org/repo#2"]
if len(queue) != 1 || second.FixerQueuePosition != 1 || second.FixerQueueAhead == nil || *second.FixerQueueAhead != 0 {
t.Fatalf("planned queue = %+v", queue)
}
}
5 changes: 5 additions & 0 deletions src/cmd/hive/main_eval_cycle.go
Original file line number Diff line number Diff line change
Expand Up @@ -574,6 +574,7 @@ func runEvalCycle(
actionable.Issues = workSourceIssuesForConfiguredCycle(ctx, cfg, ghClient, actionable.Issues, logger)
}
}
reconcileDeferredFixQueue(gov, cfg.Project.Org, actionable)
if swarmRepo := dashSrv.ActiveSwarmRepo(); swarmRepo != "" {
github.BoostActionableRepoPriority(actionable.Issues.Items, swarmRepo)
logger.Info("swarm priority boost applied", "repo", swarmRepo)
Expand Down Expand Up @@ -1003,6 +1004,7 @@ func runEvalCycle(
"agents_skipped", agentsSkipped,
)

reconcilePlannedFixQueue(gov, actionable, reviewPlan.FixKicks)
reviewKickByMessage := map[string]review.DispatchKick{}
for _, k := range append(reviewPlan.ReviewKicks, reviewPlan.FixKicks...) {
if !gov.AgentEligibleForCELKick(k.Agent) || agentMgr.IsPaused(k.Agent) {
Expand Down Expand Up @@ -1140,6 +1142,7 @@ func runEvalCycle(
// previously unreachable without a tmux session and a live governor.
var deliveredReviewKicks []review.DispatchKick
var deliveredKicks []scheduler.KickMessage
fixQueue := reviewFixKickQueue{governor: gov, kicks: reviewKickByMessage}
if len(messages) > 0 {
// governor.kick wraps the whole dispatch pass; each delivered kick
// still gets its own agent.kick child via startKickSpan below.
Expand Down Expand Up @@ -1182,11 +1185,13 @@ func runEvalCycle(
}
},
onReviewDelivered: func(msg scheduler.KickMessage) {
fixQueue.Delivered(msg)
if k, ok := reviewKickByMessage[msg.Agent+"\x00"+msg.Message]; ok {
deliveredReviewKicks = append(deliveredReviewKicks, k)
persistReviewDispatchStateInProgress(reviewPlan, deliveredReviewKicks, logger)
}
},
onDeferred: fixQueue.Deferred,
onFailed: func(msg scheduler.KickMessage, err error) {
gov.RecordKickFailure(msg.Agent, err, time.Now())
if k, ok := reviewKickByMessage[reviewKickKey(msg)]; ok {
Expand Down
15 changes: 9 additions & 6 deletions src/pkg/dashboard/api_governor_pr_models.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"time"

ghpkg "github.com/hivecommons/hive/pkg/github"
"github.com/hivecommons/hive/pkg/governor"
"github.com/hivecommons/hive/pkg/modelfamily"
)

Expand All @@ -21,12 +22,13 @@ const (
)

type governorPRModelsResponse struct {
Window string `json:"window"`
MinMergedPRs int `json:"min_merged_prs"`
Total int `json:"total"`
Buckets []governorPRModelBucket `json:"buckets"`
Unknown int `json:"unknown"`
MostReworked []governorPRMostReworkedItem `json:"most_reworked,omitempty"`
Window string `json:"window"`
MinMergedPRs int `json:"min_merged_prs"`
Total int `json:"total"`
Buckets []governorPRModelBucket `json:"buckets"`
Unknown int `json:"unknown"`
MostReworked []governorPRMostReworkedItem `json:"most_reworked,omitempty"`
FixerQueue map[string]governor.FixerQueueStatus `json:"fixer_queue,omitempty"`
}

// governorPRModelBucket is one canonical model (modelfamily.Canonical folds
Expand Down Expand Up @@ -111,6 +113,7 @@ func (s *Server) handleGovernorPRModels(w http.ResponseWriter, r *http.Request)
minMergedPRs := effectiveModelsMinMergedPRs()
actionable := s.lastActionableForPRModels()
resp := aggregateGovernorPRModels(actionable.PRs.Attributed, window, time.Now())
resp.FixerQueue = s.governorFixerQueue()
runs, _, _ := readEffectiveModelRuns(s.effectiveModelTaskRunLogPath(), governorPRModelsWindowDuration(window))
effective := aggregateEffectiveModelsFromPRAggregation(resp, aggregateEffectiveRunStats(runs, effectiveModelsFilterAll), effectiveModelsFilterAll, minMergedPRs)
applyGovernorPRModelEffectiveness(&resp, effective, minMergedPRs)
Expand Down
25 changes: 22 additions & 3 deletions src/pkg/dashboard/api_review_queue.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"time"

ghpkg "github.com/hivecommons/hive/pkg/github"
"github.com/hivecommons/hive/pkg/governor"
"github.com/hivecommons/hive/pkg/review"
"github.com/hivecommons/hive/pkg/stagelabels"
)
Expand Down Expand Up @@ -51,7 +52,19 @@ type reviewQueueResponse struct {
// needs attention (hivecommons/hive#12534).
HiveHeld int `json:"hive_held"`
History []ReviewQueueHistoryEntry `json:"history"`
Items []ghpkg.ReviewQueueEntry `json:"items"`
Items []reviewQueueItem `json:"items"`
}

type reviewQueueItem struct {
ghpkg.ReviewQueueEntry
governor.FixerQueueStatus
}

func (s *Server) governorFixerQueue() map[string]governor.FixerQueueStatus {
if s == nil || s.deps == nil {
return nil
}
return s.deps.Governor.FixerQueue()
}

// handleReviewQueue serves GET /api/review/queue?limit=N&offset=M: every open
Expand Down Expand Up @@ -110,7 +123,7 @@ func (s *Server) handleReviewQueue(w http.ResponseWriter, r *http.Request) {
Limit: limit,
Offset: offset,
History: history,
Items: []ghpkg.ReviewQueueEntry{},
Items: []reviewQueueItem{},
}
for _, e := range queue {
if e.Stage != nil && e.Stage.WaitingOnHive && e.Stage.PastDwell {
Expand All @@ -122,7 +135,13 @@ func (s *Server) handleReviewQueue(w http.ResponseWriter, r *http.Request) {
if end > len(queue) {
end = len(queue)
}
resp.Items = queue[offset:end]
fixers := s.governorFixerQueue()
for _, entry := range queue[offset:end] {
resp.Items = append(resp.Items, reviewQueueItem{
ReviewQueueEntry: entry,
FixerQueueStatus: fixers[ghpkg.ReviewQueueKey(entry.Repo, entry.Number)],
})
}
}
resp.HasMore = offset+len(resp.Items) < len(queue)
jsonResponse(w, resp)
Expand Down
Loading
Loading