Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/added-reviewer-autoscale.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- `review.reviewers.autoscale` (off by default; Settings → Review Gate → Reviewers) sizes the reviewer lane to its backlog ([#14685](https://github.com/hivecommons/hive/issues/14685)): the planner now reports the PRs it left waiting for a free reviewer or a parallel slot, and a controller adds a `reviewer-N` replica after `scale_up_cycles` cycles behind `target_backlog_per_reviewer` (or a wait older than the review-pending stage dwell), drains the top replica after `scale_down_cycles` idle cycles and retires it only once its turn ends. Scale-ups are refused by named guards — `at-max`, `cooldown`, `saturated` (kicks not reaching the CLI prompt), `provider-budget`, `provider-headroom`, `max-parallel-reviews`, cgroup `cpu` / `memory`. Dynamic replicas go through the ordinary replica expansion (same role, prompt, token, pane and cadence as `reviewer-2`) and the entrypoint pre-creates their users up to `max`. Each decision logs as `reviewer autoscale: 2→3 reason=backlog=7 oldest=41m0s`; `/api/status.reviewerAutoscale` and an *auto* badge on the lane's agent cards show current/target/min/max and the last decision.
69 changes: 41 additions & 28 deletions src/cmd/hive/main.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package main

import (
Expand Down Expand Up @@ -71,6 +71,7 @@
"github.com/hivecommons/hive/pkg/promptsrc"
"github.com/hivecommons/hive/pkg/retro"
"github.com/hivecommons/hive/pkg/review"
"github.com/hivecommons/hive/pkg/review/autoscale"
"github.com/hivecommons/hive/pkg/rfccoverage"
"github.com/hivecommons/hive/pkg/rotation"
"github.com/hivecommons/hive/pkg/scheduler"
Expand Down Expand Up @@ -1452,34 +1453,35 @@
// linearagent adapters behind the dashboard's consumer-defined
// interfaces — this composition root is the only non-test place that
// names the concrete types.
Watsonx: watsonxGateway{},
OpenRouter: openRouterGateway{},
NewLinearAgent: newLinearAgentGateway(b.logger),
LinearStoredViewerID: linearStoredViewerID,
DelegateDecider: newDelegateDeciderRunner(func() *config.Config { return b.cfg }),
ChatGrounder: newChatGroundingRunner(func() *config.Config { return b.cfg }),
Governor: b.gov,
GHClient: b.ghClient,
StageEngineStatus: func() stageengine.Status { return stageEngineStatus(b.cfg, b.ghClient, b.logger) },
ReviewEvents: b.reviewEvents,
WebhookWake: b.webhookWakeFunc(),
ApplyWebhookItems: b.applyWebhookItemsFunc(),
WebhookDelivery: b.webhookDeliveryFunc(),
EvalLoop: b.evalLoopStatusFunc(),
SentinelPREvent: b.sentinelPREventFunc(),
ProjectsV2ItemEvent: b.projectsV2ItemEventFunc(),
GHAppAuth: b.appAuth,
GHTokenScopes: b.ghAuth.TokenScopes,
Tokens: b.tokenCollector,
Knowledge: b.knowledgeAPI,
Inception: b.inceptionEngine,
Nous: b.nousState,
Scheduler: b.sched,
KnowledgePrimer: knowledgePrimerControl{b: b},
MetricsCollector: b.metricsCollector,
RotationMgr: b.rotationMgr,
QuestionAutoclose: questionAutocloseView{b: b},
HeadroomPublisher: b.quotaReadingPublisher,
Watsonx: watsonxGateway{},
OpenRouter: openRouterGateway{},
NewLinearAgent: newLinearAgentGateway(b.logger),
LinearStoredViewerID: linearStoredViewerID,
DelegateDecider: newDelegateDeciderRunner(func() *config.Config { return b.cfg }),
ChatGrounder: newChatGroundingRunner(func() *config.Config { return b.cfg }),
Governor: b.gov,
GHClient: b.ghClient,
StageEngineStatus: func() stageengine.Status { return stageEngineStatus(b.cfg, b.ghClient, b.logger) },
ReviewerAutoscaleStatus: func() autoscale.Status { return reviewerAutoscaler.Status() },
ReviewEvents: b.reviewEvents,
WebhookWake: b.webhookWakeFunc(),
ApplyWebhookItems: b.applyWebhookItemsFunc(),
WebhookDelivery: b.webhookDeliveryFunc(),
EvalLoop: b.evalLoopStatusFunc(),
SentinelPREvent: b.sentinelPREventFunc(),
ProjectsV2ItemEvent: b.projectsV2ItemEventFunc(),
GHAppAuth: b.appAuth,
GHTokenScopes: b.ghAuth.TokenScopes,
Tokens: b.tokenCollector,
Knowledge: b.knowledgeAPI,
Inception: b.inceptionEngine,
Nous: b.nousState,
Scheduler: b.sched,
KnowledgePrimer: knowledgePrimerControl{b: b},
MetricsCollector: b.metricsCollector,
RotationMgr: b.rotationMgr,
QuestionAutoclose: questionAutocloseView{b: b},
HeadroomPublisher: b.quotaReadingPublisher,
// #3972: hand the ACMM advisor the SAME cached fleet-stats collector
// the heartbeat reads, so its merge-success signal reuses the existing
// 30-minute collect loop instead of issuing a second GitHub fetch.
Expand Down Expand Up @@ -3502,6 +3504,7 @@
func (b *boot) bootSupervision() {
if b.cfg.Governor.Rotation.Enabled {
b.rotationMgr = rotation.NewManager(b.cfg.Governor.Rotation)
reviewerAutoscaler.SetHeadroomReporter(b.rotationMgr)
// Publish normalized readings to where the contributor quota guard reads
// them (kubestellar/hive#6967), off the rotation manager's own probe
// loop and the operator's configured provider→backends map.
Expand Down Expand Up @@ -3952,6 +3955,10 @@

// Preserve runtime-only fields that are not in the YAML
newCfg.HiveID = b.cfg.HiveID
// The reviewer autoscaler's current lane size survives a reload so
// an edit elsewhere in hive.yaml does not snap reviewers back to the
// declared count (hivecommons/hive#14685).
newCfg.ReviewerAutoscaleActive = b.cfg.ReviewerAutoscaleActive

// Preserve ACMM level from the agent manager — it is the
// authoritative source. The file may have a stale value if
Expand Down Expand Up @@ -7346,6 +7353,7 @@
}
return ""
})
agentsDue, agentsSkipped = reviewerAutoscaler.withholdDrainingReviewers(agentsDue, agentsSkipped)

// PROVIDER SPEND REBUFF (#4294). When the inference gateway is refusing on a
// money limit, every kick launched this cycle is a run that cannot buy a
Expand Down Expand Up @@ -7444,6 +7452,10 @@
agentsDue = sched.FilterFirstScanDeferredAgents(agentsDue)
agentsSkipped = append(agentsSkipped, firstScanDeferredSkips(dueBeforeDeferral, agentsDue)...)
reviewPlan := planReviewDispatchWith(ctx, cfg, actionable, agentMgr, beadStores, reviewPlannerGitHubFor(ghClient), logger)
// Size the reviewer lane to what the planner just left waiting
// (hivecommons/hive#14685). A replica added here takes kicks from the
// next cycle on; a retire only lands once the reviewer is idle.
reviewerAutoscaler.Run(ctx, cfg, reviewPlan, agentMgr, gov, logger, time.Now())
auditWithheldReviewFixes(reviewPlan, dashSrv, logger)
if ghClient != nil {
postReviewAcknowledgements(ctx, ghClient, reviewPlan, logger)
Expand Down Expand Up @@ -10728,6 +10740,7 @@
for name, ac := range cfg.EnabledAgents() {
cap := review.CapabilityFromConfig(name, ac)
cap.Paused = cap.Paused || (agentMgr != nil && agentMgr.IsPaused(name))
cap.Draining = reviewerAutoscaler.Draining(name)
agents = append(agents, cap)
}
reviewBotLogins := []string{}
Expand Down
220 changes: 220 additions & 0 deletions src/cmd/hive/reviewer_autoscale.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,220 @@
package main

import (
"context"
"log/slog"
"sync"
"time"

"github.com/hivecommons/hive/crux"
"github.com/hivecommons/hive/pkg/config"
"github.com/hivecommons/hive/pkg/dashboard"
"github.com/hivecommons/hive/pkg/governor"
"github.com/hivecommons/hive/pkg/review"
"github.com/hivecommons/hive/pkg/review/autoscale"
"github.com/hivecommons/hive/pkg/rotation"
"github.com/hivecommons/hive/pkg/stagelabels"
)

// reviewerAutoscaleSkipReason is the agents_skipped annotation for a
// draining reviewer withheld from the cadence kick.
const reviewerAutoscaleSkipReason = "reviewer-autoscale draining"

// unknownProviderHeadroom tells the controller no headroom reading exists.
const unknownProviderHeadroom = -1

// reviewerAutoscaler is the process-wide reviewer-lane autoscaler
// (hivecommons/hive#14685). It is package-level, like providerBudgetProbe,
// so runEvalCycle's already-long signature does not grow; the eval loop is
// single-threaded but the dashboard reads Status concurrently, hence the
// mutex.
var reviewerAutoscaler = &reviewerAutoscaleRuntime{prober: autoscale.NewProber()}

type reviewerAutoscaleRuntime struct {
mu sync.Mutex
ctrl *autoscale.Controller
prober *autoscale.Prober
headroom rotation.HeadroomReporter
status autoscale.Status
}

// SetHeadroomReporter wires the provider headroom probe when rotation is on.
func (r *reviewerAutoscaleRuntime) SetHeadroomReporter(h rotation.HeadroomReporter) {
r.mu.Lock()
defer r.mu.Unlock()
r.headroom = h
}

// Draining reports whether name is a reviewer being retired; the planner
// and the cadence kick both withhold work from it.
func (r *reviewerAutoscaleRuntime) Draining(name string) bool {
if r == nil {
return false
}
r.mu.Lock()
defer r.mu.Unlock()
return r.ctrl != nil && r.ctrl.Draining(name)
}

// Status is the dashboard snapshot; Enabled is false when autoscale is off.
func (r *reviewerAutoscaleRuntime) Status() autoscale.Status {
if r == nil {
return autoscale.Status{}
}
r.mu.Lock()
defer r.mu.Unlock()
return r.status
}

// withholdDrainingReviewers removes draining reviewers from the cadence
// kick list, reporting them in agentsSkipped like every other gate.
func (r *reviewerAutoscaleRuntime) withholdDrainingReviewers(agentsDue, agentsSkipped []string) ([]string, []string) {
if r == nil {
return agentsDue, agentsSkipped
}
kept := agentsDue[:0]
for _, name := range agentsDue {
if r.Draining(name) {
agentsSkipped = append(agentsSkipped, name+" ("+reviewerAutoscaleSkipReason+")")
continue
}
kept = append(kept, name)
}
return kept, agentsSkipped
}

// Run folds this cycle's review plan into the controller and applies its
// decision: a scale-up or retire re-expands the lane's replicas and
// reconciles the agent manager and governor the same way a dashboard
// replicas edit does; a drain only changes what Draining answers.
func (r *reviewerAutoscaleRuntime) Run(ctx context.Context, cfg *config.Config, plan review.DispatchPlan, agentMgr crux.Runtime, gov *governor.Governor, logger *slog.Logger, now time.Time) {
if r == nil || cfg == nil {
return
}
r.mu.Lock()
defer r.mu.Unlock()
if !cfg.ReviewerAutoscaleEnabled() {
if r.ctrl != nil {
r.ctrl = nil
r.status = autoscale.Status{}
if cfg.ReviewerAutoscaleActive != 0 {
cfg.ReviewerAutoscaleActive = 0
r.reconcile(ctx, cfg, agentMgr, gov, logger)
}
logger.Info("reviewer autoscale: disabled; lane back on declared replicas")
}
return
}
agent := cfg.ReviewerAutoscaleAgent()
acfg := cfg.Review.Reviewers.Autoscale
if r.ctrl == nil {
r.ctrl = autoscale.New(acfg, agent)
logger.Info("reviewer autoscale: enabled", "agent", agent, "min", acfg.EffectiveMin(), "max", acfg.EffectiveMax(), "target_backlog_per_reviewer", acfg.EffectiveTargetBacklogPerReviewer(), "cooldown", acfg.EffectiveCooldown().String())
} else {
r.ctrl.Reconfigure(acfg, agent)
}
names := cfg.ReviewerAutoscaleReplicas()
if cfg.ReviewerAutoscaleActive == 0 {
cfg.ReviewerAutoscaleActive = acfg.Clamp(len(names))
}

obs := autoscale.Observation{
Now: now,
DwellThreshold: stagelabels.Dwell(stagelabels.ReviewPending, cfg.Governor.Labels.Stage.DwellOverrides()),
MaxParallelReviews: cfg.Review.EffectiveMaxParallelReviews(),
Resources: r.prober.Probe(now),
ProviderHeadroomPct: unknownProviderHeadroom,
}
cause, _, _, _ := dashboard.InferenceBudgetExceeded()
obs.ProviderBudgetExceeded = cause != ""
if r.headroom != nil {
obs.ProviderHeadroomPct = lowestProviderHeadroom(r.headroom.HeadroomResponse())
}
lane := make(map[string]bool, len(names))
for _, name := range names {
lane[name] = true
}
for _, p := range plan.State.Pending {
if lane[p.Agent] {
obs.InFlight++
}
}
for _, name := range names {
rv := autoscale.Reviewer{Name: name, Index: cfg.Agents[name].ReplicaIndex}
if proc, err := agentMgr.GetStatusFast(name); err == nil && proc != nil {
rv.Busy = proc.State == crux.StateRunning && !proc.KickOutcome.Settled(proc.LastKick)
rv.Saturated = proc.KicksUndeliverable > 0
}
for _, p := range plan.State.Pending {
if p.Agent == name {
rv.Busy = true
break
}
}
obs.Reviewers = append(obs.Reviewers, rv)
}
for _, w := range plan.Waiting {
obs.WaitingKeys = append(obs.WaitingKeys, w.Key())
}

d := r.ctrl.Observe(obs)
switch d.Action {
case autoscale.ActionHold:
logger.Debug(d.String(), "agent", agent)
case autoscale.ActionBlocked:
logger.Info(d.String(), "agent", agent, "backlog", len(obs.WaitingKeys))
case autoscale.ActionDrain, autoscale.ActionUndrain:
logger.Info(d.String(), "agent", agent)
case autoscale.ActionScaleUp, autoscale.ActionRetire:
logger.Info(d.String(), "agent", agent)
if err := cfg.SetReviewerAutoscaleActive(d.To); err != nil {
logger.Warn("reviewer autoscale: expanding replicas failed", "agent", agent, "target", d.To, "error", err)
break
}
r.reconcile(ctx, cfg, agentMgr, gov, logger)
}
r.status = r.ctrl.Status()
}

// reconcile pushes the config's current agent set into the agent manager
// and governor. ReconcileAgents starts nothing itself and kills the tmux
// session of any replica no longer configured, so a retire needs no
// separate stop.
func (r *reviewerAutoscaleRuntime) reconcile(ctx context.Context, cfg *config.Config, agentMgr crux.Runtime, gov *governor.Governor, logger *slog.Logger) {
if agentMgr == nil {
return
}
if cfg.ReviewerAutoscaleActive == 0 {
if err := cfg.ExpandAgentReplicas(); err != nil {
logger.Warn("reviewer autoscale: expanding replicas failed", "error", err)
return
}
}
enabled := cfg.EnabledAgents()
for _, added := range agentMgr.ReconcileAgents(enabled) {
if ac, ok := cfg.Agents[added]; ok && !ac.OnDemand {
if err := agentMgr.Start(ctx, added); err != nil {
logger.Warn("reviewer autoscale: failed to start reviewer", "agent", added, "error", err)
}
}
}
if gov != nil {
gov.UpdateAgents(enabled)
}
}

// lowestProviderHeadroom is the smallest remaining quota across providers
// with a reading, or unknownProviderHeadroom when none has one.
func lowestProviderHeadroom(resp rotation.HeadroomResponse) float64 {
lowest := float64(unknownProviderHeadroom)
for _, h := range resp.Providers {
if h.CapturedAt.IsZero() && !h.Available {
continue
}
pct := float64(h.PctRemaining)
if lowest < 0 || pct < lowest {
lowest = pct
}
}
return lowest
}
Loading
Loading