From c01758e1d020675d2ea589dfc0d425418a2cfae8 Mon Sep 17 00:00:00 2001 From: Karibhou Date: Tue, 27 Oct 2026 20:38:51 +0000 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20review:=20scale=20reviewer=20concur?= =?UTF-8?q?rency=20up=20and=20down=20with=20backlog?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add review.reviewers.autoscale. The planner records the PR heads that found every plain reviewer busy (DispatchPlan.ReviewerStarved). A clock-injected ReviewerAutoscaler turns that backlog, plus how long the oldest head has waited, into an active replica count between min and max, with streaks for hysteresis and a cooldown between steps. It holds scale-up at max, while the provider budget latch is set, when cgroup headroom is short, and for one cooldown after a reviewer kick fails with "did not reach input prompt". Dynamic replicas come from ExpandAgentReplicas, which raises the scaled agent to max without rewriting its declared replicas. Replicas above the active count take no review-swarm kick (DispatchOptions.ReplicaLimits) and no cadence kick. A turn already in flight is never interrupted, so a scaled-in replica drains. Hive-Run: hivecommons/hive#14685 Hive-Plan: review-reviewer-autoscale Hive-Spec: review-swarm#reviewer-autoscale Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HRH6kjsZcEuDdED3kSWiqU Signed-off-by: Karibhou --- changelog.d/added-14685-reviewer-autoscale.md | 1 + src/cmd/hive/main.go | 8 + src/cmd/hive/review_autoscale.go | 100 ++++++ src/cmd/hive/review_autoscale_test.go | 107 +++++++ src/docs/review-swarm.md | 50 +++ src/hive.yaml.example | 6 + src/pkg/config/config.go | 11 +- src/pkg/config/review_autoscale.go | 189 ++++++++++++ src/pkg/config/review_autoscale_test.go | 119 +++++++ src/pkg/config/validate.go | 3 + src/pkg/dashboard/status_builder.go | 15 + src/pkg/review/autoscale.go | 267 ++++++++++++++++ src/pkg/review/autoscale_test.go | 290 ++++++++++++++++++ src/pkg/review/dispatch.go | 53 +++- 14 files changed, 1213 insertions(+), 6 deletions(-) create mode 100644 changelog.d/added-14685-reviewer-autoscale.md create mode 100644 src/cmd/hive/review_autoscale.go create mode 100644 src/cmd/hive/review_autoscale_test.go create mode 100644 src/pkg/config/review_autoscale.go create mode 100644 src/pkg/config/review_autoscale_test.go create mode 100644 src/pkg/review/autoscale.go create mode 100644 src/pkg/review/autoscale_test.go diff --git a/changelog.d/added-14685-reviewer-autoscale.md b/changelog.d/added-14685-reviewer-autoscale.md new file mode 100644 index 0000000000..4d87a7e807 --- /dev/null +++ b/changelog.d/added-14685-reviewer-autoscale.md @@ -0,0 +1 @@ +- The reviewer lane can scale with the PR backlog: `review.reviewers.autoscale` hands review and cadence kicks to between `min` and `max` replicas of the reviewer agent, stepping up on a sustained backlog or an over-age waiting PR and down after sustained idle, with cooldown, headroom, provider-budget and saturation guards ([#14685](https://github.com/hivecommons/hive/issues/14685)). diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 31cbbe4abb..3c683f413f 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -7346,6 +7346,9 @@ func runEvalCycle( } return "" }) + // Replicas the reviewer autoscaler has scaled in take no cadence kick + // either, or an idle reviewer-5 would still run a turn every interval. + agentsDue, agentsSkipped = withholdInactiveReviewerReplicas(agentsDue, agentsSkipped, cfg, &reviewerAutoscale) // PROVIDER SPEND REBUFF (#4294). When the inference gateway is refusing on a // money limit, every kick launched this cycle is a run that cannot buy a @@ -7653,6 +7656,7 @@ func runEvalCycle( gov.RecordKickFailure(msg.Agent, err, time.Now()) if k, ok := reviewKickByMessage[reviewKickKey(msg)]; ok { logUndeliveredReviewKick(logger, dashSrv, k, err) + noteReviewerAutoscaleKickFailure(cfg, &reviewerAutoscale, k.Agent, err, time.Now()) } }, onDelivered: func(msg scheduler.KickMessage) { @@ -10762,7 +10766,11 @@ func planReviewDispatchWith(ctx context.Context, cfg *config.Config, actionable DeltaRereview: cfg.Review.DeltaRereviewEnabled(), DeltaTrivialLines: cfg.Review.EffectiveDeltaTrivialLines(), DeltaMaxAge: cfg.Review.EffectiveDeltaMaxAge(), + ReplicaLimits: reviewerReplicaLimits(cfg, &reviewerAutoscale), }) + if cfg.Review.Reviewers.Autoscale.Enabled { + decideReviewerAutoscale(cfg, &reviewerAutoscale, plan, liveReviewerAutoscaleSignals(), logger) + } recordReviewDeltaRestamps(plan, logger) for _, k := range plan.ReviewKicks { if k.BotAuthorRouted { diff --git a/src/cmd/hive/review_autoscale.go b/src/cmd/hive/review_autoscale.go new file mode 100644 index 0000000000..f0df6ca922 --- /dev/null +++ b/src/cmd/hive/review_autoscale.go @@ -0,0 +1,100 @@ +package main + +import ( + "log/slog" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/dashboard" + "github.com/hivecommons/hive/pkg/review" +) + +// reviewerAutoscale is the reviewer lane's autoscaler (review.reviewers. +// autoscale, hivecommons/hive#14685): how many of the scaled agent's replicas +// the review swarm hands kicks to this cycle. +var reviewerAutoscale review.ReviewerAutoscaler + +// reviewerReplicaLimits is DispatchOptions.ReplicaLimits for this cycle: the +// scaled agent capped at the autoscaler's active count, nil when autoscale is +// off so every declared replica takes work as before. +func reviewerReplicaLimits(cfg *config.Config, as *review.ReviewerAutoscaler) map[string]int { + if cfg == nil || !cfg.Review.Reviewers.Autoscale.Enabled { + return nil + } + ac := cfg.Review.Reviewers.Autoscale + return map[string]int{ac.EffectiveAgent(): as.Active(ac)} +} + +// decideReviewerAutoscale feeds one planned cycle to the autoscaler and logs +// the step when it moves the active count. The new count applies from the +// next cycle's plan. +func decideReviewerAutoscale(cfg *config.Config, as *review.ReviewerAutoscaler, plan review.DispatchPlan, sig review.ReviewerAutoscaleSignals, logger *slog.Logger) review.ReviewerAutoscaleDecision { + if cfg == nil || !cfg.Review.Reviewers.Autoscale.Enabled { + return review.ReviewerAutoscaleDecision{} + } + ac := cfg.Review.Reviewers.Autoscale + sig.Waiting = plan.ReviewerStarved + if sig.Now.IsZero() { + sig.Now = time.Now().UTC() + } + d := as.Decide(ac, sig) + if d.Changed() && logger != nil { + logger.Info("reviewer autoscale: "+d.String(), + "agent", ac.EffectiveAgent(), "action", d.Action, "from", d.From, "to", d.To, + "backlog", d.Backlog, "oldest_wait", d.OldestWait.Round(time.Second).String(), + "min", ac.EffectiveMin(), "max", ac.EffectiveMax()) + } + return d +} + +// liveReviewerAutoscaleSignals are the guards read from the running hive: +// the provider spend latch and the pod's cgroup headroom. +func liveReviewerAutoscaleSignals() review.ReviewerAutoscaleSignals { + cause, _, _, _ := dashboard.InferenceBudgetExceeded() + return review.ReviewerAutoscaleSignals{ + BudgetLatched: cause != "", + Headroom: dashboard.PodHeadroom, + } +} + +// noteReviewerAutoscaleKickFailure records a review kick to one of the scaled +// agent's replicas that failed because the CLI never reached its input +// prompt: saturation, which holds scale-up for a cooldown. +func noteReviewerAutoscaleKickFailure(cfg *config.Config, as *review.ReviewerAutoscaler, agent string, err error, at time.Time) { + if cfg == nil || !cfg.Review.Reviewers.Autoscale.Enabled || !review.IsPromptNotReadyKickError(err) { + return + } + base := strings.TrimSpace(agent) + if ac, ok := cfg.Agents[base]; ok && ac.ReplicaOf != "" && !ac.IsReviewHead() { + base = ac.ReplicaOf + } + if base != cfg.Review.Reviewers.Autoscale.EffectiveAgent() { + return + } + as.NoteSaturation(at) +} + +// withholdInactiveReviewerReplicas drops the cadence kicks of the scaled +// agent's replicas numbered above the autoscaler's active count, reporting +// each as skipped "(autoscale-inactive)". With autoscale off it returns its +// inputs unchanged. +func withholdInactiveReviewerReplicas(agentsDue, skipped []string, cfg *config.Config, as *review.ReviewerAutoscaler) ([]string, []string) { + limits := reviewerReplicaLimits(cfg, as) + if len(limits) == 0 { + return agentsDue, skipped + } + kept := make([]string, 0, len(agentsDue)) + for _, key := range agentsDue { + name, _ := config.SplitCadenceTargetKey(key) + ac, ok := cfg.Agents[name] + if ok && ac.ReplicaOf != "" && !ac.IsReviewHead() { + if limit, scaled := limits[ac.ReplicaOf]; scaled && ac.ReplicaIndex > limit { + skipped = append(skipped, key+" (autoscale-inactive)") + continue + } + } + kept = append(kept, key) + } + return kept, skipped +} diff --git a/src/cmd/hive/review_autoscale_test.go b/src/cmd/hive/review_autoscale_test.go new file mode 100644 index 0000000000..d93a5619eb --- /dev/null +++ b/src/cmd/hive/review_autoscale_test.go @@ -0,0 +1,107 @@ +package main + +import ( + "errors" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/review" +) + +// Wiring for review.reviewers.autoscale (hivecommons/hive#14685). + +func autoscaleWiringConfig(enabled bool) *config.Config { + cfg := &config.Config{} + cfg.Agents = map[string]config.AgentConfig{ + "reviewer": {Enabled: true, Replicas: 1}, + "scanner": {Enabled: true, Replicas: 1}, + } + cfg.Review.Reviewers.Autoscale = config.ReviewerAutoscaleConfig{Enabled: enabled, Min: 1, Max: 3, Cooldown: "1m"} + if err := cfg.ExpandAgentReplicas(); err != nil { + panic(err) + } + return cfg +} + +func TestReviewerReplicaLimitsOffIsNil(t *testing.T) { + var as review.ReviewerAutoscaler + if got := reviewerReplicaLimits(autoscaleWiringConfig(false), &as); got != nil { + t.Fatalf("limits with autoscale off = %v, want nil", got) + } + if got := reviewerReplicaLimits(autoscaleWiringConfig(true), &as); got["reviewer"] != 1 || len(got) != 1 { + t.Fatalf("limits = %v, want reviewer:1", got) + } +} + +// Inactive replicas get no cadence kick, are reported as skipped, and every +// other agent (and repo-scoped cadence key) passes through. +func TestWithholdInactiveReviewerReplicas(t *testing.T) { + cfg := autoscaleWiringConfig(true) + var as review.ReviewerAutoscaler + due := []string{"reviewer", "reviewer-2", config.CadenceTargetKey("reviewer-3", "acme/hive"), "scanner"} + kept, skipped := withholdInactiveReviewerReplicas(due, nil, cfg, &as) + if strings.Join(kept, ",") != "reviewer,scanner" { + t.Fatalf("kept = %v, want reviewer and scanner", kept) + } + if len(skipped) != 2 || !strings.HasSuffix(skipped[0], "(autoscale-inactive)") { + t.Fatalf("skipped = %v", skipped) + } + + kept, skipped = withholdInactiveReviewerReplicas(due, nil, autoscaleWiringConfig(false), &as) + if len(kept) != len(due) || len(skipped) != 0 { + t.Fatalf("autoscale off: kept %v skipped %v, want everything kept", kept, skipped) + } +} + +func TestDecideReviewerAutoscaleScalesFromPlanBacklog(t *testing.T) { + cfg := autoscaleWiringConfig(true) + var as review.ReviewerAutoscaler + plan := review.DispatchPlan{ReviewerStarved: []string{"a#1@x", "a#2@x", "a#3@x"}} + t0 := time.Date(2026, 10, 27, 20, 0, 0, 0, time.UTC) + decideReviewerAutoscale(cfg, &as, plan, review.ReviewerAutoscaleSignals{Now: t0}, nil) + d := decideReviewerAutoscale(cfg, &as, plan, review.ReviewerAutoscaleSignals{Now: t0.Add(time.Minute)}, nil) + if d.Action != review.AutoscaleUp || d.To != 2 { + t.Fatalf("decision = %+v, want up to 2", d) + } + if got := reviewerReplicaLimits(cfg, &as)["reviewer"]; got != 2 { + t.Fatalf("next cycle limit = %d, want 2", got) + } + if d := decideReviewerAutoscale(autoscaleWiringConfig(false), &as, plan, review.ReviewerAutoscaleSignals{Now: t0}, nil); d.Action != "" { + t.Fatalf("autoscale off decided %+v", d) + } +} + +// Only a prompt-not-ready failure on the scaled agent's own replicas counts +// as saturation. +func TestNoteReviewerAutoscaleKickFailure(t *testing.T) { + cfg := autoscaleWiringConfig(true) + t0 := time.Date(2026, 10, 27, 20, 0, 0, 0, time.UTC) + busy := review.DispatchPlan{ReviewerStarved: []string{"a#1@x", "a#2@x", "a#3@x"}} + scaleUp := func(as *review.ReviewerAutoscaler) review.ReviewerAutoscaleDecision { + decideReviewerAutoscale(cfg, as, busy, review.ReviewerAutoscaleSignals{Now: t0.Add(10 * time.Second)}, nil) + return decideReviewerAutoscale(cfg, as, busy, review.ReviewerAutoscaleSignals{Now: t0.Add(20 * time.Second)}, nil) + } + promptErr := errors.New("agent reviewer-2 CLI did not reach input prompt") + + var saturated review.ReviewerAutoscaler + noteReviewerAutoscaleKickFailure(cfg, &saturated, "reviewer-2", promptErr, t0) + if d := scaleUp(&saturated); d.Changed() { + t.Fatalf("scaled up right after saturation: %+v", d) + } + + for name, tc := range map[string]struct { + agent string + err error + }{ + "other agent": {"scanner", promptErr}, + "other failure": {"reviewer-2", errors.New("tmux: no server running")}, + } { + var as review.ReviewerAutoscaler + noteReviewerAutoscaleKickFailure(cfg, &as, tc.agent, tc.err, t0) + if d := scaleUp(&as); !d.Changed() { + t.Fatalf("%s: counted as saturation, decision %+v", name, d) + } + } +} diff --git a/src/docs/review-swarm.md b/src/docs/review-swarm.md index 9264bc1354..36c7e2adff 100644 --- a/src/docs/review-swarm.md +++ b/src/docs/review-swarm.md @@ -332,6 +332,56 @@ Source: `ReviewCapable` in `src/pkg/review/dispatch.go`, which the dashboard (`dashboardAgentReviewCapable` in `src/pkg/dashboard/status_builder.go`), the relay and the merge-eligibility gate all call. +## Reviewer autoscaling (`review.reviewers.autoscale`) + +A reviewer lane with a fixed `replicas` count becomes the bottleneck when the PR backlog +grows, even with CPU and memory to spare. `review.reviewers.autoscale` lets the planner +vary how many of one reviewer agent's replicas take review kicks, between `min` and `max` +([#14685](https://github.com/hivecommons/hive/issues/14685)). + +```yaml +review: + reviewers: + autoscale: + enabled: true + agent: reviewer # base agent to scale (default "reviewer") + min: 2 # default 1 + max: 5 # default and ceiling: 5 (MaxAgentReplicas) + target_backlog_per_reviewer: 2 # scale up above this many waiting PRs per active reviewer + scale_up_cycles: 2 # consecutive cycles of pressure before a step up + scale_down_cycles: 5 # consecutive idle cycles before a step down + cooldown: 5m # minimum time between two steps + max_wait: 30m # a PR waiting this long is pressure on its own + cpu_per_reviewer: 0.5 # headroom one more reviewer needs (cores) + mem_per_reviewer_mb: 1024 # headroom one more reviewer needs (MB) +``` + +- **Replicas.** Enabling it materializes the agent's replicas up to `max` + (`reviewer-2` … `reviewer-N`) through the same machinery as `agents..replicas`, so + token minting, tmux wiring, cadence keys and the dashboard treat them as ordinary + replicas. The declared `replicas` is never rewritten. Replicas numbered above the active + count take no review-swarm kick and no cadence kick (the eval log lists them under + `agents_skipped` as `autoscale-inactive`). Review heads are not scaled. +- **Backlog.** Each cycle the planner records the PR heads that needed a plain reviewer + and found every active one busy (`DispatchPlan.ReviewerStarved`); the autoscaler tracks + how long each has waited. +- **Scale up** when the backlog exceeds `target_backlog_per_reviewer` per active reviewer, + or the oldest waiting PR is older than `max_wait`, for `scale_up_cycles` consecutive + cycles. A step is held at `max`, inside `cooldown`, while the provider budget latch is + set, when the pod's cgroup headroom is below one reviewer's `cpu_per_reviewer` / + `mem_per_reviewer_mb`, and for one `cooldown` after a reviewer kick failed with + `did not reach input prompt` (saturation, not demand). +- **Scale down** when fewer PRs wait than there are active reviewers for + `scale_down_cycles` consecutive cycles. The highest-numbered replica stops receiving new + work; a turn it already started is never interrupted and its verdict is recorded as + usual, so it drains rather than dies. +- **Observability.** Every step logs one line, e.g. + `reviewer autoscale: 2→3 reason=backlog=7 oldest=41m0s`, with the agent, min and max. + The active count is held in memory: after a restart the lane starts again at `min`. + +Source: `ReviewerAutoscaler.Decide` in `src/pkg/review/autoscale.go`; wiring in +`src/cmd/hive/review_autoscale.go`. + ## Review heads: several models per perspective `agents..review_heads` gives one base reviewer several model "heads" ([#13542](https://github.com/hivecommons/hive/issues/13542)). Each head is materialized like `replicas`: `--` copies the base agent with only `backend` and `model` replaced, is stripped on save, and is provisioned at boot by the entrypoint. `alias` defaults to a slug of `-` (without repeating the backend when the model already starts with it). The agent's Unix user and token group are `hive-`, and group names are limited to 32 characters, so `--` may be at most 27 characters; a longer name fails config load, so set a shorter `alias`. diff --git a/src/hive.yaml.example b/src/hive.yaml.example index 5a675cf660..78bd63e51a 100644 --- a/src/hive.yaml.example +++ b/src/hive.yaml.example @@ -834,6 +834,12 @@ governor: # # shadow.review_gate.would_have_held on the evidence bundle (#12104) # fan_out: false # max_parallel_reviews: 5 +# reviewers: +# autoscale: # scale the reviewer lane with the PR backlog (#14685); see docs/review-swarm.md +# enabled: false +# agent: reviewer +# min: 1 +# max: 5 # reviewer_agents: [] # optional explicit list; otherwise agents with review role/keywords are used # combined_perspectives: true # RECOMMENDED (#12126): one session + one comment covering every # # perspective; shared review boilerplate is sent once instead of once diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index a6af0979d7..ab3bda860a 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -6501,6 +6501,7 @@ func (c *Config) ExpandAgentReplicas() error { if replicas < 1 || replicas > MaxAgentReplicas { return fmt.Errorf("agent %s: replicas must be between 1 and %d", name, MaxAgentReplicas) } + replicas = c.Review.Reviewers.Autoscale.expandedReplicas(name, replicas) for i := 2; i <= replicas; i++ { derived := replicaAgentName(name, i) if existing, ok := baseAgents[derived]; ok && existing.ReplicaOf == "" { @@ -6522,8 +6523,11 @@ func (c *Config) ExpandAgentReplicas() error { } } for name, agent := range baseAgents { - replicas := normalizeReplicaCount(agent.Replicas) - agent.Replicas = replicas + agent.Replicas = normalizeReplicaCount(agent.Replicas) + // The autoscaler's max materializes the dynamic replicas + // (reviewer-3…reviewer-N) through this same path, without rewriting + // the declared replicas a save would persist. + replicas := c.Review.Reviewers.Autoscale.expandedReplicas(name, agent.Replicas) agent.ReplicaOf = "" agent.ReplicaIndex = 1 agent.ReplicaCount = replicas @@ -7502,6 +7506,9 @@ type ReviewConfig struct { // Backlog is where findings below the blocking line are filed // (hivecommons/hive#11088). See ReviewBacklogConfig. Backlog ReviewBacklogConfig `yaml:"backlog,omitempty" json:"backlog,omitempty"` + // Reviewers tunes the reviewer lane itself; today only its autoscaler + // (hivecommons/hive#14685). See ReviewReviewersConfig. + Reviewers ReviewReviewersConfig `yaml:"reviewers,omitempty" json:"reviewers,omitempty"` } // PlanMatchConfig is the switch for the plan_match review perspective diff --git a/src/pkg/config/review_autoscale.go b/src/pkg/config/review_autoscale.go new file mode 100644 index 0000000000..5d9736c08c --- /dev/null +++ b/src/pkg/config/review_autoscale.go @@ -0,0 +1,189 @@ +package config + +import ( + "fmt" + "strings" + "time" +) + +// Reviewer autoscaler defaults (hivecommons/hive#14685). A zero field in +// review.reviewers.autoscale means the matching default. +const ( + DefaultReviewerAutoscaleAgent = "reviewer" + DefaultReviewerAutoscaleMin = 1 + DefaultReviewerAutoscaleMax = MaxAgentReplicas + DefaultReviewerAutoscaleTargetBacklog = 2 + DefaultReviewerAutoscaleScaleUpCycles = 2 + DefaultReviewerAutoscaleScaleDownCycles = 5 + DefaultReviewerAutoscaleCooldown = 5 * time.Minute + DefaultReviewerAutoscaleMaxWait = 30 * time.Minute + DefaultReviewerAutoscaleCPUPerReviewer = 0.5 + DefaultReviewerAutoscaleMemPerReviewer = 1024 +) + +// ReviewReviewersConfig is review.reviewers. +type ReviewReviewersConfig struct { + Autoscale ReviewerAutoscaleConfig `yaml:"autoscale,omitempty" json:"autoscale,omitempty"` +} + +// ReviewerAutoscaleConfig is review.reviewers.autoscale: how many of one +// reviewer agent's replicas the review swarm hands work to, between Min and +// Max, driven by the backlog of PRs waiting for a reviewer. +// +// Enabling it materializes the agent's replicas up to Max through the same +// replica machinery as agents..replicas (reviewer-2…reviewer-N), so the +// planner, dashboard, kick governor and token cache see ordinary replicas. The +// autoscaler only decides how many of them receive review kicks. +type ReviewerAutoscaleConfig struct { + Enabled bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` + // Agent is the base reviewer agent this scales. Empty means "reviewer". + Agent string `yaml:"agent,omitempty" json:"agent,omitempty"` + Min int `yaml:"min,omitempty" json:"min,omitempty"` + Max int `yaml:"max,omitempty" json:"max,omitempty"` + // TargetBacklogPerReviewer is the waiting-PR count per active reviewer + // above which the lane is under-provisioned. + TargetBacklogPerReviewer int `yaml:"target_backlog_per_reviewer,omitempty" json:"target_backlog_per_reviewer,omitempty"` + // ScaleUpCycles and ScaleDownCycles are how many consecutive planner + // cycles the pressure must hold before a step up or down. + ScaleUpCycles int `yaml:"scale_up_cycles,omitempty" json:"scale_up_cycles,omitempty"` + ScaleDownCycles int `yaml:"scale_down_cycles,omitempty" json:"scale_down_cycles,omitempty"` + // Cooldown is the minimum time between two scaling actions (Go duration). + Cooldown string `yaml:"cooldown,omitempty" json:"cooldown,omitempty"` + // MaxWait is how long the oldest waiting PR may wait for a reviewer + // before that alone counts as scale-up pressure (Go duration). + MaxWait string `yaml:"max_wait,omitempty" json:"max_wait,omitempty"` + // CPUPerReviewer (cores) and MemPerReviewerMB are the per-reviewer cost + // estimate the headroom guard checks before a step up. + CPUPerReviewer float64 `yaml:"cpu_per_reviewer,omitempty" json:"cpu_per_reviewer,omitempty"` + MemPerReviewerMB int `yaml:"mem_per_reviewer_mb,omitempty" json:"mem_per_reviewer_mb,omitempty"` +} + +// EffectiveAgent is the scaled base agent name. +func (a ReviewerAutoscaleConfig) EffectiveAgent() string { + if n := strings.TrimSpace(a.Agent); n != "" { + return n + } + return DefaultReviewerAutoscaleAgent +} + +// EffectiveMax is Max, defaulted and clamped to MaxAgentReplicas. +func (a ReviewerAutoscaleConfig) EffectiveMax() int { + m := a.Max + if m <= 0 { + m = DefaultReviewerAutoscaleMax + } + if m > MaxAgentReplicas { + m = MaxAgentReplicas + } + return m +} + +// EffectiveMin is Min, defaulted and clamped to [1, EffectiveMax]. +func (a ReviewerAutoscaleConfig) EffectiveMin() int { + m := a.Min + if m <= 0 { + m = DefaultReviewerAutoscaleMin + } + if mx := a.EffectiveMax(); m > mx { + m = mx + } + return m +} + +func (a ReviewerAutoscaleConfig) EffectiveTargetBacklogPerReviewer() int { + if a.TargetBacklogPerReviewer > 0 { + return a.TargetBacklogPerReviewer + } + return DefaultReviewerAutoscaleTargetBacklog +} + +func (a ReviewerAutoscaleConfig) EffectiveScaleUpCycles() int { + if a.ScaleUpCycles > 0 { + return a.ScaleUpCycles + } + return DefaultReviewerAutoscaleScaleUpCycles +} + +func (a ReviewerAutoscaleConfig) EffectiveScaleDownCycles() int { + if a.ScaleDownCycles > 0 { + return a.ScaleDownCycles + } + return DefaultReviewerAutoscaleScaleDownCycles +} + +func (a ReviewerAutoscaleConfig) EffectiveCooldown() time.Duration { + return parseAutoscaleDuration(a.Cooldown, DefaultReviewerAutoscaleCooldown) +} + +func (a ReviewerAutoscaleConfig) EffectiveMaxWait() time.Duration { + return parseAutoscaleDuration(a.MaxWait, DefaultReviewerAutoscaleMaxWait) +} + +func (a ReviewerAutoscaleConfig) EffectiveCPUPerReviewer() float64 { + if a.CPUPerReviewer > 0 { + return a.CPUPerReviewer + } + return DefaultReviewerAutoscaleCPUPerReviewer +} + +func (a ReviewerAutoscaleConfig) EffectiveMemPerReviewerMB() int { + if a.MemPerReviewerMB > 0 { + return a.MemPerReviewerMB + } + return DefaultReviewerAutoscaleMemPerReviewer +} + +func parseAutoscaleDuration(raw string, def time.Duration) time.Duration { + raw = strings.TrimSpace(raw) + if raw == "" { + return def + } + d, err := time.ParseDuration(raw) + if err != nil || d < 0 { + return def + } + return d +} + +// expandedReplicas is how many replicas ExpandAgentReplicas materializes for +// agent name: the declared count, raised to the autoscaler's max for the +// scaled agent. +func (a ReviewerAutoscaleConfig) expandedReplicas(name string, declared int) int { + if !a.Enabled || name != a.EffectiveAgent() { + return declared + } + if mx := a.EffectiveMax(); mx > declared { + return mx + } + return declared +} + +// Validate rejects a review.reviewers.autoscale block that cannot mean what +// it says. +func (a ReviewerAutoscaleConfig) Validate() error { + if a.Min < 0 || a.Max < 0 { + return fmt.Errorf("review.reviewers.autoscale: min and max must not be negative") + } + if a.Max > MaxAgentReplicas { + return fmt.Errorf("review.reviewers.autoscale: max must be at most %d", MaxAgentReplicas) + } + if a.Min > 0 && a.Max > 0 && a.Min > a.Max { + return fmt.Errorf("review.reviewers.autoscale: min (%d) must not exceed max (%d)", a.Min, a.Max) + } + if a.TargetBacklogPerReviewer < 0 || a.ScaleUpCycles < 0 || a.ScaleDownCycles < 0 { + return fmt.Errorf("review.reviewers.autoscale: target_backlog_per_reviewer, scale_up_cycles and scale_down_cycles must not be negative") + } + if a.CPUPerReviewer < 0 || a.MemPerReviewerMB < 0 { + return fmt.Errorf("review.reviewers.autoscale: cpu_per_reviewer and mem_per_reviewer_mb must not be negative") + } + for _, f := range []struct{ name, raw string }{{"cooldown", a.Cooldown}, {"max_wait", a.MaxWait}} { + raw := strings.TrimSpace(f.raw) + if raw == "" { + continue + } + if d, err := time.ParseDuration(raw); err != nil || d < 0 { + return fmt.Errorf("review.reviewers.autoscale.%s: %q is not a non-negative duration", f.name, raw) + } + } + return nil +} diff --git a/src/pkg/config/review_autoscale_test.go b/src/pkg/config/review_autoscale_test.go new file mode 100644 index 0000000000..69deac17a5 --- /dev/null +++ b/src/pkg/config/review_autoscale_test.go @@ -0,0 +1,119 @@ +package config + +import ( + "strings" + "testing" + "time" +) + +// review.reviewers.autoscale (hivecommons/hive#14685). + +func autoscaleTestConfig(as ReviewerAutoscaleConfig) *Config { + return &Config{ + AgentRuntimeConfig: AgentRuntimeConfig{Agents: map[string]AgentConfig{ + "reviewer": {ID: "reviewer", Backend: "claude", Enabled: true, Replicas: 2}, + "scanner": {ID: "scanner", Backend: "claude", Enabled: true, Replicas: 1}, + }}, + Review: ReviewConfig{Reviewers: ReviewReviewersConfig{Autoscale: as}}, + } +} + +// Enabling autoscale materializes the scaled agent's replicas up to max +// through the ordinary replica machinery, and leaves other agents alone. +func TestExpandAgentReplicasMaterializesAutoscaleMax(t *testing.T) { + cfg := autoscaleTestConfig(ReviewerAutoscaleConfig{Enabled: true, Min: 2, Max: 4}) + if err := cfg.ExpandAgentReplicas(); err != nil { + t.Fatalf("ExpandAgentReplicas() error = %v", err) + } + for i, name := range []string{"reviewer", "reviewer-2", "reviewer-3", "reviewer-4"} { + got, ok := cfg.Agents[name] + if !ok { + t.Fatalf("missing replica %s", name) + } + if got.ReplicaIndex != i+1 || got.ReplicaCount != 4 { + t.Fatalf("%s metadata = index %d count %d, want %d/4", name, got.ReplicaIndex, got.ReplicaCount, i+1) + } + } + if _, ok := cfg.Agents["reviewer-5"]; ok { + t.Fatal("expanded past max") + } + if _, ok := cfg.Agents["scanner-2"]; ok { + t.Fatal("unscaled agent was expanded") + } + // The declared replicas stay what the operator wrote, so a save does not + // persist the autoscaler's max as a static count. + if got := cfg.Agents["reviewer"].Replicas; got != 2 { + t.Fatalf("declared replicas = %d, want 2 kept", got) + } +} + +func TestExpandAgentReplicasAutoscaleOffIsUnchanged(t *testing.T) { + cfg := autoscaleTestConfig(ReviewerAutoscaleConfig{Enabled: false, Max: 4}) + if err := cfg.ExpandAgentReplicas(); err != nil { + t.Fatalf("ExpandAgentReplicas() error = %v", err) + } + if _, ok := cfg.Agents["reviewer-3"]; ok { + t.Fatal("disabled autoscale expanded replicas") + } + if got := cfg.Agents["reviewer-2"].ReplicaCount; got != 2 { + t.Fatalf("ReplicaCount = %d, want 2", got) + } +} + +// A max below the declared replicas never removes a declared replica. +func TestExpandAgentReplicasAutoscaleNeverShrinksDeclared(t *testing.T) { + cfg := autoscaleTestConfig(ReviewerAutoscaleConfig{Enabled: true, Max: 1}) + if err := cfg.ExpandAgentReplicas(); err != nil { + t.Fatalf("ExpandAgentReplicas() error = %v", err) + } + if _, ok := cfg.Agents["reviewer-2"]; !ok { + t.Fatal("declared replica reviewer-2 was dropped") + } +} + +func TestReviewerAutoscaleDefaults(t *testing.T) { + var a ReviewerAutoscaleConfig + if a.EffectiveAgent() != "reviewer" || a.EffectiveMin() != 1 || a.EffectiveMax() != MaxAgentReplicas { + t.Fatalf("agent/min/max defaults = %s/%d/%d", a.EffectiveAgent(), a.EffectiveMin(), a.EffectiveMax()) + } + if a.EffectiveTargetBacklogPerReviewer() != 2 || a.EffectiveScaleUpCycles() != 2 || a.EffectiveScaleDownCycles() != 5 { + t.Fatal("cycle/target defaults wrong") + } + if a.EffectiveCooldown() != 5*time.Minute || a.EffectiveMaxWait() != 30*time.Minute { + t.Fatalf("cooldown/max_wait = %s/%s", a.EffectiveCooldown(), a.EffectiveMaxWait()) + } + if a.EffectiveCPUPerReviewer() != 0.5 || a.EffectiveMemPerReviewerMB() != 1024 { + t.Fatal("cost defaults wrong") + } + a = ReviewerAutoscaleConfig{Min: 4, Max: 3, Cooldown: "90s", Agent: " cr "} + if a.EffectiveMin() != 3 || a.EffectiveCooldown() != 90*time.Second || a.EffectiveAgent() != "cr" { + t.Fatalf("min clamp/cooldown/agent = %d/%s/%q", a.EffectiveMin(), a.EffectiveCooldown(), a.EffectiveAgent()) + } +} + +func TestReviewerAutoscaleValidate(t *testing.T) { + ok := []ReviewerAutoscaleConfig{ + {}, + {Enabled: true, Min: 2, Max: 4, Cooldown: "5m", MaxWait: "45m", CPUPerReviewer: 1, MemPerReviewerMB: 2048}, + } + for _, a := range ok { + if err := a.Validate(); err != nil { + t.Fatalf("Validate(%+v) = %v", a, err) + } + } + bad := map[string]ReviewerAutoscaleConfig{ + "max": {Max: MaxAgentReplicas + 1}, + "min (4)": {Min: 4, Max: 2}, + "negative": {Min: -1}, + "cooldown": {Cooldown: "soon"}, + "max_wait": {MaxWait: "-1m"}, + "cpu_per": {CPUPerReviewer: -1}, + "scale_up": {ScaleUpCycles: -1}, + } + for want, a := range bad { + err := a.Validate() + if err == nil || !strings.Contains(err.Error(), want) { + t.Fatalf("Validate(%+v) = %v, want error mentioning %q", a, err, want) + } + } +} diff --git a/src/pkg/config/validate.go b/src/pkg/config/validate.go index 8990fdb178..d525af9947 100644 --- a/src/pkg/config/validate.go +++ b/src/pkg/config/validate.go @@ -158,6 +158,9 @@ func (c *Config) ValidateWithOptions(opts ValidateOptions) error { if err := c.validateReviewHeads(); err != nil { return err } + if err := c.Review.Reviewers.Autoscale.Validate(); err != nil { + return err + } if err := c.validateGitHubActivityNotifications(); err != nil { return err } diff --git a/src/pkg/dashboard/status_builder.go b/src/pkg/dashboard/status_builder.go index fea23e62dd..4ff5606d15 100644 --- a/src/pkg/dashboard/status_builder.go +++ b/src/pkg/dashboard/status_builder.go @@ -3253,6 +3253,21 @@ func collectSystemResources() *SystemResources { return res } +// PodHeadroom is the pod's free CPU (cores) and memory (MB) from the same +// cgroup sample the status payload reports; -1 for a value that could not be +// read. The reviewer autoscaler's headroom guard (hivecommons/hive#14685). +func PodHeadroom() (cpuFreeCores, memFreeMB float64) { + res := collectSystemResources() + cpuFreeCores, memFreeMB = -1, -1 + if res.CpuCores > 0 && res.CpuPct > 0 { + cpuFreeCores = float64(res.CpuCores) * (1 - res.CpuPct/pctMultiplierSysRes) + } + if res.MemTotalMB > 0 { + memFreeMB = res.MemTotalMB - res.MemUsedMB + } + return cpuFreeCores, memFreeMB +} + // readCgroupInt64 reads a single int64 from a cgroup file. Returns -1 on error // or if the value is "max" (unlimited). func readCgroupInt64(path string) int64 { diff --git a/src/pkg/review/autoscale.go b/src/pkg/review/autoscale.go new file mode 100644 index 0000000000..163e01d59c --- /dev/null +++ b/src/pkg/review/autoscale.go @@ -0,0 +1,267 @@ +package review + +import ( + "fmt" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/pkg/config" +) + +// Reviewer autoscaler actions (hivecommons/hive#14685). +const ( + AutoscaleHold = "hold" + AutoscaleUp = "up" + AutoscaleDown = "down" +) + +// Reasons a step the pressure called for was not taken. Each is the reason +// field of the decision's log line. +const ( + AutoscaleHeldAtMax = "at_max" + AutoscaleHeldAtMin = "at_min" + AutoscaleHeldCooldown = "cooldown" + AutoscaleHeldSaturated = "saturated" + AutoscaleHeldBudget = "provider_budget" + AutoscaleHeldHeadroom = "headroom" +) + +// ReviewerAutoscaleSignals is what one planner cycle tells the autoscaler. +type ReviewerAutoscaleSignals struct { + Now time.Time + // Waiting keys the PR heads that needed a review this cycle and found no + // free reviewer (DispatchPlan.ReviewerStarved). + Waiting []string + // Saturated reports a recent reviewer kick that failed because the CLI + // did not reach its input prompt: the pod is out of capacity, and adding + // sessions would make that worse, not better. + Saturated bool + // BudgetLatched reports the provider spend latch: a new reviewer could + // not buy a token either. + BudgetLatched bool + // Headroom measures the pod's free CPU cores and memory (MB). It is only + // called when a step up is otherwise due, because sampling CPU takes a + // moment. Nil, or a negative value, means unknown, which never blocks. + Headroom func() (cpuFreeCores, memFreeMB float64) +} + +// ReviewerAutoscaleDecision is one cycle's outcome. +type ReviewerAutoscaleDecision struct { + At time.Time `json:"at"` + Action string `json:"action"` + From int `json:"from"` + To int `json:"to"` + Reason string `json:"reason,omitempty"` + Backlog int `json:"backlog"` + OldestWait time.Duration `json:"oldest_wait"` +} + +// Changed reports whether the decision moved the active count. +func (d ReviewerAutoscaleDecision) Changed() bool { return d.From != d.To } + +// String is the decision's log line body, e.g. "2→3 reason=backlog=7 oldest=41m". +func (d ReviewerAutoscaleDecision) String() string { + return fmt.Sprintf("%d→%d reason=%s", d.From, d.To, d.Reason) +} + +// ReviewerAutoscaleStatus is the dashboard view of the autoscaler. +type ReviewerAutoscaleStatus struct { + Agent string `json:"agent"` + Active int `json:"active"` + Min int `json:"min"` + Max int `json:"max"` + LastDecision ReviewerAutoscaleDecision `json:"last_decision"` + LastChange ReviewerAutoscaleDecision `json:"last_change"` +} + +// ReviewerAutoscaler holds the reviewer lane's active replica count across +// planner cycles. The zero value is ready; it starts at the configured min. +// State is in memory: a restart starts again from min and scales back up +// within scale_up_cycles if the backlog is still there. +type ReviewerAutoscaler struct { + mu sync.Mutex + active int + upStreak int + downStreak int + lastAction time.Time + waitingSince map[string]time.Time + saturatedAt time.Time + last ReviewerAutoscaleDecision + lastChange ReviewerAutoscaleDecision + min, max int + agent string +} + +// Active is the number of the scaled agent's replicas (reviewer, +// reviewer-2, …) the planner may hand review kicks to. +func (s *ReviewerAutoscaler) Active(cfg config.ReviewerAutoscaleConfig) int { + s.mu.Lock() + defer s.mu.Unlock() + return s.clampLocked(cfg) +} + +func (s *ReviewerAutoscaler) clampLocked(cfg config.ReviewerAutoscaleConfig) int { + s.min, s.max, s.agent = cfg.EffectiveMin(), cfg.EffectiveMax(), cfg.EffectiveAgent() + if s.active < s.min { + s.active = s.min + } + if s.active > s.max { + s.active = s.max + } + return s.active +} + +// NoteSaturation records a reviewer kick that failed because its CLI did not +// reach the input prompt. Scale-up is held for one cooldown after it. +func (s *ReviewerAutoscaler) NoteSaturation(at time.Time) { + s.mu.Lock() + defer s.mu.Unlock() + if at.After(s.saturatedAt) { + s.saturatedAt = at + } +} + +// Status snapshots the autoscaler for the dashboard. +func (s *ReviewerAutoscaler) Status() ReviewerAutoscaleStatus { + s.mu.Lock() + defer s.mu.Unlock() + return ReviewerAutoscaleStatus{Agent: s.agent, Active: s.active, Min: s.min, Max: s.max, LastDecision: s.last, LastChange: s.lastChange} +} + +// Decide runs one cycle of the autoscaler. +// +// Scale-up pressure is a backlog above target_backlog_per_reviewer per active +// reviewer, or a PR that has waited longer than max_wait; it must hold for +// scale_up_cycles consecutive cycles. Scale-down pressure is fewer waiting PRs +// than active reviewers (below one each), held for scale_down_cycles. The gap +// between the two thresholds and the streaks are the hysteresis; cooldown +// spaces any two steps. +// +// A step down removes the highest-numbered replica from new dispatch. It is +// never interrupted: a turn already in flight finishes and its verdict is +// recorded as usual, so the replica drains rather than dies. +func (s *ReviewerAutoscaler) Decide(cfg config.ReviewerAutoscaleConfig, sig ReviewerAutoscaleSignals) ReviewerAutoscaleDecision { + s.mu.Lock() + defer s.mu.Unlock() + active := s.clampLocked(cfg) + now := sig.Now + oldest := s.observeWaitingLocked(sig.Waiting, now) + backlog := len(sig.Waiting) + d := ReviewerAutoscaleDecision{At: now, Action: AutoscaleHold, From: active, To: active, Backlog: backlog, OldestWait: oldest} + pressure := fmt.Sprintf("backlog=%d oldest=%s", backlog, formatWait(oldest)) + + up := backlog > active*cfg.EffectiveTargetBacklogPerReviewer() || + (backlog > 0 && oldest >= cfg.EffectiveMaxWait()) + down := !up && backlog < active + switch { + case up: + s.upStreak++ + s.downStreak = 0 + case down: + s.downStreak++ + s.upStreak = 0 + default: + s.upStreak, s.downStreak = 0, 0 + } + coolingDown := !s.lastAction.IsZero() && now.Sub(s.lastAction) < cfg.EffectiveCooldown() + saturated := sig.Saturated || (!s.saturatedAt.IsZero() && now.Sub(s.saturatedAt) < cfg.EffectiveCooldown()) + + switch { + case up && s.upStreak >= cfg.EffectiveScaleUpCycles(): + held := "" + switch { + case active >= s.max: + held = AutoscaleHeldAtMax + case coolingDown: + held = AutoscaleHeldCooldown + case saturated: + held = AutoscaleHeldSaturated + case sig.BudgetLatched: + held = AutoscaleHeldBudget + case !headroomFor(cfg, sig): + held = AutoscaleHeldHeadroom + } + if held != "" { + d.Reason = held + " " + pressure + break + } + d.Action, d.To, d.Reason = AutoscaleUp, active+1, pressure + case down && s.downStreak >= cfg.EffectiveScaleDownCycles(): + held := "" + switch { + case active <= s.min: + held = AutoscaleHeldAtMin + case coolingDown: + held = AutoscaleHeldCooldown + } + if held != "" { + d.Reason = held + " " + pressure + break + } + d.Action, d.To, d.Reason = AutoscaleDown, active-1, pressure + default: + d.Reason = pressure + } + if d.Changed() { + s.active = d.To + s.lastAction = now + s.upStreak, s.downStreak = 0, 0 + s.lastChange = d + } + s.last = d + return d +} + +// observeWaitingLocked keeps the first time each waiting PR head was seen +// waiting, forgets heads no longer waiting, and returns the oldest wait. +func (s *ReviewerAutoscaler) observeWaitingLocked(keys []string, now time.Time) time.Duration { + seen := make(map[string]time.Time, len(keys)) + var oldest time.Duration + for _, k := range keys { + since, ok := s.waitingSince[k] + if !ok { + since = now + } + seen[k] = since + if w := now.Sub(since); w > oldest { + oldest = w + } + } + s.waitingSince = seen + return oldest +} + +// headroomFor reports whether the pod can afford one more reviewer session. +func headroomFor(cfg config.ReviewerAutoscaleConfig, sig ReviewerAutoscaleSignals) bool { + if sig.Headroom == nil { + return true + } + cpuFree, memFree := sig.Headroom() + if cpuFree >= 0 && cpuFree < cfg.EffectiveCPUPerReviewer() { + return false + } + if memFree >= 0 && memFree < float64(cfg.EffectiveMemPerReviewerMB()) { + return false + } + return true +} + +func formatWait(d time.Duration) string { + if d <= 0 { + return "0s" + } + return d.Round(time.Minute).String() +} + +// ReviewerStarvedKey keys a PR head in DispatchPlan.ReviewerStarved. +func ReviewerStarvedKey(repo string, number int, headSHA string) string { + return fmt.Sprintf("%s#%d@%s", strings.ToLower(strings.TrimSpace(repo)), number, strings.TrimSpace(headSHA)) +} + +// IsPromptNotReadyKickError reports a kick that failed because the agent's +// CLI never reached its input prompt (crux/agent kickPromptNotReadyErr), the +// autoscaler's saturation signal. +func IsPromptNotReadyKickError(err error) bool { + return err != nil && strings.Contains(err.Error(), "did not reach input prompt") +} diff --git a/src/pkg/review/autoscale_test.go b/src/pkg/review/autoscale_test.go new file mode 100644 index 0000000000..50b5b240ef --- /dev/null +++ b/src/pkg/review/autoscale_test.go @@ -0,0 +1,290 @@ +package review + +import ( + "errors" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" +) + +// Decision-table tests for the reviewer autoscaler (hivecommons/hive#14685). +// Every cycle is driven through an injected clock; nothing sleeps. + +var autoscaleT0 = time.Date(2026, 10, 27, 19, 20, 0, 0, time.UTC) + +func autoscaleCfg() config.ReviewerAutoscaleConfig { + return config.ReviewerAutoscaleConfig{Enabled: true, Min: 2, Max: 4} +} + +func waitingKeys(n int) []string { + out := make([]string, n) + for i := range out { + out[i] = ReviewerStarvedKey("acme/hive", i+1, "sha") + } + return out +} + +// run feeds cycles one minute apart starting at start and returns the last +// decision. +func runCycles(s *ReviewerAutoscaler, cfg config.ReviewerAutoscaleConfig, start time.Time, cycles int, sig ReviewerAutoscaleSignals) (ReviewerAutoscaleDecision, time.Time) { + var d ReviewerAutoscaleDecision + at := start + for i := 0; i < cycles; i++ { + sig.Now = at + d = s.Decide(cfg, sig) + at = at.Add(time.Minute) + } + return d, at +} + +func TestReviewerAutoscaleStartsAtMin(t *testing.T) { + var s ReviewerAutoscaler + if got := s.Active(autoscaleCfg()); got != 2 { + t.Fatalf("Active = %d, want min 2", got) + } +} + +func TestReviewerAutoscaleDecisionTable(t *testing.T) { + cases := []struct { + name string + cycles int + sig ReviewerAutoscaleSignals + wantAction string + wantTo int + wantReason string + }{ + {name: "one cycle of backlog is not sustained", cycles: 1, sig: ReviewerAutoscaleSignals{Waiting: waitingKeys(7)}, wantAction: AutoscaleHold, wantTo: 2}, + {name: "backlog above target for two cycles scales up", cycles: 2, sig: ReviewerAutoscaleSignals{Waiting: waitingKeys(7)}, wantAction: AutoscaleUp, wantTo: 3, wantReason: "backlog=7 oldest=1m0s"}, + {name: "backlog at target holds", cycles: 3, sig: ReviewerAutoscaleSignals{Waiting: waitingKeys(4)}, wantAction: AutoscaleHold, wantTo: 2}, + {name: "saturation holds scale-up", cycles: 2, sig: ReviewerAutoscaleSignals{Waiting: waitingKeys(7), Saturated: true}, wantAction: AutoscaleHold, wantTo: 2, wantReason: AutoscaleHeldSaturated + " backlog=7 oldest=1m0s"}, + {name: "provider budget latch holds scale-up", cycles: 2, sig: ReviewerAutoscaleSignals{Waiting: waitingKeys(7), BudgetLatched: true}, wantAction: AutoscaleHold, wantTo: 2, wantReason: AutoscaleHeldBudget + " backlog=7 oldest=1m0s"}, + {name: "no cpu headroom holds scale-up", cycles: 2, sig: ReviewerAutoscaleSignals{Waiting: waitingKeys(7), Headroom: func() (float64, float64) { return 0.2, 8000 }}, wantAction: AutoscaleHold, wantTo: 2, wantReason: AutoscaleHeldHeadroom + " backlog=7 oldest=1m0s"}, + {name: "no memory headroom holds scale-up", cycles: 2, sig: ReviewerAutoscaleSignals{Waiting: waitingKeys(7), Headroom: func() (float64, float64) { return 4, 512 }}, wantAction: AutoscaleHold, wantTo: 2, wantReason: AutoscaleHeldHeadroom + " backlog=7 oldest=1m0s"}, + {name: "unknown headroom never blocks", cycles: 2, sig: ReviewerAutoscaleSignals{Waiting: waitingKeys(7), Headroom: func() (float64, float64) { return -1, -1 }}, wantAction: AutoscaleUp, wantTo: 3}, + {name: "idle below one per reviewer for five cycles stays at min", cycles: 5, sig: ReviewerAutoscaleSignals{Waiting: waitingKeys(1)}, wantAction: AutoscaleHold, wantTo: 2, wantReason: AutoscaleHeldAtMin + " backlog=1 oldest=4m0s"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var s ReviewerAutoscaler + d, _ := runCycles(&s, autoscaleCfg(), autoscaleT0, tc.cycles, tc.sig) + if d.Action != tc.wantAction || d.To != tc.wantTo { + t.Fatalf("decision = %+v, want action %s to %d", d, tc.wantAction, tc.wantTo) + } + if tc.wantReason != "" && d.Reason != tc.wantReason { + t.Fatalf("reason = %q, want %q", d.Reason, tc.wantReason) + } + if got := s.Active(autoscaleCfg()); got != tc.wantTo { + t.Fatalf("Active = %d, want %d", got, tc.wantTo) + } + }) + } +} + +// A small backlog that has waited past max_wait is scale-up pressure on its +// own: the oldest PR is the stage-dwell signal from the issue. +func TestReviewerAutoscaleOldestWaitScalesUp(t *testing.T) { + cfg := autoscaleCfg() + cfg.MaxWait = "30m" + var s ReviewerAutoscaler + waiting := waitingKeys(2) + s.Decide(cfg, ReviewerAutoscaleSignals{Now: autoscaleT0, Waiting: waiting}) + d := s.Decide(cfg, ReviewerAutoscaleSignals{Now: autoscaleT0.Add(29 * time.Minute), Waiting: waiting}) + if d.Changed() { + t.Fatalf("scaled before max_wait: %+v", d) + } + d = s.Decide(cfg, ReviewerAutoscaleSignals{Now: autoscaleT0.Add(31 * time.Minute), Waiting: waiting}) + if d.Changed() { + t.Fatalf("first over-age cycle must not act alone (scale_up_cycles=2): %+v", d) + } + d = s.Decide(cfg, ReviewerAutoscaleSignals{Now: autoscaleT0.Add(32 * time.Minute), Waiting: waiting}) + if d.Action != AutoscaleUp || d.To != 3 || d.OldestWait != 32*time.Minute { + t.Fatalf("decision = %+v, want up to 3 with oldest 32m", d) + } + if d.String() != "2→3 reason=backlog=2 oldest=32m0s" { + t.Fatalf("log line = %q", d.String()) + } +} + +// A head that stops waiting is forgotten, so its age does not linger. +func TestReviewerAutoscaleForgetsServedPRs(t *testing.T) { + var s ReviewerAutoscaler + cfg := autoscaleCfg() + s.Decide(cfg, ReviewerAutoscaleSignals{Now: autoscaleT0, Waiting: waitingKeys(1)}) + s.Decide(cfg, ReviewerAutoscaleSignals{Now: autoscaleT0.Add(10 * time.Minute)}) + d := s.Decide(cfg, ReviewerAutoscaleSignals{Now: autoscaleT0.Add(20 * time.Minute), Waiting: waitingKeys(1)}) + if d.OldestWait != 0 { + t.Fatalf("oldest wait = %s, want 0 for a head seen waiting again", d.OldestWait) + } +} + +func TestReviewerAutoscaleCooldownAndMax(t *testing.T) { + cfg := autoscaleCfg() + var s ReviewerAutoscaler + busy := ReviewerAutoscaleSignals{Waiting: waitingKeys(20)} + d, at := runCycles(&s, cfg, autoscaleT0, 2, busy) + if d.To != 3 { + t.Fatalf("first step = %+v, want 3", d) + } + // Pressure keeps holding, but the next step waits for the 5m cooldown. + d, at = runCycles(&s, cfg, at, 3, busy) + if d.Changed() || d.Reason != AutoscaleHeldCooldown+" backlog=20 oldest=4m0s" { + t.Fatalf("inside cooldown = %+v, want held cooldown", d) + } + d, at = runCycles(&s, cfg, at, 2, busy) + if d.Action != AutoscaleUp || d.To != 4 { + t.Fatalf("after cooldown = %+v, want up to 4", d) + } + d, _ = runCycles(&s, cfg, at.Add(10*time.Minute), 2, busy) + if d.Changed() || d.To != 4 || d.Reason != AutoscaleHeldAtMax+" backlog=20 oldest=18m0s" { + t.Fatalf("at max = %+v, want held at_max 4", d) + } +} + +func TestReviewerAutoscaleScalesDownAfterSustainedIdle(t *testing.T) { + cfg := autoscaleCfg() + var s ReviewerAutoscaler + _, at := runCycles(&s, cfg, autoscaleT0, 2, ReviewerAutoscaleSignals{Waiting: waitingKeys(20)}) + if s.Active(cfg) != 3 { + t.Fatal("setup: expected 3 active") + } + at = at.Add(10 * time.Minute) // past the cooldown + // Four idle cycles are not enough (scale_down_cycles=5). + d, at := runCycles(&s, cfg, at, 4, ReviewerAutoscaleSignals{Waiting: waitingKeys(2)}) + if d.Changed() { + t.Fatalf("scaled down after 4 cycles: %+v", d) + } + // Backlog of 3 at 3 reviewers is not below one each: it resets the streak. + runCycles(&s, cfg, at, 1, ReviewerAutoscaleSignals{Waiting: waitingKeys(3)}) + d, at = runCycles(&s, cfg, at.Add(time.Minute), 4, ReviewerAutoscaleSignals{}) + if d.Changed() { + t.Fatalf("streak was not reset by a busy cycle: %+v", d) + } + d, _ = runCycles(&s, cfg, at, 1, ReviewerAutoscaleSignals{}) + if d.Action != AutoscaleDown || d.From != 3 || d.To != 2 { + t.Fatalf("decision = %+v, want down 3→2", d) + } + if st := s.Status(); st.Active != 2 || st.LastChange.Action != AutoscaleDown || st.Min != 2 || st.Max != 4 { + t.Fatalf("status = %+v", st) + } +} + +// A kick that failed with "did not reach input prompt" holds scale-up for one +// cooldown, then stops counting. +func TestReviewerAutoscaleNoteSaturationExpires(t *testing.T) { + cfg := autoscaleCfg() + var s ReviewerAutoscaler + s.NoteSaturation(autoscaleT0) + busy := ReviewerAutoscaleSignals{Waiting: waitingKeys(20)} + d, _ := runCycles(&s, cfg, autoscaleT0.Add(time.Minute), 2, busy) + if d.Changed() || d.Reason[:len(AutoscaleHeldSaturated)] != AutoscaleHeldSaturated { + t.Fatalf("within cooldown of saturation = %+v, want held saturated", d) + } + d, _ = runCycles(&s, cfg, autoscaleT0.Add(6*time.Minute), 1, busy) + if d.Action != AutoscaleUp { + t.Fatalf("after saturation cooldown = %+v, want up", d) + } +} + +// Lowering max in config clamps the active count on the next read. +func TestReviewerAutoscaleClampsToConfigChange(t *testing.T) { + cfg := autoscaleCfg() + var s ReviewerAutoscaler + runCycles(&s, cfg, autoscaleT0, 2, ReviewerAutoscaleSignals{Waiting: waitingKeys(20)}) + cfg.Max = 2 + if got := s.Active(cfg); got != 2 { + t.Fatalf("Active after max lowered = %d, want 2", got) + } +} + +func TestIsPromptNotReadyKickError(t *testing.T) { + if !IsPromptNotReadyKickError(errors.New("agent reviewer-2 CLI did not reach input prompt: busy")) { + t.Fatal("prompt-not-ready error not recognized") + } + if IsPromptNotReadyKickError(errors.New("tmux: no server running")) || IsPromptNotReadyKickError(nil) { + t.Fatal("unrelated error recognized as saturation") + } +} + +func autoscaleFleet() []AgentCapability { + r2 := replica("reviewer-2", "reviewer") + r2.ReplicaIndex = 2 + r3 := replica("reviewer-3", "reviewer") + r3.ReplicaIndex = 3 + base := reviewer("reviewer") + base.ReplicaIndex = 1 + return []AgentCapability{base, r2, r3} +} + +func autoscaleDispatchOpts(limits map[string]int) DispatchOptions { + return DispatchOptions{ + RequireApproval: true, FanOut: true, CombinedPerspectives: true, + MaxParallelReviews: 10, ProjectOrg: "acme", + Agents: autoscaleFleet(), ReplicaLimits: limits, + Now: autoscaleT0, + } +} + +// Replicas above the autoscaler's active count take no new review kick, and +// the PR left without a reviewer is reported as the backlog. +func TestPlanDispatchHonorsReplicaLimits(t *testing.T) { + prs := []PullRequest{dispatchPRNum(1, "sha1"), dispatchPRNum(2, "sha2"), dispatchPRNum(3, "sha3")} + plan := PlanDispatch(prs, Artifact{}, DispatchState{}, autoscaleDispatchOpts(map[string]int{"reviewer": 2})) + agents := map[string]bool{} + for _, k := range plan.ReviewKicks { + agents[k.Agent] = true + } + if len(plan.ReviewKicks) != 2 || !agents["reviewer"] || !agents["reviewer-2"] || agents["reviewer-3"] { + t.Fatalf("kicks = %+v, want reviewer and reviewer-2 only", plan.ReviewKicks) + } + want := ReviewerStarvedKey("acme/hive", 3, "sha3") + if len(plan.ReviewerStarved) != 1 || plan.ReviewerStarved[0] != want { + t.Fatalf("ReviewerStarved = %v, want [%s]", plan.ReviewerStarved, want) + } + + plan = PlanDispatch(prs, Artifact{}, DispatchState{}, autoscaleDispatchOpts(nil)) + if len(plan.ReviewKicks) != 3 || len(plan.ReviewerStarved) != 0 { + t.Fatalf("unlimited plan = %d kicks, starved %v; want 3 kicks, none starved", len(plan.ReviewKicks), plan.ReviewerStarved) + } +} + +// A scaled-in replica keeps the turn it is already running: its pending entry +// is neither pruned nor redispatched, so the verdict still lands. +func TestPlanDispatchDrainsScaledInReplica(t *testing.T) { + pending := []PendingReview{{Repo: "acme/hive", Number: 1, HeadSHA: "sha1", Perspective: PerspectiveCorrectness, Agent: "reviewer-3", Dispatched: autoscaleT0.Add(-5 * time.Minute)}} + opts := autoscaleDispatchOpts(map[string]int{"reviewer": 2}) + plan := PlanDispatch([]PullRequest{dispatchPRNum(1, "sha1")}, Artifact{}, DispatchState{Pending: pending}, opts) + kept := false + for _, p := range plan.State.Pending { + if p.Agent == "reviewer-3" && p.Number == 1 { + kept = true + } + } + if !kept { + t.Fatalf("draining replica's in-flight entry was dropped: %+v", plan.State.Pending) + } + for _, k := range plan.ReviewKicks { + if k.Agent == "reviewer-3" { + t.Fatalf("scaled-in replica got a new kick: %+v", k) + } + } +} + +// Review heads carry ReplicaOf too, but are their own lanes: the limit must +// not hide them. +func TestOverReplicaLimitIgnoresReviewHeads(t *testing.T) { + head := headReviewer("reviewer-sonnet-3", "reviewer-sonnet", "claude", "m") + head.ReplicaOf = "reviewer" + head.ReplicaIndex = 3 + if overReplicaLimit(head, map[string]int{"reviewer": 1}) { + t.Fatal("review head was limited") + } + plain := replica("reviewer-3", "reviewer") + plain.ReplicaIndex = 3 + if !overReplicaLimit(plain, map[string]int{"reviewer": 2}) { + t.Fatal("plain replica 3 not limited at 2") + } + if overReplicaLimit(plain, map[string]int{"other": 1}) { + t.Fatal("replica of an unscaled base was limited") + } +} diff --git a/src/pkg/review/dispatch.go b/src/pkg/review/dispatch.go index d817fbf77e..4cd00100bd 100644 --- a/src/pkg/review/dispatch.go +++ b/src/pkg/review/dispatch.go @@ -72,7 +72,10 @@ var LegacyReviewDispatchStatePath = filepath.Join(outputschema.AgentReportDir, R type AgentCapability struct { Name string // ReplicaOf is the base agent a derived replica was expanded from. - ReplicaOf string + ReplicaOf string + // ReplicaIndex is the replica's position under its base (1 for the base + // itself, N for base-N); what DispatchOptions.ReplicaLimits compares. + ReplicaIndex int Enabled bool Paused bool OnDemand bool @@ -108,6 +111,7 @@ func CapabilityFromConfig(name string, ac config.AgentConfig) AgentCapability { return AgentCapability{ Name: name, ReplicaOf: ac.ReplicaOf, + ReplicaIndex: ac.ReplicaIndex, Enabled: ac.Enabled, Paused: ac.Paused, OnDemand: ac.OnDemand, @@ -246,8 +250,14 @@ type DispatchOptions struct { DeltaRereview bool DeltaTrivialLines int DeltaMaxAge time.Duration - Agents []AgentCapability - Now time.Time + // ReplicaLimits caps, per base agent, which of its plain replicas take + // new review kicks: replica N is offered work only while N <= the limit + // (review.reviewers.autoscale, hivecommons/hive#14685). A base absent + // from the map is unlimited. Review heads are not affected. A replica + // over the limit keeps any kick already in flight; it only drains. + ReplicaLimits map[string]int + Agents []AgentCapability + Now time.Time } type DispatchState struct { @@ -500,7 +510,11 @@ type DispatchPlan struct { // ForcedFixes are the changes_requested heads whose fix kick was forced // this cycle after RevisitStarvationCycles revisit-suppressed cycles. ForcedFixes []ForcedFix - State DispatchState + // ReviewerStarved keys (ReviewerStarvedKey) the PR heads that needed a + // plain reviewer this cycle and found every one busy: the reviewer + // autoscaler's backlog (hivecommons/hive#14685). + ReviewerStarved []string + State DispatchState } type FixDispatchExclusion struct { @@ -777,6 +791,7 @@ func PlanDispatch(prs []PullRequest, artifact Artifact, state DispatchState, opt if opts.CombinedPerspectives { reviewer, free := freeReviewer(prReviewers, plan.inFlight(opts, now)) if !free { + plan.noteReviewerStarved(pr) continue } agent := reviewer.Name @@ -851,6 +866,9 @@ func PlanDispatch(prs []PullRequest, artifact Artifact, state DispatchState, opt for i := 0; i < limit; i++ { reviewer, free := freeReviewer(prReviewers, plan.inFlight(opts, now)) if !free { + if i == 0 { + plan.noteReviewerStarved(pr) + } break } agent := reviewer.Name @@ -886,6 +904,12 @@ func PlanDispatch(prs []PullRequest, artifact Artifact, state DispatchState, opt return plan } +// noteReviewerStarved records a PR head that waited this cycle because every +// plain reviewer was busy. +func (p *DispatchPlan) noteReviewerStarved(pr PullRequest) { + p.ReviewerStarved = append(p.ReviewerStarved, ReviewerStarvedKey(pr.Repo, pr.Number, pr.HeadSHA)) +} + // deltaPreviousHead is the head a delta kick is judged against, "" for a // full review. func deltaPreviousHead(d *DeltaContext) string { @@ -1686,6 +1710,9 @@ func reviewCapableAgents(opts DispatchOptions) []AgentCapability { if a.Name == "" || !a.Enabled || a.Paused || a.OnDemand || !a.UsesKick { continue } + if overReplicaLimit(a, opts.ReplicaLimits) { + continue + } if ReviewCapable(a, allowed) { out = append(out, a) } @@ -1694,6 +1721,24 @@ func reviewCapableAgents(opts DispatchOptions) []AgentCapability { return out } +// overReplicaLimit reports a plain replica numbered above its base's +// DispatchOptions.ReplicaLimits entry. Review heads carry ReplicaOf too, but +// are their own lanes and never limited. +func overReplicaLimit(a AgentCapability, limits map[string]int) bool { + if len(limits) == 0 || a.ReviewHead != "" { + return false + } + limit, ok := limits[baseOf(a)] + if !ok { + return false + } + index := a.ReplicaIndex + if index <= 0 { + index = 1 + } + return index > limit +} + // ReviewCapable reports whether one agent is a reviewer under the same rule // the dispatcher uses to hand out review kicks: when review.reviewer_agents // names lanes, membership in that set decides; otherwise the agent's name,