diff --git a/changelog.d/added-14685-reviewer-autoscale.md b/changelog.d/added-14685-reviewer-autoscale.md new file mode 100644 index 0000000000..4d87a7e807 --- /dev/null +++ b/changelog.d/added-14685-reviewer-autoscale.md @@ -0,0 +1 @@ +- The reviewer lane can scale with the PR backlog: `review.reviewers.autoscale` hands review and cadence kicks to between `min` and `max` replicas of the reviewer agent, stepping up on a sustained backlog or an over-age waiting PR and down after sustained idle, with cooldown, headroom, provider-budget and saturation guards ([#14685](https://github.com/hivecommons/hive/issues/14685)). diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 31cbbe4abb..3c683f413f 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -7346,6 +7346,9 @@ func runEvalCycle( } return "" }) + // Replicas the reviewer autoscaler has scaled in take no cadence kick + // either, or an idle reviewer-5 would still run a turn every interval. + agentsDue, agentsSkipped = withholdInactiveReviewerReplicas(agentsDue, agentsSkipped, cfg, &reviewerAutoscale) // PROVIDER SPEND REBUFF (#4294). When the inference gateway is refusing on a // money limit, every kick launched this cycle is a run that cannot buy a @@ -7653,6 +7656,7 @@ func runEvalCycle( gov.RecordKickFailure(msg.Agent, err, time.Now()) if k, ok := reviewKickByMessage[reviewKickKey(msg)]; ok { logUndeliveredReviewKick(logger, dashSrv, k, err) + noteReviewerAutoscaleKickFailure(cfg, &reviewerAutoscale, k.Agent, err, time.Now()) } }, onDelivered: func(msg scheduler.KickMessage) { @@ -10762,7 +10766,11 @@ func planReviewDispatchWith(ctx context.Context, cfg *config.Config, actionable DeltaRereview: cfg.Review.DeltaRereviewEnabled(), DeltaTrivialLines: cfg.Review.EffectiveDeltaTrivialLines(), DeltaMaxAge: cfg.Review.EffectiveDeltaMaxAge(), + ReplicaLimits: reviewerReplicaLimits(cfg, &reviewerAutoscale), }) + if cfg.Review.Reviewers.Autoscale.Enabled { + decideReviewerAutoscale(cfg, &reviewerAutoscale, plan, liveReviewerAutoscaleSignals(), logger) + } recordReviewDeltaRestamps(plan, logger) for _, k := range plan.ReviewKicks { if k.BotAuthorRouted { diff --git a/src/cmd/hive/review_autoscale.go b/src/cmd/hive/review_autoscale.go new file mode 100644 index 0000000000..f0df6ca922 --- /dev/null +++ b/src/cmd/hive/review_autoscale.go @@ -0,0 +1,100 @@ +package main + +import ( + "log/slog" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/dashboard" + "github.com/hivecommons/hive/pkg/review" +) + +// reviewerAutoscale is the reviewer lane's autoscaler (review.reviewers. +// autoscale, hivecommons/hive#14685): how many of the scaled agent's replicas +// the review swarm hands kicks to this cycle. +var reviewerAutoscale review.ReviewerAutoscaler + +// reviewerReplicaLimits is DispatchOptions.ReplicaLimits for this cycle: the +// scaled agent capped at the autoscaler's active count, nil when autoscale is +// off so every declared replica takes work as before. +func reviewerReplicaLimits(cfg *config.Config, as *review.ReviewerAutoscaler) map[string]int { + if cfg == nil || !cfg.Review.Reviewers.Autoscale.Enabled { + return nil + } + ac := cfg.Review.Reviewers.Autoscale + return map[string]int{ac.EffectiveAgent(): as.Active(ac)} +} + +// decideReviewerAutoscale feeds one planned cycle to the autoscaler and logs +// the step when it moves the active count. The new count applies from the +// next cycle's plan. +func decideReviewerAutoscale(cfg *config.Config, as *review.ReviewerAutoscaler, plan review.DispatchPlan, sig review.ReviewerAutoscaleSignals, logger *slog.Logger) review.ReviewerAutoscaleDecision { + if cfg == nil || !cfg.Review.Reviewers.Autoscale.Enabled { + return review.ReviewerAutoscaleDecision{} + } + ac := cfg.Review.Reviewers.Autoscale + sig.Waiting = plan.ReviewerStarved + if sig.Now.IsZero() { + sig.Now = time.Now().UTC() + } + d := as.Decide(ac, sig) + if d.Changed() && logger != nil { + logger.Info("reviewer autoscale: "+d.String(), + "agent", ac.EffectiveAgent(), "action", d.Action, "from", d.From, "to", d.To, + "backlog", d.Backlog, "oldest_wait", d.OldestWait.Round(time.Second).String(), + "min", ac.EffectiveMin(), "max", ac.EffectiveMax()) + } + return d +} + +// liveReviewerAutoscaleSignals are the guards read from the running hive: +// the provider spend latch and the pod's cgroup headroom. +func liveReviewerAutoscaleSignals() review.ReviewerAutoscaleSignals { + cause, _, _, _ := dashboard.InferenceBudgetExceeded() + return review.ReviewerAutoscaleSignals{ + BudgetLatched: cause != "", + Headroom: dashboard.PodHeadroom, + } +} + +// noteReviewerAutoscaleKickFailure records a review kick to one of the scaled +// agent's replicas that failed because the CLI never reached its input +// prompt: saturation, which holds scale-up for a cooldown. +func noteReviewerAutoscaleKickFailure(cfg *config.Config, as *review.ReviewerAutoscaler, agent string, err error, at time.Time) { + if cfg == nil || !cfg.Review.Reviewers.Autoscale.Enabled || !review.IsPromptNotReadyKickError(err) { + return + } + base := strings.TrimSpace(agent) + if ac, ok := cfg.Agents[base]; ok && ac.ReplicaOf != "" && !ac.IsReviewHead() { + base = ac.ReplicaOf + } + if base != cfg.Review.Reviewers.Autoscale.EffectiveAgent() { + return + } + as.NoteSaturation(at) +} + +// withholdInactiveReviewerReplicas drops the cadence kicks of the scaled +// agent's replicas numbered above the autoscaler's active count, reporting +// each as skipped "(autoscale-inactive)". With autoscale off it returns its +// inputs unchanged. +func withholdInactiveReviewerReplicas(agentsDue, skipped []string, cfg *config.Config, as *review.ReviewerAutoscaler) ([]string, []string) { + limits := reviewerReplicaLimits(cfg, as) + if len(limits) == 0 { + return agentsDue, skipped + } + kept := make([]string, 0, len(agentsDue)) + for _, key := range agentsDue { + name, _ := config.SplitCadenceTargetKey(key) + ac, ok := cfg.Agents[name] + if ok && ac.ReplicaOf != "" && !ac.IsReviewHead() { + if limit, scaled := limits[ac.ReplicaOf]; scaled && ac.ReplicaIndex > limit { + skipped = append(skipped, key+" (autoscale-inactive)") + continue + } + } + kept = append(kept, key) + } + return kept, skipped +} diff --git a/src/cmd/hive/review_autoscale_test.go b/src/cmd/hive/review_autoscale_test.go new file mode 100644 index 0000000000..d93a5619eb --- /dev/null +++ b/src/cmd/hive/review_autoscale_test.go @@ -0,0 +1,107 @@ +package main + +import ( + "errors" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/review" +) + +// Wiring for review.reviewers.autoscale (hivecommons/hive#14685). + +func autoscaleWiringConfig(enabled bool) *config.Config { + cfg := &config.Config{} + cfg.Agents = map[string]config.AgentConfig{ + "reviewer": {Enabled: true, Replicas: 1}, + "scanner": {Enabled: true, Replicas: 1}, + } + cfg.Review.Reviewers.Autoscale = config.ReviewerAutoscaleConfig{Enabled: enabled, Min: 1, Max: 3, Cooldown: "1m"} + if err := cfg.ExpandAgentReplicas(); err != nil { + panic(err) + } + return cfg +} + +func TestReviewerReplicaLimitsOffIsNil(t *testing.T) { + var as review.ReviewerAutoscaler + if got := reviewerReplicaLimits(autoscaleWiringConfig(false), &as); got != nil { + t.Fatalf("limits with autoscale off = %v, want nil", got) + } + if got := reviewerReplicaLimits(autoscaleWiringConfig(true), &as); got["reviewer"] != 1 || len(got) != 1 { + t.Fatalf("limits = %v, want reviewer:1", got) + } +} + +// Inactive replicas get no cadence kick, are reported as skipped, and every +// other agent (and repo-scoped cadence key) passes through. +func TestWithholdInactiveReviewerReplicas(t *testing.T) { + cfg := autoscaleWiringConfig(true) + var as review.ReviewerAutoscaler + due := []string{"reviewer", "reviewer-2", config.CadenceTargetKey("reviewer-3", "acme/hive"), "scanner"} + kept, skipped := withholdInactiveReviewerReplicas(due, nil, cfg, &as) + if strings.Join(kept, ",") != "reviewer,scanner" { + t.Fatalf("kept = %v, want reviewer and scanner", kept) + } + if len(skipped) != 2 || !strings.HasSuffix(skipped[0], "(autoscale-inactive)") { + t.Fatalf("skipped = %v", skipped) + } + + kept, skipped = withholdInactiveReviewerReplicas(due, nil, autoscaleWiringConfig(false), &as) + if len(kept) != len(due) || len(skipped) != 0 { + t.Fatalf("autoscale off: kept %v skipped %v, want everything kept", kept, skipped) + } +} + +func TestDecideReviewerAutoscaleScalesFromPlanBacklog(t *testing.T) { + cfg := autoscaleWiringConfig(true) + var as review.ReviewerAutoscaler + plan := review.DispatchPlan{ReviewerStarved: []string{"a#1@x", "a#2@x", "a#3@x"}} + t0 := time.Date(2026, 10, 27, 20, 0, 0, 0, time.UTC) + decideReviewerAutoscale(cfg, &as, plan, review.ReviewerAutoscaleSignals{Now: t0}, nil) + d := decideReviewerAutoscale(cfg, &as, plan, review.ReviewerAutoscaleSignals{Now: t0.Add(time.Minute)}, nil) + if d.Action != review.AutoscaleUp || d.To != 2 { + t.Fatalf("decision = %+v, want up to 2", d) + } + if got := reviewerReplicaLimits(cfg, &as)["reviewer"]; got != 2 { + t.Fatalf("next cycle limit = %d, want 2", got) + } + if d := decideReviewerAutoscale(autoscaleWiringConfig(false), &as, plan, review.ReviewerAutoscaleSignals{Now: t0}, nil); d.Action != "" { + t.Fatalf("autoscale off decided %+v", d) + } +} + +// Only a prompt-not-ready failure on the scaled agent's own replicas counts +// as saturation. +func TestNoteReviewerAutoscaleKickFailure(t *testing.T) { + cfg := autoscaleWiringConfig(true) + t0 := time.Date(2026, 10, 27, 20, 0, 0, 0, time.UTC) + busy := review.DispatchPlan{ReviewerStarved: []string{"a#1@x", "a#2@x", "a#3@x"}} + scaleUp := func(as *review.ReviewerAutoscaler) review.ReviewerAutoscaleDecision { + decideReviewerAutoscale(cfg, as, busy, review.ReviewerAutoscaleSignals{Now: t0.Add(10 * time.Second)}, nil) + return decideReviewerAutoscale(cfg, as, busy, review.ReviewerAutoscaleSignals{Now: t0.Add(20 * time.Second)}, nil) + } + promptErr := errors.New("agent reviewer-2 CLI did not reach input prompt") + + var saturated review.ReviewerAutoscaler + noteReviewerAutoscaleKickFailure(cfg, &saturated, "reviewer-2", promptErr, t0) + if d := scaleUp(&saturated); d.Changed() { + t.Fatalf("scaled up right after saturation: %+v", d) + } + + for name, tc := range map[string]struct { + agent string + err error + }{ + "other agent": {"scanner", promptErr}, + "other failure": {"reviewer-2", errors.New("tmux: no server running")}, + } { + var as review.ReviewerAutoscaler + noteReviewerAutoscaleKickFailure(cfg, &as, tc.agent, tc.err, t0) + if d := scaleUp(&as); !d.Changed() { + t.Fatalf("%s: counted as saturation, decision %+v", name, d) + } + } +} diff --git a/src/docs/review-swarm.md b/src/docs/review-swarm.md index 9264bc1354..36c7e2adff 100644 --- a/src/docs/review-swarm.md +++ b/src/docs/review-swarm.md @@ -332,6 +332,56 @@ Source: `ReviewCapable` in `src/pkg/review/dispatch.go`, which the dashboard (`dashboardAgentReviewCapable` in `src/pkg/dashboard/status_builder.go`), the relay and the merge-eligibility gate all call. +## Reviewer autoscaling (`review.reviewers.autoscale`) + +A reviewer lane with a fixed `replicas` count becomes the bottleneck when the PR backlog +grows, even with CPU and memory to spare. `review.reviewers.autoscale` lets the planner +vary how many of one reviewer agent's replicas take review kicks, between `min` and `max` +([#14685](https://github.com/hivecommons/hive/issues/14685)). + +```yaml +review: + reviewers: + autoscale: + enabled: true + agent: reviewer # base agent to scale (default "reviewer") + min: 2 # default 1 + max: 5 # default and ceiling: 5 (MaxAgentReplicas) + target_backlog_per_reviewer: 2 # scale up above this many waiting PRs per active reviewer + scale_up_cycles: 2 # consecutive cycles of pressure before a step up + scale_down_cycles: 5 # consecutive idle cycles before a step down + cooldown: 5m # minimum time between two steps + max_wait: 30m # a PR waiting this long is pressure on its own + cpu_per_reviewer: 0.5 # headroom one more reviewer needs (cores) + mem_per_reviewer_mb: 1024 # headroom one more reviewer needs (MB) +``` + +- **Replicas.** Enabling it materializes the agent's replicas up to `max` + (`reviewer-2` … `reviewer-N`) through the same machinery as `agents..replicas`, so + token minting, tmux wiring, cadence keys and the dashboard treat them as ordinary + replicas. The declared `replicas` is never rewritten. Replicas numbered above the active + count take no review-swarm kick and no cadence kick (the eval log lists them under + `agents_skipped` as `autoscale-inactive`). Review heads are not scaled. +- **Backlog.** Each cycle the planner records the PR heads that needed a plain reviewer + and found every active one busy (`DispatchPlan.ReviewerStarved`); the autoscaler tracks + how long each has waited. +- **Scale up** when the backlog exceeds `target_backlog_per_reviewer` per active reviewer, + or the oldest waiting PR is older than `max_wait`, for `scale_up_cycles` consecutive + cycles. A step is held at `max`, inside `cooldown`, while the provider budget latch is + set, when the pod's cgroup headroom is below one reviewer's `cpu_per_reviewer` / + `mem_per_reviewer_mb`, and for one `cooldown` after a reviewer kick failed with + `did not reach input prompt` (saturation, not demand). +- **Scale down** when fewer PRs wait than there are active reviewers for + `scale_down_cycles` consecutive cycles. The highest-numbered replica stops receiving new + work; a turn it already started is never interrupted and its verdict is recorded as + usual, so it drains rather than dies. +- **Observability.** Every step logs one line, e.g. + `reviewer autoscale: 2→3 reason=backlog=7 oldest=41m0s`, with the agent, min and max. + The active count is held in memory: after a restart the lane starts again at `min`. + +Source: `ReviewerAutoscaler.Decide` in `src/pkg/review/autoscale.go`; wiring in +`src/cmd/hive/review_autoscale.go`. + ## Review heads: several models per perspective `agents..review_heads` gives one base reviewer several model "heads" ([#13542](https://github.com/hivecommons/hive/issues/13542)). Each head is materialized like `replicas`: `--` copies the base agent with only `backend` and `model` replaced, is stripped on save, and is provisioned at boot by the entrypoint. `alias` defaults to a slug of `-` (without repeating the backend when the model already starts with it). The agent's Unix user and token group are `hive-`, and group names are limited to 32 characters, so `--` may be at most 27 characters; a longer name fails config load, so set a shorter `alias`. diff --git a/src/hive.yaml.example b/src/hive.yaml.example index 5a675cf660..78bd63e51a 100644 --- a/src/hive.yaml.example +++ b/src/hive.yaml.example @@ -834,6 +834,12 @@ governor: # # shadow.review_gate.would_have_held on the evidence bundle (#12104) # fan_out: false # max_parallel_reviews: 5 +# reviewers: +# autoscale: # scale the reviewer lane with the PR backlog (#14685); see docs/review-swarm.md +# enabled: false +# agent: reviewer +# min: 1 +# max: 5 # reviewer_agents: [] # optional explicit list; otherwise agents with review role/keywords are used # combined_perspectives: true # RECOMMENDED (#12126): one session + one comment covering every # # perspective; shared review boilerplate is sent once instead of once diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index a6af0979d7..ab3bda860a 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -6501,6 +6501,7 @@ func (c *Config) ExpandAgentReplicas() error { if replicas < 1 || replicas > MaxAgentReplicas { return fmt.Errorf("agent %s: replicas must be between 1 and %d", name, MaxAgentReplicas) } + replicas = c.Review.Reviewers.Autoscale.expandedReplicas(name, replicas) for i := 2; i <= replicas; i++ { derived := replicaAgentName(name, i) if existing, ok := baseAgents[derived]; ok && existing.ReplicaOf == "" { @@ -6522,8 +6523,11 @@ func (c *Config) ExpandAgentReplicas() error { } } for name, agent := range baseAgents { - replicas := normalizeReplicaCount(agent.Replicas) - agent.Replicas = replicas + agent.Replicas = normalizeReplicaCount(agent.Replicas) + // The autoscaler's max materializes the dynamic replicas + // (reviewer-3…reviewer-N) through this same path, without rewriting + // the declared replicas a save would persist. + replicas := c.Review.Reviewers.Autoscale.expandedReplicas(name, agent.Replicas) agent.ReplicaOf = "" agent.ReplicaIndex = 1 agent.ReplicaCount = replicas @@ -7502,6 +7506,9 @@ type ReviewConfig struct { // Backlog is where findings below the blocking line are filed // (hivecommons/hive#11088). See ReviewBacklogConfig. Backlog ReviewBacklogConfig `yaml:"backlog,omitempty" json:"backlog,omitempty"` + // Reviewers tunes the reviewer lane itself; today only its autoscaler + // (hivecommons/hive#14685). See ReviewReviewersConfig. + Reviewers ReviewReviewersConfig `yaml:"reviewers,omitempty" json:"reviewers,omitempty"` } // PlanMatchConfig is the switch for the plan_match review perspective diff --git a/src/pkg/config/review_autoscale.go b/src/pkg/config/review_autoscale.go new file mode 100644 index 0000000000..5d9736c08c --- /dev/null +++ b/src/pkg/config/review_autoscale.go @@ -0,0 +1,189 @@ +package config + +import ( + "fmt" + "strings" + "time" +) + +// Reviewer autoscaler defaults (hivecommons/hive#14685). A zero field in +// review.reviewers.autoscale means the matching default. +const ( + DefaultReviewerAutoscaleAgent = "reviewer" + DefaultReviewerAutoscaleMin = 1 + DefaultReviewerAutoscaleMax = MaxAgentReplicas + DefaultReviewerAutoscaleTargetBacklog = 2 + DefaultReviewerAutoscaleScaleUpCycles = 2 + DefaultReviewerAutoscaleScaleDownCycles = 5 + DefaultReviewerAutoscaleCooldown = 5 * time.Minute + DefaultReviewerAutoscaleMaxWait = 30 * time.Minute + DefaultReviewerAutoscaleCPUPerReviewer = 0.5 + DefaultReviewerAutoscaleMemPerReviewer = 1024 +) + +// ReviewReviewersConfig is review.reviewers. +type ReviewReviewersConfig struct { + Autoscale ReviewerAutoscaleConfig `yaml:"autoscale,omitempty" json:"autoscale,omitempty"` +} + +// ReviewerAutoscaleConfig is review.reviewers.autoscale: how many of one +// reviewer agent's replicas the review swarm hands work to, between Min and +// Max, driven by the backlog of PRs waiting for a reviewer. +// +// Enabling it materializes the agent's replicas up to Max through the same +// replica machinery as agents..replicas (reviewer-2…reviewer-N), so the +// planner, dashboard, kick governor and token cache see ordinary replicas. The +// autoscaler only decides how many of them receive review kicks. +type ReviewerAutoscaleConfig struct { + Enabled bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` + // Agent is the base reviewer agent this scales. Empty means "reviewer". + Agent string `yaml:"agent,omitempty" json:"agent,omitempty"` + Min int `yaml:"min,omitempty" json:"min,omitempty"` + Max int `yaml:"max,omitempty" json:"max,omitempty"` + // TargetBacklogPerReviewer is the waiting-PR count per active reviewer + // above which the lane is under-provisioned. + TargetBacklogPerReviewer int `yaml:"target_backlog_per_reviewer,omitempty" json:"target_backlog_per_reviewer,omitempty"` + // ScaleUpCycles and ScaleDownCycles are how many consecutive planner + // cycles the pressure must hold before a step up or down. + ScaleUpCycles int `yaml:"scale_up_cycles,omitempty" json:"scale_up_cycles,omitempty"` + ScaleDownCycles int `yaml:"scale_down_cycles,omitempty" json:"scale_down_cycles,omitempty"` + // Cooldown is the minimum time between two scaling actions (Go duration). + Cooldown string `yaml:"cooldown,omitempty" json:"cooldown,omitempty"` + // MaxWait is how long the oldest waiting PR may wait for a reviewer + // before that alone counts as scale-up pressure (Go duration). + MaxWait string `yaml:"max_wait,omitempty" json:"max_wait,omitempty"` + // CPUPerReviewer (cores) and MemPerReviewerMB are the per-reviewer cost + // estimate the headroom guard checks before a step up. + CPUPerReviewer float64 `yaml:"cpu_per_reviewer,omitempty" json:"cpu_per_reviewer,omitempty"` + MemPerReviewerMB int `yaml:"mem_per_reviewer_mb,omitempty" json:"mem_per_reviewer_mb,omitempty"` +} + +// EffectiveAgent is the scaled base agent name. +func (a ReviewerAutoscaleConfig) EffectiveAgent() string { + if n := strings.TrimSpace(a.Agent); n != "" { + return n + } + return DefaultReviewerAutoscaleAgent +} + +// EffectiveMax is Max, defaulted and clamped to MaxAgentReplicas. +func (a ReviewerAutoscaleConfig) EffectiveMax() int { + m := a.Max + if m <= 0 { + m = DefaultReviewerAutoscaleMax + } + if m > MaxAgentReplicas { + m = MaxAgentReplicas + } + return m +} + +// EffectiveMin is Min, defaulted and clamped to [1, EffectiveMax]. +func (a ReviewerAutoscaleConfig) EffectiveMin() int { + m := a.Min + if m <= 0 { + m = DefaultReviewerAutoscaleMin + } + if mx := a.EffectiveMax(); m > mx { + m = mx + } + return m +} + +func (a ReviewerAutoscaleConfig) EffectiveTargetBacklogPerReviewer() int { + if a.TargetBacklogPerReviewer > 0 { + return a.TargetBacklogPerReviewer + } + return DefaultReviewerAutoscaleTargetBacklog +} + +func (a ReviewerAutoscaleConfig) EffectiveScaleUpCycles() int { + if a.ScaleUpCycles > 0 { + return a.ScaleUpCycles + } + return DefaultReviewerAutoscaleScaleUpCycles +} + +func (a ReviewerAutoscaleConfig) EffectiveScaleDownCycles() int { + if a.ScaleDownCycles > 0 { + return a.ScaleDownCycles + } + return DefaultReviewerAutoscaleScaleDownCycles +} + +func (a ReviewerAutoscaleConfig) EffectiveCooldown() time.Duration { + return parseAutoscaleDuration(a.Cooldown, DefaultReviewerAutoscaleCooldown) +} + +func (a ReviewerAutoscaleConfig) EffectiveMaxWait() time.Duration { + return parseAutoscaleDuration(a.MaxWait, DefaultReviewerAutoscaleMaxWait) +} + +func (a ReviewerAutoscaleConfig) EffectiveCPUPerReviewer() float64 { + if a.CPUPerReviewer > 0 { + return a.CPUPerReviewer + } + return DefaultReviewerAutoscaleCPUPerReviewer +} + +func (a ReviewerAutoscaleConfig) EffectiveMemPerReviewerMB() int { + if a.MemPerReviewerMB > 0 { + return a.MemPerReviewerMB + } + return DefaultReviewerAutoscaleMemPerReviewer +} + +func parseAutoscaleDuration(raw string, def time.Duration) time.Duration { + raw = strings.TrimSpace(raw) + if raw == "" { + return def + } + d, err := time.ParseDuration(raw) + if err != nil || d < 0 { + return def + } + return d +} + +// expandedReplicas is how many replicas ExpandAgentReplicas materializes for +// agent name: the declared count, raised to the autoscaler's max for the +// scaled agent. +func (a ReviewerAutoscaleConfig) expandedReplicas(name string, declared int) int { + if !a.Enabled || name != a.EffectiveAgent() { + return declared + } + if mx := a.EffectiveMax(); mx > declared { + return mx + } + return declared +} + +// Validate rejects a review.reviewers.autoscale block that cannot mean what +// it says. +func (a ReviewerAutoscaleConfig) Validate() error { + if a.Min < 0 || a.Max < 0 { + return fmt.Errorf("review.reviewers.autoscale: min and max must not be negative") + } + if a.Max > MaxAgentReplicas { + return fmt.Errorf("review.reviewers.autoscale: max must be at most %d", MaxAgentReplicas) + } + if a.Min > 0 && a.Max > 0 && a.Min > a.Max { + return fmt.Errorf("review.reviewers.autoscale: min (%d) must not exceed max (%d)", a.Min, a.Max) + } + if a.TargetBacklogPerReviewer < 0 || a.ScaleUpCycles < 0 || a.ScaleDownCycles < 0 { + return fmt.Errorf("review.reviewers.autoscale: target_backlog_per_reviewer, scale_up_cycles and scale_down_cycles must not be negative") + } + if a.CPUPerReviewer < 0 || a.MemPerReviewerMB < 0 { + return fmt.Errorf("review.reviewers.autoscale: cpu_per_reviewer and mem_per_reviewer_mb must not be negative") + } + for _, f := range []struct{ name, raw string }{{"cooldown", a.Cooldown}, {"max_wait", a.MaxWait}} { + raw := strings.TrimSpace(f.raw) + if raw == "" { + continue + } + if d, err := time.ParseDuration(raw); err != nil || d < 0 { + return fmt.Errorf("review.reviewers.autoscale.%s: %q is not a non-negative duration", f.name, raw) + } + } + return nil +} diff --git a/src/pkg/config/review_autoscale_test.go b/src/pkg/config/review_autoscale_test.go new file mode 100644 index 0000000000..69deac17a5 --- /dev/null +++ b/src/pkg/config/review_autoscale_test.go @@ -0,0 +1,119 @@ +package config + +import ( + "strings" + "testing" + "time" +) + +// review.reviewers.autoscale (hivecommons/hive#14685). + +func autoscaleTestConfig(as ReviewerAutoscaleConfig) *Config { + return &Config{ + AgentRuntimeConfig: AgentRuntimeConfig{Agents: map[string]AgentConfig{ + "reviewer": {ID: "reviewer", Backend: "claude", Enabled: true, Replicas: 2}, + "scanner": {ID: "scanner", Backend: "claude", Enabled: true, Replicas: 1}, + }}, + Review: ReviewConfig{Reviewers: ReviewReviewersConfig{Autoscale: as}}, + } +} + +// Enabling autoscale materializes the scaled agent's replicas up to max +// through the ordinary replica machinery, and leaves other agents alone. +func TestExpandAgentReplicasMaterializesAutoscaleMax(t *testing.T) { + cfg := autoscaleTestConfig(ReviewerAutoscaleConfig{Enabled: true, Min: 2, Max: 4}) + if err := cfg.ExpandAgentReplicas(); err != nil { + t.Fatalf("ExpandAgentReplicas() error = %v", err) + } + for i, name := range []string{"reviewer", "reviewer-2", "reviewer-3", "reviewer-4"} { + got, ok := cfg.Agents[name] + if !ok { + t.Fatalf("missing replica %s", name) + } + if got.ReplicaIndex != i+1 || got.ReplicaCount != 4 { + t.Fatalf("%s metadata = index %d count %d, want %d/4", name, got.ReplicaIndex, got.ReplicaCount, i+1) + } + } + if _, ok := cfg.Agents["reviewer-5"]; ok { + t.Fatal("expanded past max") + } + if _, ok := cfg.Agents["scanner-2"]; ok { + t.Fatal("unscaled agent was expanded") + } + // The declared replicas stay what the operator wrote, so a save does not + // persist the autoscaler's max as a static count. + if got := cfg.Agents["reviewer"].Replicas; got != 2 { + t.Fatalf("declared replicas = %d, want 2 kept", got) + } +} + +func TestExpandAgentReplicasAutoscaleOffIsUnchanged(t *testing.T) { + cfg := autoscaleTestConfig(ReviewerAutoscaleConfig{Enabled: false, Max: 4}) + if err := cfg.ExpandAgentReplicas(); err != nil { + t.Fatalf("ExpandAgentReplicas() error = %v", err) + } + if _, ok := cfg.Agents["reviewer-3"]; ok { + t.Fatal("disabled autoscale expanded replicas") + } + if got := cfg.Agents["reviewer-2"].ReplicaCount; got != 2 { + t.Fatalf("ReplicaCount = %d, want 2", got) + } +} + +// A max below the declared replicas never removes a declared replica. +func TestExpandAgentReplicasAutoscaleNeverShrinksDeclared(t *testing.T) { + cfg := autoscaleTestConfig(ReviewerAutoscaleConfig{Enabled: true, Max: 1}) + if err := cfg.ExpandAgentReplicas(); err != nil { + t.Fatalf("ExpandAgentReplicas() error = %v", err) + } + if _, ok := cfg.Agents["reviewer-2"]; !ok { + t.Fatal("declared replica reviewer-2 was dropped") + } +} + +func TestReviewerAutoscaleDefaults(t *testing.T) { + var a ReviewerAutoscaleConfig + if a.EffectiveAgent() != "reviewer" || a.EffectiveMin() != 1 || a.EffectiveMax() != MaxAgentReplicas { + t.Fatalf("agent/min/max defaults = %s/%d/%d", a.EffectiveAgent(), a.EffectiveMin(), a.EffectiveMax()) + } + if a.EffectiveTargetBacklogPerReviewer() != 2 || a.EffectiveScaleUpCycles() != 2 || a.EffectiveScaleDownCycles() != 5 { + t.Fatal("cycle/target defaults wrong") + } + if a.EffectiveCooldown() != 5*time.Minute || a.EffectiveMaxWait() != 30*time.Minute { + t.Fatalf("cooldown/max_wait = %s/%s", a.EffectiveCooldown(), a.EffectiveMaxWait()) + } + if a.EffectiveCPUPerReviewer() != 0.5 || a.EffectiveMemPerReviewerMB() != 1024 { + t.Fatal("cost defaults wrong") + } + a = ReviewerAutoscaleConfig{Min: 4, Max: 3, Cooldown: "90s", Agent: " cr "} + if a.EffectiveMin() != 3 || a.EffectiveCooldown() != 90*time.Second || a.EffectiveAgent() != "cr" { + t.Fatalf("min clamp/cooldown/agent = %d/%s/%q", a.EffectiveMin(), a.EffectiveCooldown(), a.EffectiveAgent()) + } +} + +func TestReviewerAutoscaleValidate(t *testing.T) { + ok := []ReviewerAutoscaleConfig{ + {}, + {Enabled: true, Min: 2, Max: 4, Cooldown: "5m", MaxWait: "45m", CPUPerReviewer: 1, MemPerReviewerMB: 2048}, + } + for _, a := range ok { + if err := a.Validate(); err != nil { + t.Fatalf("Validate(%+v) = %v", a, err) + } + } + bad := map[string]ReviewerAutoscaleConfig{ + "max": {Max: MaxAgentReplicas + 1}, + "min (4)": {Min: 4, Max: 2}, + "negative": {Min: -1}, + "cooldown": {Cooldown: "soon"}, + "max_wait": {MaxWait: "-1m"}, + "cpu_per": {CPUPerReviewer: -1}, + "scale_up": {ScaleUpCycles: -1}, + } + for want, a := range bad { + err := a.Validate() + if err == nil || !strings.Contains(err.Error(), want) { + t.Fatalf("Validate(%+v) = %v, want error mentioning %q", a, err, want) + } + } +} diff --git a/src/pkg/config/validate.go b/src/pkg/config/validate.go index 8990fdb178..d525af9947 100644 --- a/src/pkg/config/validate.go +++ b/src/pkg/config/validate.go @@ -158,6 +158,9 @@ func (c *Config) ValidateWithOptions(opts ValidateOptions) error { if err := c.validateReviewHeads(); err != nil { return err } + if err := c.Review.Reviewers.Autoscale.Validate(); err != nil { + return err + } if err := c.validateGitHubActivityNotifications(); err != nil { return err } diff --git a/src/pkg/dashboard/status_builder.go b/src/pkg/dashboard/status_builder.go index fea23e62dd..4ff5606d15 100644 --- a/src/pkg/dashboard/status_builder.go +++ b/src/pkg/dashboard/status_builder.go @@ -3253,6 +3253,21 @@ func collectSystemResources() *SystemResources { return res } +// PodHeadroom is the pod's free CPU (cores) and memory (MB) from the same +// cgroup sample the status payload reports; -1 for a value that could not be +// read. The reviewer autoscaler's headroom guard (hivecommons/hive#14685). +func PodHeadroom() (cpuFreeCores, memFreeMB float64) { + res := collectSystemResources() + cpuFreeCores, memFreeMB = -1, -1 + if res.CpuCores > 0 && res.CpuPct > 0 { + cpuFreeCores = float64(res.CpuCores) * (1 - res.CpuPct/pctMultiplierSysRes) + } + if res.MemTotalMB > 0 { + memFreeMB = res.MemTotalMB - res.MemUsedMB + } + return cpuFreeCores, memFreeMB +} + // readCgroupInt64 reads a single int64 from a cgroup file. Returns -1 on error // or if the value is "max" (unlimited). func readCgroupInt64(path string) int64 { diff --git a/src/pkg/review/autoscale.go b/src/pkg/review/autoscale.go new file mode 100644 index 0000000000..163e01d59c --- /dev/null +++ b/src/pkg/review/autoscale.go @@ -0,0 +1,267 @@ +package review + +import ( + "fmt" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/pkg/config" +) + +// Reviewer autoscaler actions (hivecommons/hive#14685). +const ( + AutoscaleHold = "hold" + AutoscaleUp = "up" + AutoscaleDown = "down" +) + +// Reasons a step the pressure called for was not taken. Each is the reason +// field of the decision's log line. +const ( + AutoscaleHeldAtMax = "at_max" + AutoscaleHeldAtMin = "at_min" + AutoscaleHeldCooldown = "cooldown" + AutoscaleHeldSaturated = "saturated" + AutoscaleHeldBudget = "provider_budget" + AutoscaleHeldHeadroom = "headroom" +) + +// ReviewerAutoscaleSignals is what one planner cycle tells the autoscaler. +type ReviewerAutoscaleSignals struct { + Now time.Time + // Waiting keys the PR heads that needed a review this cycle and found no + // free reviewer (DispatchPlan.ReviewerStarved). + Waiting []string + // Saturated reports a recent reviewer kick that failed because the CLI + // did not reach its input prompt: the pod is out of capacity, and adding + // sessions would make that worse, not better. + Saturated bool + // BudgetLatched reports the provider spend latch: a new reviewer could + // not buy a token either. + BudgetLatched bool + // Headroom measures the pod's free CPU cores and memory (MB). It is only + // called when a step up is otherwise due, because sampling CPU takes a + // moment. Nil, or a negative value, means unknown, which never blocks. + Headroom func() (cpuFreeCores, memFreeMB float64) +} + +// ReviewerAutoscaleDecision is one cycle's outcome. +type ReviewerAutoscaleDecision struct { + At time.Time `json:"at"` + Action string `json:"action"` + From int `json:"from"` + To int `json:"to"` + Reason string `json:"reason,omitempty"` + Backlog int `json:"backlog"` + OldestWait time.Duration `json:"oldest_wait"` +} + +// Changed reports whether the decision moved the active count. +func (d ReviewerAutoscaleDecision) Changed() bool { return d.From != d.To } + +// String is the decision's log line body, e.g. "2→3 reason=backlog=7 oldest=41m". +func (d ReviewerAutoscaleDecision) String() string { + return fmt.Sprintf("%d→%d reason=%s", d.From, d.To, d.Reason) +} + +// ReviewerAutoscaleStatus is the dashboard view of the autoscaler. +type ReviewerAutoscaleStatus struct { + Agent string `json:"agent"` + Active int `json:"active"` + Min int `json:"min"` + Max int `json:"max"` + LastDecision ReviewerAutoscaleDecision `json:"last_decision"` + LastChange ReviewerAutoscaleDecision `json:"last_change"` +} + +// ReviewerAutoscaler holds the reviewer lane's active replica count across +// planner cycles. The zero value is ready; it starts at the configured min. +// State is in memory: a restart starts again from min and scales back up +// within scale_up_cycles if the backlog is still there. +type ReviewerAutoscaler struct { + mu sync.Mutex + active int + upStreak int + downStreak int + lastAction time.Time + waitingSince map[string]time.Time + saturatedAt time.Time + last ReviewerAutoscaleDecision + lastChange ReviewerAutoscaleDecision + min, max int + agent string +} + +// Active is the number of the scaled agent's replicas (reviewer, +// reviewer-2, …) the planner may hand review kicks to. +func (s *ReviewerAutoscaler) Active(cfg config.ReviewerAutoscaleConfig) int { + s.mu.Lock() + defer s.mu.Unlock() + return s.clampLocked(cfg) +} + +func (s *ReviewerAutoscaler) clampLocked(cfg config.ReviewerAutoscaleConfig) int { + s.min, s.max, s.agent = cfg.EffectiveMin(), cfg.EffectiveMax(), cfg.EffectiveAgent() + if s.active < s.min { + s.active = s.min + } + if s.active > s.max { + s.active = s.max + } + return s.active +} + +// NoteSaturation records a reviewer kick that failed because its CLI did not +// reach the input prompt. Scale-up is held for one cooldown after it. +func (s *ReviewerAutoscaler) NoteSaturation(at time.Time) { + s.mu.Lock() + defer s.mu.Unlock() + if at.After(s.saturatedAt) { + s.saturatedAt = at + } +} + +// Status snapshots the autoscaler for the dashboard. +func (s *ReviewerAutoscaler) Status() ReviewerAutoscaleStatus { + s.mu.Lock() + defer s.mu.Unlock() + return ReviewerAutoscaleStatus{Agent: s.agent, Active: s.active, Min: s.min, Max: s.max, LastDecision: s.last, LastChange: s.lastChange} +} + +// Decide runs one cycle of the autoscaler. +// +// Scale-up pressure is a backlog above target_backlog_per_reviewer per active +// reviewer, or a PR that has waited longer than max_wait; it must hold for +// scale_up_cycles consecutive cycles. Scale-down pressure is fewer waiting PRs +// than active reviewers (below one each), held for scale_down_cycles. The gap +// between the two thresholds and the streaks are the hysteresis; cooldown +// spaces any two steps. +// +// A step down removes the highest-numbered replica from new dispatch. It is +// never interrupted: a turn already in flight finishes and its verdict is +// recorded as usual, so the replica drains rather than dies. +func (s *ReviewerAutoscaler) Decide(cfg config.ReviewerAutoscaleConfig, sig ReviewerAutoscaleSignals) ReviewerAutoscaleDecision { + s.mu.Lock() + defer s.mu.Unlock() + active := s.clampLocked(cfg) + now := sig.Now + oldest := s.observeWaitingLocked(sig.Waiting, now) + backlog := len(sig.Waiting) + d := ReviewerAutoscaleDecision{At: now, Action: AutoscaleHold, From: active, To: active, Backlog: backlog, OldestWait: oldest} + pressure := fmt.Sprintf("backlog=%d oldest=%s", backlog, formatWait(oldest)) + + up := backlog > active*cfg.EffectiveTargetBacklogPerReviewer() || + (backlog > 0 && oldest >= cfg.EffectiveMaxWait()) + down := !up && backlog < active + switch { + case up: + s.upStreak++ + s.downStreak = 0 + case down: + s.downStreak++ + s.upStreak = 0 + default: + s.upStreak, s.downStreak = 0, 0 + } + coolingDown := !s.lastAction.IsZero() && now.Sub(s.lastAction) < cfg.EffectiveCooldown() + saturated := sig.Saturated || (!s.saturatedAt.IsZero() && now.Sub(s.saturatedAt) < cfg.EffectiveCooldown()) + + switch { + case up && s.upStreak >= cfg.EffectiveScaleUpCycles(): + held := "" + switch { + case active >= s.max: + held = AutoscaleHeldAtMax + case coolingDown: + held = AutoscaleHeldCooldown + case saturated: + held = AutoscaleHeldSaturated + case sig.BudgetLatched: + held = AutoscaleHeldBudget + case !headroomFor(cfg, sig): + held = AutoscaleHeldHeadroom + } + if held != "" { + d.Reason = held + " " + pressure + break + } + d.Action, d.To, d.Reason = AutoscaleUp, active+1, pressure + case down && s.downStreak >= cfg.EffectiveScaleDownCycles(): + held := "" + switch { + case active <= s.min: + held = AutoscaleHeldAtMin + case coolingDown: + held = AutoscaleHeldCooldown + } + if held != "" { + d.Reason = held + " " + pressure + break + } + d.Action, d.To, d.Reason = AutoscaleDown, active-1, pressure + default: + d.Reason = pressure + } + if d.Changed() { + s.active = d.To + s.lastAction = now + s.upStreak, s.downStreak = 0, 0 + s.lastChange = d + } + s.last = d + return d +} + +// observeWaitingLocked keeps the first time each waiting PR head was seen +// waiting, forgets heads no longer waiting, and returns the oldest wait. +func (s *ReviewerAutoscaler) observeWaitingLocked(keys []string, now time.Time) time.Duration { + seen := make(map[string]time.Time, len(keys)) + var oldest time.Duration + for _, k := range keys { + since, ok := s.waitingSince[k] + if !ok { + since = now + } + seen[k] = since + if w := now.Sub(since); w > oldest { + oldest = w + } + } + s.waitingSince = seen + return oldest +} + +// headroomFor reports whether the pod can afford one more reviewer session. +func headroomFor(cfg config.ReviewerAutoscaleConfig, sig ReviewerAutoscaleSignals) bool { + if sig.Headroom == nil { + return true + } + cpuFree, memFree := sig.Headroom() + if cpuFree >= 0 && cpuFree < cfg.EffectiveCPUPerReviewer() { + return false + } + if memFree >= 0 && memFree < float64(cfg.EffectiveMemPerReviewerMB()) { + return false + } + return true +} + +func formatWait(d time.Duration) string { + if d <= 0 { + return "0s" + } + return d.Round(time.Minute).String() +} + +// ReviewerStarvedKey keys a PR head in DispatchPlan.ReviewerStarved. +func ReviewerStarvedKey(repo string, number int, headSHA string) string { + return fmt.Sprintf("%s#%d@%s", strings.ToLower(strings.TrimSpace(repo)), number, strings.TrimSpace(headSHA)) +} + +// IsPromptNotReadyKickError reports a kick that failed because the agent's +// CLI never reached its input prompt (crux/agent kickPromptNotReadyErr), the +// autoscaler's saturation signal. +func IsPromptNotReadyKickError(err error) bool { + return err != nil && strings.Contains(err.Error(), "did not reach input prompt") +} diff --git a/src/pkg/review/autoscale_test.go b/src/pkg/review/autoscale_test.go new file mode 100644 index 0000000000..50b5b240ef --- /dev/null +++ b/src/pkg/review/autoscale_test.go @@ -0,0 +1,290 @@ +package review + +import ( + "errors" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" +) + +// Decision-table tests for the reviewer autoscaler (hivecommons/hive#14685). +// Every cycle is driven through an injected clock; nothing sleeps. + +var autoscaleT0 = time.Date(2026, 10, 27, 19, 20, 0, 0, time.UTC) + +func autoscaleCfg() config.ReviewerAutoscaleConfig { + return config.ReviewerAutoscaleConfig{Enabled: true, Min: 2, Max: 4} +} + +func waitingKeys(n int) []string { + out := make([]string, n) + for i := range out { + out[i] = ReviewerStarvedKey("acme/hive", i+1, "sha") + } + return out +} + +// run feeds cycles one minute apart starting at start and returns the last +// decision. +func runCycles(s *ReviewerAutoscaler, cfg config.ReviewerAutoscaleConfig, start time.Time, cycles int, sig ReviewerAutoscaleSignals) (ReviewerAutoscaleDecision, time.Time) { + var d ReviewerAutoscaleDecision + at := start + for i := 0; i < cycles; i++ { + sig.Now = at + d = s.Decide(cfg, sig) + at = at.Add(time.Minute) + } + return d, at +} + +func TestReviewerAutoscaleStartsAtMin(t *testing.T) { + var s ReviewerAutoscaler + if got := s.Active(autoscaleCfg()); got != 2 { + t.Fatalf("Active = %d, want min 2", got) + } +} + +func TestReviewerAutoscaleDecisionTable(t *testing.T) { + cases := []struct { + name string + cycles int + sig ReviewerAutoscaleSignals + wantAction string + wantTo int + wantReason string + }{ + {name: "one cycle of backlog is not sustained", cycles: 1, sig: ReviewerAutoscaleSignals{Waiting: waitingKeys(7)}, wantAction: AutoscaleHold, wantTo: 2}, + {name: "backlog above target for two cycles scales up", cycles: 2, sig: ReviewerAutoscaleSignals{Waiting: waitingKeys(7)}, wantAction: AutoscaleUp, wantTo: 3, wantReason: "backlog=7 oldest=1m0s"}, + {name: "backlog at target holds", cycles: 3, sig: ReviewerAutoscaleSignals{Waiting: waitingKeys(4)}, wantAction: AutoscaleHold, wantTo: 2}, + {name: "saturation holds scale-up", cycles: 2, sig: ReviewerAutoscaleSignals{Waiting: waitingKeys(7), Saturated: true}, wantAction: AutoscaleHold, wantTo: 2, wantReason: AutoscaleHeldSaturated + " backlog=7 oldest=1m0s"}, + {name: "provider budget latch holds scale-up", cycles: 2, sig: ReviewerAutoscaleSignals{Waiting: waitingKeys(7), BudgetLatched: true}, wantAction: AutoscaleHold, wantTo: 2, wantReason: AutoscaleHeldBudget + " backlog=7 oldest=1m0s"}, + {name: "no cpu headroom holds scale-up", cycles: 2, sig: ReviewerAutoscaleSignals{Waiting: waitingKeys(7), Headroom: func() (float64, float64) { return 0.2, 8000 }}, wantAction: AutoscaleHold, wantTo: 2, wantReason: AutoscaleHeldHeadroom + " backlog=7 oldest=1m0s"}, + {name: "no memory headroom holds scale-up", cycles: 2, sig: ReviewerAutoscaleSignals{Waiting: waitingKeys(7), Headroom: func() (float64, float64) { return 4, 512 }}, wantAction: AutoscaleHold, wantTo: 2, wantReason: AutoscaleHeldHeadroom + " backlog=7 oldest=1m0s"}, + {name: "unknown headroom never blocks", cycles: 2, sig: ReviewerAutoscaleSignals{Waiting: waitingKeys(7), Headroom: func() (float64, float64) { return -1, -1 }}, wantAction: AutoscaleUp, wantTo: 3}, + {name: "idle below one per reviewer for five cycles stays at min", cycles: 5, sig: ReviewerAutoscaleSignals{Waiting: waitingKeys(1)}, wantAction: AutoscaleHold, wantTo: 2, wantReason: AutoscaleHeldAtMin + " backlog=1 oldest=4m0s"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var s ReviewerAutoscaler + d, _ := runCycles(&s, autoscaleCfg(), autoscaleT0, tc.cycles, tc.sig) + if d.Action != tc.wantAction || d.To != tc.wantTo { + t.Fatalf("decision = %+v, want action %s to %d", d, tc.wantAction, tc.wantTo) + } + if tc.wantReason != "" && d.Reason != tc.wantReason { + t.Fatalf("reason = %q, want %q", d.Reason, tc.wantReason) + } + if got := s.Active(autoscaleCfg()); got != tc.wantTo { + t.Fatalf("Active = %d, want %d", got, tc.wantTo) + } + }) + } +} + +// A small backlog that has waited past max_wait is scale-up pressure on its +// own: the oldest PR is the stage-dwell signal from the issue. +func TestReviewerAutoscaleOldestWaitScalesUp(t *testing.T) { + cfg := autoscaleCfg() + cfg.MaxWait = "30m" + var s ReviewerAutoscaler + waiting := waitingKeys(2) + s.Decide(cfg, ReviewerAutoscaleSignals{Now: autoscaleT0, Waiting: waiting}) + d := s.Decide(cfg, ReviewerAutoscaleSignals{Now: autoscaleT0.Add(29 * time.Minute), Waiting: waiting}) + if d.Changed() { + t.Fatalf("scaled before max_wait: %+v", d) + } + d = s.Decide(cfg, ReviewerAutoscaleSignals{Now: autoscaleT0.Add(31 * time.Minute), Waiting: waiting}) + if d.Changed() { + t.Fatalf("first over-age cycle must not act alone (scale_up_cycles=2): %+v", d) + } + d = s.Decide(cfg, ReviewerAutoscaleSignals{Now: autoscaleT0.Add(32 * time.Minute), Waiting: waiting}) + if d.Action != AutoscaleUp || d.To != 3 || d.OldestWait != 32*time.Minute { + t.Fatalf("decision = %+v, want up to 3 with oldest 32m", d) + } + if d.String() != "2→3 reason=backlog=2 oldest=32m0s" { + t.Fatalf("log line = %q", d.String()) + } +} + +// A head that stops waiting is forgotten, so its age does not linger. +func TestReviewerAutoscaleForgetsServedPRs(t *testing.T) { + var s ReviewerAutoscaler + cfg := autoscaleCfg() + s.Decide(cfg, ReviewerAutoscaleSignals{Now: autoscaleT0, Waiting: waitingKeys(1)}) + s.Decide(cfg, ReviewerAutoscaleSignals{Now: autoscaleT0.Add(10 * time.Minute)}) + d := s.Decide(cfg, ReviewerAutoscaleSignals{Now: autoscaleT0.Add(20 * time.Minute), Waiting: waitingKeys(1)}) + if d.OldestWait != 0 { + t.Fatalf("oldest wait = %s, want 0 for a head seen waiting again", d.OldestWait) + } +} + +func TestReviewerAutoscaleCooldownAndMax(t *testing.T) { + cfg := autoscaleCfg() + var s ReviewerAutoscaler + busy := ReviewerAutoscaleSignals{Waiting: waitingKeys(20)} + d, at := runCycles(&s, cfg, autoscaleT0, 2, busy) + if d.To != 3 { + t.Fatalf("first step = %+v, want 3", d) + } + // Pressure keeps holding, but the next step waits for the 5m cooldown. + d, at = runCycles(&s, cfg, at, 3, busy) + if d.Changed() || d.Reason != AutoscaleHeldCooldown+" backlog=20 oldest=4m0s" { + t.Fatalf("inside cooldown = %+v, want held cooldown", d) + } + d, at = runCycles(&s, cfg, at, 2, busy) + if d.Action != AutoscaleUp || d.To != 4 { + t.Fatalf("after cooldown = %+v, want up to 4", d) + } + d, _ = runCycles(&s, cfg, at.Add(10*time.Minute), 2, busy) + if d.Changed() || d.To != 4 || d.Reason != AutoscaleHeldAtMax+" backlog=20 oldest=18m0s" { + t.Fatalf("at max = %+v, want held at_max 4", d) + } +} + +func TestReviewerAutoscaleScalesDownAfterSustainedIdle(t *testing.T) { + cfg := autoscaleCfg() + var s ReviewerAutoscaler + _, at := runCycles(&s, cfg, autoscaleT0, 2, ReviewerAutoscaleSignals{Waiting: waitingKeys(20)}) + if s.Active(cfg) != 3 { + t.Fatal("setup: expected 3 active") + } + at = at.Add(10 * time.Minute) // past the cooldown + // Four idle cycles are not enough (scale_down_cycles=5). + d, at := runCycles(&s, cfg, at, 4, ReviewerAutoscaleSignals{Waiting: waitingKeys(2)}) + if d.Changed() { + t.Fatalf("scaled down after 4 cycles: %+v", d) + } + // Backlog of 3 at 3 reviewers is not below one each: it resets the streak. + runCycles(&s, cfg, at, 1, ReviewerAutoscaleSignals{Waiting: waitingKeys(3)}) + d, at = runCycles(&s, cfg, at.Add(time.Minute), 4, ReviewerAutoscaleSignals{}) + if d.Changed() { + t.Fatalf("streak was not reset by a busy cycle: %+v", d) + } + d, _ = runCycles(&s, cfg, at, 1, ReviewerAutoscaleSignals{}) + if d.Action != AutoscaleDown || d.From != 3 || d.To != 2 { + t.Fatalf("decision = %+v, want down 3→2", d) + } + if st := s.Status(); st.Active != 2 || st.LastChange.Action != AutoscaleDown || st.Min != 2 || st.Max != 4 { + t.Fatalf("status = %+v", st) + } +} + +// A kick that failed with "did not reach input prompt" holds scale-up for one +// cooldown, then stops counting. +func TestReviewerAutoscaleNoteSaturationExpires(t *testing.T) { + cfg := autoscaleCfg() + var s ReviewerAutoscaler + s.NoteSaturation(autoscaleT0) + busy := ReviewerAutoscaleSignals{Waiting: waitingKeys(20)} + d, _ := runCycles(&s, cfg, autoscaleT0.Add(time.Minute), 2, busy) + if d.Changed() || d.Reason[:len(AutoscaleHeldSaturated)] != AutoscaleHeldSaturated { + t.Fatalf("within cooldown of saturation = %+v, want held saturated", d) + } + d, _ = runCycles(&s, cfg, autoscaleT0.Add(6*time.Minute), 1, busy) + if d.Action != AutoscaleUp { + t.Fatalf("after saturation cooldown = %+v, want up", d) + } +} + +// Lowering max in config clamps the active count on the next read. +func TestReviewerAutoscaleClampsToConfigChange(t *testing.T) { + cfg := autoscaleCfg() + var s ReviewerAutoscaler + runCycles(&s, cfg, autoscaleT0, 2, ReviewerAutoscaleSignals{Waiting: waitingKeys(20)}) + cfg.Max = 2 + if got := s.Active(cfg); got != 2 { + t.Fatalf("Active after max lowered = %d, want 2", got) + } +} + +func TestIsPromptNotReadyKickError(t *testing.T) { + if !IsPromptNotReadyKickError(errors.New("agent reviewer-2 CLI did not reach input prompt: busy")) { + t.Fatal("prompt-not-ready error not recognized") + } + if IsPromptNotReadyKickError(errors.New("tmux: no server running")) || IsPromptNotReadyKickError(nil) { + t.Fatal("unrelated error recognized as saturation") + } +} + +func autoscaleFleet() []AgentCapability { + r2 := replica("reviewer-2", "reviewer") + r2.ReplicaIndex = 2 + r3 := replica("reviewer-3", "reviewer") + r3.ReplicaIndex = 3 + base := reviewer("reviewer") + base.ReplicaIndex = 1 + return []AgentCapability{base, r2, r3} +} + +func autoscaleDispatchOpts(limits map[string]int) DispatchOptions { + return DispatchOptions{ + RequireApproval: true, FanOut: true, CombinedPerspectives: true, + MaxParallelReviews: 10, ProjectOrg: "acme", + Agents: autoscaleFleet(), ReplicaLimits: limits, + Now: autoscaleT0, + } +} + +// Replicas above the autoscaler's active count take no new review kick, and +// the PR left without a reviewer is reported as the backlog. +func TestPlanDispatchHonorsReplicaLimits(t *testing.T) { + prs := []PullRequest{dispatchPRNum(1, "sha1"), dispatchPRNum(2, "sha2"), dispatchPRNum(3, "sha3")} + plan := PlanDispatch(prs, Artifact{}, DispatchState{}, autoscaleDispatchOpts(map[string]int{"reviewer": 2})) + agents := map[string]bool{} + for _, k := range plan.ReviewKicks { + agents[k.Agent] = true + } + if len(plan.ReviewKicks) != 2 || !agents["reviewer"] || !agents["reviewer-2"] || agents["reviewer-3"] { + t.Fatalf("kicks = %+v, want reviewer and reviewer-2 only", plan.ReviewKicks) + } + want := ReviewerStarvedKey("acme/hive", 3, "sha3") + if len(plan.ReviewerStarved) != 1 || plan.ReviewerStarved[0] != want { + t.Fatalf("ReviewerStarved = %v, want [%s]", plan.ReviewerStarved, want) + } + + plan = PlanDispatch(prs, Artifact{}, DispatchState{}, autoscaleDispatchOpts(nil)) + if len(plan.ReviewKicks) != 3 || len(plan.ReviewerStarved) != 0 { + t.Fatalf("unlimited plan = %d kicks, starved %v; want 3 kicks, none starved", len(plan.ReviewKicks), plan.ReviewerStarved) + } +} + +// A scaled-in replica keeps the turn it is already running: its pending entry +// is neither pruned nor redispatched, so the verdict still lands. +func TestPlanDispatchDrainsScaledInReplica(t *testing.T) { + pending := []PendingReview{{Repo: "acme/hive", Number: 1, HeadSHA: "sha1", Perspective: PerspectiveCorrectness, Agent: "reviewer-3", Dispatched: autoscaleT0.Add(-5 * time.Minute)}} + opts := autoscaleDispatchOpts(map[string]int{"reviewer": 2}) + plan := PlanDispatch([]PullRequest{dispatchPRNum(1, "sha1")}, Artifact{}, DispatchState{Pending: pending}, opts) + kept := false + for _, p := range plan.State.Pending { + if p.Agent == "reviewer-3" && p.Number == 1 { + kept = true + } + } + if !kept { + t.Fatalf("draining replica's in-flight entry was dropped: %+v", plan.State.Pending) + } + for _, k := range plan.ReviewKicks { + if k.Agent == "reviewer-3" { + t.Fatalf("scaled-in replica got a new kick: %+v", k) + } + } +} + +// Review heads carry ReplicaOf too, but are their own lanes: the limit must +// not hide them. +func TestOverReplicaLimitIgnoresReviewHeads(t *testing.T) { + head := headReviewer("reviewer-sonnet-3", "reviewer-sonnet", "claude", "m") + head.ReplicaOf = "reviewer" + head.ReplicaIndex = 3 + if overReplicaLimit(head, map[string]int{"reviewer": 1}) { + t.Fatal("review head was limited") + } + plain := replica("reviewer-3", "reviewer") + plain.ReplicaIndex = 3 + if !overReplicaLimit(plain, map[string]int{"reviewer": 2}) { + t.Fatal("plain replica 3 not limited at 2") + } + if overReplicaLimit(plain, map[string]int{"other": 1}) { + t.Fatal("replica of an unscaled base was limited") + } +} diff --git a/src/pkg/review/dispatch.go b/src/pkg/review/dispatch.go index d817fbf77e..4cd00100bd 100644 --- a/src/pkg/review/dispatch.go +++ b/src/pkg/review/dispatch.go @@ -72,7 +72,10 @@ var LegacyReviewDispatchStatePath = filepath.Join(outputschema.AgentReportDir, R type AgentCapability struct { Name string // ReplicaOf is the base agent a derived replica was expanded from. - ReplicaOf string + ReplicaOf string + // ReplicaIndex is the replica's position under its base (1 for the base + // itself, N for base-N); what DispatchOptions.ReplicaLimits compares. + ReplicaIndex int Enabled bool Paused bool OnDemand bool @@ -108,6 +111,7 @@ func CapabilityFromConfig(name string, ac config.AgentConfig) AgentCapability { return AgentCapability{ Name: name, ReplicaOf: ac.ReplicaOf, + ReplicaIndex: ac.ReplicaIndex, Enabled: ac.Enabled, Paused: ac.Paused, OnDemand: ac.OnDemand, @@ -246,8 +250,14 @@ type DispatchOptions struct { DeltaRereview bool DeltaTrivialLines int DeltaMaxAge time.Duration - Agents []AgentCapability - Now time.Time + // ReplicaLimits caps, per base agent, which of its plain replicas take + // new review kicks: replica N is offered work only while N <= the limit + // (review.reviewers.autoscale, hivecommons/hive#14685). A base absent + // from the map is unlimited. Review heads are not affected. A replica + // over the limit keeps any kick already in flight; it only drains. + ReplicaLimits map[string]int + Agents []AgentCapability + Now time.Time } type DispatchState struct { @@ -500,7 +510,11 @@ type DispatchPlan struct { // ForcedFixes are the changes_requested heads whose fix kick was forced // this cycle after RevisitStarvationCycles revisit-suppressed cycles. ForcedFixes []ForcedFix - State DispatchState + // ReviewerStarved keys (ReviewerStarvedKey) the PR heads that needed a + // plain reviewer this cycle and found every one busy: the reviewer + // autoscaler's backlog (hivecommons/hive#14685). + ReviewerStarved []string + State DispatchState } type FixDispatchExclusion struct { @@ -777,6 +791,7 @@ func PlanDispatch(prs []PullRequest, artifact Artifact, state DispatchState, opt if opts.CombinedPerspectives { reviewer, free := freeReviewer(prReviewers, plan.inFlight(opts, now)) if !free { + plan.noteReviewerStarved(pr) continue } agent := reviewer.Name @@ -851,6 +866,9 @@ func PlanDispatch(prs []PullRequest, artifact Artifact, state DispatchState, opt for i := 0; i < limit; i++ { reviewer, free := freeReviewer(prReviewers, plan.inFlight(opts, now)) if !free { + if i == 0 { + plan.noteReviewerStarved(pr) + } break } agent := reviewer.Name @@ -886,6 +904,12 @@ func PlanDispatch(prs []PullRequest, artifact Artifact, state DispatchState, opt return plan } +// noteReviewerStarved records a PR head that waited this cycle because every +// plain reviewer was busy. +func (p *DispatchPlan) noteReviewerStarved(pr PullRequest) { + p.ReviewerStarved = append(p.ReviewerStarved, ReviewerStarvedKey(pr.Repo, pr.Number, pr.HeadSHA)) +} + // deltaPreviousHead is the head a delta kick is judged against, "" for a // full review. func deltaPreviousHead(d *DeltaContext) string { @@ -1686,6 +1710,9 @@ func reviewCapableAgents(opts DispatchOptions) []AgentCapability { if a.Name == "" || !a.Enabled || a.Paused || a.OnDemand || !a.UsesKick { continue } + if overReplicaLimit(a, opts.ReplicaLimits) { + continue + } if ReviewCapable(a, allowed) { out = append(out, a) } @@ -1694,6 +1721,24 @@ func reviewCapableAgents(opts DispatchOptions) []AgentCapability { return out } +// overReplicaLimit reports a plain replica numbered above its base's +// DispatchOptions.ReplicaLimits entry. Review heads carry ReplicaOf too, but +// are their own lanes and never limited. +func overReplicaLimit(a AgentCapability, limits map[string]int) bool { + if len(limits) == 0 || a.ReviewHead != "" { + return false + } + limit, ok := limits[baseOf(a)] + if !ok { + return false + } + index := a.ReplicaIndex + if index <= 0 { + index = 1 + } + return index > limit +} + // ReviewCapable reports whether one agent is a reviewer under the same rule // the dispatcher uses to hand out review kicks: when review.reviewer_agents // names lanes, membership in that set decides; otherwise the agent's name,