From 3849da1ba59467d1a1bcc5a9f873ba0d99040704 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 27 Oct 2026 15:59:18 -0400 Subject: [PATCH 1/4] =?UTF-8?q?=E2=9C=A8=20review:=20reviewer=20lane=20sca?= =?UTF-8?q?les=20concurrency=20with=20backlog?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add review.reviewers.autoscale: a controller that each governor cycle reads the PRs the dispatch planner left waiting for a free reviewer or a parallel slot and raises or lowers the reviewer lane's materialised replica count between min and max. Scale-up fires after scale_up_cycles cycles behind target_backlog_per_reviewer (or a wait older than the review-pending stage dwell) and is refused by named guards: at-max, cooldown, saturated kicks (CLI did not reach input prompt), provider-budget, provider-headroom, max-parallel-reviews, and cgroup cpu/memory. Scale-down drains the top replica after scale_down_cycles idle cycles and retires it only when its turn ends. Dynamic replicas go through ExpandAgentReplicas, so reviewer-3..N share role, prompt, token, pane and cadence with reviewer-2; the entrypoint pre-creates users up to max. Decisions log as 'reviewer autoscale: 2→3 reason=backlog=7 oldest=41m', are exposed on /api/status.reviewerAutoscale, and show as an auto badge on the lane's agent cards. Settings → Review Gate → Reviewers exposes the knobs. Fixes #14685 Signed-off-by: Andy Anderson --- changelog.d/added-reviewer-autoscale.md | 1 + src/cmd/hive/main.go | 69 +-- src/cmd/hive/reviewer_autoscale.go | 220 ++++++++++ src/deploy/entrypoint.sh | 51 ++- src/deploy/test_entrypoint_replica_names.sh | 46 ++ src/docs/agent-configuration.md | 2 +- src/docs/operator-reference.md | 1 + src/docs/review-swarm.md | 30 ++ src/hive.yaml.example | 12 + src/pkg/config/config.go | 24 +- src/pkg/config/reviewer_autoscale.go | 277 ++++++++++++ src/pkg/config/reviewer_autoscale_test.go | 145 ++++++ src/pkg/config/validate.go | 3 + src/pkg/dashboard/api_config_review.go | 53 +++ src/pkg/dashboard/deps.go | 21 +- src/pkg/dashboard/overview_status.go | 5 + src/pkg/dashboard/server.go | 4 + src/pkg/dashboard/static/index.html | 81 +++- src/pkg/review/autoscale/cgroup.go | 140 ++++++ src/pkg/review/autoscale/controller.go | 464 ++++++++++++++++++++ src/pkg/review/autoscale/controller_test.go | 251 +++++++++++ src/pkg/review/dispatch.go | 56 ++- src/pkg/review/dispatch_waiting_test.go | 92 ++++ 23 files changed, 1995 insertions(+), 53 deletions(-) create mode 100644 changelog.d/added-reviewer-autoscale.md create mode 100644 src/cmd/hive/reviewer_autoscale.go create mode 100644 src/pkg/config/reviewer_autoscale.go create mode 100644 src/pkg/config/reviewer_autoscale_test.go create mode 100644 src/pkg/review/autoscale/cgroup.go create mode 100644 src/pkg/review/autoscale/controller.go create mode 100644 src/pkg/review/autoscale/controller_test.go create mode 100644 src/pkg/review/dispatch_waiting_test.go diff --git a/changelog.d/added-reviewer-autoscale.md b/changelog.d/added-reviewer-autoscale.md new file mode 100644 index 0000000000..2ed2ec2fa0 --- /dev/null +++ b/changelog.d/added-reviewer-autoscale.md @@ -0,0 +1 @@ +- `review.reviewers.autoscale` (off by default; Settings → Review Gate → Reviewers) sizes the reviewer lane to its backlog ([#14685](https://github.com/hivecommons/hive/issues/14685)): the planner now reports the PRs it left waiting for a free reviewer or a parallel slot, and a controller adds a `reviewer-N` replica after `scale_up_cycles` cycles behind `target_backlog_per_reviewer` (or a wait older than the review-pending stage dwell), drains the top replica after `scale_down_cycles` idle cycles and retires it only once its turn ends. Scale-ups are refused by named guards — `at-max`, `cooldown`, `saturated` (kicks not reaching the CLI prompt), `provider-budget`, `provider-headroom`, `max-parallel-reviews`, cgroup `cpu` / `memory`. Dynamic replicas go through the ordinary replica expansion (same role, prompt, token, pane and cadence as `reviewer-2`) and the entrypoint pre-creates their users up to `max`. Each decision logs as `reviewer autoscale: 2→3 reason=backlog=7 oldest=41m0s`; `/api/status.reviewerAutoscale` and an *auto* badge on the lane's agent cards show current/target/min/max and the last decision. diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 31cbbe4abb..3e93841170 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -71,6 +71,7 @@ import ( "github.com/hivecommons/hive/pkg/promptsrc" "github.com/hivecommons/hive/pkg/retro" "github.com/hivecommons/hive/pkg/review" + "github.com/hivecommons/hive/pkg/review/autoscale" "github.com/hivecommons/hive/pkg/rfccoverage" "github.com/hivecommons/hive/pkg/rotation" "github.com/hivecommons/hive/pkg/scheduler" @@ -1452,34 +1453,35 @@ func (b *boot) wireBootClosures() { // linearagent adapters behind the dashboard's consumer-defined // interfaces — this composition root is the only non-test place that // names the concrete types. - Watsonx: watsonxGateway{}, - OpenRouter: openRouterGateway{}, - NewLinearAgent: newLinearAgentGateway(b.logger), - LinearStoredViewerID: linearStoredViewerID, - DelegateDecider: newDelegateDeciderRunner(func() *config.Config { return b.cfg }), - ChatGrounder: newChatGroundingRunner(func() *config.Config { return b.cfg }), - Governor: b.gov, - GHClient: b.ghClient, - StageEngineStatus: func() stageengine.Status { return stageEngineStatus(b.cfg, b.ghClient, b.logger) }, - ReviewEvents: b.reviewEvents, - WebhookWake: b.webhookWakeFunc(), - ApplyWebhookItems: b.applyWebhookItemsFunc(), - WebhookDelivery: b.webhookDeliveryFunc(), - EvalLoop: b.evalLoopStatusFunc(), - SentinelPREvent: b.sentinelPREventFunc(), - ProjectsV2ItemEvent: b.projectsV2ItemEventFunc(), - GHAppAuth: b.appAuth, - GHTokenScopes: b.ghAuth.TokenScopes, - Tokens: b.tokenCollector, - Knowledge: b.knowledgeAPI, - Inception: b.inceptionEngine, - Nous: b.nousState, - Scheduler: b.sched, - KnowledgePrimer: knowledgePrimerControl{b: b}, - MetricsCollector: b.metricsCollector, - RotationMgr: b.rotationMgr, - QuestionAutoclose: questionAutocloseView{b: b}, - HeadroomPublisher: b.quotaReadingPublisher, + Watsonx: watsonxGateway{}, + OpenRouter: openRouterGateway{}, + NewLinearAgent: newLinearAgentGateway(b.logger), + LinearStoredViewerID: linearStoredViewerID, + DelegateDecider: newDelegateDeciderRunner(func() *config.Config { return b.cfg }), + ChatGrounder: newChatGroundingRunner(func() *config.Config { return b.cfg }), + Governor: b.gov, + GHClient: b.ghClient, + StageEngineStatus: func() stageengine.Status { return stageEngineStatus(b.cfg, b.ghClient, b.logger) }, + ReviewerAutoscaleStatus: func() autoscale.Status { return reviewerAutoscaler.Status() }, + ReviewEvents: b.reviewEvents, + WebhookWake: b.webhookWakeFunc(), + ApplyWebhookItems: b.applyWebhookItemsFunc(), + WebhookDelivery: b.webhookDeliveryFunc(), + EvalLoop: b.evalLoopStatusFunc(), + SentinelPREvent: b.sentinelPREventFunc(), + ProjectsV2ItemEvent: b.projectsV2ItemEventFunc(), + GHAppAuth: b.appAuth, + GHTokenScopes: b.ghAuth.TokenScopes, + Tokens: b.tokenCollector, + Knowledge: b.knowledgeAPI, + Inception: b.inceptionEngine, + Nous: b.nousState, + Scheduler: b.sched, + KnowledgePrimer: knowledgePrimerControl{b: b}, + MetricsCollector: b.metricsCollector, + RotationMgr: b.rotationMgr, + QuestionAutoclose: questionAutocloseView{b: b}, + HeadroomPublisher: b.quotaReadingPublisher, // #3972: hand the ACMM advisor the SAME cached fleet-stats collector // the heartbeat reads, so its merge-success signal reuses the existing // 30-minute collect loop instead of issuing a second GitHub fetch. @@ -3502,6 +3504,7 @@ func (b *boot) bootKnowledgeWith(deps bootKnowledgeDeps) { func (b *boot) bootSupervision() { if b.cfg.Governor.Rotation.Enabled { b.rotationMgr = rotation.NewManager(b.cfg.Governor.Rotation) + reviewerAutoscaler.SetHeadroomReporter(b.rotationMgr) // Publish normalized readings to where the contributor quota guard reads // them (kubestellar/hive#6967), off the rotation manager's own probe // loop and the operator's configured provider→backends map. @@ -3952,6 +3955,10 @@ func (b *boot) bootWatchersWith(deps bootWatchersDeps) { // Preserve runtime-only fields that are not in the YAML newCfg.HiveID = b.cfg.HiveID + // The reviewer autoscaler's current lane size survives a reload so + // an edit elsewhere in hive.yaml does not snap reviewers back to the + // declared count (hivecommons/hive#14685). + newCfg.ReviewerAutoscaleActive = b.cfg.ReviewerAutoscaleActive // Preserve ACMM level from the agent manager — it is the // authoritative source. The file may have a stale value if @@ -7346,6 +7353,7 @@ func runEvalCycle( } return "" }) + agentsDue, agentsSkipped = reviewerAutoscaler.withholdDrainingReviewers(agentsDue, agentsSkipped) // PROVIDER SPEND REBUFF (#4294). When the inference gateway is refusing on a // money limit, every kick launched this cycle is a run that cannot buy a @@ -7444,6 +7452,10 @@ func runEvalCycle( agentsDue = sched.FilterFirstScanDeferredAgents(agentsDue) agentsSkipped = append(agentsSkipped, firstScanDeferredSkips(dueBeforeDeferral, agentsDue)...) reviewPlan := planReviewDispatchWith(ctx, cfg, actionable, agentMgr, beadStores, reviewPlannerGitHubFor(ghClient), logger) + // Size the reviewer lane to what the planner just left waiting + // (hivecommons/hive#14685). A replica added here takes kicks from the + // next cycle on; a retire only lands once the reviewer is idle. + reviewerAutoscaler.Run(ctx, cfg, reviewPlan, agentMgr, gov, logger, time.Now()) auditWithheldReviewFixes(reviewPlan, dashSrv, logger) if ghClient != nil { postReviewAcknowledgements(ctx, ghClient, reviewPlan, logger) @@ -10728,6 +10740,7 @@ func planReviewDispatchWith(ctx context.Context, cfg *config.Config, actionable for name, ac := range cfg.EnabledAgents() { cap := review.CapabilityFromConfig(name, ac) cap.Paused = cap.Paused || (agentMgr != nil && agentMgr.IsPaused(name)) + cap.Draining = reviewerAutoscaler.Draining(name) agents = append(agents, cap) } reviewBotLogins := []string{} diff --git a/src/cmd/hive/reviewer_autoscale.go b/src/cmd/hive/reviewer_autoscale.go new file mode 100644 index 0000000000..e923a793f6 --- /dev/null +++ b/src/cmd/hive/reviewer_autoscale.go @@ -0,0 +1,220 @@ +package main + +import ( + "context" + "log/slog" + "sync" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/dashboard" + "github.com/hivecommons/hive/pkg/governor" + "github.com/hivecommons/hive/pkg/review" + "github.com/hivecommons/hive/pkg/review/autoscale" + "github.com/hivecommons/hive/pkg/rotation" + "github.com/hivecommons/hive/pkg/stagelabels" +) + +// reviewerAutoscaleSkipReason is the agents_skipped annotation for a +// draining reviewer withheld from the cadence kick. +const reviewerAutoscaleSkipReason = "reviewer-autoscale draining" + +// unknownProviderHeadroom tells the controller no headroom reading exists. +const unknownProviderHeadroom = -1 + +// reviewerAutoscaler is the process-wide reviewer-lane autoscaler +// (hivecommons/hive#14685). It is package-level, like providerBudgetProbe, +// so runEvalCycle's already-long signature does not grow; the eval loop is +// single-threaded but the dashboard reads Status concurrently, hence the +// mutex. +var reviewerAutoscaler = &reviewerAutoscaleRuntime{prober: autoscale.NewProber()} + +type reviewerAutoscaleRuntime struct { + mu sync.Mutex + ctrl *autoscale.Controller + prober *autoscale.Prober + headroom rotation.HeadroomReporter + status autoscale.Status +} + +// SetHeadroomReporter wires the provider headroom probe when rotation is on. +func (r *reviewerAutoscaleRuntime) SetHeadroomReporter(h rotation.HeadroomReporter) { + r.mu.Lock() + defer r.mu.Unlock() + r.headroom = h +} + +// Draining reports whether name is a reviewer being retired; the planner +// and the cadence kick both withhold work from it. +func (r *reviewerAutoscaleRuntime) Draining(name string) bool { + if r == nil { + return false + } + r.mu.Lock() + defer r.mu.Unlock() + return r.ctrl != nil && r.ctrl.Draining(name) +} + +// Status is the dashboard snapshot; Enabled is false when autoscale is off. +func (r *reviewerAutoscaleRuntime) Status() autoscale.Status { + if r == nil { + return autoscale.Status{} + } + r.mu.Lock() + defer r.mu.Unlock() + return r.status +} + +// withholdDrainingReviewers removes draining reviewers from the cadence +// kick list, reporting them in agentsSkipped like every other gate. +func (r *reviewerAutoscaleRuntime) withholdDrainingReviewers(agentsDue, agentsSkipped []string) ([]string, []string) { + if r == nil { + return agentsDue, agentsSkipped + } + kept := agentsDue[:0] + for _, name := range agentsDue { + if r.Draining(name) { + agentsSkipped = append(agentsSkipped, name+" ("+reviewerAutoscaleSkipReason+")") + continue + } + kept = append(kept, name) + } + return kept, agentsSkipped +} + +// Run folds this cycle's review plan into the controller and applies its +// decision: a scale-up or retire re-expands the lane's replicas and +// reconciles the agent manager and governor the same way a dashboard +// replicas edit does; a drain only changes what Draining answers. +func (r *reviewerAutoscaleRuntime) Run(ctx context.Context, cfg *config.Config, plan review.DispatchPlan, agentMgr crux.Runtime, gov *governor.Governor, logger *slog.Logger, now time.Time) { + if r == nil || cfg == nil { + return + } + r.mu.Lock() + defer r.mu.Unlock() + if !cfg.ReviewerAutoscaleEnabled() { + if r.ctrl != nil { + r.ctrl = nil + r.status = autoscale.Status{} + if cfg.ReviewerAutoscaleActive != 0 { + cfg.ReviewerAutoscaleActive = 0 + r.reconcile(ctx, cfg, agentMgr, gov, logger) + } + logger.Info("reviewer autoscale: disabled; lane back on declared replicas") + } + return + } + agent := cfg.ReviewerAutoscaleAgent() + acfg := cfg.Review.Reviewers.Autoscale + if r.ctrl == nil { + r.ctrl = autoscale.New(acfg, agent) + logger.Info("reviewer autoscale: enabled", "agent", agent, "min", acfg.EffectiveMin(), "max", acfg.EffectiveMax(), "target_backlog_per_reviewer", acfg.EffectiveTargetBacklogPerReviewer(), "cooldown", acfg.EffectiveCooldown().String()) + } else { + r.ctrl.Reconfigure(acfg, agent) + } + names := cfg.ReviewerAutoscaleReplicas() + if cfg.ReviewerAutoscaleActive == 0 { + cfg.ReviewerAutoscaleActive = acfg.Clamp(len(names)) + } + + obs := autoscale.Observation{ + Now: now, + DwellThreshold: stagelabels.Dwell(stagelabels.ReviewPending, cfg.Governor.Labels.Stage.DwellOverrides()), + MaxParallelReviews: cfg.Review.EffectiveMaxParallelReviews(), + Resources: r.prober.Probe(now), + ProviderHeadroomPct: unknownProviderHeadroom, + } + cause, _, _, _ := dashboard.InferenceBudgetExceeded() + obs.ProviderBudgetExceeded = cause != "" + if r.headroom != nil { + obs.ProviderHeadroomPct = lowestProviderHeadroom(r.headroom.HeadroomResponse()) + } + lane := make(map[string]bool, len(names)) + for _, name := range names { + lane[name] = true + } + for _, p := range plan.State.Pending { + if lane[p.Agent] { + obs.InFlight++ + } + } + for _, name := range names { + rv := autoscale.Reviewer{Name: name, Index: cfg.Agents[name].ReplicaIndex} + if proc, err := agentMgr.GetStatusFast(name); err == nil && proc != nil { + rv.Busy = proc.State == crux.StateRunning && !proc.KickOutcome.Settled(proc.LastKick) + rv.Saturated = proc.KicksUndeliverable > 0 + } + for _, p := range plan.State.Pending { + if p.Agent == name { + rv.Busy = true + break + } + } + obs.Reviewers = append(obs.Reviewers, rv) + } + for _, w := range plan.Waiting { + obs.WaitingKeys = append(obs.WaitingKeys, w.Key()) + } + + d := r.ctrl.Observe(obs) + switch d.Action { + case autoscale.ActionHold: + logger.Debug(d.String(), "agent", agent) + case autoscale.ActionBlocked: + logger.Info(d.String(), "agent", agent, "backlog", len(obs.WaitingKeys)) + case autoscale.ActionDrain, autoscale.ActionUndrain: + logger.Info(d.String(), "agent", agent) + case autoscale.ActionScaleUp, autoscale.ActionRetire: + logger.Info(d.String(), "agent", agent) + if err := cfg.SetReviewerAutoscaleActive(d.To); err != nil { + logger.Warn("reviewer autoscale: expanding replicas failed", "agent", agent, "target", d.To, "error", err) + break + } + r.reconcile(ctx, cfg, agentMgr, gov, logger) + } + r.status = r.ctrl.Status() +} + +// reconcile pushes the config's current agent set into the agent manager +// and governor. ReconcileAgents starts nothing itself and kills the tmux +// session of any replica no longer configured, so a retire needs no +// separate stop. +func (r *reviewerAutoscaleRuntime) reconcile(ctx context.Context, cfg *config.Config, agentMgr crux.Runtime, gov *governor.Governor, logger *slog.Logger) { + if agentMgr == nil { + return + } + if cfg.ReviewerAutoscaleActive == 0 { + if err := cfg.ExpandAgentReplicas(); err != nil { + logger.Warn("reviewer autoscale: expanding replicas failed", "error", err) + return + } + } + enabled := cfg.EnabledAgents() + for _, added := range agentMgr.ReconcileAgents(enabled) { + if ac, ok := cfg.Agents[added]; ok && !ac.OnDemand { + if err := agentMgr.Start(ctx, added); err != nil { + logger.Warn("reviewer autoscale: failed to start reviewer", "agent", added, "error", err) + } + } + } + if gov != nil { + gov.UpdateAgents(enabled) + } +} + +// lowestProviderHeadroom is the smallest remaining quota across providers +// with a reading, or unknownProviderHeadroom when none has one. +func lowestProviderHeadroom(resp rotation.HeadroomResponse) float64 { + lowest := float64(unknownProviderHeadroom) + for _, h := range resp.Providers { + if h.CapturedAt.IsZero() && !h.Available { + continue + } + pct := float64(h.PctRemaining) + if lowest < 0 || pct < lowest { + lowest = pct + } + } + return lowest +} diff --git a/src/deploy/entrypoint.sh b/src/deploy/entrypoint.sh index 411ecbd735..ab9c357b83 100644 --- a/src/deploy/entrypoint.sh +++ b/src/deploy/entrypoint.sh @@ -431,23 +431,55 @@ hive_ensure_agent_user() { # -- per head and replica the same way, with alias the head's # `alias` or a slug of -. head_names mirrors # config.ReviewHeadAgentName / reviewHeadSlug / ReviewHead.EffectiveAlias. +# +# Reviewer autoscale (#14685): review.reviewers.autoscale grows the lane's +# base agent up to `max` replicas at runtime, so those names are pre-created +# here too. autoscale_lane mirrors config.ReviewerAutoscaleAgent (the +# configured agent, else the first name-sorted enabled role:reviewer agent, +# else review.reviewer_agents[0]) and the default/ceiling mirror +# DefaultReviewerAutoscaleMax / MaxAgentReplicas. hive_collect_agent_names() { HIVE_COLLECT_CONFIG="$1" python3 - <<'PY' import glob, os, yaml MAX_REPLICAS = 5 # config.MaxAgentReplicas +AUTOSCALE_DEFAULT_MAX = 4 # config.DefaultReviewerAutoscaleMax -def replica_names(name, spec): +def replica_names(name, spec, floor=0): if not isinstance(spec, dict): return [] try: n = int(spec.get('replicas') or 0) except (TypeError, ValueError): return [] + n = max(n, floor) if n < 2 or n > MAX_REPLICAS: return [] return ['%s-%d' % (name, i) for i in range(2, n + 1)] +def autoscale_lane(cfg, agent_specs): + review = cfg.get('review') if isinstance(cfg.get('review'), dict) else {} + reviewers = review.get('reviewers') if isinstance(review.get('reviewers'), dict) else {} + auto = reviewers.get('autoscale') if isinstance(reviewers.get('autoscale'), dict) else {} + if not auto.get('enabled'): + return None, 0 + try: + mx = int(auto.get('max') or 0) or AUTOSCALE_DEFAULT_MAX + except (TypeError, ValueError): + mx = AUTOSCALE_DEFAULT_MAX + mx = min(mx, MAX_REPLICAS) + lane = str(auto.get('agent') or '').strip() + if lane: + return (lane if lane in agent_specs else None), mx + for name in sorted(agent_specs): + spec = agent_specs[name] + if isinstance(spec, dict) and spec.get('enabled', True) is not False and str(spec.get('role') or '').strip().lower() == 'reviewer': + return name, mx + for name in review.get('reviewer_agents') or []: + if name in agent_specs: + return name, mx + return None, 0 + def head_slug(s): out, dash = [], False for ch in str(s or '').strip().lower(): @@ -493,17 +525,16 @@ names = set() with open(os.environ['HIVE_COLLECT_CONFIG']) as f: cfg = yaml.safe_load(f) or {} agents = cfg.get('agents', {}) +agent_specs = {} if isinstance(agents, dict): - for name, spec in agents.items(): - names.add(name) - names.update(replica_names(name, spec)) - names.update(head_names(name, spec)) + agent_specs = dict(agents) elif isinstance(agents, list): - for a in agents: - if isinstance(a, dict) and 'name' in a: - names.add(a['name']) - names.update(replica_names(a['name'], a)) - names.update(head_names(a['name'], a)) + agent_specs = {a['name']: a for a in agents if isinstance(a, dict) and 'name' in a} +lane, lane_max = autoscale_lane(cfg, agent_specs) +for name, spec in agent_specs.items(): + names.add(name) + names.update(replica_names(name, spec, lane_max if name == lane else 0)) + names.update(head_names(name, spec)) # Also check pack YAML if HIVE_LEVEL is set if os.environ.get('HIVE_LEVEL', ''): for p in glob.glob('/opt/hive/packs/level-*.yaml') + glob.glob('/data/packs/level-*.yaml'): diff --git a/src/deploy/test_entrypoint_replica_names.sh b/src/deploy/test_entrypoint_replica_names.sh index 68252f36fe..72951c89ca 100755 --- a/src/deploy/test_entrypoint_replica_names.sh +++ b/src/deploy/test_entrypoint_replica_names.sh @@ -93,6 +93,52 @@ agents: - not-a-head YAML +# #14685: review.reviewers.autoscale grows the lane's base agent up to `max` +# at runtime, so the entrypoint pre-creates those names (mirrors +# config.ReviewerAutoscaleAgent and DefaultReviewerAutoscaleMax). +expect "autoscale expands the role:reviewer lane to max" "reviewer reviewer-2 reviewer-3 scanner" <<'YAML' +agents: + scanner: {backend: codex} + reviewer: {backend: claude, role: reviewer, replicas: 2} +review: + reviewers: + autoscale: {enabled: true, max: 3} +YAML + +expect "autoscale default max is 4, declared replicas above it still win" "reviewer reviewer-2 reviewer-3 reviewer-4 reviewer-5" <<'YAML' +agents: + reviewer: {backend: claude, role: reviewer, replicas: 5} +review: + reviewers: + autoscale: {enabled: true} +YAML + +expect "autoscale honours an explicit lane agent and the replica cap" "critic critic-2 critic-3 critic-4 critic-5 reviewer" <<'YAML' +agents: + reviewer: {backend: claude, role: reviewer} + critic: {backend: codex} +review: + reviewers: + autoscale: {enabled: true, agent: critic, max: 9} +YAML + +expect "autoscale falls back to review.reviewer_agents[0]" "qa qa-2 qa-3 qa-4" <<'YAML' +agents: + qa: {backend: claude} +review: + reviewer_agents: [qa] + reviewers: + autoscale: {enabled: true} +YAML + +expect "autoscale off expands nothing extra" "reviewer reviewer-2" <<'YAML' +agents: + reviewer: {backend: claude, role: reviewer, replicas: 2} +review: + reviewers: + autoscale: {enabled: false, max: 4} +YAML + expect "an empty config prints nothing" "" <<'YAML' YAML diff --git a/src/docs/agent-configuration.md b/src/docs/agent-configuration.md index b39c882ee7..c7844c2c41 100644 --- a/src/docs/agent-configuration.md +++ b/src/docs/agent-configuration.md @@ -370,7 +370,7 @@ Presets map onto modes (`advisory` denies issue and PR creation, `issues-only` d ### Replicated agents -Set `replicas: N` on a declared agent to run a small pool with the same prompt, backend, model, mode, channel, tool, and metadata settings. `N` defaults to `1` and is capped at `5`; config load fails if it is outside `1..5`. Hive materializes derived names as `-2`, `-3`, ... (`scanner`, `scanner-2`, `scanner-3`). Do not declare those derived names yourself: a real `scanner-2` entry collides with `scanner: { replicas: 2 }` and is rejected. Derived replicas get their own IDs and bead directories (`/data/beads/scanner-2`) but inherit the base agent's kick template/prompt selection. Runtime-derived replicas are stripped before saving and recreated on the next load. +Set `replicas: N` on a declared agent to run a small pool with the same prompt, backend, model, mode, channel, tool, and metadata settings. `N` defaults to `1` and is capped at `5`; config load fails if it is outside `1..5`. Hive materializes derived names as `-2`, `-3`, ... (`scanner`, `scanner-2`, `scanner-3`). Do not declare those derived names yourself: a real `scanner-2` entry collides with `scanner: { replicas: 2 }` and is rejected. Derived replicas get their own IDs and bead directories (`/data/beads/scanner-2`) but inherit the base agent's kick template/prompt selection. Runtime-derived replicas are stripped before saving and recreated on the next load. For the reviewer lane, `review.reviewers.autoscale` can raise and lower the materialised count at runtime between `min` and `max`; the declared `replicas` then only seeds the initial size. See [review-swarm.md](review-swarm.md#reviewer-autoscale-reviewreviewersautoscale). ### Trigger channels diff --git a/src/docs/operator-reference.md b/src/docs/operator-reference.md index 269e3fab9c..7970cbcb2a 100644 --- a/src/docs/operator-reference.md +++ b/src/docs/operator-reference.md @@ -242,6 +242,7 @@ Go callers keep reading `cfg.Agents`, `cfg.Governor.Gateways` and so on through | `review.severity.comment_below` / `backlog_below` | `true` / `true`; owner-only. | Whether findings below the blocking line are posted as non-blocking comments and filed to `review.backlog`. | | `review.backlog.destination` | `github_issue`; owner-only. | `github_issue`, `github_project`, `linear` or `jira`. The last three need the matching `governor.work_source.type` (`github_projects`, `linear`, `jira`); on a mismatch the hive falls back to `github_issue`. Destination fields: `project_column_id` (required for `github_project`), `linear_state`, `jira_status`. | | `review.backlog.labels` / `max_per_pr_per_day` | `[agent/reviewer]` / `10`; owner-only. `from-review`, `origin/review`, and `origin/reviewer` are accepted and mapped to `agent/reviewer`. | Labels on every backlog item and the per-PR-per-day cap (negative fails load; 0 = default). Reviewer is an agent lane, so `agent/reviewer` is the routing/ownership/provenance label. Non-lane hive issue filers continue to use `origin/selfreport`, `origin/governor`, `origin/dashboard`, `origin/forge`, or `origin/convergence`; the one-time startup migration (`/data/origin-label-migration-v4.done`) removes redundant lane-origin labels and backfills only non-lane origins. | +| `review.reviewers.autoscale.*` | `enabled: false`; when on: `min: 1`, `max: 4` (≤ the replicas cap 5), `target_backlog_per_reviewer: 2`, `scale_up_cycles: 2`, `scale_down_cycles: 5`, `cooldown: 5m`, `cpu_per_reviewer: 1.0`, `mem_per_reviewer_mb: 2048`, `agent` unset (first `role: reviewer` agent, else `reviewer_agents[0]`); owner-only. | Sizes the reviewer lane to its backlog ([#14685](https://github.com/hivecommons/hive/issues/14685)): adds a `reviewer-N` replica after `scale_up_cycles` cycles with more PRs waiting for a free reviewer than `target_backlog_per_reviewer` per reviewer (or a wait older than the review-pending stage dwell), drains the top replica after `scale_down_cycles` idle cycles and retires it only when its turn ends. Scale-ups are refused by named guards (`at-max`, `cooldown`, `saturated` kicks, `provider-budget`, `provider-headroom`, `max-parallel-reviews`, cgroup `cpu` / `memory`). Decisions log as `reviewer autoscale: 2→3 reason=…`; state is in `/api/status.reviewerAutoscale`. See [review-swarm.md](review-swarm.md#reviewer-autoscale-reviewreviewersautoscale). | | `review.contributor_prs.base_sync` | Off by default; owner-only. | Lets PR follow-up use GitHub's update-branch API on hive-authored lane PRs whose head is in a contributor fork, whose `mergeable_state` is `behind`/`dirty`, and whose author enabled "allow edits by maintainers". `behind` PRs get a GitHub-authored merge commit; `dirty`/422 falls back to the `hive-base-moved` repair note and kick. This requires an agent/proxy mode with `ISSUES_PRS_MERGE` because `PUT /pulls/{n}/update-branch` is merge-scoped. It does not rewrite history and does not add a human/bot-authored DCO commit; the repository's "Sign-off survives the squash" check examines the PR's own commits, not GitHub's update-branch merge commit. | | `governor.advisory.post_retry_backoff_s` | `3600` seconds. `HIVE_ADVISORY_POST_RETRY_BACKOFF_S` overrides the default when the key is unset; a negative value disables the pause. | How long the advisory digest write is paused after GitHub refuses it with `403 Resource not accessible by integration`. That refusal is structural (foreign-authored comment, repo dropped from the installation, or repo transferred to another owner), so per-cycle retries only burn API quota. The failed edit first falls back to creating a fresh App-authored comment; if the create is refused too, the error is recorded for the hub health row — naming the new `owner/repo` when the repository has moved — and attempts pause for this window. See [advisory.md](advisory.md#when-the-write-is-refused). | | `governor.trajectory.enabled` | Defaults to enabled. | The lane no-ops until a reviewer endpoint and model resolve from `governor.trajectory` or `governor.litellm`. | diff --git a/src/docs/review-swarm.md b/src/docs/review-swarm.md index 9264bc1354..186961328f 100644 --- a/src/docs/review-swarm.md +++ b/src/docs/review-swarm.md @@ -381,6 +381,36 @@ Phase 2 adds dispatch state in `/data/review-dispatch-state.json`. Pending revie **One kick in flight per agent.** An agent's CLI works one prompt at a time, so the planner never sends a review kick (per-perspective, head, combined or decision) to an agent that still has one pending, i.e. dispatched and not yet answered or expired. It picks the first reviewer in name order that is free; when every eligible reviewer (or every replica of a head) is busy, that work waits for a later cycle. Waiting records nothing, so it costs no `max_redispatch` or `head_max_retries`, and `pending_ttl` starts when the kick actually goes to an idle agent. Load is counted in kicks, not PR heads: counting PR heads let replicas holding the same PR tie, and one PR's remaining perspectives all went to the first replica, four kicks typed into one Claude session within three minutes, three of whose verdicts never arrived. +### Reviewer autoscale (`review.reviewers.autoscale`) + +By default the reviewer lane is a fixed pool: `agents.reviewer.replicas: 2` runs `reviewer` and `reviewer-2`, and when both have a kick in flight every other PR waits. `review.reviewers.autoscale` ([#14685](https://github.com/hivecommons/hive/issues/14685)) instead lets the lane grow and shrink with that wait. It is off by default. + +```yaml +review: + reviewers: + autoscale: + enabled: true + agent: reviewer # optional; default: first role:reviewer agent, else reviewer_agents[0] + min: 1 + max: 4 # capped at the replicas limit (5) + target_backlog_per_reviewer: 2 + scale_up_cycles: 2 + scale_down_cycles: 5 + cooldown: 5m + cpu_per_reviewer: 1.0 + mem_per_reviewer_mb: 2048 +``` + +**Demand.** Each governor cycle the planner records the PRs it left *waiting*: PRs with perspectives still to review on their current head for which every eligible reviewer was busy (`no-free-reviewer`) or the `max_parallel_reviews` budget was spent (`no-parallel-slot`). Drafts, held PRs and PRs whose review is already dispatched are not waiting. The controller ages each wait across cycles, keyed on repo, number and head, so a push restarts the clock. + +**Scale-up.** The lane is *behind* when the backlog exceeds `target_backlog_per_reviewer` × active reviewers, or when the oldest wait is older than the `stage/050-review-pending` dwell (`governor.labels.stage.dwell` overrides apply). Behind for `scale_up_cycles` consecutive cycles adds one replica. Guards refuse a wanted scale-up, in this order, and the refusal is logged with the guard's name: `at-max`; `cooldown` (less than `cooldown` since the last action); `saturated` (half or more of the active reviewers' last kicks were undeliverable — `CLI did not reach input prompt` — which is a saturated pod, not demand); `provider-budget` (the inference budget latch is on); `provider-headroom` (any provider under 10% remaining, when rotation publishes readings); `max-parallel-reviews` (more reviewers than slots is pointless); `cpu` / `memory` (the pod cgroup — `cpu.max`, `cpu.stat`, `memory.max`, `memory.current` — has less free than one reviewer's reservation; a host without cgroup limits never blocks). A refused scale-up keeps its streak, so it fires as soon as the guard clears. + +**Scale-down.** The lane is *idle* when nothing is waiting and at least one reviewer has no kick in flight. Idle for `scale_down_cycles` consecutive cycles, with more than `min` replicas active and the cooldown elapsed, marks the highest-numbered replica *draining*: the planner and the cadence kick both skip it (the eval log shows `reviewer-3 (reviewer-autoscale draining)`), and it is retired — its tmux session ended and its config entry dropped — only once its turn has ended and it has no review pending. A reviewer is never interrupted mid-turn. If demand returns while a replica is draining it is simply un-drained rather than a fresh replica being added. + +**What a dynamic replica is.** Nothing new: the controller raises or lowers the lane's materialised replica count and the ordinary `ExpandAgentReplicas` path creates `reviewer-3`, `reviewer-4` … with the same role, prompt, beads directory, scoped token, tmux pane and cadence key as a configured `reviewer-2`. The planner, kick governor, dashboard agent list and token cache cannot tell them apart. The entrypoint pre-creates the per-agent user and token cache for every name up to `max`, so a replica added at 03:00 can read its token. The declared `replicas:` only seeds the initial count (clamped into `[min, max]`) and is what is persisted; the live count survives a config reload and is never written to `hive.yaml`. + +**Observability.** One log line per decision — `reviewer autoscale: 2→3 reason=backlog=7 oldest=41m0s`, `reviewer autoscale: 3→2 drain reviewer-3 reason=idle for 5 cycles`, `reviewer autoscale: 3→2 retire reviewer-3 reason=idle`, `reviewer autoscale: 2→3 blocked by cpu`. `/api/status.reviewerAutoscale` carries `enabled`, `agent`, `min`, `max`, `current`, `target`, `draining`, `backlog`, `oldestWait`, the streaks and `lastDecision`; the Agents section shows the lane's reviewer count with an *auto* badge (replicas show *auto* / *draining*) whose tooltip carries the last decision. The knobs live in Settings → Review Gate → Reviewers → *Scale reviewers with the backlog* (owner only; `PUT /api/config/review` sub-object `reviewer_autoscale`). + ### Pending-review expiry (`review.pending_ttl`, `review.max_redispatch`) A pending entry is the planner's only record that a perspective was asked, so a reviewer whose kick turn dies — model timeout, proxy read timeout, pod restart — leaves entries that post no verdict and, with no expiry, made every perspective look covered until someone pushed a new commit (hivecommons/hive#12045 and #12025 sat at queue position 1 for hours with CI green and the contributor lane stuck at `no-verdict`). Each cycle the planner therefore expires pending entries older than `pending_ttl` (Go duration string, default `45m`; a negative value disables expiry) whose head still has no verdict in the artifact: the expired perspectives are dropped from state and re-dispatched in the same cycle, the governor logs `review pending expired repo=… pr=… head=… perspective=… age=… expiries=N`, and the state file's `redispatches[]` counts expiries per head. Entries younger than the TTL are left alone, so a review genuinely in flight is never re-kicked, and a verdict for the head settles it however old the entries are. The same expiry releases a `revise_verdicts_before` revisit whose reviewer died. diff --git a/src/hive.yaml.example b/src/hive.yaml.example index 5a675cf660..9e5d7d541c 100644 --- a/src/hive.yaml.example +++ b/src/hive.yaml.example @@ -835,6 +835,18 @@ governor: # fan_out: false # max_parallel_reviews: 5 # reviewer_agents: [] # optional explicit list; otherwise agents with review role/keywords are used +# reviewers: +# autoscale: # size the reviewer lane to its backlog (#14685); off by default +# enabled: false +# agent: "" # base reviewer to scale; default: first role:reviewer agent, else reviewer_agents[0] +# min: 1 # floor of active replicas (the base agent is never retired) +# max: 4 # ceiling; capped at the agents..replicas limit (5) +# target_backlog_per_reviewer: 2 # PRs waiting per reviewer before the lane is "behind" +# scale_up_cycles: 2 # consecutive behind cycles before adding a reviewer +# scale_down_cycles: 5 # consecutive idle cycles before draining the top replica +# cooldown: 5m # minimum gap between scaling actions +# cpu_per_reviewer: 1.0 # cgroup CPU (cores) reserved per added reviewer +# mem_per_reviewer_mb: 2048 # cgroup memory (MiB) reserved per added reviewer # combined_perspectives: true # RECOMMENDED (#12126): one session + one comment covering every # # perspective; shared review boilerplate is sent once instead of once # # per perspective. Verdicts stay per-perspective. Default false. diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index a6af0979d7..b6fc8d4fdb 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -218,6 +218,12 @@ type Config struct { RemovedAgents []string `yaml:"removed_agents,omitempty" json:"removed_agents,omitempty"` SourcePath string `yaml:"-" json:"-"` + // ReviewerAutoscaleActive is the reviewer autoscaler's current target + // replica count for the autoscaled lane (hivecommons/hive#14685). + // Runtime-only: it is never persisted, is carried across config reloads + // by the caller, and zero means "use the declared replicas". Written + // via SetReviewerAutoscaleActive. + ReviewerAutoscaleActive int `yaml:"-" json:"-"` } // UnmarshalYAML accepts the operator-facing agents.github_api_hourly_cap @@ -6495,12 +6501,23 @@ func (c *Config) ExpandAgentReplicas() error { delete(c.Agents, name) } } + autoscaled := "" + if c.Review.Reviewers.Autoscale.Enabled { + autoscaled = c.ReviewerAutoscaleAgent() + } + effectiveReplicas := func(name string, agent AgentConfig) int { + if autoscaled != "" && name == autoscaled { + return c.reviewerAutoscaleReplicaCount(agent.Replicas) + } + return normalizeReplicaCount(agent.Replicas) + } derivedNames := map[string]string{} for name, agent := range baseAgents { replicas := normalizeReplicaCount(agent.Replicas) if replicas < 1 || replicas > MaxAgentReplicas { return fmt.Errorf("agent %s: replicas must be between 1 and %d", name, MaxAgentReplicas) } + replicas = effectiveReplicas(name, agent) for i := 2; i <= replicas; i++ { derived := replicaAgentName(name, i) if existing, ok := baseAgents[derived]; ok && existing.ReplicaOf == "" { @@ -6522,8 +6539,8 @@ func (c *Config) ExpandAgentReplicas() error { } } for name, agent := range baseAgents { - replicas := normalizeReplicaCount(agent.Replicas) - agent.Replicas = replicas + replicas := effectiveReplicas(name, agent) + agent.Replicas = normalizeReplicaCount(agent.Replicas) agent.ReplicaOf = "" agent.ReplicaIndex = 1 agent.ReplicaCount = replicas @@ -7502,6 +7519,9 @@ type ReviewConfig struct { // Backlog is where findings below the blocking line are filed // (hivecommons/hive#11088). See ReviewBacklogConfig. Backlog ReviewBacklogConfig `yaml:"backlog,omitempty" json:"backlog,omitempty"` + // Reviewers configures the reviewer agents themselves; see + // ReviewersConfig and review.reviewers.autoscale (hivecommons/hive#14685). + Reviewers ReviewersConfig `yaml:"reviewers,omitempty" json:"reviewers,omitzero"` } // PlanMatchConfig is the switch for the plan_match review perspective diff --git a/src/pkg/config/reviewer_autoscale.go b/src/pkg/config/reviewer_autoscale.go new file mode 100644 index 0000000000..1689e4a663 --- /dev/null +++ b/src/pkg/config/reviewer_autoscale.go @@ -0,0 +1,277 @@ +package config + +import ( + "fmt" + "sort" + "strings" + "time" +) + +// Reviewer autoscale defaults (hivecommons/hive#14685). Every threshold the +// controller consults is either one of these or a config key below, so there +// is one place to read when asking "why did the lane grow?". +const ( + // DefaultReviewerAutoscaleMin is the floor of active reviewer replicas. + DefaultReviewerAutoscaleMin = 1 + // DefaultReviewerAutoscaleMax is the ceiling of active reviewer replicas. + // It is below MaxAgentReplicas so a hive that opts in gets a safe band + // without also raising the fleet-wide replica cap. + DefaultReviewerAutoscaleMax = 4 + // DefaultReviewerAutoscaleTargetBacklog is how many PRs may wait per + // active reviewer before the lane is considered behind. + DefaultReviewerAutoscaleTargetBacklog = 2 + // DefaultReviewerAutoscaleScaleUpCycles is how many consecutive cycles + // the lane must be behind before a reviewer is added. Small on purpose: + // a backlog costs merge latency every cycle it persists. + DefaultReviewerAutoscaleScaleUpCycles = 2 + // DefaultReviewerAutoscaleScaleDownCycles is how many consecutive idle + // cycles precede retiring a reviewer. Larger than scale-up so a lull + // between pushes does not thrash the lane. + DefaultReviewerAutoscaleScaleDownCycles = 5 + // DefaultReviewerAutoscaleCooldown is the minimum gap between two + // scaling actions in either direction. + DefaultReviewerAutoscaleCooldown = 5 * time.Minute + // DefaultReviewerAutoscaleCPUPerReviewer is the CPU (cores) one reviewer + // is assumed to need; scale-up is refused when the cgroup cannot carry it. + DefaultReviewerAutoscaleCPUPerReviewer = 1.0 + // DefaultReviewerAutoscaleMemPerReviewerMB is the memory one reviewer is + // assumed to need, in MiB. + DefaultReviewerAutoscaleMemPerReviewerMB = 2048 +) + +// ReviewersConfig groups settings about the reviewer agents themselves (as +// opposed to what they review). Lives under review.reviewers. +type ReviewersConfig struct { + // Autoscale lets the reviewer lane grow and shrink with the review + // backlog instead of running a fixed replica count. + Autoscale ReviewerAutoscaleConfig `yaml:"autoscale,omitempty" json:"autoscale,omitzero"` +} + +// ReviewerAutoscaleConfig is review.reviewers.autoscale +// (hivecommons/hive#14685). Opt-in: the zero value keeps the static +// agents..replicas behaviour. +// +// When enabled, the lane's base agent (Agent, or the first reviewer-role +// agent) is expanded to the controller's current target instead of its +// declared replicas; the declared value only seeds the initial count, +// clamped into [Min, Max]. Dynamic replicas are ordinary replicas — +// reviewer-3, reviewer-4 — with the same role, prompt, beads dir, scoped +// token and cadence key family as reviewer-2, so nothing downstream can +// tell them apart. +type ReviewerAutoscaleConfig struct { + Enabled bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` + // Agent names the base reviewer agent that scales. Empty picks the + // first (name-sorted) enabled agent with role "reviewer", falling back + // to review.reviewer_agents[0]. + Agent string `yaml:"agent,omitempty" json:"agent,omitempty"` + Min int `yaml:"min,omitempty" json:"min,omitempty"` + Max int `yaml:"max,omitempty" json:"max,omitempty"` + // TargetBacklogPerReviewer is the waiting-PRs-per-reviewer ratio above + // which the lane is behind. + TargetBacklogPerReviewer int `yaml:"target_backlog_per_reviewer,omitempty" json:"target_backlog_per_reviewer,omitempty"` + // ScaleUpCycles / ScaleDownCycles are the sustained-signal lengths. + ScaleUpCycles int `yaml:"scale_up_cycles,omitempty" json:"scale_up_cycles,omitempty"` + ScaleDownCycles int `yaml:"scale_down_cycles,omitempty" json:"scale_down_cycles,omitempty"` + // Cooldown is a Go duration string; the minimum gap between actions. + Cooldown string `yaml:"cooldown,omitempty" json:"cooldown,omitempty"` + // CPUPerReviewer / MemPerReviewerMB are the per-reviewer resource cost + // the cgroup guard reserves before adding one. + CPUPerReviewer float64 `yaml:"cpu_per_reviewer,omitempty" json:"cpu_per_reviewer,omitempty"` + MemPerReviewerMB int `yaml:"mem_per_reviewer_mb,omitempty" json:"mem_per_reviewer_mb,omitempty"` +} + +// EffectiveMin returns Min or its default. +func (a ReviewerAutoscaleConfig) EffectiveMin() int { + if a.Min > 0 { + return a.Min + } + return DefaultReviewerAutoscaleMin +} + +// EffectiveMax returns Max or its default, never above MaxAgentReplicas. +func (a ReviewerAutoscaleConfig) EffectiveMax() int { + max := a.Max + if max <= 0 { + max = DefaultReviewerAutoscaleMax + } + if max > MaxAgentReplicas { + max = MaxAgentReplicas + } + if max < a.EffectiveMin() { + max = a.EffectiveMin() + } + return max +} + +// EffectiveTargetBacklogPerReviewer returns the ratio or its default. +func (a ReviewerAutoscaleConfig) EffectiveTargetBacklogPerReviewer() int { + if a.TargetBacklogPerReviewer > 0 { + return a.TargetBacklogPerReviewer + } + return DefaultReviewerAutoscaleTargetBacklog +} + +// EffectiveScaleUpCycles returns the sustained scale-up length or its default. +func (a ReviewerAutoscaleConfig) EffectiveScaleUpCycles() int { + if a.ScaleUpCycles > 0 { + return a.ScaleUpCycles + } + return DefaultReviewerAutoscaleScaleUpCycles +} + +// EffectiveScaleDownCycles returns the sustained scale-down length or its default. +func (a ReviewerAutoscaleConfig) EffectiveScaleDownCycles() int { + if a.ScaleDownCycles > 0 { + return a.ScaleDownCycles + } + return DefaultReviewerAutoscaleScaleDownCycles +} + +// EffectiveCooldown parses Cooldown, falling back to the default when unset +// or malformed (Validate rejects malformed values at load). +func (a ReviewerAutoscaleConfig) EffectiveCooldown() time.Duration { + raw := strings.TrimSpace(a.Cooldown) + if raw == "" { + return DefaultReviewerAutoscaleCooldown + } + d, err := time.ParseDuration(raw) + if err != nil || d <= 0 { + return DefaultReviewerAutoscaleCooldown + } + return d +} + +// EffectiveCPUPerReviewer returns the per-reviewer CPU reservation. +func (a ReviewerAutoscaleConfig) EffectiveCPUPerReviewer() float64 { + if a.CPUPerReviewer > 0 { + return a.CPUPerReviewer + } + return DefaultReviewerAutoscaleCPUPerReviewer +} + +// EffectiveMemPerReviewerMB returns the per-reviewer memory reservation. +func (a ReviewerAutoscaleConfig) EffectiveMemPerReviewerMB() int { + if a.MemPerReviewerMB > 0 { + return a.MemPerReviewerMB + } + return DefaultReviewerAutoscaleMemPerReviewerMB +} + +// Clamp bounds n into [EffectiveMin, EffectiveMax]. +func (a ReviewerAutoscaleConfig) Clamp(n int) int { + if n < a.EffectiveMin() { + return a.EffectiveMin() + } + if n > a.EffectiveMax() { + return a.EffectiveMax() + } + return n +} + +// Validate rejects a band that cannot be honoured. +func (a ReviewerAutoscaleConfig) Validate() error { + if !a.Enabled { + return nil + } + if a.Min < 0 || a.Max < 0 { + return fmt.Errorf("review.reviewers.autoscale: min and max must not be negative") + } + if a.Max > MaxAgentReplicas { + return fmt.Errorf("review.reviewers.autoscale: max %d exceeds the replica cap %d", a.Max, MaxAgentReplicas) + } + if a.Min > 0 && a.Max > 0 && a.Min > a.Max { + return fmt.Errorf("review.reviewers.autoscale: min %d is greater than max %d", a.Min, a.Max) + } + if a.TargetBacklogPerReviewer < 0 || a.ScaleUpCycles < 0 || a.ScaleDownCycles < 0 || a.MemPerReviewerMB < 0 || a.CPUPerReviewer < 0 { + return fmt.Errorf("review.reviewers.autoscale: thresholds must not be negative") + } + if raw := strings.TrimSpace(a.Cooldown); raw != "" { + d, err := time.ParseDuration(raw) + if err != nil || d <= 0 { + return fmt.Errorf("review.reviewers.autoscale: cooldown %q is not a positive duration (e.g. 5m)", a.Cooldown) + } + } + return nil +} + +// ReviewerAutoscaleAgent resolves the base agent the autoscaler manages: +// the configured name when it is a declared base agent, else the first +// name-sorted enabled base agent with role "reviewer", else +// review.reviewer_agents[0]. Empty when nothing qualifies. +func (c *Config) ReviewerAutoscaleAgent() string { + if c == nil { + return "" + } + a := c.Review.Reviewers.Autoscale + if name := strings.TrimSpace(a.Agent); name != "" { + if agent, ok := c.Agents[name]; ok && agent.ReplicaOf == "" { + return name + } + return "" + } + names := make([]string, 0, len(c.Agents)) + for name, agent := range c.Agents { + if agent.ReplicaOf != "" || !agent.Enabled { + continue + } + if strings.EqualFold(strings.TrimSpace(agent.Role), "reviewer") { + names = append(names, name) + } + } + if len(names) > 0 { + sort.Strings(names) + return names[0] + } + for _, name := range c.Review.ReviewerAgents { + if agent, ok := c.Agents[name]; ok && agent.ReplicaOf == "" { + return name + } + } + return "" +} + +// ReviewerAutoscaleEnabled reports whether the lane is under autoscale +// control and resolves to a real base agent. +func (c *Config) ReviewerAutoscaleEnabled() bool { + return c != nil && c.Review.Reviewers.Autoscale.Enabled && c.ReviewerAutoscaleAgent() != "" +} + +// reviewerAutoscaleReplicaCount is the replica count ExpandAgentReplicas +// materialises for the autoscaled lane: the controller's current target when +// it has set one, else the declared replicas clamped into the band. +func (c *Config) reviewerAutoscaleReplicaCount(declared int) int { + a := c.Review.Reviewers.Autoscale + if c.ReviewerAutoscaleActive > 0 { + return a.Clamp(c.ReviewerAutoscaleActive) + } + return a.Clamp(normalizeReplicaCount(declared)) +} + +// SetReviewerAutoscaleActive records the controller's target replica count +// and re-expands agents so the lane materialises it. It is the single write +// path for ReviewerAutoscaleActive. +func (c *Config) SetReviewerAutoscaleActive(n int) error { + if c == nil { + return nil + } + c.ReviewerAutoscaleActive = c.Review.Reviewers.Autoscale.Clamp(n) + return c.ExpandAgentReplicas() +} + +// ReviewerAutoscaleReplicas lists the lane's current agent names in replica +// order (base first), or nil when autoscale is off. +func (c *Config) ReviewerAutoscaleReplicas() []string { + if !c.ReviewerAutoscaleEnabled() { + return nil + } + base := c.ReviewerAutoscaleAgent() + out := []string{base} + count := c.Agents[base].ReplicaCount + for i := 2; i <= count; i++ { + name := replicaAgentName(base, i) + if _, ok := c.Agents[name]; ok { + out = append(out, name) + } + } + return out +} diff --git a/src/pkg/config/reviewer_autoscale_test.go b/src/pkg/config/reviewer_autoscale_test.go new file mode 100644 index 0000000000..c59bcc8690 --- /dev/null +++ b/src/pkg/config/reviewer_autoscale_test.go @@ -0,0 +1,145 @@ +package config + +import ( + "sort" + "strings" + "testing" + "time" +) + +func autoscaleTestConfig(enabled bool, replicas int) *Config { + cfg := &Config{AgentRuntimeConfig: AgentRuntimeConfig{ + Agents: map[string]AgentConfig{ + "scanner": {ID: "scanner", Enabled: true}, + "reviewer": {ID: "reviewer", Enabled: true, Role: "reviewer", Replicas: replicas}, + }, + }} + cfg.Review.Reviewers.Autoscale = ReviewerAutoscaleConfig{Enabled: enabled, Min: 1, Max: 4} + return cfg +} + +func agentNames(cfg *Config) []string { + out := make([]string, 0, len(cfg.Agents)) + for n := range cfg.Agents { + out = append(out, n) + } + sort.Strings(out) + return out +} + +func TestReviewerAutoscaleDefaults(t *testing.T) { + var a ReviewerAutoscaleConfig + if a.EffectiveMin() != DefaultReviewerAutoscaleMin || a.EffectiveMax() != DefaultReviewerAutoscaleMax { + t.Fatalf("band = %d..%d", a.EffectiveMin(), a.EffectiveMax()) + } + if a.EffectiveCooldown() != DefaultReviewerAutoscaleCooldown || a.EffectiveScaleUpCycles() != DefaultReviewerAutoscaleScaleUpCycles || a.EffectiveScaleDownCycles() != DefaultReviewerAutoscaleScaleDownCycles { + t.Fatalf("cycle defaults wrong: %+v", a) + } + a.Max = MaxAgentReplicas + 3 + if a.EffectiveMax() != MaxAgentReplicas { + t.Fatalf("max not capped: %d", a.EffectiveMax()) + } + a.Cooldown = "90s" + if a.EffectiveCooldown() != 90*time.Second { + t.Fatalf("cooldown = %s", a.EffectiveCooldown()) + } +} + +func TestReviewerAutoscaleValidate(t *testing.T) { + cases := []struct { + name string + cfg ReviewerAutoscaleConfig + want string + }{ + {"disabled anything goes", ReviewerAutoscaleConfig{Min: 9, Max: 1}, ""}, + {"min above max", ReviewerAutoscaleConfig{Enabled: true, Min: 3, Max: 2}, "min 3 is greater than max 2"}, + {"max above cap", ReviewerAutoscaleConfig{Enabled: true, Max: MaxAgentReplicas + 1}, "exceeds the replica cap"}, + {"bad cooldown", ReviewerAutoscaleConfig{Enabled: true, Cooldown: "soon"}, "cooldown"}, + {"ok", ReviewerAutoscaleConfig{Enabled: true, Min: 1, Max: 4, Cooldown: "5m"}, ""}, + } + for _, tc := range cases { + err := tc.cfg.Validate() + if tc.want == "" && err != nil { + t.Fatalf("%s: unexpected %v", tc.name, err) + } + if tc.want != "" && (err == nil || !strings.Contains(err.Error(), tc.want)) { + t.Fatalf("%s: err = %v, want %q", tc.name, err, tc.want) + } + } +} + +func TestReviewerAutoscaleAgentResolution(t *testing.T) { + cfg := autoscaleTestConfig(true, 1) + if got := cfg.ReviewerAutoscaleAgent(); got != "reviewer" { + t.Fatalf("role lookup = %q", got) + } + cfg.Review.Reviewers.Autoscale.Agent = "scanner" + if got := cfg.ReviewerAutoscaleAgent(); got != "scanner" { + t.Fatalf("explicit = %q", got) + } + cfg.Review.Reviewers.Autoscale.Agent = "ghost" + if got := cfg.ReviewerAutoscaleAgent(); got != "" { + t.Fatalf("unknown explicit = %q", got) + } + cfg.Review.Reviewers.Autoscale.Agent = "" + cfg.Agents["reviewer"] = AgentConfig{ID: "reviewer", Enabled: true} + cfg.Review.ReviewerAgents = []string{"scanner"} + if got := cfg.ReviewerAutoscaleAgent(); got != "scanner" { + t.Fatalf("reviewer_agents fallback = %q", got) + } +} + +func TestExpandAgentReplicasUsesAutoscaleActiveCount(t *testing.T) { + cfg := autoscaleTestConfig(true, 2) + if err := cfg.ExpandAgentReplicas(); err != nil { + t.Fatal(err) + } + if got := agentNames(cfg); strings.Join(got, " ") != "reviewer reviewer-2 scanner" { + t.Fatalf("initial expansion = %v", got) + } + if err := cfg.SetReviewerAutoscaleActive(4); err != nil { + t.Fatal(err) + } + if got := agentNames(cfg); strings.Join(got, " ") != "reviewer reviewer-2 reviewer-3 reviewer-4 scanner" { + t.Fatalf("scaled up = %v", got) + } + r4 := cfg.Agents["reviewer-4"] + if r4.ReplicaOf != "reviewer" || r4.ReplicaIndex != 4 || r4.ReplicaCount != 4 || r4.Role != "reviewer" || r4.BeadsDir != "/data/beads/reviewer-4" { + t.Fatalf("reviewer-4 = %+v", r4) + } + // The declared value is what persists; only the runtime count moved. + if cfg.Agents["reviewer"].Replicas != 2 { + t.Fatalf("declared replicas changed: %d", cfg.Agents["reviewer"].Replicas) + } + if got := cfg.ReviewerAutoscaleReplicas(); strings.Join(got, " ") != "reviewer reviewer-2 reviewer-3 reviewer-4" { + t.Fatalf("lane = %v", got) + } + if err := cfg.SetReviewerAutoscaleActive(1); err != nil { + t.Fatal(err) + } + if got := agentNames(cfg); strings.Join(got, " ") != "reviewer scanner" { + t.Fatalf("scaled down = %v", got) + } + // Out-of-band targets clamp into the band. + if err := cfg.SetReviewerAutoscaleActive(9); err != nil || cfg.ReviewerAutoscaleActive != 4 { + t.Fatalf("clamp: active=%d err=%v", cfg.ReviewerAutoscaleActive, err) + } +} + +func TestExpandAgentReplicasAutoscaleClampsDeclared(t *testing.T) { + cfg := autoscaleTestConfig(true, 5) + if err := cfg.ExpandAgentReplicas(); err != nil { + t.Fatal(err) + } + if got := agentNames(cfg); strings.Join(got, " ") != "reviewer reviewer-2 reviewer-3 reviewer-4 scanner" { + t.Fatalf("declared above max = %v", got) + } + cfg = autoscaleTestConfig(false, 5) + cfg.ReviewerAutoscaleActive = 2 + if err := cfg.ExpandAgentReplicas(); err != nil { + t.Fatal(err) + } + if got := agentNames(cfg); strings.Join(got, " ") != "reviewer reviewer-2 reviewer-3 reviewer-4 reviewer-5 scanner" { + t.Fatalf("autoscale off ignores active: %v", got) + } +} diff --git a/src/pkg/config/validate.go b/src/pkg/config/validate.go index 8990fdb178..762f71dffe 100644 --- a/src/pkg/config/validate.go +++ b/src/pkg/config/validate.go @@ -146,6 +146,9 @@ func (c *Config) ValidateWithOptions(opts ValidateOptions) error { if err := c.Review.ValidatePendingReviewTTL(); err != nil { return err } + if err := c.Review.Reviewers.Autoscale.Validate(); err != nil { + return err + } if err := c.Review.ValidateDeltaMaxAge(); err != nil { return err } diff --git a/src/pkg/dashboard/api_config_review.go b/src/pkg/dashboard/api_config_review.go index da4f56382d..6964ee228d 100644 --- a/src/pkg/dashboard/api_config_review.go +++ b/src/pkg/dashboard/api_config_review.go @@ -177,6 +177,21 @@ func (s *Server) handleReviewConfigPut(w http.ResponseWriter, r *http.Request) { Labels *[]string `json:"labels"` MaxPerPRPerDay *int `json:"max_per_pr_per_day"` } `json:"backlog"` + // ReviewerAutoscale is review.reviewers.autoscale + // (hivecommons/hive#14685). Pointer fields keep "absent means + // unchanged"; the band is validated before anything is mutated. + ReviewerAutoscale *struct { + Enabled *bool `json:"enabled"` + Agent *string `json:"agent"` + Min *int `json:"min"` + Max *int `json:"max"` + TargetBacklogPerReviewer *int `json:"target_backlog_per_reviewer"` + ScaleUpCycles *int `json:"scale_up_cycles"` + ScaleDownCycles *int `json:"scale_down_cycles"` + Cooldown *string `json:"cooldown"` + CPUPerReviewer *float64 `json:"cpu_per_reviewer"` + MemPerReviewerMB *int `json:"mem_per_reviewer_mb"` + } `json:"reviewer_autoscale"` } if err := decodeBody(r, &body); err != nil { jsonError(w, "invalid body", http.StatusBadRequest) @@ -248,6 +263,43 @@ func (s *Server) handleReviewConfigPut(w http.ResponseWriter, r *http.Request) { // Validate the loader runs, so the error names the field and a bad value // rejects the whole request before anything is mutated. severity, backlog := cfg.Review.Severity, cfg.Review.Backlog + autoscaleCfg := cfg.Review.Reviewers.Autoscale + if a := body.ReviewerAutoscale; a != nil { + if a.Enabled != nil { + autoscaleCfg.Enabled = *a.Enabled + } + if a.Agent != nil { + autoscaleCfg.Agent = sanitizeString(*a.Agent) + } + if a.Min != nil { + autoscaleCfg.Min = *a.Min + } + if a.Max != nil { + autoscaleCfg.Max = *a.Max + } + if a.TargetBacklogPerReviewer != nil { + autoscaleCfg.TargetBacklogPerReviewer = *a.TargetBacklogPerReviewer + } + if a.ScaleUpCycles != nil { + autoscaleCfg.ScaleUpCycles = *a.ScaleUpCycles + } + if a.ScaleDownCycles != nil { + autoscaleCfg.ScaleDownCycles = *a.ScaleDownCycles + } + if a.Cooldown != nil { + autoscaleCfg.Cooldown = sanitizeString(*a.Cooldown) + } + if a.CPUPerReviewer != nil { + autoscaleCfg.CPUPerReviewer = *a.CPUPerReviewer + } + if a.MemPerReviewerMB != nil { + autoscaleCfg.MemPerReviewerMB = *a.MemPerReviewerMB + } + if err := autoscaleCfg.Validate(); err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + } if body.Severity != nil { if body.Severity.BlockAt != nil { b, ok := config.NormalizeReviewSeverityBlockAt(*body.Severity.BlockAt) @@ -518,6 +570,7 @@ func (s *Server) handleReviewConfigPut(w http.ResponseWriter, r *http.Request) { cfg.Review.Severity = severity cfg.Review.Backlog = backlog + cfg.Review.Reviewers.Autoscale = autoscaleCfg // Written into hive.yaml's review_bots block only; EffectiveReviewBots // lets it override the project file's threshold without copying the diff --git a/src/pkg/dashboard/deps.go b/src/pkg/dashboard/deps.go index e47a47f2b8..c8babd77b8 100644 --- a/src/pkg/dashboard/deps.go +++ b/src/pkg/dashboard/deps.go @@ -7,6 +7,8 @@ import ( "sync" "time" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/toolapprove" "github.com/hivecommons/hive/pkg/beads" "github.com/hivecommons/hive/pkg/celtrigger" "github.com/hivecommons/hive/pkg/claims" @@ -15,7 +17,6 @@ import ( "github.com/hivecommons/hive/pkg/convergence/mutation" "github.com/hivecommons/hive/pkg/convergence/outcome" "github.com/hivecommons/hive/pkg/convergence/proof" - "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/dashboard/collect" "github.com/hivecommons/hive/pkg/delegatedecision" ghpkg "github.com/hivecommons/hive/pkg/github" @@ -24,12 +25,12 @@ import ( "github.com/hivecommons/hive/pkg/hooks" "github.com/hivecommons/hive/pkg/knowledge" "github.com/hivecommons/hive/pkg/nous" + "github.com/hivecommons/hive/pkg/review/autoscale" "github.com/hivecommons/hive/pkg/review/eventdispatch" "github.com/hivecommons/hive/pkg/rotation" "github.com/hivecommons/hive/pkg/scheduler" "github.com/hivecommons/hive/pkg/stageengine" "github.com/hivecommons/hive/pkg/tokens" - "github.com/hivecommons/hive/crux/toolapprove" ) type Dependencies struct { @@ -56,12 +57,16 @@ type Dependencies struct { // counters for /api/status.stageEngine and the activity tracker's // status strip. nil leaves the block out of /api/status. StageEngineStatus func() stageengine.Status - Tokens *tokens.Collector - Knowledge *knowledge.KnowledgeAPI - Inception *knowledge.InceptionEngine - Nous *NousState - Scheduler SchedulerControl - MetricsCollector *MetricsCollector + // ReviewerAutoscaleStatus reports the reviewer-lane autoscaler + // (hivecommons/hive#14685) for /api/status.reviewerAutoscale. nil, or a + // Status with Enabled=false, leaves the block out. + ReviewerAutoscaleStatus func() autoscale.Status + Tokens *tokens.Collector + Knowledge *knowledge.KnowledgeAPI + Inception *knowledge.InceptionEngine + Nous *NousState + Scheduler SchedulerControl + MetricsCollector *MetricsCollector // KnowledgePrimer builds, publishes and reports the scheduler's kick // primer — the one thing knowledge.enabled actually gates (#9231). Nil // (bare test deps) means the toggle can only persist the flag, so it diff --git a/src/pkg/dashboard/overview_status.go b/src/pkg/dashboard/overview_status.go index 522106ec60..aa3c6dc42e 100644 --- a/src/pkg/dashboard/overview_status.go +++ b/src/pkg/dashboard/overview_status.go @@ -42,6 +42,11 @@ func (s *Server) statusWithOverviewBands(status *StatusPayload, now time.Time) * st := s.deps.StageEngineStatus() out.StageEngine = &st } + if s.deps != nil && s.deps.ReviewerAutoscaleStatus != nil { + if st := s.deps.ReviewerAutoscaleStatus(); st.Enabled { + out.ReviewerAutoscale = &st + } + } out.Repos = append([]FrontendRepo(nil), status.Repos...) issues := func(items []any, held bool, repoName string) []any { if items == nil { diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index 3a013cdc4d..faad2bc3a3 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -33,6 +33,7 @@ import ( "github.com/hivecommons/hive/pkg/hiveadvisor" spoke "github.com/hivecommons/hive/pkg/hub/spoke" "github.com/hivecommons/hive/pkg/planning" + "github.com/hivecommons/hive/pkg/review/autoscale" "github.com/hivecommons/hive/pkg/stageengine" "github.com/hivecommons/hive/pkg/stagelabels" "github.com/hivecommons/hive/pkg/timeline" @@ -487,6 +488,9 @@ type StatusPayload struct { // (governor.labels.stage.engine.mode). Filled at serve time from // Dependencies.StageEngineStatus; absent when the hive has no engine. StageEngine *stageengine.Status `json:"stageEngine,omitempty"` + // ReviewerAutoscale is the reviewer-lane autoscaler's snapshot + // (hivecommons/hive#14685); absent when autoscale is off. + ReviewerAutoscale *autoscale.Status `json:"reviewerAutoscale,omitempty"` // SLA is the fleet-wide Delivery SLA roll-up (spans, 30-day history, // stale in-progress issues, targets); per-repo blocks sit on each repo. SLA *FrontendSLA `json:"sla,omitempty"` diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index d4dd07baac..15b072c359 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -17889,7 +17889,13 @@

All labels

} const indEl = agentIndicators(a.name); const compactCls = isAgentCompact(a.name) ? ' compact' : ''; - const replicaBadge = (a.replicaBase && a.replicaIndex > 1) ? ` ${esc(a.replicaBase)} · replica ${a.replicaIndex} of ${a.replicaCount}` : ''; + const autoscale = (window._lastStatus || {}).reviewerAutoscale; + const autoLane = autoscale && autoscale.enabled && autoscale.agent === (a.replicaBase || a.name); + const autoTitle = autoLane ? ` — reviewer autoscale: ${autoscale.current} active, target ${autoscale.target}, band ${autoscale.min}–${autoscale.max}${autoscale.lastDecision ? '; last ' + reviewerAutoscaleDecisionText(autoscale.lastDecision) : ''}` : ''; + const autoDraining = autoLane && Array.isArray(autoscale.draining) && autoscale.draining.includes(a.name); + const replicaBadge = (a.replicaBase && a.replicaIndex > 1) + ? ` ${esc(a.replicaBase)} · replica ${a.replicaIndex} of ${a.replicaCount}${autoLane ? ' · auto' : ''}${autoDraining ? ' · draining' : ''}` + : (autoLane ? ` ${autoscale.current} reviewer${autoscale.current === 1 ? '' : 's'} · auto` : ''); const configTarget = a.replicaBase || a.name; /* Says why the card is grey. Without it the only visible difference between "switched off" and "crashed" is the absence of a session, @@ -50166,6 +50172,7 @@

${esc(s.title)}

+ ${renderReviewerAutoscale(rv)}
Dispatch
@@ -52333,6 +52340,78 @@

${esc(s.title)}

rerenderReviewSeverityBacklog(); } + // review.reviewers.autoscale (hivecommons/hive#14685) is a sub-object on + // PUT /api/config/review, sent as reviewer_autoscale with only the edited + // keys so an absent key leaves the file's value alone. + const REVIEWER_AUTOSCALE_DEFAULTS = { min: 1, max: 4, target_backlog_per_reviewer: 2, scale_up_cycles: 2, scale_down_cycles: 5, cooldown: '5m', cpu_per_reviewer: 1, mem_per_reviewer_mb: 2048 }; + function reviewerAutoscaleState() { + const d = _configState.data || (_configState.data = {}); + d.review = d.review || {}; + d.review.reviewers = d.review.reviewers || {}; + d.review.reviewers.autoscale = d.review.reviewers.autoscale || {}; + return d.review.reviewers.autoscale; + } + function markDirtyReviewerAutoscale(key, value) { + const a = reviewerAutoscaleState(); + if (['min', 'max', 'target_backlog_per_reviewer', 'scale_up_cycles', 'scale_down_cycles', 'mem_per_reviewer_mb'].includes(key)) { + value = Number.isFinite(value) && value > 0 ? Math.floor(value) : 0; + } else if (key === 'cpu_per_reviewer') { + value = Number.isFinite(value) && value > 0 ? value : 0; + } + a[key] = value; + markDirty('review', 'reviewer_autoscale', Object.assign({}, (_configState.dirty.review || {}).reviewer_autoscale, { [key]: value })); + } + function toggleReviewerAutoscaleEnabled() { + if (!dashboardRoleAtLeast(window._hiveRole || 'read', 'owner')) return; + const a = reviewerAutoscaleState(); + markDirtyReviewerAutoscale('enabled', !a.enabled); + const host = document.getElementById('review-reviewer-autoscale'); + if (host) host.outerHTML = renderReviewerAutoscale((_configState.data || {}).review); + } + function renderReviewerAutoscale(rv) { + const a = (rv && rv.reviewers && rv.reviewers.autoscale) || {}; + const canEdit = dashboardRoleAtLeast(window._hiveRole || 'read', 'owner'); + const dis = canEdit ? '' : ' disabled'; + const live = (window._lastStatus || {}).reviewerAutoscale; + const liveLine = live && live.enabled + ? `

Now: ${live.current} reviewer${live.current === 1 ? '' : 's'} (target ${live.target}, band ${live.min}–${live.max})${live.draining && live.draining.length ? ', draining ' + live.draining.map(esc).join(', ') : ''}; backlog ${live.backlog}${live.oldestWait ? ', oldest ' + esc(live.oldestWait) : ''}${live.lastDecision ? ' — last: ' + esc(reviewerAutoscaleDecisionText(live.lastDecision)) : ''}

` + : ''; + const num = (key, label, help, step) => `
+
`; + const fields = a.enabled ? ` +
+ ${num('min', 'Min reviewers', 'The floor of active reviewer replicas. The base reviewer agent is never retired.')} + ${num('max', 'Max reviewers', 'The ceiling of active reviewer replicas; capped at the fleet replica limit (5). Each replica is an ordinary reviewer-N agent with its own pane, token and cadence.')} + ${num('target_backlog_per_reviewer', 'Target backlog per reviewer', 'How many PRs may wait for a reviewer, per active reviewer, before the lane is behind. A PR counts as waiting when it has perspectives to review and every eligible reviewer is busy.')} +
+
+ ${num('scale_up_cycles', 'Scale-up cycles', 'Consecutive governor cycles the lane must be behind (backlog over target, or the oldest wait past the review-pending stage dwell) before a reviewer is added.')} + ${num('scale_down_cycles', 'Scale-down cycles', 'Consecutive idle cycles (nothing waiting and at least one reviewer free) before the highest replica is drained. A draining reviewer gets no new work and is retired only once its turn ends.')} +
+
+
+
+ ${num('cpu_per_reviewer', 'CPU per reviewer (cores)', 'The CPU one reviewer is assumed to need. A scale-up is refused when the pod cgroup has less than this free; unknown limits (no cgroup) never block.', '0.1')} + ${num('mem_per_reviewer_mb', 'Memory per reviewer (MiB)', 'The memory one reviewer is assumed to need. A scale-up is refused when the pod cgroup has less than this free.')} +
+
+
` : ''; + return `
+
+
+ Scale reviewers with the backlog ${yamlHint('review.reviewers.autoscale.enabled')} ${settingHelpMark('review-swarm', 'reviewer-autoscale-reviewreviewersautoscale', 'Scale reviewers with the backlog', `Off by default. On, the reviewer lane grows from Min to Max replicas while PRs wait for a free reviewer, and drains back when idle. Replicas added this way are ordinary reviewer-N agents (same role, prompt, scoped token, pane and cadence as a configured replica). Scale-ups are refused when the pod cgroup lacks CPU or memory for another reviewer, when the provider budget latch is on or provider headroom is under 10%, or when reviewer kicks are failing to reach the CLI prompt (a saturated pod, not demand). Each decision is logged as "reviewer autoscale: 2→3 reason=…". Owner only.`)} +
+ ${liveLine}${fields} +
`; + } + function reviewerAutoscaleDecisionText(d) { + if (!d) return ''; + const when = d.at ? ' at ' + new Date(d.at).toLocaleTimeString() : ''; + if (d.action === 'blocked') return `${d.from}→${d.to} blocked by ${d.reason}${when}`; + if (d.agent) return `${d.from}→${d.to} ${d.action} ${d.agent} (${d.reason})${when}`; + return `${d.from}→${d.to} ${d.action} (${d.reason})${when}`; + } + // review_bots is a sub-object on PUT /api/config/review; only // min_priority is editable from the dashboard (logins stay config-only). function markDirtyReviewBotsMinPriority(value) { diff --git a/src/pkg/review/autoscale/cgroup.go b/src/pkg/review/autoscale/cgroup.go new file mode 100644 index 0000000000..0c23d2ffcc --- /dev/null +++ b/src/pkg/review/autoscale/cgroup.go @@ -0,0 +1,140 @@ +package autoscale + +import ( + "os" + "path/filepath" + "strconv" + "strings" + "time" +) + +// Cgroup paths the resource probe reads (cgroup v2, which every supported +// pod runtime mounts). Missing files mean Resources.Known=false. +const ( + defaultCgroupRoot = "/sys/fs/cgroup" + cpuMaxFile = "cpu.max" + cpuStatFile = "cpu.stat" + memMaxFile = "memory.max" + memCurrentFile = "memory.current" + cpuStatUsageKey = "usage_usec" + microsPerSecond = 1e6 +) + +// Prober reads the pod's cgroup budget once per cycle. CPU use is the +// usage_usec delta since the previous call divided by wall time, so the first +// call reports zero use and nothing ever sleeps. +type Prober struct { + root string + lastUsage float64 + lastAt time.Time +} + +// NewProber reads from the default cgroup root. +func NewProber() *Prober { return &Prober{root: defaultCgroupRoot} } + +// NewProberAt reads from root; tests point it at a fixture directory. +func NewProberAt(root string) *Prober { return &Prober{root: root} } + +// Probe returns the current budget. now is the sample time. +func (p *Prober) Probe(now time.Time) Resources { + if p == nil { + return Resources{} + } + var res Resources + if limit, ok := p.cpuLimit(); ok { + res.Known = true + res.CPULimit = limit + } + if usage, ok := p.cpuUsageSeconds(); ok { + if !p.lastAt.IsZero() { + if wall := now.Sub(p.lastAt).Seconds(); wall > 0 { + res.CPUUsed = (usage - p.lastUsage) / wall + if res.CPUUsed < 0 { + res.CPUUsed = 0 + } + } + } + p.lastUsage, p.lastAt = usage, now + } + if limitMB, set, ok := p.memLimitMB(); ok { + res.Known = true + res.MemLimitSet = set + res.MemLimitMB = limitMB + } + if cur, ok := p.readUint(memCurrentFile); ok { + res.MemUsedMB = int(cur / mibBytes) + } + return res +} + +func (p *Prober) read(name string) (string, bool) { + b, err := os.ReadFile(filepath.Join(p.root, name)) + if err != nil { + return "", false + } + return strings.TrimSpace(string(b)), true +} + +func (p *Prober) readUint(name string) (uint64, bool) { + s, ok := p.read(name) + if !ok { + return 0, false + } + v, err := strconv.ParseUint(s, 10, 64) + if err != nil { + return 0, false + } + return v, true +} + +// cpuLimit parses cpu.max ("quota period" or "max period") into cores. +func (p *Prober) cpuLimit() (float64, bool) { + s, ok := p.read(cpuMaxFile) + if !ok { + return 0, false + } + fields := strings.Fields(s) + if len(fields) < 2 || fields[0] == "max" { + return 0, false + } + quota, err1 := strconv.ParseFloat(fields[0], 64) + period, err2 := strconv.ParseFloat(fields[1], 64) + if err1 != nil || err2 != nil || period <= 0 { + return 0, false + } + return quota / period, true +} + +func (p *Prober) cpuUsageSeconds() (float64, bool) { + s, ok := p.read(cpuStatFile) + if !ok { + return 0, false + } + for _, line := range strings.Split(s, "\n") { + fields := strings.Fields(line) + if len(fields) == 2 && fields[0] == cpuStatUsageKey { + v, err := strconv.ParseFloat(fields[1], 64) + if err != nil { + return 0, false + } + return v / microsPerSecond, true + } + } + return 0, false +} + +// memLimitMB parses memory.max; "max" means unlimited (set=false). +func (p *Prober) memLimitMB() (limitMB int, set bool, ok bool) { + s, found := p.read(memMaxFile) + if !found { + return 0, false, false + } + if s == "max" { + return 0, false, true + } + v, err := strconv.ParseUint(s, 10, 64) + if err != nil { + return 0, false, false + } + return int(v / mibBytes), true, true +} diff --git a/src/pkg/review/autoscale/controller.go b/src/pkg/review/autoscale/controller.go new file mode 100644 index 0000000000..1ad6c4162e --- /dev/null +++ b/src/pkg/review/autoscale/controller.go @@ -0,0 +1,464 @@ +// Package autoscale sizes the reviewer lane to its backlog +// (hivecommons/hive#14685). +// +// The Controller is a pure decision function over one Observation per +// governor cycle. It never touches agents itself: the caller materialises a +// scale-up by raising the lane's replica count, honours a drain by +// withholding kicks from the named reviewer, and retires a drained reviewer +// only when the controller says its turn has ended. Time comes in on the +// Observation, so tests drive it with a fake clock and no sleeps. +package autoscale + +import ( + "fmt" + "sort" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/config" +) + +// Action is what the caller should do after an Observe. +type Action string + +const ( + // ActionHold means no change this cycle. + ActionHold Action = "hold" + // ActionScaleUp means raise the lane to Decision.To replicas. + ActionScaleUp Action = "scale-up" + // ActionDrain means stop kicking Decision.Agent; it retires when idle. + ActionDrain Action = "drain" + // ActionUndrain means Decision.Agent may be kicked again: demand returned + // before its turn ended, so keeping it is cheaper than a fresh replica. + ActionUndrain Action = "undrain" + // ActionRetire means Decision.Agent is idle and may now be removed, + // lowering the lane to Decision.To replicas. + ActionRetire Action = "retire" + // ActionBlocked means a scale-up was wanted and a guard refused it; + // Decision.Reason names the guard. + ActionBlocked Action = "blocked" +) + +// Guard names recorded in Decision.Reason when a wanted scale-up is refused. +const ( + GuardAtMax = "at-max" + GuardCooldown = "cooldown" + GuardSaturated = "saturated" + GuardProviderBudget = "provider-budget" + GuardProviderHeadroom = "provider-headroom" + GuardMaxParallelReviews = "max-parallel-reviews" + GuardCPU = "cpu" + GuardMemory = "memory" +) + +const ( + // SaturationFraction is the share of active reviewers whose last kick + // failed to reach the CLI prompt above which the lane is treated as + // saturated: more reviewers would fail the same way, so demand is not the + // problem. + SaturationFraction = 0.5 + // MinProviderHeadroomPct is the provider quota remaining below which a + // scale-up is refused; a new reviewer would only burn the last of it. + MinProviderHeadroomPct = 10.0 + // mibBytes converts the cgroup's byte counters to the MiB config unit. + mibBytes = 1024 * 1024 +) + +// Reviewer is one active reviewer replica as the caller sees it this cycle. +type Reviewer struct { + Name string + // Index is the replica index (base agent is 1). Drains pick the highest. + Index int + // Busy is true while the agent is mid-turn or has a review kick in + // flight; a draining reviewer is never retired while Busy. + Busy bool + // Saturated is true when the agent's most recent kicks were undeliverable + // ("CLI did not reach input prompt"). + Saturated bool +} + +// Resources is the pod's cgroup budget as of this cycle. Known is false when +// no cgroup limits could be read, which disables the resource guard rather +// than blocking (a dev laptop has no cgroup, a pod always does). +type Resources struct { + Known bool + CPULimit float64 + CPUUsed float64 + MemLimitMB int + MemUsedMB int + MemLimitSet bool +} + +// FreeCPU is the unused share of the CPU quota in cores. +func (r Resources) FreeCPU() float64 { return r.CPULimit - r.CPUUsed } + +// FreeMemMB is the unused memory in MiB; unbounded when no limit is set. +func (r Resources) FreeMemMB() int { + if !r.MemLimitSet { + return int(^uint(0) >> 1) + } + return r.MemLimitMB - r.MemUsedMB +} + +// Observation is everything the controller looks at for one cycle. +type Observation struct { + Now time.Time + Reviewers []Reviewer + // WaitingKeys identify PRs that needed a reviewer this cycle and got + // none (repo#number@head). The controller ages them across cycles. + WaitingKeys []string + // InFlight is the number of review kicks currently pending on the lane. + InFlight int + // DwellThreshold is the stage dwell after which a waiting PR counts as + // late regardless of the backlog ratio; zero disables the age signal. + DwellThreshold time.Duration + // MaxParallelReviews is review.max_parallel_reviews; zero disables the + // slot guard. + MaxParallelReviews int + Resources Resources + // ProviderBudgetExceeded is the inference budget latch. + ProviderBudgetExceeded bool + // ProviderHeadroomPct is the lowest remaining quota across providers; + // negative when unknown. + ProviderHeadroomPct float64 +} + +// Decision is the controller's answer for one cycle. +type Decision struct { + Action Action + From int + To int + Agent string + Reason string + At time.Time +} + +// String renders the one-line audit form, e.g. +// "reviewer autoscale: 2→3 reason=backlog=7 oldest=41m". +func (d Decision) String() string { + switch d.Action { + case ActionHold: + return fmt.Sprintf("reviewer autoscale: hold at %d reason=%s", d.From, d.Reason) + case ActionDrain, ActionUndrain, ActionRetire: + return fmt.Sprintf("reviewer autoscale: %d→%d %s %s reason=%s", d.From, d.To, d.Action, d.Agent, d.Reason) + case ActionBlocked: + return fmt.Sprintf("reviewer autoscale: %d→%d blocked by %s", d.From, d.To, d.Reason) + default: + return fmt.Sprintf("reviewer autoscale: %d→%d reason=%s", d.From, d.To, d.Reason) + } +} + +// Status is the snapshot the dashboard shows. +type Status struct { + Enabled bool `json:"enabled"` + Agent string `json:"agent,omitempty"` + Min int `json:"min"` + Max int `json:"max"` + Current int `json:"current"` + Target int `json:"target"` + Draining []string `json:"draining,omitempty"` + Backlog int `json:"backlog"` + OldestWait string `json:"oldestWait,omitempty"` + UpStreak int `json:"upStreak"` + DownStreak int `json:"downStreak"` + LastDecision *Decision `json:"lastDecision,omitempty"` +} + +// Controller holds the sustained-signal state between cycles. +type Controller struct { + cfg config.ReviewerAutoscaleConfig + agent string + firstSeen map[string]time.Time + draining map[string]bool + upStreak int + downStreak int + lastAction time.Time + last *Decision + backlog int + oldest time.Duration + current int + target int +} + +// New builds a controller for the named base agent. +func New(cfg config.ReviewerAutoscaleConfig, agent string) *Controller { + return &Controller{ + cfg: cfg, + agent: agent, + firstSeen: map[string]time.Time{}, + draining: map[string]bool{}, + } +} + +// Reconfigure swaps the band and thresholds without losing streaks. +func (c *Controller) Reconfigure(cfg config.ReviewerAutoscaleConfig, agent string) { + c.cfg = cfg + c.agent = agent +} + +// Draining reports whether kicks to name should be withheld. +func (c *Controller) Draining(name string) bool { + return c != nil && c.draining[name] +} + +// DrainingNames lists draining reviewers, sorted. +func (c *Controller) DrainingNames() []string { + if c == nil || len(c.draining) == 0 { + return nil + } + out := make([]string, 0, len(c.draining)) + for n := range c.draining { + out = append(out, n) + } + sort.Strings(out) + return out +} + +// Status returns the dashboard snapshot. +func (c *Controller) Status() Status { + if c == nil { + return Status{} + } + st := Status{ + Enabled: true, + Agent: c.agent, + Min: c.cfg.EffectiveMin(), + Max: c.cfg.EffectiveMax(), + Current: c.current, + Target: c.target, + Draining: c.DrainingNames(), + Backlog: c.backlog, + UpStreak: c.upStreak, + DownStreak: c.downStreak, + } + if c.oldest > 0 { + st.OldestWait = c.oldest.Round(time.Second).String() + } + if c.last != nil { + d := *c.last + st.LastDecision = &d + } + return st +} + +// Observe folds one cycle into the controller and returns what to do. +func (c *Controller) Observe(obs Observation) Decision { + d := c.observe(obs) + d.At = obs.Now + if d.Action != ActionHold { + c.last = &d + } + return d +} + +func (c *Controller) observe(obs Observation) Decision { + c.ageBacklog(obs) + active := len(obs.Reviewers) + c.current = active + c.pruneDraining(obs.Reviewers) + serving := active - len(c.draining) + if serving < 0 { + serving = 0 + } + c.target = serving + min, max := c.cfg.EffectiveMin(), c.cfg.EffectiveMax() + + behind := c.behind(serving, obs.DwellThreshold) + idle := c.backlog == 0 && obs.InFlight < active + if behind { + c.upStreak++ + c.downStreak = 0 + } else if idle { + c.downStreak++ + c.upStreak = 0 + } else { + c.upStreak = 0 + c.downStreak = 0 + } + + // A drained reviewer whose turn ended is retired before anything else: + // it completes the earlier decision rather than being a new one. + if name, ok := c.retirable(obs.Reviewers); ok { + delete(c.draining, name) + c.lastAction = obs.Now + c.target = active - 1 + return Decision{Action: ActionRetire, From: active, To: active - 1, Agent: name, Reason: "idle"} + } + + reason := c.reason() + if behind && c.upStreak >= c.cfg.EffectiveScaleUpCycles() { + // Demand came back while a reviewer was draining: keep it. + if name, ok := c.highestDraining(obs.Reviewers); ok { + delete(c.draining, name) + c.upStreak = 0 + c.lastAction = obs.Now + c.target = serving + 1 + return Decision{Action: ActionUndrain, From: serving, To: serving + 1, Agent: name, Reason: reason} + } + if guard := c.scaleUpGuard(obs, active, serving, max); guard != "" { + return Decision{Action: ActionBlocked, From: serving, To: serving + 1, Reason: guard} + } + c.upStreak = 0 + c.lastAction = obs.Now + c.target = active + 1 + return Decision{Action: ActionScaleUp, From: active, To: active + 1, Reason: reason} + } + + if idle && c.downStreak >= c.cfg.EffectiveScaleDownCycles() && serving > min && len(c.draining) == 0 { + if !obs.Now.Before(c.lastAction.Add(c.cfg.EffectiveCooldown())) { + if name, ok := c.highestActive(obs.Reviewers); ok { + c.draining[name] = true + c.downStreak = 0 + c.lastAction = obs.Now + c.target = serving - 1 + return Decision{Action: ActionDrain, From: serving, To: serving - 1, Agent: name, Reason: fmt.Sprintf("idle for %d cycles", c.cfg.EffectiveScaleDownCycles())} + } + } + } + return Decision{Action: ActionHold, From: active, To: active, Reason: reason} +} + +func (c *Controller) ageBacklog(obs Observation) { + seen := make(map[string]bool, len(obs.WaitingKeys)) + for _, k := range obs.WaitingKeys { + k = strings.TrimSpace(k) + if k == "" { + continue + } + seen[k] = true + if _, ok := c.firstSeen[k]; !ok { + c.firstSeen[k] = obs.Now + } + } + for k := range c.firstSeen { + if !seen[k] { + delete(c.firstSeen, k) + } + } + c.backlog = len(seen) + c.oldest = 0 + for _, t := range c.firstSeen { + if age := obs.Now.Sub(t); age > c.oldest { + c.oldest = age + } + } +} + +func (c *Controller) behind(serving int, dwell time.Duration) bool { + if c.backlog == 0 { + return false + } + if serving == 0 { + return true + } + if c.backlog > c.cfg.EffectiveTargetBacklogPerReviewer()*serving { + return true + } + return dwell > 0 && c.oldest > dwell +} + +func (c *Controller) reason() string { + if c.oldest > 0 { + return fmt.Sprintf("backlog=%d oldest=%s", c.backlog, c.oldest.Round(time.Second)) + } + return fmt.Sprintf("backlog=%d", c.backlog) +} + +func (c *Controller) scaleUpGuard(obs Observation, active, serving, max int) string { + if serving >= max || active >= max { + return GuardAtMax + } + if obs.Now.Before(c.lastAction.Add(c.cfg.EffectiveCooldown())) { + return GuardCooldown + } + if saturated(obs.Reviewers) { + return GuardSaturated + } + if obs.ProviderBudgetExceeded { + return GuardProviderBudget + } + if obs.ProviderHeadroomPct >= 0 && obs.ProviderHeadroomPct < MinProviderHeadroomPct { + return GuardProviderHeadroom + } + if obs.MaxParallelReviews > 0 && active >= obs.MaxParallelReviews { + return GuardMaxParallelReviews + } + if obs.Resources.Known { + if obs.Resources.CPULimit > 0 && obs.Resources.FreeCPU() < c.cfg.EffectiveCPUPerReviewer() { + return GuardCPU + } + if obs.Resources.FreeMemMB() < c.cfg.EffectiveMemPerReviewerMB() { + return GuardMemory + } + } + return "" +} + +func saturated(reviewers []Reviewer) bool { + if len(reviewers) == 0 { + return false + } + n := 0 + for _, r := range reviewers { + if r.Saturated { + n++ + } + } + return float64(n) >= SaturationFraction*float64(len(reviewers)) +} + +func (c *Controller) pruneDraining(reviewers []Reviewer) { + present := make(map[string]bool, len(reviewers)) + for _, r := range reviewers { + present[r.Name] = true + } + for n := range c.draining { + if !present[n] { + delete(c.draining, n) + } + } +} + +func (c *Controller) retirable(reviewers []Reviewer) (string, bool) { + var best *Reviewer + for i := range reviewers { + r := &reviewers[i] + if !c.draining[r.Name] || r.Busy { + continue + } + if best == nil || r.Index > best.Index { + best = r + } + } + if best == nil { + return "", false + } + return best.Name, true +} + +func (c *Controller) highestDraining(reviewers []Reviewer) (string, bool) { + return c.highest(reviewers, true) +} + +func (c *Controller) highestActive(reviewers []Reviewer) (string, bool) { + return c.highest(reviewers, false) +} + +// highest picks the highest-index reviewer in the draining (or not) set, +// never the base replica (index 1), which is the lane's floor. +func (c *Controller) highest(reviewers []Reviewer, draining bool) (string, bool) { + var best *Reviewer + for i := range reviewers { + r := &reviewers[i] + if c.draining[r.Name] != draining || r.Index <= 1 { + continue + } + if best == nil || r.Index > best.Index { + best = r + } + } + if best == nil { + return "", false + } + return best.Name, true +} diff --git a/src/pkg/review/autoscale/controller_test.go b/src/pkg/review/autoscale/controller_test.go new file mode 100644 index 0000000000..5930faa9a0 --- /dev/null +++ b/src/pkg/review/autoscale/controller_test.go @@ -0,0 +1,251 @@ +package autoscale + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" +) + +var t0 = time.Date(2026, 10, 27, 19, 0, 0, 0, time.UTC) + +func testCfg() config.ReviewerAutoscaleConfig { + return config.ReviewerAutoscaleConfig{ + Enabled: true, + Min: 1, + Max: 3, + TargetBacklogPerReviewer: 2, + ScaleUpCycles: 2, + ScaleDownCycles: 3, + Cooldown: "5m", + } +} + +func reviewers(n int) []Reviewer { + out := make([]Reviewer, 0, n) + for i := 1; i <= n; i++ { + name := "reviewer" + if i > 1 { + name = "reviewer-" + string(rune('0'+i)) + } + out = append(out, Reviewer{Name: name, Index: i}) + } + return out +} + +func keys(n int) []string { + out := make([]string, 0, n) + for i := 0; i < n; i++ { + out = append(out, "o/r#"+string(rune('a'+i))+"@sha") + } + return out +} + +func obsAt(step int, rs []Reviewer, backlog int) Observation { + return Observation{ + Now: t0.Add(time.Duration(step) * time.Minute), + Reviewers: rs, + WaitingKeys: keys(backlog), + InFlight: 0, + ProviderHeadroomPct: -1, + } +} + +// drive feeds n identical cycles and returns the last decision. +func drive(c *Controller, from, n int, rs []Reviewer, backlog int, mutate func(*Observation)) Decision { + var d Decision + for i := 0; i < n; i++ { + o := obsAt(from+i, rs, backlog) + if mutate != nil { + mutate(&o) + } + d = c.Observe(o) + } + return d +} + +func TestScaleUpDecisionTable(t *testing.T) { + cases := []struct { + name string + reviewers []Reviewer + backlog int + cycles int + mutate func(*Observation) + want Action + reason string + }{ + {name: "sustained backlog scales up", reviewers: reviewers(2), backlog: 5, cycles: 2, want: ActionScaleUp}, + {name: "single behind cycle holds", reviewers: reviewers(2), backlog: 5, cycles: 1, want: ActionHold}, + {name: "backlog within target holds", reviewers: reviewers(2), backlog: 4, cycles: 3, want: ActionHold}, + {name: "never beyond max", reviewers: reviewers(3), backlog: 9, cycles: 2, want: ActionBlocked, reason: GuardAtMax}, + {name: "cpu guard blocks", reviewers: reviewers(2), backlog: 5, cycles: 2, want: ActionBlocked, reason: GuardCPU, + mutate: func(o *Observation) { o.Resources = Resources{Known: true, CPULimit: 8, CPUUsed: 7.5} }}, + {name: "memory guard blocks", reviewers: reviewers(2), backlog: 5, cycles: 2, want: ActionBlocked, reason: GuardMemory, + mutate: func(o *Observation) { + o.Resources = Resources{Known: true, CPULimit: 8, MemLimitSet: true, MemLimitMB: 4096, MemUsedMB: 3000} + }}, + {name: "unknown resources do not block", reviewers: reviewers(2), backlog: 5, cycles: 2, want: ActionScaleUp, + mutate: func(o *Observation) { o.Resources = Resources{} }}, + {name: "saturated kicks block", reviewers: []Reviewer{{Name: "reviewer", Index: 1, Saturated: true}, {Name: "reviewer-2", Index: 2}}, + backlog: 5, cycles: 2, want: ActionBlocked, reason: GuardSaturated}, + {name: "provider budget latch blocks", reviewers: reviewers(2), backlog: 5, cycles: 2, want: ActionBlocked, reason: GuardProviderBudget, + mutate: func(o *Observation) { o.ProviderBudgetExceeded = true }}, + {name: "low provider headroom blocks", reviewers: reviewers(2), backlog: 5, cycles: 2, want: ActionBlocked, reason: GuardProviderHeadroom, + mutate: func(o *Observation) { o.ProviderHeadroomPct = 5 }}, + {name: "max parallel reviews blocks", reviewers: reviewers(2), backlog: 5, cycles: 2, want: ActionBlocked, reason: GuardMaxParallelReviews, + mutate: func(o *Observation) { o.MaxParallelReviews = 2 }}, + {name: "old waiting PR scales up below ratio", reviewers: reviewers(2), backlog: 1, cycles: 13, want: ActionScaleUp, + mutate: func(o *Observation) { o.DwellThreshold = 10 * time.Minute }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := New(testCfg(), "reviewer") + d := drive(c, 0, tc.cycles, tc.reviewers, tc.backlog, tc.mutate) + if d.Action != tc.want { + t.Fatalf("action = %s (%s), want %s", d.Action, d.Reason, tc.want) + } + if tc.reason != "" && d.Reason != tc.reason { + t.Fatalf("reason = %q, want %q", d.Reason, tc.reason) + } + if tc.want == ActionScaleUp && d.To != len(tc.reviewers)+1 { + t.Fatalf("to = %d, want %d", d.To, len(tc.reviewers)+1) + } + }) + } +} + +func TestCooldownBlocksSecondScaleUp(t *testing.T) { + c := New(testCfg(), "reviewer") + if d := drive(c, 0, 2, reviewers(1), 5, nil); d.Action != ActionScaleUp { + t.Fatalf("first: %s", d.Action) + } + // Two more behind cycles one minute later: wanted, but inside cooldown. + if d := drive(c, 2, 2, reviewers(2), 9, nil); d.Action != ActionBlocked || d.Reason != GuardCooldown { + t.Fatalf("second: %s %s", d.Action, d.Reason) + } + // After the cooldown the streak is still live, so it scales. + if d := drive(c, 10, 1, reviewers(2), 9, nil); d.Action != ActionScaleUp { + t.Fatalf("third: %s %s", d.Action, d.Reason) + } +} + +func TestScaleDownDrainsThenRetiresOnlyWhenIdle(t *testing.T) { + c := New(testCfg(), "reviewer") + rs := reviewers(3) + // Idle for fewer cycles than required: hold. + if d := drive(c, 10, 2, rs, 0, nil); d.Action != ActionHold { + t.Fatalf("early: %s", d.Action) + } + d := drive(c, 12, 1, rs, 0, nil) + if d.Action != ActionDrain || d.Agent != "reviewer-3" || d.To != 2 { + t.Fatalf("drain: %+v", d) + } + if !c.Draining("reviewer-3") { + t.Fatal("reviewer-3 should be draining") + } + // Mid-turn: never retired. + busy := []Reviewer{rs[0], rs[1], {Name: "reviewer-3", Index: 3, Busy: true}} + for i := 0; i < 4; i++ { + if d := drive(c, 13+i, 1, busy, 0, nil); d.Action != ActionHold { + t.Fatalf("busy cycle %d: %s", i, d.Action) + } + } + // Turn ended: retire. + d = drive(c, 20, 1, rs, 0, nil) + if d.Action != ActionRetire || d.Agent != "reviewer-3" || d.To != 2 { + t.Fatalf("retire: %+v", d) + } + if c.Draining("reviewer-3") { + t.Fatal("retired reviewer still draining") + } +} + +func TestNeverDrainsBelowMin(t *testing.T) { + cfg := testCfg() + cfg.Min = 2 + c := New(cfg, "reviewer") + if d := drive(c, 0, 10, reviewers(2), 0, nil); d.Action != ActionHold { + t.Fatalf("got %s", d.Action) + } + if st := c.Status(); st.Target != 2 || st.Current != 2 { + t.Fatalf("status %+v", st) + } +} + +func TestInFlightWorkIsNotIdle(t *testing.T) { + c := New(testCfg(), "reviewer") + d := drive(c, 0, 10, reviewers(2), 0, func(o *Observation) { o.InFlight = 2 }) + if d.Action != ActionHold { + t.Fatalf("got %s", d.Action) + } +} + +func TestDemandDuringDrainUndrains(t *testing.T) { + c := New(testCfg(), "reviewer") + rs := reviewers(2) + if d := drive(c, 0, 3, rs, 0, nil); d.Action != ActionDrain { + t.Fatalf("drain: %s", d.Action) + } + busy := []Reviewer{rs[0], {Name: "reviewer-2", Index: 2, Busy: true}} + d := drive(c, 3, 2, busy, 5, nil) + if d.Action != ActionUndrain || d.Agent != "reviewer-2" { + t.Fatalf("undrain: %+v", d) + } + if c.Draining("reviewer-2") { + t.Fatal("still draining") + } +} + +func TestOldestWaitAgesAcrossCycles(t *testing.T) { + c := New(testCfg(), "reviewer") + drive(c, 0, 5, reviewers(2), 1, nil) + st := c.Status() + if st.Backlog != 1 || st.OldestWait != "4m0s" { + t.Fatalf("status %+v", st) + } + // Key leaves the backlog: age resets. + drive(c, 5, 1, reviewers(2), 0, nil) + if st := c.Status(); st.OldestWait != "" || st.Backlog != 0 { + t.Fatalf("status after clear %+v", st) + } +} + +func TestDecisionString(t *testing.T) { + d := Decision{Action: ActionScaleUp, From: 2, To: 3, Reason: "backlog=7 oldest=41m0s"} + if got := d.String(); got != "reviewer autoscale: 2→3 reason=backlog=7 oldest=41m0s" { + t.Fatalf("got %q", got) + } +} + +func TestProberReadsCgroupV2(t *testing.T) { + dir := t.TempDir() + write := func(name, body string) { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + write(cpuMaxFile, "800000 100000\n") + write(cpuStatFile, "usage_usec 1000000\nuser_usec 1\n") + write(memMaxFile, "4294967296\n") + write(memCurrentFile, "1073741824\n") + p := NewProberAt(dir) + r := p.Probe(t0) + if !r.Known || r.CPULimit != 8 || r.MemLimitMB != 4096 || r.MemUsedMB != 1024 || r.CPUUsed != 0 { + t.Fatalf("first %+v", r) + } + write(cpuStatFile, "usage_usec 61000000\n") + r = p.Probe(t0.Add(time.Minute)) + if r.CPUUsed != 1 { + t.Fatalf("cpu used = %v, want 1", r.CPUUsed) + } + write(memMaxFile, "max\n") + r = p.Probe(t0.Add(2 * time.Minute)) + if r.MemLimitSet || r.FreeMemMB() < 1<<40 { + t.Fatalf("unlimited mem %+v", r) + } + if r := NewProberAt(filepath.Join(dir, "missing")).Probe(t0); r.Known { + t.Fatalf("missing cgroup should be unknown: %+v", r) + } +} diff --git a/src/pkg/review/dispatch.go b/src/pkg/review/dispatch.go index d817fbf77e..d50e5f6e46 100644 --- a/src/pkg/review/dispatch.go +++ b/src/pkg/review/dispatch.go @@ -97,6 +97,10 @@ type AgentCapability struct { ReviewHead string ReviewHeadOf string HasReviewHeads bool + // Draining marks a reviewer the autoscaler is retiring + // (hivecommons/hive#14685): it finishes what it has but is handed no + // new review, exactly as if it were paused. + Draining bool } // CapabilityFromConfig builds the dispatcher's view of one configured lane @@ -500,7 +504,42 @@ type DispatchPlan struct { // ForcedFixes are the changes_requested heads whose fix kick was forced // this cycle after RevisitStarvationCycles revisit-suppressed cycles. ForcedFixes []ForcedFix - State DispatchState + // Waiting are the PRs that needed a review kick this cycle and got none + // because every eligible reviewer was busy or the parallel budget was + // spent. They are the reviewer autoscaler's demand signal + // (hivecommons/hive#14685): one entry per PR, on its current head. + Waiting []ReviewWait + State DispatchState +} + +// ReviewWait is one PR left waiting for a reviewer this cycle. +type ReviewWait struct { + Repo string + Number int + HeadSHA string + Reason string +} + +// Key identifies the wait across cycles: a push to the PR starts a new wait. +func (w ReviewWait) Key() string { return fmt.Sprintf("%s#%d@%s", w.Repo, w.Number, w.HeadSHA) } + +const ( + // ReviewWaitNoFreeReviewer: every reviewer eligible for the PR has a + // review kick in flight. + ReviewWaitNoFreeReviewer = "no-free-reviewer" + // ReviewWaitNoParallelSlot: review.max_parallel_reviews is spent. + ReviewWaitNoParallelSlot = "no-parallel-slot" +) + +// recordWait notes a PR that still has perspectives to review but could not +// be dispatched this cycle. Idempotent per PR. +func (plan *DispatchPlan) recordWait(pr PullRequest, reason string) { + for _, w := range plan.Waiting { + if w.Repo == pr.Repo && w.Number == pr.Number { + return + } + } + plan.Waiting = append(plan.Waiting, ReviewWait{Repo: pr.Repo, Number: pr.Number, HeadSHA: pr.HeadSHA, Reason: reason}) } type FixDispatchExclusion struct { @@ -747,6 +786,9 @@ func PlanDispatch(prs []PullRequest, artifact Artifact, state DispatchState, opt } } if availableSlots <= 0 { + if len(pendingMissingPerspectives(plan.State, pr, opts.Perspectives.List(), delivered)) > 0 { + plan.recordWait(pr, ReviewWaitNoParallelSlot) + } continue } authorBase := prAuthorBase(pr, opts.Agents) @@ -762,13 +804,17 @@ func PlanDispatch(prs []PullRequest, artifact Artifact, state DispatchState, opt // validateReviewHeads with heads on) take the perspectives the heads // left unasked, on the ordinary path. prReviewers := reviewersForPR(reviewers, pr.AuthorAgent, authorBase) - if len(prReviewers) == 0 || availableSlots <= 0 { + if len(prReviewers) == 0 { continue } missing := pendingMissingPerspectives(plan.State, pr, opts.Perspectives.List(), delivered) if len(missing) == 0 { continue } + if availableSlots <= 0 { + plan.recordWait(pr, ReviewWaitNoParallelSlot) + continue + } // One kick, every outstanding perspective, one comment. The per-PR cap // is deliberately not applied here: it exists to bound how many review // COMMENTS one PR collects, and a combined review produces exactly one @@ -777,6 +823,7 @@ func PlanDispatch(prs []PullRequest, artifact Artifact, state DispatchState, opt if opts.CombinedPerspectives { reviewer, free := freeReviewer(prReviewers, plan.inFlight(opts, now)) if !free { + plan.recordWait(pr, ReviewWaitNoFreeReviewer) continue } agent := reviewer.Name @@ -851,6 +898,9 @@ func PlanDispatch(prs []PullRequest, artifact Artifact, state DispatchState, opt for i := 0; i < limit; i++ { reviewer, free := freeReviewer(prReviewers, plan.inFlight(opts, now)) if !free { + if i == 0 { + plan.recordWait(pr, ReviewWaitNoFreeReviewer) + } break } agent := reviewer.Name @@ -1683,7 +1733,7 @@ func reviewCapableAgents(opts DispatchOptions) []AgentCapability { allowed := ReviewerAgentSet(opts.ReviewerAgents) var out []AgentCapability for _, a := range opts.Agents { - if a.Name == "" || !a.Enabled || a.Paused || a.OnDemand || !a.UsesKick { + if a.Name == "" || !a.Enabled || a.Paused || a.Draining || a.OnDemand || !a.UsesKick { continue } if ReviewCapable(a, allowed) { diff --git a/src/pkg/review/dispatch_waiting_test.go b/src/pkg/review/dispatch_waiting_test.go new file mode 100644 index 0000000000..52fa0ce47c --- /dev/null +++ b/src/pkg/review/dispatch_waiting_test.go @@ -0,0 +1,92 @@ +package review + +import ( + "testing" +) + +// hivecommons/hive#14685: the planner reports the PRs it could not hand to a +// reviewer so the reviewer autoscaler can size the lane to them. + +func waitingNumbers(plan DispatchPlan) map[int]string { + out := map[int]string{} + for _, w := range plan.Waiting { + out[w.Number] = w.Reason + } + return out +} + +func TestPlanDispatchReportsPRsWaitingForAFreeReviewer(t *testing.T) { + prs := []PullRequest{dispatchPRNum(1, "sha1"), dispatchPRNum(2, "sha2"), dispatchPRNum(3, "sha3")} + plan := PlanDispatch(prs, Artifact{}, DispatchState{}, DispatchOptions{ + RequireApproval: true, + CombinedPerspectives: true, + MaxParallelReviews: 10, + ProjectOrg: "acme", + Agents: []AgentCapability{reviewer("r1"), reviewer("r2")}, + }) + if len(plan.ReviewKicks) != 2 { + t.Fatalf("kicks = %d, want 2 (one per reviewer): %+v", len(plan.ReviewKicks), plan.ReviewKicks) + } + got := waitingNumbers(plan) + if len(got) != 1 || got[3] != ReviewWaitNoFreeReviewer { + t.Fatalf("waiting = %+v, want PR 3 %s", plan.Waiting, ReviewWaitNoFreeReviewer) + } + if key := plan.Waiting[0].Key(); key != "hive#3@sha3" { + t.Fatalf("key = %q", key) + } +} + +func TestPlanDispatchReportsPRsWaitingForAParallelSlot(t *testing.T) { + prs := []PullRequest{dispatchPRNum(1, "sha1"), dispatchPRNum(2, "sha2")} + plan := PlanDispatch(prs, Artifact{}, DispatchState{}, DispatchOptions{ + RequireApproval: true, + CombinedPerspectives: true, + MaxParallelReviews: 1, + ProjectOrg: "acme", + Agents: []AgentCapability{reviewer("r1"), reviewer("r2")}, + }) + if len(plan.ReviewKicks) != 1 { + t.Fatalf("kicks = %d, want 1", len(plan.ReviewKicks)) + } + if got := waitingNumbers(plan); len(got) != 1 || got[2] != ReviewWaitNoParallelSlot { + t.Fatalf("waiting = %+v", plan.Waiting) + } +} + +func TestPlanDispatchNothingWaitingWhenReviewersCover(t *testing.T) { + prs := []PullRequest{dispatchPRNum(1, "sha1")} + plan := PlanDispatch(prs, Artifact{}, DispatchState{}, DispatchOptions{ + RequireApproval: true, + CombinedPerspectives: true, + MaxParallelReviews: 10, + ProjectOrg: "acme", + Agents: []AgentCapability{reviewer("r1"), reviewer("r2")}, + }) + if len(plan.Waiting) != 0 { + t.Fatalf("waiting = %+v, want none", plan.Waiting) + } +} + +func TestDrainingReviewerGetsNoNewReview(t *testing.T) { + draining := reviewer("r2") + draining.Draining = true + prs := []PullRequest{dispatchPRNum(1, "sha1"), dispatchPRNum(2, "sha2")} + plan := PlanDispatch(prs, Artifact{}, DispatchState{}, DispatchOptions{ + RequireApproval: true, + CombinedPerspectives: true, + MaxParallelReviews: 10, + ProjectOrg: "acme", + Agents: []AgentCapability{reviewer("r1"), draining}, + }) + for _, k := range plan.ReviewKicks { + if k.Agent == "r2" { + t.Fatalf("draining reviewer was dispatched: %+v", plan.ReviewKicks) + } + } + if len(plan.ReviewKicks) != 1 { + t.Fatalf("kicks = %d, want 1 from r1", len(plan.ReviewKicks)) + } + if got := waitingNumbers(plan); got[2] != ReviewWaitNoFreeReviewer { + t.Fatalf("waiting = %+v", plan.Waiting) + } +} From c21d888a551c8e83daffd4525e15a1454e37cf34 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 27 Oct 2026 16:10:53 -0400 Subject: [PATCH 2/4] review autoscale: satisfy dashboard guards and planner test setup - settings parity: cite each review.reviewers.autoscale.* path literally - style ratchet: drop the inline display style on the autoscale wrapper - timezone: format the last decision time through formatDashboardTime - api-reference.md: resync server.go citations shifted by StatusPayload - dispatch_waiting_test: enable FanOut, which gates review dispatch Signed-off-by: Andy Anderson --- src/docs/api-reference.md | 34 ++++++++++++------------- src/pkg/dashboard/static/index.html | 20 +++++++-------- src/pkg/review/dispatch_waiting_test.go | 4 +++ 3 files changed, 31 insertions(+), 27 deletions(-) diff --git a/src/docs/api-reference.md b/src/docs/api-reference.md index 20ce96a058..0677f9aa8c 100644 --- a/src/docs/api-reference.md +++ b/src/docs/api-reference.md @@ -13,17 +13,17 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)) | `pkg/dashboard/api.go:42` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=&to=` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:43` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:64` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1498` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1499` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1500` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1506` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced}` is the governor strip's `running N · $X this hour` figure; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376) | `pkg/dashboard/server.go:1511` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1512` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1513` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1514` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1515` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1516` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1517` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1502` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1503` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1504` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1510` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced}` is the governor strip's `running N · $X this hour` figure; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376) | `pkg/dashboard/server.go:1515` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1516` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1517` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1518` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1519` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1520` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1521` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | @@ -66,7 +66,7 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:44` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](branding.md)) | `pkg/dashboard/server.go:1578` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](branding.md)) | `pkg/dashboard/server.go:1582` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:77` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:78` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:79` | @@ -94,9 +94,9 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1518` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1519` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1520` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1522` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1523` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1524` | ## Configuration @@ -569,7 +569,7 @@ always resolved server-side from the validated token. | `GET` | `/api/budget-ignore` | Dashboard auth/session | Budget Ignore Get | `pkg/dashboard/api.go:170` | | `POST` | `/api/budget-ignore` | Owner only | Budget Ignore Set | `pkg/dashboard/api.go:171` | | `GET` | `/api/summaries` | Dashboard auth/session | Summaries | `pkg/dashboard/api.go:183` | -| `GET` | `/api/stage-tracker` | Dashboard auth/session | Stage Tracker: every open issue and pull request of the watched repos for the Hive activity monitor — actionable, held, claimed, in-progress, covered-by-PR, filtered (`needs-*`), draft and exempt-labelled items alike, each with `stage`, `stage_since`, `labels`, `created_at`, `author_kind`, `modifiers`, (PRs) `sentinel_alert` / `sentinel_findings` / `sentinel_reason` when the sentinel flagged the PR, and (issues) `linked_prs`; items with no stage yet report stage `0` "Open"; `needs_human_labels` is the "a person must act" label set behind the monitor's Needs human facet (the same list as `/api/status.labelPresets[needs-human]`); `recently_merged` lists the PRs (`repo`, `number`, `merged_at`) whose `pr_merged` audit entry is under 30 minutes old, so a client diffing two payloads can tell a merge from a close. Served from a server-side cache rebuilt at most every `dashboard.stage_tracker.cache_ttl_s` (default `governor.eval_interval_s`) from the spoke's existing snapshot — a request never calls GitHub. Carries `generated_at`, `source` (`cache`, `fresh`, or `stale` — the previous body served at once while one rebuild runs; at most one rebuild is ever in flight, and a status publish or webhook only marks the cache stale, it never waits), `cache_ttl_s`, an `ETag` and `Cache-Control: private, max-age=` (`0` for `stale`); `If-None-Match` yields 304. A request that arrives before any body exists waits for the in-flight build and answers 503 if its own connection closes first. Each hit is recorded under REST caller `dashboard:stage-tracker` so `/api/gh-rate-limits` shows it charged nothing. | `pkg/dashboard/api.go:184` | +| `GET` | `/api/stage-tracker` | Dashboard auth/session | Stage Tracker: every open issue and pull request of the watched repos for the Hive activity monitor — actionable, held, claimed, in-progress, covered-by-PR, filtered (`needs-*`), draft and exempt-labelled items alike, each with `stage`, `stage_since`, `labels`, `created_at`, `author_kind`, `modifiers`, (PRs) `sentinel_alert` / `sentinel_findings` / `sentinel_reason` when the sentinel flagged the PR, and (issues) `linked_prs`; items with no stage yet report stage `0` "Open"; `needs_human_labels` is the "a person must act" label set behind the monitor's Needs human facet (the same list as `/api/status.labelPresets[needs-human]`); `recently_merged` lists the PRs (`repo`, `number`, `merged_at`) whose `pr_merged` audit entry is under 30 minutes old, so a client diffing two payloads can tell a merge from a close. Served from a server-side cache rebuilt at most every `dashboard.stage_tracker.cache_ttl_s` (default `governor.eval_interval_s`) from the spoke's existing snapshot — a request never calls GitHub. Carries `generated_at`, `source` (`cache` or `fresh`), `cache_ttl_s`, an `ETag` and `Cache-Control: private, max-age=`; `If-None-Match` yields 304. Each hit is recorded under REST caller `dashboard:stage-tracker` so `/api/gh-rate-limits` shows it charged nothing. | `pkg/dashboard/api.go:184` | | `POST` | `/api/prs/{owner}/{repo}/{number}/queue-automerge` | Merger/owner role | Queue PRAuto Merge | `pkg/dashboard/api.go:185` | | `GET` | `/api/inference/models/{backend}` | Dashboard auth/session | Inference Models | `pkg/dashboard/api.go:355` | | `GET` | `/api/hive-id` | Dashboard auth/session | Hive IDGet | `pkg/dashboard/api.go:417` | @@ -593,7 +593,7 @@ always resolved server-side from the validated token. | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:275` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:276` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:277` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1525` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1529` | ## Hub SaaS diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 15b072c359..b2eb885d08 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -52376,27 +52376,27 @@

${esc(s.title)}

const liveLine = live && live.enabled ? `

Now: ${live.current} reviewer${live.current === 1 ? '' : 's'} (target ${live.target}, band ${live.min}–${live.max})${live.draining && live.draining.length ? ', draining ' + live.draining.map(esc).join(', ') : ''}; backlog ${live.backlog}${live.oldestWait ? ', oldest ' + esc(live.oldestWait) : ''}${live.lastDecision ? ' — last: ' + esc(reviewerAutoscaleDecisionText(live.lastDecision)) : ''}

` : ''; - const num = (key, label, help, step) => `
+ const num = (key, yamlPath, label, help, step) => `
`; const fields = a.enabled ? `
- ${num('min', 'Min reviewers', 'The floor of active reviewer replicas. The base reviewer agent is never retired.')} - ${num('max', 'Max reviewers', 'The ceiling of active reviewer replicas; capped at the fleet replica limit (5). Each replica is an ordinary reviewer-N agent with its own pane, token and cadence.')} - ${num('target_backlog_per_reviewer', 'Target backlog per reviewer', 'How many PRs may wait for a reviewer, per active reviewer, before the lane is behind. A PR counts as waiting when it has perspectives to review and every eligible reviewer is busy.')} + ${num('min', 'review.reviewers.autoscale.min', 'Min reviewers', 'The floor of active reviewer replicas. The base reviewer agent is never retired.')} + ${num('max', 'review.reviewers.autoscale.max', 'Max reviewers', 'The ceiling of active reviewer replicas; capped at the fleet replica limit (5). Each replica is an ordinary reviewer-N agent with its own pane, token and cadence.')} + ${num('target_backlog_per_reviewer', 'review.reviewers.autoscale.target_backlog_per_reviewer', 'Target backlog per reviewer', 'How many PRs may wait for a reviewer, per active reviewer, before the lane is behind. A PR counts as waiting when it has perspectives to review and every eligible reviewer is busy.')}
- ${num('scale_up_cycles', 'Scale-up cycles', 'Consecutive governor cycles the lane must be behind (backlog over target, or the oldest wait past the review-pending stage dwell) before a reviewer is added.')} - ${num('scale_down_cycles', 'Scale-down cycles', 'Consecutive idle cycles (nothing waiting and at least one reviewer free) before the highest replica is drained. A draining reviewer gets no new work and is retired only once its turn ends.')} + ${num('scale_up_cycles', 'review.reviewers.autoscale.scale_up_cycles', 'Scale-up cycles', 'Consecutive governor cycles the lane must be behind (backlog over target, or the oldest wait past the review-pending stage dwell) before a reviewer is added.')} + ${num('scale_down_cycles', 'review.reviewers.autoscale.scale_down_cycles', 'Scale-down cycles', 'Consecutive idle cycles (nothing waiting and at least one reviewer free) before the highest replica is drained. A draining reviewer gets no new work and is retired only once its turn ends.')}
- ${num('cpu_per_reviewer', 'CPU per reviewer (cores)', 'The CPU one reviewer is assumed to need. A scale-up is refused when the pod cgroup has less than this free; unknown limits (no cgroup) never block.', '0.1')} - ${num('mem_per_reviewer_mb', 'Memory per reviewer (MiB)', 'The memory one reviewer is assumed to need. A scale-up is refused when the pod cgroup has less than this free.')} + ${num('cpu_per_reviewer', 'review.reviewers.autoscale.cpu_per_reviewer', 'CPU per reviewer (cores)', 'The CPU one reviewer is assumed to need. A scale-up is refused when the pod cgroup has less than this free; unknown limits (no cgroup) never block.', '0.1')} + ${num('mem_per_reviewer_mb', 'review.reviewers.autoscale.mem_per_reviewer_mb', 'Memory per reviewer (MiB)', 'The memory one reviewer is assumed to need. A scale-up is refused when the pod cgroup has less than this free.')}
` : ''; - return `
+ return `
Scale reviewers with the backlog ${yamlHint('review.reviewers.autoscale.enabled')} ${settingHelpMark('review-swarm', 'reviewer-autoscale-reviewreviewersautoscale', 'Scale reviewers with the backlog', `Off by default. On, the reviewer lane grows from Min to Max replicas while PRs wait for a free reviewer, and drains back when idle. Replicas added this way are ordinary reviewer-N agents (same role, prompt, scoped token, pane and cadence as a configured replica). Scale-ups are refused when the pod cgroup lacks CPU or memory for another reviewer, when the provider budget latch is on or provider headroom is under 10%, or when reviewer kicks are failing to reach the CLI prompt (a saturated pod, not demand). Each decision is logged as "reviewer autoscale: 2→3 reason=…". Owner only.`)} @@ -52406,7 +52406,7 @@

${esc(s.title)}

} function reviewerAutoscaleDecisionText(d) { if (!d) return ''; - const when = d.at ? ' at ' + new Date(d.at).toLocaleTimeString() : ''; + const when = d.at ? ' at ' + formatDashboardTime(d.at, { hour:'2-digit', minute:'2-digit', timeZoneName:'short' }) : ''; if (d.action === 'blocked') return `${d.from}→${d.to} blocked by ${d.reason}${when}`; if (d.agent) return `${d.from}→${d.to} ${d.action} ${d.agent} (${d.reason})${when}`; return `${d.from}→${d.to} ${d.action} (${d.reason})${when}`; diff --git a/src/pkg/review/dispatch_waiting_test.go b/src/pkg/review/dispatch_waiting_test.go index 52fa0ce47c..1ff1ba2d83 100644 --- a/src/pkg/review/dispatch_waiting_test.go +++ b/src/pkg/review/dispatch_waiting_test.go @@ -19,6 +19,7 @@ func TestPlanDispatchReportsPRsWaitingForAFreeReviewer(t *testing.T) { prs := []PullRequest{dispatchPRNum(1, "sha1"), dispatchPRNum(2, "sha2"), dispatchPRNum(3, "sha3")} plan := PlanDispatch(prs, Artifact{}, DispatchState{}, DispatchOptions{ RequireApproval: true, + FanOut: true, CombinedPerspectives: true, MaxParallelReviews: 10, ProjectOrg: "acme", @@ -40,6 +41,7 @@ func TestPlanDispatchReportsPRsWaitingForAParallelSlot(t *testing.T) { prs := []PullRequest{dispatchPRNum(1, "sha1"), dispatchPRNum(2, "sha2")} plan := PlanDispatch(prs, Artifact{}, DispatchState{}, DispatchOptions{ RequireApproval: true, + FanOut: true, CombinedPerspectives: true, MaxParallelReviews: 1, ProjectOrg: "acme", @@ -57,6 +59,7 @@ func TestPlanDispatchNothingWaitingWhenReviewersCover(t *testing.T) { prs := []PullRequest{dispatchPRNum(1, "sha1")} plan := PlanDispatch(prs, Artifact{}, DispatchState{}, DispatchOptions{ RequireApproval: true, + FanOut: true, CombinedPerspectives: true, MaxParallelReviews: 10, ProjectOrg: "acme", @@ -73,6 +76,7 @@ func TestDrainingReviewerGetsNoNewReview(t *testing.T) { prs := []PullRequest{dispatchPRNum(1, "sha1"), dispatchPRNum(2, "sha2")} plan := PlanDispatch(prs, Artifact{}, DispatchState{}, DispatchOptions{ RequireApproval: true, + FanOut: true, CombinedPerspectives: true, MaxParallelReviews: 10, ProjectOrg: "acme", From 37996cd1c2e7714224b09b7ae15d1b3741725abd Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 27 Oct 2026 16:19:08 -0400 Subject: [PATCH 3/4] review autoscale: fix wait key expectation and dialog-ratchet false positive Signed-off-by: Andy Anderson --- src/pkg/dashboard/static/index.html | 2 +- src/pkg/review/dispatch_waiting_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index b2eb885d08..8bd6bc02bf 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -52399,7 +52399,7 @@

${esc(s.title)}

return `
- Scale reviewers with the backlog ${yamlHint('review.reviewers.autoscale.enabled')} ${settingHelpMark('review-swarm', 'reviewer-autoscale-reviewreviewersautoscale', 'Scale reviewers with the backlog', `Off by default. On, the reviewer lane grows from Min to Max replicas while PRs wait for a free reviewer, and drains back when idle. Replicas added this way are ordinary reviewer-N agents (same role, prompt, scoped token, pane and cadence as a configured replica). Scale-ups are refused when the pod cgroup lacks CPU or memory for another reviewer, when the provider budget latch is on or provider headroom is under 10%, or when reviewer kicks are failing to reach the CLI prompt (a saturated pod, not demand). Each decision is logged as "reviewer autoscale: 2→3 reason=…". Owner only.`)} + Scale reviewers with the backlog ${yamlHint('review.reviewers.autoscale.enabled')} ${settingHelpMark('review-swarm', 'reviewer-autoscale-reviewreviewersautoscale', 'Scale reviewers with the backlog', `Off by default. On, the reviewer lane grows from Min to Max replicas while PRs wait for a free reviewer, and drains back when idle. Replicas added this way are ordinary reviewer-N agents (same role, prompt, scoped token, pane and cadence as a configured replica). Scale-ups are refused when the pod cgroup lacks CPU or memory for another reviewer, when the provider budget latch is on or provider headroom is under 10%, or when reviewer kicks are failing to reach the CLI input prompt — a saturated pod, not demand. Each decision is logged as "reviewer autoscale: 2→3 reason=…". Owner only.`)}
${liveLine}${fields}
`; diff --git a/src/pkg/review/dispatch_waiting_test.go b/src/pkg/review/dispatch_waiting_test.go index 1ff1ba2d83..1eee7b8274 100644 --- a/src/pkg/review/dispatch_waiting_test.go +++ b/src/pkg/review/dispatch_waiting_test.go @@ -32,7 +32,7 @@ func TestPlanDispatchReportsPRsWaitingForAFreeReviewer(t *testing.T) { if len(got) != 1 || got[3] != ReviewWaitNoFreeReviewer { t.Fatalf("waiting = %+v, want PR 3 %s", plan.Waiting, ReviewWaitNoFreeReviewer) } - if key := plan.Waiting[0].Key(); key != "hive#3@sha3" { + if key := plan.Waiting[0].Key(); key != "acme/hive#3@sha3" { t.Fatalf("key = %q", key) } } From 112c27f0afac72b9cacc1c279c0270318600b6ca Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 27 Oct 2026 16:28:52 -0400 Subject: [PATCH 4/4] review autoscale: cover prober error paths and controller accessors Signed-off-by: Andy Anderson --- src/pkg/review/autoscale/controller_test.go | 104 ++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/src/pkg/review/autoscale/controller_test.go b/src/pkg/review/autoscale/controller_test.go index 5930faa9a0..e1d8f1702e 100644 --- a/src/pkg/review/autoscale/controller_test.go +++ b/src/pkg/review/autoscale/controller_test.go @@ -249,3 +249,107 @@ func TestProberReadsCgroupV2(t *testing.T) { t.Fatalf("missing cgroup should be unknown: %+v", r) } } + +func TestNilControllerAndProberAreInert(t *testing.T) { + var c *Controller + if c.Draining("reviewer") || c.DrainingNames() != nil || c.Status().Enabled { + t.Fatal("nil controller must report nothing") + } + var p *Prober + if r := p.Probe(t0); r.Known { + t.Fatalf("nil prober must be unknown: %+v", r) + } + if NewProber().root != defaultCgroupRoot { + t.Fatal("NewProber must read the default cgroup root") + } +} + +func TestReconfigureAndDrainingNamesAndPrune(t *testing.T) { + c := New(testCfg(), "reviewer") + rs := reviewers(3) + if d := drive(c, 0, 3, rs, 0, nil); d.Action != ActionDrain || d.Agent != "reviewer-3" { + t.Fatalf("drain: %+v", d) + } + if got := c.DrainingNames(); len(got) != 1 || got[0] != "reviewer-3" { + t.Fatalf("draining names = %v", got) + } + st := c.Status() + if st.Draining == nil || st.LastDecision == nil || st.LastDecision.Action != ActionDrain { + t.Fatalf("status = %+v", st) + } + if s := st.LastDecision.String(); s != "reviewer autoscale: 3→2 drain reviewer-3 reason=idle for 3 cycles" { + t.Fatalf("drain string = %q", s) + } + + cfg := testCfg() + cfg.Max = 2 + c.Reconfigure(cfg, "reviewer-x") + if st := c.Status(); st.Max != 2 || st.Agent != "reviewer-x" { + t.Fatalf("reconfigure not reflected: %+v", st) + } + + // The draining replica disappears from the roster (an operator removed + // it); the controller forgets it rather than retiring a ghost. + if d := drive(c, 3, 1, reviewers(2), 0, nil); d.Action == ActionRetire { + t.Fatalf("retired a reviewer that is not present: %+v", d) + } + if c.Draining("reviewer-3") { + t.Fatal("absent reviewer should be pruned from draining") + } +} + +func TestDecisionStringVariants(t *testing.T) { + cases := map[Decision]string{ + {Action: ActionHold, From: 2, Reason: "idle"}: "reviewer autoscale: hold at 2 reason=idle", + {Action: ActionBlocked, From: 2, To: 3, Reason: GuardCPU}: "reviewer autoscale: 2→3 blocked by cpu", + {Action: ActionRetire, From: 3, To: 2, Agent: "reviewer-3", Reason: "idle"}: "reviewer autoscale: 3→2 retire reviewer-3 reason=idle", + {Action: ActionScaleUp, From: 2, To: 3, Reason: "backlog=7"}: "reviewer autoscale: 2→3 reason=backlog=7", + } + for d, want := range cases { + if got := d.String(); got != want { + t.Errorf("%+v => %q, want %q", d, got, want) + } + } +} + +func TestProberToleratesMalformedCgroupFiles(t *testing.T) { + dir := t.TempDir() + write := func(name, body string) { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + // "max" quota, unparsable stat, bad memory values: nothing known, nothing crashes. + write(cpuMaxFile, "max 100000\n") + write(cpuStatFile, "usage_usec notanumber\n") + write(memMaxFile, "lots\n") + write(memCurrentFile, "many\n") + if r := NewProberAt(dir).Probe(t0); r.Known || r.CPUUsed != 0 || r.MemUsedMB != 0 { + t.Fatalf("malformed cgroup should be unknown: %+v", r) + } + write(cpuMaxFile, "abc 100000\n") + write(cpuStatFile, "nr_periods 5\n") + if r := NewProberAt(dir).Probe(t0); r.Known { + t.Fatalf("unparsable quota should be unknown: %+v", r) + } + write(cpuMaxFile, "100000 0\n") + if r := NewProberAt(dir).Probe(t0); r.Known { + t.Fatalf("zero period should be unknown: %+v", r) + } + write(cpuMaxFile, "400000\n") + if r := NewProberAt(dir).Probe(t0); r.Known { + t.Fatalf("single field should be unknown: %+v", r) + } + // A usage counter that goes backwards clamps to zero rather than going negative. + write(cpuMaxFile, "400000 100000\n") + write(cpuStatFile, "usage_usec 5000000\n") + p := NewProberAt(dir) + p.Probe(t0) + write(cpuStatFile, "usage_usec 1000000\n") + if r := p.Probe(t0.Add(time.Second)); r.CPUUsed != 0 || r.CPULimit != 4 { + t.Fatalf("backwards counter: %+v", r) + } + if r := p.Probe(t0.Add(time.Second)); r.CPUUsed != 0 { + t.Fatalf("zero wall time must not divide: %+v", r) + } +}