diff --git a/changelog.d/changed-15585-main-split.md b/changelog.d/changed-15585-main-split.md new file mode 100644 index 0000000000..88131e6a91 --- /dev/null +++ b/changelog.d/changed-15585-main-split.md @@ -0,0 +1 @@ +- refactor: split `src/cmd/hive/main.go` into 4 files (`main.go`, `main_github_app.go`, `main_eval_cycle.go`, `main_sweeps.go`) to cut rebase churn on a merge hotspot; no behaviour change ([#15585](https://github.com/hivecommons/hive/issues/15585)). diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index b601143429..d5ad531f8e 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -4,17 +4,12 @@ import ( "context" "crypto/rand" "encoding/json" - "errors" "fmt" "io" - "io/fs" "log/slog" - "net/http" "net/url" "os" "path/filepath" - "regexp" - "slices" "sort" "strconv" "strings" @@ -30,12 +25,10 @@ import ( "github.com/hivecommons/hive/crux/inference" "github.com/hivecommons/hive/crux/logscrub" "github.com/hivecommons/hive/crux/proxy" - "github.com/hivecommons/hive/crux/toolapprove" "github.com/hivecommons/hive/crux/trajectory" "github.com/hivecommons/hive/crux/watchdog" "github.com/hivecommons/hive/crux/watsonx" "github.com/hivecommons/hive/pkg/advisory" - "github.com/hivecommons/hive/pkg/apphealth" "github.com/hivecommons/hive/pkg/beads" "github.com/hivecommons/hive/pkg/celtrigger" "github.com/hivecommons/hive/pkg/classify" @@ -52,7 +45,6 @@ import ( "github.com/hivecommons/hive/pkg/extwork" "github.com/hivecommons/hive/pkg/fleetreport" "github.com/hivecommons/hive/pkg/fleetreport/slogfeed" - "github.com/hivecommons/hive/pkg/forge" "github.com/hivecommons/hive/pkg/forgestate" "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/github/automerge" @@ -64,8 +56,6 @@ import ( "github.com/hivecommons/hive/pkg/ioscan" "github.com/hivecommons/hive/pkg/jev" "github.com/hivecommons/hive/pkg/knowledge" - "github.com/hivecommons/hive/pkg/loginscan" - "github.com/hivecommons/hive/pkg/mint" "github.com/hivecommons/hive/pkg/notify" "github.com/hivecommons/hive/pkg/planning" "github.com/hivecommons/hive/pkg/policies" @@ -79,16 +69,12 @@ import ( "github.com/hivecommons/hive/pkg/scheduler" "github.com/hivecommons/hive/pkg/sessionprune" "github.com/hivecommons/hive/pkg/snapshot" - "github.com/hivecommons/hive/pkg/spokealerts" "github.com/hivecommons/hive/pkg/stageengine" "github.com/hivecommons/hive/pkg/stageengine/prstage" - "github.com/hivecommons/hive/pkg/stagelabels" - "github.com/hivecommons/hive/pkg/timeline" "github.com/hivecommons/hive/pkg/tokens" "github.com/hivecommons/hive/pkg/tracing" "github.com/hivecommons/hive/pkg/worksource" "github.com/hivecommons/hive/pkg/worksource/wavefront" // registers the additive Wavefront source (#8362) - "go.opentelemetry.io/otel/attribute" "gopkg.in/natefinch/lumberjack.v2" ) @@ -400,392 +386,6 @@ func agentCanProduceJudgedOutput(acmmLevel int, a spoke.AgentSummary) bool { } } -// prospectiveGitHubIdentity returns the GitHub identity the spoke WOULD hold -// after adopting ghCfg, or nil when the push speaks to no identity field and -// there is nothing to validate. -// -// It mirrors the adoption rules in the GitHubAppConfigCallback exactly — a -// zero app_id and an empty app_slug both mean "not speaking to this field", and -// the placeholder sentinel is never adopted over a real App. Mirroring rather -// than validating ghCfg alone is what makes the check correct: the damaging -// state is a combination of PUSHED and EXISTING fields (a GHE app_id landing -// beside the spoke's own empty api_url), and validating the push in isolation -// cannot see it. -func prospectiveGitHubIdentity(cur config.GitHubConfig, ghCfg *spoke.HeartbeatGitHubAppConfig) *config.GitHubConfig { - if ghCfg == nil { - return nil - } - touched := false - next := cur - if ghCfg.AppID != 0 && ghCfg.AppID != config.PlaceholderAppID { - next.AppID = ghCfg.AppID - touched = true - } - if ghCfg.AppSlug != "" && ghCfg.AppSlug != cur.AppSlug { - next.AppSlug = ghCfg.AppSlug - touched = true - } - // The forge URLs are part of the SAME set as the App above, so they are - // adopted here and validated with it rather than arriving separately on the - // project-config channel. An App ID presented to the wrong forge returns - // "404 Integration not found", so applying one half without the other is the - // live failure this function exists to prevent. - // - // Empty means "unchanged", matching AppSlug. It cannot mean "make me - // public": empty URLs are also the correct steady state for a public hive - // (~41 of 50 spokes), so silence here is indistinguishable from "no opinion" - // and must never blank a working GHE URL. A hive moving TO public gets that - // from its app_id, which the resolver derives from its forge. - if ghCfg.APIURL != "" && ghCfg.APIURL != cur.APIURL { - next.APIURL = ghCfg.APIURL - touched = true - } - if ghCfg.BaseURL != "" && ghCfg.BaseURL != cur.BaseURL { - next.BaseURL = ghCfg.BaseURL - touched = true - } - if !touched { - return nil - } - return &next -} - -// nextInstallationID decides what a hive's installation_id becomes after a -// hub delivery, and reports whether the change is an operator RESET. -// -// Three cases, and the difference between the last two is load-bearing: -// -// ResetInstallation -> 0. The operator clicked "Reset App". Clearing makes -// HasUsableApp() false, which raises githubAppRequired: -// the owner is prompted to install the App again and the -// self-heal ticker starts, whose RediscoverAndAdopt -// adopts the correct installation for whatever they -// install. -// non-zero pushed -> adopt it. -// zero pushed -> KEEP the current value. Zero means "the hub is not -// speaking to this field", not "clear it". The -// cluster-wide key reconcile sends zero on every beat -// because it repairs KEYS on hives whose installation the -// hub does not track; reading that as a clear would blank -// a working installation fleet-wide and turn a key-only -// fault into a total auth outage. -// -// docs_installation_id is DELIBERATELY untouched by all three cases. It is the -// optional docs-org add-on installation, has no rediscovery flow (zero simply -// disables the docs token refresh, permanently), and a stale value is -// non-fatal: the periodic docs mint warns and retries. After an app-changing -// delivery it can therefore briefly equal the PREVIOUS app's installation id — -// which looks like the old installation_id was "parked" there, but is just the -// provisioned docs value (public hives commonly provision both fields with the -// same installation) surviving a reset that correctly cleared only -// installation_id. On a flip-back to the original App it becomes valid again -// on its own. -func nextInstallationID(current int64, ghCfg *spoke.HeartbeatGitHubAppConfig) (next int64, reset bool) { - if ghCfg == nil { - return current, false - } - if ghCfg.ResetInstallation { - return 0, current != 0 - } - if ghCfg.InstallationID != 0 { - return ghCfg.InstallationID, false - } - return current, false -} - -var githubAppTokenCachePath = github.TokenCachePath - -// resumeKickHeldAlerts tracks crash-restarted agents the resume-kick gate -// refused, so the eval loop can raise and later clear their dashboard alert. -var resumeKickHeldAlerts = spokealerts.NewResumeKickHeld() - -func githubAppTokenHeartbeatFields(cfg *config.Config, detail string) (status, lastMintAt, lastErr string) { - if cfg == nil || !cfg.GitHub.HasApp() { - return "", "", "" - } - info, err := os.Stat(githubAppTokenCachePath) - if err != nil { - if os.IsNotExist(err) { - return spoke.GitHubAppTokenStatusMissing, "", detail - } - return spoke.GitHubAppTokenStatusError, "", err.Error() - } - lastMintAt = info.ModTime().UTC().Format(time.RFC3339) - if time.Since(info.ModTime()) > spoke.GitHubAppTokenStaleAfter { - return spoke.GitHubAppTokenStatusStale, lastMintAt, detail - } - return spoke.GitHubAppTokenStatusOK, lastMintAt, "" -} - -var githubHTTPStatusRe = regexp.MustCompile(`\b([1-5][0-9]{2})\b`) - -func firstNonEmpty(vals ...string) string { - for _, v := range vals { - if strings.TrimSpace(v) != "" { - return v - } - } - return "" -} - -func githubAppStructuredFailure(state, detail string) (class string, httpStatus int) { - switch strings.TrimSpace(state) { - case github.AppStateNotInstalled.String(): - class = "not-installed" - case github.AppStateWrongInstallation.String(): - class = "wrong-installation" - case github.AppStateInsufficientPerms.String(): - class = "insufficient-permissions" - case github.AppStateKeyMissing.String(): - class = "key-missing" - case github.AppStateKeyInvalid.String(): - class = "key-invalid" - case github.AppStateNoAppAssigned.String(): - class = "no-app-assigned" - case github.AppStateRepoNotCovered.String(): - class = "repo-not-covered" - case github.AppStateRepoMoved.String(): - class = "repo-moved" - case github.AppStateWriteForbidden.String(): - class = "write-forbidden" - } - if class == "" && strings.TrimSpace(detail) != "" { - class = "token-error" - } - for _, m := range githubHTTPStatusRe.FindAllStringSubmatch(detail, -1) { - if len(m) == 2 { - if n, err := strconv.Atoi(m[1]); err == nil { - httpStatus = n - } - } - } - if httpStatus != 0 { - if (class == "token-error" || class == "not-installed") && httpStatus == http.StatusNotFound { - class = "installation-not-found" - } - } - return class, httpStatus -} - -// githubAuth is the outcome of resolving this hive's GitHub credentials at -// startup. Every field is optional: a hive with no usable credentials is a -// legitimate, bootable state. -type githubAuth struct { - // Client is nil when no credentials could be resolved. Callers must treat a - // nil Client as "GitHub is unavailable", never as a fatal condition. - Client *github.Client - // AppAuth is non-nil only when App auth was successfully initialized. - AppAuth *github.AppAuth - // Failure, when non-empty, is the operator-facing reason there is no - // working GitHub client. It is shown in the dashboard's GitHub App banner. - Failure string - // State classifies Failure so the banner and the hub's journey nudges can - // tell an operator-side fault (a key that was never delivered) from a - // user-actionable one (the App is not installed). Escalating against an - // owner for a key WE failed to provision is precisely the mistake - // github.AppAuthState exists to prevent. - State github.AppAuthState - // TokenScopes is the boot-time PAT scope probe (see - // github.CheckTokenScopes). Zero value / ScopeStatusSkipped on the App - // path. It is advisory: a ScopeStatusMissing result never blocks startup, - // it only gives github_auth a specific detail string instead of leaving the - // operator to decode a runtime 403. - TokenScopes github.ScopeResult -} - -// initGitHubAuth resolves this hive's GitHub credentials. -// -// It NEVER exits the process. A hive that cannot authenticate to GitHub must -// still boot and serve its dashboard, because the dashboard is the only place -// its owner can see what is wrong and fix it. Exiting here — which is what this -// code used to do on a key-read failure — happens before the HTTP listener -// binds, so the pod crashloops with the diagnosis visible only in kubectl logs: -// the hub shows the hive offline, the heartbeat never starts, and a rollout -// hangs forever because the new pod never goes Ready. -// -// The placeholder app_id is the reason that path was reachable at all. See -// config.PlaceholderAppID. -func initGitHubAuth(ctx context.Context, cfg *config.Config, logger *slog.Logger) githubAuth { - var out githubAuth - appKeyFile := appKeys.Resolve(cfg.GitHub.KeyFile, os.Getenv("GH_APP_KEY_FILE"), cfg.GitHub.AppID) - - // HasApp() rejects config.PlaceholderAppID. Build AppAuth as soon as a real - // app_id and key are present, even when installation_id is still empty: the - // App JWT is exactly what automatic installation-ID discovery needs. - if cfg.GitHub.HasApp() { - appAuth, err := github.NewAppAuth(cfg.GitHub.AppID, cfg.GitHub.InstallationID, appKeyFile, logger, cfg.GitHub.ResolvedAPIURL()) - if err != nil { - // A genuinely-configured App whose key is missing or malformed is a - // real, actionable fault — but not a reason to refuse to boot. - out.Failure = describeAppKeyFailure(cfg.GitHub.KeyFile, os.Getenv("GH_APP_KEY_FILE"), appKeyFile, err) - // Both states are operator-actionable: the hive's owner cannot - // deliver a key. Absent vs. unparseable is the distinction the hub - // needs to tell "never pushed" from "pushed something broken". - out.State = github.AppStateKeyInvalid - if errors.Is(err, fs.ErrNotExist) { - out.State = github.AppStateKeyMissing - } - logger.Error("GitHub App auth unavailable — starting in dashboard-only mode", - "app_id", cfg.GitHub.AppID, - "installation_id", cfg.GitHub.InstallationID, - "key_file", appKeyFile, - "state", out.State.String(), - "detail", out.Failure, - "error", err, - ) - } else { - out.AppAuth = appAuth - } - } - - if out.AppAuth != nil { - logger.Info("using GitHub App authentication", "app_id", cfg.GitHub.AppID) - if cfg.GitHub.InstallationID == 0 { - out.Failure = "The GitHub App is configured but has no installation. Install the app on your org; this hive will discover the installation automatically." - out.State = github.AppStateNotInstalled - logger.Warn("GitHub App configured without installation_id — starting in dashboard-only mode while auto-discovery polls") - return out - } - // Correct a stale/wrong installation_id BEFORE building the client, so - // the very first token this process mints is scoped to the right org - // rather than 403ing on every write until the self-heal tick runs. - healGitHubAppInstallation(ctx, out.AppAuth, cfg, logger) - // Hooks (hold labels, #6203 repo pause, #6204 agent scope, and the - // rest) are installed by bootGitHub through configureGitHubClient, the - // same function every rebuild path uses (#9614). - out.Client = github.NewClientFromAppWithBotLogin(out.AppAuth, cfg.Project.Org, cfg.Project.Repos, logger, cfg.GitHub.BotLogin()) - startDocsTokenRefresh(ctx, cfg, appKeyFile, logger) - return out - } - - ghToken := cfg.GitHub.Token - if ghToken == "" { - ghToken = os.Getenv("HIVE_GITHUB_TOKEN") - } - switch { - case ghToken != "": - out.Client = github.NewClient(ghToken, cfg.Project.Org, cfg.Project.Repos, logger, cfg.GitHub.ResolvedAPIURL()) // hooks: see the App branch above - // PAT path only: introspect the token's granted scopes ONCE, here, so a - // too-narrow token is named at boot instead of surfacing hours later as - // a generic 403 inside an agent — or, worse, as an empty backlog that - // looks like "no work to do". Fail-soft and bounded (see - // CheckTokenScopes); it never blocks or fails startup. The App branch - // above returns before this point: Apps have permissions, not scopes. - // An unset acmm_level is passed through as github.ACMMLevelUnset rather - // than inferACMMLevel's L1 default: L1 requires no scopes at all, so - // defaulting to it would silently suppress every warning on exactly the - // hives whose intent we cannot read. See ACMMLevelUnset. - scopeLevel := github.ACMMLevelUnset - if cfg.ACMMLevel != nil { - scopeLevel = *cfg.ACMMLevel - } - out.TokenScopes = out.Client.LogTokenScopeCheck(ctx, logger, scopeLevel) - case out.Failure != "": - // Real App, unusable key. Already logged; leave Client nil so nothing - // tries to act on GitHub with credentials that do not work. - case cfg.GitHub.IsPlaceholderApp(): - // OPERATOR-actionable, not user-actionable. This hive was provisioned as - // a placeholder and never assigned a real app_id, so the JWT it signs - // names no App and fails before any installation is consulted. Nothing - // the owner enters in the installation-ID box can fix it — reporting this - // as AppStateNotInstalled sent owners to re-install an App that was - // already installed and to re-enter an installation_id that was already - // correct. - out.Failure = "This hive was never assigned a GitHub App ID: it still carries the placeholder github.app_id, " + - "which does not name a real GitHub App. Setting an installation ID cannot resolve this — " + - "the hub operator must assign the real App ID." - out.State = github.AppStateNoAppAssigned - logger.Warn("placeholder github.app_id — hive starting in dashboard-only mode", - "placeholder_app_id", config.PlaceholderAppID, - "installation_id", cfg.GitHub.InstallationID, - ) - case cfg.GitHub.AppID != 0: - out.Failure = "The GitHub App is configured but has no installation. Install the app on your org to enable agents." - out.State = github.AppStateNotInstalled - logger.Warn("GitHub App configured without credentials — hive starting in dashboard-only mode. Install the app and provide installation_id + key to enable agents.") - default: - // Neither a token nor any app_id at all. config.validate() rejects this - // at load, so reaching it means the config was mutated afterwards. - // Still a degraded boot rather than an exit: the dashboard is where an - // operator fixes it. - out.Failure = "No GitHub credentials configured. Set github.token, or github.app_id plus an App installation." - logger.Error("no GitHub token configured (set github.token or github.app_id in config) — starting in dashboard-only mode") - } - return out -} - -// startDocsTokenRefresh mints and periodically refreshes a token for the -// separate docs-org installation, when one is configured. A failure here is -// always non-fatal: the docs org is an add-on, not this hive's primary auth. -func startDocsTokenRefresh(ctx context.Context, cfg *config.Config, appKeyFile string, logger *slog.Logger) { - if cfg.GitHub.DocsInstallationID == 0 { - return - } - docsAuth, err := github.NewAppAuthWithCache( - cfg.GitHub.AppID, cfg.GitHub.DocsInstallationID, - appKeyFile, github.DocsTokenCachePath, logger, cfg.GitHub.ResolvedAPIURL(), - ) - if err != nil { - logger.Warn("failed to init docs org token", "error", err) - return - } - go func() { - // Mint the initial docs token in the BACKGROUND, not on the startup - // path. The docs org is an add-on; blocking here on a docs-installation - // mint that hangs (unreachable/uninstalled GHE) would delay the whole - // process reaching MarkReady — the #2439 readiness-stall pattern. The - // mint is bounded (tokenMintTimeout) and non-fatal regardless. - if _, err := docsAuth.Token(ctx); err != nil { - logger.Warn("failed to generate initial docs org token", "error", err) - } else { - logger.Info("docs org token cached", "installation_id", cfg.GitHub.DocsInstallationID) - } - const docsTokenRefreshInterval = 45 * time.Minute - ticker := time.NewTicker(docsTokenRefreshInterval) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - if _, err := docsAuth.Token(ctx); err != nil { - logger.Warn("docs token refresh failed", "error", err) - } - } - } - }() -} - -// buildAgentMinter constructs the opt-in per-agent mint credential issuer from -// config. It loads (or creates) the signing key at cfg.Mint.KeyPath, builds a -// Minter with the configured issuer/hive-id/TTL, and wraps it as an AgentMinter. -// Callers gate on cfg.Mint.Enabled before calling. The signing key path comes -// from config (never hardcoded); a missing key file is created with 0600 perms -// by the mint package. -func buildAgentMinter(cfg *config.Config, logger *slog.Logger) (*mint.AgentMinter, error) { - if cfg.Mint.KeyPath == "" { - return nil, fmt.Errorf("mint.key_path is required when mint is enabled") - } - if cfg.Mint.Issuer == "" { - return nil, fmt.Errorf("mint.issuer is required when mint is enabled") - } - key, err := mint.LoadOrCreateKey(cfg.Mint.KeyPath) - if err != nil { - return nil, fmt.Errorf("loading mint signing key: %w", err) - } - maxTTL := time.Duration(cfg.Mint.MaxTTLSeconds) * time.Second - minter, err := mint.NewMinter(key, cfg.Mint.Issuer, - mint.WithHiveID(cfg.HiveID), - mint.WithMaxTTL(maxTTL), - ) - if err != nil { - return nil, fmt.Errorf("building minter: %w", err) - } - logger.Debug("mint signing key loaded", "key_path", cfg.Mint.KeyPath) - // ttl<=0 lets the minter fall back to its configured max on each Mint. - return mint.NewAgentMinter(minter, maxTTL), nil -} - // reporterName identifies this spoke PROCESS to the hub (HeartbeatPayload // Reporter) as "/". In-cluster the hostname IS the pod name; // the PID suffix exists because two hive processes were observed beating from @@ -6583,4868 +6183,413 @@ func (b *boot) runLoopWith(deps runLoopDeps) { } -// buildRepoActivityWire maps the dashboard activity collector's per-repo -// snapshot into the plain hub wire structs the heartbeat carries. Kept here (in -// the one package that imports both hub and dashboard) so pkg/hub never has to -// import pkg/dashboard back — that would be an import cycle, since dashboard -// already imports hub. A field-by-field copy, mirroring how the fleet-stat -// scalars are lifted out of their snapshot at the beat's build site. -func buildRepoActivityWire(repos []collect.RepoActivity) []spoke.RepoActivityWire { - if len(repos) == 0 { - return nil - } - stat := func(s collect.ActivityActionStat) spoke.ActivityStatWire { - return spoke.ActivityStatWire{Count: s.Count, NewestAt: s.NewestAt} - } - out := make([]spoke.RepoActivityWire, 0, len(repos)) - for _, r := range repos { - agents := make([]spoke.AgentRepoActivityWire, 0, len(r.Agents)) - for _, a := range r.Agents { - agents = append(agents, spoke.AgentRepoActivityWire{ - Agent: a.Agent, - Issues: stat(a.Issues), - PRs: stat(a.PRs), - Comments: stat(a.Comments), - Merges: stat(a.Merges), - Claims: stat(a.Claims), - Reviews: stat(a.Reviews), - Advisory: stat(a.Advisory), - Reconciled: stat(a.Reconciled), - }) +func convertKnowledgeLayers(cfgLayers []config.KnowledgeLayer) []knowledge.LayerConfig { + layers := make([]knowledge.LayerConfig, len(cfgLayers)) + for i, l := range cfgLayers { + layers[i] = knowledge.LayerConfig{ + Type: knowledge.LayerType(l.Type), + Path: l.Path, + URL: l.URL, + Shared: l.Shared, } - out = append(out, spoke.RepoActivityWire{ - Repo: r.Repo, - Issues: stat(r.Issues), - PRs: stat(r.PRs), - Comments: stat(r.Comments), - Merges: stat(r.Merges), - Claims: stat(r.Claims), - Reviews: stat(r.Reviews), - Advisory: stat(r.Advisory), - Reconciled: stat(r.Reconciled), - Agents: agents, - }) } - return out -} - -// providerBudgetNotify is the one-shot guard for the provider spend-rebuff -// notification (#4294). Package-level because runEvalCycle is a function called -// once per tick with no state of its own; mutex-guarded inside pkg/governor. -var providerBudgetNotify governor.ProviderBudgetNotifyState - -// providerBudgetProbe remembers when the last probe kick was released while a -// provider spend rebuff is latched. Package-level for the same reason as -// providerBudgetNotify: runEvalCycle has no state of its own. -var providerBudgetProbe governor.ProviderBudgetProbeState - -// governorKickFairness remembers, per agent, how many cycles in a row the -// governor kick was deferred behind a delivered review-fix kick -// (hivecommons/hive#15166). Package-level for the same reason as -// providerBudgetProbe: runEvalCycle has no state of its own. -var governorKickFairness = newKickLaneFairness() - -var evalAPIBudgetCycle atomic.Uint64 - -// applyBudgetAlerts is a thin wrapper around spokealerts.ApplyBudget, kept so -// call sites in this file do not need the package-qualified name (same -// pattern as applyModeUnscheduledAlert below). -func applyBudgetAlerts(gov *governor.Governor, trans governor.BudgetTransitions, dashSrv *dashboard.Server, notifier *notify.Notifier) { - spokealerts.ApplyBudget(gov, trans, dashSrv, notifier) -} - -// applyNoCadenceAlert is a thin wrapper around spokealerts.ApplyNoCadence -// (see that package for the banner's design: #5577). -func applyNoCadenceAlert(gov *governor.Governor, dashSrv *dashboard.Server) { - spokealerts.ApplyNoCadence(gov, dashSrv) -} - -func applyModeUnscheduledAlert(gov *governor.Governor, dashSrv *dashboard.Server) { - spokealerts.ApplyModeUnscheduled(gov, dashSrv) + return layers } -func activeIssueRefsFromLastActionable(actionable *github.ActionableResult) (map[string]struct{}, map[string]struct{}) { - if actionable == nil { - return nil, nil - } - refs := make(map[string]struct{}) - coveredRepos := make(map[string]struct{}) - for repo := range actionable.TotalByRepo { - coveredRepos[repo] = struct{}{} - } - for _, issue := range actionable.Issues.Items { - if ref := planning.IssueRef(issue); ref != "" { - refs[ref] = struct{}{} - } - } - for _, held := range actionable.Hold.Items { - if held.Type != "issue" || held.Repo == "" || held.Number <= 0 { - continue - } - ref := planning.IssueRef(github.Issue{Repo: held.Repo, Number: held.Number, Title: held.Title}) - if ref != "" { - refs[ref] = struct{}{} - } +// curatorConfigFromHive maps the hive.yaml curator block onto the knowledge +// package's own config. Enabled is carried across as a pointer so "absent" +// stays distinguishable from "explicitly false" — the scheduled promotion loop +// treats absent as OFF, and flattening it to a bool here would quietly turn +// unreviewed promotion on fleet-wide (#5430). +func curatorConfigFromHive(c config.KnowledgeCurator) knowledge.CuratorConfig { + return knowledge.CuratorConfig{ + Enabled: c.Enabled, + Schedule: c.Schedule, + ExtractFrom: c.ExtractFrom, + AutoPromoteThreshold: c.AutoPromoteThreshold, + PromoteFrom: c.PromoteFrom, + PromoteTo: c.PromoteTo, } - return refs, coveredRepos } -// agentKicker adapts crux.Runtime to planning.Kicker for the Phase 3 -// stall-replan lane. Kick delegates to SendKick, which takes the manager lock -// ITSELF and is only ever called here from the governor tick (never from the -// agent-launch path), so it cannot re-enter a held manager lock. This is the -// same out-of-band kick path the eval loop already uses for governor kicks. -type agentKicker struct{ mgr crux.Runtime } - -func (k agentKicker) Kick(agent, message string) error { - if k.mgr == nil { - return fmt.Errorf("agent manager unavailable: cannot kick %s", agent) - } - return k.mgr.SendKick(agent, message) -} +// hiveIDFilePath is the persistent file where the Hive ID is stored across restarts. +var hiveIDFilePath = "/data/hive-id" -// planFromLabeledIssues is Phase 4 Part B: for each actionable issue carrying a -// `plan`/`epic` label, mint an epic (idempotent) and hand it to the architect, -// RESPECTING the architect's pause. It is a plain synchronous call on the eval -// tick — no goroutine — and only ever touches the manager via SendKick/IsPaused, -// exactly like every governor kick, so it never re-enters the launch-path mutex. -// Epics are minted into the architect store (falling back to any store) so the -// dashboard plan-review flow and replan lane find them the same way. -// -// A minted epic is decompose_pending (plan_status=draft). While the architect is -// paused, the request stays queued and visible (the PLANNING tile shows it as -// pending) — we never force-unpause. Once the architect is available, we kick it -// each cycle until it decomposes and clears the pending marker. -func planFromLabeledIssues( - actionable *github.ActionableResult, - beadStores map[string]*beads.Store, - agentMgr crux.Runtime, - gov *governor.Governor, - dashSrv *dashboard.Server, - logger *slog.Logger, - cfg *config.Config, - acmmLevel int, -) { - if actionable == nil || len(beadStores) == 0 { - return - } - store, ok := beadStores[planning.ArchitectAgentName] - if !ok { - for name := range beadStores { - store = beadStores[name] - break +func loadOrGenerateHiveID(logger *slog.Logger) string { + if envID := os.Getenv("HIVE_ID"); envID != "" { + if err := os.WriteFile(hiveIDFilePath, []byte(envID+"\n"), 0o644); err == nil { + logger.Info("hive ID set from HIVE_ID env var", "id", envID) } - } - if store == nil { - return + return envID } - designCfg := planning.DesignConfig{ - PlanLabels: cfg.Planning.PlanLabelsOrDefault(), - DesignLabels: cfg.Planning.DesignLabelsOrDefault(), - ApprovedLabel: cfg.Planning.DesignApprovedLabelOrDefault(), - MaxRevisions: cfg.Planning.MaxDesignRevisionsOrDefault(), - MaxConcurrent: cfg.Planning.MaxConcurrentDesignsOrDefault(), - } - issues := actionable.Issues.Items - if cfg == nil || !cfg.Planning.PlanFromLabelEnabled(acmmLevel) { - issues = nil - for _, issue := range actionable.Issues.Items { - if store.FindByExternalRef(planning.IssueRef(issue)) != nil { - issues = append(issues, issue) - } - } - } - sink := labelPlanSink{gov: gov, dashSrv: dashSrv, logger: logger} - planning.PlanIssuesFromLabelsWithConfig(store, agentMgr, issues, designCfg, sink, - func(ref string, err error) { - logger.Warn("plan-from-label: minting epic failed", "issue", ref, "error", err) - }, acmmLevel) - if config.PlanAutoApproveForLevel(acmmLevel) { - for _, id := range planning.AutoApproveDrafts(store) { - if dashSrv != nil { - dashSrv.AuditLog("planning", "plan_auto_approved", "epic="+id, planning.ArchitectAgentName) - } - logger.Info("audit: plan auto-approved by ACMM pack", "epic", id, "acmm_level", acmmLevel) + if data, err := os.ReadFile(hiveIDFilePath); err == nil { + id := strings.TrimSpace(string(data)) + if id != "" { + logger.Info("hive ID loaded from disk", "id", id) + return id } } -} - -// labelPlanSink adapts the governor/dashboard/logger to planning.LabelPlanSink so -// the label-trigger core lives (and is tested) in pkg/planning. -type labelPlanSink struct { - gov *governor.Governor - dashSrv *dashboard.Server - logger *slog.Logger -} -func (s labelPlanSink) KickedPlan(epic *beads.Bead) { - s.gov.RecordKick(planning.ArchitectAgentName) - if s.dashSrv != nil { - s.dashSrv.AuditLog("planning", "plan_from_label", "epic="+epic.ID+" ref="+epic.ExternalRef, planning.ArchitectAgentName) - } - s.logger.Info("audit: plan requested from labeled issue", "epic", epic.ID, "ref", epic.ExternalRef) -} + id := "hive-" + randomName() -func (s labelPlanSink) FailedPlan(epic *beads.Bead) { - if s.dashSrv != nil { - s.dashSrv.AuditLog("planning", "plan_decompose_failed", "epic="+epic.ID+" ref="+epic.ExternalRef+" attempts="+strconv.Itoa(planning.DecomposeMaxAttempts), planning.ArchitectAgentName) + if err := os.WriteFile(hiveIDFilePath, []byte(id+"\n"), 0o644); err != nil { + logger.Warn("failed to persist hive ID", "error", err) + } else { + logger.Info("generated new hive ID", "id", id) } - s.logger.Warn("plan-from-label: architect produced no plan after max attempts — epic marked stuck; re-request it from the dashboard", - "epic", epic.ID, "ref", epic.ExternalRef, "attempts", planning.DecomposeMaxAttempts) -} -func (s labelPlanSink) QueuedPlan(epic *beads.Bead, paused bool) { - if paused { - // Architect deliberately paused — queue, do not unpause, log once. - s.logger.Info("plan-from-label: architect paused, plan queued", "epic", epic.ID, "ref", epic.ExternalRef) - return - } - s.logger.Warn("plan-from-label: architect unavailable, plan queued", "epic", epic.ID, "ref", epic.ExternalRef) + return id } -func (s labelPlanSink) KickedDesign(epic *beads.Bead, revision int) { - s.gov.RecordKick(planning.ArchitectAgentName) - if s.dashSrv != nil { - s.dashSrv.AuditLog("planning", "design_kicked", "epic="+epic.ID+" ref="+epic.ExternalRef+" revision="+strconv.Itoa(revision), planning.ArchitectAgentName) +// randomName generates a Docker-style adjective-noun name. +func randomName() string { + adjectives := []string{ + "bold", "calm", "cool", "dark", "deep", "fair", "fast", "keen", + "kind", "loud", "mild", "neat", "pale", "pure", "rare", "rich", + "safe", "slim", "soft", "tall", "thin", "true", "vast", "warm", + "wise", "able", "busy", "easy", "epic", "free", "glad", "good", + "idle", "just", "lazy", "lean", "live", "long", "lost", "main", + "next", "open", "real", "sure", "wild", "worn", "zero", "blue", } - s.logger.Info("audit: design requested from labeled issue", "epic", epic.ID, "ref", epic.ExternalRef, "revision", revision) -} - -func (s labelPlanSink) ApprovedDesign(epic *beads.Bead) { - if s.dashSrv != nil { - s.dashSrv.AuditLog("planning", "design_approved", "epic="+epic.ID+" ref="+epic.ExternalRef, planning.ArchitectAgentName) + nouns := []string{ + "ant", "ape", "bat", "bee", "cow", "doe", "eel", "elk", + "fox", "gnu", "hen", "jay", "kit", "lark", "moth", "newt", + "owl", "pug", "ram", "ray", "seal", "swan", "toad", "wren", + "bear", "colt", "crow", "deer", "dove", "duck", "fawn", "frog", + "goat", "gull", "hare", "hawk", "ibis", "lynx", "mink", "mole", + "orca", "pike", "puma", "slug", "stag", "wolf", "yak", "wasp", } - s.logger.Info("audit: design approved from labeled issue", "epic", epic.ID, "ref", epic.ExternalRef) -} -func (s labelPlanSink) DesignNeedsHuman(epic *beads.Bead, revisions int) { - if s.dashSrv != nil { - s.dashSrv.AuditLog("planning", "design_needs_human", "epic="+epic.ID+" ref="+epic.ExternalRef+" revisions="+strconv.Itoa(revisions), planning.ArchitectAgentName) + buf := make([]byte, 2) + if _, err := rand.Read(buf); err != nil { + return "bold-ant" } - s.logger.Warn("plan-from-label: design revision cap reached — epic needs a human", "epic", epic.ID, "ref", epic.ExternalRef, "revisions", revisions) + adj := adjectives[int(buf[0])%len(adjectives)] + noun := nouns[int(buf[1])%len(nouns)] + return adj + "-" + noun } -// healGitHubAppInstallation self-heals a hive whose github.installation_id -// points at the WRONG account — the failure mode diagnoseGitHubApp -// already detects and reports ("installation N belongs to 'X', not 'Y'"). It -// asks pkg/github to rediscover the installation covering cfg.Project.Org via -// the App JWT and, only on an unambiguous match, adopts it in place and -// persists it so the fix survives a pod restart. -// -// Every failure path is soft and silent-ish: a hive with no App key is not -// App-authenticated (skip), an API error or an ambiguous/absent discovery -// result leaves installation_id exactly as configured so the existing -// "check github.installation_id" banner still stands. It never returns an -// error to the caller and never blocks startup or a heartbeat. -// -// Rediscovery is rate-limited by pkg/github's discovery cache -// (github.InstallationDiscoveryTTL), so calling this from the self-heal tick -// is cheap even when the App is genuinely not installed on the org. -func healGitHubAppInstallation(ctx context.Context, appAuth *github.AppAuth, cfg *config.Config, logger *slog.Logger) { - if appAuth == nil || !appAuth.HasKey() || cfg == nil { - return - } - org := cfg.Project.Org - if org == "" { - return - } - newID, err := appAuth.RediscoverAndAdopt(ctx, org, logger) - if err != nil { - logger.Debug("github app installation rediscovery did not adopt a new id", - "org", org, "error", err) - return - } - if newID == 0 { - return // already correct, or nothing safe to adopt +// watchdogAuthProbes builds the per-provider credential probes for the +// watchdog by adapting the rotation package's provider probers (#4608) — +// the same machinery the #4645 probe rewrite targets, so that rewrite reaches +// the watchdog automatically. +func watchdogAuthProbes(cfg *config.Config) map[string]watchdog.AuthProbe { + threshold := cfg.Governor.Rotation.EffectiveThreshold() + probers := []rotation.Prober{ + rotation.ClaudeProber{ThresholdPct: threshold}, + rotation.CodexProber{ThresholdPct: threshold}, + rotation.AgyProber{ThresholdPct: threshold}, + rotation.DeepSeekProber{}, } - cfg.GitHub.InstallationID = newID - if err := cfg.Save(); err != nil { - logger.Error("adopted rediscovered installation_id but failed to persist it — "+ - "it will revert on the next pod restart", - "installation_id", newID, "error", err) - return + out := make(map[string]watchdog.AuthProbe, len(probers)) + for _, p := range probers { + out[p.Provider()] = watchdog.RotationAuthProbe{Prober: usageProber{p}} } - logger.Info("persisted rediscovered github app installation_id", - "installation_id", newID, "org", org) + return out } -// maxTimelineEnumeratePerCycle bounds how many enumerated-issue events a single -// eval cycle records into the lifecycle timeline, keeping the recording loop -// O(1)-bounded so it never slows the eval cycle. Since #5656 the store dedupes -// by (ref, kind) — re-enumeration refreshes the journey instead of appending — -// so the cap no longer protects the ring from eviction floods; it matches the -// endpoint's default journey limit so every renderable journey gets refreshed. -const maxTimelineEnumeratePerCycle = 200 - -// lifecycleRecorder narrows *dashboard.Server to just the timeline accessor the -// recording helpers need, so they stay trivially testable with a fake and never -// depend on the rest of the Server surface. -type lifecycleRecorder interface { - LifecycleTimeline() *timeline.Store +// usageProber adapts a rotation prober to the watchdog's UsageProber slice +// (provider key + sanitised probe error); the watchdog does not import +// pkg/rotation. +type usageProber struct{ rotation.Prober } + +func (u usageProber) ProbeError(ctx context.Context) string { + return u.Prober.Probe(ctx).ProbeError() } -// recordEnumeratedIssues records a KindEnumerated event for each enumerated -// actionable issue, bounded by maxTimelineEnumeratePerCycle. The store dedupes -// by (ref, kind), so each cycle refreshes the journeys' enumerated stage -// rather than appending a flood (#5656). It is fully guarded: a nil recorder, -// nil store, or nil actionable set is a no-op, and Record itself is nil-safe. -// This must never slow or break the eval loop, so it does no blocking I/O -// (journey persistence is throttled and atomic inside the store). +// turnLossToSnapshot converts the manager's in-memory turn-loss accumulation +// into its persisted form, or nil when nothing has been recorded. // -// PR-open/merge lifecycle spans are emitted by the same tracing mapper when -// callers record those timeline events; this helper only has enumerated issues. -func recordEnumeratedIssues(ctx context.Context, rec lifecycleRecorder, actionable *github.ActionableResult) { - if ctx == nil { - ctx = context.Background() - } - if rec == nil || actionable == nil { - return - } - store := rec.LifecycleTimeline() - if store == nil { - return +// Nil rather than a zero struct on purpose: `turn_loss` is omitempty, so an +// agent that has never been interrupted adds nothing to /data/hive-state.json. +// The overwhelming majority of agents are in that state, and a measurement that +// bloated every hive's state file with empty records would be its own argument +// for removing it. +func turnLossToSnapshot(loss crux.TurnLoss) *snapshot.AgentTurnLoss { + if loss.Interruptions == 0 && len(loss.Recent) == 0 { + return nil } - limit := maxTimelineEnumeratePerCycle - if len(actionable.Issues.Items) < limit { - limit = len(actionable.Issues.Items) + out := &snapshot.AgentTurnLoss{ + Interruptions: loss.Interruptions, + Producing: loss.Producing, + UpperBoundS: loss.UpperBound.Seconds(), + Bytes: loss.Bytes, } - for i := 0; i < limit; i++ { - issue := actionable.Issues.Items[i] - event := timeline.Event{ - IssueRef: issueRef(issue.Repo, issue.Number), - Kind: timeline.KindEnumerated, + for _, r := range loss.Recent { + rec := snapshot.AgentTurnInterruption{ + At: r.At, + Reason: r.Reason, + SinceKickS: r.SinceKick.Seconds(), + Producing: r.Producing, + Bytes: r.Bytes, } - // The GitHub created_at is the open→triage / open→merge SLA origin; - // retained here so the SLA never re-fetches the issue. - if !issue.CreatedAt.IsZero() { - event.Attrs = map[string]string{timeline.AttrIssueCreatedAt: issue.CreatedAt.UTC().Format(time.RFC3339)} + if r.SinceOutput != nil { + s := r.SinceOutput.Seconds() + rec.SinceOutputS = &s } - _, span := tracing.StartTimelineSpan(ctx, event) - store.Record(event) - span.End() + out.Recent = append(out.Recent, rec) } + return out } -// recordKick records KindKicked events for the given agent. When issue refs are -// supplied it records one issue-scoped event per ref so post-completion lanes -// can reconstruct per-bead kick counts. With no refs, it records one -// agent-scoped event. Guarded and nil-safe; no I/O. -func recordKick(ctx context.Context, rec lifecycleRecorder, agent string, issueRefs ...string) { - if ctx == nil { - ctx = context.Background() - } - if rec == nil { - return - } - store := rec.LifecycleTimeline() - if store == nil { - return +func restartEventsToSnapshot(events []crux.RestartEvent) []snapshot.AgentRestartEvent { + if len(events) == 0 { + return nil } - if len(issueRefs) > 0 { - for _, ref := range issueRefs { - if ref == "" { - continue - } - event := timeline.Event{ - IssueRef: ref, - Kind: timeline.KindKicked, - Agent: agent, - } - _, span := tracing.StartTimelineSpan(ctx, event) - store.Record(event) - span.End() + out := make([]snapshot.AgentRestartEvent, 0, len(events)) + cutoff := time.Now().Add(-24 * time.Hour) + for _, ev := range events { + if ev.At.IsZero() || ev.At.Before(cutoff) { + continue } - return - } - event := timeline.Event{ - Kind: timeline.KindKicked, - Agent: agent, - } - _, span := tracing.StartTimelineSpan(ctx, event) - store.Record(event) - span.End() -} - -// issueRef renders the canonical "repo#number" reference the timeline uses to -// group events by issue. An empty repo yields an empty ref (the store tolerates -// it). -func issueRef(repo string, number int) string { - if repo == "" { - return "" + out = append(out, snapshot.AgentRestartEvent{At: ev.At, Reason: ev.Reason}) } - return fmt.Sprintf("%s#%d", repo, number) + return out } -func actionableIssueRef(issue github.Issue) string { - ref := worksource.Ref{ - SourceType: issue.SourceType, - Repo: issue.Repo, - ExternalID: issue.ExternalID, - Number: issue.Number, - URL: issue.URL, - } - if key := ref.Key(); key != "" { - return key +func restartEventsFromSnapshot(events []snapshot.AgentRestartEvent) []crux.RestartEvent { + if len(events) == 0 { + return nil } - if issue.Repo != "" { - return issue.Repo + out := make([]crux.RestartEvent, 0, len(events)) + cutoff := time.Now().Add(-24 * time.Hour) + for _, ev := range events { + if ev.At.IsZero() || ev.At.Before(cutoff) { + continue + } + out = append(out, crux.RestartEvent{At: ev.At, Reason: ev.Reason}) } - return issue.ExternalID -} - -// githubRateLimitErrText is the substring GitHub's client surfaces on a rate or -// abuse limit. Matching text is acceptable ONLY here: a rate limit is a reason -// to skip classification entirely, never a reason to accuse anyone of anything, -// so a false negative costs one extra (correct) classification round-trip. -const githubRateLimitErrText = "rate limit" - -// isGitHubRateLimitText reports whether an error looks like a GitHub rate limit. -func isGitHubRateLimitText(err error) bool { - return err != nil && strings.Contains(strings.ToLower(err.Error()), githubRateLimitErrText) + return out } -// The GitHub App credential classifiers moved to pkg/apphealth (#7238). These -// wrappers exist so call sites keep their current shape and so the App key -// paths -- vars that tests repoint at a temp dir -- are read HERE, at call -// time, rather than captured once at init. -func classifyGitHubAppFailure(ctx context.Context, appAuth *github.AppAuth, expectedOwner string, logger *slog.Logger) (bool, string, github.AppAuthState) { - return apphealth.ClassifyFailure(ctx, appAuth, expectedOwner, appKeyPaths(appAuthID(appAuth)), logger) -} +// persistPathsForRuntime resolves the file set persistState writes. It is a +// package var (the same seam pattern as githubAppTokenCachePath) rather than a +// direct defaultPersistPaths() call so the wrapper's delegation can be pinned +// by a test without writing the live /data files the default set hardwires +// (#6846). persistState is synchronous, so a test that swaps and restores +// this var cannot race a goroutine still reading it. +var persistPathsForRuntime = defaultPersistPaths -func classifyGitHubAppWriteForbidden(ctx context.Context, appAuth *github.AppAuth, expectedOwner, repo string) (string, github.AppAuthState) { - return apphealth.ClassifyWriteForbidden(ctx, appAuth, expectedOwner, repo, appKeyPaths(appAuthID(appAuth))) +func persistState(agentMgr crux.Runtime, gov *governor.Governor, cfg *config.Config, path string, logger *slog.Logger, dashSrv *dashboard.Server, wd *watchdog.Reconciler) { + persistStateWithPaths(agentMgr, gov, cfg, path, logger, dashSrv, wd, persistPathsForRuntime()) } -func appAuthID(appAuth *github.AppAuth) int64 { - if appAuth == nil { - return 0 +func persistStateWithPaths(agentMgr crux.Runtime, gov *governor.Governor, cfg *config.Config, path string, logger *slog.Logger, dashSrv *dashboard.Server, wd *watchdog.Reconciler, paths persistPaths) { + statuses := agentMgr.AllStatuses() + agents := make(map[string]snapshot.AgentState, len(statuses)) + for name, proc := range statuses { + as := snapshot.AgentState{ + Paused: proc.Paused, + PinnedCLI: proc.PinnedCLI, + PinnedModel: proc.PinnedModel, + ModelOverride: proc.ModelOverride, + BackendOverride: proc.BackendOverride, + RestartCount: proc.RestartCount, + RestartEvents: restartEventsToSnapshot(proc.RestartEvents), + LastRestartReason: proc.LastRestartReason, + LastKick: proc.LastKick, + PausedReason: proc.PausedReason, + PausedTrigger: proc.PausedTrigger, + PausedBy: proc.PausedBy, + TurnLoss: turnLossToSnapshot(proc.TurnLoss), + } + if !proc.PausedAt.IsZero() { + t := proc.PausedAt + as.PausedAt = &t + } + if len(proc.KickHistory) > 0 { + as.KickHistory = make([]snapshot.AgentKickEntry, len(proc.KickHistory)) + for i, kr := range proc.KickHistory { + as.KickHistory[i] = snapshot.AgentKickEntry{ + Timestamp: kr.Timestamp, + Agent: kr.Agent, + Snippet: kr.Snippet, + } + } + } + if agentCfg, ok := cfg.Agents[name]; ok { + as.DisplayName = agentCfg.DisplayName + as.Description = agentCfg.Description + enabled := agentCfg.Enabled + as.Enabled = &enabled + clearOnKick := agentCfg.ClearOnKick + as.ClearOnKick = &clearOnKick + staleTimeout := agentCfg.StaleTimeout + as.StaleTimeout = &staleTimeout + as.RestartStrategy = agentCfg.RestartStrategy + as.LaunchCmd = agentCfg.LaunchCmd + } + agents[name] = as } - return appAuth.AppID() -} - -func classifyGitHubAppRepoCoverage(ctx context.Context, appAuth *github.AppAuth, org string, repos []string, logger *slog.Logger) (bool, string, github.AppAuthState) { - return apphealth.ClassifyRepoCoverage(ctx, appAuth, org, repos, logger) -} - -// The advisory digest posting policy moved to pkg/advisory (#7238 stage 2). -// These wrappers keep the existing call sites unchanged; advisoryPostGate is -// now an owned instance rather than a package-level struct tests reset. -var advisoryPostGate = advisory.NewPostGate() - -func primaryAdvisoryRepo(cfg *config.Config) string { - return advisory.PrimaryRepo(cfg) -} - -func advisoryIssueUnresolved(advisoryIssues map[string]int, repo string) bool { - return advisory.IssueUnresolved(advisoryIssues, repo) -} - -func advisoryIssueNumber(advisoryIssues map[string]int, repo string) (int, bool) { - return advisory.IssueNumber(advisoryIssues, repo) -} -func shouldBuildAdvisoryDigest(beadStores map[string]*beads.Store, ghClient *github.Client, hasExistingPinnedIssue bool) bool { - return advisory.ShouldBuildDigest(beadStores, ghClient != nil, hasExistingPinnedIssue) -} + cadenceOverrides := make(map[string]map[string]config.Cadence) + for modeName, mode := range cfg.Governor.Modes { + if len(mode.Cadences) > 0 { + cadenceOverrides[modeName] = make(map[string]config.Cadence, len(mode.Cadences)) + for agentName, cadence := range mode.Cadences { + cadenceOverrides[modeName][agentName] = cadence + } + } + } -func shouldPostAdvisoryDigest(digest *advisory.Digest, ghClient *github.Client, hasPinnedIssue bool) bool { - return advisory.ShouldPostDigest(digest, ghClient != nil, hasPinnedIssue) -} + budget := gov.GetBudget() + govState := gov.GetState() -func advisoryPostDue(advCfg config.AdvisoryConfig, repo string, now time.Time, logger *slog.Logger) bool { - return advisoryPostGate.Due(advCfg, repo, now, logger) -} + govKickHistory := gov.KickHistory() + kickEntries := make([]snapshot.GovKickEntry, len(govKickHistory)) + for i, kr := range govKickHistory { + kickEntries[i] = snapshot.GovKickEntry{Timestamp: kr.Timestamp, Agent: kr.Agent, Outcome: kr.Outcome, OutcomeReason: kr.OutcomeReason} + } -func recordAdvisoryPostSuccess(repo string, now time.Time) { - advisoryPostGate.RecordSuccess(repo, now) -} + state := &snapshot.PersistedState{ + Agents: agents, + GovernorMode: string(govState.Mode), + BudgetLimit: budget.WeeklyLimit, + BudgetIgnored: budget.IgnoredAgents, + BudgetIgnoreAll: budget.IgnoreAll, + CadenceOverrides: cadenceOverrides, + LastKicks: govState.LastKick, + BudgetSpend: budget.CurrentSpend, + BudgetResetAt: budget.ResetAt, + BudgetByAgent: budget.ByAgent, + BudgetByModel: budget.ByModel, + CoinBudgetLimit: budget.CoinLimit, + CoinBudgetSpend: budget.CoinSpend, + CoinBudgetLabel: budget.CoinLabel, + USDBudgetLimit: budget.USDLimit, + USDBudgetSpend: budget.USDSpend, + BudgetWindowBaseline: budget.WindowBaseline, + CoinWindowBaseline: budget.CoinWindowBaseline, + USDWindowBaseline: budget.USDWindowBaseline, + KickHistory: kickEntries, + LastEval: govState.LastEval, + ACMMLevel: cfg.ACMMLevel, + } -func recordAdvisoryPostForbidden(repo string, now time.Time, backoff time.Duration) { - advisoryPostGate.RecordWriteForbidden(repo, now, backoff) -} + // Persist the fleet breaker so an engaged kill-switch survives a restart. + // Only written when engaged — a never-thrown breaker adds nothing. + if engaged, breakerPaused := agentMgr.BreakerState(); engaged { + state.Breaker = &snapshot.BreakerState{Engaged: true, Paused: breakerPaused} + } -// advisoryIssueMissingError is the error recorded (and reported to the hub) when -// a hive has findings to publish but no advisory issue to publish them to. It is -// deliberately an ERROR rather than a silent skip: the hub's staleness gate -// treats a hive reporting neither a post time nor an error as "not an advisory -// participant" and never alarms, which is how a wedged digest went unnoticed for -// six days in #4167. -func advisoryIssueMissingError(repo string, cause error) string { - base := fmt.Sprintf("no advisory issue resolved for %s — digest not posted", repo) - // Issues-disabled is the one ensure failure with a remedy the OPERATOR of - // the target repo owns (#4329): flipping a repo setting, not fixing App - // auth. Fold its actionable message into the alert text so the fleet - // stale-advisory pill says so instead of reading like an auth failure. - var disabled *github.IssuesDisabledError - if errors.As(cause, &disabled) { - return base + ": " + disabled.Error() - } - return base -} + // Persist the watchdog's backoff/crash-loop/condition state (RFC #4665 + // open question 2: it rides the existing state file). + if wd != nil { + if wdState := wd.Snapshot(); len(wdState) > 0 { + state.Watchdog = wdState + } + } -// actionableAfterGitHubEnumerate decides whether an eval cycle survives a -// failed GitHub enumeration. On the default (GitHub) work source the answer -// is no: an all-repos failure usually means a rate limit or outage, and a -// zero-count result would idle the agents, so the cycle keeps prior state. -// -// On a non-default work source (e.g. Linear), or a default GitHub source with -// additive work enabled (run stages / Wavefront), the GitHub issue call is not -// the only backlog authority. Aborting here meant a Linear-sourced hive whose -// GitHub App could not list issues (403 "Resource not accessible by -// integration", an Issues permission a Linear hive should not need) never -// enumerated its Linear backlog at all and sat at queue 0; the same rule keeps -// local additive sources reachable during a transient GitHub issue outage. Such -// a hive continues with whatever partial result GitHub returned (nil becomes -// an empty result; PRs are kept when obtainable) and lets the overlay populate -// issues. -func actionableAfterGitHubEnumerate(cfg *config.Config, actionable *github.ActionableResult, err error, logger *slog.Logger) (*github.ActionableResult, bool) { - if err == nil { - return actionable, true - } - if !workSourceOverlayEnabled(cfg.Governor.WorkSource) { - logger.Error("failed to enumerate actionable items", "error", err) - return nil, false - } - logger.Warn("GitHub enumeration failed; continuing so the configured work source can still populate issues", - "work_source", cfg.Governor.WorkSource.Type, "run_stages", cfg.Governor.WorkSource.RunStages, - "wavefront", cfg.Governor.WorkSource.Wavefront.Enabled, "error", err) - if actionable == nil { - actionable = &github.ActionableResult{GeneratedAt: time.Now()} - } - // The GitHub share of this result is missing, not zero: every configured - // repo is marked stale so the dashboard shows the gap as stale rather - // than as an empty hive (a cold cache has no last-good to fall back on). - markConfiguredReposStale(actionable, cfg.Project.Repos) - return actionable, true -} + if err := snapshot.SaveState(path, state, logger); err != nil { + logger.Error("failed to persist state", "error", err) + } -// markConfiguredReposStale flags every configured repo the result does not -// already cover as stale, with a zero count entry so the repo still appears. -func markConfiguredReposStale(actionable *github.ActionableResult, repos []string) { - if actionable == nil { - return + // Component reach counters (#3993) ride the SAME save cadence as the main + // state file but live in their own file (reachStatePath — resolved OQ-2 of + // #3973), so a reach write failure never corrupts agent/governor state. + if err := tracing.SaveReachState(paths.ReachState); err != nil { + logger.Error("failed to persist reach state", "error", err) } - if actionable.TotalByRepo == nil { - actionable.TotalByRepo = map[string]github.RepoCounts{} + + // Reconcile the persisted pause field from the authoritative live manager + // state and save, atomically under the config save mutex. persistState runs + // async (go PersistFunc()) on every pause/resume; doing the c.Agents update + // and Save under saveMu (via ReconcilePausedAndSave) means it can neither + // race the pause callback's map write nor clobber its file write with a + // stale paused=false. livePaused is built from AllStatuses(), read above. + livePaused := make(map[string]bool, len(agents)) + for name, as := range agents { + livePaused[name] = as.Paused } - for _, repo := range repos { - if _, ok := actionable.TotalByRepo[repo]; !ok { - actionable.TotalByRepo[repo] = github.RepoCounts{} + if err := cfg.ReconcilePausedAndSave(livePaused); err != nil { + logger.Error("failed to persist config to yaml", "error", err) + if dashSrv != nil { + dashSrv.AddSystemAlert("config-save-failed", "error", + "Config save failed — runtime state (ACMM level, agent config) will be lost on restart: "+err.Error()) } + } else if dashSrv != nil { + dashSrv.ClearSystemAlert("config-save-failed") } - actionable.MarkAllStale() -} -func tokensForBackend(summary *tokens.AggregateSummary, backend string) int64 { - if summary == nil { - return 0 - } - var total int64 - for _, sess := range summary.Sessions { - if strings.EqualFold(sess.Backend, backend) { - total += sess.TotalTokens + history := gov.EvalHistory() + if len(history) > 0 { + historyData, err := json.Marshal(history) + if err == nil { + atomicWrite(paths.SparklineHistory, historyData) } } - return total -} -func runEvalCycle( - ctx context.Context, - cfg *config.Config, - ghClient *github.Client, - gov *governor.Governor, - sched *scheduler.Scheduler, - agentMgr crux.Runtime, - dashSrv *dashboard.Server, - notifier *notify.Notifier, - beadStores map[string]*beads.Store, - tokenCollector *tokens.Collector, - metricsCollector *dashboard.MetricsCollector, - nousState *dashboard.NousState, - lastActionable *atomic.Pointer[github.ActionableResult], - advisoryStore *advisory.Store, - advisoryIssues map[string]int, - restartedAgents []string, - approvalDesk *toolapprove.Desk, - source evalCycleSource, - logger *slog.Logger, -) { - // Governor eval-cycle span. When tracing is disabled (the default) this is - // a no-op span with no allocation of note and no export — see pkg/tracing. - ctx, span := tracing.StartSpan(ctx, "governor.eval_cycle", - attribute.String("hive.id", cfg.HiveID), - attribute.Int(tracing.AttrHiveACMMLevel, inferACMMLevel(cfg))) - defer span.End() - - // A hive running without GitHub credentials (placeholder app_id, or a real - // App whose key could not be read) has nothing to enumerate. Return before - // the first API call rather than logging a misleading enumeration failure - // once per eval interval — the dashboard banner already states the cause. - if ghClient == nil { - logger.Debug("skipping eval cycle: hive is running without GitHub credentials") - return + modeHistory := gov.ModeHistory() + if len(modeHistory) > 0 { + modeData, err := json.Marshal(modeHistory) + if err == nil { + atomicWrite(paths.ModeHistory, modeData) + } } - budgetMode, _ := ghClient.APIBudgetMode() - budgetDecision := decideEvalBudgetWork(budgetMode, cfg.Governor.OptionalSweepEveryNCycles, evalAPIBudgetCycle.Add(1)) - - // A cached cycle (webhook or re-kick wake) plans from the stored - // snapshot: no listing, and none of the enrichment or unstick sweeps - // that walk every item — those belong to the reconcile pass. The first - // cycle has no snapshot yet and enumerates regardless of source. - cachedSnapshot := cachedSnapshotFor(source, lastActionable) - if cachedSnapshot != nil { - budgetDecision.SkipOptional = true - budgetDecision.SkipUnstick = true - budgetDecision.SkippedSteps = append(budgetDecision.SkippedSteps, cachedSnapshotSkippedStep) - } else if source == evalSourceCached { - logger.Info("eval cycle: no cached snapshot yet; enumerating") - } - ghClient.RecordAPIBudgetSkippedSteps(budgetDecision.SkippedSteps) - - // Re-ensure the pinned advisory issue whenever it is still unresolved, not - // only while the App banner is up (#4167). The startup ensure can fail for - // reasons that deliberately do NOT raise that banner — a rate limit, a 5xx, - // a search-API blip — and the old gate meant such a hive kept an empty - // advisoryIssues map for the rest of the process lifetime: every later - // digest silently found no issue to post to, so the pinned comment froze at - // whatever it last said. Retrying here is cheap (one search per eval cycle - // only while unresolved, nothing once resolved) and is the difference - // between a transient boot error and a permanently wedged digest. - // - // advisoryEnsureErr keeps this cycle's ensure failure so the post-path - // error recorded below can name the CAUSE (e.g. Issues disabled on a fork, - // #4329) instead of only the symptom. - primaryRepoAtCycleStart := primaryAdvisoryRepo(cfg) - _, hadPinnedAdvisoryIssueAtCycleStart := advisoryIssueNumber(advisoryIssues, primaryRepoAtCycleStart) - advisoryEnsureDepsForCycle := advisoryEnsureDeps{setenv: os.Setenv} - if ghClient != nil { - advisoryEnsureDepsForCycle.ensure = ghClient.EnsureAdvisoryIssue - } - advisoryEnsureErr := error(nil) - if !budgetDecision.SkipOptional { - advisoryEnsureErr = ensurePinnedAdvisoryIssue( - ctx, advisoryIssues, primaryRepoAtCycleStart, advisoryEnsureDepsForCycle, logger) - } - - var actionable *github.ActionableResult - if cachedSnapshot != nil { - actionable = cachedSnapshot - logger.Info("eval cycle planning from cached snapshot", - "snapshot_age", time.Since(actionable.GeneratedAt).Round(time.Second).String(), - "issues", actionable.Issues.Count, "prs", actionable.PRs.Count, "hold", actionable.Hold.Total, - "stale", actionable.Stale()) - } else { - enumCtx, enumSpan := tracing.StartSpan(ctx, "governor.enumerate_actionable") - fresh, err := ghClient.EnumerateActionable(enumCtx) - enumSpan.End() - var ok bool - actionable, ok = actionableAfterGitHubEnumerate(cfg, fresh, err, logger) - if !ok { - if kept := keepLastGoodAfterFailedEnumerate(lastActionable, logger); kept != nil && dashSrv != nil { - dashSrv.MarkStatusStale(kept.StaleSince) + if dashSrv != nil { + tokenHistory := dashSrv.TokenSparklineHistory() + if len(tokenHistory) > 0 { + tokenData, err := json.Marshal(tokenHistory) + if err == nil { + atomicWrite(paths.TokenSparklineHistory, tokenData) } - return } - // If a non-default or additive work source is configured, overlay its - // issues onto the actionable result. PRs always come from GitHub. A - // cached cycle keeps the overlay the reconcile pass already applied. - if workSourceOverlayEnabled(cfg.Governor.WorkSource) { - actionable.Issues = workSourceIssuesForConfiguredCycle(ctx, cfg, ghClient, actionable.Issues, logger) - } - } - if swarmRepo := dashSrv.ActiveSwarmRepo(); swarmRepo != "" { - github.BoostActionableRepoPriority(actionable.Issues.Items, swarmRepo) - logger.Info("swarm priority boost applied", "repo", swarmRepo) - } - - ghClient.EnrichCIStatus(ctx, actionable.PRs.Items) - // Held PRs need CI status too, or a red held PR can never be repaired: - // the hold label kept it out of PRs.Items, so nothing ever learned it was - // red and no agent was ever told to fix it (hivecommons/hive#7438). This - // enriches the held list ONLY for the repair path — held PRs still never - // reach the merge sweep, escalation or the queue counts. - if budgetMode != github.APIBudgetCritical { - ghClient.EnrichCIStatus(ctx, actionable.PRs.Held) - } - // Stale drafts sit in the same dashboard PR column; they get the - // review/link signals only — no mergeability or check-run fetches, a - // draft is not a merge candidate (hivecommons/hive#8968). - if !budgetDecision.SkipOptional { - // Sidelined PRs (drafts, exempt labels) sit in the same column and - // need their closing references to pair with the issue they fix - // (hivecommons/hive#13421); one shared collector keeps the budget at - // one query per repo for both. - ghClient.EnrichReviewSignalsAndWorkPRs(ctx, actionable.PRs.StaleDrafts, actionable.WorkDetailsByRepo) - // A closing reference whose issue is already closed (a superseded - // PR's `Fixes #N`) has no open row to borrow a title from; fill it - // once per issue, from a long-lived memo, so the activity monitor - // never shows a bare "#N". - ghClient.EnrichLinkedIssueTitles(ctx, actionable.WorkDetailsByRepo, actionable.PRs.Items, actionable.PRs.Held, actionable.PRs.StaleDrafts) - } - - // Publish the human-facing "what should I merge next?" digest. This reads - // the PR set enumerated and CI-enriched immediately above, so it must stay - // after those two calls: Mergeable and the failing-check names it sorts on - // are populated by EnrichCIStatus, not by EnumerateActionable. - if !budgetDecision.SkipOptional { - postRecommendationsForCycle(ctx, cfg, ghClient, actionable, logger) - } - - // Fold this pass's CI state into the fix-loop staleness clock BEFORE any - // consumer reads it, so the claim-suppression guard (#3), the merge watcher - // (#2), and the stuck-PR reaper (#4) all key off a consistent, current - // signal within the same tick. This only records first-seen-red times; the - // distinct-SHA attempt counting still happens in runEscalationSweep below. - recordRedStaleness(cfg, actionable) - - // Duplicate-PR guard: drop issues an open hive-authored PR already claims, - // before the governor counts the queue or the scheduler builds kicks. A - // restart storm otherwise re-offers the same issue on every fresh agent - // start, and the agent — having no memory of the PR it just filed — files - // another. Backed by a PVC ledger so it survives those restarts, and fails - // closed (keeps the last known claims) when the GitHub API is unavailable. - if !budgetDecision.SkipOptional { - applyDuplicatePRGuard(ctx, cfg, ghClient, actionable, logger) - } - // Every cycle, cached ones included: heal issue stage records the - // claim ledger says an open PR covers but the store still holds below - // 040 (a claim release written over the cover between reconcile passes). - reconcileLedgerCoverRecords(ctx, cfg, ghClient, actionable, logger) - - // PR review queue (#9590): stamp each PR's rank, priority and reasons - // onto the snapshot so last-actionable.json carries them next to - // review_class. Additive only: Items and Held keep their order, and - // nothing that gates work reads the stamp. - reviewQueue := stampReviewQueue(cfg, actionable, logger) - - // Only the reconcile pass publishes the snapshot: a cached cycle works - // on a copy, and storing it back would overwrite a webhook patch that - // landed while the cycle ran. - if cachedSnapshot == nil { - publishEnumeratedSnapshot(lastActionable, actionable, logger) - if data, err := json.Marshal(actionable); err == nil { - atomicWrite(lastActionablePath, data) - } - } - publishStageSharedCIIncidents(actionable) - - // Record enumerated issues into the lifecycle timeline so the dashboard's - // lifecycle view has real data. Cheap and fully guarded: a nil dashboard or - // nil store is a no-op (timeline.Store.Record is nil-safe), the loop is - // bounded by maxTimelineEnumeratePerCycle, and the store dedupes by - // (ref, kind) so this per-cycle sweep refreshes journeys instead of - // flooding them (#5656). - recordEnumeratedIssues(ctx, dashSrv, actionable) - - escalatedPRs := map[string]bool{} - if !budgetDecision.SkipUnstick { - escalatedPRs = runEscalationSweep(ctx, cfg, governorForge(cfg, ghClient, logger), actionable, notifier, dashSrv, logger) - } - - intentVerdicts := writeIntentVerdicts(ctx, cfg, ghClient, actionable, beadStores, logger) - refreshReviewVerdicts(cfg, logger) - recordReviewOutcomes(ctx, cfg, ghClient, actionable, logger) - requiredCheckSet, _ := cfg.AutoMerge.RequiredCheckSetForRepo("") - - // Hold guard (#5589): snapshot hold-gated PR heads, and when a hold lifts - // on a branch that moved, block the merge lanes and force a fresh review. - // Runs before writeMergeEligible so drifted PRs are excluded from the very - // tick their hold lifted — no window for the sweep to race the re-hold. - selfAuthorizationHoldEnabled := func(repo string) bool { - level := cfg.ACMMLevelOrZero() - if agentMgr != nil { - level = agentMgr.GetACMMLevel() - } - return cfg.SelfAuthorizationHoldEnabledForRepoAtLevel(repo, level) - } - holdDriftPRs := enforceHoldGuard(ctx, cfg, ghClient, governorForge(cfg, ghClient, logger), actionable, logger, selfAuthorizationHoldEnabled) - - // The per-PR verdicts come back so the dashboard's PR pills can be - // painted from the sweep's own classification rather than a looser - // reading of GitHub's mergeable flag (hivecommons/hive#7478). - mergeVerdicts := writeMergeEligible(actionable, actionable.Hold, cfg.Project.Org, escalatedPRs, cfg.Intent.Enforce, intentVerdicts, cfg.Review.RequireApproval, requiredCheckSet, holdDriftPRs, cfg, logger) - - // Review-bot threads (hivecommons/hive#7360): list every unresolved - // external-review-bot thread on an open hive-authored PR into - // review-threads.json, attributed to the agent that opened the PR the - // same way ci-failing.json is, so the scheduler can route each PR back - // to its author for a fix + in-thread replies before any new work. - if !budgetDecision.SkipOptional { - writeReviewThreads(ctx, ghClient, actionable, cfg.Project.Org, escalatedPRs, logger) - } - - // PR follow-up session resume (hivecommons/hive#9583, default off): feed - // CI failures, changes-requested reviews and new review-bot threads on a - // PR this hive opened back into the CLI session that authored it, while - // that session is still live. Anything it cannot resume stays on the - // fix-before-new path above, unchanged. - routePRFollowUps(ctx, cfg, ghClient, actionable, escalatedPRs, agentMgr, logger) - - // Stuck-PR reaper (backstop): DELIVER a targeted FIX-BEFORE-NEW kick for any - // hive-authored PR that is red on a required check AND stale (its red head - // SHA unchanged past RedPRStaleAfter). writeMergeEligible already surfaces - // every red PR into ci-failing.json (the CI_FAILING kick block), but relying - // on the owner's next cadence kick to carry it meant a red PR could be - // escalated as "6 automated fix re-dispatches" having been mentioned to - // nobody (hivecommons/hive#9472); the reaper now kicks the owner — or the - // fallback fixer when the owner is paused or unreachable — and charges the - // re-engagement budget only for kicks that were accepted. Composes with the - // merge watcher (#2): both go through the same per-red-SHA cap, so the same - // PR is never double-dispatched within a red-SHA's budget. - reapStuckRedPRs(cfg, actionable, escalatedPRs, agentKicker{mgr: agentMgr}, - agentAvailability(cfg, agentMgr), logger) - - // Release sentinel (hivecommons/hive#9585), opt-in and default OFF: when - // the current v tag's CI fails, dispatch a bounded repair round - // through the same kick path as the reaper above, or escalate to a human - // when the failure is a setting no commit can fix. Never pushes a branch; - // moves the tag to a merged fix PR only with retag_enabled (also opt-in). - runReleaseSentinel(ctx, cfg, ghClient, agentKicker{mgr: agentMgr}, - agentAvailability(cfg, agentMgr), notifier, logger) - - // Upstream watch (hivecommons/hive#9967), opt-in and default OFF: at most - // once per upstream_watch.interval, file a labelled fork issue for each - // upstream merged PR or release that still applies to the fork. Opens - // issues only, never PRs; the watermark and dedupe index live on the PVC. - runUpstreamWatch(ctx, cfg, ghClient, logger) - - // Merge-hotspot advisor: at most hourly, score the files concurrent - // PRs keep colliding on (from the merge train's own records) and file - // one modularize issue per hotspot for the ci-maintainer lane through - // the writeback gate. Skipped with the other optional sweeps when the - // GitHub budget is tight. - if !budgetDecision.SkipOptional { - runHotspotAdvisor(ctx, cfg, ghClient, actionable, logger) - } - - if !budgetDecision.SkipUnstick { - shaResult, shaErr := ghClient.EnforceSHAHold(ctx, github.SHAHoldConfig{ - PrimaryRepo: cfg.Project.PrimaryRepo, - AIAuthor: cfg.Project.AIAuthor, - InternalAuthors: []string{"hivecommons-hive[bot]", "kubestellar-hive[bot]", "hivecommons-hive-ghe[bot]", "kubestellar-hive-ghe[bot]", "github-actions[bot]", "dependabot[bot]", "copilot-swe-agent[bot]"}, - // Trusted reporters get the soft needs-sha label, not the hard - // hold (hivecommons/hive#12002). - TrustedAssociations: cfg.Project.IssueFilter.ReporterTrust.EffectiveTrustedAssociations(), - TrustedLabel: cfg.Issues.EffectiveSHAHoldTrustedLabel(), - }) - if shaErr != nil { - logger.Warn("SHA hold enforcement failed", "error", shaErr) - } else { - logger.Info("SHA hold enforcement complete", - "held", shaResult.Held, - "unheld", shaResult.Unheld, - "softened", shaResult.Softened, - "skipped", shaResult.Skipped, - ) - } - runHardSuppressReconcile(ctx, cfg, ghClient, logger) - } - - // Refresh budget spend from lifetime token totals before Evaluate so - // the kick gate sees current-window numbers. The banner itself is - // level-triggered every cycle (#9612 follow-up): a collector summary may - // not be available yet (right after a restart, before the first session - // scan completes), and the one-shot Crossed edge that used to gate the - // raise is lost on restart too, so gate ONLY the spend refresh on the - // summary and re-derive the banner from whatever budget state the - // governor currently holds (persisted spend if the summary is not in - // yet, refreshed spend once it is) on every cycle. - trans := gov.BudgetLevel() - if tokenCollector != nil { - if summary := tokenCollector.Summary(); summary != nil { - var bobCoins, bobUSD float64 - if coinCfg, ok := cfg.Governor.Budget.CoinConfig(tokens.BackendBob); ok { - bobCoins = coinCfg.TokensToCoins(tokensForBackend(summary, tokens.BackendBob)) - bobUSD = coinCfg.CoinsToUSD(bobCoins) + factHist := dashSrv.FactHistory() + if len(factHist) > 0 { + factData, err := json.Marshal(factHist) + if err == nil { + atomicWrite(paths.FactHistory, factData) } - trans = gov.UpdateBudgetFromTotalsCoinsAndUSD(summary.TotalTokens, summary.ByAgent, summary.ByModel, bobCoins, bobUSD) } - } - applyBudgetAlerts(gov, trans, dashSrv, notifier) - - // Cause+fix banner for the never-kicked class (#5577): the dashboard's - // not-producing warnings name the SYMPTOM (agent idle, zero tokens); this - // names the cause — enabled agent, no cadence in any mode, never kicked — - // and the fix. Computed from the spoke's own governor config, no hub - // round-trip; self-clears the moment a cadence is set or any kick lands. - applyNoCadenceAlert(gov, dashSrv) - - agentsDue := gov.EvaluateWithRepoDepths( - actionable.Issues.Count, - actionable.PRs.Count, - actionable.Hold.Total, - actionable.Issues.SLAViolations, - governor.RepoDepthsFromActionable(actionable), - ) - - // The weaker sibling of the banner above (#7474): an agent SOME mode - // schedules but the mode the fleet is now in does not — a reviewer with a - // cadence only in surge goes silent the moment its own work drives the - // backlog below the surge threshold, and every other signal calls it - // healthy. Applied after Evaluate so it reads the mode this tick settled - // on; self-clears when the mode changes back or the operator fills the - // gap. - applyModeUnscheduledAlert(gov, dashSrv) - - // Crash-restarted agents may get a "resume" kick ahead of their cadence - // slot so work interrupted mid-task resumes promptly — but ONLY through - // the governor's gate (#2573). Unconditionally kicking every restarted - // agent meant a crash-looping CLI was kicked on every eval cycle, - // burning backend tokens far faster than any configured cadence and - // bypassing the budget gate; AllowResumeKick bounds resume kicks to one - // per cadence interval and respects mode pauses and the budget. - // Refused restarts are not silent: an interval-throttled agent sits at - // an empty prompt until its slot, so raise a per-agent alert (with an OOM - // hint when the cgroup killed the CLI). Every cycle, reconcile clears it - // once it no longer describes a problem (#9612): kicked, paused/removed, - // no longer expected to run, working again, or past its max age. - var resumeHeld []spokealerts.ResumeKickHeldAgent - agentsDue, resumeHeld = mergeResumeKicks(agentsDue, restartedAgents, gov.AllowResumeKickReason, logger) - resumeKickHeldAlerts.Apply(dashSrv, resumeHeld, agentMgr.CrashOOMSuspected) - resumeKickHeldAlerts.Reconcile(dashSrv, resumeKickHeldFacts(agentMgr, gov.ResumeKickVerdict)) - govState := gov.GetState() - span.SetAttributes( - attribute.String(tracing.AttrHiveGovernorMode, string(govState.Mode)), - attribute.Int("hive.queue.issues", govState.QueueIssues), - attribute.Int("hive.queue.prs", govState.QueuePRs), - attribute.Int("hive.queue.hold", govState.QueueHold), - ) - // cadence.Paused (cadence: "pause" in config) means "don't kick this agent - // in this mode" — it does NOT force-pause the agent. Manual pause/resume - // via the dashboard is always respected; the governor only controls kicks. - - // Filter out on-demand agents — they are only triggered explicitly - // Operator-paused agents must consume NOTHING (#2573); see - // filterKickableAgents for the full gate. This runs BEFORE the eval-cycle - // log so "agents_due" names the agents that are actually about to be - // kicked; anything gated out is reported in "agents_skipped" with its - // reason rather than vanishing silently. - agentsDue, agentsSkipped := partitionKickableAgents(agentsDue, cfg.Agents, config.OnDemandAgentsFromPacks(), agentMgr.IsPaused, func(name string) string { - if st, err := agentMgr.GetStatusFast(name); err == nil && st != nil { - return st.PausedTrigger + costHist := dashSrv.CostHistory() + if len(costHist) > 0 { + costData, err := json.Marshal(costHist) + if err == nil { + atomicWrite(paths.CostHistory, costData) + } } - return "" - }) - agentsDue, agentsSkipped = reviewerAutoscaler.withholdDrainingReviewers(agentsDue, agentsSkipped) - // PROVIDER SPEND REBUFF (#4294). When the inference gateway is refusing on a - // money limit, every kick launched this cycle is a run that cannot buy a - // single token — precisely the failure this addresses: a hive that kept - // firing its cadence into a gateway rejecting 100% of requests all day, - // silently, until the provider's spend window happened to roll over. - // - // The alert is raised HERE, before kick assembly, so an operator is told - // even on a cycle where nothing happened to be due. The actual suppression - // happens after every kick source has contributed — see below. - providerBudgetCause, providerBudgetSince, providerBudgetLastRebuff, providerBudgetRebuffs := dashboard.InferenceBudgetExceeded() - // Suppress only while the latch is FRESH. Withholding kicks also withholds - // the inference calls that clear the latch, so a hive whose only kick source - // is the governor cadence would never learn the provider's window reset and - // would stay muted forever — the exact topology in the field report. Once - // the last rebuff is older than the probe interval, this cycle's kicks go - // through as a probe: still clipped re-freshens the stamp and suppression - // resumes, served clears the latch outright. - providerBudgetProbeInterval := cfg.Governor.ProviderBudget.EffectiveProbeInterval() - providerBudgetLatched := providerBudgetCause != "" - suppressKicks := governor.ProviderBudgetSuppresses(providerBudgetLatched, - providerBudgetProbe.Freshest(providerBudgetLastRebuff), time.Now(), providerBudgetProbeInterval) - budgetAlert := decideProviderBudgetAlert(providerBudgetLatched, suppressKicks, - providerBudgetCause, providerBudgetSince, providerBudgetRebuffs, - func() string { - return hub.QuotaExhaustedAgentReason(hub.QuotaExhaustedProcessCount(agentMgr.AllStatuses())) - }) - if budgetAlert.Message != "" { - dashSrv.AddSystemAlert(providerBudgetAlertID, "error", budgetAlert.Message) - } - if budgetAlert.Clear { - dashSrv.ClearSystemAlert(providerBudgetAlertID) - } - if budgetAlert.Cause != "" { - providerBudgetCause = budgetAlert.Cause - } - // Notify ONCE per latch, not once per cycle. The deduped banner above - // already carries the ongoing state; a high-priority notification repeated - // every eval cycle for as long as the provider stays clipped is a day of - // pages saying the same thing. Keyed on the latch time, which recordRebuff - // deliberately does not move forward, so a genuinely new clip after a - // recovery notifies again. Matches applyBudgetAlerts, which notifies on the - // crossing rather than on the condition. - notifyProviderBudget := providerBudgetLatched && providerBudgetNotify.ShouldSend(providerBudgetSince) - if !providerBudgetLatched { - providerBudgetProbe.Reset() - // The RECOVERY crossing: the latch a notification went out for has - // cleared (a probe's inference call succeeded), so tell the operator - // once that the hive resumed — the counterpart of the entering page, - // without which the only signal of recovery is a banner quietly - // vanishing. Every later healthy cycle is silent. - if providerBudgetNotify.Reset() { - logger.Info("provider spending limit lifted: agent kicks resumed") - notifier.Send("Provider spending limit lifted", - "the inference provider is serving again — agent kicks have resumed", - notify.PriorityDefault) - } - } - - // ADDITIVE CEL routing: evaluate operator-defined CEL trigger rules - // (cfg.Triggers, pkg/celtrigger) against the items enumerated this cycle and - // UNION any matched, gated agents into agentsDue. This runs alongside — never - // instead of — the label/governor selection above: unionAgents only ever adds - // names, and celMatchedAgents enforces the same pause/budget/on-demand gates, - // so a CEL match can neither remove a governor-selected agent nor bypass a - // paused agent or exhausted budget. Fully guarded/cheap: with no `triggers:` - // configured, celEngineFor returns nil and celMatchedAgents does zero work. - if celEngine := celEngineFor(cfg, logger); celEngine != nil { - celAgents := celMatchedAgents(celEngine, actionable, cfg, gov, agentMgr.IsPaused, logger) - if len(celAgents) > 0 { - before := len(agentsDue) - agentsDue = unionAgents(agentsDue, celAgents) - if len(agentsDue) > before { - logger.Info("celtrigger: unioned CEL-matched agents into due set", - "added", agentsDue[before:], "cel_matched", celAgents) + costRollup := dashSrv.CostDailyRollups() + if len(costRollup) > 0 { + rollupData, err := json.Marshal(costRollup) + if err == nil { + atomicWrite(paths.CostDailyRollup, rollupData) } } - } - // #4247/#4263 (parent #3845): apply the shared convergence admission at the - // internal-kick dispatch boundary — AFTER governor policy evaluated the raw - // queue, BEFORE the scheduler caches and renders issues. Gated by the - // runtime convergence rollout mode, captured once per pass: with mode "off" - // (the DEFAULT) this is entirely inert and kickActionable IS actionable; - // "shadow" logs and records what would be withheld but still dispatches the - // raw population; only "enforce" gates the scheduled/cached issue payloads - // below. The raw actionable population stays authoritative for governor - // policy, dashboard status, PR/review dispatch, escalation, and every path - // not explicitly enrolled. - kickActionable := applyConvergenceKickAdmission(cfg, dashSrv, actionable, notifier, logger) - - sched.SetLastActionable(kickActionable) - // The filter compacts agentsDue in place, so snapshot it first to report - // the targets it withholds alongside the other skips below. - dueBeforeDeferral := append([]string(nil), agentsDue...) - agentsDue = sched.FilterFirstScanDeferredAgents(agentsDue) - agentsSkipped = append(agentsSkipped, firstScanDeferredSkips(dueBeforeDeferral, agentsDue)...) - reviewPlan := planReviewDispatchWith(ctx, cfg, actionable, agentMgr, beadStores, reviewPlannerGitHubFor(ghClient), logger) - // Size the reviewer lane to what the planner just left waiting - // (hivecommons/hive#14685). A replica added here takes kicks from the - // next cycle on; a retire only lands once the reviewer is idle. - reviewerAutoscaler.Run(ctx, cfg, reviewPlan, agentMgr, gov, logger, time.Now()) - auditWithheldReviewFixes(reviewPlan, dashSrv, logger) - if ghClient != nil { - postReviewAcknowledgements(ctx, ghClient, reviewPlan, logger) - } - applyHumanDecisionLabels(ctx, cfg, ghClient, actionable, reviewPlan, logger) - if ghClient != nil { - applyReviewPriorityLabels(ctx, cfg, ghClient, reviewQueue, logger) - applyReviewVerdictLabels(ctx, cfg, ghClient, actionable, reviewPlan, logger) - sweepClosedPRVerdictLabels(ctx, cfg, ghClient, time.Now(), logger) - } - messages, messageSkips := sched.BuildKickMessagesWithSkips(kickActionable, agentsDue) - agentsSkipped = append(agentsSkipped, messageSkips...) - // Message construction can withhold a due kick (notably an idle reviewer - // lane). Report only assembled targets as due. Skipped targets dispatch - // nothing, so (as before) no LastKick is recorded for them and new lane - // work stays eligible on the next evaluation. - assembledTargets := make([]string, 0, len(messages)) - for _, msg := range messages { - assembledTargets = append(assembledTargets, config.CadenceTargetKey(msg.Agent, msg.Repo)) - } - logger.Info("governor eval complete", - "mode", govState.Mode, - "issues", govState.QueueIssues, - "prs", govState.QueuePRs, - "agents_due", assembledTargets, - "agents_skipped", agentsSkipped, - ) - - reviewKickByMessage := map[string]review.DispatchKick{} - for _, k := range append(reviewPlan.ReviewKicks, reviewPlan.FixKicks...) { - if !gov.AgentEligibleForCELKick(k.Agent) || agentMgr.IsPaused(k.Agent) { - logger.Info("review swarm kick suppressed by governor gate", "agent", k.Agent, "pr", k.PRRef) - continue - } - messages = append(messages, scheduler.KickMessage{Agent: k.Agent, Message: k.Message, IssueRefs: []string{k.PRRef}}) - reviewKickByMessage[k.Agent+"\x00"+k.Message] = k - } - // A fix kick (a maintainer's changes_requested / `/hive decision`, or - // the swarm's) goes ahead of the governor kick for the same agent, or - // the governor kick owns the agent and the fix kick times out every - // cycle (hivecommons/hive#13100). - for _, agent := range pendingKickAgentsFromMessages(messages) { - if governorKickFairness.GovernorFirst(agent) { - logger.Warn("governor kick displaced by fix kicks on consecutive cycles; governor kick goes first this cycle", - "agent", agent, "displaced_cycles", governorKickFairness.Displaced(agent), "limit", governorKickDisplacementLimit) - } - } - messages = orderReviewFixKicksFirst(messages, reviewKickByMessage, governorKickFairness.GovernorFirst) - // Freshness bound before a kick (#13375): a ref older than - // github.DecisionFreshnessBound in a cache-served or last-good snapshot - // is confirmed with one targeted GET; a stale repo's refs are withheld. - messages = freshenKickMessages(ctx, ghClient.SnapshotFreshnessGuard(actionable, time.Now), actionable, messages, logger) - // #4294: drop EVERY assembled kick while the provider is refusing on a - // spending limit. Placed after all three sources have contributed — - // governor-due agents, the CEL union, and the review swarm — because gating - // only `agentsDue` earlier would still let a CEL match or a review kick fire - // into the same clipped key. - // - if len(messages) > 0 { - filtered := messages[:0] - for _, msg := range messages { - if remaining, class, line, ok := agentMgr.ProviderErrorBackoffRemaining(msg.Agent); ok { - logger.Warn("provider inference error: withholding agent kick during backoff", - "agent", msg.Agent, - "class", class, - "retry_in", remaining.Round(time.Second), - "error", line) - gov.RecordKickFailure(msg.Agent, fmt.Errorf("provider inference backoff (%s): %s", class, line), time.Now()) - continue + trendHist := dashSrv.TrendHistory() + if len(trendHist) > 0 { + trendData, err := json.Marshal(trendHist) + if err == nil { + atomicWrite(paths.TrendHistory, trendData) } - filtered = append(filtered, msg) } - messages = filtered - } - // Suppression is total rather than per-agent because the limit is on the - // KEY: no agent can succeed while it is clipped. It self-heals — the first - // inference call that succeeds after the provider's window resets clears the - // signal — so there is no timer to tune and no operator action required. - // - // Deliberately does NOT force-pause agents. Operator pause state is a human - // decision (#2573) and must not be forged by an automatic signal that will - // clear itself; withholding kicks achieves the saving without leaving paused - // agents behind for a human to un-pause by hand. - // - // Probe cycle: when the last rebuff has gone stale, ONE kick is - // deliberately allowed through to find out whether the provider is - // serving again. Its inference calls are what clear the latch (on a - // 2xx) or re-freshen it (on another rebuff) — nothing else can. Only - // one: the question is "is the window still clipped", and every kick - // beyond the first spends a run to learn the same answer. Releasing it - // re-arms suppression immediately, so the cycles while the probe's run - // is still in flight withhold again rather than leaking more kicks. - kickGate := gateKickMessagesForProviderBudget(messages, suppressKicks, providerBudgetLatched) - releaseProviderBudgetProbe := kickGate.ReleaseProbe - if suppressKicks && len(messages) > 0 { - logger.Warn("provider spending limit: withholding agent kicks", - "withheld", kickGate.Withheld, "rebuffs", providerBudgetRebuffs, "since", providerBudgetSince, - "next_probe_in", (providerBudgetProbeInterval - time.Since(providerBudgetProbe.Freshest(providerBudgetLastRebuff))).Truncate(time.Second)) - } else if releaseProviderBudgetProbe { - if len(kickGate.Withheld) > 0 { - logger.Warn("provider spending limit: withholding all but the probe kick", - "withheld", kickGate.Withheld, "rebuffs", providerBudgetRebuffs, "since", providerBudgetSince) - } - logger.Info("provider spending limit: releasing a single probe kick", - "probe_agent", kickGate.Kept[0].Agent, "rebuffs", providerBudgetRebuffs, "since", providerBudgetSince, - "last_rebuff", providerBudgetLastRebuff, "probe_interval", providerBudgetProbeInterval) - } - messages = kickGate.Kept - // Self-upgrade drain (#11876): a hub-instructed upgrade is waiting for - // in-flight kicks to end, so starting new turns would only feed the - // restart. Withheld kicks stay due and go out from the new process. - if target, draining := selfUpgradeDrain.Active(selfUpgradeDrainNow()); draining && len(messages) > 0 { - logger.Info("self-upgrade drain: withholding new agent kicks", - "target", target, "withheld", pendingKickAgentsFromMessages(messages)) - messages = nil - } - govState.PendingKicks = pendingKickAgentsFromMessages(messages) - gov.SetPendingKicks(govState.PendingKicks) - if notifyProviderBudget { - notifier.Send("Provider spending limit reached", providerBudgetCause, notify.PriorityHigh) - } - - // buildStatusSnapshot assembles the full dashboard snapshot from this - // cycle's evaluation. It is called twice: once BEFORE the kick dispatch - // below and once at the end of the cycle. Each kick can wait up to the - // input-prompt timeout for a busy pane; before dispatch ran one agent at - // a time, a dozen due agents took 10-15 minutes on hosted hives — during - // which /api/status (and everything derived from it: stage tracker, agent - // states, governor.nextKickAt) kept showing the previous cycle's data, and - // a rolling upgrade landing mid-dispatch meant the evaluation never - // reached the dashboard at all. Dispatch is now concurrent across agents - // (dispatchAgentKicks), so the wait is one agent's, but the early publish - // stays: the evaluation should be visible before any kick is typed. - buildStatusSnapshot := func() *dashboard.StatusPayload { - payload := dashboard.BuildFrontendStatus( - govState, - actionable, - agentMgr.AllStatuses(), - cfg, - tokenCollector, - gov, - beadStores, - ghClient, - ctx, - metricsCollector, - ) - // Green on a PR pill means "the sweep would merge this now" — the - // verdict writeMergeEligible reached for this same actionable set, - // not a re-derivation from mergeable_state (#7478). - dashboard.AttachMergeVerdicts(payload, mergeVerdicts) - // A PR pill also carries the hive's OWN review on that PR, so the - // queue view shows where the hive has already spoken. Read from the - // durable ledger, so this costs no GitHub call per PR; a missing or - // unreadable ledger simply means no review pills this cycle. - attachReviewLinksForDashboard(payload, logger) - return payload - } - publishStatusBeforeKicks(dashSrv, len(messages), buildStatusSnapshot, logger) - - // Kick dispatch lives behind a seam (#7232): the skip rules and the - // single-probe rule are the decisions worth testing, and they were - // previously unreachable without a tmux session and a live governor. - var deliveredReviewKicks []review.DispatchKick - var deliveredKicks []scheduler.KickMessage - if len(messages) > 0 { - // governor.kick wraps the whole dispatch pass; each delivered kick - // still gets its own agent.kick child via startKickSpan below. - kickCtx, kickPassSpan := tracing.StartSpan(ctx, "governor.kick", - attribute.Int("hive.kicks.assembled", len(messages))) - ctx := kickCtx - dispatchAgentKicks(messages, releaseProviderBudgetProbe, kickDispatchDeps{ - backoffRemaining: agentMgr.ProviderErrorBackoffRemaining, - sendKick: agentMgr.SendKick, - startStoppedAgent: func(agentName string) error { - if !agentMgr.GovernorMayStart(agentName) { - return nil - } - agentMgr.MarkStartupLaunchQueued([]string{agentName}) - if err := agentMgr.Start(ctx, agentName); err != nil { - if stoppedAgentStartErrorIsBenign(err) { - logger.Info("stopped agent start raced with existing launch before governor kick", "agent", agentName, "error", err) - return nil - } - return fmt.Errorf("start stopped agent: %w", err) - } - logger.Info("started stopped agent before governor kick", "agent", agentName) - return nil - }, - startKickSpan: func(agentName string) func(error) { - agentCfg := cfg.Agents[agentName] - _, kickSpan := tracing.StartSpan(ctx, "agent.kick", tracing.AgentKickAttributes( - agentName, - agentCfg.Backend, - agentCfg.Model, - agentCfg.Role, - string(govState.Mode), - inferACMMLevel(cfg), - )...) - return func(err error) { - if err != nil { - kickSpan.RecordError(err) - } - kickSpan.End() - } - }, - onReviewDelivered: func(msg scheduler.KickMessage) { - if k, ok := reviewKickByMessage[msg.Agent+"\x00"+msg.Message]; ok { - deliveredReviewKicks = append(deliveredReviewKicks, k) - persistReviewDispatchStateInProgress(reviewPlan, deliveredReviewKicks, logger) - } - }, - onFailed: func(msg scheduler.KickMessage, err error) { - gov.RecordKickFailure(msg.Agent, err, time.Now()) - if k, ok := reviewKickByMessage[reviewKickKey(msg)]; ok { - logUndeliveredReviewKick(logger, dashSrv, k, err) - } - }, - onDelivered: func(msg scheduler.KickMessage, deliveredAt time.Time) { - deliveredKicks = append(deliveredKicks, msg) - gov.RecordKickForRepoAt(msg.Agent, msg.Repo, deliveredAt) - dashSrv.AuditLog("governor", "kick", "trigger=governor-eval", msg.Agent) - - // Record issue-scoped kicks into the lifecycle timeline. Cheap, - // guarded, and nil-safe (Record no-ops on a nil dashboard/store). - recordKick(ctx, dashSrv, msg.Agent, msg.IssueRefs...) - // A review-swarm kick (review or fix) names a pull request, - // not an issue: its IssueRefs is the PR ref. Listing it in - // the issue-claims ledger made the author's next comment on - // its own PR an issue "start signal", so the PR was claimed - // as an issue and staged stage/030-pr-in-progress — an - // issue-only stage — until the PR sweep healed it. - reviewKick, isReview := reviewKickByMessage[msg.Agent+"\x00"+msg.Message] - if !isReview { - // #8380/#10527: list these issues for the kicked agent so - // contributors and other agents back off while the kick is - // live; the claim itself is recorded on the agent's first - // start signal (recordAgentStart), not here. - recordAgentKickListings(dashSrv, cfg.Project.Org, msg.Agent, msg.IssueRefs, logger) - } - - // #12128: record which issue/PR/head SHA this kick was launched - // for, so /api/launch-cost can place the turn's spend. Review - // kicks carry the PR head; governor kicks carry the issue refs. - if tokenCollector != nil { - tokenCollector.RecordLaunch(launchRecordForKick(msg, reviewKick, isReview, time.Now())) - } - - // Log token state at time of kick for cost attribution - if tokenCollector != nil { - if summary := tokenCollector.Summary(); summary != nil { - agentTokens := summary.ByAgent[msg.Agent] - logger.Info("kick token snapshot", - "agent", msg.Agent, - "agent_tokens", agentTokens, - "total_tokens", summary.TotalTokens, - "total_sessions", summary.SessionCount, - ) - } - } - }, - markProbeReleased: providerBudgetProbe.MarkReleased, - now: time.Now, - }, logger) - kickPassSpan.SetAttributes(attribute.Int("hive.kicks.review_delivered", len(deliveredReviewKicks))) - kickPassSpan.End() - } - governorKickFairness.ObserveCycle(messages, deliveredKicks, reviewKickByMessage) - persistReviewDispatchState(reviewPlan, deliveredReviewKicks, logger) - - if actionable.Issues.SLAViolations > 0 { - toNotify, capped := selectSLABreachNotifications(actionable.Issues.Items) - for _, issue := range toNotify { - notifier.Send( - "SLA 2x breach", - fmt.Sprintf("%s age %dm: %s\n%s", actionableIssueRef(issue), issue.AgeMinutes, issue.Title, issue.URL), - notify.PriorityHigh, - ) - } - if capped { - logger.Info("SLA notification cap reached, skipping remaining", "remaining", actionable.Issues.SLAViolations-len(toNotify)) - } - } - - // Scan agent panes for login-required patterns and pause + notify if detected - loginscan.Scan(ctx, cfg, agentMgr, notifier, dashSrv, logger, loginSightings) - - // Epoch captured before reading agent/governor state so a mutation that - // lands mid-build (restart-count/budget reset) drops this snapshot instead - // of letting it revert the mutation on the dashboard (#4348). - buildEpoch := dashSrv.BeginStatusSnapshot() - statusPayload := buildStatusSnapshot() - statusPublished := false - // Ingest any JSONL findings agents wrote and persist them as beads. - if advisoryStore != nil { - findings, err := advisoryStore.ReadNewFindings() - if err != nil { - logger.Warn("failed to read advisory findings", "error", err) - } else if len(findings) > 0 { - // The canary gate's decisions live behind a seam (#7232); only the - // effects — audit entry, critical bead — are supplied here. - var scanCanary func(agent, reportText, source string) (ioscan.CanaryLeak, bool) - if cfg.Ioscan.IsEnabled() && cfg.Ioscan.CanariesEnabled() { - scanCanary = ioscan.DefaultCanaries.Scan - } - safeFindings := gateAdvisoryFindings(findings, advisoryIngestDeps{ - scanCanary: scanCanary, - failClosed: cfg.Ioscan.FailClosed(), - auditLog: dashSrv.AuditLog, - recordLeakBead: func(leak ioscan.CanaryLeak) { - if store, ok := beadStores[leak.Agent]; ok && store != nil { - if b, berr := store.Create("Canary token leaked via "+leak.Source, beads.TypeAdvisory, beads.PriorityCritical, leak.Agent, ""); berr == nil { - _ = store.SetMetadata(b.ID, "rule", ioscan.CanaryLeakRule) - _ = store.SetMetadata(b.ID, "source", leak.Source) - } - } - }, - }, logger) - if persisted := advisory.PersistAsBeads(safeFindings, beadStores); persisted > 0 { - logger.Info("advisory findings persisted as beads", "count", persisted) - } - } - } - - // Reload bead stores from disk before building the digest. Agents write - // beads via the bd CLI which persists directly to disk, so the in-memory - // stores can become stale between eval cycles. Reload failures are deduped - // (WARN once per distinct error, then DEBUG) — see beads_reload.go (#5505). - reloadBeadStores(beadStores, logger) - - // Phase 4 Part B: `plan`/`epic` label trigger. An actionable issue carrying a - // plan label auto-mints an epic and requests decomposition — the same flow as - // the dashboard "Plan this issue" click, but triggered by the label. It runs - // AFTER the store reload so FindByExternalRef sees current state (idempotent: - // no duplicate epic if one already exists). Cheap, synchronous, adds NO - // goroutine, and drives the architect only via SendKick (never the launch - // path). Gated by config/ACMM so low-maturity hives stay advisory-only. - if acmmLvl := inferACMMLevel(cfg); planning.PlanningAllowedAtLevel(acmmLvl) && - approvalDeskAllowsLegacyOperation(ctx, approvalDesk, cfg, toolapprove.KindPlanFromLabel, "plan-from-label", planning.ArchitectAgentName, logger) { - planFromLabeledIssues(actionable, beadStores, agentMgr, gov, dashSrv, logger, cfg, acmmLvl) - } - - // Advisory digest: build from beads (the source of truth) before status broadcast. - primaryRepo := primaryAdvisoryRepo(cfg) - issueNum, hasPinnedAdvisoryIssue := advisoryIssueNumber(advisoryIssues, primaryRepo) - hasExistingPinnedIssueForEmptyDigest := hasPinnedAdvisoryIssue && - primaryRepo == primaryRepoAtCycleStart && - hadPinnedAdvisoryIssueAtCycleStart - if shouldBuildAdvisoryDigest(beadStores, ghClient, hasExistingPinnedIssueForEmptyDigest) { - // Mark findings no agent has re-reported inside the staleness window - // BEFORE the digest is built, so stale evidence is captioned. Silence is - // not proof a finding healed: an agent run can be partial, truncated, or - // non-deterministic, so absence cannot move an item to Recently Resolved. - advCfg := cfg.Governor.Advisory - if advCfg.StalenessDays > 0 { - if marked := advisory.MarkStaleAdvisoryBeads(beadStores, time.Duration(advCfg.StalenessDays)*24*time.Hour); len(marked) > 0 { - logger.Info("marked advisory findings not re-reported within the staleness window", - "count", len(marked), "staleness_days", advCfg.StalenessDays, "titles", strings.Join(marked, "; ")) - } - } - // Repo entries may be org-qualified ("org/repo"); the digest linkifier - // needs the bare repo name alongside the org. - org, repoName := cfg.Project.Org, primaryRepo - if parts := strings.SplitN(primaryRepo, "/", 2); len(parts) == 2 { - org, repoName = parts[0], parts[1] - } - - // #3704: pin the digest to ONE repo commit. Resolve the target repo's - // latest commit ONCE here (invariants 1 & 3), cite it in the rendered - // comment (invariant 2, via the footer FormatDigestMarkdown emits when - // AnalyzedSnapshot is set), and verify each finding's file path against - // that exact commit so a since-removed path (e.g. "docs/install.md") is - // flagged as outdated rather than cited as live. Best-effort: if the SHA - // cannot be resolved, fall back to the previous unpinned behavior rather - // than skip the digest. - // - // This is resolved BEFORE the digest is built because the top-N cap - // consumes it: ranking cannot prefer a live finding over a since-removed - // one unless it knows which is which at ranking time (#2364). - digestOpts := advisory.DigestOptions{ - MaxFindings: advCfg.MaxFindings, - ShowAll: advCfg.ShowAll, - } - if ghClient != nil { - pinDigestSnapshot(ctx, &digestOpts, org, repoName, primaryRepo, cfg.Policies.Branch, digestSnapshotDeps{ - defaultBranch: func(ctx context.Context, owner, repo string) (string, error) { - r, _, err := ghClient.GetRepo(ctx, owner, repo) - if err != nil { - return "", err - } - return r.GetDefaultBranch(), nil - }, - latestCommit: ghClient.LatestCommitHash, - pathExists: ghClient.PathExistsAtRef, - issueClosedAt: ghClient.IssueClosedAt, - }, logger) - } - digest := advisory.BuildDigestFromBeads(beadStores, string(govState.Mode), digestOpts) - enrichAdvisoryLinkedWork(ctx, ghClient, digest, org, repoName, logger) - if advisoryStore != nil { - advisoryStore.SetLatestDigest(digest) - } - dashSrv.SetAdvisoryDigest(digest) - statusPayload.AdvisoryDigest = digest - statusPublished = dashSrv.UpdateStatusIfFresh(statusPayload, buildEpoch) - if !statusPublished { - return - } - hiveAdvice := statusPayload.HiveAdvice - - // Post whenever there is something CURRENT to say: open findings, - // recently resolved ones, or an empty evaluation for a hive that already - // has a pinned advisory issue. The resolved and empty cases matter for - // freshness: otherwise the pinned comment and AdvisoryLastPostedAt - // freeze after the last finding disappears, and the hub reports a stale - // advisory loop even though the agents are running cleanly. - // - // advisoryPostDue additionally paces the GitHub write to the - // operator's governor.advisory.update_interval_s (#4820); 0/unset - // keeps this exact per-cycle cadence. cfg is read live each cycle — - // the same pattern as the staleness/max-findings knobs above — so a - // dashboard edit applies from the next cycle without a restart. The - // digest itself and the dashboard state above still refresh every - // cycle; only the comment write is throttled. Note the #4821 - // write-through counts consecutive unchanged post ATTEMPTS, so its - // forced full rewrite stretches with this interval (60 attempts × - // interval) — acceptable, since it only heals out-of-band comment - // edits, and documented in the settings tooltip. - // governor.advisory.target routes the comment write: GitHub (default, - // the unchanged path below) or a designated Linear issue. For the - // Linear route the configured issue plays the pinned issue's role in - // the empty-digest freshness rule, so a clean Linear-sourced hive - // keeps refreshing its comment exactly as a GitHub one does. - advisoryTarget, advisoryLinearIssue, advisoryRouteErr := resolveAdvisoryDigestRoute(cfg) - hasDigestHome := hasExistingPinnedIssueForEmptyDigest || - (advisoryTarget == config.AdvisoryTargetLinear && advisoryRouteErr == nil) - if shouldPostAdvisoryDigest(digest, ghClient, hasDigestHome) && - advisoryPostDue(advCfg, primaryRepo, time.Now(), logger) { - bySeverity, agentNames := summarizeDigestForLog(digest) - logger.Info("advisory digest built", - "total_findings", digest.TotalCount, - "critical", bySeverity["critical"], - "high", bySeverity["high"], - "medium", bySeverity["medium"], - "low", bySeverity["low"], - "agents", agentNames, - "resolved_count", len(digest.RecentlyResolved), - ) - if digest.TotalCount == 0 && len(digest.RecentlyResolved) == 0 { - logger.Info("advisory digest empty — posting freshness marker", - "repo", primaryRepo, "issue", issueNum) - } - - md := advisory.FormatDigestMarkdown(digest, advisory.DigestOptions{ - MaxFindings: digestOpts.MaxFindings, - ShowAll: digestOpts.ShowAll, - Org: org, - ShowEmpty: digest.TotalCount == 0 && len(digest.RecentlyResolved) == 0, - PrimaryRepo: repoName, - Advice: hiveAdvice, - DashboardURL: advisoryDashboardOrigin(cfg), - }) - // The routing/classification decisions live in - // publishAdvisoryDigest (#7232); only the effects are wired here. - // The App is the sole advisory-digest writer (#1927): a user-token - // failure must never drive the App banner, so every banner hook - // below is fed exclusively by the App's own error. - publishAdvisoryDigest(ctx, md, digest, advisoryPublishRoute{ - target: advisoryTarget, - linearIssue: advisoryLinearIssue, - routeErr: advisoryRouteErr, - primaryRepo: primaryRepo, - issueNum: issueNum, - hasPinnedIssue: hasPinnedAdvisoryIssue, - ensureErr: advisoryEnsureErr, - }, advisoryPublishDeps{ - postGitHub: ghClient.PostAdvisoryDigest, - postLinear: func(ctx context.Context, linearIssue, md string) error { - return postAdvisoryDigestToLinear(ctx, cfg, linearIssue, md) - }, - recordError: dashSrv.RecordAdvisoryError, - recordPosted: func(findings, overflow int) { - dashSrv.RecordAdvisoryPost(findings) - recordAdvisoryPostSuccess(primaryRepo, time.Now()) - dashSrv.RecordAdvisoryOverflow(overflow) - }, - onWriteForbidden: func(ctx context.Context) { - // App is installed (we found the issue) but a real WRITE - // was forbidden. #2353: attribute this honestly — surface - // a DISTINCT write-forbidden state naming the likeliest - // cause instead of faking a permission gap the diagnosis - // just disproved. - msg, state := classifyGitHubAppWriteForbidden(ctx, ghClient.AppAuth(), cfg.Project.Org, primaryRepo) - dashSrv.SetGitHubAppRequired(true) - dashSrv.SetGitHubAppPermIssue(msg) - dashSrv.SetGitHubAppState(state.String()) - logger.Warn("GitHub App write failed — cannot write issue comments", - "repo", primaryRepo, "state", state.String(), - "operator_actionable", state.OperatorActionable(), "detail", msg) - }, - onRepoMoved: func(ctx context.Context, moved *github.RepoMovedError) { - // #11791: the write was refused because the repository - // now lives under another owner. The installation is - // healthy, so the generic write-forbidden remedy would - // misdirect the operator; name the move instead. - dashSrv.SetGitHubAppRequired(true) - dashSrv.SetGitHubAppPermIssue(moved.Error()) - dashSrv.SetGitHubAppState(github.AppStateRepoMoved.String()) - logger.Warn("GitHub App write failed — repository transferred to another owner", - "configured", moved.Configured, "current", moved.Current) - }, - recordRepoPaused: func(repo string) { - recordAdvisoryPostSuccess(repo, time.Now()) - }, - recordWriteForbidden: func(repo string) { - backoff := advCfg.PostRetryBackoff() - recordAdvisoryPostForbidden(repo, time.Now(), backoff) - if backoff > 0 { - logger.Warn("advisory digest write forbidden — pausing post attempts", - "repo", repo, "retry_after", backoff.String(), - "knob", "governor.advisory.post_retry_backoff_s") - } - }, - onAuthFailure: func(ctx context.Context) { - // Same verdict function as boot and Re-check, so a healthy - // or unclassifiable probe cannot raise the banner here. - raise, msg, state := classifyGitHubAppFailure(ctx, ghClient.AppAuth(), cfg.Project.Org, logger) - if !raise { - return - } - dashSrv.SetGitHubAppRequired(true) - if msg != "" { - dashSrv.SetGitHubAppPermIssue(msg) - } - dashSrv.SetGitHubAppState(state.String()) - logger.Warn("GitHub App authentication failed posting advisory digest", - "repo", primaryRepo, "state", state.String(), - "operator_actionable", state.OperatorActionable()) - }, - onWriteProven: func(ctx context.Context) { - // A successful write proves the app is installed AND has - // write access — clear BOTH the perm issue and the - // app-required flag, or the "Not Installed" banner sticks - // forever despite tokens working. - dashSrv.SetGitHubAppPermIssue("") - dashSrv.SetGitHubAppRequired(false) - dashSrv.ClearPendingGitHubAppInstall() - // The same proof retires stale ACCESS findings (#2575). - if healed := advisory.CloseHealedAppAuthFindings(beadStores); len(healed) > 0 { - logger.Info("closed healed GitHub App access findings after successful App digest post", - "count", len(healed), "titles", strings.Join(healed, "; ")) - } - // Repo-ACCESS findings need their own proof: a digest post - // only proves issues:WRITE, so verify with a real - // advisor-scoped Contents read, memoized per repo. - readVerified := map[string]bool{} - canRead := func(ownerRepo string) bool { - target := ownerRepo - if target == "" { - target = primaryRepo - } - owner, name := cfg.Project.Org, target - if i := strings.LastIndex(target, "/"); i > 0 { - owner, name = target[:i], target[i+1:] - } - if owner == "" || name == "" { - return false - } - key := owner + "/" + name - if v, ok := readVerified[key]; ok { - return v - } - appAuth := ghClient.AppAuth() - if appAuth == nil { - // Static-token client: no advisor-tier token can be - // minted, so leave the finding open. - return false - } - err := appAuth.VerifyRepoRead(ctx, owner, name) - if err != nil { - logger.Info("repo-access finding left open: advisor read probe failed", - "repo", key, "error", err) - } - readVerified[key] = err == nil - return readVerified[key] - } - if healed := advisory.CloseHealedRepoAccessFindings(beadStores, canRead); len(healed) > 0 { - logger.Info("closed healed repo-access findings after verified advisory read path", - "count", len(healed), "titles", strings.Join(healed, "; ")) - } - }, - }, logger) - } - } else if d := dashSrv.GetAdvisoryDigest(); d != nil { - statusPayload.AdvisoryDigest = d - } - - if !statusPublished { - dashSrv.UpdateStatusIfFresh(statusPayload, buildEpoch) - } - - publishFleetReportsWith(ctx, logger, dashSrv, statusPayload.FleetReport, newSelfReportPublishDeps(cfg, ghClient, dashSrv)) - - if agentStats := dashboard.CollectAgentStats(statusPayload); len(agentStats) > 0 { - gov.AttachAgentStats(agentStats) - } - - if repoSnaps := dashboard.CollectRepoSnapshots(statusPayload); len(repoSnaps) > 0 { - gov.AttachRepoSnapshots(repoSnaps) - } - - if nousState != nil { - var tokenSummary *tokens.AggregateSummary - if tokenCollector != nil { - tokenSummary = tokenCollector.Summary() - } - if err := nousState.RecordSnapshot(govState, actionable, agentsDue, agentMgr.AllStatuses(), tokenSummary); err != nil { - logger.Warn("failed to record nous snapshot", "error", err) - } - } -} - -func convertKnowledgeLayers(cfgLayers []config.KnowledgeLayer) []knowledge.LayerConfig { - layers := make([]knowledge.LayerConfig, len(cfgLayers)) - for i, l := range cfgLayers { - layers[i] = knowledge.LayerConfig{ - Type: knowledge.LayerType(l.Type), - Path: l.Path, - URL: l.URL, - Shared: l.Shared, - } - } - return layers -} - -// curatorConfigFromHive maps the hive.yaml curator block onto the knowledge -// package's own config. Enabled is carried across as a pointer so "absent" -// stays distinguishable from "explicitly false" — the scheduled promotion loop -// treats absent as OFF, and flattening it to a bool here would quietly turn -// unreviewed promotion on fleet-wide (#5430). -func curatorConfigFromHive(c config.KnowledgeCurator) knowledge.CuratorConfig { - return knowledge.CuratorConfig{ - Enabled: c.Enabled, - Schedule: c.Schedule, - ExtractFrom: c.ExtractFrom, - AutoPromoteThreshold: c.AutoPromoteThreshold, - PromoteFrom: c.PromoteFrom, - PromoteTo: c.PromoteTo, - } -} - -// hiveIDFilePath is the persistent file where the Hive ID is stored across restarts. -var hiveIDFilePath = "/data/hive-id" - -func loadOrGenerateHiveID(logger *slog.Logger) string { - if envID := os.Getenv("HIVE_ID"); envID != "" { - if err := os.WriteFile(hiveIDFilePath, []byte(envID+"\n"), 0o644); err == nil { - logger.Info("hive ID set from HIVE_ID env var", "id", envID) - } - return envID - } - - if data, err := os.ReadFile(hiveIDFilePath); err == nil { - id := strings.TrimSpace(string(data)) - if id != "" { - logger.Info("hive ID loaded from disk", "id", id) - return id - } - } - - id := "hive-" + randomName() - - if err := os.WriteFile(hiveIDFilePath, []byte(id+"\n"), 0o644); err != nil { - logger.Warn("failed to persist hive ID", "error", err) - } else { - logger.Info("generated new hive ID", "id", id) - } - - return id -} - -// randomName generates a Docker-style adjective-noun name. -func randomName() string { - adjectives := []string{ - "bold", "calm", "cool", "dark", "deep", "fair", "fast", "keen", - "kind", "loud", "mild", "neat", "pale", "pure", "rare", "rich", - "safe", "slim", "soft", "tall", "thin", "true", "vast", "warm", - "wise", "able", "busy", "easy", "epic", "free", "glad", "good", - "idle", "just", "lazy", "lean", "live", "long", "lost", "main", - "next", "open", "real", "sure", "wild", "worn", "zero", "blue", - } - nouns := []string{ - "ant", "ape", "bat", "bee", "cow", "doe", "eel", "elk", - "fox", "gnu", "hen", "jay", "kit", "lark", "moth", "newt", - "owl", "pug", "ram", "ray", "seal", "swan", "toad", "wren", - "bear", "colt", "crow", "deer", "dove", "duck", "fawn", "frog", - "goat", "gull", "hare", "hawk", "ibis", "lynx", "mink", "mole", - "orca", "pike", "puma", "slug", "stag", "wolf", "yak", "wasp", - } - - buf := make([]byte, 2) - if _, err := rand.Read(buf); err != nil { - return "bold-ant" - } - adj := adjectives[int(buf[0])%len(adjectives)] - noun := nouns[int(buf[1])%len(nouns)] - return adj + "-" + noun -} - -// watchdogAuthProbes builds the per-provider credential probes for the -// watchdog by adapting the rotation package's provider probers (#4608) — -// the same machinery the #4645 probe rewrite targets, so that rewrite reaches -// the watchdog automatically. -func watchdogAuthProbes(cfg *config.Config) map[string]watchdog.AuthProbe { - threshold := cfg.Governor.Rotation.EffectiveThreshold() - probers := []rotation.Prober{ - rotation.ClaudeProber{ThresholdPct: threshold}, - rotation.CodexProber{ThresholdPct: threshold}, - rotation.AgyProber{ThresholdPct: threshold}, - rotation.DeepSeekProber{}, - } - out := make(map[string]watchdog.AuthProbe, len(probers)) - for _, p := range probers { - out[p.Provider()] = watchdog.RotationAuthProbe{Prober: usageProber{p}} - } - return out -} - -// usageProber adapts a rotation prober to the watchdog's UsageProber slice -// (provider key + sanitised probe error); the watchdog does not import -// pkg/rotation. -type usageProber struct{ rotation.Prober } - -func (u usageProber) ProbeError(ctx context.Context) string { - return u.Prober.Probe(ctx).ProbeError() -} - -// turnLossToSnapshot converts the manager's in-memory turn-loss accumulation -// into its persisted form, or nil when nothing has been recorded. -// -// Nil rather than a zero struct on purpose: `turn_loss` is omitempty, so an -// agent that has never been interrupted adds nothing to /data/hive-state.json. -// The overwhelming majority of agents are in that state, and a measurement that -// bloated every hive's state file with empty records would be its own argument -// for removing it. -func turnLossToSnapshot(loss crux.TurnLoss) *snapshot.AgentTurnLoss { - if loss.Interruptions == 0 && len(loss.Recent) == 0 { - return nil - } - out := &snapshot.AgentTurnLoss{ - Interruptions: loss.Interruptions, - Producing: loss.Producing, - UpperBoundS: loss.UpperBound.Seconds(), - Bytes: loss.Bytes, - } - for _, r := range loss.Recent { - rec := snapshot.AgentTurnInterruption{ - At: r.At, - Reason: r.Reason, - SinceKickS: r.SinceKick.Seconds(), - Producing: r.Producing, - Bytes: r.Bytes, - } - if r.SinceOutput != nil { - s := r.SinceOutput.Seconds() - rec.SinceOutputS = &s - } - out.Recent = append(out.Recent, rec) - } - return out -} - -func restartEventsToSnapshot(events []crux.RestartEvent) []snapshot.AgentRestartEvent { - if len(events) == 0 { - return nil - } - out := make([]snapshot.AgentRestartEvent, 0, len(events)) - cutoff := time.Now().Add(-24 * time.Hour) - for _, ev := range events { - if ev.At.IsZero() || ev.At.Before(cutoff) { - continue - } - out = append(out, snapshot.AgentRestartEvent{At: ev.At, Reason: ev.Reason}) - } - return out -} - -func restartEventsFromSnapshot(events []snapshot.AgentRestartEvent) []crux.RestartEvent { - if len(events) == 0 { - return nil - } - out := make([]crux.RestartEvent, 0, len(events)) - cutoff := time.Now().Add(-24 * time.Hour) - for _, ev := range events { - if ev.At.IsZero() || ev.At.Before(cutoff) { - continue - } - out = append(out, crux.RestartEvent{At: ev.At, Reason: ev.Reason}) - } - return out -} - -// persistPathsForRuntime resolves the file set persistState writes. It is a -// package var (the same seam pattern as githubAppTokenCachePath) rather than a -// direct defaultPersistPaths() call so the wrapper's delegation can be pinned -// by a test without writing the live /data files the default set hardwires -// (#6846). persistState is synchronous, so a test that swaps and restores -// this var cannot race a goroutine still reading it. -var persistPathsForRuntime = defaultPersistPaths - -func persistState(agentMgr crux.Runtime, gov *governor.Governor, cfg *config.Config, path string, logger *slog.Logger, dashSrv *dashboard.Server, wd *watchdog.Reconciler) { - persistStateWithPaths(agentMgr, gov, cfg, path, logger, dashSrv, wd, persistPathsForRuntime()) -} - -func persistStateWithPaths(agentMgr crux.Runtime, gov *governor.Governor, cfg *config.Config, path string, logger *slog.Logger, dashSrv *dashboard.Server, wd *watchdog.Reconciler, paths persistPaths) { - statuses := agentMgr.AllStatuses() - agents := make(map[string]snapshot.AgentState, len(statuses)) - for name, proc := range statuses { - as := snapshot.AgentState{ - Paused: proc.Paused, - PinnedCLI: proc.PinnedCLI, - PinnedModel: proc.PinnedModel, - ModelOverride: proc.ModelOverride, - BackendOverride: proc.BackendOverride, - RestartCount: proc.RestartCount, - RestartEvents: restartEventsToSnapshot(proc.RestartEvents), - LastRestartReason: proc.LastRestartReason, - LastKick: proc.LastKick, - PausedReason: proc.PausedReason, - PausedTrigger: proc.PausedTrigger, - PausedBy: proc.PausedBy, - TurnLoss: turnLossToSnapshot(proc.TurnLoss), - } - if !proc.PausedAt.IsZero() { - t := proc.PausedAt - as.PausedAt = &t - } - if len(proc.KickHistory) > 0 { - as.KickHistory = make([]snapshot.AgentKickEntry, len(proc.KickHistory)) - for i, kr := range proc.KickHistory { - as.KickHistory[i] = snapshot.AgentKickEntry{ - Timestamp: kr.Timestamp, - Agent: kr.Agent, - Snippet: kr.Snippet, - } - } - } - if agentCfg, ok := cfg.Agents[name]; ok { - as.DisplayName = agentCfg.DisplayName - as.Description = agentCfg.Description - enabled := agentCfg.Enabled - as.Enabled = &enabled - clearOnKick := agentCfg.ClearOnKick - as.ClearOnKick = &clearOnKick - staleTimeout := agentCfg.StaleTimeout - as.StaleTimeout = &staleTimeout - as.RestartStrategy = agentCfg.RestartStrategy - as.LaunchCmd = agentCfg.LaunchCmd - } - agents[name] = as - } - - cadenceOverrides := make(map[string]map[string]config.Cadence) - for modeName, mode := range cfg.Governor.Modes { - if len(mode.Cadences) > 0 { - cadenceOverrides[modeName] = make(map[string]config.Cadence, len(mode.Cadences)) - for agentName, cadence := range mode.Cadences { - cadenceOverrides[modeName][agentName] = cadence - } - } - } - - budget := gov.GetBudget() - govState := gov.GetState() - - govKickHistory := gov.KickHistory() - kickEntries := make([]snapshot.GovKickEntry, len(govKickHistory)) - for i, kr := range govKickHistory { - kickEntries[i] = snapshot.GovKickEntry{Timestamp: kr.Timestamp, Agent: kr.Agent, Outcome: kr.Outcome, OutcomeReason: kr.OutcomeReason} - } - - state := &snapshot.PersistedState{ - Agents: agents, - GovernorMode: string(govState.Mode), - BudgetLimit: budget.WeeklyLimit, - BudgetIgnored: budget.IgnoredAgents, - BudgetIgnoreAll: budget.IgnoreAll, - CadenceOverrides: cadenceOverrides, - LastKicks: govState.LastKick, - BudgetSpend: budget.CurrentSpend, - BudgetResetAt: budget.ResetAt, - BudgetByAgent: budget.ByAgent, - BudgetByModel: budget.ByModel, - CoinBudgetLimit: budget.CoinLimit, - CoinBudgetSpend: budget.CoinSpend, - CoinBudgetLabel: budget.CoinLabel, - USDBudgetLimit: budget.USDLimit, - USDBudgetSpend: budget.USDSpend, - BudgetWindowBaseline: budget.WindowBaseline, - CoinWindowBaseline: budget.CoinWindowBaseline, - USDWindowBaseline: budget.USDWindowBaseline, - KickHistory: kickEntries, - LastEval: govState.LastEval, - ACMMLevel: cfg.ACMMLevel, - } - - // Persist the fleet breaker so an engaged kill-switch survives a restart. - // Only written when engaged — a never-thrown breaker adds nothing. - if engaged, breakerPaused := agentMgr.BreakerState(); engaged { - state.Breaker = &snapshot.BreakerState{Engaged: true, Paused: breakerPaused} - } - - // Persist the watchdog's backoff/crash-loop/condition state (RFC #4665 - // open question 2: it rides the existing state file). - if wd != nil { - if wdState := wd.Snapshot(); len(wdState) > 0 { - state.Watchdog = wdState - } - } - - if err := snapshot.SaveState(path, state, logger); err != nil { - logger.Error("failed to persist state", "error", err) - } - - // Component reach counters (#3993) ride the SAME save cadence as the main - // state file but live in their own file (reachStatePath — resolved OQ-2 of - // #3973), so a reach write failure never corrupts agent/governor state. - if err := tracing.SaveReachState(paths.ReachState); err != nil { - logger.Error("failed to persist reach state", "error", err) - } - - // Reconcile the persisted pause field from the authoritative live manager - // state and save, atomically under the config save mutex. persistState runs - // async (go PersistFunc()) on every pause/resume; doing the c.Agents update - // and Save under saveMu (via ReconcilePausedAndSave) means it can neither - // race the pause callback's map write nor clobber its file write with a - // stale paused=false. livePaused is built from AllStatuses(), read above. - livePaused := make(map[string]bool, len(agents)) - for name, as := range agents { - livePaused[name] = as.Paused - } - if err := cfg.ReconcilePausedAndSave(livePaused); err != nil { - logger.Error("failed to persist config to yaml", "error", err) - if dashSrv != nil { - dashSrv.AddSystemAlert("config-save-failed", "error", - "Config save failed — runtime state (ACMM level, agent config) will be lost on restart: "+err.Error()) - } - } else if dashSrv != nil { - dashSrv.ClearSystemAlert("config-save-failed") - } - - history := gov.EvalHistory() - if len(history) > 0 { - historyData, err := json.Marshal(history) - if err == nil { - atomicWrite(paths.SparklineHistory, historyData) - } - } - - modeHistory := gov.ModeHistory() - if len(modeHistory) > 0 { - modeData, err := json.Marshal(modeHistory) - if err == nil { - atomicWrite(paths.ModeHistory, modeData) - } - } - - if dashSrv != nil { - tokenHistory := dashSrv.TokenSparklineHistory() - if len(tokenHistory) > 0 { - tokenData, err := json.Marshal(tokenHistory) - if err == nil { - atomicWrite(paths.TokenSparklineHistory, tokenData) - } - } - - factHist := dashSrv.FactHistory() - if len(factHist) > 0 { - factData, err := json.Marshal(factHist) - if err == nil { - atomicWrite(paths.FactHistory, factData) - } - } - - costHist := dashSrv.CostHistory() - if len(costHist) > 0 { - costData, err := json.Marshal(costHist) - if err == nil { - atomicWrite(paths.CostHistory, costData) - } - } - - costRollup := dashSrv.CostDailyRollups() - if len(costRollup) > 0 { - rollupData, err := json.Marshal(costRollup) - if err == nil { - atomicWrite(paths.CostDailyRollup, rollupData) - } - } - - trendHist := dashSrv.TrendHistory() - if len(trendHist) > 0 { - trendData, err := json.Marshal(trendHist) - if err == nil { - atomicWrite(paths.TrendHistory, trendData) - } - } - - // #4298: per-budget-window history. Written on the same cadence as the - // other series so a pod roll cannot lose more of one than the others. - budgetHist := dashSrv.BudgetWindowHistory() - if len(budgetHist) > 0 { - budgetData, err := json.Marshal(budgetHist) - if err == nil { - atomicWrite(paths.BudgetWindowHistory, budgetData) - } - } - - // #4263: convergence soak telemetry, written atomically on the same - // cadence as the other series so a pod roll cannot lose more of one - // than the others. - soakHist := dashSrv.ConvergenceSoakHistory() - if len(soakHist) > 0 { - soakData, err := json.Marshal(soakHist) - if err == nil { - atomicWrite(paths.ConvergenceSoak, soakData) - } - } - } -} - -var ( - escalationStoreOnce sync.Once - escalationStore *escalation.Store -) - -const escalationLedgerPath = "/data/metrics/fix-streaks.json" - -// getEscalationStore lazily loads the shared fix-loop ledger (staleness clock + -// distinct-SHA attempt counts + re-engagement caps). It is the SINGLE -// loop-safety/dedup authority shared by the re-engagement paths (#2 merge -// watcher, #3 claim release, #4 reaper) and the human-escalation sweep, so they -// all agree on which red PRs are stale, how many times each has been -// re-nudged, and which have crossed the human-escalation threshold. -func getEscalationStore() *escalation.Store { - escalationStoreOnce.Do(func() { - escalationStore = escalation.Load(escalationLedgerPath) - }) - return escalationStore -} - -// hivePRObservations projects the enumerated hive-authored PRs into escalation -// observations (repo fully-qualified, Red == a required check failed). Shared -// by recordRedStaleness and the reaper so both classify PRs identically. -// -// owners maps "org/repo#number" to the agent that opened the PR (see -// auditPRAgents); pass nil when ownership is not needed. Shared/base CI -// breakage is classified across every PR in the pass before human and -// non-Hive observations are removed. A green PR is counter-evidence, and an -// independent human-authored PR must confirm the shared failure. -func hivePRObservations(cfg *config.Config, actionable *github.ActionableResult, owners map[string]string) []escalation.Observation { - if actionable == nil { - return nil - } - all := allEscalationObservations(cfg, actionable, owners) - agentObs := all[:0] - for i, pr := range actionable.PRs.Items { - if isHiveAgentAuthor(cfg, pr.Author) { - agentObs = append(agentObs, all[i]) - } - } - return agentObs -} - -func allEscalationObservations(cfg *config.Config, actionable *github.ActionableResult, owners map[string]string) []escalation.Observation { - all := make([]escalation.Observation, 0, len(actionable.PRs.Items)+len(actionable.PRs.Held)) - for _, pr := range actionable.PRs.Items { - all = append(all, escalationObservation(cfg, pr, owners)) - } - for _, pr := range actionable.PRs.Held { - all = append(all, escalationObservation(cfg, pr, owners)) - } - escalation.MarkSharedFailures(all) - return all -} - -// escalationObservation projects one enumerated PR into the fix-loop ledger's -// view of it. Red means a required check concluded failure. Pending means this -// pass could not conclude CI at all — checks still running, no check runs, or -// (per EnrichCIStatus) the check-run fetch errored — which the ledger must -// treat as "no information", never as "went green". Labeled mirrors the forge's -// needs-human label so the ledger and the label can never disagree about -// whether a PR has already been handed to a human. Labels carry the PR's -// current forge labels so Sweep can reconcile reviewer-lane verdicts -// (label-only edits: needs-human removed, reviewer-passed added) back into -// the ledger (#5511, gap G1). -func escalationObservation(cfg *config.Config, pr github.PullRequest, owners map[string]string) escalation.Observation { - repo := pr.Repo - if !strings.Contains(repo, "/") && cfg.Project.Org != "" { - repo = cfg.Project.Org + "/" + repo - } - red := pr.HasFailingRequiredCheck() - return escalation.Observation{ - Repo: repo, - Number: pr.Number, - HeadSHA: pr.HeadSHA, - HeadTree: pr.HeadTree, - Red: red, - Pending: !red && pr.CIStatus != "success", - CIRunning: red && pr.CIChecksRunning, - Labeled: escalation.HasNeedsHumanLabel(pr.Labels), - Excerpt: pr.CIFailureExcerpt, - Labels: pr.Labels, - FailingChecks: pr.FailingChecks, - BaseRedChecks: pr.BaseRedChecks, - SharedCIIncident: pr.SharedCIIncident, - Agent: owners[escalation.Key(repo, pr.Number)], - IndependentControl: isIndependentCIControl(cfg, pr), - } -} - -// redPRFixOwner resolves who should be asked to repair a red PR: the agent the -// audit trail attributes it to, or — when that agent is unattributed — the -// fleet's primary PR creator, exactly the default formatRedPRFixData applies -// to an unattributed ci-failing.json row. Keeping the two in step is what lets -// a hook predicate (`t.agent == "scanner"`) and the FIX-BEFORE-NEW block name -// the same owner (hivecommons/hive#9474). -func redPRFixOwner(agent string) string { - if agent = strings.TrimSpace(agent); agent != "" { - return agent - } - return review.DefaultFixerAgent -} - -// dependencyBots are forge bots whose PRs are dependency bumps, not hive fix -// attempts. They carry the "[bot]" suffix that otherwise marks a PR as -// agent-authored, but nothing in the hive opened them and no hive agent is -// iterating on them, so a red one is not a fix loop to break: escalating it -// only pages a human with "1 distinct fix attempts" about a crate bump that -// renovate will rebase on its own. Mirrors the hub's default contribute -// deny-authors list (config: hub.contribute_deny_authors). -var dependencyBots = map[string]bool{ - "renovate[bot]": true, - "dependabot[bot]": true, - "mergeraptor[bot]": true, -} - -// isHiveAgentAuthor reports whether a PR author is one of OUR agents — the -// configured ai_author, the App bot identity agents author as, or another -// bot account — excluding the dependency bots above. Shared by every fix-loop -// path (staleness clock, reaper, escalation sweep) so they classify PRs -// identically. -func isHiveAgentAuthor(cfg *config.Config, author string) bool { - if author == "" { - return false - } - if author == cfg.Project.AIAuthor { - return true - } - if eff := cfg.EffectiveAIAuthor(); eff != "" && author == eff { - return true - } - return strings.HasSuffix(author, "[bot]") && !dependencyBots[author] -} - -// isIndependentCIControl reports whether a PR supplies human-authored evidence -// independent of Hive. Bot-authored and Hive-attributed contributor-relay PRs -// cannot prove that a failure also affects unrelated human work. -func isIndependentCIControl(cfg *config.Config, pr github.PullRequest) bool { - return !isHiveAgentAuthor(cfg, pr.Author) && !pr.AuthorNonHuman && !pr.HiveAttributed -} - -// recordRedStaleness updates the shared staleness clock (first-seen-red per red -// head SHA) for every hive-authored PR in this pass. It must run before the -// claim guard and the reaper so their StaleRed() reads reflect the current tick. -// A disabled escalation subsystem skips it (the whole fix-loop machinery is off). -// -// It also fires the escalation_red hook for each red PR, WITH the owning agent -// on the payload: the catalog and hooks.md have always documented `agent` on -// this transition, but nothing ever set it, so a predicate like -// `t.agent == "scanner"` could never match (hivecommons/hive#9474). -func recordRedStaleness(cfg *config.Config, actionable *github.ActionableResult) { - if cfg.Escalation.Disabled || actionable == nil { - return - } - owners := auditPRAgents(cfg.Project.Org, time.Now().Add(-auditPRAttributionWindow), "") - obs := hivePRObservations(cfg, actionable, owners) - getEscalationStore().ObserveRed(obs) - for _, ob := range obs { - if !ob.Red { - continue - } - hookDispatcher().Fire(context.Background(), hooks.Payload{ - Transition: hooks.TransitionEscalationRed, - Repo: ob.Repo, - Agent: redPRFixOwner(ob.Agent), - Reason: "required CI check red", - Attrs: map[string]string{ - hooks.AttrPR: strconv.Itoa(ob.Number), - "head_sha": ob.HeadSHA, - "excerpt": ob.Excerpt, - }, - }) - } -} - -// fixKicker is the out-of-band kick path the re-engagement lanes deliver -// through. crux.Runtime satisfies it via agentKicker, which is the same -// adapter the governor tick's replan lane uses. -type fixKicker interface { - Kick(agent, message string) error -} - -// agentAvailability reports, per agent, whether a kick sent right now could -// reach it: the agent must not be paused by the manager, and at least one -// governor mode must schedule it with a non-paused cadence. An agent paused in -// every mode (the v5 L6 pack pauses `strategist` everywhere) can never answer a -// FIX-BEFORE-NEW kick, so charging its red PRs' re-engagement budget only -// escalates them to a human having asked nobody (hivecommons/hive#9472). -func agentAvailability(cfg *config.Config, mgr crux.Runtime) func(string) bool { - return func(name string) bool { - if name == "" { - return false - } - if mgr != nil && mgr.IsPaused(name) { - return false - } - return cfg == nil || cadenceReachable(cfg, name) - } -} - -// cadenceReachable reports whether ANY governor mode gives the agent a cadence -// that is not paused/off. An agent with no cadence entry at all is treated as -// reachable: "nothing schedules it" is a fleet-configuration signal of its own -// (NoCadenceAgents), and refusing to route repairs to it would silently widen -// this gate into an agent-disablement policy. -func cadenceReachable(cfg *config.Config, name string) bool { - configured := false - for mode := range cfg.Governor.Modes { - cad := cfg.CadenceValueForMode(name, mode) - if cad == "" { - continue - } - configured = true - if !cad.IsPaused() { - return true - } - } - return !configured -} - -// ownerReEngageCooldown is the minimum spacing between two re-engagements of a -// PR owned by this agent: its slowest configured cadence, so a budget of six -// attempts cannot be spent inside a single cadence window where no kick the -// owner could answer would ever fit. The escalation store floors it at -// ReEngageCooldown and caps it at MaxReEngageCooldown. -func ownerReEngageCooldown(cfg *config.Config, name string) time.Duration { - if cfg == nil || name == "" { - return 0 - } - var slowest time.Duration - for mode := range cfg.Governor.Modes { - cad := cfg.CadenceValueForMode(name, mode) - if cad == "" || cad.IsPaused() { - continue - } - d, err := time.ParseDuration(strings.TrimSpace(cad.Interval())) - if err == nil && d > slowest { - slowest = d - } - } - return slowest -} - -// resolveRedPRFixer picks the agent a stuck red PR's fix kick should go to: its -// owner, else the configured review fixer agent, else the fleet's default -// fixer — the same chain the review-fix kick uses (src/docs/review-swarm.md). -// It returns "" when no candidate is reachable, which tells the caller to leave -// the PR alone rather than spend budget nobody can answer. -func resolveRedPRFixer(cfg *config.Config, owner string, available func(string) bool) string { - candidates := []string{redPRFixOwner(owner), strings.TrimSpace(cfg.Review.FixerAgent), review.DefaultFixerAgent} - for _, c := range candidates { - if c == "" { - continue - } - if available == nil || available(c) { - return c - } - } - return "" -} - -// redPRFixKick renders the targeted FIX-BEFORE-NEW kick a re-engagement -// delivers. It names ONE PR — the stuck one — with its evidence, so the kick is -// answerable on its own rather than relying on the owner's next cadence kick -// happening to carry the ci-failing.json block. -func redPRFixKick(o escalation.Observation, rerouted bool) string { - var b strings.Builder - fmt.Fprintf(&b, "🔴 FIX-BEFORE-NEW — %s#%d is red and has not moved\n\n", o.Repo, o.Number) - if rerouted { - fmt.Fprintf(&b, "Its owning agent (%s) is not reachable right now, so this repair is routed to you.\n\n", redPRFixOwner(o.Agent)) - } - b.WriteString("Repair it BEFORE claiming new issues or opening any new PR:\n") - if slices.Contains(o.Labels, stagelabels.NoDCOLabel) { - fmt.Fprintf(&b, " FIRST (required): the PR is %s — it inherited unsigned commits. Re-sign them: gh pr checkout %d --repo %s → git rebase --signoff (original authors preserved) → git push --force-with-lease\n", stagelabels.NoDCOLabel, o.Number, o.Repo) - b.WriteString(" THEN fix any failing CI the same way.\n") - } - fmt.Fprintf(&b, " gh pr checkout %d --repo %s → fix → git commit -s → git push\n", o.Number, o.Repo) - b.WriteString("Push to the SAME branch. Do NOT open a replacement PR.\n") - b.WriteString("After the push, STOP: do not watch, poll, or sleep on CI (no gh run watch/view loops).\n") - b.WriteString("The automerge sweep merges the PR once it is green; if the failure is infrastructure\n") - b.WriteString("(runner lost, no space left, checks still queued) leave one comment and end the turn.\n") - if o.HeadSHA != "" { - fmt.Fprintf(&b, "\nhead: %s\n", o.HeadSHA) - } - if len(o.FailingChecks) > 0 { - fmt.Fprintf(&b, "failing: %s\n", strings.Join(o.FailingChecks, ", ")) - } - if excerpt := strings.TrimSpace(o.Excerpt); excerpt != "" { - if runes := []rune(excerpt); len(runes) > redPRFixKickExcerptRunes { - excerpt = string(runes[:redPRFixKickExcerptRunes]) + "…" - } - fmt.Fprintf(&b, "\nevidence:\n%s\n", excerpt) - } - return b.String() -} - -// redPRFixKickExcerptRunes bounds the CI evidence carried in a targeted fix -// kick, matching the FIX-BEFORE-NEW block's own budget. -const redPRFixKickExcerptRunes = 400 - -// mergeReEngageHook builds the Fix #2 re-engagement callback for the merge -// watcher. It normalizes the repo, resolves the PR's owner, DELIVERS a targeted -// fix kick to it (or to the fallback fixer when the owner is unreachable), and -// only then records the attempt under the shared escalation store's per-red-SHA -// cap. Passing an empty head SHA tells the store to reuse the red head SHA it -// last observed for this PR (the eval cycle's ObserveRed keeps it current), so -// the merge watcher does not need to re-fetch the head. Returns false when the -// cap is exhausted or no kick could be delivered, so the watcher can log that -// the escalation path now owns the PR. A disabled escalation subsystem yields a -// nil hook (watcher falls back to quarantine-only). -// -// Delivery is the point: before hivecommons/hive#9472 this only incremented a -// counter, so a PR could be escalated as "6 automated fix re-dispatches" having -// never been mentioned to any agent. -func mergeReEngageHook(cfg *config.Config, kicker fixKicker, available func(string) bool, logger *slog.Logger) github.MergeReEngageFunc { - if cfg.Escalation.Disabled { - return nil - } - fullRepo := func(repo string) string { - if !strings.Contains(repo, "/") && cfg.Project.Org != "" { - return cfg.Project.Org + "/" + repo - } - return repo - } - return func(repo string, number int) bool { - full := fullRepo(repo) - owners := auditPRAgents(cfg.Project.Org, time.Now().Add(-auditPRAttributionWindow), "") - o := escalation.Observation{ - Repo: full, - Number: number, - Agent: owners[escalation.Key(full, number)], - } - // Empty head SHA: reuse the store's tracked current red SHA for this PR - // (do not reset the cap counter). The eval cycle records it via - // ObserveRed; if the store has never seen this PR red, the store still - // allows the first MaxReEngagements nudges. - return deliverReEngagement(cfg, getEscalationStore(), o, "", kicker, available, logger) - } -} - -// deliverReEngagement is the one place a fix re-dispatch is both SENT and -// COUNTED. It resolves a reachable fixer, checks the budget without spending -// it, delivers the kick, and only records the attempt once the kick was -// accepted: an undeliverable kick (agent not running, provider backoff, -// restart) must not consume a PR's budget and strand it in needs-human -// (hivecommons/hive#9472). -func deliverReEngagement( - cfg *config.Config, - store *escalation.Store, - o escalation.Observation, - headSHA string, - kicker fixKicker, - available func(string) bool, - logger *slog.Logger, -) bool { - fixer := resolveRedPRFixer(cfg, o.Agent, available) - if fixer == "" { - // Nobody can be asked right now. Charging budget here is what escalated - // red strategist PRs on a pack that pauses strategist in every mode. - if logger != nil { - logger.Info("re-engagement skipped: no reachable fixer agent", - "repo", o.Repo, "pr", o.Number, "owner", redPRFixOwner(o.Agent)) - } - return false - } - if !store.ReEngageEligible(o.Repo, o.Number, headSHA, ownerReEngageCooldown(cfg, fixer)) { - return false - } - if kicker == nil { - return false - } - if err := kicker.Kick(fixer, redPRFixKick(o, fixer != redPRFixOwner(o.Agent))); err != nil { - if logger != nil { - level := slog.LevelWarn - // Pool contention is expected backpressure: retry on a later sweep. - // Keep it below WARN so slogfeed does not file a runtime defect - // for a healthy shared pool refusing another session (#15350). - if isSharedPoolBusyErr(err) { - level = slog.LevelInfo - } - logger.Log(context.Background(), level, "re-engagement kick undeliverable; budget not charged", - "repo", o.Repo, "pr", o.Number, "agent", fixer, "error", err) - } - return false - } - store.RecordReEngagement(o.Repo, o.Number) - return true -} - -// reapStuckRedPRs is Fix #4: the governor's backstop sweep. For each -// hive-authored PR that is red on a required check AND stale (StaleRed) AND not -// already escalated to a human, it DELIVERS a targeted FIX-BEFORE-NEW kick to -// the PR's owner (or the fallback fixer when the owner is unreachable) and -// records the re-engagement only if that kick was accepted — deduped and capped -// via the escalation store. Entirely generic: it keys only off check state + -// staleness, never a linter. -func reapStuckRedPRs( - cfg *config.Config, - actionable *github.ActionableResult, - escalatedPRs map[string]bool, - kicker fixKicker, - available func(string) bool, - logger *slog.Logger, -) { - if cfg.Escalation.Disabled || actionable == nil { - return - } - store := getEscalationStore() - owners := auditPRAgents(cfg.Project.Org, time.Now().Add(-auditPRAttributionWindow), "") - for _, o := range hivePRObservations(cfg, actionable, owners) { - if !o.Red { - continue - } - if o.Shared { - // Every failing check is red across the fleet: the PR is not - // broken, a shared incident is. Nudging its owner to "fix" it - // wastes budget on work that cannot succeed (hivecommons/hive#9473). - continue - } - key := escalation.Key(o.Repo, o.Number) - if escalatedPRs[key] { - // Already handed to a human (needs-human label); kick builders skip - // it and we must not re-dispatch automated fixes. - continue - } - if !store.StaleRed(o.Repo, o.Number, o.HeadSHA) { - continue // still churning (fresh red SHA) — leave it to the fix agent - } - if !deliverReEngagement(cfg, store, o, o.HeadSHA, kicker, available, logger) { - // Cap reached, cooldown not elapsed, or nothing could be delivered. - // The distinct-SHA escalation path owns it from here. - continue - } - logger.Info("reaper: re-dispatched fix for stuck red PR", - "repo", o.Repo, "pr", o.Number, "head_sha", o.HeadSHA, - "re_engagements", store.ReEngagements(o.Repo, o.Number)) - } -} - -// runEscalationSweep folds this enumeration pass into the fix-loop breaker -// ledger and fires the one-time escalation actions (evidence comment + -// needs-human label + ntfy) for any agent-authored PR that just crossed the -// threshold of distinct failed fix attempts. Returns the full set of -// escalated PR keys so the work-list writers can flag them. Deterministic by -// design: no agent judgment is involved in counting, evidence, or the -// stop-order. Human-authored PRs are never escalated. -// -// The two forge writes go through forge.IssueWriter rather than *github.Client -// so the evidence lands on whichever forge the hive is actually configured for -// (see governorForge in forgewire.go). On a GitHub hive the writer IS the -// *github.Client this used to take, so nothing about that path changed. -// -// rec receives a KindBlocked lifecycle event for each newly-escalated PR -// (#5656); a nil rec is a no-op, matching the other timeline producers. -func runEscalationSweep( - ctx context.Context, - cfg *config.Config, - writer forge.IssueWriter, - actionable *github.ActionableResult, - notifier *notify.Notifier, - rec lifecycleRecorder, - logger *slog.Logger, -) map[string]bool { - escalated := map[string]bool{} - if cfg.Escalation.Disabled || writer == nil || actionable == nil { - return escalated - } - getEscalationStore() - - type prMeta struct{ checks []string } - meta := map[string]prMeta{} - owners := auditPRAgents(cfg.Project.Org, time.Now().Add(-auditPRAttributionWindow), "") - all := allEscalationObservations(cfg, actionable, owners) - itemCount := len(actionable.PRs.Items) - - obs := all[:0] - for i, o := range all { - var pr *github.PullRequest - if i < itemCount { - pr = &actionable.PRs.Items[i] - } else { - pr = &actionable.PRs.Held[i-itemCount] - } - if pr.ReporterTrustReason != "" { - escalationStore.SetReporterTrustReason(o.Repo, o.Number, pr.ReporterTrustReason) - } - if (i >= itemCount || !isHiveAgentAuthor(cfg, pr.Author)) && pr.ReporterTrustReason == "" { - continue - } - obs = append(obs, o) - meta[escalation.Key(o.Repo, o.Number)] = prMeta{checks: o.FailingChecks} - } - results := escalationStore.Sweep(obs, cfg.Escalation.EffectiveThreshold()) - - for _, o := range obs { - key := escalation.Key(o.Repo, o.Number) - r, ok := results[key] - if !ok { - continue - } - if r.Unparked { - // The sweep returned this PR to the automated lane by itself: its - // head went green, or the shared incident it escalated over - // cleared. Take the label off (retried on later passes until a - // pass sees it gone) and say why, once. - unparkEscalatedPR(ctx, writer, o, r, logger) - continue - } - if r.Escalated { - escalated[key] = true - } - if r.BaseRedHold { - // Every failing check is red on the base branch too: say so - // once per head, count nothing, kick no one. The hold clears - // by itself when the base recovers and the head is re-evaluated. - body := escalation.BaseRedHoldCommentBody(o.BaseRedChecks, o.SharedCIIncident) - if err := writer.CreateIssueComment(ctx, o.Repo, o.Number, body); err != nil { - logger.Warn("base-branch-red hold comment failed; will retry next pass", "repo", o.Repo, "pr", o.Number, "error", err) - } else { - escalationStore.MarkBaseRedHeld(o.Repo, o.Number, o.HeadSHA) - logger.Info("fix loop holding: base branch red", "repo", o.Repo, "pr", o.Number, "head_sha", o.HeadSHA, "checks", strings.Join(o.BaseRedChecks, ",")) - } - } - if r.NeedsLabel && !o.Labeled { - // Escalated on an earlier pass but the label never landed (the - // AddLabels call failed). Retry the LABEL ONLY — the evidence - // comment already reached the human and must not be repeated. - if err := writer.AddLabels(ctx, o.Repo, o.Number, []string{escalation.NeedsHumanLabel}); err != nil { - logger.Warn("escalation label retry failed", "repo", o.Repo, "pr", o.Number, "error", err) - } else { - escalationStore.MarkLabelApplied(o.Repo, o.Number) - } - } - if !r.NewlyEscala { - continue - } - escalated[key] = true - excerpt := o.Excerpt - if excerpt == "" { - excerpt = escalationStore.Excerpt(o.Repo, o.Number) - } - // A PR that re-escalates AFTER a reviewer-lane pass reaches its human - // with a structured hand-off note (#5617 item 3): what the reviewer - // left on the branch, when, and that no second automated pass is - // coming. Before this, the only thing distinguishing that hand-off - // from a first escalation was the label set. - ev := escalation.Evidence{ - Attempts: r.Attempts, - FailingChecks: meta[key].checks, - SharedChecks: r.SharedChecks, - Excerpt: excerpt, - Exhausted: r.Exhausted, - ReEngagements: r.ReEngagements, - } - body := escalation.CommentBody(ev) - afterReviewerPass := false - if sha, at, ok := escalationStore.ReviewerPass(o.Repo, o.Number); ok { - body = escalation.HandoffCommentBody(ev, escalation.ReviewerHandoff{SHA: sha, At: at}) - afterReviewerPass = true - } - if err := writer.CreateIssueComment(ctx, o.Repo, o.Number, body); err != nil { - // Retry next pass rather than marking escalated with no comment: - // the whole point is that the evidence reaches a human. - logger.Warn("escalation comment failed; will retry next pass", - "repo", o.Repo, "pr", o.Number, "error", err) - continue - } - // Mark escalated BEFORE the label call: once the comment is on the - // PR, nothing may post it again, whatever happens to the label. - escalationStore.MarkEscalated(o.Repo, o.Number) - if err := writer.AddLabels(ctx, o.Repo, o.Number, []string{escalation.NeedsHumanLabel}); err != nil { - logger.Warn("escalation label failed; will retry next pass", "repo", o.Repo, "pr", o.Number, "error", err) - } else { - escalationStore.MarkLabelApplied(o.Repo, o.Number) - } - // The escalation IS the real "blocked" lifecycle signal (#5656): a PR - // out of automated fix attempts, handed to a human. Record it on the - // item's journey so the panel's Blocked counter reflects reality, not - // just hook annotations. - recordBlocked(ctx, rec, cfg.Project.Org, o.Repo, o.Number, r.Attempts, meta[key].checks) - logger.Info("fix loop escalated to human", - "repo", o.Repo, "pr", o.Number, "attempts", r.Attempts, - "failing_checks", strings.Join(meta[key].checks, ",")) - if notifier != nil { - title := "Fix loop escalated" - detail := fmt.Sprintf("%s#%d red on %d fix attempts — needs a human (see PR comment for the raw error)", o.Repo, o.Number, r.Attempts) - if afterReviewerPass { - // Materially more urgent than a first escalation: a reviewer - // has already had its one pass, so nothing automated remains - // behind this page. - title = "Fix loop escalated after reviewer pass" - detail = fmt.Sprintf("%s#%d red again on %d fix attempts since the reviewer's pass — no further automated pass will run (see PR comment)", o.Repo, o.Number, r.Attempts) - } - notifier.Send(title, detail, notify.PriorityHigh) - } - } - return escalated -} - -// labelRemover is the optional forge capability the auto-un-park path needs. -// Every real adapter (GitHub, GitLab, Gitea) implements RemoveLabel on -// forge.Forge; the narrow IssueWriter the sweep takes does not declare it, so -// a writer that cannot remove labels simply leaves the PR labeled and logs. -type labelRemover interface { - RemoveLabel(ctx context.Context, repo string, number int, label string) error -} - -// unparkEscalatedPR performs the forge side of an automatic un-park: the -// needs-human label comes off, and — on the pass that decided it — a one-line -// comment says why. Before hivecommons/hive#9473 nothing did this: a PR -// escalated over a base-branch or runner outage stayed parked after the -// incident cleared, and a labeled PR that went green stayed parked forever, -// because the labeled branch returned before the green branch. -func unparkEscalatedPR(ctx context.Context, writer forge.IssueWriter, o escalation.Observation, r escalation.Result, logger *slog.Logger) { - remover, ok := writer.(labelRemover) - if !ok { - logger.Warn("cannot un-park escalated PR: forge writer cannot remove labels", - "repo", o.Repo, "pr", o.Number) - return - } - if err := remover.RemoveLabel(ctx, o.Repo, o.Number, escalation.NeedsHumanLabel); err != nil { - // Retried on the next pass: the ledger keeps asking until a pass - // observes the label gone. - logger.Warn("un-park label removal failed; will retry next pass", - "repo", o.Repo, "pr", o.Number, "error", err) - return - } - if r.UnparkReason == "" { - return // label-removal retry: the comment was posted on the deciding pass - } - if err := writer.CreateIssueComment(ctx, o.Repo, o.Number, escalation.UnparkCommentBody(r.UnparkReason)); err != nil { - logger.Warn("un-park comment failed", "repo", o.Repo, "pr", o.Number, "error", err) - } - logger.Info("escalation un-parked", "repo", o.Repo, "pr", o.Number, "reason", r.UnparkReason) -} - -// autoMergeSweepInterval is the minimum spacing between label-queued -// auto-merge sweeps. The sweep piggybacks on the governor eval tick, which can -// fire much more often than once a minute; this floor keeps the sweep from -// hammering the GitHub API on short eval intervals. -const autoMergeSweepInterval = time.Minute - -// autoMergeSweepTickBudget is how long the queued automerge sweep may wait -// for its first pacing slot before the sweep is shed to the next cycle. -const autoMergeSweepTickBudget = 5 * time.Second - -// taskListSweepInterval is the minimum spacing between task-list sweeps. The -// sweep enumerates every open issue in every repo — much heavier than the -// auto-merge sweep's label-scoped query — and "done" moves at PR-merge cadence, -// so a 15-minute floor keeps the API cost modest while still closing completed -// epics on the same day the last box gets ticked. -const taskListSweepInterval = 15 * time.Minute - -// supersessionSweepInterval is the minimum spacing between sweeps that retire -// open PRs whose claimed issue was already closed by a different merged PR. -// It walks open PRs and may inspect issue timelines and file lists, so it uses -// the same conservative cadence as the task-list sweep. -const supersessionSweepInterval = 15 * time.Minute - -// duplicateFixSweepInterval is the minimum spacing between sweeps that -// resolve two open PRs fixing the same issue (hivecommons/hive#14799). It -// reuses the open-PR listing the pre-dispatch guard caches and reads check -// runs per pair, so it runs on the supersession cadence. -const duplicateFixSweepInterval = 15 * time.Minute - -// sharedCIRerunInterval is the minimum spacing between shared-CI rerun -// sweeps. Each pass lists recently closed issues per repo and only walks open -// PRs when a [shared-ci] incident closed inside the lookback window, so it -// shares the supersession sweep's cadence. -const sharedCIRerunInterval = 15 * time.Minute - -// duplicateSweepInterval is the minimum spacing between duplicate sweeps. It -// is an hour rather than the task-list sweep's fifteen minutes for two -// reasons. Cost: the sweep fingerprints the changed-file set of every open PR, -// which on a large queue is the heaviest read the hive performs (the head-SHA -// cache makes steady-state passes nearly free, but a cold pass is not). -// Signal: duplicates accumulate at human-PR-opening cadence, so nothing is -// lost by noticing one an hour later, while a tighter loop only multiplies the -// chance of editing a suggestion under a reader's cursor. -const duplicateSweepInterval = time.Hour - -// runAutoMergeSweepIfDue drains the label-queued auto-merge queue (the human -// "Approved ... for Hive auto-merge" path) at most once per -// autoMergeSweepInterval. All merge-eligibility decisions — queue-approval -// trust, the trusted-merger tier gate (SetMergerAuthorizer), check -// verification — live inside SweepQueuedAutoMerges; this function is only the -// scheduler and the dashboard audit sink. -// rotationTrigger is the PausedTrigger stamped on strand-pauses so rotation's -// auto-resume never resumes a pause it did not create. -const rotationTrigger = "provider-rotation" - -// runRotationCheck applies RFC #3958 provider rotation after an eval cycle: -// for each agent not mid-task whose provider was positively measured as -// exhausted, move it to a backend with headroom at the same tier; when -// nothing has headroom, pause it loudly (strand). Stranded agents are -// auto-resumed when their provider recovers headroom. Never runs mid-task: -// only idle agents are candidates. -func runRotationCheck(ctx context.Context, cfg *config.Config, rotMgr *rotation.Manager, gov *governor.Governor, agentMgr crux.Runtime, logger *slog.Logger) { - if rotMgr == nil || !cfg.Governor.Rotation.Enabled { - return - } - govState := gov.GetState() - for name, proc := range agentMgr.AllStatuses() { - backend := proc.Config.Backend - if proc.BackendOverride != "" { - backend = proc.BackendOverride - } - - // Auto-resume: a stranded agent whose provider recovered. - if proc.Paused && proc.PausedTrigger == rotationTrigger { - if rotMgr.StrandRecovered(backend) { - if err := agentMgr.Resume(ctx, name, rotationTrigger, "provider headroom recovered"); err != nil { - logger.Warn("rotation: auto-resume failed", "agent", name, "error", err) - } else { - logger.Info("rotation: auto-resumed stranded agent", "agent", name, "backend", backend) - } - } - continue - } - if proc.Paused { - continue // never touch an operator pause - } - // Never rotate mid-task: only idle agents are candidates. - if proc.State == crux.StateRunning { - continue - } - - cadenceS := 0 - if interval := governorShortestActiveInterval(govState, name); interval > 0 { - cadenceS = int(interval / time.Second) - } - - if !rotMgr.Exhausted(backend) { - continue - } - next := rotMgr.NextBackendForCadence(name, backend, cadenceS) - if next == "" { - // Strand loudly: pause so the agent burns nothing until a - // provider recovers; the loop above auto-resumes it. - if err := agentMgr.Pause(name, rotationTrigger, "no provider has headroom (RFC #3958)"); err != nil { - logger.Warn("rotation: strand-pause failed", "agent", name, "error", err) - } else { - logger.Info("rotation: stranding agent, no headroom anywhere", "agent", name, "backend", backend) - } - continue - } - if err := agentMgr.SetBackendOverride(name, next); err != nil { - logger.Warn("rotation: backend override failed", "agent", name, "to", next, "error", err) - continue - } - logger.Info("rotation: moved agent to new backend", "agent", name, "from", backend, "to", next) - } -} - -func runAutoMergeSweepIfDue(ctx context.Context, ghClient *github.Client, cfg *config.Config, dashSrv *dashboard.Server, lastRun *time.Time, logger *slog.Logger) { - if ghClient == nil { - return - } - now := time.Now() - if lastRun != nil && !lastRun.IsZero() && now.Sub(*lastRun) < autoMergeSweepInterval { - return - } - // Housekeeping against the pacer: while a caution window has the pacing - // queue deeper than this sweep could wait, skip the sweep (it is due - // again next cycle) rather than queue behind the enumeration. - if github.ShedLowPriorityTick("automerge:queued-sweep", autoMergeSweepTickBudget) { - return - } - if lastRun != nil { - *lastRun = now - } - ctx = github.WithPacingClass(ctx, github.PacingLow) - // governor.automerge covers the queued, trusted-author and contributor- - // lane sweeps plus the stage-label pass that follows them. The sweep runs - // on its own cadence in the run loop, so this is a sibling of - // governor.eval_cycle rather than a child. - ctx, sweepSpan := tracing.StartSpan(ctx, "governor.automerge") - defer sweepSpan.End() - opts := automerge.Options{ - Logger: logger, - MergerAuthorizer: trustedMergerFunc(cfg), - HumanChangeRequests: review.DefaultHumanChangeRequests(), - TrustedAuthorizer: trustedAuthorFunc(cfg), - TrustedAuthorPolicy: trustedAuthorPolicyFunc(cfg), - ContributorLanePolicy: contributorLanePolicyFunc(cfg), - ContributorLaneVerdict: contributorLaneVerdictFunc(cfg, logger), - ContributorRegistryTier: contributorRegistryTierFunc(), - StageLabelPolicy: stageLabelPolicyFunc(cfg), - StageEngine: stageEngineOptions(cfg, logger), - } - syncStageEngineMode(cfg, ghClient, logger) - // hivecommons/hive#12436: the contribute hub is the stage-072 nudge sink - // (relay fix-pr tasks). Guarded so a nil hub never becomes a typed-nil - // interface. - if hub := dashSrv.ContributeHub(); hub != nil { - opts.ContributorNudge = hub - } - if cfg != nil { - opts.SentinelLabel = func() string { return cfg.Sentinel.LabelOrDefault() } - if set, ok := cfg.AutoMerge.RequiredCheckSet(); ok { - opts.RequiredChecks = set - } - opts.RequiredChecksForRepo = cfg.AutoMerge.RequiredCheckSetForRepo - opts.MinHeadAge = cfg.AutoMerge.EffectiveMinHeadAge() - opts.RepoAutoMergeEnabled = func(repo string) bool { return cfg.RepoAutoMergeEnabled(repo) } - opts.RepoOnGitHub = func(repo string) bool { return cfg.RepoOnGitHub(repo) } - opts.HumanMergePaths = func(repo string) []string { return humanMergePathsFor(cfg, repo) } - opts.HumanMergeRule = func() prstage.HumanMergeRule { return humanMergeRuleFor(cfg) } - opts.RequireCurrentBase = func(repo string) bool { return cfg.AutoMerge.RequireCurrentBaseFor(repo) } - opts.RequireCurrentBaseStrict = func(repo string) bool { return cfg.AutoMerge.RequireCurrentBaseStrictFor(repo) } - opts.MergeForwardTrustedHuman = func(repo string) bool { return cfg.AutoMerge.MergeForwardTrustedHumanFor(repo) } - opts.MergeTrain = func() automerge.MergeTrainPolicy { - return automerge.MergeTrainPolicy{Enabled: cfg.AutoMerge.MergeTrain.IsEnabled(), MaxRebaseAttempts: cfg.AutoMerge.MergeTrain.EffectiveMaxRebaseAttempts()} - } - opts.TrustedBotAuthors = func() map[string]bool { return cfg.AutoMerge.TrustedBotAuthorSet() } - } - audit := func(event automerge.AutoMergeSweepEvent) { - if dashSrv == nil { - return - } - detail := fmt.Sprintf("repo=%s, pr=%d, author=%s, queued_by=%s, label=%s, tier=%s, head_sha=%s, merge_sha=%s", - event.Repo, event.Number, event.Author, event.QueuedBy, event.Label, event.Tier, event.HeadSHA, event.MergeSHA) - dashSrv.AuditLogRecord("system", "automerge-sweep-merged", detail, "", event.Repo, event.Number) - } - result, err := automerge.SweepQueuedAutoMerges(ctx, ghClient, opts, automerge.AutoMergeSweepOptions{ - MaxMerges: automerge.DefaultAutoMergeSweepMaxMerges, - Audit: audit, - }) - if err != nil { - logger.Warn("automerge sweep failed", "error", err) - return - } - if len(result.Merged) > 0 || result.Seen > 0 { - logger.Info("automerge sweep complete", "seen", result.Seen, "merged", len(result.Merged), "skipped", result.Skipped) - } - sweepSpan.SetAttributes( - attribute.Int("hive.automerge.seen", result.Seen), - attribute.Int("hive.automerge.merged", len(result.Merged)), - attribute.Int("hive.automerge.skipped", result.Skipped)) - remaining := automerge.DefaultAutoMergeSweepMaxMerges - len(result.Merged) - if remaining > 0 && opts.TrustedAuthorPolicy().Enabled { - trustedResult, trustedErr := automerge.SweepTrustedAuthorAutoMerges(ctx, ghClient, opts, automerge.AutoMergeSweepOptions{ - MaxMerges: remaining, - Audit: audit, - SelfAuthoredLaneActive: cfg != nil && cfg.AutoMerge.SelfAuthoredAutoMergeAllowed(cfg.ACMMLevel), - }) - if trustedErr != nil { - logger.Warn("trusted-author automerge sweep failed", "error", trustedErr) - } else { - result.Seen += trustedResult.Seen - result.Skipped += trustedResult.Skipped - result.Merged = append(result.Merged, trustedResult.Merged...) - if len(trustedResult.Merged) > 0 || trustedResult.Seen > 0 { - logger.Info("trusted-author automerge sweep complete", "seen", trustedResult.Seen, "merged", len(trustedResult.Merged), "skipped", trustedResult.Skipped) - } - } - } - remaining = automerge.DefaultAutoMergeSweepMaxMerges - len(result.Merged) - if remaining > 0 && opts.ContributorLanePolicy().Enabled { - laneResult, laneErr := automerge.SweepContributorLaneAutoMerges(ctx, ghClient, opts, automerge.AutoMergeSweepOptions{ - MaxMerges: remaining, - Audit: audit, - }) - if laneErr != nil { - logger.Warn("contributor-lane automerge sweep failed", "error", laneErr) - } else { - result.Seen += laneResult.Seen - result.Skipped += laneResult.Skipped - result.Merged = append(result.Merged, laneResult.Merged...) - if len(laneResult.Merged) > 0 || laneResult.Seen > 0 { - logger.Info("contributor-lane automerge sweep complete", "seen", laneResult.Seen, "merged", len(laneResult.Merged), "skipped", laneResult.Skipped) - } - } - } - // Stage labels (#12429): the lanes above each recorded their per-PR - // reason into the process-shared tracker; one pass writes the labels. - if stageResult, stageErr := automerge.ApplyStageLabels(ctx, ghClient, opts); stageErr != nil { - logger.Warn("stage labels reconcile failed", "error", stageErr, "items", stageResult.Items, "writes", stageResult.Writes) - } else if stageResult.Writes+stageResult.Comments > 0 { - logger.Info("stage labels reconciled", "items", stageResult.Items, "writes", stageResult.Writes, "comments", stageResult.Comments, "healed", stageResult.Healed) - } - hookDispatcher().Fire(context.Background(), hooks.Payload{ - Transition: hooks.TransitionSweepCompleted, - Reason: "queued automerge sweep complete", - Attrs: map[string]string{ - "seen": strconv.Itoa(result.Seen), - "merged": strconv.Itoa(len(result.Merged)), - "skipped": strconv.Itoa(result.Skipped), - }, - }) -} - -// legacyFleetReportSweepInterval is how often the hive that watches the -// fleet-report repo looks for standalone "Fleet report:" issues to close. -const legacyFleetReportSweepInterval = github.LegacyFleetReportSweepInterval - -// hiveWatchesRepo reports whether repo is one of project.repos. -func hiveWatchesRepo(cfg *config.Config, repo string) bool { - if cfg == nil { - return false - } - for _, r := range cfg.Project.Repos { - if strings.EqualFold(strings.TrimSpace(r), strings.TrimSpace(repo)) { - return true - } - } - return false -} - -// runLegacyFleetReportSweepIfDue closes standalone "Fleet report:" issues a -// spoke still on a pre-#13470 build filed directly in the report repo -// (hivecommons/hive#13567). Only the hive that watches that repo runs it — -// it is the one with write access there — at most once per -// legacyFleetReportSweepInterval. -func runLegacyFleetReportSweepIfDue(ctx context.Context, ghClient *github.Client, cfg *config.Config, dashSrv *dashboard.Server, lastRun *time.Time, logger *slog.Logger) { - if ghClient == nil || cfg == nil { - return - } - repo := cfg.Governor.FleetReport.EffectiveRepo() - if !hiveWatchesRepo(cfg, repo) { - return - } - now := time.Now() - if lastRun != nil && !lastRun.IsZero() && now.Sub(*lastRun) < legacyFleetReportSweepInterval { - return - } - if lastRun != nil { - *lastRun = now - } - result, err := ghClient.SweepLegacyFleetReportIssues(ctx, github.LegacyFleetReportSweepOptions{ - Repo: repo, - Audit: func(event github.LegacyFleetReportSweepEvent) { - if dashSrv == nil { - return - } - detail := fmt.Sprintf("repo=%s, issue=%d, author=%s, title=%s", event.Repo, event.Number, event.Author, event.Title) - dashSrv.AuditLogRecord("system", "legacy-fleet-report-closed", detail, "", event.Repo, event.Number) - }, - }) - if err != nil { - logger.Warn("legacy fleet report sweep failed", "error", err) - return - } - for _, e := range result.Errors { - logger.Warn("legacy fleet report sweep: issue skipped", "detail", e) - } - if len(result.Closed) > 0 { - logger.Info("legacy fleet report sweep complete", "scanned", result.Scanned, "closed", len(result.Closed), "skipped", result.Skipped) - } -} - -// runTaskListSweepIfDue closes hive-filed issues whose task-list bodies are -// fully ticked, at most once per taskListSweepInterval. The finding-granularity -// policy (guidance now in every issue-filing template) tells agents to encode -// multi-part deliverables as `- [ ]` boxes; this is the sink that turns those -// boxes into closures. All safety gates — hive-filed only, at-least-one-box, -// all-boxes-ticked, hold-label respected, per-tick cap — live inside -// SweepCompletedTaskListIssues; this function is only the scheduler and the -// dashboard audit sink, mirroring runAutoMergeSweepIfDue above. -func runTaskListSweepIfDue(ctx context.Context, ghClient *github.Client, dashSrv *dashboard.Server, lastRun *time.Time, logger *slog.Logger) { - if ghClient == nil { - return - } - now := time.Now() - if lastRun != nil && !lastRun.IsZero() && now.Sub(*lastRun) < taskListSweepInterval { - return - } - if lastRun != nil { - *lastRun = now - } - result, err := ghClient.SweepCompletedTaskListIssues(ctx, github.TaskListSweepOptions{ - MaxCloses: github.DefaultTaskListSweepMaxCloses, - Audit: func(event github.TaskListSweepEvent) { - if dashSrv == nil { - return - } - detail := fmt.Sprintf("repo=%s, issue=%d, author=%s, boxes=%d", - event.Repo, event.Number, event.Author, event.TotalBoxes) - dashSrv.AuditLogRecord("system", "task-list-sweep-closed", detail, "", event.Repo, event.Number) - }, - }) - if err != nil { - logger.Warn("task-list sweep failed", "error", err) - return - } - if len(result.Closed) > 0 || result.Seen > 0 { - logger.Info("task-list sweep complete", "seen", result.Seen, "closed", len(result.Closed), "skipped", result.Skipped) - } - hookDispatcher().Fire(context.Background(), hooks.Payload{ - Transition: hooks.TransitionSweepCompleted, - Reason: "task-list sweep complete", - Attrs: map[string]string{ - "seen": strconv.Itoa(result.Seen), - "closed": strconv.Itoa(len(result.Closed)), - "skipped": strconv.Itoa(result.Skipped), - }, - }) -} - -func runSupersessionSweepIfDue(ctx context.Context, ghClient *github.Client, cfg *config.Config, dashSrv *dashboard.Server, lastRun *time.Time, logger *slog.Logger) { - if ghClient == nil { - return - } - now := time.Now() - if lastRun != nil && !lastRun.IsZero() && now.Sub(*lastRun) < supersessionSweepInterval { - return - } - if lastRun != nil { - *lastRun = now - } - var supersessionCfg config.SupersessionSweepConfig - if cfg != nil { - supersessionCfg = cfg.SupersessionSweep - } - result, err := ghClient.SweepSupersededOpenPRs(ctx, github.SupersessionSweepOptions{ - MaxActions: github.DefaultSupersessionSweepMaxActions, - CloseContributorPRs: supersessionCfg.CloseContributorPRs, - GracePeriod: supersessionCfg.GracePeriod, - AutoCloseSuperseded: func() bool { return cfg == nil || cfg.Governor.AutoCloseSupersededEnabled() }, - ACMMLevelForRepo: func(repo string) int { - if cfg == nil { - return 0 - } - return cfg.EffectiveACMMLevelForRepo(repo) - }, - Audit: func(event github.SupersessionSweepEvent) { - if dashSrv == nil { - return - } - detail := fmt.Sprintf("repo=%s, pr=%d, issue=%s#%d, closer_pr=%d, action=%s, reason=%s", - event.Repo, event.Number, event.IssueRepo, event.Issue, event.CloserPR, event.Action, event.Reason) - dashSrv.AuditLogRecord("system", github.SupersessionSweepAuditPrefix+event.Action, detail, "", event.Repo, event.Number) - }, - }) - if err != nil { - logger.Warn("supersession sweep failed", "error", err) - return - } - if len(result.Closed) > 0 || len(result.Commented) > 0 || len(result.Resolved) > 0 || result.Seen > 0 { - logger.Info("supersession sweep complete", "seen", result.Seen, "closed", len(result.Closed), "commented", len(result.Commented), "resolved", len(result.Resolved), "skipped", result.Skipped) - } - hookDispatcher().Fire(context.Background(), hooks.Payload{ - Transition: hooks.TransitionSweepCompleted, - Reason: "supersession sweep complete", - Attrs: map[string]string{ - "seen": strconv.Itoa(result.Seen), - "closed": strconv.Itoa(len(result.Closed)), - "commented": strconv.Itoa(len(result.Commented)), - "resolved": strconv.Itoa(len(result.Resolved)), - "skipped": strconv.Itoa(result.Skipped), - }, - }) -} - -// runDuplicateFixSweepIfDue resolves same-issue duplicate open PRs -// (hivecommons/hive#14799) at most once per duplicateFixSweepInterval: the -// earlier green-or-pending PR stays, the later hive PR is closed as -// duplicate-of:, a person's later PR gets the comment only, and the -// earlier-red-later-green case is parked for the owner. It follows the -// pre-dispatch guard's switch: scanner.skip_issues_with_open_pr off turns -// this off too, since both exist to keep one issue on one PR. -func runDuplicateFixSweepIfDue(ctx context.Context, ghClient *github.Client, cfg *config.Config, dashSrv *dashboard.Server, lastRun *time.Time, logger *slog.Logger) { - if ghClient == nil { - return - } - if cfg != nil && !cfg.Scanner.IsSkipIssuesWithOpenPR() { - return - } - now := time.Now() - if lastRun != nil && !lastRun.IsZero() && now.Sub(*lastRun) < duplicateFixSweepInterval { - return - } - if lastRun != nil { - *lastRun = now - } - result, err := ghClient.SweepDuplicateFixPRs(ctx, github.DuplicateFixSweepOptions{ - MaxActions: github.DefaultDuplicateFixSweepMaxActions, - Audit: func(event github.DuplicateFixEvent) { - if dashSrv == nil { - return - } - detail := fmt.Sprintf("repo=%s, pr=%d, issue=%s#%d, keeper_pr=%d, action=%s, reason=%s", - event.Repo, event.Number, event.IssueRepo, event.Issue, event.KeeperPR, event.Action, event.Reason) - dashSrv.AuditLogRecord("system", github.DuplicateFixSweepAuditPrefix+event.Action, detail, "", event.Repo, event.Number) - }, - }) - if err != nil { - logger.Warn("duplicate-fix sweep failed", "error", err) - return - } - if result.Pairs > 0 { - logger.Info("duplicate-fix sweep complete", "pairs", result.Pairs, "closed", len(result.Closed), "commented", len(result.Commented), "escalated", len(result.Escalated), "skipped", result.Skipped) - } -} - -// runSharedCIRerunSweepIfDue re-runs failed jobs on open PRs deferred to a -// [shared-ci] incident that has since closed (hivecommons/hive#12253), at most -// once per sharedCIRerunInterval. Gates and dedupe live in -// SweepSharedCIReruns; this is the scheduler and dashboard audit sink. -func runSharedCIRerunSweepIfDue(ctx context.Context, ghClient *github.Client, cfg *config.Config, dashSrv *dashboard.Server, lastRun *time.Time, logger *slog.Logger) { - if ghClient == nil { - return - } - if cfg != nil && !cfg.SharedCIRerun.IsEnabled() { - return - } - now := time.Now() - if lastRun != nil && !lastRun.IsZero() && now.Sub(*lastRun) < sharedCIRerunInterval { - return - } - if lastRun != nil { - *lastRun = now - } - result, err := ghClient.SweepSharedCIReruns(ctx, github.SharedCIRerunOptions{ - MaxPRs: github.DefaultSharedCIRerunMaxPRs, - Audit: func(event github.SharedCIRerunEvent) { - if dashSrv == nil { - return - } - incidents := make([]string, len(event.Incidents)) - for i, n := range event.Incidents { - incidents[i] = "#" + strconv.Itoa(n) - } - detail := fmt.Sprintf("repo=%s, pr=%d, incident=%s, head=%s, runs=%d", - event.Repo, event.Number, strings.Join(incidents, ","), event.HeadSHA, len(event.RunIDs)) - dashSrv.AuditLogRecord("system", "shared-ci-rerun", detail, "", event.Repo, event.Number) - }, - }) - if err != nil { - logger.Warn("shared-ci rerun sweep failed", "error", err) - return - } - if len(result.Rerun) > 0 { - logger.Info("shared-ci rerun sweep complete", "incidents", result.Incidents, "seen", result.Seen, "rerun", len(result.Rerun), "skipped", result.Skipped) - } -} - -// blockedIssuesSweepInterval is how often the periodic sweep reconciles open -// issues carrying the `blocked` label. -const blockedIssuesSweepInterval = 5 * time.Minute - -func runBlockedIssuesSweepIfDue(ctx context.Context, ghClient *github.Client, cfg *config.Config, dashSrv *dashboard.Server, lastRun *time.Time, logger *slog.Logger) { - if ghClient == nil { - return - } - now := time.Now() - if lastRun != nil && !lastRun.IsZero() && now.Sub(*lastRun) < blockedIssuesSweepInterval { - return - } - if lastRun != nil { - *lastRun = now - } - var repos []string - if cfg != nil { - repos = cfg.Project.Repos - } - res, err := ghClient.SweepBlockedIssues(ctx, github.BlockedSweepOptions{Repos: repos}) - if err != nil { - logger.Warn("blocked issues reconciliation sweep failed", "error", err) - return - } - if res.Unblocked+res.Retargeted+res.Stripped+res.Errors > 0 { - logger.Info("blocked issues reconciliation sweep complete", "items", res.Items, "unblocked", res.Unblocked, "retargeted", res.Retargeted, "help_wanted_stripped", res.Stripped, "errors", res.Errors) - } -} - -// runDuplicateSweepIfDue clusters open PRs by changed-file set and suggests -// which one to keep, at most once per duplicateSweepInterval -// (hivecommons/hive#7469 capability B). -// -// Fail-closed and opt-in at both levels. `duplicate_sweep.enabled` is off by -// default, so a hive that has not asked for this performs no extra API calls -// at all; `duplicate_sweep.post_comments` is a SECOND, separately-off grant -// for the write, so the natural first configuration is a report-only pass an -// operator can read in the log before the hive says anything on a -// contributor's PR. -// -// The sweep only ever comments. It does not close, label, approve or merge, -// and the reviewer gains no permission from its existence: the write is the -// hive's own App token going through the same canary-gated, scrubbed comment -// path every other hive-authored comment uses. -func runDuplicateSweepIfDue(ctx context.Context, cfg *config.Config, ghClient *github.Client, dashSrv *dashboard.Server, lastRun *time.Time, logger *slog.Logger) { - if ghClient == nil || cfg == nil || !cfg.DuplicateSweep.Enabled { - return - } - now := time.Now() - if lastRun != nil && !lastRun.IsZero() && now.Sub(*lastRun) < duplicateSweepInterval { - return - } - if lastRun != nil { - *lastRun = now - } - result, err := ghClient.SweepDuplicatePRs(ctx, github.DuplicateSweepOptions{ - PostComments: cfg.DuplicateSweep.PostComments, - MaxComments: cfg.DuplicateSweep.MaxComments, - MaxPRsPerRepo: cfg.DuplicateSweep.MaxPRsPerRepo, - BotAuthors: cfg.DuplicateSweep.BotAuthors, - Audit: func(event github.DuplicateSweepEvent) { - if dashSrv == nil { - return - } - detail := fmt.Sprintf("repo=%s, survivor=%d, superseded=%d, confidence=%s, commented=%d", - event.Repo, event.Survivor, len(event.Superseded), event.Confidence, len(event.Commented)) - dashSrv.AuditLogRecord("system", "duplicate-sweep-suggested", detail, "", event.Repo, event.Survivor) - }, - }) - if err != nil { - logger.Warn("duplicate sweep reported a problem", "error", err) - } - if result == nil { - return - } - if len(result.Clusters) > 0 || result.Scanned > 0 { - logger.Info("duplicate sweep complete", - "scanned", result.Scanned, - "skipped", result.Skipped, - "clusters", len(result.Clusters), - "commented", result.Commented, - "post_comments", cfg.DuplicateSweep.PostComments) - } - hookDispatcher().Fire(context.Background(), hooks.Payload{ - Transition: hooks.TransitionSweepCompleted, - Reason: "duplicate sweep complete", - Attrs: map[string]string{ - "scanned": strconv.Itoa(result.Scanned), - "clusters": strconv.Itoa(len(result.Clusters)), - "commented": strconv.Itoa(result.Commented), - }, - }) -} - -// sentinelSweepInterval is the reconcile cadence of the full sentinel pass -// (#13379). `pull_request` opened/synchronize/reopened webhooks evaluate the -// named PR as they arrive (runSentinelPREvent); the full pass enumerates -// open PRs from the forgestate cache, skips every head it already evaluated -// without a GitHub call, and exists to catch a PR whose delivery was lost. -// Fifteen minutes bounds how long such a PR sits unlabelled. -const sentinelSweepInterval = github.SentinelReconcileInterval - -// sentinelSweepOptions is the one option set both sentinel entry points use, -// so the webhook path and the reconcile pass cannot disagree on label, -// trust or rules. -func sentinelSweepOptions(cfg *config.Config, ghClient *github.Client, dashSrv *dashboard.Server) github.SentinelSweepOptions { - sc := cfg.Sentinel - return github.SentinelSweepOptions{ - Label: sc.LabelOrDefault(), - LabelColor: config.DefaultSentinelLabelColor, - LabelDescription: config.DefaultSentinelLabelDescription, - Evaluator: sc.EvaluatorConfig(), - TrustedAuthor: sentinelTrustedAuthorFunc(cfg, ghClient), - TrustedAuthorsBlock: sc.TrustedAuthorsBlock, - RepoAllowed: sc.RepoAllowed, - MaxActions: sc.MaxActionsOrDefault(), - Audit: func(event github.SentinelSweepEvent) { - if dashSrv == nil { - return - } - // Findings the write-back gate refused still reach the Security - // tab and /api/sentinel: the scan happened, only the GitHub - // write did not (#writeback containment). - dashSrv.RecordSentinelFinding(event) - detail := fmt.Sprintf("repo=%s, pr=%d, author=%s, head=%s, rules=%s", - event.Repo, event.Number, event.Author, event.HeadSHA, strings.Join(event.Rules(), ",")) - action := "sentinel-alert" - if event.Suppressed { - action = "sentinel-finding-suppressed" - detail += ", suppressed=" + event.SuppressedReason - } - dashSrv.AuditLogRecord("system", action, detail, "", event.Repo, event.Number) - }, - } -} - -// runSentinelPREvent is the dashboard.Dependencies.SentinelPREvent hook: a -// verified `pull_request` opened/synchronize/reopened delivery evaluates -// that one PR now, reading only it and its files. Default on with the -// sweep; `sentinel.enabled: false` disables both. -func runSentinelPREvent(ctx context.Context, ghClient *github.Client, cfg *config.Config, dashSrv *dashboard.Server, repo string, number int, logger *slog.Logger) { - if ghClient == nil || cfg == nil || !cfg.Sentinel.IsEnabled() { - return - } - result, err := ghClient.SweepSentinelPR(ctx, repo, number, sentinelSweepOptions(cfg, ghClient, dashSrv)) - if err != nil { - logger.Warn("sentinel webhook evaluation failed", "repo", repo, "pr", number, "error", err) - return - } - logSentinelResult(result, logger) -} - -// runSentinelSweepIfDue flags open PRs that look like security overrides, -// privilege escalation or codebase damage (pkg/sentinel) at most once per -// sentinelSweepInterval. Default on; `sentinel.enabled: false` disables it. -func runSentinelSweepIfDue(ctx context.Context, ghClient *github.Client, cfg *config.Config, dashSrv *dashboard.Server, lastRun *time.Time, logger *slog.Logger) { - if ghClient == nil || cfg == nil || !cfg.Sentinel.IsEnabled() { - return - } - now := time.Now() - if lastRun != nil && !lastRun.IsZero() && now.Sub(*lastRun) < sentinelSweepInterval { - return - } - if lastRun != nil { - *lastRun = now - } - result, err := ghClient.SweepSentinel(ctx, sentinelSweepOptions(cfg, ghClient, dashSrv)) - if err != nil { - logger.Warn("sentinel sweep failed", "error", err) - return - } - logSentinelResult(result, logger) - if len(result.Flagged) > 0 || len(result.Notified) > 0 || len(result.Remediated) > 0 || result.Seen > 0 { - logger.Info("sentinel sweep complete", "seen", result.Seen, "flagged", len(result.Flagged), "notified", len(result.Notified), "suppressed", len(result.Suppressed), "remediated", len(result.Remediated), "skipped", result.Skipped, "errors", len(result.Errors), "cached_repos", result.CachedRepos, "listed_repos", result.ListedRepos) - } - hookDispatcher().Fire(context.Background(), hooks.Payload{ - Transition: hooks.TransitionSweepCompleted, - Reason: "sentinel sweep complete", - Attrs: map[string]string{ - "seen": strconv.Itoa(result.Seen), - "flagged": strconv.Itoa(len(result.Flagged)), - "notified": strconv.Itoa(len(result.Notified)), - "remediated": strconv.Itoa(len(result.Remediated)), - "skipped": strconv.Itoa(result.Skipped), - }, - }) -} - -func logSentinelResult(result *github.SentinelSweepResult, logger *slog.Logger) { - for _, e := range result.Errors { - logger.Warn("sentinel sweep: PR skipped", "detail", e) - } - for _, ev := range result.Flagged { - logger.Warn("sentinel alert", "repo", ev.Repo, "pr", ev.Number, "author", ev.Author, "rules", strings.Join(ev.Rules(), ",")) - } - for _, ev := range result.Notified { - logger.Info("sentinel notice", "repo", ev.Repo, "pr", ev.Number, "author", ev.Author, "trusted_reason", ev.TrustedReason, "rules", strings.Join(ev.Rules(), ",")) - } - for _, ev := range result.Remediated { - logger.Info("sentinel unflagged trusted author PR", "repo", ev.Repo, "pr", ev.Number, "author", ev.Author, "trusted_reason", ev.TrustedReason, "head", ev.HeadSHA) - } -} - -// issueUnparkSweepInterval is the minimum spacing between un-park sweeps. A -// maintainer who replies "/hive approve" is waiting for something to happen, so -// this runs more often than the task-list sweep; it only reads the comments of -// issues that are actually parked, which keeps the API cost proportional to the -// backlog a person is waiting on. -const issueUnparkSweepInterval = 5 * time.Minute - -// issueUnparkSweepWebhookInterval is the minimum spacing when an `issues` -// webhook has marked a label change (hivecommons/hive#11313): a maintainer who -// just added `triage/accepted` should not wait out the polling interval, but a -// burst of label events must not turn into a sweep per event. -const issueUnparkSweepWebhookInterval = 30 * time.Second - -// runIssueUnparkSweepIfDue re-queues parked issues a maintainer has answered -// with a `/hive` command, and keeps the "What to reply" block current on every -// parked issue (hivecommons/hive#9879). Every gate — human author, live write -// permission, first-line command, recent unedited comment, per-tick cap — lives -// inside SweepIssueUnparkCommands; this function is only the scheduler and the -// dashboard audit sink, mirroring runTaskListSweepIfDue above. Repos with -// project.repo_policies[].label_driven skip the notice and keep their parking -// labels (hivecommons/hive#10537). -func runIssueUnparkSweepIfDue(ctx context.Context, ghClient *github.Client, cfg *config.Config, dashSrv *dashboard.Server, lastRun *time.Time, logger *slog.Logger) { - if ghClient == nil { - return - } - now := time.Now() - if lastRun != nil && !lastRun.IsZero() { - since := now.Sub(*lastRun) - if since < issueUnparkSweepWebhookInterval { - return - } - if since < issueUnparkSweepInterval && github.TakeDirtyIssueCount() == 0 { - return - } - } - if lastRun != nil { - *lastRun = now - } - // `/hive approve` on a PR is answered with what the merge lane will do - // with it (hivecommons/hive#13087); the engine here only reads policy. - approveOpts := automerge.Options{ - MergerAuthorizer: trustedMergerFunc(cfg), - TrustedAuthorizer: trustedAuthorFunc(cfg), - TrustedAuthorPolicy: trustedAuthorPolicyFunc(cfg), - ContributorLanePolicy: contributorLanePolicyFunc(cfg), - ContributorLaneVerdict: contributorLaneVerdictFunc(cfg, logger), - } - if cfg != nil { - approveOpts.RepoAutoMergeEnabled = cfg.RepoAutoMergeEnabled - approveOpts.HumanMergePaths = func(repo string) []string { return humanMergePathsFor(cfg, repo) } - approveOpts.HumanMergeRule = func() prstage.HumanMergeRule { return humanMergeRuleFor(cfg) } - approveOpts.TrustedBotAuthors = func() map[string]bool { return cfg.AutoMerge.TrustedBotAuthorSet() } - } - approveEngine := automerge.New(ghClient, approveOpts) - result, err := ghClient.SweepIssueUnparkCommands(ctx, github.IssueUnparkSweepOptions{ - PRApproveReply: approveEngine.PRApproveReply, - MaxActions: github.DefaultIssueUnparkSweepMaxActions, - // cfg is nil only in tests; nil leaves every repo on the default path. - LabelDriven: func(repo string) bool { return cfg.RepoLabelDriven(repo) }, - LabelApproval: labelApprovalPolicyResolver(cfg), - LabelApprovalAudit: func(event github.IssueLabelApprovalEvent) { - if dashSrv == nil { - return - } - detail := fmt.Sprintf("repo=%s, issue=%d, label=%s, actor=%s, trust=%s, removed=%s, released=%t, pending=%t", - event.Repo, event.Number, event.Label, event.Actor, event.TrustReason, strings.Join(event.Removed, ","), event.Released, event.Pending) - dashSrv.AuditLogRecord("system", "issue-approved-by-label", detail, "", event.Repo, event.Number) - }, - Audit: func(event github.IssueUnparkEvent) { - if dashSrv == nil { - return - } - detail := fmt.Sprintf("repo=%s, issue=%d, actor=%s, decision=%s", - event.Repo, event.Number, event.Actor, event.Decision) - dashSrv.AuditLogRecord("system", "issue-unparked-by-reply", detail, "", event.Repo, event.Number) - }, - ClosedAudit: func(event github.IssueUnparkEvent) { - if dashSrv == nil { - return - } - detail := fmt.Sprintf("repo=%s, issue=%d, actor=%s", event.Repo, event.Number, event.Actor) - dashSrv.AuditLogRecord("system", "issue-closed-by-fixed", detail, "", event.Repo, event.Number) - }, - // `/hive decision` on a hive-authored PR at 085 (hivecommons/hive#13041) - // is routed to the fix lane under the automerge lane's merger trust. - HumanChangeRequests: review.DefaultHumanChangeRequests(), - TrustedMerger: trustedMergerFunc(cfg), - PRAudit: func(event github.PullRequestDecisionEvent) { - if dashSrv == nil { - return - } - detail := fmt.Sprintf("repo=%s, pr=%d, head=%s, actor=%s, decision=%s", - event.Repo, event.Number, event.HeadSHA, event.Actor, event.Decision) - dashSrv.AuditLogRecord("system", "pr-decision-routed-to-fix", detail, "", event.Repo, event.Number) - }, - }) - if err != nil { - logger.Warn("issue un-park sweep failed", "error", err) - return - } - if result == nil { - return - } - if len(result.Unparked) > 0 || len(result.Closed) > 0 || result.Replies > 0 || len(result.LabelApproved) > 0 || len(result.PRDecisions) > 0 { - logger.Info("issue un-park sweep complete", - "seen", result.Seen, "unparked", len(result.Unparked), "closed", len(result.Closed), "label_approved", len(result.LabelApproved), "pr_decisions", len(result.PRDecisions), "replies", result.Replies, "skipped", result.Skipped) - } -} - -// closedIssueStageSweepInterval spaces the closed-issue clean-up. A closed -// issue wearing stage/045 is a stale dashboard row, not a person waiting, so -// this runs at the task-list cadence; each pass is one listing per repo. -const closedIssueStageSweepInterval = 10 * time.Minute - -// closedIssueStageSweepSlack widens each pass's `since` so an issue that -// closed while the previous pass was running is not missed. -const closedIssueStageSweepSlack = 10 * time.Minute - -// runClosedIssueStageSweepIfDue strips hive stage and park labels from issues -// that closed while carrying them (hivecommons/homebrew-hive#127): the -// issue-stage reconciler only visits open issues, so a `Fixes #N` merge or a -// by-hand close left `stage/045-needs-decision` behind and the tracker kept -// reading it as a pending decision. Every rule lives in -// SweepClosedIssueStageLabels; this is the scheduler and audit sink. -func runClosedIssueStageSweepIfDue(ctx context.Context, ghClient *github.Client, dashSrv *dashboard.Server, lastRun *time.Time, logger *slog.Logger) { - if ghClient == nil { - return - } - now := time.Now() - var since time.Time - if lastRun != nil && !lastRun.IsZero() { - if now.Sub(*lastRun) < closedIssueStageSweepInterval { - return - } - since = lastRun.Add(-closedIssueStageSweepSlack) - } - if lastRun != nil { - *lastRun = now - } - result, err := ghClient.SweepClosedIssueStageLabels(ctx, github.ClosedIssueStageSweepOptions{ - Since: since, - MaxActions: github.DefaultClosedIssueStageSweepMaxActions, - Audit: func(event github.ClosedIssueStageEvent) { - if dashSrv == nil { - return - } - detail := fmt.Sprintf("repo=%s, issue=%d, stage=%s, needs_human=%t", - event.Repo, event.Number, strings.Join(event.Stage, ","), event.NeedsHuman) - dashSrv.AuditLogRecord("system", "closed-issue-stage-cleared", detail, "", event.Repo, event.Number) - }, - }) - if err != nil { - logger.Warn("closed-issue stage sweep failed", "error", err) - } else if result != nil && (len(result.Cleaned) > 0 || result.Skipped > 0) { - logger.Info("closed-issue stage sweep complete", "seen", result.Seen, "cleaned", len(result.Cleaned), "skipped", result.Skipped) - } - runReviewBacklogSplitSweep(ctx, ghClient, dashSrv, since, logger) -} - -// runReviewBacklogSplitSweep re-files still-unchecked review findings folded -// into an issue that has since closed (hivecommons/hive#13914). It shares the -// closed-issue sweep's cadence and window; every rule lives in -// SweepReviewBacklogSplits. It then runs the one-off -// BackfillReviewBacklogSplits for hosts closed before that window -// (hivecommons/hive#13947), a no-op once the ledger records it complete. -func runReviewBacklogSplitSweep(ctx context.Context, ghClient *github.Client, dashSrv *dashboard.Server, since time.Time, logger *slog.Logger) { - audit := func(event github.ReviewBacklogSplitEvent) { - if dashSrv == nil { - return - } - detail := fmt.Sprintf("repo=%s, host=%d, issue=%d, reused=%t, key=%s", - event.Repo, event.Host, event.Number, event.Reused, event.Key) - dashSrv.AuditLogRecord("system", "review-backlog-split", detail, "", event.Repo, event.Host) - } - result, err := ghClient.SweepReviewBacklogSplits(ctx, github.ReviewBacklogSplitSweepOptions{ - Since: since, - MaxActions: github.DefaultReviewBacklogSplitMaxActions, - Audit: audit, - }) - if err != nil { - logger.Warn("review backlog split sweep failed", "error", err) - return - } - if result != nil && (len(result.Split) > 0 || result.Skipped > 0 || result.Deferred > 0) { - logger.Info("review backlog split sweep complete", "hosts", result.Hosts, "split", len(result.Split), "skipped", result.Skipped, "deferred", result.Deferred) - } - backfill, err := ghClient.BackfillReviewBacklogSplits(ctx, github.ReviewBacklogSplitSweepOptions{ - MaxActions: github.DefaultReviewBacklogSplitMaxActions, - Audit: audit, - }) - if err != nil { - logger.Warn("review backlog split backfill failed", "error", err) - return - } - if backfill != nil && (len(backfill.Split) > 0 || backfill.Skipped > 0 || backfill.Deferred > 0) { - logger.Info("review backlog split backfill pass complete", "hosts", backfill.Hosts, "split", len(backfill.Split), "skipped", backfill.Skipped, "deferred", backfill.Deferred) - } -} - -// labelApprovalPolicyResolver reads label-driven approval settings -// (hivecommons/hive#11313) live from cfg. It answers ok=false for every repo -// without project.repo_policies[].label_driven, which keeps the feature off -// there; a nil cfg (tests) opts nothing in. -func labelApprovalPolicyResolver(cfg *config.Config) func(repo string) (github.LabelApprovalPolicy, bool) { - return func(repo string) (github.LabelApprovalPolicy, bool) { - if cfg == nil { - return github.LabelApprovalPolicy{}, false - } - labels := cfg.LabelDrivenApprovalLabels(repo) - if len(labels) == 0 { - return github.LabelApprovalPolicy{}, false - } - return github.LabelApprovalPolicy{ - Labels: labels, - TrustOrgMembers: cfg.LabelDrivenTrustOrgMembers(), - TrustedBots: cfg.LabelDrivenTrustedBots(), - SupersedeParkActors: cfg.LabelDrivenSupersedeParkActors(repo), - }, true - } -} - -var ( - ciFailingPath = "/var/run/hive-metrics/ci-failing.json" - intentVerdictsPath = "/var/run/hive-metrics/intent-verdicts.json" -) - -// claimLedger holds the duplicate-PR guard's persisted issue→PR claim mapping -// across eval cycles. It is loaded lazily on first use (and retried on a load -// failure) rather than at startup, so a missing or corrupt /data ledger can -// never block the hive from booting. -var ( - claimLedgerOnce sync.Once - claimLedger *github.ClaimLedger - claimLedgerPath = github.ClaimLedgerPath - claimLedgerLoader = github.LoadClaimLedger -) - -// hiveIdentity determines which PR authors count as "this hive", so only our -// own PRs suppress work. Two accounts can open PRs on our behalf: -// - project.ai_author — the account agents push and open PRs as -// - the GitHub App bot login ("[bot]") when the hive authenticates -// as an installation, which is what actually authors PRs in that mode -func hiveIdentity(cfg *config.Config) github.HiveIdentity { - id := github.HiveIdentity{AIAuthor: cfg.Project.AIAuthor} - if slug := cfg.GitHub.ResolvedAppSlug(); slug != "" { - id.AppLogin = slug + "[bot]" - } - return id -} - -// issueStageSyncTimeout bounds one eval pass's issue stage-label sync so a -// rate-limit pacing queue cannot stall the run loop (hivecommons/hive#12983). -const issueStageSyncTimeout = 2 * time.Minute - -// recheckSoakHubURL is the hub the soak triage probes, "" when this hive has -// no hub (the probe then reads the spoke's own /api/version). -func recheckSoakHubURL(cfg *config.Config) string { - if cfg == nil || !cfg.Hub.Enabled { - return "" - } - return cfg.Hub.URL -} - -// applyDuplicatePRGuard filters issues already claimed by an open hive-authored -// PR out of the actionable set. Failures are logged, never fatal: the guard is -// a safety net, and a broken net must not take the hive down with it. -// reconcileLedgerCoverRecords runs github.BackfillCoveredByLedger with the -// cycle's red+stale valve. It is not an optional step: with webhook-driven -// cached cycles the reconcile pass (and the stage sweep with it) can be -// hours apart, and a record at 010 under an open covering PR re-offers the -// issue to the scanner for all of that time. -func reconcileLedgerCoverRecords(ctx context.Context, cfg *config.Config, ghClient *github.Client, actionable *github.ActionableResult, logger *slog.Logger) { - ledger := getClaimLedger(logger) - if ledger == nil || ledger.Len() == 0 || ghClient == nil { - return - } - backfillCtx, cancel := context.WithTimeout(ctx, issueStageSyncTimeout) - defer cancel() - res := ghClient.BackfillCoveredByLedger(backfillCtx, ledger, claimingPRRedStale(cfg, actionable)) - if res.Covered+res.Errors > 0 && logger != nil { - logger.Info("stage backfill: issue records reconciled with the claim ledger", - "checked", res.Checked, "covered", res.Covered, "skipped_red_stale", res.SkippedRedStale, - "skipped_not_open", res.SkippedNotOpen, "errors", res.Errors) - } -} - -func applyDuplicatePRGuard( - ctx context.Context, - cfg *config.Config, - ghClient *github.Client, - actionable *github.ActionableResult, - logger *slog.Logger, -) { - // The PR-claim guard runs FIRST: an open PR that fixes the issue is the - // stronger evidence, and the guards below only walk what is still in - // the actionable set. With the live-claim filter ahead of it, an issue - // whose agent claim was still live when its PR opened was withheld as - // "claimed" before the ledger could bucket it "claimed_by_pr", so every - // sweep re-derived 020 under the PR-opened 040 and the stage store - // recorded a spurious regression (hivecommons/hive#13174 et al.). The - // ledger's open-issue set (OpenIssuesOf) likewise needs the full list. - ledger := getClaimLedger(logger) - if ledger != nil { - github.ApplyDuplicatePRGuard(ctx, ghClient, ledger, hiveIdentity(cfg), actionable, claimingPRRedStale(cfg, actionable), logger) - } - // #8380: a LIVE issue claim covers an issue the same way an open PR does, - // for every kick prompt (the scanner's included). The claim fields are - // only ever set while governor.claims.enabled is on, so with the feature - // off this touches nothing. - // hivecommons/hive#13579: a contributor claim on an issue in flap - // cool-off, or one that has starved hive agents past the configured - // window with no PR, is advisory — the issue stays actionable. - if cfg != nil && cfg.Governor.Claims.Enabled { - if withheld := contributorClaimGuard.Load().FilterLiveIssueClaims(actionable, time.Now(), logger); withheld > 0 { - logger.Info("issue-claim guard applied", "withheld", withheld) - } - } - if ledger == nil { - return - } - // The in-progress sweep runs after the PR-claim guard so an issue whose - // PR just opened sheds the start mark here, and a lapsed mark frees the - // issue before kicks are built. Independent of governor.claims: the - // label is read from the issue, the marker from the comment, on every hive. - if withheld := github.SweepInProgressIssues(ctx, ghClient, actionable, time.Now(), logger); withheld > 0 { - logger.Info("in-progress guard applied", "withheld", withheld) - } - // hivecommons/hive#12436: with every bucket final, mirror the cycle's - // decision about each open issue onto its stage/NNN label (and strip the - // retired claimed / hive/in-progress / 🐝 covered by PR labels). - // hivecommons/hive#12983: the run loop holds cfgReloadMu for the whole - // pass, so bound the sync; a paced request whose slot would miss the - // deadline fails fast and the best-effort sync retries next cycle. - stageCtx, stageCancel := context.WithTimeout(ctx, issueStageSyncTimeout) - stageCtx, stageSpan := tracing.StartSpan(stageCtx, "governor.stage_labels") - syncStageEngineMode(cfg, ghClient, logger) - res := github.SyncIssueStageLabelsWith(stageCtx, ghClient, actionable, nil, time.Now(), logger, stageEngineOptions(cfg, logger)) - stageSpan.SetAttributes( - attribute.Int("hive.stage_labels.items", res.Items), - attribute.Int("hive.stage_labels.writes", res.Writes)) - stageSpan.End() - stageCancel() - if res.Writes+res.Comments > 0 { - logger.Info("issue stage labels applied", "items", res.Items, "writes", res.Writes, "comments", res.Comments, "healed", res.Healed) - } - // hivecommons/hive#13093: issues parked on an unmerged PR come back once - // that PR settles, and never advertise `help wanted` while parked. - // hivecommons/hive#13311: ops-tracking soak checklists past their - // recheck-after deadline get evidence from the hub (or this spoke). - github.SetRecheckSoakProbe(github.NewHTTPSoakProbe(recheckSoakHubURL(cfg), fmt.Sprintf("http://localhost:%d", cfg.Dashboard.Port), nil)) - blockedCtx, blockedCancel := context.WithTimeout(ctx, issueStageSyncTimeout) - // hivecommons/hive#13833: review-backlog items wait on the PR they were - // filed from in every destination; lift them when it merges, and when it - // closes unmerged unblock them with a re-triage note (never auto-close). - // Runs first so the items get the sweep's own note rather than the - // generic one from the sweep below. - if rb := github.ReconcileReviewBacklogBlocks(blockedCtx, ghClient, logger); rb.Lifted+rb.Retriaged+rb.Errors > 0 { - logger.Info("review backlog blocks reconciled", "checked", rb.Checked, "lifted", rb.Lifted, "retriaged", rb.Retriaged, "errors", rb.Errors) - } - if br := github.ReconcileBlockedIssues(blockedCtx, ghClient, actionable, logger); br.Unblocked+br.Retargeted+br.Stripped+br.SoakTriaged+br.Errors > 0 { - logger.Info("blocked issues reconciled", "items", br.Items, "unblocked", br.Unblocked, "retargeted", br.Retargeted, "help_wanted_stripped", br.Stripped, "soak_triaged", br.SoakTriaged, "errors", br.Errors) - } - blockedCancel() - // Closing an issue is irreversible: refuse while the snapshot is stale and - // re-read an item whose copy is older than DecisionFreshnessBound. - runCloseOnMergeSweep(github.WithIrreversibleGuard(ctx, ghClient.SnapshotFreshnessGuard(actionable, time.Now)), cfg, ghClient, ledger, logger) -} - -// inProgressSetting resolves the pr-in-progress stage label setting from config. -// The label follows the claims label mirror: an operator who set -// governor.claims.label: false asked for no hive-managed claim labels, and -// gets none here either; otherwise the start signal is always labelled. The -// mark lasts the agent claim TTL, so both the ledger and the label agree on -// when an agent's hold lapses. -func inProgressSetting(cfg *config.Config) github.InProgressSetting { - if cfg == nil { - return github.InProgressSetting{} - } - _, agentTTL, _, _ := cfg.Governor.Claims.EffectiveTTLs() - return github.InProgressSetting{ - Enabled: !(cfg.Governor.Claims.IsEnabled() && !cfg.Governor.Claims.LabelEnabled()), - HiveID: cfg.HiveID, - TTL: agentTTL, - } -} - -func logCloseOnMergeResults(logger *slog.Logger, msg string, results []github.CloseOnMergeResult) { - if logger == nil || len(results) == 0 { - return - } - var closed, awaiting, skipped, noop int - for _, r := range results { - switch r.Action { - case github.CloseOnMergeClosed: - closed++ - case github.CloseOnMergeAwaiting: - awaiting++ - case github.CloseOnMergeSkipped: - skipped++ - default: - noop++ - } - } - if closed > 0 || awaiting > 0 || skipped > 0 { - logger.Info(msg, "closed", closed, "awaiting_confirmation", awaiting, "skipped", skipped, "noop", noop) - } -} - -// getClaimLedger lazily loads the persisted claim ledger on first use (and -// keeps a usable empty ledger on a load failure), so a missing or corrupt -// /data ledger can never block the hive from booting. It is shared by the -// eval-cycle guard above and by the dashboard's IssueClaimed hook (#3768); the -// sync.Once publication makes the pointer safe to read from either goroutine, -// and the ledger itself is internally locked. -func getClaimLedger(logger *slog.Logger) *github.ClaimLedger { - claimLedgerOnce.Do(func() { - ledger, err := claimLedgerLoader(claimLedgerPath, logger) - if err != nil { - // LoadClaimLedger always returns a usable (possibly empty) ledger - // alongside the error, so we keep it and just report the problem. - logger.Warn("duplicate-PR guard: could not load persisted claim ledger, starting empty", - "path", claimLedgerPath, "error", err) - } - claimLedger = ledger - // A person reopening an issue after its closing PR merged voids the - // merged claim right away (hivecommons/hive#9945 sat likely-done for - // hours after a /reopen); the enumeration pass is the fallback when - // webhooks are not delivered. - github.SetIssueReopenedHook(issueReopenedHookFor(ledger, logger)) - }) - return claimLedger -} - -// issueReopenedHookFor builds the webhook hook that voids a merged claim on a -// reopened issue and persists the ledger, logging `reopened issue re-queued`. -func issueReopenedHookFor(ledger *github.ClaimLedger, logger *slog.Logger) github.IssueReopenedHook { - return func(repo string, number int, at time.Time) { - if ledger == nil || !ledger.MarkIssueReopened(repo, number, at) { - return - } - claim, _ := ledger.LookupAny(repo, number) - if logger != nil { - logger.Info("reopened issue re-queued: merged PR claim voided", - "repo", repo, "issue", number, "claimed_by_pr", claim.PRNumber, "pr_repo", claim.PRRepo, - "pr_url", claim.PRURL, "merged_at", claim.MergedAt, "reopened_at", at, "source", "webhook") - } - if err := ledger.Save(); err != nil && logger != nil { - logger.Warn("reopened issue: failed to persist claim ledger", "repo", repo, "issue", number, "error", err) - } - } -} - -// claimingPRRedStale builds the Fix #3 release predicate: given a claiming PR -// (prRepo, prNumber), report whether it is red on a required check AND stale. -// It looks up the PR's live CI state + head SHA from this pass's enumeration and -// consults the shared staleness clock. A PR not found in the enumeration, or one -// that is green/pending, or one whose red head only just appeared, returns false -// — so a HEALTHY claiming PR still suppresses its issue. Returns a nil func when -// escalation is disabled, preserving the original unconditional-suppress -// behavior. GENERIC: keys only off check state + staleness. -func claimingPRRedStale(cfg *config.Config, actionable *github.ActionableResult) github.RedStaleFunc { - if cfg.Escalation.Disabled || actionable == nil { - return nil - } - fullRepo := func(repo string) string { - if !strings.Contains(repo, "/") && cfg.Project.Org != "" { - return cfg.Project.Org + "/" + repo - } - return repo - } - // Index this pass's PRs by bare-repo#number so a claim's PRRepo (which may - // be bare or "owner/repo") resolves regardless of prefix. - type prState struct { - red bool - headSHA string - repo string - } - index := map[string]prState{} - for _, pr := range actionable.PRs.Items { - index[fmt.Sprintf("%s#%d", bareRepoName(pr.Repo), pr.Number)] = prState{ - red: pr.HasFailingRequiredCheck(), - headSHA: pr.HeadSHA, - repo: fullRepo(pr.Repo), - } - } - store := getEscalationStore() - return func(prRepo string, prNumber int) bool { - st, ok := index[fmt.Sprintf("%s#%d", bareRepoName(prRepo), prNumber)] - if !ok || !st.red { - return false // not enumerated, or healthy → keep suppressing - } - return store.StaleRed(st.repo, prNumber, st.headSHA) - } -} - -func fullRepoName(repo, org string) string { - if strings.Contains(repo, "/") || org == "" { - return repo - } - return org + "/" + repo -} - -// reviewAgentCapability describes one configured lane the way the review -// dispatcher sees it, so the relay judges reviewer-ness by the same rule. -// An unknown lane has only its name to go on. -func reviewAgentCapability(cfg *config.Config, agent string) review.AgentCapability { - if cfg != nil { - if ac, ok := cfg.Agents[agent]; ok { - return review.CapabilityFromConfig(agent, ac) - } - } - return review.AgentCapability{Name: agent} -} - -// reviewAuthorBase resolves an authoring lane to its base agent over the full -// configured agent set, enabled or not. A replica name the current config no -// longer materialises (its replicas: count was reduced) still maps to the -// declared base it was derived from, so the base and its remaining replicas -// stay excluded from reviewing it. -func reviewAuthorBase(cfg *config.Config, agent string) string { - agent = strings.TrimSpace(agent) - if cfg == nil || agent == "" { - return agent - } - if _, ok := cfg.Agents[agent]; ok { - return cfg.BaseAgentName(agent) - } - if i := strings.LastIndex(agent, "-"); i > 0 { - if n, err := strconv.Atoi(agent[i+1:]); err == nil && n >= 2 { - if ac, ok := cfg.Agents[agent[:i]]; ok && ac.ReplicaOf == "" { - return agent[:i] - } - } - } - return agent -} - -// installReviewRelaySettings gives a (possibly rebuilt) GitHub client the -// review-relay knobs that live in cfg.Review: which repos may be revised in -// place, which perspectives a verdict may name, and whether comments carry a -// confidence score (hivecommons/hive#8182). One place, so an app-auth rebuild -// cannot silently drop a setting the first client had — the perspective set -// was previously installed only on the boot path. -func installReviewRelaySettings(client *github.Client, cfg *config.Config, logger *slog.Logger) { - if client == nil || cfg == nil { - return - } - client.SetReviseRepos(cfg.Review.ReviseRepos) - client.SetPerspectives(reviewPerspectiveSet(cfg, logger)) - client.SetReviewerAgentFunc(func(agent string) bool { - return review.ReviewCapable(reviewAgentCapability(cfg, agent), review.ReviewerAgentSet(cfg.Review.ReviewerAgents)) - }) - client.SetAgentBaseFunc(func(agent string) string { - return reviewAuthorBase(cfg, agent) - }) - client.SetConfidenceScore(func() bool { return cfg.Review.ConfidenceScore }) - client.SetReviewBacklog(func() (bool, int) { - return !cfg.Review.OutOfScopeBacklogDisabled, cfg.Review.MaxOutOfScopeBacklogIssues - }) - client.SetReviewBacklogRouting(func() github.ReviewBacklogRouting { - return reviewBacklogRouting(client, cfg, logger) - }) - client.SetReviewEvidence(func(repo string) github.ReviewEvidenceSettings { - return reviewEvidenceSettings(cfg, repo) - }) - // hivecommons/hive#14581: issue-author credit on every merge path, read - // live so a reload of auto_merge.credit_issue_authors applies. - client.SetCreditIssueAuthors(func(string) bool { return cfg.AutoMerge.CreditIssueAuthorsEnabled() }) - // hivecommons/hive#13241: an issue a person reopens after its closing PR - // merged is a change failure on that PR's evidence record, from both the - // issues.reopened webhook and the enumeration reopen check. - github.SetIssueReopenRecorder(client.RecordReviewEvidenceIssueReopen) - // #8380: issue claims are read at enumeration time only while - // governor.claims.enabled is on; the setting is read live so the Features - // toggle applies without a client rebuild. - client.SetIssueClaims(func() (bool, time.Duration) { - return cfg.Governor.Claims.Enabled, cfg.Governor.Claims.EffectiveTTL() - }) - client.SetInProgress(func() github.InProgressSetting { return inProgressSetting(cfg) }) - // hivecommons/hive#12436: the issue stage reconciler follows the same - // governor.labels.stage policy as the PR one, read live. - client.SetIssueStageLabels(func() github.IssueStagePolicy { - p := stageLabelPolicyFunc(cfg)() - return github.IssueStagePolicy{Enabled: p.Enabled, Dwell: p.Dwell} - }) - client.SetReviewCadenceLimits( - func() bool { return cfg.Review.CombinedPerspectives }, - func() int { return cfg.Review.MaxReviewsPerHead }, - ) - // review.require_labels (PR approval allow-list): the relay's half of the - // gate, read live so a Features-tab edit applies to the next request. - client.SetReviewRequireLabels(func() []string { return cfg.Review.RequireLabels }) -} - -// reviewBacklogRouting resolves review.severity / review.backlog and the -// backlog sink for the active work source (hivecommons/hive#11089). A -// destination whose work source cannot be built falls back to GitHub issues; -// the reason is logged so the operator can fix the work-source config. -func reviewBacklogRouting(client *github.Client, cfg *config.Config, logger *slog.Logger) github.ReviewBacklogRouting { - ghToken := cfg.GitHub.Token - if ghToken == "" { - ghToken = os.Getenv("HIVE_GITHUB_TOKEN") - } - sink, err := worksource.NewReviewBacklogSink(cfg.Governor.WorkSource, cfg.Review.Backlog, client, ghToken, cfg.Project.Org, logger) - if err != nil && logger != nil { - logger.Warn("review backlog: destination unavailable, filing GitHub issues", "error", err) - } - return github.ReviewBacklogRouting{Severity: cfg.Review.Severity, Backlog: cfg.Review.Backlog, Sink: sink} -} - -// installReviewBots installs classification.review_bots on a (possibly -// rebuilt) GitHub client (hivecommons/hive#7360). hive.yaml's block wins; -// otherwise the same key is read from hive-project.yaml -// (config.DefaultProjectYAMLPath, overridable via HIVE_PROJECT_YAML — the -// path the bash pipeline stages already honour). Nil-safe: a hive without -// GitHub credentials runs with a nil client for the life of the process. -// Also installs review.fix_human_prs (hivecommons/hive#9361), so the -// review-thread reconciler follows PRs the hive did not open whenever the -// fixer is already allowed to push to them. -func installReviewBots(client *github.Client, cfg *config.Config, logger *slog.Logger) { - if client == nil || cfg == nil { - return - } - rb, err := cfg.EffectiveReviewBots(os.Getenv("HIVE_PROJECT_YAML")) - if err != nil && logger != nil { - logger.Warn("classification.review_bots: project file unreadable; review-thread reconciler stays off", "error", err) - } - client.SetReviewBots(rb) - client.SetFixHumanPRs(cfg.Review.FixHumanPRsEnabled()) - if logger != nil && rb.Enabled() { - logger.Info("review-thread reconciler enabled", - "review_bots", rb.Logins, - "max_attempts_per_thread", rb.MaxAttempts(), - "resolve_after_fix", rb.ResolveAfterFixEnabled(), - "fix_human_prs", cfg.Review.FixHumanPRsEnabled()) - } -} - -// reviewThreadsRefreshInterval throttles the review-thread monitor: the eval -// tick runs about once a minute, but every hive-authored open PR costs one -// GraphQL query per pass, and bot threads move on the scale of minutes, not -// seconds. A var so tests can drive it. -var reviewThreadsRefreshInterval = 5 * time.Minute - -// reviewThreadsLastRefresh is the wall-clock of the last CollectReviewThreads -// pass (zero = never). Package-level because the eval tick is a free -// function; only the eval goroutine touches it. -var reviewThreadsLastRefresh time.Time - -// writeReviewThreads is the eval-tick half of the #7360 reconciler. It runs -// CollectReviewThreads over the governor's actionable PRs at most once per -// reviewThreadsRefreshInterval, stamps each PR with the agent whose relay -// request opened it (auditPRAgents — the same attribution ci-failing.json -// carries) and with the escalation sweep's needs-human verdict, and writes -// review-threads.json. The feature-off case still writes the (empty) file -// each refresh so a reader can tell "off" from "never ran". -func writeReviewThreads(ctx context.Context, client *github.Client, actionable *github.ActionableResult, org string, escalatedPRs map[string]bool, logger *slog.Logger) { - if client == nil || actionable == nil { - return - } - now := time.Now() - if !reviewThreadsLastRefresh.IsZero() && now.Sub(reviewThreadsLastRefresh) < reviewThreadsRefreshInterval { - return - } - reviewThreadsLastRefresh = now - - report := client.CollectReviewThreads(ctx, actionable.PRs.Items, now) - if len(report.PRs) > 0 { - prAgents := auditPRAgents(org, now.Add(-auditPRAttributionWindow), "") - for i := range report.PRs { - pr := &report.PRs[i] - pr.Agent = prAgents[fmt.Sprintf("%s#%d", pr.Repo, pr.Number)] - pr.Escalated = escalatedPRs[escalation.Key(pr.Repo, pr.Number)] - } - } - if err := github.WriteReviewThreadsReport("", report); err != nil { - logger.Warn("failed to write review-threads.json", "error", err) - return - } - if report.Enabled { - logger.Info("review-threads.json refreshed", "prs", len(report.PRs), "threads", report.TotalThreads) - } -} - -// auditPRAttributionWindow bounds how far back the audit trail is scanned to -// map open PRs to the agent that opened them. Red PRs older than this fall -// back to scanner ownership in the kick builders — acceptable: 14d exceeds any -// PR the fleet should still be iterating on. -const auditPRAttributionWindow = 14 * 24 * time.Hour - -// auditPRAgents maps "org/repo#number" → agent name from the audit trail's -// agent_pr_created entries (attribution.go records one per relay-opened PR, -// reuses included). Reading the on-disk log per eval tick keeps this -// stateless; OutputActionsSince touches no receiver state, so a zero-value -// AuditLog is safe here. -func auditPRAgents(org string, since time.Time, auditPath string) map[string]string { - entries := (&dashboard.AuditLog{}).OutputActionsSince(since, - map[string]bool{github.AuditActionAgentPRCreated: true}, auditPath) - out := make(map[string]string, len(entries)) - for _, e := range entries { - if e.Agent == "" { - continue - } - var repo, number string - for _, part := range strings.Split(e.Detail, ",") { - if k, v, ok := strings.Cut(strings.TrimSpace(part), "="); ok { - switch k { - case "repo": - repo = v - case "number": - number = v - } - } - } - if repo == "" || number == "" { - continue - } - if !strings.Contains(repo, "/") && org != "" { - repo = org + "/" + repo - } - out[repo+"#"+number] = e.Agent - } - return out -} - -// applyHumanDecisionLabels mirrors the review swarm's "a human must decide" -// holds onto the operator's existing triage label, so they can be filtered -// from the PR list instead of being found by opening threads. -// -// This is strictly additive to the marker the reviewer writes into the review -// body. Every failure path here — no label configured, label absent from the -// repo, API refusal — is logged and skipped, because a mislabelled hold is a -// smaller problem than a hold that never got reported. -func applyHumanDecisionLabels(ctx context.Context, cfg *config.Config, ghClient *github.Client, actionable *github.ActionableResult, plan review.DispatchPlan, logger *slog.Logger) { - if cfg == nil || ghClient == nil { - return - } - label := strings.TrimSpace(cfg.Review.HumanDecisionLabel) - if label == "" || len(plan.State.Human) == 0 { - return - } - // Holds persist across cycles, so re-deriving them every pass would re-ask - // GitHub to apply a label the PR already carries. The enumeration already - // fetched each PR's labels, so skipping the settled ones costs nothing. - // - // Holds carry the owner/repo form (they descend from review reports, which - // record GitHub's full name) while PRs.Items carry whatever governor.repos - // says — usually the bare repo under project.org. Key both through - // fullRepoName or the skip never matches and every cycle re-labels every - // hold (observed on a hive whose repos were configured bare). - org := cfg.Project.Org - labeled := map[string]bool{} - if actionable != nil { - for _, pr := range actionable.PRs.Items { - for _, have := range pr.Labels { - if strings.EqualFold(strings.TrimSpace(have), label) { - labeled[strings.ToLower(fmt.Sprintf("%s#%d", fullRepoName(pr.Repo, org), pr.Number))] = true - } - } - } - } - failing := map[string]struct{}{} - defer func() { humanLabelWarn.retain(failing) }() - for _, hold := range plan.State.Human { - if labeled[strings.ToLower(fmt.Sprintf("%s#%d", fullRepoName(hold.Repo, org), hold.Number))] { - continue - } - if err := ghClient.ApplyHumanDecisionLabel(ctx, hold.Repo, hold.Number, label); err != nil { - // Missing optional labels are configuration notices, not runtime - // failures: first warnings across different PRs can trigger fleet - // reports even with per-PR deduplication (#13857). - key := prWarnKey(hold.Repo, hold.Number) - missing := errors.Is(err, github.ErrHumanDecisionLabelMissing) - if missing { - key += ":missing-label" - } - failing[key] = struct{}{} - level := humanLabelWarn.level(key) - if missing && level == slog.LevelWarn { - level = slog.LevelInfo - } - logger.Log(ctx, level, "human decision label not applied; review marker still stands", - "repo", hold.Repo, "pr", hold.Number, "label", label, "error", err) - continue - } - logger.Info("human decision label applied", "repo", hold.Repo, "pr", hold.Number, "label", label, "reason", hold.Reason) - } -} - -// parseReviseCutoff reads Review.ReviseVerdictsBefore. A malformed value is -// logged and ignored rather than defaulting to "now", because a cutoff that -// silently becomes the current time would re-open every verdict in the -// artifact at once — the opposite of the narrow, deliberate correction this -// setting exists for. -// reviewPerspectiveSet resolves the hive's configured review perspectives. -// -// A bad configuration falls back to the built-in set and says so, rather than -// disabling review. Silently reviewing nothing would look identical to a healthy -// hive with an empty queue; reviewing with the defaults while logging the error -// keeps coverage up and makes the mistake findable. -func reviewPerspectiveSet(cfg *config.Config, logger *slog.Logger) review.PerspectiveSet { - if cfg == nil { - return review.PerspectiveSet{} - } - set, err := review.NewPerspectiveSet(cfg.Review.Perspectives, cfg.Review.PerspectivePrompts) - if err != nil { - if logger != nil { - logger.Warn("review.perspectives is invalid; reviewing with the built-in set until it is fixed", - "error", err) - } - set = review.PerspectiveSet{} - } - // plan_match (#8317) is a separate switch from the selection so turning - // it on never requires spelling out the whole set. Applied on the - // fallback path too: a typo elsewhere in the selection must not silently - // turn plan matching off. - if cfg.Review.PlanMatch.Enabled { - set = set.WithPlanMatch() - } - return set -} - -// planWaveFor renders the approved plan a PR's run trailers name, for the -// plan_match perspective (#8317). The Hive-Plan ref is tried first, then the -// Hive-Run key, since either may carry the epic ID or the source issue ref. -// Empty when plan_match is off, the PR has no trailer, or no store holds the -// plan -- the kick tells the reviewer which of those it is. -func planWaveFor(cfg *config.Config, stores map[string]*beads.Store, planRef, runKey string) string { - if cfg == nil || !cfg.Review.PlanMatch.Enabled { - return "" - } - for _, ref := range []string{planRef, runKey} { - if tree, ok := planning.FindPlanTree(stores, ref); ok { - return tree.WaveText() - } - } - return "" -} - -func parseReviseCutoff(raw string, logger *slog.Logger) time.Time { - trimmed := strings.TrimSpace(raw) - if trimmed == "" { - return time.Time{} - } - cutoff, err := time.Parse(time.RFC3339, trimmed) - if err != nil { - if logger != nil { - logger.Warn("review revise_verdicts_before is not RFC3339; ignoring", - "value", trimmed, "error", err) - } - return time.Time{} - } - return cutoff -} - -// reviewPlannerGitHub is the slice of the GitHub client the review planner -// reads: a PR's changed paths for the bot-author model route -// (review.bot_author_full_review_paths) and the compare between the head -// the hive last judged and the current one for the delta re-review -// (review.delta_rereview). -type reviewPlannerGitHub interface { - ReviewPRChangedFiles(ctx context.Context, repo string, number int) ([]string, bool) - ReviewDeltaDiff(ctx context.Context, repo, previousHead, head string) review.DeltaDiff -} - -func planReviewDispatch(cfg *config.Config, actionable *github.ActionableResult, agentMgr crux.Runtime, beadStores map[string]*beads.Store, logger *slog.Logger) review.DispatchPlan { - return planReviewDispatchWith(context.Background(), cfg, actionable, agentMgr, beadStores, nil, logger) -} - -// reviewBotAuthorRoute is review.bot_authors / bot_author_model / -// bot_author_full_review_paths resolved for the planner. -func reviewBotAuthorRoute(cfg *config.Config) review.BotAuthorRoute { - backend, model := cfg.Review.BotAuthorModelRef() - return review.BotAuthorRoute{Authors: cfg.Review.EffectiveBotAuthors(), Backend: backend, Model: model, FullReviewPaths: cfg.Review.EffectiveBotAuthorFullReviewPaths()} -} - -// reviewBotRouteFilesFor fetches a PR's changed paths when the bot-author -// route may apply to it: route on, author a listed bot, no verdict at this -// head and no review in flight (one listing per head, not per cycle). nil, -// false otherwise, which the planner reads as unknown: no downgrade. -func reviewBotRouteFilesFor(ctx context.Context, cfg *config.Config, lister reviewPlannerGitHub, route review.BotAuthorRoute, artifact review.Artifact, state review.DispatchState, repo string, number int, author, headSHA, bodyDigest string) ([]string, bool) { - if lister == nil || !route.Enabled() || !review.IsBotAuthor(author, route.Authors, cfg.EffectiveAIAuthor()) { - return nil, false - } - if _, judged := artifact.AggregateForBody(repo, number, headSHA, bodyDigest); judged { - return nil, false - } - if !state.PendingSince(repo, number, headSHA).IsZero() { - return nil, false - } - return lister.ReviewPRChangedFiles(ctx, repo, number) -} - -// reviewDeltaDiffFor fetches the delta compare for one PR when it is a -// delta re-review candidate: delta mode on, no verdict yet at this head, no -// dispatch entry for it (a compare per cycle for a review in flight would be -// API spend for nothing), and a prior verdict at another head to diff from. -// nil otherwise, which the planner reads as "not fetched". -func reviewDeltaDiffFor(ctx context.Context, cfg *config.Config, differ reviewPlannerGitHub, artifact review.Artifact, state review.DispatchState, repo string, number int, headSHA, bodyDigest string) *review.DeltaDiff { - if differ == nil || !cfg.Review.DeltaRereviewEnabled() || strings.TrimSpace(headSHA) == "" { - return nil - } - if _, judged := artifact.AggregateForBody(repo, number, headSHA, bodyDigest); judged { - return nil - } - if !state.PendingSince(repo, number, headSHA).IsZero() { - return nil - } - prior, ok := artifact.PriorAggregateFor(repo, number, headSHA) - if !ok { - return nil - } - diff := differ.ReviewDeltaDiff(ctx, repo, prior.HeadSHA, headSHA) - return &diff -} - -func planReviewDispatchWith(ctx context.Context, cfg *config.Config, actionable *github.ActionableResult, agentMgr crux.Runtime, beadStores map[string]*beads.Store, ghr reviewPlannerGitHub, logger *slog.Logger) review.DispatchPlan { - if cfg == nil || actionable == nil || !cfg.Review.RequireApproval || !cfg.Review.FanOut { - return review.DispatchPlan{} - } - botRoute := reviewBotAuthorRoute(cfg) - state, err := review.LoadDispatchState("") - if err != nil && !os.IsNotExist(err) { - logger.Warn("review dispatch state unavailable; starting fresh", "error", err) - } - // classification.review_bots, read the same way installReviewBots reads - // it for the thread reconciler (#7360): hive.yaml wins, hive-project.yaml - // is the fallback. Feeds the reviewer's read step (#9360); empty/off - // leaves ReviewBotLogins empty and the prompt section is skipped. - reviewBots, rbErr := cfg.EffectiveReviewBots(os.Getenv("HIVE_PROJECT_YAML")) - if rbErr != nil && logger != nil { - logger.Warn("classification.review_bots: project file unreadable; reviewer will not see bot findings this pass", "error", rbErr) - } - artifact, err := review.LoadArtifact("") - if err != nil && !os.IsNotExist(err) { - logger.Warn("review verdict artifact unavailable for dispatch planning", "error", err) - } - prAgents := auditPRAgents(cfg.Project.Org, time.Now().Add(-auditPRAttributionWindow), "") - candidates := make([]github.PullRequest, 0, len(actionable.PRs.Items)+len(actionable.PRs.Held)) - seenPRs := map[string]bool{} - labelGated := 0 - addCandidates := func(items []github.PullRequest) { - for _, pr := range items { - fullRepo := fullRepoName(pr.Repo, cfg.Project.Org) - key := github.ReviewQueueKey(fullRepo, pr.Number) - if fullRepo == "" || pr.Number <= 0 || seenPRs[key] { - continue - } - seenPRs[key] = true - // review.require_labels: an unlabelled PR is never dispatched - // to the swarm. The relay refuses it independently. - if !cfg.Review.AdmitsPRLabels(pr.Labels) { - labelGated++ - continue - } - candidates = append(candidates, pr) - } - } - addCandidates(actionable.PRs.Items) - addCandidates(actionable.PRs.Held) - if labelGated > 0 && logger != nil { - logger.Info("review label gate withheld PRs from review dispatch", - "withheld", labelGated, "candidates", len(candidates), - "require_labels", strings.Join(cfg.Review.NormalizedRequireLabels(), ",")) - } - sort.SliceStable(candidates, func(i, j int) bool { - ri, rj := candidates[i].ReviewRank, candidates[j].ReviewRank - if ri > 0 && rj > 0 && ri != rj { - return ri < rj - } - return ri > 0 && rj <= 0 - }) - - scopeReader := reviewScopeLinearReader(cfg, logger) - prs := make([]review.PullRequest, 0, len(candidates)) - for _, pr := range candidates { - lane := classify.Classify(github.Issue{Title: pr.Title, Labels: pr.Labels}).Lane - fullRepo := fullRepoName(pr.Repo, cfg.Project.Org) - item := review.PullRequest{ - Repo: pr.Repo, - Number: pr.Number, - Title: pr.Title, - Author: pr.Author, - Labels: append([]string(nil), pr.Labels...), - HeadSHA: pr.HeadSHA, - UpdatedAt: pr.UpdatedAt, - BodyDigest: pr.BodyDigest, - URL: pr.URL, - Lane: string(lane), - // Attribution trailer, carried for consumers and the audit trail. - // Not a planner input: the fix gate decides on the login author - // alone (#8421, #13124). - HiveAttributed: pr.HiveAttributed, - // The authoring model from the same trailer: what review_models - // exclusions compare against, and stamped into the verdict as - // author_model (hivecommons/hive#13243). - AuthorModel: pr.HiveModel, - // Grounding anchor for the review prompt. Repo access is the - // measured active ingredient in review quality (17%→67% hit rate, - // 61% fewer false positives), so the reviewer is told which commit - // to read rather than being left to infer from the diff. - MergeBase: pr.BaseSHA, - AuthorAgent: prAgents[fmt.Sprintf("%s#%d", fullRepo, pr.Number)], - // The author's base over every configured lane, enabled or not, - // so a disabled or retired authoring replica still excludes its - // family from reviewing it (hivecommons/hive#13552). - AuthorBase: reviewAuthorBase(cfg, prAgents[fmt.Sprintf("%s#%d", fullRepo, pr.Number)]), - // Run trailers (#8310) and the plan they name, for plan_match - // (#8317). Identifiers and planner output only: the PR body - // itself still never reaches the prompt. - RunKey: pr.HiveRun, - PlanRef: pr.HivePlan, - PlanWave: planWaveFor(cfg, beadStores, pr.HivePlan, pr.HiveRun), - ScopeContract: pr.ScopeContract, - // A maintainer's changes_requested review or `/hive decision` - // on this head (hivecommons/hive#13041): the planner kicks the - // fix lane with it instead of reviewing the head again. - HumanChangeRequest: review.HumanChangeRequestFor(review.DefaultHumanChangeRequests(), fullRepo, pr.Number, pr.HeadSHA), - // The compare from the last judged head, for the delta - // re-review and its trivial skip (review.delta_rereview). - DeltaDiff: reviewDeltaDiffFor(ctx, cfg, ghr, artifact, state, fullRepo, pr.Number, pr.HeadSHA, pr.BodyDigest), - BaseDiffEmpty: pr.BaseDiffEmpty, - } - // The linked Linear issue's text for the scope contract, read here - // so reviewers never need a Linear token (hivecommons/hive#14906); - // only for a PR about to be reviewed at this head. - if _, judged := artifact.AggregateForBody(fullRepo, pr.Number, pr.HeadSHA, pr.BodyDigest); !judged && state.PendingSince(fullRepo, pr.Number, pr.HeadSHA).IsZero() { - reviewScopeContractFor(ctx, scopeReader, &item, logger) - } - // Changed paths for the bot-author model route's path exceptions - // (review.bot_author_full_review_paths); fetched only for a listed - // bot's PR that is about to be reviewed. - item.ChangedFiles, item.ChangedFilesKnown = reviewBotRouteFilesFor(ctx, cfg, ghr, botRoute, artifact, state, fullRepo, pr.Number, pr.Author, pr.HeadSHA, pr.BodyDigest) - prs = append(prs, item) - } - // PRs a review webhook just fired for go first, so the early eval cycle - // it triggered spends its review budget on them (hivecommons/hive#11091). - prs = prioritizeEventPRs(prs, activeReviewEvents.Load()) - agents := make([]review.AgentCapability, 0, len(cfg.Agents)) - for name, ac := range cfg.EnabledAgents() { - cap := review.CapabilityFromConfig(name, ac) - cap.Paused = cap.Paused || (agentMgr != nil && agentMgr.IsPaused(name)) - cap.Draining = reviewerAutoscaler.Draining(name) - agents = append(agents, cap) - } - reviewBotLogins := []string{} - if reviewBots.Enabled() { - reviewBotLogins = reviewBots.Logins - } - plan := review.PlanDispatch(prs, artifact, state, review.DispatchOptions{ - RequireApproval: cfg.Review.RequireApproval, - FanOut: cfg.Review.FanOut, - MaxParallelReviews: cfg.Review.EffectiveMaxParallelReviews(), - MaxPerspectivesPerPR: cfg.Review.MaxPerspectivesPerPR, - Perspectives: reviewPerspectiveSet(cfg, logger), - CombinedPerspectives: cfg.Review.CombinedPerspectives, - ReviewerAgents: cfg.Review.ReviewerAgents, - FixerAgent: cfg.Review.FixerAgent, - PostComments: cfg.Review.PostComments, - WritingGuideSection: cfg.Project.WritingGuideSection(), - FixHumanPRs: cfg.Review.FixHumanPRsEnabled(), - AcknowledgeNoFindings: cfg.Review.AcknowledgeNoFindings, - ReviseRepos: cfg.Review.ReviseRepos, - ReviseVerdictsBefore: parseReviseCutoff(cfg.Review.ReviseVerdictsBefore, logger), - PendingTTL: cfg.Review.PendingReviewTTL(), - PendingFixTTL: cfg.Review.PendingReviewFixTTL(), - MaxRedispatch: cfg.Review.EffectiveMaxRedispatch(), - HeadMaxRetries: cfg.Review.HeadMaxRetries, - ProjectOrg: cfg.Project.Org, - AIAuthor: cfg.EffectiveAIAuthor(), - Agents: agents, - ReviewBotLogins: reviewBotLogins, - ReviewBotMinPriority: reviewBots.MinPriority, - PerspectiveModels: cfg.Review.PerspectiveModels, - BotAuthorRoute: botRoute, - DeltaRereview: cfg.Review.DeltaRereviewEnabled(), - DeltaTrivialLines: cfg.Review.EffectiveDeltaTrivialLines(), - DeltaMaxAge: cfg.Review.EffectiveDeltaMaxAge(), - }) - recordReviewDeltaRestamps(plan, logger) - for _, k := range plan.ReviewKicks { - if k.BotAuthorRouted { - logger.Info("review routed bot-authored PR to bot_author_model", "repo", k.Repo, "pr", k.Number, "head", k.HeadSHA, "agent", k.Agent, "review_model", k.ReviewModel, "review_backend", k.ReviewBackend) - } - } - for _, sk := range plan.SkippedReviewModels { - logger.Info("review model fallback skipped perspective", "repo", sk.Repo, "pr", sk.Number, "head", sk.HeadSHA, "perspective", sk.Perspective, "agent", sk.Agent, "author_model", sk.AuthorModel) - } - expiredNow := map[string]struct{}{} - for _, e := range plan.Expired { - // Warn once per PR while it keeps expiring (#13687). - key := prWarnKey(e.Repo, e.Number) - expiredNow[key] = struct{}{} - logger.Log(context.Background(), reviewExpiredWarn.level(key), "review pending expired", "repo", e.Repo, "pr", e.Number, "head", e.HeadSHA, "perspective", e.Perspective, "agent", e.Agent, "age", e.Age.Round(time.Second).String(), "expiries", e.Count) - } - reviewExpiredWarn.retain(expiredNow) - for _, h := range plan.Exhausted { - logger.Warn("review dispatch exhausted; holding PR for a human", "repo", h.Repo, "pr", h.Number, "head", h.HeadSHA, "reason", h.Reason) - } - for _, f := range plan.ForcedFixes { - logger.Warn("review fix forced: changes_requested head starved by the revisit path", "repo", f.Repo, "pr", f.Number, "head", f.HeadSHA, "reason", f.Reason, "cycles", f.Cycles, "limit", review.RevisitStarvationCycles) - } - for _, w := range plan.Promoted { - logger.Warn("review dispatch promoted: head starved of a reviewer walked ahead of the queue", "repo", w.Repo, "pr", w.Number, "head", w.HeadSHA, "reason", w.Reason, "cycles", w.Cycles, "since", w.FirstAt, "limit", review.ReviewWaitStarvationCycles) - } - for _, w := range plan.Waiting { - logger.Info("review dispatch waiting: perspectives still owed, no reviewer this cycle", "repo", w.Repo, "pr", w.Number, "head", w.HeadSHA, "reason", w.Reason) - } - if len(plan.ReviewKicks)+len(plan.FixKicks)+len(plan.FixExclusions) > 0 { - logger.Info("review swarm dispatch planned", "review_kicks", len(plan.ReviewKicks), "fix_kicks", len(plan.FixKicks), "fix_excluded", len(plan.FixExclusions), "fix_excluded_reasons", fixExclusionReasonSummary(plan.FixExclusions)) - } - return plan -} - -// fixExclusionReasonSummary renders the cycle's fix exclusions as -// "reason=count,..." (sorted) for the dispatch summary line, so fix_kicks: 0 -// is never printed without saying why (hivecommons/hive#14665). -func fixExclusionReasonSummary(exclusions []review.FixDispatchExclusion) string { - if len(exclusions) == 0 { - return "" - } - counts := map[string]int{} - for _, x := range exclusions { - counts[x.Reason]++ - } - reasons := make([]string, 0, len(counts)) - for r := range counts { - reasons = append(reasons, r) - } - sort.Strings(reasons) - parts := make([]string, 0, len(reasons)) - for _, r := range reasons { - parts = append(parts, fmt.Sprintf("%s=%d", r, counts[r])) - } - return strings.Join(parts, ",") -} - -// recordReviewDeltaRestamps writes the verdicts the planner carried forward -// onto a new head without a session (trivial delta, review.delta_rereview) -// into the verdict artifact, and logs each one. A write failure is logged, -// not fatal: the next cycle plans the same restamp again. -func recordReviewDeltaRestamps(plan review.DispatchPlan, logger *slog.Logger) { - if len(plan.DeltaRestamps) == 0 { - return - } - for _, r := range plan.DeltaRestamps { - note := "" - if r.Aggregate.Delta != nil { - note = r.Aggregate.Delta.Note - } - logger.Info("review delta re-stamped verdict without a session", "repo", r.Repo, "pr", r.Number, "head", r.HeadSHA, "previous_head", r.PreviousHead, "verdict", r.Aggregate.Verdict, "note", note) - } - if _, err := review.RecordDeltaRestamps("", plan.DeltaRestamps, time.Now().UTC()); err != nil { - logger.Warn("failed to record review delta re-stamps", "error", err, "count", len(plan.DeltaRestamps)) - } -} - -// reviewPlannerGitHubFor returns the GitHub client as a reviewPlannerGitHub, -// or nil for a nil client: a typed nil in the interface would not read as nil. -func reviewPlannerGitHubFor(c *github.Client) reviewPlannerGitHub { - if c == nil { - return nil - } - return c -} - -func refreshReviewVerdicts(cfg *config.Config, logger *slog.Logger) { - if cfg == nil || !shouldRefreshReviewVerdicts(cfg) { - return - } - artifact, err := review.CollectAndMerge("", "", review.AggregateOptions{ - // Unanimity is judged against what a PR was eligible to receive. Without - // this the cap makes approve unreachable and every PR aggregates to - // requires_human. - MaxPerspectivesPerPR: cfg.Review.MaxPerspectivesPerPR, - Perspectives: reviewPerspectiveSet(cfg, logger), - SelfAuthoredDecisionPolicy: cfg.Review.EffectiveSelfAuthoredDecisionPolicy(), - SensitivePaths: cfg.Security.SensitivePaths, - ModelStamps: reviewModelStamps(logger), - ExpectedHeads: cfg.ExpectedReviewHeads(), - MissingHeads: reviewMissingHeads(), - }, time.Now().UTC()) - if err != nil { - if !os.IsNotExist(err) { - logger.Warn("failed to refresh review verdicts", "error", err) - } - return - } - logger.Info("review verdict artifact refreshed", "aggregates", len(artifact.Items)) -} - -// reviewModelStamps reads the dispatch state so a verdict whose report -// omitted author_model / review_model is stamped with what the planner knew -// when it dispatched that review (hivecommons/hive#13243). No state is no -// stamps, never an error: the artifact is still correct, just unstamped. -func reviewModelStamps(logger *slog.Logger) review.ModelStamps { - state, err := review.LoadDispatchState("") - if err != nil { - if !os.IsNotExist(err) { - logger.Warn("review dispatch state unavailable for model stamping", "error", err) - } - return nil - } - return review.ModelStampsFromState(state) -} - -func shouldRefreshReviewVerdicts(cfg *config.Config) bool { - if cfg == nil { - return false - } - if cfg.Review.RequireApproval { - return true - } - entries, err := os.ReadDir(review.ReportDir("")) - if err != nil { - return false - } - for _, entry := range entries { - name := entry.Name() - if !entry.IsDir() && strings.HasPrefix(name, review.ReviewReportFilePrefix) && strings.HasSuffix(name, review.ReviewReportFileSuffix) { - return true - } - } - return false -} - -// reviewOutcomeResolveCap bounds how many vanished PRs one eval cycle asks -// GitHub about. A hive pointed at a busy org can see dozens of PRs merge -// between cycles; the rest are resolved next cycle, and an unresolved PR -// reads as still open, never as merged. -const reviewOutcomeResolveCap = 40 - -// prStateGetter is the slice of the GitHub client the outcome ledger needs. -type prStateGetter interface { - GetPRState(ctx context.Context, repo string, number int) (github.PRState, error) -} - -// reviewEvidenceOutcomeRecorder is the optional slice of the GitHub client -// that stamps a resolved outcome onto the PR's evidence bundle, so a PR this -// hive reviewed but did not merge stops reading as "outcome unknown". -type reviewEvidenceOutcomeRecorder interface { - RecordReviewEvidenceOutcome(repo string, number int, st github.PRState, now time.Time) -} - -// recordReviewOutcomes folds this cycle's open-PR enumeration into the -// review-outcome ledger (pkg/review/outcomes.go): every open PR is upserted, -// review evidence from the verdict artifact and the posted-review ledger is -// attached, and PRs that left the list are resolved to merged/closed with one -// GET each. It is what lets the dashboard say whether reviewed PRs merge -// faster than unreviewed ones on THIS hive, rather than counting verdicts. -func recordReviewOutcomes(ctx context.Context, cfg *config.Config, gh prStateGetter, actionable *github.ActionableResult, logger *slog.Logger) { - if cfg == nil || actionable == nil { - return - } - recordReviewOutcomesAt(ctx, cfg, gh, actionable, "", time.Now().UTC(), logger) -} - -func recordReviewOutcomesAt(ctx context.Context, cfg *config.Config, gh prStateGetter, actionable *github.ActionableResult, path string, now time.Time, logger *slog.Logger) { - ledger, err := review.LoadOutcomeLedger(path) - if err != nil { - logger.Warn("review outcomes: ledger unreadable, starting fresh", "error", err) - } - aiAuthor := cfg.EffectiveAIAuthor() - open := make([]review.OpenPR, 0, len(actionable.PRs.Items)) - for _, pr := range actionable.PRs.Items { - full := config.QualifyRepo(cfg.Project.Org, pr.Repo) - open = append(open, review.OpenPR{ - Repo: full, - Number: pr.Number, - Author: pr.Author, - AgentAuthored: pr.HiveAttributed || pr.AppAuthored || (aiAuthor != "" && strings.EqualFold(pr.Author, aiAuthor)), - CreatedAt: pr.CreatedAt, - }) - } - - // Review evidence: the verdict artifact knows the verdict; the links - // ledger knows a review was posted even when its verdict could not bind - // (queue-lane reviews of undispatched PRs). Earliest time wins. - signals := map[string]review.ReviewSignal{} - if art, err := review.LoadArtifact(""); err == nil { - for _, it := range art.Items { - key := it.Repo + "#" + strconv.Itoa(it.Number) - prev, ok := signals[key] - if !ok || (!it.RecordedAt.IsZero() && it.RecordedAt.Before(prev.At)) { - signals[key] = review.ReviewSignal{At: it.RecordedAt, Verdict: it.Verdict} - } else if ok && prev.Verdict == "" { - prev.Verdict = it.Verdict - signals[key] = prev - } - } - } - if links, err := github.LoadReviewLinks(""); err == nil { - for key, link := range links { - at := link.At - if at.IsZero() { - continue - } - prev, ok := signals[key] - if !ok { - signals[key] = review.ReviewSignal{At: at} - } else if at.Before(prev.At) { - prev.At = at - signals[key] = prev + // #4298: per-budget-window history. Written on the same cadence as the + // other series so a pod roll cannot lose more of one than the others. + budgetHist := dashSrv.BudgetWindowHistory() + if len(budgetHist) > 0 { + budgetData, err := json.Marshal(budgetHist) + if err == nil { + atomicWrite(paths.BudgetWindowHistory, budgetData) } } - } - missing := ledger.Observe(now, open, signals) - resolved := 0 - if gh != nil { - for i, m := range missing { - if i >= reviewOutcomeResolveCap { - break - } - st, err := gh.GetPRState(ctx, m.Repo, m.Number) - if err != nil { - logger.Debug("review outcomes: could not resolve vanished PR", "repo", m.Repo, "number", m.Number, "error", err) - continue - } - ledger.Resolve(m.Repo, m.Number, st.State, st.MergedAt, st.ClosedAt) - resolved++ - if rec, ok := gh.(reviewEvidenceOutcomeRecorder); ok { - rec.RecordReviewEvidenceOutcome(m.Repo, m.Number, st, now) + // #4263: convergence soak telemetry, written atomically on the same + // cadence as the other series so a pod roll cannot lose more of one + // than the others. + soakHist := dashSrv.ConvergenceSoakHistory() + if len(soakHist) > 0 { + soakData, err := json.Marshal(soakHist) + if err == nil { + atomicWrite(paths.ConvergenceSoak, soakData) } } } - ledger.Snapshot(now) - if err := ledger.Save(path, now); err != nil { - logger.Warn("review outcomes: save failed", "error", err) - return - } - if len(missing) > 0 { - logger.Info("review outcomes recorded", "open", len(open), "left_queue", len(missing), "resolved", resolved) - } -} - -// reviewFixAuditor is the slice of the dashboard the withheld-fix audit -// needs, so the function is testable without a live server. -type reviewFixAuditor interface { - AuditLog(user, action, detail, agent string) -} - -// auditWithheldReviewFixes puts every fix the planner refused this cycle on -// the audit trail (#8421). A refusal is silent by design — the PR just does -// not get a bot commit — and silence is the one thing an operator cannot -// investigate, so each one names the PR, who opened it, and the -// review.fix_human_prs value at the time (on no longer permits the push: -// hivecommons/hive#12897). The planner records the exclusion and -// the withheld fix only on the cycle that first withholds a head (a persisted -// State.Withheld entry suppresses later cycles), so this does not repeat on -// every eval cycle. -func auditWithheldReviewFixes(plan review.DispatchPlan, auditor reviewFixAuditor, logger *slog.Logger) { - // One WARN per changes_requested head that ends the cycle without a fix - // kick: a PR shown as "Fixing" that nobody is fixing must be loud - // (hivecommons/hive#14665). - for _, x := range plan.FixExclusions { - level := slog.LevelWarn - if x.Reason == review.FixDispatchExcludedNotHiveAuthored || x.Reason == review.FixDispatchExcludedNotHiveAuthoredDCO { - // Policy skip (the hive never pushes to PRs it did not author), not a fault. - level = slog.LevelInfo - } - logger.Log(context.Background(), level, "review fix dispatch excluded", - "repo", x.Repo, "pr", x.Number, "head_sha", x.HeadSHA, "author", x.Author, "reason", x.Reason, "attempts", x.Attempts) - } - for _, w := range plan.WithheldFixes { - logger.Info("review fix withheld: PR not hive-authored; review published, nothing pushed", - "repo", w.Repo, "pr", w.Number, "head_sha", w.HeadSHA, "author", w.Author, "setting", w.Setting, "setting_enabled", w.SettingEnabled) - if auditor == nil { - continue - } - auditor.AuditLog("governor", "review_fix_withheld", - fmt.Sprintf("pr=%s#%d head=%s author=%s setting=%s=%t reason=%s", w.Repo, w.Number, w.HeadSHA, w.Author, w.Setting, w.SettingEnabled, w.Reason), "") - } -} - -// reviewMissingHeads reads the review heads that ran out of retries while -// another head answered (hivecommons/hive#13542), so their perspectives count -// as settled for the decision step. No state is none. -func reviewMissingHeads() map[string][]string { - state, err := review.LoadDispatchState("") - if err != nil { - return nil - } - return review.MissingHeadsFromState(state) -} - -// reviewCommentPoster is the GitHub write postReviewAcknowledgements needs. -type reviewCommentPoster interface { - CreateIssueCommentBudgeted(ctx context.Context, repo string, number int, body string) error -} - -// postReviewAcknowledgements posts the one-line no-findings comment for each -// PR head the review-heads skip rule settled this cycle: every head approved -// every perspective with no findings, so no decision session ran to post it -// (hivecommons/hive#13542). The planner records each head once, so a failed -// post is logged rather than repeated. -func postReviewAcknowledgements(ctx context.Context, poster reviewCommentPoster, plan review.DispatchPlan, logger *slog.Logger) { - if len(plan.Acknowledgements) == 0 || poster == nil { - return - } - for _, a := range plan.Acknowledgements { - if err := poster.CreateIssueCommentBudgeted(ctx, a.Repo, a.Number, a.Body); err != nil { - logger.Warn("review heads: could not post the no-findings acknowledgement", "repo", a.Repo, "pr", a.Number, "head_sha", a.HeadSHA, "error", err) - continue - } - logger.Info("review heads: posted the no-findings acknowledgement", "repo", a.Repo, "pr", a.Number, "head_sha", a.HeadSHA) - } -} - -func persistReviewDispatchState(plan review.DispatchPlan, delivered []review.DispatchKick, logger *slog.Logger) { - writeReviewDispatchState(plan, func(planned []review.DispatchKick) review.DispatchState { - return review.ConfirmDelivered(plan.State, planned, delivered) - }, logger) -} - -// persistReviewDispatchStateInProgress is the persist made while the kick pass -// is still delivering: it keeps the pending rows of planned kicks not yet -// confirmed, so a verdict from a kick still in flight binds to its dispatch -// (hivecommons/hive#15559). The end-of-pass persistReviewDispatchState drops -// the kicks that never landed. -func persistReviewDispatchStateInProgress(plan review.DispatchPlan, delivered []review.DispatchKick, logger *slog.Logger) { - writeReviewDispatchState(plan, func([]review.DispatchKick) review.DispatchState { - return review.ConfirmDeliveredInProgress(plan.State, delivered) - }, logger) -} - -func writeReviewDispatchState(plan review.DispatchPlan, confirm func(planned []review.DispatchKick) review.DispatchState, logger *slog.Logger) { - planned := append(append([]review.DispatchKick(nil), plan.ReviewKicks...), plan.FixKicks...) - if plan.State.GeneratedAt.IsZero() && len(planned) == 0 { - return - } - state := confirm(planned) - if err := review.WriteDispatchState("", state); err != nil { - logger.Warn("failed to persist review dispatch state", "error", err) - } -} - -func atomicWrite(path string, data []byte) { - tmp := path + ".tmp" - if os.WriteFile(tmp, data, 0o644) == nil { - _ = os.Rename(tmp, path) - } } func applyConfigOverrides(cfg *config.Config, o *snapshot.ConfigOverrides) { diff --git a/src/cmd/hive/main_eval_cycle.go b/src/cmd/hive/main_eval_cycle.go new file mode 100644 index 0000000000..c5ad25fbec --- /dev/null +++ b/src/cmd/hive/main_eval_cycle.go @@ -0,0 +1,1574 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "os" + "strconv" + "strings" + "sync/atomic" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/toolapprove" + "github.com/hivecommons/hive/pkg/advisory" + "github.com/hivecommons/hive/pkg/beads" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/dashboard" + "github.com/hivecommons/hive/pkg/dashboard/collect" + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/governor" + "github.com/hivecommons/hive/pkg/hub" + spoke "github.com/hivecommons/hive/pkg/hub/spoke" + "github.com/hivecommons/hive/pkg/ioscan" + "github.com/hivecommons/hive/pkg/loginscan" + "github.com/hivecommons/hive/pkg/notify" + "github.com/hivecommons/hive/pkg/planning" + "github.com/hivecommons/hive/pkg/review" + "github.com/hivecommons/hive/pkg/scheduler" + "github.com/hivecommons/hive/pkg/spokealerts" + "github.com/hivecommons/hive/pkg/timeline" + "github.com/hivecommons/hive/pkg/tokens" + "github.com/hivecommons/hive/pkg/tracing" + "github.com/hivecommons/hive/pkg/worksource" + "go.opentelemetry.io/otel/attribute" +) + +// buildRepoActivityWire maps the dashboard activity collector's per-repo +// snapshot into the plain hub wire structs the heartbeat carries. Kept here (in +// the one package that imports both hub and dashboard) so pkg/hub never has to +// import pkg/dashboard back — that would be an import cycle, since dashboard +// already imports hub. A field-by-field copy, mirroring how the fleet-stat +// scalars are lifted out of their snapshot at the beat's build site. +func buildRepoActivityWire(repos []collect.RepoActivity) []spoke.RepoActivityWire { + if len(repos) == 0 { + return nil + } + stat := func(s collect.ActivityActionStat) spoke.ActivityStatWire { + return spoke.ActivityStatWire{Count: s.Count, NewestAt: s.NewestAt} + } + out := make([]spoke.RepoActivityWire, 0, len(repos)) + for _, r := range repos { + agents := make([]spoke.AgentRepoActivityWire, 0, len(r.Agents)) + for _, a := range r.Agents { + agents = append(agents, spoke.AgentRepoActivityWire{ + Agent: a.Agent, + Issues: stat(a.Issues), + PRs: stat(a.PRs), + Comments: stat(a.Comments), + Merges: stat(a.Merges), + Claims: stat(a.Claims), + Reviews: stat(a.Reviews), + Advisory: stat(a.Advisory), + Reconciled: stat(a.Reconciled), + }) + } + out = append(out, spoke.RepoActivityWire{ + Repo: r.Repo, + Issues: stat(r.Issues), + PRs: stat(r.PRs), + Comments: stat(r.Comments), + Merges: stat(r.Merges), + Claims: stat(r.Claims), + Reviews: stat(r.Reviews), + Advisory: stat(r.Advisory), + Reconciled: stat(r.Reconciled), + Agents: agents, + }) + } + return out +} + +// providerBudgetNotify is the one-shot guard for the provider spend-rebuff +// notification (#4294). Package-level because runEvalCycle is a function called +// once per tick with no state of its own; mutex-guarded inside pkg/governor. +var providerBudgetNotify governor.ProviderBudgetNotifyState + +// providerBudgetProbe remembers when the last probe kick was released while a +// provider spend rebuff is latched. Package-level for the same reason as +// providerBudgetNotify: runEvalCycle has no state of its own. +var providerBudgetProbe governor.ProviderBudgetProbeState + +// governorKickFairness remembers, per agent, how many cycles in a row the +// governor kick was deferred behind a delivered review-fix kick +// (hivecommons/hive#15166). Package-level for the same reason as +// providerBudgetProbe: runEvalCycle has no state of its own. +var governorKickFairness = newKickLaneFairness() + +var evalAPIBudgetCycle atomic.Uint64 + +// applyBudgetAlerts is a thin wrapper around spokealerts.ApplyBudget, kept so +// call sites in this file do not need the package-qualified name (same +// pattern as applyModeUnscheduledAlert below). +func applyBudgetAlerts(gov *governor.Governor, trans governor.BudgetTransitions, dashSrv *dashboard.Server, notifier *notify.Notifier) { + spokealerts.ApplyBudget(gov, trans, dashSrv, notifier) +} + +// applyNoCadenceAlert is a thin wrapper around spokealerts.ApplyNoCadence +// (see that package for the banner's design: #5577). +func applyNoCadenceAlert(gov *governor.Governor, dashSrv *dashboard.Server) { + spokealerts.ApplyNoCadence(gov, dashSrv) +} + +func applyModeUnscheduledAlert(gov *governor.Governor, dashSrv *dashboard.Server) { + spokealerts.ApplyModeUnscheduled(gov, dashSrv) +} + +func activeIssueRefsFromLastActionable(actionable *github.ActionableResult) (map[string]struct{}, map[string]struct{}) { + if actionable == nil { + return nil, nil + } + refs := make(map[string]struct{}) + coveredRepos := make(map[string]struct{}) + for repo := range actionable.TotalByRepo { + coveredRepos[repo] = struct{}{} + } + for _, issue := range actionable.Issues.Items { + if ref := planning.IssueRef(issue); ref != "" { + refs[ref] = struct{}{} + } + } + for _, held := range actionable.Hold.Items { + if held.Type != "issue" || held.Repo == "" || held.Number <= 0 { + continue + } + ref := planning.IssueRef(github.Issue{Repo: held.Repo, Number: held.Number, Title: held.Title}) + if ref != "" { + refs[ref] = struct{}{} + } + } + return refs, coveredRepos +} + +// agentKicker adapts crux.Runtime to planning.Kicker for the Phase 3 +// stall-replan lane. Kick delegates to SendKick, which takes the manager lock +// ITSELF and is only ever called here from the governor tick (never from the +// agent-launch path), so it cannot re-enter a held manager lock. This is the +// same out-of-band kick path the eval loop already uses for governor kicks. +type agentKicker struct{ mgr crux.Runtime } + +func (k agentKicker) Kick(agent, message string) error { + if k.mgr == nil { + return fmt.Errorf("agent manager unavailable: cannot kick %s", agent) + } + return k.mgr.SendKick(agent, message) +} + +// planFromLabeledIssues is Phase 4 Part B: for each actionable issue carrying a +// `plan`/`epic` label, mint an epic (idempotent) and hand it to the architect, +// RESPECTING the architect's pause. It is a plain synchronous call on the eval +// tick — no goroutine — and only ever touches the manager via SendKick/IsPaused, +// exactly like every governor kick, so it never re-enters the launch-path mutex. +// Epics are minted into the architect store (falling back to any store) so the +// dashboard plan-review flow and replan lane find them the same way. +// +// A minted epic is decompose_pending (plan_status=draft). While the architect is +// paused, the request stays queued and visible (the PLANNING tile shows it as +// pending) — we never force-unpause. Once the architect is available, we kick it +// each cycle until it decomposes and clears the pending marker. +func planFromLabeledIssues( + actionable *github.ActionableResult, + beadStores map[string]*beads.Store, + agentMgr crux.Runtime, + gov *governor.Governor, + dashSrv *dashboard.Server, + logger *slog.Logger, + cfg *config.Config, + acmmLevel int, +) { + if actionable == nil || len(beadStores) == 0 { + return + } + store, ok := beadStores[planning.ArchitectAgentName] + if !ok { + for name := range beadStores { + store = beadStores[name] + break + } + } + if store == nil { + return + } + + designCfg := planning.DesignConfig{ + PlanLabels: cfg.Planning.PlanLabelsOrDefault(), + DesignLabels: cfg.Planning.DesignLabelsOrDefault(), + ApprovedLabel: cfg.Planning.DesignApprovedLabelOrDefault(), + MaxRevisions: cfg.Planning.MaxDesignRevisionsOrDefault(), + MaxConcurrent: cfg.Planning.MaxConcurrentDesignsOrDefault(), + } + issues := actionable.Issues.Items + if cfg == nil || !cfg.Planning.PlanFromLabelEnabled(acmmLevel) { + issues = nil + for _, issue := range actionable.Issues.Items { + if store.FindByExternalRef(planning.IssueRef(issue)) != nil { + issues = append(issues, issue) + } + } + } + sink := labelPlanSink{gov: gov, dashSrv: dashSrv, logger: logger} + planning.PlanIssuesFromLabelsWithConfig(store, agentMgr, issues, designCfg, sink, + func(ref string, err error) { + logger.Warn("plan-from-label: minting epic failed", "issue", ref, "error", err) + }, acmmLevel) + if config.PlanAutoApproveForLevel(acmmLevel) { + for _, id := range planning.AutoApproveDrafts(store) { + if dashSrv != nil { + dashSrv.AuditLog("planning", "plan_auto_approved", "epic="+id, planning.ArchitectAgentName) + } + logger.Info("audit: plan auto-approved by ACMM pack", "epic", id, "acmm_level", acmmLevel) + } + } +} + +// labelPlanSink adapts the governor/dashboard/logger to planning.LabelPlanSink so +// the label-trigger core lives (and is tested) in pkg/planning. +type labelPlanSink struct { + gov *governor.Governor + dashSrv *dashboard.Server + logger *slog.Logger +} + +func (s labelPlanSink) KickedPlan(epic *beads.Bead) { + s.gov.RecordKick(planning.ArchitectAgentName) + if s.dashSrv != nil { + s.dashSrv.AuditLog("planning", "plan_from_label", "epic="+epic.ID+" ref="+epic.ExternalRef, planning.ArchitectAgentName) + } + s.logger.Info("audit: plan requested from labeled issue", "epic", epic.ID, "ref", epic.ExternalRef) +} + +func (s labelPlanSink) FailedPlan(epic *beads.Bead) { + if s.dashSrv != nil { + s.dashSrv.AuditLog("planning", "plan_decompose_failed", "epic="+epic.ID+" ref="+epic.ExternalRef+" attempts="+strconv.Itoa(planning.DecomposeMaxAttempts), planning.ArchitectAgentName) + } + s.logger.Warn("plan-from-label: architect produced no plan after max attempts — epic marked stuck; re-request it from the dashboard", + "epic", epic.ID, "ref", epic.ExternalRef, "attempts", planning.DecomposeMaxAttempts) +} + +func (s labelPlanSink) QueuedPlan(epic *beads.Bead, paused bool) { + if paused { + // Architect deliberately paused — queue, do not unpause, log once. + s.logger.Info("plan-from-label: architect paused, plan queued", "epic", epic.ID, "ref", epic.ExternalRef) + return + } + s.logger.Warn("plan-from-label: architect unavailable, plan queued", "epic", epic.ID, "ref", epic.ExternalRef) +} + +func (s labelPlanSink) KickedDesign(epic *beads.Bead, revision int) { + s.gov.RecordKick(planning.ArchitectAgentName) + if s.dashSrv != nil { + s.dashSrv.AuditLog("planning", "design_kicked", "epic="+epic.ID+" ref="+epic.ExternalRef+" revision="+strconv.Itoa(revision), planning.ArchitectAgentName) + } + s.logger.Info("audit: design requested from labeled issue", "epic", epic.ID, "ref", epic.ExternalRef, "revision", revision) +} + +func (s labelPlanSink) ApprovedDesign(epic *beads.Bead) { + if s.dashSrv != nil { + s.dashSrv.AuditLog("planning", "design_approved", "epic="+epic.ID+" ref="+epic.ExternalRef, planning.ArchitectAgentName) + } + s.logger.Info("audit: design approved from labeled issue", "epic", epic.ID, "ref", epic.ExternalRef) +} + +func (s labelPlanSink) DesignNeedsHuman(epic *beads.Bead, revisions int) { + if s.dashSrv != nil { + s.dashSrv.AuditLog("planning", "design_needs_human", "epic="+epic.ID+" ref="+epic.ExternalRef+" revisions="+strconv.Itoa(revisions), planning.ArchitectAgentName) + } + s.logger.Warn("plan-from-label: design revision cap reached — epic needs a human", "epic", epic.ID, "ref", epic.ExternalRef, "revisions", revisions) +} + +// maxTimelineEnumeratePerCycle bounds how many enumerated-issue events a single +// eval cycle records into the lifecycle timeline, keeping the recording loop +// O(1)-bounded so it never slows the eval cycle. Since #5656 the store dedupes +// by (ref, kind) — re-enumeration refreshes the journey instead of appending — +// so the cap no longer protects the ring from eviction floods; it matches the +// endpoint's default journey limit so every renderable journey gets refreshed. +const maxTimelineEnumeratePerCycle = 200 + +// lifecycleRecorder narrows *dashboard.Server to just the timeline accessor the +// recording helpers need, so they stay trivially testable with a fake and never +// depend on the rest of the Server surface. +type lifecycleRecorder interface { + LifecycleTimeline() *timeline.Store +} + +// recordEnumeratedIssues records a KindEnumerated event for each enumerated +// actionable issue, bounded by maxTimelineEnumeratePerCycle. The store dedupes +// by (ref, kind), so each cycle refreshes the journeys' enumerated stage +// rather than appending a flood (#5656). It is fully guarded: a nil recorder, +// nil store, or nil actionable set is a no-op, and Record itself is nil-safe. +// This must never slow or break the eval loop, so it does no blocking I/O +// (journey persistence is throttled and atomic inside the store). +// +// PR-open/merge lifecycle spans are emitted by the same tracing mapper when +// callers record those timeline events; this helper only has enumerated issues. +func recordEnumeratedIssues(ctx context.Context, rec lifecycleRecorder, actionable *github.ActionableResult) { + if ctx == nil { + ctx = context.Background() + } + if rec == nil || actionable == nil { + return + } + store := rec.LifecycleTimeline() + if store == nil { + return + } + limit := maxTimelineEnumeratePerCycle + if len(actionable.Issues.Items) < limit { + limit = len(actionable.Issues.Items) + } + for i := 0; i < limit; i++ { + issue := actionable.Issues.Items[i] + event := timeline.Event{ + IssueRef: issueRef(issue.Repo, issue.Number), + Kind: timeline.KindEnumerated, + } + // The GitHub created_at is the open→triage / open→merge SLA origin; + // retained here so the SLA never re-fetches the issue. + if !issue.CreatedAt.IsZero() { + event.Attrs = map[string]string{timeline.AttrIssueCreatedAt: issue.CreatedAt.UTC().Format(time.RFC3339)} + } + _, span := tracing.StartTimelineSpan(ctx, event) + store.Record(event) + span.End() + } +} + +// recordKick records KindKicked events for the given agent. When issue refs are +// supplied it records one issue-scoped event per ref so post-completion lanes +// can reconstruct per-bead kick counts. With no refs, it records one +// agent-scoped event. Guarded and nil-safe; no I/O. +func recordKick(ctx context.Context, rec lifecycleRecorder, agent string, issueRefs ...string) { + if ctx == nil { + ctx = context.Background() + } + if rec == nil { + return + } + store := rec.LifecycleTimeline() + if store == nil { + return + } + if len(issueRefs) > 0 { + for _, ref := range issueRefs { + if ref == "" { + continue + } + event := timeline.Event{ + IssueRef: ref, + Kind: timeline.KindKicked, + Agent: agent, + } + _, span := tracing.StartTimelineSpan(ctx, event) + store.Record(event) + span.End() + } + return + } + event := timeline.Event{ + Kind: timeline.KindKicked, + Agent: agent, + } + _, span := tracing.StartTimelineSpan(ctx, event) + store.Record(event) + span.End() +} + +// issueRef renders the canonical "repo#number" reference the timeline uses to +// group events by issue. An empty repo yields an empty ref (the store tolerates +// it). +func issueRef(repo string, number int) string { + if repo == "" { + return "" + } + return fmt.Sprintf("%s#%d", repo, number) +} + +func actionableIssueRef(issue github.Issue) string { + ref := worksource.Ref{ + SourceType: issue.SourceType, + Repo: issue.Repo, + ExternalID: issue.ExternalID, + Number: issue.Number, + URL: issue.URL, + } + if key := ref.Key(); key != "" { + return key + } + if issue.Repo != "" { + return issue.Repo + } + return issue.ExternalID +} + +// actionableAfterGitHubEnumerate decides whether an eval cycle survives a +// failed GitHub enumeration. On the default (GitHub) work source the answer +// is no: an all-repos failure usually means a rate limit or outage, and a +// zero-count result would idle the agents, so the cycle keeps prior state. +// +// On a non-default work source (e.g. Linear), or a default GitHub source with +// additive work enabled (run stages / Wavefront), the GitHub issue call is not +// the only backlog authority. Aborting here meant a Linear-sourced hive whose +// GitHub App could not list issues (403 "Resource not accessible by +// integration", an Issues permission a Linear hive should not need) never +// enumerated its Linear backlog at all and sat at queue 0; the same rule keeps +// local additive sources reachable during a transient GitHub issue outage. Such +// a hive continues with whatever partial result GitHub returned (nil becomes +// an empty result; PRs are kept when obtainable) and lets the overlay populate +// issues. +func actionableAfterGitHubEnumerate(cfg *config.Config, actionable *github.ActionableResult, err error, logger *slog.Logger) (*github.ActionableResult, bool) { + if err == nil { + return actionable, true + } + if !workSourceOverlayEnabled(cfg.Governor.WorkSource) { + logger.Error("failed to enumerate actionable items", "error", err) + return nil, false + } + logger.Warn("GitHub enumeration failed; continuing so the configured work source can still populate issues", + "work_source", cfg.Governor.WorkSource.Type, "run_stages", cfg.Governor.WorkSource.RunStages, + "wavefront", cfg.Governor.WorkSource.Wavefront.Enabled, "error", err) + if actionable == nil { + actionable = &github.ActionableResult{GeneratedAt: time.Now()} + } + // The GitHub share of this result is missing, not zero: every configured + // repo is marked stale so the dashboard shows the gap as stale rather + // than as an empty hive (a cold cache has no last-good to fall back on). + markConfiguredReposStale(actionable, cfg.Project.Repos) + return actionable, true +} + +// markConfiguredReposStale flags every configured repo the result does not +// already cover as stale, with a zero count entry so the repo still appears. +func markConfiguredReposStale(actionable *github.ActionableResult, repos []string) { + if actionable == nil { + return + } + if actionable.TotalByRepo == nil { + actionable.TotalByRepo = map[string]github.RepoCounts{} + } + for _, repo := range repos { + if _, ok := actionable.TotalByRepo[repo]; !ok { + actionable.TotalByRepo[repo] = github.RepoCounts{} + } + } + actionable.MarkAllStale() +} + +func tokensForBackend(summary *tokens.AggregateSummary, backend string) int64 { + if summary == nil { + return 0 + } + var total int64 + for _, sess := range summary.Sessions { + if strings.EqualFold(sess.Backend, backend) { + total += sess.TotalTokens + } + } + return total +} + +func runEvalCycle( + ctx context.Context, + cfg *config.Config, + ghClient *github.Client, + gov *governor.Governor, + sched *scheduler.Scheduler, + agentMgr crux.Runtime, + dashSrv *dashboard.Server, + notifier *notify.Notifier, + beadStores map[string]*beads.Store, + tokenCollector *tokens.Collector, + metricsCollector *dashboard.MetricsCollector, + nousState *dashboard.NousState, + lastActionable *atomic.Pointer[github.ActionableResult], + advisoryStore *advisory.Store, + advisoryIssues map[string]int, + restartedAgents []string, + approvalDesk *toolapprove.Desk, + source evalCycleSource, + logger *slog.Logger, +) { + // Governor eval-cycle span. When tracing is disabled (the default) this is + // a no-op span with no allocation of note and no export — see pkg/tracing. + ctx, span := tracing.StartSpan(ctx, "governor.eval_cycle", + attribute.String("hive.id", cfg.HiveID), + attribute.Int(tracing.AttrHiveACMMLevel, inferACMMLevel(cfg))) + defer span.End() + + // A hive running without GitHub credentials (placeholder app_id, or a real + // App whose key could not be read) has nothing to enumerate. Return before + // the first API call rather than logging a misleading enumeration failure + // once per eval interval — the dashboard banner already states the cause. + if ghClient == nil { + logger.Debug("skipping eval cycle: hive is running without GitHub credentials") + return + } + + budgetMode, _ := ghClient.APIBudgetMode() + budgetDecision := decideEvalBudgetWork(budgetMode, cfg.Governor.OptionalSweepEveryNCycles, evalAPIBudgetCycle.Add(1)) + + // A cached cycle (webhook or re-kick wake) plans from the stored + // snapshot: no listing, and none of the enrichment or unstick sweeps + // that walk every item — those belong to the reconcile pass. The first + // cycle has no snapshot yet and enumerates regardless of source. + cachedSnapshot := cachedSnapshotFor(source, lastActionable) + if cachedSnapshot != nil { + budgetDecision.SkipOptional = true + budgetDecision.SkipUnstick = true + budgetDecision.SkippedSteps = append(budgetDecision.SkippedSteps, cachedSnapshotSkippedStep) + } else if source == evalSourceCached { + logger.Info("eval cycle: no cached snapshot yet; enumerating") + } + ghClient.RecordAPIBudgetSkippedSteps(budgetDecision.SkippedSteps) + + // Re-ensure the pinned advisory issue whenever it is still unresolved, not + // only while the App banner is up (#4167). The startup ensure can fail for + // reasons that deliberately do NOT raise that banner — a rate limit, a 5xx, + // a search-API blip — and the old gate meant such a hive kept an empty + // advisoryIssues map for the rest of the process lifetime: every later + // digest silently found no issue to post to, so the pinned comment froze at + // whatever it last said. Retrying here is cheap (one search per eval cycle + // only while unresolved, nothing once resolved) and is the difference + // between a transient boot error and a permanently wedged digest. + // + // advisoryEnsureErr keeps this cycle's ensure failure so the post-path + // error recorded below can name the CAUSE (e.g. Issues disabled on a fork, + // #4329) instead of only the symptom. + primaryRepoAtCycleStart := primaryAdvisoryRepo(cfg) + _, hadPinnedAdvisoryIssueAtCycleStart := advisoryIssueNumber(advisoryIssues, primaryRepoAtCycleStart) + advisoryEnsureDepsForCycle := advisoryEnsureDeps{setenv: os.Setenv} + if ghClient != nil { + advisoryEnsureDepsForCycle.ensure = ghClient.EnsureAdvisoryIssue + } + advisoryEnsureErr := error(nil) + if !budgetDecision.SkipOptional { + advisoryEnsureErr = ensurePinnedAdvisoryIssue( + ctx, advisoryIssues, primaryRepoAtCycleStart, advisoryEnsureDepsForCycle, logger) + } + + var actionable *github.ActionableResult + if cachedSnapshot != nil { + actionable = cachedSnapshot + logger.Info("eval cycle planning from cached snapshot", + "snapshot_age", time.Since(actionable.GeneratedAt).Round(time.Second).String(), + "issues", actionable.Issues.Count, "prs", actionable.PRs.Count, "hold", actionable.Hold.Total, + "stale", actionable.Stale()) + } else { + enumCtx, enumSpan := tracing.StartSpan(ctx, "governor.enumerate_actionable") + fresh, err := ghClient.EnumerateActionable(enumCtx) + enumSpan.End() + var ok bool + actionable, ok = actionableAfterGitHubEnumerate(cfg, fresh, err, logger) + if !ok { + if kept := keepLastGoodAfterFailedEnumerate(lastActionable, logger); kept != nil && dashSrv != nil { + dashSrv.MarkStatusStale(kept.StaleSince) + } + return + } + + // If a non-default or additive work source is configured, overlay its + // issues onto the actionable result. PRs always come from GitHub. A + // cached cycle keeps the overlay the reconcile pass already applied. + if workSourceOverlayEnabled(cfg.Governor.WorkSource) { + actionable.Issues = workSourceIssuesForConfiguredCycle(ctx, cfg, ghClient, actionable.Issues, logger) + } + } + if swarmRepo := dashSrv.ActiveSwarmRepo(); swarmRepo != "" { + github.BoostActionableRepoPriority(actionable.Issues.Items, swarmRepo) + logger.Info("swarm priority boost applied", "repo", swarmRepo) + } + + ghClient.EnrichCIStatus(ctx, actionable.PRs.Items) + // Held PRs need CI status too, or a red held PR can never be repaired: + // the hold label kept it out of PRs.Items, so nothing ever learned it was + // red and no agent was ever told to fix it (hivecommons/hive#7438). This + // enriches the held list ONLY for the repair path — held PRs still never + // reach the merge sweep, escalation or the queue counts. + if budgetMode != github.APIBudgetCritical { + ghClient.EnrichCIStatus(ctx, actionable.PRs.Held) + } + // Stale drafts sit in the same dashboard PR column; they get the + // review/link signals only — no mergeability or check-run fetches, a + // draft is not a merge candidate (hivecommons/hive#8968). + if !budgetDecision.SkipOptional { + // Sidelined PRs (drafts, exempt labels) sit in the same column and + // need their closing references to pair with the issue they fix + // (hivecommons/hive#13421); one shared collector keeps the budget at + // one query per repo for both. + ghClient.EnrichReviewSignalsAndWorkPRs(ctx, actionable.PRs.StaleDrafts, actionable.WorkDetailsByRepo) + // A closing reference whose issue is already closed (a superseded + // PR's `Fixes #N`) has no open row to borrow a title from; fill it + // once per issue, from a long-lived memo, so the activity monitor + // never shows a bare "#N". + ghClient.EnrichLinkedIssueTitles(ctx, actionable.WorkDetailsByRepo, actionable.PRs.Items, actionable.PRs.Held, actionable.PRs.StaleDrafts) + } + + // Publish the human-facing "what should I merge next?" digest. This reads + // the PR set enumerated and CI-enriched immediately above, so it must stay + // after those two calls: Mergeable and the failing-check names it sorts on + // are populated by EnrichCIStatus, not by EnumerateActionable. + if !budgetDecision.SkipOptional { + postRecommendationsForCycle(ctx, cfg, ghClient, actionable, logger) + } + + // Fold this pass's CI state into the fix-loop staleness clock BEFORE any + // consumer reads it, so the claim-suppression guard (#3), the merge watcher + // (#2), and the stuck-PR reaper (#4) all key off a consistent, current + // signal within the same tick. This only records first-seen-red times; the + // distinct-SHA attempt counting still happens in runEscalationSweep below. + recordRedStaleness(cfg, actionable) + + // Duplicate-PR guard: drop issues an open hive-authored PR already claims, + // before the governor counts the queue or the scheduler builds kicks. A + // restart storm otherwise re-offers the same issue on every fresh agent + // start, and the agent — having no memory of the PR it just filed — files + // another. Backed by a PVC ledger so it survives those restarts, and fails + // closed (keeps the last known claims) when the GitHub API is unavailable. + if !budgetDecision.SkipOptional { + applyDuplicatePRGuard(ctx, cfg, ghClient, actionable, logger) + } + // Every cycle, cached ones included: heal issue stage records the + // claim ledger says an open PR covers but the store still holds below + // 040 (a claim release written over the cover between reconcile passes). + reconcileLedgerCoverRecords(ctx, cfg, ghClient, actionable, logger) + + // PR review queue (#9590): stamp each PR's rank, priority and reasons + // onto the snapshot so last-actionable.json carries them next to + // review_class. Additive only: Items and Held keep their order, and + // nothing that gates work reads the stamp. + reviewQueue := stampReviewQueue(cfg, actionable, logger) + + // Only the reconcile pass publishes the snapshot: a cached cycle works + // on a copy, and storing it back would overwrite a webhook patch that + // landed while the cycle ran. + if cachedSnapshot == nil { + publishEnumeratedSnapshot(lastActionable, actionable, logger) + if data, err := json.Marshal(actionable); err == nil { + atomicWrite(lastActionablePath, data) + } + } + publishStageSharedCIIncidents(actionable) + + // Record enumerated issues into the lifecycle timeline so the dashboard's + // lifecycle view has real data. Cheap and fully guarded: a nil dashboard or + // nil store is a no-op (timeline.Store.Record is nil-safe), the loop is + // bounded by maxTimelineEnumeratePerCycle, and the store dedupes by + // (ref, kind) so this per-cycle sweep refreshes journeys instead of + // flooding them (#5656). + recordEnumeratedIssues(ctx, dashSrv, actionable) + + escalatedPRs := map[string]bool{} + if !budgetDecision.SkipUnstick { + escalatedPRs = runEscalationSweep(ctx, cfg, governorForge(cfg, ghClient, logger), actionable, notifier, dashSrv, logger) + } + + intentVerdicts := writeIntentVerdicts(ctx, cfg, ghClient, actionable, beadStores, logger) + refreshReviewVerdicts(cfg, logger) + recordReviewOutcomes(ctx, cfg, ghClient, actionable, logger) + requiredCheckSet, _ := cfg.AutoMerge.RequiredCheckSetForRepo("") + + // Hold guard (#5589): snapshot hold-gated PR heads, and when a hold lifts + // on a branch that moved, block the merge lanes and force a fresh review. + // Runs before writeMergeEligible so drifted PRs are excluded from the very + // tick their hold lifted — no window for the sweep to race the re-hold. + selfAuthorizationHoldEnabled := func(repo string) bool { + level := cfg.ACMMLevelOrZero() + if agentMgr != nil { + level = agentMgr.GetACMMLevel() + } + return cfg.SelfAuthorizationHoldEnabledForRepoAtLevel(repo, level) + } + holdDriftPRs := enforceHoldGuard(ctx, cfg, ghClient, governorForge(cfg, ghClient, logger), actionable, logger, selfAuthorizationHoldEnabled) + + // The per-PR verdicts come back so the dashboard's PR pills can be + // painted from the sweep's own classification rather than a looser + // reading of GitHub's mergeable flag (hivecommons/hive#7478). + mergeVerdicts := writeMergeEligible(actionable, actionable.Hold, cfg.Project.Org, escalatedPRs, cfg.Intent.Enforce, intentVerdicts, cfg.Review.RequireApproval, requiredCheckSet, holdDriftPRs, cfg, logger) + + // Review-bot threads (hivecommons/hive#7360): list every unresolved + // external-review-bot thread on an open hive-authored PR into + // review-threads.json, attributed to the agent that opened the PR the + // same way ci-failing.json is, so the scheduler can route each PR back + // to its author for a fix + in-thread replies before any new work. + if !budgetDecision.SkipOptional { + writeReviewThreads(ctx, ghClient, actionable, cfg.Project.Org, escalatedPRs, logger) + } + + // PR follow-up session resume (hivecommons/hive#9583, default off): feed + // CI failures, changes-requested reviews and new review-bot threads on a + // PR this hive opened back into the CLI session that authored it, while + // that session is still live. Anything it cannot resume stays on the + // fix-before-new path above, unchanged. + routePRFollowUps(ctx, cfg, ghClient, actionable, escalatedPRs, agentMgr, logger) + + // Stuck-PR reaper (backstop): DELIVER a targeted FIX-BEFORE-NEW kick for any + // hive-authored PR that is red on a required check AND stale (its red head + // SHA unchanged past RedPRStaleAfter). writeMergeEligible already surfaces + // every red PR into ci-failing.json (the CI_FAILING kick block), but relying + // on the owner's next cadence kick to carry it meant a red PR could be + // escalated as "6 automated fix re-dispatches" having been mentioned to + // nobody (hivecommons/hive#9472); the reaper now kicks the owner — or the + // fallback fixer when the owner is paused or unreachable — and charges the + // re-engagement budget only for kicks that were accepted. Composes with the + // merge watcher (#2): both go through the same per-red-SHA cap, so the same + // PR is never double-dispatched within a red-SHA's budget. + reapStuckRedPRs(cfg, actionable, escalatedPRs, agentKicker{mgr: agentMgr}, + agentAvailability(cfg, agentMgr), logger) + + // Release sentinel (hivecommons/hive#9585), opt-in and default OFF: when + // the current v tag's CI fails, dispatch a bounded repair round + // through the same kick path as the reaper above, or escalate to a human + // when the failure is a setting no commit can fix. Never pushes a branch; + // moves the tag to a merged fix PR only with retag_enabled (also opt-in). + runReleaseSentinel(ctx, cfg, ghClient, agentKicker{mgr: agentMgr}, + agentAvailability(cfg, agentMgr), notifier, logger) + + // Upstream watch (hivecommons/hive#9967), opt-in and default OFF: at most + // once per upstream_watch.interval, file a labelled fork issue for each + // upstream merged PR or release that still applies to the fork. Opens + // issues only, never PRs; the watermark and dedupe index live on the PVC. + runUpstreamWatch(ctx, cfg, ghClient, logger) + + // Merge-hotspot advisor: at most hourly, score the files concurrent + // PRs keep colliding on (from the merge train's own records) and file + // one modularize issue per hotspot for the ci-maintainer lane through + // the writeback gate. Skipped with the other optional sweeps when the + // GitHub budget is tight. + if !budgetDecision.SkipOptional { + runHotspotAdvisor(ctx, cfg, ghClient, actionable, logger) + } + + if !budgetDecision.SkipUnstick { + shaResult, shaErr := ghClient.EnforceSHAHold(ctx, github.SHAHoldConfig{ + PrimaryRepo: cfg.Project.PrimaryRepo, + AIAuthor: cfg.Project.AIAuthor, + InternalAuthors: []string{"hivecommons-hive[bot]", "kubestellar-hive[bot]", "hivecommons-hive-ghe[bot]", "kubestellar-hive-ghe[bot]", "github-actions[bot]", "dependabot[bot]", "copilot-swe-agent[bot]"}, + // Trusted reporters get the soft needs-sha label, not the hard + // hold (hivecommons/hive#12002). + TrustedAssociations: cfg.Project.IssueFilter.ReporterTrust.EffectiveTrustedAssociations(), + TrustedLabel: cfg.Issues.EffectiveSHAHoldTrustedLabel(), + }) + if shaErr != nil { + logger.Warn("SHA hold enforcement failed", "error", shaErr) + } else { + logger.Info("SHA hold enforcement complete", + "held", shaResult.Held, + "unheld", shaResult.Unheld, + "softened", shaResult.Softened, + "skipped", shaResult.Skipped, + ) + } + runHardSuppressReconcile(ctx, cfg, ghClient, logger) + } + + // Refresh budget spend from lifetime token totals before Evaluate so + // the kick gate sees current-window numbers. The banner itself is + // level-triggered every cycle (#9612 follow-up): a collector summary may + // not be available yet (right after a restart, before the first session + // scan completes), and the one-shot Crossed edge that used to gate the + // raise is lost on restart too, so gate ONLY the spend refresh on the + // summary and re-derive the banner from whatever budget state the + // governor currently holds (persisted spend if the summary is not in + // yet, refreshed spend once it is) on every cycle. + trans := gov.BudgetLevel() + if tokenCollector != nil { + if summary := tokenCollector.Summary(); summary != nil { + var bobCoins, bobUSD float64 + if coinCfg, ok := cfg.Governor.Budget.CoinConfig(tokens.BackendBob); ok { + bobCoins = coinCfg.TokensToCoins(tokensForBackend(summary, tokens.BackendBob)) + bobUSD = coinCfg.CoinsToUSD(bobCoins) + } + trans = gov.UpdateBudgetFromTotalsCoinsAndUSD(summary.TotalTokens, summary.ByAgent, summary.ByModel, bobCoins, bobUSD) + } + } + applyBudgetAlerts(gov, trans, dashSrv, notifier) + + // Cause+fix banner for the never-kicked class (#5577): the dashboard's + // not-producing warnings name the SYMPTOM (agent idle, zero tokens); this + // names the cause — enabled agent, no cadence in any mode, never kicked — + // and the fix. Computed from the spoke's own governor config, no hub + // round-trip; self-clears the moment a cadence is set or any kick lands. + applyNoCadenceAlert(gov, dashSrv) + + agentsDue := gov.EvaluateWithRepoDepths( + actionable.Issues.Count, + actionable.PRs.Count, + actionable.Hold.Total, + actionable.Issues.SLAViolations, + governor.RepoDepthsFromActionable(actionable), + ) + + // The weaker sibling of the banner above (#7474): an agent SOME mode + // schedules but the mode the fleet is now in does not — a reviewer with a + // cadence only in surge goes silent the moment its own work drives the + // backlog below the surge threshold, and every other signal calls it + // healthy. Applied after Evaluate so it reads the mode this tick settled + // on; self-clears when the mode changes back or the operator fills the + // gap. + applyModeUnscheduledAlert(gov, dashSrv) + + // Crash-restarted agents may get a "resume" kick ahead of their cadence + // slot so work interrupted mid-task resumes promptly — but ONLY through + // the governor's gate (#2573). Unconditionally kicking every restarted + // agent meant a crash-looping CLI was kicked on every eval cycle, + // burning backend tokens far faster than any configured cadence and + // bypassing the budget gate; AllowResumeKick bounds resume kicks to one + // per cadence interval and respects mode pauses and the budget. + // Refused restarts are not silent: an interval-throttled agent sits at + // an empty prompt until its slot, so raise a per-agent alert (with an OOM + // hint when the cgroup killed the CLI). Every cycle, reconcile clears it + // once it no longer describes a problem (#9612): kicked, paused/removed, + // no longer expected to run, working again, or past its max age. + var resumeHeld []spokealerts.ResumeKickHeldAgent + agentsDue, resumeHeld = mergeResumeKicks(agentsDue, restartedAgents, gov.AllowResumeKickReason, logger) + resumeKickHeldAlerts.Apply(dashSrv, resumeHeld, agentMgr.CrashOOMSuspected) + resumeKickHeldAlerts.Reconcile(dashSrv, resumeKickHeldFacts(agentMgr, gov.ResumeKickVerdict)) + + govState := gov.GetState() + span.SetAttributes( + attribute.String(tracing.AttrHiveGovernorMode, string(govState.Mode)), + attribute.Int("hive.queue.issues", govState.QueueIssues), + attribute.Int("hive.queue.prs", govState.QueuePRs), + attribute.Int("hive.queue.hold", govState.QueueHold), + ) + // cadence.Paused (cadence: "pause" in config) means "don't kick this agent + // in this mode" — it does NOT force-pause the agent. Manual pause/resume + // via the dashboard is always respected; the governor only controls kicks. + + // Filter out on-demand agents — they are only triggered explicitly + // Operator-paused agents must consume NOTHING (#2573); see + // filterKickableAgents for the full gate. This runs BEFORE the eval-cycle + // log so "agents_due" names the agents that are actually about to be + // kicked; anything gated out is reported in "agents_skipped" with its + // reason rather than vanishing silently. + agentsDue, agentsSkipped := partitionKickableAgents(agentsDue, cfg.Agents, config.OnDemandAgentsFromPacks(), agentMgr.IsPaused, func(name string) string { + if st, err := agentMgr.GetStatusFast(name); err == nil && st != nil { + return st.PausedTrigger + } + return "" + }) + agentsDue, agentsSkipped = reviewerAutoscaler.withholdDrainingReviewers(agentsDue, agentsSkipped) + + // PROVIDER SPEND REBUFF (#4294). When the inference gateway is refusing on a + // money limit, every kick launched this cycle is a run that cannot buy a + // single token — precisely the failure this addresses: a hive that kept + // firing its cadence into a gateway rejecting 100% of requests all day, + // silently, until the provider's spend window happened to roll over. + // + // The alert is raised HERE, before kick assembly, so an operator is told + // even on a cycle where nothing happened to be due. The actual suppression + // happens after every kick source has contributed — see below. + providerBudgetCause, providerBudgetSince, providerBudgetLastRebuff, providerBudgetRebuffs := dashboard.InferenceBudgetExceeded() + // Suppress only while the latch is FRESH. Withholding kicks also withholds + // the inference calls that clear the latch, so a hive whose only kick source + // is the governor cadence would never learn the provider's window reset and + // would stay muted forever — the exact topology in the field report. Once + // the last rebuff is older than the probe interval, this cycle's kicks go + // through as a probe: still clipped re-freshens the stamp and suppression + // resumes, served clears the latch outright. + providerBudgetProbeInterval := cfg.Governor.ProviderBudget.EffectiveProbeInterval() + providerBudgetLatched := providerBudgetCause != "" + suppressKicks := governor.ProviderBudgetSuppresses(providerBudgetLatched, + providerBudgetProbe.Freshest(providerBudgetLastRebuff), time.Now(), providerBudgetProbeInterval) + budgetAlert := decideProviderBudgetAlert(providerBudgetLatched, suppressKicks, + providerBudgetCause, providerBudgetSince, providerBudgetRebuffs, + func() string { + return hub.QuotaExhaustedAgentReason(hub.QuotaExhaustedProcessCount(agentMgr.AllStatuses())) + }) + if budgetAlert.Message != "" { + dashSrv.AddSystemAlert(providerBudgetAlertID, "error", budgetAlert.Message) + } + if budgetAlert.Clear { + dashSrv.ClearSystemAlert(providerBudgetAlertID) + } + if budgetAlert.Cause != "" { + providerBudgetCause = budgetAlert.Cause + } + // Notify ONCE per latch, not once per cycle. The deduped banner above + // already carries the ongoing state; a high-priority notification repeated + // every eval cycle for as long as the provider stays clipped is a day of + // pages saying the same thing. Keyed on the latch time, which recordRebuff + // deliberately does not move forward, so a genuinely new clip after a + // recovery notifies again. Matches applyBudgetAlerts, which notifies on the + // crossing rather than on the condition. + notifyProviderBudget := providerBudgetLatched && providerBudgetNotify.ShouldSend(providerBudgetSince) + if !providerBudgetLatched { + providerBudgetProbe.Reset() + // The RECOVERY crossing: the latch a notification went out for has + // cleared (a probe's inference call succeeded), so tell the operator + // once that the hive resumed — the counterpart of the entering page, + // without which the only signal of recovery is a banner quietly + // vanishing. Every later healthy cycle is silent. + if providerBudgetNotify.Reset() { + logger.Info("provider spending limit lifted: agent kicks resumed") + notifier.Send("Provider spending limit lifted", + "the inference provider is serving again — agent kicks have resumed", + notify.PriorityDefault) + } + } + + // ADDITIVE CEL routing: evaluate operator-defined CEL trigger rules + // (cfg.Triggers, pkg/celtrigger) against the items enumerated this cycle and + // UNION any matched, gated agents into agentsDue. This runs alongside — never + // instead of — the label/governor selection above: unionAgents only ever adds + // names, and celMatchedAgents enforces the same pause/budget/on-demand gates, + // so a CEL match can neither remove a governor-selected agent nor bypass a + // paused agent or exhausted budget. Fully guarded/cheap: with no `triggers:` + // configured, celEngineFor returns nil and celMatchedAgents does zero work. + if celEngine := celEngineFor(cfg, logger); celEngine != nil { + celAgents := celMatchedAgents(celEngine, actionable, cfg, gov, agentMgr.IsPaused, logger) + if len(celAgents) > 0 { + before := len(agentsDue) + agentsDue = unionAgents(agentsDue, celAgents) + if len(agentsDue) > before { + logger.Info("celtrigger: unioned CEL-matched agents into due set", + "added", agentsDue[before:], "cel_matched", celAgents) + } + } + } + + // #4247/#4263 (parent #3845): apply the shared convergence admission at the + // internal-kick dispatch boundary — AFTER governor policy evaluated the raw + // queue, BEFORE the scheduler caches and renders issues. Gated by the + // runtime convergence rollout mode, captured once per pass: with mode "off" + // (the DEFAULT) this is entirely inert and kickActionable IS actionable; + // "shadow" logs and records what would be withheld but still dispatches the + // raw population; only "enforce" gates the scheduled/cached issue payloads + // below. The raw actionable population stays authoritative for governor + // policy, dashboard status, PR/review dispatch, escalation, and every path + // not explicitly enrolled. + kickActionable := applyConvergenceKickAdmission(cfg, dashSrv, actionable, notifier, logger) + + sched.SetLastActionable(kickActionable) + // The filter compacts agentsDue in place, so snapshot it first to report + // the targets it withholds alongside the other skips below. + dueBeforeDeferral := append([]string(nil), agentsDue...) + agentsDue = sched.FilterFirstScanDeferredAgents(agentsDue) + agentsSkipped = append(agentsSkipped, firstScanDeferredSkips(dueBeforeDeferral, agentsDue)...) + reviewPlan := planReviewDispatchWith(ctx, cfg, actionable, agentMgr, beadStores, reviewPlannerGitHubFor(ghClient), logger) + // Size the reviewer lane to what the planner just left waiting + // (hivecommons/hive#14685). A replica added here takes kicks from the + // next cycle on; a retire only lands once the reviewer is idle. + reviewerAutoscaler.Run(ctx, cfg, reviewPlan, agentMgr, gov, logger, time.Now()) + auditWithheldReviewFixes(reviewPlan, dashSrv, logger) + if ghClient != nil { + postReviewAcknowledgements(ctx, ghClient, reviewPlan, logger) + } + applyHumanDecisionLabels(ctx, cfg, ghClient, actionable, reviewPlan, logger) + if ghClient != nil { + applyReviewPriorityLabels(ctx, cfg, ghClient, reviewQueue, logger) + applyReviewVerdictLabels(ctx, cfg, ghClient, actionable, reviewPlan, logger) + sweepClosedPRVerdictLabels(ctx, cfg, ghClient, time.Now(), logger) + } + messages, messageSkips := sched.BuildKickMessagesWithSkips(kickActionable, agentsDue) + agentsSkipped = append(agentsSkipped, messageSkips...) + // Message construction can withhold a due kick (notably an idle reviewer + // lane). Report only assembled targets as due. Skipped targets dispatch + // nothing, so (as before) no LastKick is recorded for them and new lane + // work stays eligible on the next evaluation. + assembledTargets := make([]string, 0, len(messages)) + for _, msg := range messages { + assembledTargets = append(assembledTargets, config.CadenceTargetKey(msg.Agent, msg.Repo)) + } + logger.Info("governor eval complete", + "mode", govState.Mode, + "issues", govState.QueueIssues, + "prs", govState.QueuePRs, + "agents_due", assembledTargets, + "agents_skipped", agentsSkipped, + ) + + reviewKickByMessage := map[string]review.DispatchKick{} + for _, k := range append(reviewPlan.ReviewKicks, reviewPlan.FixKicks...) { + if !gov.AgentEligibleForCELKick(k.Agent) || agentMgr.IsPaused(k.Agent) { + logger.Info("review swarm kick suppressed by governor gate", "agent", k.Agent, "pr", k.PRRef) + continue + } + messages = append(messages, scheduler.KickMessage{Agent: k.Agent, Message: k.Message, IssueRefs: []string{k.PRRef}}) + reviewKickByMessage[k.Agent+"\x00"+k.Message] = k + } + // A fix kick (a maintainer's changes_requested / `/hive decision`, or + // the swarm's) goes ahead of the governor kick for the same agent, or + // the governor kick owns the agent and the fix kick times out every + // cycle (hivecommons/hive#13100). + for _, agent := range pendingKickAgentsFromMessages(messages) { + if governorKickFairness.GovernorFirst(agent) { + logger.Warn("governor kick displaced by fix kicks on consecutive cycles; governor kick goes first this cycle", + "agent", agent, "displaced_cycles", governorKickFairness.Displaced(agent), "limit", governorKickDisplacementLimit) + } + } + messages = orderReviewFixKicksFirst(messages, reviewKickByMessage, governorKickFairness.GovernorFirst) + // Freshness bound before a kick (#13375): a ref older than + // github.DecisionFreshnessBound in a cache-served or last-good snapshot + // is confirmed with one targeted GET; a stale repo's refs are withheld. + messages = freshenKickMessages(ctx, ghClient.SnapshotFreshnessGuard(actionable, time.Now), actionable, messages, logger) + // #4294: drop EVERY assembled kick while the provider is refusing on a + // spending limit. Placed after all three sources have contributed — + // governor-due agents, the CEL union, and the review swarm — because gating + // only `agentsDue` earlier would still let a CEL match or a review kick fire + // into the same clipped key. + // + if len(messages) > 0 { + filtered := messages[:0] + for _, msg := range messages { + if remaining, class, line, ok := agentMgr.ProviderErrorBackoffRemaining(msg.Agent); ok { + logger.Warn("provider inference error: withholding agent kick during backoff", + "agent", msg.Agent, + "class", class, + "retry_in", remaining.Round(time.Second), + "error", line) + gov.RecordKickFailure(msg.Agent, fmt.Errorf("provider inference backoff (%s): %s", class, line), time.Now()) + continue + } + filtered = append(filtered, msg) + } + messages = filtered + } + + // Suppression is total rather than per-agent because the limit is on the + // KEY: no agent can succeed while it is clipped. It self-heals — the first + // inference call that succeeds after the provider's window resets clears the + // signal — so there is no timer to tune and no operator action required. + // + // Deliberately does NOT force-pause agents. Operator pause state is a human + // decision (#2573) and must not be forged by an automatic signal that will + // clear itself; withholding kicks achieves the saving without leaving paused + // agents behind for a human to un-pause by hand. + // + // Probe cycle: when the last rebuff has gone stale, ONE kick is + // deliberately allowed through to find out whether the provider is + // serving again. Its inference calls are what clear the latch (on a + // 2xx) or re-freshen it (on another rebuff) — nothing else can. Only + // one: the question is "is the window still clipped", and every kick + // beyond the first spends a run to learn the same answer. Releasing it + // re-arms suppression immediately, so the cycles while the probe's run + // is still in flight withhold again rather than leaking more kicks. + kickGate := gateKickMessagesForProviderBudget(messages, suppressKicks, providerBudgetLatched) + releaseProviderBudgetProbe := kickGate.ReleaseProbe + if suppressKicks && len(messages) > 0 { + logger.Warn("provider spending limit: withholding agent kicks", + "withheld", kickGate.Withheld, "rebuffs", providerBudgetRebuffs, "since", providerBudgetSince, + "next_probe_in", (providerBudgetProbeInterval - time.Since(providerBudgetProbe.Freshest(providerBudgetLastRebuff))).Truncate(time.Second)) + } else if releaseProviderBudgetProbe { + if len(kickGate.Withheld) > 0 { + logger.Warn("provider spending limit: withholding all but the probe kick", + "withheld", kickGate.Withheld, "rebuffs", providerBudgetRebuffs, "since", providerBudgetSince) + } + logger.Info("provider spending limit: releasing a single probe kick", + "probe_agent", kickGate.Kept[0].Agent, "rebuffs", providerBudgetRebuffs, "since", providerBudgetSince, + "last_rebuff", providerBudgetLastRebuff, "probe_interval", providerBudgetProbeInterval) + } + messages = kickGate.Kept + // Self-upgrade drain (#11876): a hub-instructed upgrade is waiting for + // in-flight kicks to end, so starting new turns would only feed the + // restart. Withheld kicks stay due and go out from the new process. + if target, draining := selfUpgradeDrain.Active(selfUpgradeDrainNow()); draining && len(messages) > 0 { + logger.Info("self-upgrade drain: withholding new agent kicks", + "target", target, "withheld", pendingKickAgentsFromMessages(messages)) + messages = nil + } + govState.PendingKicks = pendingKickAgentsFromMessages(messages) + gov.SetPendingKicks(govState.PendingKicks) + if notifyProviderBudget { + notifier.Send("Provider spending limit reached", providerBudgetCause, notify.PriorityHigh) + } + + // buildStatusSnapshot assembles the full dashboard snapshot from this + // cycle's evaluation. It is called twice: once BEFORE the kick dispatch + // below and once at the end of the cycle. Each kick can wait up to the + // input-prompt timeout for a busy pane; before dispatch ran one agent at + // a time, a dozen due agents took 10-15 minutes on hosted hives — during + // which /api/status (and everything derived from it: stage tracker, agent + // states, governor.nextKickAt) kept showing the previous cycle's data, and + // a rolling upgrade landing mid-dispatch meant the evaluation never + // reached the dashboard at all. Dispatch is now concurrent across agents + // (dispatchAgentKicks), so the wait is one agent's, but the early publish + // stays: the evaluation should be visible before any kick is typed. + buildStatusSnapshot := func() *dashboard.StatusPayload { + payload := dashboard.BuildFrontendStatus( + govState, + actionable, + agentMgr.AllStatuses(), + cfg, + tokenCollector, + gov, + beadStores, + ghClient, + ctx, + metricsCollector, + ) + // Green on a PR pill means "the sweep would merge this now" — the + // verdict writeMergeEligible reached for this same actionable set, + // not a re-derivation from mergeable_state (#7478). + dashboard.AttachMergeVerdicts(payload, mergeVerdicts) + // A PR pill also carries the hive's OWN review on that PR, so the + // queue view shows where the hive has already spoken. Read from the + // durable ledger, so this costs no GitHub call per PR; a missing or + // unreadable ledger simply means no review pills this cycle. + attachReviewLinksForDashboard(payload, logger) + return payload + } + publishStatusBeforeKicks(dashSrv, len(messages), buildStatusSnapshot, logger) + + // Kick dispatch lives behind a seam (#7232): the skip rules and the + // single-probe rule are the decisions worth testing, and they were + // previously unreachable without a tmux session and a live governor. + var deliveredReviewKicks []review.DispatchKick + var deliveredKicks []scheduler.KickMessage + if len(messages) > 0 { + // governor.kick wraps the whole dispatch pass; each delivered kick + // still gets its own agent.kick child via startKickSpan below. + kickCtx, kickPassSpan := tracing.StartSpan(ctx, "governor.kick", + attribute.Int("hive.kicks.assembled", len(messages))) + ctx := kickCtx + dispatchAgentKicks(messages, releaseProviderBudgetProbe, kickDispatchDeps{ + backoffRemaining: agentMgr.ProviderErrorBackoffRemaining, + sendKick: agentMgr.SendKick, + startStoppedAgent: func(agentName string) error { + if !agentMgr.GovernorMayStart(agentName) { + return nil + } + agentMgr.MarkStartupLaunchQueued([]string{agentName}) + if err := agentMgr.Start(ctx, agentName); err != nil { + if stoppedAgentStartErrorIsBenign(err) { + logger.Info("stopped agent start raced with existing launch before governor kick", "agent", agentName, "error", err) + return nil + } + return fmt.Errorf("start stopped agent: %w", err) + } + logger.Info("started stopped agent before governor kick", "agent", agentName) + return nil + }, + startKickSpan: func(agentName string) func(error) { + agentCfg := cfg.Agents[agentName] + _, kickSpan := tracing.StartSpan(ctx, "agent.kick", tracing.AgentKickAttributes( + agentName, + agentCfg.Backend, + agentCfg.Model, + agentCfg.Role, + string(govState.Mode), + inferACMMLevel(cfg), + )...) + return func(err error) { + if err != nil { + kickSpan.RecordError(err) + } + kickSpan.End() + } + }, + onReviewDelivered: func(msg scheduler.KickMessage) { + if k, ok := reviewKickByMessage[msg.Agent+"\x00"+msg.Message]; ok { + deliveredReviewKicks = append(deliveredReviewKicks, k) + persistReviewDispatchStateInProgress(reviewPlan, deliveredReviewKicks, logger) + } + }, + onFailed: func(msg scheduler.KickMessage, err error) { + gov.RecordKickFailure(msg.Agent, err, time.Now()) + if k, ok := reviewKickByMessage[reviewKickKey(msg)]; ok { + logUndeliveredReviewKick(logger, dashSrv, k, err) + } + }, + onDelivered: func(msg scheduler.KickMessage, deliveredAt time.Time) { + deliveredKicks = append(deliveredKicks, msg) + gov.RecordKickForRepoAt(msg.Agent, msg.Repo, deliveredAt) + dashSrv.AuditLog("governor", "kick", "trigger=governor-eval", msg.Agent) + + // Record issue-scoped kicks into the lifecycle timeline. Cheap, + // guarded, and nil-safe (Record no-ops on a nil dashboard/store). + recordKick(ctx, dashSrv, msg.Agent, msg.IssueRefs...) + // A review-swarm kick (review or fix) names a pull request, + // not an issue: its IssueRefs is the PR ref. Listing it in + // the issue-claims ledger made the author's next comment on + // its own PR an issue "start signal", so the PR was claimed + // as an issue and staged stage/030-pr-in-progress — an + // issue-only stage — until the PR sweep healed it. + reviewKick, isReview := reviewKickByMessage[msg.Agent+"\x00"+msg.Message] + if !isReview { + // #8380/#10527: list these issues for the kicked agent so + // contributors and other agents back off while the kick is + // live; the claim itself is recorded on the agent's first + // start signal (recordAgentStart), not here. + recordAgentKickListings(dashSrv, cfg.Project.Org, msg.Agent, msg.IssueRefs, logger) + } + + // #12128: record which issue/PR/head SHA this kick was launched + // for, so /api/launch-cost can place the turn's spend. Review + // kicks carry the PR head; governor kicks carry the issue refs. + if tokenCollector != nil { + tokenCollector.RecordLaunch(launchRecordForKick(msg, reviewKick, isReview, time.Now())) + } + + // Log token state at time of kick for cost attribution + if tokenCollector != nil { + if summary := tokenCollector.Summary(); summary != nil { + agentTokens := summary.ByAgent[msg.Agent] + logger.Info("kick token snapshot", + "agent", msg.Agent, + "agent_tokens", agentTokens, + "total_tokens", summary.TotalTokens, + "total_sessions", summary.SessionCount, + ) + } + } + }, + markProbeReleased: providerBudgetProbe.MarkReleased, + now: time.Now, + }, logger) + kickPassSpan.SetAttributes(attribute.Int("hive.kicks.review_delivered", len(deliveredReviewKicks))) + kickPassSpan.End() + } + governorKickFairness.ObserveCycle(messages, deliveredKicks, reviewKickByMessage) + persistReviewDispatchState(reviewPlan, deliveredReviewKicks, logger) + + if actionable.Issues.SLAViolations > 0 { + toNotify, capped := selectSLABreachNotifications(actionable.Issues.Items) + for _, issue := range toNotify { + notifier.Send( + "SLA 2x breach", + fmt.Sprintf("%s age %dm: %s\n%s", actionableIssueRef(issue), issue.AgeMinutes, issue.Title, issue.URL), + notify.PriorityHigh, + ) + } + if capped { + logger.Info("SLA notification cap reached, skipping remaining", "remaining", actionable.Issues.SLAViolations-len(toNotify)) + } + } + + // Scan agent panes for login-required patterns and pause + notify if detected + loginscan.Scan(ctx, cfg, agentMgr, notifier, dashSrv, logger, loginSightings) + + // Epoch captured before reading agent/governor state so a mutation that + // lands mid-build (restart-count/budget reset) drops this snapshot instead + // of letting it revert the mutation on the dashboard (#4348). + buildEpoch := dashSrv.BeginStatusSnapshot() + statusPayload := buildStatusSnapshot() + statusPublished := false + // Ingest any JSONL findings agents wrote and persist them as beads. + if advisoryStore != nil { + findings, err := advisoryStore.ReadNewFindings() + if err != nil { + logger.Warn("failed to read advisory findings", "error", err) + } else if len(findings) > 0 { + // The canary gate's decisions live behind a seam (#7232); only the + // effects — audit entry, critical bead — are supplied here. + var scanCanary func(agent, reportText, source string) (ioscan.CanaryLeak, bool) + if cfg.Ioscan.IsEnabled() && cfg.Ioscan.CanariesEnabled() { + scanCanary = ioscan.DefaultCanaries.Scan + } + safeFindings := gateAdvisoryFindings(findings, advisoryIngestDeps{ + scanCanary: scanCanary, + failClosed: cfg.Ioscan.FailClosed(), + auditLog: dashSrv.AuditLog, + recordLeakBead: func(leak ioscan.CanaryLeak) { + if store, ok := beadStores[leak.Agent]; ok && store != nil { + if b, berr := store.Create("Canary token leaked via "+leak.Source, beads.TypeAdvisory, beads.PriorityCritical, leak.Agent, ""); berr == nil { + _ = store.SetMetadata(b.ID, "rule", ioscan.CanaryLeakRule) + _ = store.SetMetadata(b.ID, "source", leak.Source) + } + } + }, + }, logger) + if persisted := advisory.PersistAsBeads(safeFindings, beadStores); persisted > 0 { + logger.Info("advisory findings persisted as beads", "count", persisted) + } + } + } + + // Reload bead stores from disk before building the digest. Agents write + // beads via the bd CLI which persists directly to disk, so the in-memory + // stores can become stale between eval cycles. Reload failures are deduped + // (WARN once per distinct error, then DEBUG) — see beads_reload.go (#5505). + reloadBeadStores(beadStores, logger) + + // Phase 4 Part B: `plan`/`epic` label trigger. An actionable issue carrying a + // plan label auto-mints an epic and requests decomposition — the same flow as + // the dashboard "Plan this issue" click, but triggered by the label. It runs + // AFTER the store reload so FindByExternalRef sees current state (idempotent: + // no duplicate epic if one already exists). Cheap, synchronous, adds NO + // goroutine, and drives the architect only via SendKick (never the launch + // path). Gated by config/ACMM so low-maturity hives stay advisory-only. + if acmmLvl := inferACMMLevel(cfg); planning.PlanningAllowedAtLevel(acmmLvl) && + approvalDeskAllowsLegacyOperation(ctx, approvalDesk, cfg, toolapprove.KindPlanFromLabel, "plan-from-label", planning.ArchitectAgentName, logger) { + planFromLabeledIssues(actionable, beadStores, agentMgr, gov, dashSrv, logger, cfg, acmmLvl) + } + + // Advisory digest: build from beads (the source of truth) before status broadcast. + primaryRepo := primaryAdvisoryRepo(cfg) + issueNum, hasPinnedAdvisoryIssue := advisoryIssueNumber(advisoryIssues, primaryRepo) + hasExistingPinnedIssueForEmptyDigest := hasPinnedAdvisoryIssue && + primaryRepo == primaryRepoAtCycleStart && + hadPinnedAdvisoryIssueAtCycleStart + if shouldBuildAdvisoryDigest(beadStores, ghClient, hasExistingPinnedIssueForEmptyDigest) { + // Mark findings no agent has re-reported inside the staleness window + // BEFORE the digest is built, so stale evidence is captioned. Silence is + // not proof a finding healed: an agent run can be partial, truncated, or + // non-deterministic, so absence cannot move an item to Recently Resolved. + advCfg := cfg.Governor.Advisory + if advCfg.StalenessDays > 0 { + if marked := advisory.MarkStaleAdvisoryBeads(beadStores, time.Duration(advCfg.StalenessDays)*24*time.Hour); len(marked) > 0 { + logger.Info("marked advisory findings not re-reported within the staleness window", + "count", len(marked), "staleness_days", advCfg.StalenessDays, "titles", strings.Join(marked, "; ")) + } + } + // Repo entries may be org-qualified ("org/repo"); the digest linkifier + // needs the bare repo name alongside the org. + org, repoName := cfg.Project.Org, primaryRepo + if parts := strings.SplitN(primaryRepo, "/", 2); len(parts) == 2 { + org, repoName = parts[0], parts[1] + } + + // #3704: pin the digest to ONE repo commit. Resolve the target repo's + // latest commit ONCE here (invariants 1 & 3), cite it in the rendered + // comment (invariant 2, via the footer FormatDigestMarkdown emits when + // AnalyzedSnapshot is set), and verify each finding's file path against + // that exact commit so a since-removed path (e.g. "docs/install.md") is + // flagged as outdated rather than cited as live. Best-effort: if the SHA + // cannot be resolved, fall back to the previous unpinned behavior rather + // than skip the digest. + // + // This is resolved BEFORE the digest is built because the top-N cap + // consumes it: ranking cannot prefer a live finding over a since-removed + // one unless it knows which is which at ranking time (#2364). + digestOpts := advisory.DigestOptions{ + MaxFindings: advCfg.MaxFindings, + ShowAll: advCfg.ShowAll, + } + if ghClient != nil { + pinDigestSnapshot(ctx, &digestOpts, org, repoName, primaryRepo, cfg.Policies.Branch, digestSnapshotDeps{ + defaultBranch: func(ctx context.Context, owner, repo string) (string, error) { + r, _, err := ghClient.GetRepo(ctx, owner, repo) + if err != nil { + return "", err + } + return r.GetDefaultBranch(), nil + }, + latestCommit: ghClient.LatestCommitHash, + pathExists: ghClient.PathExistsAtRef, + issueClosedAt: ghClient.IssueClosedAt, + }, logger) + } + digest := advisory.BuildDigestFromBeads(beadStores, string(govState.Mode), digestOpts) + enrichAdvisoryLinkedWork(ctx, ghClient, digest, org, repoName, logger) + if advisoryStore != nil { + advisoryStore.SetLatestDigest(digest) + } + dashSrv.SetAdvisoryDigest(digest) + statusPayload.AdvisoryDigest = digest + statusPublished = dashSrv.UpdateStatusIfFresh(statusPayload, buildEpoch) + if !statusPublished { + return + } + hiveAdvice := statusPayload.HiveAdvice + + // Post whenever there is something CURRENT to say: open findings, + // recently resolved ones, or an empty evaluation for a hive that already + // has a pinned advisory issue. The resolved and empty cases matter for + // freshness: otherwise the pinned comment and AdvisoryLastPostedAt + // freeze after the last finding disappears, and the hub reports a stale + // advisory loop even though the agents are running cleanly. + // + // advisoryPostDue additionally paces the GitHub write to the + // operator's governor.advisory.update_interval_s (#4820); 0/unset + // keeps this exact per-cycle cadence. cfg is read live each cycle — + // the same pattern as the staleness/max-findings knobs above — so a + // dashboard edit applies from the next cycle without a restart. The + // digest itself and the dashboard state above still refresh every + // cycle; only the comment write is throttled. Note the #4821 + // write-through counts consecutive unchanged post ATTEMPTS, so its + // forced full rewrite stretches with this interval (60 attempts × + // interval) — acceptable, since it only heals out-of-band comment + // edits, and documented in the settings tooltip. + // governor.advisory.target routes the comment write: GitHub (default, + // the unchanged path below) or a designated Linear issue. For the + // Linear route the configured issue plays the pinned issue's role in + // the empty-digest freshness rule, so a clean Linear-sourced hive + // keeps refreshing its comment exactly as a GitHub one does. + advisoryTarget, advisoryLinearIssue, advisoryRouteErr := resolveAdvisoryDigestRoute(cfg) + hasDigestHome := hasExistingPinnedIssueForEmptyDigest || + (advisoryTarget == config.AdvisoryTargetLinear && advisoryRouteErr == nil) + if shouldPostAdvisoryDigest(digest, ghClient, hasDigestHome) && + advisoryPostDue(advCfg, primaryRepo, time.Now(), logger) { + bySeverity, agentNames := summarizeDigestForLog(digest) + logger.Info("advisory digest built", + "total_findings", digest.TotalCount, + "critical", bySeverity["critical"], + "high", bySeverity["high"], + "medium", bySeverity["medium"], + "low", bySeverity["low"], + "agents", agentNames, + "resolved_count", len(digest.RecentlyResolved), + ) + if digest.TotalCount == 0 && len(digest.RecentlyResolved) == 0 { + logger.Info("advisory digest empty — posting freshness marker", + "repo", primaryRepo, "issue", issueNum) + } + + md := advisory.FormatDigestMarkdown(digest, advisory.DigestOptions{ + MaxFindings: digestOpts.MaxFindings, + ShowAll: digestOpts.ShowAll, + Org: org, + ShowEmpty: digest.TotalCount == 0 && len(digest.RecentlyResolved) == 0, + PrimaryRepo: repoName, + Advice: hiveAdvice, + DashboardURL: advisoryDashboardOrigin(cfg), + }) + // The routing/classification decisions live in + // publishAdvisoryDigest (#7232); only the effects are wired here. + // The App is the sole advisory-digest writer (#1927): a user-token + // failure must never drive the App banner, so every banner hook + // below is fed exclusively by the App's own error. + publishAdvisoryDigest(ctx, md, digest, advisoryPublishRoute{ + target: advisoryTarget, + linearIssue: advisoryLinearIssue, + routeErr: advisoryRouteErr, + primaryRepo: primaryRepo, + issueNum: issueNum, + hasPinnedIssue: hasPinnedAdvisoryIssue, + ensureErr: advisoryEnsureErr, + }, advisoryPublishDeps{ + postGitHub: ghClient.PostAdvisoryDigest, + postLinear: func(ctx context.Context, linearIssue, md string) error { + return postAdvisoryDigestToLinear(ctx, cfg, linearIssue, md) + }, + recordError: dashSrv.RecordAdvisoryError, + recordPosted: func(findings, overflow int) { + dashSrv.RecordAdvisoryPost(findings) + recordAdvisoryPostSuccess(primaryRepo, time.Now()) + dashSrv.RecordAdvisoryOverflow(overflow) + }, + onWriteForbidden: func(ctx context.Context) { + // App is installed (we found the issue) but a real WRITE + // was forbidden. #2353: attribute this honestly — surface + // a DISTINCT write-forbidden state naming the likeliest + // cause instead of faking a permission gap the diagnosis + // just disproved. + msg, state := classifyGitHubAppWriteForbidden(ctx, ghClient.AppAuth(), cfg.Project.Org, primaryRepo) + dashSrv.SetGitHubAppRequired(true) + dashSrv.SetGitHubAppPermIssue(msg) + dashSrv.SetGitHubAppState(state.String()) + logger.Warn("GitHub App write failed — cannot write issue comments", + "repo", primaryRepo, "state", state.String(), + "operator_actionable", state.OperatorActionable(), "detail", msg) + }, + onRepoMoved: func(ctx context.Context, moved *github.RepoMovedError) { + // #11791: the write was refused because the repository + // now lives under another owner. The installation is + // healthy, so the generic write-forbidden remedy would + // misdirect the operator; name the move instead. + dashSrv.SetGitHubAppRequired(true) + dashSrv.SetGitHubAppPermIssue(moved.Error()) + dashSrv.SetGitHubAppState(github.AppStateRepoMoved.String()) + logger.Warn("GitHub App write failed — repository transferred to another owner", + "configured", moved.Configured, "current", moved.Current) + }, + recordRepoPaused: func(repo string) { + recordAdvisoryPostSuccess(repo, time.Now()) + }, + recordWriteForbidden: func(repo string) { + backoff := advCfg.PostRetryBackoff() + recordAdvisoryPostForbidden(repo, time.Now(), backoff) + if backoff > 0 { + logger.Warn("advisory digest write forbidden — pausing post attempts", + "repo", repo, "retry_after", backoff.String(), + "knob", "governor.advisory.post_retry_backoff_s") + } + }, + onAuthFailure: func(ctx context.Context) { + // Same verdict function as boot and Re-check, so a healthy + // or unclassifiable probe cannot raise the banner here. + raise, msg, state := classifyGitHubAppFailure(ctx, ghClient.AppAuth(), cfg.Project.Org, logger) + if !raise { + return + } + dashSrv.SetGitHubAppRequired(true) + if msg != "" { + dashSrv.SetGitHubAppPermIssue(msg) + } + dashSrv.SetGitHubAppState(state.String()) + logger.Warn("GitHub App authentication failed posting advisory digest", + "repo", primaryRepo, "state", state.String(), + "operator_actionable", state.OperatorActionable()) + }, + onWriteProven: func(ctx context.Context) { + // A successful write proves the app is installed AND has + // write access — clear BOTH the perm issue and the + // app-required flag, or the "Not Installed" banner sticks + // forever despite tokens working. + dashSrv.SetGitHubAppPermIssue("") + dashSrv.SetGitHubAppRequired(false) + dashSrv.ClearPendingGitHubAppInstall() + // The same proof retires stale ACCESS findings (#2575). + if healed := advisory.CloseHealedAppAuthFindings(beadStores); len(healed) > 0 { + logger.Info("closed healed GitHub App access findings after successful App digest post", + "count", len(healed), "titles", strings.Join(healed, "; ")) + } + // Repo-ACCESS findings need their own proof: a digest post + // only proves issues:WRITE, so verify with a real + // advisor-scoped Contents read, memoized per repo. + readVerified := map[string]bool{} + canRead := func(ownerRepo string) bool { + target := ownerRepo + if target == "" { + target = primaryRepo + } + owner, name := cfg.Project.Org, target + if i := strings.LastIndex(target, "/"); i > 0 { + owner, name = target[:i], target[i+1:] + } + if owner == "" || name == "" { + return false + } + key := owner + "/" + name + if v, ok := readVerified[key]; ok { + return v + } + appAuth := ghClient.AppAuth() + if appAuth == nil { + // Static-token client: no advisor-tier token can be + // minted, so leave the finding open. + return false + } + err := appAuth.VerifyRepoRead(ctx, owner, name) + if err != nil { + logger.Info("repo-access finding left open: advisor read probe failed", + "repo", key, "error", err) + } + readVerified[key] = err == nil + return readVerified[key] + } + if healed := advisory.CloseHealedRepoAccessFindings(beadStores, canRead); len(healed) > 0 { + logger.Info("closed healed repo-access findings after verified advisory read path", + "count", len(healed), "titles", strings.Join(healed, "; ")) + } + }, + }, logger) + } + } else if d := dashSrv.GetAdvisoryDigest(); d != nil { + statusPayload.AdvisoryDigest = d + } + + if !statusPublished { + dashSrv.UpdateStatusIfFresh(statusPayload, buildEpoch) + } + + publishFleetReportsWith(ctx, logger, dashSrv, statusPayload.FleetReport, newSelfReportPublishDeps(cfg, ghClient, dashSrv)) + + if agentStats := dashboard.CollectAgentStats(statusPayload); len(agentStats) > 0 { + gov.AttachAgentStats(agentStats) + } + + if repoSnaps := dashboard.CollectRepoSnapshots(statusPayload); len(repoSnaps) > 0 { + gov.AttachRepoSnapshots(repoSnaps) + } + + if nousState != nil { + var tokenSummary *tokens.AggregateSummary + if tokenCollector != nil { + tokenSummary = tokenCollector.Summary() + } + if err := nousState.RecordSnapshot(govState, actionable, agentsDue, agentMgr.AllStatuses(), tokenSummary); err != nil { + logger.Warn("failed to record nous snapshot", "error", err) + } + } +} diff --git a/src/cmd/hive/main_github_app.go b/src/cmd/hive/main_github_app.go new file mode 100644 index 0000000000..5bb5fa99dd --- /dev/null +++ b/src/cmd/hive/main_github_app.go @@ -0,0 +1,544 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io/fs" + "log/slog" + "net/http" + "os" + "regexp" + "strconv" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/advisory" + "github.com/hivecommons/hive/pkg/apphealth" + "github.com/hivecommons/hive/pkg/beads" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" + spoke "github.com/hivecommons/hive/pkg/hub/spoke" + "github.com/hivecommons/hive/pkg/mint" + "github.com/hivecommons/hive/pkg/spokealerts" +) + +// prospectiveGitHubIdentity returns the GitHub identity the spoke WOULD hold +// after adopting ghCfg, or nil when the push speaks to no identity field and +// there is nothing to validate. +// +// It mirrors the adoption rules in the GitHubAppConfigCallback exactly — a +// zero app_id and an empty app_slug both mean "not speaking to this field", and +// the placeholder sentinel is never adopted over a real App. Mirroring rather +// than validating ghCfg alone is what makes the check correct: the damaging +// state is a combination of PUSHED and EXISTING fields (a GHE app_id landing +// beside the spoke's own empty api_url), and validating the push in isolation +// cannot see it. +func prospectiveGitHubIdentity(cur config.GitHubConfig, ghCfg *spoke.HeartbeatGitHubAppConfig) *config.GitHubConfig { + if ghCfg == nil { + return nil + } + touched := false + next := cur + if ghCfg.AppID != 0 && ghCfg.AppID != config.PlaceholderAppID { + next.AppID = ghCfg.AppID + touched = true + } + if ghCfg.AppSlug != "" && ghCfg.AppSlug != cur.AppSlug { + next.AppSlug = ghCfg.AppSlug + touched = true + } + // The forge URLs are part of the SAME set as the App above, so they are + // adopted here and validated with it rather than arriving separately on the + // project-config channel. An App ID presented to the wrong forge returns + // "404 Integration not found", so applying one half without the other is the + // live failure this function exists to prevent. + // + // Empty means "unchanged", matching AppSlug. It cannot mean "make me + // public": empty URLs are also the correct steady state for a public hive + // (~41 of 50 spokes), so silence here is indistinguishable from "no opinion" + // and must never blank a working GHE URL. A hive moving TO public gets that + // from its app_id, which the resolver derives from its forge. + if ghCfg.APIURL != "" && ghCfg.APIURL != cur.APIURL { + next.APIURL = ghCfg.APIURL + touched = true + } + if ghCfg.BaseURL != "" && ghCfg.BaseURL != cur.BaseURL { + next.BaseURL = ghCfg.BaseURL + touched = true + } + if !touched { + return nil + } + return &next +} + +// nextInstallationID decides what a hive's installation_id becomes after a +// hub delivery, and reports whether the change is an operator RESET. +// +// Three cases, and the difference between the last two is load-bearing: +// +// ResetInstallation -> 0. The operator clicked "Reset App". Clearing makes +// HasUsableApp() false, which raises githubAppRequired: +// the owner is prompted to install the App again and the +// self-heal ticker starts, whose RediscoverAndAdopt +// adopts the correct installation for whatever they +// install. +// non-zero pushed -> adopt it. +// zero pushed -> KEEP the current value. Zero means "the hub is not +// speaking to this field", not "clear it". The +// cluster-wide key reconcile sends zero on every beat +// because it repairs KEYS on hives whose installation the +// hub does not track; reading that as a clear would blank +// a working installation fleet-wide and turn a key-only +// fault into a total auth outage. +// +// docs_installation_id is DELIBERATELY untouched by all three cases. It is the +// optional docs-org add-on installation, has no rediscovery flow (zero simply +// disables the docs token refresh, permanently), and a stale value is +// non-fatal: the periodic docs mint warns and retries. After an app-changing +// delivery it can therefore briefly equal the PREVIOUS app's installation id — +// which looks like the old installation_id was "parked" there, but is just the +// provisioned docs value (public hives commonly provision both fields with the +// same installation) surviving a reset that correctly cleared only +// installation_id. On a flip-back to the original App it becomes valid again +// on its own. +func nextInstallationID(current int64, ghCfg *spoke.HeartbeatGitHubAppConfig) (next int64, reset bool) { + if ghCfg == nil { + return current, false + } + if ghCfg.ResetInstallation { + return 0, current != 0 + } + if ghCfg.InstallationID != 0 { + return ghCfg.InstallationID, false + } + return current, false +} + +var githubAppTokenCachePath = github.TokenCachePath + +// resumeKickHeldAlerts tracks crash-restarted agents the resume-kick gate +// refused, so the eval loop can raise and later clear their dashboard alert. +var resumeKickHeldAlerts = spokealerts.NewResumeKickHeld() + +func githubAppTokenHeartbeatFields(cfg *config.Config, detail string) (status, lastMintAt, lastErr string) { + if cfg == nil || !cfg.GitHub.HasApp() { + return "", "", "" + } + info, err := os.Stat(githubAppTokenCachePath) + if err != nil { + if os.IsNotExist(err) { + return spoke.GitHubAppTokenStatusMissing, "", detail + } + return spoke.GitHubAppTokenStatusError, "", err.Error() + } + lastMintAt = info.ModTime().UTC().Format(time.RFC3339) + if time.Since(info.ModTime()) > spoke.GitHubAppTokenStaleAfter { + return spoke.GitHubAppTokenStatusStale, lastMintAt, detail + } + return spoke.GitHubAppTokenStatusOK, lastMintAt, "" +} + +var githubHTTPStatusRe = regexp.MustCompile(`\b([1-5][0-9]{2})\b`) + +func firstNonEmpty(vals ...string) string { + for _, v := range vals { + if strings.TrimSpace(v) != "" { + return v + } + } + return "" +} + +func githubAppStructuredFailure(state, detail string) (class string, httpStatus int) { + switch strings.TrimSpace(state) { + case github.AppStateNotInstalled.String(): + class = "not-installed" + case github.AppStateWrongInstallation.String(): + class = "wrong-installation" + case github.AppStateInsufficientPerms.String(): + class = "insufficient-permissions" + case github.AppStateKeyMissing.String(): + class = "key-missing" + case github.AppStateKeyInvalid.String(): + class = "key-invalid" + case github.AppStateNoAppAssigned.String(): + class = "no-app-assigned" + case github.AppStateRepoNotCovered.String(): + class = "repo-not-covered" + case github.AppStateRepoMoved.String(): + class = "repo-moved" + case github.AppStateWriteForbidden.String(): + class = "write-forbidden" + } + if class == "" && strings.TrimSpace(detail) != "" { + class = "token-error" + } + for _, m := range githubHTTPStatusRe.FindAllStringSubmatch(detail, -1) { + if len(m) == 2 { + if n, err := strconv.Atoi(m[1]); err == nil { + httpStatus = n + } + } + } + if httpStatus != 0 { + if (class == "token-error" || class == "not-installed") && httpStatus == http.StatusNotFound { + class = "installation-not-found" + } + } + return class, httpStatus +} + +// githubAuth is the outcome of resolving this hive's GitHub credentials at +// startup. Every field is optional: a hive with no usable credentials is a +// legitimate, bootable state. +type githubAuth struct { + // Client is nil when no credentials could be resolved. Callers must treat a + // nil Client as "GitHub is unavailable", never as a fatal condition. + Client *github.Client + // AppAuth is non-nil only when App auth was successfully initialized. + AppAuth *github.AppAuth + // Failure, when non-empty, is the operator-facing reason there is no + // working GitHub client. It is shown in the dashboard's GitHub App banner. + Failure string + // State classifies Failure so the banner and the hub's journey nudges can + // tell an operator-side fault (a key that was never delivered) from a + // user-actionable one (the App is not installed). Escalating against an + // owner for a key WE failed to provision is precisely the mistake + // github.AppAuthState exists to prevent. + State github.AppAuthState + // TokenScopes is the boot-time PAT scope probe (see + // github.CheckTokenScopes). Zero value / ScopeStatusSkipped on the App + // path. It is advisory: a ScopeStatusMissing result never blocks startup, + // it only gives github_auth a specific detail string instead of leaving the + // operator to decode a runtime 403. + TokenScopes github.ScopeResult +} + +// initGitHubAuth resolves this hive's GitHub credentials. +// +// It NEVER exits the process. A hive that cannot authenticate to GitHub must +// still boot and serve its dashboard, because the dashboard is the only place +// its owner can see what is wrong and fix it. Exiting here — which is what this +// code used to do on a key-read failure — happens before the HTTP listener +// binds, so the pod crashloops with the diagnosis visible only in kubectl logs: +// the hub shows the hive offline, the heartbeat never starts, and a rollout +// hangs forever because the new pod never goes Ready. +// +// The placeholder app_id is the reason that path was reachable at all. See +// config.PlaceholderAppID. +func initGitHubAuth(ctx context.Context, cfg *config.Config, logger *slog.Logger) githubAuth { + var out githubAuth + appKeyFile := appKeys.Resolve(cfg.GitHub.KeyFile, os.Getenv("GH_APP_KEY_FILE"), cfg.GitHub.AppID) + + // HasApp() rejects config.PlaceholderAppID. Build AppAuth as soon as a real + // app_id and key are present, even when installation_id is still empty: the + // App JWT is exactly what automatic installation-ID discovery needs. + if cfg.GitHub.HasApp() { + appAuth, err := github.NewAppAuth(cfg.GitHub.AppID, cfg.GitHub.InstallationID, appKeyFile, logger, cfg.GitHub.ResolvedAPIURL()) + if err != nil { + // A genuinely-configured App whose key is missing or malformed is a + // real, actionable fault — but not a reason to refuse to boot. + out.Failure = describeAppKeyFailure(cfg.GitHub.KeyFile, os.Getenv("GH_APP_KEY_FILE"), appKeyFile, err) + // Both states are operator-actionable: the hive's owner cannot + // deliver a key. Absent vs. unparseable is the distinction the hub + // needs to tell "never pushed" from "pushed something broken". + out.State = github.AppStateKeyInvalid + if errors.Is(err, fs.ErrNotExist) { + out.State = github.AppStateKeyMissing + } + logger.Error("GitHub App auth unavailable — starting in dashboard-only mode", + "app_id", cfg.GitHub.AppID, + "installation_id", cfg.GitHub.InstallationID, + "key_file", appKeyFile, + "state", out.State.String(), + "detail", out.Failure, + "error", err, + ) + } else { + out.AppAuth = appAuth + } + } + + if out.AppAuth != nil { + logger.Info("using GitHub App authentication", "app_id", cfg.GitHub.AppID) + if cfg.GitHub.InstallationID == 0 { + out.Failure = "The GitHub App is configured but has no installation. Install the app on your org; this hive will discover the installation automatically." + out.State = github.AppStateNotInstalled + logger.Warn("GitHub App configured without installation_id — starting in dashboard-only mode while auto-discovery polls") + return out + } + // Correct a stale/wrong installation_id BEFORE building the client, so + // the very first token this process mints is scoped to the right org + // rather than 403ing on every write until the self-heal tick runs. + healGitHubAppInstallation(ctx, out.AppAuth, cfg, logger) + // Hooks (hold labels, #6203 repo pause, #6204 agent scope, and the + // rest) are installed by bootGitHub through configureGitHubClient, the + // same function every rebuild path uses (#9614). + out.Client = github.NewClientFromAppWithBotLogin(out.AppAuth, cfg.Project.Org, cfg.Project.Repos, logger, cfg.GitHub.BotLogin()) + startDocsTokenRefresh(ctx, cfg, appKeyFile, logger) + return out + } + + ghToken := cfg.GitHub.Token + if ghToken == "" { + ghToken = os.Getenv("HIVE_GITHUB_TOKEN") + } + switch { + case ghToken != "": + out.Client = github.NewClient(ghToken, cfg.Project.Org, cfg.Project.Repos, logger, cfg.GitHub.ResolvedAPIURL()) // hooks: see the App branch above + // PAT path only: introspect the token's granted scopes ONCE, here, so a + // too-narrow token is named at boot instead of surfacing hours later as + // a generic 403 inside an agent — or, worse, as an empty backlog that + // looks like "no work to do". Fail-soft and bounded (see + // CheckTokenScopes); it never blocks or fails startup. The App branch + // above returns before this point: Apps have permissions, not scopes. + // An unset acmm_level is passed through as github.ACMMLevelUnset rather + // than inferACMMLevel's L1 default: L1 requires no scopes at all, so + // defaulting to it would silently suppress every warning on exactly the + // hives whose intent we cannot read. See ACMMLevelUnset. + scopeLevel := github.ACMMLevelUnset + if cfg.ACMMLevel != nil { + scopeLevel = *cfg.ACMMLevel + } + out.TokenScopes = out.Client.LogTokenScopeCheck(ctx, logger, scopeLevel) + case out.Failure != "": + // Real App, unusable key. Already logged; leave Client nil so nothing + // tries to act on GitHub with credentials that do not work. + case cfg.GitHub.IsPlaceholderApp(): + // OPERATOR-actionable, not user-actionable. This hive was provisioned as + // a placeholder and never assigned a real app_id, so the JWT it signs + // names no App and fails before any installation is consulted. Nothing + // the owner enters in the installation-ID box can fix it — reporting this + // as AppStateNotInstalled sent owners to re-install an App that was + // already installed and to re-enter an installation_id that was already + // correct. + out.Failure = "This hive was never assigned a GitHub App ID: it still carries the placeholder github.app_id, " + + "which does not name a real GitHub App. Setting an installation ID cannot resolve this — " + + "the hub operator must assign the real App ID." + out.State = github.AppStateNoAppAssigned + logger.Warn("placeholder github.app_id — hive starting in dashboard-only mode", + "placeholder_app_id", config.PlaceholderAppID, + "installation_id", cfg.GitHub.InstallationID, + ) + case cfg.GitHub.AppID != 0: + out.Failure = "The GitHub App is configured but has no installation. Install the app on your org to enable agents." + out.State = github.AppStateNotInstalled + logger.Warn("GitHub App configured without credentials — hive starting in dashboard-only mode. Install the app and provide installation_id + key to enable agents.") + default: + // Neither a token nor any app_id at all. config.validate() rejects this + // at load, so reaching it means the config was mutated afterwards. + // Still a degraded boot rather than an exit: the dashboard is where an + // operator fixes it. + out.Failure = "No GitHub credentials configured. Set github.token, or github.app_id plus an App installation." + logger.Error("no GitHub token configured (set github.token or github.app_id in config) — starting in dashboard-only mode") + } + return out +} + +// startDocsTokenRefresh mints and periodically refreshes a token for the +// separate docs-org installation, when one is configured. A failure here is +// always non-fatal: the docs org is an add-on, not this hive's primary auth. +func startDocsTokenRefresh(ctx context.Context, cfg *config.Config, appKeyFile string, logger *slog.Logger) { + if cfg.GitHub.DocsInstallationID == 0 { + return + } + docsAuth, err := github.NewAppAuthWithCache( + cfg.GitHub.AppID, cfg.GitHub.DocsInstallationID, + appKeyFile, github.DocsTokenCachePath, logger, cfg.GitHub.ResolvedAPIURL(), + ) + if err != nil { + logger.Warn("failed to init docs org token", "error", err) + return + } + go func() { + // Mint the initial docs token in the BACKGROUND, not on the startup + // path. The docs org is an add-on; blocking here on a docs-installation + // mint that hangs (unreachable/uninstalled GHE) would delay the whole + // process reaching MarkReady — the #2439 readiness-stall pattern. The + // mint is bounded (tokenMintTimeout) and non-fatal regardless. + if _, err := docsAuth.Token(ctx); err != nil { + logger.Warn("failed to generate initial docs org token", "error", err) + } else { + logger.Info("docs org token cached", "installation_id", cfg.GitHub.DocsInstallationID) + } + const docsTokenRefreshInterval = 45 * time.Minute + ticker := time.NewTicker(docsTokenRefreshInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if _, err := docsAuth.Token(ctx); err != nil { + logger.Warn("docs token refresh failed", "error", err) + } + } + } + }() +} + +// buildAgentMinter constructs the opt-in per-agent mint credential issuer from +// config. It loads (or creates) the signing key at cfg.Mint.KeyPath, builds a +// Minter with the configured issuer/hive-id/TTL, and wraps it as an AgentMinter. +// Callers gate on cfg.Mint.Enabled before calling. The signing key path comes +// from config (never hardcoded); a missing key file is created with 0600 perms +// by the mint package. +func buildAgentMinter(cfg *config.Config, logger *slog.Logger) (*mint.AgentMinter, error) { + if cfg.Mint.KeyPath == "" { + return nil, fmt.Errorf("mint.key_path is required when mint is enabled") + } + if cfg.Mint.Issuer == "" { + return nil, fmt.Errorf("mint.issuer is required when mint is enabled") + } + key, err := mint.LoadOrCreateKey(cfg.Mint.KeyPath) + if err != nil { + return nil, fmt.Errorf("loading mint signing key: %w", err) + } + maxTTL := time.Duration(cfg.Mint.MaxTTLSeconds) * time.Second + minter, err := mint.NewMinter(key, cfg.Mint.Issuer, + mint.WithHiveID(cfg.HiveID), + mint.WithMaxTTL(maxTTL), + ) + if err != nil { + return nil, fmt.Errorf("building minter: %w", err) + } + logger.Debug("mint signing key loaded", "key_path", cfg.Mint.KeyPath) + // ttl<=0 lets the minter fall back to its configured max on each Mint. + return mint.NewAgentMinter(minter, maxTTL), nil +} + +// healGitHubAppInstallation self-heals a hive whose github.installation_id +// points at the WRONG account — the failure mode diagnoseGitHubApp +// already detects and reports ("installation N belongs to 'X', not 'Y'"). It +// asks pkg/github to rediscover the installation covering cfg.Project.Org via +// the App JWT and, only on an unambiguous match, adopts it in place and +// persists it so the fix survives a pod restart. +// +// Every failure path is soft and silent-ish: a hive with no App key is not +// App-authenticated (skip), an API error or an ambiguous/absent discovery +// result leaves installation_id exactly as configured so the existing +// "check github.installation_id" banner still stands. It never returns an +// error to the caller and never blocks startup or a heartbeat. +// +// Rediscovery is rate-limited by pkg/github's discovery cache +// (github.InstallationDiscoveryTTL), so calling this from the self-heal tick +// is cheap even when the App is genuinely not installed on the org. +func healGitHubAppInstallation(ctx context.Context, appAuth *github.AppAuth, cfg *config.Config, logger *slog.Logger) { + if appAuth == nil || !appAuth.HasKey() || cfg == nil { + return + } + org := cfg.Project.Org + if org == "" { + return + } + newID, err := appAuth.RediscoverAndAdopt(ctx, org, logger) + if err != nil { + logger.Debug("github app installation rediscovery did not adopt a new id", + "org", org, "error", err) + return + } + if newID == 0 { + return // already correct, or nothing safe to adopt + } + cfg.GitHub.InstallationID = newID + if err := cfg.Save(); err != nil { + logger.Error("adopted rediscovered installation_id but failed to persist it — "+ + "it will revert on the next pod restart", + "installation_id", newID, "error", err) + return + } + logger.Info("persisted rediscovered github app installation_id", + "installation_id", newID, "org", org) +} + +// githubRateLimitErrText is the substring GitHub's client surfaces on a rate or +// abuse limit. Matching text is acceptable ONLY here: a rate limit is a reason +// to skip classification entirely, never a reason to accuse anyone of anything, +// so a false negative costs one extra (correct) classification round-trip. +const githubRateLimitErrText = "rate limit" + +// isGitHubRateLimitText reports whether an error looks like a GitHub rate limit. +func isGitHubRateLimitText(err error) bool { + return err != nil && strings.Contains(strings.ToLower(err.Error()), githubRateLimitErrText) +} + +// The GitHub App credential classifiers moved to pkg/apphealth (#7238). These +// wrappers exist so call sites keep their current shape and so the App key +// paths -- vars that tests repoint at a temp dir -- are read HERE, at call +// time, rather than captured once at init. +func classifyGitHubAppFailure(ctx context.Context, appAuth *github.AppAuth, expectedOwner string, logger *slog.Logger) (bool, string, github.AppAuthState) { + return apphealth.ClassifyFailure(ctx, appAuth, expectedOwner, appKeyPaths(appAuthID(appAuth)), logger) +} + +func classifyGitHubAppWriteForbidden(ctx context.Context, appAuth *github.AppAuth, expectedOwner, repo string) (string, github.AppAuthState) { + return apphealth.ClassifyWriteForbidden(ctx, appAuth, expectedOwner, repo, appKeyPaths(appAuthID(appAuth))) +} + +func appAuthID(appAuth *github.AppAuth) int64 { + if appAuth == nil { + return 0 + } + return appAuth.AppID() +} + +func classifyGitHubAppRepoCoverage(ctx context.Context, appAuth *github.AppAuth, org string, repos []string, logger *slog.Logger) (bool, string, github.AppAuthState) { + return apphealth.ClassifyRepoCoverage(ctx, appAuth, org, repos, logger) +} + +// The advisory digest posting policy moved to pkg/advisory (#7238 stage 2). +// These wrappers keep the existing call sites unchanged; advisoryPostGate is +// now an owned instance rather than a package-level struct tests reset. +var advisoryPostGate = advisory.NewPostGate() + +func primaryAdvisoryRepo(cfg *config.Config) string { + return advisory.PrimaryRepo(cfg) +} + +func advisoryIssueUnresolved(advisoryIssues map[string]int, repo string) bool { + return advisory.IssueUnresolved(advisoryIssues, repo) +} + +func advisoryIssueNumber(advisoryIssues map[string]int, repo string) (int, bool) { + return advisory.IssueNumber(advisoryIssues, repo) +} + +func shouldBuildAdvisoryDigest(beadStores map[string]*beads.Store, ghClient *github.Client, hasExistingPinnedIssue bool) bool { + return advisory.ShouldBuildDigest(beadStores, ghClient != nil, hasExistingPinnedIssue) +} + +func shouldPostAdvisoryDigest(digest *advisory.Digest, ghClient *github.Client, hasPinnedIssue bool) bool { + return advisory.ShouldPostDigest(digest, ghClient != nil, hasPinnedIssue) +} + +func advisoryPostDue(advCfg config.AdvisoryConfig, repo string, now time.Time, logger *slog.Logger) bool { + return advisoryPostGate.Due(advCfg, repo, now, logger) +} + +func recordAdvisoryPostSuccess(repo string, now time.Time) { + advisoryPostGate.RecordSuccess(repo, now) +} + +func recordAdvisoryPostForbidden(repo string, now time.Time, backoff time.Duration) { + advisoryPostGate.RecordWriteForbidden(repo, now, backoff) +} + +// advisoryIssueMissingError is the error recorded (and reported to the hub) when +// a hive has findings to publish but no advisory issue to publish them to. It is +// deliberately an ERROR rather than a silent skip: the hub's staleness gate +// treats a hive reporting neither a post time nor an error as "not an advisory +// participant" and never alarms, which is how a wedged digest went unnoticed for +// six days in #4167. +func advisoryIssueMissingError(repo string, cause error) string { + base := fmt.Sprintf("no advisory issue resolved for %s — digest not posted", repo) + // Issues-disabled is the one ensure failure with a remedy the OPERATOR of + // the target repo owns (#4329): flipping a repo setting, not fixing App + // auth. Fold its actionable message into the alert text so the fleet + // stale-advisory pill says so instead of reading like an auth failure. + var disabled *github.IssuesDisabledError + if errors.As(cause, &disabled) { + return base + ": " + disabled.Error() + } + return base +} diff --git a/src/cmd/hive/main_sweeps.go b/src/cmd/hive/main_sweeps.go new file mode 100644 index 0000000000..5adbb646aa --- /dev/null +++ b/src/cmd/hive/main_sweeps.go @@ -0,0 +1,2820 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "slices" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/pkg/beads" + "github.com/hivecommons/hive/pkg/classify" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/dashboard" + "github.com/hivecommons/hive/pkg/escalation" + "github.com/hivecommons/hive/pkg/forge" + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/github/automerge" + "github.com/hivecommons/hive/pkg/governor" + "github.com/hivecommons/hive/pkg/hooks" + "github.com/hivecommons/hive/pkg/notify" + "github.com/hivecommons/hive/pkg/planning" + "github.com/hivecommons/hive/pkg/review" + "github.com/hivecommons/hive/pkg/rotation" + "github.com/hivecommons/hive/pkg/stageengine/prstage" + "github.com/hivecommons/hive/pkg/stagelabels" + "github.com/hivecommons/hive/pkg/tracing" + "github.com/hivecommons/hive/pkg/worksource" + "go.opentelemetry.io/otel/attribute" +) + +var ( + escalationStoreOnce sync.Once + escalationStore *escalation.Store +) + +const escalationLedgerPath = "/data/metrics/fix-streaks.json" + +// getEscalationStore lazily loads the shared fix-loop ledger (staleness clock + +// distinct-SHA attempt counts + re-engagement caps). It is the SINGLE +// loop-safety/dedup authority shared by the re-engagement paths (#2 merge +// watcher, #3 claim release, #4 reaper) and the human-escalation sweep, so they +// all agree on which red PRs are stale, how many times each has been +// re-nudged, and which have crossed the human-escalation threshold. +func getEscalationStore() *escalation.Store { + escalationStoreOnce.Do(func() { + escalationStore = escalation.Load(escalationLedgerPath) + }) + return escalationStore +} + +// hivePRObservations projects the enumerated hive-authored PRs into escalation +// observations (repo fully-qualified, Red == a required check failed). Shared +// by recordRedStaleness and the reaper so both classify PRs identically. +// +// owners maps "org/repo#number" to the agent that opened the PR (see +// auditPRAgents); pass nil when ownership is not needed. Shared/base CI +// breakage is classified across every PR in the pass before human and +// non-Hive observations are removed. A green PR is counter-evidence, and an +// independent human-authored PR must confirm the shared failure. +func hivePRObservations(cfg *config.Config, actionable *github.ActionableResult, owners map[string]string) []escalation.Observation { + if actionable == nil { + return nil + } + all := allEscalationObservations(cfg, actionable, owners) + agentObs := all[:0] + for i, pr := range actionable.PRs.Items { + if isHiveAgentAuthor(cfg, pr.Author) { + agentObs = append(agentObs, all[i]) + } + } + return agentObs +} + +func allEscalationObservations(cfg *config.Config, actionable *github.ActionableResult, owners map[string]string) []escalation.Observation { + all := make([]escalation.Observation, 0, len(actionable.PRs.Items)+len(actionable.PRs.Held)) + for _, pr := range actionable.PRs.Items { + all = append(all, escalationObservation(cfg, pr, owners)) + } + for _, pr := range actionable.PRs.Held { + all = append(all, escalationObservation(cfg, pr, owners)) + } + escalation.MarkSharedFailures(all) + return all +} + +// escalationObservation projects one enumerated PR into the fix-loop ledger's +// view of it. Red means a required check concluded failure. Pending means this +// pass could not conclude CI at all — checks still running, no check runs, or +// (per EnrichCIStatus) the check-run fetch errored — which the ledger must +// treat as "no information", never as "went green". Labeled mirrors the forge's +// needs-human label so the ledger and the label can never disagree about +// whether a PR has already been handed to a human. Labels carry the PR's +// current forge labels so Sweep can reconcile reviewer-lane verdicts +// (label-only edits: needs-human removed, reviewer-passed added) back into +// the ledger (#5511, gap G1). +func escalationObservation(cfg *config.Config, pr github.PullRequest, owners map[string]string) escalation.Observation { + repo := pr.Repo + if !strings.Contains(repo, "/") && cfg.Project.Org != "" { + repo = cfg.Project.Org + "/" + repo + } + red := pr.HasFailingRequiredCheck() + return escalation.Observation{ + Repo: repo, + Number: pr.Number, + HeadSHA: pr.HeadSHA, + HeadTree: pr.HeadTree, + Red: red, + Pending: !red && pr.CIStatus != "success", + CIRunning: red && pr.CIChecksRunning, + Labeled: escalation.HasNeedsHumanLabel(pr.Labels), + Excerpt: pr.CIFailureExcerpt, + Labels: pr.Labels, + FailingChecks: pr.FailingChecks, + BaseRedChecks: pr.BaseRedChecks, + SharedCIIncident: pr.SharedCIIncident, + Agent: owners[escalation.Key(repo, pr.Number)], + IndependentControl: isIndependentCIControl(cfg, pr), + } +} + +// redPRFixOwner resolves who should be asked to repair a red PR: the agent the +// audit trail attributes it to, or — when that agent is unattributed — the +// fleet's primary PR creator, exactly the default formatRedPRFixData applies +// to an unattributed ci-failing.json row. Keeping the two in step is what lets +// a hook predicate (`t.agent == "scanner"`) and the FIX-BEFORE-NEW block name +// the same owner (hivecommons/hive#9474). +func redPRFixOwner(agent string) string { + if agent = strings.TrimSpace(agent); agent != "" { + return agent + } + return review.DefaultFixerAgent +} + +// dependencyBots are forge bots whose PRs are dependency bumps, not hive fix +// attempts. They carry the "[bot]" suffix that otherwise marks a PR as +// agent-authored, but nothing in the hive opened them and no hive agent is +// iterating on them, so a red one is not a fix loop to break: escalating it +// only pages a human with "1 distinct fix attempts" about a crate bump that +// renovate will rebase on its own. Mirrors the hub's default contribute +// deny-authors list (config: hub.contribute_deny_authors). +var dependencyBots = map[string]bool{ + "renovate[bot]": true, + "dependabot[bot]": true, + "mergeraptor[bot]": true, +} + +// isHiveAgentAuthor reports whether a PR author is one of OUR agents — the +// configured ai_author, the App bot identity agents author as, or another +// bot account — excluding the dependency bots above. Shared by every fix-loop +// path (staleness clock, reaper, escalation sweep) so they classify PRs +// identically. +func isHiveAgentAuthor(cfg *config.Config, author string) bool { + if author == "" { + return false + } + if author == cfg.Project.AIAuthor { + return true + } + if eff := cfg.EffectiveAIAuthor(); eff != "" && author == eff { + return true + } + return strings.HasSuffix(author, "[bot]") && !dependencyBots[author] +} + +// isIndependentCIControl reports whether a PR supplies human-authored evidence +// independent of Hive. Bot-authored and Hive-attributed contributor-relay PRs +// cannot prove that a failure also affects unrelated human work. +func isIndependentCIControl(cfg *config.Config, pr github.PullRequest) bool { + return !isHiveAgentAuthor(cfg, pr.Author) && !pr.AuthorNonHuman && !pr.HiveAttributed +} + +// recordRedStaleness updates the shared staleness clock (first-seen-red per red +// head SHA) for every hive-authored PR in this pass. It must run before the +// claim guard and the reaper so their StaleRed() reads reflect the current tick. +// A disabled escalation subsystem skips it (the whole fix-loop machinery is off). +// +// It also fires the escalation_red hook for each red PR, WITH the owning agent +// on the payload: the catalog and hooks.md have always documented `agent` on +// this transition, but nothing ever set it, so a predicate like +// `t.agent == "scanner"` could never match (hivecommons/hive#9474). +func recordRedStaleness(cfg *config.Config, actionable *github.ActionableResult) { + if cfg.Escalation.Disabled || actionable == nil { + return + } + owners := auditPRAgents(cfg.Project.Org, time.Now().Add(-auditPRAttributionWindow), "") + obs := hivePRObservations(cfg, actionable, owners) + getEscalationStore().ObserveRed(obs) + for _, ob := range obs { + if !ob.Red { + continue + } + hookDispatcher().Fire(context.Background(), hooks.Payload{ + Transition: hooks.TransitionEscalationRed, + Repo: ob.Repo, + Agent: redPRFixOwner(ob.Agent), + Reason: "required CI check red", + Attrs: map[string]string{ + hooks.AttrPR: strconv.Itoa(ob.Number), + "head_sha": ob.HeadSHA, + "excerpt": ob.Excerpt, + }, + }) + } +} + +// fixKicker is the out-of-band kick path the re-engagement lanes deliver +// through. crux.Runtime satisfies it via agentKicker, which is the same +// adapter the governor tick's replan lane uses. +type fixKicker interface { + Kick(agent, message string) error +} + +// agentAvailability reports, per agent, whether a kick sent right now could +// reach it: the agent must not be paused by the manager, and at least one +// governor mode must schedule it with a non-paused cadence. An agent paused in +// every mode (the v5 L6 pack pauses `strategist` everywhere) can never answer a +// FIX-BEFORE-NEW kick, so charging its red PRs' re-engagement budget only +// escalates them to a human having asked nobody (hivecommons/hive#9472). +func agentAvailability(cfg *config.Config, mgr crux.Runtime) func(string) bool { + return func(name string) bool { + if name == "" { + return false + } + if mgr != nil && mgr.IsPaused(name) { + return false + } + return cfg == nil || cadenceReachable(cfg, name) + } +} + +// cadenceReachable reports whether ANY governor mode gives the agent a cadence +// that is not paused/off. An agent with no cadence entry at all is treated as +// reachable: "nothing schedules it" is a fleet-configuration signal of its own +// (NoCadenceAgents), and refusing to route repairs to it would silently widen +// this gate into an agent-disablement policy. +func cadenceReachable(cfg *config.Config, name string) bool { + configured := false + for mode := range cfg.Governor.Modes { + cad := cfg.CadenceValueForMode(name, mode) + if cad == "" { + continue + } + configured = true + if !cad.IsPaused() { + return true + } + } + return !configured +} + +// ownerReEngageCooldown is the minimum spacing between two re-engagements of a +// PR owned by this agent: its slowest configured cadence, so a budget of six +// attempts cannot be spent inside a single cadence window where no kick the +// owner could answer would ever fit. The escalation store floors it at +// ReEngageCooldown and caps it at MaxReEngageCooldown. +func ownerReEngageCooldown(cfg *config.Config, name string) time.Duration { + if cfg == nil || name == "" { + return 0 + } + var slowest time.Duration + for mode := range cfg.Governor.Modes { + cad := cfg.CadenceValueForMode(name, mode) + if cad == "" || cad.IsPaused() { + continue + } + d, err := time.ParseDuration(strings.TrimSpace(cad.Interval())) + if err == nil && d > slowest { + slowest = d + } + } + return slowest +} + +// resolveRedPRFixer picks the agent a stuck red PR's fix kick should go to: its +// owner, else the configured review fixer agent, else the fleet's default +// fixer — the same chain the review-fix kick uses (src/docs/review-swarm.md). +// It returns "" when no candidate is reachable, which tells the caller to leave +// the PR alone rather than spend budget nobody can answer. +func resolveRedPRFixer(cfg *config.Config, owner string, available func(string) bool) string { + candidates := []string{redPRFixOwner(owner), strings.TrimSpace(cfg.Review.FixerAgent), review.DefaultFixerAgent} + for _, c := range candidates { + if c == "" { + continue + } + if available == nil || available(c) { + return c + } + } + return "" +} + +// redPRFixKick renders the targeted FIX-BEFORE-NEW kick a re-engagement +// delivers. It names ONE PR — the stuck one — with its evidence, so the kick is +// answerable on its own rather than relying on the owner's next cadence kick +// happening to carry the ci-failing.json block. +func redPRFixKick(o escalation.Observation, rerouted bool) string { + var b strings.Builder + fmt.Fprintf(&b, "🔴 FIX-BEFORE-NEW — %s#%d is red and has not moved\n\n", o.Repo, o.Number) + if rerouted { + fmt.Fprintf(&b, "Its owning agent (%s) is not reachable right now, so this repair is routed to you.\n\n", redPRFixOwner(o.Agent)) + } + b.WriteString("Repair it BEFORE claiming new issues or opening any new PR:\n") + if slices.Contains(o.Labels, stagelabels.NoDCOLabel) { + fmt.Fprintf(&b, " FIRST (required): the PR is %s — it inherited unsigned commits. Re-sign them: gh pr checkout %d --repo %s → git rebase --signoff (original authors preserved) → git push --force-with-lease\n", stagelabels.NoDCOLabel, o.Number, o.Repo) + b.WriteString(" THEN fix any failing CI the same way.\n") + } + fmt.Fprintf(&b, " gh pr checkout %d --repo %s → fix → git commit -s → git push\n", o.Number, o.Repo) + b.WriteString("Push to the SAME branch. Do NOT open a replacement PR.\n") + b.WriteString("After the push, STOP: do not watch, poll, or sleep on CI (no gh run watch/view loops).\n") + b.WriteString("The automerge sweep merges the PR once it is green; if the failure is infrastructure\n") + b.WriteString("(runner lost, no space left, checks still queued) leave one comment and end the turn.\n") + if o.HeadSHA != "" { + fmt.Fprintf(&b, "\nhead: %s\n", o.HeadSHA) + } + if len(o.FailingChecks) > 0 { + fmt.Fprintf(&b, "failing: %s\n", strings.Join(o.FailingChecks, ", ")) + } + if excerpt := strings.TrimSpace(o.Excerpt); excerpt != "" { + if runes := []rune(excerpt); len(runes) > redPRFixKickExcerptRunes { + excerpt = string(runes[:redPRFixKickExcerptRunes]) + "…" + } + fmt.Fprintf(&b, "\nevidence:\n%s\n", excerpt) + } + return b.String() +} + +// redPRFixKickExcerptRunes bounds the CI evidence carried in a targeted fix +// kick, matching the FIX-BEFORE-NEW block's own budget. +const redPRFixKickExcerptRunes = 400 + +// mergeReEngageHook builds the Fix #2 re-engagement callback for the merge +// watcher. It normalizes the repo, resolves the PR's owner, DELIVERS a targeted +// fix kick to it (or to the fallback fixer when the owner is unreachable), and +// only then records the attempt under the shared escalation store's per-red-SHA +// cap. Passing an empty head SHA tells the store to reuse the red head SHA it +// last observed for this PR (the eval cycle's ObserveRed keeps it current), so +// the merge watcher does not need to re-fetch the head. Returns false when the +// cap is exhausted or no kick could be delivered, so the watcher can log that +// the escalation path now owns the PR. A disabled escalation subsystem yields a +// nil hook (watcher falls back to quarantine-only). +// +// Delivery is the point: before hivecommons/hive#9472 this only incremented a +// counter, so a PR could be escalated as "6 automated fix re-dispatches" having +// never been mentioned to any agent. +func mergeReEngageHook(cfg *config.Config, kicker fixKicker, available func(string) bool, logger *slog.Logger) github.MergeReEngageFunc { + if cfg.Escalation.Disabled { + return nil + } + fullRepo := func(repo string) string { + if !strings.Contains(repo, "/") && cfg.Project.Org != "" { + return cfg.Project.Org + "/" + repo + } + return repo + } + return func(repo string, number int) bool { + full := fullRepo(repo) + owners := auditPRAgents(cfg.Project.Org, time.Now().Add(-auditPRAttributionWindow), "") + o := escalation.Observation{ + Repo: full, + Number: number, + Agent: owners[escalation.Key(full, number)], + } + // Empty head SHA: reuse the store's tracked current red SHA for this PR + // (do not reset the cap counter). The eval cycle records it via + // ObserveRed; if the store has never seen this PR red, the store still + // allows the first MaxReEngagements nudges. + return deliverReEngagement(cfg, getEscalationStore(), o, "", kicker, available, logger) + } +} + +// deliverReEngagement is the one place a fix re-dispatch is both SENT and +// COUNTED. It resolves a reachable fixer, checks the budget without spending +// it, delivers the kick, and only records the attempt once the kick was +// accepted: an undeliverable kick (agent not running, provider backoff, +// restart) must not consume a PR's budget and strand it in needs-human +// (hivecommons/hive#9472). +func deliverReEngagement( + cfg *config.Config, + store *escalation.Store, + o escalation.Observation, + headSHA string, + kicker fixKicker, + available func(string) bool, + logger *slog.Logger, +) bool { + fixer := resolveRedPRFixer(cfg, o.Agent, available) + if fixer == "" { + // Nobody can be asked right now. Charging budget here is what escalated + // red strategist PRs on a pack that pauses strategist in every mode. + if logger != nil { + logger.Info("re-engagement skipped: no reachable fixer agent", + "repo", o.Repo, "pr", o.Number, "owner", redPRFixOwner(o.Agent)) + } + return false + } + if !store.ReEngageEligible(o.Repo, o.Number, headSHA, ownerReEngageCooldown(cfg, fixer)) { + return false + } + if kicker == nil { + return false + } + if err := kicker.Kick(fixer, redPRFixKick(o, fixer != redPRFixOwner(o.Agent))); err != nil { + if logger != nil { + level := slog.LevelWarn + // Pool contention is expected backpressure: retry on a later sweep. + // Keep it below WARN so slogfeed does not file a runtime defect + // for a healthy shared pool refusing another session (#15350). + if isSharedPoolBusyErr(err) { + level = slog.LevelInfo + } + logger.Log(context.Background(), level, "re-engagement kick undeliverable; budget not charged", + "repo", o.Repo, "pr", o.Number, "agent", fixer, "error", err) + } + return false + } + store.RecordReEngagement(o.Repo, o.Number) + return true +} + +// reapStuckRedPRs is Fix #4: the governor's backstop sweep. For each +// hive-authored PR that is red on a required check AND stale (StaleRed) AND not +// already escalated to a human, it DELIVERS a targeted FIX-BEFORE-NEW kick to +// the PR's owner (or the fallback fixer when the owner is unreachable) and +// records the re-engagement only if that kick was accepted — deduped and capped +// via the escalation store. Entirely generic: it keys only off check state + +// staleness, never a linter. +func reapStuckRedPRs( + cfg *config.Config, + actionable *github.ActionableResult, + escalatedPRs map[string]bool, + kicker fixKicker, + available func(string) bool, + logger *slog.Logger, +) { + if cfg.Escalation.Disabled || actionable == nil { + return + } + store := getEscalationStore() + owners := auditPRAgents(cfg.Project.Org, time.Now().Add(-auditPRAttributionWindow), "") + for _, o := range hivePRObservations(cfg, actionable, owners) { + if !o.Red { + continue + } + if o.Shared { + // Every failing check is red across the fleet: the PR is not + // broken, a shared incident is. Nudging its owner to "fix" it + // wastes budget on work that cannot succeed (hivecommons/hive#9473). + continue + } + key := escalation.Key(o.Repo, o.Number) + if escalatedPRs[key] { + // Already handed to a human (needs-human label); kick builders skip + // it and we must not re-dispatch automated fixes. + continue + } + if !store.StaleRed(o.Repo, o.Number, o.HeadSHA) { + continue // still churning (fresh red SHA) — leave it to the fix agent + } + if !deliverReEngagement(cfg, store, o, o.HeadSHA, kicker, available, logger) { + // Cap reached, cooldown not elapsed, or nothing could be delivered. + // The distinct-SHA escalation path owns it from here. + continue + } + logger.Info("reaper: re-dispatched fix for stuck red PR", + "repo", o.Repo, "pr", o.Number, "head_sha", o.HeadSHA, + "re_engagements", store.ReEngagements(o.Repo, o.Number)) + } +} + +// runEscalationSweep folds this enumeration pass into the fix-loop breaker +// ledger and fires the one-time escalation actions (evidence comment + +// needs-human label + ntfy) for any agent-authored PR that just crossed the +// threshold of distinct failed fix attempts. Returns the full set of +// escalated PR keys so the work-list writers can flag them. Deterministic by +// design: no agent judgment is involved in counting, evidence, or the +// stop-order. Human-authored PRs are never escalated. +// +// The two forge writes go through forge.IssueWriter rather than *github.Client +// so the evidence lands on whichever forge the hive is actually configured for +// (see governorForge in forgewire.go). On a GitHub hive the writer IS the +// *github.Client this used to take, so nothing about that path changed. +// +// rec receives a KindBlocked lifecycle event for each newly-escalated PR +// (#5656); a nil rec is a no-op, matching the other timeline producers. +func runEscalationSweep( + ctx context.Context, + cfg *config.Config, + writer forge.IssueWriter, + actionable *github.ActionableResult, + notifier *notify.Notifier, + rec lifecycleRecorder, + logger *slog.Logger, +) map[string]bool { + escalated := map[string]bool{} + if cfg.Escalation.Disabled || writer == nil || actionable == nil { + return escalated + } + getEscalationStore() + + type prMeta struct{ checks []string } + meta := map[string]prMeta{} + owners := auditPRAgents(cfg.Project.Org, time.Now().Add(-auditPRAttributionWindow), "") + all := allEscalationObservations(cfg, actionable, owners) + itemCount := len(actionable.PRs.Items) + + obs := all[:0] + for i, o := range all { + var pr *github.PullRequest + if i < itemCount { + pr = &actionable.PRs.Items[i] + } else { + pr = &actionable.PRs.Held[i-itemCount] + } + if pr.ReporterTrustReason != "" { + escalationStore.SetReporterTrustReason(o.Repo, o.Number, pr.ReporterTrustReason) + } + if (i >= itemCount || !isHiveAgentAuthor(cfg, pr.Author)) && pr.ReporterTrustReason == "" { + continue + } + obs = append(obs, o) + meta[escalation.Key(o.Repo, o.Number)] = prMeta{checks: o.FailingChecks} + } + results := escalationStore.Sweep(obs, cfg.Escalation.EffectiveThreshold()) + + for _, o := range obs { + key := escalation.Key(o.Repo, o.Number) + r, ok := results[key] + if !ok { + continue + } + if r.Unparked { + // The sweep returned this PR to the automated lane by itself: its + // head went green, or the shared incident it escalated over + // cleared. Take the label off (retried on later passes until a + // pass sees it gone) and say why, once. + unparkEscalatedPR(ctx, writer, o, r, logger) + continue + } + if r.Escalated { + escalated[key] = true + } + if r.BaseRedHold { + // Every failing check is red on the base branch too: say so + // once per head, count nothing, kick no one. The hold clears + // by itself when the base recovers and the head is re-evaluated. + body := escalation.BaseRedHoldCommentBody(o.BaseRedChecks, o.SharedCIIncident) + if err := writer.CreateIssueComment(ctx, o.Repo, o.Number, body); err != nil { + logger.Warn("base-branch-red hold comment failed; will retry next pass", "repo", o.Repo, "pr", o.Number, "error", err) + } else { + escalationStore.MarkBaseRedHeld(o.Repo, o.Number, o.HeadSHA) + logger.Info("fix loop holding: base branch red", "repo", o.Repo, "pr", o.Number, "head_sha", o.HeadSHA, "checks", strings.Join(o.BaseRedChecks, ",")) + } + } + if r.NeedsLabel && !o.Labeled { + // Escalated on an earlier pass but the label never landed (the + // AddLabels call failed). Retry the LABEL ONLY — the evidence + // comment already reached the human and must not be repeated. + if err := writer.AddLabels(ctx, o.Repo, o.Number, []string{escalation.NeedsHumanLabel}); err != nil { + logger.Warn("escalation label retry failed", "repo", o.Repo, "pr", o.Number, "error", err) + } else { + escalationStore.MarkLabelApplied(o.Repo, o.Number) + } + } + if !r.NewlyEscala { + continue + } + escalated[key] = true + excerpt := o.Excerpt + if excerpt == "" { + excerpt = escalationStore.Excerpt(o.Repo, o.Number) + } + // A PR that re-escalates AFTER a reviewer-lane pass reaches its human + // with a structured hand-off note (#5617 item 3): what the reviewer + // left on the branch, when, and that no second automated pass is + // coming. Before this, the only thing distinguishing that hand-off + // from a first escalation was the label set. + ev := escalation.Evidence{ + Attempts: r.Attempts, + FailingChecks: meta[key].checks, + SharedChecks: r.SharedChecks, + Excerpt: excerpt, + Exhausted: r.Exhausted, + ReEngagements: r.ReEngagements, + } + body := escalation.CommentBody(ev) + afterReviewerPass := false + if sha, at, ok := escalationStore.ReviewerPass(o.Repo, o.Number); ok { + body = escalation.HandoffCommentBody(ev, escalation.ReviewerHandoff{SHA: sha, At: at}) + afterReviewerPass = true + } + if err := writer.CreateIssueComment(ctx, o.Repo, o.Number, body); err != nil { + // Retry next pass rather than marking escalated with no comment: + // the whole point is that the evidence reaches a human. + logger.Warn("escalation comment failed; will retry next pass", + "repo", o.Repo, "pr", o.Number, "error", err) + continue + } + // Mark escalated BEFORE the label call: once the comment is on the + // PR, nothing may post it again, whatever happens to the label. + escalationStore.MarkEscalated(o.Repo, o.Number) + if err := writer.AddLabels(ctx, o.Repo, o.Number, []string{escalation.NeedsHumanLabel}); err != nil { + logger.Warn("escalation label failed; will retry next pass", "repo", o.Repo, "pr", o.Number, "error", err) + } else { + escalationStore.MarkLabelApplied(o.Repo, o.Number) + } + // The escalation IS the real "blocked" lifecycle signal (#5656): a PR + // out of automated fix attempts, handed to a human. Record it on the + // item's journey so the panel's Blocked counter reflects reality, not + // just hook annotations. + recordBlocked(ctx, rec, cfg.Project.Org, o.Repo, o.Number, r.Attempts, meta[key].checks) + logger.Info("fix loop escalated to human", + "repo", o.Repo, "pr", o.Number, "attempts", r.Attempts, + "failing_checks", strings.Join(meta[key].checks, ",")) + if notifier != nil { + title := "Fix loop escalated" + detail := fmt.Sprintf("%s#%d red on %d fix attempts — needs a human (see PR comment for the raw error)", o.Repo, o.Number, r.Attempts) + if afterReviewerPass { + // Materially more urgent than a first escalation: a reviewer + // has already had its one pass, so nothing automated remains + // behind this page. + title = "Fix loop escalated after reviewer pass" + detail = fmt.Sprintf("%s#%d red again on %d fix attempts since the reviewer's pass — no further automated pass will run (see PR comment)", o.Repo, o.Number, r.Attempts) + } + notifier.Send(title, detail, notify.PriorityHigh) + } + } + return escalated +} + +// labelRemover is the optional forge capability the auto-un-park path needs. +// Every real adapter (GitHub, GitLab, Gitea) implements RemoveLabel on +// forge.Forge; the narrow IssueWriter the sweep takes does not declare it, so +// a writer that cannot remove labels simply leaves the PR labeled and logs. +type labelRemover interface { + RemoveLabel(ctx context.Context, repo string, number int, label string) error +} + +// unparkEscalatedPR performs the forge side of an automatic un-park: the +// needs-human label comes off, and — on the pass that decided it — a one-line +// comment says why. Before hivecommons/hive#9473 nothing did this: a PR +// escalated over a base-branch or runner outage stayed parked after the +// incident cleared, and a labeled PR that went green stayed parked forever, +// because the labeled branch returned before the green branch. +func unparkEscalatedPR(ctx context.Context, writer forge.IssueWriter, o escalation.Observation, r escalation.Result, logger *slog.Logger) { + remover, ok := writer.(labelRemover) + if !ok { + logger.Warn("cannot un-park escalated PR: forge writer cannot remove labels", + "repo", o.Repo, "pr", o.Number) + return + } + if err := remover.RemoveLabel(ctx, o.Repo, o.Number, escalation.NeedsHumanLabel); err != nil { + // Retried on the next pass: the ledger keeps asking until a pass + // observes the label gone. + logger.Warn("un-park label removal failed; will retry next pass", + "repo", o.Repo, "pr", o.Number, "error", err) + return + } + if r.UnparkReason == "" { + return // label-removal retry: the comment was posted on the deciding pass + } + if err := writer.CreateIssueComment(ctx, o.Repo, o.Number, escalation.UnparkCommentBody(r.UnparkReason)); err != nil { + logger.Warn("un-park comment failed", "repo", o.Repo, "pr", o.Number, "error", err) + } + logger.Info("escalation un-parked", "repo", o.Repo, "pr", o.Number, "reason", r.UnparkReason) +} + +// autoMergeSweepInterval is the minimum spacing between label-queued +// auto-merge sweeps. The sweep piggybacks on the governor eval tick, which can +// fire much more often than once a minute; this floor keeps the sweep from +// hammering the GitHub API on short eval intervals. +const autoMergeSweepInterval = time.Minute + +// autoMergeSweepTickBudget is how long the queued automerge sweep may wait +// for its first pacing slot before the sweep is shed to the next cycle. +const autoMergeSweepTickBudget = 5 * time.Second + +// taskListSweepInterval is the minimum spacing between task-list sweeps. The +// sweep enumerates every open issue in every repo — much heavier than the +// auto-merge sweep's label-scoped query — and "done" moves at PR-merge cadence, +// so a 15-minute floor keeps the API cost modest while still closing completed +// epics on the same day the last box gets ticked. +const taskListSweepInterval = 15 * time.Minute + +// supersessionSweepInterval is the minimum spacing between sweeps that retire +// open PRs whose claimed issue was already closed by a different merged PR. +// It walks open PRs and may inspect issue timelines and file lists, so it uses +// the same conservative cadence as the task-list sweep. +const supersessionSweepInterval = 15 * time.Minute + +// duplicateFixSweepInterval is the minimum spacing between sweeps that +// resolve two open PRs fixing the same issue (hivecommons/hive#14799). It +// reuses the open-PR listing the pre-dispatch guard caches and reads check +// runs per pair, so it runs on the supersession cadence. +const duplicateFixSweepInterval = 15 * time.Minute + +// sharedCIRerunInterval is the minimum spacing between shared-CI rerun +// sweeps. Each pass lists recently closed issues per repo and only walks open +// PRs when a [shared-ci] incident closed inside the lookback window, so it +// shares the supersession sweep's cadence. +const sharedCIRerunInterval = 15 * time.Minute + +// duplicateSweepInterval is the minimum spacing between duplicate sweeps. It +// is an hour rather than the task-list sweep's fifteen minutes for two +// reasons. Cost: the sweep fingerprints the changed-file set of every open PR, +// which on a large queue is the heaviest read the hive performs (the head-SHA +// cache makes steady-state passes nearly free, but a cold pass is not). +// Signal: duplicates accumulate at human-PR-opening cadence, so nothing is +// lost by noticing one an hour later, while a tighter loop only multiplies the +// chance of editing a suggestion under a reader's cursor. +const duplicateSweepInterval = time.Hour + +// runAutoMergeSweepIfDue drains the label-queued auto-merge queue (the human +// "Approved ... for Hive auto-merge" path) at most once per +// autoMergeSweepInterval. All merge-eligibility decisions — queue-approval +// trust, the trusted-merger tier gate (SetMergerAuthorizer), check +// verification — live inside SweepQueuedAutoMerges; this function is only the +// scheduler and the dashboard audit sink. +// rotationTrigger is the PausedTrigger stamped on strand-pauses so rotation's +// auto-resume never resumes a pause it did not create. +const rotationTrigger = "provider-rotation" + +// runRotationCheck applies RFC #3958 provider rotation after an eval cycle: +// for each agent not mid-task whose provider was positively measured as +// exhausted, move it to a backend with headroom at the same tier; when +// nothing has headroom, pause it loudly (strand). Stranded agents are +// auto-resumed when their provider recovers headroom. Never runs mid-task: +// only idle agents are candidates. +func runRotationCheck(ctx context.Context, cfg *config.Config, rotMgr *rotation.Manager, gov *governor.Governor, agentMgr crux.Runtime, logger *slog.Logger) { + if rotMgr == nil || !cfg.Governor.Rotation.Enabled { + return + } + govState := gov.GetState() + for name, proc := range agentMgr.AllStatuses() { + backend := proc.Config.Backend + if proc.BackendOverride != "" { + backend = proc.BackendOverride + } + + // Auto-resume: a stranded agent whose provider recovered. + if proc.Paused && proc.PausedTrigger == rotationTrigger { + if rotMgr.StrandRecovered(backend) { + if err := agentMgr.Resume(ctx, name, rotationTrigger, "provider headroom recovered"); err != nil { + logger.Warn("rotation: auto-resume failed", "agent", name, "error", err) + } else { + logger.Info("rotation: auto-resumed stranded agent", "agent", name, "backend", backend) + } + } + continue + } + if proc.Paused { + continue // never touch an operator pause + } + // Never rotate mid-task: only idle agents are candidates. + if proc.State == crux.StateRunning { + continue + } + + cadenceS := 0 + if interval := governorShortestActiveInterval(govState, name); interval > 0 { + cadenceS = int(interval / time.Second) + } + + if !rotMgr.Exhausted(backend) { + continue + } + next := rotMgr.NextBackendForCadence(name, backend, cadenceS) + if next == "" { + // Strand loudly: pause so the agent burns nothing until a + // provider recovers; the loop above auto-resumes it. + if err := agentMgr.Pause(name, rotationTrigger, "no provider has headroom (RFC #3958)"); err != nil { + logger.Warn("rotation: strand-pause failed", "agent", name, "error", err) + } else { + logger.Info("rotation: stranding agent, no headroom anywhere", "agent", name, "backend", backend) + } + continue + } + if err := agentMgr.SetBackendOverride(name, next); err != nil { + logger.Warn("rotation: backend override failed", "agent", name, "to", next, "error", err) + continue + } + logger.Info("rotation: moved agent to new backend", "agent", name, "from", backend, "to", next) + } +} + +func runAutoMergeSweepIfDue(ctx context.Context, ghClient *github.Client, cfg *config.Config, dashSrv *dashboard.Server, lastRun *time.Time, logger *slog.Logger) { + if ghClient == nil { + return + } + now := time.Now() + if lastRun != nil && !lastRun.IsZero() && now.Sub(*lastRun) < autoMergeSweepInterval { + return + } + // Housekeeping against the pacer: while a caution window has the pacing + // queue deeper than this sweep could wait, skip the sweep (it is due + // again next cycle) rather than queue behind the enumeration. + if github.ShedLowPriorityTick("automerge:queued-sweep", autoMergeSweepTickBudget) { + return + } + if lastRun != nil { + *lastRun = now + } + ctx = github.WithPacingClass(ctx, github.PacingLow) + // governor.automerge covers the queued, trusted-author and contributor- + // lane sweeps plus the stage-label pass that follows them. The sweep runs + // on its own cadence in the run loop, so this is a sibling of + // governor.eval_cycle rather than a child. + ctx, sweepSpan := tracing.StartSpan(ctx, "governor.automerge") + defer sweepSpan.End() + opts := automerge.Options{ + Logger: logger, + MergerAuthorizer: trustedMergerFunc(cfg), + HumanChangeRequests: review.DefaultHumanChangeRequests(), + TrustedAuthorizer: trustedAuthorFunc(cfg), + TrustedAuthorPolicy: trustedAuthorPolicyFunc(cfg), + ContributorLanePolicy: contributorLanePolicyFunc(cfg), + ContributorLaneVerdict: contributorLaneVerdictFunc(cfg, logger), + ContributorRegistryTier: contributorRegistryTierFunc(), + StageLabelPolicy: stageLabelPolicyFunc(cfg), + StageEngine: stageEngineOptions(cfg, logger), + } + syncStageEngineMode(cfg, ghClient, logger) + // hivecommons/hive#12436: the contribute hub is the stage-072 nudge sink + // (relay fix-pr tasks). Guarded so a nil hub never becomes a typed-nil + // interface. + if hub := dashSrv.ContributeHub(); hub != nil { + opts.ContributorNudge = hub + } + if cfg != nil { + opts.SentinelLabel = func() string { return cfg.Sentinel.LabelOrDefault() } + if set, ok := cfg.AutoMerge.RequiredCheckSet(); ok { + opts.RequiredChecks = set + } + opts.RequiredChecksForRepo = cfg.AutoMerge.RequiredCheckSetForRepo + opts.MinHeadAge = cfg.AutoMerge.EffectiveMinHeadAge() + opts.RepoAutoMergeEnabled = func(repo string) bool { return cfg.RepoAutoMergeEnabled(repo) } + opts.RepoOnGitHub = func(repo string) bool { return cfg.RepoOnGitHub(repo) } + opts.HumanMergePaths = func(repo string) []string { return humanMergePathsFor(cfg, repo) } + opts.HumanMergeRule = func() prstage.HumanMergeRule { return humanMergeRuleFor(cfg) } + opts.RequireCurrentBase = func(repo string) bool { return cfg.AutoMerge.RequireCurrentBaseFor(repo) } + opts.RequireCurrentBaseStrict = func(repo string) bool { return cfg.AutoMerge.RequireCurrentBaseStrictFor(repo) } + opts.MergeForwardTrustedHuman = func(repo string) bool { return cfg.AutoMerge.MergeForwardTrustedHumanFor(repo) } + opts.MergeTrain = func() automerge.MergeTrainPolicy { + return automerge.MergeTrainPolicy{Enabled: cfg.AutoMerge.MergeTrain.IsEnabled(), MaxRebaseAttempts: cfg.AutoMerge.MergeTrain.EffectiveMaxRebaseAttempts()} + } + opts.TrustedBotAuthors = func() map[string]bool { return cfg.AutoMerge.TrustedBotAuthorSet() } + } + audit := func(event automerge.AutoMergeSweepEvent) { + if dashSrv == nil { + return + } + detail := fmt.Sprintf("repo=%s, pr=%d, author=%s, queued_by=%s, label=%s, tier=%s, head_sha=%s, merge_sha=%s", + event.Repo, event.Number, event.Author, event.QueuedBy, event.Label, event.Tier, event.HeadSHA, event.MergeSHA) + dashSrv.AuditLogRecord("system", "automerge-sweep-merged", detail, "", event.Repo, event.Number) + } + result, err := automerge.SweepQueuedAutoMerges(ctx, ghClient, opts, automerge.AutoMergeSweepOptions{ + MaxMerges: automerge.DefaultAutoMergeSweepMaxMerges, + Audit: audit, + }) + if err != nil { + logger.Warn("automerge sweep failed", "error", err) + return + } + if len(result.Merged) > 0 || result.Seen > 0 { + logger.Info("automerge sweep complete", "seen", result.Seen, "merged", len(result.Merged), "skipped", result.Skipped) + } + sweepSpan.SetAttributes( + attribute.Int("hive.automerge.seen", result.Seen), + attribute.Int("hive.automerge.merged", len(result.Merged)), + attribute.Int("hive.automerge.skipped", result.Skipped)) + remaining := automerge.DefaultAutoMergeSweepMaxMerges - len(result.Merged) + if remaining > 0 && opts.TrustedAuthorPolicy().Enabled { + trustedResult, trustedErr := automerge.SweepTrustedAuthorAutoMerges(ctx, ghClient, opts, automerge.AutoMergeSweepOptions{ + MaxMerges: remaining, + Audit: audit, + SelfAuthoredLaneActive: cfg != nil && cfg.AutoMerge.SelfAuthoredAutoMergeAllowed(cfg.ACMMLevel), + }) + if trustedErr != nil { + logger.Warn("trusted-author automerge sweep failed", "error", trustedErr) + } else { + result.Seen += trustedResult.Seen + result.Skipped += trustedResult.Skipped + result.Merged = append(result.Merged, trustedResult.Merged...) + if len(trustedResult.Merged) > 0 || trustedResult.Seen > 0 { + logger.Info("trusted-author automerge sweep complete", "seen", trustedResult.Seen, "merged", len(trustedResult.Merged), "skipped", trustedResult.Skipped) + } + } + } + remaining = automerge.DefaultAutoMergeSweepMaxMerges - len(result.Merged) + if remaining > 0 && opts.ContributorLanePolicy().Enabled { + laneResult, laneErr := automerge.SweepContributorLaneAutoMerges(ctx, ghClient, opts, automerge.AutoMergeSweepOptions{ + MaxMerges: remaining, + Audit: audit, + }) + if laneErr != nil { + logger.Warn("contributor-lane automerge sweep failed", "error", laneErr) + } else { + result.Seen += laneResult.Seen + result.Skipped += laneResult.Skipped + result.Merged = append(result.Merged, laneResult.Merged...) + if len(laneResult.Merged) > 0 || laneResult.Seen > 0 { + logger.Info("contributor-lane automerge sweep complete", "seen", laneResult.Seen, "merged", len(laneResult.Merged), "skipped", laneResult.Skipped) + } + } + } + // Stage labels (#12429): the lanes above each recorded their per-PR + // reason into the process-shared tracker; one pass writes the labels. + if stageResult, stageErr := automerge.ApplyStageLabels(ctx, ghClient, opts); stageErr != nil { + logger.Warn("stage labels reconcile failed", "error", stageErr, "items", stageResult.Items, "writes", stageResult.Writes) + } else if stageResult.Writes+stageResult.Comments > 0 { + logger.Info("stage labels reconciled", "items", stageResult.Items, "writes", stageResult.Writes, "comments", stageResult.Comments, "healed", stageResult.Healed) + } + hookDispatcher().Fire(context.Background(), hooks.Payload{ + Transition: hooks.TransitionSweepCompleted, + Reason: "queued automerge sweep complete", + Attrs: map[string]string{ + "seen": strconv.Itoa(result.Seen), + "merged": strconv.Itoa(len(result.Merged)), + "skipped": strconv.Itoa(result.Skipped), + }, + }) +} + +// legacyFleetReportSweepInterval is how often the hive that watches the +// fleet-report repo looks for standalone "Fleet report:" issues to close. +const legacyFleetReportSweepInterval = github.LegacyFleetReportSweepInterval + +// hiveWatchesRepo reports whether repo is one of project.repos. +func hiveWatchesRepo(cfg *config.Config, repo string) bool { + if cfg == nil { + return false + } + for _, r := range cfg.Project.Repos { + if strings.EqualFold(strings.TrimSpace(r), strings.TrimSpace(repo)) { + return true + } + } + return false +} + +// runLegacyFleetReportSweepIfDue closes standalone "Fleet report:" issues a +// spoke still on a pre-#13470 build filed directly in the report repo +// (hivecommons/hive#13567). Only the hive that watches that repo runs it — +// it is the one with write access there — at most once per +// legacyFleetReportSweepInterval. +func runLegacyFleetReportSweepIfDue(ctx context.Context, ghClient *github.Client, cfg *config.Config, dashSrv *dashboard.Server, lastRun *time.Time, logger *slog.Logger) { + if ghClient == nil || cfg == nil { + return + } + repo := cfg.Governor.FleetReport.EffectiveRepo() + if !hiveWatchesRepo(cfg, repo) { + return + } + now := time.Now() + if lastRun != nil && !lastRun.IsZero() && now.Sub(*lastRun) < legacyFleetReportSweepInterval { + return + } + if lastRun != nil { + *lastRun = now + } + result, err := ghClient.SweepLegacyFleetReportIssues(ctx, github.LegacyFleetReportSweepOptions{ + Repo: repo, + Audit: func(event github.LegacyFleetReportSweepEvent) { + if dashSrv == nil { + return + } + detail := fmt.Sprintf("repo=%s, issue=%d, author=%s, title=%s", event.Repo, event.Number, event.Author, event.Title) + dashSrv.AuditLogRecord("system", "legacy-fleet-report-closed", detail, "", event.Repo, event.Number) + }, + }) + if err != nil { + logger.Warn("legacy fleet report sweep failed", "error", err) + return + } + for _, e := range result.Errors { + logger.Warn("legacy fleet report sweep: issue skipped", "detail", e) + } + if len(result.Closed) > 0 { + logger.Info("legacy fleet report sweep complete", "scanned", result.Scanned, "closed", len(result.Closed), "skipped", result.Skipped) + } +} + +// runTaskListSweepIfDue closes hive-filed issues whose task-list bodies are +// fully ticked, at most once per taskListSweepInterval. The finding-granularity +// policy (guidance now in every issue-filing template) tells agents to encode +// multi-part deliverables as `- [ ]` boxes; this is the sink that turns those +// boxes into closures. All safety gates — hive-filed only, at-least-one-box, +// all-boxes-ticked, hold-label respected, per-tick cap — live inside +// SweepCompletedTaskListIssues; this function is only the scheduler and the +// dashboard audit sink, mirroring runAutoMergeSweepIfDue above. +func runTaskListSweepIfDue(ctx context.Context, ghClient *github.Client, dashSrv *dashboard.Server, lastRun *time.Time, logger *slog.Logger) { + if ghClient == nil { + return + } + now := time.Now() + if lastRun != nil && !lastRun.IsZero() && now.Sub(*lastRun) < taskListSweepInterval { + return + } + if lastRun != nil { + *lastRun = now + } + result, err := ghClient.SweepCompletedTaskListIssues(ctx, github.TaskListSweepOptions{ + MaxCloses: github.DefaultTaskListSweepMaxCloses, + Audit: func(event github.TaskListSweepEvent) { + if dashSrv == nil { + return + } + detail := fmt.Sprintf("repo=%s, issue=%d, author=%s, boxes=%d", + event.Repo, event.Number, event.Author, event.TotalBoxes) + dashSrv.AuditLogRecord("system", "task-list-sweep-closed", detail, "", event.Repo, event.Number) + }, + }) + if err != nil { + logger.Warn("task-list sweep failed", "error", err) + return + } + if len(result.Closed) > 0 || result.Seen > 0 { + logger.Info("task-list sweep complete", "seen", result.Seen, "closed", len(result.Closed), "skipped", result.Skipped) + } + hookDispatcher().Fire(context.Background(), hooks.Payload{ + Transition: hooks.TransitionSweepCompleted, + Reason: "task-list sweep complete", + Attrs: map[string]string{ + "seen": strconv.Itoa(result.Seen), + "closed": strconv.Itoa(len(result.Closed)), + "skipped": strconv.Itoa(result.Skipped), + }, + }) +} + +func runSupersessionSweepIfDue(ctx context.Context, ghClient *github.Client, cfg *config.Config, dashSrv *dashboard.Server, lastRun *time.Time, logger *slog.Logger) { + if ghClient == nil { + return + } + now := time.Now() + if lastRun != nil && !lastRun.IsZero() && now.Sub(*lastRun) < supersessionSweepInterval { + return + } + if lastRun != nil { + *lastRun = now + } + var supersessionCfg config.SupersessionSweepConfig + if cfg != nil { + supersessionCfg = cfg.SupersessionSweep + } + result, err := ghClient.SweepSupersededOpenPRs(ctx, github.SupersessionSweepOptions{ + MaxActions: github.DefaultSupersessionSweepMaxActions, + CloseContributorPRs: supersessionCfg.CloseContributorPRs, + GracePeriod: supersessionCfg.GracePeriod, + AutoCloseSuperseded: func() bool { return cfg == nil || cfg.Governor.AutoCloseSupersededEnabled() }, + ACMMLevelForRepo: func(repo string) int { + if cfg == nil { + return 0 + } + return cfg.EffectiveACMMLevelForRepo(repo) + }, + Audit: func(event github.SupersessionSweepEvent) { + if dashSrv == nil { + return + } + detail := fmt.Sprintf("repo=%s, pr=%d, issue=%s#%d, closer_pr=%d, action=%s, reason=%s", + event.Repo, event.Number, event.IssueRepo, event.Issue, event.CloserPR, event.Action, event.Reason) + dashSrv.AuditLogRecord("system", github.SupersessionSweepAuditPrefix+event.Action, detail, "", event.Repo, event.Number) + }, + }) + if err != nil { + logger.Warn("supersession sweep failed", "error", err) + return + } + if len(result.Closed) > 0 || len(result.Commented) > 0 || len(result.Resolved) > 0 || result.Seen > 0 { + logger.Info("supersession sweep complete", "seen", result.Seen, "closed", len(result.Closed), "commented", len(result.Commented), "resolved", len(result.Resolved), "skipped", result.Skipped) + } + hookDispatcher().Fire(context.Background(), hooks.Payload{ + Transition: hooks.TransitionSweepCompleted, + Reason: "supersession sweep complete", + Attrs: map[string]string{ + "seen": strconv.Itoa(result.Seen), + "closed": strconv.Itoa(len(result.Closed)), + "commented": strconv.Itoa(len(result.Commented)), + "resolved": strconv.Itoa(len(result.Resolved)), + "skipped": strconv.Itoa(result.Skipped), + }, + }) +} + +// runDuplicateFixSweepIfDue resolves same-issue duplicate open PRs +// (hivecommons/hive#14799) at most once per duplicateFixSweepInterval: the +// earlier green-or-pending PR stays, the later hive PR is closed as +// duplicate-of:, a person's later PR gets the comment only, and the +// earlier-red-later-green case is parked for the owner. It follows the +// pre-dispatch guard's switch: scanner.skip_issues_with_open_pr off turns +// this off too, since both exist to keep one issue on one PR. +func runDuplicateFixSweepIfDue(ctx context.Context, ghClient *github.Client, cfg *config.Config, dashSrv *dashboard.Server, lastRun *time.Time, logger *slog.Logger) { + if ghClient == nil { + return + } + if cfg != nil && !cfg.Scanner.IsSkipIssuesWithOpenPR() { + return + } + now := time.Now() + if lastRun != nil && !lastRun.IsZero() && now.Sub(*lastRun) < duplicateFixSweepInterval { + return + } + if lastRun != nil { + *lastRun = now + } + result, err := ghClient.SweepDuplicateFixPRs(ctx, github.DuplicateFixSweepOptions{ + MaxActions: github.DefaultDuplicateFixSweepMaxActions, + Audit: func(event github.DuplicateFixEvent) { + if dashSrv == nil { + return + } + detail := fmt.Sprintf("repo=%s, pr=%d, issue=%s#%d, keeper_pr=%d, action=%s, reason=%s", + event.Repo, event.Number, event.IssueRepo, event.Issue, event.KeeperPR, event.Action, event.Reason) + dashSrv.AuditLogRecord("system", github.DuplicateFixSweepAuditPrefix+event.Action, detail, "", event.Repo, event.Number) + }, + }) + if err != nil { + logger.Warn("duplicate-fix sweep failed", "error", err) + return + } + if result.Pairs > 0 { + logger.Info("duplicate-fix sweep complete", "pairs", result.Pairs, "closed", len(result.Closed), "commented", len(result.Commented), "escalated", len(result.Escalated), "skipped", result.Skipped) + } +} + +// runSharedCIRerunSweepIfDue re-runs failed jobs on open PRs deferred to a +// [shared-ci] incident that has since closed (hivecommons/hive#12253), at most +// once per sharedCIRerunInterval. Gates and dedupe live in +// SweepSharedCIReruns; this is the scheduler and dashboard audit sink. +func runSharedCIRerunSweepIfDue(ctx context.Context, ghClient *github.Client, cfg *config.Config, dashSrv *dashboard.Server, lastRun *time.Time, logger *slog.Logger) { + if ghClient == nil { + return + } + if cfg != nil && !cfg.SharedCIRerun.IsEnabled() { + return + } + now := time.Now() + if lastRun != nil && !lastRun.IsZero() && now.Sub(*lastRun) < sharedCIRerunInterval { + return + } + if lastRun != nil { + *lastRun = now + } + result, err := ghClient.SweepSharedCIReruns(ctx, github.SharedCIRerunOptions{ + MaxPRs: github.DefaultSharedCIRerunMaxPRs, + Audit: func(event github.SharedCIRerunEvent) { + if dashSrv == nil { + return + } + incidents := make([]string, len(event.Incidents)) + for i, n := range event.Incidents { + incidents[i] = "#" + strconv.Itoa(n) + } + detail := fmt.Sprintf("repo=%s, pr=%d, incident=%s, head=%s, runs=%d", + event.Repo, event.Number, strings.Join(incidents, ","), event.HeadSHA, len(event.RunIDs)) + dashSrv.AuditLogRecord("system", "shared-ci-rerun", detail, "", event.Repo, event.Number) + }, + }) + if err != nil { + logger.Warn("shared-ci rerun sweep failed", "error", err) + return + } + if len(result.Rerun) > 0 { + logger.Info("shared-ci rerun sweep complete", "incidents", result.Incidents, "seen", result.Seen, "rerun", len(result.Rerun), "skipped", result.Skipped) + } +} + +// blockedIssuesSweepInterval is how often the periodic sweep reconciles open +// issues carrying the `blocked` label. +const blockedIssuesSweepInterval = 5 * time.Minute + +func runBlockedIssuesSweepIfDue(ctx context.Context, ghClient *github.Client, cfg *config.Config, dashSrv *dashboard.Server, lastRun *time.Time, logger *slog.Logger) { + if ghClient == nil { + return + } + now := time.Now() + if lastRun != nil && !lastRun.IsZero() && now.Sub(*lastRun) < blockedIssuesSweepInterval { + return + } + if lastRun != nil { + *lastRun = now + } + var repos []string + if cfg != nil { + repos = cfg.Project.Repos + } + res, err := ghClient.SweepBlockedIssues(ctx, github.BlockedSweepOptions{Repos: repos}) + if err != nil { + logger.Warn("blocked issues reconciliation sweep failed", "error", err) + return + } + if res.Unblocked+res.Retargeted+res.Stripped+res.Errors > 0 { + logger.Info("blocked issues reconciliation sweep complete", "items", res.Items, "unblocked", res.Unblocked, "retargeted", res.Retargeted, "help_wanted_stripped", res.Stripped, "errors", res.Errors) + } +} + +// runDuplicateSweepIfDue clusters open PRs by changed-file set and suggests +// which one to keep, at most once per duplicateSweepInterval +// (hivecommons/hive#7469 capability B). +// +// Fail-closed and opt-in at both levels. `duplicate_sweep.enabled` is off by +// default, so a hive that has not asked for this performs no extra API calls +// at all; `duplicate_sweep.post_comments` is a SECOND, separately-off grant +// for the write, so the natural first configuration is a report-only pass an +// operator can read in the log before the hive says anything on a +// contributor's PR. +// +// The sweep only ever comments. It does not close, label, approve or merge, +// and the reviewer gains no permission from its existence: the write is the +// hive's own App token going through the same canary-gated, scrubbed comment +// path every other hive-authored comment uses. +func runDuplicateSweepIfDue(ctx context.Context, cfg *config.Config, ghClient *github.Client, dashSrv *dashboard.Server, lastRun *time.Time, logger *slog.Logger) { + if ghClient == nil || cfg == nil || !cfg.DuplicateSweep.Enabled { + return + } + now := time.Now() + if lastRun != nil && !lastRun.IsZero() && now.Sub(*lastRun) < duplicateSweepInterval { + return + } + if lastRun != nil { + *lastRun = now + } + result, err := ghClient.SweepDuplicatePRs(ctx, github.DuplicateSweepOptions{ + PostComments: cfg.DuplicateSweep.PostComments, + MaxComments: cfg.DuplicateSweep.MaxComments, + MaxPRsPerRepo: cfg.DuplicateSweep.MaxPRsPerRepo, + BotAuthors: cfg.DuplicateSweep.BotAuthors, + Audit: func(event github.DuplicateSweepEvent) { + if dashSrv == nil { + return + } + detail := fmt.Sprintf("repo=%s, survivor=%d, superseded=%d, confidence=%s, commented=%d", + event.Repo, event.Survivor, len(event.Superseded), event.Confidence, len(event.Commented)) + dashSrv.AuditLogRecord("system", "duplicate-sweep-suggested", detail, "", event.Repo, event.Survivor) + }, + }) + if err != nil { + logger.Warn("duplicate sweep reported a problem", "error", err) + } + if result == nil { + return + } + if len(result.Clusters) > 0 || result.Scanned > 0 { + logger.Info("duplicate sweep complete", + "scanned", result.Scanned, + "skipped", result.Skipped, + "clusters", len(result.Clusters), + "commented", result.Commented, + "post_comments", cfg.DuplicateSweep.PostComments) + } + hookDispatcher().Fire(context.Background(), hooks.Payload{ + Transition: hooks.TransitionSweepCompleted, + Reason: "duplicate sweep complete", + Attrs: map[string]string{ + "scanned": strconv.Itoa(result.Scanned), + "clusters": strconv.Itoa(len(result.Clusters)), + "commented": strconv.Itoa(result.Commented), + }, + }) +} + +// sentinelSweepInterval is the reconcile cadence of the full sentinel pass +// (#13379). `pull_request` opened/synchronize/reopened webhooks evaluate the +// named PR as they arrive (runSentinelPREvent); the full pass enumerates +// open PRs from the forgestate cache, skips every head it already evaluated +// without a GitHub call, and exists to catch a PR whose delivery was lost. +// Fifteen minutes bounds how long such a PR sits unlabelled. +const sentinelSweepInterval = github.SentinelReconcileInterval + +// sentinelSweepOptions is the one option set both sentinel entry points use, +// so the webhook path and the reconcile pass cannot disagree on label, +// trust or rules. +func sentinelSweepOptions(cfg *config.Config, ghClient *github.Client, dashSrv *dashboard.Server) github.SentinelSweepOptions { + sc := cfg.Sentinel + return github.SentinelSweepOptions{ + Label: sc.LabelOrDefault(), + LabelColor: config.DefaultSentinelLabelColor, + LabelDescription: config.DefaultSentinelLabelDescription, + Evaluator: sc.EvaluatorConfig(), + TrustedAuthor: sentinelTrustedAuthorFunc(cfg, ghClient), + TrustedAuthorsBlock: sc.TrustedAuthorsBlock, + RepoAllowed: sc.RepoAllowed, + MaxActions: sc.MaxActionsOrDefault(), + Audit: func(event github.SentinelSweepEvent) { + if dashSrv == nil { + return + } + // Findings the write-back gate refused still reach the Security + // tab and /api/sentinel: the scan happened, only the GitHub + // write did not (#writeback containment). + dashSrv.RecordSentinelFinding(event) + detail := fmt.Sprintf("repo=%s, pr=%d, author=%s, head=%s, rules=%s", + event.Repo, event.Number, event.Author, event.HeadSHA, strings.Join(event.Rules(), ",")) + action := "sentinel-alert" + if event.Suppressed { + action = "sentinel-finding-suppressed" + detail += ", suppressed=" + event.SuppressedReason + } + dashSrv.AuditLogRecord("system", action, detail, "", event.Repo, event.Number) + }, + } +} + +// runSentinelPREvent is the dashboard.Dependencies.SentinelPREvent hook: a +// verified `pull_request` opened/synchronize/reopened delivery evaluates +// that one PR now, reading only it and its files. Default on with the +// sweep; `sentinel.enabled: false` disables both. +func runSentinelPREvent(ctx context.Context, ghClient *github.Client, cfg *config.Config, dashSrv *dashboard.Server, repo string, number int, logger *slog.Logger) { + if ghClient == nil || cfg == nil || !cfg.Sentinel.IsEnabled() { + return + } + result, err := ghClient.SweepSentinelPR(ctx, repo, number, sentinelSweepOptions(cfg, ghClient, dashSrv)) + if err != nil { + logger.Warn("sentinel webhook evaluation failed", "repo", repo, "pr", number, "error", err) + return + } + logSentinelResult(result, logger) +} + +// runSentinelSweepIfDue flags open PRs that look like security overrides, +// privilege escalation or codebase damage (pkg/sentinel) at most once per +// sentinelSweepInterval. Default on; `sentinel.enabled: false` disables it. +func runSentinelSweepIfDue(ctx context.Context, ghClient *github.Client, cfg *config.Config, dashSrv *dashboard.Server, lastRun *time.Time, logger *slog.Logger) { + if ghClient == nil || cfg == nil || !cfg.Sentinel.IsEnabled() { + return + } + now := time.Now() + if lastRun != nil && !lastRun.IsZero() && now.Sub(*lastRun) < sentinelSweepInterval { + return + } + if lastRun != nil { + *lastRun = now + } + result, err := ghClient.SweepSentinel(ctx, sentinelSweepOptions(cfg, ghClient, dashSrv)) + if err != nil { + logger.Warn("sentinel sweep failed", "error", err) + return + } + logSentinelResult(result, logger) + if len(result.Flagged) > 0 || len(result.Notified) > 0 || len(result.Remediated) > 0 || result.Seen > 0 { + logger.Info("sentinel sweep complete", "seen", result.Seen, "flagged", len(result.Flagged), "notified", len(result.Notified), "suppressed", len(result.Suppressed), "remediated", len(result.Remediated), "skipped", result.Skipped, "errors", len(result.Errors), "cached_repos", result.CachedRepos, "listed_repos", result.ListedRepos) + } + hookDispatcher().Fire(context.Background(), hooks.Payload{ + Transition: hooks.TransitionSweepCompleted, + Reason: "sentinel sweep complete", + Attrs: map[string]string{ + "seen": strconv.Itoa(result.Seen), + "flagged": strconv.Itoa(len(result.Flagged)), + "notified": strconv.Itoa(len(result.Notified)), + "remediated": strconv.Itoa(len(result.Remediated)), + "skipped": strconv.Itoa(result.Skipped), + }, + }) +} + +func logSentinelResult(result *github.SentinelSweepResult, logger *slog.Logger) { + for _, e := range result.Errors { + logger.Warn("sentinel sweep: PR skipped", "detail", e) + } + for _, ev := range result.Flagged { + logger.Warn("sentinel alert", "repo", ev.Repo, "pr", ev.Number, "author", ev.Author, "rules", strings.Join(ev.Rules(), ",")) + } + for _, ev := range result.Notified { + logger.Info("sentinel notice", "repo", ev.Repo, "pr", ev.Number, "author", ev.Author, "trusted_reason", ev.TrustedReason, "rules", strings.Join(ev.Rules(), ",")) + } + for _, ev := range result.Remediated { + logger.Info("sentinel unflagged trusted author PR", "repo", ev.Repo, "pr", ev.Number, "author", ev.Author, "trusted_reason", ev.TrustedReason, "head", ev.HeadSHA) + } +} + +// issueUnparkSweepInterval is the minimum spacing between un-park sweeps. A +// maintainer who replies "/hive approve" is waiting for something to happen, so +// this runs more often than the task-list sweep; it only reads the comments of +// issues that are actually parked, which keeps the API cost proportional to the +// backlog a person is waiting on. +const issueUnparkSweepInterval = 5 * time.Minute + +// issueUnparkSweepWebhookInterval is the minimum spacing when an `issues` +// webhook has marked a label change (hivecommons/hive#11313): a maintainer who +// just added `triage/accepted` should not wait out the polling interval, but a +// burst of label events must not turn into a sweep per event. +const issueUnparkSweepWebhookInterval = 30 * time.Second + +// runIssueUnparkSweepIfDue re-queues parked issues a maintainer has answered +// with a `/hive` command, and keeps the "What to reply" block current on every +// parked issue (hivecommons/hive#9879). Every gate — human author, live write +// permission, first-line command, recent unedited comment, per-tick cap — lives +// inside SweepIssueUnparkCommands; this function is only the scheduler and the +// dashboard audit sink, mirroring runTaskListSweepIfDue above. Repos with +// project.repo_policies[].label_driven skip the notice and keep their parking +// labels (hivecommons/hive#10537). +func runIssueUnparkSweepIfDue(ctx context.Context, ghClient *github.Client, cfg *config.Config, dashSrv *dashboard.Server, lastRun *time.Time, logger *slog.Logger) { + if ghClient == nil { + return + } + now := time.Now() + if lastRun != nil && !lastRun.IsZero() { + since := now.Sub(*lastRun) + if since < issueUnparkSweepWebhookInterval { + return + } + if since < issueUnparkSweepInterval && github.TakeDirtyIssueCount() == 0 { + return + } + } + if lastRun != nil { + *lastRun = now + } + // `/hive approve` on a PR is answered with what the merge lane will do + // with it (hivecommons/hive#13087); the engine here only reads policy. + approveOpts := automerge.Options{ + MergerAuthorizer: trustedMergerFunc(cfg), + TrustedAuthorizer: trustedAuthorFunc(cfg), + TrustedAuthorPolicy: trustedAuthorPolicyFunc(cfg), + ContributorLanePolicy: contributorLanePolicyFunc(cfg), + ContributorLaneVerdict: contributorLaneVerdictFunc(cfg, logger), + } + if cfg != nil { + approveOpts.RepoAutoMergeEnabled = cfg.RepoAutoMergeEnabled + approveOpts.HumanMergePaths = func(repo string) []string { return humanMergePathsFor(cfg, repo) } + approveOpts.HumanMergeRule = func() prstage.HumanMergeRule { return humanMergeRuleFor(cfg) } + approveOpts.TrustedBotAuthors = func() map[string]bool { return cfg.AutoMerge.TrustedBotAuthorSet() } + } + approveEngine := automerge.New(ghClient, approveOpts) + result, err := ghClient.SweepIssueUnparkCommands(ctx, github.IssueUnparkSweepOptions{ + PRApproveReply: approveEngine.PRApproveReply, + MaxActions: github.DefaultIssueUnparkSweepMaxActions, + // cfg is nil only in tests; nil leaves every repo on the default path. + LabelDriven: func(repo string) bool { return cfg.RepoLabelDriven(repo) }, + LabelApproval: labelApprovalPolicyResolver(cfg), + LabelApprovalAudit: func(event github.IssueLabelApprovalEvent) { + if dashSrv == nil { + return + } + detail := fmt.Sprintf("repo=%s, issue=%d, label=%s, actor=%s, trust=%s, removed=%s, released=%t, pending=%t", + event.Repo, event.Number, event.Label, event.Actor, event.TrustReason, strings.Join(event.Removed, ","), event.Released, event.Pending) + dashSrv.AuditLogRecord("system", "issue-approved-by-label", detail, "", event.Repo, event.Number) + }, + Audit: func(event github.IssueUnparkEvent) { + if dashSrv == nil { + return + } + detail := fmt.Sprintf("repo=%s, issue=%d, actor=%s, decision=%s", + event.Repo, event.Number, event.Actor, event.Decision) + dashSrv.AuditLogRecord("system", "issue-unparked-by-reply", detail, "", event.Repo, event.Number) + }, + ClosedAudit: func(event github.IssueUnparkEvent) { + if dashSrv == nil { + return + } + detail := fmt.Sprintf("repo=%s, issue=%d, actor=%s", event.Repo, event.Number, event.Actor) + dashSrv.AuditLogRecord("system", "issue-closed-by-fixed", detail, "", event.Repo, event.Number) + }, + // `/hive decision` on a hive-authored PR at 085 (hivecommons/hive#13041) + // is routed to the fix lane under the automerge lane's merger trust. + HumanChangeRequests: review.DefaultHumanChangeRequests(), + TrustedMerger: trustedMergerFunc(cfg), + PRAudit: func(event github.PullRequestDecisionEvent) { + if dashSrv == nil { + return + } + detail := fmt.Sprintf("repo=%s, pr=%d, head=%s, actor=%s, decision=%s", + event.Repo, event.Number, event.HeadSHA, event.Actor, event.Decision) + dashSrv.AuditLogRecord("system", "pr-decision-routed-to-fix", detail, "", event.Repo, event.Number) + }, + }) + if err != nil { + logger.Warn("issue un-park sweep failed", "error", err) + return + } + if result == nil { + return + } + if len(result.Unparked) > 0 || len(result.Closed) > 0 || result.Replies > 0 || len(result.LabelApproved) > 0 || len(result.PRDecisions) > 0 { + logger.Info("issue un-park sweep complete", + "seen", result.Seen, "unparked", len(result.Unparked), "closed", len(result.Closed), "label_approved", len(result.LabelApproved), "pr_decisions", len(result.PRDecisions), "replies", result.Replies, "skipped", result.Skipped) + } +} + +// closedIssueStageSweepInterval spaces the closed-issue clean-up. A closed +// issue wearing stage/045 is a stale dashboard row, not a person waiting, so +// this runs at the task-list cadence; each pass is one listing per repo. +const closedIssueStageSweepInterval = 10 * time.Minute + +// closedIssueStageSweepSlack widens each pass's `since` so an issue that +// closed while the previous pass was running is not missed. +const closedIssueStageSweepSlack = 10 * time.Minute + +// runClosedIssueStageSweepIfDue strips hive stage and park labels from issues +// that closed while carrying them (hivecommons/homebrew-hive#127): the +// issue-stage reconciler only visits open issues, so a `Fixes #N` merge or a +// by-hand close left `stage/045-needs-decision` behind and the tracker kept +// reading it as a pending decision. Every rule lives in +// SweepClosedIssueStageLabels; this is the scheduler and audit sink. +func runClosedIssueStageSweepIfDue(ctx context.Context, ghClient *github.Client, dashSrv *dashboard.Server, lastRun *time.Time, logger *slog.Logger) { + if ghClient == nil { + return + } + now := time.Now() + var since time.Time + if lastRun != nil && !lastRun.IsZero() { + if now.Sub(*lastRun) < closedIssueStageSweepInterval { + return + } + since = lastRun.Add(-closedIssueStageSweepSlack) + } + if lastRun != nil { + *lastRun = now + } + result, err := ghClient.SweepClosedIssueStageLabels(ctx, github.ClosedIssueStageSweepOptions{ + Since: since, + MaxActions: github.DefaultClosedIssueStageSweepMaxActions, + Audit: func(event github.ClosedIssueStageEvent) { + if dashSrv == nil { + return + } + detail := fmt.Sprintf("repo=%s, issue=%d, stage=%s, needs_human=%t", + event.Repo, event.Number, strings.Join(event.Stage, ","), event.NeedsHuman) + dashSrv.AuditLogRecord("system", "closed-issue-stage-cleared", detail, "", event.Repo, event.Number) + }, + }) + if err != nil { + logger.Warn("closed-issue stage sweep failed", "error", err) + } else if result != nil && (len(result.Cleaned) > 0 || result.Skipped > 0) { + logger.Info("closed-issue stage sweep complete", "seen", result.Seen, "cleaned", len(result.Cleaned), "skipped", result.Skipped) + } + runReviewBacklogSplitSweep(ctx, ghClient, dashSrv, since, logger) +} + +// runReviewBacklogSplitSweep re-files still-unchecked review findings folded +// into an issue that has since closed (hivecommons/hive#13914). It shares the +// closed-issue sweep's cadence and window; every rule lives in +// SweepReviewBacklogSplits. It then runs the one-off +// BackfillReviewBacklogSplits for hosts closed before that window +// (hivecommons/hive#13947), a no-op once the ledger records it complete. +func runReviewBacklogSplitSweep(ctx context.Context, ghClient *github.Client, dashSrv *dashboard.Server, since time.Time, logger *slog.Logger) { + audit := func(event github.ReviewBacklogSplitEvent) { + if dashSrv == nil { + return + } + detail := fmt.Sprintf("repo=%s, host=%d, issue=%d, reused=%t, key=%s", + event.Repo, event.Host, event.Number, event.Reused, event.Key) + dashSrv.AuditLogRecord("system", "review-backlog-split", detail, "", event.Repo, event.Host) + } + result, err := ghClient.SweepReviewBacklogSplits(ctx, github.ReviewBacklogSplitSweepOptions{ + Since: since, + MaxActions: github.DefaultReviewBacklogSplitMaxActions, + Audit: audit, + }) + if err != nil { + logger.Warn("review backlog split sweep failed", "error", err) + return + } + if result != nil && (len(result.Split) > 0 || result.Skipped > 0 || result.Deferred > 0) { + logger.Info("review backlog split sweep complete", "hosts", result.Hosts, "split", len(result.Split), "skipped", result.Skipped, "deferred", result.Deferred) + } + backfill, err := ghClient.BackfillReviewBacklogSplits(ctx, github.ReviewBacklogSplitSweepOptions{ + MaxActions: github.DefaultReviewBacklogSplitMaxActions, + Audit: audit, + }) + if err != nil { + logger.Warn("review backlog split backfill failed", "error", err) + return + } + if backfill != nil && (len(backfill.Split) > 0 || backfill.Skipped > 0 || backfill.Deferred > 0) { + logger.Info("review backlog split backfill pass complete", "hosts", backfill.Hosts, "split", len(backfill.Split), "skipped", backfill.Skipped, "deferred", backfill.Deferred) + } +} + +// labelApprovalPolicyResolver reads label-driven approval settings +// (hivecommons/hive#11313) live from cfg. It answers ok=false for every repo +// without project.repo_policies[].label_driven, which keeps the feature off +// there; a nil cfg (tests) opts nothing in. +func labelApprovalPolicyResolver(cfg *config.Config) func(repo string) (github.LabelApprovalPolicy, bool) { + return func(repo string) (github.LabelApprovalPolicy, bool) { + if cfg == nil { + return github.LabelApprovalPolicy{}, false + } + labels := cfg.LabelDrivenApprovalLabels(repo) + if len(labels) == 0 { + return github.LabelApprovalPolicy{}, false + } + return github.LabelApprovalPolicy{ + Labels: labels, + TrustOrgMembers: cfg.LabelDrivenTrustOrgMembers(), + TrustedBots: cfg.LabelDrivenTrustedBots(), + SupersedeParkActors: cfg.LabelDrivenSupersedeParkActors(repo), + }, true + } +} + +var ( + ciFailingPath = "/var/run/hive-metrics/ci-failing.json" + intentVerdictsPath = "/var/run/hive-metrics/intent-verdicts.json" +) + +// claimLedger holds the duplicate-PR guard's persisted issue→PR claim mapping +// across eval cycles. It is loaded lazily on first use (and retried on a load +// failure) rather than at startup, so a missing or corrupt /data ledger can +// never block the hive from booting. +var ( + claimLedgerOnce sync.Once + claimLedger *github.ClaimLedger + claimLedgerPath = github.ClaimLedgerPath + claimLedgerLoader = github.LoadClaimLedger +) + +// hiveIdentity determines which PR authors count as "this hive", so only our +// own PRs suppress work. Two accounts can open PRs on our behalf: +// - project.ai_author — the account agents push and open PRs as +// - the GitHub App bot login ("[bot]") when the hive authenticates +// as an installation, which is what actually authors PRs in that mode +func hiveIdentity(cfg *config.Config) github.HiveIdentity { + id := github.HiveIdentity{AIAuthor: cfg.Project.AIAuthor} + if slug := cfg.GitHub.ResolvedAppSlug(); slug != "" { + id.AppLogin = slug + "[bot]" + } + return id +} + +// issueStageSyncTimeout bounds one eval pass's issue stage-label sync so a +// rate-limit pacing queue cannot stall the run loop (hivecommons/hive#12983). +const issueStageSyncTimeout = 2 * time.Minute + +// recheckSoakHubURL is the hub the soak triage probes, "" when this hive has +// no hub (the probe then reads the spoke's own /api/version). +func recheckSoakHubURL(cfg *config.Config) string { + if cfg == nil || !cfg.Hub.Enabled { + return "" + } + return cfg.Hub.URL +} + +// applyDuplicatePRGuard filters issues already claimed by an open hive-authored +// PR out of the actionable set. Failures are logged, never fatal: the guard is +// a safety net, and a broken net must not take the hive down with it. +// reconcileLedgerCoverRecords runs github.BackfillCoveredByLedger with the +// cycle's red+stale valve. It is not an optional step: with webhook-driven +// cached cycles the reconcile pass (and the stage sweep with it) can be +// hours apart, and a record at 010 under an open covering PR re-offers the +// issue to the scanner for all of that time. +func reconcileLedgerCoverRecords(ctx context.Context, cfg *config.Config, ghClient *github.Client, actionable *github.ActionableResult, logger *slog.Logger) { + ledger := getClaimLedger(logger) + if ledger == nil || ledger.Len() == 0 || ghClient == nil { + return + } + backfillCtx, cancel := context.WithTimeout(ctx, issueStageSyncTimeout) + defer cancel() + res := ghClient.BackfillCoveredByLedger(backfillCtx, ledger, claimingPRRedStale(cfg, actionable)) + if res.Covered+res.Errors > 0 && logger != nil { + logger.Info("stage backfill: issue records reconciled with the claim ledger", + "checked", res.Checked, "covered", res.Covered, "skipped_red_stale", res.SkippedRedStale, + "skipped_not_open", res.SkippedNotOpen, "errors", res.Errors) + } +} + +func applyDuplicatePRGuard( + ctx context.Context, + cfg *config.Config, + ghClient *github.Client, + actionable *github.ActionableResult, + logger *slog.Logger, +) { + // The PR-claim guard runs FIRST: an open PR that fixes the issue is the + // stronger evidence, and the guards below only walk what is still in + // the actionable set. With the live-claim filter ahead of it, an issue + // whose agent claim was still live when its PR opened was withheld as + // "claimed" before the ledger could bucket it "claimed_by_pr", so every + // sweep re-derived 020 under the PR-opened 040 and the stage store + // recorded a spurious regression (hivecommons/hive#13174 et al.). The + // ledger's open-issue set (OpenIssuesOf) likewise needs the full list. + ledger := getClaimLedger(logger) + if ledger != nil { + github.ApplyDuplicatePRGuard(ctx, ghClient, ledger, hiveIdentity(cfg), actionable, claimingPRRedStale(cfg, actionable), logger) + } + // #8380: a LIVE issue claim covers an issue the same way an open PR does, + // for every kick prompt (the scanner's included). The claim fields are + // only ever set while governor.claims.enabled is on, so with the feature + // off this touches nothing. + // hivecommons/hive#13579: a contributor claim on an issue in flap + // cool-off, or one that has starved hive agents past the configured + // window with no PR, is advisory — the issue stays actionable. + if cfg != nil && cfg.Governor.Claims.Enabled { + if withheld := contributorClaimGuard.Load().FilterLiveIssueClaims(actionable, time.Now(), logger); withheld > 0 { + logger.Info("issue-claim guard applied", "withheld", withheld) + } + } + if ledger == nil { + return + } + // The in-progress sweep runs after the PR-claim guard so an issue whose + // PR just opened sheds the start mark here, and a lapsed mark frees the + // issue before kicks are built. Independent of governor.claims: the + // label is read from the issue, the marker from the comment, on every hive. + if withheld := github.SweepInProgressIssues(ctx, ghClient, actionable, time.Now(), logger); withheld > 0 { + logger.Info("in-progress guard applied", "withheld", withheld) + } + // hivecommons/hive#12436: with every bucket final, mirror the cycle's + // decision about each open issue onto its stage/NNN label (and strip the + // retired claimed / hive/in-progress / 🐝 covered by PR labels). + // hivecommons/hive#12983: the run loop holds cfgReloadMu for the whole + // pass, so bound the sync; a paced request whose slot would miss the + // deadline fails fast and the best-effort sync retries next cycle. + stageCtx, stageCancel := context.WithTimeout(ctx, issueStageSyncTimeout) + stageCtx, stageSpan := tracing.StartSpan(stageCtx, "governor.stage_labels") + syncStageEngineMode(cfg, ghClient, logger) + res := github.SyncIssueStageLabelsWith(stageCtx, ghClient, actionable, nil, time.Now(), logger, stageEngineOptions(cfg, logger)) + stageSpan.SetAttributes( + attribute.Int("hive.stage_labels.items", res.Items), + attribute.Int("hive.stage_labels.writes", res.Writes)) + stageSpan.End() + stageCancel() + if res.Writes+res.Comments > 0 { + logger.Info("issue stage labels applied", "items", res.Items, "writes", res.Writes, "comments", res.Comments, "healed", res.Healed) + } + // hivecommons/hive#13093: issues parked on an unmerged PR come back once + // that PR settles, and never advertise `help wanted` while parked. + // hivecommons/hive#13311: ops-tracking soak checklists past their + // recheck-after deadline get evidence from the hub (or this spoke). + github.SetRecheckSoakProbe(github.NewHTTPSoakProbe(recheckSoakHubURL(cfg), fmt.Sprintf("http://localhost:%d", cfg.Dashboard.Port), nil)) + blockedCtx, blockedCancel := context.WithTimeout(ctx, issueStageSyncTimeout) + // hivecommons/hive#13833: review-backlog items wait on the PR they were + // filed from in every destination; lift them when it merges, and when it + // closes unmerged unblock them with a re-triage note (never auto-close). + // Runs first so the items get the sweep's own note rather than the + // generic one from the sweep below. + if rb := github.ReconcileReviewBacklogBlocks(blockedCtx, ghClient, logger); rb.Lifted+rb.Retriaged+rb.Errors > 0 { + logger.Info("review backlog blocks reconciled", "checked", rb.Checked, "lifted", rb.Lifted, "retriaged", rb.Retriaged, "errors", rb.Errors) + } + if br := github.ReconcileBlockedIssues(blockedCtx, ghClient, actionable, logger); br.Unblocked+br.Retargeted+br.Stripped+br.SoakTriaged+br.Errors > 0 { + logger.Info("blocked issues reconciled", "items", br.Items, "unblocked", br.Unblocked, "retargeted", br.Retargeted, "help_wanted_stripped", br.Stripped, "soak_triaged", br.SoakTriaged, "errors", br.Errors) + } + blockedCancel() + // Closing an issue is irreversible: refuse while the snapshot is stale and + // re-read an item whose copy is older than DecisionFreshnessBound. + runCloseOnMergeSweep(github.WithIrreversibleGuard(ctx, ghClient.SnapshotFreshnessGuard(actionable, time.Now)), cfg, ghClient, ledger, logger) +} + +// inProgressSetting resolves the pr-in-progress stage label setting from config. +// The label follows the claims label mirror: an operator who set +// governor.claims.label: false asked for no hive-managed claim labels, and +// gets none here either; otherwise the start signal is always labelled. The +// mark lasts the agent claim TTL, so both the ledger and the label agree on +// when an agent's hold lapses. +func inProgressSetting(cfg *config.Config) github.InProgressSetting { + if cfg == nil { + return github.InProgressSetting{} + } + _, agentTTL, _, _ := cfg.Governor.Claims.EffectiveTTLs() + return github.InProgressSetting{ + Enabled: !(cfg.Governor.Claims.IsEnabled() && !cfg.Governor.Claims.LabelEnabled()), + HiveID: cfg.HiveID, + TTL: agentTTL, + } +} + +func logCloseOnMergeResults(logger *slog.Logger, msg string, results []github.CloseOnMergeResult) { + if logger == nil || len(results) == 0 { + return + } + var closed, awaiting, skipped, noop int + for _, r := range results { + switch r.Action { + case github.CloseOnMergeClosed: + closed++ + case github.CloseOnMergeAwaiting: + awaiting++ + case github.CloseOnMergeSkipped: + skipped++ + default: + noop++ + } + } + if closed > 0 || awaiting > 0 || skipped > 0 { + logger.Info(msg, "closed", closed, "awaiting_confirmation", awaiting, "skipped", skipped, "noop", noop) + } +} + +// getClaimLedger lazily loads the persisted claim ledger on first use (and +// keeps a usable empty ledger on a load failure), so a missing or corrupt +// /data ledger can never block the hive from booting. It is shared by the +// eval-cycle guard above and by the dashboard's IssueClaimed hook (#3768); the +// sync.Once publication makes the pointer safe to read from either goroutine, +// and the ledger itself is internally locked. +func getClaimLedger(logger *slog.Logger) *github.ClaimLedger { + claimLedgerOnce.Do(func() { + ledger, err := claimLedgerLoader(claimLedgerPath, logger) + if err != nil { + // LoadClaimLedger always returns a usable (possibly empty) ledger + // alongside the error, so we keep it and just report the problem. + logger.Warn("duplicate-PR guard: could not load persisted claim ledger, starting empty", + "path", claimLedgerPath, "error", err) + } + claimLedger = ledger + // A person reopening an issue after its closing PR merged voids the + // merged claim right away (hivecommons/hive#9945 sat likely-done for + // hours after a /reopen); the enumeration pass is the fallback when + // webhooks are not delivered. + github.SetIssueReopenedHook(issueReopenedHookFor(ledger, logger)) + }) + return claimLedger +} + +// issueReopenedHookFor builds the webhook hook that voids a merged claim on a +// reopened issue and persists the ledger, logging `reopened issue re-queued`. +func issueReopenedHookFor(ledger *github.ClaimLedger, logger *slog.Logger) github.IssueReopenedHook { + return func(repo string, number int, at time.Time) { + if ledger == nil || !ledger.MarkIssueReopened(repo, number, at) { + return + } + claim, _ := ledger.LookupAny(repo, number) + if logger != nil { + logger.Info("reopened issue re-queued: merged PR claim voided", + "repo", repo, "issue", number, "claimed_by_pr", claim.PRNumber, "pr_repo", claim.PRRepo, + "pr_url", claim.PRURL, "merged_at", claim.MergedAt, "reopened_at", at, "source", "webhook") + } + if err := ledger.Save(); err != nil && logger != nil { + logger.Warn("reopened issue: failed to persist claim ledger", "repo", repo, "issue", number, "error", err) + } + } +} + +// claimingPRRedStale builds the Fix #3 release predicate: given a claiming PR +// (prRepo, prNumber), report whether it is red on a required check AND stale. +// It looks up the PR's live CI state + head SHA from this pass's enumeration and +// consults the shared staleness clock. A PR not found in the enumeration, or one +// that is green/pending, or one whose red head only just appeared, returns false +// — so a HEALTHY claiming PR still suppresses its issue. Returns a nil func when +// escalation is disabled, preserving the original unconditional-suppress +// behavior. GENERIC: keys only off check state + staleness. +func claimingPRRedStale(cfg *config.Config, actionable *github.ActionableResult) github.RedStaleFunc { + if cfg.Escalation.Disabled || actionable == nil { + return nil + } + fullRepo := func(repo string) string { + if !strings.Contains(repo, "/") && cfg.Project.Org != "" { + return cfg.Project.Org + "/" + repo + } + return repo + } + // Index this pass's PRs by bare-repo#number so a claim's PRRepo (which may + // be bare or "owner/repo") resolves regardless of prefix. + type prState struct { + red bool + headSHA string + repo string + } + index := map[string]prState{} + for _, pr := range actionable.PRs.Items { + index[fmt.Sprintf("%s#%d", bareRepoName(pr.Repo), pr.Number)] = prState{ + red: pr.HasFailingRequiredCheck(), + headSHA: pr.HeadSHA, + repo: fullRepo(pr.Repo), + } + } + store := getEscalationStore() + return func(prRepo string, prNumber int) bool { + st, ok := index[fmt.Sprintf("%s#%d", bareRepoName(prRepo), prNumber)] + if !ok || !st.red { + return false // not enumerated, or healthy → keep suppressing + } + return store.StaleRed(st.repo, prNumber, st.headSHA) + } +} + +func fullRepoName(repo, org string) string { + if strings.Contains(repo, "/") || org == "" { + return repo + } + return org + "/" + repo +} + +// reviewAgentCapability describes one configured lane the way the review +// dispatcher sees it, so the relay judges reviewer-ness by the same rule. +// An unknown lane has only its name to go on. +func reviewAgentCapability(cfg *config.Config, agent string) review.AgentCapability { + if cfg != nil { + if ac, ok := cfg.Agents[agent]; ok { + return review.CapabilityFromConfig(agent, ac) + } + } + return review.AgentCapability{Name: agent} +} + +// reviewAuthorBase resolves an authoring lane to its base agent over the full +// configured agent set, enabled or not. A replica name the current config no +// longer materialises (its replicas: count was reduced) still maps to the +// declared base it was derived from, so the base and its remaining replicas +// stay excluded from reviewing it. +func reviewAuthorBase(cfg *config.Config, agent string) string { + agent = strings.TrimSpace(agent) + if cfg == nil || agent == "" { + return agent + } + if _, ok := cfg.Agents[agent]; ok { + return cfg.BaseAgentName(agent) + } + if i := strings.LastIndex(agent, "-"); i > 0 { + if n, err := strconv.Atoi(agent[i+1:]); err == nil && n >= 2 { + if ac, ok := cfg.Agents[agent[:i]]; ok && ac.ReplicaOf == "" { + return agent[:i] + } + } + } + return agent +} + +// installReviewRelaySettings gives a (possibly rebuilt) GitHub client the +// review-relay knobs that live in cfg.Review: which repos may be revised in +// place, which perspectives a verdict may name, and whether comments carry a +// confidence score (hivecommons/hive#8182). One place, so an app-auth rebuild +// cannot silently drop a setting the first client had — the perspective set +// was previously installed only on the boot path. +func installReviewRelaySettings(client *github.Client, cfg *config.Config, logger *slog.Logger) { + if client == nil || cfg == nil { + return + } + client.SetReviseRepos(cfg.Review.ReviseRepos) + client.SetPerspectives(reviewPerspectiveSet(cfg, logger)) + client.SetReviewerAgentFunc(func(agent string) bool { + return review.ReviewCapable(reviewAgentCapability(cfg, agent), review.ReviewerAgentSet(cfg.Review.ReviewerAgents)) + }) + client.SetAgentBaseFunc(func(agent string) string { + return reviewAuthorBase(cfg, agent) + }) + client.SetConfidenceScore(func() bool { return cfg.Review.ConfidenceScore }) + client.SetReviewBacklog(func() (bool, int) { + return !cfg.Review.OutOfScopeBacklogDisabled, cfg.Review.MaxOutOfScopeBacklogIssues + }) + client.SetReviewBacklogRouting(func() github.ReviewBacklogRouting { + return reviewBacklogRouting(client, cfg, logger) + }) + client.SetReviewEvidence(func(repo string) github.ReviewEvidenceSettings { + return reviewEvidenceSettings(cfg, repo) + }) + // hivecommons/hive#14581: issue-author credit on every merge path, read + // live so a reload of auto_merge.credit_issue_authors applies. + client.SetCreditIssueAuthors(func(string) bool { return cfg.AutoMerge.CreditIssueAuthorsEnabled() }) + // hivecommons/hive#13241: an issue a person reopens after its closing PR + // merged is a change failure on that PR's evidence record, from both the + // issues.reopened webhook and the enumeration reopen check. + github.SetIssueReopenRecorder(client.RecordReviewEvidenceIssueReopen) + // #8380: issue claims are read at enumeration time only while + // governor.claims.enabled is on; the setting is read live so the Features + // toggle applies without a client rebuild. + client.SetIssueClaims(func() (bool, time.Duration) { + return cfg.Governor.Claims.Enabled, cfg.Governor.Claims.EffectiveTTL() + }) + client.SetInProgress(func() github.InProgressSetting { return inProgressSetting(cfg) }) + // hivecommons/hive#12436: the issue stage reconciler follows the same + // governor.labels.stage policy as the PR one, read live. + client.SetIssueStageLabels(func() github.IssueStagePolicy { + p := stageLabelPolicyFunc(cfg)() + return github.IssueStagePolicy{Enabled: p.Enabled, Dwell: p.Dwell} + }) + client.SetReviewCadenceLimits( + func() bool { return cfg.Review.CombinedPerspectives }, + func() int { return cfg.Review.MaxReviewsPerHead }, + ) + // review.require_labels (PR approval allow-list): the relay's half of the + // gate, read live so a Features-tab edit applies to the next request. + client.SetReviewRequireLabels(func() []string { return cfg.Review.RequireLabels }) +} + +// reviewBacklogRouting resolves review.severity / review.backlog and the +// backlog sink for the active work source (hivecommons/hive#11089). A +// destination whose work source cannot be built falls back to GitHub issues; +// the reason is logged so the operator can fix the work-source config. +func reviewBacklogRouting(client *github.Client, cfg *config.Config, logger *slog.Logger) github.ReviewBacklogRouting { + ghToken := cfg.GitHub.Token + if ghToken == "" { + ghToken = os.Getenv("HIVE_GITHUB_TOKEN") + } + sink, err := worksource.NewReviewBacklogSink(cfg.Governor.WorkSource, cfg.Review.Backlog, client, ghToken, cfg.Project.Org, logger) + if err != nil && logger != nil { + logger.Warn("review backlog: destination unavailable, filing GitHub issues", "error", err) + } + return github.ReviewBacklogRouting{Severity: cfg.Review.Severity, Backlog: cfg.Review.Backlog, Sink: sink} +} + +// installReviewBots installs classification.review_bots on a (possibly +// rebuilt) GitHub client (hivecommons/hive#7360). hive.yaml's block wins; +// otherwise the same key is read from hive-project.yaml +// (config.DefaultProjectYAMLPath, overridable via HIVE_PROJECT_YAML — the +// path the bash pipeline stages already honour). Nil-safe: a hive without +// GitHub credentials runs with a nil client for the life of the process. +// Also installs review.fix_human_prs (hivecommons/hive#9361), so the +// review-thread reconciler follows PRs the hive did not open whenever the +// fixer is already allowed to push to them. +func installReviewBots(client *github.Client, cfg *config.Config, logger *slog.Logger) { + if client == nil || cfg == nil { + return + } + rb, err := cfg.EffectiveReviewBots(os.Getenv("HIVE_PROJECT_YAML")) + if err != nil && logger != nil { + logger.Warn("classification.review_bots: project file unreadable; review-thread reconciler stays off", "error", err) + } + client.SetReviewBots(rb) + client.SetFixHumanPRs(cfg.Review.FixHumanPRsEnabled()) + if logger != nil && rb.Enabled() { + logger.Info("review-thread reconciler enabled", + "review_bots", rb.Logins, + "max_attempts_per_thread", rb.MaxAttempts(), + "resolve_after_fix", rb.ResolveAfterFixEnabled(), + "fix_human_prs", cfg.Review.FixHumanPRsEnabled()) + } +} + +// reviewThreadsRefreshInterval throttles the review-thread monitor: the eval +// tick runs about once a minute, but every hive-authored open PR costs one +// GraphQL query per pass, and bot threads move on the scale of minutes, not +// seconds. A var so tests can drive it. +var reviewThreadsRefreshInterval = 5 * time.Minute + +// reviewThreadsLastRefresh is the wall-clock of the last CollectReviewThreads +// pass (zero = never). Package-level because the eval tick is a free +// function; only the eval goroutine touches it. +var reviewThreadsLastRefresh time.Time + +// writeReviewThreads is the eval-tick half of the #7360 reconciler. It runs +// CollectReviewThreads over the governor's actionable PRs at most once per +// reviewThreadsRefreshInterval, stamps each PR with the agent whose relay +// request opened it (auditPRAgents — the same attribution ci-failing.json +// carries) and with the escalation sweep's needs-human verdict, and writes +// review-threads.json. The feature-off case still writes the (empty) file +// each refresh so a reader can tell "off" from "never ran". +func writeReviewThreads(ctx context.Context, client *github.Client, actionable *github.ActionableResult, org string, escalatedPRs map[string]bool, logger *slog.Logger) { + if client == nil || actionable == nil { + return + } + now := time.Now() + if !reviewThreadsLastRefresh.IsZero() && now.Sub(reviewThreadsLastRefresh) < reviewThreadsRefreshInterval { + return + } + reviewThreadsLastRefresh = now + + report := client.CollectReviewThreads(ctx, actionable.PRs.Items, now) + if len(report.PRs) > 0 { + prAgents := auditPRAgents(org, now.Add(-auditPRAttributionWindow), "") + for i := range report.PRs { + pr := &report.PRs[i] + pr.Agent = prAgents[fmt.Sprintf("%s#%d", pr.Repo, pr.Number)] + pr.Escalated = escalatedPRs[escalation.Key(pr.Repo, pr.Number)] + } + } + if err := github.WriteReviewThreadsReport("", report); err != nil { + logger.Warn("failed to write review-threads.json", "error", err) + return + } + if report.Enabled { + logger.Info("review-threads.json refreshed", "prs", len(report.PRs), "threads", report.TotalThreads) + } +} + +// auditPRAttributionWindow bounds how far back the audit trail is scanned to +// map open PRs to the agent that opened them. Red PRs older than this fall +// back to scanner ownership in the kick builders — acceptable: 14d exceeds any +// PR the fleet should still be iterating on. +const auditPRAttributionWindow = 14 * 24 * time.Hour + +// auditPRAgents maps "org/repo#number" → agent name from the audit trail's +// agent_pr_created entries (attribution.go records one per relay-opened PR, +// reuses included). Reading the on-disk log per eval tick keeps this +// stateless; OutputActionsSince touches no receiver state, so a zero-value +// AuditLog is safe here. +func auditPRAgents(org string, since time.Time, auditPath string) map[string]string { + entries := (&dashboard.AuditLog{}).OutputActionsSince(since, + map[string]bool{github.AuditActionAgentPRCreated: true}, auditPath) + out := make(map[string]string, len(entries)) + for _, e := range entries { + if e.Agent == "" { + continue + } + var repo, number string + for _, part := range strings.Split(e.Detail, ",") { + if k, v, ok := strings.Cut(strings.TrimSpace(part), "="); ok { + switch k { + case "repo": + repo = v + case "number": + number = v + } + } + } + if repo == "" || number == "" { + continue + } + if !strings.Contains(repo, "/") && org != "" { + repo = org + "/" + repo + } + out[repo+"#"+number] = e.Agent + } + return out +} + +// applyHumanDecisionLabels mirrors the review swarm's "a human must decide" +// holds onto the operator's existing triage label, so they can be filtered +// from the PR list instead of being found by opening threads. +// +// This is strictly additive to the marker the reviewer writes into the review +// body. Every failure path here — no label configured, label absent from the +// repo, API refusal — is logged and skipped, because a mislabelled hold is a +// smaller problem than a hold that never got reported. +func applyHumanDecisionLabels(ctx context.Context, cfg *config.Config, ghClient *github.Client, actionable *github.ActionableResult, plan review.DispatchPlan, logger *slog.Logger) { + if cfg == nil || ghClient == nil { + return + } + label := strings.TrimSpace(cfg.Review.HumanDecisionLabel) + if label == "" || len(plan.State.Human) == 0 { + return + } + // Holds persist across cycles, so re-deriving them every pass would re-ask + // GitHub to apply a label the PR already carries. The enumeration already + // fetched each PR's labels, so skipping the settled ones costs nothing. + // + // Holds carry the owner/repo form (they descend from review reports, which + // record GitHub's full name) while PRs.Items carry whatever governor.repos + // says — usually the bare repo under project.org. Key both through + // fullRepoName or the skip never matches and every cycle re-labels every + // hold (observed on a hive whose repos were configured bare). + org := cfg.Project.Org + labeled := map[string]bool{} + if actionable != nil { + for _, pr := range actionable.PRs.Items { + for _, have := range pr.Labels { + if strings.EqualFold(strings.TrimSpace(have), label) { + labeled[strings.ToLower(fmt.Sprintf("%s#%d", fullRepoName(pr.Repo, org), pr.Number))] = true + } + } + } + } + failing := map[string]struct{}{} + defer func() { humanLabelWarn.retain(failing) }() + for _, hold := range plan.State.Human { + if labeled[strings.ToLower(fmt.Sprintf("%s#%d", fullRepoName(hold.Repo, org), hold.Number))] { + continue + } + if err := ghClient.ApplyHumanDecisionLabel(ctx, hold.Repo, hold.Number, label); err != nil { + // Missing optional labels are configuration notices, not runtime + // failures: first warnings across different PRs can trigger fleet + // reports even with per-PR deduplication (#13857). + key := prWarnKey(hold.Repo, hold.Number) + missing := errors.Is(err, github.ErrHumanDecisionLabelMissing) + if missing { + key += ":missing-label" + } + failing[key] = struct{}{} + level := humanLabelWarn.level(key) + if missing && level == slog.LevelWarn { + level = slog.LevelInfo + } + logger.Log(ctx, level, "human decision label not applied; review marker still stands", + "repo", hold.Repo, "pr", hold.Number, "label", label, "error", err) + continue + } + logger.Info("human decision label applied", "repo", hold.Repo, "pr", hold.Number, "label", label, "reason", hold.Reason) + } +} + +// parseReviseCutoff reads Review.ReviseVerdictsBefore. A malformed value is +// logged and ignored rather than defaulting to "now", because a cutoff that +// silently becomes the current time would re-open every verdict in the +// artifact at once — the opposite of the narrow, deliberate correction this +// setting exists for. +// reviewPerspectiveSet resolves the hive's configured review perspectives. +// +// A bad configuration falls back to the built-in set and says so, rather than +// disabling review. Silently reviewing nothing would look identical to a healthy +// hive with an empty queue; reviewing with the defaults while logging the error +// keeps coverage up and makes the mistake findable. +func reviewPerspectiveSet(cfg *config.Config, logger *slog.Logger) review.PerspectiveSet { + if cfg == nil { + return review.PerspectiveSet{} + } + set, err := review.NewPerspectiveSet(cfg.Review.Perspectives, cfg.Review.PerspectivePrompts) + if err != nil { + if logger != nil { + logger.Warn("review.perspectives is invalid; reviewing with the built-in set until it is fixed", + "error", err) + } + set = review.PerspectiveSet{} + } + // plan_match (#8317) is a separate switch from the selection so turning + // it on never requires spelling out the whole set. Applied on the + // fallback path too: a typo elsewhere in the selection must not silently + // turn plan matching off. + if cfg.Review.PlanMatch.Enabled { + set = set.WithPlanMatch() + } + return set +} + +// planWaveFor renders the approved plan a PR's run trailers name, for the +// plan_match perspective (#8317). The Hive-Plan ref is tried first, then the +// Hive-Run key, since either may carry the epic ID or the source issue ref. +// Empty when plan_match is off, the PR has no trailer, or no store holds the +// plan -- the kick tells the reviewer which of those it is. +func planWaveFor(cfg *config.Config, stores map[string]*beads.Store, planRef, runKey string) string { + if cfg == nil || !cfg.Review.PlanMatch.Enabled { + return "" + } + for _, ref := range []string{planRef, runKey} { + if tree, ok := planning.FindPlanTree(stores, ref); ok { + return tree.WaveText() + } + } + return "" +} + +func parseReviseCutoff(raw string, logger *slog.Logger) time.Time { + trimmed := strings.TrimSpace(raw) + if trimmed == "" { + return time.Time{} + } + cutoff, err := time.Parse(time.RFC3339, trimmed) + if err != nil { + if logger != nil { + logger.Warn("review revise_verdicts_before is not RFC3339; ignoring", + "value", trimmed, "error", err) + } + return time.Time{} + } + return cutoff +} + +// reviewPlannerGitHub is the slice of the GitHub client the review planner +// reads: a PR's changed paths for the bot-author model route +// (review.bot_author_full_review_paths) and the compare between the head +// the hive last judged and the current one for the delta re-review +// (review.delta_rereview). +type reviewPlannerGitHub interface { + ReviewPRChangedFiles(ctx context.Context, repo string, number int) ([]string, bool) + ReviewDeltaDiff(ctx context.Context, repo, previousHead, head string) review.DeltaDiff +} + +func planReviewDispatch(cfg *config.Config, actionable *github.ActionableResult, agentMgr crux.Runtime, beadStores map[string]*beads.Store, logger *slog.Logger) review.DispatchPlan { + return planReviewDispatchWith(context.Background(), cfg, actionable, agentMgr, beadStores, nil, logger) +} + +// reviewBotAuthorRoute is review.bot_authors / bot_author_model / +// bot_author_full_review_paths resolved for the planner. +func reviewBotAuthorRoute(cfg *config.Config) review.BotAuthorRoute { + backend, model := cfg.Review.BotAuthorModelRef() + return review.BotAuthorRoute{Authors: cfg.Review.EffectiveBotAuthors(), Backend: backend, Model: model, FullReviewPaths: cfg.Review.EffectiveBotAuthorFullReviewPaths()} +} + +// reviewBotRouteFilesFor fetches a PR's changed paths when the bot-author +// route may apply to it: route on, author a listed bot, no verdict at this +// head and no review in flight (one listing per head, not per cycle). nil, +// false otherwise, which the planner reads as unknown: no downgrade. +func reviewBotRouteFilesFor(ctx context.Context, cfg *config.Config, lister reviewPlannerGitHub, route review.BotAuthorRoute, artifact review.Artifact, state review.DispatchState, repo string, number int, author, headSHA, bodyDigest string) ([]string, bool) { + if lister == nil || !route.Enabled() || !review.IsBotAuthor(author, route.Authors, cfg.EffectiveAIAuthor()) { + return nil, false + } + if _, judged := artifact.AggregateForBody(repo, number, headSHA, bodyDigest); judged { + return nil, false + } + if !state.PendingSince(repo, number, headSHA).IsZero() { + return nil, false + } + return lister.ReviewPRChangedFiles(ctx, repo, number) +} + +// reviewDeltaDiffFor fetches the delta compare for one PR when it is a +// delta re-review candidate: delta mode on, no verdict yet at this head, no +// dispatch entry for it (a compare per cycle for a review in flight would be +// API spend for nothing), and a prior verdict at another head to diff from. +// nil otherwise, which the planner reads as "not fetched". +func reviewDeltaDiffFor(ctx context.Context, cfg *config.Config, differ reviewPlannerGitHub, artifact review.Artifact, state review.DispatchState, repo string, number int, headSHA, bodyDigest string) *review.DeltaDiff { + if differ == nil || !cfg.Review.DeltaRereviewEnabled() || strings.TrimSpace(headSHA) == "" { + return nil + } + if _, judged := artifact.AggregateForBody(repo, number, headSHA, bodyDigest); judged { + return nil + } + if !state.PendingSince(repo, number, headSHA).IsZero() { + return nil + } + prior, ok := artifact.PriorAggregateFor(repo, number, headSHA) + if !ok { + return nil + } + diff := differ.ReviewDeltaDiff(ctx, repo, prior.HeadSHA, headSHA) + return &diff +} + +func planReviewDispatchWith(ctx context.Context, cfg *config.Config, actionable *github.ActionableResult, agentMgr crux.Runtime, beadStores map[string]*beads.Store, ghr reviewPlannerGitHub, logger *slog.Logger) review.DispatchPlan { + if cfg == nil || actionable == nil || !cfg.Review.RequireApproval || !cfg.Review.FanOut { + return review.DispatchPlan{} + } + botRoute := reviewBotAuthorRoute(cfg) + state, err := review.LoadDispatchState("") + if err != nil && !os.IsNotExist(err) { + logger.Warn("review dispatch state unavailable; starting fresh", "error", err) + } + // classification.review_bots, read the same way installReviewBots reads + // it for the thread reconciler (#7360): hive.yaml wins, hive-project.yaml + // is the fallback. Feeds the reviewer's read step (#9360); empty/off + // leaves ReviewBotLogins empty and the prompt section is skipped. + reviewBots, rbErr := cfg.EffectiveReviewBots(os.Getenv("HIVE_PROJECT_YAML")) + if rbErr != nil && logger != nil { + logger.Warn("classification.review_bots: project file unreadable; reviewer will not see bot findings this pass", "error", rbErr) + } + artifact, err := review.LoadArtifact("") + if err != nil && !os.IsNotExist(err) { + logger.Warn("review verdict artifact unavailable for dispatch planning", "error", err) + } + prAgents := auditPRAgents(cfg.Project.Org, time.Now().Add(-auditPRAttributionWindow), "") + candidates := make([]github.PullRequest, 0, len(actionable.PRs.Items)+len(actionable.PRs.Held)) + seenPRs := map[string]bool{} + labelGated := 0 + addCandidates := func(items []github.PullRequest) { + for _, pr := range items { + fullRepo := fullRepoName(pr.Repo, cfg.Project.Org) + key := github.ReviewQueueKey(fullRepo, pr.Number) + if fullRepo == "" || pr.Number <= 0 || seenPRs[key] { + continue + } + seenPRs[key] = true + // review.require_labels: an unlabelled PR is never dispatched + // to the swarm. The relay refuses it independently. + if !cfg.Review.AdmitsPRLabels(pr.Labels) { + labelGated++ + continue + } + candidates = append(candidates, pr) + } + } + addCandidates(actionable.PRs.Items) + addCandidates(actionable.PRs.Held) + if labelGated > 0 && logger != nil { + logger.Info("review label gate withheld PRs from review dispatch", + "withheld", labelGated, "candidates", len(candidates), + "require_labels", strings.Join(cfg.Review.NormalizedRequireLabels(), ",")) + } + sort.SliceStable(candidates, func(i, j int) bool { + ri, rj := candidates[i].ReviewRank, candidates[j].ReviewRank + if ri > 0 && rj > 0 && ri != rj { + return ri < rj + } + return ri > 0 && rj <= 0 + }) + + scopeReader := reviewScopeLinearReader(cfg, logger) + prs := make([]review.PullRequest, 0, len(candidates)) + for _, pr := range candidates { + lane := classify.Classify(github.Issue{Title: pr.Title, Labels: pr.Labels}).Lane + fullRepo := fullRepoName(pr.Repo, cfg.Project.Org) + item := review.PullRequest{ + Repo: pr.Repo, + Number: pr.Number, + Title: pr.Title, + Author: pr.Author, + Labels: append([]string(nil), pr.Labels...), + HeadSHA: pr.HeadSHA, + UpdatedAt: pr.UpdatedAt, + BodyDigest: pr.BodyDigest, + URL: pr.URL, + Lane: string(lane), + // Attribution trailer, carried for consumers and the audit trail. + // Not a planner input: the fix gate decides on the login author + // alone (#8421, #13124). + HiveAttributed: pr.HiveAttributed, + // The authoring model from the same trailer: what review_models + // exclusions compare against, and stamped into the verdict as + // author_model (hivecommons/hive#13243). + AuthorModel: pr.HiveModel, + // Grounding anchor for the review prompt. Repo access is the + // measured active ingredient in review quality (17%→67% hit rate, + // 61% fewer false positives), so the reviewer is told which commit + // to read rather than being left to infer from the diff. + MergeBase: pr.BaseSHA, + AuthorAgent: prAgents[fmt.Sprintf("%s#%d", fullRepo, pr.Number)], + // The author's base over every configured lane, enabled or not, + // so a disabled or retired authoring replica still excludes its + // family from reviewing it (hivecommons/hive#13552). + AuthorBase: reviewAuthorBase(cfg, prAgents[fmt.Sprintf("%s#%d", fullRepo, pr.Number)]), + // Run trailers (#8310) and the plan they name, for plan_match + // (#8317). Identifiers and planner output only: the PR body + // itself still never reaches the prompt. + RunKey: pr.HiveRun, + PlanRef: pr.HivePlan, + PlanWave: planWaveFor(cfg, beadStores, pr.HivePlan, pr.HiveRun), + ScopeContract: pr.ScopeContract, + // A maintainer's changes_requested review or `/hive decision` + // on this head (hivecommons/hive#13041): the planner kicks the + // fix lane with it instead of reviewing the head again. + HumanChangeRequest: review.HumanChangeRequestFor(review.DefaultHumanChangeRequests(), fullRepo, pr.Number, pr.HeadSHA), + // The compare from the last judged head, for the delta + // re-review and its trivial skip (review.delta_rereview). + DeltaDiff: reviewDeltaDiffFor(ctx, cfg, ghr, artifact, state, fullRepo, pr.Number, pr.HeadSHA, pr.BodyDigest), + BaseDiffEmpty: pr.BaseDiffEmpty, + } + // The linked Linear issue's text for the scope contract, read here + // so reviewers never need a Linear token (hivecommons/hive#14906); + // only for a PR about to be reviewed at this head. + if _, judged := artifact.AggregateForBody(fullRepo, pr.Number, pr.HeadSHA, pr.BodyDigest); !judged && state.PendingSince(fullRepo, pr.Number, pr.HeadSHA).IsZero() { + reviewScopeContractFor(ctx, scopeReader, &item, logger) + } + // Changed paths for the bot-author model route's path exceptions + // (review.bot_author_full_review_paths); fetched only for a listed + // bot's PR that is about to be reviewed. + item.ChangedFiles, item.ChangedFilesKnown = reviewBotRouteFilesFor(ctx, cfg, ghr, botRoute, artifact, state, fullRepo, pr.Number, pr.Author, pr.HeadSHA, pr.BodyDigest) + prs = append(prs, item) + } + // PRs a review webhook just fired for go first, so the early eval cycle + // it triggered spends its review budget on them (hivecommons/hive#11091). + prs = prioritizeEventPRs(prs, activeReviewEvents.Load()) + agents := make([]review.AgentCapability, 0, len(cfg.Agents)) + for name, ac := range cfg.EnabledAgents() { + cap := review.CapabilityFromConfig(name, ac) + cap.Paused = cap.Paused || (agentMgr != nil && agentMgr.IsPaused(name)) + cap.Draining = reviewerAutoscaler.Draining(name) + agents = append(agents, cap) + } + reviewBotLogins := []string{} + if reviewBots.Enabled() { + reviewBotLogins = reviewBots.Logins + } + plan := review.PlanDispatch(prs, artifact, state, review.DispatchOptions{ + RequireApproval: cfg.Review.RequireApproval, + FanOut: cfg.Review.FanOut, + MaxParallelReviews: cfg.Review.EffectiveMaxParallelReviews(), + MaxPerspectivesPerPR: cfg.Review.MaxPerspectivesPerPR, + Perspectives: reviewPerspectiveSet(cfg, logger), + CombinedPerspectives: cfg.Review.CombinedPerspectives, + ReviewerAgents: cfg.Review.ReviewerAgents, + FixerAgent: cfg.Review.FixerAgent, + PostComments: cfg.Review.PostComments, + WritingGuideSection: cfg.Project.WritingGuideSection(), + FixHumanPRs: cfg.Review.FixHumanPRsEnabled(), + AcknowledgeNoFindings: cfg.Review.AcknowledgeNoFindings, + ReviseRepos: cfg.Review.ReviseRepos, + ReviseVerdictsBefore: parseReviseCutoff(cfg.Review.ReviseVerdictsBefore, logger), + PendingTTL: cfg.Review.PendingReviewTTL(), + PendingFixTTL: cfg.Review.PendingReviewFixTTL(), + MaxRedispatch: cfg.Review.EffectiveMaxRedispatch(), + HeadMaxRetries: cfg.Review.HeadMaxRetries, + ProjectOrg: cfg.Project.Org, + AIAuthor: cfg.EffectiveAIAuthor(), + Agents: agents, + ReviewBotLogins: reviewBotLogins, + ReviewBotMinPriority: reviewBots.MinPriority, + PerspectiveModels: cfg.Review.PerspectiveModels, + BotAuthorRoute: botRoute, + DeltaRereview: cfg.Review.DeltaRereviewEnabled(), + DeltaTrivialLines: cfg.Review.EffectiveDeltaTrivialLines(), + DeltaMaxAge: cfg.Review.EffectiveDeltaMaxAge(), + }) + recordReviewDeltaRestamps(plan, logger) + for _, k := range plan.ReviewKicks { + if k.BotAuthorRouted { + logger.Info("review routed bot-authored PR to bot_author_model", "repo", k.Repo, "pr", k.Number, "head", k.HeadSHA, "agent", k.Agent, "review_model", k.ReviewModel, "review_backend", k.ReviewBackend) + } + } + for _, sk := range plan.SkippedReviewModels { + logger.Info("review model fallback skipped perspective", "repo", sk.Repo, "pr", sk.Number, "head", sk.HeadSHA, "perspective", sk.Perspective, "agent", sk.Agent, "author_model", sk.AuthorModel) + } + expiredNow := map[string]struct{}{} + for _, e := range plan.Expired { + // Warn once per PR while it keeps expiring (#13687). + key := prWarnKey(e.Repo, e.Number) + expiredNow[key] = struct{}{} + logger.Log(context.Background(), reviewExpiredWarn.level(key), "review pending expired", "repo", e.Repo, "pr", e.Number, "head", e.HeadSHA, "perspective", e.Perspective, "agent", e.Agent, "age", e.Age.Round(time.Second).String(), "expiries", e.Count) + } + reviewExpiredWarn.retain(expiredNow) + for _, h := range plan.Exhausted { + logger.Warn("review dispatch exhausted; holding PR for a human", "repo", h.Repo, "pr", h.Number, "head", h.HeadSHA, "reason", h.Reason) + } + for _, f := range plan.ForcedFixes { + logger.Warn("review fix forced: changes_requested head starved by the revisit path", "repo", f.Repo, "pr", f.Number, "head", f.HeadSHA, "reason", f.Reason, "cycles", f.Cycles, "limit", review.RevisitStarvationCycles) + } + for _, w := range plan.Promoted { + logger.Warn("review dispatch promoted: head starved of a reviewer walked ahead of the queue", "repo", w.Repo, "pr", w.Number, "head", w.HeadSHA, "reason", w.Reason, "cycles", w.Cycles, "since", w.FirstAt, "limit", review.ReviewWaitStarvationCycles) + } + for _, w := range plan.Waiting { + logger.Info("review dispatch waiting: perspectives still owed, no reviewer this cycle", "repo", w.Repo, "pr", w.Number, "head", w.HeadSHA, "reason", w.Reason) + } + if len(plan.ReviewKicks)+len(plan.FixKicks)+len(plan.FixExclusions) > 0 { + logger.Info("review swarm dispatch planned", "review_kicks", len(plan.ReviewKicks), "fix_kicks", len(plan.FixKicks), "fix_excluded", len(plan.FixExclusions), "fix_excluded_reasons", fixExclusionReasonSummary(plan.FixExclusions)) + } + return plan +} + +// fixExclusionReasonSummary renders the cycle's fix exclusions as +// "reason=count,..." (sorted) for the dispatch summary line, so fix_kicks: 0 +// is never printed without saying why (hivecommons/hive#14665). +func fixExclusionReasonSummary(exclusions []review.FixDispatchExclusion) string { + if len(exclusions) == 0 { + return "" + } + counts := map[string]int{} + for _, x := range exclusions { + counts[x.Reason]++ + } + reasons := make([]string, 0, len(counts)) + for r := range counts { + reasons = append(reasons, r) + } + sort.Strings(reasons) + parts := make([]string, 0, len(reasons)) + for _, r := range reasons { + parts = append(parts, fmt.Sprintf("%s=%d", r, counts[r])) + } + return strings.Join(parts, ",") +} + +// recordReviewDeltaRestamps writes the verdicts the planner carried forward +// onto a new head without a session (trivial delta, review.delta_rereview) +// into the verdict artifact, and logs each one. A write failure is logged, +// not fatal: the next cycle plans the same restamp again. +func recordReviewDeltaRestamps(plan review.DispatchPlan, logger *slog.Logger) { + if len(plan.DeltaRestamps) == 0 { + return + } + for _, r := range plan.DeltaRestamps { + note := "" + if r.Aggregate.Delta != nil { + note = r.Aggregate.Delta.Note + } + logger.Info("review delta re-stamped verdict without a session", "repo", r.Repo, "pr", r.Number, "head", r.HeadSHA, "previous_head", r.PreviousHead, "verdict", r.Aggregate.Verdict, "note", note) + } + if _, err := review.RecordDeltaRestamps("", plan.DeltaRestamps, time.Now().UTC()); err != nil { + logger.Warn("failed to record review delta re-stamps", "error", err, "count", len(plan.DeltaRestamps)) + } +} + +// reviewPlannerGitHubFor returns the GitHub client as a reviewPlannerGitHub, +// or nil for a nil client: a typed nil in the interface would not read as nil. +func reviewPlannerGitHubFor(c *github.Client) reviewPlannerGitHub { + if c == nil { + return nil + } + return c +} + +func refreshReviewVerdicts(cfg *config.Config, logger *slog.Logger) { + if cfg == nil || !shouldRefreshReviewVerdicts(cfg) { + return + } + artifact, err := review.CollectAndMerge("", "", review.AggregateOptions{ + // Unanimity is judged against what a PR was eligible to receive. Without + // this the cap makes approve unreachable and every PR aggregates to + // requires_human. + MaxPerspectivesPerPR: cfg.Review.MaxPerspectivesPerPR, + Perspectives: reviewPerspectiveSet(cfg, logger), + SelfAuthoredDecisionPolicy: cfg.Review.EffectiveSelfAuthoredDecisionPolicy(), + SensitivePaths: cfg.Security.SensitivePaths, + ModelStamps: reviewModelStamps(logger), + ExpectedHeads: cfg.ExpectedReviewHeads(), + MissingHeads: reviewMissingHeads(), + }, time.Now().UTC()) + if err != nil { + if !os.IsNotExist(err) { + logger.Warn("failed to refresh review verdicts", "error", err) + } + return + } + logger.Info("review verdict artifact refreshed", "aggregates", len(artifact.Items)) +} + +// reviewModelStamps reads the dispatch state so a verdict whose report +// omitted author_model / review_model is stamped with what the planner knew +// when it dispatched that review (hivecommons/hive#13243). No state is no +// stamps, never an error: the artifact is still correct, just unstamped. +func reviewModelStamps(logger *slog.Logger) review.ModelStamps { + state, err := review.LoadDispatchState("") + if err != nil { + if !os.IsNotExist(err) { + logger.Warn("review dispatch state unavailable for model stamping", "error", err) + } + return nil + } + return review.ModelStampsFromState(state) +} + +func shouldRefreshReviewVerdicts(cfg *config.Config) bool { + if cfg == nil { + return false + } + if cfg.Review.RequireApproval { + return true + } + entries, err := os.ReadDir(review.ReportDir("")) + if err != nil { + return false + } + for _, entry := range entries { + name := entry.Name() + if !entry.IsDir() && strings.HasPrefix(name, review.ReviewReportFilePrefix) && strings.HasSuffix(name, review.ReviewReportFileSuffix) { + return true + } + } + return false +} + +// reviewOutcomeResolveCap bounds how many vanished PRs one eval cycle asks +// GitHub about. A hive pointed at a busy org can see dozens of PRs merge +// between cycles; the rest are resolved next cycle, and an unresolved PR +// reads as still open, never as merged. +const reviewOutcomeResolveCap = 40 + +// prStateGetter is the slice of the GitHub client the outcome ledger needs. +type prStateGetter interface { + GetPRState(ctx context.Context, repo string, number int) (github.PRState, error) +} + +// reviewEvidenceOutcomeRecorder is the optional slice of the GitHub client +// that stamps a resolved outcome onto the PR's evidence bundle, so a PR this +// hive reviewed but did not merge stops reading as "outcome unknown". +type reviewEvidenceOutcomeRecorder interface { + RecordReviewEvidenceOutcome(repo string, number int, st github.PRState, now time.Time) +} + +// recordReviewOutcomes folds this cycle's open-PR enumeration into the +// review-outcome ledger (pkg/review/outcomes.go): every open PR is upserted, +// review evidence from the verdict artifact and the posted-review ledger is +// attached, and PRs that left the list are resolved to merged/closed with one +// GET each. It is what lets the dashboard say whether reviewed PRs merge +// faster than unreviewed ones on THIS hive, rather than counting verdicts. +func recordReviewOutcomes(ctx context.Context, cfg *config.Config, gh prStateGetter, actionable *github.ActionableResult, logger *slog.Logger) { + if cfg == nil || actionable == nil { + return + } + recordReviewOutcomesAt(ctx, cfg, gh, actionable, "", time.Now().UTC(), logger) +} + +func recordReviewOutcomesAt(ctx context.Context, cfg *config.Config, gh prStateGetter, actionable *github.ActionableResult, path string, now time.Time, logger *slog.Logger) { + ledger, err := review.LoadOutcomeLedger(path) + if err != nil { + logger.Warn("review outcomes: ledger unreadable, starting fresh", "error", err) + } + aiAuthor := cfg.EffectiveAIAuthor() + open := make([]review.OpenPR, 0, len(actionable.PRs.Items)) + for _, pr := range actionable.PRs.Items { + full := config.QualifyRepo(cfg.Project.Org, pr.Repo) + open = append(open, review.OpenPR{ + Repo: full, + Number: pr.Number, + Author: pr.Author, + AgentAuthored: pr.HiveAttributed || pr.AppAuthored || (aiAuthor != "" && strings.EqualFold(pr.Author, aiAuthor)), + CreatedAt: pr.CreatedAt, + }) + } + + // Review evidence: the verdict artifact knows the verdict; the links + // ledger knows a review was posted even when its verdict could not bind + // (queue-lane reviews of undispatched PRs). Earliest time wins. + signals := map[string]review.ReviewSignal{} + if art, err := review.LoadArtifact(""); err == nil { + for _, it := range art.Items { + key := it.Repo + "#" + strconv.Itoa(it.Number) + prev, ok := signals[key] + if !ok || (!it.RecordedAt.IsZero() && it.RecordedAt.Before(prev.At)) { + signals[key] = review.ReviewSignal{At: it.RecordedAt, Verdict: it.Verdict} + } else if ok && prev.Verdict == "" { + prev.Verdict = it.Verdict + signals[key] = prev + } + } + } + if links, err := github.LoadReviewLinks(""); err == nil { + for key, link := range links { + at := link.At + if at.IsZero() { + continue + } + prev, ok := signals[key] + if !ok { + signals[key] = review.ReviewSignal{At: at} + } else if at.Before(prev.At) { + prev.At = at + signals[key] = prev + } + } + } + + missing := ledger.Observe(now, open, signals) + resolved := 0 + if gh != nil { + for i, m := range missing { + if i >= reviewOutcomeResolveCap { + break + } + st, err := gh.GetPRState(ctx, m.Repo, m.Number) + if err != nil { + logger.Debug("review outcomes: could not resolve vanished PR", "repo", m.Repo, "number", m.Number, "error", err) + continue + } + ledger.Resolve(m.Repo, m.Number, st.State, st.MergedAt, st.ClosedAt) + resolved++ + if rec, ok := gh.(reviewEvidenceOutcomeRecorder); ok { + rec.RecordReviewEvidenceOutcome(m.Repo, m.Number, st, now) + } + } + } + ledger.Snapshot(now) + if err := ledger.Save(path, now); err != nil { + logger.Warn("review outcomes: save failed", "error", err) + return + } + if len(missing) > 0 { + logger.Info("review outcomes recorded", "open", len(open), "left_queue", len(missing), "resolved", resolved) + } +} + +// reviewFixAuditor is the slice of the dashboard the withheld-fix audit +// needs, so the function is testable without a live server. +type reviewFixAuditor interface { + AuditLog(user, action, detail, agent string) +} + +// auditWithheldReviewFixes puts every fix the planner refused this cycle on +// the audit trail (#8421). A refusal is silent by design — the PR just does +// not get a bot commit — and silence is the one thing an operator cannot +// investigate, so each one names the PR, who opened it, and the +// review.fix_human_prs value at the time (on no longer permits the push: +// hivecommons/hive#12897). The planner records the exclusion and +// the withheld fix only on the cycle that first withholds a head (a persisted +// State.Withheld entry suppresses later cycles), so this does not repeat on +// every eval cycle. +func auditWithheldReviewFixes(plan review.DispatchPlan, auditor reviewFixAuditor, logger *slog.Logger) { + // One WARN per changes_requested head that ends the cycle without a fix + // kick: a PR shown as "Fixing" that nobody is fixing must be loud + // (hivecommons/hive#14665). + for _, x := range plan.FixExclusions { + level := slog.LevelWarn + if x.Reason == review.FixDispatchExcludedNotHiveAuthored || x.Reason == review.FixDispatchExcludedNotHiveAuthoredDCO { + // Policy skip (the hive never pushes to PRs it did not author), not a fault. + level = slog.LevelInfo + } + logger.Log(context.Background(), level, "review fix dispatch excluded", + "repo", x.Repo, "pr", x.Number, "head_sha", x.HeadSHA, "author", x.Author, "reason", x.Reason, "attempts", x.Attempts) + } + for _, w := range plan.WithheldFixes { + logger.Info("review fix withheld: PR not hive-authored; review published, nothing pushed", + "repo", w.Repo, "pr", w.Number, "head_sha", w.HeadSHA, "author", w.Author, "setting", w.Setting, "setting_enabled", w.SettingEnabled) + if auditor == nil { + continue + } + auditor.AuditLog("governor", "review_fix_withheld", + fmt.Sprintf("pr=%s#%d head=%s author=%s setting=%s=%t reason=%s", w.Repo, w.Number, w.HeadSHA, w.Author, w.Setting, w.SettingEnabled, w.Reason), "") + } +} + +// reviewMissingHeads reads the review heads that ran out of retries while +// another head answered (hivecommons/hive#13542), so their perspectives count +// as settled for the decision step. No state is none. +func reviewMissingHeads() map[string][]string { + state, err := review.LoadDispatchState("") + if err != nil { + return nil + } + return review.MissingHeadsFromState(state) +} + +// reviewCommentPoster is the GitHub write postReviewAcknowledgements needs. +type reviewCommentPoster interface { + CreateIssueCommentBudgeted(ctx context.Context, repo string, number int, body string) error +} + +// postReviewAcknowledgements posts the one-line no-findings comment for each +// PR head the review-heads skip rule settled this cycle: every head approved +// every perspective with no findings, so no decision session ran to post it +// (hivecommons/hive#13542). The planner records each head once, so a failed +// post is logged rather than repeated. +func postReviewAcknowledgements(ctx context.Context, poster reviewCommentPoster, plan review.DispatchPlan, logger *slog.Logger) { + if len(plan.Acknowledgements) == 0 || poster == nil { + return + } + for _, a := range plan.Acknowledgements { + if err := poster.CreateIssueCommentBudgeted(ctx, a.Repo, a.Number, a.Body); err != nil { + logger.Warn("review heads: could not post the no-findings acknowledgement", "repo", a.Repo, "pr", a.Number, "head_sha", a.HeadSHA, "error", err) + continue + } + logger.Info("review heads: posted the no-findings acknowledgement", "repo", a.Repo, "pr", a.Number, "head_sha", a.HeadSHA) + } +} + +func persistReviewDispatchState(plan review.DispatchPlan, delivered []review.DispatchKick, logger *slog.Logger) { + writeReviewDispatchState(plan, func(planned []review.DispatchKick) review.DispatchState { + return review.ConfirmDelivered(plan.State, planned, delivered) + }, logger) +} + +// persistReviewDispatchStateInProgress is the persist made while the kick pass +// is still delivering: it keeps the pending rows of planned kicks not yet +// confirmed, so a verdict from a kick still in flight binds to its dispatch +// (hivecommons/hive#15559). The end-of-pass persistReviewDispatchState drops +// the kicks that never landed. +func persistReviewDispatchStateInProgress(plan review.DispatchPlan, delivered []review.DispatchKick, logger *slog.Logger) { + writeReviewDispatchState(plan, func([]review.DispatchKick) review.DispatchState { + return review.ConfirmDeliveredInProgress(plan.State, delivered) + }, logger) +} + +func writeReviewDispatchState(plan review.DispatchPlan, confirm func(planned []review.DispatchKick) review.DispatchState, logger *slog.Logger) { + planned := append(append([]review.DispatchKick(nil), plan.ReviewKicks...), plan.FixKicks...) + if plan.State.GeneratedAt.IsZero() && len(planned) == 0 { + return + } + state := confirm(planned) + if err := review.WriteDispatchState("", state); err != nil { + logger.Warn("failed to persist review dispatch state", "error", err) + } +} + +func atomicWrite(path string, data []byte) { + tmp := path + ".tmp" + if os.WriteFile(tmp, data, 0o644) == nil { + _ = os.Rename(tmp, path) + } +} diff --git a/src/docs/design/agent-turn-model.md b/src/docs/design/agent-turn-model.md index 65367227b0..b348fe42c6 100644 --- a/src/docs/design/agent-turn-model.md +++ b/src/docs/design/agent-turn-model.md @@ -136,12 +136,12 @@ Kick *timing* lives in `pkg/governor`; kick *text* is built in `pkg/scheduler`. - `Governor.Evaluate` (`src/pkg/governor/governor.go:377`) calls it at `src/pkg/governor/governor.go:485` and returns the due list. - The driving loop is a single ticker in `main`: - `deps.newTicker(initialEvalInterval)` at `src/cmd/hive/main.go:5889`, loop at - `src/cmd/hive/main.go:6126`, evaluation via `deps.runEval` at - `src/cmd/hive/main.go:6045` (`runEvalCycle`, `src/cmd/hive/main.go:6710`), - message assembly via `sched.BuildKickMessagesWithSkips` at `src/cmd/hive/main.go:7138` + `deps.newTicker(initialEvalInterval)` at `src/cmd/hive/main.go:5772`, loop at + `src/cmd/hive/main.go:6112`, evaluation via `deps.runEval` at + `src/cmd/hive/main.go:5956` (`runEvalCycle`, `src/cmd/hive/main_eval_cycle.go:471`), + message assembly via `sched.BuildKickMessagesWithSkips` at `src/cmd/hive/main_eval_cycle.go:964` (`src/pkg/scheduler/kickmessage.go:261`), and delivery via - `agentMgr.SendKick` wired into `dispatchAgentKicks` at `src/cmd/hive/main.go:7241`. + `agentMgr.SendKick` wired into `dispatchAgentKicks` at `src/cmd/hive/main_eval_cycle.go:1127`. This matters for the RFC: the scheduler is already **stateless with respect to turns**. It does not hold a continuation, does not await turn *N* before diff --git a/src/docs/design/master-key-rotation.md b/src/docs/design/master-key-rotation.md index b13c668ccb..236a8dba6b 100644 --- a/src/docs/design/master-key-rotation.md +++ b/src/docs/design/master-key-rotation.md @@ -361,7 +361,7 @@ empty master. It is not a bug. Those three sites are inside `spokeDomainKey`, `SpokeHeartbeatKey`, and `SpokeSSOPublicKey` — all SPOKE-side resolvers. The `hive` binary serves both roles but selects between them at startup: -`runHub()` (`src/cmd/hive/main.go:8000`) is the only caller of +`runHub()` (`src/cmd/hive/main.go:6899`) is the only caller of `hub.NewHubServer`, and it is a distinct mode from the spoke path. On a hub pod those three functions are never called, so the empty `HIVE_HUB_SECRET` they would read is never consulted. diff --git a/src/docs/duplicate-sweep.md b/src/docs/duplicate-sweep.md index 0b15697715..46e996248d 100644 --- a/src/docs/duplicate-sweep.md +++ b/src/docs/duplicate-sweep.md @@ -79,7 +79,7 @@ duplicate_sweep: says anything on a contributor's PR. `post_comments` has no effect without it. The sweep runs at most once per hour (`duplicateSweepInterval`, -`src/cmd/hive/main.go`) and its comments are idempotent: each cluster carries an +`src/cmd/hive/main_sweeps.go`) and its comments are idempotent: each cluster carries an invisible marker, so a later pass edits its own prior suggestion in place — and elides the write entirely when the body is unchanged — rather than stacking a fresh comment every hour. @@ -98,4 +98,4 @@ fresh comment every hour. Source: `src/pkg/dupsweep` (clustering and rendering, no GitHub dependency), `src/pkg/github/duplicate_sweep.go` (the scan and the comment path), -`runDuplicateSweepIfDue` in `src/cmd/hive/main.go` (the cadence). +`runDuplicateSweepIfDue` in `src/cmd/hive/main_sweeps.go` (the cadence). diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index e9e9295e4f..1232fcbb16 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -190,7 +190,7 @@ change the agents' model — not a repeated runtime defect The re-engagement sweep logs `re-engagement kick undeliverable; budget not charged` at INFO rather than WARN when the cause is `shared session pool is -busy` (`deliverReEngagement`, `src/cmd/hive/main.go`): a saturated shared CLI +busy` (`deliverReEngagement`, `src/cmd/hive/main_sweeps.go`): a saturated shared CLI session pool refusing another session is backpressure. The kick can be retried on a later sweep and consumes no repair budget, so a burst across PRs is not a repeated runtime defect (hivecommons/hive#15350). Other undeliverable-kick diff --git a/src/docs/notifications.md b/src/docs/notifications.md index 19791d42d8..80788a0fff 100644 --- a/src/docs/notifications.md +++ b/src/docs/notifications.md @@ -137,14 +137,14 @@ The Hive Go process sends notifications for these events: | Event | Priority | Source | |---|---|---| -| Weekly token budget crosses the warning threshold. | default | `applyBudgetAlerts` in `src/cmd/hive/main.go` | -| Weekly token budget is exhausted and non-exempt kicks are suspended. | high | `applyBudgetAlerts` in `src/cmd/hive/main.go` | -| Actionable issues exceed the SLA threshold; up to three `SLA 2x breach` notifications are sent per eval cycle for issues older than 60 minutes (`maxSLANotificationsPerCycle`, `doubleSLAMinutes`). | high | `selectSLABreachNotifications` in `src/cmd/hive/eval_cycle_seams.go`, sent from `runEvalCycle` in `src/cmd/hive/main.go` | +| Weekly token budget crosses the warning threshold. | default | `applyBudgetAlerts` in `src/cmd/hive/main_eval_cycle.go` | +| Weekly token budget is exhausted and non-exempt kicks are suspended. | high | `applyBudgetAlerts` in `src/cmd/hive/main_eval_cycle.go` | +| Actionable issues exceed the SLA threshold; up to three `SLA 2x breach` notifications are sent per eval cycle for issues older than 60 minutes (`maxSLANotificationsPerCycle`, `doubleSLAMinutes`). | high | `selectSLABreachNotifications` in `src/cmd/hive/eval_cycle_seams.go`, sent from `runEvalCycle` in `src/cmd/hive/main_eval_cycle.go` | | A running agent pane matches a configured login-required pattern; Hive pauses that agent and sends the backend-specific login instruction (`Login required: `). | high | `scanForLoginRequired` in `src/cmd/hive/main.go` | | Trajectory review detects divergence and either pauses the agent or flags the divergence. | high | `src/pkg/dashboard/trajectory_sink.go` and `src/pkg/trajectory/lane.go` | -| The inference provider starts rebuffing kicks with spending-limit errors; sent once per crossing of the latch, not per cycle (`Provider spending limit reached`). | high | provider-budget kick gate in `runEvalCycle`, `src/cmd/hive/main.go` | -| A probe kick succeeds after a spending-limit clip and agent kicks resume (`Provider spending limit lifted`). | default | provider-budget recovery path in `runEvalCycle`, `src/cmd/hive/main.go` | -| A PR stays red after exhausting its automated fix-attempt budget and is escalated to a human (`Fix loop escalated`). | high | `runEscalationSweep` in `src/cmd/hive/main.go` | +| The inference provider starts rebuffing kicks with spending-limit errors; sent once per crossing of the latch, not per cycle (`Provider spending limit reached`). | high | provider-budget kick gate in `runEvalCycle`, `src/cmd/hive/main_eval_cycle.go` | +| A probe kick succeeds after a spending-limit clip and agent kicks resume (`Provider spending limit lifted`). | default | provider-budget recovery path in `runEvalCycle`, `src/cmd/hive/main_eval_cycle.go` | +| A PR stays red after exhausting its automated fix-attempt budget and is escalated to a human (`Fix loop escalated`). | high | `runEscalationSweep` in `src/cmd/hive/main_sweeps.go` | | The planning stall-replan lane re-kicks the architect on a stalled plan (`Plan replan`) or hits the replan cap (`Plan replan-cap`). | high | `src/pkg/planning/replan.go` via `src/pkg/dashboard/replan_sink.go` | | The convergence rollout mode changes at runtime (settings PUT, YAML reload, or env override); sent once per transition, not per cycle (`Convergence mode changed`). | default | `applyConvergenceKickAdmission` in `src/cmd/hive/convergence_kick.go` | diff --git a/src/docs/running-at-level-6.md b/src/docs/running-at-level-6.md index 54c14e0377..123286970d 100644 --- a/src/docs/running-at-level-6.md +++ b/src/docs/running-at-level-6.md @@ -334,7 +334,7 @@ config**; no per-hive dashboard/YAML override exists. **Limit:** the plan sink reads L6's pack directly and allows child tasks without human plan approval. Pack approval is not design correctness. Evidence: `PlanAutoApproveForLevel` in [pack config](../pkg/config/acmm_packs.go), -`planFromLabeledIssues` in [runtime wiring](../cmd/hive/main.go). +`planFromLabeledIssues` in [runtime wiring](../cmd/hive/main_eval_cycle.go). #### H10. Keep input scanning fail-closed @@ -546,7 +546,7 @@ These are confirmations of the practices above, not additional product gates. | One repo: Auto-merge off | All Hive merge paths for that repo; PR creation can continue. Existing PRs **cannot** merge through Hive. Repo Pause additionally removes repo work enumeration/sweep activity. | Owner or repo-write user may switch off/pause; **owner** required to restore auto-merge. Resume repo and separately restore chosen merge policy. Confirm card + refused test merge. [Repo API](../pkg/dashboard/api_repo_pause.go), [repo policy](../pkg/config/repo_policy.go). | | One agent: Pause | That worker's execution/scheduling; other workers and App sweep remain. Existing green PRs **can** still merge. | Dashboard owner; Resume reverses manual pause. Confirm agent paused state. [Agent control](../pkg/dashboard/api_agent_control.go), [pause implementation](../crux/agent/manager_pause.go). | | Every agent: fleet breaker, plus explicitly pause any remaining on-demand workers | Breaker pauses currently running non-on-demand agents, preserving already-paused states; it is **not a merge brake**. Existing PRs **can** merge via sweep. | Owner, using the fleet breaker button in the dashboard's top bar. Release resumes only breaker-owned pauses; manually paused agents stay paused. Confirm captured set and each agent status. [Breaker API](../pkg/dashboard/api_breaker.go), [pause implementation](../crux/agent/manager_pause.go). | -| All automatic merging: disable Auto-merge on every repo, or step down below L6 | Repo gate blocks Hive merge authority on existing and new PRs. Workers can still create work according to the remaining level/modes. Disabling only `auto_merge.self_authored` stops **only the App sweep**, not the scanner lane; restarting relays may be necessary for startup-captured sweep flags. | Owner for level change; owner/repo-write for repo off. Restore selected repo switches as owner; re-applying L6 enables active repos again. Confirm each repository card and the **Audit Log**. [Repo policy](../pkg/config/repo_policy.go), [pack API](../pkg/dashboard/api_packs.go), [sweep wiring](../cmd/hive/main.go). | +| All automatic merging: disable Auto-merge on every repo, or step down below L6 | Repo gate blocks Hive merge authority on existing and new PRs. Workers can still create work according to the remaining level/modes. Disabling only `auto_merge.self_authored` stops **only the App sweep**, not the scanner lane; restarting relays may be necessary for startup-captured sweep flags. | Owner for level change; owner/repo-write for repo off. Restore selected repo switches as owner; re-applying L6 enables active repos again. Confirm each repository card and the **Audit Log**. [Repo policy](../pkg/config/repo_policy.go), [pack API](../pkg/dashboard/api_packs.go), [sweep wiring](../cmd/hive/main_sweeps.go). | | Contributor work: Suspend contributions, revoke/stop active relays if necessary | Stops offering/admitting contribution work as configured, not Hive's other workers or automatic merges. Existing contributor PRs **can** merge; active work must be handled separately. | Owner in contribution admin; contributor can stop their own relay. Unsuspend/re-authorize only after triage; confirm queue paused and active leases/workers accounted for. [Contribution API](../pkg/dashboard/api_contribute.go), [relay reference](contributor-relay.md). | These controls stop **Hive**, not a separate GitHub bot or a maintainer manually merging. For an immediate repo-wide incident boundary, a GitHub administrator can also revoke/suspend the App installation's access; coordinate restoration and verify credential failures, rather than silently weakening branch rules. **JUDGMENT CALL:** use this larger boundary if Hive's runtime state cannot be trusted. No Hive setting enforces that incident policy. diff --git a/src/docs/troubleshooting.md b/src/docs/troubleshooting.md index b672bdbb1a..a94e34ef0b 100644 --- a/src/docs/troubleshooting.md +++ b/src/docs/troubleshooting.md @@ -137,7 +137,7 @@ podman exec hive hive validate # Podman ## GitHub credentials are missing or invalid -When no token or App credentials are usable, Hive starts the dashboard but disables write-capable GitHub work. The code logs these exact messages from `src/cmd/hive/main.go` depending on the state: +When no token or App credentials are usable, Hive starts the dashboard but disables write-capable GitHub work. The code logs these exact messages from `src/cmd/hive/main_github_app.go` depending on the state: - `no GitHub token configured (set github.token or github.app_id in config) — starting in dashboard-only mode` - `GitHub App configured without credentials — hive starting in dashboard-only mode. Install the app and provide installation_id + key to enable agents.` diff --git a/src/pkg/github/automerge/f3_trusted_merger_source_test.go b/src/pkg/github/automerge/f3_trusted_merger_source_test.go index 075dff93d4..f56be045a0 100644 --- a/src/pkg/github/automerge/f3_trusted_merger_source_test.go +++ b/src/pkg/github/automerge/f3_trusted_merger_source_test.go @@ -191,17 +191,17 @@ func TestF3IsTrustedMergerFailsClosedInSource(t *testing.T) { // silently stops merging — or, if the fail-closed branch were also lost, merges // everything. func TestF3AuthorizerIsWiredInMain(t *testing.T) { - src := f3ReadSource(t, "../../../cmd/hive/main.go") + pkg := f3ReadPackage(t, "../../../cmd/hive") // Whitespace-tolerant: gofmt column-aligns the key when sibling fields // (TrustedAuthorizer, TrustedAuthorPolicy) are longer. - if !regexp.MustCompile(`MergerAuthorizer:\s+trustedMergerFunc\(cfg\)`).MatchString(src) { + if !regexp.MustCompile(`MergerAuthorizer:\s+trustedMergerFunc\(cfg\)`).MatchString(pkg) { t.Error("cmd/hive main() does not install the trusted-merger authorizer — the F3 gate in " + "trySweepQueuedPR is present but INERT (audit F3, standing). Restore " + "MergerAuthorizer: trustedMergerFunc(cfg) beside StartMergeRequestWatcher.") } - body := f3FuncBody(t, f3ReadPackage(t, "../../../cmd/hive"), "func trustedMergerFunc(", "trustedMergerFunc") + body := f3FuncBody(t, pkg, "func trustedMergerFunc(", "trustedMergerFunc") if !strings.Contains(body, "config.RoleAtLeast(role, config.RoleMerger)") { t.Error("trustedMergerFunc no longer requires at least config.RoleMerger — the sweep would " + "admit a tier below the one the dashboard queue endpoint enforces (audit F3)") diff --git a/src/pkg/writeback/testdata/direct-writes-baseline.txt b/src/pkg/writeback/testdata/direct-writes-baseline.txt index 518755806b..8ec92dc4a1 100644 --- a/src/pkg/writeback/testdata/direct-writes-baseline.txt +++ b/src/pkg/writeback/testdata/direct-writes-baseline.txt @@ -55,7 +55,7 @@ wrap cmd/hive/claimescalation.go 2 wrap cmd/hive/claimswire.go 3 wrap cmd/hive/forgewire.go 2 wrap cmd/hive/github_client_provider.go 2 -wrap cmd/hive/main.go 6 +wrap cmd/hive/main_sweeps.go 6 wrap cmd/hive/review_queue.go 1 wrap cmd/hive/rfccoverage_wire.go 1 wrap pkg/convergence/publish/channel.go 1