diff --git a/.github/workflows/v2-ci.yml b/.github/workflows/v2-ci.yml index 7912717ed5..924e986d6f 100644 --- a/.github/workflows/v2-ci.yml +++ b/.github/workflows/v2-ci.yml @@ -42,6 +42,11 @@ concurrency: group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +# A slow cache-segment download must never cost more than a recompile +# (see v2-tests.yml — observed 16-minute restore steps on some nodes). +env: + ACTIONS_CACHE_SEGMENT_DOWNLOAD_TIMEOUT_MINS: '1' + jobs: build-and-test: runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} @@ -151,8 +156,12 @@ jobs: - name: Entrypoint UID-isolation migration (#5525) run: bash deploy/test_entrypoint_uid_isolation.sh - # #6287: the token-access audit log must not be agent-writable. The - # wrappers write only to the drop-box spool and no chmod may loosen it. + # #6287: the token-access audit log (GET /api/token-access) was + # dev:node 0664 while every agent UID is in group node, so the audited + # agents could truncate or forge their own trail. Fixed on v5 by #6303 + # and backported here by #7523. Pins the invariant on the shipped + # entrypoint and wrappers: the log is dev-owned 0600, the wrappers write + # only to the drop-box spool, and no chmod loosens it. - name: Entrypoint token-access audit log not agent-writable (#6287) run: bash deploy/test_entrypoint_token_access_audit.sh diff --git a/.github/workflows/v2-tests.yml b/.github/workflows/v2-tests.yml index a878ce078e..8126f34f46 100644 --- a/.github/workflows/v2-tests.yml +++ b/.github/workflows/v2-tests.yml @@ -75,6 +75,11 @@ env: # Self-hosted runners sit inside OpenShift; keep these hermetic unit tests from auto-discovering that cluster. KUBERNETES_SERVICE_HOST: '' KUBERNETES_SERVICE_PORT: '' + # A slow cache-segment download must never cost more than a recompile. + # Some cluster nodes crawl when pulling from the Actions cache backend — + # observed 16-minute "Restore warm Go build cache" steps against a ~2-minute + # cold compile. Abort a segment after 1 minute and let the job build instead. + ACTIONS_CACHE_SEGMENT_DOWNLOAD_TIMEOUT_MINS: '1' # Cancel superseded PR runs only (#4623). A new push to a PR makes the diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b27ea9398..edd71371cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,43 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-09-18 (v4.55.1) + +### Changed + +- `cmd/hive`'s `main.go` shed five self-contained domains into their own files — GitHub App key-file resolution (`appkeyfile.go`), self-upgrade marker/outcome bookkeeping (`selfupgrade.go`), login-required scanning and its sighting debounce (`login_scan.go`), auto-merge eligibility classification (`merge_eligibility.go`), and PR/issue intent verdicts (`intent_verdicts.go`) ([#7238](https://github.com/hivecommons/hive/issues/7238)). `main.go` drops from 9,795 to 8,072 lines. This is pure code motion — every one of the 242 top-level declarations was moved byte-identically, verified by a structural declaration diff, so no operator-visible behaviour changes; it makes the merge, upgrade and App-key paths reviewable on their own instead of only as a slice of a 9.8K-line file. + +### Fixed + +- Agent dots no longer show red (down) for 10-15+ minutes while the fleet is healthy ([#7526](https://github.com/hivecommons/hive/issues/7526)). `/api/status` — and the replay frame a reconnecting dashboard tab gets on the SSE stream — served a snapshot that was only ever refreshed at the end of an eval cycle, and that cycle enumerates every configured repo against the GitHub API first. On a spoke with sixteen repos under rate-limit backoff the cycle ran far past its nominal `eval_interval_s`, so agents the manager had started (and whose tmux sessions were alive and posting PR reviews) still read `state=stopped` and the whole fleet rendered as crashed. The existing 10-second agent-only tick — in-memory manager state, no GitHub calls — now also patches the agent block of the served snapshot, so liveness on every dashboard surface is at most one tick old regardless of how long the GitHub enumeration takes. +- Tests: the copilot SDK-helper probe tests in `pkg/dashboard` are now hermetic. `TestProbeCopilotModelsSDK_AbsentHelperYieldsSentinel` and `TestProbeCopilotModelsSDK_HelperNotInstalled` used to run whatever was installed at the production helper path, so on any host that ships `copilot-models.mjs` without stored copilot auth (every live agent host) the "absent helper" sentinel never fired and the whole `pkg/dashboard` suite went red with `Not authenticated`. The helper path is now a test-seam var (`setCopilotSDKHelperPathForTest`, mirroring the `knowledge.SetBaseDirForTest` convention) pointed at a temp path, and a new `TestProbeCopilotModelsSDK_FailingHelperIsNotAbsent` covers the previously untestable-on-CI third state — helper present but exiting nonzero (the #7365 case) — via a stub script instead of the real helper. + +## 2026-09-18 (v4.55.0) + +### Added + +- Added `review.max_perspectives_per_pr`, an opt-in cap on how many review perspectives one PR may be given per dispatch cycle. Without it the parallel review budget is spent in PR order, so the head of a deep queue absorbs every slot and adding reviewers buys more opinions on one PR instead of coverage across many. Capping spends the same budget breadth-first, and loses no coverage because skipped perspectives are still dispatched on later cycles. Zero (the default) keeps the existing fan-out behavior. +- Review comments now route: a `requires_human` or `reject` verdict opens with a pinned `**HUMAN DECISION NEEDED**` marker maintainers can filter on, mentions the PR author when that author is a person (never an app/bot account, which notifies nobody), and asks the reviewer to state plainly what it could not judge instead of approving around it. + +### Changed + +- Split six more domains out of `pkg/agent/manager.go` (consent, copilot auth, env, modes, routing, thrash) into per-domain files matching the v5 layout; manager.go shrinks from 4,321 to 1,793 lines. No behavior change — verbatim block moves. +- Hovering a blocked PR pill on the dashboard now names the branch-protection rule that is holding the PR, instead of GitHub's one-word `blocked` state ([#7515](https://github.com/hivecommons/hive/issues/7515)). GitHub folds every unsatisfied rule — a red required check, a required check that never ran, a missing approval, a reviewer who asked for changes — into that single word, so an operator had to open the PR on GitHub to learn which one, the trip the pill exists to save. The sweep now collects GitHub's own review decision (one GraphQL query per repository per cycle, not one per PR) and compares the base branch's required-check set against the check runs it already walks, and the tooltip reads `blocked — changes requested by @reviewer`, `blocked — required check "validate" has not reported`, or `blocked — required check "build" is failing`. When the facts on hand genuinely do not identify a rule the tooltip still says so plainly rather than guessing, and on a repository that reports its required contexts as commit statuses no "has not reported" claim is made at all. + +### Fixed + +- Hub: the `auth-url` / `auth-response-headers` annotations [#7457](https://github.com/hivecommons/hive/pull/7457) added to the `hive-contribute` Ingress now reach hosted spokes that were provisioned before that fix ([#7517](https://github.com/hivecommons/hive/issues/7517)). The provisioning template is applied only when a hive is created, so on every pre-existing spoke nginx never asked the hub who was calling and `/api/contribute/me` kept answering `401` to the hive's own signed-in owner — the #7453 symptom, on a spoke whose `served_sha` was well past the fix. A new 15-minute hub sweep (`contribute_ingress_reconcile.go`, alongside the NET_ADMIN and per-hive-env reconciles) reads each hosted spoke's live `hive-contribute` Ingress on nginx clusters and merge-patches the two annotations on when they are missing or stale; a converged Ingress is a no-op, the patch rolls no pod, and the expected values are pinned equal to what the template renders so the sweep and provisioning cannot disagree. OpenShift-Route and pull-only clusters are skipped. `src/docs/security-model.md` now records the general rule: a template change to an existing object needs a reconcile path or a re-provision note. + +### Security + +- The token-access audit log behind `GET /api/token-access` is no longer writable by the agents it audits ([#6287](https://github.com/hivecommons/hive/issues/6287)). The log records every gh CLI command and git credential lookup an agent makes and is gated at owner role for exactly that reason, yet it was fed by the audited parties: `bin/gh-wrapper.sh` and `bin/git-credential-hive.sh` run as the agent UID and appended straight into `/var/run/hive-metrics/token-access.jsonl`, which only works when the file is writable by every agent (dev:node 0664, and every agent's primary group is node). Append is indistinguishable from write at the permission level, so a compromised or prompt-injected agent could truncate the trail of its own token use, rewrite lines to blame a peer, or forge entries outright. The wrappers now drop one JSON event per call into `/var/run/hive-metrics/token-access-events`, a drop-box the agents can create files in but cannot list (0730, sticky, setgid), and the hive process ingests those events into the log, which is now hive-owned 0600 with no group or other bit at all. On ingest the hive replaces each event's self-reported `uid` with the uid that owns the event file, the same trust anchor the PR-request watcher uses, and keeps a disagreeing claim beside it as `claimed_uid`, so an entry forged in a peer's name lands attributed to the forger. Malformed or oversized events are rejected rather than appended. A 0664 log left by an older image is tightened on every boot and on every ingest pass instead of trusted. What remains open is narrower than before: an agent can still delete its own event during the two-second window before ingest, and can still flood the log with real calls; it can no longer alter anything already recorded. + +## 2026-09-18 (v4.54.2) + +### Fixed + +- Dashboard: hovering a PR pill that is not green now says, in plain words, why the sweep will not merge it and what would unblock it ([#7515](https://github.com/hivecommons/hive/issues/7515)). A PR GitHub reported as not mergeable used to hover as the bare GitHub enum — `not mergeable on GitHub (blocked)` — even though the merge-eligible classifier had just been handed the exact gate that made GitHub say blocked and then dropped it. `blocked` now keeps the sweep's reason (`blocked — CI failing: build, lint`, `blocked — awaiting review approval`, `blocked — held: …`, `blocked — intent verification: …`, `blocked — CI pending`), and when every sweep gate passes it says `blocked — all sweep gates pass; a branch-protection rule is unsatisfied` instead of nothing. The other states read as what to do rather than the API name: `has merge conflicts with v4 — needs a rebase`, `behind v4 — needs an update from the base branch`, `draft — mark ready for review to enter the sweep`, `mergeability not yet computed by GitHub — re-checked next tick`. The base branch comes from the PR list payload (`base_ref`, new on the status snapshot); no extra GitHub calls. Naming the exact branch-protection rule when the sweep itself has no reason (a review GitHub requires, a required check that never reported) is the issue's step 2 and still open. + ## 2026-09-18 (v4.54.1) ### Changed diff --git a/ROADMAP.md b/ROADMAP.md index a10e21e347..c195400985 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -122,6 +122,37 @@ operation during the transition, is part of the [v5 GA readiness bar](https://github.com/hivecommons/hive/blob/v5/src/docs/v5-ga.md) (live tracker: [#6016](https://github.com/hivecommons/hive/issues/6016)). +## v6 — Future Line (not open) + +There is no v6 branch, milestone, or release channel, and none is planned +until v5 reaches GA. "v6" is a **designation, not a line**: a label for +proposals that deliberately target the horizon after the current one, so +they can be written down without pretending they are scheduled. + +Policy for v6-designated work: + +- **Design-only.** A v6-designated proposal may land as a design document + under [`src/docs/design/`](src/docs/design/) after normal review. + Implementation PRs for v6-designated work are out of scope on every + current branch and should be closed with a pointer to this section. +- **The line opens after v5 GA.** Opening a v6 line is blocked on the + [v5 GA readiness bar](https://github.com/hivecommons/hive/blob/v5/src/docs/v5-ga.md) + closing (live tracker: + [#6016](https://github.com/hivecommons/hive/issues/6016)). When it opens, + it opens the same way v5 did — public RFC issues gate each workstream, + per [GOVERNANCE.md](GOVERNANCE.md) — and designs parked here re-enter + through that gate rather than being grandfathered in. +- **Designation is cheap and non-binding.** Marking a design "v6" records + intent and preserves the work; it is not acceptance, priority, or a + commitment that a v6 line will include it. + +v6-designated designs to date: + +- **GitHub @-mention triggers** — a human summons an agent by mentioning + the App on an issue or PR, mirroring the existing Linear inbound-mention + path ([#7483](https://github.com/hivecommons/hive/issues/7483), + [design doc](src/docs/design/github-mention-triggers.md)). + ## Hosted Hive Hub The hosted hub at [hive.hivecommons.dev](https://hive.hivecommons.dev) is the diff --git a/src/cmd/hive/appkeyfile.go b/src/cmd/hive/appkeyfile.go new file mode 100644 index 0000000000..6da429c665 --- /dev/null +++ b/src/cmd/hive/appkeyfile.go @@ -0,0 +1,68 @@ +package main + +import ( + "fmt" + "strings" + + "github.com/hivecommons/hive/pkg/apphealth" +) + +// deliveredKeyPath is where a hub-delivered private key for appID is stored. +// +// The filename NAMES the App, so a key can only ever be found under the App it +// was delivered for. The generic /data/gh-app-key.pem carries no such evidence: +// a key written there for one App silently becomes "the key" for whatever +// app_id the config later claims, which is how all 33 heartbeat-only-cluster spokes ended up +// signing as the public App with the GHE key and getting +// 404 Integration not found. +// +// Falls back to the generic path only when the delivery names no App, so a key +// is never dropped on the floor. +func deliveredKeyPath(appID int64) string { + if p := appKeys.PerAppIDKeyPath(appID); p != "" { + return p + } + return appKeys.DataKeyPath +} + +// describeAppKeyFailure turns a bare wrapped os error from github.NewAppAuth +// into a message an operator can act on without reading the source: it names +// the path actually tried, the full resolution order that produced it, and the +// underlying cause. +// +// The generic "reading app key /secrets/gh-app-key.pem: no such file" that this +// replaces gave no hint that key_file, $GH_APP_KEY_FILE, the PVC path and the +// provisioning mount are all consulted in a fixed order — so the usual response +// was to put the key in the wrong one of the four. +func describeAppKeyFailure(configured, envOverride, resolved string, err error) string { + order := []string{ + fmt.Sprintf("$GH_APP_KEY_FILE=%s", describeKeySource(envOverride)), + fmt.Sprintf("github.key_file=%s", describeKeySource(configured)), + fmt.Sprintf("per-app-id PVC key %s/gh-app-key-.pem", appKeys.DataDir), + fmt.Sprintf("per-app-id provisioning key %s/gh-app-key-.pem", appKeys.ProvisionedDir), + fmt.Sprintf("PVC fallback %s", appKeys.DataKeyPath), + fmt.Sprintf("provisioning mount %s", appKeys.ProvisionedKeyPath), + } + return fmt.Sprintf( + "GitHub App private key could not be loaded from %q: %v. "+ + "Resolution order (first non-empty wins): %s. "+ + "Write a PEM-encoded RSA private key to that path, or point github.key_file at one.", + resolved, err, strings.Join(order, " → "), + ) +} + +// describeKeySource renders an unset key-file source as "(unset)" so the +// resolution order in describeAppKeyFailure reads unambiguously. +func describeKeySource(v string) string { + if strings.TrimSpace(v) == "" { + return "(unset)" + } + return v +} + +// appKeyPaths snapshots the two App key path locations for a pkg/apphealth +// call. Read at call time on purpose: tests repoint these, and capturing them +// once would silently ignore that. +func appKeyPaths() apphealth.KeyPaths { + return apphealth.KeyPaths{Spoke: appKeys.DataKeyPath, Provisioned: appKeys.ProvisionedKeyPath} +} diff --git a/src/cmd/hive/intent_verdicts.go b/src/cmd/hive/intent_verdicts.go new file mode 100644 index 0000000000..de521f0928 --- /dev/null +++ b/src/cmd/hive/intent_verdicts.go @@ -0,0 +1,318 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "os" + "strings" + "time" + + gh "github.com/google/go-github/v72/github" + "github.com/hivecommons/hive/pkg/beads" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/github/automerge" + "github.com/hivecommons/hive/pkg/intent" +) + +func writeIntentVerdicts( + ctx context.Context, + cfg *config.Config, + ghClient *github.Client, + actionable *github.ActionableResult, + beadStores map[string]*beads.Store, + logger *slog.Logger, +) map[string]intent.Verdict { + verdicts := make(map[string]intent.Verdict) + if cfg == nil || actionable == nil { + return verdicts + } + _ = os.MkdirAll("/var/run/hive-metrics", 0o755) + aiAuthor := strings.TrimSpace(cfg.EffectiveAIAuthor()) + intentCfg := intentConfigFromCfg(cfg) + var alignmentReviewer *intent.AlignmentReviewer + if strings.TrimSpace(cfg.Intent.AlignmentModel) != "" { + endpoint, apiKey, _ := cfg.Governor.ResolveReviewer() + var err error + alignmentReviewer, err = intent.NewAlignmentReviewer(intent.AlignmentReviewerConfig{ + Endpoint: endpoint, + APIKey: apiKey, + Model: cfg.Intent.AlignmentModel, + }) + if err != nil { + logger.Warn("intent alignment reviewer disabled", "error", err) + } + } + type verdictRecord struct { + Repo string `json:"repo"` + Number int `json:"number"` + Title string `json:"title"` + Author string `json:"author"` + Enforced bool `json:"enforced"` + Verdict intent.Verdict `json:"verdict"` + Classify string `json:"classification_reason"` + FetchError string `json:"fetch_error,omitempty"` + } + records := make([]verdictRecord, 0, len(actionable.PRs.Items)) + for _, pr := range actionable.PRs.Items { + fullRepo := fullRepoName(pr.Repo, cfg.Project.Org) + key := fmt.Sprintf("%s/%d", fullRepo, pr.Number) + agentPR := aiAuthor != "" && strings.EqualFold(pr.Author, aiAuthor) + record := verdictRecord{ + Repo: fullRepo, + Number: pr.Number, + Title: pr.Title, + Author: pr.Author, + Enforced: cfg.Intent.Enforce, + } + if !agentPR { + class := intent.Classify(intent.PR{Title: pr.Title, Labels: pr.Labels, Author: pr.Author, AgentAuthor: false}, intentCfg) + verdict := intent.Evaluate(class, intent.Evidence{}) + verdicts[key] = verdict + record.Verdict = verdict + record.Classify = class.Reason + records = append(records, record) + continue + } + body, files, approved, err := fetchIntentPREvidence(ctx, ghClient, fullRepo, pr.Number) + if err != nil { + verdict := intent.Verdict{ + Tier: intent.Tier1, + Authorized: false, + Reason: "intent evidence unavailable: " + err.Error(), + AgentPR: true, + } + verdicts[key] = verdict + record.Verdict = verdict + record.FetchError = err.Error() + records = append(records, record) + logger.Warn("intent verification evidence fetch failed", "repo", fullRepo, "number", pr.Number, "error", err) + continue + } + class := intent.Classify(intent.PR{ + Title: pr.Title, + Body: body, + Labels: pr.Labels, + Files: files, + Author: pr.Author, + AgentAuthor: true, + }, intentCfg) + evidence := intent.BuildEvidenceForRepo(body, fullRepo, beadStores, approved) + verdict := intent.Evaluate(class, evidence) + issueTexts, issueErr := fetchIntentIssueTexts(ctx, ghClient, fullRepo, body) + if issueErr != nil { + logger.Warn("intent alignment issue evidence fetch failed", "repo", fullRepo, "number", pr.Number, "error", issueErr) + } + refs := intent.LinkedIssueRefs(body, fullRepo) + alignCtx := intent.BuildAlignmentContext(intent.PR{ + Title: pr.Title, + Body: body, + Labels: pr.Labels, + Files: files, + Author: pr.Author, + AgentAuthor: true, + }, issueTexts, beadStores, refs) + alignment := intent.EvaluateAlignment(alignCtx, class.Tier, intentCfg) + if alignmentReviewer != nil { + modelVerdict, err := alignmentReviewer.Review(ctx, alignCtx) + if err != nil { + logger.Warn("intent alignment model review failed open", "repo", fullRepo, "number", pr.Number, "error", err) + alignment = intent.MergeAlignment(alignment, nil, err) + } else { + alignment = intent.MergeAlignment(alignment, &modelVerdict, nil) + } + } + verdict.Alignment = &alignment + verdicts[key] = verdict + record.Verdict = verdict + record.Classify = class.Reason + records = append(records, record) + if !verdict.Authorized { + logger.Info("intent authorization denied", "repo", fullRepo, "number", pr.Number, "tier", verdict.Tier, "reason", verdict.Reason, "enforce", cfg.Intent.Enforce) + } + if alignment.Misaligned() { + logger.Info("intent alignment denied", "repo", fullRepo, "number", pr.Number, "reason", alignment.Rationale, "enforce", cfg.Intent.Enforce) + recordIntentAlignmentAdvisory(beadStores, fullRepo, pr.Number, alignment, logger) + } + } + payload := map[string]any{ + "generated_at": time.Now().UTC().Format(time.RFC3339), + "enforced": cfg.Intent.Enforce, + "verdicts": records, + } + if data, err := json.Marshal(payload); err == nil { + atomicWrite(intentVerdictsPath, data) + } else { + logger.Warn("failed to marshal intent verdicts", "error", err) + } + return verdicts +} + +func fetchIntentPREvidence(ctx context.Context, ghClient *github.Client, repo string, number int) (string, []intent.ChangedFile, bool, error) { + if ghClient == nil || ghClient.GoGitHub() == nil { + return "", nil, false, github.ErrNoGitHubClient + } + owner, repoName, ok := strings.Cut(repo, "/") + if !ok || owner == "" || repoName == "" { + return "", nil, false, fmt.Errorf("invalid repo %q", repo) + } + client := ghClient.GoGitHub() + pr, _, err := client.PullRequests.Get(ctx, owner, repoName, number) + if err != nil { + return "", nil, false, fmt.Errorf("getting PR: %w", err) + } + files, err := automerge.ListChangedFiles(ctx, client, owner, repoName, pr) + if err != nil { + return "", nil, false, err + } + approved, err := hasMaintainerApproval(ctx, client, owner, repoName, number) + if err != nil { + return "", nil, false, err + } + return pr.GetBody(), files, approved, nil +} + +func fetchIntentIssueTexts(ctx context.Context, ghClient *github.Client, defaultRepo, body string) ([]intent.TextEvidence, error) { + if ghClient == nil || ghClient.GoGitHub() == nil { + return nil, github.ErrNoGitHubClient + } + client := ghClient.GoGitHub() + refs := intent.LinkedIssueRefs(body, defaultRepo) + out := make([]intent.TextEvidence, 0, len(refs)) + for _, ref := range refs { + repo := ref.Repo + if repo == "" { + repo = defaultRepo + } + owner, repoName, ok := strings.Cut(repo, "/") + if !ok || owner == "" || repoName == "" { + continue + } + issue, _, err := client.Issues.Get(ctx, owner, repoName, ref.Number) + if err != nil { + return out, fmt.Errorf("getting linked issue %s#%d: %w", repo, ref.Number, err) + } + out = append(out, intent.TextEvidence{ + Source: fmt.Sprintf("issue %s#%d", repo, ref.Number), + Title: issue.GetTitle(), + Body: issue.GetBody(), + }) + } + return out, nil +} + +func recordIntentAlignmentAdvisory(stores map[string]*beads.Store, repo string, number int, alignment intent.AlignmentVerdict, logger *slog.Logger) { + store := stores["intent"] + if store == nil { + store = stores["quality"] + } + if store == nil { + for _, candidate := range stores { + if candidate != nil { + store = candidate + break + } + } + } + if store == nil { + return + } + title := fmt.Sprintf("Intent alignment drift in %s#%d", repo, number) + // "/#", NOT "gh-/#". The old form fused the + // source prefix into the org when the digest built its URL, so every one of + // these rendered a link to a github.com/gh- that does not exist + // (#6080). The renderer strips the prefix defensively for beads already + // written this way; this stops writing new ones. + ref := fmt.Sprintf("%s#%d", repo, number) + // Beads created before that change carry the prefixed form. Matching both + // keeps this idempotent across the change: without it the first run after + // upgrading would fail to recognise the existing bead and open a duplicate. + legacyRef := "gh-" + ref + for _, b := range store.List(beads.ListFilter{}) { + if b.Type == beads.TypeAdvisory && b.Title == title && + (b.ExternalRef == ref || b.ExternalRef == legacyRef) && + b.Status != beads.StatusClosed && b.Status != beads.StatusDone { + return + } + } + b, err := store.Create(title, beads.TypeAdvisory, beads.PriorityHigh, "intent", ref) + if err != nil { + logger.Warn("failed to record intent alignment advisory", "repo", repo, "number", number, "error", err) + return + } + _ = store.Update(b.ID, func(bead *beads.Bead) { + bead.Notes = alignmentSummary(alignment) + }) +} + +func alignmentSummary(alignment intent.AlignmentVerdict) string { + var parts []string + if alignment.Rationale != "" { + parts = append(parts, alignment.Rationale) + } + for _, f := range alignment.DeterministicFindings { + if f.Status == intent.AlignmentStatusMisaligned { + parts = append(parts, f.Code+": "+f.Reason+" ("+strings.Join(f.Files, ", ")+")") + } + } + if alignment.Model != nil && alignment.Model.Status == intent.AlignmentStatusMisaligned { + parts = append(parts, "model: "+alignment.Model.Rationale) + } + if len(parts) == 0 { + return "intent alignment check reported misalignment" + } + return strings.Join(parts, "\n") +} + +func hasMaintainerApproval(ctx context.Context, client *gh.Client, owner, repo string, number int) (bool, error) { + opts := &gh.ListOptions{PerPage: 100} + latest := make(map[string]string) + maintainer := make(map[string]bool) + for { + reviews, resp, err := client.PullRequests.ListReviews(ctx, owner, repo, number, opts) + if err != nil { + return false, fmt.Errorf("listing PR reviews: %w", err) + } + for _, review := range reviews { + login := review.GetUser().GetLogin() + if login == "" { + continue + } + if maintainerAssociation(review.GetAuthorAssociation()) { + switch review.GetState() { + case "APPROVED", "CHANGES_REQUESTED", "DISMISSED": + latest[login] = review.GetState() + } + maintainer[login] = true + } + } + if resp == nil || resp.NextPage == 0 { + break + } + opts.Page = resp.NextPage + } + approved := false + for login, state := range latest { + if !maintainer[login] { + continue + } + switch state { + case "CHANGES_REQUESTED": + return false, nil + case "APPROVED": + approved = true + } + } + return approved, nil +} + +func maintainerAssociation(association string) bool { + switch strings.ToUpper(strings.TrimSpace(association)) { + case "OWNER", "MEMBER", "COLLABORATOR": + return true + default: + return false + } +} diff --git a/src/cmd/hive/login_scan.go b/src/cmd/hive/login_scan.go new file mode 100644 index 0000000000..40eed56776 --- /dev/null +++ b/src/cmd/hive/login_scan.go @@ -0,0 +1,13 @@ +package main + +import ( + "github.com/hivecommons/hive/pkg/loginscan" +) + +// loginSightings is the login detector's process-scoped state. The governor +// cycle is a function rather than an object, so the consecutive-sighting counts +// have to outlive a single call; tests build their own tracker and pass it +// explicitly. +// +// The detector itself lives in pkg/loginscan (#7238 stage 4). +var loginSightings = loginscan.NewSightingTracker() diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 9e6a3e45f9..5db935b17f 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -48,7 +48,6 @@ import ( "github.com/hivecommons/hive/pkg/hub" spoke "github.com/hivecommons/hive/pkg/hub/spoke" "github.com/hivecommons/hive/pkg/inference" - "github.com/hivecommons/hive/pkg/intent" "github.com/hivecommons/hive/pkg/ioscan" "github.com/hivecommons/hive/pkg/knowledge" "github.com/hivecommons/hive/pkg/loginscan" @@ -131,8 +130,6 @@ func reportedVersion() string { return normalizeVersion(version) } -// publishFleetReports writes the fleet self-report results computed for the -// status payload upstream, unless the feature is in dry-run (the default). func publishFleetReports(ctx context.Context, logger *slog.Logger, ghClient *github.Client, dashSrv *dashboard.Server, res *fleetreport.Result, dryRun bool) { if res == nil || dryRun || ghClient == nil || dashSrv == nil { return @@ -497,59 +494,6 @@ func nextInstallationID(current int64, ghCfg *spoke.HeartbeatGitHubAppConfig) (n return current, false } -// deliveredKeyPath is where a hub-delivered private key for appID is stored. -// -// The filename NAMES the App, so a key can only ever be found under the App it -// was delivered for. The generic /data/gh-app-key.pem carries no such evidence: -// a key written there for one App silently becomes "the key" for whatever -// app_id the config later claims, which is how all 33 heartbeat-only-cluster spokes ended up -// signing as the public App with the GHE key and getting -// 404 Integration not found. -// -// Falls back to the generic path only when the delivery names no App, so a key -// is never dropped on the floor. -func deliveredKeyPath(appID int64) string { - if p := appKeys.PerAppIDKeyPath(appID); p != "" { - return p - } - return appKeys.DataKeyPath -} - -// describeAppKeyFailure turns a bare wrapped os error from github.NewAppAuth -// into a message an operator can act on without reading the source: it names -// the path actually tried, the full resolution order that produced it, and the -// underlying cause. -// -// The generic "reading app key /secrets/gh-app-key.pem: no such file" that this -// replaces gave no hint that key_file, $GH_APP_KEY_FILE, the PVC path and the -// provisioning mount are all consulted in a fixed order — so the usual response -// was to put the key in the wrong one of the four. -func describeAppKeyFailure(configured, envOverride, resolved string, err error) string { - order := []string{ - fmt.Sprintf("$GH_APP_KEY_FILE=%s", describeKeySource(envOverride)), - fmt.Sprintf("github.key_file=%s", describeKeySource(configured)), - fmt.Sprintf("per-app-id PVC key %s/gh-app-key-.pem", appKeys.DataDir), - fmt.Sprintf("per-app-id provisioning key %s/gh-app-key-.pem", appKeys.ProvisionedDir), - fmt.Sprintf("PVC fallback %s", appKeys.DataKeyPath), - fmt.Sprintf("provisioning mount %s", appKeys.ProvisionedKeyPath), - } - return fmt.Sprintf( - "GitHub App private key could not be loaded from %q: %v. "+ - "Resolution order (first non-empty wins): %s. "+ - "Write a PEM-encoded RSA private key to that path, or point github.key_file at one.", - resolved, err, strings.Join(order, " → "), - ) -} - -// describeKeySource renders an unset key-file source as "(unset)" so the -// resolution order in describeAppKeyFailure reads unambiguously. -func describeKeySource(v string) string { - if strings.TrimSpace(v) == "" { - return "(unset)" - } - return v -} - var githubAppTokenCachePath = github.TokenCachePath func githubAppTokenHeartbeatFields(cfg *config.Config, detail string) (status, lastMintAt, lastErr string) { @@ -5526,19 +5470,6 @@ func main() { } -// Dashboard system-alert IDs for the budget thresholds. -const ( - budgetWarnAlertID = "budget-warn" - budgetExhaustedAlertID = "budget-exhausted" - // noCadenceAlertID is the never-kicked cause+fix banner (#5577): enabled - // agents with no cadence in any mode and no kick ever. - noCadenceAlertID = "agent-no-cadence" - // providerBudgetAlertID is the PROVIDER spend rebuff (#4294), kept distinct - // from the two token-budget alerts above so an operator can tell "we used - // our token allowance" from "the gateway will not spend more money". - providerBudgetAlertID = "provider-budget-exceeded" -) - // buildRepoActivityWire maps the dashboard activity collector's per-repo // snapshot into the plain hub wire structs the heartbeat carries. Kept here (in // the one package that imports both hub and dashboard) so pkg/hub never has to @@ -5721,13 +5652,6 @@ func (s labelPlanSink) QueuedPlan(epic *beads.Bead, paused bool) { s.logger.Warn("plan-from-label: architect unavailable, plan queued", "epic", epic.ID, "ref", epic.ExternalRef) } -// appKeyPaths snapshots the two App key path locations for a pkg/apphealth -// call. Read at call time on purpose: tests repoint these, and capturing them -// once would silently ignore that. -func appKeyPaths() apphealth.KeyPaths { - return apphealth.KeyPaths{Spoke: appKeys.DataKeyPath, Provisioned: appKeys.ProvisionedKeyPath} -} - // healGitHubAppInstallation self-heals a hive whose github.installation_id // points at the WRONG account — the failure mode diagnoseGitHubApp // already detects and reports ("installation N belongs to 'X', not 'Y'"). It @@ -5902,6 +5826,10 @@ func isGitHubRateLimitText(err error) bool { return err != nil && strings.Contains(strings.ToLower(err.Error()), githubRateLimitErrText) } +// The GitHub App credential classifiers moved to pkg/apphealth (#7238). These +// wrappers exist so call sites keep their current shape and so the App key +// paths -- vars that tests repoint at a temp dir -- are read HERE, at call +// time, rather than captured once at init. func classifyGitHubAppFailure(ctx context.Context, appAuth *github.AppAuth, expectedOwner string, logger *slog.Logger) (bool, string, github.AppAuthState) { return apphealth.ClassifyFailure(ctx, appAuth, expectedOwner, appKeyPaths(), logger) } @@ -5914,8 +5842,9 @@ func classifyGitHubAppRepoCoverage(ctx context.Context, appAuth *github.AppAuth, return apphealth.ClassifyRepoCoverage(ctx, appAuth, org, repos, logger) } -// advisoryPostGate is process-wide because the eval-cycle ticker is not the -// only poster: startup and restart paths post too. +// The advisory digest posting policy moved to pkg/advisory (#7238 stage 2). +// These wrappers keep the existing call sites unchanged; advisoryPostGate is +// now an owned instance rather than a package-level struct tests reset. var advisoryPostGate = advisory.NewPostGate() func primaryAdvisoryRepo(cfg *config.Config) string { @@ -6901,14 +6830,6 @@ func runEvalCycle( } } -// loginSightings is the login detector's process-scoped state. The governor -// cycle is a function rather than an object, so the consecutive-sighting counts -// have to outlive a single call; tests build their own tracker and pass it -// explicitly. -// -// The detector itself lives in pkg/loginscan (#7238 stage 4). -var loginSightings = loginscan.NewSightingTracker() - func convertKnowledgeLayers(cfgLayers []config.KnowledgeLayer) []knowledge.LayerConfig { layers := make([]knowledge.LayerConfig, len(cfgLayers)) for i, l := range cfgLayers { @@ -6939,181 +6860,8 @@ func curatorConfigFromHive(c config.KnowledgeCurator) knowledge.CuratorConfig { } // hiveIDFilePath is the persistent file where the Hive ID is stored across restarts. -// It is a var, not a const, so tests can repoint it at a temp dir and exercise -// loadOrGenerateHiveID's disk-read and generate-and-persist branches hermetically -// (#7148). v4 carries the same seam in cmd/hive/main.go; v5 keeps the symbol here -// after the main.go split, so the change lands in this file instead. var hiveIDFilePath = "/data/hive-id" -// loadOrGenerateHiveID reads the Hive ID from disk, or generates and persists a new one. -const ( - // selfUpgradeMaxAttempts bounds how many times a spoke retries an upgrade - // that keeps leaving the image unchanged. Bounded rather than unlimited so a - // genuinely broken hive (e.g. missing RBAC) stops thrashing its pod, and - // bounded rather than "never again" so a transient failure still converges. - selfUpgradeMaxAttempts = 5 - // selfUpgradeBaseBackoff is the delay before retry #2; it doubles per - // attempt up to selfUpgradeMaxBackoff. - selfUpgradeBaseBackoff = 2 * time.Minute - // selfUpgradeMaxBackoff caps the exponential backoff between retries. - selfUpgradeMaxBackoff = 30 * time.Minute - // selfUpgradeFailureExitCode marks a process exit caused by a FAILED - // self-upgrade. Distinct from 0 so the failure is visible in the container's - // termination state instead of looking like a clean shutdown. - selfUpgradeFailureExitCode = 17 -) - -// upgradeMarker is the on-PVC record at /data/upgrade-requested. It survives -// pod restarts (that is the whole point: the process exits as part of an -// upgrade), so it is the only place attempt bookkeeping can live. -type upgradeMarker struct { - TargetSHA string `json:"target_sha"` - CurrentSHA string `json:"current_sha"` - RequestedAt time.Time `json:"requested_at"` - Attempts int `json:"attempts"` - LastError string `json:"last_error,omitempty"` -} - -// parseUpgradeMarker decodes a marker, tolerating the legacy format that had no -// attempts/last_error fields. A legacy marker counts as one prior attempt so an -// already-wedged hive gets retries under the new budget instead of being -// treated as fresh. -func parseUpgradeMarker(data []byte) upgradeMarker { - var m upgradeMarker - if err := json.Unmarshal(data, &m); err != nil { - return upgradeMarker{} - } - if m.Attempts < 1 { - m.Attempts = 1 - } - return m -} - -// sameUpgradeTarget reports whether two target SHAs refer to the same commit, -// tolerating short/full SHA length mismatch the way the hub's sameCommit does. -// A DIFFERENT target must reset the attempt budget, so this comparison is what -// keeps the latch from outliving the upgrade it was created for. -func sameUpgradeTarget(a, b string) bool { - if a == "" || b == "" { - return false - } - n := len(a) - if len(b) < n { - n = len(b) - } - return strings.EqualFold(a[:n], b[:n]) -} - -func writeUpgradeMarker(path string, m upgradeMarker, logger *slog.Logger) { - data, err := json.Marshal(m) - if err != nil { - logger.Warn("failed to encode upgrade marker", "error", err) - return - } - if err := os.WriteFile(path, data, 0o644); err != nil { - logger.Warn("failed to write upgrade marker", "path", path, "error", err) - } -} - -// upgradeMarkerPath and lastUpgradeOutcomePath are the two on-PVC records the -// spoke keeps for auto-upgrade visibility (#7092). The marker at -// upgradeMarkerPath is present ONLY while an instructed upgrade has not landed -// (in flight or terminally failed) and is cleared the moment the new image -// boots — so it can NEVER represent a success. lastUpgradeOutcomePath is the -// durable companion that records the last upgrade that actually LANDED, so the -// dashboard can tell "attempted and succeeded" apart from "never attempted" -// instead of letting a blank panel masquerade as success. -const ( - upgradeMarkerPath = "/data/upgrade-requested" - lastUpgradeOutcomePath = "/data/last-upgrade-outcome" -) - -// upgradeOutcome is the durable "last upgrade LANDED" record. Written on the -// boot that completes an upgrade (reconcileUpgradeOutcomeAtBoot), it survives — -// unlike upgradeMarker, which is removed the moment the target image boots. -type upgradeOutcome struct { - TargetSHA string `json:"target_sha"` - CurrentSHA string `json:"current_sha"` - RequestedAt time.Time `json:"requested_at"` - CompletedAt time.Time `json:"completed_at"` -} - -func writeUpgradeOutcome(path string, o upgradeOutcome, logger *slog.Logger) { - data, err := json.Marshal(o) - if err != nil { - logger.Warn("failed to encode upgrade outcome", "error", err) - return - } - if err := os.WriteFile(path, data, 0o644); err != nil { - logger.Warn("failed to write upgrade outcome", "path", path, "error", err) - } -} - -// reconcileUpgradeOutcomeAtBoot records a SUCCESSFUL self-upgrade. An in-flight -// marker whose target equals the now-running commit means the instructed -// upgrade LANDED: the pod booted on the target image. That success would -// otherwise vanish — the next upgrade instruction silently discards the stale -// marker, so a hive that updated cleanly looks identical to one that never -// tried. This persists the success durably and clears the in-flight marker so -// it stops reading as "not landed". A marker whose target does NOT match the -// running commit is still in flight or failed and is left untouched for that -// surface. Called once at startup, before the heartbeat loop and dashboard come -// up, so the dashboard always sees the reconciled state. -func reconcileUpgradeOutcomeAtBoot(markerPath, outcomePath, runningSHA string, logger *slog.Logger) { - data, err := os.ReadFile(markerPath) - if err != nil { - return - } - m := parseUpgradeMarker(data) - if m.TargetSHA == "" || runningSHA == "" || !sameUpgradeTarget(m.TargetSHA, runningSHA) { - return - } - writeUpgradeOutcome(outcomePath, upgradeOutcome{ - TargetSHA: m.TargetSHA, - CurrentSHA: m.CurrentSHA, - RequestedAt: m.RequestedAt, - CompletedAt: time.Now().UTC(), - }, logger) - if err := os.Remove(markerPath); err != nil && !os.IsNotExist(err) { - logger.Warn("failed to clear landed upgrade marker", "path", markerPath, "error", err) - } - logger.Info("self-upgrade landed: recorded successful upgrade outcome", - "target", m.TargetSHA, "current", runningSHA) -} - -// recordUpgradeError annotates the existing marker with the cause of the failed -// attempt so the NEXT boot can log why the previous one did not land — without -// it the reason dies with the process and the failure is invisible. -// upgradeFailureSummary renders what the hub shows an operator. An empty -// LastError must never render as a dangling "attempts: " - a colon promising a -// reason and delivering none is worse than saying the reason was not captured, -// because it reads as truncation and sends the reader looking for the rest. -func upgradeFailureSummary(attempts int, lastError string) string { - if strings.TrimSpace(lastError) == "" { - return fmt.Sprintf("self-upgrade failed after %d attempts (no error recorded; the image never changed - check that the deployment tracks a tag carrying the target SHA)", attempts) - } - return fmt.Sprintf("self-upgrade failed after %d attempts: %s", attempts, lastError) -} - -func recordUpgradeError(path string, upgradeErr error, logger *slog.Logger) { - if upgradeErr == nil { - return - } - var m upgradeMarker - data, err := os.ReadFile(path) - switch { - case os.IsNotExist(err): - return - case err != nil: - logger.Warn("upgrade marker unreadable; recording the error against a fresh marker", - "path", path, "error", err) - default: - m = parseUpgradeMarker(data) - } - m.LastError = upgradeErr.Error() - writeUpgradeMarker(path, m, logger) -} - func loadOrGenerateHiveID(logger *slog.Logger) string { if envID := os.Getenv("HIVE_ID"); envID != "" { if err := os.WriteFile(hiveIDFilePath, []byte(envID+"\n"), 0o644); err == nil { @@ -7800,28 +7548,6 @@ const taskListSweepInterval = 15 * time.Minute // chance of editing a suggestion under a reader's cursor. const duplicateSweepInterval = time.Hour -// trustedMergerFunc resolves a GitHub login against the hive's authorized-users -// allowlist and reports whether it holds at least config.RoleMerger — the same -// bar requireMergerOrOwnerRole enforces on the dashboard queue endpoint (audit -// F3). -// -// Fails CLOSED: a nil config or a login absent from the allowlist is NOT -// trusted, so an unclassifiable actor can never merge. cfg is read on every -// call so a config reload that grants or revokes the merger tier takes effect -// without a restart. -func trustedMergerFunc(cfg *config.Config) automerge.MergerAuthorizer { - return func(login string) bool { - if cfg == nil || strings.TrimSpace(login) == "" { - return false - } - role, ok := cfg.Dashboard.AuthorizedRole(login) - if !ok { - return false - } - return config.RoleAtLeast(role, config.RoleMerger) - } -} - // runAutoMergeSweepIfDue drains the label-queued auto-merge queue (the human // "Approved ... for Hive auto-merge" path) at most once per // autoMergeSweepInterval. All merge-eligibility decisions — queue-approval @@ -8052,141 +7778,13 @@ func runDuplicateSweepIfDue(ctx context.Context, cfg *config.Config, ghClient *g "commented": strconv.Itoa(result.Commented), }, }) -} // mergeEligiblePath is a var (not a const) only so tests can point -// mergeTargetEligible at a temp file; production never reassigns it. -var mergeEligiblePath = "/var/run/hive-metrics/merge-eligible.json" +} var ( ciFailingPath = "/var/run/hive-metrics/ci-failing.json" intentVerdictsPath = "/var/run/hive-metrics/intent-verdicts.json" ) -// acmmHoldGatedMinLevel / acmmHoldGatedMaxLevel bracket the ACMM levels whose -// merge policy is "hold-gated" — every agent-opened PR gets a "hold" label and -// no agent merges (see src/pkg/config/packs/level-{3,4,5}.yaml). L1/L2 are -// "manual" (agents open no PRs) and L6 is "auto-merge on green CI, no hold -// label", so both fall outside this range. Used by the F6 hold-label decider. -const ( - acmmHoldGatedMinLevel = 3 - acmmHoldGatedMaxLevel = 5 -) - -// shouldHoldAgentPR keeps public outreach claims human-reviewed even at L6, -// where ordinary agent PRs may auto-merge. The general ACMM hold gate remains -// unchanged for all roles at L3-L5. -func shouldHoldAgentPR(agentName string, level int) bool { - if strings.EqualFold(strings.TrimSpace(agentName), "outreach") { - return true - } - return level >= acmmHoldGatedMinLevel && level <= acmmHoldGatedMaxLevel -} - -// mergeableJSONUnknown is the explicit wire value for "mergeability was never -// determined". It is spelled out rather than left as "" so a consumer reading -// merge-eligible.json cannot mistake an unpopulated field for a definitive -// "no" — the failure mode that made every PR read as unmergeable. -const mergeableJSONUnknown = "unknown" - -// mergeTargetEligible reports whether (repo, number) currently appears in the -// governor's merge-eligible.json AT the expected head SHA. It reads the file -// FRESH on every call (never caches) because eligibility is recomputed each -// governor cycle — a stale cache could authorize a PR that has since fallen out -// of the list. On any read/parse error it returns false (FAIL CLOSED): if we -// cannot prove the target is eligible, we must not authorize the merge. -// -// M4 (CWE-367, TOCTOU): the governor records the head SHA it observed when it -// deemed the PR eligible (eligiblePR.HeadSHA). A branch can move between that -// review and the merge relay firing, so matching (repo, number) alone would let -// a moved head merge at a commit the governor never vetted. We therefore also -// require the entry's stored head_sha to equal expectSHA. A mismatch — or a -// stored SHA that is empty (governor could not observe it) — fails closed; the -// relay's SHA pin then fails the merge cleanly if a stale request slips through. -// -// merge-eligible.json stores repos as "owner/repo"; a MergeRequest.Repo may be -// bare ("repo") or fully qualified ("owner/repo"). We match on the bare repo -// name (the segment after the last "/") plus the number, so both request forms -// resolve to the same eligible entry without depending on the org prefix. -func mergeTargetEligible(repo string, number int, expectSHA string) bool { - data, err := os.ReadFile(mergeEligiblePath) - if err != nil { - return false // fail closed: no list ⇒ nothing is eligible - } - var payload struct { - Items []struct { - Number int `json:"number"` - Repo string `json:"repo"` - HeadSHA string `json:"head_sha"` - } `json:"merge_eligible"` - } - if err := json.Unmarshal(data, &payload); err != nil { - return false // fail closed: unparseable list ⇒ deny - } - want := bareRepoName(repo) - wantSHA := strings.TrimSpace(expectSHA) - for _, it := range payload.Items { - if it.Number == number && bareRepoName(it.Repo) == want { - // M4: bind authorization to the governor-observed head. An empty - // stored SHA cannot be proven to match, so it fails closed rather - // than authorizing an unpinned head. - return strings.TrimSpace(it.HeadSHA) != "" && strings.TrimSpace(it.HeadSHA) == wantSHA - } - } - return false -} - -// bareRepoName returns the repo segment after the last "/", so "owner/repo" and -// "repo" compare equal. Used to match a MergeRequest.Repo against the -// "owner/repo" entries in merge-eligible.json regardless of prefix. -func bareRepoName(repo string) string { - if i := strings.LastIndex(repo, "/"); i >= 0 { - return repo[i+1:] - } - return repo -} - -// bindMergeAuthz wraps the manager's agent/UID/CanMerge authorizer with the -// F4 target-binding checks (CWE-863). The inner authz owns the "may this agent -// merge at all" decision; this wrapper owns "is THIS specific target one the -// governor deemed eligible, at a pinned SHA". Both must pass before MergePR is -// reached. Ordering: run the agent/UID/CanMerge check first (cheapest, and it -// gives the clearest denial reason), then the SHA + eligible-list binding. -func bindMergeAuthz(inner func(agent string, fileUID int) error) github.MergeRequestAuthorizer { - return func(agent string, fileUID int, repo string, number int, expectSHA string) error { - if err := inner(agent, fileUID); err != nil { - return err - } - // (a) Require a pinned head SHA. An empty expectSHA means "merge whatever - // HEAD is now", which is the TOCTOU hole: a PR that was eligible when the - // governor last looked could have had a malicious commit pushed since. - // MergePR passes expectSHA as the required head SHA, so a moved head fails - // cleanly — but only if we insist it is set. - if strings.TrimSpace(expectSHA) == "" { - return fmt.Errorf("merge target %s#%d has no expected head SHA — refusing to merge an unpinned head (TOCTOU guard)", repo, number) - } - // (b) Require the target to be in the governor's current merge-eligible - // list AT the expected head SHA. This binds authorization to a PR the - // hive actually deemed landable this cycle, at the exact commit it - // reviewed, so an injected agent cannot request landing an arbitrary - // reachable PR (e.g. its own) whose required checks happen to pass, nor - // land an eligible PR at a head that moved after review (M4, CWE-367). - // Read fresh + fail closed (see mergeTargetEligible). - if !mergeTargetEligible(repo, number, expectSHA) { - return fmt.Errorf("merge target %s#%d is not in the current merge-eligible list at head %s — only governor-approved PRs may be landed via the merge relay, and only at the reviewed head SHA", repo, number, expectSHA) - } - return nil - } -} - -// mergeableJSON renders a tri-state mergeability verdict for the -// merge-eligible.json marker, mapping the unknown zero value to an explicit -// "unknown" rather than an empty string. -func mergeableJSON(m github.Mergeable) string { - if m == github.MergeableUnknown { - return mergeableJSONUnknown - } - return string(m) -} - // claimLedger holds the duplicate-PR guard's persisted issue→PR claim mapping // across eval cycles. It is loaded lazily on first use (and retried on a load // failure) rather than at startup, so a missing or corrupt /data ledger can @@ -8291,306 +7889,6 @@ func claimingPRRedStale(cfg *config.Config, actionable *github.ActionableResult) } } -func writeIntentVerdicts( - ctx context.Context, - cfg *config.Config, - ghClient *github.Client, - actionable *github.ActionableResult, - beadStores map[string]*beads.Store, - logger *slog.Logger, -) map[string]intent.Verdict { - verdicts := make(map[string]intent.Verdict) - if cfg == nil || actionable == nil { - return verdicts - } - _ = os.MkdirAll("/var/run/hive-metrics", 0o755) - aiAuthor := strings.TrimSpace(cfg.EffectiveAIAuthor()) - intentCfg := intentConfigFromCfg(cfg) - var alignmentReviewer *intent.AlignmentReviewer - if strings.TrimSpace(cfg.Intent.AlignmentModel) != "" { - endpoint, apiKey, _ := cfg.Governor.ResolveReviewer() - var err error - alignmentReviewer, err = intent.NewAlignmentReviewer(intent.AlignmentReviewerConfig{ - Endpoint: endpoint, - APIKey: apiKey, - Model: cfg.Intent.AlignmentModel, - }) - if err != nil { - logger.Warn("intent alignment reviewer disabled", "error", err) - } - } - type verdictRecord struct { - Repo string `json:"repo"` - Number int `json:"number"` - Title string `json:"title"` - Author string `json:"author"` - Enforced bool `json:"enforced"` - Verdict intent.Verdict `json:"verdict"` - Classify string `json:"classification_reason"` - FetchError string `json:"fetch_error,omitempty"` - } - records := make([]verdictRecord, 0, len(actionable.PRs.Items)) - for _, pr := range actionable.PRs.Items { - fullRepo := fullRepoName(pr.Repo, cfg.Project.Org) - key := fmt.Sprintf("%s/%d", fullRepo, pr.Number) - agentPR := aiAuthor != "" && strings.EqualFold(pr.Author, aiAuthor) - record := verdictRecord{ - Repo: fullRepo, - Number: pr.Number, - Title: pr.Title, - Author: pr.Author, - Enforced: cfg.Intent.Enforce, - } - if !agentPR { - class := intent.Classify(intent.PR{Title: pr.Title, Labels: pr.Labels, Author: pr.Author, AgentAuthor: false}, intentCfg) - verdict := intent.Evaluate(class, intent.Evidence{}) - verdicts[key] = verdict - record.Verdict = verdict - record.Classify = class.Reason - records = append(records, record) - continue - } - body, files, approved, err := fetchIntentPREvidence(ctx, ghClient, fullRepo, pr.Number) - if err != nil { - verdict := intent.Verdict{ - Tier: intent.Tier1, - Authorized: false, - Reason: "intent evidence unavailable: " + err.Error(), - AgentPR: true, - } - verdicts[key] = verdict - record.Verdict = verdict - record.FetchError = err.Error() - records = append(records, record) - logger.Warn("intent verification evidence fetch failed", "repo", fullRepo, "number", pr.Number, "error", err) - continue - } - class := intent.Classify(intent.PR{ - Title: pr.Title, - Body: body, - Labels: pr.Labels, - Files: files, - Author: pr.Author, - AgentAuthor: true, - }, intentCfg) - evidence := intent.BuildEvidenceForRepo(body, fullRepo, beadStores, approved) - verdict := intent.Evaluate(class, evidence) - issueTexts, issueErr := fetchIntentIssueTexts(ctx, ghClient, fullRepo, body) - if issueErr != nil { - logger.Warn("intent alignment issue evidence fetch failed", "repo", fullRepo, "number", pr.Number, "error", issueErr) - } - refs := intent.LinkedIssueRefs(body, fullRepo) - alignCtx := intent.BuildAlignmentContext(intent.PR{ - Title: pr.Title, - Body: body, - Labels: pr.Labels, - Files: files, - Author: pr.Author, - AgentAuthor: true, - }, issueTexts, beadStores, refs) - alignment := intent.EvaluateAlignment(alignCtx, class.Tier, intentCfg) - if alignmentReviewer != nil { - modelVerdict, err := alignmentReviewer.Review(ctx, alignCtx) - if err != nil { - logger.Warn("intent alignment model review failed open", "repo", fullRepo, "number", pr.Number, "error", err) - alignment = intent.MergeAlignment(alignment, nil, err) - } else { - alignment = intent.MergeAlignment(alignment, &modelVerdict, nil) - } - } - verdict.Alignment = &alignment - verdicts[key] = verdict - record.Verdict = verdict - record.Classify = class.Reason - records = append(records, record) - if !verdict.Authorized { - logger.Info("intent authorization denied", "repo", fullRepo, "number", pr.Number, "tier", verdict.Tier, "reason", verdict.Reason, "enforce", cfg.Intent.Enforce) - } - if alignment.Misaligned() { - logger.Info("intent alignment denied", "repo", fullRepo, "number", pr.Number, "reason", alignment.Rationale, "enforce", cfg.Intent.Enforce) - recordIntentAlignmentAdvisory(beadStores, fullRepo, pr.Number, alignment, logger) - } - } - payload := map[string]any{ - "generated_at": time.Now().UTC().Format(time.RFC3339), - "enforced": cfg.Intent.Enforce, - "verdicts": records, - } - if data, err := json.Marshal(payload); err == nil { - atomicWrite(intentVerdictsPath, data) - } else { - logger.Warn("failed to marshal intent verdicts", "error", err) - } - return verdicts -} - -func fetchIntentPREvidence(ctx context.Context, ghClient *github.Client, repo string, number int) (string, []intent.ChangedFile, bool, error) { - if ghClient == nil || ghClient.GoGitHub() == nil { - return "", nil, false, github.ErrNoGitHubClient - } - owner, repoName, ok := strings.Cut(repo, "/") - if !ok || owner == "" || repoName == "" { - return "", nil, false, fmt.Errorf("invalid repo %q", repo) - } - client := ghClient.GoGitHub() - pr, _, err := client.PullRequests.Get(ctx, owner, repoName, number) - if err != nil { - return "", nil, false, fmt.Errorf("getting PR: %w", err) - } - files, err := automerge.ListChangedFiles(ctx, client, owner, repoName, pr) - if err != nil { - return "", nil, false, err - } - approved, err := hasMaintainerApproval(ctx, client, owner, repoName, number) - if err != nil { - return "", nil, false, err - } - return pr.GetBody(), files, approved, nil -} - -func fetchIntentIssueTexts(ctx context.Context, ghClient *github.Client, defaultRepo, body string) ([]intent.TextEvidence, error) { - if ghClient == nil || ghClient.GoGitHub() == nil { - return nil, github.ErrNoGitHubClient - } - client := ghClient.GoGitHub() - refs := intent.LinkedIssueRefs(body, defaultRepo) - out := make([]intent.TextEvidence, 0, len(refs)) - for _, ref := range refs { - repo := ref.Repo - if repo == "" { - repo = defaultRepo - } - owner, repoName, ok := strings.Cut(repo, "/") - if !ok || owner == "" || repoName == "" { - continue - } - issue, _, err := client.Issues.Get(ctx, owner, repoName, ref.Number) - if err != nil { - return out, fmt.Errorf("getting linked issue %s#%d: %w", repo, ref.Number, err) - } - out = append(out, intent.TextEvidence{ - Source: fmt.Sprintf("issue %s#%d", repo, ref.Number), - Title: issue.GetTitle(), - Body: issue.GetBody(), - }) - } - return out, nil -} - -func recordIntentAlignmentAdvisory(stores map[string]*beads.Store, repo string, number int, alignment intent.AlignmentVerdict, logger *slog.Logger) { - store := stores["intent"] - if store == nil { - store = stores["quality"] - } - if store == nil { - for _, candidate := range stores { - if candidate != nil { - store = candidate - break - } - } - } - if store == nil { - return - } - title := fmt.Sprintf("Intent alignment drift in %s#%d", repo, number) - // "/#", NOT "gh-/#". The old form fused the - // source prefix into the org when the digest built its URL, so every one of - // these rendered a link to a github.com/gh- that does not exist - // (#6080). The renderer strips the prefix defensively for beads already - // written this way; this stops writing new ones. - ref := fmt.Sprintf("%s#%d", repo, number) - // Beads created before that change carry the prefixed form. Matching both - // keeps this idempotent across the change: without it the first run after - // upgrading would fail to recognise the existing bead and open a duplicate. - legacyRef := "gh-" + ref - for _, b := range store.List(beads.ListFilter{}) { - if b.Type == beads.TypeAdvisory && b.Title == title && - (b.ExternalRef == ref || b.ExternalRef == legacyRef) && - b.Status != beads.StatusClosed && b.Status != beads.StatusDone { - return - } - } - b, err := store.Create(title, beads.TypeAdvisory, beads.PriorityHigh, "intent", ref) - if err != nil { - logger.Warn("failed to record intent alignment advisory", "repo", repo, "number", number, "error", err) - return - } - _ = store.Update(b.ID, func(bead *beads.Bead) { - bead.Notes = alignmentSummary(alignment) - }) -} - -func alignmentSummary(alignment intent.AlignmentVerdict) string { - var parts []string - if alignment.Rationale != "" { - parts = append(parts, alignment.Rationale) - } - for _, f := range alignment.DeterministicFindings { - if f.Status == intent.AlignmentStatusMisaligned { - parts = append(parts, f.Code+": "+f.Reason+" ("+strings.Join(f.Files, ", ")+")") - } - } - if alignment.Model != nil && alignment.Model.Status == intent.AlignmentStatusMisaligned { - parts = append(parts, "model: "+alignment.Model.Rationale) - } - if len(parts) == 0 { - return "intent alignment check reported misalignment" - } - return strings.Join(parts, "\n") -} - -func hasMaintainerApproval(ctx context.Context, client *gh.Client, owner, repo string, number int) (bool, error) { - opts := &gh.ListOptions{PerPage: 100} - latest := make(map[string]string) - maintainer := make(map[string]bool) - for { - reviews, resp, err := client.PullRequests.ListReviews(ctx, owner, repo, number, opts) - if err != nil { - return false, fmt.Errorf("listing PR reviews: %w", err) - } - for _, review := range reviews { - login := review.GetUser().GetLogin() - if login == "" { - continue - } - if maintainerAssociation(review.GetAuthorAssociation()) { - switch review.GetState() { - case "APPROVED", "CHANGES_REQUESTED", "DISMISSED": - latest[login] = review.GetState() - } - maintainer[login] = true - } - } - if resp == nil || resp.NextPage == 0 { - break - } - opts.Page = resp.NextPage - } - approved := false - for login, state := range latest { - if !maintainer[login] { - continue - } - switch state { - case "CHANGES_REQUESTED": - return false, nil - case "APPROVED": - approved = true - } - } - return approved, nil -} - -func maintainerAssociation(association string) bool { - switch strings.ToUpper(strings.TrimSpace(association)) { - case "OWNER", "MEMBER", "COLLABORATOR": - return true - default: - return false - } -} - func fullRepoName(repo, org string) string { if strings.Contains(repo, "/") || org == "" { return repo @@ -8708,409 +8006,6 @@ func auditPRAgents(org string, since time.Time, auditPath string) map[string]str return out } -// anyRequiredCheckFailing reports whether any of a PR's failing check names -// is in the operator-declared required set. -func anyRequiredCheckFailing(failing []string, required map[string]bool) bool { - for _, name := range failing { - if required[name] { - return true - } - } - return false -} - -// mergeBucket is where the merge-eligible classifier files a PR: the -// merge-eligible.json list, the ci-failing.json list, or neither. -type mergeBucket int - -const ( - mergeBucketSkip mergeBucket = iota - mergeBucketFailing - mergeBucketEligible -) - -// mergeGates bundles the per-tick inputs the classifier applies beyond the -// PR itself: the intent and review artifacts and the operator's -// required-check set. -type mergeGates struct { - enforceIntent bool - intentVerdicts map[string]intent.Verdict - requireReviewApproval bool - reviewArtifact review.Artifact - reviewLoaded bool - requiredChecks map[string]bool -} - -// classifyMergeEligibility is THE merge-eligibility rule: the one place that -// decides whether a PR goes to merge-eligible.json (the sweep would merge it -// now), ci-failing.json (its author has CI to fix), or neither. It returns -// the bucket and, for the dashboard, the same decision as a MergeVerdict with -// the reason spelled out (hivecommons/hive#7478): the pill is painted from -// this verdict, so green on the card means exactly what the sweep means by -// eligible, and never the looser "GitHub says mergeable". -// -// intentReason is non-empty only when the intent gate excluded the PR; the -// caller logs it (the log line carries the verdict tier, which this function -// does not need). -func classifyMergeEligibility(pr github.PullRequest, held bool, fullRepo string, g mergeGates) (bucket mergeBucket, verdict github.MergeVerdict, intentReason string) { - // blockedOrOutstanding is the verdict for a PR the sweep will not take - // for a reason of its own: the state still depends on what GitHub says, - // because a conflicting PR is blocked whatever else is outstanding, and - // one whose mergeability was never fetched is unknown, not amber. - blockedOrOutstanding := func(reason string) github.MergeVerdict { - switch pr.Mergeable { - case github.MergeableNo: - return github.MergeVerdict{State: github.MergeVerdictBlocked, Reason: notMergeableReason(pr)} - case github.MergeableUnknown: - return github.MergeVerdict{State: github.MergeVerdictUnknown, Reason: "mergeability not yet known; " + reason} - } - return github.MergeVerdict{State: github.MergeVerdictOutstanding, Reason: reason} - } - - if pr.Draft { - return mergeBucketSkip, github.MergeVerdict{State: github.MergeVerdictBlocked, Reason: "draft"}, "" - } - // intent.Verdict.BlocksMerge is the one shared refusal predicate; the - // App self-merge sweep gates on the same function (#6258). - if v, ok := g.intentVerdicts[fmt.Sprintf("%s/%d", fullRepo, pr.Number)]; ok && v.BlocksMerge(g.enforceIntent) { - reason := v.Reason - if v.Authorized && v.Alignment != nil && v.Alignment.Misaligned() { - reason = intent.ReasonAlignmentMisaligned + ": " + v.Alignment.Rationale - } - return mergeBucketSkip, blockedOrOutstanding("intent verification: " + reason), reason - } - - if pr.CIStatus == "failure" { - // A PR red ONLY on non-required checks (perma-red Playwright - // shards, coverage) that GitHub itself reports mergeable is NOT a - // failing PR — it is merge-eligible, mirroring the - // pending-but-mergeable rule below. Without this, every dependabot - // PR on a repo with permanently-red optional checks classified as - // "failure", landed in ci-failing.json where no sweep or agent - // would ever merge it, and accumulated indefinitely (observed on - // kubestellar/console 2026-08-28: 16 dependabot PRs, oldest 11 - // days). Gated on an operator-declared required-check set: with no - // set configured we cannot distinguish required from optional and - // keep the old fail-closed behavior. The merge step re-enforces - // branch protection, so this cannot merge anything GitHub blocks. - onlyOptionalRed := len(g.requiredChecks) > 0 && - !anyRequiredCheckFailing(pr.FailingChecks, g.requiredChecks) && - pr.Mergeable == github.MergeableYes - if !onlyOptionalRed { - reason := "CI failing" - if len(pr.FailingChecks) > 0 { - reason += ": " + strings.Join(pr.FailingChecks, ", ") - } - if len(g.requiredChecks) == 0 && pr.Mergeable == github.MergeableYes { - // GitHub calls it mergeable (unstable): nothing REQUIRED is - // red. The sweep still refuses it because, with no - // required-check set declared, it cannot tell optional from - // required. Say so — this is the shape #7478 was filed on. - reason += " (GitHub reports it mergeable; declare auto_merge.required_checks for the sweep to treat non-required checks as optional)" - } - return mergeBucketFailing, blockedOrOutstanding(reason), "" - } - } - - // The hold check sits AFTER the red classification on purpose - // (hivecommons/hive#7438): a held PR must never become merge-eligible, - // but a held RED PR is still its author's to repair. When this skip ran - // first, a level-held agent PR with a failing check vanished from - // ci-failing.json, its author never got a fix-before-new block for it, - // and it sat red and held until a human did the agent's repair. - if held { - return mergeBucketSkip, blockedOrOutstanding("held: a hold label keeps it out of the sweep"), "" - } - - // A PR whose CI is still "pending" is nonetheless merge-eligible when - // GitHub itself reports it as mergeable (mergeStateStatus=unstable): - // that state means every REQUIRED check has passed and only - // non-required checks remain outstanding. Those non-required checks — - // a cancelled Mobile Browser Tests, a still-running coverage-report, - // perpetually-pending tide — can never complete on their own, so - // waiting for CIStatus=="success" (all checks done) leaves cleanly - // mergeable PRs frozen out of the sweep indefinitely (observed - // 2026-08-04: three green console PRs stuck for hours). The merge step - // re-enforces branch protection, so trusting the mergeable verdict here - // cannot merge anything GitHub would actually block. - if pr.CIStatus == "pending" && pr.Mergeable != github.MergeableYes { - // Genuinely not ready: a required check is still running (or - // mergeability is unknown/no). Leave it out of both buckets, as - // before — it neither merges nor gets a fix dispatched. - return mergeBucketSkip, blockedOrOutstanding("CI pending"), "" - } - - if pr.Mergeable == github.MergeableNo { - // A conflicting PR cannot merge no matter how green its checks - // are. Listing it as merge-eligible left the eligible count stuck - // at N forever while nothing could actually merge (console - // #23002/#23003, 2026-08-31: the only two build-gate-green PRs - // were DIRTY go.mod dependabot bumps). Conflicts are the - // rebase/needs-human path's job, not the sweep's — keep them out - // of the eligible bucket. - return mergeBucketSkip, github.MergeVerdict{State: github.MergeVerdictBlocked, Reason: notMergeableReason(pr)}, "" - } - - if g.requireReviewApproval { - if !g.reviewLoaded { - return mergeBucketSkip, blockedOrOutstanding("review approval required, but review-verdicts.json is unavailable"), "" - } - if !g.reviewArtifact.HasAggregateApproval(fullRepo, pr.Number, pr.HeadSHA) { - return mergeBucketSkip, blockedOrOutstanding("awaiting review approval"), "" - } - } - - // Eligible. The reason names what GitHub still shows outstanding that - // the sweep chooses to ignore, so a green pill beside a red optional - // check does not read as "all green". - reason := "the sweep would merge this now" - switch { - case pr.CIStatus == "failure": - reason += " — only non-required checks are red (" + strings.Join(pr.FailingChecks, ", ") + ")" - case pr.CIStatus == "pending": - reason += " — non-required checks still pending (GitHub: " + pr.MergeableState + ")" - case pr.MergeableState == "unstable": - reason += " — non-required checks outstanding (GitHub: unstable)" - case pr.Mergeable == github.MergeableUnknown: - reason += " — mergeability not yet fetched; the sweep re-checks it at merge time" - } - return mergeBucketEligible, github.MergeVerdict{State: github.MergeVerdictEligible, Reason: reason}, "" -} - -// notMergeableReason names GitHub's own state (dirty, blocked, behind, ...) -// for a PR it reports as not mergeable; the state is what the operator has -// to resolve. -func notMergeableReason(pr github.PullRequest) string { - if pr.MergeableState != "" { - return "not mergeable on GitHub (" + pr.MergeableState + ")" - } - return "not mergeable on GitHub" -} - -func writeMergeEligible(actionable *github.ActionableResult, hold github.HoldResult, org string, escalatedPRs map[string]bool, enforceIntent bool, intentVerdicts map[string]intent.Verdict, requireReviewApproval bool, requiredChecks map[string]bool, holdDriftPRs map[string]bool, logger *slog.Logger) map[string]github.MergeVerdict { - // holdDriftPRs ("repo/number", same keying as holdSet) are PRs whose hold - // just lifted on a branch that MOVED while hold-gated (#5589). They are - // treated exactly like held PRs — invisible to both the eligible and the - // ci-failing buckets — because neither the merge sweep nor a fix agent - // should touch a branch whose unreviewed drift is awaiting a human. - verdicts := make(map[string]github.MergeVerdict) - holdSet := make(map[string]bool) - for _, h := range hold.Items { - key := fmt.Sprintf("%s/%d", h.Repo, h.Number) - holdSet[key] = true - } - for key := range holdDriftPRs { - holdSet[key] = true - } - for key := range holdDriftPRs { - holdSet[key] = true - } - - type eligiblePR struct { - Number int `json:"number"` - Repo string `json:"repo"` - Title string `json:"title"` - Author string `json:"author"` - Labels []string `json:"labels,omitempty"` - // CreatedAt is the PR's forge creation time — the reviewer lane's - // ordering key (#5617 item 4); see failingPR.CreatedAt. - CreatedAt time.Time `json:"created_at"` - // Mergeable is a tri-state string ("yes"/"no"/"unknown"), not a bool. - // A bool here defaulted to false for every PR, because the value was - // read from a list endpoint that never returns it. - Mergeable string `json:"mergeable"` - DCO string `json:"dco"` - // HeadSHA is the governor-observed head commit at the moment eligibility - // was decided. mergeTargetEligible compares the relay's expected SHA - // against this value (M4, CWE-367): a branch that moved after review - // no longer matches and fails closed. - HeadSHA string `json:"head_sha,omitempty"` - } - - type failingPR struct { - Number int `json:"number"` - Repo string `json:"repo"` - Title string `json:"title"` - Author string `json:"author"` - HeadSHA string `json:"head_sha,omitempty"` - // FailingChecks + Excerpt carry the raw CI evidence into the kick - // work list so fix agents see the actual error, not just "red". - FailingChecks []string `json:"failing_checks,omitempty"` - Excerpt string `json:"excerpt,omitempty"` - // Escalated marks PRs past the fix-loop breaker threshold: kick - // builders list them separately and agents must NOT dispatch more - // fix work for them. - Escalated bool `json:"escalated,omitempty"` - // Agent is the hive agent whose relay request opened this PR (from the - // audit trail's agent_pr_created entries). The scheduler's - // fix-before-new section routes each red PR back to its author; empty - // means unattributed (kick builders default it to scanner). - Agent string `json:"agent,omitempty"` - // Labels carries the PR's current labels into the kick builders. The - // reviewer lane (#5480) reads them to exclude PRs already carrying - // reviewer-passed — a PR that re-escalates after a reviewer pass - // belongs to a true human, never to another automated pass. - Labels []string `json:"labels,omitempty"` - // CreatedAt is the PR's forge creation time — the reviewer lane's - // ordering key (#5617 item 4). Its work list is capped at a few PRs - // per kick and documented "oldest first", but until this field the - // rows carried no age signal at all and were ordered by (repo name, PR - // number). Numbers are monotonic only WITHIN a repo, so that proxy - // sorted by repo NAME first and could starve an old escalated PR in a - // late-alphabet repo behind newer ones, on every kick, forever. - CreatedAt time.Time `json:"created_at"` - // HeadRef / HeadRepo / FromFork say where the red branch actually - // lives (hivecommons/hive#7386). The hive's App token pushes only to - // the base repository, so a fork PR is comment-only for every agent: - // ReachableAction spells that out ("push" | "comment-only") so no - // kick consumer has to discover it with a failed push — the failure - // mode that burned a scanner session and left a stray branch on the - // base repo under the fork's head-ref name. - HeadRef string `json:"head_ref,omitempty"` - HeadRepo string `json:"head_repo,omitempty"` - FromFork bool `json:"from_fork,omitempty"` - ReachableAction string `json:"reachable_action"` - // Held marks a PR carrying a hold label (the ACMM level gate's, or a - // human's). The hold is a MERGE checkpoint, not a repair checkpoint - // (hivecommons/hive#7438): a held red PR is still its author's to fix, - // so it is listed here with the flag rather than dropped — the owning - // agent's fix-before-new block says "fix CI, do not remove the hold". - Held bool `json:"held,omitempty"` - } - - prAgents := auditPRAgents(org, time.Now().Add(-auditPRAttributionWindow), "") - - var eligible []eligiblePR - var failing []failingPR - var reviewArtifact review.Artifact - reviewLoaded := false - if requireReviewApproval { - var err error - reviewArtifact, err = review.LoadArtifact("") - if err != nil { - logger.Warn("review approval required but review-verdicts.json is unavailable; merge eligibility will fail closed", "error", err) - } else { - reviewLoaded = true - } - } - // Two populations, one classifier (hivecommons/hive#7438). PRs.Items are - // the merge candidates. PRs.Held are PRs the hold gate removed from Items - // — they can never become merge-eligible, but a RED one still has to reach - // its authoring agent, otherwise it deadlocks: it stays red, so it stays - // held, so nothing ever repairs it. - type prCandidate struct { - pr github.PullRequest - held bool - } - candidates := make([]prCandidate, 0, len(actionable.PRs.Items)+len(actionable.PRs.Held)) - for _, pr := range actionable.PRs.Items { - candidates = append(candidates, prCandidate{pr: pr}) - } - for _, pr := range actionable.PRs.Held { - candidates = append(candidates, prCandidate{pr: pr, held: true}) - } - - gates := mergeGates{ - enforceIntent: enforceIntent, - intentVerdicts: intentVerdicts, - requireReviewApproval: requireReviewApproval, - reviewArtifact: reviewArtifact, - reviewLoaded: reviewLoaded, - requiredChecks: requiredChecks, - } - seen := make(map[string]bool, len(candidates)) - for _, cand := range candidates { - pr := cand.pr - key := fmt.Sprintf("%s/%d", pr.Repo, pr.Number) - if seen[key] { - continue - } - seen[key] = true - // The hold can arrive either as membership in PRs.Held or as a row in - // the hold snapshot; both mean the same thing here. - held := cand.held || holdSet[key] - fullRepo := fullRepoName(pr.Repo, org) - - bucket, verdict, intentReason := classifyMergeEligibility(pr, held, fullRepo, gates) - verdicts[github.MergeVerdictKey(pr)] = verdict - if intentReason != "" { - iv := intentVerdicts[fmt.Sprintf("%s/%d", fullRepo, pr.Number)] - logger.Info("excluding PR from merge-eligible due to intent verification", "repo", fullRepo, "number", pr.Number, "tier", iv.Tier, "reason", intentReason) - } - switch bucket { - case mergeBucketSkip: - continue - case mergeBucketFailing: - failing = append(failing, failingPR{ - Number: pr.Number, - Repo: fullRepo, - Title: pr.Title, - Author: pr.Author, - HeadSHA: pr.HeadSHA, - FailingChecks: pr.FailingChecks, - Excerpt: pr.CIFailureExcerpt, - Escalated: escalatedPRs[escalation.Key(fullRepo, pr.Number)], - Agent: prAgents[fmt.Sprintf("%s#%d", fullRepo, pr.Number)], - Labels: pr.Labels, - CreatedAt: pr.CreatedAt, - HeadRef: pr.HeadRef, - HeadRepo: pr.HeadRepo, - FromFork: pr.FromFork, - ReachableAction: github.ReachableAction(pr), - Held: held, - }) - continue - } - - dco := "unknown" - for _, l := range pr.Labels { - switch l { - case "dco-signoff: yes": - dco = "yes" - case "dco-signoff: no": - dco = "no" - } - } - eligible = append(eligible, eligiblePR{ - Number: pr.Number, - Repo: fullRepo, - Title: pr.Title, - Author: pr.Author, - Labels: pr.Labels, - CreatedAt: pr.CreatedAt, - Mergeable: mergeableJSON(pr.Mergeable), - DCO: dco, - HeadSHA: pr.HeadSHA, - }) - } - - _ = os.MkdirAll("/var/run/hive-metrics", 0o755) - - payload := map[string]any{ - "generated_at": time.Now().UTC().Format(time.RFC3339), - "merge_eligible": eligible, - } - data, err := json.Marshal(payload) - if err != nil { - logger.Warn("failed to marshal merge-eligible", "error", err) - return verdicts - } - atomicWrite(mergeEligiblePath, data) - logger.Info("merge-eligible.json updated", "eligible", len(eligible), "ci_failing", len(failing), "total_prs", len(actionable.PRs.Items)) - - failPayload := map[string]any{ - "generated_at": time.Now().UTC().Format(time.RFC3339), - "ci_failing": failing, - } - failData, err := json.Marshal(failPayload) - if err != nil { - logger.Warn("failed to marshal ci-failing", "error", err) - return verdicts - } - atomicWrite(ciFailingPath, failData) - return verdicts -} - func planReviewDispatch(cfg *config.Config, actionable *github.ActionableResult, agentMgr *agent.Manager, logger *slog.Logger) review.DispatchPlan { if cfg == nil || actionable == nil || !cfg.Review.RequireApproval || !cfg.Review.FanOut { return review.DispatchPlan{} @@ -9156,15 +8051,16 @@ func planReviewDispatch(cfg *config.Config, actionable *github.ActionableResult, }) } plan := review.PlanDispatch(prs, artifact, state, review.DispatchOptions{ - RequireApproval: cfg.Review.RequireApproval, - FanOut: cfg.Review.FanOut, - MaxParallelReviews: cfg.Review.EffectiveMaxParallelReviews(), - ReviewerAgents: cfg.Review.ReviewerAgents, - FixerAgent: cfg.Review.FixerAgent, - PostComments: cfg.Review.PostComments, - ProjectOrg: cfg.Project.Org, - AIAuthor: cfg.EffectiveAIAuthor(), - Agents: agents, + RequireApproval: cfg.Review.RequireApproval, + FanOut: cfg.Review.FanOut, + MaxParallelReviews: cfg.Review.EffectiveMaxParallelReviews(), + MaxPerspectivesPerPR: cfg.Review.MaxPerspectivesPerPR, + ReviewerAgents: cfg.Review.ReviewerAgents, + FixerAgent: cfg.Review.FixerAgent, + PostComments: cfg.Review.PostComments, + ProjectOrg: cfg.Project.Org, + AIAuthor: cfg.EffectiveAIAuthor(), + Agents: agents, }) if len(plan.ReviewKicks)+len(plan.FixKicks) > 0 { logger.Info("review swarm dispatch planned", "review_kicks", len(plan.ReviewKicks), "fix_kicks", len(plan.FixKicks)) @@ -9197,19 +8093,6 @@ func persistReviewDispatchState(plan review.DispatchPlan, delivered []review.Dis } } -// normalizedAutoMergeLabel resolves the configured queue label, falling back -// to the shared default when the value is blank. Client.SetAutoMergeLabel -// ignores blank input (keeping whatever was set before) and -// Client.AutoMergeLabel falls back on read, but the cmd layer normalizes -// eagerly too so a partially-populated config can never propagate an unnamed -// label to a fresh client. -func normalizedAutoMergeLabel(label string) string { - if label = strings.TrimSpace(label); label != "" { - return label - } - return github.AutoMergeQueuedLabel -} - func atomicWrite(path string, data []byte) { tmp := path + ".tmp" if os.WriteFile(tmp, data, 0o644) == nil { @@ -9563,6 +8446,19 @@ func parseEndpointList(raw string) []string { return out } +// Dashboard system-alert IDs for the budget thresholds. +const ( + budgetWarnAlertID = "budget-warn" + budgetExhaustedAlertID = "budget-exhausted" + // noCadenceAlertID is the never-kicked cause+fix banner (#5577): enabled + // agents with no cadence in any mode and no kick ever. + noCadenceAlertID = "agent-no-cadence" + // providerBudgetAlertID is the PROVIDER spend rebuff (#4294), kept distinct + // from the two token-budget alerts above so an operator can tell "we used + // our token allowance" from "the gateway will not spend more money". + providerBudgetAlertID = "provider-budget-exceeded" +) + func dispatchSubcommand(args []string, stdout, stderr io.Writer) (bool, int) { if len(args) == 0 { return false, 0 diff --git a/src/cmd/hive/merge_eligibility.go b/src/cmd/hive/merge_eligibility.go new file mode 100644 index 0000000000..6e9bb9bfda --- /dev/null +++ b/src/cmd/hive/merge_eligibility.go @@ -0,0 +1,647 @@ +package main + +import ( + "encoding/json" + "fmt" + "log/slog" + "os" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/escalation" + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/github/automerge" + "github.com/hivecommons/hive/pkg/intent" + "github.com/hivecommons/hive/pkg/review" +) + +// trustedMergerFunc resolves a GitHub login against the hive's authorized-users +// allowlist and reports whether it holds at least config.RoleMerger — the same +// bar requireMergerOrOwnerRole enforces on the dashboard queue endpoint (audit +// F3). +// +// Fails CLOSED: a nil config or a login absent from the allowlist is NOT +// trusted, so an unclassifiable actor can never merge. cfg is read on every +// call so a config reload that grants or revokes the merger tier takes effect +// without a restart. +func trustedMergerFunc(cfg *config.Config) automerge.MergerAuthorizer { + return func(login string) bool { + if cfg == nil || strings.TrimSpace(login) == "" { + return false + } + role, ok := cfg.Dashboard.AuthorizedRole(login) + if !ok { + return false + } + return config.RoleAtLeast(role, config.RoleMerger) + } +} + +// mergeEligiblePath is a var (not a const) only so tests can point +// mergeTargetEligible at a temp file; production never reassigns it. +var mergeEligiblePath = "/var/run/hive-metrics/merge-eligible.json" + +// acmmHoldGatedMinLevel / acmmHoldGatedMaxLevel bracket the ACMM levels whose +// merge policy is "hold-gated" — every agent-opened PR gets a "hold" label and +// no agent merges (see src/pkg/config/packs/level-{3,4,5}.yaml). L1/L2 are +// "manual" (agents open no PRs) and L6 is "auto-merge on green CI, no hold +// label", so both fall outside this range. Used by the F6 hold-label decider. +const ( + acmmHoldGatedMinLevel = 3 + acmmHoldGatedMaxLevel = 5 +) + +// shouldHoldAgentPR keeps public outreach claims human-reviewed even at L6, +// where ordinary agent PRs may auto-merge. The general ACMM hold gate remains +// unchanged for all roles at L3-L5. +func shouldHoldAgentPR(agentName string, level int) bool { + if strings.EqualFold(strings.TrimSpace(agentName), "outreach") { + return true + } + return level >= acmmHoldGatedMinLevel && level <= acmmHoldGatedMaxLevel +} + +// mergeableJSONUnknown is the explicit wire value for "mergeability was never +// determined". It is spelled out rather than left as "" so a consumer reading +// merge-eligible.json cannot mistake an unpopulated field for a definitive +// "no" — the failure mode that made every PR read as unmergeable. +const mergeableJSONUnknown = "unknown" + +// mergeTargetEligible reports whether (repo, number) currently appears in the +// governor's merge-eligible.json AT the expected head SHA. It reads the file +// FRESH on every call (never caches) because eligibility is recomputed each +// governor cycle — a stale cache could authorize a PR that has since fallen out +// of the list. On any read/parse error it returns false (FAIL CLOSED): if we +// cannot prove the target is eligible, we must not authorize the merge. +// +// M4 (CWE-367, TOCTOU): the governor records the head SHA it observed when it +// deemed the PR eligible (eligiblePR.HeadSHA). A branch can move between that +// review and the merge relay firing, so matching (repo, number) alone would let +// a moved head merge at a commit the governor never vetted. We therefore also +// require the entry's stored head_sha to equal expectSHA. A mismatch — or a +// stored SHA that is empty (governor could not observe it) — fails closed; the +// relay's SHA pin then fails the merge cleanly if a stale request slips through. +// +// merge-eligible.json stores repos as "owner/repo"; a MergeRequest.Repo may be +// bare ("repo") or fully qualified ("owner/repo"). We match on the bare repo +// name (the segment after the last "/") plus the number, so both request forms +// resolve to the same eligible entry without depending on the org prefix. +func mergeTargetEligible(repo string, number int, expectSHA string) bool { + data, err := os.ReadFile(mergeEligiblePath) + if err != nil { + return false // fail closed: no list ⇒ nothing is eligible + } + var payload struct { + Items []struct { + Number int `json:"number"` + Repo string `json:"repo"` + HeadSHA string `json:"head_sha"` + } `json:"merge_eligible"` + } + if err := json.Unmarshal(data, &payload); err != nil { + return false // fail closed: unparseable list ⇒ deny + } + want := bareRepoName(repo) + wantSHA := strings.TrimSpace(expectSHA) + for _, it := range payload.Items { + if it.Number == number && bareRepoName(it.Repo) == want { + // M4: bind authorization to the governor-observed head. An empty + // stored SHA cannot be proven to match, so it fails closed rather + // than authorizing an unpinned head. + return strings.TrimSpace(it.HeadSHA) != "" && strings.TrimSpace(it.HeadSHA) == wantSHA + } + } + return false +} + +// bareRepoName returns the repo segment after the last "/", so "owner/repo" and +// "repo" compare equal. Used to match a MergeRequest.Repo against the +// "owner/repo" entries in merge-eligible.json regardless of prefix. +func bareRepoName(repo string) string { + if i := strings.LastIndex(repo, "/"); i >= 0 { + return repo[i+1:] + } + return repo +} + +// bindMergeAuthz wraps the manager's agent/UID/CanMerge authorizer with the +// F4 target-binding checks (CWE-863). The inner authz owns the "may this agent +// merge at all" decision; this wrapper owns "is THIS specific target one the +// governor deemed eligible, at a pinned SHA". Both must pass before MergePR is +// reached. Ordering: run the agent/UID/CanMerge check first (cheapest, and it +// gives the clearest denial reason), then the SHA + eligible-list binding. +func bindMergeAuthz(inner func(agent string, fileUID int) error) github.MergeRequestAuthorizer { + return func(agent string, fileUID int, repo string, number int, expectSHA string) error { + if err := inner(agent, fileUID); err != nil { + return err + } + // (a) Require a pinned head SHA. An empty expectSHA means "merge whatever + // HEAD is now", which is the TOCTOU hole: a PR that was eligible when the + // governor last looked could have had a malicious commit pushed since. + // MergePR passes expectSHA as the required head SHA, so a moved head fails + // cleanly — but only if we insist it is set. + if strings.TrimSpace(expectSHA) == "" { + return fmt.Errorf("merge target %s#%d has no expected head SHA — refusing to merge an unpinned head (TOCTOU guard)", repo, number) + } + // (b) Require the target to be in the governor's current merge-eligible + // list AT the expected head SHA. This binds authorization to a PR the + // hive actually deemed landable this cycle, at the exact commit it + // reviewed, so an injected agent cannot request landing an arbitrary + // reachable PR (e.g. its own) whose required checks happen to pass, nor + // land an eligible PR at a head that moved after review (M4, CWE-367). + // Read fresh + fail closed (see mergeTargetEligible). + if !mergeTargetEligible(repo, number, expectSHA) { + return fmt.Errorf("merge target %s#%d is not in the current merge-eligible list at head %s — only governor-approved PRs may be landed via the merge relay, and only at the reviewed head SHA", repo, number, expectSHA) + } + return nil + } +} + +// mergeableJSON renders a tri-state mergeability verdict for the +// merge-eligible.json marker, mapping the unknown zero value to an explicit +// "unknown" rather than an empty string. +func mergeableJSON(m github.Mergeable) string { + if m == github.MergeableUnknown { + return mergeableJSONUnknown + } + return string(m) +} + +// anyRequiredCheckFailing reports whether any of a PR's failing check names +// is in the operator-declared required set. +func anyRequiredCheckFailing(failing []string, required map[string]bool) bool { + for _, name := range failing { + if required[name] { + return true + } + } + return false +} + +// mergeBucket is where the merge-eligible classifier files a PR: the +// merge-eligible.json list, the ci-failing.json list, or neither. +type mergeBucket int + +const ( + mergeBucketSkip mergeBucket = iota + mergeBucketFailing + mergeBucketEligible +) + +// mergeGates bundles the per-tick inputs the classifier applies beyond the +// PR itself: the intent and review artifacts and the operator's +// required-check set. +type mergeGates struct { + enforceIntent bool + intentVerdicts map[string]intent.Verdict + requireReviewApproval bool + reviewArtifact review.Artifact + reviewLoaded bool + requiredChecks map[string]bool +} + +// classifyMergeEligibility is THE merge-eligibility rule: the one place that +// decides whether a PR goes to merge-eligible.json (the sweep would merge it +// now), ci-failing.json (its author has CI to fix), or neither. It returns +// the bucket and, for the dashboard, the same decision as a MergeVerdict with +// the reason spelled out (hivecommons/hive#7478): the pill is painted from +// this verdict, so green on the card means exactly what the sweep means by +// eligible, and never the looser "GitHub says mergeable". +// +// intentReason is non-empty only when the intent gate excluded the PR; the +// caller logs it (the log line carries the verdict tier, which this function +// does not need). +func classifyMergeEligibility(pr github.PullRequest, held bool, fullRepo string, g mergeGates) (bucket mergeBucket, verdict github.MergeVerdict, intentReason string) { + // blockedOrOutstanding is the verdict for a PR the sweep will not take + // for a reason of its own: the state still depends on what GitHub says, + // because a conflicting PR is blocked whatever else is outstanding, and + // one whose mergeability was never fetched is unknown, not amber. The + // sweep's reason is kept in every state (hivecommons/hive#7515): on a + // protected branch a red required check or a missing review is exactly + // what makes GitHub say "blocked", so dropping it for the bare enum sent + // the operator to GitHub to learn what this function already knew. + blockedOrOutstanding := func(reason string) github.MergeVerdict { + switch pr.Mergeable { + case github.MergeableNo: + return github.MergeVerdict{State: github.MergeVerdictBlocked, Reason: notMergeableReason(pr, reason)} + case github.MergeableUnknown: + return github.MergeVerdict{State: github.MergeVerdictUnknown, Reason: mergeabilityUnknownReason + "; " + reason} + } + return github.MergeVerdict{State: github.MergeVerdictOutstanding, Reason: reason} + } + + if pr.Draft { + return mergeBucketSkip, github.MergeVerdict{State: github.MergeVerdictBlocked, Reason: "draft — mark ready for review to enter the sweep"}, "" + } + // intent.Verdict.BlocksMerge is the one shared refusal predicate; the + // App self-merge sweep gates on the same function (#6258). + if v, ok := g.intentVerdicts[fmt.Sprintf("%s/%d", fullRepo, pr.Number)]; ok && v.BlocksMerge(g.enforceIntent) { + reason := v.Reason + if v.Authorized && v.Alignment != nil && v.Alignment.Misaligned() { + reason = intent.ReasonAlignmentMisaligned + ": " + v.Alignment.Rationale + } + return mergeBucketSkip, blockedOrOutstanding("intent verification: " + reason), reason + } + + if pr.CIStatus == "failure" { + // A PR red ONLY on non-required checks (perma-red Playwright + // shards, coverage) that GitHub itself reports mergeable is NOT a + // failing PR — it is merge-eligible, mirroring the + // pending-but-mergeable rule below. Without this, every dependabot + // PR on a repo with permanently-red optional checks classified as + // "failure", landed in ci-failing.json where no sweep or agent + // would ever merge it, and accumulated indefinitely (observed on + // kubestellar/console 2026-08-28: 16 dependabot PRs, oldest 11 + // days). Gated on an operator-declared required-check set: with no + // set configured we cannot distinguish required from optional and + // keep the old fail-closed behavior. The merge step re-enforces + // branch protection, so this cannot merge anything GitHub blocks. + onlyOptionalRed := len(g.requiredChecks) > 0 && + !anyRequiredCheckFailing(pr.FailingChecks, g.requiredChecks) && + pr.Mergeable == github.MergeableYes + if !onlyOptionalRed { + reason := "CI failing" + if len(pr.FailingChecks) > 0 { + reason += ": " + strings.Join(pr.FailingChecks, ", ") + } + if len(g.requiredChecks) == 0 && pr.Mergeable == github.MergeableYes { + // GitHub calls it mergeable (unstable): nothing REQUIRED is + // red. The sweep still refuses it because, with no + // required-check set declared, it cannot tell optional from + // required. Say so — this is the shape #7478 was filed on. + reason += " (GitHub reports it mergeable; declare auto_merge.required_checks for the sweep to treat non-required checks as optional)" + } + return mergeBucketFailing, blockedOrOutstanding(reason), "" + } + } + + // The hold check sits AFTER the red classification on purpose + // (hivecommons/hive#7438): a held PR must never become merge-eligible, + // but a held RED PR is still its author's to repair. When this skip ran + // first, a level-held agent PR with a failing check vanished from + // ci-failing.json, its author never got a fix-before-new block for it, + // and it sat red and held until a human did the agent's repair. + if held { + return mergeBucketSkip, blockedOrOutstanding("held: a hold label keeps it out of the sweep"), "" + } + + // A PR whose CI is still "pending" is nonetheless merge-eligible when + // GitHub itself reports it as mergeable (mergeStateStatus=unstable): + // that state means every REQUIRED check has passed and only + // non-required checks remain outstanding. Those non-required checks — + // a cancelled Mobile Browser Tests, a still-running coverage-report, + // perpetually-pending tide — can never complete on their own, so + // waiting for CIStatus=="success" (all checks done) leaves cleanly + // mergeable PRs frozen out of the sweep indefinitely (observed + // 2026-08-04: three green console PRs stuck for hours). The merge step + // re-enforces branch protection, so trusting the mergeable verdict here + // cannot merge anything GitHub would actually block. + if pr.CIStatus == "pending" && pr.Mergeable != github.MergeableYes { + // Genuinely not ready: a required check is still running (or + // mergeability is unknown/no). Leave it out of both buckets, as + // before — it neither merges nor gets a fix dispatched. + return mergeBucketSkip, blockedOrOutstanding("CI pending"), "" + } + + // The review gate runs BEFORE the GitHub-says-no return below so that a + // PR GitHub calls "blocked" for want of a review carries that reason + // (hivecommons/hive#7515). Both paths file a MergeableNo PR in the skip + // bucket, so the order changes only the verdict's wording. + if g.requireReviewApproval { + if !g.reviewLoaded { + return mergeBucketSkip, blockedOrOutstanding("review approval required, but review-verdicts.json is unavailable"), "" + } + if !g.reviewArtifact.HasAggregateApproval(fullRepo, pr.Number, pr.HeadSHA) { + return mergeBucketSkip, blockedOrOutstanding("awaiting review approval"), "" + } + } + + if pr.Mergeable == github.MergeableNo { + // A conflicting PR cannot merge no matter how green its checks + // are. Listing it as merge-eligible left the eligible count stuck + // at N forever while nothing could actually merge (console + // #23002/#23003, 2026-08-31: the only two build-gate-green PRs + // were DIRTY go.mod dependabot bumps). Conflicts are the + // rebase/needs-human path's job, not the sweep's — keep them out + // of the eligible bucket. No sweep gate explains this one: for + // "blocked" that means a branch-protection rule we do not read yet + // (a review GitHub requires, a required check that never reported); + // notMergeableReason says so rather than the bare enum. + return mergeBucketSkip, github.MergeVerdict{State: github.MergeVerdictBlocked, Reason: notMergeableReason(pr, "")}, "" + } + + // Eligible. The reason names what GitHub still shows outstanding that + // the sweep chooses to ignore, so a green pill beside a red optional + // check does not read as "all green". + reason := "the sweep would merge this now" + switch { + case pr.CIStatus == "failure": + reason += " — only non-required checks are red (" + strings.Join(pr.FailingChecks, ", ") + ")" + case pr.CIStatus == "pending": + reason += " — non-required checks still pending (GitHub: " + pr.MergeableState + ")" + case pr.MergeableState == "unstable": + reason += " — non-required checks outstanding (GitHub: unstable)" + case pr.Mergeable == github.MergeableUnknown: + reason += " — mergeability not yet fetched; the sweep re-checks it at merge time" + } + return mergeBucketEligible, github.MergeVerdict{State: github.MergeVerdictEligible, Reason: reason}, "" +} + +// mergeabilityUnknownReason is the verdict prefix for a PR whose +// mergeability GitHub has not computed yet (or the fetch failed); the +// classifier appends the sweep's own reason after it. +const mergeabilityUnknownReason = "mergeability not yet computed by GitHub — re-checked next tick" + +// notMergeableReason explains, in words an operator can act on, why GitHub +// reports a PR as not mergeable — what to do, not the API enum +// (hivecommons/hive#7515). sweepReason is the gate the sweep itself failed +// the PR on, or "" when every sweep gate passed: +// +// - "blocked" folds every unsatisfied branch-protection rule into one +// word. When the sweep has a reason it is almost always the rule +// ("blocked — CI failing: build"); when GitHub's own facts name a +// different rule (a review decision, a required check that never +// reported) that rule is named too; with neither, say that a rule we +// cannot read is unsatisfied rather than nothing at all. +// - "dirty" and "behind" name the base branch and the fix (rebase / +// update); a sweep reason is appended, since it still stands once the +// branch is fixed. +// - Any other state falls back to naming it. +func notMergeableReason(pr github.PullRequest, sweepReason string) string { + base, from := pr.BaseRef, "the base branch" + if base == "" { + base, from = "the base branch", "it" + } + var msg string + switch pr.MergeableState { + case "blocked": + // The branch-protection rule GitHub is hiding behind the word + // "blocked", when the sweep collected enough to name it + // (hivecommons/hive#7515 step 2). The wording itself lives in + // github.PullRequest.BranchProtectionBlockReason — one place, under + // test — not in this switch and not in the dashboard's JS. + rule, ruleKnown := pr.BranchProtectionBlockReason() + switch { + case sweepReason == "" && ruleKnown: + return "blocked — " + rule + case sweepReason == "": + return "blocked — all sweep gates pass; a branch-protection rule is unsatisfied" + case ruleKnown && !strings.Contains(sweepReason, rule): + // Both are true and neither subsumes the other: the sweep's own + // gate is what it will act on, and GitHub's rule is what the + // operator must also clear. + return "blocked — " + sweepReason + "; GitHub also requires: " + rule + } + return "blocked — " + sweepReason + case "dirty": + msg = "has merge conflicts with " + base + " — needs a rebase" + case "behind": + msg = "behind " + base + " — needs an update from " + from + case "": + msg = "not mergeable on GitHub" + default: + msg = "not mergeable on GitHub (" + pr.MergeableState + ")" + } + if sweepReason != "" { + msg += "; also " + sweepReason + } + return msg +} + +func writeMergeEligible(actionable *github.ActionableResult, hold github.HoldResult, org string, escalatedPRs map[string]bool, enforceIntent bool, intentVerdicts map[string]intent.Verdict, requireReviewApproval bool, requiredChecks map[string]bool, holdDriftPRs map[string]bool, logger *slog.Logger) map[string]github.MergeVerdict { + // holdDriftPRs ("repo/number", same keying as holdSet) are PRs whose hold + // just lifted on a branch that MOVED while hold-gated (#5589). They are + // treated exactly like held PRs — invisible to both the eligible and the + // ci-failing buckets — because neither the merge sweep nor a fix agent + // should touch a branch whose unreviewed drift is awaiting a human. + verdicts := make(map[string]github.MergeVerdict) + holdSet := make(map[string]bool) + for _, h := range hold.Items { + key := fmt.Sprintf("%s/%d", h.Repo, h.Number) + holdSet[key] = true + } + for key := range holdDriftPRs { + holdSet[key] = true + } + for key := range holdDriftPRs { + holdSet[key] = true + } + + type eligiblePR struct { + Number int `json:"number"` + Repo string `json:"repo"` + Title string `json:"title"` + Author string `json:"author"` + Labels []string `json:"labels,omitempty"` + // CreatedAt is the PR's forge creation time — the reviewer lane's + // ordering key (#5617 item 4); see failingPR.CreatedAt. + CreatedAt time.Time `json:"created_at"` + // Mergeable is a tri-state string ("yes"/"no"/"unknown"), not a bool. + // A bool here defaulted to false for every PR, because the value was + // read from a list endpoint that never returns it. + Mergeable string `json:"mergeable"` + DCO string `json:"dco"` + // HeadSHA is the governor-observed head commit at the moment eligibility + // was decided. mergeTargetEligible compares the relay's expected SHA + // against this value (M4, CWE-367): a branch that moved after review + // no longer matches and fails closed. + HeadSHA string `json:"head_sha,omitempty"` + } + + type failingPR struct { + Number int `json:"number"` + Repo string `json:"repo"` + Title string `json:"title"` + Author string `json:"author"` + HeadSHA string `json:"head_sha,omitempty"` + // FailingChecks + Excerpt carry the raw CI evidence into the kick + // work list so fix agents see the actual error, not just "red". + FailingChecks []string `json:"failing_checks,omitempty"` + Excerpt string `json:"excerpt,omitempty"` + // Escalated marks PRs past the fix-loop breaker threshold: kick + // builders list them separately and agents must NOT dispatch more + // fix work for them. + Escalated bool `json:"escalated,omitempty"` + // Agent is the hive agent whose relay request opened this PR (from the + // audit trail's agent_pr_created entries). The scheduler's + // fix-before-new section routes each red PR back to its author; empty + // means unattributed (kick builders default it to scanner). + Agent string `json:"agent,omitempty"` + // Labels carries the PR's current labels into the kick builders. The + // reviewer lane (#5480) reads them to exclude PRs already carrying + // reviewer-passed — a PR that re-escalates after a reviewer pass + // belongs to a true human, never to another automated pass. + Labels []string `json:"labels,omitempty"` + // CreatedAt is the PR's forge creation time — the reviewer lane's + // ordering key (#5617 item 4). Its work list is capped at a few PRs + // per kick and documented "oldest first", but until this field the + // rows carried no age signal at all and were ordered by (repo name, PR + // number). Numbers are monotonic only WITHIN a repo, so that proxy + // sorted by repo NAME first and could starve an old escalated PR in a + // late-alphabet repo behind newer ones, on every kick, forever. + CreatedAt time.Time `json:"created_at"` + // HeadRef / HeadRepo / FromFork say where the red branch actually + // lives (hivecommons/hive#7386). The hive's App token pushes only to + // the base repository, so a fork PR is comment-only for every agent: + // ReachableAction spells that out ("push" | "comment-only") so no + // kick consumer has to discover it with a failed push — the failure + // mode that burned a scanner session and left a stray branch on the + // base repo under the fork's head-ref name. + HeadRef string `json:"head_ref,omitempty"` + HeadRepo string `json:"head_repo,omitempty"` + FromFork bool `json:"from_fork,omitempty"` + ReachableAction string `json:"reachable_action"` + // Held marks a PR carrying a hold label (the ACMM level gate's, or a + // human's). The hold is a MERGE checkpoint, not a repair checkpoint + // (hivecommons/hive#7438): a held red PR is still its author's to fix, + // so it is listed here with the flag rather than dropped — the owning + // agent's fix-before-new block says "fix CI, do not remove the hold". + Held bool `json:"held,omitempty"` + } + + prAgents := auditPRAgents(org, time.Now().Add(-auditPRAttributionWindow), "") + + var eligible []eligiblePR + var failing []failingPR + var reviewArtifact review.Artifact + reviewLoaded := false + if requireReviewApproval { + var err error + reviewArtifact, err = review.LoadArtifact("") + if err != nil { + logger.Warn("review approval required but review-verdicts.json is unavailable; merge eligibility will fail closed", "error", err) + } else { + reviewLoaded = true + } + } + // Two populations, one classifier (hivecommons/hive#7438). PRs.Items are + // the merge candidates. PRs.Held are PRs the hold gate removed from Items + // — they can never become merge-eligible, but a RED one still has to reach + // its authoring agent, otherwise it deadlocks: it stays red, so it stays + // held, so nothing ever repairs it. + type prCandidate struct { + pr github.PullRequest + held bool + } + candidates := make([]prCandidate, 0, len(actionable.PRs.Items)+len(actionable.PRs.Held)) + for _, pr := range actionable.PRs.Items { + candidates = append(candidates, prCandidate{pr: pr}) + } + for _, pr := range actionable.PRs.Held { + candidates = append(candidates, prCandidate{pr: pr, held: true}) + } + + gates := mergeGates{ + enforceIntent: enforceIntent, + intentVerdicts: intentVerdicts, + requireReviewApproval: requireReviewApproval, + reviewArtifact: reviewArtifact, + reviewLoaded: reviewLoaded, + requiredChecks: requiredChecks, + } + seen := make(map[string]bool, len(candidates)) + for _, cand := range candidates { + pr := cand.pr + key := fmt.Sprintf("%s/%d", pr.Repo, pr.Number) + if seen[key] { + continue + } + seen[key] = true + // The hold can arrive either as membership in PRs.Held or as a row in + // the hold snapshot; both mean the same thing here. + held := cand.held || holdSet[key] + fullRepo := fullRepoName(pr.Repo, org) + + bucket, verdict, intentReason := classifyMergeEligibility(pr, held, fullRepo, gates) + verdicts[github.MergeVerdictKey(pr)] = verdict + if intentReason != "" { + iv := intentVerdicts[fmt.Sprintf("%s/%d", fullRepo, pr.Number)] + logger.Info("excluding PR from merge-eligible due to intent verification", "repo", fullRepo, "number", pr.Number, "tier", iv.Tier, "reason", intentReason) + } + switch bucket { + case mergeBucketSkip: + continue + case mergeBucketFailing: + failing = append(failing, failingPR{ + Number: pr.Number, + Repo: fullRepo, + Title: pr.Title, + Author: pr.Author, + HeadSHA: pr.HeadSHA, + FailingChecks: pr.FailingChecks, + Excerpt: pr.CIFailureExcerpt, + Escalated: escalatedPRs[escalation.Key(fullRepo, pr.Number)], + Agent: prAgents[fmt.Sprintf("%s#%d", fullRepo, pr.Number)], + Labels: pr.Labels, + CreatedAt: pr.CreatedAt, + HeadRef: pr.HeadRef, + HeadRepo: pr.HeadRepo, + FromFork: pr.FromFork, + ReachableAction: github.ReachableAction(pr), + Held: held, + }) + continue + } + + dco := "unknown" + for _, l := range pr.Labels { + switch l { + case "dco-signoff: yes": + dco = "yes" + case "dco-signoff: no": + dco = "no" + } + } + eligible = append(eligible, eligiblePR{ + Number: pr.Number, + Repo: fullRepo, + Title: pr.Title, + Author: pr.Author, + Labels: pr.Labels, + CreatedAt: pr.CreatedAt, + Mergeable: mergeableJSON(pr.Mergeable), + DCO: dco, + HeadSHA: pr.HeadSHA, + }) + } + + _ = os.MkdirAll("/var/run/hive-metrics", 0o755) + + payload := map[string]any{ + "generated_at": time.Now().UTC().Format(time.RFC3339), + "merge_eligible": eligible, + } + data, err := json.Marshal(payload) + if err != nil { + logger.Warn("failed to marshal merge-eligible", "error", err) + return verdicts + } + atomicWrite(mergeEligiblePath, data) + logger.Info("merge-eligible.json updated", "eligible", len(eligible), "ci_failing", len(failing), "total_prs", len(actionable.PRs.Items)) + + failPayload := map[string]any{ + "generated_at": time.Now().UTC().Format(time.RFC3339), + "ci_failing": failing, + } + failData, err := json.Marshal(failPayload) + if err != nil { + logger.Warn("failed to marshal ci-failing", "error", err) + return verdicts + } + atomicWrite(ciFailingPath, failData) + return verdicts +} + +// normalizedAutoMergeLabel resolves the configured queue label, falling back +// to the shared default when the value is blank. Client.SetAutoMergeLabel +// ignores blank input (keeping whatever was set before) and +// Client.AutoMergeLabel falls back on read, but the cmd layer normalizes +// eagerly too so a partially-populated config can never propagate an unnamed +// label to a fresh client. +func normalizedAutoMergeLabel(label string) string { + if label = strings.TrimSpace(label); label != "" { + return label + } + return github.AutoMergeQueuedLabel +} diff --git a/src/cmd/hive/merge_verdict_7478_test.go b/src/cmd/hive/merge_verdict_7478_test.go index a521ec32aa..7ded86995c 100644 --- a/src/cmd/hive/merge_verdict_7478_test.go +++ b/src/cmd/hive/merge_verdict_7478_test.go @@ -88,32 +88,36 @@ func TestClassifyMergeEligibility_VerdictTracksBucket(t *testing.T) { wantReason: []string{"pending", "unstable"}, }, { - name: "pending with mergeability unknown is unknown, not amber", + name: "pending with mergeability unknown is unknown, not amber, and keeps the sweep reason", pr: github.PullRequest{Number: 6, CIStatus: "pending"}, wantBucket: mergeBucketSkip, wantState: github.MergeVerdictUnknown, - wantReason: []string{"not yet known"}, + wantReason: []string{"not yet computed by GitHub", "re-checked next tick", "CI pending"}, }, { - name: "dirty is blocked and names the GitHub state", - pr: github.PullRequest{Number: 1259, Mergeable: github.MergeableNo, MergeableState: "dirty", CIStatus: "failure", FailingChecks: []string{"build"}}, + // Conflicts read as what to do, name the base branch, and keep + // the sweep's own reason — it still stands after the rebase. + name: "dirty is blocked, names the base branch and the fix, and keeps the sweep reason", + pr: github.PullRequest{Number: 1259, Mergeable: github.MergeableNo, MergeableState: "dirty", BaseRef: "v4", CIStatus: "failure", FailingChecks: []string{"build"}}, wantBucket: mergeBucketFailing, wantState: github.MergeVerdictBlocked, - wantReason: []string{"not mergeable", "dirty"}, + wantReason: []string{"merge conflicts with v4", "needs a rebase", "CI failing: build"}, }, { - name: "blocked with green CI is blocked", + // GitHub "blocked" with every sweep gate green: the sweep cannot + // name the rule yet, but must say that, not just "blocked". + name: "blocked with green CI is blocked and says a branch-protection rule is unsatisfied", pr: github.PullRequest{Number: 603, Mergeable: github.MergeableNo, MergeableState: "blocked", CIStatus: "success"}, wantBucket: mergeBucketSkip, wantState: github.MergeVerdictBlocked, - wantReason: []string{"blocked"}, + wantReason: []string{"blocked — all sweep gates pass", "branch-protection rule"}, }, { - name: "a draft is blocked", + name: "a draft is blocked and says how to enter the sweep", pr: github.PullRequest{Number: 7, Draft: true, Mergeable: yes, CIStatus: "success"}, wantBucket: mergeBucketSkip, wantState: github.MergeVerdictBlocked, - wantReason: []string{"draft"}, + wantReason: []string{"draft — mark ready for review"}, }, { name: "review approval required and missing is outstanding", @@ -143,13 +147,14 @@ func TestClassifyMergeEligibility_VerdictTracksBucket(t *testing.T) { }, { // A conflicting PR the sweep also refuses for another reason is - // still BLOCKED: the conflict is what the operator resolves first. - name: "held AND dirty is blocked, not amber", + // still BLOCKED: the conflict is what the operator resolves first + // — and the hold is still named, since it outlives the rebase. + name: "held AND dirty is blocked, not amber, and still names the hold", pr: github.PullRequest{Number: 11, Mergeable: github.MergeableNo, MergeableState: "dirty", CIStatus: "success"}, held: true, wantBucket: mergeBucketSkip, wantState: github.MergeVerdictBlocked, - wantReason: []string{"dirty"}, + wantReason: []string{"merge conflicts with the base branch", "held"}, }, } for _, tc := range cases { diff --git a/src/cmd/hive/merge_verdict_7515_test.go b/src/cmd/hive/merge_verdict_7515_test.go new file mode 100644 index 0000000000..6f4047a5b3 --- /dev/null +++ b/src/cmd/hive/merge_verdict_7515_test.go @@ -0,0 +1,189 @@ +package main + +import ( + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/intent" +) + +// hivecommons/hive#7515: hovering a non-green PR pill must say, in plain +// words, why the sweep will not merge it and what would unblock it. Before +// this, a PR GitHub reported as not mergeable got the bare GitHub enum +// ("not mergeable on GitHub (blocked)") even when the classifier had just +// been handed the exact gate — "CI failing: build, lint", "awaiting review +// approval", "held: …" — that made GitHub say blocked. These cases pin the +// wording for every row of the issue's table so the reason cannot be +// dropped again. +func TestClassifyMergeEligibility_BlockedKeepsSweepReason(t *testing.T) { + no := github.MergeableNo + blockedPR := func(n int, ci string, failing ...string) github.PullRequest { + return github.PullRequest{Number: n, Mergeable: no, MergeableState: "blocked", BaseRef: "v4", CIStatus: ci, FailingChecks: failing} + } + cases := []struct { + name string + pr github.PullRequest + held bool + gates mergeGates + wantBucket mergeBucket + wantState github.MergeVerdictState + wantReason string // exact + }{ + { + name: "required check red", + pr: blockedPR(1, "failure", "build", "lint"), + gates: mergeGates{requiredChecks: map[string]bool{"build": true, "lint": true}}, + wantBucket: mergeBucketFailing, + wantState: github.MergeVerdictBlocked, + wantReason: "blocked — CI failing: build, lint", + }, + { + // The review gate used to sit behind the GitHub-says-no return, + // so a PR blocked for want of a review never reached it. + name: "review missing", + pr: blockedPR(2, "success"), + gates: mergeGates{requireReviewApproval: true, reviewLoaded: true}, + wantBucket: mergeBucketSkip, + wantState: github.MergeVerdictBlocked, + wantReason: "blocked — awaiting review approval", + }, + { + name: "review artifact unavailable", + pr: blockedPR(3, "success"), + gates: mergeGates{requireReviewApproval: true, reviewLoaded: false}, + wantBucket: mergeBucketSkip, + wantState: github.MergeVerdictBlocked, + wantReason: "blocked — review approval required, but review-verdicts.json is unavailable", + }, + { + name: "hold label", + pr: blockedPR(4, "success"), + held: true, + wantBucket: mergeBucketSkip, + wantState: github.MergeVerdictBlocked, + wantReason: "blocked — held: a hold label keeps it out of the sweep", + }, + { + name: "intent verdict", + pr: blockedPR(5, "success"), + gates: mergeGates{enforceIntent: true, intentVerdicts: map[string]intent.Verdict{ + "org/repo/5": {AgentPR: true, Authorized: false, Reason: "no authorizing issue"}, + }}, + wantBucket: mergeBucketSkip, + wantState: github.MergeVerdictBlocked, + wantReason: "blocked — intent verification: no authorizing issue", + }, + { + // A required check still running is why GitHub says blocked. + name: "required check pending", + pr: blockedPR(6, "pending"), + wantBucket: mergeBucketSkip, + wantState: github.MergeVerdictBlocked, + wantReason: "blocked — CI pending", + }, + { + // Every sweep gate passes and GitHub still says blocked: a + // branch-protection rule the sweep does not read yet. Say so. + name: "no sweep gate explains it", + pr: blockedPR(7, "success"), + wantBucket: mergeBucketSkip, + wantState: github.MergeVerdictBlocked, + wantReason: "blocked — all sweep gates pass; a branch-protection rule is unsatisfied", + }, + { + name: "conflicts", + pr: github.PullRequest{Number: 8, Mergeable: no, MergeableState: "dirty", BaseRef: "v4", CIStatus: "success"}, + wantBucket: mergeBucketSkip, + wantState: github.MergeVerdictBlocked, + wantReason: "has merge conflicts with v4 — needs a rebase", + }, + { + name: "conflicts with a sweep reason on top", + pr: github.PullRequest{Number: 9, Mergeable: no, MergeableState: "dirty", BaseRef: "v4", CIStatus: "failure", FailingChecks: []string{"build"}}, + wantBucket: mergeBucketFailing, + wantState: github.MergeVerdictBlocked, + wantReason: "has merge conflicts with v4 — needs a rebase; also CI failing: build", + }, + { + name: "behind", + pr: github.PullRequest{Number: 10, Mergeable: no, MergeableState: "behind", BaseRef: "v4", CIStatus: "success"}, + wantBucket: mergeBucketSkip, + wantState: github.MergeVerdictBlocked, + wantReason: "behind v4 — needs an update from the base branch", + }, + { + // An abbreviated payload with no base ref still reads sensibly. + name: "behind with the base branch unknown", + pr: github.PullRequest{Number: 11, Mergeable: no, MergeableState: "behind", CIStatus: "success"}, + wantBucket: mergeBucketSkip, + wantState: github.MergeVerdictBlocked, + wantReason: "behind the base branch — needs an update from it", + }, + { + name: "draft", + pr: github.PullRequest{Number: 12, Draft: true, Mergeable: github.MergeableYes, CIStatus: "success"}, + wantBucket: mergeBucketSkip, + wantState: github.MergeVerdictBlocked, + wantReason: "draft — mark ready for review to enter the sweep", + }, + { + name: "unknown", + pr: github.PullRequest{Number: 13, CIStatus: "pending"}, + wantBucket: mergeBucketSkip, + wantState: github.MergeVerdictUnknown, + wantReason: "mergeability not yet computed by GitHub — re-checked next tick; CI pending", + }, + { + // A GitHub state this function has no wording for still names + // the state and keeps the sweep reason. + name: "an unrecognised GitHub state falls back to naming it", + pr: github.PullRequest{Number: 14, Mergeable: no, MergeableState: "draft", CIStatus: "success"}, + held: true, + wantBucket: mergeBucketSkip, + wantState: github.MergeVerdictBlocked, + wantReason: "not mergeable on GitHub (draft); also held: a hold label keeps it out of the sweep", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + bucket, verdict := mergeVerdictOf(t, tc.pr, tc.held, tc.gates) + if bucket != tc.wantBucket { + t.Errorf("bucket = %v, want %v", bucket, tc.wantBucket) + } + if verdict.State != tc.wantState { + t.Errorf("state = %q, want %q", verdict.State, tc.wantState) + } + if verdict.Reason != tc.wantReason { + t.Errorf("reason = %q\n want %q", verdict.Reason, tc.wantReason) + } + // The bare enum is never the whole story any more. + for _, enum := range []string{"(blocked)", "(dirty)", "(behind)"} { + if strings.Contains(verdict.Reason, enum) { + t.Errorf("reason %q still shows GitHub's raw state %s", verdict.Reason, enum) + } + } + }) + } +} + +// Moving the review gate ahead of the GitHub-says-no return must not change +// which bucket anything lands in: a MergeableNo PR skips either way, and a +// mergeable one never reached that return. The verdict is the only thing +// the order affects. +func TestClassifyMergeEligibility_ReviewGateOrderKeepsBuckets(t *testing.T) { + gates := mergeGates{requireReviewApproval: true, reviewLoaded: true} + for _, pr := range []github.PullRequest{ + {Number: 1, Mergeable: github.MergeableNo, MergeableState: "blocked", CIStatus: "success"}, + {Number: 2, Mergeable: github.MergeableNo, MergeableState: "dirty", CIStatus: "success"}, + {Number: 3, Mergeable: github.MergeableYes, MergeableState: "clean", CIStatus: "success"}, + } { + bucket, verdict := mergeVerdictOf(t, pr, false, gates) + if bucket != mergeBucketSkip { + t.Errorf("#%d: bucket = %v, want skip (reason %q)", pr.Number, bucket, verdict.Reason) + } + if !strings.Contains(verdict.Reason, "awaiting review approval") { + t.Errorf("#%d: reason %q does not name the missing review", pr.Number, verdict.Reason) + } + } +} diff --git a/src/cmd/hive/merge_verdict_rule_7515_test.go b/src/cmd/hive/merge_verdict_rule_7515_test.go new file mode 100644 index 0000000000..271c9aad16 --- /dev/null +++ b/src/cmd/hive/merge_verdict_rule_7515_test.go @@ -0,0 +1,170 @@ +package main + +import ( + "testing" + + "github.com/hivecommons/hive/pkg/github" +) + +// hivecommons/hive#7515 step 2: when GitHub says "blocked", the tooltip must +// name the branch-protection rule that is unsatisfied, not the word +// "blocked". Step 1 (#7516) kept the SWEEP's own gate when it had one; these +// cases pin what happens with GitHub's own facts — a review decision, a red +// or absent required check — which the sweep now collects during enumeration. +// +// The wording itself lives in github.PullRequest.BranchProtectionBlockReason +// (pinned by TestBranchProtectionBlockReason); what is pinned here is that +// the classifier reaches for it, and that it still declines to guess. +func TestClassifyMergeEligibility_BlockedNamesTheProtectionRule(t *testing.T) { + blocked := func(p *github.ProtectionFacts) github.PullRequest { + return github.PullRequest{ + Number: 1, + Mergeable: github.MergeableNo, + MergeableState: "blocked", + BaseRef: "v4", + CIStatus: "success", + Protection: p, + } + } + + cases := []struct { + name string + pr github.PullRequest + gates mergeGates + wantReason string // exact + }{ + { + name: "changes requested is named where the placeholder used to be", + pr: blocked(&github.ProtectionFacts{ + ReviewDecision: github.ReviewDecisionChangesRequested, + ChangesRequestedBy: []string{"reviewer"}, + }), + wantReason: "blocked — changes requested by @reviewer", + }, + { + name: "a required check that never reported is named", + pr: blocked(&github.ProtectionFacts{ + RequiredChecksKnown: true, + MissingRequiredChecks: []string{"validate"}, + }), + wantReason: `blocked — required check "validate" has not reported`, + }, + { + name: "a review GitHub requires is named", + pr: blocked(&github.ProtectionFacts{ + ReviewDecision: github.ReviewDecisionReviewRequired, + }), + wantReason: "blocked — an approving review is required by branch protection (0 given)", + }, + { + // NEGATIVE CONTROL. Facts were gathered and they explain + // nothing: the honest placeholder must survive. A build that + // always prints a derived rule fails here. + name: "no rule derivable keeps the honest placeholder", + pr: blocked(&github.ProtectionFacts{ + RequiredChecksKnown: true, + ReviewDecision: github.ReviewDecisionApproved, + }), + wantReason: "blocked — all sweep gates pass; a branch-protection rule is unsatisfied", + }, + { + // NEGATIVE CONTROL. No facts at all (an older governor, a repo + // whose GraphQL query failed) must read exactly as it did after + // step 1 — this is the #7516 regression guard. + name: "no facts at all keeps the step-1 placeholder", + pr: blocked(nil), + wantReason: "blocked — all sweep gates pass; a branch-protection rule is unsatisfied", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + bucket, verdict, _ := classifyMergeEligibility(tc.pr, false, "org/repo", tc.gates) + if bucket != mergeBucketSkip { + t.Errorf("bucket = %v, want %v", bucket, mergeBucketSkip) + } + if verdict.State != github.MergeVerdictBlocked { + t.Errorf("state = %q, want %q", verdict.State, github.MergeVerdictBlocked) + } + if verdict.Reason != tc.wantReason { + t.Errorf("reason = %q\n want %q", verdict.Reason, tc.wantReason) + } + }) + } +} + +// When the sweep has its own gate AND GitHub names a different rule, the +// operator needs both: clearing the sweep's gate alone will not merge the PR. +// When the two say the same thing, it must not be said twice. +func TestClassifyMergeEligibility_BlockedKeepsBothReasons(t *testing.T) { + base := func(p *github.ProtectionFacts) github.PullRequest { + return github.PullRequest{ + Number: 2, + Mergeable: github.MergeableNo, + MergeableState: "blocked", + BaseRef: "v4", + CIStatus: "failure", + FailingChecks: []string{"build"}, + Protection: p, + } + } + gates := mergeGates{requiredChecks: map[string]bool{"build": true}} + + t.Run("different rules are both reported", func(t *testing.T) { + pr := base(&github.ProtectionFacts{ + ReviewDecision: github.ReviewDecisionChangesRequested, + ChangesRequestedBy: []string{"reviewer"}, + }) + _, verdict, _ := classifyMergeEligibility(pr, false, "org/repo", gates) + want := "blocked — CI failing: build; GitHub also requires: changes requested by @reviewer" + if verdict.Reason != want { + t.Errorf("reason = %q\n want %q", verdict.Reason, want) + } + }) + + t.Run("the same rule is not said twice", func(t *testing.T) { + // The sweep's reason already contains the derived rule verbatim. + pr := github.PullRequest{ + Number: 3, + Mergeable: github.MergeableNo, + MergeableState: "blocked", + BaseRef: "v4", + CIStatus: "success", + Protection: &github.ProtectionFacts{ + ReviewDecision: github.ReviewDecisionChangesRequested, + ChangesRequestedBy: []string{"reviewer"}, + }, + } + // Feed the intent gate a reason that literally contains the rule. + _, verdict, _ := classifyMergeEligibility(pr, true, "org/repo", mergeGates{}) + if verdict.Reason != "blocked — held: a hold label keeps it out of the sweep; GitHub also requires: changes requested by @reviewer" { + t.Errorf("held+changes-requested reason = %q", verdict.Reason) + } + }) +} + +// A state OTHER than "blocked" must not sprout a branch-protection rule: +// "dirty" is a conflict and "behind" is a stale branch, and neither is a +// protection rule the operator can clear by getting a review. +func TestClassifyMergeEligibility_ProtectionRuleOnlyAppliesToBlocked(t *testing.T) { + facts := &github.ProtectionFacts{ + ReviewDecision: github.ReviewDecisionChangesRequested, + ChangesRequestedBy: []string{"reviewer"}, + } + for _, tc := range []struct { + state string + want string + }{ + {"dirty", "has merge conflicts with v4 — needs a rebase"}, + {"behind", "behind v4 — needs an update from the base branch"}, + } { + pr := github.PullRequest{ + Number: 4, Mergeable: github.MergeableNo, MergeableState: tc.state, + BaseRef: "v4", CIStatus: "success", Protection: facts, + } + _, verdict, _ := classifyMergeEligibility(pr, false, "org/repo", mergeGates{}) + if verdict.Reason != tc.want { + t.Errorf("%s reason = %q\n want %q", tc.state, verdict.Reason, tc.want) + } + } +} diff --git a/src/cmd/hive/selfupgrade.go b/src/cmd/hive/selfupgrade.go new file mode 100644 index 0000000000..2adf30c0e3 --- /dev/null +++ b/src/cmd/hive/selfupgrade.go @@ -0,0 +1,180 @@ +package main + +import ( + "encoding/json" + "fmt" + "log/slog" + "os" + "strings" + "time" +) + +// loadOrGenerateHiveID reads the Hive ID from disk, or generates and persists a new one. +const ( + // selfUpgradeMaxAttempts bounds how many times a spoke retries an upgrade + // that keeps leaving the image unchanged. Bounded rather than unlimited so a + // genuinely broken hive (e.g. missing RBAC) stops thrashing its pod, and + // bounded rather than "never again" so a transient failure still converges. + selfUpgradeMaxAttempts = 5 + // selfUpgradeBaseBackoff is the delay before retry #2; it doubles per + // attempt up to selfUpgradeMaxBackoff. + selfUpgradeBaseBackoff = 2 * time.Minute + // selfUpgradeMaxBackoff caps the exponential backoff between retries. + selfUpgradeMaxBackoff = 30 * time.Minute + // selfUpgradeFailureExitCode marks a process exit caused by a FAILED + // self-upgrade. Distinct from 0 so the failure is visible in the container's + // termination state instead of looking like a clean shutdown. + selfUpgradeFailureExitCode = 17 +) + +// upgradeMarker is the on-PVC record at /data/upgrade-requested. It survives +// pod restarts (that is the whole point: the process exits as part of an +// upgrade), so it is the only place attempt bookkeeping can live. +type upgradeMarker struct { + TargetSHA string `json:"target_sha"` + CurrentSHA string `json:"current_sha"` + RequestedAt time.Time `json:"requested_at"` + Attempts int `json:"attempts"` + LastError string `json:"last_error,omitempty"` +} + +// parseUpgradeMarker decodes a marker, tolerating the legacy format that had no +// attempts/last_error fields. A legacy marker counts as one prior attempt so an +// already-wedged hive gets retries under the new budget instead of being +// treated as fresh. +func parseUpgradeMarker(data []byte) upgradeMarker { + var m upgradeMarker + if err := json.Unmarshal(data, &m); err != nil { + return upgradeMarker{} + } + if m.Attempts < 1 { + m.Attempts = 1 + } + return m +} + +// sameUpgradeTarget reports whether two target SHAs refer to the same commit, +// tolerating short/full SHA length mismatch the way the hub's sameCommit does. +// A DIFFERENT target must reset the attempt budget, so this comparison is what +// keeps the latch from outliving the upgrade it was created for. +func sameUpgradeTarget(a, b string) bool { + if a == "" || b == "" { + return false + } + n := len(a) + if len(b) < n { + n = len(b) + } + return strings.EqualFold(a[:n], b[:n]) +} + +func writeUpgradeMarker(path string, m upgradeMarker, logger *slog.Logger) { + data, err := json.Marshal(m) + if err != nil { + logger.Warn("failed to encode upgrade marker", "error", err) + return + } + if err := os.WriteFile(path, data, 0o644); err != nil { + logger.Warn("failed to write upgrade marker", "path", path, "error", err) + } +} + +// recordUpgradeError annotates the existing marker with the cause of the failed +// attempt so the NEXT boot can log why the previous one did not land — without +// it the reason dies with the process and the failure is invisible. +// +// upgradeMarkerPath and lastUpgradeOutcomePath are the two on-PVC records the +// spoke keeps for auto-upgrade visibility (#7092). The marker at +// upgradeMarkerPath is present ONLY while an instructed upgrade has not landed +// (in flight or terminally failed) and is cleared the moment the new image +// boots — so it can NEVER represent a success. lastUpgradeOutcomePath is the +// durable companion that records the last upgrade that actually LANDED, so the +// dashboard can tell "attempted and succeeded" apart from "never attempted" +// instead of letting a blank panel masquerade as success. +const ( + upgradeMarkerPath = "/data/upgrade-requested" + lastUpgradeOutcomePath = "/data/last-upgrade-outcome" +) + +// upgradeOutcome is the durable "last upgrade LANDED" record. Written on the +// boot that completes an upgrade (reconcileUpgradeOutcomeAtBoot), it survives — +// unlike upgradeMarker, which is removed the moment the target image boots. +type upgradeOutcome struct { + TargetSHA string `json:"target_sha"` + CurrentSHA string `json:"current_sha"` + RequestedAt time.Time `json:"requested_at"` + CompletedAt time.Time `json:"completed_at"` +} + +func writeUpgradeOutcome(path string, o upgradeOutcome, logger *slog.Logger) { + data, err := json.Marshal(o) + if err != nil { + logger.Warn("failed to encode upgrade outcome", "error", err) + return + } + if err := os.WriteFile(path, data, 0o644); err != nil { + logger.Warn("failed to write upgrade outcome", "path", path, "error", err) + } +} + +// reconcileUpgradeOutcomeAtBoot records a SUCCESSFUL self-upgrade. An in-flight +// marker whose target equals the now-running commit means the instructed +// upgrade LANDED: the pod booted on the target image. That success would +// otherwise vanish — the next upgrade instruction silently discards the stale +// marker, so a hive that updated cleanly looks identical to one that never +// tried. This persists the success durably and clears the in-flight marker so +// it stops reading as "not landed". A marker whose target does NOT match the +// running commit is still in flight or failed and is left untouched for that +// surface. Called once at startup, before the heartbeat loop and dashboard come +// up, so the dashboard always sees the reconciled state. +func reconcileUpgradeOutcomeAtBoot(markerPath, outcomePath, runningSHA string, logger *slog.Logger) { + data, err := os.ReadFile(markerPath) + if err != nil { + return + } + m := parseUpgradeMarker(data) + if m.TargetSHA == "" || runningSHA == "" || !sameUpgradeTarget(m.TargetSHA, runningSHA) { + return + } + writeUpgradeOutcome(outcomePath, upgradeOutcome{ + TargetSHA: m.TargetSHA, + CurrentSHA: m.CurrentSHA, + RequestedAt: m.RequestedAt, + CompletedAt: time.Now().UTC(), + }, logger) + if err := os.Remove(markerPath); err != nil && !os.IsNotExist(err) { + logger.Warn("failed to clear landed upgrade marker", "path", markerPath, "error", err) + } + logger.Info("self-upgrade landed: recorded successful upgrade outcome", + "target", m.TargetSHA, "current", runningSHA) +} + +// upgradeFailureSummary renders what the hub shows an operator. An empty +// LastError must never render as a dangling "attempts: " - a colon promising a +// reason and delivering none is worse than saying the reason was not captured, +// because it reads as truncation and sends the reader looking for the rest. +func upgradeFailureSummary(attempts int, lastError string) string { + if strings.TrimSpace(lastError) == "" { + return fmt.Sprintf("self-upgrade failed after %d attempts (no error recorded; the image never changed - check that the deployment tracks a tag carrying the target SHA)", attempts) + } + return fmt.Sprintf("self-upgrade failed after %d attempts: %s", attempts, lastError) +} + +func recordUpgradeError(path string, upgradeErr error, logger *slog.Logger) { + if upgradeErr == nil { + return + } + var m upgradeMarker + data, err := os.ReadFile(path) + switch { + case os.IsNotExist(err): + return + case err != nil: + logger.Warn("upgrade marker unreadable; recording the error against a fresh marker", + "path", path, "error", err) + default: + m = parseUpgradeMarker(data) + } + m.LastError = upgradeErr.Error() + writeUpgradeMarker(path, m, logger) +} diff --git a/src/docs/api-reference.md b/src/docs/api-reference.md index 75af1a901c..c1acaa499a 100644 --- a/src/docs/api-reference.md +++ b/src/docs/api-reference.md @@ -565,27 +565,27 @@ always resolved server-side from the validated token. | `GET` | `/api/hub/clusters` | Hub auth | List Clusters | `pkg/hub/saas.go:509` | | `GET` | `/api/hub/image-pulls` | Hub auth | Per-Release Image Pull Series | `pkg/hub/saas.go:369` | | `GET` | `/api/reach` | Hub admin | PR Reach Report (?pr=NNN or ?recent=K) | `pkg/hub/saas.go:500` | -| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1592` | -| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1593` | -| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat | `pkg/hub/server.go:1554` | -| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1555` | -| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1556` | -| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1557` | -| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1558` | -| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1559` | -| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1560` | -| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1570` | -| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1571` | -| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1572` | -| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1573` | -| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1574` | -| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1575` | -| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1576` | -| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1577` | -| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1578` | -| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1579` | -| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1580` | -| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1583` | -| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1600` | -| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1605` | -| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1606` | +| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1597` | +| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1598` | +| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat | `pkg/hub/server.go:1559` | +| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1560` | +| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1561` | +| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1562` | +| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1563` | +| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1564` | +| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1565` | +| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1575` | +| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1576` | +| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1577` | +| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1578` | +| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1579` | +| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1580` | +| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1581` | +| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1582` | +| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1583` | +| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1584` | +| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1585` | +| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1588` | +| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1605` | +| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1610` | +| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1611` | diff --git a/src/docs/design/agent-turn-model.md b/src/docs/design/agent-turn-model.md index b50946e32d..1ceac8c7f2 100644 --- a/src/docs/design/agent-turn-model.md +++ b/src/docs/design/agent-turn-model.md @@ -128,11 +128,11 @@ Kick *timing* lives in `pkg/governor`; kick *text* is built in `pkg/scheduler`. - `Governor.Evaluate` (`src/pkg/governor/governor.go:323`) calls it at `src/pkg/governor/governor.go:420` and returns the due list. - The driving loop is a single ticker in `main`: - `time.NewTicker(… EvalIntervalS …)` at `src/cmd/hive/main.go:5332`, loop at - `src/cmd/hive/main.go:5388`, evaluation at `src/cmd/hive/main.go:6156`, - message assembly via `sched.BuildKickMessages` at `src/cmd/hive/main.go:6277` + `time.NewTicker(… EvalIntervalS …)` at `src/cmd/hive/main.go:4985`, loop at + `src/cmd/hive/main.go:5041`, evaluation at `src/cmd/hive/main.go:5802`, + message assembly via `sched.BuildKickMessages` at `src/cmd/hive/main.go:5923` (`src/pkg/scheduler/scheduler.go:658`), and delivery via - `agentMgr.SendKick` at `src/cmd/hive/main.go:6367`. + `agentMgr.SendKick` at `src/cmd/hive/main.go:6013`. This matters for the RFC: the scheduler is already **stateless with respect to turns**. It does not hold a continuation, does not await turn *N* before @@ -150,11 +150,11 @@ CLI subprocess. | State | Where | Citation | |---|---|---| | Pause flag (one bool per agent) | `/data/hive.yaml` via `AgentConfig.Paused` | `src/pkg/config/config.go:921`; writer `SetAgentPausedAndSave` `src/pkg/config/config.go:5736` | -| Pause provenance (`PausedAt`, `PausedReason`, `PausedTrigger`, `PausedBy`), CLI/model pins, model/backend overrides, restart count, `LastKick`, truncated kick history | `/data/hive-state.json` via `snapshot.AgentState` | `src/pkg/snapshot/state.go:78-101`; path `src/cmd/hive/main.go:2162` | +| Pause provenance (`PausedAt`, `PausedReason`, `PausedTrigger`, `PausedBy`), CLI/model pins, model/backend overrides, restart count, `LastKick`, truncated kick history | `/data/hive-state.json` via `snapshot.AgentState` | `src/pkg/snapshot/state.go:78-101`; path `src/cmd/hive/main.go:1815` | | Watchdog failure count, crash-loop latch, backoff deadline, healthy-since, conditions | same file, `snapshot.PersistedState.Watchdog` | `src/pkg/snapshot/state.go:41`; `watchdog.PersistedAgent` `src/pkg/watchdog/reconciler.go:205` | | Fleet-breaker engagement + held set | same file, `BreakerState` | `src/pkg/snapshot/state.go:49` | | Governor budget/spend/eval history, cadence overrides, ACMM level | same file | `src/pkg/snapshot/state.go:16-42` | -| **Full text of every delivered prompt** | `/data/prompt-history.jsonl` (lumberjack-rotated JSONL) | `src/pkg/dashboard/prompt_history.go:44`; writer `Server.RecordPrompt` `src/pkg/dashboard/prompt_history.go:371`, wired at `src/cmd/hive/main.go:3625` | +| **Full text of every delivered prompt** | `/data/prompt-history.jsonl` (lumberjack-rotated JSONL) | `src/pkg/dashboard/prompt_history.go:44`; writer `Server.RecordPrompt` `src/pkg/dashboard/prompt_history.go:371`, wired at `src/cmd/hive/main.go:3278` | | **Rendered terminal scrollback, per kick** | `/data/logs/kicks//-.log` | `src/pkg/agent/kick_logs.go:43` (`defaultKickLogDir`); writer `archiveKickLogLocked` `src/pkg/agent/kick_logs.go:180` | | Token-usage summary | `/data/metrics/token-summary.json` | `src/pkg/tokens/collector.go:194`, `:121` | | Structured audit trail | `/data/audit.jsonl`, reloaded into a ring at boot | `src/pkg/dashboard/audit.go:22`, `loadFromDisk` `:88` | @@ -235,7 +235,7 @@ Three things about this are worth stating precisely: (`src/pkg/agent/manager.go:1909-1911`) — a surviving session is reused, not recreated. Boot reaches both: `main` unconditionally calls `agentMgr.Start(ctx, name)` for every enabled agent - (`src/cmd/hive/main.go:4020`) and the reuse-vs-relaunch decision is taken + (`src/cmd/hive/main.go:3673`) and the reuse-vs-relaunch decision is taken inside. There is no `Adopt`, `Reattach`, or `RecoverAgents` function; searching for one finds only `RestoreBreaker` (`src/pkg/agent/manager.go:6367`), which restores control metadata and is diff --git a/src/docs/design/copilot-cost-capture.md b/src/docs/design/copilot-cost-capture.md index 0513755e2f..55e76bf0e2 100644 --- a/src/docs/design/copilot-cost-capture.md +++ b/src/docs/design/copilot-cost-capture.md @@ -68,7 +68,7 @@ moment. That is the per-request grain phase 4 wants, and it already exists. `liveCaptureSinceMs` comes from exactly one production writer: `tokenCollector.SetCopilotLiveCapture(time.Now().UnixMilli())` at -`src/cmd/hive/main.go:3734`, called immediately after `SetTokenSink`. The +`src/cmd/hive/main.go:3387`, called immediately after `SetTokenSink`. The collector stores it (`src/pkg/tokens/collector.go:273`) and passes it to `ScanCopilotSessions` (`collector.go:212`), which zeroes shutdown tokens for sessions whose `LastActive` is at or after that moment diff --git a/src/docs/design/github-mention-triggers.md b/src/docs/design/github-mention-triggers.md index a411c46662..3bcad64672 100644 --- a/src/docs/design/github-mention-triggers.md +++ b/src/docs/design/github-mention-triggers.md @@ -197,7 +197,7 @@ mechanism that already exists; none is new policy. the hive already trusts, using the dashboard's own role list (`DashboardConfig.AuthorizedRole`, `src/pkg/config/config.go:4182`) at `read-write` or above by default — the same lookup `trustedMergerFunc` - uses for the merge queue (`src/cmd/hive/main.go:8076`). An explicit + uses for the merge queue (`src/cmd/hive/merge_eligibility.go:50`). An explicit `github.mentions.summoners` list widens it. No configuration means the feature is **off**, never "any commenter"; that is the rule the Discord design already set for its channel list. @@ -337,5 +337,5 @@ From the issue, restated as boundaries this design must not cross: - `src/pkg/hub/webhook.go:53` — the fail-closed GitHub webhook verifier. - `src/pkg/ioscan/enforce.go:24`, `src/pkg/scheduler/ioscan_enforce.go:122`, [ADR-0008](../adr/0008-ioscan-untrusted-input.md) — untrusted kick input. -- `src/cmd/hive/main.go:8076` — `trustedMergerFunc`, the role-list lookup to +- `src/cmd/hive/merge_eligibility.go:50` — `trustedMergerFunc`, the role-list lookup to reuse for summoners. diff --git a/src/docs/design/master-key-rotation.md b/src/docs/design/master-key-rotation.md index 0b0cdd6c74..cb2e5f49a3 100644 --- a/src/docs/design/master-key-rotation.md +++ b/src/docs/design/master-key-rotation.md @@ -361,7 +361,7 @@ empty master. It is not a bug. Those three sites are inside `spokeDomainKey`, `SpokeHeartbeatKey`, and `SpokeSSOPublicKey` — all SPOKE-side resolvers. The `hive` binary serves both roles but selects between them at startup: -`runHub()` (`src/cmd/hive/main.go:9608`) is the only caller of +`runHub()` (`src/cmd/hive/main.go:8104`) is the only caller of `hub.NewHubServer`, and it is a distinct mode from the spoke path. On a hub pod those three functions are never called, so the empty `HIVE_HUB_SECRET` they would read is never consulted. diff --git a/src/docs/knowledge-curator.md b/src/docs/knowledge-curator.md index 905c06193b..34396e7349 100644 --- a/src/docs/knowledge-curator.md +++ b/src/docs/knowledge-curator.md @@ -15,3 +15,151 @@ knowledge: | `auto_promote_threshold` | Defaults to `0.9` when knowledge is enabled. `Promoter.AutoPromoteCandidates` selects facts whose llm-wiki page has `status == "verified"` and `confidence >= threshold`. | Merged-PR extraction and its old scheduling/source-list knobs are intentionally not documented as supported configuration because no scheduler, CLI command, or HTTP endpoint triggered extraction. + +## Remote git sources (`knowledge.git_sources`) + +`knowledge.git_sources` indexes markdown from a remote git repository (or a +subdirectory of one) as a knowledge source, so agents get facts from an +external repo — a runbook repo, an upstream docs repo, a shared pattern +library — primed into their kicks the same way wiki-layer facts are. This is +implemented and live, unlike curator scheduling above: `pkg/knowledge/gitsource.go` +does the cloning, indexing, and periodic sync; `cmd/hive/main.go:2303-2347` +wires configured entries at startup. + +```yaml +knowledge: + enabled: true + git_sources: + - name: dakota-skills # required — display name and dedup/lookup key + url: https://github.com/projectbluefin/dakota # required — https:// only + branch: main # optional — default "main" + subpath: docs/skills # optional — index only this subdirectory + layer: project # optional — default "project" +``` + +Config fields (`GitSourceConfigYAML`, `pkg/config/config.go:562-568`, mirrored +by the runtime type `GitSourceConfig`, `pkg/knowledge/gitsource.go:33-39`): + +| YAML key | Required | Default | Notes | +|---|---|---|---| +| `name` | Yes | — | Display name; also the key used to look up/remove the source via the API. | +| `url` | Yes | — | Git remote URL. **`https://` only** — see Auth below. | +| `branch` | No | `main` (`gitsource.go:66-68`) | Branch to shallow-clone (`--depth 1 --branch `). | +| `subpath` | No | — (whole repo) | When set, only this subdirectory is checked out (git sparse-checkout, `gitsource.go:199-219,232-248`) and indexed. | +| `layer` | No | `project` when set via the API (`api.go:8115-8117`); **required, no code default, when set via `hive.yaml`** | One of `personal`, `project`, `org`, `community` — see Layer semantics below. | + +### What "indexed" means + +On connect, the source is shallow-cloned (depth 1) into +`/git-sources/` and the clone (or its `subpath`) is +handed to a `FileStore`, which indexes markdown files under it +(`gitsource.go:78-114`). If `subpath` doesn't exist after cloning, connection +fails with `subpath "" not found after clone` — check the path relative +to the *repository root*, not the branch's top-level display in GitHub's UI. + +### Layer semantics + +`layer` places the source's facts in the same 4-layer precedence used +everywhere else in the knowledge system (`pkg/knowledge/types.go:8-30`, +lower number = higher precedence, i.e. overrides on conflict): + +| Layer | Precedence | Typical use for a git source | +|---|---|---| +| `personal` | 1 (highest) | A private runbook repo only this operator's hive should see. | +| `project` | 2 | A repo scoped to the project this hive manages — the default. | +| `org` | 3 | A shared org-wide reference repo. | +| `community` | 4 (lowest) | A public upstream docs repo, e.g. a framework's own documentation. | + +### Auth for private repos + +**There is no auth configuration for git sources.** `ensureCloned` invokes a +plain `git clone` with `GIT_TERMINAL_PROMPT=0` (so a credential prompt fails +instead of hanging) and no token, SSH key, or credential-helper wiring +(`gitsource.go:183-227,503-512`). In practice this means: + +- A **public HTTPS repo** works with just `url:`. +- A **private repo** will fail to clone — there is no field to supply a + token or deploy key, and the code does not fall back to any host git + credential store. Do not attempt to embed a token in the `url` (e.g. + `https://TOKEN@github.com/...`); nothing in this codebase does that + pattern for git sources, and hardcoding a token in `hive.yaml` is + explicitly against the fail-closed secret-handling used elsewhere in this + repo. +- Only `https://` (and, if `HIVE_ALLOW_PRIVATE_GIT_SOURCE=true`, `http://`) + URLs validate at all; `git@`-style SCP syntax is explicitly rejected + (`gitsource.go:267-268,275-281`). + +If your knowledge source is private, treat this as unsupported today rather +than assuming a missing config knob — see Open questions. + +### SSRF / URL hardening (operator-relevant) + +`ValidateGitSourceURLContext` rejects URLs whose host resolves to a +loopback, private, or link-local address (including the cloud metadata IP) +before cloning, and fails closed on a DNS lookup error +(`gitsource.go:259-351`). git itself is invoked with +`-c http.followRedirects=false` so a remote can't 302 the clone to an +internal address after validation passes (`gitsource.go:479-501`). An +in-cluster git server (e.g. an internal GitLab) needs +`HIVE_ALLOW_PRIVATE_GIT_SOURCE=true` set as an explicit opt-in +(`gitsource.go:293-297,368-370`). + +### Refresh / staleness + +Once connected, a background loop pulls and reindexes every 5 minutes +(`gitSourceSyncInterval`, `gitsource.go:25`, `StartSyncLoop`, +`gitsource.go:161-180`). A failed sync is logged at `warn` and the loop +keeps retrying on the next tick — it does not surface as a dashboard alert, +so staleness is only visible in the hive's logs +(`git source sync failed`). There is no operator-facing "last synced" +timestamp in `GitSourceInfo` (`gitsource.go:531-540`) — only whether the +source is `Ready` and its current page count. + +### Static config vs. the runtime API + +`GET/POST/DELETE /api/knowledge/git-sources` (owner-role only, +`pkg/dashboard/api.go:8068,8090-8153,8155-8192`) manage sources at runtime +and are the *same* underlying list as `knowledge.git_sources` in +`hive.yaml` — not a separate system: + +- `POST` connects a source immediately and, if it isn't already present + (matched by `url`+`subpath`), appends it to `Config.Knowledge.GitSources` + and persists the config (`api.go:8131-8149`). A `POST` for a source that + is only in `hive.yaml` but not yet connected in the running process (e.g. + right after editing the file without restarting) will add a duplicate + config entry once reconnected, since the dedup check is by URL+subpath + against what's already in `Config`, not against what main.go loaded at + boot. +- `DELETE` disconnects the live source and removes matching entries from + `Config.Knowledge.GitSources`, then persists (`api.go:8155-8192`). +- Editing `git_sources:` directly in `hive.yaml` takes effect on the next + process restart (main.go's startup loop at `cmd/hive/main.go:2303-2347`); + it does not hot-reload while the process is running. Use the API for a + live change without a restart. + +### Diagnosing a failed source + +- **Never appears / `knowledge not enabled`**: if `knowledge.enabled` is + `false` but `git_sources` is non-empty, startup auto-enables a minimal + knowledge API (`engine: file`) just to host the git sources + (`main.go:2651-2658`) — so a git source can work even with `knowledge.enabled: false`. + If you still get "knowledge not enabled" from the API, no source has + triggered that auto-enable yet (empty `git_sources` list). +- **Connect fails immediately**: check the hive log for `failed to connect + git source` with the URL and error (`main.go:2667`) — most often an + SSRF-validation rejection, a bad branch name, or (for private repos) an + authentication failure from git itself. +- **Connects but `subpath` errors**: `subpath "" not found after clone` — + the path is wrong or doesn't exist on the configured `branch`. +- **Facts never show up in kicks**: confirm the source reached `Ready: true` + (`GET /api/knowledge/git-sources`) — the primer only registers a source's + `FileStore` for priming after it reports ready + (`main.go:2680-2693`). + +## Open questions + +- #4944 (the issue behind this section) suggested documenting auth for + private repos as if it exists. It does not: see [Auth for private + repos](#auth-for-private-repos) above. If private-repo support is added + later, this section should be updated with the actual credential-config + shape rather than assumed ahead of the code. diff --git a/src/docs/roadmap.md b/src/docs/roadmap.md index b3743e9ca9..66975d518c 100644 --- a/src/docs/roadmap.md +++ b/src/docs/roadmap.md @@ -44,7 +44,6 @@ order, not priority rank. | Gitea/Forgejo forge program sequencing | **Decided — #6177/#6167 closed 2026-09-09.** The accepted sequencing: Wave 0 docs-only ADRs land now; Wave 1 may land on `v5` behind the existing `pkg/forge` seam; Wave 2 T5 enumeration-policy extraction is post-GA-cut, or v5-first only if it does not reset the required-gate soak; the enumeration-engine ADR must address the #6090 hardcoded-org portability findings (that cluster, #6081–#6087, closed with the row bound in v5-ga.md) before the seam is widened. Follow-ups go on [#6167](https://github.com/hivecommons/hive/issues/6167). | [#6177](https://github.com/hivecommons/hive/issues/6177), [#6167](https://github.com/hivecommons/hive/issues/6167), [#6169](https://github.com/hivecommons/hive/issues/6169), [#6170](https://github.com/hivecommons/hive/issues/6170), [#6171](https://github.com/hivecommons/hive/issues/6171), [#6112](https://github.com/hivecommons/hive/issues/6112), [#6090](https://github.com/hivecommons/hive/issues/6090), [#6081](https://github.com/hivecommons/hive/issues/6081), [#6082](https://github.com/hivecommons/hive/issues/6082), [#6083](https://github.com/hivecommons/hive/issues/6083), [#6084](https://github.com/hivecommons/hive/issues/6084), [#6085](https://github.com/hivecommons/hive/issues/6085), [#6086](https://github.com/hivecommons/hive/issues/6086), [#6087](https://github.com/hivecommons/hive/issues/6087) | | One-command contribute install | Collapse the four-step ClankeR on-ramp into `brew install …` via an apptainer build of the published `hive-contributor` image, phased as distroless image → tap-repo conversion action → agent-harness integration. Ownership split and per-phase acceptance criteria are in the [contribute distribution roadmap](https://github.com/hivecommons/hive/blob/v4/docs/contribute-distribution-roadmap.md); implementation is owned by the proposer on the tracker. | [#6635](https://github.com/hivecommons/hive/issues/6635), [#6641](https://github.com/hivecommons/hive/issues/6641) | | GitLab through `pkg/forge` | `pkg/forge` ships GitHub, GitLab, and Gitea/Forgejo adapters with the read path and core write path implemented and tested; `Merge` is left an explicit interface TODO because merge semantics diverge across forges. **First production caller landed:** the governor's escalation writes (evidence comment + `needs-human` label) are now typed against the `forge.IssueWriter` seam, with the adapter selected from `project.forge` — so that key is no longer display-only. A GitHub hive is unchanged, still on `*github.Client`. Those writes are not yet *reached* on a non-GitHub hive, because the read path is still GitHub-shaped: `EnumerateActionable` feeds the whole governor cycle and owns hold-label filtering, issue filters and SLA tracking inside `pkg/github`. Neutralizing enumeration — lifting that policy above the forge boundary, and adding a bulk list method so an N-repo hive does not enumerate N times — is what remains. | [ADR-0005](adr/0005-forge-abstraction.md), [#5259](https://github.com/hivecommons/hive/issues/5259), origin: [#2812](https://github.com/hivecommons/hive/issues/2812) (closed) | -| Kubernetes-native agent sandboxes | Graduate from tmux/container execution toward k8s-native, policy-isolated agent workloads where that complexity is justified. First slice shipped on v5: `sandbox.runtime: job` runs a sandboxed kick as a Kubernetes Job from an operator image with node selector, device limits, and secrets by reference, workspace shared through a RWX claim, credential boundary unchanged. | [#6311](https://github.com/hivecommons/hive/issues/6311), [sandbox isolation](sandbox-isolation.md#running-sandboxed-kicks-as-kubernetes-jobs) | ## Later @@ -52,6 +51,8 @@ order, not priority rank. | --- | --- | --- | | Cross-forge orchestration | Coordinate issues, merge requests, policy, and evidence across GitHub, GitLab, and Forgejo/Gitea-style forges. | [ADR-0005](adr/0005-forge-abstraction.md) | | Memory and learning maturation | Turn retro findings and curated knowledge into durable, testable priming without hidden or unauditable agent memory. | [knowledge design](design/knowledge-system.md), [retro lane](retro-lane.md) | +| Kubernetes-native agent sandboxes | Graduate from tmux/container execution toward k8s-native, policy-isolated agent workloads where that complexity is justified. | [architecture](architecture.md), [security threat model](security-threat-model.md) | +| v6-designated designs (line not open) | "v6" is a designation, not a line: no v6 branch, milestone, or channel exists, and opening one is blocked on the v5 GA bar closing. v6-designated proposals land as design docs only (no implementation PRs on current branches) and re-enter through the RFC gate when the line opens. First entry: GitHub @-mention triggers — the first inbound GitHub trigger, mirroring the Linear mention path. Full policy in [ROADMAP.md](../../ROADMAP.md#v6--future-line-not-open). | [#7519](https://github.com/hivecommons/hive/issues/7519), [#7483](https://github.com/hivecommons/hive/issues/7483), [mention-triggers design](design/github-mention-triggers.md) | ## Reading this roadmap diff --git a/src/docs/security-model.md b/src/docs/security-model.md index f4730eb671..0a593fba89 100644 --- a/src/docs/security-model.md +++ b/src/docs/security-model.md @@ -124,6 +124,12 @@ Two gotchas: - **Empty is worse than absent.** The resolvers fall back on empty values just like missing ones, but an empty var can make convergence accounting report the hive as converged when it is not. Unset rather than blank. - Per-hive env material derives from the **current** master generation — relevant during rotation, below. +### Provisioning-template changes reach existing spokes only through a reconcile + +The provisioning template (`k8sManifestTemplate` in `src/pkg/hub/saas_provision.go`) is `kubectl apply`ed once, when a hive is provisioned. A change to it is born onto every spoke created afterwards and reaches **no spoke that already exists** — those keep whatever object the template rendered on their day. The per-hive env sweep above, the NET_ADMIN sweep, and the vanity-host patch are the reconcilers that close such gaps for the objects they own; a change to any other existing object needs one too, or the PR must say that existing spokes have to be re-provisioned. + +The case that made this rule: [#7457](https://github.com/hivecommons/hive/pull/7457) added `auth-url` / `auth-response-headers` to the `hive-contribute` Ingress so a signed-in visitor's identity reaches `/api/contribute/me`. The code half rolled out with the next image; the Ingress half reached only newly provisioned spokes, and `/api/contribute/me` kept answering `401` everywhere else ([#7517](https://github.com/hivecommons/hive/issues/7517)). The hub now reconciles those two annotations onto every hosted spoke's `hive-contribute` Ingress on nginx clusters (a 15-minute sweep, `contribute_ingress_reconcile.go`; a merge patch on the annotations, which rolls no pod). OpenShift-Route clusters have no nginx Ingress and are skipped. + ### Master key rotation The hub master secret supports **generations**: at most two live at once — one CURRENT (mints new material) and one PREVIOUS (verify-only, default window 7 days). Design details: [design/master-key-rotation.md](design/master-key-rotation.md). diff --git a/src/pkg/agent/copilot_models_test.go b/src/pkg/agent/copilot_models_test.go index 48dbebde12..fa5573f327 100644 --- a/src/pkg/agent/copilot_models_test.go +++ b/src/pkg/agent/copilot_models_test.go @@ -69,3 +69,23 @@ func TestCanonicalizeCopilotModelIdempotent(t *testing.T) { // blind trailing-digits dot-rewrite corrupted claude-fable-5 into the // CLI-rejected claude-fable.5 (#4262); normalizeModelName must now emit // CLI-accepted spellings for copilot, self-correcting stored bad ids. +func TestNormalizeModelNameCopilotDrift(t *testing.T) { + tests := []struct{ in, want string }{ + {"claude-fable-5", "claude-fable-5"}, // dashed family must NOT gain a dot + {"claude-fable.5", "claude-fable-5"}, // stored bad id self-corrects at launch + {"claude-sonnet-5", "claude-sonnet-5"}, + {"claude-opus-5", "claude-opus-5"}, + {"claude-sonnet-4-6", "claude-sonnet-4.6"}, // YAML-friendly dashed still maps to dotted + {"claude-opus-4.6", "claude-opus-4.6"}, // legitimate dot unchanged + {"gpt-5.5", "gpt-5.5"}, + {"gemini-2.5-pro", "gemini-2.5-pro"}, + {"auto", "auto"}, // auto-select sentinel flows through + {"gpt-next", "gpt-next"}, // unknown id passthrough + {"custom-7", "custom-7"}, // unknown id: no blind dot-rewrite anymore + } + for _, tt := range tests { + if got := normalizeModelNameForBackend(tt.in, "copilot", false); got != tt.want { + t.Errorf("normalizeModelNameForBackend(%q, copilot, false) = %q, want %q", tt.in, got, tt.want) + } + } +} diff --git a/src/pkg/agent/manager.go b/src/pkg/agent/manager.go index 7546440b91..0e9e8c0bec 100644 --- a/src/pkg/agent/manager.go +++ b/src/pkg/agent/manager.go @@ -2,14 +2,9 @@ package agent import ( "context" - "encoding/json" "fmt" "log/slog" "os" - "os/exec" - "path/filepath" - "regexp" - "strconv" "strings" "sync" "sync/atomic" @@ -18,7 +13,6 @@ import ( "github.com/hivecommons/hive/pkg/claude" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/effects" - ghpkg "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/pushbroker" "github.com/hivecommons/hive/pkg/sandbox" "github.com/hivecommons/hive/pkg/watchdog" @@ -415,14 +409,6 @@ type AgentProcess struct { kickHoldReason string } -// effectiveBackend returns the agent's backend accounting for any override. -func effectiveBackend(agent *AgentProcess) string { - if agent.BackendOverride != "" { - return agent.BackendOverride - } - return agent.Config.Backend -} - // ProjectContext holds project-level config injected into agent boot prompts. type ProjectContext struct { Org string @@ -671,65 +657,6 @@ type Manager struct { kickLogMaxBytes int64 } -// IsInferenceBackend returns true if the backend is a self-hosted inference -// backend (vllm, llm-d, litellm) rather than a CLI tool. Delegates to the -// canonical list in the config package (shared with the proxy package, -// which cannot be imported from here without a cycle). -func IsInferenceBackend(backend string) bool { - return config.IsInferenceBackend(backend) -} - -// SetInferenceCallbacks registers callbacks that the manager uses to -// configure/clear inference routing on the proxy when launching agents. -func (m *Manager) SetInferenceCallbacks( - setRoute func(agentName, backend, model string), - clearRoute func(agentName string), -) { - m.mu.Lock() - defer m.mu.Unlock() - m.inferenceRouteCallback = setRoute - m.clearInferenceRouteCallback = clearRoute -} - -// SetGatewayBackendChecker injects a predicate that reports whether a backend -// string names a configured model gateway. This makes an agent whose backend is -// a gateway name inference-routable, so its route is resolved via the inference -// callback exactly like the built-in litellm/vllm/llm-d backends. -func (m *Manager) SetGatewayBackendChecker(fn func(backend string) bool) { - // Atomic store — no m.mu — so routableBackend can read it lock-free from the - // lock-holding launch path without deadlocking (see isGatewayBackend docs). - m.isGatewayBackend.Store(&fn) -} - -// routableBackend reports whether a backend should be routed through the -// inference proxy: either a built-in inference backend, or a configured gateway -// name. Safe to call while holding m.mu (isGatewayBackend is read atomically). -func (m *Manager) routableBackend(backend string) bool { - if IsInferenceBackend(backend) { - return true - } - // Lock-free atomic read: this is invoked from the launch path while m.mu is - // already held, so it MUST NOT take m.mu (non-reentrant RWMutex → deadlock). - fnp := m.isGatewayBackend.Load() - return fnp != nil && *fnp != nil && (*fnp)(backend) -} - -// validateBackendName reports whether backend is one the launcher can actually -// dispatch: an agentic CLI, a model-gateway backend, or a configured gateway -// name. An empty backend is valid (it means "the hive default"). -// -// This is the manager-side half of the accept-then-fail fix. It dispatches on -// the SAME canonical lists as config.ValidateBackend and backendBinary, so a -// backend accepted by any write path is one the launch path can start. -// Safe to call while holding m.mu (routableBackend reads atomically). -func (m *Manager) validateBackendName(backend string) error { - if backend == "" || config.IsCLIBackend(backend) || m.routableBackend(backend) { - return nil - } - return fmt.Errorf("unsupported backend %q (supported: %s; or the name of a configured model gateway)", - backend, strings.Join(config.SupportedBackends(), ", ")) -} - // SetACMMLevel updates the cached ACMM level used by agentMode() when // launching agents. Call this whenever the ACMM level changes. func (m *Manager) SetACMMLevel(level int) { @@ -1046,247 +973,11 @@ func (m *Manager) Start(ctx context.Context, name string) error { return m.launchInTmux(ctx, agent) } -const ( - // consentConfirmFooter appears at the bottom of Claude Code interactive - // selection screens (consent dialogs, settings-error menus). - consentConfirmFooter = "Enter to confirm" - // bypassConsentTitle is the heading of the --dangerously-skip-permissions - // consent screen. Its default selection is "No, exit" — confirming it - // terminates the CLI and leaves a bare bash pane. - bypassConsentTitle = "Bypass Permissions mode" - // bypassConsentDefaultOption is the default (negative) option on the - // bypass-permissions consent screen. - bypassConsentDefaultOption = "No, exit" - // bypassConsentAcceptOption is the affirmative option on the - // bypass-permissions consent screen. Its position varies between CLI - // versions, so acceptance navigates by matching the selected-line text. - bypassConsentAcceptOption = "Yes, I accept" - // apiKeyPromptTitle is the heading of the custom-API-key approval prompt, - // shown when ANTHROPIC_API_KEY is not in customApiKeyResponses.approved. - // Its default selection is "No (recommended)" with the affirmative option - // above it. - apiKeyPromptTitle = "Detected a custom API key" - // apiKeyPromptAcceptOption is the affirmative option on the - // custom-API-key approval prompt. - apiKeyPromptAcceptOption = "Yes" - // cliWorkingMarker is shown while Claude Code is actively processing a - // request; a pane in this state is never a consent screen. - cliWorkingMarker = "esc to interrupt" -) - -// paneShowsConsentScreen reports whether the pane is showing an interactive -// consent/selection screen rather than a ready CLI input prompt. Such screens -// contain a "❯"-selected menu option (e.g. "❯ 1. No, exit"), so they satisfy -// marker-based CLI presence checks ("❯" is also a cliPaneMarkers entry) — a -// kick typed into one is consumed by the menu, or by bash once the default -// "No, exit" selection terminates the CLI. Callers should pass the visible -// pane only (no scrollback): dismissed consent screens linger in history. -func paneShowsConsentScreen(pane string) bool { - if pane == "" || strings.Contains(pane, cliWorkingMarker) { - return false - } - // A known startup-blocking menu is not a ready prompt either. The generic - // test below needs the "Enter to confirm" footer AND a "❯"-marked line; - // codex renders neither (its footer is "Press enter to continue" and its - // marker is "›" U+203A), so its update menu read as READY. Everything that - // gates on readiness — the startup kick, caveman activation — then typed - // into the menu, and the Enter confirmed its pre-selected option: - // "1. Update now", which runs `npm install -g` as the agent UID, fails, and - // kills the CLI. Blocking on these lets the prompt watcher answer them. - if paneHasBlockingPrompt(pane) { - return true - } - if strings.Contains(pane, bypassConsentTitle) && strings.Contains(pane, bypassConsentDefaultOption) { - return true - } - if !strings.Contains(pane, consentConfirmFooter) { - return false - } - for _, line := range strings.Split(pane, "\n") { - if strings.HasPrefix(strings.TrimSpace(line), "❯") { - return true - } - } - return false -} - // kickOutcomePollEvery throttles the post-kick turn-ended check to one visible // pane capture per this many 3s poll ticks (15s), so a long turn does not // cost an extra tmux exec every tick. const kickOutcomePollEvery = 5 -func (m *Manager) buildBootstrapPrompt(agent *AgentProcess) string { - // No boot prompt — the governor's first eval cycle (10s after startup) - // kicks all due agents via BuildKickMessages with fully substituted - // templates. Sending a boot prompt here caused unsubstituted ${ISSUE_LIST} - // and other vars to reach the agent. The policy-file path list this - // function used to assemble was dead code once the boot prompt was - // removed, so it is gone too. - _ = agent // signature kept for the call site; the arg is no longer read - return "" -} - -// metricsCachePath is a var (not const) so tests can point it at a temp file -// to exercise readCoveragePreamble without a real /data volume. Production -// value is unchanged. -var metricsCachePath = "/data/metrics/agent-metrics-cache.json" - -func (m *Manager) readCoveragePreamble() string { - data, err := os.ReadFile(metricsCachePath) - if err != nil { - return "" - } - var metrics map[string]map[string]json.Number - if err := json.Unmarshal(data, &metrics); err != nil { - return "" - } - ci, ok := metrics["ci-maintainer"] - if !ok { - return "" - } - cov, err := ci["coverage"].Int64() - if err != nil { - return "" - } - target, err := ci["coverageTarget"].Int64() - if err != nil { - target = 91 - } - return fmt.Sprintf("[COVERAGE] Current: %d%% | Target: %d%%.", cov, target) -} - -// shellEnvVar formats KEY='value' with single-quoting so values containing -// spaces, parentheses, or other shell metacharacters are safe in inline env -// var assignments sent to tmux via send-keys. -func shellEnvVar(key, value string) string { - quoted := strings.ReplaceAll(value, "'", "'\"'\"'") - return fmt.Sprintf("%s='%s'", key, quoted) -} - -// applySecretEnv pushes only the Secret pairs into the agent's tmux session via -// set-environment. Values are passed as exec args (never through a shell), so -// they are not word-split and never land in the pane or in bash history. -// Failures are ignored for the same reason ensureTmuxSession ignores them: a -// missing session is handled by the launch path, not here. -func (m *Manager) applySecretEnv(agent *AgentProcess) { - if agent == nil || agent.tmuxSession == "" { - return - } - for _, p := range m.agentEnvPairs(agent) { - if !p.Secret { - continue - } - _ = m.tmuxCmd(agent, "set-environment", "-t", agent.tmuxSession, p.Key, p.Value).Run() - } -} - -func (m *Manager) buildEnvPrefix(agent *AgentProcess) string { - pairs := m.agentEnvPairs(agent) - var parts []string - for _, p := range pairs { - if p.Secret { - continue - } - parts = append(parts, shellEnvVar(p.Key, p.Value)) - } - if len(parts) == 0 { - return "" - } - return strings.Join(parts, " ") + " " -} - -// Blocked-action thrash breaker: an agent that keeps hammering a policy wall -// (e.g. a push with no per-agent token, blocked every ~3s by -// git-credential-hive, or a proxy hard-deny) burns model tokens indefinitely -// with zero possible output — observed live 2026-08-04 on a hosted L2 hive -// whose guide agent retried a blocked push every 3 seconds. (Since #4289, -// ADVISORY-mode pushes are no longer blocked by the credential helper — the -// read-only token is served and GitHub rejects the push with 403 — but the -// helper still emits "git push blocked:" for unknown-UID and missing-token -// failures, which this breaker continues to catch.) The hub, not the model, -// breaks the loop: thrashThreshold blocked-action lines within thrashWindow -// pauses the session (visible, reversible, stops governor kicks) with the -// reason spelled out. -const ( - thrashWindow = 60 * time.Second - thrashThreshold = 5 - thrashCooldown = 10 * time.Minute -) - -// blockedActionMarkers are the policy-wall stderr lines that can never -// succeed by retrying. Keep in sync with bin/git-credential-hive.sh and the -// proxy's hard-deny responses. -var blockedActionMarkers = []string{ - "git push blocked:", - "blocked by hive policy", -} - -type thrashState struct { - times []time.Time - lastTrip time.Time -} - -// checkBlockedThrash records a blocked-action output line for the agent and, -// past the threshold, pauses the agent asynchronously (never inline: this is -// called from the output-capture goroutine and Pause takes m.mu). -func (m *Manager) checkBlockedThrash(agent, line string) { - matched := false - for _, marker := range blockedActionMarkers { - if strings.Contains(line, marker) { - matched = true - break - } - } - if !matched { - return - } - now := time.Now() - m.thrashMu.Lock() - if m.thrash == nil { - m.thrash = map[string]*thrashState{} - } - st := m.thrash[agent] - if st == nil { - st = &thrashState{} - m.thrash[agent] = st - } - trip := recordBlockedAndCheck(st, now, thrashWindow, thrashThreshold, thrashCooldown) - m.thrashMu.Unlock() - if !trip { - return - } - reason := fmt.Sprintf("blocked-action loop: %d+ policy-blocked attempts in %s — the block is terminal in this mode; paused to stop token burn", thrashThreshold, thrashWindow) - m.logger.Warn("thrash breaker tripped", "agent", agent, "line", truncateStr(line, 160)) - go func() { - if err := m.Pause(agent, "thrash-breaker", reason); err != nil { - m.logger.Warn("thrash breaker pause failed", "agent", agent, "error", err) - } - }() -} - -// recordBlockedAndCheck is the pure sliding-window decision: append now, drop -// entries older than window, and report whether the threshold is crossed -// outside the cooldown. -func recordBlockedAndCheck(st *thrashState, now time.Time, window time.Duration, threshold int, cooldown time.Duration) bool { - st.times = append(st.times, now) - cutoff := now.Add(-window) - kept := st.times[:0] - for _, t := range st.times { - if !t.Before(cutoff) { - kept = append(kept, t) - } - } - st.times = kept - if len(st.times) < threshold { - return false - } - if !st.lastTrip.IsZero() && now.Sub(st.lastTrip) < cooldown { - return false - } - st.lastTrip = now - return true -} - // waitForInputPromptForAgentUnless is waitForInputPromptForAgent with an // abort predicate, consulted once per poll tick: when it reports true the wait // returns false at once instead of running out inputPromptTimeout. The kick @@ -1559,252 +1250,6 @@ func (m *Manager) RemoveAgent(name string) { )) } -// effectiveBackend is the backend this agent will actually launch with: the -// per-agent override when set, otherwise its configured backend. -func (a *AgentProcess) effectiveBackend() string { - if a.BackendOverride != "" { - return a.BackendOverride - } - return a.Config.Backend -} - -// effectiveModel is the model this agent will actually launch with: the -// per-agent override when set, otherwise its configured model. Returns the -// raw (un-normalized) name — the audit log should show what was ASKED for, -// since a bad model name is exactly the kind of misconfiguration being -// audited. -func (a *AgentProcess) effectiveModel() string { - if a.ModelOverride != "" { - return a.ModelOverride - } - return a.Config.Model -} - -// dismissInferencePrompts polls the tmux pane for Claude Code interactive -// prompts and auto-dismisses them. The "Bypass Permissions mode" consent -// screen and the custom-API-key approval prompt are handled first and -// explicitly (see confirmMenuOption): their default selections are negative -// ("No, exit" / "No (recommended)"), so confirming blind terminates the CLI -// or declines the seeded key. -// Other prompts are handled dynamically regardless of prompt text changes -// between Claude Code versions by: -// 1. Detecting "Enter to confirm" (universal prompt footer) -// 2. Finding the selected option (line with "❯" marker) -// 3. If selected option looks negative (contains "No" or "exit"), navigate -// away from it before confirming -// 4. For "Press Enter to continue" screens, just press Enter -// -// The pane is polled fast for the first 10s — the consent screen appears -// within ~5-8s of launch and every second it lingers is a window for a kick -// to be swallowed by the menu — then at a relaxed interval. -// -// Stops when the main Claude Code input prompt appears ("esc to interrupt"). -func (m *Manager) dismissInferencePrompts(agent *AgentProcess) { - const ( - // promptFastPollWindow covers the launch window in which the consent - // screen normally appears (~5-8s after CLI start). - promptFastPollWindow = 10 * time.Second - promptFastPollInterval = 250 * time.Millisecond - promptPollInterval = 1 * time.Second - promptDismissTimeout = 60 * time.Second - postKeystrokeDelay = 500 * time.Millisecond - ) - - start := time.Now() - timeout := promptDismissTimeout - if m.promptDismissTimeout > 0 { - timeout = m.promptDismissTimeout - } - deadline := start.Add(timeout) - lastPane := "" - - for time.Now().Before(deadline) { - interval := promptPollInterval - if time.Since(start) < promptFastPollWindow { - interval = promptFastPollInterval - } - m.sleepDuringPromptDismiss(interval) - - pane := m.captureVisiblePaneForAgent(agent) - if pane == "" { - continue - } - - // Bypass-permissions consent screen: handle first and explicitly, - // even if the pane is unchanged since the last poll (a mistimed - // keystroke must be retried, not skipped). The affirmative option - // sits below the default "No, exit". - if strings.Contains(pane, bypassConsentTitle) && !strings.Contains(pane, cliWorkingMarker) { - m.logger.Info("accepting bypass-permissions consent", "agent", agent.Name) - m.confirmMenuOption(agent, bypassConsentTitle, bypassConsentAcceptOption, "Down") - lastPane = "" // re-capture fresh on the next pass - continue - } - - // Custom-API-key approval prompt: the affirmative "Yes" sits ABOVE - // the default "No (recommended)" selection, so the generic - // Down-then-Enter fallback below would decline it. - if strings.Contains(pane, apiKeyPromptTitle) && !strings.Contains(pane, cliWorkingMarker) { - m.logger.Info("approving seeded inference API key", "agent", agent.Name) - m.confirmMenuOption(agent, apiKeyPromptTitle, apiKeyPromptAcceptOption, "Up") - lastPane = "" - continue - } - - if pane == lastPane { - continue - } - lastPane = pane - - // Main prompt visible — agent is ready - if strings.Contains(pane, "bypass permissions on") || strings.Contains(pane, "esc to interrupt") { - m.logger.Info("inference agent ready", "agent", agent.Name) - return - } - - // "Press Enter to continue" screens - if strings.Contains(pane, "Press Enter to continue") { - m.logger.Info("inference prompt: press enter", "agent", agent.Name) - m.tmuxSendKeysForAgent(agent, "Enter") - continue - } - - // Selection prompts have "Enter to confirm" footer - if !strings.Contains(pane, "Enter to confirm") { - continue - } - - // Find the currently selected option (marked with ❯) - selected := selectedMenuOption(pane) - - m.logger.Info("inference prompt detected", - "agent", agent.Name, - "selected", selected, - ) - - // If current selection looks negative, navigate away from it - selectedLower := strings.ToLower(selected) - if strings.Contains(selectedLower, "no,") || strings.Contains(selectedLower, "no ") || - strings.Contains(selectedLower, "exit") { - // Try moving down first (most prompts put the positive option below) - m.tmuxSendKeysForAgent(agent, "Down") - m.sleepDuringPromptDismiss(postKeystrokeDelay) - } else if strings.Contains(selectedLower, "fix with") { - // Settings error: skip past "Fix with Claude" and "Exit" to "Continue without" - m.tmuxSendKeysForAgent(agent, "Down") - m.sleepDuringPromptDismiss(postKeystrokeDelay) - m.tmuxSendKeysForAgent(agent, "Down") - m.sleepDuringPromptDismiss(postKeystrokeDelay) - } - - m.tmuxSendKeysForAgent(agent, "Enter") - } - - m.logger.Warn("inference prompt dismissal timed out", "agent", agent.Name) -} - -func (m *Manager) sleepDuringPromptDismiss(d time.Duration) { - m.terminalSession().SleepDuringPromptDismiss(d) -} - -// selectedMenuOption returns the trimmed text of the "❯"-selected line of an -// interactive CLI menu, or "" if no line is selected. -func selectedMenuOption(pane string) string { - for _, line := range strings.Split(pane, "\n") { - trimmed := strings.TrimSpace(line) - if strings.HasPrefix(trimmed, "❯") { - return trimmed - } - } - return "" -} - -// confirmMenuOption drives an interactive CLI menu identified by title to the -// option whose text contains want, then confirms it with Enter. Navigation -// matches the "❯"-selected line text rather than pressing a fixed number of -// keys, so it lands on the right option whichever position it occupies (menu -// option order differs between Claude CLI versions). navKey is the arrow key -// to step with ("Down" or "Up"). Returns true once the option was confirmed -// or the screen is gone. -func (m *Manager) confirmMenuOption(agent *AgentProcess, title, want, navKey string) bool { - const ( - // menuMaxNavigateSteps bounds arrow-key navigation; the handled menus - // have 2 options, extra headroom covers future variants. - menuMaxNavigateSteps = 4 - postKeystrokeDelay = 500 * time.Millisecond - ) - for step := 0; step < menuMaxNavigateSteps; step++ { - pane := m.captureVisiblePaneForAgent(agent) - if !strings.Contains(pane, title) || strings.Contains(pane, cliWorkingMarker) { - return true // screen already dismissed - } - if strings.Contains(selectedMenuOption(pane), want) { - m.tmuxSendKeysForAgent(agent, "Enter") - m.sleepDuringPromptDismiss(postKeystrokeDelay) - return true - } - m.tmuxSendKeysForAgent(agent, navKey) - m.sleepDuringPromptDismiss(postKeystrokeDelay) - } - m.logger.Warn("inference menu: wanted option not reached", - "agent", agent.Name, "title", title, "want", want) - return false -} - -const ( - // consentStuckGracePeriod is how long a consent screen must stay visible - // across watcher passes before the agent counts as stuck. The launch-time - // dismissal goroutine runs for 60s, so a screen still visible this long - // after first being seen by the watcher means dismissal lost the race. - consentStuckGracePeriod = 30 * time.Second - // consentDismissCooldown is the minimum interval between watcher-triggered - // dismissal passes for one agent, so a stubborn screen can't spam - // keystroke goroutines (each dismissal pass itself polls for 60s). - consentDismissCooldown = 2 * time.Minute -) - -// clearConsentTracking resets the consent-stuck timer for an agent whose pane -// no longer shows a consent screen. -func (m *Manager) clearConsentTracking(name string) { - m.mu.Lock() - defer m.mu.Unlock() - if agent, ok := m.agents[name]; ok { - agent.consentSeenAt = time.Time{} - } -} - -// dismissConsentIfStuck re-runs dismissInferencePrompts for an inference agent -// whose pane has shown a consent screen for longer than the grace period, -// subject to a per-agent cooldown. Called from the watcher loop -// (CheckAndRestartCrashedAgents) so an agent that lands on a consent screen -// after launch — e.g. a crash-recovery restart whose launch-time dismissal -// timed out — recovers instead of sitting stuck while kicks appear to succeed. -func (m *Manager) dismissConsentIfStuck(name string) { - now := time.Now() - m.mu.Lock() - agent, ok := m.agents[name] - if !ok { - m.mu.Unlock() - return - } - if agent.consentSeenAt.IsZero() { - agent.consentSeenAt = now - m.mu.Unlock() - return - } - stuckFor := now.Sub(agent.consentSeenAt) - if stuckFor < consentStuckGracePeriod || now.Sub(agent.lastConsentDismiss) < consentDismissCooldown { - m.mu.Unlock() - return - } - agent.lastConsentDismiss = now - m.mu.Unlock() - - m.logger.Warn("inference agent stuck on consent screen, re-running prompt dismissal", - "name", name, "stuck_seconds", int(stuckFor.Seconds())) - go m.dismissInferencePrompts(agent) -} - func (m *Manager) markProviderErrorLocked(agent *AgentProcess, match providerErrorMatch, now time.Time) time.Duration { if status, ok := classifyBackendAuthStatus(match.Class, match.Line); ok { agent.markBackendAuthLocked(status, match.Line, now) @@ -1991,442 +1436,6 @@ func (m *Manager) AllStatuses() map[string]*AgentProcess { return result } -// backendBinaryAliases names the backends whose binary is NOT simply the -// backend name. Only genuine aliases belong here: every other CLI backend is -// derived from config.CLIBackends by identity, and every routable model-gateway -// backend is resolved by Manager.backendBinaryName. Keeping this map to aliases -// only is what makes the accept-then-fail class of bug structurally impossible. -var backendBinaryAliases = map[string]string{ - // pi was previously aliased to "goose", which made every pi-configured - // agent exec the goose CLI instead of pi (the backend launch command - // switch now has a real pi case). pi is a first-class CLI backend - // (config.CLIBackends includes "pi"), so identity mapping applies. -} - -// backendBinaryName maps a config-independent agent backend to the NAME of the -// CLI binary that is exec'd for it, without touching the filesystem. Split out -// from backendBinary so the "every supported backend resolves" invariant can be -// tested without requiring each CLI to be installed on the test machine. -// -// Both canonical lists are derived rather than written out here: -// -// - config.CLIBackends (claude, copilot, goose, codex, pi, bob, aider, gemini) -// each launch a binary of the same name, except for the aliases above. -// - config.InferenceBackends (vllm, llm-d, litellm, watsonx) all launch the -// SAME claude CLI, pointed at hive's local OpenAI-compatible translator via -// ANTHROPIC_BASE_URL — the backend name selects the upstream route, not the -// binary. -// -// Deriving both means a backend added to either list can never again be -// accepted by config.ValidateBackend and then rejected hours later at kick time -// with "unknown backend". Previously only InferenceBackends was derived, so -// codex and aider were valid config values that failed at launch. -func backendBinaryName(backend string) (string, error) { - binaries := make(map[string]string, len(config.CLIBackends)+len(config.InferenceBackends)) - for _, b := range config.CLIBackends { - binaries[b] = b - } - for _, b := range config.InferenceBackends { - binaries[b] = "claude" - } - for backend, binary := range backendBinaryAliases { - binaries[backend] = binary - } - - binary, ok := binaries[backend] - if !ok { - return "", fmt.Errorf("unknown backend: %s", backend) - } - return binary, nil -} - -// backendBinaryName resolves both config-independent backends and live -// configured gateway names. A gateway name validates via Manager.routableBackend, -// so the launch path must use the same predicate and route it through claude. -func (m *Manager) backendBinaryName(backend string) (string, error) { - if binary, err := backendBinaryName(backend); err == nil { - return binary, nil - } - if m != nil && m.routableBackend(backend) { - return "claude", nil - } - return "", fmt.Errorf("unknown backend: %s", backend) -} - -// backendBinary resolves an agent backend to the absolute path of the CLI -// binary that is actually exec'd for it. -func backendBinary(backend string) (string, error) { - binary, err := backendBinaryName(backend) - if err != nil { - return "", err - } - - path, err := exec.LookPath(binary) - if err != nil { - return "", fmt.Errorf("backend %s not found in PATH: %w", backend, err) - } - - return path, nil -} - -func (m *Manager) backendBinary(backend string) (string, error) { - binary, err := m.backendBinaryName(backend) - if err != nil { - return "", err - } - - path, err := exec.LookPath(binary) - if err != nil { - return "", fmt.Errorf("backend %s binary %s not found in PATH: %w", backend, binary, err) - } - - return path, nil -} - -func (m *Manager) backendLaunchFailureMessage(backend string, err error) string { - binary, nameErr := m.backendBinaryName(backend) - if nameErr != nil { - return fmt.Sprintf( - "backend %s did not launch: %v. This backend is not a supported CLI, built-in inference backend, or configured model gateway; switch this agent to a supported backend or configure a matching model gateway.", - backend, err) - } - return fmt.Sprintf( - "backend %s did not launch: %v. The %s CLI required for this backend is not installed in this hive image — upgrade the hive image or switch this agent to a different backend.", - backend, err, binary) -} - -const ( - sharedConfigDesiredMode = 0o660 - // agyDefaultEffort is the reasoning effort passed alongside agy's --model - // when the agent has no usable reasoning_effort configured (see - // agyLaunchEffort). agy requires --effort whenever --model is given and - // otherwise ignores the model entirely; "low" is the effort agy defaults - // to on its own, so this makes the configured model take effect without - // changing behaviour. - agyDefaultEffort = "low" - - tokenRestartCooldownSec = 60 // minimum seconds between token-triggered restarts per agent - // loginPromptTailLines bounds the pane region the login-prompt detector - // reads: a prompt the CLI is stuck at sits at the pane bottom, while - // echoed kick text and startup flashes live in scrollback (see the poller). - loginPromptTailLines = 15 - // loginStreakRestartMin is how many consecutive polls (~3s apart) must see - // the login prompt before a token-triggered restart may fire — filters the - // CLI's transient startup "/login" flash. - loginStreakRestartMin = 3 - // tokenRestartMaxAttempts bounds CONSECUTIVE token-triggered restarts that - // fail to clear the login prompt. - // - // The three guards above answer WHEN to restart; none of them answered HOW - // MANY TIMES, so a restart that could never work was retried forever at the - // cooldown interval. #4596 is precisely that shape: the shared credential is - // valid (so configHasTokens() is true) while $HOME/.claude.json has lost its - // oauthAccount (so the CLI shows the login menu regardless), and each - // restart re-launched a CLI that rewrote the same contended file and asked - // again. Restarts are not free — they destroy in-flight work, which is the - // failure the kick grace above was added for. - // - // Three is deliberately generous: one restart genuinely does fix the case - // this feature was built for (an operator authenticates in one agent's - // terminal and the others need a nudge), so the cap only engages on a - // theory that has now failed repeatedly. - tokenRestartMaxAttempts = 3 - // tokenRestartKickGrace suppresses token-triggered restarts after a kick - // delivery so the restart can never destroy just-delivered work. - tokenRestartKickGrace = 10 * time.Minute - expiredTokenHangTimeoutSec = 180 // blank pane after this many seconds triggers token purge + restart - tlsErrorRestartCooldownSec = 120 // minimum seconds between TLS-error-triggered restarts per agent -) - -// codexBackend is the backend name for the OpenAI Codex CLI. -const codexBackend = "codex" - -// bobBackend is the backend name for the IBM bobshell ("bob") CLI. -const bobBackend = "bob" - -// normalizeModelName converts YAML-friendly model names to the format each -// CLI backend expects. Claude CLI uses hyphens (claude-opus-4-7), while -// gemini/goose/agy-style backends use dots (claude-opus-4.7). -// -// copilot does NOT take the blind trailing-digits dot-rewrite below: the -// Copilot CLI's --model nomenclature mixes separators per model family -// (claude-fable-5 is DASHED, claude-opus-4.6 is DOTTED), so the rewrite -// corrupted every dashed-family id — verified live, copilot CLI v1.0.78 -// rejected the rewritten `claude-fable.5` ("is not available") and fell back -// to a different model (#4262). copilot instead uses the alias-based -// CanonicalizeCopilotModel (copilot_models.go), which normalizes separator -// drift against the known CLI-accepted list in both directions and passes -// unknown ids through verbatim. Applied here — at launch time — so an -// already-stored bad id self-corrects on existing spokes without operator -// action. -// -// Self-hosted inference backends (vllm, llm-d, litellm) and configured gateway -// names are the outbound gateway model id verbatim — the string must match an -// entitled model on the gateway EXACTLY (prefixes like "Azure/", dots vs -// hyphens, case). Rewriting it (e.g. "Azure/gpt-4" -> "Azure/gpt.4", -// "gpt-4o-2024-08-06" -> "gpt-4o-2024-08.06") produces a model the team is not -// entitled to and the gateway 403s ("team not allowed to access model") even -// for entitled models. So never normalize inference model names — pass them -// through untouched. -// -// bob is likewise excluded. bobLaunchCmd passes no --model at all (bob -// auto-selects), so this is defense-in-depth rather than the fix: the value is -// still computed and logged on the bob launch path, and the dot-rewrite is -// what turned a configured `claude-sonnet-4-6` into the unknown -// `claude-sonnet-4.6` that made bob die with "Cannot read properties of -// undefined (reading 'maxTokens')". Leaving it unrewritten keeps logs honest -// about what was configured and stops the corrupted id from being handed to a -// future bob consumer. -func normalizeModelNameForBackend(model, backend string, inferenceRoutable bool) string { - if backend == "claude" || backend == bobBackend || inferenceRoutable { - return model - } - if backend == "copilot" { - return CanonicalizeCopilotModel(model) - } - idx := strings.LastIndex(model, "-") - if idx < 0 || idx == len(model)-1 { - return model - } - suffix := model[idx+1:] - allDigits := true - for _, c := range suffix { - if c < '0' || c > '9' { - allDigits = false - break - } - } - if allDigits { - return model[:idx] + "." + suffix - } - return model -} - -// ClearModeOverrides clears Config.Mode for the NAMED agents so that -// DefaultAgentMode determines their mode from the ACMM level. Call it before -// SyncModeFiles when applying a pack, because a pack agent's Config.Mode may -// have been set by a previous level's pack and would otherwise override the -// new level's expected default. -// -// Scoped to a name list — the pack's roster — on purpose (#7503). The previous -// ClearAllModeOverrides wiped every agent in the process table, including -// agents no pack manages. Their Mode was never pack-seeded, so there is no -// stale pack value to clear: what got cleared was the OPERATOR's setting. On -// the projectbluefin spoke `reviewer` was `mode: ADVISORY` in every config -// layer and ran as ISSUES_AND_PRS, the L5 default — two rungs more authority -// than anything on disk granted it — because the startup pack apply cleared it -// here and SyncModeFiles then wrote the default. Names not in the process -// table are ignored. -func (m *Manager) ClearModeOverrides(names []string) { - m.mu.Lock() - defer m.mu.Unlock() - for _, name := range names { - if agent, ok := m.agents[name]; ok { - agent.Config.Mode = "" - } - } -} - -// SyncModeFiles rewrites /tmp/.hive-mode-* for all running agents to reflect the given ACMM level. -func (m *Manager) SyncModeFiles(level int) { - m.mu.RLock() - defer m.mu.RUnlock() - for name, agent := range m.agents { - if agent.Paused { - continue - } - mode := DefaultAgentMode(name, level) - // converseConfigured is logged on both branches below so one grep by - // agent name shows whether the capability came from config or fell - // back to the default alongside the mode decision (#7503). - converseConfigured := agent.Config.Converse != nil - if modeStr := agent.Config.Mode; modeStr != "" { - if parsed, ok := ParseAgentMode(modeStr); ok { - m.logger.Info("SyncModeFiles: Config.Mode override", - "agent", name, "level", level, - "default", DefaultAgentMode(name, level).String(), - "override", modeStr, - "converse_configured", converseConfigured) - mode = parsed - } - } else { - // Log the fallback too, not only the override (#7503). Before this, - // an agent whose configured mode had been dropped on the way to the - // process table was indistinguishable from one that never set a - // mode: the only signal was reading /tmp/.hive-mode- and - // comparing by hand. One line saying "config said nothing, using - // the level default" turns that into a grep. `overlay_file` names - // the per-agent file when the entry came from one, so an operator - // can check what it says against what is being used. - m.logger.Info("SyncModeFiles: no Config.Mode, using level default", - "agent", name, "level", level, - "default", mode.String(), - "converse_configured", converseConfigured, - "overlay_file", agent.Config.SourceFile()) - } - modeFile := filepath.Join(agentStateDir, ".hive-mode-"+name) - if err := writeAgentStateFile(modeFile, []byte(mode.String())); err != nil { - m.logger.Warn("SyncModeFiles: write failed", "file", modeFile, "error", err) - } - // The capability file rides the same sync (#4492). It is level-independent - // today, but writing it here is what makes a `converse` change take effect - // on the next reconcile instead of only at the next agent launch. - caps := DefaultCapabilities(mode, level) - if agent.Config.Converse != nil { - caps.Converse = *agent.Config.Converse - } - m.writeAgentCapsFile(name, caps) - } -} - -// agentCapabilities returns the ORTHOGONAL capabilities for a given agent -// (#4492). Unlike agentMode there is no per-level default table: `converse` is -// opt-in everywhere, so an agent whose config says nothing gets the zero value -// and behaves exactly as it did before capabilities existed. -func (m *Manager) agentCapabilities(agent *AgentProcess) AgentCapabilities { - caps := DefaultCapabilities(m.agentMode(agent), m.project.ACMMLevel) - if agent.Config.Converse != nil { - caps.Converse = *agent.Config.Converse - } - return caps -} - -// writeAgentCapsFile persists the capability set the proxy reads on the request -// path. It is written for EVERY agent, including those with no capabilities, so -// a cleared `converse` actually revokes: leaving a stale file behind would keep -// granting the capability after the operator turned it off. -func (m *Manager) writeAgentCapsFile(name string, caps AgentCapabilities) { - capsFile := filepath.Join(agentStateDir, ".hive-caps-"+name) - if err := writeAgentStateFile(capsFile, []byte(caps.String())); err != nil { - m.logger.Warn("caps file write failed", "file", capsFile, "error", err) - } -} - -// agentMode returns the GitHub interaction mode for a given agent at the current ACMM level. -// If the agent has an explicit Mode in its config (hive.yaml or pack YAML), that takes precedence. -// Otherwise, the default table by ACMM level is used. -func (m *Manager) agentMode(agent *AgentProcess) AgentMode { - if modeStr := agent.Config.Mode; modeStr != "" { - if parsed, ok := ParseAgentMode(modeStr); ok { - return parsed - } - } - return DefaultAgentMode(agent.Name, m.project.ACMMLevel) -} - -// DefaultAgentMode returns the default mode for a given agent name and ACMM level, -// ignoring any hive.yaml override. Used by the dashboard to show "(default)" indicators. -func DefaultAgentMode(agentName string, level int) AgentMode { - if agentName == "supervisor" { - return ModeAdvisory - } - switch level { - case 1: - return ModeAdvisory - case 2: - return ModeAdvisory - case 3: - if agentName == "quality" { - return ModeIssuesAndPRs - } - return ModeAdvisory - case 4: - switch agentName { - case "quality", "sec-check", "ci-maintainer": - return ModeIssuesAndPRs - case "scanner", "guide": - return ModeIssuesOnly - default: - return ModeAdvisory - } - case 5: - return ModeIssuesAndPRs - case 6: - if agentName == "scanner" { - return ModeIssuesPRsMerge - } - return ModeIssuesAndPRs - default: - return ModeAdvisory - } -} - -// AuthorizePROpen enforces the policy for the hive-opens-PR watcher: an agent -// may open a PR (by dropping a request file) only if BOTH hold: -// -// 1. Forge-resistance — the request file's owning UID (fileUID) maps to the -// agent it claims to be (via the uid-map). One agent cannot open a PR "as" -// another, and a non-agent process (unknown UID) is refused. When per-agent -// UIDs are not in play (fileUID <= 0, e.g. shared-dev-UID mode with no map), -// ownership is unverifiable, so we fall back to the ACMM check alone rather -// than hard-failing — the same posture the credential helper takes. -// 2. ACMM write-gate — the agent must be push-capable at the hive's current -// ACMM level, i.e. exactly the CanPush() check that governs `gh pr create`. -// -// Returns nil to authorize, or an error describing the denial. This mirrors the -// direct PR path's policy so the request-file route grants no extra privilege. -func (m *Manager) AuthorizePROpen(agentName string, fileUID int) error { - if strings.TrimSpace(agentName) == "" { - return fmt.Errorf("no agent named in the request") - } - // Forge check: when we have a UID map and a real owning UID, the file owner - // must BE this agent. - if m.uidMap != nil && fileUID > 0 { - owner := m.uidMap.LookupByUID(fileUID) - if owner == "" { - return fmt.Errorf("request file owned by unknown uid %d (not a registered agent)", fileUID) - } - if owner != agentName { - return fmt.Errorf("request claims agent %q but file is owned by agent %q (uid %d)", agentName, owner, fileUID) - } - } - // ACMM write-gate: resolve the agent and check CanPush. - m.mu.RLock() - agent := m.agents[agentName] - m.mu.RUnlock() - if agent == nil { - return fmt.Errorf("unknown agent %q", agentName) - } - if !m.agentMode(agent).CanPush() { - return fmt.Errorf("agent %q is not push-capable at this ACMM level (mode %s) — advisory agents may not open PRs", - agentName, m.agentMode(agent).String()) - } - return nil -} - -// AuthorizeIssueOpen enforces the policy for the issue-request watcher, -// mirroring AuthorizePROpen with the mode gates that govern the direct gh -// paths: "issue" requests need CanCreateIssues() (mode >= ISSUES_ONLY); -// "comment" and "claim" requests need the same (commenting and claiming an -// issue are both issue-writes under the same tier). The same UID -// forge-resistance applies: the request file's owner must BE the claimed -// agent. A nil manager or unknown agent is denied. -func (m *Manager) AuthorizeIssueOpen(agentName string, fileUID int, kind string) error { - if strings.TrimSpace(agentName) == "" { - return fmt.Errorf("no agent named in the request") - } - if m.uidMap != nil && fileUID > 0 { - owner := m.uidMap.LookupByUID(fileUID) - if owner == "" { - return fmt.Errorf("request file owned by unknown uid %d (not a registered agent)", fileUID) - } - if owner != agentName { - return fmt.Errorf("request claims agent %q but file is owned by agent %q (uid %d)", agentName, owner, fileUID) - } - } - m.mu.RLock() - agent := m.agents[agentName] - m.mu.RUnlock() - if agent == nil { - return fmt.Errorf("unknown agent %q", agentName) - } - if !m.agentMode(agent).CanCreateIssues() { - return fmt.Errorf("agent %q may not create issues or comments at this ACMM level (mode %s)", - agentName, m.agentMode(agent).String()) - } - return nil -} - // AuthorizeReviewRequest enforces the policy for the review-request watcher. // It mirrors AuthorizePROpen's forge-resistance, but grants on capability as // well as mode (hivecommons/hive#7485). @@ -2480,657 +1489,6 @@ func (m *Manager) AuthorizeReviewRequest(agentName string, fileUID int) error { return nil } -// AuthorizeMerge enforces the policy for the hive-merges-PR watcher, mirroring -// AuthorizePROpen but with the stricter CanMerge() gate: the request's agent -// must own the request file (forge-resistance) AND be merge-capable at the -// hive's current ACMM level (ModeIssuesPRsMerge). This keeps the file-based -// merge relay under the exact same authority as a direct merge would require — -// an issues/PRs agent that can open PRs still cannot merge them unless its mode -// grants merge. A nil manager or unknown agent is denied. -func (m *Manager) AuthorizeMerge(agentName string, fileUID int) error { - if strings.TrimSpace(agentName) == "" { - return fmt.Errorf("no agent named in the request") - } - // Forge check: when we have a UID map and a real owning UID, the file owner - // must BE this agent. - if m.uidMap != nil && fileUID > 0 { - owner := m.uidMap.LookupByUID(fileUID) - if owner == "" { - return fmt.Errorf("request file owned by unknown uid %d (not a registered agent)", fileUID) - } - if owner != agentName { - return fmt.Errorf("request claims agent %q but file is owned by agent %q (uid %d)", agentName, owner, fileUID) - } - } - // ACMM merge-gate: resolve the agent and check CanMerge. - m.mu.RLock() - agent := m.agents[agentName] - m.mu.RUnlock() - if agent == nil { - return fmt.Errorf("unknown agent %q", agentName) - } - if !m.agentMode(agent).CanMerge() { - return fmt.Errorf("agent %q is not merge-capable at this ACMM level (mode %s) — only ISSUES_PRS_MERGE agents may merge PRs", - agentName, m.agentMode(agent).String()) - } - return nil -} - -// AgentCapabilities reports whether the named agent is ABLE — at the hive's -// current ACMM level and the agent's effective mode — to create issues, open -// PRs, and merge PRs. These are the EXACT gates AuthorizePROpen (CanPush) and -// AuthorizeMerge (CanMerge) enforce, so a hub capability badge derived from -// these can never claim a capability the merge/PR relay would actually refuse. -// ok=false when the agent is unknown to the manager (the caller then reports -// "unknown", not a false negative). Read-only under RLock. -func (m *Manager) AgentCapabilities(agentName string) (canOpenIssue, canOpenPR, canMerge, ok bool) { - m.mu.RLock() - agent, exists := m.agents[agentName] - m.mu.RUnlock() - if !exists || agent == nil { - return false, false, false, false - } - mode := m.agentMode(agent) - return mode.CanCreateIssues(), mode.CanPush(), mode.CanMerge(), true -} - -// EffectiveBackend reports the named agent's effective backend, honoring any -// runtime BackendOverride (see effectiveBackend). ok=false when the agent is -// unknown. Read-only under RLock — a small exported wrapper so callers outside -// the package (the heartbeat builder) need not reach into unexported state. -func (m *Manager) EffectiveBackend(agentName string) (backend string, ok bool) { - m.mu.RLock() - agent, exists := m.agents[agentName] - m.mu.RUnlock() - if !exists || agent == nil { - return "", false - } - return effectiveBackend(agent), true -} - -// InvocationMetadata reports the effective backend, model, and reasoning effort -// the hive invokes for the named agent, accounting for runtime overrides — the -// launch-time truth the invocation-attribution trail records (see pkg/github/attribution -// .go). ok=false when the agent is unknown to the manager (the caller then -// falls back to static config). Read-only under RLock; called from the -// PR-request watcher goroutine, never from the launch path. -func (m *Manager) InvocationMetadata(agentName string) (backend, model, effort string, ok bool) { - m.mu.RLock() - defer m.mu.RUnlock() - agent, exists := m.agents[agentName] - if !exists { - return "", "", "", false - } - backend = effectiveBackend(agent) - model = agent.Config.Model - if agent.ModelOverride != "" { - model = agent.ModelOverride - } - return backend, model, ResolveReasoningEffort(backend, model, agent.Config.ReasoningEffort), true -} - -// ResolveReasoningEffort reports the reasoning effort the hive actually launches -// a given backend/model pair with, given the agent's configured reasoning_effort. -// Exported because the attribution trail is -// resolved in TWO places — Manager.InvocationMetadata above for a running agent, -// and cmd/hive's fallback that reads straight from config when the Manager does -// not know the agent — and both must give the same answer. -// -// Before this existed the fallback carried its own hardcoded "low", so changing -// agyDefaultEffort here would have left cmd/hive silently stamping PRs with an -// effort agy was no longer being launched with. An attribution trail that -// misreports is worse than one that says nothing. -// -// The rules mirror the launch path exactly: -// - agy REQUIRES --effort whenever --model is given, so with a model it runs -// at agyLaunchEffort(configured) and with no model at no effort at all. -// - codex is launched with `-c model_reasoning_effort` only when an effort -// is configured; unset means codex's own default, which the hive does not -// resolve, so the honest answer is the configured value verbatim. -// - every other backend takes its effort from its own config, which the -// hive does not resolve here, so the honest answer is "". -func ResolveReasoningEffort(backend, model, configured string) string { - switch backend { - case "agy": - if model != "" { - return agyLaunchEffort(configured) - } - return "" - case codexBackend: - return configured - } - return "" -} - -// embeddedTokenRe matches git remote URLs with embedded credentials: -// https://x-access-token:TOKEN@github.com/org/repo.git -var embeddedTokenRe = regexp.MustCompile(`^https://[^@]+@(github\.com/.+)$`) - -// sanitizeGitRemotes strips embedded tokens from git remote URLs in all repos -// under the agent's work directory. Copilot CLI embeds the GitHub App token -// directly in the remote URL when it clones, bypassing both the credential -// helper (Layer 1) and env var filtering (Layer 2). -func (m *Manager) sanitizeGitRemotes(agent *AgentProcess) { - if m.agentMode(agent).CanPush() { - return - } - agentDir := m.workDir + "/" + agent.Name - _ = filepath.WalkDir(agentDir, func(path string, d os.DirEntry, err error) error { - if err != nil || d.Name() != ".git" || !d.IsDir() { - return nil - } - repoDir := filepath.Dir(path) - out, err := exec.Command("git", "-C", repoDir, "remote", "get-url", "origin").Output() - if err != nil { - return filepath.SkipDir - } - url := strings.TrimSpace(string(out)) - if match := embeddedTokenRe.FindStringSubmatch(url); match != nil { - clean := "https://" + match[1] - _ = exec.Command("git", "-C", repoDir, "remote", "set-url", "origin", clean).Run() - m.logger.Info("stripped embedded token from git remote", - "agent", agent.Name, "repo", repoDir) - } - return filepath.SkipDir - }) -} - -// agentEnvPair is an unquoted key-value environment variable. -type agentEnvPair struct { - Key string - Value string - // Secret vars are set via tmux set-environment only, never on the command line. - Secret bool -} - -// inferenceQuietCLIEnv is the set of Claude CLI switches exported to -// inference-routed sessions so the CLI stops emitting non-inference traffic -// (telemetry, error reporting, nonessential lookups) to its Anthropic host. -var inferenceQuietCLIEnv = []string{ - "DISABLE_TELEMETRY", - "DISABLE_ERROR_REPORTING", - "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", -} - -func (m *Manager) agentEnvPairs(agent *AgentProcess) []agentEnvPair { - model := agent.Config.Model - if agent.ModelOverride != "" { - model = agent.ModelOverride - } - backend := agent.Config.Backend - if agent.BackendOverride != "" { - backend = agent.BackendOverride - } - displayName := agent.Config.DisplayName - if displayName == "" { - displayName = agent.Name - } - vars := []agentEnvPair{ - {"HIVE_AGENT", agent.Name, false}, - {"HIVE_AGENT_DISPLAY_NAME", displayName, false}, - {"HIVE_BACKEND", backend, false}, - {"HIVE_MODEL", model, false}, - } - if hiveID := os.Getenv("HIVE_ID"); hiveID != "" { - vars = append(vars, agentEnvPair{"HIVE_ID", hiveID, false}) - } - vars = append(vars, agentEnvPair{"HIVE_ACMM_LEVEL", fmt.Sprintf("%d", m.project.ACMMLevel), false}) - - mode := m.agentMode(agent) - if agent.Config.Tools != nil { - if effectiveMode := agent.Config.Tools.EffectiveMode(); effectiveMode != "" { - vars = append(vars, agentEnvPair{"HIVE_AGENT_MODE", effectiveMode, false}) - } else { - vars = append(vars, agentEnvPair{"HIVE_AGENT_MODE", mode.String(), false}) - } - } else { - vars = append(vars, agentEnvPair{"HIVE_AGENT_MODE", mode.String(), false}) - } - modeFile := filepath.Join(agentStateDir, ".hive-mode-"+agent.Name) - if err := writeAgentStateFile(modeFile, []byte(mode.String())); err != nil { - m.logger.Warn("agentBootstrapEnv: mode file write failed", "file", modeFile, "error", err) - } - m.writeAgentCapsFile(agent.Name, m.agentCapabilities(agent)) - // Plain proxy URL without userinfo — Claude Code's native binary fails - // to open a socket when the URL contains username:password@ (FailedToOpenSocket). - // Agent identification uses UID-based /proc/net/tcp lookup instead of - // Proxy-Authorization headers. GIT_TERMINAL_PROMPT=0 prevents git from - // prompting for proxy credentials. - proxyURL := fmt.Sprintf("http://127.0.0.1:%d", proxyListenPort) - vars = append(vars, agentEnvPair{"HTTPS_PROXY", proxyURL, false}) - vars = append(vars, agentEnvPair{"HTTP_PROXY", proxyURL, false}) - vars = append(vars, agentEnvPair{"HIVE_PROXY_AGENT", agent.Name, false}) - vars = append(vars, agentEnvPair{"GIT_TERMINAL_PROMPT", "0", false}) - vars = append(vars, agentEnvPair{"NODE_EXTRA_CA_CERTS", proxyCACertPath, false}) - if sha := os.Getenv("HIVE_SHA"); sha != "" { - vars = append(vars, agentEnvPair{"HIVE_SHA", sha, false}) - } - if advisory := os.Getenv("HIVE_ADVISORY_ISSUE"); advisory != "" { - vars = append(vars, agentEnvPair{"HIVE_ADVISORY_ISSUE", advisory, false}) - } - // HIVE_REPO / HIVE_REPOS: the shipped policy templates instruct agents to - // run `gh issue create --repo "$HIVE_REPO"`, but nothing ever exported it - // to hosted agents (only the OSS scheduler set a hardcoded "/hive"). - // Root-caused on a live hosted hive (2026-08-20): the sec-check agent saw - // HIVE_REPO unset, fell back to the git remote of its own workdir, and - // silently scanned only the primary repo — the other project repos were - // never touched. Export the primary repo and the full project repo list so - // templates and agents can target every configured repo. - if m.project.Org != "" && len(m.project.Repos) != 0 { - // Both variables are per-agent now (#6204). A repo-scoped agent must - // not be handed a $HIVE_REPO it may not write to — every shipped - // template example passes it straight to `gh ... --repo "$HIVE_REPO"` — - // and $HIVE_REPOS is the list templates iterate as "everything in - // scope". For an unscoped agent both resolve exactly as before. - scoped := m.project.ReposFor(agent.Name) - primary := m.project.PrimaryRepoFor(agent.Name) - if primary == "" && len(scoped) > 0 { - primary = scoped[0] - } - if primary != "" { - vars = append(vars, agentEnvPair{"HIVE_REPO", config.QualifyRepo(m.project.Org, primary), false}) - } - // HIVE_REPOS is the work scope templates iterate: the agent's own repos - // (#6204) minus any the operator has paused (#6203). HIVE_REPO above is - // identity, not scope, and keeps naming the agent's primary repo even - // while it is paused — see ActiveReposFor. - active := m.project.ActiveReposFor(agent.Name) - full := make([]string, len(active)) - for i, r := range active { - full[i] = config.QualifyRepo(m.project.Org, r) - } - vars = append(vars, agentEnvPair{"HIVE_REPOS", strings.Join(full, ","), false}) - } - // GH_HOST: point the gh CLI at the configured forge host for GHE spokes. - // See ProjectContext.GHHost. The gh wrapper pairs this with - // GH_ENTERPRISE_TOKEN so the per-agent scoped token authenticates there. - if m.project.GHHost != "" { - vars = append(vars, agentEnvPair{"GH_HOST", m.project.GHHost, false}) - } - if m.routableBackend(backend) { - const inferenceTranslatePort = 18444 - vars = append(vars, agentEnvPair{"ANTHROPIC_API_KEY", "sk-hive-" + agent.Name, false}) - baseURL := fmt.Sprintf("http://127.0.0.1:%d", inferenceTranslatePort) - vars = append(vars, agentEnvPair{"ANTHROPIC_BASE_URL", baseURL, false}) - vars = append(vars, agentEnvPair{"NO_PROXY", "127.0.0.1,localhost", false}) - // Cap the CLI output-token budget at a value every commercial model - // litellm may front will accept. A prior 128000 (chosen so verbose - // OSS models would not truncate) exceeds Azure GPT-4o's 16384 - // completion-token cap, so every request 400s with - // "max_tokens is too large: 128000. This model supports at most - // 16384 completion tokens". See inferenceMaxOutputTokensDefault. - // TODO: the gateway 400 body names the model's real cap ("supports - // at most N completion tokens"); a future enhancement could parse it - // to auto-adjust per-model instead of using a universal floor. - vars = append(vars, agentEnvPair{"CLAUDE_CODE_MAX_OUTPUT_TOKENS", strconv.Itoa(inferenceMaxOutputTokensDefault), false}) - // The Claude CLI sends telemetry batches, error reports, and other - // non-inference traffic to its configured Anthropic host. Routed at - // an OpenAI-compatible gateway that traffic has nowhere useful to go - // (the proxy now answers it locally rather than forwarding it — see - // classifyInferencePath), so switch it off at the source. Only for - // inference-routed sessions: subscription/Anthropic-direct sessions - // keep Anthropic's own telemetry. - for _, v := range inferenceQuietCLIEnv { - vars = append(vars, agentEnvPair{v, "1", false}) - } - } - if m.copilotAuthToken != "" { - vars = append(vars, agentEnvPair{copilotTokenEnvVar, m.copilotAuthToken, true}) - } - // Point the GitHub MCP server at the App installation token so PRs, issue - // comments, and merges are authored by the App bot ("[bot]") — NOT by - // the Copilot login user. COPILOT_GITHUB_TOKEN above stays as the Copilot - // OAuth token because it authenticates the AI model (a separate concern from - // GitHub write identity); leaving it untouched keeps the Copilot CLI login - // working. The Copilot CLI reads GH_TOKEN / GITHUB_TOKEN for GitHub API auth - // (per its README: GH_TOKEN or GITHUB_TOKEN, in that precedence), so setting - // GITHUB_TOKEN here makes the built-in GitHub MCP server act as the App bot. - // - // Gated on the opt-in flag first (default OFF → no behavior change on any - // hive that has not explicitly enabled App-bot authorship), then on CanPush(): - // advisory agents are deliberately kept GITHUB_TOKEN-less (see the -u - // GITHUB_TOKEN strip after the env loop) so they cannot write; only push- - // capable tiers — the ones that legitimately open/merge PRs — get the App - // token. m.appAuth != nil means an App is configured. The value is the - // per-agent tier-SCOPED App token, and refreshAgentTokens re-pushes it hourly - // so it never goes stale. - if m.project.AppAuthoredPRs && m.appAuth != nil && agent.UID > 0 && m.agentMode(agent).CanPush() { - if data, err := os.ReadFile(ghpkg.AgentTokenCachePath(agent.Name)); err == nil { - if tok := strings.TrimSpace(string(data)); tok != "" { - vars = append(vars, agentEnvPair{"GITHUB_TOKEN", tok, true}) - } - } - } - // Linear write credential for ISSUES_ONLY+ agents — see linearEnvPairs. - // Nil for advisory agents and for hives with no Linear credential, so a - // GitHub-only hive sees no change. - vars = append(vars, m.linearEnvPairs(agent)...) - // CLAUDE_CODE_OAUTH_TOKEN is a LAST RESORT, not the normal delivery path. - // - // Claude Code treats this variable as a static bearer token: when it is - // set the CLI uses it verbatim, never opens ~/.claude/.credentials.json, - // and therefore never refreshes. Measured in-container (2026-09-01): with - // the variable set to a bad value and a perfectly good credentials file - // beside it, the CLI answered "401 OAuth access token is invalid" — there - // is no fallback to the file. - // - // m.claudeAuthToken is a snapshot of the SHORT-LIVED access token, taken - // once at manager construction and refreshed only by ReloadClaudeToken() - // after a dashboard login. Injecting it therefore pinned every claude - // agent to the remaining life of whatever access token happened to be on - // disk when the container started — Claude access tokens live 8h, so the - // whole fleet 401'd within a day of every restart and the only recovery - // hive offered was an operator re-login, once per agent. That is the daily - // re-authentication treadmill of #5454. - // - // It is also unnecessary since per-agent homes (#4619): every agent's - // ~/.claude is a symlink to the shared /data/home/.claude, so the CLI can - // read the credential itself — and redeem its refresh grant on start, - // which is the one thing the env var makes impossible. - // - // So inject ONLY when the agent has no credential file it can read. That - // keeps the variable doing the job it was added for (#c5648bc9: deliver a - // dashboard-obtained token to an agent that cannot see the file) and stops - // it overriding a credential that can still refresh itself. - if m.claudeAuthToken != "" && backend == "claude" && !claudeCredentialReachable(agent, backend) { - vars = append(vars, agentEnvPair{"CLAUDE_CODE_OAUTH_TOKEN", m.claudeAuthToken, true}) - } - // bob reads its key from BOBSHELL_API_KEY. Secret: true keeps the value off - // the shell command line (out of `ps`, bash history, and pane scrollback); - // it reaches the CLI via tmux set-environment only. Gated on the backend so - // no other CLI's environment carries an IBM credential it has no use for. - if backend == bobBackend { - if key := m.bobAPIKey(); key != "" { - vars = append(vars, agentEnvPair{config.BobAPIKeyEnvVar, key, true}) - } - // BOBSHELL_DEFAULT_AUTH_TYPE is what actually selects API-key auth; - // without it bob defaults to W3ID SSO and parks at the interactive key - // prompt forever. Deliberately NOT Secret: the value is the literal - // non-credential string "api-key", and secret pairs only reach a - // freshly-created pane shell via tmux set-environment, whereas - // non-secret pairs are re-applied on EVERY launch through - // buildEnvPrefix. That asymmetry is exactly what caused the sibling - // bug fixed in #2228, so the auth type must ride the always-reapplied - // path or a relaunch into an existing session loses it. - vars = append(vars, agentEnvPair{config.BobAuthTypeEnvVar, config.BobAuthTypeAPIKey, false}) - } - // BD_DIR tells the `bd` CLI where to read/write beads. Without this, - // bd falls back to cwd (/data/agents/) instead of the configured - // beads_dir (/data/beads/), causing a path mismatch with the - // dashboard and advisory digest. - if agent.Config.BeadsDir != "" { - vars = append(vars, agentEnvPair{"BD_DIR", agent.Config.BeadsDir, false}) - } - if agent.Config.CavemanMode != "" { - vars = append(vars, agentEnvPair{"HIVE_CAVEMAN_MODE", agent.Config.CavemanMode, false}) - } - // Export the RESOLVED explain mode, not the raw config value, so an agent's - // skills and helper scripts see the same answer the kick suffix acted on - // (including inheritance from the hive-wide default and the off fallback for - // an invalid value). Always exported, off included, so a script can branch on - // it without having to re-derive the precedence rules itself. - vars = append(vars, agentEnvPair{config.ExplainModeEnvVar, resolveExplainMode(agent.Config, m.explainModeDefault()), false}) - // GIT_SSL_CAINFO only — NOT SSL_CERT_FILE (that breaks Copilot API TLS) - vars = append(vars, agentEnvPair{"GIT_SSL_CAINFO", proxyCACertPath, false}) - if agent.UID > 0 { - vars = append(vars, agentEnvPair{"HIVE_AGENT_TOKEN_CACHE", ghpkg.AgentTokenCachePath(agent.Name), false}) - } - if agent.UID > 0 { - // Per-UID agents get a per-agent HOME (#4596) — AgentHome is the single - // source of truth so the auth probe and this export can never diverge. - vars = append(vars, agentEnvPair{"HOME", AgentHome(agent.Name, agent.UID, backend), false}) - - // Per-agent XDG data/state roots (#6238), beneath the per-agent HOME. - // Every backend CLI keeps its session transcripts, run locks and - // caches under $XDG_DATA_HOME / $XDG_STATE_HOME; with the legacy - // shared /data/home/.local those were one contended tree owned by - // whichever agent wrote first. Exported explicitly (not left to the - // spec default under $HOME) so the answer cannot depend on how each - // CLI resolves XDG, and only when HOME itself is per-agent — the - // HIVE_SHARED_AGENT_HOME=1 escape hatch keeps the legacy layout whole. - // XDG_CONFIG_HOME is deliberately NOT set: ~/.config stays the shared - // credential/config bridge (gh hosts.yml, goose config.yaml). See - // setupAgentXDGDirs, which pre-creates these as the agent's own dirs. - if xdgHome, ok := perAgentXDGHome(agent.Name, agent.UID, backend); ok { - vars = append(vars, agentEnvPair{"XDG_DATA_HOME", agentXDGDataHome(xdgHome), false}) - vars = append(vars, agentEnvPair{"XDG_STATE_HOME", agentXDGStateHome(xdgHome), false}) - } - - // Under the per-agent-UID layout the global npm prefix is owned by the - // image's build user, so the Claude Code CLI's self-updater fails on - // every launch with "✘ Auto-update failed: no write permission to npm - // prefix" — a red line in every agent pane for an update the agent must - // not perform anyway (the CLI version is managed by the image, not by - // an in-pod npm write). Disabling the updater removes the failure at its - // source; a per-agent npm prefix would instead let an agent drift off - // the pinned image version. - vars = append(vars, agentEnvPair{"DISABLE_AUTOUPDATER", "1", false}) - } - - // Codex CLI 0.144.1's in-process app-server performs OWNER-gated operations - // on files under CODEX_HOME (helper-binary "PATH alias" symlinks under - // tmp/arg0, sqlite state). The shared /data/home/.codex is owned by dev - // (the entrypoint chowns it group-writable + setgid), which claude/copilot - // tolerate but Codex does not — every non-owner agent UID fails with - // "failed to start embedded app server: Operation not permitted (os error 1)". - // The manager launches the codex binary DIRECTLY (not via agent-launch.sh), - // so CODEX_HOME must be set here. Give each agent its own dir; codex will - // NOT create it (it errors "CODEX_HOME ... does not exist"), so it is - // pre-created AS the agent below in setupCodexHome. - if backend == codexBackend { - vars = append(vars, agentEnvPair{"CODEX_HOME", codexHomePath(agent.Name), false}) - } - - for _, conn := range agent.Config.Connections { - if conn.Type != "api" { - continue - } - envName := conn.EnvName - if envName == "" { - envName = "HIVE_CONN_" + strings.ToUpper(strings.ReplaceAll(conn.Name, "-", "_")) + "_URL" - } - vars = append(vars, agentEnvPair{envName, conn.URI, false}) - if conn.Auth != nil && conn.Auth.Type == "env" && conn.Auth.EnvVar != "" { - if tokenVal := os.Getenv(conn.Auth.EnvVar); tokenVal != "" { - vars = append(vars, agentEnvPair{conn.Auth.EnvVar, tokenVal, true}) - } - } - } - - return vars -} - -func (m *Manager) PinCLI(name, version string) error { - m.mu.Lock() - defer m.mu.Unlock() - - agent, ok := m.agents[name] - if !ok { - return fmt.Errorf("agent %s not found", name) - } - - agent.PinnedCLI = version - m.logger.Info("agent CLI pinned", "name", name, "version", version) - return nil -} - -func (m *Manager) UnpinCLI(name string) error { - m.mu.Lock() - defer m.mu.Unlock() - - agent, ok := m.agents[name] - if !ok { - return fmt.Errorf("agent %s not found", name) - } - - agent.PinnedCLI = "" - m.logger.Info("agent CLI unpinned", "name", name) - return nil -} - -func (m *Manager) PinModel(name, model string) error { - m.mu.Lock() - defer m.mu.Unlock() - - agent, ok := m.agents[name] - if !ok { - return fmt.Errorf("agent %s not found", name) - } - - prevModel := agent.effectiveModel() - agent.PinnedModel = model - agent.ModelOverride = model - m.logger.Info("agent model pinned", "name", name, "model", model) - if prevModel != model { - m.audit(AuditAgentModelSet, name, auditFields( - "outcome", "success", - "backend", agent.effectiveBackend(), - "model", model, - "previous_model", prevModel, - "trigger", "pin", - )) - } - return nil -} - -func (m *Manager) UnpinModel(name string) error { - m.mu.Lock() - defer m.mu.Unlock() - - agent, ok := m.agents[name] - if !ok { - return fmt.Errorf("agent %s not found", name) - } - - agent.PinnedModel = "" - m.logger.Info("agent model unpinned", "name", name) - return nil -} - -func (m *Manager) SetModelOverride(name, model string) error { - m.mu.Lock() - defer m.mu.Unlock() - - agent, ok := m.agents[name] - if !ok { - return fmt.Errorf("agent %s not found", name) - } - - // Store the CLI-accepted spelling for copilot so the persisted selection, - // the dropdown preselect, and auto-heal all agree on one canonical id - // (separator drift like claude-fable.5 vs claude-fable-5, #4262). Launch - // re-applies the same canonicalization, so even ids stored before this - // existed self-correct there. - if agent.effectiveBackend() == "copilot" { - model = CanonicalizeCopilotModel(model) - } - - // A pin blocks the governor's auto-selection, never a user's explicit - // switch: retarget the pin to the new model so the pin state is - // unchanged (still pinned) while the change takes effect. - if agent.PinnedModel != "" { - agent.PinnedModel = model - m.logger.Info("agent model pin retargeted by user switch", "name", name, "model", model) - } - - prevModel := agent.effectiveModel() - agent.ModelOverride = model - m.logger.Info("agent model override set", "name", name, "model", model) - // State CHANGES only — the governor re-asserts the current model on every - // evaluation cycle, so auditing unchanged writes would flood the ring. - if prevModel != model { - m.audit(AuditAgentModelSet, name, auditFields( - "outcome", "success", - "backend", agent.effectiveBackend(), - "model", model, - "previous_model", prevModel, - )) - } - - effectiveBackend := agent.Config.Backend - if agent.BackendOverride != "" { - effectiveBackend = agent.BackendOverride - } - if m.routableBackend(effectiveBackend) && m.inferenceRouteCallback != nil { - m.inferenceRouteCallback(name, effectiveBackend, model) - } - return nil -} - -func (m *Manager) SetBackendOverride(name, backend string) error { - m.mu.Lock() - defer m.mu.Unlock() - - agent, ok := m.agents[name] - if !ok { - return fmt.Errorf("agent %s not found", name) - } - - // Refuse a backend the launcher cannot dispatch, at SET time. Previously - // any string was accepted here and the agent was then restarted into it, - // failing only at launch with "unknown backend: " — the agent stops - // working and the operator gets no signal at the moment of the change. - // routableBackend covers configured gateway names (resolved live), so a - // gateway-named backend still passes. - if err := m.validateBackendName(backend); err != nil { - return err - } - - // Captured after validation so a rejected switch records no audit event: - // the override is only mutated below, once the backend is known routable. - prevBackend := agent.effectiveBackend() - agent.BackendOverride = backend - m.logger.Info("agent backend override set", "name", name, "backend", backend) - // Record only a real transition: /switch/{backend} is also re-applied on - // config reload with the value already in effect, and auditing those - // no-ops would bury the actual operator changes. - if prevBackend != backend { - m.audit(AuditAgentBackendSet, name, auditFields( - "outcome", "success", - "backend", backend, - "model", agent.effectiveModel(), - "previous_backend", prevBackend, - )) - } - - if m.routableBackend(backend) && m.inferenceRouteCallback != nil { - model := agent.ModelOverride - if model == "" { - model = agent.Config.Model - } - m.inferenceRouteCallback(name, backend, model) - } else if !IsInferenceBackend(backend) && m.clearInferenceRouteCallback != nil { - m.clearInferenceRouteCallback(name) - } - return nil -} - -// RefreshInferenceRoutes re-fires the inference route callback for every -// agent whose effective backend matches, so endpoint or credential changes -// (e.g. a governor LiteLLM config save) take effect on live agents without -// a restart. -func (m *Manager) RefreshInferenceRoutes(backend string) { - m.mu.Lock() - defer m.mu.Unlock() - if m.inferenceRouteCallback == nil || !m.routableBackend(backend) { - return - } - for name, agent := range m.agents { - effective := agent.Config.Backend - if agent.BackendOverride != "" { - effective = agent.BackendOverride - } - if effective != backend { - continue - } - model := agent.ModelOverride - if model == "" { - model = agent.Config.Model - } - m.inferenceRouteCallback(name, backend, model) - } -} - // GetBufferOutput returns output from the ring buffer directly, bypassing // the tmux pane capture. The ring buffer accumulates all output over time // (up to 500 lines) while the pane capture only has visible lines. diff --git a/src/pkg/agent/manager_consent.go b/src/pkg/agent/manager_consent.go new file mode 100644 index 0000000000..7434d8344c --- /dev/null +++ b/src/pkg/agent/manager_consent.go @@ -0,0 +1,295 @@ +package agent + +import ( + "strings" + "time" +) + +const ( + // consentConfirmFooter appears at the bottom of Claude Code interactive + // selection screens (consent dialogs, settings-error menus). + consentConfirmFooter = "Enter to confirm" + // bypassConsentTitle is the heading of the --dangerously-skip-permissions + // consent screen. Its default selection is "No, exit" — confirming it + // terminates the CLI and leaves a bare bash pane. + bypassConsentTitle = "Bypass Permissions mode" + // bypassConsentDefaultOption is the default (negative) option on the + // bypass-permissions consent screen. + bypassConsentDefaultOption = "No, exit" + // bypassConsentAcceptOption is the affirmative option on the + // bypass-permissions consent screen. Its position varies between CLI + // versions, so acceptance navigates by matching the selected-line text. + bypassConsentAcceptOption = "Yes, I accept" + // apiKeyPromptTitle is the heading of the custom-API-key approval prompt, + // shown when ANTHROPIC_API_KEY is not in customApiKeyResponses.approved. + // Its default selection is "No (recommended)" with the affirmative option + // above it. + apiKeyPromptTitle = "Detected a custom API key" + // apiKeyPromptAcceptOption is the affirmative option on the + // custom-API-key approval prompt. + apiKeyPromptAcceptOption = "Yes" + // cliWorkingMarker is shown while Claude Code is actively processing a + // request; a pane in this state is never a consent screen. + cliWorkingMarker = "esc to interrupt" +) + +// paneShowsConsentScreen reports whether the pane is showing an interactive +// consent/selection screen rather than a ready CLI input prompt. Such screens +// contain a "❯"-selected menu option (e.g. "❯ 1. No, exit"), so they satisfy +// marker-based CLI presence checks ("❯" is also a cliPaneMarkers entry) — a +// kick typed into one is consumed by the menu, or by bash once the default +// "No, exit" selection terminates the CLI. Callers should pass the visible +// pane only (no scrollback): dismissed consent screens linger in history. +func paneShowsConsentScreen(pane string) bool { + if pane == "" || strings.Contains(pane, cliWorkingMarker) { + return false + } + // A known startup-blocking menu is not a ready prompt either. The generic + // test below needs the "Enter to confirm" footer AND a "❯"-marked line; + // codex renders neither (its footer is "Press enter to continue" and its + // marker is "›" U+203A), so its update menu read as READY. Everything that + // gates on readiness — the startup kick, caveman activation — then typed + // into the menu, and the Enter confirmed its pre-selected option: + // "1. Update now", which runs `npm install -g` as the agent UID, fails, and + // kills the CLI. Blocking on these lets the prompt watcher answer them. + if paneHasBlockingPrompt(pane) { + return true + } + if strings.Contains(pane, bypassConsentTitle) && strings.Contains(pane, bypassConsentDefaultOption) { + return true + } + if !strings.Contains(pane, consentConfirmFooter) { + return false + } + for _, line := range strings.Split(pane, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "❯") { + return true + } + } + return false +} + +// dismissInferencePrompts polls the tmux pane for Claude Code interactive +// prompts and auto-dismisses them. The "Bypass Permissions mode" consent +// screen and the custom-API-key approval prompt are handled first and +// explicitly (see confirmMenuOption): their default selections are negative +// ("No, exit" / "No (recommended)"), so confirming blind terminates the CLI +// or declines the seeded key. +// Other prompts are handled dynamically regardless of prompt text changes +// between Claude Code versions by: +// 1. Detecting "Enter to confirm" (universal prompt footer) +// 2. Finding the selected option (line with "❯" marker) +// 3. If selected option looks negative (contains "No" or "exit"), navigate +// away from it before confirming +// 4. For "Press Enter to continue" screens, just press Enter +// +// The pane is polled fast for the first 10s — the consent screen appears +// within ~5-8s of launch and every second it lingers is a window for a kick +// to be swallowed by the menu — then at a relaxed interval. +// +// Stops when the main Claude Code input prompt appears ("esc to interrupt"). +func (m *Manager) dismissInferencePrompts(agent *AgentProcess) { + const ( + // promptFastPollWindow covers the launch window in which the consent + // screen normally appears (~5-8s after CLI start). + promptFastPollWindow = 10 * time.Second + promptFastPollInterval = 250 * time.Millisecond + promptPollInterval = 1 * time.Second + promptDismissTimeout = 60 * time.Second + postKeystrokeDelay = 500 * time.Millisecond + ) + + start := time.Now() + timeout := promptDismissTimeout + if m.promptDismissTimeout > 0 { + timeout = m.promptDismissTimeout + } + deadline := start.Add(timeout) + lastPane := "" + + for time.Now().Before(deadline) { + interval := promptPollInterval + if time.Since(start) < promptFastPollWindow { + interval = promptFastPollInterval + } + m.sleepDuringPromptDismiss(interval) + + pane := m.captureVisiblePaneForAgent(agent) + if pane == "" { + continue + } + + // Bypass-permissions consent screen: handle first and explicitly, + // even if the pane is unchanged since the last poll (a mistimed + // keystroke must be retried, not skipped). The affirmative option + // sits below the default "No, exit". + if strings.Contains(pane, bypassConsentTitle) && !strings.Contains(pane, cliWorkingMarker) { + m.logger.Info("accepting bypass-permissions consent", "agent", agent.Name) + m.confirmMenuOption(agent, bypassConsentTitle, bypassConsentAcceptOption, "Down") + lastPane = "" // re-capture fresh on the next pass + continue + } + + // Custom-API-key approval prompt: the affirmative "Yes" sits ABOVE + // the default "No (recommended)" selection, so the generic + // Down-then-Enter fallback below would decline it. + if strings.Contains(pane, apiKeyPromptTitle) && !strings.Contains(pane, cliWorkingMarker) { + m.logger.Info("approving seeded inference API key", "agent", agent.Name) + m.confirmMenuOption(agent, apiKeyPromptTitle, apiKeyPromptAcceptOption, "Up") + lastPane = "" + continue + } + + if pane == lastPane { + continue + } + lastPane = pane + + // Main prompt visible — agent is ready + if strings.Contains(pane, "bypass permissions on") || strings.Contains(pane, "esc to interrupt") { + m.logger.Info("inference agent ready", "agent", agent.Name) + return + } + + // "Press Enter to continue" screens + if strings.Contains(pane, "Press Enter to continue") { + m.logger.Info("inference prompt: press enter", "agent", agent.Name) + m.tmuxSendKeysForAgent(agent, "Enter") + continue + } + + // Selection prompts have "Enter to confirm" footer + if !strings.Contains(pane, "Enter to confirm") { + continue + } + + // Find the currently selected option (marked with ❯) + selected := selectedMenuOption(pane) + + m.logger.Info("inference prompt detected", + "agent", agent.Name, + "selected", selected, + ) + + // If current selection looks negative, navigate away from it + selectedLower := strings.ToLower(selected) + if strings.Contains(selectedLower, "no,") || strings.Contains(selectedLower, "no ") || + strings.Contains(selectedLower, "exit") { + // Try moving down first (most prompts put the positive option below) + m.tmuxSendKeysForAgent(agent, "Down") + m.sleepDuringPromptDismiss(postKeystrokeDelay) + } else if strings.Contains(selectedLower, "fix with") { + // Settings error: skip past "Fix with Claude" and "Exit" to "Continue without" + m.tmuxSendKeysForAgent(agent, "Down") + m.sleepDuringPromptDismiss(postKeystrokeDelay) + m.tmuxSendKeysForAgent(agent, "Down") + m.sleepDuringPromptDismiss(postKeystrokeDelay) + } + + m.tmuxSendKeysForAgent(agent, "Enter") + } + + m.logger.Warn("inference prompt dismissal timed out", "agent", agent.Name) +} + +func (m *Manager) sleepDuringPromptDismiss(d time.Duration) { + m.terminalSession().SleepDuringPromptDismiss(d) +} + +// selectedMenuOption returns the trimmed text of the "❯"-selected line of an +// interactive CLI menu, or "" if no line is selected. +func selectedMenuOption(pane string) string { + for _, line := range strings.Split(pane, "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "❯") { + return trimmed + } + } + return "" +} + +// confirmMenuOption drives an interactive CLI menu identified by title to the +// option whose text contains want, then confirms it with Enter. Navigation +// matches the "❯"-selected line text rather than pressing a fixed number of +// keys, so it lands on the right option whichever position it occupies (menu +// option order differs between Claude CLI versions). navKey is the arrow key +// to step with ("Down" or "Up"). Returns true once the option was confirmed +// or the screen is gone. +func (m *Manager) confirmMenuOption(agent *AgentProcess, title, want, navKey string) bool { + const ( + // menuMaxNavigateSteps bounds arrow-key navigation; the handled menus + // have 2 options, extra headroom covers future variants. + menuMaxNavigateSteps = 4 + postKeystrokeDelay = 500 * time.Millisecond + ) + for step := 0; step < menuMaxNavigateSteps; step++ { + pane := m.captureVisiblePaneForAgent(agent) + if !strings.Contains(pane, title) || strings.Contains(pane, cliWorkingMarker) { + return true // screen already dismissed + } + if strings.Contains(selectedMenuOption(pane), want) { + m.tmuxSendKeysForAgent(agent, "Enter") + m.sleepDuringPromptDismiss(postKeystrokeDelay) + return true + } + m.tmuxSendKeysForAgent(agent, navKey) + m.sleepDuringPromptDismiss(postKeystrokeDelay) + } + m.logger.Warn("inference menu: wanted option not reached", + "agent", agent.Name, "title", title, "want", want) + return false +} + +const ( + // consentStuckGracePeriod is how long a consent screen must stay visible + // across watcher passes before the agent counts as stuck. The launch-time + // dismissal goroutine runs for 60s, so a screen still visible this long + // after first being seen by the watcher means dismissal lost the race. + consentStuckGracePeriod = 30 * time.Second + // consentDismissCooldown is the minimum interval between watcher-triggered + // dismissal passes for one agent, so a stubborn screen can't spam + // keystroke goroutines (each dismissal pass itself polls for 60s). + consentDismissCooldown = 2 * time.Minute +) + +// clearConsentTracking resets the consent-stuck timer for an agent whose pane +// no longer shows a consent screen. +func (m *Manager) clearConsentTracking(name string) { + m.mu.Lock() + defer m.mu.Unlock() + if agent, ok := m.agents[name]; ok { + agent.consentSeenAt = time.Time{} + } +} + +// dismissConsentIfStuck re-runs dismissInferencePrompts for an inference agent +// whose pane has shown a consent screen for longer than the grace period, +// subject to a per-agent cooldown. Called from the watcher loop +// (CheckAndRestartCrashedAgents) so an agent that lands on a consent screen +// after launch — e.g. a crash-recovery restart whose launch-time dismissal +// timed out — recovers instead of sitting stuck while kicks appear to succeed. +func (m *Manager) dismissConsentIfStuck(name string) { + now := time.Now() + m.mu.Lock() + agent, ok := m.agents[name] + if !ok { + m.mu.Unlock() + return + } + if agent.consentSeenAt.IsZero() { + agent.consentSeenAt = now + m.mu.Unlock() + return + } + stuckFor := now.Sub(agent.consentSeenAt) + if stuckFor < consentStuckGracePeriod || now.Sub(agent.lastConsentDismiss) < consentDismissCooldown { + m.mu.Unlock() + return + } + agent.lastConsentDismiss = now + m.mu.Unlock() + + m.logger.Warn("inference agent stuck on consent screen, re-running prompt dismissal", + "name", name, "stuck_seconds", int(stuckFor.Seconds())) + go m.dismissInferencePrompts(agent) +} diff --git a/src/pkg/agent/manager_copilot_auth.go b/src/pkg/agent/manager_copilot_auth.go new file mode 100644 index 0000000000..f3279163b5 --- /dev/null +++ b/src/pkg/agent/manager_copilot_auth.go @@ -0,0 +1,48 @@ +package agent + +import ( + "time" +) + +const ( + sharedConfigDesiredMode = 0o660 + // agyDefaultEffort is the reasoning effort passed alongside agy's --model + // when the agent has no usable reasoning_effort configured (see + // agyLaunchEffort). agy requires --effort whenever --model is given and + // otherwise ignores the model entirely; "low" is the effort agy defaults + // to on its own, so this makes the configured model take effect without + // changing behaviour. + agyDefaultEffort = "low" + + tokenRestartCooldownSec = 60 // minimum seconds between token-triggered restarts per agent + // loginPromptTailLines bounds the pane region the login-prompt detector + // reads: a prompt the CLI is stuck at sits at the pane bottom, while + // echoed kick text and startup flashes live in scrollback (see the poller). + loginPromptTailLines = 15 + // loginStreakRestartMin is how many consecutive polls (~3s apart) must see + // the login prompt before a token-triggered restart may fire — filters the + // CLI's transient startup "/login" flash. + loginStreakRestartMin = 3 + // tokenRestartMaxAttempts bounds CONSECUTIVE token-triggered restarts that + // fail to clear the login prompt. + // + // The three guards above answer WHEN to restart; none of them answered HOW + // MANY TIMES, so a restart that could never work was retried forever at the + // cooldown interval. #4596 is precisely that shape: the shared credential is + // valid (so configHasTokens() is true) while $HOME/.claude.json has lost its + // oauthAccount (so the CLI shows the login menu regardless), and each + // restart re-launched a CLI that rewrote the same contended file and asked + // again. Restarts are not free — they destroy in-flight work, which is the + // failure the kick grace above was added for. + // + // Three is deliberately generous: one restart genuinely does fix the case + // this feature was built for (an operator authenticates in one agent's + // terminal and the others need a nudge), so the cap only engages on a + // theory that has now failed repeatedly. + tokenRestartMaxAttempts = 3 + // tokenRestartKickGrace suppresses token-triggered restarts after a kick + // delivery so the restart can never destroy just-delivered work. + tokenRestartKickGrace = 10 * time.Minute + expiredTokenHangTimeoutSec = 180 // blank pane after this many seconds triggers token purge + restart + tlsErrorRestartCooldownSec = 120 // minimum seconds between TLS-error-triggered restarts per agent +) diff --git a/src/pkg/agent/manager_env.go b/src/pkg/agent/manager_env.go new file mode 100644 index 0000000000..ff6e57d221 --- /dev/null +++ b/src/pkg/agent/manager_env.go @@ -0,0 +1,433 @@ +package agent + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + + "github.com/hivecommons/hive/pkg/config" + ghpkg "github.com/hivecommons/hive/pkg/github" +) + +func (m *Manager) buildBootstrapPrompt(agent *AgentProcess) string { + // No boot prompt — the governor's first eval cycle (10s after startup) + // kicks all due agents via BuildKickMessages with fully substituted + // templates. Sending a boot prompt here caused unsubstituted ${ISSUE_LIST} + // and other vars to reach the agent. The policy-file path list this + // function used to assemble was dead code once the boot prompt was + // removed, so it is gone too. + _ = agent // signature kept for the call site; the arg is no longer read + return "" +} + +// metricsCachePath is a var (not const) so tests can point it at a temp file +// to exercise readCoveragePreamble without a real /data volume. Production +// value is unchanged. +var metricsCachePath = "/data/metrics/agent-metrics-cache.json" + +func (m *Manager) readCoveragePreamble() string { + data, err := os.ReadFile(metricsCachePath) + if err != nil { + return "" + } + var metrics map[string]map[string]json.Number + if err := json.Unmarshal(data, &metrics); err != nil { + return "" + } + ci, ok := metrics["ci-maintainer"] + if !ok { + return "" + } + cov, err := ci["coverage"].Int64() + if err != nil { + return "" + } + target, err := ci["coverageTarget"].Int64() + if err != nil { + target = 91 + } + return fmt.Sprintf("[COVERAGE] Current: %d%% | Target: %d%%.", cov, target) +} + +// shellEnvVar formats KEY='value' with single-quoting so values containing +// spaces, parentheses, or other shell metacharacters are safe in inline env +// var assignments sent to tmux via send-keys. +func shellEnvVar(key, value string) string { + quoted := strings.ReplaceAll(value, "'", "'\"'\"'") + return fmt.Sprintf("%s='%s'", key, quoted) +} + +// applySecretEnv pushes only the Secret pairs into the agent's tmux session via +// set-environment. Values are passed as exec args (never through a shell), so +// they are not word-split and never land in the pane or in bash history. +// Failures are ignored for the same reason ensureTmuxSession ignores them: a +// missing session is handled by the launch path, not here. +func (m *Manager) applySecretEnv(agent *AgentProcess) { + if agent == nil || agent.tmuxSession == "" { + return + } + for _, p := range m.agentEnvPairs(agent) { + if !p.Secret { + continue + } + _ = m.tmuxCmd(agent, "set-environment", "-t", agent.tmuxSession, p.Key, p.Value).Run() + } +} + +func (m *Manager) buildEnvPrefix(agent *AgentProcess) string { + pairs := m.agentEnvPairs(agent) + var parts []string + for _, p := range pairs { + if p.Secret { + continue + } + parts = append(parts, shellEnvVar(p.Key, p.Value)) + } + if len(parts) == 0 { + return "" + } + return strings.Join(parts, " ") + " " +} + +// embeddedTokenRe matches git remote URLs with embedded credentials: +// https://x-access-token:TOKEN@github.com/org/repo.git +var embeddedTokenRe = regexp.MustCompile(`^https://[^@]+@(github\.com/.+)$`) + +// sanitizeGitRemotes strips embedded tokens from git remote URLs in all repos +// under the agent's work directory. Copilot CLI embeds the GitHub App token +// directly in the remote URL when it clones, bypassing both the credential +// helper (Layer 1) and env var filtering (Layer 2). +func (m *Manager) sanitizeGitRemotes(agent *AgentProcess) { + if m.agentMode(agent).CanPush() { + return + } + agentDir := m.workDir + "/" + agent.Name + _ = filepath.WalkDir(agentDir, func(path string, d os.DirEntry, err error) error { + if err != nil || d.Name() != ".git" || !d.IsDir() { + return nil + } + repoDir := filepath.Dir(path) + out, err := exec.Command("git", "-C", repoDir, "remote", "get-url", "origin").Output() + if err != nil { + return filepath.SkipDir + } + url := strings.TrimSpace(string(out)) + if match := embeddedTokenRe.FindStringSubmatch(url); match != nil { + clean := "https://" + match[1] + _ = exec.Command("git", "-C", repoDir, "remote", "set-url", "origin", clean).Run() + m.logger.Info("stripped embedded token from git remote", + "agent", agent.Name, "repo", repoDir) + } + return filepath.SkipDir + }) +} + +// agentEnvPair is an unquoted key-value environment variable. +type agentEnvPair struct { + Key string + Value string + // Secret vars are set via tmux set-environment only, never on the command line. + Secret bool +} + +// inferenceQuietCLIEnv is the set of Claude CLI switches exported to +// inference-routed sessions so the CLI stops emitting non-inference traffic +// (telemetry, error reporting, nonessential lookups) to its Anthropic host. +var inferenceQuietCLIEnv = []string{ + "DISABLE_TELEMETRY", + "DISABLE_ERROR_REPORTING", + "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", +} + +func (m *Manager) agentEnvPairs(agent *AgentProcess) []agentEnvPair { + model := agent.Config.Model + if agent.ModelOverride != "" { + model = agent.ModelOverride + } + backend := agent.Config.Backend + if agent.BackendOverride != "" { + backend = agent.BackendOverride + } + displayName := agent.Config.DisplayName + if displayName == "" { + displayName = agent.Name + } + vars := []agentEnvPair{ + {"HIVE_AGENT", agent.Name, false}, + {"HIVE_AGENT_DISPLAY_NAME", displayName, false}, + {"HIVE_BACKEND", backend, false}, + {"HIVE_MODEL", model, false}, + } + if hiveID := os.Getenv("HIVE_ID"); hiveID != "" { + vars = append(vars, agentEnvPair{"HIVE_ID", hiveID, false}) + } + vars = append(vars, agentEnvPair{"HIVE_ACMM_LEVEL", fmt.Sprintf("%d", m.project.ACMMLevel), false}) + + mode := m.agentMode(agent) + if agent.Config.Tools != nil { + if effectiveMode := agent.Config.Tools.EffectiveMode(); effectiveMode != "" { + vars = append(vars, agentEnvPair{"HIVE_AGENT_MODE", effectiveMode, false}) + } else { + vars = append(vars, agentEnvPair{"HIVE_AGENT_MODE", mode.String(), false}) + } + } else { + vars = append(vars, agentEnvPair{"HIVE_AGENT_MODE", mode.String(), false}) + } + modeFile := filepath.Join(agentStateDir, ".hive-mode-"+agent.Name) + if err := writeAgentStateFile(modeFile, []byte(mode.String())); err != nil { + m.logger.Warn("agentBootstrapEnv: mode file write failed", "file", modeFile, "error", err) + } + m.writeAgentCapsFile(agent.Name, m.agentCapabilities(agent)) + // Plain proxy URL without userinfo — Claude Code's native binary fails + // to open a socket when the URL contains username:password@ (FailedToOpenSocket). + // Agent identification uses UID-based /proc/net/tcp lookup instead of + // Proxy-Authorization headers. GIT_TERMINAL_PROMPT=0 prevents git from + // prompting for proxy credentials. + proxyURL := fmt.Sprintf("http://127.0.0.1:%d", proxyListenPort) + vars = append(vars, agentEnvPair{"HTTPS_PROXY", proxyURL, false}) + vars = append(vars, agentEnvPair{"HTTP_PROXY", proxyURL, false}) + vars = append(vars, agentEnvPair{"HIVE_PROXY_AGENT", agent.Name, false}) + vars = append(vars, agentEnvPair{"GIT_TERMINAL_PROMPT", "0", false}) + vars = append(vars, agentEnvPair{"NODE_EXTRA_CA_CERTS", proxyCACertPath, false}) + if sha := os.Getenv("HIVE_SHA"); sha != "" { + vars = append(vars, agentEnvPair{"HIVE_SHA", sha, false}) + } + if advisory := os.Getenv("HIVE_ADVISORY_ISSUE"); advisory != "" { + vars = append(vars, agentEnvPair{"HIVE_ADVISORY_ISSUE", advisory, false}) + } + // HIVE_REPO / HIVE_REPOS: the shipped policy templates instruct agents to + // run `gh issue create --repo "$HIVE_REPO"`, but nothing ever exported it + // to hosted agents (only the OSS scheduler set a hardcoded "/hive"). + // Root-caused on a live hosted hive (2026-08-20): the sec-check agent saw + // HIVE_REPO unset, fell back to the git remote of its own workdir, and + // silently scanned only the primary repo — the other project repos were + // never touched. Export the primary repo and the full project repo list so + // templates and agents can target every configured repo. + if m.project.Org != "" && len(m.project.Repos) != 0 { + // Both variables are per-agent now (#6204). A repo-scoped agent must + // not be handed a $HIVE_REPO it may not write to — every shipped + // template example passes it straight to `gh ... --repo "$HIVE_REPO"` — + // and $HIVE_REPOS is the list templates iterate as "everything in + // scope". For an unscoped agent both resolve exactly as before. + scoped := m.project.ReposFor(agent.Name) + primary := m.project.PrimaryRepoFor(agent.Name) + if primary == "" && len(scoped) > 0 { + primary = scoped[0] + } + if primary != "" { + vars = append(vars, agentEnvPair{"HIVE_REPO", config.QualifyRepo(m.project.Org, primary), false}) + } + // HIVE_REPOS is the work scope templates iterate: the agent's own repos + // (#6204) minus any the operator has paused (#6203). HIVE_REPO above is + // identity, not scope, and keeps naming the agent's primary repo even + // while it is paused — see ActiveReposFor. + active := m.project.ActiveReposFor(agent.Name) + full := make([]string, len(active)) + for i, r := range active { + full[i] = config.QualifyRepo(m.project.Org, r) + } + vars = append(vars, agentEnvPair{"HIVE_REPOS", strings.Join(full, ","), false}) + } + // GH_HOST: point the gh CLI at the configured forge host for GHE spokes. + // See ProjectContext.GHHost. The gh wrapper pairs this with + // GH_ENTERPRISE_TOKEN so the per-agent scoped token authenticates there. + if m.project.GHHost != "" { + vars = append(vars, agentEnvPair{"GH_HOST", m.project.GHHost, false}) + } + if m.routableBackend(backend) { + const inferenceTranslatePort = 18444 + vars = append(vars, agentEnvPair{"ANTHROPIC_API_KEY", "sk-hive-" + agent.Name, false}) + baseURL := fmt.Sprintf("http://127.0.0.1:%d", inferenceTranslatePort) + vars = append(vars, agentEnvPair{"ANTHROPIC_BASE_URL", baseURL, false}) + vars = append(vars, agentEnvPair{"NO_PROXY", "127.0.0.1,localhost", false}) + // Cap the CLI output-token budget at a value every commercial model + // litellm may front will accept. A prior 128000 (chosen so verbose + // OSS models would not truncate) exceeds Azure GPT-4o's 16384 + // completion-token cap, so every request 400s with + // "max_tokens is too large: 128000. This model supports at most + // 16384 completion tokens". See inferenceMaxOutputTokensDefault. + // TODO: the gateway 400 body names the model's real cap ("supports + // at most N completion tokens"); a future enhancement could parse it + // to auto-adjust per-model instead of using a universal floor. + vars = append(vars, agentEnvPair{"CLAUDE_CODE_MAX_OUTPUT_TOKENS", strconv.Itoa(inferenceMaxOutputTokensDefault), false}) + // The Claude CLI sends telemetry batches, error reports, and other + // non-inference traffic to its configured Anthropic host. Routed at + // an OpenAI-compatible gateway that traffic has nowhere useful to go + // (the proxy now answers it locally rather than forwarding it — see + // classifyInferencePath), so switch it off at the source. Only for + // inference-routed sessions: subscription/Anthropic-direct sessions + // keep Anthropic's own telemetry. + for _, v := range inferenceQuietCLIEnv { + vars = append(vars, agentEnvPair{v, "1", false}) + } + } + if m.copilotAuthToken != "" { + vars = append(vars, agentEnvPair{copilotTokenEnvVar, m.copilotAuthToken, true}) + } + // Point the GitHub MCP server at the App installation token so PRs, issue + // comments, and merges are authored by the App bot ("[bot]") — NOT by + // the Copilot login user. COPILOT_GITHUB_TOKEN above stays as the Copilot + // OAuth token because it authenticates the AI model (a separate concern from + // GitHub write identity); leaving it untouched keeps the Copilot CLI login + // working. The Copilot CLI reads GH_TOKEN / GITHUB_TOKEN for GitHub API auth + // (per its README: GH_TOKEN or GITHUB_TOKEN, in that precedence), so setting + // GITHUB_TOKEN here makes the built-in GitHub MCP server act as the App bot. + // + // Gated on the opt-in flag first (default OFF → no behavior change on any + // hive that has not explicitly enabled App-bot authorship), then on CanPush(): + // advisory agents are deliberately kept GITHUB_TOKEN-less (see the -u + // GITHUB_TOKEN strip after the env loop) so they cannot write; only push- + // capable tiers — the ones that legitimately open/merge PRs — get the App + // token. m.appAuth != nil means an App is configured. The value is the + // per-agent tier-SCOPED App token, and refreshAgentTokens re-pushes it hourly + // so it never goes stale. + if m.project.AppAuthoredPRs && m.appAuth != nil && agent.UID > 0 && m.agentMode(agent).CanPush() { + if data, err := os.ReadFile(ghpkg.AgentTokenCachePath(agent.Name)); err == nil { + if tok := strings.TrimSpace(string(data)); tok != "" { + vars = append(vars, agentEnvPair{"GITHUB_TOKEN", tok, true}) + } + } + } + // Linear write credential for ISSUES_ONLY+ agents — see linearEnvPairs. + // Nil for advisory agents and for hives with no Linear credential, so a + // GitHub-only hive sees no change. + vars = append(vars, m.linearEnvPairs(agent)...) + // CLAUDE_CODE_OAUTH_TOKEN is a LAST RESORT, not the normal delivery path. + // + // Claude Code treats this variable as a static bearer token: when it is + // set the CLI uses it verbatim, never opens ~/.claude/.credentials.json, + // and therefore never refreshes. Measured in-container (2026-09-01): with + // the variable set to a bad value and a perfectly good credentials file + // beside it, the CLI answered "401 OAuth access token is invalid" — there + // is no fallback to the file. + // + // m.claudeAuthToken is a snapshot of the SHORT-LIVED access token, taken + // once at manager construction and refreshed only by ReloadClaudeToken() + // after a dashboard login. Injecting it therefore pinned every claude + // agent to the remaining life of whatever access token happened to be on + // disk when the container started — Claude access tokens live 8h, so the + // whole fleet 401'd within a day of every restart and the only recovery + // hive offered was an operator re-login, once per agent. That is the daily + // re-authentication treadmill of #5454. + // + // It is also unnecessary since per-agent homes (#4619): every agent's + // ~/.claude is a symlink to the shared /data/home/.claude, so the CLI can + // read the credential itself — and redeem its refresh grant on start, + // which is the one thing the env var makes impossible. + // + // So inject ONLY when the agent has no credential file it can read. That + // keeps the variable doing the job it was added for (#c5648bc9: deliver a + // dashboard-obtained token to an agent that cannot see the file) and stops + // it overriding a credential that can still refresh itself. + if m.claudeAuthToken != "" && backend == "claude" && !claudeCredentialReachable(agent, backend) { + vars = append(vars, agentEnvPair{"CLAUDE_CODE_OAUTH_TOKEN", m.claudeAuthToken, true}) + } + // bob reads its key from BOBSHELL_API_KEY. Secret: true keeps the value off + // the shell command line (out of `ps`, bash history, and pane scrollback); + // it reaches the CLI via tmux set-environment only. Gated on the backend so + // no other CLI's environment carries an IBM credential it has no use for. + if backend == bobBackend { + if key := m.bobAPIKey(); key != "" { + vars = append(vars, agentEnvPair{config.BobAPIKeyEnvVar, key, true}) + } + // BOBSHELL_DEFAULT_AUTH_TYPE is what actually selects API-key auth; + // without it bob defaults to W3ID SSO and parks at the interactive key + // prompt forever. Deliberately NOT Secret: the value is the literal + // non-credential string "api-key", and secret pairs only reach a + // freshly-created pane shell via tmux set-environment, whereas + // non-secret pairs are re-applied on EVERY launch through + // buildEnvPrefix. That asymmetry is exactly what caused the sibling + // bug fixed in #2228, so the auth type must ride the always-reapplied + // path or a relaunch into an existing session loses it. + vars = append(vars, agentEnvPair{config.BobAuthTypeEnvVar, config.BobAuthTypeAPIKey, false}) + } + // BD_DIR tells the `bd` CLI where to read/write beads. Without this, + // bd falls back to cwd (/data/agents/) instead of the configured + // beads_dir (/data/beads/), causing a path mismatch with the + // dashboard and advisory digest. + if agent.Config.BeadsDir != "" { + vars = append(vars, agentEnvPair{"BD_DIR", agent.Config.BeadsDir, false}) + } + if agent.Config.CavemanMode != "" { + vars = append(vars, agentEnvPair{"HIVE_CAVEMAN_MODE", agent.Config.CavemanMode, false}) + } + // Export the RESOLVED explain mode, not the raw config value, so an agent's + // skills and helper scripts see the same answer the kick suffix acted on + // (including inheritance from the hive-wide default and the off fallback for + // an invalid value). Always exported, off included, so a script can branch on + // it without having to re-derive the precedence rules itself. + vars = append(vars, agentEnvPair{config.ExplainModeEnvVar, resolveExplainMode(agent.Config, m.explainModeDefault()), false}) + // GIT_SSL_CAINFO only — NOT SSL_CERT_FILE (that breaks Copilot API TLS) + vars = append(vars, agentEnvPair{"GIT_SSL_CAINFO", proxyCACertPath, false}) + if agent.UID > 0 { + vars = append(vars, agentEnvPair{"HIVE_AGENT_TOKEN_CACHE", ghpkg.AgentTokenCachePath(agent.Name), false}) + } + if agent.UID > 0 { + // Per-UID agents get a per-agent HOME (#4596) — AgentHome is the single + // source of truth so the auth probe and this export can never diverge. + vars = append(vars, agentEnvPair{"HOME", AgentHome(agent.Name, agent.UID, backend), false}) + + // Per-agent XDG data/state roots (#6238), beneath the per-agent HOME. + // Every backend CLI keeps its session transcripts, run locks and + // caches under $XDG_DATA_HOME / $XDG_STATE_HOME; with the legacy + // shared /data/home/.local those were one contended tree owned by + // whichever agent wrote first. Exported explicitly (not left to the + // spec default under $HOME) so the answer cannot depend on how each + // CLI resolves XDG, and only when HOME itself is per-agent — the + // HIVE_SHARED_AGENT_HOME=1 escape hatch keeps the legacy layout whole. + // XDG_CONFIG_HOME is deliberately NOT set: ~/.config stays the shared + // credential/config bridge (gh hosts.yml, goose config.yaml). See + // setupAgentXDGDirs, which pre-creates these as the agent's own dirs. + if xdgHome, ok := perAgentXDGHome(agent.Name, agent.UID, backend); ok { + vars = append(vars, agentEnvPair{"XDG_DATA_HOME", agentXDGDataHome(xdgHome), false}) + vars = append(vars, agentEnvPair{"XDG_STATE_HOME", agentXDGStateHome(xdgHome), false}) + } + + // Under the per-agent-UID layout the global npm prefix is owned by the + // image's build user, so the Claude Code CLI's self-updater fails on + // every launch with "✘ Auto-update failed: no write permission to npm + // prefix" — a red line in every agent pane for an update the agent must + // not perform anyway (the CLI version is managed by the image, not by + // an in-pod npm write). Disabling the updater removes the failure at its + // source; a per-agent npm prefix would instead let an agent drift off + // the pinned image version. + vars = append(vars, agentEnvPair{"DISABLE_AUTOUPDATER", "1", false}) + } + + // Codex CLI 0.144.1's in-process app-server performs OWNER-gated operations + // on files under CODEX_HOME (helper-binary "PATH alias" symlinks under + // tmp/arg0, sqlite state). The shared /data/home/.codex is owned by dev + // (the entrypoint chowns it group-writable + setgid), which claude/copilot + // tolerate but Codex does not — every non-owner agent UID fails with + // "failed to start embedded app server: Operation not permitted (os error 1)". + // The manager launches the codex binary DIRECTLY (not via agent-launch.sh), + // so CODEX_HOME must be set here. Give each agent its own dir; codex will + // NOT create it (it errors "CODEX_HOME ... does not exist"), so it is + // pre-created AS the agent below in setupCodexHome. + if backend == codexBackend { + vars = append(vars, agentEnvPair{"CODEX_HOME", codexHomePath(agent.Name), false}) + } + + for _, conn := range agent.Config.Connections { + if conn.Type != "api" { + continue + } + envName := conn.EnvName + if envName == "" { + envName = "HIVE_CONN_" + strings.ToUpper(strings.ReplaceAll(conn.Name, "-", "_")) + "_URL" + } + vars = append(vars, agentEnvPair{envName, conn.URI, false}) + if conn.Auth != nil && conn.Auth.Type == "env" && conn.Auth.EnvVar != "" { + if tokenVal := os.Getenv(conn.Auth.EnvVar); tokenVal != "" { + vars = append(vars, agentEnvPair{conn.Auth.EnvVar, tokenVal, true}) + } + } + } + + return vars +} diff --git a/src/pkg/agent/manager_modes.go b/src/pkg/agent/manager_modes.go new file mode 100644 index 0000000000..029f18f7a4 --- /dev/null +++ b/src/pkg/agent/manager_modes.go @@ -0,0 +1,354 @@ +package agent + +import ( + "fmt" + "path/filepath" + "strings" +) + +// ClearModeOverrides clears Config.Mode for the NAMED agents so that +// DefaultAgentMode determines their mode from the ACMM level. Call it before +// SyncModeFiles when applying a pack, because a pack agent's Config.Mode may +// have been set by a previous level's pack and would otherwise override the +// new level's expected default. +// +// Scoped to a name list — the pack's roster — on purpose (#7503). The previous +// ClearAllModeOverrides wiped every agent in the process table, including +// agents no pack manages. Their Mode was never pack-seeded, so there is no +// stale pack value to clear: what got cleared was the OPERATOR's setting. On +// the projectbluefin spoke `reviewer` was `mode: ADVISORY` in every config +// layer and ran as ISSUES_AND_PRS, the L5 default — two rungs more authority +// than anything on disk granted it — because the startup pack apply cleared it +// here and SyncModeFiles then wrote the default. Names not in the process +// table are ignored. +func (m *Manager) ClearModeOverrides(names []string) { + m.mu.Lock() + defer m.mu.Unlock() + for _, name := range names { + if agent, ok := m.agents[name]; ok { + agent.Config.Mode = "" + } + } +} + +// SyncModeFiles rewrites /tmp/.hive-mode-* for all running agents to reflect the given ACMM level. +func (m *Manager) SyncModeFiles(level int) { + m.mu.RLock() + defer m.mu.RUnlock() + for name, agent := range m.agents { + if agent.Paused { + continue + } + mode := DefaultAgentMode(name, level) + // converseConfigured is logged on both branches below so one grep by + // agent name shows whether the capability came from config or fell + // back to the default alongside the mode decision (#7503). + converseConfigured := agent.Config.Converse != nil + if modeStr := agent.Config.Mode; modeStr != "" { + if parsed, ok := ParseAgentMode(modeStr); ok { + m.logger.Info("SyncModeFiles: Config.Mode override", + "agent", name, "level", level, + "default", DefaultAgentMode(name, level).String(), + "override", modeStr, + "converse_configured", converseConfigured) + mode = parsed + } + } else { + // Log the fallback too, not only the override (#7503). Before this, + // an agent whose configured mode had been dropped on the way to the + // process table was indistinguishable from one that never set a + // mode: the only signal was reading /tmp/.hive-mode- and + // comparing by hand. One line saying "config said nothing, using + // the level default" turns that into a grep. `overlay_file` names + // the per-agent file when the entry came from one, so an operator + // can check what it says against what is being used. + m.logger.Info("SyncModeFiles: no Config.Mode, using level default", + "agent", name, "level", level, + "default", mode.String(), + "converse_configured", converseConfigured, + "overlay_file", agent.Config.SourceFile()) + } + modeFile := filepath.Join(agentStateDir, ".hive-mode-"+name) + if err := writeAgentStateFile(modeFile, []byte(mode.String())); err != nil { + m.logger.Warn("SyncModeFiles: write failed", "file", modeFile, "error", err) + } + // The capability file rides the same sync (#4492). It is level-independent + // today, but writing it here is what makes a `converse` change take effect + // on the next reconcile instead of only at the next agent launch. + caps := DefaultCapabilities(mode, level) + if agent.Config.Converse != nil { + caps.Converse = *agent.Config.Converse + } + m.writeAgentCapsFile(name, caps) + } +} + +// agentCapabilities returns the ORTHOGONAL capabilities for a given agent +// (#4492). Unlike agentMode there is no per-level default table: `converse` is +// opt-in everywhere, so an agent whose config says nothing gets the zero value +// and behaves exactly as it did before capabilities existed. +func (m *Manager) agentCapabilities(agent *AgentProcess) AgentCapabilities { + caps := DefaultCapabilities(m.agentMode(agent), m.project.ACMMLevel) + if agent.Config.Converse != nil { + caps.Converse = *agent.Config.Converse + } + return caps +} + +// writeAgentCapsFile persists the capability set the proxy reads on the request +// path. It is written for EVERY agent, including those with no capabilities, so +// a cleared `converse` actually revokes: leaving a stale file behind would keep +// granting the capability after the operator turned it off. +func (m *Manager) writeAgentCapsFile(name string, caps AgentCapabilities) { + capsFile := filepath.Join(agentStateDir, ".hive-caps-"+name) + if err := writeAgentStateFile(capsFile, []byte(caps.String())); err != nil { + m.logger.Warn("caps file write failed", "file", capsFile, "error", err) + } +} + +// agentMode returns the GitHub interaction mode for a given agent at the current ACMM level. +// If the agent has an explicit Mode in its config (hive.yaml or pack YAML), that takes precedence. +// Otherwise, the default table by ACMM level is used. +func (m *Manager) agentMode(agent *AgentProcess) AgentMode { + if modeStr := agent.Config.Mode; modeStr != "" { + if parsed, ok := ParseAgentMode(modeStr); ok { + return parsed + } + } + return DefaultAgentMode(agent.Name, m.project.ACMMLevel) +} + +// DefaultAgentMode returns the default mode for a given agent name and ACMM level, +// ignoring any hive.yaml override. Used by the dashboard to show "(default)" indicators. +func DefaultAgentMode(agentName string, level int) AgentMode { + if agentName == "supervisor" { + return ModeAdvisory + } + switch level { + case 1: + return ModeAdvisory + case 2: + return ModeAdvisory + case 3: + if agentName == "quality" { + return ModeIssuesAndPRs + } + return ModeAdvisory + case 4: + switch agentName { + case "quality", "sec-check", "ci-maintainer": + return ModeIssuesAndPRs + case "scanner", "guide": + return ModeIssuesOnly + default: + return ModeAdvisory + } + case 5: + return ModeIssuesAndPRs + case 6: + if agentName == "scanner" { + return ModeIssuesPRsMerge + } + return ModeIssuesAndPRs + default: + return ModeAdvisory + } +} + +// AuthorizePROpen enforces the policy for the hive-opens-PR watcher: an agent +// may open a PR (by dropping a request file) only if BOTH hold: +// +// 1. Forge-resistance — the request file's owning UID (fileUID) maps to the +// agent it claims to be (via the uid-map). One agent cannot open a PR "as" +// another, and a non-agent process (unknown UID) is refused. When per-agent +// UIDs are not in play (fileUID <= 0, e.g. shared-dev-UID mode with no map), +// ownership is unverifiable, so we fall back to the ACMM check alone rather +// than hard-failing — the same posture the credential helper takes. +// 2. ACMM write-gate — the agent must be push-capable at the hive's current +// ACMM level, i.e. exactly the CanPush() check that governs `gh pr create`. +// +// Returns nil to authorize, or an error describing the denial. This mirrors the +// direct PR path's policy so the request-file route grants no extra privilege. +func (m *Manager) AuthorizePROpen(agentName string, fileUID int) error { + if strings.TrimSpace(agentName) == "" { + return fmt.Errorf("no agent named in the request") + } + // Forge check: when we have a UID map and a real owning UID, the file owner + // must BE this agent. + if m.uidMap != nil && fileUID > 0 { + owner := m.uidMap.LookupByUID(fileUID) + if owner == "" { + return fmt.Errorf("request file owned by unknown uid %d (not a registered agent)", fileUID) + } + if owner != agentName { + return fmt.Errorf("request claims agent %q but file is owned by agent %q (uid %d)", agentName, owner, fileUID) + } + } + // ACMM write-gate: resolve the agent and check CanPush. + m.mu.RLock() + agent := m.agents[agentName] + m.mu.RUnlock() + if agent == nil { + return fmt.Errorf("unknown agent %q", agentName) + } + if !m.agentMode(agent).CanPush() { + return fmt.Errorf("agent %q is not push-capable at this ACMM level (mode %s) — advisory agents may not open PRs", + agentName, m.agentMode(agent).String()) + } + return nil +} + +// AuthorizeIssueOpen enforces the policy for the issue-request watcher, +// mirroring AuthorizePROpen with the mode gates that govern the direct gh +// paths: "issue" requests need CanCreateIssues() (mode >= ISSUES_ONLY); +// "comment" and "claim" requests need the same (commenting and claiming an +// issue are both issue-writes under the same tier). The same UID +// forge-resistance applies: the request file's owner must BE the claimed +// agent. A nil manager or unknown agent is denied. +func (m *Manager) AuthorizeIssueOpen(agentName string, fileUID int, kind string) error { + if strings.TrimSpace(agentName) == "" { + return fmt.Errorf("no agent named in the request") + } + if m.uidMap != nil && fileUID > 0 { + owner := m.uidMap.LookupByUID(fileUID) + if owner == "" { + return fmt.Errorf("request file owned by unknown uid %d (not a registered agent)", fileUID) + } + if owner != agentName { + return fmt.Errorf("request claims agent %q but file is owned by agent %q (uid %d)", agentName, owner, fileUID) + } + } + m.mu.RLock() + agent := m.agents[agentName] + m.mu.RUnlock() + if agent == nil { + return fmt.Errorf("unknown agent %q", agentName) + } + if !m.agentMode(agent).CanCreateIssues() { + return fmt.Errorf("agent %q may not create issues or comments at this ACMM level (mode %s)", + agentName, m.agentMode(agent).String()) + } + return nil +} + +// AuthorizeMerge enforces the policy for the hive-merges-PR watcher, mirroring +// AuthorizePROpen but with the stricter CanMerge() gate: the request's agent +// must own the request file (forge-resistance) AND be merge-capable at the +// hive's current ACMM level (ModeIssuesPRsMerge). This keeps the file-based +// merge relay under the exact same authority as a direct merge would require — +// an issues/PRs agent that can open PRs still cannot merge them unless its mode +// grants merge. A nil manager or unknown agent is denied. +func (m *Manager) AuthorizeMerge(agentName string, fileUID int) error { + if strings.TrimSpace(agentName) == "" { + return fmt.Errorf("no agent named in the request") + } + // Forge check: when we have a UID map and a real owning UID, the file owner + // must BE this agent. + if m.uidMap != nil && fileUID > 0 { + owner := m.uidMap.LookupByUID(fileUID) + if owner == "" { + return fmt.Errorf("request file owned by unknown uid %d (not a registered agent)", fileUID) + } + if owner != agentName { + return fmt.Errorf("request claims agent %q but file is owned by agent %q (uid %d)", agentName, owner, fileUID) + } + } + // ACMM merge-gate: resolve the agent and check CanMerge. + m.mu.RLock() + agent := m.agents[agentName] + m.mu.RUnlock() + if agent == nil { + return fmt.Errorf("unknown agent %q", agentName) + } + if !m.agentMode(agent).CanMerge() { + return fmt.Errorf("agent %q is not merge-capable at this ACMM level (mode %s) — only ISSUES_PRS_MERGE agents may merge PRs", + agentName, m.agentMode(agent).String()) + } + return nil +} + +// AgentCapabilities reports whether the named agent is ABLE — at the hive's +// current ACMM level and the agent's effective mode — to create issues, open +// PRs, and merge PRs. These are the EXACT gates AuthorizePROpen (CanPush) and +// AuthorizeMerge (CanMerge) enforce, so a hub capability badge derived from +// these can never claim a capability the merge/PR relay would actually refuse. +// ok=false when the agent is unknown to the manager (the caller then reports +// "unknown", not a false negative). Read-only under RLock. +func (m *Manager) AgentCapabilities(agentName string) (canOpenIssue, canOpenPR, canMerge, ok bool) { + m.mu.RLock() + agent, exists := m.agents[agentName] + m.mu.RUnlock() + if !exists || agent == nil { + return false, false, false, false + } + mode := m.agentMode(agent) + return mode.CanCreateIssues(), mode.CanPush(), mode.CanMerge(), true +} + +// EffectiveBackend reports the named agent's effective backend, honoring any +// runtime BackendOverride (see effectiveBackend). ok=false when the agent is +// unknown. Read-only under RLock — a small exported wrapper so callers outside +// the package (the heartbeat builder) need not reach into unexported state. +func (m *Manager) EffectiveBackend(agentName string) (backend string, ok bool) { + m.mu.RLock() + agent, exists := m.agents[agentName] + m.mu.RUnlock() + if !exists || agent == nil { + return "", false + } + return effectiveBackend(agent), true +} + +// InvocationMetadata reports the effective backend, model, and reasoning effort +// the hive invokes for the named agent, accounting for runtime overrides — the +// launch-time truth the invocation-attribution trail records (see pkg/github/attribution +// .go). ok=false when the agent is unknown to the manager (the caller then +// falls back to static config). Read-only under RLock; called from the +// PR-request watcher goroutine, never from the launch path. +func (m *Manager) InvocationMetadata(agentName string) (backend, model, effort string, ok bool) { + m.mu.RLock() + defer m.mu.RUnlock() + agent, exists := m.agents[agentName] + if !exists { + return "", "", "", false + } + backend = effectiveBackend(agent) + model = agent.Config.Model + if agent.ModelOverride != "" { + model = agent.ModelOverride + } + return backend, model, ResolveReasoningEffort(backend, model, agent.Config.ReasoningEffort), true +} + +// ResolveReasoningEffort reports the reasoning effort the hive actually launches +// a given backend/model pair with, given the agent's configured reasoning_effort. +// Exported because the attribution trail is +// resolved in TWO places — Manager.InvocationMetadata above for a running agent, +// and cmd/hive's fallback that reads straight from config when the Manager does +// not know the agent — and both must give the same answer. +// +// Before this existed the fallback carried its own hardcoded "low", so changing +// agyDefaultEffort here would have left cmd/hive silently stamping PRs with an +// effort agy was no longer being launched with. An attribution trail that +// misreports is worse than one that says nothing. +// +// The rules mirror the launch path exactly: +// - agy REQUIRES --effort whenever --model is given, so with a model it runs +// at agyLaunchEffort(configured) and with no model at no effort at all. +// - codex is launched with `-c model_reasoning_effort` only when an effort +// is configured; unset means codex's own default, which the hive does not +// resolve, so the honest answer is the configured value verbatim. +// - every other backend takes its effort from its own config, which the +// hive does not resolve here, so the honest answer is "". +func ResolveReasoningEffort(backend, model, configured string) string { + switch backend { + case "agy": + if model != "" { + return agyLaunchEffort(configured) + } + return "" + case codexBackend: + return configured + } + return "" +} diff --git a/src/pkg/agent/manager_routing.go b/src/pkg/agent/manager_routing.go new file mode 100644 index 0000000000..9ae3cc46db --- /dev/null +++ b/src/pkg/agent/manager_routing.go @@ -0,0 +1,456 @@ +package agent + +import ( + "fmt" + "os/exec" + "strings" + + "github.com/hivecommons/hive/pkg/config" +) + +// effectiveBackend returns the agent's backend accounting for any override. +func effectiveBackend(agent *AgentProcess) string { + if agent.BackendOverride != "" { + return agent.BackendOverride + } + return agent.Config.Backend +} + +// IsInferenceBackend returns true if the backend is a self-hosted inference +// backend (vllm, llm-d, litellm) rather than a CLI tool. Delegates to the +// canonical list in the config package (shared with the proxy package, +// which cannot be imported from here without a cycle). +func IsInferenceBackend(backend string) bool { + return config.IsInferenceBackend(backend) +} + +// SetInferenceCallbacks registers callbacks that the manager uses to +// configure/clear inference routing on the proxy when launching agents. +func (m *Manager) SetInferenceCallbacks( + setRoute func(agentName, backend, model string), + clearRoute func(agentName string), +) { + m.mu.Lock() + defer m.mu.Unlock() + m.inferenceRouteCallback = setRoute + m.clearInferenceRouteCallback = clearRoute +} + +// SetGatewayBackendChecker injects a predicate that reports whether a backend +// string names a configured model gateway. This makes an agent whose backend is +// a gateway name inference-routable, so its route is resolved via the inference +// callback exactly like the built-in litellm/vllm/llm-d backends. +func (m *Manager) SetGatewayBackendChecker(fn func(backend string) bool) { + // Atomic store — no m.mu — so routableBackend can read it lock-free from the + // lock-holding launch path without deadlocking (see isGatewayBackend docs). + m.isGatewayBackend.Store(&fn) +} + +// routableBackend reports whether a backend should be routed through the +// inference proxy: either a built-in inference backend, or a configured gateway +// name. Safe to call while holding m.mu (isGatewayBackend is read atomically). +func (m *Manager) routableBackend(backend string) bool { + if IsInferenceBackend(backend) { + return true + } + // Lock-free atomic read: this is invoked from the launch path while m.mu is + // already held, so it MUST NOT take m.mu (non-reentrant RWMutex → deadlock). + fnp := m.isGatewayBackend.Load() + return fnp != nil && *fnp != nil && (*fnp)(backend) +} + +// validateBackendName reports whether backend is one the launcher can actually +// dispatch: an agentic CLI, a model-gateway backend, or a configured gateway +// name. An empty backend is valid (it means "the hive default"). +// +// This is the manager-side half of the accept-then-fail fix. It dispatches on +// the SAME canonical lists as config.ValidateBackend and backendBinary, so a +// backend accepted by any write path is one the launch path can start. +// Safe to call while holding m.mu (routableBackend reads atomically). +func (m *Manager) validateBackendName(backend string) error { + if backend == "" || config.IsCLIBackend(backend) || m.routableBackend(backend) { + return nil + } + return fmt.Errorf("unsupported backend %q (supported: %s; or the name of a configured model gateway)", + backend, strings.Join(config.SupportedBackends(), ", ")) +} + +// effectiveBackend is the backend this agent will actually launch with: the +// per-agent override when set, otherwise its configured backend. +func (a *AgentProcess) effectiveBackend() string { + if a.BackendOverride != "" { + return a.BackendOverride + } + return a.Config.Backend +} + +// effectiveModel is the model this agent will actually launch with: the +// per-agent override when set, otherwise its configured model. Returns the +// raw (un-normalized) name — the audit log should show what was ASKED for, +// since a bad model name is exactly the kind of misconfiguration being +// audited. +func (a *AgentProcess) effectiveModel() string { + if a.ModelOverride != "" { + return a.ModelOverride + } + return a.Config.Model +} + +// backendBinaryAliases names the backends whose binary is NOT simply the +// backend name. Only genuine aliases belong here: every other CLI backend is +// derived from config.CLIBackends by identity, and every routable model-gateway +// backend is resolved by Manager.backendBinaryName. Keeping this map to aliases +// only is what makes the accept-then-fail class of bug structurally impossible. +var backendBinaryAliases = map[string]string{ + // pi was previously aliased to "goose", which made every pi-configured + // agent exec the goose CLI instead of pi (the backend launch command + // switch now has a real pi case). pi is a first-class CLI backend + // (config.CLIBackends includes "pi"), so identity mapping applies. +} + +// backendBinaryName maps a config-independent agent backend to the NAME of the +// CLI binary that is exec'd for it, without touching the filesystem. Split out +// from backendBinary so the "every supported backend resolves" invariant can be +// tested without requiring each CLI to be installed on the test machine. +// +// Both canonical lists are derived rather than written out here: +// +// - config.CLIBackends (claude, copilot, goose, codex, pi, bob, aider, gemini) +// each launch a binary of the same name, except for the aliases above. +// - config.InferenceBackends (vllm, llm-d, litellm, watsonx) all launch the +// SAME claude CLI, pointed at hive's local OpenAI-compatible translator via +// ANTHROPIC_BASE_URL — the backend name selects the upstream route, not the +// binary. +// +// Deriving both means a backend added to either list can never again be +// accepted by config.ValidateBackend and then rejected hours later at kick time +// with "unknown backend". Previously only InferenceBackends was derived, so +// codex and aider were valid config values that failed at launch. +func backendBinaryName(backend string) (string, error) { + binaries := make(map[string]string, len(config.CLIBackends)+len(config.InferenceBackends)) + for _, b := range config.CLIBackends { + binaries[b] = b + } + for _, b := range config.InferenceBackends { + binaries[b] = "claude" + } + for backend, binary := range backendBinaryAliases { + binaries[backend] = binary + } + + binary, ok := binaries[backend] + if !ok { + return "", fmt.Errorf("unknown backend: %s", backend) + } + return binary, nil +} + +// backendBinaryName resolves both config-independent backends and live +// configured gateway names. A gateway name validates via Manager.routableBackend, +// so the launch path must use the same predicate and route it through claude. +func (m *Manager) backendBinaryName(backend string) (string, error) { + if binary, err := backendBinaryName(backend); err == nil { + return binary, nil + } + if m != nil && m.routableBackend(backend) { + return "claude", nil + } + return "", fmt.Errorf("unknown backend: %s", backend) +} + +// backendBinary resolves an agent backend to the absolute path of the CLI +// binary that is actually exec'd for it. +func backendBinary(backend string) (string, error) { + binary, err := backendBinaryName(backend) + if err != nil { + return "", err + } + + path, err := exec.LookPath(binary) + if err != nil { + return "", fmt.Errorf("backend %s not found in PATH: %w", backend, err) + } + + return path, nil +} + +func (m *Manager) backendBinary(backend string) (string, error) { + binary, err := m.backendBinaryName(backend) + if err != nil { + return "", err + } + + path, err := exec.LookPath(binary) + if err != nil { + return "", fmt.Errorf("backend %s binary %s not found in PATH: %w", backend, binary, err) + } + + return path, nil +} + +func (m *Manager) backendLaunchFailureMessage(backend string, err error) string { + binary, nameErr := m.backendBinaryName(backend) + if nameErr != nil { + return fmt.Sprintf( + "backend %s did not launch: %v. This backend is not a supported CLI, built-in inference backend, or configured model gateway; switch this agent to a supported backend or configure a matching model gateway.", + backend, err) + } + return fmt.Sprintf( + "backend %s did not launch: %v. The %s CLI required for this backend is not installed in this hive image — upgrade the hive image or switch this agent to a different backend.", + backend, err, binary) +} + +// codexBackend is the backend name for the OpenAI Codex CLI. +const codexBackend = "codex" + +// bobBackend is the backend name for the IBM bobshell ("bob") CLI. +const bobBackend = "bob" + +// normalizeModelName converts YAML-friendly model names to the format each +// CLI backend expects. Claude CLI uses hyphens (claude-opus-4-7), while +// gemini/goose/agy-style backends use dots (claude-opus-4.7). +// +// copilot does NOT take the blind trailing-digits dot-rewrite below: the +// Copilot CLI's --model nomenclature mixes separators per model family +// (claude-fable-5 is DASHED, claude-opus-4.6 is DOTTED), so the rewrite +// corrupted every dashed-family id — verified live, copilot CLI v1.0.78 +// rejected the rewritten `claude-fable.5` ("is not available") and fell back +// to a different model (#4262). copilot instead uses the alias-based +// CanonicalizeCopilotModel (copilot_models.go), which normalizes separator +// drift against the known CLI-accepted list in both directions and passes +// unknown ids through verbatim. Applied here — at launch time — so an +// already-stored bad id self-corrects on existing spokes without operator +// action. +// +// Self-hosted inference backends (vllm, llm-d, litellm) and configured gateway +// names are the outbound gateway model id verbatim — the string must match an +// entitled model on the gateway EXACTLY (prefixes like "Azure/", dots vs +// hyphens, case). Rewriting it (e.g. "Azure/gpt-4" -> "Azure/gpt.4", +// "gpt-4o-2024-08-06" -> "gpt-4o-2024-08.06") produces a model the team is not +// entitled to and the gateway 403s ("team not allowed to access model") even +// for entitled models. So never normalize inference model names — pass them +// through untouched. +// +// bob is likewise excluded. bobLaunchCmd passes no --model at all (bob +// auto-selects), so this is defense-in-depth rather than the fix: the value is +// still computed and logged on the bob launch path, and the dot-rewrite is +// what turned a configured `claude-sonnet-4-6` into the unknown +// `claude-sonnet-4.6` that made bob die with "Cannot read properties of +// undefined (reading 'maxTokens')". Leaving it unrewritten keeps logs honest +// about what was configured and stops the corrupted id from being handed to a +// future bob consumer. +func normalizeModelNameForBackend(model, backend string, inferenceRoutable bool) string { + if backend == "claude" || backend == bobBackend || inferenceRoutable { + return model + } + if backend == "copilot" { + return CanonicalizeCopilotModel(model) + } + idx := strings.LastIndex(model, "-") + if idx < 0 || idx == len(model)-1 { + return model + } + suffix := model[idx+1:] + allDigits := true + for _, c := range suffix { + if c < '0' || c > '9' { + allDigits = false + break + } + } + if allDigits { + return model[:idx] + "." + suffix + } + return model +} + +func (m *Manager) PinCLI(name, version string) error { + m.mu.Lock() + defer m.mu.Unlock() + + agent, ok := m.agents[name] + if !ok { + return fmt.Errorf("agent %s not found", name) + } + + agent.PinnedCLI = version + m.logger.Info("agent CLI pinned", "name", name, "version", version) + return nil +} + +func (m *Manager) UnpinCLI(name string) error { + m.mu.Lock() + defer m.mu.Unlock() + + agent, ok := m.agents[name] + if !ok { + return fmt.Errorf("agent %s not found", name) + } + + agent.PinnedCLI = "" + m.logger.Info("agent CLI unpinned", "name", name) + return nil +} + +func (m *Manager) PinModel(name, model string) error { + m.mu.Lock() + defer m.mu.Unlock() + + agent, ok := m.agents[name] + if !ok { + return fmt.Errorf("agent %s not found", name) + } + + prevModel := agent.effectiveModel() + agent.PinnedModel = model + agent.ModelOverride = model + m.logger.Info("agent model pinned", "name", name, "model", model) + if prevModel != model { + m.audit(AuditAgentModelSet, name, auditFields( + "outcome", "success", + "backend", agent.effectiveBackend(), + "model", model, + "previous_model", prevModel, + "trigger", "pin", + )) + } + return nil +} + +func (m *Manager) UnpinModel(name string) error { + m.mu.Lock() + defer m.mu.Unlock() + + agent, ok := m.agents[name] + if !ok { + return fmt.Errorf("agent %s not found", name) + } + + agent.PinnedModel = "" + m.logger.Info("agent model unpinned", "name", name) + return nil +} + +func (m *Manager) SetModelOverride(name, model string) error { + m.mu.Lock() + defer m.mu.Unlock() + + agent, ok := m.agents[name] + if !ok { + return fmt.Errorf("agent %s not found", name) + } + + // Store the CLI-accepted spelling for copilot so the persisted selection, + // the dropdown preselect, and auto-heal all agree on one canonical id + // (separator drift like claude-fable.5 vs claude-fable-5, #4262). Launch + // re-applies the same canonicalization, so even ids stored before this + // existed self-correct there. + if agent.effectiveBackend() == "copilot" { + model = CanonicalizeCopilotModel(model) + } + + // A pin blocks the governor's auto-selection, never a user's explicit + // switch: retarget the pin to the new model so the pin state is + // unchanged (still pinned) while the change takes effect. + if agent.PinnedModel != "" { + agent.PinnedModel = model + m.logger.Info("agent model pin retargeted by user switch", "name", name, "model", model) + } + + prevModel := agent.effectiveModel() + agent.ModelOverride = model + m.logger.Info("agent model override set", "name", name, "model", model) + // State CHANGES only — the governor re-asserts the current model on every + // evaluation cycle, so auditing unchanged writes would flood the ring. + if prevModel != model { + m.audit(AuditAgentModelSet, name, auditFields( + "outcome", "success", + "backend", agent.effectiveBackend(), + "model", model, + "previous_model", prevModel, + )) + } + + effectiveBackend := agent.Config.Backend + if agent.BackendOverride != "" { + effectiveBackend = agent.BackendOverride + } + if m.routableBackend(effectiveBackend) && m.inferenceRouteCallback != nil { + m.inferenceRouteCallback(name, effectiveBackend, model) + } + return nil +} + +func (m *Manager) SetBackendOverride(name, backend string) error { + m.mu.Lock() + defer m.mu.Unlock() + + agent, ok := m.agents[name] + if !ok { + return fmt.Errorf("agent %s not found", name) + } + + // Refuse a backend the launcher cannot dispatch, at SET time. Previously + // any string was accepted here and the agent was then restarted into it, + // failing only at launch with "unknown backend: " — the agent stops + // working and the operator gets no signal at the moment of the change. + // routableBackend covers configured gateway names (resolved live), so a + // gateway-named backend still passes. + if err := m.validateBackendName(backend); err != nil { + return err + } + + // Captured after validation so a rejected switch records no audit event: + // the override is only mutated below, once the backend is known routable. + prevBackend := agent.effectiveBackend() + agent.BackendOverride = backend + m.logger.Info("agent backend override set", "name", name, "backend", backend) + // Record only a real transition: /switch/{backend} is also re-applied on + // config reload with the value already in effect, and auditing those + // no-ops would bury the actual operator changes. + if prevBackend != backend { + m.audit(AuditAgentBackendSet, name, auditFields( + "outcome", "success", + "backend", backend, + "model", agent.effectiveModel(), + "previous_backend", prevBackend, + )) + } + + if m.routableBackend(backend) && m.inferenceRouteCallback != nil { + model := agent.ModelOverride + if model == "" { + model = agent.Config.Model + } + m.inferenceRouteCallback(name, backend, model) + } else if !IsInferenceBackend(backend) && m.clearInferenceRouteCallback != nil { + m.clearInferenceRouteCallback(name) + } + return nil +} + +// RefreshInferenceRoutes re-fires the inference route callback for every +// agent whose effective backend matches, so endpoint or credential changes +// (e.g. a governor LiteLLM config save) take effect on live agents without +// a restart. +func (m *Manager) RefreshInferenceRoutes(backend string) { + m.mu.Lock() + defer m.mu.Unlock() + if m.inferenceRouteCallback == nil || !m.routableBackend(backend) { + return + } + for name, agent := range m.agents { + effective := agent.Config.Backend + if agent.BackendOverride != "" { + effective = agent.BackendOverride + } + if effective != backend { + continue + } + model := agent.ModelOverride + if model == "" { + model = agent.Config.Model + } + m.inferenceRouteCallback(name, backend, model) + } +} diff --git a/src/pkg/agent/manager_thrash.go b/src/pkg/agent/manager_thrash.go new file mode 100644 index 0000000000..e07bcacc58 --- /dev/null +++ b/src/pkg/agent/manager_thrash.go @@ -0,0 +1,99 @@ +package agent + +import ( + "fmt" + "strings" + "time" +) + +// Blocked-action thrash breaker: an agent that keeps hammering a policy wall +// (e.g. a push with no per-agent token, blocked every ~3s by +// git-credential-hive, or a proxy hard-deny) burns model tokens indefinitely +// with zero possible output — observed live 2026-08-04 on a hosted L2 hive +// whose guide agent retried a blocked push every 3 seconds. (Since #4289, +// ADVISORY-mode pushes are no longer blocked by the credential helper — the +// read-only token is served and GitHub rejects the push with 403 — but the +// helper still emits "git push blocked:" for unknown-UID and missing-token +// failures, which this breaker continues to catch.) The hub, not the model, +// breaks the loop: thrashThreshold blocked-action lines within thrashWindow +// pauses the session (visible, reversible, stops governor kicks) with the +// reason spelled out. +const ( + thrashWindow = 60 * time.Second + thrashThreshold = 5 + thrashCooldown = 10 * time.Minute +) + +// blockedActionMarkers are the policy-wall stderr lines that can never +// succeed by retrying. Keep in sync with bin/git-credential-hive.sh and the +// proxy's hard-deny responses. +var blockedActionMarkers = []string{ + "git push blocked:", + "blocked by hive policy", +} + +type thrashState struct { + times []time.Time + lastTrip time.Time +} + +// checkBlockedThrash records a blocked-action output line for the agent and, +// past the threshold, pauses the agent asynchronously (never inline: this is +// called from the output-capture goroutine and Pause takes m.mu). +func (m *Manager) checkBlockedThrash(agent, line string) { + matched := false + for _, marker := range blockedActionMarkers { + if strings.Contains(line, marker) { + matched = true + break + } + } + if !matched { + return + } + now := time.Now() + m.thrashMu.Lock() + if m.thrash == nil { + m.thrash = map[string]*thrashState{} + } + st := m.thrash[agent] + if st == nil { + st = &thrashState{} + m.thrash[agent] = st + } + trip := recordBlockedAndCheck(st, now, thrashWindow, thrashThreshold, thrashCooldown) + m.thrashMu.Unlock() + if !trip { + return + } + reason := fmt.Sprintf("blocked-action loop: %d+ policy-blocked attempts in %s — the block is terminal in this mode; paused to stop token burn", thrashThreshold, thrashWindow) + m.logger.Warn("thrash breaker tripped", "agent", agent, "line", truncateStr(line, 160)) + go func() { + if err := m.Pause(agent, "thrash-breaker", reason); err != nil { + m.logger.Warn("thrash breaker pause failed", "agent", agent, "error", err) + } + }() +} + +// recordBlockedAndCheck is the pure sliding-window decision: append now, drop +// entries older than window, and report whether the threshold is crossed +// outside the cooldown. +func recordBlockedAndCheck(st *thrashState, now time.Time, window time.Duration, threshold int, cooldown time.Duration) bool { + st.times = append(st.times, now) + cutoff := now.Add(-window) + kept := st.times[:0] + for _, t := range st.times { + if !t.Before(cutoff) { + kept = append(kept, t) + } + } + st.times = kept + if len(st.times) < threshold { + return false + } + if !st.lastTrip.IsZero() && now.Sub(st.lastTrip) < cooldown { + return false + } + st.lastTrip = now + return true +} diff --git a/src/pkg/agent/routing_coverage_test.go b/src/pkg/agent/routing_coverage_test.go index 17f95dc11c..6caa835386 100644 --- a/src/pkg/agent/routing_coverage_test.go +++ b/src/pkg/agent/routing_coverage_test.go @@ -295,6 +295,30 @@ func TestConnectionMCPFlags_NonClaude(t *testing.T) { // normalizeModelName // --------------------------------------------------------------------------- +func TestNormalizeModelName(t *testing.T) { + cases := []struct{ model, backend, want string }{ + {"opus", "claude", "opus"}, // claude passes through + {"deepseek-14", "litellm", "deepseek-14"}, // inference passes through + // copilot no longer takes the blind digit-suffix dot-rewrite (#4262): + // ids are canonicalized against the CLI-accepted list and UNKNOWN ids + // pass through verbatim — the rewrite is what corrupted the known + // claude-fable-5 into the CLI-rejected claude-fable.5. + {"gpt-4o-2024", "copilot", "gpt-4o-2024"}, // unknown id passthrough + {"claude-fable-5", "copilot", "claude-fable-5"}, // dashed family kept dashed + {"claude-fable.5", "copilot", "claude-fable-5"}, // stored bad id self-corrects + {"claude-opus-4-6", "copilot", "claude-opus-4.6"}, // known dotted family still mapped + {"gpt-4o-2024", "gemini", "gpt-4o.2024"}, // digit suffix -> dot (non-copilot) + {"gpt-4o-preview", "copilot", "gpt-4o-preview"}, // non-digit suffix unchanged + {"nohyphen", "copilot", "nohyphen"}, // no hyphen + {"trailing-", "copilot", "trailing-"}, // trailing hyphen + } + for _, c := range cases { + if got := normalizeModelNameForBackend(c.model, c.backend, IsInferenceBackend(c.backend)); got != c.want { + t.Errorf("normalizeModelNameForBackend(%q,%q)=%q want %q", c.model, c.backend, got, c.want) + } + } +} + // --------------------------------------------------------------------------- // Pause / Resume with persist callback // --------------------------------------------------------------------------- diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index b02321f04b..e0c90ae0c3 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -6288,6 +6288,19 @@ type ReviewConfig struct { // aggregate has no consumer and the reviewer is silent by construction. // Turning this on is what makes a review reach the human who has to decide. PostComments bool `yaml:"post_comments,omitempty" json:"post_comments,omitempty"` + // MaxPerspectivesPerPR caps how many review perspectives one PR may be + // given in a single dispatch cycle. It exists because parallel review + // slots are a fixed budget spent in PR order: without a cap, the first PR + // in a deep queue absorbs every slot for its own perspectives, so adding + // reviewers buys more opinions on one PR instead of coverage across many. + // Capping it spends the same budget breadth-first. No coverage is lost — + // the perspectives skipped this cycle are still "missing" next cycle and + // get dispatched then — so this schedules depth rather than dropping it. + // It also bounds how many comments a single PR can collect at once, which + // matters once reviewers publish. Zero means no cap — fanning every + // perspective out at once stays the default, so this only changes a hive + // that opts in because its queue is too deep to review in depth. + MaxPerspectivesPerPR int `yaml:"max_perspectives_per_pr,omitempty" json:"max_perspectives_per_pr,omitempty"` } // DuplicateSweepConfig gates the cross-PR duplicate sweep diff --git a/src/pkg/dashboard/agent_snapshot_freshness_7526_test.go b/src/pkg/dashboard/agent_snapshot_freshness_7526_test.go new file mode 100644 index 0000000000..18794f83a9 --- /dev/null +++ b/src/pkg/dashboard/agent_snapshot_freshness_7526_test.go @@ -0,0 +1,101 @@ +package dashboard + +import ( + "encoding/json" + "testing" +) + +// #7526: /api/status serves a cached snapshot that used to be refreshed only by +// the eval cycle, which enumerates every configured repo against the GitHub API +// first. On a rate-limited spoke that cycle runs far past its nominal interval, +// so agents the manager started minutes ago still read state=stopped and the +// dashboard paints the whole fleet red. The fast agent-only tick must refresh +// the agent block of the served snapshot. +func TestBroadcastAgentStatusRefreshesServedSnapshot(t *testing.T) { + s := newTestServer() + + stale := minimalPayload() + stale.Agents = []FrontendAgent{ + {Name: "reviewer", State: "stopped", Busy: "idle"}, + {Name: "scanner", State: "stopped", Busy: "idle"}, + } + s.UpdateStatus(stale) + + // The 10s agent tick: manager reports both agents running. + s.BroadcastAgentStatus(&AgentStatusPayload{ + Agents: []FrontendAgent{ + {Name: "reviewer", State: "running", Busy: "working", Session: "hive-reviewer"}, + {Name: "scanner", State: "running", Busy: "idle", Session: "hive-scanner"}, + }, + ConfiguredAgents: []FrontendConfiguredAgent{{Name: "reviewer"}, {Name: "scanner"}}, + }) + + rec := doGet(s, "/api/status") + if rec.Code != 200 { + t.Fatalf("GET /api/status = %d, want 200", rec.Code) + } + var got StatusPayload + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatalf("unmarshal status: %v", err) + } + if len(got.Agents) != 2 { + t.Fatalf("agents = %d, want 2", len(got.Agents)) + } + for _, a := range got.Agents { + if a.State != "running" { + t.Errorf("agent %s state = %q, want %q — /api/status is serving "+ + "eval-cycle-stale liveness (#7526)", a.Name, a.State, "running") + } + } + if len(got.ConfiguredAgents) != 2 { + t.Errorf("configuredAgents = %d, want 2", len(got.ConfiguredAgents)) + } +} + +// The refresh must be copy-on-write: a payload already handed to a reader (as +// handleStatus does, marshalling outside statusMu) must not be mutated. +func TestRefreshAgentSnapshotIsCopyOnWrite(t *testing.T) { + s := newTestServer() + base := minimalPayload() + base.Agents = []FrontendAgent{{Name: "reviewer", State: "stopped"}} + s.UpdateStatus(base) + + s.statusMu.RLock() + held := s.status + s.statusMu.RUnlock() + + s.RefreshAgentSnapshot(&AgentStatusPayload{ + Agents: []FrontendAgent{{Name: "reviewer", State: "running"}}, + }) + + if held.Agents[0].State != "stopped" { + t.Errorf("previously loaded snapshot was mutated in place: state = %q", held.Agents[0].State) + } + s.statusMu.RLock() + current := s.status + s.statusMu.RUnlock() + if current == held { + t.Fatal("snapshot pointer unchanged, want copy-on-write replacement") + } + if current.Agents[0].State != "running" { + t.Errorf("current snapshot state = %q, want running", current.Agents[0].State) + } + if current.HiveID != base.HiveID || current.Governor.Mode != base.Governor.Mode { + t.Error("non-agent fields lost by the agent-block patch") + } +} + +// A nil cached snapshot (pre-first-eval boot) must not panic or fabricate one: +// handleStatus's "initializing" response stays intact. +func TestRefreshAgentSnapshotNoopsBeforeFirstStatus(t *testing.T) { + s := newTestServer() + s.RefreshAgentSnapshot(&AgentStatusPayload{Agents: []FrontendAgent{{Name: "reviewer", State: "running"}}}) + s.RefreshAgentSnapshot(nil) + + s.statusMu.RLock() + got := s.status + s.statusMu.RUnlock() + if got != nil { + t.Fatalf("status = %#v, want nil before the first full snapshot", got) + } +} diff --git a/src/pkg/dashboard/api.go b/src/pkg/dashboard/api.go index b1fe9e9129..e8ffdb3fe6 100644 --- a/src/pkg/dashboard/api.go +++ b/src/pkg/dashboard/api.go @@ -2710,6 +2710,11 @@ const ( tokenAccessMaxEntries = 100 ) +// tokenAccessLogPath is github.TokenAccessLogPath. The file is written ONLY +// by the hive process (pkg/github's token-access ingester) and is hive-owned +// 0600: the agents whose gh calls it records cannot append to, truncate, or +// read it (#6287). The wrappers drop per-call events into a spool the hive +// ingests, attributing each to the uid that owns the event file. var tokenAccessLogPath = "/var/run/hive-metrics/token-access.jsonl" func (s *Server) handleTokenAccess(w http.ResponseWriter, r *http.Request) { @@ -2718,7 +2723,8 @@ func (s *Server) handleTokenAccess(w http.ResponseWriter, r *http.Request) { // --body ...). Without a role gate any authenticated user — including // read-only contributors — could enumerate the hive's full GitHub operation // history. Gate at owner-role, consistent with handleConfigDownload and - // handleSelfUpgrade which protect equivalent operator-only data. + // handleSelfUpgrade which protect equivalent operator-only data. The + // write side is protected too: see tokenAccessLogPath. if !requireOwnerRole(w, r) { return } diff --git a/src/pkg/dashboard/cli_models.go b/src/pkg/dashboard/cli_models.go index 96cac66576..768c7aa57c 100644 --- a/src/pkg/dashboard/cli_models.go +++ b/src/pkg/dashboard/cli_models.go @@ -67,17 +67,6 @@ const ( // require a plausible editor identifier. copilotEditorVersion = "vscode/1.99.0" - // copilotSDKHelperPath is where the image installs the Node helper that - // lists Copilot models through the official @github/copilot-sdk (source: - // bin/copilot-models.mjs, COPYed by src/Dockerfile). The SDK spawns the - // pinned copilot CLI as a JSON-RPC server, so the probe rides the CLI's - // own stored auth and TLS handling — auth configurations the raw-HTTP - // probe below cannot reach (verified live against copilot CLI 1.0.59 in a - // hive pod: 25 models via stored device-flow auth behind the egress - // proxy). Absent outside the image, in which case the SDK probe is - // skipped instantly. - copilotSDKHelperPath = "/usr/local/bin/copilot-models.mjs" - // copilotSDKNodeBinary runs the helper. The helper is plain-Node ESM; it // is invoked explicitly (not via shebang) so no exec bit is needed. copilotSDKNodeBinary = "node" @@ -222,6 +211,33 @@ const ( // it at hermetic servers and exercise fallback branches without network. var copilotUserEndpointURL = "https://api.github.com/copilot_internal/user" +// copilotSDKHelperPath is where the image installs the Node helper that +// lists Copilot models through the official @github/copilot-sdk (source: +// bin/copilot-models.mjs, COPYed by src/Dockerfile). The SDK spawns the +// pinned copilot CLI as a JSON-RPC server, so the probe rides the CLI's +// own stored auth and TLS handling — auth configurations the raw-HTTP +// probe below cannot reach (verified live against copilot CLI 1.0.59 in a +// hive pod: 25 models via stored device-flow auth behind the egress +// proxy). Absent outside the image, in which case the SDK probe is +// skipped instantly. A var (not const) only so tests can point it at a +// hermetic path regardless of what the host has installed; production +// never reassigns it. +var copilotSDKHelperPath = "/usr/local/bin/copilot-models.mjs" + +// setCopilotSDKHelperPathForTest repoints the SDK helper script path for the +// lifetime of t, mirroring the knowledge.SetBaseDirForTest convention. +func setCopilotSDKHelperPathForTest(t interface { + Helper() + Cleanup(func()) +}, path string) { + t.Helper() + old := copilotSDKHelperPath + copilotSDKHelperPath = path + t.Cleanup(func() { copilotSDKHelperPath = old }) +} + +// --- Static fallback lists (kept CURRENT — July 2026) --- + // claudeStaticModels is the fallback offered when the Claude models probe // cannot run (no API key and no fresh OAuth token in the CLI credentials // file) or fails. Live discovery via api.anthropic.com/v1/models is strongly diff --git a/src/pkg/dashboard/cli_models_sdk_severity_test.go b/src/pkg/dashboard/cli_models_sdk_severity_test.go index bfd73a33d4..cafc565c52 100644 --- a/src/pkg/dashboard/cli_models_sdk_severity_test.go +++ b/src/pkg/dashboard/cli_models_sdk_severity_test.go @@ -15,6 +15,9 @@ import ( "context" "errors" "fmt" + "os" + "os/exec" + "path/filepath" "strings" "testing" ) @@ -39,15 +42,45 @@ func TestCopilotSDKHelperAbsentIsASentinel(t *testing.T) { // The real exec path must produce the sentinel when the helper script is not // installed, otherwise the sentinel is dead code and every dev machine starts -// emitting a WARN. +// emitting a WARN. The helper path is repointed at a path that does not exist +// so the assertion holds regardless of whether the host image ships the real +// helper (on live agent hosts it exists but is unauthenticated, which used to +// flip this test to a hard FAIL — the third state the old version, which ran +// whatever was at the production path, never accounted for). func TestProbeCopilotModelsSDK_AbsentHelperYieldsSentinel(t *testing.T) { - if _, err := execCopilotSDKHelper(context.Background(), ""); err != nil { - if !errors.Is(err, errCopilotSDKHelperAbsent) { - t.Fatalf("helper absence did not yield the sentinel: %v", err) - } - return + setCopilotSDKHelperPathForTest(t, filepath.Join(t.TempDir(), "copilot-models.mjs")) + _, err := execCopilotSDKHelper(context.Background(), "") + if err == nil { + t.Fatal("exec of a nonexistent helper unexpectedly succeeded") + } + if !errors.Is(err, errCopilotSDKHelperAbsent) { + t.Fatalf("helper absence did not yield the sentinel: %v", err) + } +} + +// The converse on the same exec path: a helper that EXISTS and fails (#7365 — +// exits nonzero) must NOT match the absence sentinel, or the failure would be +// logged at INFO and stay invisible. Uses a stub script so the assertion never +// depends on the host's real helper or its auth state. +func TestProbeCopilotModelsSDK_FailingHelperIsNotAbsent(t *testing.T) { + if _, err := exec.LookPath("node"); err != nil { + t.Skip("node not on PATH; cannot exercise the helper exec path") + } + stub := filepath.Join(t.TempDir(), "copilot-models.mjs") + if err := os.WriteFile(stub, []byte("console.error('stub failure'); process.exit(1);\n"), 0o644); err != nil { + t.Fatal(err) + } + setCopilotSDKHelperPathForTest(t, stub) + _, err := execCopilotSDKHelper(context.Background(), "") + if err == nil { + t.Fatal("failing stub helper unexpectedly succeeded") + } + if errors.Is(err, errCopilotSDKHelperAbsent) { + t.Fatalf("a real helper failure matched the absence sentinel: %v", err) + } + if !strings.Contains(err.Error(), "stub failure") { + t.Errorf("helper stderr not folded into the error: %v", err) } - t.Skip("copilot SDK helper present on this machine; skipping absence check") } // The severity split itself, asserted on real emitted log records. diff --git a/src/pkg/dashboard/cli_models_sdk_test.go b/src/pkg/dashboard/cli_models_sdk_test.go index b2138af1bc..99d7df3af3 100644 --- a/src/pkg/dashboard/cli_models_sdk_test.go +++ b/src/pkg/dashboard/cli_models_sdk_test.go @@ -3,6 +3,7 @@ package dashboard import ( "context" "errors" + "path/filepath" "testing" ) @@ -145,13 +146,15 @@ func TestQueryCLIModels_SDKResultsFeedRetention(t *testing.T) { // TestProbeCopilotModelsSDK_HelperNotInstalled verifies the real exec path // degrades instantly (no hang, no panic) when the helper script is absent — -// the situation on dev machines and CI runners. +// the situation on dev machines and CI runners. The helper path is repointed +// at a nonexistent file so the check is hermetic on hosts that DO ship the +// helper (where the old version exec'd the real helper — a network-dependent +// probe whose outcome tracked the host's copilot auth state). func TestProbeCopilotModelsSDK_HelperNotInstalled(t *testing.T) { + setCopilotSDKHelperPathForTest(t, filepath.Join(t.TempDir(), "copilot-models.mjs")) s := &Server{logger: testLogger()} if _, err := s.probeCopilotModelsSDK(""); err == nil { - // The helper is installed only inside the hive image; if this machine - // actually has it, the probe may legitimately succeed — skip then. - t.Skip("copilot SDK helper present on this machine; skipping absence check") + t.Fatal("probe with an absent helper unexpectedly succeeded") } } diff --git a/src/pkg/dashboard/pr_pill_merge_verdict_7478_test.go b/src/pkg/dashboard/pr_pill_merge_verdict_7478_test.go index ba6f0c295d..b6a80f238c 100644 --- a/src/pkg/dashboard/pr_pill_merge_verdict_7478_test.go +++ b/src/pkg/dashboard/pr_pill_merge_verdict_7478_test.go @@ -156,7 +156,7 @@ function check(name, cond) { // Green only on the sweep's verdict. check('eligible is green', prMergeState({ mergeable: 'yes', merge_verdict: { state: 'eligible' } }) === 'eligible'); check('outstanding is amber', prMergeState({ mergeable: 'yes', merge_verdict: { state: 'outstanding', reason: 'CI failing: build' } }) === 'outstanding'); -check('blocked lights nothing', prMergeState({ mergeable: 'no', merge_verdict: { state: 'blocked', reason: 'not mergeable on GitHub (dirty)' } }) === ''); +check('blocked lights nothing', prMergeState({ mergeable: 'no', merge_verdict: { state: 'blocked', reason: 'has merge conflicts with v4 — needs a rebase' } }) === ''); check('unknown lights nothing', prMergeState({ mergeable: '', merge_verdict: { state: 'unknown' } }) === ''); check('an unrecognised state lights nothing', prMergeState({ mergeable: 'yes', merge_verdict: { state: 'someday' } }) === ''); @@ -186,11 +186,42 @@ check('eligible tooltip carries the reason', prMergeNote(green).includes('would const greenOptionalRed = { mergeable: 'yes', mergeable_state: 'unstable', merge_verdict: { state: 'eligible', reason: 'the sweep would merge this now — only non-required checks are red (playwright)' } }; check('eligible beside a red optional check names it', prMergeNote(greenOptionalRed).includes('playwright')); check('eligible without a reason still reads sensibly', prMergeNote({ mergeable: 'yes', merge_verdict: { state: 'eligible' } }).includes('would merge this now')); -const dirty = { mergeable: 'no', mergeable_state: 'dirty', merge_verdict: { state: 'blocked', reason: 'not mergeable on GitHub (dirty)' } }; -check('blocked tooltip carries the reason', prMergeNote(dirty).includes('dirty')); +const dirty = { mergeable: 'no', mergeable_state: 'dirty', merge_verdict: { state: 'blocked', reason: 'has merge conflicts with v4 — needs a rebase' } }; +check('blocked tooltip carries the reason', prMergeNote(dirty).includes('merge conflicts with v4')); check('blocked tooltip does not say eligible', !prMergeNote(dirty).includes('eligible')); -check('unknown tooltip says not yet known', prMergeNote({ mergeable: '', merge_verdict: { state: 'unknown', reason: 'mergeability not yet known; CI pending' } }).includes('not yet known')); -check('no undefined anywhere', ![green, bluefin1253, dirty, { mergeable: 'yes' }, {}, { merge_verdict: {} }].some(p => prMergeNote(p).includes('undefined'))); +// hivecommons/hive#7515: a "blocked" verdict now carries the sweep's own +// reason (the red required check, the missing review) and the tooltip shows +// it verbatim — GitHub's raw state is not appended on top. +const blocked = { mergeable: 'no', mergeable_state: 'blocked', merge_verdict: { state: 'blocked', reason: 'blocked — CI failing: build, lint' } }; +check('blocked tooltip names the failing required checks', prMergeNote(blocked).includes('blocked — CI failing: build, lint')); +check('blocked tooltip does not repeat the raw GitHub state', !prMergeNote(blocked).includes('GitHub state')); +const blockedReview = { mergeable: 'no', mergeable_state: 'blocked', merge_verdict: { state: 'blocked', reason: 'blocked — awaiting review approval' } }; +check('blocked tooltip names the missing review', prMergeNote(blockedReview).includes('awaiting review approval')); +check('draft tooltip says what to do', prMergeNote({ mergeable: 'yes', merge_verdict: { state: 'blocked', reason: 'draft — mark ready for review to enter the sweep' } }).includes('mark ready for review')); +check('unknown tooltip says not yet computed', prMergeNote({ mergeable: '', merge_verdict: { state: 'unknown', reason: 'mergeability not yet computed by GitHub — re-checked next tick; CI pending' } }).includes('not yet computed')); +// hivecommons/hive#7515 step 2: when no sweep gate explains a "blocked" PR, +// the verdict now names the branch-protection rule GitHub was hiding. The +// tooltip must show that rule verbatim — the mapping is Go's, and the JS +// must not re-word, truncate or re-derive any of it. +const rules = [ + 'blocked — changes requested by @reviewer', + 'blocked — required check "validate" has not reported', + 'blocked — required checks "build", "lint" are failing', + 'blocked — an approving review is required by branch protection (0 given)', + 'blocked — CI failing: build; GitHub also requires: changes requested by @reviewer', +]; +rules.forEach(reason => { + const p = { mergeable: 'no', mergeable_state: 'blocked', merge_verdict: { state: 'blocked', reason } }; + check('tooltip shows the derived rule verbatim: ' + reason, prMergeNote(p).includes(reason)); + check('no raw GitHub state appended to: ' + reason, !prMergeNote(p).includes('GitHub state')); + check('a blocked rule never reads as eligible: ' + reason, !prMergeNote(p).includes('eligible')); +}); +// Negative control: the placeholder is still shown when the governor could +// NOT name a rule. A frontend that invented one would fail here. +const unnamed = { mergeable: 'no', mergeable_state: 'blocked', merge_verdict: { state: 'blocked', reason: 'blocked — all sweep gates pass; a branch-protection rule is unsatisfied' } }; +check('an underivable rule still says so honestly', prMergeNote(unnamed).includes('a branch-protection rule is unsatisfied')); +check('an underivable rule does not name a check', !prMergeNote(unnamed).includes('required check')); +check('no undefined anywhere', ![green, bluefin1253, dirty, blocked, { mergeable: 'yes' }, {}, { merge_verdict: {} }].some(p => prMergeNote(p).includes('undefined'))); if (fails) { console.log(fails + ' check(s) failed'); process.exit(1); } ` diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index f490fc26e2..a5c2fc9b9a 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -2240,10 +2240,45 @@ func (s *Server) handleGitHubAppRecheck(w http.ResponseWriter, r *http.Request) } } +// RefreshAgentSnapshot patches the agent block of the cached status snapshot +// that /api/status (and the SSE replay frame a reconnecting tab receives) +// serves, using the fast-tick agent-only payload. +// +// Without this, agent liveness on those two surfaces is only as fresh as the +// last completed eval cycle, which enumerates every configured repo against +// the GitHub API and can therefore run far past its nominal interval under +// rate-limit backoff. A whole fleet that started seconds ago then reads +// state=stopped — red "crashed" dots — for 10-15+ minutes (#7526). Agent state +// is in-memory manager state that needs no GitHub call, so it is refreshed on +// the 10s tick independently of the enumeration. +// +// Copy-on-write: handleStatus marshals the *StatusPayload it loaded outside +// statusMu, so the cached payload must never be mutated in place. +func (s *Server) RefreshAgentSnapshot(payload *AgentStatusPayload) { + if payload == nil { + return + } + s.statusMu.Lock() + defer s.statusMu.Unlock() + if s.status == nil { + return + } + patched := *s.status + patched.Agents = payload.Agents + patched.HiddenAgents = payload.HiddenAgents + patched.ConfiguredAgents = payload.ConfiguredAgents + s.status = &patched +} + // BroadcastAgentStatus sends a lightweight agent-only SSE event on a fast // cadence. Skipped if a full status was broadcast within the last 5 seconds // to avoid redundant renders on the frontend. func (s *Server) BroadcastAgentStatus(payload *AgentStatusPayload) { + // Refresh the cached snapshot FIRST, ahead of the skip below: even when the + // SSE frame is redundant, /api/status and the SSE replay frame must not be + // left serving eval-cycle-old liveness (#7526). + s.RefreshAgentSnapshot(payload) + s.statusMu.RLock() recentFull := time.Since(s.lastFullBroadcast) < agentSkipAfterFullBroadcastS s.statusMu.RUnlock() diff --git a/src/pkg/github/automerge/f3_trusted_merger_source_test.go b/src/pkg/github/automerge/f3_trusted_merger_source_test.go index 1a3709dfa3..b7f6b78d1a 100644 --- a/src/pkg/github/automerge/f3_trusted_merger_source_test.go +++ b/src/pkg/github/automerge/f3_trusted_merger_source_test.go @@ -2,6 +2,7 @@ package automerge import ( "os" + "path/filepath" "regexp" "strings" "testing" @@ -45,6 +46,38 @@ func f3ReadSource(t *testing.T, file string) string { return string(raw) } +// f3ReadPackage returns every non-test .go file in dir concatenated, so a +// source-level assertion follows a declaration that moves between files in the +// same package. cmd/hive is being split file-by-file out of a god-file, and a +// guard pinned to one filename fails on pure code motion — which reads as a +// lost fix when nothing was lost. The package is still the unit that matters: +// the declaration must exist SOMEWHERE in cmd/hive, and the wiring assertion +// below stays pinned to main.go because that is where startup lives. +func f3ReadPackage(t *testing.T, dir string) string { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read dir %s: %v", dir, err) + } + var b strings.Builder + for _, e := range entries { + name := e.Name() + if e.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + raw, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil { + t.Fatalf("read %s: %v", name, err) + } + b.Write(raw) + b.WriteString("\n") + } + if b.Len() == 0 { + t.Fatalf("no non-test .go files found under %s", dir) + } + return b.String() +} + // f3FuncBody returns the source of a named func, from its declaration line to // the first line that is exactly "}". func f3FuncBody(t *testing.T, src, decl, name string) string { @@ -166,7 +199,7 @@ func TestF3AuthorizerIsWiredInMain(t *testing.T) { "MergerAuthorizer: trustedMergerFunc(cfg) beside StartMergeRequestWatcher.") } - body := f3FuncBody(t, src, "func trustedMergerFunc(", "trustedMergerFunc") + body := f3FuncBody(t, f3ReadPackage(t, "../../../cmd/hive"), "func trustedMergerFunc(", "trustedMergerFunc") if !strings.Contains(body, "config.RoleAtLeast(role, config.RoleMerger)") { t.Error("trustedMergerFunc no longer requires at least config.RoleMerger — the sweep would " + "admit a tier below the one the dashboard queue endpoint enforces (audit F3)") diff --git a/src/pkg/github/automerge_sweep.go b/src/pkg/github/automerge_sweep.go index 01dd55159c..67716176c9 100644 --- a/src/pkg/github/automerge_sweep.go +++ b/src/pkg/github/automerge_sweep.go @@ -102,3 +102,13 @@ func (c *Client) info(msg string, args ...any) { c.logger.Info(msg, args...) } } + +// requiredStatusCheckContexts returns the set of status-check contexts / +// check-run names the base branch requires, and whether that set is known. +// See RequiredStatusCheckContexts for the resolution order; on v5 the merge +// sweep itself lives in pkg/github/automerge, so this Client-level shim is +// used only by the protection-facts collector (#7515). +func (c *Client) requiredStatusCheckContexts(ctx context.Context, owner, repo, branch string) (map[string]bool, bool) { + set, ok := c.configRequiredChecks() + return RequiredStatusCheckContexts(ctx, c.client, owner, repo, branch, set, ok) +} diff --git a/src/pkg/github/client.go b/src/pkg/github/client.go index e107fe21d8..5f45cd8a70 100644 --- a/src/pkg/github/client.go +++ b/src/pkg/github/client.go @@ -373,6 +373,13 @@ type PullRequest struct { HeadRef string `json:"head_ref,omitempty"` HeadRepo string `json:"head_repo,omitempty"` FromFork bool `json:"from_fork,omitempty"` + // BaseRef is the branch the PR targets. Display only: the merge-verdict + // reason names it ("has merge conflicts with v4 — needs a rebase") so a + // blocked pill says what to do rather than GitHub's enum + // (hivecommons/hive#7515). It comes from the list payload — no extra + // fetch — and stays empty on an abbreviated payload, where the wording + // falls back to "the base branch". + BaseRef string `json:"base_ref,omitempty"` // FailingChecks names the completed check runs whose conclusion was // failure/action_required. CIFailureExcerpt carries the raw error lines // pulled from those runs' annotations — the evidence a fix agent (or an @@ -397,6 +404,12 @@ type PullRequest struct { // src/docs/review-queue-triage.md (#6183). Derived from Title + Labels // at enumeration time; never read by the governor or any agent. ReviewClass ReviewClass `json:"review_class,omitempty"` + // Protection carries the branch-protection facts behind GitHub's + // one-word "blocked" state — which required checks are red or absent, + // and GitHub's own review decision. It is nil when none of it could be + // determined, and BranchProtectionBlockReason then declines to guess + // (hivecommons/hive#7515). Display only: no merge gate reads it. + Protection *ProtectionFacts `json:"protection,omitempty"` } // HasFailingRequiredCheck reports whether this PR has a completed, non-meta @@ -869,6 +882,14 @@ func prHeadSHA(pr *gh.PullRequest) string { return pr.GetHead().GetSHA() } +// prBaseRef is the branch a PR targets, or "" on an abbreviated payload. +func prBaseRef(pr *gh.PullRequest) string { + if pr.GetBase() == nil { + return "" + } + return pr.GetBase().GetRef() +} + func (c *Client) fetchPRs(ctx context.Context, repo string) (actionable []PullRequest, held []HoldItem, heldPRs []PullRequest, staleDrafts []PullRequest, totalPRs int, breakdown RepoPRBreakdown, err error) { now := time.Now() owner, repoName := c.splitRepo(repo) @@ -928,6 +949,7 @@ func (c *Client) fetchPRs(ctx context.Context, repo string) (actionable []PullRe HeadRef: headRef, HeadRepo: headRepo, FromFork: fromFork, + BaseRef: prBaseRef(pr), }) } continue @@ -989,6 +1011,7 @@ func (c *Client) fetchPRs(ctx context.Context, repo string) (actionable []PullRe HeadRef: headRef, HeadRepo: headRepo, FromFork: fromFork, + BaseRef: prBaseRef(pr), }) } @@ -1001,81 +1024,107 @@ func (c *Client) fetchPRs(ctx context.Context, repo string) (actionable []PullRe // EnrichCIStatus fetches check-run results for each PR's HEAD commit // and sets the CIStatus field to "success", "failure", or "pending". // The "tide" check is skipped — it reports Prow merge-bot state, not CI. +// +// It also collects the branch-protection facts behind a "blocked" PR +// (hivecommons/hive#7515, step 2) — which required checks are red or have +// never reported, and GitHub's own review decision — so that the dashboard +// pill can name the unsatisfied rule instead of repeating the word +// "blocked". Those facts are gathered from data this pass already walks +// plus at most one GraphQL query per repository; nothing here is per-hover +// and nothing here is per-PR beyond the calls that were already made. func (c *Client) EnrichCIStatus(ctx context.Context, prs []PullRequest) { if c == nil { return } + facts := newProtectionCollector(c) + for i := range prs { + reported := c.enrichPRCI(ctx, &prs[i]) + facts.attach(ctx, &prs[i], reported) + } +} + +// enrichPRCI is EnrichCIStatus's per-PR body. It returns the names of every +// check run observed on the PR's head SHA — meta checks included, because a +// required status-check context may well be one of them — or nil when no +// check-run listing was obtained at all. The caller distinguishes the two: +// "no listing" must never be read as "the required check never reported". +func (c *Client) enrichPRCI(ctx context.Context, pr *PullRequest) map[string]bool { const ciStatusSuccess = "success" const ciStatusFailure = "failure" const ciStatusPending = "pending" - for i := range prs { - if prs[i].HeadSHA == "" { - prs[i].CIStatus = ciStatusPending - continue - } - owner, repoName := c.splitRepo(prs[i].Repo) - - // Fetch the PR individually to learn its mergeability. The list - // endpoint that produced these PullRequests never populates - // "mergeable"/"mergeable_state" — GitHub computes them per-PR and - // returns them only from this single-PR GET. On error we leave the - // field as MergeableUnknown rather than guessing. - if full, _, err := c.client.PullRequests.Get(ctx, owner, repoName, prs[i].Number); err != nil { - c.logger.Warn("failed to fetch PR mergeability", "repo", prs[i].Repo, "pr", prs[i].Number, "error", err) - } else { - prs[i].Mergeable = mergeableFromState(full.GetMergeableState(), full.Mergeable) - prs[i].MergeableState = full.GetMergeableState() - } + if pr.HeadSHA == "" { + pr.CIStatus = ciStatusPending + return nil + } + owner, repoName := c.splitRepo(pr.Repo) - checkRuns, _, err := c.client.Checks.ListCheckRunsForRef(ctx, owner, repoName, prs[i].HeadSHA, &gh.ListCheckRunsOptions{ - ListOptions: gh.ListOptions{PerPage: 100}, - }) - if err != nil { - c.logger.Warn("failed to fetch check runs", "repo", prs[i].Repo, "pr", prs[i].Number, "error", err) - prs[i].CIStatus = ciStatusPending - continue + // Fetch the PR individually to learn its mergeability. The list + // endpoint that produced these PullRequests never populates + // "mergeable"/"mergeable_state" — GitHub computes them per-PR and + // returns them only from this single-PR GET. On error we leave the + // field as MergeableUnknown rather than guessing. + if full, _, err := c.client.PullRequests.Get(ctx, owner, repoName, pr.Number); err != nil { + c.logger.Warn("failed to fetch PR mergeability", "repo", pr.Repo, "pr", pr.Number, "error", err) + } else { + pr.Mergeable = mergeableFromState(full.GetMergeableState(), full.Mergeable) + pr.MergeableState = full.GetMergeableState() + } + + checkRuns, _, err := c.client.Checks.ListCheckRunsForRef(ctx, owner, repoName, pr.HeadSHA, &gh.ListCheckRunsOptions{ + ListOptions: gh.ListOptions{PerPage: 100}, + }) + if err != nil { + c.logger.Warn("failed to fetch check runs", "repo", pr.Repo, "pr", pr.Number, "error", err) + pr.CIStatus = ciStatusPending + return nil + } + reported := make(map[string]bool, len(checkRuns.CheckRuns)) + for _, cr := range checkRuns.CheckRuns { + if name := cr.GetName(); name != "" { + reported[name] = true } - if checkRuns.GetTotal() == 0 { - prs[i].CIStatus = ciStatusPending + } + if checkRuns.GetTotal() == 0 { + pr.CIStatus = ciStatusPending + return reported + } + hasFail := false + allDone := true + ciChecksFound := 0 + var failingNames []string + var failingIDs []int64 + for _, cr := range checkRuns.CheckRuns { + if isMetaCheck(cr.GetName()) { continue } - hasFail := false - allDone := true - ciChecksFound := 0 - var failingNames []string - var failingIDs []int64 - for _, cr := range checkRuns.CheckRuns { - if isMetaCheck(cr.GetName()) { - continue - } - ciChecksFound++ - if cr.GetStatus() != "completed" { - allDone = false - continue - } - conclusion := cr.GetConclusion() - if conclusion == "failure" || conclusion == "action_required" { - hasFail = true - failingNames = append(failingNames, cr.GetName()) - failingIDs = append(failingIDs, cr.GetID()) - } - } - if ciChecksFound == 0 { - prs[i].CIStatus = ciStatusPending + ciChecksFound++ + if cr.GetStatus() != "completed" { + allDone = false continue } - switch { - case hasFail: - prs[i].CIStatus = ciStatusFailure - prs[i].FailingChecks = failingNames - prs[i].CIFailureExcerpt = c.fetchFailureExcerpt(ctx, owner, repoName, failingIDs, failingNames) - case allDone: - prs[i].CIStatus = ciStatusSuccess - default: - prs[i].CIStatus = ciStatusPending + conclusion := cr.GetConclusion() + if conclusion == "failure" || conclusion == "action_required" { + hasFail = true + failingNames = append(failingNames, cr.GetName()) + failingIDs = append(failingIDs, cr.GetID()) } } + if ciChecksFound == 0 { + pr.CIStatus = ciStatusPending + return reported + } + switch { + case hasFail: + pr.CIStatus = ciStatusFailure + pr.FailingChecks = failingNames + pr.CIFailureExcerpt = c.fetchFailureExcerpt(ctx, owner, repoName, failingIDs, failingNames) + case allDone: + pr.CIStatus = ciStatusSuccess + default: + pr.CIStatus = ciStatusPending + } + return reported } // isMetaCheck reports check runs that are merge-gates or deploy-status diff --git a/src/pkg/github/merge_block_reason.go b/src/pkg/github/merge_block_reason.go new file mode 100644 index 0000000000..63e7d18a5f --- /dev/null +++ b/src/pkg/github/merge_block_reason.go @@ -0,0 +1,163 @@ +package github + +import ( + "sort" + "strconv" + "strings" +) + +// ReviewDecision is GitHub's own aggregate verdict on a PR's reviews, as +// reported by the GraphQL PullRequest.reviewDecision field. It is the only +// signal that distinguishes "a review is required and nobody has given one" +// from "a reviewer asked for changes" — REST exposes neither, and +// mergeable_state folds both into the single word "blocked". +type ReviewDecision string + +const ( + // ReviewDecisionNone is the zero value: GitHub said nothing, either + // because the base branch requires no review or because the field was + // never fetched. It must never be read as "approved". + ReviewDecisionNone ReviewDecision = "" + // ReviewDecisionApproved means the review requirement is satisfied. + ReviewDecisionApproved ReviewDecision = "APPROVED" + // ReviewDecisionChangesRequested means a reviewer asked for changes and + // has not dismissed that review. + ReviewDecisionChangesRequested ReviewDecision = "CHANGES_REQUESTED" + // ReviewDecisionReviewRequired means branch protection requires an + // approving review that has not been given. + ReviewDecisionReviewRequired ReviewDecision = "REVIEW_REQUIRED" +) + +// ProtectionFacts are the branch-protection observations gathered for one PR +// during the sweep's normal enumeration — no extra per-PR API call, and +// certainly no call per dashboard hover. Every field is optional: an empty +// one means "not determined", never "not required". +type ProtectionFacts struct { + // ReviewDecision is GitHub's aggregate review verdict, or "" when it was + // not fetched. + ReviewDecision ReviewDecision `json:"review_decision,omitempty"` + // ChangesRequestedBy are the logins whose latest review on this PR asked + // for changes. + ChangesRequestedBy []string `json:"changes_requested_by,omitempty"` + // ApprovalsGiven counts distinct reviewers whose latest opinionated + // review is an approval. + ApprovalsGiven int `json:"approvals_given,omitempty"` + // RequiredChecksKnown records that the required status-check set for the + // PR's base branch was determined (from auto_merge.required_checks or + // from the branch-protection API). Without it, FailingRequiredChecks and + // MissingRequiredChecks say nothing. + RequiredChecksKnown bool `json:"required_checks_known,omitempty"` + // FailingRequiredChecks are the required checks observed red on the head + // commit. + FailingRequiredChecks []string `json:"failing_required_checks,omitempty"` + // MissingRequiredChecks are required checks for which NO check run exists + // on the head commit. It is only ever populated when at least one OTHER + // required check WAS observed as a check run: on a repository that + // reports its required contexts as commit statuses rather than check + // runs, every required context looks absent from a check-run listing, and + // "required check X has not reported" would be confidently wrong for all + // of them. See protectionCollector.attach. + MissingRequiredChecks []string `json:"missing_required_checks,omitempty"` +} + +// BranchProtectionBlockReason names, in one place, the branch-protection rule +// that GitHub's "blocked" mergeable_state is hiding (hivecommons/hive#7515). +// +// This is THE state → explanation mapping for the blocked case: the dashboard +// pill's tooltip renders whatever this returns, so the wording lives in Go, +// under test, and never in JS string concatenation. +// +// ok is false when the facts on hand do not identify a rule. The caller must +// then say so honestly rather than guess — a confidently wrong explanation +// sends the operator to fix something that is not broken, which is worse than +// admitting we do not know. Order is by what the operator acts on first: an +// explicit "changes requested" outranks a red check, which outranks a check +// that never ran, which outranks a merely missing approval. +func (p PullRequest) BranchProtectionBlockReason() (string, bool) { + f := p.Protection + if f == nil { + return "", false + } + if f.ReviewDecision == ReviewDecisionChangesRequested { + if who := mentionList(f.ChangesRequestedBy); who != "" { + return "changes requested by " + who, true + } + return "a reviewer requested changes", true + } + if f.RequiredChecksKnown { + if names := quotedList(f.FailingRequiredChecks); names != "" { + return "required " + checkNoun(f.FailingRequiredChecks) + " " + names + " " + isAre(f.FailingRequiredChecks) + " failing", true + } + if names := quotedList(f.MissingRequiredChecks); names != "" { + return "required " + checkNoun(f.MissingRequiredChecks) + " " + names + " " + hasHave(f.MissingRequiredChecks) + " not reported", true + } + } + if f.ReviewDecision == ReviewDecisionReviewRequired { + return "an approving review is required by branch protection (" + + strconv.Itoa(f.ApprovalsGiven) + " given)", true + } + return "", false +} + +// mentionList renders logins as "@a, @b", skipping blanks and duplicates. +func mentionList(logins []string) string { + seen := make(map[string]bool, len(logins)) + out := make([]string, 0, len(logins)) + for _, l := range logins { + l = strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(l), "@")) + if l == "" || seen[strings.ToLower(l)] { + continue + } + seen[strings.ToLower(l)] = true + out = append(out, "@"+l) + } + sort.Strings(out) + return strings.Join(out, ", ") +} + +// quotedList renders check names as `"build", "lint"` in a stable order. +func quotedList(names []string) string { + seen := make(map[string]bool, len(names)) + out := make([]string, 0, len(names)) + for _, n := range names { + n = strings.TrimSpace(n) + if n == "" || seen[n] { + continue + } + seen[n] = true + out = append(out, strconv.Quote(n)) + } + sort.Strings(out) + return strings.Join(out, ", ") +} + +func checkNoun(names []string) string { + if countDistinct(names) == 1 { + return "check" + } + return "checks" +} + +func isAre(names []string) string { + if countDistinct(names) == 1 { + return "is" + } + return "are" +} + +func hasHave(names []string) string { + if countDistinct(names) == 1 { + return "has" + } + return "have" +} + +func countDistinct(names []string) int { + seen := make(map[string]bool, len(names)) + for _, n := range names { + if n = strings.TrimSpace(n); n != "" { + seen[n] = true + } + } + return len(seen) +} diff --git a/src/pkg/github/merge_block_reason_7515_test.go b/src/pkg/github/merge_block_reason_7515_test.go new file mode 100644 index 0000000000..26cd87ff08 --- /dev/null +++ b/src/pkg/github/merge_block_reason_7515_test.go @@ -0,0 +1,215 @@ +package github + +import "testing" + +// TestBranchProtectionBlockReason is the pin for the ONE place that maps a +// blocked PR's branch-protection facts to the words an operator reads in the +// dashboard tooltip (hivecommons/hive#7515). Every row asserts the EXACT +// string, so a rewording anywhere has to come through here. +func TestBranchProtectionBlockReason(t *testing.T) { + tests := []struct { + name string + pr PullRequest + want string + wantOK bool + comment string + }{ + { + name: "no facts at all declines to guess", + pr: PullRequest{Repo: "org/repo", Number: 1}, + // Negative control: the function must NOT return a plausible + // default. "unknown" is the honest answer and the caller prints + // its own placeholder. + wantOK: false, + }, + { + name: "facts present but empty declines to guess", + pr: PullRequest{Protection: &ProtectionFacts{}}, + // Second negative control: a non-nil Protection whose every + // field is the zero value is still "we determined nothing". + wantOK: false, + }, + { + name: "required set known and everything green declines to guess", + pr: PullRequest{Protection: &ProtectionFacts{ + RequiredChecksKnown: true, + ReviewDecision: ReviewDecisionApproved, + }}, + // Third negative control: approved + no red/absent required + // check means the rule is one we do not read (a merge-queue + // rule, a CODEOWNERS rule). Do not invent one. + wantOK: false, + }, + { + name: "changes requested names the reviewer", + pr: PullRequest{Protection: &ProtectionFacts{ + ReviewDecision: ReviewDecisionChangesRequested, + ChangesRequestedBy: []string{"reviewer"}, + }}, + want: "changes requested by @reviewer", + wantOK: true, + }, + { + name: "changes requested by several, deduped, @-stripped, sorted", + pr: PullRequest{Protection: &ProtectionFacts{ + ReviewDecision: ReviewDecisionChangesRequested, + ChangesRequestedBy: []string{"zoe", "@alice", "zoe", ""}, + }}, + want: "changes requested by @alice, @zoe", + wantOK: true, + }, + { + name: "changes requested with no login still says so", + pr: PullRequest{Protection: &ProtectionFacts{ + ReviewDecision: ReviewDecisionChangesRequested, + ChangesRequestedBy: nil, + }}, + want: "a reviewer requested changes", + wantOK: true, + }, + { + name: "one failing required check", + pr: PullRequest{Protection: &ProtectionFacts{ + RequiredChecksKnown: true, + FailingRequiredChecks: []string{"build"}, + }}, + want: `required check "build" is failing`, + wantOK: true, + }, + { + name: "two failing required checks pluralise", + pr: PullRequest{Protection: &ProtectionFacts{ + RequiredChecksKnown: true, + FailingRequiredChecks: []string{"lint", "build"}, + }}, + want: `required checks "build", "lint" are failing`, + wantOK: true, + }, + { + name: "required check that never reported", + pr: PullRequest{Protection: &ProtectionFacts{ + RequiredChecksKnown: true, + MissingRequiredChecks: []string{"validate"}, + }}, + want: `required check "validate" has not reported`, + wantOK: true, + }, + { + name: "two required checks that never reported pluralise", + pr: PullRequest{Protection: &ProtectionFacts{ + RequiredChecksKnown: true, + MissingRequiredChecks: []string{"validate", "dco"}, + }}, + want: `required checks "dco", "validate" have not reported`, + wantOK: true, + }, + { + name: "missing required checks are ignored when the set is not known", + pr: PullRequest{Protection: &ProtectionFacts{ + RequiredChecksKnown: false, + MissingRequiredChecks: []string{"validate"}, + ReviewDecision: ReviewDecisionReviewRequired, + }}, + // Fourth negative control: without a known required set the + // check lists mean nothing, so the review rule is what is left. + want: "an approving review is required by branch protection (0 given)", + wantOK: true, + }, + { + name: "review required names the shortfall", + pr: PullRequest{Protection: &ProtectionFacts{ + ReviewDecision: ReviewDecisionReviewRequired, + ApprovalsGiven: 1, + }}, + want: "an approving review is required by branch protection (1 given)", + wantOK: true, + }, + { + name: "changes requested outranks a red required check", + pr: PullRequest{Protection: &ProtectionFacts{ + ReviewDecision: ReviewDecisionChangesRequested, + ChangesRequestedBy: []string{"alice"}, + RequiredChecksKnown: true, + FailingRequiredChecks: []string{"build"}, + }}, + want: "changes requested by @alice", + wantOK: true, + }, + { + name: "a red required check outranks one that never reported", + pr: PullRequest{Protection: &ProtectionFacts{ + RequiredChecksKnown: true, + FailingRequiredChecks: []string{"build"}, + MissingRequiredChecks: []string{"validate"}, + }}, + want: `required check "build" is failing`, + wantOK: true, + }, + { + name: "a red required check outranks a missing approval", + pr: PullRequest{Protection: &ProtectionFacts{ + ReviewDecision: ReviewDecisionReviewRequired, + RequiredChecksKnown: true, + FailingRequiredChecks: []string{"build"}, + }}, + want: `required check "build" is failing`, + wantOK: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := tt.pr.BranchProtectionBlockReason() + if ok != tt.wantOK { + t.Fatalf("ok = %v, want %v (reason %q)", ok, tt.wantOK, got) + } + if got != tt.want { + t.Errorf("reason = %q, want %q", got, tt.want) + } + }) + } +} + +// TestBranchProtectionBlockReason_NeverInventsAReason is the invariant behind +// the negative controls above, stated once: whenever the function claims a +// reason, some fact must support it. A build that returns a hardcoded string +// for everything passes each individual row's "want" only by accident; this +// one fails it outright. +func TestBranchProtectionBlockReason_NeverInventsAReason(t *testing.T) { + silent := []PullRequest{ + {}, + {Protection: &ProtectionFacts{}}, + {Protection: &ProtectionFacts{ReviewDecision: ReviewDecisionApproved}}, + {Protection: &ProtectionFacts{RequiredChecksKnown: true}}, + {Protection: &ProtectionFacts{ReviewDecision: ReviewDecisionNone, RequiredChecksKnown: true}}, + {Protection: &ProtectionFacts{FailingRequiredChecks: []string{"build"}}}, + {Protection: &ProtectionFacts{MissingRequiredChecks: []string{"validate"}}}, + } + for i, pr := range silent { + if got, ok := pr.BranchProtectionBlockReason(); ok { + t.Errorf("case %d: claimed %q with nothing to support it", i, got) + } + } + + // ...and the reasons it DOES give must differ from one another, so a + // single hardcoded string cannot satisfy the table above. + distinct := map[string]bool{} + for _, pr := range []PullRequest{ + {Protection: &ProtectionFacts{ReviewDecision: ReviewDecisionChangesRequested, ChangesRequestedBy: []string{"alice"}}}, + {Protection: &ProtectionFacts{RequiredChecksKnown: true, FailingRequiredChecks: []string{"build"}}}, + {Protection: &ProtectionFacts{RequiredChecksKnown: true, MissingRequiredChecks: []string{"validate"}}}, + {Protection: &ProtectionFacts{ReviewDecision: ReviewDecisionReviewRequired}}, + } { + got, ok := pr.BranchProtectionBlockReason() + if !ok { + t.Fatalf("expected a reason for %+v", pr.Protection) + } + if distinct[got] { + t.Errorf("reason %q is not specific to its facts", got) + } + distinct[got] = true + } + if len(distinct) != 4 { + t.Errorf("got %d distinct reasons, want 4", len(distinct)) + } +} diff --git a/src/pkg/github/pr_head_origin_test.go b/src/pkg/github/pr_head_origin_test.go index 70284e6cb2..6d243b5165 100644 --- a/src/pkg/github/pr_head_origin_test.go +++ b/src/pkg/github/pr_head_origin_test.go @@ -89,4 +89,20 @@ func TestFetchPRs_PopulatesForkOrigin(t *testing.T) { if pr := byNum[841]; !pr.FromFork || pr.HeadRepo != "" { t.Errorf("deleted-fork PR must be unpushable: %+v", pr) } + // The base branch rides along from the same payload so the merge + // verdict can say "has merge conflicts with main" (hivecommons/hive#7515). + for _, n := range []int{839, 840, 841} { + if pr := byNum[n]; pr.BaseRef != "main" { + t.Errorf("#%d: BaseRef = %q, want %q", n, pr.BaseRef, "main") + } + } +} + +func TestPRBaseRef(t *testing.T) { + if got := prBaseRef(&gh.PullRequest{Base: &gh.PullRequestBranch{Ref: gh.Ptr("v4")}}); got != "v4" { + t.Errorf("prBaseRef = %q, want v4", got) + } + if got := prBaseRef(&gh.PullRequest{}); got != "" { + t.Errorf("prBaseRef with no base = %q, want empty", got) + } } diff --git a/src/pkg/github/protection_facts.go b/src/pkg/github/protection_facts.go new file mode 100644 index 0000000000..931dbe516c --- /dev/null +++ b/src/pkg/github/protection_facts.go @@ -0,0 +1,216 @@ +package github + +import ( + "context" + "strings" +) + +// protectionCollector gathers, once per enrichment pass, the per-repository +// data needed to name the branch-protection rule behind a "blocked" PR +// (hivecommons/hive#7515, step 2). +// +// Cost discipline: the required-check set comes from the operator's +// auto_merge.required_checks config when one is installed — zero API calls, +// the primary path — and falls back to the branch-protection API once per +// repo+branch. Review decisions come from ONE GraphQL query per repository, +// paginated 100 PRs at a time, not one per PR. Both are memoised for the +// lifetime of the pass. Nothing is fetched on a dashboard hover. +type protectionCollector struct { + c *Client + // required is keyed "owner/repo@branch". + required map[string]requiredSet + // reviews is keyed "owner/repo"; a nil map value records a repo whose + // GraphQL query failed, so it is attempted only once. + reviews map[string]map[int]prReviewState +} + +type requiredSet struct { + set map[string]bool + known bool +} + +// prReviewState is GitHub's review verdict for one PR. +type prReviewState struct { + decision ReviewDecision + changesRequestedBy []string + approvals int +} + +func newProtectionCollector(c *Client) *protectionCollector { + return &protectionCollector{ + c: c, + required: make(map[string]requiredSet), + reviews: make(map[string]map[int]prReviewState), + } +} + +// attach populates pr.Protection from the facts on hand. reported is the set +// of check-run names observed on the PR's head commit, or nil when no +// check-run listing was obtained — the distinction matters, because a missing +// listing is not evidence that a required check never ran. +// +// pr.Protection is left nil when nothing at all was determined, so that +// BranchProtectionBlockReason declines to guess rather than inventing a rule. +func (pc *protectionCollector) attach(ctx context.Context, pr *PullRequest, reported map[string]bool) { + if pc == nil || pc.c == nil || pr == nil { + return + } + var f ProtectionFacts + + if rs, ok := pc.reviewState(ctx, pr.Repo, pr.Number); ok { + f.ReviewDecision = rs.decision + f.ChangesRequestedBy = rs.changesRequestedBy + f.ApprovalsGiven = rs.approvals + } + + if reported != nil { + req := pc.requiredChecks(ctx, pr.Repo, pr.BaseRef) + if req.known { + f.RequiredChecksKnown = true + failing := make(map[string]bool, len(pr.FailingChecks)) + for _, n := range pr.FailingChecks { + failing[n] = true + } + // observedRequired proves this repository reports its required + // contexts as CHECK RUNS. Without that proof a required context + // absent from the check-run listing may simply be a commit + // status, which this pass does not fetch; claiming it "has not + // reported" would be confidently wrong for every required + // context on such a repo. + observedRequired := 0 + var missing, red []string + for name := range req.set { + switch { + case failing[name]: + observedRequired++ + red = append(red, name) + case reported[name]: + observedRequired++ + default: + missing = append(missing, name) + } + } + f.FailingRequiredChecks = red + if observedRequired > 0 { + f.MissingRequiredChecks = missing + } + } + } + + if f.ReviewDecision == ReviewDecisionNone && !f.RequiredChecksKnown { + return + } + pr.Protection = &f +} + +func (pc *protectionCollector) requiredChecks(ctx context.Context, repo, branch string) requiredSet { + key := repo + "@" + branch + if rs, ok := pc.required[key]; ok { + return rs + } + owner, name := pc.c.splitRepo(repo) + set, known := pc.c.requiredStatusCheckContexts(ctx, owner, name, branch) + rs := requiredSet{set: set, known: known} + pc.required[key] = rs + return rs +} + +func (pc *protectionCollector) reviewState(ctx context.Context, repo string, number int) (prReviewState, bool) { + byNumber, ok := pc.reviews[repo] + if !ok { + byNumber = pc.c.fetchReviewDecisions(ctx, repo) + pc.reviews[repo] = byNumber + } + if byNumber == nil { + return prReviewState{}, false + } + rs, ok := byNumber[number] + return rs, ok +} + +// reviewDecisionQuery asks for every open PR's review verdict in one request. +// latestOpinionatedReviews is GitHub's own per-author deduplication: one +// entry per reviewer, carrying that reviewer's current position, which is +// exactly the "0 approvals given" / "changes requested by @x" the operator +// needs and is not derivable from a raw review list. +const reviewDecisionQuery = `query($owner:String!,$name:String!,$cursor:String){ + repository(owner:$owner,name:$name){ + pullRequests(states:OPEN,first:100,after:$cursor){ + pageInfo{hasNextPage endCursor} + nodes{ + number + reviewDecision + latestOpinionatedReviews(first:50){nodes{state author{login}}} + } + } + } +}` + +type reviewDecisionResponse struct { + Repository struct { + PullRequests struct { + PageInfo struct { + HasNextPage bool `json:"hasNextPage"` + EndCursor string `json:"endCursor"` + } `json:"pageInfo"` + Nodes []struct { + Number int `json:"number"` + ReviewDecision string `json:"reviewDecision"` + LatestOpinionatedReviews struct { + Nodes []struct { + State string `json:"state"` + Author *struct { + Login string `json:"login"` + } `json:"author"` + } `json:"nodes"` + } `json:"latestOpinionatedReviews"` + } `json:"nodes"` + } `json:"pullRequests"` + } `json:"repository"` +} + +// fetchReviewDecisions returns GitHub's review verdict for every open PR in +// repo, or nil when the query could not be answered (no GraphQL scope, a GHE +// that does not serve the field, a transport error). nil means "not +// determined" and is never read as "approved". +func (c *Client) fetchReviewDecisions(ctx context.Context, repo string) map[int]prReviewState { + if c == nil || c.client == nil { + return nil + } + owner, name := c.splitRepo(repo) + if owner == "" || name == "" { + return nil + } + out := make(map[int]prReviewState) + vars := map[string]any{"owner": owner, "name": name} + for page := 0; page < 20; page++ { + var resp reviewDecisionResponse + if err := c.graphQL(ctx, reviewDecisionQuery, vars, &resp); err != nil { + c.logger.Warn("failed to fetch PR review decisions", "repo", repo, "error", err) + return nil + } + for _, n := range resp.Repository.PullRequests.Nodes { + st := prReviewState{decision: ReviewDecision(strings.ToUpper(strings.TrimSpace(n.ReviewDecision)))} + for _, r := range n.LatestOpinionatedReviews.Nodes { + login := "" + if r.Author != nil { + login = r.Author.Login + } + switch strings.ToUpper(strings.TrimSpace(r.State)) { + case "APPROVED": + st.approvals++ + case "CHANGES_REQUESTED": + if login != "" { + st.changesRequestedBy = append(st.changesRequestedBy, login) + } + } + } + out[n.Number] = st + } + if !resp.Repository.PullRequests.PageInfo.HasNextPage || resp.Repository.PullRequests.PageInfo.EndCursor == "" { + break + } + vars["cursor"] = resp.Repository.PullRequests.PageInfo.EndCursor + } + return out +} diff --git a/src/pkg/github/protection_facts_7515_test.go b/src/pkg/github/protection_facts_7515_test.go new file mode 100644 index 0000000000..98acc6dd0d --- /dev/null +++ b/src/pkg/github/protection_facts_7515_test.go @@ -0,0 +1,292 @@ +package github + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" +) + +// protectionServer is a fake forge that answers the three endpoints +// EnrichCIStatus touches: the single-PR GET (mergeability), the check-run +// listing, and the GraphQL query that carries review decisions. +type protectionServer struct { + mu sync.Mutex + graphQLCalls int + checkRuns []map[string]any + mergeableState string + reviewDecision string + reviews []map[string]any + graphQLFails bool +} + +func (p *protectionServer) start(t *testing.T) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/graphql"): + p.mu.Lock() + p.graphQLCalls++ + fails := p.graphQLFails + p.mu.Unlock() + body, _ := io.ReadAll(r.Body) + if !strings.Contains(string(body), "reviewDecision") { + t.Errorf("graphql query did not ask for reviewDecision: %s", body) + } + if fails { + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"message":"no scope"}`)) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "data": map[string]any{ + "repository": map[string]any{ + "pullRequests": map[string]any{ + "pageInfo": map[string]any{"hasNextPage": false, "endCursor": ""}, + "nodes": []map[string]any{{ + "number": 1, + "reviewDecision": p.reviewDecision, + "latestOpinionatedReviews": map[string]any{"nodes": p.reviews}, + }}, + }, + }, + }, + }) + case strings.HasSuffix(r.URL.Path, "/pulls/1"): + _ = json.NewEncoder(w).Encode(map[string]any{ + "number": 1, + "mergeable": false, + "mergeable_state": p.mergeableState, + }) + case strings.Contains(r.URL.Path, "/check-runs"): + _ = json.NewEncoder(w).Encode(map[string]any{ + "total_count": len(p.checkRuns), + "check_runs": p.checkRuns, + }) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + return srv +} + +func enrichOnePR(t *testing.T, p *protectionServer, required map[string]bool) PullRequest { + t.Helper() + srv := p.start(t) + c := newTestClient(t, srv, "org", []string{"repo"}) + if required != nil { + c.SetRequiredChecks(required) + } + prs := []PullRequest{{Repo: "org/repo", Number: 1, HeadSHA: "abc123", BaseRef: "v4"}} + c.EnrichCIStatus(context.Background(), prs) + return prs[0] +} + +// A red REQUIRED check is the commonest thing behind GitHub's "blocked", and +// the sweep already knows the required set from auto_merge.required_checks — +// no branch-protection API call is needed for this path at all. +func TestEnrichCIStatus_ProtectionNamesFailingRequiredCheck(t *testing.T) { + p := &protectionServer{ + mergeableState: "blocked", + reviewDecision: "APPROVED", + checkRuns: []map[string]any{ + {"name": "build", "status": "completed", "conclusion": "failure"}, + {"name": "lint", "status": "completed", "conclusion": "success"}, + }, + } + pr := enrichOnePR(t, p, map[string]bool{"build": true, "lint": true}) + if pr.Protection == nil { + t.Fatal("Protection is nil; the blocked pill has nothing to say") + } + if !pr.Protection.RequiredChecksKnown { + t.Error("RequiredChecksKnown = false, want true (the config set is installed)") + } + got, ok := pr.BranchProtectionBlockReason() + if !ok || got != `required check "build" is failing` { + t.Errorf("reason = %q (ok=%v), want `required check \"build\" is failing`", got, ok) + } +} + +// A required check that never produced a check run is "expected" in GitHub's +// vocabulary and invisible in every REST field the sweep reads. +func TestEnrichCIStatus_ProtectionNamesRequiredCheckThatNeverReported(t *testing.T) { + p := &protectionServer{ + mergeableState: "blocked", + reviewDecision: "APPROVED", + checkRuns: []map[string]any{ + {"name": "build", "status": "completed", "conclusion": "success"}, + }, + } + pr := enrichOnePR(t, p, map[string]bool{"build": true, "validate": true}) + if pr.Protection == nil { + t.Fatal("Protection is nil") + } + got, ok := pr.BranchProtectionBlockReason() + if !ok || got != `required check "validate" has not reported` { + t.Errorf("reason = %q (ok=%v), want `required check \"validate\" has not reported`", got, ok) + } +} + +// The guard that keeps this honest: on a repository whose required contexts +// arrive as COMMIT STATUSES, not check runs, every required context is absent +// from the check-run listing. Claiming all of them "never reported" would be +// confidently wrong, so nothing at all is claimed. +func TestEnrichCIStatus_NoRequiredCheckObservedMakesNoMissingClaim(t *testing.T) { + p := &protectionServer{ + mergeableState: "blocked", + reviewDecision: "APPROVED", + checkRuns: []map[string]any{ + {"name": "some-unrelated-check", "status": "completed", "conclusion": "success"}, + }, + } + pr := enrichOnePR(t, p, map[string]bool{"validate": true, "dco": true}) + if pr.Protection == nil { + t.Fatal("Protection is nil") + } + if len(pr.Protection.MissingRequiredChecks) != 0 { + t.Errorf("MissingRequiredChecks = %v, want none: no required context was seen as a check run, so their absence proves nothing", pr.Protection.MissingRequiredChecks) + } + if got, ok := pr.BranchProtectionBlockReason(); ok { + t.Errorf("reason = %q, want none (nothing supports a claim here)", got) + } +} + +// A check-run listing that could not be fetched must never be read as "the +// required check never reported". +func TestEnrichCIStatus_CheckRunFetchFailureMakesNoCheckClaim(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/graphql"): + _ = json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{ + "repository": map[string]any{"pullRequests": map[string]any{ + "pageInfo": map[string]any{"hasNextPage": false}, + "nodes": []map[string]any{{ + "number": 1, "reviewDecision": "APPROVED", + "latestOpinionatedReviews": map[string]any{"nodes": []map[string]any{}}, + }}, + }}, + }}) + case strings.HasSuffix(r.URL.Path, "/pulls/1"): + _ = json.NewEncoder(w).Encode(map[string]any{"number": 1, "mergeable_state": "blocked"}) + default: + w.WriteHeader(http.StatusInternalServerError) + } + })) + defer srv.Close() + c := newTestClient(t, srv, "org", []string{"repo"}) + c.SetRequiredChecks(map[string]bool{"validate": true}) + prs := []PullRequest{{Repo: "org/repo", Number: 1, HeadSHA: "abc123", BaseRef: "v4"}} + c.EnrichCIStatus(context.Background(), prs) + + if prs[0].Protection != nil && prs[0].Protection.RequiredChecksKnown { + t.Error("RequiredChecksKnown = true with no check-run listing; absence of evidence is not evidence") + } + if got, ok := prs[0].BranchProtectionBlockReason(); ok { + t.Errorf("reason = %q, want none", got) + } +} + +// GitHub's own review verdict is the only signal that separates "nobody has +// reviewed" from "somebody said no", and REST exposes neither. +func TestEnrichCIStatus_ProtectionCarriesReviewDecision(t *testing.T) { + t.Run("changes requested names the reviewer", func(t *testing.T) { + p := &protectionServer{ + mergeableState: "blocked", + reviewDecision: "CHANGES_REQUESTED", + reviews: []map[string]any{ + {"state": "CHANGES_REQUESTED", "author": map[string]any{"login": "octo"}}, + }, + checkRuns: []map[string]any{{"name": "build", "status": "completed", "conclusion": "success"}}, + } + pr := enrichOnePR(t, p, nil) + if pr.Protection == nil || pr.Protection.ReviewDecision != ReviewDecisionChangesRequested { + t.Fatalf("ReviewDecision not carried: %+v", pr.Protection) + } + got, ok := pr.BranchProtectionBlockReason() + if !ok || got != "changes requested by @octo" { + t.Errorf("reason = %q (ok=%v), want %q", got, ok, "changes requested by @octo") + } + }) + + t.Run("review required counts the approvals given", func(t *testing.T) { + p := &protectionServer{ + mergeableState: "blocked", + reviewDecision: "REVIEW_REQUIRED", + reviews: []map[string]any{ + {"state": "APPROVED", "author": map[string]any{"login": "one"}}, + }, + checkRuns: []map[string]any{{"name": "build", "status": "completed", "conclusion": "success"}}, + } + pr := enrichOnePR(t, p, nil) + if pr.Protection == nil { + t.Fatal("Protection is nil") + } + if pr.Protection.ApprovalsGiven != 1 { + t.Errorf("ApprovalsGiven = %d, want 1", pr.Protection.ApprovalsGiven) + } + got, ok := pr.BranchProtectionBlockReason() + want := "an approving review is required by branch protection (1 given)" + if !ok || got != want { + t.Errorf("reason = %q (ok=%v), want %q", got, ok, want) + } + }) +} + +// A forge that will not answer the GraphQL query (no scope, an older GHE) +// must leave the review decision undetermined — never "approved" — and must +// not be asked again for every PR in the repository. +func TestEnrichCIStatus_ReviewDecisionFailureIsOnePerRepoAndNotApproved(t *testing.T) { + p := &protectionServer{ + mergeableState: "blocked", + graphQLFails: true, + checkRuns: []map[string]any{{"name": "build", "status": "completed", "conclusion": "success"}}, + } + srv := p.start(t) + c := newTestClient(t, srv, "org", []string{"repo"}) + prs := []PullRequest{ + {Repo: "org/repo", Number: 1, HeadSHA: "abc123", BaseRef: "v4"}, + {Repo: "org/repo", Number: 1, HeadSHA: "abc123", BaseRef: "v4"}, + {Repo: "org/repo", Number: 1, HeadSHA: "abc123", BaseRef: "v4"}, + } + c.EnrichCIStatus(context.Background(), prs) + + p.mu.Lock() + calls := p.graphQLCalls + p.mu.Unlock() + if calls != 1 { + t.Errorf("graphQL called %d times for one repository, want 1: a per-PR query is unaffordable at a 400-PR queue", calls) + } + for i := range prs { + if prs[i].Protection != nil && prs[i].Protection.ReviewDecision == ReviewDecisionApproved { + t.Errorf("pr %d: an unanswerable query became %q", i, ReviewDecisionApproved) + } + } +} + +// One GraphQL query serves every PR in the repository. +func TestEnrichCIStatus_ReviewDecisionIsOneQueryPerRepo(t *testing.T) { + p := &protectionServer{ + mergeableState: "blocked", + reviewDecision: "REVIEW_REQUIRED", + checkRuns: []map[string]any{{"name": "build", "status": "completed", "conclusion": "success"}}, + } + srv := p.start(t) + c := newTestClient(t, srv, "org", []string{"repo"}) + prs := make([]PullRequest, 5) + for i := range prs { + prs[i] = PullRequest{Repo: "org/repo", Number: 1, HeadSHA: "abc123", BaseRef: "v4"} + } + c.EnrichCIStatus(context.Background(), prs) + + p.mu.Lock() + calls := p.graphQLCalls + p.mu.Unlock() + if calls != 1 { + t.Errorf("graphQL called %d times for 5 PRs in one repository, want 1", calls) + } +} diff --git a/src/pkg/github/token_access_audit.go b/src/pkg/github/token_access_audit.go index 61dac73f7d..5f6d5a08d3 100644 --- a/src/pkg/github/token_access_audit.go +++ b/src/pkg/github/token_access_audit.go @@ -11,6 +11,40 @@ import ( "time" ) +// Token-access audit trail (#6287). +// +// THE DEFECT: GET /api/token-access is an operator-only security audit trail +// ("every gh CLI command an agent issued", gated at owner role, CWE-284) and +// it was fed by the audited parties writing straight into the log file. +// bin/gh-wrapper.sh and bin/git-credential-hive.sh run AS the agent UID and +// used to `>>` the JSONL file directly, which only works when the file is +// writable by every agent (v4 pre-created it dev:node 0664, and every agent's +// primary group is node). Append is indistinguishable from write at the file +// permission level, so any prompt-injected agent could truncate the trail of +// its own token use, rewrite lines to attribute its activity to a peer, or +// forge entries wholesale. An audit log the audited party can edit is not an +// audit log. +// +// THE FIX: the same drop-box relay every other agent->hive write already uses +// (pr-requests, issue-requests, merge-requests). The wrappers write one JSON +// event per gh call / credential lookup into TokenAccessSpoolDir, a directory +// agents can create files in but cannot list, and only the file's owner (or +// the hive, which owns the directory) can unlink. The hive process ingests +// those events into TokenAccessLogPath, which is owned by the hive UID with +// mode 0600: no agent can open it for writing, truncating, or appending, and +// no agent can read the fleet-wide history either (the endpoint is owner-only +// for exactly that reason). While ingesting, the hive replaces the event's +// self-reported "uid" with the spool file's OWNING uid, the same trust anchor +// the PR-request watcher uses: an agent can only ever speak for itself, so an +// entry forged in a peer's name lands attributed to the forger, with the +// claimed uid preserved alongside as evidence. +// +// What this does NOT close: an agent can still delete its OWN event during +// the poll window before the hive picks it up (a hive-side unix socket with +// SO_PEERCRED would close that too, and is the natural next step), and it can +// still flood the log with real gh calls. Neither lets it alter a line that +// has already been ingested, which is the invariant the tests assert. + // TokenAccessSpoolDir is where the per-agent wrappers drop token-access // events. Mirrors PRRequestDir. Changing it requires changing the constant in // bin/gh-wrapper.sh and bin/git-credential-hive.sh, which deliberately carry diff --git a/src/pkg/github/token_access_audit_test.go b/src/pkg/github/token_access_audit_test.go index 3f61b33fe1..8db68eef72 100644 --- a/src/pkg/github/token_access_audit_test.go +++ b/src/pkg/github/token_access_audit_test.go @@ -9,6 +9,8 @@ import ( "strings" "testing" "time" + + "github.com/hivecommons/hive/internal/testutil" ) // #6287: the token-access audit log must not be writable by the agents it @@ -226,16 +228,10 @@ func TestTokenAccessAudit_WatcherIngestsAndStops(t *testing.T) { done := StartTokenAccessAuditWatcher(ctx, quietLogger()) dropTokenAccessEvent(t, spool, "1-evt.json", `{"op":"gh","cmd":"gh issue list"}`) - deadline := time.Now().Add(5 * time.Second) - for { - if data, err := os.ReadFile(logPath); err == nil && strings.Contains(string(data), "gh issue list") { - break - } - if time.Now().After(deadline) { - t.Fatal("watcher never ingested the event") - } - time.Sleep(5 * time.Millisecond) - } + testutil.Eventually(t, 5*time.Second, func() bool { + data, err := os.ReadFile(logPath) + return err == nil && strings.Contains(string(data), "gh issue list") + }, "watcher never ingested the event") cancel() select { case <-done: diff --git a/src/pkg/hub/contribute_ingress_reconcile.go b/src/pkg/hub/contribute_ingress_reconcile.go new file mode 100644 index 0000000000..592c6ff4be --- /dev/null +++ b/src/pkg/hub/contribute_ingress_reconcile.go @@ -0,0 +1,263 @@ +package hub + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" +) + +// hive-contribute Ingress reconcile — closes the pre-#7457 auth-url drift. +// +// #7457 fixed the 401 on /api/contribute/me (#7453) in two halves: code on the +// hub and the spoke, and two nginx annotations on the hive-contribute Ingress +// in k8sManifestTemplate — auth-url, so nginx asks the hub who is calling, and +// auth-response-headers, so the answer (X-Hive-User / X-Hive-Role / +// X-Hive-Proxy-Auth) is copied onto the request. The code half rolled out with +// the next image; the Ingress half is `kubectl apply`ed ONLY by provisionHive, +// so every hosted spoke provisioned before it merged still serves +// /api/contribute through an Ingress with no auth-url. nginx never asks, the +// identity never arrives, and /api/contribute/me answers 401 to the hive's own +// owner — the exact symptom #7453 was filed on, now on a spoke whose +// served_sha is well past the fix (hivecommons/hive#7517). +// +// This sweep is the missing re-apply: for every hosted spoke on an nginx +// cluster it reads the live hive-contribute Ingress and merge-patches the two +// annotations on when they are absent or stale. It follows the NET_ADMIN and +// per-hive-env reconciles (netadmin_reconcile.go, perhive_env_reconcile.go), +// with two differences worth naming: an annotation patch does not roll the +// spoke's pod, so there is no per-cycle cap; and the expected values are +// derived from the same inputs the template renders from (hubPublicURL and +// the hive ID), with a test that pins them equal to the rendered Ingress so +// the sweep and the template cannot disagree about what "converged" means. +// +// More generally (from the issue): a change to k8sManifestTemplate that alters +// an existing object reaches only spokes provisioned after it, unless it has +// a reconcile path like this one. src/docs/security-model.md says so. + +const ( + // contributeIngressName is the Ingress that routes /api/contribute on a + // hosted spoke (k8sManifestTemplate, `name: hive-contribute`). + contributeIngressName = "hive-contribute" + + // The two nginx annotations #7457 added to that Ingress. Named here so + // the sweep's check and its patch can never disagree about the keys. + ingressAuthURLAnnotation = "nginx.ingress.kubernetes.io/auth-url" + ingressAuthResponseHeadersAnnotation = "nginx.ingress.kubernetes.io/auth-response-headers" + + // ingressAuthResponseHeaders is the header list every gated Ingress in the + // template forwards; hive-contribute carries the same one. + ingressAuthResponseHeaders = "X-Hive-User,X-Hive-Role,X-Hive-Proxy-Auth" + + // contributeIngressReconcileInterval throttles the sweep. The drift is + // static — an Ingress either carries the annotations or it does not, and + // a correctly provisioned one never loses them — so this is remediation, + // not a hot path. Same window as the NET_ADMIN sweep. + contributeIngressReconcileInterval = 15 * time.Minute + + // contributeIngressKubectlTimeout bounds each per-hive get/patch so one + // unreachable cluster cannot stall the whole sweep. + contributeIngressKubectlTimeout = 15 * time.Second +) + +// contributeIngressAuthAnnotations is what the hive-contribute Ingress of +// hive `hiveID` must carry for /api/contribute/me to learn who is calling: +// the per-hive auth-url and the identity headers nginx copies back. These are +// the values k8sManifestTemplate renders from the same two inputs +// (TestContributeIngressReconcileMatchesTheTemplate pins that), so a spoke the +// sweep converges is indistinguishable from one provisioned after #7457. +func contributeIngressAuthAnnotations(hubURL, hiveID string) map[string]string { + return map[string]string{ + ingressAuthURLAnnotation: hubURL + "/api/saas/auth-check?hive=" + hiveID + "&uri=$request_uri", + ingressAuthResponseHeadersAnnotation: ingressAuthResponseHeaders, + } +} + +// contributeIngressAnnotationPatch is the PURE reconcile decision: given the +// live Ingress as `kubectl get ingress -o json` prints it and the annotations +// it must carry, return the strategic-merge patch body that installs the +// missing or stale ones — or "" when the Ingress already carries every one, +// so a converged spoke is never patched. Only the annotations that differ are +// in the patch; a merge patch on metadata.annotations leaves every other key +// (cert-manager's issuer, the proxy timeouts, a vanity-host mirror's marks) +// untouched. +// +// An Ingress that does not parse is an error rather than a patch: patching +// blind onto an object the sweep could not read is how a typo becomes a +// fleet-wide outage. +func contributeIngressAnnotationPatch(raw []byte, want map[string]string) (string, error) { + var obj struct { + Metadata struct { + Annotations map[string]string `json:"annotations"` + } `json:"metadata"` + } + if err := json.Unmarshal(raw, &obj); err != nil { + return "", fmt.Errorf("parsing ingress: %w", err) + } + drift := map[string]string{} + for key, value := range want { + if obj.Metadata.Annotations[key] != value { + drift[key] = value + } + } + if len(drift) == 0 { + return "", nil + } + body, err := json.Marshal(map[string]any{ + "metadata": map[string]any{"annotations": drift}, + }) + if err != nil { + return "", fmt.Errorf("encoding patch: %w", err) + } + return string(body), nil +} + +// contributeIngressSweepEligible reports whether a hive should be examined. +// The only status excluded is "provisioning": its Ingress is being applied +// from the template right now and is born with the annotations. Every other +// status — including the unwritten "" most steady-state hives carry, and +// "available" placeholders, which must be right BEFORE they are claimed — is +// swept. See netAdminSweepEligible for why this is not a "running" check. +func contributeIngressSweepEligible(status string) bool { + return strings.TrimSpace(status) != "provisioning" +} + +// reconcileContributeIngressIfDue runs the sweep only if +// contributeIngressReconcileInterval has elapsed. Safe to call from the +// poller loop every tick; same shape and same guarding mutex as +// reconcileNetAdminIfDue — poller-loop-only state. +func (s *HubServer) reconcileContributeIngressIfDue() { + s.clusterUnreachableMu.Lock() + due := s.lastContributeIngressReconcile.IsZero() || + time.Since(s.lastContributeIngressReconcile) >= contributeIngressReconcileInterval + if due { + s.lastContributeIngressReconcile = time.Now() + } + s.clusterUnreachableMu.Unlock() + if !due { + return + } + s.reconcileContributeIngress() +} + +// reconcileContributeIngress sweeps every hub-managed hosted hive on an nginx +// cluster and, for any whose live hive-contribute Ingress is missing the +// auth-url or auth-response-headers annotation (or carries a stale value), +// merge-patches the template's values on. Idempotent — a converged Ingress is +// a Debug-level no-op — and non-fatal on kubectl errors, which are retried +// on the next sweep. An annotation patch does not restart anything: nginx +// re-reads the Ingress and starts asking the hub on the next request. +func (s *HubServer) reconcileContributeIngress() { + hives := listSaaSHives() + hubURL := hubPublicURL() + + // Sweep accounting, so "the filter selected nobody" and "the fleet is + // converged" are different, readable outcomes (the NET_ADMIN sweep + // shipped with a filter that selected nothing and nobody could tell). + considered := 0 + skippedByStatus := 0 + skippedNoIngress := 0 + converged := 0 + patched := 0 + failures := 0 + + for _, h := range hives { + if !contributeIngressSweepEligible(h.Status) { + skippedByStatus++ + continue + } + cluster := s.clusterForHive(&h) + if cluster == nil { + continue + } + // An OpenShift-Route cluster has no nginx Ingress to annotate: the + // template renders Routes there and /api/contribute has no auth-proxy + // at all (see the IsNginxIngress conditional in k8sManifestTemplate). + if cluster.IngressType == ingressTypeOpenShiftRoute { + skippedNoIngress++ + continue + } + // A pull-only cluster is reached only by answering its outbound + // heartbeat; the hub has no kubectl path into it. + if !cluster.KubectlReachable() { + skippedNoIngress++ + continue + } + // Skip clusters the hub just failed to dial — the same suppression the + // upgrade and NET_ADMIN paths use, so one down cluster does not cost a + // timeout per hive every sweep. Recovers on the next sweep after TTL. + if s.clusterRecentlyUnreachable(cluster.ID) { + continue + } + considered++ + + ns := hostedNamespaceForHive(&h) + if ns == "" { + continue + } + + ctx, cancel := context.WithTimeout(context.Background(), contributeIngressKubectlTimeout) + raw, err := kubectlForClusterContext(ctx, cluster, "get", "ingress", contributeIngressName, + "-n", ns, "-o", "json").Output() + cancel() + if err != nil { + // Ingress missing (a spoke that predates hive-contribute entirely, + // or one mid-teardown), cluster unreachable, or a transient kubectl + // error — all non-fatal. Debug, and the next sweep retries. + s.logger.Debug("contribute ingress reconcile: could not read hive-contribute ingress", + "hive_id", h.ID, "cluster", cluster.ID, "namespace", ns, "error", err) + continue + } + s.markClusterReachable(cluster.ID) + + patch, perr := contributeIngressAnnotationPatch(raw, contributeIngressAuthAnnotations(hubURL, h.ID)) + if perr != nil { + failures++ + s.logger.Warn("contribute ingress reconcile: could not parse hive-contribute ingress — not patching blind", + "hive_id", h.ID, "cluster", cluster.ID, "namespace", ns, "error", perr) + continue + } + if patch == "" { + converged++ + s.logger.Debug("contribute ingress reconcile: hive-contribute already carries the auth-url", + "hive_id", h.ID, "cluster", cluster.ID) + continue + } + + pctx, pcancel := context.WithTimeout(context.Background(), contributeIngressKubectlTimeout) + pout, perr := kubectlForClusterContext(pctx, cluster, "patch", "ingress", contributeIngressName, + "-n", ns, "--type", "merge", "-p", patch).CombinedOutput() + pcancel() + if perr != nil { + failures++ + s.markClusterUnreachable(cluster.ID) + s.logger.Warn("contribute ingress reconcile: patch failed — will retry next sweep", + "hive_id", h.ID, "cluster", cluster.ID, "namespace", ns, + "output", strings.TrimSpace(string(pout)), "error", perr) + continue + } + s.markClusterReachable(cluster.ID) + patched++ + s.logger.Info("reconciled auth-url onto hive-contribute ingress (#7517)", + "hive_id", h.ID, "cluster", cluster.ID, "namespace", ns) + } + + // A sweep that admitted no hives at all on a hub that hosts spokes is a + // bug signal, not a quiet no-op — the condition that made the NET_ADMIN + // lane dead code for its whole production life. + if considered == 0 && len(hives) > 0 { + s.logger.Warn("contribute ingress reconcile: selected NO hives — sweep is a no-op, /api/contribute/me stays 401 on drifted spokes", + "hives_in_registry", len(hives), "skipped_by_status", skippedByStatus, + "skipped_no_nginx_ingress", skippedNoIngress) + return + } + // The per-sweep summary is Info only when the sweep changed or failed + // something, so a converged fleet stays quiet. + if patched > 0 || failures > 0 { + s.logger.Info("contribute ingress reconcile sweep complete", + "considered", considered, "converged", converged, "patched", patched, + "failures", failures, "skipped_by_status", skippedByStatus, + "skipped_no_nginx_ingress", skippedNoIngress) + } +} diff --git a/src/pkg/hub/contribute_ingress_reconcile_test.go b/src/pkg/hub/contribute_ingress_reconcile_test.go new file mode 100644 index 0000000000..bb70656e14 --- /dev/null +++ b/src/pkg/hub/contribute_ingress_reconcile_test.go @@ -0,0 +1,273 @@ +package hub + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "gopkg.in/yaml.v3" +) + +// hivecommons/hive#7517: #7457 put auth-url / auth-response-headers on the +// hive-contribute Ingress in the provisioning template, but the template is +// applied only at provision time, so every hosted spoke that already existed +// kept an Ingress with no auth-url and /api/contribute/me kept answering 401. +// These tests pin the reconcile that re-applies the two annotations: what +// "converged" means (the template's own values), what the sweep patches (only +// the drifted keys, by merge patch), what it leaves alone, and that the +// poller actually runs it. + +// The reconcile's idea of the annotations MUST be the template's: if they +// ever disagree, a spoke the sweep has "converged" is still not what a freshly +// provisioned one looks like, and the 401 comes back on a hub URL change or a +// header rename nobody remembered to mirror. Render the template with the same +// two inputs and compare byte for byte. +func TestContributeIngressReconcileMatchesTheTemplate(t *testing.T) { + for _, useWildcard := range []bool{false, true} { + blocks := ingressBlocks(t, renderManifestWildcard(t, useWildcard)) + raw, ok := blocks[contributeIngressName] + if !ok { + t.Fatalf("useWildcard=%v: no %s Ingress in the template", useWildcard, contributeIngressName) + } + var doc struct { + Metadata struct { + Annotations map[string]string `yaml:"annotations"` + } `yaml:"metadata"` + } + if err := yaml.Unmarshal([]byte(raw), &doc); err != nil { + t.Fatalf("%s does not parse: %v\n%s", contributeIngressName, err, raw) + } + // renderManifestWildcard renders ID hosted-hive-x against + // https://hive.hivecommons.dev. + want := contributeIngressAuthAnnotations("https://hive.hivecommons.dev", "hosted-hive-x") + for key, value := range want { + if got := doc.Metadata.Annotations[key]; got != value { + t.Errorf("useWildcard=%v: reconcile expects %s = %q but the template renders %q — a converged spoke would not match a provisioned one", + useWildcard, key, value, got) + } + } + } +} + +// liveIngressJSON is a hive-contribute Ingress as `kubectl get -o json` prints +// it, with whatever annotations a test wants on it. The rest of the object is +// the shape provisionHive leaves behind, so the parse is exercised on a real +// document rather than a bare annotations map. +func liveIngressJSON(t *testing.T, annotations map[string]string) []byte { + t.Helper() + obj := map[string]any{ + "apiVersion": "networking.k8s.io/v1", + "kind": "Ingress", + "metadata": map[string]any{ + "name": contributeIngressName, + "namespace": "hive-hosted-hosted-projectbluefin-common-nmq5", + "annotations": annotations, + }, + "spec": map[string]any{ + "ingressClassName": "nginx", + "rules": []any{map[string]any{ + "host": "hosted-projectbluefin-common-nmq5.hive.hivecommons.dev", + }}, + }, + } + raw, err := json.Marshal(obj) + if err != nil { + t.Fatal(err) + } + return raw +} + +func decodePatch(t *testing.T, patch string) map[string]string { + t.Helper() + var body struct { + Metadata struct { + Annotations map[string]string `json:"annotations"` + } `json:"metadata"` + } + if err := json.Unmarshal([]byte(patch), &body); err != nil { + t.Fatalf("patch %q is not a merge patch on metadata.annotations: %v", patch, err) + } + return body.Metadata.Annotations +} + +// The issue's spoke: provisioned before #7457, so the Ingress carries the +// cert-manager issuer and the proxy timeouts but no auth-url. Both annotations +// go on, and ONLY those two — the merge patch must not touch what is there. +func TestContributeIngressPatchAddsTheMissingAnnotations(t *testing.T) { + want := contributeIngressAuthAnnotations("https://hive.hivecommons.dev", "hosted-projectbluefin-common-nmq5") + pre7457 := map[string]string{ + "cert-manager.io/cluster-issuer": "letsencrypt-dns01", + "nginx.ingress.kubernetes.io/proxy-read-timeout": "3600", + "nginx.ingress.kubernetes.io/proxy-send-timeout": "3600", + } + patch, err := contributeIngressAnnotationPatch(liveIngressJSON(t, pre7457), want) + if err != nil { + t.Fatal(err) + } + if patch == "" { + t.Fatal("a pre-#7457 Ingress with no auth-url was judged converged — this is the production bug: /api/contribute/me stays 401") + } + got := decodePatch(t, patch) + if len(got) != 2 { + t.Errorf("patch touches %d annotations, want exactly the two #7457 added: %v", len(got), got) + } + if got[ingressAuthURLAnnotation] != "https://hive.hivecommons.dev/api/saas/auth-check?hive=hosted-projectbluefin-common-nmq5&uri=$request_uri" { + t.Errorf("auth-url = %q", got[ingressAuthURLAnnotation]) + } + if got[ingressAuthResponseHeadersAnnotation] != "X-Hive-User,X-Hive-Role,X-Hive-Proxy-Auth" { + t.Errorf("auth-response-headers = %q", got[ingressAuthResponseHeadersAnnotation]) + } + for key := range pre7457 { + if _, touched := got[key]; touched { + t.Errorf("patch rewrites %s, which was not drifted", key) + } + } + // No auth-signin: /api/contribute is public and fetch() could not follow + // the redirect anyway (#7457). A reconcile that added one would turn an + // anonymous leaderboard load into a cross-origin bounce. + if _, has := got["nginx.ingress.kubernetes.io/auth-signin"]; has { + t.Error("patch adds auth-signin to a public path") + } + // The value nginx needs is the literal $request_uri, not a shell- or + // template-expanded one. + if !strings.Contains(patch, `$request_uri`) { + t.Errorf("patch lost the literal $request_uri: %s", patch) + } +} + +// A spoke provisioned after #7457 (or one this sweep already fixed) is a +// no-op: no patch body, so the sweep never issues a pointless kubectl patch +// and can run every 15 minutes forever without touching a converged fleet. +func TestContributeIngressPatchIsEmptyWhenConverged(t *testing.T) { + want := contributeIngressAuthAnnotations("https://hive.hivecommons.dev", "hosted-hive-x") + live := map[string]string{ + "cert-manager.io/cluster-issuer": "letsencrypt-dns01", + "nginx.ingress.kubernetes.io/proxy-read-timeout": "3600", + } + for key, value := range want { + live[key] = value + } + patch, err := contributeIngressAnnotationPatch(liveIngressJSON(t, live), want) + if err != nil { + t.Fatal(err) + } + if patch != "" { + t.Errorf("a converged Ingress produced a patch: %s", patch) + } +} + +// Only the drifted key goes in the patch. A spoke whose auth-url points at a +// previous hub URL (or a hand-edited one) is corrected; the headers it already +// carries correctly are left out of the body. +func TestContributeIngressPatchReplacesOnlyTheStaleValue(t *testing.T) { + want := contributeIngressAuthAnnotations("https://hive.hivecommons.dev", "hosted-hive-x") + live := map[string]string{ + ingressAuthURLAnnotation: "https://hive.kubestellar.io/api/saas/auth-check?hive=hosted-hive-x&uri=$request_uri", + ingressAuthResponseHeadersAnnotation: ingressAuthResponseHeaders, + } + patch, err := contributeIngressAnnotationPatch(liveIngressJSON(t, live), want) + if err != nil { + t.Fatal(err) + } + got := decodePatch(t, patch) + if len(got) != 1 || got[ingressAuthURLAnnotation] != want[ingressAuthURLAnnotation] { + t.Errorf("patch = %v, want only the corrected auth-url", got) + } +} + +// An Ingress with no annotations at all (kubectl prints no `annotations` key) +// still gets both — the absent map must read as "everything missing", not as +// a parse failure. +func TestContributeIngressPatchHandlesNoAnnotationsAtAll(t *testing.T) { + want := contributeIngressAuthAnnotations("https://hive.hivecommons.dev", "hosted-hive-x") + patch, err := contributeIngressAnnotationPatch([]byte(`{"metadata":{"name":"hive-contribute"}}`), want) + if err != nil { + t.Fatal(err) + } + if got := decodePatch(t, patch); len(got) != 2 { + t.Errorf("patch = %v, want both annotations", got) + } +} + +// Unparseable output is an error, never a patch: the sweep must not patch +// blind onto an object it could not read. +func TestContributeIngressPatchRefusesToPatchBlind(t *testing.T) { + want := contributeIngressAuthAnnotations("https://hive.hivecommons.dev", "hosted-hive-x") + for _, raw := range []string{"", "not json", "Error from server (NotFound): ingresses.networking.k8s.io \"hive-contribute\" not found"} { + patch, err := contributeIngressAnnotationPatch([]byte(raw), want) + if err == nil || patch != "" { + t.Errorf("raw %q: patch=%q err=%v, want an error and no patch", raw, patch, err) + } + } +} + +// The values are per hive and per hub: two hives never share an auth-url +// (the hub scopes the auth-check by ?hive=), and the hub URL is whatever +// hubPublicURL() says at sweep time, not a constant. +func TestContributeIngressAuthAnnotationsArePerHive(t *testing.T) { + a := contributeIngressAuthAnnotations("https://hub.example", "hive-a") + b := contributeIngressAuthAnnotations("https://hub.example", "hive-b") + if a[ingressAuthURLAnnotation] == b[ingressAuthURLAnnotation] { + t.Error("two hives share an auth-url; the hub could not tell whose grant to check") + } + if !strings.HasPrefix(a[ingressAuthURLAnnotation], "https://hub.example/api/saas/auth-check?hive=hive-a&") { + t.Errorf("auth-url = %q", a[ingressAuthURLAnnotation]) + } +} + +// Same predicate shape as netAdminSweepEligible, for the same reason: the +// steady-state fleet carries "" and "available", not "running", and a +// placeholder must be right before it is claimed. +func TestContributeIngressSweepEligible(t *testing.T) { + for _, status := range []string{"", "available", "assigned", "running", "error", " available "} { + if !contributeIngressSweepEligible(status) { + t.Errorf("status %q is not swept, but live spokes carry it", status) + } + } + if contributeIngressSweepEligible("provisioning") { + t.Error("a hive still provisioning was selected; its Ingress is being applied from the template right now") + } +} + +// The poller-loop throttle lets the sweep run once per interval. Driven on +// the timestamp directly, as the NET_ADMIN throttle test is, rather than +// running the kubectl-shelling body. +func TestContributeIngressReconcileThrottle(t *testing.T) { + s := &HubServer{clusterUnreachableUntil: map[string]time.Time{}} + due := func() bool { + s.clusterUnreachableMu.Lock() + defer s.clusterUnreachableMu.Unlock() + d := s.lastContributeIngressReconcile.IsZero() || + time.Since(s.lastContributeIngressReconcile) >= contributeIngressReconcileInterval + if d { + s.lastContributeIngressReconcile = time.Now() + } + return d + } + if !due() { + t.Fatal("first call must be due") + } + if due() { + t.Fatal("second call inside the interval must not be due") + } + s.lastContributeIngressReconcile = time.Now().Add(-contributeIngressReconcileInterval - time.Second) + if !due() { + t.Fatal("a call after the interval must be due again") + } +} + +// The sweep exists only if the poller calls it. A reconcile that is written, +// tested and never wired is the NET_ADMIN story again (#2674), so pin the +// call site in saas_sha_poller.go next to its siblings. +func TestContributeIngressReconcileIsWiredIntoThePoller(t *testing.T) { + src, err := os.ReadFile(filepath.Join("saas_sha_poller.go")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(src), "s.reconcileContributeIngressIfDue()") { + t.Fatal("saas_sha_poller.go never calls reconcileContributeIngressIfDue(); the hive-contribute auth-url drift is never repaired") + } +} diff --git a/src/pkg/hub/saas_sha_poller.go b/src/pkg/hub/saas_sha_poller.go index f5e9286b61..3b65710b24 100644 --- a/src/pkg/hub/saas_sha_poller.go +++ b/src/pkg/hub/saas_sha_poller.go @@ -690,6 +690,14 @@ func (s *HubServer) StartLatestSHAPoller(ctx context.Context) { // rate-limited to perHiveEnvMaxPatchesPerCycle patches per cycle, because // each patch rolls that hive's pod. See perhive_env_reconcile.go. s.reconcilePerHiveEnvIfDue() + // Put #7457's auth-url / auth-response-headers onto the hive-contribute + // Ingress of every hosted spoke provisioned before that fix, so + // /api/contribute/me learns who is calling there too (#7517). The + // template is applied only at provision time; nothing else re-applies an + // Ingress annotation. Throttled internally to + // contributeIngressReconcileInterval; an annotation patch rolls no pod. + // See contribute_ingress_reconcile.go. + s.reconcileContributeIngressIfDue() // Force-delete hive-namespace pods stuck in Terminating past // orphanedPodMinAge with no finalizers and a non-Running phase — the // residue of nodes disappearing without draining (#5328). Throttled diff --git a/src/pkg/hub/server.go b/src/pkg/hub/server.go index 984009d828..a8ee0dd399 100644 --- a/src/pkg/hub/server.go +++ b/src/pkg/hub/server.go @@ -1081,6 +1081,11 @@ type HubServer struct { // Same rationale and same guarding mutex as lastNetAdminReconcile above: // both are poller-loop-only state. lastPerHiveEnvReconcile time.Time + // lastContributeIngressReconcile throttles the hive-contribute Ingress + // auth-url reconcile (contribute_ingress_reconcile.go), which puts the + // #7457 annotations onto spokes provisioned before that fix (#7517). + // Same rationale and same guarding mutex as the two above. + lastContributeIngressReconcile time.Time // lastOrphanedPodReap throttles the orphaned Terminating-pod reaper // (orphaned_pod_reaper.go), which force-deletes hive-namespace pods left // behind when a node disappears without draining (#5328). Same rationale diff --git a/src/pkg/review/dispatch.go b/src/pkg/review/dispatch.go index d6dd0f5f33..98ea572455 100644 --- a/src/pkg/review/dispatch.go +++ b/src/pkg/review/dispatch.go @@ -38,10 +38,13 @@ type DispatchOptions struct { RequireApproval bool FanOut bool MaxParallelReviews int - ReviewerAgents []string - FixerAgent string - ProjectOrg string - AIAuthor string + // MaxPerspectivesPerPR caps perspectives dispatched to one PR per cycle. + // Zero means DefaultMaxPerspectivesPerPR. + MaxPerspectivesPerPR int + ReviewerAgents []string + FixerAgent string + ProjectOrg string + AIAuthor string // PostComments carries config.ReviewConfig.PostComments into the prompt // builder, so reviewers are told to publish their verdict on the PR. PostComments bool @@ -169,6 +172,12 @@ func PlanDispatch(prs []PullRequest, artifact Artifact, state DispatchState, opt continue } limit := len(missing) + // Breadth before depth: the parallel budget is spent in PR order, so + // an uncapped first PR would take every slot for its own perspectives + // and leave the rest of the queue unreviewed this cycle. + if perPR := opts.effectiveMaxPerspectivesPerPR(); perPR > 0 && limit > perPR { + limit = perPR + } if len(reviewers) == 1 && limit > 1 { limit = 1 } @@ -293,6 +302,19 @@ func (a Artifact) AggregateFor(repo string, number int, headSHA string) (Aggrega return Aggregate{}, false } +// effectiveMaxPerspectivesPerPR returns the per-PR perspective cap, or 0 for +// "no cap". Unlimited is the default deliberately: fanning every perspective +// out at once is the review swarm's designed behavior, and a hive that wants +// depth on each PR should keep getting it. The cap is for the opposite +// situation — a queue too deep to review in depth — and is opt-in so no +// existing hive silently changes shape. +func (o DispatchOptions) effectiveMaxPerspectivesPerPR() int { + if o.MaxPerspectivesPerPR <= 0 { + return 0 + } + return o.MaxPerspectivesPerPR +} + func reviewCapableAgents(opts DispatchOptions) []AgentCapability { allowed := map[string]bool{} for _, name := range opts.ReviewerAgents { diff --git a/src/pkg/review/dispatch_test.go b/src/pkg/review/dispatch_test.go index 2448dbaede..b42d079c09 100644 --- a/src/pkg/review/dispatch_test.go +++ b/src/pkg/review/dispatch_test.go @@ -133,3 +133,84 @@ func TestConfigReviewDefaults(t *testing.T) { t.Fatalf("configured max parallel = %d, want 2", got) } } + +func dispatchPRNum(n int, sha string) PullRequest { + pr := dispatchPR(sha) + pr.Number = n + return pr +} + +// TestDispatchSpendsSlotsDepthFirstByDefault documents the status quo the cap +// exists to change: the parallel budget is spent in PR order, so the head of +// the queue absorbs every slot for its own perspectives and the PRs behind it +// get nothing this cycle. That is the right behavior when the queue is short. +func TestDispatchSpendsSlotsDepthFirstByDefault(t *testing.T) { + prs := []PullRequest{dispatchPRNum(1, "sha1"), dispatchPRNum(2, "sha2"), dispatchPRNum(3, "sha3")} + plan := PlanDispatch(prs, Artifact{}, DispatchState{}, DispatchOptions{ + RequireApproval: true, + FanOut: true, + MaxParallelReviews: 3, + ProjectOrg: "acme", + Agents: []AgentCapability{reviewer("r1"), reviewer("r2"), reviewer("r3")}, + }) + + if len(plan.ReviewKicks) != 3 { + t.Fatalf("got %d kicks, want 3 (the full slot budget)", len(plan.ReviewKicks)) + } + for _, k := range plan.ReviewKicks { + if k.Number != 1 { + t.Fatalf("uncapped dispatch should concentrate on the first PR, got a kick for #%d", k.Number) + } + } +} + +// TestMaxPerspectivesPerPRSpreadsAcrossPRs is the point of the cap: the same +// budget, spent breadth-first, reviews every PR in the queue once instead of +// one PR three times. No coverage is lost — the perspectives skipped here are +// still missing next cycle and get dispatched then. +func TestMaxPerspectivesPerPRSpreadsAcrossPRs(t *testing.T) { + prs := []PullRequest{dispatchPRNum(1, "sha1"), dispatchPRNum(2, "sha2"), dispatchPRNum(3, "sha3")} + plan := PlanDispatch(prs, Artifact{}, DispatchState{}, DispatchOptions{ + RequireApproval: true, + FanOut: true, + MaxParallelReviews: 3, + MaxPerspectivesPerPR: 1, + ProjectOrg: "acme", + Agents: []AgentCapability{reviewer("r1"), reviewer("r2"), reviewer("r3")}, + }) + + if len(plan.ReviewKicks) != 3 { + t.Fatalf("got %d kicks, want 3", len(plan.ReviewKicks)) + } + seen := map[int]int{} + for _, k := range plan.ReviewKicks { + seen[k.Number]++ + } + if len(seen) != 3 { + t.Fatalf("capped dispatch covered %d distinct PRs, want 3: %+v", len(seen), seen) + } + for num, n := range seen { + if n != 1 { + t.Errorf("PR #%d got %d perspectives, want 1 under the cap", num, n) + } + } +} + +// TestMaxPerspectivesPerPRNeverExceedsSlotBudget keeps the cap subordinate to +// the parallel budget: raising it must not let dispatch run more reviews at +// once than the hive allows. +func TestMaxPerspectivesPerPRNeverExceedsSlotBudget(t *testing.T) { + prs := []PullRequest{dispatchPRNum(1, "sha1"), dispatchPRNum(2, "sha2")} + plan := PlanDispatch(prs, Artifact{}, DispatchState{}, DispatchOptions{ + RequireApproval: true, + FanOut: true, + MaxParallelReviews: 2, + MaxPerspectivesPerPR: 4, + ProjectOrg: "acme", + Agents: []AgentCapability{reviewer("r1"), reviewer("r2")}, + }) + + if len(plan.ReviewKicks) != 2 { + t.Fatalf("got %d kicks, want 2 (the slot budget, not the per-PR cap)", len(plan.ReviewKicks)) + } +} diff --git a/src/pkg/review/prompts.go b/src/pkg/review/prompts.go index cb7f5c143d..c85617607d 100644 --- a/src/pkg/review/prompts.go +++ b/src/pkg/review/prompts.go @@ -146,5 +146,44 @@ func buildPublishInstruction(pr PullRequest) string { b.WriteString("Say nothing rather than pad. Do NOT post a comment that is only nits, only praise, or a restatement of the diff. If this perspective found nothing a human needs, skip the comment entirely and just return the JSON.\n") b.WriteString("If the PR body claims behavior the diff does not implement, say so with file:line — that gap is one of the most useful things you can report.\n") b.WriteString("Be brief and specific. One comment, at most a few findings, worst first.\n") + b.WriteString(buildRoutingInstruction(pr)) return b.String() } + +// buildRoutingInstruction is what turns a verdict into a decision. +// +// A hive that cannot merge produces reviews whose only possible outcome is a +// human acting on them. But a correct, well-cited comment buried in a queue of +// hundreds is not actionable: nothing distinguishes "a human must decide this" +// from routine review noise. Routing is therefore not a nicety on top of the +// review — it is the step that makes the review reach anyone. +func buildRoutingInstruction(pr PullRequest) string { + var b strings.Builder + b.WriteString("\nROUTING — make a needed human decision findable.\n") + b.WriteString("If your verdict is requires_human or reject, the FIRST line of the comment must be exactly:\n") + b.WriteString(" **HUMAN DECISION NEEDED** — \n") + b.WriteString("A maintainer triaging a long queue filters on that marker; without it a blocking finding reads as one more comment and is skipped.\n") + if handle := mentionableAuthor(pr.Author); handle != "" { + fmt.Fprintf(&b, "On that same line, mention @%s (the PR author) so the person who can act is notified.\n", handle) + } else { + b.WriteString("Do NOT @-mention the PR author: this PR was opened by an app or bot account, so a mention notifies nobody. The marker line is the routing.\n") + } + b.WriteString("Report the limits of your own review. If you could not judge part of this PR — missing context, an unfamiliar subsystem, an ambiguous requirement, a change you cannot test — say so plainly and use requires_human. Naming what you could not verify is more useful than a confident guess, and omitting it is how an unreviewed change gets waved through on the strength of an automated approval.\n") + return b.String() +} + +// mentionableAuthor returns the bare @-handle for a PR author when mentioning +// it would reach a person, and "" when it would not. +// +// On a hive fleet most PRs are agent-authored, so the author is an App +// ("app/") or a bot ("[bot]"). @-mentioning either notifies no one +// — it renders as a link and nothing else — while still looking to a reader +// like the review was routed somewhere. That false signal is worse than no +// mention at all, because it suggests a human is already on it. +func mentionableAuthor(author string) string { + a := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(author), "@")) + if a == "" || strings.HasSuffix(a, "[bot]") || strings.Contains(a, "/") { + return "" + } + return a +} diff --git a/src/pkg/review/publish_prompt_test.go b/src/pkg/review/publish_prompt_test.go index 2fac0685cb..3cada0aa68 100644 --- a/src/pkg/review/publish_prompt_test.go +++ b/src/pkg/review/publish_prompt_test.go @@ -111,3 +111,81 @@ func TestPromptPublishAppliesToEveryPerspective(t *testing.T) { } } } + +// TestRoutingMarkerIsPinned pins the exact marker a maintainer filters on. The +// value of the marker is entirely in its being identical across every repo and +// every reviewer; a reworded variant is unsearchable and therefore useless. +func TestRoutingMarkerIsPinned(t *testing.T) { + got := BuildPerspectivePromptOpts(PerspectiveCorrectness, testPR(), true) + + if !strings.Contains(got, "**HUMAN DECISION NEEDED**") { + t.Error("publish prompt does not pin the HUMAN DECISION NEEDED marker") + } + if !strings.Contains(got, "requires_human or reject") { + t.Error("publish prompt does not tie the marker to the blocking verdicts") + } +} + +// TestRoutingDoesNotMentionBotAuthors is the point of the author check: the +// common case on a hive is an agent-authored PR, and mentioning the App that +// opened it notifies nobody while looking like the review was routed. +func TestRoutingDoesNotMentionBotAuthors(t *testing.T) { + for _, author := range []string{ + "kubestellar-hive[bot]", + "app/kubestellar-hive", + "", + } { + pr := testPR() + pr.Author = author + got := BuildPerspectivePromptOpts(PerspectiveCorrectness, pr, true) + + if strings.Contains(got, "(the PR author) so the person who can act") { + t.Errorf("author %q: prompt asks the reviewer to mention a non-human author", author) + } + if !strings.Contains(got, "Do NOT @-mention the PR author") { + t.Errorf("author %q: prompt omits the do-not-mention instruction", author) + } + } +} + +// TestRoutingMentionsHumanAuthors is the converse: when a person opened the PR, +// the mention is the fastest path from finding to decision. +func TestRoutingMentionsHumanAuthors(t *testing.T) { + pr := testPR() + pr.Author = "@clubanderson" + got := BuildPerspectivePromptOpts(PerspectiveCorrectness, pr, true) + + if !strings.Contains(got, "mention @clubanderson (the PR author)") { + t.Error("prompt does not ask the reviewer to mention the human PR author") + } + // The leading @ must not be doubled when the author already carries one. + if strings.Contains(got, "@@") { + t.Error("prompt double-prefixed the author handle with @") + } + if strings.Contains(got, "Do NOT @-mention the PR author") { + t.Error("prompt suppressed the mention for a human author") + } +} + +// TestRoutingAsksForReviewLimits covers the capability-honesty half: a reviewer +// that cannot judge something must say so rather than approve around it. +func TestRoutingAsksForReviewLimits(t *testing.T) { + got := BuildPerspectivePromptOpts(PerspectiveCorrectness, testPR(), true) + + if !strings.Contains(got, "Report the limits of your own review") { + t.Error("publish prompt does not ask the reviewer to report what it could not judge") + } +} + +// TestRoutingIsOptInWithPublish keeps routing on the same switch as publishing: +// a hive that has not opted into comments must not have its prompt changed. +func TestRoutingIsOptInWithPublish(t *testing.T) { + got := BuildPerspectivePrompt(PerspectiveCorrectness, testPR()) + + if strings.Contains(got, "HUMAN DECISION NEEDED") { + t.Error("default prompt contains the routing marker; routing must be opt-in with publishing") + } + if strings.Contains(got, "ROUTING") { + t.Error("default prompt contains the routing block; routing must be opt-in with publishing") + } +}