From 784e7017f4f2f0495ea2dcbab13db1bbd67ba847 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 14:37:39 +0000 Subject: [PATCH 01/17] feat(github): enforce pin-only exception in merge-request relay (#16607) The merge-request relay was the last App merge path that ignored auto_merge.human_merge_pin_only. Wire the policy into the relay's human-merge-path gate: a PR whose author is in trusted_authors, whose head equals the requested head (before and after listing files), whose complete changed-file list classifies as pin replacements, whose every commit is a verified trusted-author commit, and whose action SHAs are anchored upstream is admitted without a hold. Anything else keeps the existing hold, marker comment and refusal; non-GitHub forges stay fail-closed. The provenance checks move from the automerge package into pkg/github/pin_only_provenance.go so the sweep, /approve and the relay share one implementation. Docs and config comments now state the relay enforces the exception. Closes #16265 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../added-pin-only-human-merge-relay.md | 1 + .../added-pin-only-human-merge-sweep.md | 2 +- src/cmd/hive/notifywire.go | 7 + src/docs/hive-merge.md | 23 +- src/docs/operator-reference/README.md | 10 +- src/pkg/config/automerge_pin_only.go | 4 +- src/pkg/config/config.go | 4 +- .../github/automerge/automerge_human_paths.go | 146 +---------- src/pkg/github/client.go | 5 +- src/pkg/github/human_merge_paths.go | 98 +++++++- src/pkg/github/human_merge_paths_test.go | 230 +++++++++++++++++- src/pkg/github/merge_request_watcher.go | 7 +- src/pkg/github/pin_only_provenance.go | 167 +++++++++++++ 13 files changed, 535 insertions(+), 169 deletions(-) create mode 100644 changelog.d/added-pin-only-human-merge-relay.md create mode 100644 src/pkg/github/pin_only_provenance.go diff --git a/changelog.d/added-pin-only-human-merge-relay.md b/changelog.d/added-pin-only-human-merge-relay.md new file mode 100644 index 0000000000..5f3a2181d4 --- /dev/null +++ b/changelog.d/added-pin-only-human-merge-relay.md @@ -0,0 +1 @@ +- The `hive-merge` relay now honours the `auto_merge.human_merge_pin_only` exception, with the same checks as the automerge sweep plus an exact requested-head match; anything short of a proven pin-only change by a trusted author stays held (#16265). diff --git a/changelog.d/added-pin-only-human-merge-sweep.md b/changelog.d/added-pin-only-human-merge-sweep.md index 6f68c92ee2..53de8298b1 100644 --- a/changelog.d/added-pin-only-human-merge-sweep.md +++ b/changelog.d/added-pin-only-human-merge-sweep.md @@ -1 +1 @@ -- Enforce the `auto_merge.human_merge_pin_only` exception in the automerge sweep lanes and `/hive approve` (#16266): a PR by a listed trusted author whose complete diff is pin replacements on guarded paths is no longer held for a person. Any other changed file, including an unguarded changelog or doc, keeps the hold (#16326). Repos without an enabled entry are unchanged, and the `hive-merge` relay does not apply the exception yet (#16265). +- Enforce the `auto_merge.human_merge_pin_only` exception in the automerge sweep lanes and `/hive approve` (#16266): a PR by a listed trusted author whose complete diff is pin replacements on guarded paths is no longer held for a person. Any other changed file, including an unguarded changelog or doc, keeps the hold (#16326). Repos without an enabled entry are unchanged; the `hive-merge` relay enforces the same exception as of #16265. diff --git a/src/cmd/hive/notifywire.go b/src/cmd/hive/notifywire.go index cb6b6698fa..cda8a8d6a2 100644 --- a/src/cmd/hive/notifywire.go +++ b/src/cmd/hive/notifywire.go @@ -35,6 +35,13 @@ func syncAutoMergePolicyToGitHubClient(cfg *config.Config, ghClient *github.Clie ghClient.SetMergeRequestNoCIAllowedRepos(cfg.AutoMerge.NoCIOKSet()) // The merge-request relay refuses PRs touching these paths (#11039). ghClient.SetHumanMergePaths(cfg.AutoMerge.HumanMergePaths) + pinOnly := make(map[string]github.PinOnlySpec) + for repo := range cfg.AutoMerge.HumanMergePinOnly { + if ex, enabled := cfg.AutoMerge.HumanMergePinOnlyFor(repo); enabled { + pinOnly[repo] = github.PinOnlySpec{Enabled: true, Paths: ex.Paths, Kinds: ex.Kinds, TrustedAuthors: ex.TrustedAuthors} + } + } + ghClient.SetHumanMergePinOnly(pinOnly) ghClient.SetAutoMergeMinHeadAge(cfg.AutoMerge.EffectiveMinHeadAge()) return set, ok } diff --git a/src/docs/hive-merge.md b/src/docs/hive-merge.md index 4e12d582f7..d56f26b65d 100644 --- a/src/docs/hive-merge.md +++ b/src/docs/hive-merge.md @@ -170,13 +170,12 @@ sweep lanes whatever the intent tier. If the changed-file list for a configured repo cannot be fetched completely, the merge is withheld (fail closed). See [operator-reference.md](operator-reference.md). -**Pin-only policy (enforced by the automerge sweep and `/approve`; relay pending).** +**Pin-only policy (enforced by the automerge sweep, `/approve`, and relay).** `auto_merge.human_merge_pin_only` provides the schema and conservative diff classifier for #16242 / #16262. **The automerge sweep lanes and `/approve` -enforce it (#16266); the `hive-merge` relay does not yet** (#16265) and still -requires the existing human-merge path policy. The native merge queue is entered -only after a lane's human-merge-path gate, so queued PRs inherit the exception. A sweep lane +(#16266) and the `hive-merge` relay (#16265) enforce it.** The native merge +queue is entered only after a lane's human-merge-path gate, so queued PRs inherit the exception. A sweep lane does not hold a PR on its human-merge-path matches when the repo has an enabled entry, the PR author is in `trusted_authors` (case-insensitive login match), GitHub's reported changed-file count is present and equals the fetched list, @@ -236,8 +235,10 @@ This classifier proves **syntax only**, not authenticated identity, exact-head binding, file-list completeness, semantic interpretation or review of upstream references. The sweep supplies the author (GitHub's PR author login), the verified-commit provenance of the head, file-list completeness and the -upstream anchoring of each action SHA itself (#16399); the relay integration -remains #16265; +upstream anchoring of each action SHA itself (#16399). The relay applies +the same checks through `Client.SetHumanMergePinOnly` (#16265) and additionally +requires a nonempty requested head SHA matching the PR head before and after +fetching files; attribution and safe hold release are tracked in #16269. None of the schema or classifier APIs grants permission to merge or clears a hold. Native required checks and producer bindings, freshness, current reviews, substantive @@ -245,9 +246,9 @@ objections, unrelated holds and configured soak must remain independent gates. Operators stage the key in the controller's configured `hive.yaml` seed, not in the image repository (reload behaviour is documented in the operator reference -linked above). On an image that contains #16266, reloading the seed enables -the sweep and `/approve` exception at once; the `hive-merge` relay still -ignores the key until #16265 lands, so relay merges keep holding matching PRs. +linked above). On an image that contains #16266 and #16265, reloading the seed +enables the sweep, `/approve`, and `hive-merge` relay exception at once. Non-GitHub +forge relays retain the existing fail-closed human-merge-path policy. There is **no minimum deployable version enabling this lane yet**. Utah rollout and current-head positive/negative deployment evidence remain tracked by @@ -257,8 +258,8 @@ projectbluefin/utah#1082 and #16242; projectbluefin/utah#1068 is not pre-authori `auto_merge.human_merge_pin_only`; see the [operator reference](operator-reference/README.md#human-merge-pin-only-rollout) for its configuration and deployment prerequisites. The schema/classifier -(#16262), relay admission (#16265), and sweep admission (#16266, landed) are -separate changes: this attribution change alone does **not** enable the exception. +(#16262), relay admission (#16265), and sweep admission (#16266) are +separate, landed changes: this attribution change alone does **not** enable the exception. New path-hold comments retain `` and append `` only when Hive observes that exact head without a pre-existing hold. Legacy comments and diff --git a/src/docs/operator-reference/README.md b/src/docs/operator-reference/README.md index 5a72c0fd7d..0803386188 100644 --- a/src/docs/operator-reference/README.md +++ b/src/docs/operator-reference/README.md @@ -255,7 +255,7 @@ Go callers keep reading `cfg.Agents`, `cfg.Governor.Gateways` and so on through | `config.refuse_gate_regression` | `false` | Safety-gate regression guard ([#16438](https://github.com/hivecommons/hive/issues/16438)). At boot the hive compares the effective config against any previous runtime overlay still on the PVC (`/data/hive.yaml.runtime`, an operator's `hive.yaml.runtime.bak`, `/data/hive.yaml.dashboard`) for `review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos` and `writeback.per_repo` modes. A gate the overlay carried that the effective config lacks or loosens — typically a boot from the ConfigMap seed after the overlay was moved aside — is logged once per gate as `WARN safety gate dropped: (was ) — effective config has `, listed in `/api/status` under `config.gate_regressions`, and shown as a banner at the top of Settings. `true` makes it fatal: the hive exits (code 21) with the gates named instead of running with weaker gates. Set it in the overlay itself; the ConfigMap seed is the file the regression usually comes from. Legacy `/data/hive.yaml.bak` is not consulted — it is frozen at the rename and would warn forever about gates removed on purpose. | | `auto_merge.allow_unprotected_base` | Deprecated no-op. | Accepted so older configs keep loading. `hive-merge` now merges into any protected or unprotected branch the App can write, subject to the CI-evidence gate and GitHub's own merge rules. | | `auto_merge.no_ci_ok` | Empty by default. | Explicit repo list whose zero-CI merge-request verdict may pass; failing or pending CI evidence is still enforced. | -| `auto_merge.human_merge_pin_only` | Unset (off); enforced by the automerge sweep lanes and `/approve` (#16266), **not yet by the `hive-merge` relay** (#16265). | Per-repo schema for `enabled`, `paths`, `kinds` (`action-sha`, `image-digest`, `version-pin`) and `trusted_authors`. Enabled entries require all three lists. The sweep does not place a human-merge-path hold on a PR whose author is in `trusted_authors`, whose complete diff (every changed file, guarded or not) is pin replacements, whose every commit is authored by a `trusted_authors` login with a GitHub-verified signature, and whose every new action SHA resolves in its upstream action repository (the `# vN` tag points at it, or it is on the default branch; #16399); it never clears an existing hold. See [hive-merge.md](../hive-merge.md) for grammar, restrictions and outstanding integration/deployment requirements (#16242). | +| `auto_merge.human_merge_pin_only` | Unset (off); enforced by the automerge sweep lanes and `/approve` (#16266), and by the `hive-merge` relay (#16265), which additionally requires the requested head SHA to match the PR head before and after fetching files. | Per-repo schema for `enabled`, `paths`, `kinds` (`action-sha`, `image-digest`, `version-pin`) and `trusted_authors`. Enabled entries require all three lists. The sweep does not place a human-merge-path hold on a PR whose author is in `trusted_authors`, whose complete diff (every changed file, guarded or not) is pin replacements, whose every commit is authored by a `trusted_authors` login with a GitHub-verified signature, and whose every new action SHA resolves in its upstream action repository (the `# vN` tag points at it, or it is on the default branch; #16399); it never clears an existing hold. See [hive-merge.md](../hive-merge.md) for grammar, restrictions and outstanding integration/deployment requirements (#16242). | | `auto_merge.human_merge_paths` | Unset (off). | Per-repo map of `owner/repo` to glob patterns (same syntax as `intent.guardrail_path_patterns`) for paths a person must merge. Applies to every merge lane (App self-merge, trusted-author, label-queued, merge-request relay) whatever the intent tier; a matching PR is held with one comment, and an incomplete changed-file list fails closed. Details in the sweep key table below. | | `auto_merge.human_merge_title_prefixes` | `["⚠️", "⚠"]` | Title prefixes (after trimming) that mark a PR as a breaking change a person must merge. Every auto-merge lane and the merge train skip the PR; the tracker parks it at 090 *Ready for human merge · human merge*. Set to `[]` to disable. | | `auto_merge.human_merge_labels` | `["needs-owner-merge"]` | Labels (case-insensitive) that mark a PR a person must merge; same effect as the title prefixes. Set to `[]` to disable. | @@ -474,7 +474,7 @@ advanced into the merge path. | `auto_merge.repos..required_checks` | inherits default | Per-repo override for heterogeneous hives. `` may be the bare name from `project.repos` or `owner/repo`; an explicit empty list declares that GitHub has no required contexts, while Hive still requires positive CI evidence from at least one real successful check-run before merge. | | `auto_merge.allow_unprotected_base` | deprecated no-op | Accepted for compatibility only. [`hive-merge`](../hive-merge.md) no longer refuses solely because a base branch has no GitHub branch protection; it may merge into any branch the App can write after positive CI evidence. | | `auto_merge.no_ci_ok` | unset (refuse) | **Merge-request watcher key, not a sweep key.** Per-repo opt-in that downgrades only the "unverified" CI verdict (zero statuses, check runs, and workflow runs) to green, for adopted repos with no CI by design. Red and pending verdicts are never downgraded (#6281). See [hive-merge.md](../hive-merge.md). | -| `auto_merge.human_merge_paths` | unset | Per-repo map of `owner/repo` to glob patterns (same syntax as `intent.guardrail_path_patterns`) for paths a person must merge, e.g. `Danathar/goodreads-mcp: [".claude/settings.json", ".claude/hooks/**"]`. Every App merge path honors it: the automerge sweep lanes (App self-merge, trusted-author, label-queued) and the agent merge-request relay refuse a PR that touches a listed path, add `hold`, and post one `` comment naming the path(s); a person must merge it. The sweep lanes skip that hold only for a PR admitted by `auto_merge.human_merge_pin_only` (#16266); the relay does not apply that exception yet. If the changed-file list cannot be fetched completely for a configured repo, the merge is withheld (#11038, #11039). | +| `auto_merge.human_merge_paths` | unset | Per-repo map of `owner/repo` to glob patterns (same syntax as `intent.guardrail_path_patterns`) for paths a person must merge, e.g. `Danathar/goodreads-mcp: [".claude/settings.json", ".claude/hooks/**"]`. Every App merge path honors it: the automerge sweep lanes (App self-merge, trusted-author, label-queued) and the agent merge-request relay refuse a PR that touches a listed path, add `hold`, and post one `` comment naming the path(s); a person must merge it. The sweep lanes and GitHub relay skip that hold only for a PR admitted by `auto_merge.human_merge_pin_only` (#16266, #16265); the relay also requires the exact requested head before and after fetching files. If the changed-file list cannot be fetched completely for a configured repo, the merge is withheld (#11038, #11039). | | `auto_merge.human_merge_title_prefixes` | `["⚠️", "⚠"]` | PR-title prefixes that mean "breaking change — owner merges" (U+26A0 with or without the VS16 selector by default, matched after trimming leading whitespace). One shared predicate, `automerge.RequiresHumanMerge(pr)`, runs where each lane computes eligibility — App self-merge, label-queued, trusted-author and contributor lanes, plus the refetch before a merge — so a matching PR is never merged by hive and never fronts a merge train, however green and however approved. The PR parks at stage 090 *Ready for human merge* with the `human merge` chip and reason *breaking change — owner merges*; the owner merges it with **Approve & merge as maintainer** or on GitHub. Hive posts one writeback-gated comment saying it will wait for a human. Set to `[]` to turn the title rule off. Added after a ⚠️-titled PR was auto-merged by the trusted-author lane while its owner was still deciding whether to ship it. | | `auto_merge.human_merge_labels` | `["needs-owner-merge"]` | Labels (case-insensitive, trimmed) that carry the same "a person merges this" meaning as the title prefixes above; either match is enough. Set to `[]` to turn the label rule off. | | `auto_merge.require_current_base` | unset | Repo list (`owner/repo`) with an explicit **on** override for the current-base merge gate (Settings → Repos → "Merge only on current base"). On a gated repo the App self-merge, trusted-author and contributor lanes only merge a head that contains the current base tip. A green head tested against an older base is updated with GitHub update-branch (a merge commit; no author commit is rewritten) and merges once CI is green on the new head, so a semantic conflict with work merged since turns that PR red instead of breaking the base ([#13634](https://github.com/hivecommons/hive/issues/13634), [#15335](https://github.com/hivecommons/hive/issues/15335)). Costs one compare call per merge candidate. Listing a repo here is also **strict**: a behind head the App cannot update (fork without maintainer edits, branch protection or a missing `contents: write`) is parked (`fork-not-editable` / `update-branch`) rather than merged as tested. A dependency bot's branch (`renovate/*` head ref, or a Renovate, Dependabot or Mergeraptor author) is never updated: on a listed repo, or when GitHub itself reports it `behind`, it holds as `behind` until the bot rebases it; a repo gated only by the default merges it as tested (projectbluefin/utah#1097). Wins over `skip_current_base` and over the default. | @@ -515,8 +515,8 @@ hosted spoke changes survive restarts without editing `hive.yaml`. ### Human-merge pin-only rollout -**Availability:** the automerge sweep lanes and `/approve` enforce the -exception (#16266); the relay does not yet. Minimum +**Availability:** the automerge sweep lanes and `/approve` (#16266) and the +GitHub relay (#16265) enforce the exception. Minimum supported version: the first deployed v5 release containing the schema/classifier (#16262), relay (#16265), sweep (#16266), and hold-release admission integration (#16269). No released version is established here; do not assume an existing @@ -525,7 +525,7 @@ digest/version and live positive/negative evidence in projectbluefin/utah#1082 before enabling that rollout. The additive configuration (schema #16262; enforced by the automerge sweep lanes -and `/approve` per #16266, relay pending #16265) is: +and `/approve` per #16266, and the GitHub relay per #16265) is: ```yaml auto_merge: diff --git a/src/pkg/config/automerge_pin_only.go b/src/pkg/config/automerge_pin_only.go index 8a2088d3f6..11cd07ce39 100644 --- a/src/pkg/config/automerge_pin_only.go +++ b/src/pkg/config/automerge_pin_only.go @@ -13,8 +13,8 @@ import ( // and commit provenance: every commit behind the head must have a trusted // author login, a committer that is GitHub itself (web-flow) or a trusted // login, and a GitHub-verified signature (#16399). It does not prove upstream -// review (separate gates handle that). The hive-merge relay does not yet apply -// it (#16265). +// review (separate gates handle that). The hive-merge relay applies the same +// policy through Client.SetHumanMergePinOnly (#16265). type PinOnlyException struct { Enabled bool `yaml:"enabled" json:"enabled"` Paths []string `yaml:"paths" json:"paths"` diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index bb3dff4b68..85a446d85d 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -8401,8 +8401,8 @@ type AutoMergeConfig struct { HumanMergePaths map[string][]string `yaml:"human_merge_paths,omitempty" json:"human_merge_paths,omitempty"` // HumanMergePinOnly holds the opt-in pin-only exception, enforced by the - // automerge sweep and /approve gates via humanMergePinOnlyExempt (#16266); - // the hive-merge relay does not apply it yet (#16265). + // automerge sweep and /approve gates (#16266), and the hive-merge relay + // (#16265). HumanMergePinOnly map[string]PinOnlyException `yaml:"human_merge_pin_only,omitempty" json:"human_merge_pin_only,omitempty"` // HumanMergeTitlePrefixes lists title prefixes (matched after trimming // the title) that mark a PR as one the owner merges by hand; no lane and diff --git a/src/pkg/github/automerge/automerge_human_paths.go b/src/pkg/github/automerge/automerge_human_paths.go index 08025015a2..a5cb507228 100644 --- a/src/pkg/github/automerge/automerge_human_paths.go +++ b/src/pkg/github/automerge/automerge_human_paths.go @@ -2,7 +2,6 @@ package automerge import ( "context" - "fmt" "strings" gh "github.com/google/go-github/v72/github" @@ -10,7 +9,6 @@ import ( hgithub "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/intent" "github.com/hivecommons/hive/pkg/stageengine/prstage" - "github.com/hivecommons/hive/pkg/stagelabels" ) // Skip reasons reported by the human-merge-path gate (#11038). @@ -122,148 +120,20 @@ func (c *Engine) humanMergePinOnlyExempt(ctx context.Context, owner, repo string return true, "all changes are approved pin replacements by verified trusted-author commits, anchored upstream" } -// pinOnlyMaxTagDepth bounds annotated-tag dereferencing (a tag of a tag). -const pinOnlyMaxTagDepth = 3 - -// pinOnlyActionPinsUpstream returns "" when every accepted action-sha pin is -// a commit of its upstream action repository. GitHub resolves a `uses:` SHA -// through the whole fork network, so a 40-hex commit pushed to any fork of -// owner/repo is a valid pin; the classifier sees syntax only. A pin with a -// version comment must be exactly what tags/ (dereferenced through -// annotated tags) points at; one without must be an ancestor of the upstream -// default branch. Any API error, 404 or mismatch fails closed with a reason. -// Image digests are content-addressed and need no such anchor. +// pinOnlyActionPinsUpstream, pinOnlyCommitsTrusted and pinOnlyTrustedLogin +// delegate to the pkg/github helpers the merge-request relay also uses +// (#16265), so the sweep, /approve and the relay apply one provenance policy +// (#16399). func (c *Engine) pinOnlyActionPinsUpstream(ctx context.Context, pins []intent.ActionPin) string { - if len(pins) == 0 { - return "" - } - ctx = hgithub.WithRESTCaller(ctx, "hive:automerge_pin_only") - for _, pin := range pins { - if why := c.pinOnlyActionPinUpstream(ctx, pin); why != "" { - return fmt.Sprintf("action %s@%s: %s", pin.Repo, stagelabels.ShortSHA(pin.SHA), why) - } - } - return "" -} - -func (c *Engine) pinOnlyActionPinUpstream(ctx context.Context, pin intent.ActionPin) string { - owner, repo, ok := strings.Cut(pin.Repo, "/") - if !ok || owner == "" || repo == "" || pin.SHA == "" { - return "upstream repository or commit unknown" - } - if pin.Version != "" { - ref, _, err := c.gh.Git.GetRef(ctx, owner, repo, "tags/"+pin.Version) - if err != nil { - return fmt.Sprintf("tag %s could not be resolved upstream: %v", pin.Version, err) - } - obj := ref.GetObject() - for depth := 0; obj.GetType() == "tag"; depth++ { - if depth == pinOnlyMaxTagDepth { - return fmt.Sprintf("tag %s nests too deeply", pin.Version) - } - tag, _, err := c.gh.Git.GetTag(ctx, owner, repo, obj.GetSHA()) - if err != nil { - return fmt.Sprintf("annotated tag %s could not be resolved upstream: %v", pin.Version, err) - } - obj = tag.GetObject() - } - if obj.GetType() != "commit" || !strings.EqualFold(obj.GetSHA(), pin.SHA) { - return fmt.Sprintf("upstream tag %s does not point at the pinned commit", pin.Version) - } - return "" - } - upstream, _, err := c.gh.Repositories.Get(ctx, owner, repo) - if err != nil { - return fmt.Sprintf("upstream repository could not be read: %v", err) - } - branch := upstream.GetDefaultBranch() - if branch == "" { - return "upstream default branch unknown" - } - // base=pin, head=default: "ahead" or "identical" means the pinned commit - // is an ancestor of (reachable from) the upstream default branch. - cmp, _, err := c.gh.Repositories.CompareCommits(ctx, owner, repo, pin.SHA, branch, &gh.ListOptions{PerPage: 1}) - if err != nil { - return fmt.Sprintf("pinned commit could not be compared with upstream %s: %v", branch, err) - } - if status := cmp.GetStatus(); status != "ahead" && status != "identical" { - return fmt.Sprintf("pinned commit is not on upstream %s (compare status %q)", branch, status) - } - return "" + return hgithub.PinOnlyActionPinsUpstream(ctx, c.gh, pins) } -func pinOnlyTrustedLogin(policy config.PinOnlyException, login string) bool { - login = strings.TrimSpace(login) - if login == "" { - return false - } - for _, trusted := range policy.TrustedAuthors { - if strings.EqualFold(strings.TrimSpace(trusted), login) { - return true - } - } - return false -} - -// pinOnlyCommitsTrusted returns "" when every commit of pr is attributed to a -// trusted author, was committed by a trusted login or by GitHub itself -// (web-flow), carries a signature GitHub verified, and the current head is -// the newest listed commit. GitHub verifies a signature against the -// committer's email only, so a verified commit proves the committer, not the -// author: a write-access principal with their own signing key can craft a -// commit whose author is a bot's noreply address and sign it as themselves -// (see pr_merge_state.go on verification reasons). Requiring the committer to -// be web-flow (GitHub-created and GitHub-signed, so GitHub set the author) or -// a trusted login (a human trusted author signing locally) closes that; a -// spoofed committer email fails verification because it is not on the -// signer's account. Any listing error, an empty or truncated list, or a head -// not in the list fails closed with a reason. func (c *Engine) pinOnlyCommitsTrusted(ctx context.Context, owner, repo string, pr *gh.PullRequest, policy config.PinOnlyException) string { - head := pr.GetHead().GetSHA() - if head == "" { - return "PR head SHA unknown" - } - ctx = hgithub.WithRESTCaller(ctx, "hive:automerge_pin_only") - all, complete, err := hgithub.ListPRRepositoryCommits(ctx, c.gh, owner, repo, pr.GetNumber()) - if err != nil { - return fmt.Sprintf("PR commits could not be listed: %v", err) - } - if !complete { - return "PR commit list too long to verify" - } - if len(all) == 0 { - return "PR has no commits to verify" - } - if all[len(all)-1].GetSHA() != head { - return "PR commit list does not end at the current head" - } - for _, rc := range all { - sha := rc.GetSHA() - if sha == "" { - return "PR commit without a SHA" - } - if login := hgithub.SafeGetLogin(rc.GetAuthor()); !pinOnlyTrustedLogin(policy, login) { - return fmt.Sprintf("commit %s author %q is not in trusted_authors", stagelabels.ShortSHA(sha), login) - } - if committer := hgithub.SafeGetLogin(rc.GetCommitter()); !pinOnlyTrustedCommitter(policy, committer) { - return fmt.Sprintf("commit %s committer %q is neither GitHub nor in trusted_authors", stagelabels.ShortSHA(sha), committer) - } - if !rc.GetCommit().GetVerification().GetVerified() { - return fmt.Sprintf("commit %s has no GitHub-verified signature", stagelabels.ShortSHA(sha)) - } - } - return "" + return hgithub.PinOnlyCommitsTrusted(ctx, c.gh, owner, repo, pr, policy.TrustedAuthors) } -// pinOnlyWebFlowLogin is the login GitHub attaches as committer to commits it -// creates and signs itself (web UI, Apps such as Dependabot and Renovate). -const pinOnlyWebFlowLogin = "web-flow" - -// pinOnlyTrustedCommitter accepts GitHub's own committer identity or a -// trusted author committing (and signing) locally; the signature check then -// binds to this identity. -func pinOnlyTrustedCommitter(policy config.PinOnlyException, login string) bool { - return strings.EqualFold(strings.TrimSpace(login), pinOnlyWebFlowLogin) || pinOnlyTrustedLogin(policy, login) +func pinOnlyTrustedLogin(policy config.PinOnlyException, login string) bool { + return hgithub.PinOnlyTrustedLogin(policy.TrustedAuthors, login) } // humanMergePathGate refuses a PR that touches a path a person must merge. diff --git a/src/pkg/github/client.go b/src/pkg/github/client.go index a02ea7755e..aa9725c2ac 100644 --- a/src/pkg/github/client.go +++ b/src/pkg/github/client.go @@ -155,8 +155,9 @@ type Client struct { mergePolicyMu sync.RWMutex allowUnprotectedBaseRepos map[string]bool noCIAllowedRepos map[string]bool - humanMergePaths map[string][]string // auto_merge.human_merge_paths; see SetHumanMergePaths - sentinelHeads sentinelHeadsSeen // head SHAs the sentinel sweep already evaluated + humanMergePaths map[string][]string // auto_merge.human_merge_paths; see SetHumanMergePaths + humanMergePinOnly map[string]PinOnlySpec // guarded by mergePolicyMu + sentinelHeads sentinelHeadsSeen // head SHAs the sentinel sweep already evaluated autoMergeMinHeadAge time.Duration mergeAlertMu sync.Mutex mergeAlertSink MergeFailureAlertSink diff --git a/src/pkg/github/human_merge_paths.go b/src/pkg/github/human_merge_paths.go index 2531bcf8e5..5082e66f73 100644 --- a/src/pkg/github/human_merge_paths.go +++ b/src/pkg/github/human_merge_paths.go @@ -13,7 +13,8 @@ import ( // Human-merge paths (#11027): auto_merge.human_merge_paths lists, per repo, // paths a person must merge. Every App merge path (the automerge sweep lanes // and the merge-request relay) refuses a PR that touches one, adds `hold`, -// and leaves one marker-stamped comment naming the path(s). +// and leaves one marker-stamped comment naming the path(s), unless the +// configured pin-only exception is proven. const ( // HumanMergePathsConfigKey is the config key named in every refusal. HumanMergePathsConfigKey = "auto_merge.human_merge_paths" @@ -192,9 +193,58 @@ func (c *Client) humanMergePathsFor(repo string) []string { return c.humanMergePaths[repoPolicyKey(c.org, repo)] } +// PinOnlySpec is the relay's config-independent pin-only exception policy. +// It only lifts the human-merge-path gate, never any other merge requirement. +type PinOnlySpec struct { + Enabled bool + Paths []string + Kinds []string + TrustedAuthors []string +} + +func (p PinOnlySpec) clone() PinOnlySpec { + p.Paths = append([]string(nil), p.Paths...) + p.Kinds = append([]string(nil), p.Kinds...) + p.TrustedAuthors = append([]string(nil), p.TrustedAuthors...) + return p +} + +// SetHumanMergePinOnly replaces the relay policy on config reload; nil clears it. +// Duplicate normalized repository keys fail closed. +func (c *Client) SetHumanMergePinOnly(policies map[string]PinOnlySpec) { + if c == nil { + return + } + normalized := make(map[string]PinOnlySpec, len(policies)) + for repo, policy := range policies { + key := repoPolicyKey(c.org, repo) + if key == "" { + continue + } + if _, duplicate := normalized[key]; duplicate { + normalized[key] = PinOnlySpec{} + continue + } + normalized[key] = policy.clone() + } + c.mergePolicyMu.Lock() + defer c.mergePolicyMu.Unlock() + c.humanMergePinOnly = normalized +} + +func (c *Client) humanMergePinOnlyFor(repo string) PinOnlySpec { + if c == nil { + return PinOnlySpec{} + } + c.mergePolicyMu.RLock() + defer c.mergePolicyMu.RUnlock() + return c.humanMergePinOnly[repoPolicyKey(c.org, repo)].clone() +} + // mergeRequestHumanMergePathRefusal checks a merge request against the // repo's human-merge paths. It returns a non-empty refusal (terminal) when -// the PR touches one — after adding `hold` and the single marker comment — +// the PR touches one without a proven pin-only exception — after adding +// `hold` and the single marker comment — // and an error when the repo has paths configured but the complete // changed-file list cannot be established (fail closed). An unconfigured // repo costs no API call. @@ -216,8 +266,52 @@ func (c *Client) mergeRequestHumanMergePathRefusal(ctx context.Context, req Merg if len(matches) == 0 { return "", nil } + if ex := c.humanMergePinOnlyFor(req.Repo); ex.Enabled { + reason := "PR author is not trusted" + if PinOnlyTrustedLogin(ex.TrustedAuthors, SafeGetLogin(pr.GetUser())) { + switch { + case req.ExpectSHA == "" || pr.GetHead().GetSHA() != req.ExpectSHA: + reason = "PR head does not match requested head" + case pr.ChangedFiles == nil || pr.GetChangedFiles() != len(files): + reason = "complete changed-file count is unavailable" + default: + reason = c.pinOnlyProvenance(ctx, owner, name, req, pr, files, matches, ex) + if reason == "" { + return "", nil + } + } + } + c.warn("merge-request watcher: pin-only exception rejected", "repo", req.Repo, "pr", req.Number, "reason", reason) + } if err := HoldForHumanMergePaths(ctx, c.client, owner, name, req.Number, matches, pr.GetHead().GetSHA()); err != nil { c.warn("merge-request watcher: human-merge-path hold failed", "repo", req.Repo, "pr", req.Number, "error", err) } return HumanMergePathReason(matches), nil } + +// pinOnlyProvenance applies the relay's share of the pin-only exception once +// the author, requested head and file count have qualified: the whole diff +// must classify as pin replacements, every commit must be a verified +// trusted-author commit, every new action SHA must be anchored upstream +// (#16399, the same checks the sweep applies), and the head must still be the +// requested SHA after the file listing. It returns "" when the PR qualifies +// and the refusal reason otherwise. +func (c *Client) pinOnlyProvenance(ctx context.Context, owner, name string, req MergeRequest, pr *gh.PullRequest, files []intent.ChangedFile, matches []string, ex PinOnlySpec) string { + ok, reason, pins := intent.ClassifyPinOnlyActions(files, matches, intent.PinOnlySpec{Paths: ex.Paths, Kinds: ex.Kinds}) + if !ok { + return reason + } + if why := PinOnlyCommitsTrusted(ctx, c.client, owner, name, pr, ex.TrustedAuthors); why != "" { + return why + } + if why := PinOnlyActionPinsUpstream(ctx, c.client, pins); why != "" { + return why + } + // The files endpoint is not SHA-addressable. Ensure the head did not + // move while paging; the eventual merge also uses req.ExpectSHA. + current, _, err := c.client.PullRequests.Get(ctx, owner, name, req.Number) + if err != nil || current.GetHead().GetSHA() != req.ExpectSHA { + return "PR head could not be confirmed after listing files" + } + return "" +} diff --git a/src/pkg/github/human_merge_paths_test.go b/src/pkg/github/human_merge_paths_test.go index cf45762c5e..28c614e65e 100644 --- a/src/pkg/github/human_merge_paths_test.go +++ b/src/pkg/github/human_merge_paths_test.go @@ -19,6 +19,16 @@ import ( type humanPathMergeFixture struct { mu sync.Mutex files []string + fileDetails []map[string]any + prPayload map[string]any + movedHead string + // commits is served for GET pulls/42/commits; nil serves one verified + // web-flow commit by alice at the head "abc" (what the pin-only exception + // needs). upstream maps "/repos//" to a payload (or + // an int status) for the action-pin anchoring reads. + commits []map[string]any + upstream map[string]any + ci *ciFixture prStatus int filesStatus int filesHits int @@ -29,7 +39,10 @@ type humanPathMergeFixture struct { func newHumanPathMergeServer(t *testing.T, f *humanPathMergeFixture) *httptest.Server { t.Helper() - green := greenFixture() + green := f.ci + if green == nil { + green = greenFixture() + } return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { f.mu.Lock() defer f.mu.Unlock() @@ -38,13 +51,26 @@ func newHumanPathMergeServer(t *testing.T, f *humanPathMergeFixture) *httptest.S case r.Method == http.MethodGet && p == "/repos/o/r/pulls/42" && f.prStatus != 0: w.WriteHeader(f.prStatus) return + case r.Method == http.MethodGet && p == "/repos/o/r/pulls/42" && f.prPayload != nil: + if f.movedHead != "" && f.filesHits > 0 { + f.prPayload["head"] = map[string]any{"sha": f.movedHead} + } + _ = json.NewEncoder(w).Encode(f.prPayload) + return + case r.Method == http.MethodGet && p == "/repos/o/r/pulls/42/commits": + commits := f.commits + if commits == nil { + commits = []map[string]any{pinOnlyTestCommit("abc", "alice", "web-flow", true)} + } + _ = json.NewEncoder(w).Encode(commits) + return case r.Method == http.MethodGet && p == "/repos/o/r/pulls/42/files": f.filesHits++ if f.filesStatus != 0 { w.WriteHeader(f.filesStatus) return } - var out []map[string]any + out := f.fileDetails for _, name := range f.files { out = append(out, map[string]any{"filename": name, "status": "modified"}) } @@ -75,6 +101,19 @@ func newHumanPathMergeServer(t *testing.T, f *humanPathMergeFixture) *httptest.S if green.serveCI(w, r) { return } + if r.Method == http.MethodGet && f.upstream != nil { + for suffix, payload := range f.upstream { + if p != "/repos/upstream/action"+suffix { + continue + } + if status, ok := payload.(int); ok { + w.WriteHeader(status) + return + } + _ = json.NewEncoder(w).Encode(payload) + return + } + } if r.Method == http.MethodPut && strings.HasSuffix(p, "/merge") { f.merges++ w.Header().Set("Content-Type", "application/json") @@ -85,12 +124,15 @@ func newHumanPathMergeServer(t *testing.T, f *humanPathMergeFixture) *httptest.S })) } -func runHumanPathMergeRequest(t *testing.T, f *humanPathMergeFixture, paths map[string][]string) (string, MergeResponse) { +func runHumanPathMergeRequest(t *testing.T, f *humanPathMergeFixture, paths map[string][]string, pinOnly ...map[string]PinOnlySpec) (string, MergeResponse) { t.Helper() srv := newHumanPathMergeServer(t, f) t.Cleanup(srv.Close) c := testMergeClient(t, srv.URL) c.SetHumanMergePaths(paths) + if len(pinOnly) > 0 { + c.SetHumanMergePinOnly(pinOnly[0]) + } dir := t.TempDir() mergeRequestDirForTest = dir @@ -227,3 +269,185 @@ func TestHumanMergePathHelpersGuardNilInputs(t *testing.T) { t.Fatalf("reason = %q", reason) } } + +// pinOnlyTestCommit builds a pulls/N/commits entry with the author login, +// committer login and GitHub signature verification the exception inspects. +func pinOnlyTestCommit(sha, author, committer string, verified bool) map[string]any { + return map[string]any{ + "sha": sha, + "author": map[string]any{"login": author}, + "committer": map[string]any{"login": committer}, + "commit": map[string]any{"verification": map[string]any{"verified": verified}}, + } +} + +const ( + pinOnlyTestOldSHA = "1111111111111111111111111111111111111111" + pinOnlyTestNewSHA = "2222222222222222222222222222222222222222" +) + +// pinOnlyActionPatch is a single-line uses: SHA bump of upstream/action with a +// # v5 version comment. +func pinOnlyActionPatch(version string) string { + suffix := "" + if version != "" { + suffix = " # " + version + } + return "@@ -1 +1 @@\n- - uses: upstream/action@" + pinOnlyTestOldSHA + suffix + "\n+ - uses: upstream/action@" + pinOnlyTestNewSHA + suffix +} + +func pinOnlyRelayFixture() *humanPathMergeFixture { + return &humanPathMergeFixture{ + prPayload: map[string]any{"number": 42, "state": "open", "user": map[string]any{"login": "alice", "type": "User"}, "head": map[string]any{"sha": "abc", "repo": map[string]any{"full_name": "o/r"}}, "base": map[string]any{"ref": "main", "repo": map[string]any{"full_name": "o/r"}}, "changed_files": 1}, + fileDetails: []map[string]any{{"filename": "Containerfile", "status": "modified", "additions": 1, "deletions": 1, "patch": "@@ -1 +1 @@\n-FROM image@sha256:" + strings.Repeat("a", 64) + "\n+FROM image@sha256:" + strings.Repeat("b", 64)}}, + } +} + +func TestMergeRequestPinOnlyException(t *testing.T) { + for _, tc := range []struct { + name string + change func(*humanPathMergeFixture, *PinOnlySpec) + wantPass bool + wantUnknown bool + }{ + {name: "trusted pin", wantPass: true}, + {name: "disabled", change: func(f *humanPathMergeFixture, ex *PinOnlySpec) { ex.Enabled = false }}, + {name: "untrusted", change: func(f *humanPathMergeFixture, ex *PinOnlySpec) { ex.TrustedAuthors = []string{"bob"} }}, + {name: "empty trust list", change: func(f *humanPathMergeFixture, ex *PinOnlySpec) { ex.TrustedAuthors = nil }}, + {name: "mixed diff", change: func(f *humanPathMergeFixture, ex *PinOnlySpec) { + f.fileDetails[0]["patch"] = strings.Replace(f.fileDetails[0]["patch"].(string), "@@ -1 +1 @@", "@@ -1 +1,2 @@", 1) + "\n+RUN echo unsafe" + f.fileDetails[0]["additions"] = 2 + }}, + {name: "unguarded extra file", change: func(f *humanPathMergeFixture, ex *PinOnlySpec) { + f.prPayload["changed_files"] = 2 + f.files = []string{"README.md"} + }}, + {name: "outside exception paths", change: func(f *humanPathMergeFixture, ex *PinOnlySpec) { ex.Paths = []string{".github/workflows/*.yml"} }}, + {name: "missing patch", change: func(f *humanPathMergeFixture, ex *PinOnlySpec) { delete(f.fileDetails[0], "patch") }}, + {name: "missing count", change: func(f *humanPathMergeFixture, ex *PinOnlySpec) { delete(f.prPayload, "changed_files") }}, + {name: "incomplete list", wantUnknown: true, change: func(f *humanPathMergeFixture, ex *PinOnlySpec) { f.prPayload["changed_files"] = 2 }}, + {name: "wrong head", change: func(f *humanPathMergeFixture, ex *PinOnlySpec) { f.prPayload["head"] = map[string]any{"sha": "other"} }}, + {name: "head moves during file fetch", change: func(f *humanPathMergeFixture, ex *PinOnlySpec) { f.movedHead = "other" }}, + {name: "commit by untrusted author", change: func(f *humanPathMergeFixture, ex *PinOnlySpec) { + f.commits = []map[string]any{pinOnlyTestCommit("abc", "mallory", "web-flow", true)} + }}, + {name: "commit by untrusted committer", change: func(f *humanPathMergeFixture, ex *PinOnlySpec) { + f.commits = []map[string]any{pinOnlyTestCommit("abc", "alice", "mallory", true)} + }}, + {name: "unverified commit", change: func(f *humanPathMergeFixture, ex *PinOnlySpec) { + f.commits = []map[string]any{pinOnlyTestCommit("abc", "alice", "web-flow", false)} + }}, + {name: "later untrusted commit", change: func(f *humanPathMergeFixture, ex *PinOnlySpec) { + f.commits = []map[string]any{pinOnlyTestCommit("aaa", "alice", "web-flow", true), pinOnlyTestCommit("abc", "mallory", "mallory", true)} + }}, + {name: "commit list not at head", change: func(f *humanPathMergeFixture, ex *PinOnlySpec) { + f.commits = []map[string]any{pinOnlyTestCommit("stale", "alice", "web-flow", true)} + }}, + {name: "empty commit list", change: func(f *humanPathMergeFixture, ex *PinOnlySpec) { + f.commits = []map[string]any{} + }}, + {name: "trusted author signing locally", wantPass: true, change: func(f *humanPathMergeFixture, ex *PinOnlySpec) { + f.commits = []map[string]any{pinOnlyTestCommit("abc", "alice", "alice", true)} + }}, + {name: "action pin anchored by upstream tag", wantPass: true, change: func(f *humanPathMergeFixture, ex *PinOnlySpec) { + pinOnlyActionCase(f, ex, "v5", map[string]any{"/git/ref/tags/v5": map[string]any{"object": map[string]any{"type": "commit", "sha": pinOnlyTestNewSHA}}}) + }}, + {name: "action pin tag points elsewhere", change: func(f *humanPathMergeFixture, ex *PinOnlySpec) { + pinOnlyActionCase(f, ex, "v5", map[string]any{"/git/ref/tags/v5": map[string]any{"object": map[string]any{"type": "commit", "sha": pinOnlyTestOldSHA}}}) + }}, + {name: "action pin tag missing upstream", change: func(f *humanPathMergeFixture, ex *PinOnlySpec) { + pinOnlyActionCase(f, ex, "v5", map[string]any{"/git/ref/tags/v5": http.StatusNotFound}) + }}, + {name: "action pin on upstream default branch", wantPass: true, change: func(f *humanPathMergeFixture, ex *PinOnlySpec) { + pinOnlyActionCase(f, ex, "", map[string]any{"": map[string]any{"default_branch": "main"}, "/compare/" + pinOnlyTestNewSHA + "...main": map[string]any{"status": "ahead"}}) + }}, + {name: "action pin off upstream default branch", change: func(f *humanPathMergeFixture, ex *PinOnlySpec) { + pinOnlyActionCase(f, ex, "", map[string]any{"": map[string]any{"default_branch": "main"}, "/compare/" + pinOnlyTestNewSHA + "...main": map[string]any{"status": "diverged"}}) + }}, + } { + t.Run(tc.name, func(t *testing.T) { + f := pinOnlyRelayFixture() + ex := PinOnlySpec{Enabled: true, Paths: []string{"Containerfile"}, Kinds: []string{"image-digest"}, TrustedAuthors: []string{"ALICE"}} + if tc.change != nil { + tc.change(f, &ex) + } + _, resp := runHumanPathMergeRequest(t, f, map[string][]string{"o/r": {"Containerfile", ".github/workflows/*.yml"}}, map[string]PinOnlySpec{"O/R": ex}) + if tc.wantPass { + if !resp.OK || f.merges != 1 || len(f.labels) != 0 || len(f.comments) != 0 { + t.Fatalf("result=%+v merges=%d labels=%v comments=%v; want merge without hold", resp, f.merges, f.labels, f.comments) + } + } else { + if resp.OK || f.merges != 0 { + t.Fatalf("result=%+v merges=%d; want refusal", resp, f.merges) + } + if tc.wantUnknown { + if !strings.Contains(resp.Error, "fail closed") || len(f.labels) != 0 || len(f.comments) != 0 { + t.Fatalf("unknown list must fail closed without a hold: %+v", resp) + } + } else if len(f.labels) != 1 || len(f.comments) != 1 || !strings.Contains(f.comments[0], HumanMergePathMarker) { + t.Fatalf("labels=%v comments=%v; want hold and marker", f.labels, f.comments) + } + } + }) + } +} + +// pinOnlyActionCase turns the fixture into a workflow action-sha bump of +// upstream/action and serves upstream for the anchoring reads. +func pinOnlyActionCase(f *humanPathMergeFixture, ex *PinOnlySpec, version string, upstream map[string]any) { + f.fileDetails = []map[string]any{{"filename": ".github/workflows/ci.yml", "status": "modified", "additions": 1, "deletions": 1, "patch": pinOnlyActionPatch(version)}} + f.upstream = upstream + ex.Paths = []string{".github/workflows/*.yml"} + ex.Kinds = []string{"action-sha"} +} + +func TestMergeRequestPinOnlyStillRequiresGreenCI(t *testing.T) { + f := pinOnlyRelayFixture() + f.ci = greenFixture() + f.ci.checks[0].conclusion = "failure" + _, resp := runHumanPathMergeRequest(t, f, map[string][]string{"o/r": {"Containerfile"}}, map[string]PinOnlySpec{"r": {Enabled: true, Paths: []string{"Containerfile"}, Kinds: []string{"image-digest"}, TrustedAuthors: []string{"alice"}}}) + if resp.OK || f.merges != 0 || len(f.labels) != 0 || len(f.comments) != 0 { + t.Fatalf("result=%+v merges=%d labels=%v comments=%v; pin exception must still fail CI", resp, f.merges, f.labels, f.comments) + } +} + +func TestSetHumanMergePinOnlyReload(t *testing.T) { + c := NewClientForTest("http://127.0.0.1:0", "o", []string{"r"}, slog.New(slog.NewTextHandler(io.Discard, nil))) + ex := PinOnlySpec{Enabled: true, Paths: []string{"Containerfile"}, Kinds: []string{"image-digest"}, TrustedAuthors: []string{"alice"}} + c.SetHumanMergePinOnly(map[string]PinOnlySpec{" O/R ": ex}) + ex.Paths[0] = "unsafe" + got := c.humanMergePinOnlyFor("r") + if !got.Enabled || got.Paths[0] != "Containerfile" { + t.Fatalf("policy not normalized or copied: %+v", got) + } + got.Paths[0] = "unsafe" + if c.humanMergePinOnlyFor("r").Paths[0] != "Containerfile" { + t.Fatal("lookup must return a copy") + } + c.SetHumanMergePinOnly(map[string]PinOnlySpec{"r": ex, "O/R": ex}) + if c.humanMergePinOnlyFor("r").Enabled { + t.Fatal("duplicate normalized keys must fail closed") + } + c.SetHumanMergePinOnly(nil) + if c.humanMergePinOnlyFor("r").Enabled { + t.Fatal("nil must clear policy") + } + var absent *Client + absent.SetHumanMergePinOnly(nil) + if absent.humanMergePinOnlyFor("r").Enabled { + t.Fatal("nil client must have no policy") + } +} + +func TestMergeRequestPinOnlyRequiresExpectedHead(t *testing.T) { + f := pinOnlyRelayFixture() + srv := newHumanPathMergeServer(t, f) + defer srv.Close() + c := testMergeClient(t, srv.URL) + c.SetHumanMergePaths(map[string][]string{"o/r": {"Containerfile"}}) + c.SetHumanMergePinOnly(map[string]PinOnlySpec{"o/r": {Enabled: true, Paths: []string{"Containerfile"}, Kinds: []string{"image-digest"}, TrustedAuthors: []string{"alice"}}}) + refusal, err := c.mergeRequestHumanMergePathRefusal(context.Background(), MergeRequest{Repo: "o/r", Number: 42}) + if err != nil || refusal == "" || len(f.labels) != 1 || len(f.comments) != 1 { + t.Fatalf("refusal=%q err=%v labels=%v comments=%v; missing requested head must hold", refusal, err, f.labels, f.comments) + } +} diff --git a/src/pkg/github/merge_request_watcher.go b/src/pkg/github/merge_request_watcher.go index c7a8d6eeaf..c7a8f0b67d 100644 --- a/src/pkg/github/merge_request_watcher.go +++ b/src/pkg/github/merge_request_watcher.go @@ -423,9 +423,10 @@ func (c *Client) handleOneMergeRequest(ctx context.Context, path string, nowFn f return } // Human-merge paths (#11039): a PR touching a path listed in - // auto_merge.human_merge_paths for its repo is never merged by the App, - // whatever agent asks. A match is terminal (the request is refused, the - // PR gets `hold` and one marker comment); an unknown changed-file list on + // auto_merge.human_merge_paths for its repo requires a person, whatever + // agent asks, unless auto_merge.human_merge_pin_only is proven (#16265). + // A match is terminal (the request is refused, the PR gets `hold` and one + // marker comment); an unknown changed-file list on // a configured repo fails closed through the bounded retry path. Checked // before the optional branch-update so a refused PR receives no push. if refusal, err := c.mergeRequestHumanMergePathRefusal(ctx, req); err != nil { diff --git a/src/pkg/github/pin_only_provenance.go b/src/pkg/github/pin_only_provenance.go new file mode 100644 index 0000000000..aa37bd7f32 --- /dev/null +++ b/src/pkg/github/pin_only_provenance.go @@ -0,0 +1,167 @@ +package github + +import ( + "context" + "fmt" + "strings" + + gh "github.com/google/go-github/v72/github" + "github.com/hivecommons/hive/pkg/intent" + "github.com/hivecommons/hive/pkg/stagelabels" +) + +// Pin-only exception provenance (#16399): once the classifier accepts a diff +// as pin replacements, every App merge path (the automerge sweep lanes, +// /approve and the merge-request relay) must also prove that every commit +// behind the head is the trusted author's and that every new action SHA is +// anchored in its upstream repository. The helpers live here so the sweep and +// the relay share one implementation and can never disagree. + +// pinOnlyMaxTagDepth bounds annotated-tag dereferencing (a tag of a tag). +const pinOnlyMaxTagDepth = 3 + +// PinOnlyWebFlowLogin is the login GitHub attaches as committer to commits it +// creates and signs itself (web UI, Apps such as Dependabot and Renovate). +const PinOnlyWebFlowLogin = "web-flow" + +// pinOnlyRESTCaller labels the REST reads the provenance checks make. +const pinOnlyRESTCaller = "hive:automerge_pin_only" + +// PinOnlyTrustedLogin reports whether login is one of trusted (case-insensitive, +// whitespace-trimmed). An empty login is never trusted. +func PinOnlyTrustedLogin(trusted []string, login string) bool { + login = strings.TrimSpace(login) + if login == "" { + return false + } + for _, t := range trusted { + if strings.EqualFold(strings.TrimSpace(t), login) { + return true + } + } + return false +} + +// PinOnlyTrustedCommitter accepts GitHub's own committer identity or a +// trusted author committing (and signing) locally; the signature check then +// binds to this identity. +func PinOnlyTrustedCommitter(trusted []string, login string) bool { + return strings.EqualFold(strings.TrimSpace(login), PinOnlyWebFlowLogin) || PinOnlyTrustedLogin(trusted, login) +} + +// PinOnlyCommitsTrusted returns "" when every commit of pr is attributed to a +// trusted author, was committed by a trusted login or by GitHub itself +// (web-flow), carries a signature GitHub verified, and the current head is +// the newest listed commit. GitHub verifies a signature against the +// committer's email only, so a verified commit proves the committer, not the +// author: a write-access principal with their own signing key can craft a +// commit whose author is a bot's noreply address and sign it as themselves +// (see pr_merge_state.go on verification reasons). Requiring the committer to +// be web-flow (GitHub-created and GitHub-signed, so GitHub set the author) or +// a trusted login (a human trusted author signing locally) closes that; a +// spoofed committer email fails verification because it is not on the +// signer's account. Any listing error, an empty or truncated list, or a head +// not in the list fails closed with a reason. +func PinOnlyCommitsTrusted(ctx context.Context, client *gh.Client, owner, repo string, pr *gh.PullRequest, trusted []string) string { + head := pr.GetHead().GetSHA() + if head == "" { + return "PR head SHA unknown" + } + ctx = WithRESTCaller(ctx, pinOnlyRESTCaller) + all, complete, err := ListPRRepositoryCommits(ctx, client, owner, repo, pr.GetNumber()) + if err != nil { + return fmt.Sprintf("PR commits could not be listed: %v", err) + } + if !complete { + return "PR commit list too long to verify" + } + if len(all) == 0 { + return "PR has no commits to verify" + } + if all[len(all)-1].GetSHA() != head { + return "PR commit list does not end at the current head" + } + for _, rc := range all { + sha := rc.GetSHA() + if sha == "" { + return "PR commit without a SHA" + } + if login := SafeGetLogin(rc.GetAuthor()); !PinOnlyTrustedLogin(trusted, login) { + return fmt.Sprintf("commit %s author %q is not in trusted_authors", stagelabels.ShortSHA(sha), login) + } + if committer := SafeGetLogin(rc.GetCommitter()); !PinOnlyTrustedCommitter(trusted, committer) { + return fmt.Sprintf("commit %s committer %q is neither GitHub nor in trusted_authors", stagelabels.ShortSHA(sha), committer) + } + if !rc.GetCommit().GetVerification().GetVerified() { + return fmt.Sprintf("commit %s has no GitHub-verified signature", stagelabels.ShortSHA(sha)) + } + } + return "" +} + +// PinOnlyActionPinsUpstream returns "" when every accepted action-sha pin is +// a commit of its upstream action repository. GitHub resolves a `uses:` SHA +// through the whole fork network, so a 40-hex commit pushed to any fork of +// owner/repo is a valid pin; the classifier sees syntax only. A pin with a +// version comment must be exactly what tags/ (dereferenced through +// annotated tags) points at; one without must be an ancestor of the upstream +// default branch. Any API error, 404 or mismatch fails closed with a reason. +// Image digests are content-addressed and need no such anchor. +func PinOnlyActionPinsUpstream(ctx context.Context, client *gh.Client, pins []intent.ActionPin) string { + if len(pins) == 0 { + return "" + } + ctx = WithRESTCaller(ctx, pinOnlyRESTCaller) + for _, pin := range pins { + if why := pinOnlyActionPinUpstream(ctx, client, pin); why != "" { + return fmt.Sprintf("action %s@%s: %s", pin.Repo, stagelabels.ShortSHA(pin.SHA), why) + } + } + return "" +} + +func pinOnlyActionPinUpstream(ctx context.Context, client *gh.Client, pin intent.ActionPin) string { + owner, repo, ok := strings.Cut(pin.Repo, "/") + if !ok || owner == "" || repo == "" || pin.SHA == "" { + return "upstream repository or commit unknown" + } + if pin.Version != "" { + ref, _, err := client.Git.GetRef(ctx, owner, repo, "tags/"+pin.Version) + if err != nil { + return fmt.Sprintf("tag %s could not be resolved upstream: %v", pin.Version, err) + } + obj := ref.GetObject() + for depth := 0; obj.GetType() == "tag"; depth++ { + if depth == pinOnlyMaxTagDepth { + return fmt.Sprintf("tag %s nests too deeply", pin.Version) + } + tag, _, err := client.Git.GetTag(ctx, owner, repo, obj.GetSHA()) + if err != nil { + return fmt.Sprintf("annotated tag %s could not be resolved upstream: %v", pin.Version, err) + } + obj = tag.GetObject() + } + if obj.GetType() != "commit" || !strings.EqualFold(obj.GetSHA(), pin.SHA) { + return fmt.Sprintf("upstream tag %s does not point at the pinned commit", pin.Version) + } + return "" + } + upstream, _, err := client.Repositories.Get(ctx, owner, repo) + if err != nil { + return fmt.Sprintf("upstream repository could not be read: %v", err) + } + branch := upstream.GetDefaultBranch() + if branch == "" { + return "upstream default branch unknown" + } + // base=pin, head=default: "ahead" or "identical" means the pinned commit + // is an ancestor of (reachable from) the upstream default branch. + cmp, _, err := client.Repositories.CompareCommits(ctx, owner, repo, pin.SHA, branch, &gh.ListOptions{PerPage: 1}) + if err != nil { + return fmt.Sprintf("pinned commit could not be compared with upstream %s: %v", branch, err) + } + if status := cmp.GetStatus(); status != "ahead" && status != "identical" { + return fmt.Sprintf("pinned commit is not on upstream %s (compare status %q)", branch, status) + } + return "" +} From e13e891b13c68ff4b2c3a2173a06d6a373986232 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 14:49:19 +0000 Subject: [PATCH 02/17] fix(crux/proxy): canary scan refuses write bodies it could not read (#16895) inspectCanaryEgress drained and closed the request body, then returned allow when the read failed, forwarding an empty, unscanned body. Fail closed instead, as enforceLinear and the oversized-body branch already do, and restore what was read so the denial path sees a well-formed request. Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-crux-proxy-canary-body-read.md | 1 + src/crux/proxy/canary_test.go | 41 +++++++++++++++++++ src/crux/proxy/github_proxy_policy.go | 8 +++- 3 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixed-crux-proxy-canary-body-read.md diff --git a/changelog.d/fixed-crux-proxy-canary-body-read.md b/changelog.d/fixed-crux-proxy-canary-body-read.md new file mode 100644 index 0000000000..1fbe4a7611 --- /dev/null +++ b/changelog.d/fixed-crux-proxy-canary-body-read.md @@ -0,0 +1 @@ +- The crux proxy's canary scan of GitHub write bodies now refuses a request whose body could not be fully read, matching the Linear gate and the oversized-body path. Before, a read error consumed the body and still let the request through, so it reached GitHub empty and unscanned. diff --git a/src/crux/proxy/canary_test.go b/src/crux/proxy/canary_test.go index 98909d297c..f60da4d5b4 100644 --- a/src/crux/proxy/canary_test.go +++ b/src/crux/proxy/canary_test.go @@ -1,6 +1,7 @@ package proxy import ( + "errors" "io" "log/slog" "net/http" @@ -140,6 +141,46 @@ func TestInspectCanaryEgressBlocksOversizedBodies(t *testing.T) { } } +// brokenBody fails partway through the read, the way a client that drops the +// connection mid-request does. +type brokenBody struct { + io.Reader + closed bool +} + +func (b *brokenBody) Read(p []byte) (int, error) { + n, err := b.Reader.Read(p) + if err == io.EOF { + return n, errors.New("unexpected EOF") + } + return n, err +} + +func (b *brokenBody) Close() error { b.closed = true; return nil } + +func TestInspectCanaryEgressDeniesUnreadableBodies(t *testing.T) { + r := newFakeCanaryScanner() + p := &GitHubProxy{logger: slog.Default()} + p.SetCanaryScanner(true, false, r, nil) + body := &brokenBody{Reader: strings.NewReader(`{"body":"partial`)} + req, _ := http.NewRequest(http.MethodPost, "https://api.github.com/repos/org/repo/issues/1/comments", nil) + req.Body = body + reason, deny, detected := p.inspectCanaryEgress("scanner", req) + if !detected || !deny || !strings.Contains(reason, "unreadable") { + t.Fatalf("got detected=%v deny=%v reason=%q, want fail-closed block", detected, deny, reason) + } + if len(r.seen) != 0 { + t.Fatalf("unreadable body was handed to the scanner") + } + if !body.closed { + t.Fatalf("original body was not closed") + } + rest, err := io.ReadAll(req.Body) + if err != nil || string(rest) != `{"body":"partial` { + t.Fatalf("request body was not restored: %q, %v", rest, err) + } +} + func TestInspectCanaryEgressCoversDirectPullCreatePath(t *testing.T) { r := newFakeCanaryScanner() tok := r.plant("quality") diff --git a/src/crux/proxy/github_proxy_policy.go b/src/crux/proxy/github_proxy_policy.go index b18a136dea..b61b8d03ab 100644 --- a/src/crux/proxy/github_proxy_policy.go +++ b/src/crux/proxy/github_proxy_policy.go @@ -267,7 +267,13 @@ func (p *GitHubProxy) inspectCanaryEgress(agentName string, req *http.Request) ( _ = req.Body.Close() } if err != nil { - return "", false, false + // Fail closed, as enforceLinear does: a body we could not fully read + // is a body we could not scan, and the original body is already + // consumed, so letting the request through would forward it empty. + req.Body = io.NopCloser(bytes.NewReader(body)) + req.ContentLength = int64(len(body)) + p.logger.Warn("ioscan canary scan: request body read failed — denying", "agent", agentName, "path", req.URL.Path) + return "ioscan canary scan body unreadable", true, true } if len(body) > maxGitHubWriteBodyScan { return "ioscan canary scan body too large", true, true From ab5de9fea32d395321ca9fa45036d2727fc9bb17 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 14:56:56 +0000 Subject: [PATCH 03/17] =?UTF-8?q?=F0=9F=90=9B=20fix(crux/proxy):=20apply?= =?UTF-8?q?=20destination=20policy=20to=20uninspected=20tunnels=20(#16785)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(crux/proxy): apply destination policy to uninspected tunnels The plain-HTTP forward path refuses non-routable destinations at dial time, but a CONNECT to a host the proxy neither inspects nor intercepts, and the transparent-TLS passthrough for such a host, still dialled whatever host:port the client named from the proxy's own uid. Route both through the same guarded dialer so the policy has one definition and one override. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> * test(crux/proxy): pin transparent-TLS refusal counts estate, not loopback handleTransparentTLS counts a refused estate-bound SNI against the caller and skips the counter for loopback; no test reached that arm, so dropping either the reason check or the countViolation call kept the suite green. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../fixed-proxy-tunnel-destination-policy.md | 1 + src/crux/proxy/github_proxy.go | 9 +- src/crux/proxy/github_proxy_connect.go | 43 ++++- src/crux/proxy/plain_forward_guard.go | 62 ++++++-- src/crux/proxy/proxy_deep2_test.go | 2 + src/crux/proxy/proxy_deep_test.go | 2 + .../proxy/tunnel_destination_guard_test.go | 149 ++++++++++++++++++ 7 files changed, 244 insertions(+), 24 deletions(-) create mode 100644 changelog.d/fixed-proxy-tunnel-destination-policy.md create mode 100644 src/crux/proxy/tunnel_destination_guard_test.go diff --git a/changelog.d/fixed-proxy-tunnel-destination-policy.md b/changelog.d/fixed-proxy-tunnel-destination-policy.md new file mode 100644 index 0000000000..82de7bf416 --- /dev/null +++ b/changelog.d/fixed-proxy-tunnel-destination-policy.md @@ -0,0 +1 @@ +- The agent proxy now applies its plain-HTTP destination policy to uninspected CONNECT and transparent-TLS tunnels as well, so all three egress legs dial only globally routable addresses; `HIVE_PROXY_ALLOW_PRIVATE_FORWARD=true` lifts it for every leg at once diff --git a/src/crux/proxy/github_proxy.go b/src/crux/proxy/github_proxy.go index 12d0b36a18..aa131f4065 100644 --- a/src/crux/proxy/github_proxy.go +++ b/src/crux/proxy/github_proxy.go @@ -198,9 +198,10 @@ type GitHubProxy struct { // allowPrivateForward snapshots HIVE_PROXY_ALLOW_PRIVATE_FORWARD at // construction (a boot-time deployment choice like proxyAdvisoryOK). The - // zero value keeps the plain-HTTP forward guard ON: non-routable - // destinations (loopback, link-local/metadata, RFC 1918, CGNAT, multicast) - // are refused on the plain path. See plain_forward_guard.go. + // zero value keeps the destination guard ON: non-routable destinations + // (loopback, link-local/metadata, RFC 1918, CGNAT, multicast) are refused + // on the plain-HTTP forward path and on uninspected CONNECT/transparent + // tunnels. See plain_forward_guard.go. allowPrivateForward bool // injectGHAuth snapshots config.ProxyInjectGHAuth() at construction (#1861, @@ -365,7 +366,7 @@ func newGitHubProxyWithCA(caCert tls.Certificate, caX509 *x509.Certificate, logg // dial the private estate on an agent's behalf. allowPrivateForward := plainForwardAllowPrivate() if allowPrivateForward { - logger.Warn("proxy: plain-HTTP forwards to non-routable destinations are ALLOWED — agents can reach loopback, link-local (cloud metadata) and private-network services through this proxy", + logger.Warn("proxy: plain-HTTP forwards and uninspected tunnels to non-routable destinations are ALLOWED — agents can reach loopback, link-local (cloud metadata) and private-network services through this proxy", "env", plainForwardAllowPrivateEnv) } diff --git a/src/crux/proxy/github_proxy_connect.go b/src/crux/proxy/github_proxy_connect.go index a9c2e23f0a..ab26876bae 100644 --- a/src/crux/proxy/github_proxy_connect.go +++ b/src/crux/proxy/github_proxy_connect.go @@ -168,6 +168,12 @@ func (p *GitHubProxy) handleTransparentTLS(conn net.Conn, peeked []byte) { // so the forced-egress redirect exempts this proxy-originated dial. upstream, err := p.dialTunnelUpstream(host) if err != nil { + var denied *plainForwardDenied + if errors.As(err, &denied) && denied.reason != plainForwardReasonLoopback { + p.logger.Warn("proxy: transparent tunnel refused — destination is not globally routable", + "agent", agentName, "host", host, "target", denied.target, "class", denied.reason) + p.countViolation(agentName) + } return } defer func() { _ = upstream.Close() }() @@ -268,12 +274,15 @@ func (p *GitHubProxy) attributeTransparentConn(conn net.Conn) string { // dialTunnelUpstream opens the raw TCP leg of a non-inspected transparent // tunnel. The socket is SO_MARKed so the forced-egress redirect exempts this -// proxy-originated dial. tunnelDial, when set, overrides it (tests). +// proxy-originated dial, and the destination is guarded like every other +// uninspected egress leg (see plain_forward_guard.go): the host here comes +// from the client's SNI, which names whatever the client chose. tunnelDial, +// when set, overrides it (tests). func (p *GitHubProxy) dialTunnelUpstream(host string) (net.Conn, error) { if p.tunnelDial != nil { return p.tunnelDial(host) } - return markDialer(transparentProxyTimeout).Dial("tcp", host+":443") + return p.tunnelDialer(transparentProxyTimeout).Dial("tcp", host+":443") } const tlsClientHelloMaxSize = 4096 @@ -598,9 +607,17 @@ func (p *GitHubProxy) logClientRead(msg string, err error, attrs ...any) { func (p *GitHubProxy) tunnelDirect(conn net.Conn, r *http.Request) { const tunnelDialTimeout = 10 * time.Second // SO_MARK the socket so the forced-egress redirect exempts this - // proxy-originated tunnel dial (CONNECT targets include :443). - upstream, err := markDialer(tunnelDialTimeout).Dial("tcp", r.Host) + // proxy-originated tunnel dial (CONNECT targets include :443). The dialer + // is destination-guarded like the plain forward path: a CONNECT names an + // arbitrary host:port, and without the guard it was a raw TCP tunnel from + // the proxy's uid to the cloud metadata service or the cluster estate. + upstream, err := p.tunnelDialer(tunnelDialTimeout).Dial("tcp", r.Host) if err != nil { + var denied *plainForwardDenied + if errors.As(err, &denied) { + p.refuseTunnel(conn, r, p.identifyAgentFromConn(conn, r), denied) + return + } p.logger.Warn("proxy: CONNECT dial failed", "host", r.Host, "error", err) _, _ = fmt.Fprintf(conn, "HTTP/1.1 502 Bad Gateway\r\n\r\nconnection failed\n") return @@ -612,6 +629,24 @@ func (p *GitHubProxy) tunnelDirect(conn net.Conn, r *http.Request) { relayTunnel(conn, upstream) } +// refuseTunnel answers a CONNECT whose destination the guard barred, with the +// same loopback-vs-estate split as refusePlainForward: loopback is a client +// that ignored NO_PROXY and is logged at Debug without a violation; every +// other class is counted against the caller. +func (p *GitHubProxy) refuseTunnel(conn net.Conn, r *http.Request, agentName string, denied *plainForwardDenied) { + if denied.reason == plainForwardReasonLoopback { + p.logger.Debug("proxy: CONNECT to loopback refused (clients must bypass the proxy for loopback; NO_PROXY)", + "agent", agentName, "host", r.Host, "target", denied.target) + } else { + p.logger.Warn("proxy: CONNECT refused — destination is not globally routable", + "agent", agentName, "host", r.Host, "target", denied.target, "class", denied.reason) + p.countViolation(agentName) + } + _, _ = fmt.Fprintf(conn, "HTTP/1.1 403 Forbidden\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n"+ + "the hive proxy does not tunnel to %s addresses (%s); set %s=true to allow it\n", + denied.reason, denied.target, plainForwardAllowPrivateEnv) +} + // forwardPlainDirect handles non-CONNECT (plain HTTP) requests on a raw // connection. // diff --git a/src/crux/proxy/plain_forward_guard.go b/src/crux/proxy/plain_forward_guard.go index f2628b4af9..1ef80f0ef7 100644 --- a/src/crux/proxy/plain_forward_guard.go +++ b/src/crux/proxy/plain_forward_guard.go @@ -9,17 +9,22 @@ import ( "os" "strings" "syscall" + "time" ) -// The plain-HTTP (non-CONNECT) forward path is the one egress path through +// The plain-HTTP (non-CONNECT) forward path and the uninspected tunnel paths +// (a CONNECT to a host the proxy neither inspects nor intercepts, and the +// transparent-TLS passthrough for such a host) are the egress paths through // this proxy that carried NO destination policy at all: handleConn hands every -// non-CONNECT request to forwardPlainDirect, which RoundTripped it unchanged. -// An agent that sets a proxy URL by hand — or simply asks its CLI for -// `http://` — therefore had the proxy open a connection on its -// behalf, from the PROXY's uid, to any address the agent named. The forced -// egress gate does not cover it: the iptables rule only redirects :443, so -// plain HTTP is reachable only through this path, and the proxy's own uid is -// exempted from the redirect precisely so its upstream dials work. +// non-CONNECT request to forwardPlainDirect, which RoundTripped it unchanged, +// and tunnelDirect dialled whatever host:port the CONNECT line named. An agent +// that sets a proxy URL by hand — or simply asks its CLI for +// `http://`, or sends `CONNECT 169.254.169.254:80` — therefore had +// the proxy open a connection on its behalf, from the PROXY's uid, to any +// address the agent named. The forced egress gate does not cover it: the +// iptables rule only redirects :443, so plain HTTP is reachable only through +// this path, and the proxy's own uid is exempted from the redirect precisely +// so its upstream dials work. // // The interesting targets are not on the public internet, where the agent's // own egress would have reached anyway. They are the addresses only the pod @@ -29,7 +34,8 @@ import ( // (node credentials on most clouds), and the cluster-internal RFC1918 estate // (the Kubernetes API, other spokes, internal registries). // -// So the plain forward path dials only globally routable addresses. The check +// So the plain forward and uninspected tunnel paths dial only globally +// routable addresses. The check // runs in the dialer's Control hook rather than on the request's host string: // Control is called AFTER name resolution with the concrete remote address, so // a hostname that resolves to 127.0.0.1 or 169.254.169.254 — a public DNS @@ -132,6 +138,36 @@ var ( unguardedPlainForwardTransport = newPlainForwardTransport(false) ) +// guardedMarkDialer is markDialer with the destination check layered into the +// Control hook, so the address refused is the resolved one the kernel is about +// to connect to. It is the single dialer both the plain forward transport and +// the uninspected tunnel paths build on; keep the policy here so the two +// paths cannot drift apart. +func guardedMarkDialer(timeout time.Duration) *net.Dialer { + dialer := markDialer(timeout) + mark := dialer.Control + dialer.Control = func(network, address string, c syscall.RawConn) error { + if reason := plainForwardDenyReason(address); reason != "" { + return &plainForwardDenied{target: address, reason: reason} + } + return mark(network, address, c) + } + return dialer +} + +// tunnelDialer returns the dialer an uninspected tunnel (CONNECT passthrough +// or transparent-TLS passthrough) opens its upstream leg with: guarded unless +// the deployment lifted the guard with HIVE_PROXY_ALLOW_PRIVATE_FORWARD, the +// same boot-time choice the plain forward path honours. The zero value of +// allowPrivateForward is guarded, so a struct-literal GitHubProxy is +// protected without opting in. +func (p *GitHubProxy) tunnelDialer(timeout time.Duration) *net.Dialer { + if p.allowPrivateForward { + return markDialer(timeout) + } + return guardedMarkDialer(timeout) +} + // newPlainForwardTransport clones the standard transport (keeping its // connection pooling and timeouts), drops its ProxyFromEnvironment so an // HTTP_PROXY/HTTPS_PROXY on the proxy process is deliberately ignored — with @@ -146,13 +182,7 @@ func newPlainForwardTransport(guard bool) *http.Transport { transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { dialer := markDialer(upstreamDialTimeout) if guard { - mark := dialer.Control - dialer.Control = func(network, address string, c syscall.RawConn) error { - if reason := plainForwardDenyReason(address); reason != "" { - return &plainForwardDenied{target: address, reason: reason} - } - return mark(network, address, c) - } + dialer = guardedMarkDialer(upstreamDialTimeout) } return dialer.DialContext(ctx, network, address) } diff --git a/src/crux/proxy/proxy_deep2_test.go b/src/crux/proxy/proxy_deep2_test.go index ebdcf8e5f0..657b884275 100644 --- a/src/crux/proxy/proxy_deep2_test.go +++ b/src/crux/proxy/proxy_deep2_test.go @@ -321,6 +321,7 @@ func TestProxyHTTPGraphQLMutationAllowed(t *testing.T) { func TestHandleConnectDirectAnthropicNoRoute(t *testing.T) { p := newTestProxy() + p.allowPrivateForward = true // the tunnel target under test is a loopback listener p.inference = newInferenceRouter() // No route set for any agent @@ -641,6 +642,7 @@ func TestProxyHTTPUpstreamWriteError(t *testing.T) { func TestTunnelDirectSuccess(t *testing.T) { p := newTestProxy() + p.allowPrivateForward = true // the tunnel target under test is a loopback listener // Create a local TCP server ln, err := net.Listen("tcp", "127.0.0.1:0") diff --git a/src/crux/proxy/proxy_deep_test.go b/src/crux/proxy/proxy_deep_test.go index d678aa0d79..09c3dc0dac 100644 --- a/src/crux/proxy/proxy_deep_test.go +++ b/src/crux/proxy/proxy_deep_test.go @@ -738,6 +738,7 @@ func TestUpdateMaxContextLenStaleGuard(t *testing.T) { func TestHandleConnCONNECTNonGitHub(t *testing.T) { p := newTestProxy() + p.allowPrivateForward = true // the tunnel target under test is a loopback listener // Start a mock TCP server to tunnel to ln, err := net.Listen("tcp", "127.0.0.1:0") @@ -957,6 +958,7 @@ func TestExtractTextFromContentInvalid(t *testing.T) { func TestHandleConnectDirectNonGitHub(t *testing.T) { p := newTestProxy() + p.allowPrivateForward = true // the tunnel target under test is a loopback listener p.inference = newInferenceRouter() // Create a local TCP server to tunnel to diff --git a/src/crux/proxy/tunnel_destination_guard_test.go b/src/crux/proxy/tunnel_destination_guard_test.go new file mode 100644 index 0000000000..68763283cb --- /dev/null +++ b/src/crux/proxy/tunnel_destination_guard_test.go @@ -0,0 +1,149 @@ +package proxy + +import ( + "bufio" + "errors" + "io" + "net" + "net/http" + "strings" + "testing" + "time" +) + +// connectRoundTrip drives handleConnectDirect over a pipe for a CONNECT to +// target and returns the response the agent-side client would see. +func connectRoundTrip(t *testing.T, p *GitHubProxy, target string) *http.Response { + t.Helper() + req, err := http.NewRequest(http.MethodConnect, "http://"+target, nil) + if err != nil { + t.Fatalf("new request: %v", err) + } + req.Host = target + req.Header.Set("Proxy-Authorization", "hive scanner") + + clientConn, proxyClient := net.Pipe() + t.Cleanup(func() { _ = clientConn.Close() }) + + go func() { + p.handleConnectDirect(proxyClient, req) + _ = proxyClient.Close() + }() + + _ = clientConn.SetReadDeadline(time.Now().Add(10 * time.Second)) + resp, err := http.ReadResponse(bufio.NewReader(clientConn), req) + if err != nil { + t.Fatalf("read response: %v", err) + } + t.Cleanup(func() { _ = resp.Body.Close() }) + return resp +} + +// TestTunnelDirectRefusesNonRoutableDestination pins that the uninspected +// CONNECT path applies the same destination policy as the plain-HTTP forward: +// a CONNECT to the cloud metadata address or the RFC 1918 estate is answered +// 403 and counted against the caller, not tunnelled from the proxy's uid. +func TestTunnelDirectRefusesNonRoutableDestination(t *testing.T) { + p := newTestProxy() + p.proxyAdvisoryOK = true // no UID map in tests; accept the header identity + + for _, target := range []string{"169.254.169.254:80", "10.255.255.1:6443", "100.64.0.1:443"} { + resp := connectRoundTrip(t, p, target) + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("CONNECT %s: status = %d, want 403", target, resp.StatusCode) + } + body, _ := io.ReadAll(resp.Body) + if !strings.Contains(string(body), plainForwardAllowPrivateEnv) { + t.Errorf("CONNECT %s: body %q should name the override env var", target, body) + } + } + if got := p.violations["scanner"]; got != 3 { + t.Errorf("violations[scanner] = %d, want 3 — each estate-bound CONNECT must be attributed and counted", got) + } +} + +// TestTunnelDirectLoopbackRefusalIsNotAViolation mirrors the plain path: a +// loopback CONNECT is a client that ignored NO_PROXY, refused but not held +// against the caller. +func TestTunnelDirectLoopbackRefusalIsNotAViolation(t *testing.T) { + p := newTestProxy() + p.proxyAdvisoryOK = true + + if resp := connectRoundTrip(t, p, "127.0.0.1:9"); resp.StatusCode != http.StatusForbidden { + t.Fatalf("status = %d, want 403 for a loopback CONNECT", resp.StatusCode) + } + if got := p.violations["scanner"]; got != 0 { + t.Errorf("violations[scanner] = %d, want 0 — loopback is a misconfiguration, not a violation", got) + } +} + +// TestTunnelDirectAllowPrivateForwardLiftsGuard pins that the same boot-time +// override that lifts the plain-forward guard lifts the tunnel guard, so a +// deployment that opted in keeps working as before. +func TestTunnelDirectAllowPrivateForwardLiftsGuard(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer func() { _ = ln.Close() }() + go func() { + conn, err := ln.Accept() + if err != nil { + return + } + _ = conn.Close() + }() + + p := newTestProxy() + p.allowPrivateForward = true + + if resp := connectRoundTrip(t, p, ln.Addr().String()); resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200 with the guard lifted", resp.StatusCode) + } +} + +// TestDialTunnelUpstreamRefusesNonRoutableSNI covers the transparent-TLS +// passthrough leg: the host there comes from the client's SNI, so an SNI that +// names a private address must be refused on the same evidence. +func TestDialTunnelUpstreamRefusesNonRoutableSNI(t *testing.T) { + p := newTestProxy() + + _, err := p.dialTunnelUpstream("10.255.255.1") + var denied *plainForwardDenied + if !errors.As(err, &denied) { + t.Fatalf("err = %v, want *plainForwardDenied", err) + } + if denied.reason != plainForwardReasonPrivate { + t.Errorf("reason = %q, want %q", denied.reason, plainForwardReasonPrivate) + } +} + +// TestHandleTransparentTLSRefusalCountsEstateNotLoopback pins the handler-level +// split in handleTransparentTLS: a refused estate-bound SNI is counted against +// the caller, a refused loopback SNI is not. Over net.Pipe attribution yields +// "", so the counter lands under that key. +func TestHandleTransparentTLSRefusalCountsEstateNotLoopback(t *testing.T) { + for _, tc := range []struct { + sni string + want int + }{{"10.255.255.1", 1}, {"127.0.0.1", 0}} { + p := newTestProxy() + clientHello := buildClientHello(tc.sni) + clientConn, proxyConn := net.Pipe() + done := make(chan struct{}) + go func() { + defer close(done) + p.handleTransparentTLS(proxyConn, clientHello[:1]) + _ = proxyConn.Close() + }() + _, _ = clientConn.Write(clientHello[1:]) + <-done + _ = clientConn.Close() + p.mu.Lock() + got := p.violations[""] + p.mu.Unlock() + if got != tc.want { + t.Errorf("SNI %s: violations[\"\"] = %d, want %d", tc.sni, got, tc.want) + } + } +} From c549296b8cfb00cbed18814c123c7f4f02a11c94 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 15:02:51 +0000 Subject: [PATCH 04/17] =?UTF-8?q?=F0=9F=90=9B=20fix(crux/toolapprove):=20r?= =?UTF-8?q?ead=20apply=5Fpatch=20target=20paths=20for=20the=20guardrail=20?= =?UTF-8?q?check=20(#16984)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(crux/toolapprove): read apply_patch target paths for the guardrail check Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * fix(crux/toolapprove): check every apply_patch path against the guardrails The guardrail check read only GetFilePath (the first *** header), so a patch naming a harmless file first and OWNERS second auto-approved. Iterate GetFilePaths instead and cover it with a Scan-level test. Hive-Fix-Attempt: 1/6 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../security-toolapprove-apply-patch-paths.md | 1 + src/crux/toolapprove/security.go | 9 ++- src/crux/toolapprove/types.go | 65 ++++++++++++++++++ .../toolapprove/types_apply_patch_test.go | 68 +++++++++++++++++++ 4 files changed, 140 insertions(+), 3 deletions(-) create mode 100644 changelog.d/security-toolapprove-apply-patch-paths.md create mode 100644 src/crux/toolapprove/types_apply_patch_test.go diff --git a/changelog.d/security-toolapprove-apply-patch-paths.md b/changelog.d/security-toolapprove-apply-patch-paths.md new file mode 100644 index 0000000000..65d2151779 --- /dev/null +++ b/changelog.d/security-toolapprove-apply-patch-paths.md @@ -0,0 +1 @@ +- Fixed the tool-approval guardrail-path check never seeing the target of a Copilot CLI `apply_patch` call ([#16979](https://github.com/hivecommons/hive/issues/16979)). The tool carries its paths in the patch text (`*** Update File:`, `*** Add File:`, `*** Delete File:`, `*** Move to:`) instead of a path argument, so `ToolRequest.GetFilePath` returned an empty string and a patch touching `.github/workflows/**`, `OWNERS`, `hive.yaml` or `policies/**` auto-approved at L5/L6. `GetFilePath` now falls back to the first path in the patch body, and the guardrail check now iterates the new `GetFilePaths` over every path the request names, so a patch that lists a harmless file first and a guardrail path later is still caught. diff --git a/src/crux/toolapprove/security.go b/src/crux/toolapprove/security.go index 01d71107da..f52b45094e 100644 --- a/src/crux/toolapprove/security.go +++ b/src/crux/toolapprove/security.go @@ -179,10 +179,13 @@ func (s *DefaultSecurityScanner) Scan(ctx context.Context, req ToolRequest) (Sca // str_replace_editor, an agent's own name for it) must not reach the // guardrail paths unseen. The gate fails closed — a read tool IsReadOnly // does not spell is denied on these paths until it is added there. + // Every path is checked, not just the first: an apply_patch body can + // name a harmless file first and OWNERS second. if req.IsSideEffectful() { - filePath := req.GetFilePath() - if filePath != "" && s.touchesGuardrailPath(filePath) { - violations = append(violations, fmt.Sprintf("tampering with guardrail-critical path: %q", filePath)) + for _, filePath := range req.GetFilePaths() { + if s.touchesGuardrailPath(filePath) { + violations = append(violations, fmt.Sprintf("tampering with guardrail-critical path: %q", filePath)) + } } } diff --git a/src/crux/toolapprove/types.go b/src/crux/toolapprove/types.go index 29b6bb4eb8..62a6ff88c0 100644 --- a/src/crux/toolapprove/types.go +++ b/src/crux/toolapprove/types.go @@ -66,10 +66,75 @@ func (r ToolRequest) GetFilePath() string { } } } + if paths := r.patchPaths(); len(paths) > 0 { + return paths[0] + } } return "" } +// GetFilePaths returns every file path this request targets: the single +// path GetFilePath finds, plus every path named in an apply_patch body. +func (r ToolRequest) GetFilePaths() []string { + var paths []string + seen := map[string]bool{} + add := func(p string) { + if p != "" && !seen[p] { + seen[p] = true + paths = append(paths, p) + } + } + if r.Target != "" { + add(r.Target) + } else if r.Arguments != nil { + for _, k := range []string{"file_path", "filePath", "path", "target", "filename", "file"} { + if s, ok := r.Arguments[k].(string); ok { + add(s) + break + } + } + } + for _, p := range r.patchPaths() { + add(p) + } + return paths +} + +var patchPathHeaders = []string{"*** Update File:", "*** Add File:", "*** Delete File:", "*** Move to:"} + +// patchPaths parses the target paths out of an apply_patch body, which +// carries them in its "*** Update File: " style headers rather than in +// a path argument. +func (r ToolRequest) patchPaths() []string { + if r.Arguments == nil { + return nil + } + isPatchTool := strings.EqualFold(strings.TrimSpace(r.Tool), "apply_patch") + var paths []string + seen := map[string]bool{} + for _, k := range []string{"input", "patch"} { + body, ok := r.Arguments[k].(string) + if !ok || (!isPatchTool && !strings.HasPrefix(strings.TrimSpace(body), "*** Begin Patch")) { + continue + } + for _, line := range strings.Split(body, "\n") { + line = strings.TrimSpace(line) + for _, h := range patchPathHeaders { + if !strings.HasPrefix(line, h) { + continue + } + p := strings.TrimSpace(strings.TrimPrefix(line, h)) + if p != "" && !seen[p] { + seen[p] = true + paths = append(paths, p) + } + break + } + } + } + return paths +} + // GetContent returns the text content or body payload associated with this request. func (r ToolRequest) GetContent() string { if r.Content != "" { diff --git a/src/crux/toolapprove/types_apply_patch_test.go b/src/crux/toolapprove/types_apply_patch_test.go new file mode 100644 index 0000000000..24280b8d74 --- /dev/null +++ b/src/crux/toolapprove/types_apply_patch_test.go @@ -0,0 +1,68 @@ +package toolapprove + +import ( + "context" + "reflect" + "strings" + "testing" +) + +func TestToolRequestApplyPatchPaths(t *testing.T) { + multi := "*** Begin Patch\n*** Update File: src/a.go\n@@\n-x\n+y\n*** Add File: OWNERS \n+a\n*** Delete File: policies/p.yaml\n*** Update File: src/a.go\n*** Move to: hive.yaml\n\n*** End Patch" + tests := []struct { + name string + req ToolRequest + wantFirst string + wantAll []string + }{ + {"input key", ToolRequest{Tool: "apply_patch", Arguments: map[string]any{"input": multi}}, "src/a.go", []string{"src/a.go", "OWNERS", "policies/p.yaml", "hive.yaml"}}, + {"patch key", ToolRequest{Tool: "apply_patch", Arguments: map[string]any{"patch": "*** Begin Patch\n*** Update File: .github/workflows/ci.yml\n*** End Patch"}}, ".github/workflows/ci.yml", []string{".github/workflows/ci.yml"}}, + {"unknown tool with patch text", ToolRequest{Tool: "editor", Arguments: map[string]any{"input": "*** Begin Patch\n*** Add File: OWNERS\n*** End Patch"}}, "OWNERS", []string{"OWNERS"}}, + {"path argument wins", ToolRequest{Tool: "apply_patch", Arguments: map[string]any{"path": "x.go", "input": multi}}, "x.go", []string{"x.go", "src/a.go", "OWNERS", "policies/p.yaml", "hive.yaml"}}, + {"non-patch text ignored", ToolRequest{Tool: "write", Arguments: map[string]any{"input": "*** Update File: OWNERS"}}, "", nil}, + {"no arguments", ToolRequest{Tool: "apply_patch"}, "", nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.req.GetFilePath(); got != tt.wantFirst { + t.Errorf("GetFilePath() = %q, want %q", got, tt.wantFirst) + } + if got := tt.req.GetFilePaths(); !reflect.DeepEqual(got, tt.wantAll) { + t.Errorf("GetFilePaths() = %v, want %v", got, tt.wantAll) + } + }) + } +} + +// TestSecurityScanner_ApplyPatchGuardrailAnyPosition drives Scan with a +// multi-file apply_patch so a guardrail path that is not the first header +// still fails the scan (#16979). +func TestSecurityScanner_ApplyPatchGuardrailAnyPosition(t *testing.T) { + scanner := NewDefaultSecurityScanner() + tests := []struct { + name string + patch string + wantPassed bool + wantPath string + }{ + {"guardrail path second", "*** Begin Patch\n*** Update File: src/a.go\n@@\n-x\n+y\n*** Add File: OWNERS\n+a\n*** End Patch", false, `"OWNERS"`}, + {"guardrail path last via move", "*** Begin Patch\n*** Update File: src/a.go\n*** Update File: src/b.go\n*** Move to: policies/p.yaml\n*** End Patch", false, `"policies/p.yaml"`}, + {"guardrail path first", "*** Begin Patch\n*** Update File: hive.yaml\n*** Update File: src/a.go\n*** End Patch", false, `"hive.yaml"`}, + {"no guardrail paths", "*** Begin Patch\n*** Update File: src/a.go\n*** Add File: src/b.go\n*** End Patch", true, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := ToolRequest{Tool: "apply_patch", Arguments: map[string]any{"input": tt.patch}} + res, err := scanner.Scan(context.Background(), req) + if err != nil { + t.Fatalf("scanner error: %v", err) + } + if res.Passed != tt.wantPassed { + t.Fatalf("Passed = %v, want %v (violations: %v)", res.Passed, tt.wantPassed, res.Violations) + } + if tt.wantPath != "" && !strings.Contains(res.Details, tt.wantPath) { + t.Errorf("Details %q does not name %s", res.Details, tt.wantPath) + } + }) + } +} From 22f9f1d593645df03758ffeea36ddd9b87950a88 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Sat, 7 Nov 2026 10:07:33 -0500 Subject: [PATCH 05/17] =?UTF-8?q?=F0=9F=90=9B=20fix(github-relay):=20repor?= =?UTF-8?q?t=20unpushed=20PR=20head=20once,=20naming=20the=20ref=20checked?= =?UTF-8?q?=20and=20the=20push=20outcome=20(#16991)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pr-request watcher logged 'head branch is not on the remote — the agent's push did not authenticate' at ERROR on every retry of the same request (30s→15m backoff), so a single unpushed branch tripped the fleetreport 5-in-10m threshold and filed a selfreport that asserted an authentication failure without evidence. The apostrophe in "agent's" was also treated as an opening quote by fleetreport.NormalizeMessage, truncating the scope to 'the agentQ'. - failPRRequest: ERROR once per request with repo, head, the exact ref and remote checked, the hive-push-branch relay outcome for that branch, and the diagnosis; retries log at Info, and the eventual success on retry is logged. - errMissingHead: describe what to check (relay result.json, credential helper, token cache) instead of asserting authentication failed. - NormalizeMessage: mid-word apostrophes are not quotes. - hive-open-pr.sh: warn (never refuse) when the head is not on the remote before writing the request. Closes #16969 Signed-off-by: Andrew Anderson --- bin/hive-open-pr.sh | 28 +++++ bin/test_hive_open_pr.sh | 32 +++++- .../fixed-pr-request-head-unpushed-report.md | 1 + src/docs/fleet-report.md | 2 +- src/docs/hive-open-pr.md | 39 +++++-- src/pkg/fleetreport/runtime.go | 12 ++- src/pkg/fleetreport/runtime_test.go | 101 +++++++++++------- src/pkg/github/pr_request_missing_head.go | 75 ++++++++++++- .../github/pr_request_missing_head_test.go | 13 +-- .../github/pr_request_retry_reporting_test.go | 74 +++++++++++++ src/pkg/github/pr_request_watcher.go | 31 +++++- src/pkg/github/request_retry.go | 12 +++ 12 files changed, 355 insertions(+), 65 deletions(-) create mode 100644 changelog.d/fixed-pr-request-head-unpushed-report.md diff --git a/bin/hive-open-pr.sh b/bin/hive-open-pr.sh index 204f7a2083..d352674e02 100644 --- a/bin/hive-open-pr.sh +++ b/bin/hive-open-pr.sh @@ -86,6 +86,33 @@ commit_has_trailer() { grep -Fqx "$trailer" } +# warn_if_head_not_on_remote tells the agent, at request time, when the branch +# it is asking a PR for is not known to be on the remote (#16969). The watcher +# cannot open a PR for a branch GitHub does not have; it retries for 24h, so +# a push that never landed surfaces only as a fleet self-report hours later. +# Evidence is the checkout's own remote-tracking ref (written by a successful +# `git push`, no network) and, failing that, one `git ls-remote` of the exact +# ref. It never refuses: a hive-push-branch relay push may still be in flight, +# and the queued request lands on retry once the branch arrives. +warn_if_head_not_on_remote() { + head="$1" + [ -n "${HIVE_OPEN_PR_SKIP_REMOTE_CHECK:-}" ] && return 0 + git rev-parse --is-inside-work-tree >/dev/null 2>&1 || return 0 + remote="$(git config --get "branch.${head}.remote" 2>/dev/null || true)" + [ -n "$remote" ] || remote=origin + git remote get-url "$remote" >/dev/null 2>&1 || return 0 + if git rev-parse --verify --quiet "refs/remotes/${remote}/${head}" >/dev/null 2>&1; then + return 0 + fi + rc=0 + GIT_TERMINAL_PROMPT=0 git ls-remote --exit-code --heads "$remote" "refs/heads/${head}" >/dev/null 2>&1 || rc=$? + # 2 is ls-remote's definitive "no such ref"; anything else (auth, network) + # is not evidence either way, so stay quiet rather than cry wolf. + [ "$rc" -eq 2 ] || return 0 + echo "hive-open-pr: WARN: branch '${head}' is NOT on remote '${remote}' ($(git remote get-url "$remote" 2>/dev/null)) — the hive cannot open a PR for a branch GitHub does not have." >&2 + echo "hive-open-pr: WARN: push it first (git push -u ${remote} ${head}, or hive-push-branch --repo ${REPO:-}) and check the push's own output; the request below stays queued and succeeds on retry once the branch arrives." >&2 +} + check_issue_author_coauthors() { repo="$1" body="$2" @@ -213,6 +240,7 @@ if [ -z "$REPO" ] || [ -z "$HEAD" ] || [ -z "$TITLE" ]; then fi check_issue_author_coauthors "$REPO" "$BODY" +warn_if_head_not_on_remote "$HEAD" # Identify the requesting agent. Prefer the UID map (the watcher re-derives the # owner from the FILE's UID anyway, so this is informational + a nicer log line); diff --git a/bin/test_hive_open_pr.sh b/bin/test_hive_open_pr.sh index 8eeb4409fa..1ac8c78677 100755 --- a/bin/test_hive_open_pr.sh +++ b/bin/test_hive_open_pr.sh @@ -144,7 +144,7 @@ check "--body-file body reaches the request" \ reset_reqs check_exit "--body-file - reads stdin" 0 \ - bash -c 'printf "stdin body" | "$@"' _ env HIVE_AGENT=a3 HIVE_OPEN_PR_REQ_DIR="$REQ_DIR" \ + bash -c 'printf "stdin body" | "$@"' _ env HIVE_AGENT=a3 HIVE_OPEN_PR_REQ_DIR="$REQ_DIR" HIVE_OPEN_PR_SKIP_REMOTE_CHECK=1 \ bash "$MODIFIED_SCRIPT" --repo o/r --title T --head h --body-file - REQ="$(find_req a3)" check "stdin body reaches the request" "stdin body" "$(json_field "$REQ" body)" @@ -373,6 +373,36 @@ else fi reset_reqs +# --- Section 10: head-on-remote preflight (#16969) --- +echo "" +echo "--- head-on-remote preflight ---" + +BARE="$TMPDIR_ROOT/remote.git" +git init --quiet --bare "$BARE" +PUSHED_REPO="$TMPDIR_ROOT/pushed" +git clone --quiet "$BARE" "$PUSHED_REPO" 2>/dev/null +( cd "$PUSHED_REPO" && git checkout --quiet -b pushed/branch 2>/dev/null && touch f && git add f && + git -c user.name=t -c user.email=t@t -c commit.gpgsign=false commit --quiet -m seed && + git push --quiet -u origin pushed/branch 2>/dev/null && + git checkout --quiet -b local/only && touch g && git add g && + git -c user.name=t -c user.email=t@t -c commit.gpgsign=false commit --quiet -m local ) + +STDERR_OUT="$( (cd "$PUSHED_REPO" && env HIVE_AGENT=a10 HIVE_OPEN_PR_REQ_DIR="$REQ_DIR" bash "$MODIFIED_SCRIPT" --repo o/r --head pushed/branch --title T --body B) 2>&1 >/dev/null )" +check "pushed branch produces no warning" "" "$STDERR_OUT" +reset_reqs + +STDERR_OUT="$( (cd "$PUSHED_REPO" && env HIVE_AGENT=a10 HIVE_OPEN_PR_REQ_DIR="$REQ_DIR" bash "$MODIFIED_SCRIPT" --repo o/r --head local/only --title T --body B) 2>&1 >/dev/null )" +case "$STDERR_OUT" in + *"branch 'local/only' is NOT on remote 'origin'"*"hive-push-branch --repo o/r"*) check "unpushed branch warns and names branch, remote and the push step" ok ok;; + *) check "unpushed branch warns and names branch, remote and the push step" "warning naming local/only, origin and hive-push-branch" "$STDERR_OUT";; +esac +check_exit "unpushed branch still queues the request (watcher retries once it lands)" 0 test -n "$(find_req a10)" +reset_reqs + +STDERR_OUT="$( (cd "$PUSHED_REPO" && env HIVE_AGENT=a10 HIVE_OPEN_PR_REQ_DIR="$REQ_DIR" HIVE_OPEN_PR_SKIP_REMOTE_CHECK=1 bash "$MODIFIED_SCRIPT" --repo o/r --head local/only --title T --body B) 2>&1 >/dev/null )" +check "HIVE_OPEN_PR_SKIP_REMOTE_CHECK silences the preflight" "" "$STDERR_OUT" +reset_reqs + # --- Summary --- echo "" echo "=== Results: $PASS passed, $FAIL failed ===" diff --git a/changelog.d/fixed-pr-request-head-unpushed-report.md b/changelog.d/fixed-pr-request-head-unpushed-report.md new file mode 100644 index 0000000000..1a08900aaa --- /dev/null +++ b/changelog.d/fixed-pr-request-head-unpushed-report.md @@ -0,0 +1 @@ +- The pr-request watcher now reports an unpushed head branch once per request instead of once per retry, states the exact ref and repository it checked and what the push relay recorded for that branch (queued, pushed, failed, or no record) rather than asserting a push-authentication failure, and logs when the request later succeeds on retry; `hive-open-pr` warns at request time when the head is not on the remote, naming the branch, remote and push step; fleet-report fingerprints no longer treat a possessive apostrophe as an opening quote, which truncated this report's scope to "the agentQ" ([#16969](https://github.com/hivecommons/hive/issues/16969)). diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index f6c6f3aa18..2676ca9d35 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -141,7 +141,7 @@ title is `[selfreport] : `. | `agent_dir_setup_failed` | the harness failed to set up the per-agent sandbox directory structure — create XDG directories, restore or tighten directory permissions on interactive/inference home directories. Log-fed: keywords include `failed to create per-agent xdg dir`, `failed to restore per-agent dir mode`, `failed to restore interactive home mode`, `failed to tighten per-agent dir mode`, `failed to tighten interactive home mode`, `failed to restore inference home mode`, `failed to tighten inference home mode` (`crux/agent/manager.go`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `agent_pane_timeout` | the agent-readiness wait timed out and the harness dumped the tmux pane (`prompt timeout — dumping pane` / `pane content at timeout`, `pkg/agent/manager.go`), and the startup kick dropped because the CLI never became ready or reached its input prompt (`startup kick dropped: …`, `crux/agent/manager_kick.go`, the same event as the pane dump). Log-fed: same ≥5-in-10-min repeat threshold and 20-min idle clear as the catch-all below; pane excerpts are scrubbed and bounded before storage | low | | `agent_login_required` | the login scanner saw the agent CLI asking for `/login` (`login required detected`, `pkg/loginscan`) — the agent's auth lapsed and it is paused until an operator re-authenticates. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | -| `pr_request_head_unpushed` | the pr-request watcher found the PR head branch missing on the remote (`head branch is not on the remote`) — the agent's push did not authenticate or never ran. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | +| `pr_request_head_unpushed` | the pr-request watcher found the PR head branch missing on the remote (`head branch is not on the remote`): the agent's push failed, never ran, or went to another ref — the line's `checked` and `push_outcome` attributes say which. Logged once per request (retries stay at INFO), so the repeat threshold counts distinct unpushed requests, not one request's retries ([#16969](https://github.com/hivecommons/hive/issues/16969)). Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `bead_fact_ingest_failed` | the knowledge system failed to ingest a bead fact from an agent (`failed to ingest bead fact`, `pkg/knowledge`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `feedback_issue_refresh_failed` | the dashboard's feedback-issue watcher failed to refresh or update a feedback issue (`feedback: issue refresh failed`, `pkg/dashboard`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `github_app_repo_not_covered` | the GitHub App recheck (`state=repo-not-covered`) found the installation does not include every configured repo — an operator configuration condition (tick the repos in the App's repository access), not a hive defect. Log-fed: same ≥5-in-10-min repeat gate and 20-minute idle clear as `agent_pane_timeout` | low | diff --git a/src/docs/hive-open-pr.md b/src/docs/hive-open-pr.md index b3c74eb7cc..86dd5a37c6 100644 --- a/src/docs/hive-open-pr.md +++ b/src/docs/hive-open-pr.md @@ -368,26 +368,37 @@ recognise it. The branch is not implicated at all. Fix the App installation — see [GitHub App setup](github-app-setup.md). -**2. The repository is visible but the head ref is genuinely absent.** This is -the push-authentication case, and the error says so: +**2. The repository is visible but the head ref is genuinely absent.** The +branch is not where the watcher looked, and the error says what it checked: -> branch `` was never pushed to `/` — the commits exist only in the agent's working copy. This is almost always a PUSH AUTHENTICATION failure, not a branch-creation problem +> branch `` is not on `/` — the commits exist only in the agent's working copy, or were pushed under another name or to another remote -It names the two causes to check, in order: +It names the three things to check, in order: -1. **The git credential helper is not reachable from the agent's UID.** +1. **The push's own outcome.** A `hive-push-branch` relay push leaves + `.result.json` under `/var/run/hive-metrics/push-requests`; a + direct `git push` printed its result in the agent's pane. The watcher's + log line (below) already carries the relay's record as `push_outcome`. +2. **The git credential helper is not reachable from the agent's UID.** `su -s /bin/sh hive- -c 'git config --get-regexp credential'` must list `/usr/local/bin/git-credential-hive.sh`. It is wired system-wide in `/etc/gitconfig` precisely because agents do not share the dev user's `$HOME`. -2. **The agent's scoped token file is absent or unreadable by that UID.** The +3. **The agent's scoped token file is absent or unreadable by that UID.** The path is in `$HIVE_AGENT_TOKEN_CACHE`. Check readability only — never print its contents. This stays a **retryable** error rather than a policy rejection: once the agent pushes the branch, the same request becomes valid and succeeds on retry. That -is exactly what the bounded retry is for, so fix the credential problem and let -the queued request land — you do not need to re-issue it. +is exactly what the bounded retry is for, so fix the push problem and let the +queued request land — you do not need to re-issue it. + +`hive-open-pr` itself warns at request time when the head is not known to be +on the remote ([#16969](https://github.com/hivecommons/hive/issues/16969)): it +checks the checkout's remote-tracking ref and, failing that, one +`git ls-remote` of the exact ref. The warning names the branch, the remote and +the push step; the request is still written, because a relay push may be in +flight. Set `HIVE_OPEN_PR_SKIP_REMOTE_CHECK=1` to silence it. **3. The head ref exists, so the 404 was about something else.** Usually the base: @@ -409,8 +420,16 @@ since the agent's session ended healthy: kubectl -n hive logs deploy/hive | grep 'pr-request watcher' ``` -Look for `head branch is not on the remote — the agent's push did not -authenticate`, which carries `repo`, `head`, `agent`, and the full `diagnosis`. +Look for `head branch is not on the remote`, which carries `repo`, `head`, +`checked` (the exact ref and repository the watcher asked GitHub for), +`push_outcome` (what the push relay recorded for that branch — queued, pushed +as ``, failed with the git error, or no record because the agent pushed +directly), `attempt`, `agent`, and the full `diagnosis`. It is logged **once +per request**: retries of the same request log `head branch still not on the +remote, retrying` at INFO, and the eventual success logs `request succeeded +on retry` ([#16969](https://github.com/hivecommons/hive/issues/16969) — a +single unpushed branch used to re-log at ERROR every retry and file a fleet +self-report before the branch arrived). For the broader symptom — an agent that completed a session with no branch and no PR, and how to tell a credential-helper failure from a mid-task token diff --git a/src/pkg/fleetreport/runtime.go b/src/pkg/fleetreport/runtime.go index d4af257b48..a08cddd162 100644 --- a/src/pkg/fleetreport/runtime.go +++ b/src/pkg/fleetreport/runtime.go @@ -8,6 +8,7 @@ import ( "strings" "sync" "time" + "unicode" "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/crux/logscrub" @@ -561,7 +562,7 @@ func NormalizeMessage(msg string) string { } b.WriteString("P") i = j - case r == '"' || r == '\'' || r == '`': + case r == '"' || r == '`' || (r == '\'' && !midWordApostrophe(runes, i)): j := i + 1 for j < len(runes) && runes[j] != r { j++ @@ -582,6 +583,15 @@ func NormalizeMessage(msg string) string { return strings.Join(strings.Fields(b.String()), " ") } +// midWordApostrophe reports whether the apostrophe at i sits between two +// letters ("agent's", "don't"). That is a contraction or possessive, not an +// opening quote: treating it as one swallowed the rest of the message into a +// single Q placeholder (#16969's scope read "… — the agentQ"), so unrelated +// messages sharing a prefix collapsed into one fingerprint. +func midWordApostrophe(runes []rune, i int) bool { + return i > 0 && i+1 < len(runes) && unicode.IsLetter(runes[i-1]) && unicode.IsLetter(runes[i+1]) +} + func runOfHex(runes []rune, i int) int { n := 0 for i+n < len(runes) { diff --git a/src/pkg/fleetreport/runtime_test.go b/src/pkg/fleetreport/runtime_test.go index b8e8918db6..958f22cb53 100644 --- a/src/pkg/fleetreport/runtime_test.go +++ b/src/pkg/fleetreport/runtime_test.go @@ -91,6 +91,25 @@ func TestNormalizeMessageAndFingerprintStable(t *testing.T) { } } +// #16969: a possessive apostrophe is not an opening quote. Before this the +// scope of a "… — the agent's push did not authenticate" report rendered as +// "… — the agentQ" and every message sharing that prefix fingerprinted alike. +func TestNormalizeMessageKeepsMidWordApostrophes(t *testing.T) { + got := NormalizeMessage("pr-request watcher: head branch is not on the remote — the agent's push did not authenticate") + if !strings.HasSuffix(got, "the agent's push did not authenticate") { + t.Fatalf("possessive swallowed the rest of the message: %q", got) + } + a := NormalizeMessage("watcher: the agent's push failed") + b := NormalizeMessage("watcher: the agent's token expired") + if a == b { + t.Fatalf("distinct messages collapsed into one key: %q", a) + } + // Real single-quoted strings still collapse to a placeholder. + if got := NormalizeMessage("refused 'fix/x' on origin"); got != "refused Q on origin" { + t.Fatalf("quoted string not collapsed: %q", got) + } +} + func TestBuildRuntimeReportCarriesMarkersAndLabels(t *testing.T) { now := time.Date(2026, 9, 24, 12, 0, 0, 0, time.UTC) ev := Evidence{Class: ClassRestartStorm, Component: "agent-runtime", Severity: "high", Count: 4, Window: 10 * time.Minute, Message: "restart storm: agent quality restarted 4 times in 10m", Diagnostics: "line1\ntoken=ghp_abcdefghijklmnopqrstuvwxyz0123456789\nline3", Attributable: true} @@ -365,7 +384,7 @@ func TestEveryRuntimeClassTriggerAndNonTriggerThresholds(t *testing.T) { {class: ClassKickFreshnessWithheld, message: "kick withheld: every ref failed the freshness check"}, {class: ClassModelsEndpointEmpty, message: "no models found from any endpoint"}, {class: ClassAgentLoginRequired, message: "login required detected"}, - {class: ClassPRRequestHeadUnpushed, message: "pr-request watcher: head branch is not on the remote — the agent's push did not authenticate"}, + {class: ClassPRRequestHeadUnpushed, message: "pr-request watcher: head branch is not on the remote"}, {class: ClassIOScanClassifierUnavail, message: "ioscan classifier failed closed; text withheld"}, {class: ClassIOScanClassifierUnavail, message: "ioscan classifier failed open; text passes on unclassified"}, {class: ClassIOScanClassifierUnavail, message: "ioscan fail-closed blocked kick"}, @@ -1081,46 +1100,46 @@ func TestUnclassifiedRepeatedHonoursWindowAndIdleClear(t *testing.T) { func TestRecurringRuntimeMessagesAreClassified(t *testing.T) { fed := map[string]string{ "startup kick dropped: agent restarted or stopped while waiting": ClassStartupKickInterrupted, - "tmux has-session failed": ClassTmuxSessionCheckFailed, - "pane content at timeout": ClassAgentPaneTimeout, - "prompt timeout — dumping pane": ClassAgentPaneTimeout, - "startup kick dropped: CLI never reached input prompt": ClassAgentPaneTimeout, - "startup kick dropped: CLI never became ready": ClassAgentPaneTimeout, - "channel commit date: fetch failed — the row will render without a timestamp": ClassGitHubFetchTransient, - "issue stage labels: provisioning failed": ClassLabelProvisioningFailed, - "agent CLI crashed or stuck on consent screen, restarting before kick": ClassAgentCLIRestartLoop, - "shared-ci rerun forbidden: the App installation lacks Actions write permission": ClassSharedCIRerunForbidden, - "SHA poll: commit message fetch non-200": ClassSHAPollCommitMsgUnavail, - "kick withheld: every ref failed the freshness check": ClassKickFreshnessWithheld, - "no models found from any endpoint": ClassModelsEndpointEmpty, - "login required detected": ClassAgentLoginRequired, - "pr-request watcher: head branch is not on the remote — the agent's push did not authenticate": ClassPRRequestHeadUnpushed, - "ioscan classifier failed closed; text withheld": ClassIOScanClassifierUnavail, - "watchdog: agent credential failure": ClassAgentLoginRequired, - "ioscan classifier failed open; text passes on unclassified": ClassIOScanClassifierUnavail, - "ioscan fail-closed blocked kick": ClassIOScanClassifierUnavail, - "ioscan fail-closed blocked held-PR coordination kick": ClassIOScanClassifierUnavail, - "proxy client TLS handshake failed": ClassProxyTLSHandshakeFailed, - "transparent proxy TLS handshake failed": ClassProxyTLSHandshakeFailed, - "failed to restore per-agent dir mode": ClassAgentDirSetupFailed, - "failed to restore interactive home mode": ClassAgentDirSetupFailed, - "failed to tighten per-agent dir mode": ClassAgentDirSetupFailed, - "failed to tighten interactive home mode": ClassAgentDirSetupFailed, - "failed to create per-agent XDG dir": ClassAgentDirSetupFailed, - "review evidence: could not write bundle": ClassReviewEvidenceWriteFail, - "feedback: issue refresh failed": ClassFeedbackIssueRefreshFailed, - "failed to restore inference home mode": ClassAgentDirSetupFailed, - "failed to tighten inference home mode": ClassAgentDirSetupFailed, - "failed to ingest bead fact": ClassBeadFactIngestFailed, - "fleet report: cleared comment failed": ClassSelfReportClearedFailed, - "copilot sniff: client TLS handshake failed": ClassProxyTLSHandshakeFailed, - "github app recheck: installation does not cover every configured repo": ClassGitHubAppRepoNotCovered, - "proxy: plain HTTP forward refused — destination is not globally routable": ClassProxyPlainForwardRefused, - "proxy: CONNECT dial failed": ClassProxyConnectDialFailed, - "inference upstream error": ClassInferenceUpstreamError, - "copilot transcript activity scan outlived its interval; agent liveness reads may lag": ClassTranscriptScanSlow, - "inference upstream failed": ClassInferenceUpstreamError, - "inference translate request failed": ClassInferenceUpstreamError, + "tmux has-session failed": ClassTmuxSessionCheckFailed, + "pane content at timeout": ClassAgentPaneTimeout, + "prompt timeout — dumping pane": ClassAgentPaneTimeout, + "startup kick dropped: CLI never reached input prompt": ClassAgentPaneTimeout, + "startup kick dropped: CLI never became ready": ClassAgentPaneTimeout, + "channel commit date: fetch failed — the row will render without a timestamp": ClassGitHubFetchTransient, + "issue stage labels: provisioning failed": ClassLabelProvisioningFailed, + "agent CLI crashed or stuck on consent screen, restarting before kick": ClassAgentCLIRestartLoop, + "shared-ci rerun forbidden: the App installation lacks Actions write permission": ClassSharedCIRerunForbidden, + "SHA poll: commit message fetch non-200": ClassSHAPollCommitMsgUnavail, + "kick withheld: every ref failed the freshness check": ClassKickFreshnessWithheld, + "no models found from any endpoint": ClassModelsEndpointEmpty, + "login required detected": ClassAgentLoginRequired, + "pr-request watcher: head branch is not on the remote": ClassPRRequestHeadUnpushed, + "ioscan classifier failed closed; text withheld": ClassIOScanClassifierUnavail, + "watchdog: agent credential failure": ClassAgentLoginRequired, + "ioscan classifier failed open; text passes on unclassified": ClassIOScanClassifierUnavail, + "ioscan fail-closed blocked kick": ClassIOScanClassifierUnavail, + "ioscan fail-closed blocked held-PR coordination kick": ClassIOScanClassifierUnavail, + "proxy client TLS handshake failed": ClassProxyTLSHandshakeFailed, + "transparent proxy TLS handshake failed": ClassProxyTLSHandshakeFailed, + "failed to restore per-agent dir mode": ClassAgentDirSetupFailed, + "failed to restore interactive home mode": ClassAgentDirSetupFailed, + "failed to tighten per-agent dir mode": ClassAgentDirSetupFailed, + "failed to tighten interactive home mode": ClassAgentDirSetupFailed, + "failed to create per-agent XDG dir": ClassAgentDirSetupFailed, + "review evidence: could not write bundle": ClassReviewEvidenceWriteFail, + "feedback: issue refresh failed": ClassFeedbackIssueRefreshFailed, + "failed to restore inference home mode": ClassAgentDirSetupFailed, + "failed to tighten inference home mode": ClassAgentDirSetupFailed, + "failed to ingest bead fact": ClassBeadFactIngestFailed, + "fleet report: cleared comment failed": ClassSelfReportClearedFailed, + "copilot sniff: client TLS handshake failed": ClassProxyTLSHandshakeFailed, + "github app recheck: installation does not cover every configured repo": ClassGitHubAppRepoNotCovered, + "proxy: plain HTTP forward refused — destination is not globally routable": ClassProxyPlainForwardRefused, + "proxy: CONNECT dial failed": ClassProxyConnectDialFailed, + "inference upstream error": ClassInferenceUpstreamError, + "copilot transcript activity scan outlived its interval; agent liveness reads may lag": ClassTranscriptScanSlow, + "inference upstream failed": ClassInferenceUpstreamError, + "inference translate request failed": ClassInferenceUpstreamError, } for msg, want := range fed { if got := LogFedClass(msg); got != want { diff --git a/src/pkg/github/pr_request_missing_head.go b/src/pkg/github/pr_request_missing_head.go index 4e0555d3f9..8e1ce05652 100644 --- a/src/pkg/github/pr_request_missing_head.go +++ b/src/pkg/github/pr_request_missing_head.go @@ -2,9 +2,12 @@ package github import ( "context" + "encoding/json" "errors" "fmt" "net/http" + "os" + "path/filepath" "strings" gh "github.com/google/go-github/v72/github" @@ -36,8 +39,8 @@ func (e *errMissingHead) Error() string { e.repo, e.head) } return fmt.Sprintf( - "branch %q was never pushed to %s — the commits exist only in the agent's working copy. This is almost always a PUSH AUTHENTICATION failure, not a branch-creation problem: check (1) that the git credential helper is reachable from the agent's UID (`su -s /bin/sh hive- -c 'git config --get-regexp credential'` must list /usr/local/bin/git-credential-hive.sh; it is wired system-wide in /etc/gitconfig because agents do not share the dev user's $HOME — see hivecommons/hive#5343), and (2) that the agent's scoped token file ($HIVE_AGENT_TOKEN_CACHE) exists and is readable by that UID. Re-push the branch and this request succeeds on retry", - e.head, e.repo) + "branch %q is not on %s — the commits exist only in the agent's working copy, or were pushed under another name or to another remote. Check, in order: (1) the push's own outcome (hive-push-branch writes .result.json under %s; a direct `git push` prints it in the agent's pane), (2) that the git credential helper is reachable from the agent's UID (`su -s /bin/sh hive- -c 'git config --get-regexp credential'` must list /usr/local/bin/git-credential-hive.sh; it is wired system-wide in /etc/gitconfig because agents do not share the dev user's $HOME — see hivecommons/hive#5343), and (3) that the agent's scoped token file ($HIVE_AGENT_TOKEN_CACHE) exists and is readable by that UID. Push the branch and this request succeeds on retry", + e.head, e.repo, PushBranchRequestDir) } // missingHeadReason renders an errMissingHead for the operator-facing surfaces @@ -50,6 +53,74 @@ func missingHeadReason(err error) (string, bool) { return missing.Error(), true } +// missingHeadRemoteRef names exactly what the watcher asked GitHub for when it +// concluded the head is absent: the ref, on the repository, as the App sees it. +// The agent may have pushed somewhere else (a fork, a differently named +// branch); naming the ref checked lets an operator compare it with the push. +func missingHeadRemoteRef(repo, head string) string { + return "refs/heads/" + strings.TrimPrefix(strings.TrimSpace(head), "refs/heads/") + " on github.com/" + strings.TrimSpace(repo) +} + +// pushRelayOutcomeNone is the push outcome reported when the push relay +// (hive-push-branch, PushBranchRequestDir) has no record of the branch: the +// agent either ran `git push` itself — whose result the hive never saw — or +// never pushed at all. +const pushRelayOutcomeNone = "no push-relay record for this branch: the agent pushed directly (result not visible to the hive) or did not push" + +// pushRelayOutcome reports what the hive's own push relay saw for branch, so +// the missing-head report states the push outcome rather than asserting one +// (#16969 reported "the agent's push did not authenticate" for a branch that +// arrived hours later). It reads only the relay's drop box: a queued request +// still waiting for its tick, or the newest result file naming the branch. +func pushRelayOutcome(dir, branch string) string { + branch = strings.TrimPrefix(strings.TrimSpace(branch), "refs/heads/") + if branch == "" { + return pushRelayOutcomeNone + } + entries, err := os.ReadDir(dir) + if err != nil { + return pushRelayOutcomeNone + } + var ( + queued bool + newest PushBranchResponse + haveRes bool + ) + for _, e := range entries { + name := e.Name() + if e.IsDir() || !strings.HasSuffix(name, ".json") { + continue + } + data, _, err := readUntrustedFile(filepath.Join(dir, name), requestMaxBytes) + if err != nil { + continue + } + if strings.HasSuffix(name, ".result.json") { + var res PushBranchResponse + if json.Unmarshal(data, &res) != nil || res.Branch != branch { + continue + } + if !haveRes || res.At > newest.At { + newest, haveRes = res, true + } + continue + } + var req PushBranchRequest + if json.Unmarshal(data, &req) == nil && strings.TrimPrefix(req.Branch, "refs/heads/") == branch { + queued = true + } + } + switch { + case queued: + return "push-relay request for this branch is still queued (not yet pushed)" + case haveRes && newest.OK: + return fmt.Sprintf("push-relay pushed %s as %s at %s", branch, newest.SHA, newest.At) + case haveRes: + return fmt.Sprintf("push-relay FAILED after %d attempt(s) at %s: %s", newest.Attempts, newest.At, newest.Error) + } + return pushRelayOutcomeNone +} + // diagnoseCompareFailure turns a base...head comparison failure into an // actionable error. Any failure that is not a definitive 404 is returned as-is // wrapped with ctx — a 403, a rate limit or a 5xx must keep its own identity so diff --git a/src/pkg/github/pr_request_missing_head_test.go b/src/pkg/github/pr_request_missing_head_test.go index 99c563f089..ed90e78ae4 100644 --- a/src/pkg/github/pr_request_missing_head_test.go +++ b/src/pkg/github/pr_request_missing_head_test.go @@ -54,9 +54,10 @@ func TestContentGateReportsUnpushedBranchAsAuthFailure(t *testing.T) { t.Fatal("a missing head ref must be an error") } - // The whole point: the operator must be pointed at credentials. + // The whole point: the operator must be pointed at the push's own + // outcome first, then at credentials — never at branch creation. got := err.Error() - for _, want := range []string{"never pushed", "PUSH AUTHENTICATION", "git-credential-hive.sh", "HIVE_AGENT_TOKEN_CACHE"} { + for _, want := range []string{"is not on o/r", "result.json", "git-credential-hive.sh", "HIVE_AGENT_TOKEN_CACHE"} { if !strings.Contains(got, want) { t.Errorf("diagnosis does not mention %q:\n%s", want, got) } @@ -94,8 +95,8 @@ func TestClaimsGateReportsUnpushedBranchAsAuthFailure(t *testing.T) { } if reason, ok := missingHeadReason(err); !ok { t.Errorf("claims gate did not diagnose the unpushed branch: %v", err) - } else if !strings.Contains(reason, "PUSH AUTHENTICATION") { - t.Errorf("claims-gate diagnosis is not the push-auth one: %s", reason) + } else if !strings.Contains(reason, "git-credential-hive.sh") { + t.Errorf("claims-gate diagnosis is not the unpushed-branch one: %s", reason) } } @@ -115,7 +116,7 @@ func TestInvisibleRepoIsNotReportedAsAPushFailure(t *testing.T) { if !strings.Contains(got, "cannot see that repository") { t.Errorf("invisible repo not diagnosed:\n%s", got) } - if strings.Contains(got, "PUSH AUTHENTICATION") { + if strings.Contains(got, "git-credential-hive.sh") { t.Errorf("invisible repo was mislabelled as a push failure:\n%s", got) } } @@ -134,7 +135,7 @@ func TestExistingHeadRefDoesNotClaimAPushFailure(t *testing.T) { t.Fatal("a 404 comparison must be an error") } got := err.Error() - if strings.Contains(got, "PUSH AUTHENTICATION") { + if strings.Contains(got, "git-credential-hive.sh") { t.Errorf("a present head ref was blamed on push auth:\n%s", got) } if !strings.Contains(got, "base branch") { diff --git a/src/pkg/github/pr_request_retry_reporting_test.go b/src/pkg/github/pr_request_retry_reporting_test.go index b9a958c64b..a4e80c34ec 100644 --- a/src/pkg/github/pr_request_retry_reporting_test.go +++ b/src/pkg/github/pr_request_retry_reporting_test.go @@ -4,6 +4,7 @@ import ( "bytes" "errors" "log/slog" + "os" "path/filepath" "strings" "testing" @@ -32,3 +33,76 @@ func TestFailPRRequestReportsMissingHeadOnce(t *testing.T) { }) } } + +// #16969: one unpushed branch retried every tick must be ONE fleet-feed +// observation, not one per retry. The first sighting is ERROR and names the +// ref checked and the push outcome seen; retries stay below the feed's WARN +// floor; the eventual success is logged so the story closes. +func TestFailPRRequestMissingHeadErrorIsOncePerRequest(t *testing.T) { + pushBranchRequestDirForTest = t.TempDir() + defer func() { pushBranchRequestDirForTest = "" }() + + var logs bytes.Buffer + c := NewClientForTest("http://127.0.0.1:0", "o", nil, slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug}))) + path := filepath.Join(t.TempDir(), "request.json") + req := PRRequest{Repo: "o/r", Head: "docs/runbook", Agent: "operations"} + err := &errMissingHead{repo: "o/r", head: "docs/runbook", repoVisible: true} + + now := time.Now() + for i := 0; i < 3; i++ { + c.failPRRequest(path, req, err, func() time.Time { return now.Add(time.Duration(i) * time.Minute) }) + } + got := logs.String() + if n := strings.Count(got, "level=ERROR msg=\"pr-request watcher: head branch is not on the remote\""); n != 1 { + t.Fatalf("missing head logged at ERROR %d times across 3 retries, want exactly 1:\n%s", n, got) + } + if n := strings.Count(got, "head branch still not on the remote, retrying"); n != 2 { + t.Fatalf("retries logged %d times, want 2:\n%s", n, got) + } + if strings.Contains(got, "did not authenticate") || strings.Contains(got, "agent's push") { + t.Fatalf("message asserts a cause the watcher did not observe:\n%s", got) + } + for _, want := range []string{ + "checked=\"refs/heads/docs/runbook on github.com/o/r\"", + "push_outcome=\"" + pushRelayOutcomeNone + "\"", + "attempt=1", "attempt=3", "agent=operations", + } { + if !strings.Contains(got, want) { + t.Errorf("log lacks %s:\n%s", want, got) + } + } + if c.prRetries.attempts(path) != 3 { + t.Fatalf("attempts = %d, want 3", c.prRetries.attempts(path)) + } +} + +func TestPushRelayOutcomeReadsTheRelayDropBox(t *testing.T) { + dir := t.TempDir() + write := func(name, body string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + if got := pushRelayOutcome(dir, "docs/x"); got != pushRelayOutcomeNone { + t.Fatalf("empty drop box: %q", got) + } + write("operations-1.result.json", `{"ok":false,"branch":"docs/x","attempts":2,"error":"git push failed: remote: Permission denied","at":"2026-11-07T05:00:00Z"}`) + if got := pushRelayOutcome(dir, "docs/x"); !strings.Contains(got, "FAILED after 2 attempt(s)") || !strings.Contains(got, "Permission denied") { + t.Fatalf("failed relay push not reported: %q", got) + } + // A newer success for the same branch wins over the older failure. + write("operations-2.result.json", `{"ok":true,"branch":"docs/x","sha":"abc123","attempts":1,"at":"2026-11-07T05:05:00Z"}`) + if got := pushRelayOutcome(dir, "docs/x"); !strings.Contains(got, "pushed docs/x as abc123") { + t.Fatalf("newest relay result not preferred: %q", got) + } + // Another branch's result never speaks for this one. + if got := pushRelayOutcome(dir, "docs/other"); got != pushRelayOutcomeNone { + t.Fatalf("unrelated branch matched: %q", got) + } + // A request still queued is reported as such, ahead of any old result. + write("operations-3.json", `{"repo":"o/r","branch":"docs/x","dir":"/data/agents/operations/r"}`) + if got := pushRelayOutcome(dir, "refs/heads/docs/x"); !strings.Contains(got, "still queued") { + t.Fatalf("queued relay request not reported: %q", got) + } +} diff --git a/src/pkg/github/pr_request_watcher.go b/src/pkg/github/pr_request_watcher.go index 25862c3473..5b061701f4 100644 --- a/src/pkg/github/pr_request_watcher.go +++ b/src/pkg/github/pr_request_watcher.go @@ -703,7 +703,14 @@ func (c *Client) handleOnePRRequest(ctx context.Context, path string, nowFn func "duplicate_reason", res.DuplicateReason) c.writePRResult(path, resp) // Success (or reuse of an existing PR) — consume the request so it isn't - // reprocessed. + // reprocessed. A request that had failed before succeeding is the retry + // policy working; say so, so the ERROR that opened the story has a + // matching close in the same log. + if failed := c.prRetries.attempts(path); failed > 0 { + c.logger.Info("pr-request watcher: request succeeded on retry", + slog.String("repo", req.Repo), slog.String("head", req.Head), + slog.Int("failed_attempts", failed), slog.Int("number", res.Number)) + } _ = os.Remove(path) c.prRetries.clear(path) if res.IsDuplicate() { @@ -733,11 +740,29 @@ func (c *Client) failPRRequest(path string, req PRRequest, err error, nowFn func // points away from its cause. Log it at ERROR on its own line — this is // work an agent already completed and cannot publish, and it is invisible // in the fleet view because the agent's session ended healthy. + // + // Once per request, not once per retry (#16969): the retry exists so a + // late push still lands, and re-logging the same request at ERROR every + // tick turned ONE unpushed branch into a "5 in 10m" fleet report before + // the branch arrived. Retries stay at INFO (below the fleet feed) so the + // fleet report fires only when distinct requests fail the same way. The + // message states what the watcher checked and what it saw — the ref, the + // repository, and the push relay's own record — not an inferred cause. reason, missingHead := missingHeadReason(err) if missingHead { - c.logger.Error("pr-request watcher: head branch is not on the remote — the agent's push did not authenticate", + attempt := c.prRetries.attempts(path) + 1 + attrs := []any{ slog.String("repo", req.Repo), slog.String("head", req.Head), - slog.String("agent", req.Agent), slog.String("diagnosis", reason)) + slog.String("checked", missingHeadRemoteRef(req.Repo, req.Head)), + slog.String("push_outcome", pushRelayOutcome(pushBranchRequestDir(), req.Head)), + slog.Int("attempt", attempt), + slog.String("agent", req.Agent), slog.String("diagnosis", reason), + } + if attempt == 1 { + c.logger.Error("pr-request watcher: head branch is not on the remote", attrs...) + } else { + c.logger.Info("pr-request watcher: head branch still not on the remote, retrying", attrs...) + } } if c.prRetries.noteFailure(path, nowFn()) { _ = os.Rename(path, path+".failed") diff --git a/src/pkg/github/request_retry.go b/src/pkg/github/request_retry.go index 402bc2243a..23e2b32626 100644 --- a/src/pkg/github/request_retry.go +++ b/src/pkg/github/request_retry.go @@ -74,6 +74,18 @@ func (t *retryTracker) noteFailure(path string, now time.Time) bool { return now.Sub(st.firstTry) > requestRetryMaxAge } +// attempts reports how many failed attempts path has accumulated so far +// (zero when it has never failed). It lets a caller tell the first sighting +// of a failure apart from its retries before noteFailure records this one. +func (t *retryTracker) attempts(path string) int { + t.mu.Lock() + defer t.mu.Unlock() + if st := t.m[path]; st != nil { + return st.attempts + } + return 0 +} + // clear drops the tracked state for path (request consumed or quarantined). func (t *retryTracker) clear(path string) { t.mu.Lock() From d426e9e91e9ed5948f60ca41b64f02beb84d0215 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Sat, 7 Nov 2026 10:22:35 -0500 Subject: [PATCH 06/17] =?UTF-8?q?=F0=9F=90=9B=20fix(loginscan):=20name=20t?= =?UTF-8?q?he=20backend=20and=20login=20command=20in=20the=20login-require?= =?UTF-8?q?d=20report=20(#16992)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix(loginscan): name the backend and login command in the login-required report The 'login required detected' WARN is what the agent_login_required fleet selfreport is built from, but it carried only agent, pattern and sightings. hivecommons/hive#16953 sat through two decision rounds because nobody could tell from the report WHICH provider (claude/copilot/gemini/ codex) the supervisor agent on hosted-ibm-alchemy-logg-ptoo had to re-authenticate. - Add backend, login_hint (CommandForBackend) and credential_valid to the WARN record. - CommandForBackend: add codex. - fleet-report.md: document the attrs and the operator action. Closes #16953 Signed-off-by: Andrew Anderson * changelog: fragment for #16992 Generated by changelog-fragment-guard from the PR title (#5675). Edit the wording freely, or replace this file with the no-changelog label if the change is not user-visible. Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> * changelog: single bullet fragment for #16953 Signed-off-by: Andrew Anderson --------- Signed-off-by: Andrew Anderson Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../fixed-login-required-report-backend.md | 1 + src/docs/fleet-report.md | 2 +- src/pkg/loginscan/command_test.go | 1 + src/pkg/loginscan/login_scan_loop_test.go | 41 ++++++++++++++++++- src/pkg/loginscan/loginscan.go | 15 +++++-- 5 files changed, 55 insertions(+), 5 deletions(-) create mode 100644 changelog.d/fixed-login-required-report-backend.md diff --git a/changelog.d/fixed-login-required-report-backend.md b/changelog.d/fixed-login-required-report-backend.md new file mode 100644 index 0000000000..afc6ba099c --- /dev/null +++ b/changelog.d/fixed-login-required-report-backend.md @@ -0,0 +1 @@ +- The `agent_login_required` selfreport now names the backend that lost its login: the login scanner's `login required detected` record carries `backend`, `login_hint` (the CLI's login command) and `credential_valid`, so the operator knows which provider to re-authenticate on which agent instead of being told only the agent name; `codex` gets a login hint (`codex login`) ([#16953](https://github.com/hivecommons/hive/issues/16953)). diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index 2676ca9d35..59073d2989 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -140,7 +140,7 @@ title is `[selfreport] : `. | `effect_ledger_op_stuck` | the mutation boundary refused a retry because the journaled external effect (issue create, PR open, …) has been `Unknown`/`Planned` for ≥30 min — the caller keeps replaying the same logical op and it never resolves. Key is the op's logical ID; message carries kind, status and age, diagnostics the attempt count, repo and claim target. Clears when the op is applied, replayed as already-applied, or reconciled | medium | | `agent_dir_setup_failed` | the harness failed to set up the per-agent sandbox directory structure — create XDG directories, restore or tighten directory permissions on interactive/inference home directories. Log-fed: keywords include `failed to create per-agent xdg dir`, `failed to restore per-agent dir mode`, `failed to restore interactive home mode`, `failed to tighten per-agent dir mode`, `failed to tighten interactive home mode`, `failed to restore inference home mode`, `failed to tighten inference home mode` (`crux/agent/manager.go`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `agent_pane_timeout` | the agent-readiness wait timed out and the harness dumped the tmux pane (`prompt timeout — dumping pane` / `pane content at timeout`, `pkg/agent/manager.go`), and the startup kick dropped because the CLI never became ready or reached its input prompt (`startup kick dropped: …`, `crux/agent/manager_kick.go`, the same event as the pane dump). Log-fed: same ≥5-in-10-min repeat threshold and 20-min idle clear as the catch-all below; pane excerpts are scrubbed and bounded before storage | low | -| `agent_login_required` | the login scanner saw the agent CLI asking for `/login` (`login required detected`, `pkg/loginscan`) — the agent's auth lapsed and it is paused until an operator re-authenticates. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | +| `agent_login_required` | the login scanner saw the agent CLI asking for `/login` (`login required detected`, `pkg/loginscan`) — the agent's auth lapsed and it is paused until an operator re-authenticates. The record carries `backend` (which provider to re-authenticate), `login_hint` (the CLI's login command) and `credential_valid`; re-login in the agent's tmux pane, then Resume. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `pr_request_head_unpushed` | the pr-request watcher found the PR head branch missing on the remote (`head branch is not on the remote`): the agent's push failed, never ran, or went to another ref — the line's `checked` and `push_outcome` attributes say which. Logged once per request (retries stay at INFO), so the repeat threshold counts distinct unpushed requests, not one request's retries ([#16969](https://github.com/hivecommons/hive/issues/16969)). Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `bead_fact_ingest_failed` | the knowledge system failed to ingest a bead fact from an agent (`failed to ingest bead fact`, `pkg/knowledge`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `feedback_issue_refresh_failed` | the dashboard's feedback-issue watcher failed to refresh or update a feedback issue (`feedback: issue refresh failed`, `pkg/dashboard`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | diff --git a/src/pkg/loginscan/command_test.go b/src/pkg/loginscan/command_test.go index 409465661c..47fc0ddff4 100644 --- a/src/pkg/loginscan/command_test.go +++ b/src/pkg/loginscan/command_test.go @@ -13,6 +13,7 @@ func TestCommandForBackend(t *testing.T) { {"copilot", "Run: copilot auth login"}, {"gemini", "Run: gemini auth login"}, {"goose", "Run: goose auth login"}, + {"codex", "Run: codex login"}, {"unknown-backend", "Run the login command for unknown-backend"}, {"", "Run the login command for "}, } diff --git a/src/pkg/loginscan/login_scan_loop_test.go b/src/pkg/loginscan/login_scan_loop_test.go index 8c6860efd1..9d0c7b8de4 100644 --- a/src/pkg/loginscan/login_scan_loop_test.go +++ b/src/pkg/loginscan/login_scan_loop_test.go @@ -1,13 +1,16 @@ package loginscan import ( + "bytes" "context" "errors" "fmt" + "log/slog" + "strings" "testing" - "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/notify" ) @@ -241,3 +244,39 @@ func indexBefore(values []string, first, second string) bool { } return firstIndex >= 0 && secondIndex >= 0 && firstIndex < secondIndex } + +// The "login required detected" WARN feeds the agent_login_required fleet +// selfreport. hivecommons/hive#16953 was parked through two decision rounds +// because the report could not say which backend had lost its login; the +// record has to carry the backend and the operator's login hint. +func TestScanLoginRequiredWarnNamesBackendAndLoginHint(t *testing.T) { + t.Parallel() + + mgr := &fakeLoginScanManager{ + statuses: map[string]*crux.Process{"supervisor": {State: crux.StateRunning}}, + outputs: map[string][]string{"supervisor": {"Please run /login"}}, + } + sightings := NewSightingTracker() + sightings.Observe("supervisor", true) + cfg := loginScanLoopConfig([]string{`(?i)Please run /login`}, map[string]string{"supervisor": "claude"}) + + var buf bytes.Buffer + logger := slog.New(slog.NewTextHandler(&buf, nil)) + + Scan(context.Background(), cfg, mgr, &fakeLoginScanNotifier{}, &fakeLoginScanAuditor{}, logger, sightings) + + line := "" + for _, l := range strings.Split(buf.String(), "\n") { + if strings.Contains(l, "login required detected") { + line = l + } + } + if line == "" { + t.Fatalf("no 'login required detected' WARN in:\n%s", buf.String()) + } + for _, want := range []string{"level=WARN", "agent=supervisor", "backend=claude", `login_hint="Run: claude login"`, "credential_valid=false", "sightings=2"} { + if !strings.Contains(line, want) { + t.Errorf("WARN record missing %q:\n%s", want, line) + } + } +} diff --git a/src/pkg/loginscan/loginscan.go b/src/pkg/loginscan/loginscan.go index 1e90e90ac3..ed53c60434 100644 --- a/src/pkg/loginscan/loginscan.go +++ b/src/pkg/loginscan/loginscan.go @@ -30,6 +30,8 @@ func CommandForBackend(backend string) string { return "Run: gemini auth login" case "goose": return "Run: goose auth login" + case "codex": + return "Run: codex login" default: return "Run the login command for " + backend } @@ -300,8 +302,18 @@ func Scan( "sightings", streak, "required", PauseMinSightings) continue case ActionPause: + // Determine the login instruction based on the agent's backend + loginCmd := CommandForBackend(backend) + + // This WARN is what the fleet selfreport for class + // agent_login_required is built from (hivecommons/hive#16953): + // without the backend and the login hint the report cannot say + // WHICH provider the operator has to re-authenticate. logger.Warn("login required detected", "agent", name, + "backend", backend, + "login_hint", loginCmd, + "credential_valid", credentialValid, "pattern", re.String(), "sightings", streak, ) @@ -326,9 +338,6 @@ func Scan( dashSrv.AuditLog("system", "pause", "trigger=login-detector", name) } - // Determine the login instruction based on the agent's backend - loginCmd := CommandForBackend(backend) - notifier.Send( fmt.Sprintf("\U0001F511 Login required: %s", name), fmt.Sprintf( From dfe70c162e26bc25078a7063c40f38b6181e7122 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Sat, 7 Nov 2026 10:30:43 -0500 Subject: [PATCH 07/17] =?UTF-8?q?=F0=9F=90=9B=20fix(automerge):=20recover?= =?UTF-8?q?=20a=20flaky=20red=20check=20on=20an=20operator-admitted=20cont?= =?UTF-8?q?ributor=20head=20(#16999)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A person's Approve stands the hive down from adopting the head, but a red head cannot be merged as-is either, and a fork author cannot re-run upstream workflows. The stale-red recovery (update-branch, else re-run failed jobs) was gated on an in-place adoption, so the flake had no owner and the hive nudged the contributor to push a new head. Run the recovery where the admission stand-down happens, and count cancelled workflow runs (runner stalls) as re-runnable. Fixes #16998 Signed-off-by: Andrew Anderson --- .../fixed-admitted-head-stale-red-recovery.md | 1 + .../automerge/automerge_adoption_recovery.go | 54 +++++++++++++++--- .../automerge_adoption_recovery_test.go | 9 +++ .../automerge_contributor_adopt_test.go | 3 +- .../automerge/automerge_operator_admission.go | 11 ++++ .../automerge_operator_admission_test.go | 55 +++++++++++++++++++ 6 files changed, 125 insertions(+), 8 deletions(-) create mode 100644 changelog.d/fixed-admitted-head-stale-red-recovery.md diff --git a/changelog.d/fixed-admitted-head-stale-red-recovery.md b/changelog.d/fixed-admitted-head-stale-red-recovery.md new file mode 100644 index 0000000000..44ed0fddb6 --- /dev/null +++ b/changelog.d/fixed-admitted-head-stale-red-recovery.md @@ -0,0 +1 @@ +- The merge sweep now re-runs cancelled or failed workflow jobs (or refreshes the branch) on an operator-approved contributor PR whose only blocker is a red check that is green on the base, instead of standing down from adoption and nudging the fork author to push a new head for a shared-CI flake they cannot re-run; cancelled runs now count as re-runnable in the adopted-PR recovery too. diff --git a/src/pkg/github/automerge/automerge_adoption_recovery.go b/src/pkg/github/automerge/automerge_adoption_recovery.go index 96a7cfd637..a7ecc2599b 100644 --- a/src/pkg/github/automerge/automerge_adoption_recovery.go +++ b/src/pkg/github/automerge/automerge_adoption_recovery.go @@ -80,6 +80,27 @@ func adoptionRecoveryCandidate(obs stageObservation, rec stagelabels.Record) boo if rec.AdoptionEscalated(obs.HeadSHA) && rec.AdoptEscalatedReason == adoptionWorkflowRefusedReason { return false } + return staleRedRecoverable(obs) +} + +// admittedRecoveryCandidate reports a contributor head a person admitted +// (hivecommons/hive#15136) whose only blocker is red CI. The admission +// makes the hive stand down from adoption, and a fork author cannot re-run +// workflows in the upstream repository, so without this nobody owns a +// shared-CI flake on that head: the hive would nudge the contributor to +// push a new head for a failure that is not theirs (hivecommons/hive#16718, +// where the red head was the hive's own base refresh). +func admittedRecoveryCandidate(obs stageObservation) bool { + if obs.HeadSHA == "" { + return false + } + return staleRedRecoverable(obs) +} + +// staleRedRecoverable reports the shared guards of both recovery +// candidates: red CI is the lane's reason, and nothing else (hold label, +// review blocker, shared-CI incident, red base, fork guard) defers it. +func staleRedRecoverable(obs stageObservation) bool { if adoptionBlocked(obs.Labels) != "" || verdictBlocks(obs.Detail) || verdictNeedsHuman(obs.Detail) { return false } @@ -178,12 +199,28 @@ func (c *Engine) recoverStaleRedAdoption(ctx context.Context, store *stagelabels if !adoptionRecoveryCandidate(obs, rec) { return adoptionRecoverySkipped("not-a-candidate") } + return c.recoverStaleRedHead(ctx, store, key, obs, rec, "adopted PR recovery") +} + +// recoverStaleRedAdmittedHead runs where a person's admission makes the +// adoption engine stand down from a red contributor head: the same +// update-branch / re-run recovery, same one-attempt-per-(head, base tip) +// and lifetime budget on the record. The stand-down itself is unchanged; +// this only gives the flake an owner. +func (c *Engine) recoverStaleRedAdmittedHead(ctx context.Context, store *stagelabels.Store, key string, obs stageObservation, rec stagelabels.Record) adoptionRecovery { + if !admittedRecoveryCandidate(obs) { + return adoptionRecoverySkipped("not-a-candidate") + } + return c.recoverStaleRedHead(ctx, store, key, obs, rec, "admitted head recovery") +} + +func (c *Engine) recoverStaleRedHead(ctx context.Context, store *stagelabels.Store, key string, obs stageObservation, rec stagelabels.Record, logMsg string) adoptionRecovery { logArgs := func(extra ...any) []any { return append([]any{"repo", obs.Display, "pr", obs.Number, "head", obs.HeadSHA, "base", obs.BaseRef, "lane_reason", nonEmpty(obs.Reason, obs.LaneReason)}, extra...) } if rec.AdoptRecoveryCount >= adoptionRecoveryMaxAttempts { out := adoptionRecoverySkipped("attempts-exhausted") - c.info("adopted PR recovery", logArgs("reason", out.Reason, "attempts", rec.AdoptRecoveryCount)...) + c.info(logMsg, logArgs("reason", out.Reason, "attempts", rec.AdoptRecoveryCount)...) return out } baseRef := strings.TrimSpace(obs.BaseRef) @@ -193,7 +230,7 @@ func (c *Engine) recoverStaleRedAdoption(ctx context.Context, store *stagelabels restCtx := hgithub.WithRESTCaller(ctx, "hive:adopted_recovery_compare") cmp, _, err := c.gh.Repositories.CompareCommits(restCtx, obs.Owner, obs.Repo, baseRef, obs.HeadSHA, &gh.ListOptions{PerPage: 1}) if err != nil { - c.warn("adopted PR recovery", logArgs("reason", adoptionRecoverySkipPrefix+"compare-failed", "error", err)...) + c.warn(logMsg, logArgs("reason", adoptionRecoverySkipPrefix+"compare-failed", "error", err)...) return adoptionRecoverySkipped("compare-failed") } baseSHA := cmp.GetBaseCommit().GetSHA() @@ -207,13 +244,13 @@ func (c *Engine) recoverStaleRedAdoption(ctx context.Context, store *stagelabels } st, err := c.baseChecks(ctx, obs.Owner, obs.Repo, baseSHA) if err != nil { - c.warn("adopted PR recovery", logArgs("reason", adoptionRecoverySkipPrefix+"base-checks-failed", "error", err)...) + c.warn(logMsg, logArgs("reason", adoptionRecoverySkipPrefix+"base-checks-failed", "error", err)...) return adoptionRecoverySkipped("base-checks-failed") } failing := adoptionFailingCheck(nonEmpty(obs.Reason, obs.LaneReason)) if ok, skip := baseClearsFailure(st, failing); !ok { out := adoptionRecoverySkipped(skip) - c.info("adopted PR recovery", logArgs("reason", out.Reason, "base_sha", baseSHA, "failing_check", failing)...) + c.info(logMsg, logArgs("reason", out.Reason, "base_sha", baseSHA, "failing_check", failing)...) return out } behind := cmp.GetBehindBy() @@ -222,7 +259,7 @@ func (c *Engine) recoverStaleRedAdoption(ctx context.Context, store *stagelabels // The bot rebases its own branch; a hive merge commit would stop // it doing so for good (projectbluefin/utah#1097). out := adoptionRecoverySkipped("dependency-bot-branch") - c.info("adopted PR recovery", logArgs("reason", out.Reason, "base_sha", baseSHA, "behind_by", behind, "head_ref", obs.HeadRef, "author", obs.Author)...) + c.info(logMsg, logArgs("reason", out.Reason, "base_sha", baseSHA, "behind_by", behind, "head_ref", obs.HeadRef, "author", obs.Author)...) store.MarkAdoptionRecovery(key, obs.HeadSHA, baseSHA) return out } @@ -312,7 +349,10 @@ func (c *Engine) recoverByRerun(ctx context.Context, obs stageObservation, logAr } // latestFailedWorkflowRuns returns, per workflow, the newest run for sha -// that completed as failure or timed_out. +// that completed as failure, timed_out or cancelled. A cancelled run is +// what a runner stall leaves behind (a job cancelled at setup after the +// workflow timeout, hivecommons/hive#16718): the CI gate reads it as red, +// and GitHub re-runs its cancelled jobs through the same endpoint. func (c *Engine) latestFailedWorkflowRuns(ctx context.Context, owner, repo, sha string) ([]*gh.WorkflowRun, error) { opts := &gh.ListWorkflowRunsOptions{HeadSHA: sha, ListOptions: gh.ListOptions{PerPage: 100}} latest := map[int64]*gh.WorkflowRun{} @@ -344,7 +384,7 @@ func (c *Engine) latestFailedWorkflowRuns(ctx context.Context, owner, repo, sha continue } switch run.GetConclusion() { - case "failure", "timed_out": + case "failure", "timed_out", "cancelled": out = append(out, run) } } diff --git a/src/pkg/github/automerge/automerge_adoption_recovery_test.go b/src/pkg/github/automerge/automerge_adoption_recovery_test.go index 2d5e02c9c4..75ec7c9f4b 100644 --- a/src/pkg/github/automerge/automerge_adoption_recovery_test.go +++ b/src/pkg/github/automerge/automerge_adoption_recovery_test.go @@ -51,6 +51,15 @@ func TestAdoptedStaleRedRecovery(t *testing.T) { }, wantReruns: 2, wantStage: stagelabels.AdoptedByHive, wantAttempt: "aaa@base1", }, + { + name: "a cancelled run (runner stall) is re-run like a failed one", + setup: func(a *adoptAPI) { + a.baseChecks = map[string]string{"coverage": "success"} + a.failedRuns = []int64{41} + a.failedRunConclusion = "cancelled" + }, + wantReruns: 1, wantStage: stagelabels.AdoptedByHive, wantAttempt: "aaa@base1", + }, { name: "base does not run the check but is fully green: still recoverable", setup: func(a *adoptAPI) { diff --git a/src/pkg/github/automerge/automerge_contributor_adopt_test.go b/src/pkg/github/automerge/automerge_contributor_adopt_test.go index 55a7183be9..4a5d08436e 100644 --- a/src/pkg/github/automerge/automerge_contributor_adopt_test.go +++ b/src/pkg/github/automerge/automerge_contributor_adopt_test.go @@ -69,6 +69,7 @@ type adoptAPI struct { baseChecks map[string]string failBaseChecks bool failedRuns []int64 + failedRunConclusion string // conclusion of failedRuns; "" means failure updateBranchStatus int updateBranchMessage string branchUpdates []string @@ -316,7 +317,7 @@ func (a *adoptAPI) handle(w http.ResponseWriter, r *http.Request) bool { sha := r.URL.Query().Get("head_sha") runs := []map[string]any{} for _, id := range a.failedRuns { - runs = append(runs, map[string]any{"id": id, "workflow_id": id, "run_number": 1, "run_attempt": 1, "head_sha": sha, "status": "completed", "conclusion": "failure"}) + runs = append(runs, map[string]any{"id": id, "workflow_id": id, "run_number": 1, "run_attempt": 1, "head_sha": sha, "status": "completed", "conclusion": nonEmpty(a.failedRunConclusion, "failure")}) } _ = json.NewEncoder(w).Encode(map[string]any{"total_count": len(runs), "workflow_runs": runs}) return true diff --git a/src/pkg/github/automerge/automerge_operator_admission.go b/src/pkg/github/automerge/automerge_operator_admission.go index 0471a562e6..54f736f127 100644 --- a/src/pkg/github/automerge/automerge_operator_admission.go +++ b/src/pkg/github/automerge/automerge_operator_admission.go @@ -81,6 +81,17 @@ func (c *Engine) adoptionRefusedByAdmission(ctx context.Context, store *stagelab } store.NoteAdoptionBlocked(key, obs.HeadSHA, stagelabels.AdoptBlockedOperatorAdmission, now) c.info("stage labels: adoption due but a person admitted this head; not adopting", "repo", obs.Display, "pr", obs.Number, "author", obs.Author, "head", obs.HeadSHA, "admitted_by", stagelabels.OperatorApprovalAdmission(login)) + // Standing down must not strand the head: a fork author cannot re-run + // upstream workflows, and the admission keeps the hive from adopting, + // so a shared-CI flake on the admitted head (often the hive's own base + // refresh) has no owner. The adopted-PR recovery — update-branch when + // behind, else re-run the failed jobs, once per (head, base tip) — runs + // here too (hivecommons/hive#16718). + if rec, ok := store.Get(key); ok { + if recovery := c.recoverStaleRedAdmittedHead(ctx, store, key, obs, rec); recovery.Attempted { + c.info("stage labels: admitted head is red for a reason the base clears; recovering instead of waiting on the author", "repo", obs.Display, "pr", obs.Number, "author", obs.Author, "head", obs.HeadSHA, "admitted_by", stagelabels.OperatorApprovalAdmission(login), "recovery", recovery.Reason) + } + } return true } diff --git a/src/pkg/github/automerge/automerge_operator_admission_test.go b/src/pkg/github/automerge/automerge_operator_admission_test.go index 058f9d8bca..9dab1a98da 100644 --- a/src/pkg/github/automerge/automerge_operator_admission_test.go +++ b/src/pkg/github/automerge/automerge_operator_admission_test.go @@ -92,6 +92,61 @@ func TestOperatorAdmissionDerivedFromHumanApproval(t *testing.T) { // Nothing admitted: a nil engine and an unknown head both answer "" and // the adoption refusal is a no-op. +// An admitted head that is red for a reason the base clears is nobody's to +// fix without this: the admission stands the hive down from adoption and a +// fork author cannot re-run upstream workflows (hivecommons/hive#16718, +// where the red head was the hive's own base refresh and the failing shard +// was cancelled at setup). The stand-down holds, and the stale-red recovery +// re-runs the failed jobs once per (head, base tip). +func TestOperatorAdmissionRecoversStaleRedHead(t *testing.T) { + h, done := newAdoptHarness(t, ContributorNudgeOutcome{Mode: "comment"}, adoptPolicy()) + defer done() + h.observe(adoptPR(7, "aaa"), recoveryReason) + key := stageKey("acme", "widget", 7) + store := h.e.stageStoreOrShared() + if !store.NoteAdmission(key, "aaa", stagelabels.OperatorApprovalAdmission("octo"), h.clock.now()) { + t.Fatalf("NoteAdmission refused a known key") + } + h.adopt.compareBase = "base1" + h.adopt.baseChecks = map[string]string{"coverage": "success"} + h.adopt.failedRuns = []int64{41} + h.adopt.failedRunConclusion = "cancelled" + labels := []string{"stage/072-returned-to-contributor", "role/contributor", "author/clanker"} + + h.api.reset() + h.clock.advance(5 * time.Hour) + h.observe(adoptPR(7, "aaa", labels...), recoveryReason) + rec, _ := store.Get(key) + if len(h.adopt.pulls) != 0 || rec.AdoptedHead != "" || rec.AdoptionBlocked("aaa") != stagelabels.AdoptBlockedOperatorAdmission { + t.Fatalf("admission no longer stands the hive down: pulls=%d rec=%+v", len(h.adopt.pulls), rec) + } + if len(h.adopt.reruns) != 1 || h.adopt.reruns[0] != 41 || len(h.adopt.branchUpdates) != 0 { + t.Fatalf("admitted red head not re-run: reruns=%v updates=%v", h.adopt.reruns, h.adopt.branchUpdates) + } + if rec.AdoptRecoveryAttempt != "aaa@base1" || rec.AdoptRecoveryCount != 1 { + t.Fatalf("recovery attempt not recorded: %+v", rec) + } + + // Still red on the same head and base tip: no second re-run. + h.api.reset() + h.clock.advance(time.Hour) + h.observe(adoptPR(7, "aaa", labels...), recoveryReason) + if len(h.adopt.reruns) != 1 { + t.Fatalf("recovery repeated on the same (head, base): %v", h.adopt.reruns) + } + + // Check red on the base too: the failure is not a flake, nothing to re-run. + h.api.reset() + h.adopt.compareBase = "base2" + h.adopt.baseChecks = map[string]string{"coverage": "failure"} + h.clock.advance(time.Hour) + h.observe(adoptPR(7, "aaa", labels...), recoveryReason) + rec, _ = store.Get(key) + if len(h.adopt.reruns) != 1 || rec.AdoptRecoveryCount != 1 { + t.Fatalf("re-ran a failure the base shares: reruns=%v rec=%+v", h.adopt.reruns, rec) + } +} + func TestOperatorAdmissionAbsent(t *testing.T) { var nilEngine *Engine if got := nilEngine.operatorAdmission(context.Background(), "acme", "widget", 7, "aaa"); got != "" { From dfe8e33d9c02a9fa97733a6f3f990a2a69113f87 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Sat, 7 Nov 2026 10:51:50 -0500 Subject: [PATCH 08/17] =?UTF-8?q?=E2=9C=A8=20crux:=20caller=20principal=20?= =?UTF-8?q?on=20serve=20routes=20for=20work=20services=20(#16996)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit serve.authenticate now attaches a Principal to every request it admits: the control token (or a token-less server's own unix socket) is the operator, named by --operator-name / CRUX_OPERATOR_NAME; a lane project credential is a project principal, accepted only on routes a work-service family registered (Server.workService) and still refused on agent RPCs. serve.CallerFrom(ctx) reads it. Prerequisite (S0) for S1 ownership and S3 gates. Refs hivecommons/crux#114 Signed-off-by: Andrew Anderson --- changelog.d/added-crux-caller-principal.md | 1 + src/crux/cli/serve.go | 8 +- src/crux/config/serve.go | 4 + src/crux/serve/principal.go | 85 +++++++++++++ src/crux/serve/principal_test.go | 132 +++++++++++++++++++++ src/crux/serve/serve.go | 24 +++- src/docs/crux-serve.md | 18 +++ 7 files changed, 266 insertions(+), 6 deletions(-) create mode 100644 changelog.d/added-crux-caller-principal.md create mode 100644 src/crux/serve/principal.go create mode 100644 src/crux/serve/principal_test.go diff --git a/changelog.d/added-crux-caller-principal.md b/changelog.d/added-crux-caller-principal.md new file mode 100644 index 0000000000..6956e0dbe3 --- /dev/null +++ b/changelog.d/added-crux-caller-principal.md @@ -0,0 +1 @@ +- crux: `serve` attaches a caller principal to every authenticated request (`serve.CallerFrom`): the control token is the operator (named by `--operator-name`/`CRUX_OPERATOR_NAME`), and lane project credentials are accepted as project principals on work-services routes only — agent RPCs still refuse them. Groundwork for the crux work services (hivecommons/crux#114 S0). diff --git a/src/crux/cli/serve.go b/src/crux/cli/serve.go index 5549ed9496..9923fb5419 100644 --- a/src/crux/cli/serve.go +++ b/src/crux/cli/serve.go @@ -33,13 +33,15 @@ func (e *Env) cmdServe(ctx context.Context, opts globalOptions, args []string) i fs.StringVar(&cfg.Token, "token", "", "bearer token every request must carry (default $"+config.ServeTokenEnvVar+")") var hiveAddr string var laneConfig string + var operatorName string + fs.StringVar(&operatorName, "operator-name", "", "name of the principal behind the control token on work-services routes (default $"+config.ServeOperatorNameEnvVar+", then "+serve.DefaultOperatorName+")") fs.StringVar(&laneConfig, "lanes-config", "", "opt-in durable lane configuration file (default $CRUX_LANES_CONFIG)") fs.StringVar(&hiveAddr, "hive-callback", "", "hive capability endpoint to dial (default $"+callback.SocketEnvVar+"; none when unset)") if code, ok := parseSub(fs, args); !ok { return code } if fs.NArg() != 0 { - fmt.Fprintln(e.Stderr, "usage: crux serve [-socket PATH] [-listen HOST:PORT] [-token T] [-hive-callback ADDR] [-lanes-config PATH]") + fmt.Fprintln(e.Stderr, "usage: crux serve [-socket PATH] [-listen HOST:PORT] [-token T] [-hive-callback ADDR] [-lanes-config PATH] [-operator-name NAME]") return ExitUsage } if cfg.Token == "" { @@ -70,6 +72,10 @@ func (e *Env) cmdServe(ctx context.Context, opts globalOptions, args []string) i srv := serve.New(rt) srv.Token = cfg.Token + if operatorName == "" { + operatorName = os.Getenv(config.ServeOperatorNameEnvVar) + } + srv.OperatorName = operatorName srv.Logger = slog.New(slog.NewTextHandler(e.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})) if laneConfig == "" { laneConfig = os.Getenv("CRUX_LANES_CONFIG") diff --git a/src/crux/config/serve.go b/src/crux/config/serve.go index 9971fa19e6..1e90a0fb96 100644 --- a/src/crux/config/serve.go +++ b/src/crux/config/serve.go @@ -25,6 +25,10 @@ const ( // ServeTokenEnvVar is the bearer token the server requires and the client // presents. A TCP listener refuses to start without one. ServeTokenEnvVar = "CRUX_TOKEN" + // ServeOperatorNameEnvVar names the principal the control token stands + // for on the work-services routes (serve.CallerFrom); the server + // defaults it when unset. + ServeOperatorNameEnvVar = "CRUX_OPERATOR_NAME" // ServeSocketName is the socket file name under ServeSocketDir. ServeSocketName = "crux.sock" // ServeSocketDirName is the per-user directory under $XDG_RUNTIME_DIR or diff --git a/src/crux/serve/principal.go b/src/crux/serve/principal.go new file mode 100644 index 0000000000..fd3f53f2da --- /dev/null +++ b/src/crux/serve/principal.go @@ -0,0 +1,85 @@ +package serve + +import ( + "context" + "net/http" + "strings" +) + +// Caller identity on serve routes (hivecommons/crux#114 S0). Every request +// that passes authenticate carries a Principal in its context: the shared +// control token (or the unauthenticated unix socket of a token-less +// server) is the operator; a lane project credential (lanes.Authenticator) +// is a project principal, accepted only on the routes a work-service +// family registered through workService. Agent RPCs keep refusing project +// credentials exactly as before — the work services are the only surface +// where a non-operator caller exists, because they own records per caller +// (S1 checkouts) and gate answers by caller (S3). + +const ( + // KindOperator is the principal behind the shared control token. It may + // act on every record regardless of owner. + KindOperator = "operator" + // KindProject is a lane project principal (lanes.Config Principals); its + // Name is the project id. + KindProject = "project" + // DefaultOperatorName is Principal.Name for the operator when the + // server was given none. + DefaultOperatorName = "operator" +) + +// Principal is who made an authenticated request. +type Principal struct { + // Kind is KindOperator or KindProject. + Kind string `json:"kind"` + // Name is the operator name or the lane project id. + Name string `json:"name"` +} + +// IsOperator reports whether p holds the shared control credential. +func (p Principal) IsOperator() bool { return p.Kind == KindOperator } + +// String renders kind:name for logs and error messages. +func (p Principal) String() string { return p.Kind + ":" + p.Name } + +// principalKey carries the Principal through the request context. +type principalKey struct{} + +// CallerFrom returns the Principal authenticate attached to ctx. ok is +// false on a context that did not pass through the server's auth gate. +func CallerFrom(ctx context.Context) (p Principal, ok bool) { + p, ok = ctx.Value(principalKey{}).(Principal) + return p, ok +} + +// withCaller returns r with p as its caller. +func withCaller(r *http.Request, p Principal) *http.Request { + return r.WithContext(context.WithValue(r.Context(), principalKey{}, p)) +} + +// operator is the principal for the shared control credential. +func (s *Server) operator() Principal { + name := s.OperatorName + if name == "" { + name = DefaultOperatorName + } + return Principal{Kind: KindOperator, Name: name} +} + +// workService marks every route under prefix (the path itself and its +// subpaths) as a work-services route: project principals are accepted there +// and nowhere else. Handler calls it for each family before the mux is +// wrapped in authenticate. +func (s *Server) workService(prefix string) { + s.workServices = append(s.workServices, prefix) +} + +// isWorkService reports whether path belongs to a work-services family. +func (s *Server) isWorkService(path string) bool { + for _, prefix := range s.workServices { + if path == prefix || strings.HasPrefix(path, prefix+"/") { + return true + } + } + return false +} diff --git a/src/crux/serve/principal_test.go b/src/crux/serve/principal_test.go new file mode 100644 index 0000000000..6a060a1680 --- /dev/null +++ b/src/crux/serve/principal_test.go @@ -0,0 +1,132 @@ +package serve + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/lanes" +) + +const testProjectToken = "pppppppppppppppppppppppppppppppp" + +// principalFixture is a token-bearing server with one lane project +// ("atlas") and a work-services route at /v1/whoami that echoes the caller. +func principalFixture(t *testing.T) (*Server, http.Handler) { + t.Helper() + cfg := lanes.Config{Principals: []lanes.Principal{{Project: "atlas", TokenEnv: "ATLAS"}}} + auth, err := lanes.Bind(cfg, func(string) string { return testProjectToken }, "agent-admin") + if err != nil { + t.Fatal(err) + } + s := New(newFakeRuntime()) + s.Token, s.LaneAuth = "agent-admin", auth + mux := http.NewServeMux() + mux.HandleFunc("GET "+api.PathHealth, s.handleHealth) + mux.HandleFunc("GET /v1/whoami", func(w http.ResponseWriter, r *http.Request) { + p, ok := CallerFrom(r.Context()) + if !ok { + s.writeError(w, http.StatusInternalServerError, api.CodeRuntime, "no caller") + return + } + s.writeJSON(w, http.StatusOK, p) + }) + mux.HandleFunc("GET /v1/whoami/{id}", func(w http.ResponseWriter, r *http.Request) { + p, _ := CallerFrom(r.Context()) + s.writeJSON(w, http.StatusOK, p) + }) + s.workService("/v1/whoami") + return s, s.authenticate(mux) +} + +func callWith(h http.Handler, path, token string) (*httptest.ResponseRecorder, Principal, api.Error) { + req := httptest.NewRequest(http.MethodGet, path, nil) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + var p Principal + var e api.Error + if rec.Code == http.StatusOK { + _ = json.Unmarshal(rec.Body.Bytes(), &p) + } else { + _ = json.Unmarshal(rec.Body.Bytes(), &e) + } + return rec, p, e +} + +func TestPrincipalControlTokenIsOperator(t *testing.T) { + s, h := principalFixture(t) + rec, p, _ := callWith(h, "/v1/whoami", "agent-admin") + if rec.Code != http.StatusOK || p != (Principal{Kind: KindOperator, Name: DefaultOperatorName}) { + t.Fatalf("control token: %d %+v", rec.Code, p) + } + if !p.IsOperator() || p.String() != "operator:operator" { + t.Fatalf("operator predicate/string: %v %q", p.IsOperator(), p) + } + s.OperatorName = "hive-prod" + if _, p, _ = callWith(h, "/v1/whoami", "agent-admin"); p.Name != "hive-prod" { + t.Fatalf("configured operator name: %+v", p) + } + // The operator is also the caller on agent RPCs, which keep working. + if rec, _, _ := callWith(h, api.PathHealth, "agent-admin"); rec.Code != http.StatusOK { + t.Fatalf("health with control token: %d", rec.Code) + } +} + +func TestPrincipalProjectTokenOnlyOnWorkServices(t *testing.T) { + _, h := principalFixture(t) + for _, path := range []string{"/v1/whoami", "/v1/whoami/chk_1"} { + rec, p, _ := callWith(h, path, testProjectToken) + if rec.Code != http.StatusOK || p != (Principal{Kind: KindProject, Name: "atlas"}) { + t.Fatalf("project token on %s: %d %+v %s", path, rec.Code, p, rec.Body) + } + if p.IsOperator() { + t.Fatal("project principal must not be the operator") + } + } + // A path that merely shares the prefix string is not a work service. + if rec, _, e := callWith(h, "/v1/whoamix", testProjectToken); rec.Code != http.StatusUnauthorized || e.Code != api.CodeUnauthorized { + t.Fatalf("prefix lookalike: %d %+v", rec.Code, e) + } + // Agent RPCs refuse project credentials exactly as before. + rec, _, e := callWith(h, api.PathHealth, testProjectToken) + if rec.Code != http.StatusUnauthorized || e.Code != api.CodeUnauthorized || !strings.Contains(e.Error, "project credentials cannot access agent RPCs") { + t.Fatalf("project token on agent RPC: %d %+v", rec.Code, e) + } +} + +func TestPrincipalMissingOrWrongTokenUnchanged(t *testing.T) { + _, h := principalFixture(t) + for _, token := range []string{"", "wrong"} { + for _, path := range []string{"/v1/whoami", api.PathHealth} { + rec, _, e := callWith(h, path, token) + if rec.Code != http.StatusUnauthorized || e.Code != api.CodeUnauthorized || rec.Header().Get("WWW-Authenticate") == "" { + t.Fatalf("token %q on %s: %d %+v", token, path, rec.Code, e) + } + } + } +} + +func TestPrincipalTokenlessUnixSocketIsOperator(t *testing.T) { + s := New(newFakeRuntime()) + var got Principal + var ok bool + h := s.authenticate(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got, ok = CallerFrom(r.Context()) + })) + req := httptest.NewRequest(http.MethodGet, api.PathHealth, nil) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req.WithContext(context.WithValue(req.Context(), networkKey, networkUnix))) + if rec.Code != http.StatusOK || !ok || got != s.operator() { + t.Fatalf("token-less unix request: %d %v %+v", rec.Code, ok, got) + } + if _, ok := CallerFrom(context.Background()); ok { + t.Fatal("a context outside the auth gate has no caller") + } +} diff --git a/src/crux/serve/serve.go b/src/crux/serve/serve.go index a88d328b12..4e79f10452 100644 --- a/src/crux/serve/serve.go +++ b/src/crux/serve/serve.go @@ -101,6 +101,9 @@ type Server struct { // lane surface. They confer no agent-control permissions. Lanes *lanes.Broker LaneAuth *lanes.Authenticator + // OperatorName names the principal behind the shared control token + // (CallerFrom); empty means DefaultOperatorName. + OperatorName string now func() time.Time started time.Time @@ -109,6 +112,9 @@ type Server struct { subscribers map[chan api.Event]struct{} lastStates map[string]string background []string + // workServices lists the route prefixes project principals may call; + // see workService. + workServices []string } // New returns a Server for rt with the defaults filled in. @@ -166,13 +172,21 @@ var agentNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]*$`) // authenticate is the fail-closed gate: when the server has no token, only // a request Serve tagged as arriving on the unix socket is allowed, so a // Handler mounted elsewhere (any TCP listener, tagged or not) is refused; -// when the server has a token, every request must present it. +// when the server has a token, every request must present it. Whoever +// passes is attached to the request as a Principal (CallerFrom): the +// control token is the operator, and a lane project credential is a +// project principal on work-services routes only — on agent RPCs it is +// refused as before. func (s *Server) authenticate(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { network, _ := r.Context().Value(networkKey).(string) token, _ := strings.CutPrefix(r.Header.Get("Authorization"), bearerPrefix) - if _, projectToken := s.LaneAuth.Project(token); projectToken { - s.writeError(w, http.StatusUnauthorized, api.CodeUnauthorized, "project credentials cannot access agent RPCs") + if project, projectToken := s.LaneAuth.Project(token); projectToken { + if !s.isWorkService(r.URL.Path) { + s.writeError(w, http.StatusUnauthorized, api.CodeUnauthorized, "project credentials cannot access agent RPCs") + return + } + next.ServeHTTP(w, withCaller(r, Principal{Kind: KindProject, Name: project})) return } if s.Token == "" { @@ -180,7 +194,7 @@ func (s *Server) authenticate(next http.Handler) http.Handler { s.writeError(w, http.StatusUnauthorized, api.CodeUnauthorized, "this server has no bearer token; only unix-socket requests are accepted") return } - next.ServeHTTP(w, r) + next.ServeHTTP(w, withCaller(r, s.operator())) return } got, ok := strings.CutPrefix(r.Header.Get("Authorization"), bearerPrefix) @@ -189,7 +203,7 @@ func (s *Server) authenticate(next http.Handler) http.Handler { s.writeError(w, http.StatusUnauthorized, api.CodeUnauthorized, "missing or invalid bearer token") return } - next.ServeHTTP(w, r) + next.ServeHTTP(w, withCaller(r, s.operator())) }) } diff --git a/src/docs/crux-serve.md b/src/docs/crux-serve.md index 288494364c..bb79e72980 100644 --- a/src/docs/crux-serve.md +++ b/src/docs/crux-serve.md @@ -33,6 +33,7 @@ runtime and every agent must arrive as a definition on `POST /v1/agents`. | `--socket PATH`, `CRUX_SOCKET` | unix socket to serve on. Default `$XDG_RUNTIME_DIR/crux/crux.sock`, else `/crux-/crux.sock`. The directory is created `0700`, the socket `0600`, and it is never reachable at that path with wider permissions (it is bound under a temporary name, chmod'ed, then renamed). A socket somebody already answers on is refused; a stale one is replaced. | | `--listen HOST:PORT` | also serve on TCP. **Refused without a token.** | | `--token T`, `CRUX_TOKEN` | bearer token every request must carry. When set it gates the socket too. | +| `--operator-name NAME`, `CRUX_OPERATOR_NAME` | the name the control token's caller is recorded under on work-services routes (see [Caller principals](#caller-principals)). Default `operator`. | | `--lanes-config PATH`, `CRUX_LANES_CONFIG` | opt into [durable event/work lanes](crux-lanes.md) on this same server, backed by NATS JetStream. Lane project credentials are separate from agent-control credentials. | | `--hive-callback ADDR`, `CRUX_HIVE_SOCKET` (+ `CRUX_HIVE_TOKEN`) | dial hive's callback endpoint (`crux/api/callback`) for the capabilities hive owns — scoped GitHub tokens, PR creation, Linear credentials, text/canary scanners, conditions — and feed the runtime's seams from it instead of leaving them nil. Health reports it under `hive_callback`; an unreachable endpoint raises the `hive_callback_unavailable` condition. (#14246) When the runtime is a manager it also polls hive's host policy from the same endpoint; see [Host policy from hive](#host-policy-from-hive-14138-stage-4). | @@ -55,6 +56,23 @@ runtime and every agent must arrive as a definition on `POST /v1/agents`. world-readable `/crux-` is an error, not a trust. - Request bodies are capped at 1 MiB and unknown JSON fields are rejected. +### Caller principals + +Every request that passes the auth gate carries a *principal* in its +context, readable by a route handler with `serve.CallerFrom(ctx)` +(hivecommons/crux#114, S0). It is the identity the work services record and +enforce ownership by; nothing on the agent-RPC routes reads it. + +| Credential presented | Principal | Where accepted | +|---|---|---| +| the control token (`--token`/`CRUX_TOKEN`), or no token on a token-less server's own unix socket | `{"kind":"operator","name":"<--operator-name>"}` | every route | +| a lane project credential (`--lanes-config` principals) | `{"kind":"project","name":""}` | work-services routes only (families registered as such, e.g. `/v1/checkouts`); on every agent RPC it is still `401 unauthorized`, as before | +| anything else | — | `401 unauthorized` | + +The operator may act on every record regardless of who owns it; a project +principal only on its own. Lane routes (`/v1/lanes…`) keep their separate +project-only gate and are not affected. + ## The API All routes are under `/v1`. Bodies and responses are JSON; agent-control errors are From e6bc5326318fcdea3bcd1641dac3303144c2dcea Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 16:11:45 +0000 Subject: [PATCH 09/17] docs: define what an ACMM pack is (#16983) Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/docs/acmm-advisor.md | 2 +- src/docs/acmm-policy-matrix.md | 14 ++++++++++++++ src/docs/agent-configuration.md | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/docs/acmm-advisor.md b/src/docs/acmm-advisor.md index 02806082e3..cb2015baa7 100644 --- a/src/docs/acmm-advisor.md +++ b/src/docs/acmm-advisor.md @@ -18,7 +18,7 @@ The advisor evaluates a fixed set of six signals (`Signals` struct, `src/pkg/acm | `HoldCount` | Count of open PRs still carrying a `hold` label awaiting human review. | | `HasQualityAgent` | Whether an active quality/coverage agent is present in the config. | -Levels run L1 (Assisted) through L6 (Autonomous), matching the level pack files and the dashboard's 1–6 bound (`src/pkg/acmmadvisor/acmmadvisor.go:34-42`). The advisor never proposes a target more than one level above the current level — advancement is always incremental (`levelStep = 1`, same file). +Levels run L1 (Assisted) through L6 (Autonomous), matching the level [pack](acmm-policy-matrix.md#what-is-a-pack) files and the dashboard's 1–6 bound (`src/pkg/acmmadvisor/acmmadvisor.go:34-42`). The advisor never proposes a target more than one level above the current level — advancement is always incremental (`levelStep = 1`, same file). ### Where the live signals come from diff --git a/src/docs/acmm-policy-matrix.md b/src/docs/acmm-policy-matrix.md index fe32ca11b1..64004a3d09 100644 --- a/src/docs/acmm-policy-matrix.md +++ b/src/docs/acmm-policy-matrix.md @@ -16,6 +16,20 @@ issues. The **KB** is the knowledge base of reusable facts; an **RFC** is a request for comments on a proposed design. For a first deployment's trust choices, start with [Securing your hive](securing-your-hive.md). +## What is a pack? + +An ACMM **pack** is the bundle of settings hive applies for one ACMM level. +Hive ships six, `level-1.yaml` through `level-6.yaml` +(`src/pkg/config/packs/`, embedded in the binary), one per level. Each pack +defines that level's agent roster, each agent's policy mode and kick template, +the governor cadences, and the merge policy. Setting `acmm_level` (or +`PUT /api/packs/level`) selects the pack, and applying it reconciles the +hive's agents to match; see [Changing a hive's ACMM level](#changing-a-hives-acmm-level). +Your explicit `backend:`, `model:`, and `enabled: false` settings win over +pack values. "Pack" is a Hive term for implementing a level; the ACMM paper +describes levels, not packs. For the full reconcile rules, see +[ACMM levels: agent rosters as packs](agent-configuration.md#acmm-levels-agent-rosters-as-packs). + ## Repository E2E detection The codebase evaluator's `acmm:prereq-e2e` prerequisite accepts the existing diff --git a/src/docs/agent-configuration.md b/src/docs/agent-configuration.md index 3b9257fa52..beee8386f9 100644 --- a/src/docs/agent-configuration.md +++ b/src/docs/agent-configuration.md @@ -832,7 +832,7 @@ agents: ## ACMM levels: agent rosters as packs -You don't have to design a roster. Hive ships six **ACMM packs** (`level-1.yaml` … `level-6.yaml`, embedded in the binary and forkable) that pair a curated agent roster with governor cadences and a merge policy: +You don't have to design a roster. Hive ships six **ACMM packs** (see [What is a pack?](acmm-policy-matrix.md#what-is-a-pack); `level-1.yaml` … `level-6.yaml`, embedded in the binary and forkable) that pair a curated agent roster with governor cadences and a merge policy: | Level | Name | Posture | |---|---|---| From 53c2e09570437f07335e46e94dad8169da6fab04 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 16:24:34 +0000 Subject: [PATCH 10/17] =?UTF-8?q?=F0=9F=90=9B=20fix(loginscan):=20auth-scr?= =?UTF-8?q?een=20patterns=20no=20longer=20match=20their=20own=20source=20t?= =?UTF-8?q?ext=20in=20agent=20panes=20(#17008)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #17006 Agents that grep or read loginscan/watchdog code paint their pane with the pattern text itself and tripped PaneShowsLogin. Skip grep-prefixed, quoted, alternation and pattern-table lines; the watchdog also only scans the pane tail. Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> --- changelog.d/fixed-17006.md | 1 + src/crux/agent/agent_unit_test.go | 24 +++++++++++++ src/crux/agent/pane_classify.go | 4 +-- src/crux/agent/source_text.go | 56 ++++++++++++++++++++++++++++++ src/crux/watchdog/classify.go | 23 +++++++++++- src/crux/watchdog/classify_test.go | 26 ++++++++++++++ src/crux/watchdog/source_text.go | 56 ++++++++++++++++++++++++++++++ 7 files changed, 187 insertions(+), 3 deletions(-) create mode 100644 changelog.d/fixed-17006.md create mode 100644 src/crux/agent/source_text.go create mode 100644 src/crux/watchdog/source_text.go diff --git a/changelog.d/fixed-17006.md b/changelog.d/fixed-17006.md new file mode 100644 index 0000000000..0930d8367c --- /dev/null +++ b/changelog.d/fixed-17006.md @@ -0,0 +1 @@ +- Agent panes that show hive's own login-pattern source or grep output (for example `file.go:NN: "Please run /login"`) are no longer misread as a login screen, so reviewers that read loginscan/watchdog code stop raising a false "needs login" banner; the watchdog also only checks the bottom of the pane for login prompts ([#17006](https://github.com/hivecommons/hive/issues/17006)). diff --git a/src/crux/agent/agent_unit_test.go b/src/crux/agent/agent_unit_test.go index 02134ebcd3..55dc9f320f 100644 --- a/src/crux/agent/agent_unit_test.go +++ b/src/crux/agent/agent_unit_test.go @@ -738,3 +738,27 @@ func TestDefaultAgentModeByRole(t *testing.T) { } } } + +func TestPaneShowsLoginPromptIgnoresSourceText(t *testing.T) { + cases := []struct { + name string + line string + want bool + }{ + {"grep prefix directive", `src/crux/watchdog/classify.go:99: "Please run /login",`, false}, + {"quoted pattern", ` "Enter one-time code",`, false}, + {"alternation", `rg 'credentialValid|Not logged in|agent_login_required'`, false}, + {"table name", `var loginPromptPatterns = []string{ // Not logged in`, false}, + {"device flow", "Enter one-time code: ABCD-1234", true}, + {"login expired", "Login expired · Please run /login", true}, + {"not logged in", "Not logged in", true}, + {"device url", "Visit github.com/login/device", true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := paneShowsLoginPrompt([]string{tc.line}); got != tc.want { + t.Fatalf("paneShowsLoginPrompt(%q) = %v, want %v", tc.line, got, tc.want) + } + }) + } +} diff --git a/src/crux/agent/pane_classify.go b/src/crux/agent/pane_classify.go index 077f361d71..bf393c1bbd 100644 --- a/src/crux/agent/pane_classify.go +++ b/src/crux/agent/pane_classify.go @@ -619,11 +619,11 @@ func paneShowsLoginPrompt(lines []string) bool { if lineShowsUpstreamAuthorizationError(line) || paneShowsQuotaExhausted([]string{line}) { continue } - if lineHasLoginDirective(line) { + if lineHasLoginDirective(line) && !authHitIsSourceText(line, "/login") { return true } for _, pat := range loginPromptPatterns { - if strings.Contains(line, pat) { + if strings.Contains(line, pat) && !authHitIsSourceText(line, pat) { return true } } diff --git a/src/crux/agent/source_text.go b/src/crux/agent/source_text.go new file mode 100644 index 0000000000..9426732af2 --- /dev/null +++ b/src/crux/agent/source_text.go @@ -0,0 +1,56 @@ +package agent + +import ( + "regexp" + "strings" +) + +// grepPrefixRe matches the "path.go:NN:" prefix grep/rg put on a hit. +var grepPrefixRe = regexp.MustCompile("^\\s*[\\w./-]+\\.\\w+:\\d+[:-]") + +// patternTableNames are identifiers whose presence on a line means the pane is +// showing hive's own login-pattern source, not a login screen (#17006). +var patternTableNames = []string{"authScreenPatterns", "loginPromptPatterns", "classify.go"} + +// authHitIsSourceText reports whether every match of pat on line is source or +// grep output quoting the pattern (a path.go:NN: prefix, a pattern-table name, +// a string literal, or a |-joined alternation) rather than a CLI login screen. +func authHitIsSourceText(line, pat string) bool { + if grepPrefixRe.MatchString(line) { + return true + } + for _, name := range patternTableNames { + if strings.Contains(line, name) { + return true + } + } + for off := 0; ; { + i := strings.Index(line[off:], pat) + if i < 0 { + return true + } + start := off + i + end := start + len(pat) + if !hitIsQuoted(line, start, end) { + return false + } + off = end + } +} + +// hitIsQuoted reports whether the match at [start,end) sits inside a quoted +// literal or beside a regex alternation bar. +func hitIsQuoted(line string, start, end int) bool { + before, after := line[:start], line[end:] + if strings.Count(before, "\"")%2 == 1 || strings.Count(before, "`")%2 == 1 { + return true + } + b := strings.TrimRight(before, " ") + a := strings.TrimLeft(after, " ") + for _, q := range []string{"'", "\"", "`", "|"} { + if strings.HasSuffix(b, q) || strings.HasPrefix(a, q) { + return true + } + } + return false +} diff --git a/src/crux/watchdog/classify.go b/src/crux/watchdog/classify.go index 916d8886a0..565a104aff 100644 --- a/src/crux/watchdog/classify.go +++ b/src/crux/watchdog/classify.go @@ -132,6 +132,27 @@ func paneMatchesAny(pane string, patterns []string) bool { return false } +// authScreenTailLines bounds where a real login prompt is looked for: it sits +// at the bottom of the pane, while source/grep output scrolls through above. +const authScreenTailLines = 15 + +// paneShowsAuthScreen reports whether the tail of the pane carries an auth +// pattern outside source/grep output (#17006). +func paneShowsAuthScreen(pane string) bool { + lines := strings.Split(strings.TrimRight(pane, " \t\r\n"), "\n") + if len(lines) > authScreenTailLines { + lines = lines[len(lines)-authScreenTailLines:] + } + for _, line := range lines { + for _, p := range authScreenPatterns { + if strings.Contains(line, p) && !authHitIsSourceText(line, p) { + return true + } + } + } + return false +} + // looksLikeShellPrompt reports whether the last non-empty line of the pane is // a bare shell prompt (short line ending in a prompt character). func looksLikeShellPrompt(pane string) bool { @@ -197,7 +218,7 @@ func Classify(obs Observation, now time.Time, s Settings) Classification { } return Classification{ClassNoSession, "tmux session missing"} } - if obs.ShowsLoginPrompt || paneMatchesAny(obs.Pane, authScreenPatterns) { + if obs.ShowsLoginPrompt || paneShowsAuthScreen(obs.Pane) { // Boot grace covers login chrome too. Interactive CLIs paint a login // screen DURING their startup auth handshake even when the seeded // credential is about to be accepted -- the pane poller already diff --git a/src/crux/watchdog/classify_test.go b/src/crux/watchdog/classify_test.go index 6ab3e8e6fe..f7eea61d59 100644 --- a/src/crux/watchdog/classify_test.go +++ b/src/crux/watchdog/classify_test.go @@ -200,3 +200,29 @@ func TestLooksLikeShellPromptSkipsTrailingBlanks(t *testing.T) { t.Fatal("empty pane is not a shell prompt") } } + +func TestPaneShowsAuthScreenIgnoresSourceText(t *testing.T) { + filler := strings.Repeat("working\n", authScreenTailLines) + cases := []struct { + name string + pane string + want bool + }{ + {"grep prefix", "classify.go:99: \"Please run /login\",", false}, + {"quoted literal", " \"Login expired\",", false}, + {"backtick literal", "see `Sign in to continue` here", false}, + {"regex alternation", "rg 'credentialValid|Login expired|Run /login'", false}, + {"table name", "var authScreenPatterns = []string{ Login expired", false}, + {"login expired chrome", "● Login expired · Please run /login", true}, + {"sign in", "Sign in to continue", true}, + {"scrolled out of tail", "Sign in to continue\n" + filler, false}, + {"real prompt under old output", filler + "Login expired · Please run /login\n", true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := paneShowsAuthScreen(tc.pane); got != tc.want { + t.Fatalf("paneShowsAuthScreen() = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/src/crux/watchdog/source_text.go b/src/crux/watchdog/source_text.go new file mode 100644 index 0000000000..a82c9e1084 --- /dev/null +++ b/src/crux/watchdog/source_text.go @@ -0,0 +1,56 @@ +package watchdog + +import ( + "regexp" + "strings" +) + +// grepPrefixRe matches the "path.go:NN:" prefix grep/rg put on a hit. +var grepPrefixRe = regexp.MustCompile("^\\s*[\\w./-]+\\.\\w+:\\d+[:-]") + +// patternTableNames are identifiers whose presence on a line means the pane is +// showing hive's own login-pattern source, not a login screen (#17006). +var patternTableNames = []string{"authScreenPatterns", "loginPromptPatterns", "classify.go"} + +// authHitIsSourceText reports whether every match of pat on line is source or +// grep output quoting the pattern (a path.go:NN: prefix, a pattern-table name, +// a string literal, or a |-joined alternation) rather than a CLI login screen. +func authHitIsSourceText(line, pat string) bool { + if grepPrefixRe.MatchString(line) { + return true + } + for _, name := range patternTableNames { + if strings.Contains(line, name) { + return true + } + } + for off := 0; ; { + i := strings.Index(line[off:], pat) + if i < 0 { + return true + } + start := off + i + end := start + len(pat) + if !hitIsQuoted(line, start, end) { + return false + } + off = end + } +} + +// hitIsQuoted reports whether the match at [start,end) sits inside a quoted +// literal or beside a regex alternation bar. +func hitIsQuoted(line string, start, end int) bool { + before, after := line[:start], line[end:] + if strings.Count(before, "\"")%2 == 1 || strings.Count(before, "`")%2 == 1 { + return true + } + b := strings.TrimRight(before, " ") + a := strings.TrimLeft(after, " ") + for _, q := range []string{"'", "\"", "`", "|"} { + if strings.HasSuffix(b, q) || strings.HasPrefix(a, q) { + return true + } + } + return false +} From a9d25531fd9d6cc6cfefb211544d4e990afa14dc Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 16:33:32 +0000 Subject: [PATCH 11/17] =?UTF-8?q?=F0=9F=90=9B=20fix(ioscan):=20apply=20Inp?= =?UTF-8?q?ut=20Defense=20saves=20without=20a=20restart=20and=20keep=20fai?= =?UTF-8?q?l=20mode=20open=20at=20L5=20(#16985)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ioscan): apply Input Defense saves without a restart and keep fail mode open at L5 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: williamdalessandro <231250207+williamdalessandro@users.noreply.github.com> * fix(ioscan): rebuild the classifier on runtime ACMM level changes and fix restart guidance Review follow-ups for #16985: - onACMMLevelChanged now calls the new boot.reinstallIoscanClassifier, so an L4<->L5 change in the dashboard (or via a hive.yaml reload) applies the level's classifier requirement and fail mode without a restart, instead of leaving the boot-time classifier and stale alert level in place. - boot.ioscanAlerts() replaces the two duplicated nil-interface adapters in main.go; IoscanConfigApplied is now the shared method value. - The Classifier backend tooltip in Settings -> Security no longer says "Takes effect after a restart"; it says saves there apply immediately and a model endpoint or hive.yaml change takes effect after a restart. - The blocked-kick alert names ioscan.fail_mode (the key the Security tab exposes) rather than only ioscan.classifier.fail_mode. - Docs and changelog fragment updated; tests cover the hook and the alert wording. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * [scanner] fix(dashboard): regenerate static/index.html for the ioscan settings change Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: williamdalessandro <231250207+williamdalessandro@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...ixed-ioscan-classifier-settings-restart.md | 1 + src/cmd/hive/github_client_provider.go | 6 +- src/cmd/hive/ioscan_classifier_boot.go | 55 +++++++++++- src/cmd/hive/ioscan_classifier_boot_test.go | 90 +++++++++++++++++++ src/cmd/hive/main.go | 7 +- src/docs/ioscan.md | 13 ++- src/pkg/dashboard/api_governor_security.go | 14 +-- .../api_governor_security_ioscan_test.go | 45 ++++++++++ src/pkg/dashboard/deps.go | 5 ++ .../static-src/js/config-watchdog-security.js | 2 +- src/pkg/dashboard/static/index.html | 2 +- 11 files changed, 222 insertions(+), 18 deletions(-) create mode 100644 changelog.d/fixed-ioscan-classifier-settings-restart.md diff --git a/changelog.d/fixed-ioscan-classifier-settings-restart.md b/changelog.d/fixed-ioscan-classifier-settings-restart.md new file mode 100644 index 0000000000..904afb5037 --- /dev/null +++ b/changelog.d/fixed-ioscan-classifier-settings-restart.md @@ -0,0 +1 @@ +- Input Defense settings saved from the dashboard now actually unblock kicks at ACMM L5+ ([#16973](https://github.com/hivecommons/hive/issues/16973)). Saving ioscan fail mode "open" was stored as unset, which at L5+ falls back to the ACMM pack's fail-closed default, so the setting snapped back to "closed" and kicks carrying untrusted text stayed withheld. "open" is now stored as given, and every ioscan save in Settings → Security → Input Defense rebuilds the semantic classifier from the new backend and fail mode immediately, clearing the blocked-kick alert when it no longer applies. A runtime ACMM level change (for example L4 → L5 in the dashboard) rebuilds the classifier the same way, so the level's classifier requirement and fail mode apply without a restart. The alert and the Classifier backend help text no longer tell operators to restart for changes the dashboard now applies live, and the alert names the `ioscan.fail_mode` key the Security tab actually exposes. diff --git a/src/cmd/hive/github_client_provider.go b/src/cmd/hive/github_client_provider.go index afdd6e5de3..018f628d7e 100644 --- a/src/cmd/hive/github_client_provider.go +++ b/src/cmd/hive/github_client_provider.go @@ -110,11 +110,15 @@ func (b *boot) armRequestRelays(client *github.Client) bool { // onACMMLevelChanged re-arms request relays when the level crosses the // self-authored auto-merge boundary. The relay generation re-reads the current // ACMM verdict when it starts, so this turns a runtime L5↔L6 change into the -// same safe hand-over used for GitHub App client rebuilds. +// same safe hand-over used for GitHub App client rebuilds. It also rebuilds +// the ioscan semantic classifier, whose enablement and fail mode follow the +// level (required and fail-closed at L5+), so an L4↔L5 change in the +// dashboard takes effect without a restart. func (b *boot) onACMMLevelChanged(prev, next int) { if b == nil || b.cfg == nil { return } + b.reinstallIoscanClassifier() before, after := b.cfg.AutoMerge.SelfAuthoredAutoMergeAllowed(&prev), b.cfg.AutoMerge.SelfAuthoredAutoMergeAllowed(&next) if before == after { return diff --git a/src/cmd/hive/ioscan_classifier_boot.go b/src/cmd/hive/ioscan_classifier_boot.go index db54b3ae61..739a424ae3 100644 --- a/src/cmd/hive/ioscan_classifier_boot.go +++ b/src/cmd/hive/ioscan_classifier_boot.go @@ -140,6 +140,55 @@ func installIoscanClassifier(sched ioscanClassifierSink, alerts ioscanClassifier } } +// reinstallIoscanClassifier re-runs installIoscanClassifier against the live +// config after an Input Defense save, so a new backend or fail mode applies +// without a restart (#16973). A run that installs nothing (classifier off, or +// fail-open with no runnable classifier) drops the classifier an earlier run +// installed, so a stale unavailable classifier cannot keep withholding text. +func reinstallIoscanClassifier(sched ioscanClassifierSink, alerts ioscanClassifierAlerter, cfg *config.Config, logger *slog.Logger, now func() time.Time) { + rec := &ioscanClassifierSetRecorder{ioscanClassifierSink: sched} + installIoscanClassifier(rec, alerts, cfg, logger, now) + if !rec.set { + sched.SetClassifier(nil, ioscan.Thresholds{}) + } +} + +// ioscanAlerts is the dashboard the ioscan classifier raises its blocked-kick +// alert on, as a nil interface (not a typed nil) when there is no dashboard. +func (b *boot) ioscanAlerts() ioscanClassifierAlerter { + if b == nil || b.dashSrv == nil { + return nil + } + return b.dashSrv +} + +// reinstallIoscanClassifier rebuilds the scheduler's semantic classifier +// from the live config: after an Input Defense save (#16973) and after a +// runtime ACMM level change, since the level decides whether the classifier +// is required and which fail mode applies. No-op before the scheduler exists. +func (b *boot) reinstallIoscanClassifier() { + if b == nil || b.sched == nil || b.cfg == nil { + return + } + logger := b.logger + if logger == nil { + logger = slog.Default() + } + reinstallIoscanClassifier(b.sched, b.ioscanAlerts(), b.cfg, logger, time.Now) +} + +// ioscanClassifierSetRecorder notes whether installIoscanClassifier installed +// a classifier. +type ioscanClassifierSetRecorder struct { + ioscanClassifierSink + set bool +} + +func (r *ioscanClassifierSetRecorder) SetClassifier(c ioscan.Classifier, t ioscan.Thresholds) { + r.set = true + r.ioscanClassifierSink.SetClassifier(c, t) +} + // ioscanClassifierFallbackBackend picks the subscription CLI the classifier // runs through when no backend is configured and no model endpoint is // resolved: the tool-less-capable backend most of the hive's agents run on, @@ -206,8 +255,10 @@ func ioscanClassifierBlockedAlertMessage(acmmLevel int, cause error) string { return fmt.Sprintf("ioscan semantic classifier is required at ACMM L%d but failed or is unavailable (%v). "+ "Kicks carrying untrusted text (issues, comments, PR bodies, reviews) can be withheld until this is fixed. "+ "Fix: configure a model endpoint (governor.trajectory or governor.litellm), set ioscan.classifier.backend to one of %s "+ - "(Settings → Security → Input Defense), or accept scanning without the classifier with ioscan.classifier.fail_mode: open. "+ - "Changes take effect after a restart.", acmmLevel, cause, strings.Join(ioscan.ToollessBackends(), ", ")) + "(Settings → Security → Input Defense), or accept scanning without the classifier by setting ioscan.fail_mode to open "+ + "(the Fail mode control in that tab; ioscan.classifier.fail_mode: open in hive.yaml overrides it for the classifier alone). "+ + "Input Defense changes saved in Settings → Security apply immediately; a model endpoint or hive.yaml edit takes effect after the hive restarts.", + acmmLevel, cause, strings.Join(ioscan.ToollessBackends(), ", ")) } func raiseIoscanClassifierAlert(alerts ioscanClassifierAlerter, acmmLevel int, cause error) { diff --git a/src/cmd/hive/ioscan_classifier_boot_test.go b/src/cmd/hive/ioscan_classifier_boot_test.go index 2a3d9c2890..482113352d 100644 --- a/src/cmd/hive/ioscan_classifier_boot_test.go +++ b/src/cmd/hive/ioscan_classifier_boot_test.go @@ -11,6 +11,7 @@ import ( "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/ioscan" + "github.com/hivecommons/hive/pkg/scheduler" ) type fakeClassifierSink struct { @@ -421,3 +422,92 @@ func TestInstallIoscanClassifier_RuntimeUnavailableAlert(t *testing.T) { t.Fatal("runtime alert remains after successful score") } } + +// A dashboard save re-runs the install against the live config (#16973): +// switching to fail-open drops the stale unavailable classifier and clears +// the alert, and naming a tool-less backend installs it, with no restart. +func TestReinstallIoscanClassifier_AppliesDashboardChanges(t *testing.T) { + sink := &fakeClassifierSink{} + alerts := &fakeClassifierAlerts{} + logger, _ := ioscanBootLogger() + cfg := ioscanBootConfigWithAgents(t, 6, "codex") + installIoscanClassifier(sink, alerts, cfg, logger, ioscanBootNow) + if _, ok := sink.classifier.(ioscan.UnavailableClassifier); !ok { + t.Fatalf("installed %T, want UnavailableClassifier", sink.classifier) + } + msg, ok := alerts.blocked() + if !ok || !strings.Contains(msg, "apply immediately") || strings.Contains(msg, "Changes take effect after a restart") { + t.Fatalf("blocked-kick alert does not say dashboard changes apply immediately: %q", msg) + } + // The Security tab exposes ioscan.fail_mode, not classifier.fail_mode; + // the remediation names the key the operator can actually set there. + if !strings.Contains(msg, "ioscan.fail_mode to open") { + t.Fatalf("blocked-kick alert does not name the dashboard-visible ioscan.fail_mode key: %q", msg) + } + + cfg.Ioscan.FailMode = "open" + reinstallIoscanClassifier(sink, alerts, cfg, logger, ioscanBootNow) + if sink.classifier != nil { + t.Fatalf("fail-open save kept %T installed", sink.classifier) + } + if _, ok := alerts.blocked(); ok { + t.Fatal("fail-open save left the blocked-kick alert raised") + } + + cfg.Ioscan.FailMode = "" + cfg.Ioscan.Classifier.Backend = "copilot" + reinstallIoscanClassifier(sink, alerts, cfg, logger, ioscanBootNow) + if _, ok := sink.classifier.(*ioscan.CachedClassifier); !ok { + t.Fatalf("backend save installed %T, want *ioscan.CachedClassifier", sink.classifier) + } + if _, ok := alerts.blocked(); ok { + t.Fatal("working backend left the blocked-kick alert raised") + } +} + +// A runtime ACMM level change rebuilds the classifier through the same hook +// the request relays use. With a backend that cannot run, L4 is fail-open +// (nothing installed) and L5 is fail-closed (the unavailable classifier is +// installed so untrusted text is withheld); raising L4->L5 in the dashboard +// must switch between them without a restart, and dropping back must undo +// it. The hook must tolerate a boot with no scheduler. +func TestOnACMMLevelChanged_ReinstallsIoscanClassifier(t *testing.T) { + t.Setenv(config.LiteLLMEndpointEnv, "") + cfg := ioscanBootConfig(4, "codex", "") + logger, _ := ioscanBootLogger() + b := &boot{cfg: cfg, logger: logger} + b.onACMMLevelChanged(4, 5) // no scheduler yet: must not panic + + b.sched = scheduler.New(cfg, logger) + installIoscanClassifier(b.sched, b.ioscanAlerts(), cfg, logger, ioscanBootNow) + if _, _, installed := b.sched.IoscanHooksForTest(); installed { + t.Fatal("refused backend installed a classifier at fail-open L4") + } + + l5 := 5 + cfg.ACMMLevel = &l5 + b.onACMMLevelChanged(4, 5) + if _, _, installed := b.sched.IoscanHooksForTest(); !installed { + t.Fatal("L4->L5 runtime change did not install the fail-closed unavailable classifier") + } + + l4 := 4 + cfg.ACMMLevel = &l4 + b.onACMMLevelChanged(5, 4) + if _, _, installed := b.sched.IoscanHooksForTest(); installed { + t.Fatal("L5->L4 runtime change left the unavailable classifier withholding text") + } +} + +// ioscanAlerts yields a nil interface, not a typed nil *dashboard.Server, +// when the boot has no dashboard, so raiseIoscanClassifierAlert's nil check +// holds. +func TestBootIoscanAlerts_NilWithoutDashboard(t *testing.T) { + var b *boot + if b.ioscanAlerts() != nil { + t.Fatal("nil boot yielded a non-nil alerter") + } + if (&boot{}).ioscanAlerts() != nil { + t.Fatal("boot without a dashboard yielded a non-nil alerter") + } +} diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 17a2b757cd..32c5b7def2 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -1277,6 +1277,7 @@ func (b *boot) wireBootClosures() { ReInitFunc: func() { initAgentConfigDrivenSystems(b.cfg) }, + IoscanConfigApplied: b.reinstallIoscanClassifier, ReviewConfigApplied: func(rc config.ReviewConfig) { if b.ghClient == nil { return @@ -2538,11 +2539,7 @@ func (b *boot) bootDashboardWith(deps bootDashboardDeps) { } } }) - var ioscanAlerts ioscanClassifierAlerter - if b.dashSrv != nil { - ioscanAlerts = b.dashSrv - } - installIoscanClassifier(b.sched, ioscanAlerts, b.cfg, b.logger, time.Now) + installIoscanClassifier(b.sched, b.ioscanAlerts(), b.cfg, b.logger, time.Now) b.agentMgr.SetSandboxAuditCallback(func(agentName, action, detail string) { b.dashSrv.AuditLog(agentName, action, detail, agentName) if action == "sandbox_broker_rejected" { diff --git a/src/docs/ioscan.md b/src/docs/ioscan.md index 95d70e75e3..7e80a613b8 100644 --- a/src/docs/ioscan.md +++ b/src/docs/ioscan.md @@ -151,8 +151,13 @@ agents all run on a subscription CLI such as Copilot has no such endpoint; with `classifier.backend` left empty, boot then runs the classifier through the tool-less CLI most of the hive's agents run on (the most common of `claude`/`copilot`/`omp` across `agents.*.backend`, ties by name), so such a hive -needs no extra setting. The classifier itself is built at boot, so changing -its backend needs a restart. +needs no extra setting. The classifier is built at boot and rebuilt whenever +an ioscan setting is saved in Settings → Security → Input Defense or the ACMM +level changes at runtime, so a new backend or fail mode saved there, or an +L4↔L5 level change, applies immediately; a model endpoint change or a direct +`hive.yaml` edit takes effect after a restart. The Security tab exposes +`ioscan.fail_mode` (which the classifier inherits), not +`classifier.fail_mode`; the alert names that dashboard key. **What still passes.** The classifier has a budget of 64 calls per scheduler pass (8 with a CLI backend, see below). Text beyond that budget is passed on with a `budget_exhausted` skip in @@ -227,7 +232,9 @@ and `omp`. `daily_quota_exhausted` skip in the audit log, in either fail mode, just as for an exhausted per-pass budget, and the scheduler logs one WARN per UTC day. The count starts again at 00:00 UTC. Cache hits do not count. -* The backend is chosen at boot; changing it takes effect after a restart. +* The backend is chosen at boot and again on every Input Defense save or + runtime ACMM level change in the dashboard; a direct `hive.yaml` edit takes + effect after a restart. **Cost.** One call per untrusted text per pass, cached by content, with an 8,000-character cap per call. At the default model that is on the order of diff --git a/src/pkg/dashboard/api_governor_security.go b/src/pkg/dashboard/api_governor_security.go index e8b0f4509f..85b3928712 100644 --- a/src/pkg/dashboard/api_governor_security.go +++ b/src/pkg/dashboard/api_governor_security.go @@ -153,11 +153,10 @@ func (s *Server) handleGovernorSecurity(w http.ResponseWriter, r *http.Request) cfg.Ioscan.Enabled = &v } if body.IoscanFailMode != nil { - mode := strings.ToLower(strings.TrimSpace(*body.IoscanFailMode)) - if mode == "open" { - mode = "" - } - cfg.Ioscan.FailMode = mode + // "open" is stored as given: an empty fail_mode follows the ACMM + // pack, which is closed at L5+, so the operator's choice would be + // silently lost there (#16973). + cfg.Ioscan.FailMode = strings.ToLower(strings.TrimSpace(*body.IoscanFailMode)) } if body.IoscanClassifierBackend != nil { cfg.Ioscan.Classifier.Backend = strings.TrimSpace(*body.IoscanClassifierBackend) @@ -271,6 +270,11 @@ func (s *Server) handleGovernorSecurity(w http.ResponseWriter, r *http.Request) if err := s.saveConfig(); err != nil { s.logger.Error("failed to persist config after security update", "error", err) } + ioscanChanged := body.IoscanEnabled != nil || body.IoscanFailMode != nil || body.IoscanClassifierBackend != nil || + body.IoscanClassifierCLIBudgetPerEval != nil || body.IoscanClassifierCLIDailyQuota != nil + if ioscanChanged && s.deps.IoscanConfigApplied != nil { + s.deps.IoscanConfigApplied() + } s.auditFromRequest(r, "config_governor_security", auditDetail("section", "security"), "") s.refreshAndPersist() okResponse(w, map[string]string{"status": "updated"}) diff --git a/src/pkg/dashboard/api_governor_security_ioscan_test.go b/src/pkg/dashboard/api_governor_security_ioscan_test.go index 7e7a2392b9..a8a7784914 100644 --- a/src/pkg/dashboard/api_governor_security_ioscan_test.go +++ b/src/pkg/dashboard/api_governor_security_ioscan_test.go @@ -63,3 +63,48 @@ func TestGovernorSecurityRejectsInvalidClassifierBackendValues(t *testing.T) { }) } } + +// At L5 the ACMM pack defaults ioscan to fail-closed, so a dashboard save of +// "open" must be stored as given and reported back as open, and every ioscan +// save must reach IoscanConfigApplied so it applies without a restart +// (#16973). +func TestGovernorSecurityIoscanFailModeOpenSticksAtL5AndApplies(t *testing.T) { + s, deps := apiServer(t) + level := 5 + deps.Config.ACMMLevel = &level + applied := 0 + deps.IoscanConfigApplied = func() { applied++ } + + sec := decodeJSON(t, doGet(s, "/api/config/governor"))["security"].(map[string]any) + if sec["ioscanFailMode"] != "closed" { + t.Fatalf("L5 default fail mode = %v, want closed", sec["ioscanFailMode"]) + } + + rec := doPut(s, "/api/config/governor/security", map[string]any{"ioscanFailMode": "open"}) + if rec.Code != http.StatusOK { + t.Fatalf("PUT security = %d, want 200: %s", rec.Code, rec.Body.String()) + } + if deps.Config.Ioscan.FailMode != "open" { + t.Fatalf("stored fail mode = %q, want open", deps.Config.Ioscan.FailMode) + } + if deps.Config.Ioscan.ClassifierFailClosedAtLevel(level) { + t.Fatal("classifier still fail-closed after saving fail mode open") + } + sec = decodeJSON(t, doGet(s, "/api/config/governor"))["security"].(map[string]any) + if sec["ioscanFailMode"] != "open" { + t.Fatalf("fail mode reported back as %v after saving open", sec["ioscanFailMode"]) + } + if applied != 1 { + t.Fatalf("IoscanConfigApplied calls after fail mode save = %d, want 1", applied) + } + + rec = doPut(s, "/api/config/governor/security", map[string]any{"ioscanClassifierBackend": "copilot"}) + if rec.Code != http.StatusOK || applied != 2 { + t.Fatalf("backend save: code=%d applied=%d, want 200 and 2", rec.Code, applied) + } + + rec = doPut(s, "/api/config/governor/security", map[string]any{"reviewFanOut": true}) + if rec.Code != http.StatusOK || applied != 2 { + t.Fatalf("non-ioscan save: code=%d applied=%d, want 200 and 2", rec.Code, applied) + } +} diff --git a/src/pkg/dashboard/deps.go b/src/pkg/dashboard/deps.go index f8244017a4..04f5167c82 100644 --- a/src/pkg/dashboard/deps.go +++ b/src/pkg/dashboard/deps.go @@ -145,6 +145,11 @@ type Dependencies struct { // OnACMMLevelChanged is called after an in-process runtime level change // has been persisted/applied. Nil means no live component needs to react. OnACMMLevelChanged func(prev, next int) + // IoscanConfigApplied is called after PUT /api/config/governor/security + // has changed an ioscan setting, so the owner of the scheduler can rebuild + // the semantic classifier from the new values without a restart. Nil means + // nothing caches. + IoscanConfigApplied func() // ReviewConfigApplied is called after PUT /api/config/review has mutated // Config.Review, so the owner of any component that caches review // settings at boot (the GitHub client's revise allowlist and perspective diff --git a/src/pkg/dashboard/static-src/js/config-watchdog-security.js b/src/pkg/dashboard/static-src/js/config-watchdog-security.js index b78626fb72..3a74632df6 100644 --- a/src/pkg/dashboard/static-src/js/config-watchdog-security.js +++ b/src/pkg/dashboard/static-src/js/config-watchdog-security.js @@ -829,7 +829,7 @@ const classifierBackend = String(sec.ioscanClassifierBackend || '').trim(); const classifierControls = Object.prototype.hasOwnProperty.call(sec, 'ioscanClassifierBackend') ? `
- + From 4b6fc4db7f998e47f28cfd7b588c41add7073cc7 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 16:45:21 +0000 Subject: [PATCH 12/17] =?UTF-8?q?=F0=9F=93=96=20docs:=20capitalize=20Hive?= =?UTF-8?q?=20in=20the=20ACMM=20pack=20definition=20(#17011)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/docs/acmm-policy-matrix.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/docs/acmm-policy-matrix.md b/src/docs/acmm-policy-matrix.md index 64004a3d09..63dc5aff6b 100644 --- a/src/docs/acmm-policy-matrix.md +++ b/src/docs/acmm-policy-matrix.md @@ -18,7 +18,7 @@ choices, start with [Securing your hive](securing-your-hive.md). ## What is a pack? -An ACMM **pack** is the bundle of settings hive applies for one ACMM level. +An ACMM **pack** is the bundle of settings Hive applies for one ACMM level. Hive ships six, `level-1.yaml` through `level-6.yaml` (`src/pkg/config/packs/`, embedded in the binary), one per level. Each pack defines that level's agent roster, each agent's policy mode and kick template, From 51de61e0dca430c7c0a6475827f2141835a1e31d Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 17:03:47 +0000 Subject: [PATCH 13/17] =?UTF-8?q?=F0=9F=90=9B=20fix(automerge):=20align=20?= =?UTF-8?q?contributor=20merge=20eligibility=20with=20relay=20policy=20(#1?= =?UTF-8?q?7009)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hive-Run: hivecommons/hive#16717 Hive-Plan: contributor-lane-eligibility-parity Hive-Spec: issue-16717#eligibility-side Signed-off-by: thatguywob Co-authored-by: thatguywob Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...xed-16717-contributor-merge-eligibility.md | 1 + src/cmd/hive/merge_eligibility.go | 41 +++++++-- src/cmd/hive/merge_eligible_lane_gate_test.go | 87 +++++++++++++++++-- src/pkg/scheduler/agent_repo_scope_test.go | 34 ++++++++ src/pkg/scheduler/lane_messages.go | 36 ++++++-- 5 files changed, 179 insertions(+), 20 deletions(-) create mode 100644 changelog.d/fixed-16717-contributor-merge-eligibility.md diff --git a/changelog.d/fixed-16717-contributor-merge-eligibility.md b/changelog.d/fixed-16717-contributor-merge-eligibility.md new file mode 100644 index 0000000000..e1ca2d525b --- /dev/null +++ b/changelog.d/fixed-16717-contributor-merge-eligibility.md @@ -0,0 +1 @@ +- Contributor PRs with hive attribution now pass the merge relay's trust gate before being listed merge-eligible; trust refusals appear separately with their reason and maintainer unblock options ([#16717](https://github.com/hivecommons/hive/issues/16717)). diff --git a/src/cmd/hive/merge_eligibility.go b/src/cmd/hive/merge_eligibility.go index 3edfe19d76..a3512f8f8c 100644 --- a/src/cmd/hive/merge_eligibility.go +++ b/src/cmd/hive/merge_eligibility.go @@ -19,6 +19,7 @@ import ( "github.com/hivecommons/hive/pkg/intent" "github.com/hivecommons/hive/pkg/rebasemerge" "github.com/hivecommons/hive/pkg/review" + "github.com/hivecommons/hive/pkg/stageengine/prstage" "github.com/hivecommons/hive/pkg/stagelabels" ) @@ -440,13 +441,14 @@ func setMergeEligibleLaneGate(g github.MergeRequestPolicyGate) { // refuses must not be listed merge-eligible: hive-merge on it is a wasted // attempt every kick, and the contributor lane merges it itself when ready. // It reuses the relay's gate (automerge.Engine.RelayMergeRefusal) rather than -// restating the rule. Hive-authored PRs bypass the lane, and an unreadable -// gate leaves the PR listed — the relay still re-checks it at merge time. +// restating the rule. The gate itself owns author exemptions: hive attribution +// describes provenance, not contributor trust. An unreadable gate leaves the +// PR listed — the relay still re-checks it at merge time. func relayLaneRefusal(pr github.PullRequest, fullRepo string, logger *slog.Logger) string { mergeEligibleLaneGateMu.RLock() gate := mergeEligibleLaneGate mergeEligibleLaneGateMu.RUnlock() - if gate == nil || hiveAuthoredPR(pr) { + if gate == nil { return "" } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) @@ -462,6 +464,21 @@ func relayLaneRefusal(pr github.PullRequest, fullRepo string, logger *slog.Logge return refusal } +// relayRefusalNeedsMaintainer reports trust admission failures using the +// stage engine's existing classification. Soak, head age and pending review +// are temporary gates, not requests for a maintainer merge. +func relayRefusalNeedsMaintainer(refusal string) bool { + if !strings.HasPrefix(refusal, automerge.RelayMergeRefusalPrefix) { + return false + } + _, reason, ok := strings.Cut(refusal, ": ") + if !ok { + return false + } + reason, _, _ = strings.Cut(reason, " (") + return prstage.HumanMergeReason(reason) == prstage.HumanMergeReasonUntrustedAuthor +} + // mergeableJSON renders a tri-state mergeability verdict for the // merge-eligible.json marker, mapping the unknown zero value to an explicit // "unknown" rather than an empty string. @@ -1096,7 +1113,15 @@ func writeMergeEligible(actionable *github.ActionableResult, hold github.HoldRes prAgents := auditPRAgents(org, time.Now().Add(-auditPRAttributionWindow), "") + type maintainerMergePR struct { + Number int `json:"number"` + Repo string `json:"repo"` + Title string `json:"title"` + HeadSHA string `json:"head_sha,omitempty"` + Reason string `json:"reason"` + } var eligible []eligiblePR + var needsMaintainer []maintainerMergePR var failing []failingPR // escalated holds the escalated (needs-human) hive PRs that did NOT land // in the failing bucket — conflicted, green or pending ones @@ -1183,6 +1208,11 @@ func writeMergeEligible(actionable *github.ActionableResult, hold github.HoldRes if refusal := relayLaneRefusal(pr, fullRepo, logger); refusal != "" { bucket = mergeBucketSkip verdict = github.MergeVerdict{State: github.MergeVerdictOutstanding, Reason: refusal} + if relayRefusalNeedsMaintainer(refusal) { + needsMaintainer = append(needsMaintainer, maintainerMergePR{ + Number: pr.Number, Repo: fullRepo, Title: pr.Title, HeadSHA: pr.HeadSHA, Reason: refusal, + }) + } } } verdicts[github.MergeVerdictKey(pr)] = verdict @@ -1347,8 +1377,9 @@ func writeMergeEligible(actionable *github.ActionableResult, hold github.HoldRes _ = os.MkdirAll("/var/run/hive-metrics", 0o755) payload := map[string]any{ - "generated_at": time.Now().UTC().Format(time.RFC3339), - "merge_eligible": eligible, + "generated_at": time.Now().UTC().Format(time.RFC3339), + "merge_eligible": eligible, + "needs_maintainer_merge": needsMaintainer, } data, err := json.Marshal(payload) if err != nil { diff --git a/src/cmd/hive/merge_eligible_lane_gate_test.go b/src/cmd/hive/merge_eligible_lane_gate_test.go index 95b7d3308c..44d76fff2c 100644 --- a/src/cmd/hive/merge_eligible_lane_gate_test.go +++ b/src/cmd/hive/merge_eligible_lane_gate_test.go @@ -3,11 +3,15 @@ package main import ( "context" "encoding/json" + "errors" "os" "path/filepath" + "strings" "testing" "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/github/automerge" + "github.com/hivecommons/hive/pkg/stageengine/prstage" ) // A PR the merge relay's contributor-lane gate refuses (a newcomer-tier @@ -23,9 +27,18 @@ func TestWriteMergeEligible_ExcludesPRsTheRelayLaneRefuses(t *testing.T) { ciFailingPath = origFail setMergeEligibleLaneGate(nil) }) - setMergeEligibleLaneGate(func(_ context.Context, _ string, number int, _ string) (string, error) { - if number == 15086 { - return "contributor-lane policy refuses the merge relay for @Karibhou's PR: contributor-lane:claim-tier-untrusted (author tier \"newcomer\", require_tiers merger, trusted)", nil + called := make(map[int]bool) + refusal := automerge.RelayMergeRefusalPrefix + " for @contributor's PR: " + prstage.ContributorLaneReasonNoClaim + ` (author tier "none", require_tiers merger, trusted)` + setMergeEligibleLaneGate(func(_ context.Context, repo string, number int, sha string) (string, error) { + called[number] = true + if repo != "hivecommons/hive" || sha != "cafe" { + t.Errorf("gate target = %s#%d@%s, want hivecommons/hive at cafe", repo, number, sha) + } + if number == 15086 || number == 15088 || number == 15089 { + return refusal, nil + } + if number == 15091 { + return automerge.RelayMergeRefusalPrefix + ": " + prstage.ContributorLaneReasonSoaking + " (10m remaining)", nil } return "", nil }) @@ -33,9 +46,19 @@ func TestWriteMergeEligible_ExcludesPRsTheRelayLaneRefuses(t *testing.T) { green := func(number int, author string) github.PullRequest { return github.PullRequest{Repo: "hive", Number: number, Author: author, CIStatus: "success", Mergeable: github.MergeableYes, HeadSHA: "cafe"} } + attributed := green(15088, "contributor") + attributed.HiveAttributed = true + agentAttributed := green(15089, "contributor") + agentAttributed.HiveAgent = "scanner" + app := green(15090, "hive[bot]") + app.AppAuthored = true actionable := &github.ActionableResult{PRs: github.PRResult{Items: []github.PullRequest{ green(15086, "Karibhou"), green(15087, "trusted-human"), + attributed, + agentAttributed, + app, + green(15091, "soaking-contributor"), }}} verdicts := writeMergeEligible(actionable, github.HoldResult{}, "hivecommons", nil, false, nil, false, nil, nil, nil, discardLogger()) @@ -47,14 +70,64 @@ func TestWriteMergeEligible_ExcludesPRsTheRelayLaneRefuses(t *testing.T) { Eligible []struct { Number int `json:"number"` } `json:"merge_eligible"` + NeedsMaintainer []struct { + Number int `json:"number"` + Repo string `json:"repo"` + HeadSHA string `json:"head_sha"` + Reason string `json:"reason"` + } `json:"needs_maintainer_merge"` } if err := json.Unmarshal(data, &out); err != nil { t.Fatal(err) } - if len(out.Eligible) != 1 || out.Eligible[0].Number != 15087 { - t.Fatalf("merge_eligible = %+v, want only #15087", out.Eligible) + if len(out.Eligible) != 2 || out.Eligible[0].Number != 15087 || out.Eligible[1].Number != 15090 { + t.Fatalf("merge_eligible = %+v, want trusted author #15087 and App #15090", out.Eligible) + } + if len(out.NeedsMaintainer) != 3 { + t.Fatalf("needs_maintainer_merge = %+v, want the three trust-refused PRs", out.NeedsMaintainer) + } + for i, number := range []int{15086, 15088, 15089} { + row := out.NeedsMaintainer[i] + if row.Number != number || row.Repo != "hivecommons/hive" || row.HeadSHA != "cafe" || row.Reason != refusal { + t.Errorf("maintainer row = %+v, want #%d with pinned head and relay refusal", row, number) + } + } + for _, pr := range actionable.PRs.Items { + if !called[pr.Number] { + t.Errorf("PR #%d bypassed the authoritative relay gate", pr.Number) + } + v := verdicts[github.MergeVerdictKey(pr)] + if pr.Number != 15087 && pr.Number != 15090 && (v.State == github.MergeVerdictEligible || !strings.Contains(v.Reason, automerge.RelayMergeRefusalPrefix)) { + t.Errorf("refused PR #%d verdict = %+v, want refusal", pr.Number, v) + } } - if v := verdicts[github.MergeVerdictKey(actionable.PRs.Items[0])]; v.State == github.MergeVerdictEligible { - t.Fatalf("refused PR verdict = %+v, want not eligible", v) +} + +func TestRelayLaneRefusalKeepsUnavailableEvidenceForRelayRecheck(t *testing.T) { + t.Cleanup(func() { setMergeEligibleLaneGate(nil) }) + setMergeEligibleLaneGate(func(context.Context, string, int, string) (string, error) { + return "", errors.New("evidence unavailable") + }) + if got := relayLaneRefusal(github.PullRequest{Number: 1}, "hivecommons/hive", discardLogger()); got != "" { + t.Fatalf("unreadable gate refusal = %q, want existing relay-recheck behavior", got) + } +} + +func TestRelayRefusalNeedsMaintainerOnlyForTrustAdmission(t *testing.T) { + for _, tc := range []struct { + reason string + want bool + }{ + {prstage.ContributorLaneReasonNoClaim, true}, + {prstage.ContributorLaneReasonClaimTier, true}, + {prstage.ContributorLaneReasonAuthorAssociation, true}, + {prstage.ContributorLaneReasonSoaking, false}, + {prstage.ContributorLaneReasonNoVerdict, false}, + {"pending: head pushed 1m ago", false}, + } { + refusal := automerge.RelayMergeRefusalPrefix + ": " + tc.reason + " (details)" + if got := relayRefusalNeedsMaintainer(refusal); got != tc.want { + t.Errorf("maintainer classification for %s = %v, want %v", tc.reason, got, tc.want) + } } } diff --git a/src/pkg/scheduler/agent_repo_scope_test.go b/src/pkg/scheduler/agent_repo_scope_test.go index 4c749aa3d6..2d08eb31e9 100644 --- a/src/pkg/scheduler/agent_repo_scope_test.go +++ b/src/pkg/scheduler/agent_repo_scope_test.go @@ -168,6 +168,40 @@ func TestFormatMergeEligibleDataFor_FiltersByRepo(t *testing.T) { } } +func TestFormatMergeEligibleDataFor_SeparatesMaintainerMerges(t *testing.T) { + data := []byte(`{"merge_eligible":[{"number":1,"repo":"console","title":"ready"}],"needs_maintainer_merge":[{"number":2,"repo":"dashboard","title":"approved contributor PR","reason":"contributor-lane:no-claim"},{"number":3,"repo":"console","title":"untrusted tier","reason":"contributor-lane:claim-tier-untrusted"},{"number":4,"repo":"console","title":"another trust hold","reason":"contributor-lane:no-claim"}]}`) + out := formatMergeEligibleData(data, 0) + eligible, maintainer, ok := strings.Cut(out, "NEEDS MAINTAINER MERGE") + if !ok { + t.Fatalf("missing maintainer section: %s", out) + } + if !strings.Contains(eligible, "#1 console") || strings.Contains(eligible, "#2") || strings.Contains(eligible, "#3") { + t.Errorf("eligible section includes refused PRs or loses ready PR: %s", eligible) + } + for _, want := range []string{"#2 dashboard", "#3 console", "#4 console", "contributor-lane:no-claim", "contributor-lane:claim-tier-untrusted", "do not request hive-merge", "merge manually", "registry tier", "verified hive claim"} { + if !strings.Contains(maintainer, want) { + t.Errorf("maintainer section missing %q: %s", want, maintainer) + } + } + // Filtering applies independently to both sections; a scoped agent can + // see maintainer work even when it has no eligible PRs. + only := formatMergeEligibleDataFor(data, func(repo string) bool { return repo == "dashboard" }, 0) + if !strings.HasPrefix(only, "(none)\n") || !strings.Contains(only, "#2 dashboard") || strings.Contains(only, "console") { + t.Errorf("dashboard-only list = %q, want only its maintainer row", only) + } + console := formatMergeEligibleDataFor(data, func(repo string) bool { return repo == "console" }, 1) + if !strings.Contains(console, "#3 console") || strings.Contains(console, "#4 console") || !strings.Contains(console, "1 more") { + t.Errorf("maintainer list did not respect the cap after repo filtering: %s", console) + } + if none := formatMergeEligibleDataFor(data, func(string) bool { return false }, 0); none != "(none)\n" { + t.Errorf("fully filtered list = %q, want (none)", none) + } + legacy := formatMergeEligibleData([]byte(`{"merge_eligible":[{"number":1,"repo":"console","title":"ready"}]}`), 0) + if strings.Contains(legacy, "NEEDS MAINTAINER MERGE") { + t.Errorf("legacy payload gained an empty maintainer section: %s", legacy) + } +} + func TestFilterIssuesForRepos(t *testing.T) { issues := []github.Issue{{Repo: "a"}, {Repo: "b"}} if got := filterIssuesForRepos(issues, nil); len(got) != 2 { diff --git a/src/pkg/scheduler/lane_messages.go b/src/pkg/scheduler/lane_messages.go index 8d4d11bd81..74742c6551 100644 --- a/src/pkg/scheduler/lane_messages.go +++ b/src/pkg/scheduler/lane_messages.go @@ -461,15 +461,18 @@ func (s *Scheduler) buildMergeEligibleListFor(keep func(repo string) bool) strin // all) and caps the rendered list at limit (governor.kick_limits.max_prs, // hivecommons/hive#7368; 0 = uncapped). func formatMergeEligibleDataFor(data []byte, keep func(repo string) bool, limit int) string { + type mergeRow struct { + Number int `json:"number"` + Repo string `json:"repo"` + Title string `json:"title"` + Queued bool `json:"queued"` + Reason string `json:"reason"` + } var payload struct { - Items []struct { - Number int `json:"number"` - Repo string `json:"repo"` - Title string `json:"title"` - Queued bool `json:"queued"` - } `json:"merge_eligible"` + Items []mergeRow `json:"merge_eligible"` + NeedsMaintainer []mergeRow `json:"needs_maintainer_merge"` } - if json.Unmarshal(data, &payload) != nil || len(payload.Items) == 0 { + if json.Unmarshal(data, &payload) != nil { return "(none)\n" } var b strings.Builder @@ -490,7 +493,24 @@ func formatMergeEligibleDataFor(data []byte, keep func(repo string) bool, limit b.WriteString(fmt.Sprintf(" #%d %s%s — %s\n", pr.Number, pr.Repo, queued, pr.Title)) } if b.Len() == 0 { - return "(none)\n" + b.WriteString("(none)\n") + } + var needsMaintainer []mergeRow + for _, pr := range payload.NeedsMaintainer { + if keep == nil || keep(pr.Repo) { + needsMaintainer = append(needsMaintainer, pr) + } + } + if len(needsMaintainer) > 0 { + b.WriteString("\nNEEDS MAINTAINER MERGE (excluded from MERGE-ELIGIBLE; do not request hive-merge):\n") + b.WriteString("A maintainer can review and merge manually. A trusted merger can admit the current head with dashboard Approve or /hive decision approve. An operator can also grant an allowed contributor registry tier (when enabled), or resolve the verified hive claim on the linked closing issue. Agent review approval alone does not grant contributor trust.\n") + for i, pr := range needsMaintainer { + if limit > 0 && i >= limit { + b.WriteString(prListOverflowLine(len(needsMaintainer)-i, limit)) + break + } + b.WriteString(fmt.Sprintf(" #%d %s — %s: %s\n", pr.Number, pr.Repo, pr.Title, pr.Reason)) + } } return b.String() } From cb06655337290ffe317b1c658b5bccb95b53c9d7 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Sat, 7 Nov 2026 12:12:16 -0500 Subject: [PATCH 14/17] =?UTF-8?q?=E2=9C=A8=20crux:=20pinned=20checkout=20s?= =?UTF-8?q?ervice=20(S1)=20(#17005)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the first crux work service from hivecommons/crux#114: a git worktree pinned at a full 40-hex SHA from an allowlisted repository, owned by the S0 caller principal. - checkout/: Service, Config (allowlist, InUse hook, MaxCheckouts), Record, typed sentinels; records under /checkouts/.json + index.jsonl - api/checkouts.go: PathCheckouts, CodeTooLarge/CodeUnsupported/CodeUpstream, request/response types, client methods - serve/checkouts.go: POST/GET/DELETE /v1/checkouts[/{id}], ownership via principal, error mapping; registered from Handler (families.go is at the size ratchet cap) and marked a work service - cli: crux checkout add|rm|list, local and remote; serve flags --checkout-allow/--checkout-dir - docs/design/crux-work-services.md documents S1 as built, including the post-hoc byte cap; TTL sweeper and InUse wiring deferred Refs hivecommons/crux#114 Signed-off-by: Andrew Anderson Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/added-crux-checkout-service.md | 1 + src/crux/api/checkouts.go | 98 +++++ src/crux/checkout/checkout.go | 426 +++++++++++++++++++ src/crux/checkout/checkout_test.go | 473 +++++++++++++++++++++ src/crux/checkout/store.go | 107 +++++ src/crux/cli/checkout.go | 255 +++++++++++ src/crux/cli/checkout_test.go | 209 +++++++++ src/crux/cli/cli.go | 15 +- src/crux/cli/remote.go | 2 + src/crux/cli/serve.go | 12 +- src/crux/cli/verbs_test.go | 2 +- src/crux/config/serve.go | 7 + src/crux/serve/checkouts.go | 166 ++++++++ src/crux/serve/checkouts_test.go | 290 +++++++++++++ src/crux/serve/serve.go | 5 + src/docs/crux-serve.md | 6 + src/docs/design/crux-work-services.md | 160 +++++++ 17 files changed, 2230 insertions(+), 4 deletions(-) create mode 100644 changelog.d/added-crux-checkout-service.md create mode 100644 src/crux/api/checkouts.go create mode 100644 src/crux/checkout/checkout.go create mode 100644 src/crux/checkout/checkout_test.go create mode 100644 src/crux/checkout/store.go create mode 100644 src/crux/cli/checkout.go create mode 100644 src/crux/cli/checkout_test.go create mode 100644 src/crux/serve/checkouts.go create mode 100644 src/crux/serve/checkouts_test.go create mode 100644 src/docs/design/crux-work-services.md diff --git a/changelog.d/added-crux-checkout-service.md b/changelog.d/added-crux-checkout-service.md new file mode 100644 index 0000000000..a04c81ce9a --- /dev/null +++ b/changelog.d/added-crux-checkout-service.md @@ -0,0 +1 @@ +- crux: pinned checkout service (hivecommons/crux#114 S1): `POST/GET/DELETE /v1/checkouts` on `crux serve` (enabled by `--checkout-allow`/`CRUX_CHECKOUT_ALLOWLIST`) and `crux checkout add|rm|list` provision a git worktree pinned at a full 40-hex SHA from an allowlisted repository, owned by the caller principal, with a recorded TTL and a post-hoc size cap. Records live under `/checkouts`; design notes in `docs/design/crux-work-services.md`. diff --git a/src/crux/api/checkouts.go b/src/crux/api/checkouts.go new file mode 100644 index 0000000000..c13c8a9df3 --- /dev/null +++ b/src/crux/api/checkouts.go @@ -0,0 +1,98 @@ +package api + +import ( + "context" + "net/url" + "time" +) + +// Pinned checkouts (hivecommons/crux#114 S1): "a working copy of this repo at +// this commit, owned by this caller, capped, deletable". Routes are a +// work-services family: lane project credentials are accepted here, with +// the project as the owner, and the control token acts as the operator. +const ( + // PathCheckouts is POST (create), GET (list, ?owner=) /v1/checkouts and + // GET/DELETE /v1/checkouts/{id}. + PathCheckouts = "/v1/checkouts" +) + +// Checkout error codes. +const ( + // CodeTooLarge is a checkout that exceeded max_bytes once measured; the + // clone has been removed (413). + CodeTooLarge = "too_large" + // CodeUnsupported is a requested limit the backend could not enforce or + // verify (501); the call was refused, never run with the limit dropped. + CodeUnsupported = "unsupported" + // CodeUpstream is a backend answer that contradicts the request, such as + // a worktree at a different commit than the one pinned (502). + CodeUpstream = "upstream_error" +) + +// CheckoutRequest is the body of POST /v1/checkouts. +type CheckoutRequest struct { + // Repo is the repository URL; it must match a configured allowlist prefix. + Repo string `json:"repo"` + // Commit is the full 40-hex SHA to pin; tags and branches are refused. + Commit string `json:"commit"` + // Owner identifies the owning run or driver. A project principal may + // only name itself (or leave it empty); the operator names anyone and + // defaults to its own name. + Owner string `json:"owner,omitempty"` + // Branches clones all refs when nil or true; false clones one branch. + Branches *bool `json:"branches,omitempty"` + // MaxBytes caps the clone plus worktree; zero means the server default. + MaxBytes int64 `json:"max_bytes,omitempty"` + // TTL is a Go duration ("72h") after which a sweeper may reclaim it. + TTL string `json:"ttl,omitempty"` +} + +// Checkout is one checkout record: the response of POST and GET. +type Checkout struct { + ID string `json:"id"` + Path string `json:"path"` + Repo string `json:"repo"` + Commit string `json:"commit"` + Owner string `json:"owner"` + Branches bool `json:"branches"` + MaxBytes int64 `json:"max_bytes"` + Bytes int64 `json:"bytes"` + TTL string `json:"ttl,omitempty"` + CreatedAt time.Time `json:"created_at"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` +} + +// CheckoutList is the response of GET /v1/checkouts. +type CheckoutList struct { + Checkouts []Checkout `json:"checkouts"` +} + +// CreateCheckout is POST /v1/checkouts. +func (c *Client) CreateCheckout(ctx context.Context, in CheckoutRequest) (Checkout, error) { + var out Checkout + err := c.do(ctx, "POST", PathCheckouts, in, &out) + return out, err +} + +// ListCheckouts is GET /v1/checkouts[?owner=]. +func (c *Client) ListCheckouts(ctx context.Context, owner string) ([]Checkout, error) { + path := PathCheckouts + if owner != "" { + path += "?owner=" + url.QueryEscape(owner) + } + var out CheckoutList + err := c.do(ctx, "GET", path, nil, &out) + return out.Checkouts, err +} + +// GetCheckout is GET /v1/checkouts/{id}. +func (c *Client) GetCheckout(ctx context.Context, id string) (Checkout, error) { + var out Checkout + err := c.do(ctx, "GET", PathCheckouts+"/"+url.PathEscape(id), nil, &out) + return out, err +} + +// DeleteCheckout is DELETE /v1/checkouts/{id}. +func (c *Client) DeleteCheckout(ctx context.Context, id string) error { + return c.do(ctx, "DELETE", PathCheckouts+"/"+url.PathEscape(id), nil, nil) +} diff --git a/src/crux/checkout/checkout.go b/src/crux/checkout/checkout.go new file mode 100644 index 0000000000..2ffaa3e8f0 --- /dev/null +++ b/src/crux/checkout/checkout.go @@ -0,0 +1,426 @@ +// Package checkout is the pinned-checkout work service (hivecommons/crux#114 +// S1): "a working copy of this repo at this commit, owned by this caller, +// capped, deletable". It builds on sandbox.WorktreeManager for the pinned +// worktree and keeps one JSON record per checkout plus an append-only +// index under Config.Dir, the way sandbox_run.go keeps run records. +// +// Trust is deny-by-default: a repository must match one of the configured +// URL prefixes (the shape of Hive's nousdeps fetch allowlist, as a prefix +// list) or it is refused before any clone is attempted. A commit must be a +// full 40-hex SHA so the pin cannot move under the caller. +// +// The size cap is honest, not pre-emptive: git cannot bound a clone before +// transferring it, so the clone and worktree are measured after they land +// and removed when they exceed the cap (ErrTooLarge). When they cannot be +// measured the checkout is removed and the call fails with ErrUnsupported +// rather than delivering an uncapped copy. +package checkout + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/url" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/hivecommons/hive/crux/sandbox" +) + +const ( + // DirName is the directory under the work dir that holds checkouts. + DirName = "checkouts" + // IndexName is the append-only create/delete history inside Dir. + IndexName = "index.jsonl" + // IDPrefix starts every checkout id. + IDPrefix = "chk_" + // DefaultMaxBytes is the size cap when a request names none: 5 GiB. + DefaultMaxBytes int64 = 5 << 30 + // DefaultMaxCheckouts bounds live worktrees under Dir. + DefaultMaxCheckouts = 64 + // MaxOwnerLen bounds the owner identifier. + MaxOwnerLen = 64 + // commitHexLen is the length of a full SHA-1 commit id. + commitHexLen = 40 + // idRandomBytes is the entropy behind an id (12 hex chars). + idRandomBytes = 6 + // sourceDirName holds the per-(repo, owner) clones worktrees pin from. + sourceDirName = "src" + // worktreeStage is the sandbox.WorktreeSpec stage name; a checkout has + // one generation, so the worktree lands at runs//pin-1. + worktreeStage = "pin" + // recordMode and dirMode keep records readable by the runtime's uid only. + recordMode = 0o600 + dirMode = 0o700 +) + +// Sentinel errors; Service methods wrap them so callers map them to typed +// API codes with errors.Is. +var ( + // ErrBadRequest is a malformed request (short SHA, bad URL, bad owner). + ErrBadRequest = errors.New("checkout: bad request") + // ErrForbidden is a repository off the allowlist or a caller that does + // not own the checkout. + ErrForbidden = errors.New("checkout: forbidden") + // ErrNotFound is an unknown checkout id. + ErrNotFound = errors.New("checkout: not found") + // ErrInUse is a delete while contained runs still use the checkout. + ErrInUse = errors.New("checkout: in use") + // ErrMismatch is a worktree whose HEAD is not the requested commit. + ErrMismatch = errors.New("checkout: commit mismatch") + // ErrTooLarge is a clone that exceeded max_bytes once measured. + ErrTooLarge = errors.New("checkout: exceeds size cap") + // ErrUnsupported is a cap the backend could not verify. + ErrUnsupported = errors.New("checkout: cannot enforce size cap") + // ErrGit is a git failure (clone, fetch, worktree). + ErrGit = errors.New("checkout: git") +) + +var ( + commitPattern = regexp.MustCompile(`^[0-9a-f]{40}$`) + ownerPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_.-]*$`) +) + +// Runner executes one command in dir and returns its combined output; the +// default runs it with os/exec. Tests substitute failures. +type Runner func(ctx context.Context, dir string, name string, args ...string) ([]byte, error) + +// Config is the administrator-owned configuration of the service. +type Config struct { + // Dir is where clones, worktrees and records live (DefaultDir). + Dir string + // AllowedRepos are URL prefixes a requested repo must start with. Empty + // denies every request. + AllowedRepos []string + // MaxCheckouts caps live worktrees; zero means DefaultMaxCheckouts. + MaxCheckouts int + // Git runs git; nil uses os/exec. + Git Runner + // Now is the clock; nil uses time.Now. + Now func() time.Time + // InUse reports whether contained runs still reference a checkout; the + // contained-run service wires it. Nil means never in use. + InUse func(id string) bool + // DiskUsage measures a tree in bytes; nil walks it. + DiskUsage func(path string) (int64, error) +} + +// DefaultDir is Dir under the runtime's work dir. +func DefaultDir(workDir string) string { return filepath.Join(workDir, DirName) } + +// Request is what a caller asks for. +type Request struct { + Repo string + Commit string + Owner string + Branches *bool + MaxBytes int64 + TTL time.Duration +} + +// Record is one checkout as stored under Dir/.json. It never carries +// environment or credentials: the repo URL is refused if it embeds any. +type Record struct { + ID string `json:"id"` + Path string `json:"path"` + Repo string `json:"repo"` + Commit string `json:"commit"` + Owner string `json:"owner"` + Branches bool `json:"branches"` + MaxBytes int64 `json:"max_bytes"` + Bytes int64 `json:"bytes"` + TTL string `json:"ttl,omitempty"` + CreatedAt time.Time `json:"created_at"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` + // Source is the shared clone the worktree pins from. + Source string `json:"source"` +} + +// Service creates, lists and deletes checkouts. +type Service struct { + cfg Config + trees sandbox.WorktreeManager + store *store +} + +// New validates cfg and returns a Service. Dir is created on first use. +func New(cfg Config) (*Service, error) { + if strings.TrimSpace(cfg.Dir) == "" { + return nil, fmt.Errorf("%w: dir is required", ErrBadRequest) + } + for i, p := range cfg.AllowedRepos { + p = strings.TrimSpace(p) + if _, err := parseRepo(p); err != nil { + return nil, fmt.Errorf("allowlist entry %q: %w", p, err) + } + cfg.AllowedRepos[i] = p + } + if cfg.MaxCheckouts <= 0 { + cfg.MaxCheckouts = DefaultMaxCheckouts + } + if cfg.Now == nil { + cfg.Now = time.Now + } + if cfg.Git == nil { + cfg.Git = execRunner + } + if cfg.DiskUsage == nil { + cfg.DiskUsage = diskUsage + } + return &Service{cfg: cfg, trees: sandbox.WorktreeManager{Runner: cfg.Git}, store: &store{dir: cfg.Dir}}, nil +} + +// InUse reports whether contained runs reference id (Config.InUse). +func (s *Service) InUse(id string) bool { + return s.cfg.InUse != nil && s.cfg.InUse(id) +} + +// Allowed reports whether repo is on the allowlist, naming the rule that +// admitted it or an ErrForbidden explaining why none did. +func (s *Service) Allowed(repo string) (rule string, err error) { + if len(s.cfg.AllowedRepos) == 0 { + return "", fmt.Errorf("%w: repository %q: the checkout allowlist is empty", ErrForbidden, repo) + } + for _, p := range s.cfg.AllowedRepos { + if strings.HasPrefix(repo, p) { + return p, nil + } + } + return "", fmt.Errorf("%w: repository %q matches none of the allowed prefixes %s", ErrForbidden, repo, strings.Join(s.cfg.AllowedRepos, ", ")) +} + +// Create provisions a checkout for req, recorded as owned by req.Owner. +func (s *Service) Create(ctx context.Context, req Request) (Record, error) { + rec, err := s.validate(req) + if err != nil { + return Record{}, err + } + if _, err := s.Allowed(rec.Repo); err != nil { + return Record{}, err + } + if err := os.MkdirAll(s.cfg.Dir, dirMode); err != nil { + return Record{}, err + } + rec.Source = s.sourceDir(rec.Repo, rec.Owner) + if err := s.ensureSource(ctx, rec); err != nil { + return Record{}, err + } + tree, err := s.trees.Add(ctx, sandbox.WorktreeSpec{ + SourceRepo: rec.Source, WorkspaceDir: s.cfg.Dir, RunKey: rec.ID, Stage: worktreeStage, Generation: 1, + TargetRef: rec.Commit, MaxWorktrees: s.cfg.MaxCheckouts, + }) + if err != nil { + return Record{}, fmt.Errorf("%w: %v", ErrGit, err) + } + rec.Path = tree.Path + if tree.SHA != rec.Commit { + _ = s.trees.Remove(ctx, rec.Source, rec.Path) + return Record{}, fmt.Errorf("%w: requested %s, worktree is at %s", ErrMismatch, rec.Commit, tree.SHA) + } + if err := s.enforceCap(&rec); err != nil { + _ = s.trees.Remove(ctx, rec.Source, rec.Path) + return Record{}, err + } + if err := s.store.put(rec); err != nil { + _ = s.trees.Remove(ctx, rec.Source, rec.Path) + return Record{}, err + } + return rec, s.store.appendIndex(indexEntry{Op: "create", ID: rec.ID, Owner: rec.Owner, Repo: rec.Repo, Commit: rec.Commit, At: rec.CreatedAt}) +} + +// Get returns the record for id. +func (s *Service) Get(id string) (Record, error) { return s.store.get(id) } + +// List returns every record, or only owner's when owner is non-empty, +// ordered by id. +func (s *Service) List(owner string) ([]Record, error) { + all, err := s.store.list() + if err != nil { + return nil, err + } + if owner == "" { + return all, nil + } + out := all[:0] + for _, r := range all { + if r.Owner == owner { + out = append(out, r) + } + } + return out, nil +} + +// Delete removes the worktree and the record. by is who asked, for the +// index; ownership is the caller's decision (see CanDelete). +func (s *Service) Delete(ctx context.Context, id, by string) error { + rec, err := s.store.get(id) + if err != nil { + return err + } + if s.InUse(id) { + return fmt.Errorf("%w: checkout %s has live contained runs", ErrInUse, id) + } + if err := s.trees.Remove(ctx, rec.Source, rec.Path); err != nil { + return fmt.Errorf("%w: remove worktree: %v", ErrGit, err) + } + if err := s.store.remove(id); err != nil { + return err + } + return s.store.appendIndex(indexEntry{Op: "delete", ID: id, Owner: rec.Owner, Repo: rec.Repo, Commit: rec.Commit, At: s.cfg.Now().UTC(), By: by}) +} + +// CanDelete is the ownership rule: the owner or an operator. +func CanDelete(rec Record, caller string, operator bool) bool { + return operator || (caller != "" && caller == rec.Owner) +} + +// validate checks req and returns the record it will produce. +func (s *Service) validate(req Request) (Record, error) { + repo := strings.TrimSpace(req.Repo) + if _, err := parseRepo(repo); err != nil { + return Record{}, err + } + commit := strings.ToLower(strings.TrimSpace(req.Commit)) + if !commitPattern.MatchString(commit) { + return Record{}, fmt.Errorf("%w: commit must be a full %d-hex SHA, not a tag or branch (got %q)", ErrBadRequest, commitHexLen, req.Commit) + } + owner := strings.TrimSpace(req.Owner) + if owner == "" || len(owner) > MaxOwnerLen || !ownerPattern.MatchString(owner) { + return Record{}, fmt.Errorf("%w: owner must be a plain identifier of at most %d characters (got %q)", ErrBadRequest, MaxOwnerLen, req.Owner) + } + if req.MaxBytes < 0 || req.TTL < 0 { + return Record{}, fmt.Errorf("%w: max_bytes and ttl must not be negative", ErrBadRequest) + } + id, err := newID() + if err != nil { + return Record{}, err + } + now := s.cfg.Now().UTC() + rec := Record{ID: id, Repo: repo, Commit: commit, Owner: owner, Branches: req.Branches == nil || *req.Branches, MaxBytes: req.MaxBytes, CreatedAt: now} + if rec.MaxBytes == 0 { + rec.MaxBytes = DefaultMaxBytes + } + if req.TTL > 0 { + rec.TTL = req.TTL.String() + exp := now.Add(req.TTL) + rec.ExpiresAt = &exp + } + return rec, nil +} + +// parseRepo accepts an https:// or file:// URL without credentials, query or +// fragment, so a token can never ride into a record. +func parseRepo(raw string) (*url.URL, error) { + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "https" && u.Scheme != "file") || u.User != nil || u.RawQuery != "" || u.Fragment != "" { + return nil, fmt.Errorf("%w: repo must be an https:// or file:// URL without credentials, query or fragment (got %q)", ErrBadRequest, raw) + } + if (u.Scheme == "https" && u.Host == "") || (u.Scheme == "file" && u.Path == "") { + return nil, fmt.Errorf("%w: repo %q names no host or path", ErrBadRequest, raw) + } + return u, nil +} + +// sourceDir is the shared clone for (repo, owner). +func (s *Service) sourceDir(repo, owner string) string { + sum := sha256.Sum256([]byte(repo)) + return filepath.Join(s.cfg.Dir, sourceDirName, owner+"-"+hex.EncodeToString(sum[:6])) +} + +// ensureSource clones rec.Repo into rec.Source once and makes sure +// rec.Commit is present, fetching it when a reused clone lacks it. +func (s *Service) ensureSource(ctx context.Context, rec Record) error { + if _, err := os.Stat(filepath.Join(rec.Source, ".git")); err != nil { + if err := os.MkdirAll(filepath.Dir(rec.Source), dirMode); err != nil { + return err + } + args := []string{"clone", "--no-checkout"} + if !rec.Branches { + args = append(args, "--single-branch") + } + args = append(args, "--", rec.Repo, rec.Source) + if out, err := s.cfg.Git(ctx, s.cfg.Dir, "git", args...); err != nil { + _ = os.RemoveAll(rec.Source) + return fmt.Errorf("%w: clone: %v: %s", ErrGit, err, strings.TrimSpace(string(out))) + } + } + if _, err := s.cfg.Git(ctx, rec.Source, "git", "cat-file", "-e", rec.Commit+"^{commit}"); err == nil { + return nil + } + if out, err := s.cfg.Git(ctx, rec.Source, "git", "fetch", "--quiet", "origin", rec.Commit); err != nil { + return fmt.Errorf("%w: commit %s is not in %s and could not be fetched: %s", ErrBadRequest, rec.Commit, rec.Repo, strings.TrimSpace(string(out))) + } + if _, err := s.cfg.Git(ctx, rec.Source, "git", "cat-file", "-e", rec.Commit+"^{commit}"); err != nil { + return fmt.Errorf("%w: commit %s is not in %s", ErrBadRequest, rec.Commit, rec.Repo) + } + return nil +} + +// enforceCap measures the clone and the worktree after the fact and refuses +// the checkout when they exceed rec.MaxBytes or cannot be measured. +func (s *Service) enforceCap(rec *Record) error { + src, err := s.cfg.DiskUsage(rec.Source) + if err != nil { + return fmt.Errorf("%w: measuring %s: %v", ErrUnsupported, rec.Source, err) + } + tree, err := s.cfg.DiskUsage(rec.Path) + if err != nil { + return fmt.Errorf("%w: measuring %s: %v", ErrUnsupported, rec.Path, err) + } + rec.Bytes = src + tree + if rec.Bytes > rec.MaxBytes { + return fmt.Errorf("%w: clone and worktree measure %d bytes, cap is %d (the cap is checked after cloning; git cannot bound a clone in advance)", ErrTooLarge, rec.Bytes, rec.MaxBytes) + } + return nil +} + +func newID() (string, error) { + b := make([]byte, idRandomBytes) + if _, err := rand.Read(b); err != nil { + return "", err + } + return IDPrefix + hex.EncodeToString(b), nil +} + +// ValidID reports whether id has the shape newID produces, so a path value +// can never escape Dir. +func ValidID(id string) bool { + rest, ok := strings.CutPrefix(id, IDPrefix) + if !ok || len(rest) != 2*idRandomBytes { + return false + } + _, err := hex.DecodeString(rest) + return err == nil +} + +func execRunner(ctx context.Context, dir string, name string, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Dir = dir + return cmd.CombinedOutput() +} + +// diskUsage sums regular file sizes under path. +func diskUsage(path string) (int64, error) { + var total int64 + err := filepath.WalkDir(path, func(_ string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.Type().IsRegular() { + info, err := d.Info() + if err != nil { + return err + } + total += info.Size() + } + return nil + }) + return total, err +} diff --git a/src/crux/checkout/checkout_test.go b/src/crux/checkout/checkout_test.go new file mode 100644 index 0000000000..95674d2fa5 --- /dev/null +++ b/src/crux/checkout/checkout_test.go @@ -0,0 +1,473 @@ +package checkout + +import ( + "context" + "encoding/json" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +// fixtureRepo creates a bare repository with two commits and returns its +// file:// URL and both SHAs (oldest first). +func fixtureRepo(t *testing.T) (repo string, shas []string) { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + work := filepath.Join(t.TempDir(), "work") + git := func(args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = work + cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@x", "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@x") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + if err := os.MkdirAll(work, 0o700); err != nil { + t.Fatal(err) + } + git("init", "-q") + for i, name := range []string{"one.txt", "two.txt"} { + if err := os.WriteFile(filepath.Join(work, name), []byte(name), 0o600); err != nil { + t.Fatal(err) + } + git("add", name) + git("commit", "-q", "-m", "c"+string(rune('0'+i))) + shas = append(shas, git("rev-parse", "HEAD")) + } + bare := filepath.Join(t.TempDir(), "fixture.git") + git("clone", "-q", "--bare", work, bare) + return "file://" + bare, shas +} + +func newTestService(t *testing.T, repo string, edit func(*Config)) *Service { + t.Helper() + cfg := Config{Dir: filepath.Join(t.TempDir(), "checkouts"), AllowedRepos: []string{"file://" + filepath.Dir(strings.TrimPrefix(repo, "file://"))}} + if edit != nil { + edit(&cfg) + } + svc, err := New(cfg) + if err != nil { + t.Fatal(err) + } + return svc +} + +func TestCreateListDeleteRoundTrip(t *testing.T) { + repo, shas := fixtureRepo(t) + now := time.Date(2026, 11, 7, 12, 0, 0, 0, time.UTC) + svc := newTestService(t, repo, func(c *Config) { c.Now = func() time.Time { return now } }) + ctx := context.Background() + + rec, err := svc.Create(ctx, Request{Repo: repo, Commit: shas[0], Owner: "nous-42", TTL: 72 * time.Hour}) + if err != nil { + t.Fatal(err) + } + if !ValidID(rec.ID) || rec.Commit != shas[0] || rec.Owner != "nous-42" || !rec.Branches || rec.MaxBytes != DefaultMaxBytes || rec.Bytes <= 0 { + t.Fatalf("record = %+v", rec) + } + if rec.TTL != "72h0m0s" || rec.ExpiresAt == nil || !rec.ExpiresAt.Equal(now.Add(72*time.Hour)) || !rec.CreatedAt.Equal(now) { + t.Fatalf("ttl/expiry = %q %v %v", rec.TTL, rec.ExpiresAt, rec.CreatedAt) + } + if _, err := os.Stat(filepath.Join(rec.Path, "one.txt")); err != nil { + t.Fatalf("worktree content: %v", err) + } + if _, err := os.Stat(filepath.Join(rec.Path, "two.txt")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("worktree is not pinned to the first commit: %v", err) + } + // Records are under Dir and the index has the create. + if _, err := os.Stat(filepath.Join(svc.cfg.Dir, rec.ID+".json")); err != nil { + t.Fatal(err) + } + got, err := svc.Get(rec.ID) + if err != nil || got.ID != rec.ID || got.Path != rec.Path || got.Source != rec.Source { + t.Fatalf("Get = %+v, %v", got, err) + } + + // A second checkout for the same (repo, owner) reuses the clone; the + // other commit is reachable without a fetch. + second, err := svc.Create(ctx, Request{Repo: repo, Commit: shas[1], Owner: "nous-42"}) + if err != nil { + t.Fatal(err) + } + if second.Source != rec.Source || second.Path == rec.Path { + t.Fatalf("second = %+v, first = %+v", second, rec) + } + // Another owner gets its own clone. + other, err := svc.Create(ctx, Request{Repo: repo, Commit: shas[1], Owner: "spotlights"}) + if err != nil { + t.Fatal(err) + } + if other.Source == rec.Source { + t.Fatal("owners must not share a clone") + } + + all, err := svc.List("") + if err != nil || len(all) != 3 { + t.Fatalf("List all = %d %v", len(all), err) + } + mine, err := svc.List("nous-42") + if err != nil || len(mine) != 2 { + t.Fatalf("List owner = %d %v", len(mine), err) + } + for i := 1; i < len(all); i++ { + if all[i-1].ID > all[i].ID { + t.Fatal("List is not ordered by id") + } + } + + if err := svc.Delete(ctx, rec.ID, "operator:operator"); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(rec.Path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("worktree still present: %v", err) + } + if _, err := svc.Get(rec.ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("Get after delete = %v", err) + } + if err := svc.Delete(ctx, rec.ID, "x"); !errors.Is(err, ErrNotFound) { + t.Fatalf("second delete = %v", err) + } + + index, err := os.ReadFile(filepath.Join(svc.cfg.Dir, IndexName)) + if err != nil { + t.Fatal(err) + } + lines := strings.Split(strings.TrimSpace(string(index)), "\n") + if len(lines) != 4 { + t.Fatalf("index has %d lines: %s", len(lines), index) + } + var last indexEntry + if err := json.Unmarshal([]byte(lines[3]), &last); err != nil { + t.Fatal(err) + } + if last.Op != "delete" || last.ID != rec.ID || last.By != "operator:operator" || last.Owner != "nous-42" { + t.Fatalf("last index entry = %+v", last) + } +} + +func TestCreateRefusals(t *testing.T) { + repo, shas := fixtureRepo(t) + svc := newTestService(t, repo, nil) + ctx := context.Background() + ok := Request{Repo: repo, Commit: shas[0], Owner: "o"} + cases := []struct { + name string + edit func(*Request) + want error + msg string + }{ + {"short sha", func(r *Request) { r.Commit = shas[0][:12] }, ErrBadRequest, "full 40-hex SHA"}, + {"branch", func(r *Request) { r.Commit = "main" }, ErrBadRequest, "not a tag or branch"}, + {"tag-like", func(r *Request) { r.Commit = "v1.0.0" }, ErrBadRequest, "full 40-hex SHA"}, + {"missing owner", func(r *Request) { r.Owner = "" }, ErrBadRequest, "owner"}, + {"bad owner", func(r *Request) { r.Owner = "Has Spaces" }, ErrBadRequest, "owner"}, + {"long owner", func(r *Request) { r.Owner = strings.Repeat("a", MaxOwnerLen+1) }, ErrBadRequest, "owner"}, + {"negative cap", func(r *Request) { r.MaxBytes = -1 }, ErrBadRequest, "max_bytes"}, + {"negative ttl", func(r *Request) { r.TTL = -time.Second }, ErrBadRequest, "ttl"}, + {"ssh url", func(r *Request) { r.Repo = "git@github.com:org/name.git" }, ErrBadRequest, "https:// or file://"}, + {"http url", func(r *Request) { r.Repo = "http://github.com/org/name.git" }, ErrBadRequest, "https:// or file://"}, + {"credentials in url", func(r *Request) { r.Repo = "https://user:tok@github.com/org/name.git" }, ErrBadRequest, "credentials"}, + {"query in url", func(r *Request) { r.Repo = "https://github.com/org/name.git?x=1" }, ErrBadRequest, "query"}, + {"no host", func(r *Request) { r.Repo = "https:///name.git" }, ErrBadRequest, "no host"}, + {"off allowlist", func(r *Request) { r.Repo = "https://github.com/org/name.git" }, ErrForbidden, "matches none of the allowed prefixes"}, + {"unknown commit", func(r *Request) { r.Commit = strings.Repeat("0", 40) }, ErrBadRequest, "could not be fetched"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + req := ok + tc.edit(&req) + _, err := svc.Create(ctx, req) + if !errors.Is(err, tc.want) || !strings.Contains(err.Error(), tc.msg) { + t.Fatalf("Create = %v, want %v containing %q", err, tc.want, tc.msg) + } + }) + } + // The forbidden error names the rule set that denied it, and an empty + // allowlist denies everything with its own message. + if _, err := svc.Allowed("https://evil.example/x.git"); err == nil || !strings.Contains(err.Error(), svc.cfg.AllowedRepos[0]) { + t.Fatalf("Allowed off-list = %v", err) + } + if rule, err := svc.Allowed(repo); err != nil || rule != svc.cfg.AllowedRepos[0] { + t.Fatalf("Allowed on-list = %q %v", rule, err) + } + empty := newTestService(t, repo, func(c *Config) { c.AllowedRepos = nil }) + if _, err := empty.Create(ctx, ok); !errors.Is(err, ErrForbidden) || !strings.Contains(err.Error(), "allowlist is empty") { + t.Fatalf("empty allowlist = %v", err) + } + // Nothing was written for refused requests. + if list, err := svc.List(""); err != nil || len(list) != 0 { + t.Fatalf("records after refusals = %v %v", list, err) + } +} + +func TestNewValidatesConfig(t *testing.T) { + if _, err := New(Config{}); !errors.Is(err, ErrBadRequest) { + t.Fatalf("New without dir = %v", err) + } + if _, err := New(Config{Dir: t.TempDir(), AllowedRepos: []string{"github.com/org/"}}); !errors.Is(err, ErrBadRequest) || !strings.Contains(err.Error(), "allowlist entry") { + t.Fatalf("New with bare prefix = %v", err) + } + svc, err := New(Config{Dir: t.TempDir(), AllowedRepos: []string{" https://github.com/org/ "}}) + if err != nil || svc.cfg.AllowedRepos[0] != "https://github.com/org/" || svc.cfg.MaxCheckouts != DefaultMaxCheckouts { + t.Fatalf("New = %+v %v", svc, err) + } + if svc.InUse("chk_000000000000") { + t.Fatal("nil InUse hook must report not in use") + } + if DefaultDir("/data/agents") != "/data/agents/checkouts" { + t.Fatal(DefaultDir("/data/agents")) + } + for id, want := range map[string]bool{"chk_0123456789ab": true, "chk_0123456789aB": true, "chk_0123456789": false, "run_0123456789ab": false, "chk_../../etc/pw": false, "": false} { + if ValidID(id) != want { + t.Errorf("ValidID(%q) = %v", id, !want) + } + } +} + +func TestSizeCapIsPostHocAndRefuses(t *testing.T) { + repo, shas := fixtureRepo(t) + svc := newTestService(t, repo, nil) + ctx := context.Background() + _, err := svc.Create(ctx, Request{Repo: repo, Commit: shas[0], Owner: "o", MaxBytes: 1}) + if !errors.Is(err, ErrTooLarge) || !strings.Contains(err.Error(), "cap is 1") || !strings.Contains(err.Error(), "after cloning") { + t.Fatalf("over cap = %v", err) + } + // The over-cap worktree is gone and nothing was recorded; the shared + // clone stays for the next request. + if list, _ := svc.List(""); len(list) != 0 { + t.Fatalf("records after refusal: %v", list) + } + if entries, _ := os.ReadDir(filepath.Join(svc.cfg.Dir, "runs")); len(entries) != 0 { + for _, e := range entries { + if sub, _ := os.ReadDir(filepath.Join(svc.cfg.Dir, "runs", e.Name())); len(sub) != 0 { + t.Fatalf("worktree left behind: %v", sub) + } + } + } + + // A cap that cannot be measured is refused as unsupported, not skipped. + broken := newTestService(t, repo, func(c *Config) { + c.DiskUsage = func(string) (int64, error) { return 0, errors.New("no du") } + }) + if _, err := broken.Create(ctx, Request{Repo: repo, Commit: shas[0], Owner: "o"}); !errors.Is(err, ErrUnsupported) { + t.Fatalf("unmeasurable = %v", err) + } + // Measuring only the worktree failing is the same refusal. + calls := 0 + broken = newTestService(t, repo, func(c *Config) { + c.DiskUsage = func(path string) (int64, error) { + calls++ + if calls == 2 { + return 0, errors.New("no du") + } + return 1, nil + } + }) + if _, err := broken.Create(ctx, Request{Repo: repo, Commit: shas[0], Owner: "o"}); !errors.Is(err, ErrUnsupported) { + t.Fatalf("unmeasurable worktree = %v", err) + } + if _, err := diskUsage(filepath.Join(t.TempDir(), "missing")); err == nil { + t.Fatal("diskUsage of a missing path must fail") + } +} + +func TestDeleteOwnershipAndInUse(t *testing.T) { + repo, shas := fixtureRepo(t) + inUse := map[string]bool{} + svc := newTestService(t, repo, func(c *Config) { c.InUse = func(id string) bool { return inUse[id] } }) + ctx := context.Background() + rec, err := svc.Create(ctx, Request{Repo: repo, Commit: shas[0], Owner: "atlas"}) + if err != nil { + t.Fatal(err) + } + if !CanDelete(rec, "atlas", false) || CanDelete(rec, "beacon", false) || !CanDelete(rec, "beacon", true) || CanDelete(rec, "", false) { + t.Fatal("ownership rule") + } + inUse[rec.ID] = true + if err := svc.Delete(ctx, rec.ID, "project:atlas"); !errors.Is(err, ErrInUse) || !svc.InUse(rec.ID) { + t.Fatalf("delete in use = %v", err) + } + inUse[rec.ID] = false + if err := svc.Delete(ctx, rec.ID, "project:atlas"); err != nil { + t.Fatal(err) + } + if _, err := svc.Get("../escape"); !errors.Is(err, ErrNotFound) { + t.Fatalf("Get with a path = %v", err) + } + if err := svc.Delete(ctx, "chk_000000000000", "x"); !errors.Is(err, ErrNotFound) { + t.Fatalf("Delete unknown = %v", err) + } +} + +func TestGitFailuresAreTyped(t *testing.T) { + repo, shas := fixtureRepo(t) + ctx := context.Background() + real := execRunner + // Clone fails: typed ErrGit and the half-made source removed. + failClone := newTestService(t, repo, func(c *Config) { + c.Git = func(ctx context.Context, dir, name string, args ...string) ([]byte, error) { + if args[0] == "clone" { + return []byte("fatal: nope"), errors.New("exit 128") + } + return real(ctx, dir, name, args...) + } + }) + _, err := failClone.Create(ctx, Request{Repo: repo, Commit: shas[0], Owner: "o"}) + if !errors.Is(err, ErrGit) || !strings.Contains(err.Error(), "fatal: nope") { + t.Fatalf("clone failure = %v", err) + } + if entries, _ := os.ReadDir(filepath.Join(failClone.cfg.Dir, sourceDirName)); len(entries) != 0 { + t.Fatalf("source left behind: %v", entries) + } + + // Worktree add fails. + failTree := newTestService(t, repo, func(c *Config) { + c.Git = func(ctx context.Context, dir, name string, args ...string) ([]byte, error) { + if args[0] == "worktree" { + return []byte("boom"), errors.New("exit 1") + } + return real(ctx, dir, name, args...) + } + }) + if _, err := failTree.Create(ctx, Request{Repo: repo, Commit: shas[0], Owner: "o"}); !errors.Is(err, ErrGit) || !strings.Contains(err.Error(), "boom") { + t.Fatalf("worktree failure = %v", err) + } + + // HEAD disagrees with the pin: 502-class mismatch naming both SHAs. + mismatch := newTestService(t, repo, func(c *Config) { + c.Git = func(ctx context.Context, dir, name string, args ...string) ([]byte, error) { + if args[0] == "rev-parse" { + return []byte(shas[1] + "\n"), nil + } + return real(ctx, dir, name, args...) + } + }) + _, err = mismatch.Create(ctx, Request{Repo: repo, Commit: shas[0], Owner: "o"}) + if !errors.Is(err, ErrMismatch) || !strings.Contains(err.Error(), shas[0]) || !strings.Contains(err.Error(), shas[1]) { + t.Fatalf("mismatch = %v", err) + } + + // A fetch that "succeeds" without delivering the commit is still a + // bad request, not a checkout at the wrong commit. + phantom := newTestService(t, repo, func(c *Config) { + c.Git = func(ctx context.Context, dir, name string, args ...string) ([]byte, error) { + if args[0] == "fetch" { + return nil, nil + } + return real(ctx, dir, name, args...) + } + }) + if _, err := phantom.Create(ctx, Request{Repo: repo, Commit: strings.Repeat("a", 40), Owner: "o"}); !errors.Is(err, ErrBadRequest) || !strings.Contains(err.Error(), "is not in") { + t.Fatalf("phantom fetch = %v", err) + } + + // Single-branch clones are requested when branches is false. + var cloneArgs []string + single := newTestService(t, repo, func(c *Config) { + c.Git = func(ctx context.Context, dir, name string, args ...string) ([]byte, error) { + if args[0] == "clone" { + cloneArgs = args + } + return real(ctx, dir, name, args...) + } + }) + no := false + rec, err := single.Create(ctx, Request{Repo: repo, Commit: shas[1], Owner: "o", Branches: &no}) + if err != nil || rec.Branches { + t.Fatalf("single branch = %+v %v", rec, err) + } + if !contains(cloneArgs, "--single-branch") { + t.Fatalf("clone args = %v", cloneArgs) + } + + // Removing a worktree whose git state is gone falls back to deleting + // the directory; a runner that fails on remove is reported. + failRemove := newTestService(t, repo, func(c *Config) { + c.Git = func(ctx context.Context, dir, name string, args ...string) ([]byte, error) { + if args[0] == "worktree" && args[1] == "remove" { + return nil, errors.New("locked") + } + return real(ctx, dir, name, args...) + } + }) + rec, err = failRemove.Create(ctx, Request{Repo: repo, Commit: shas[0], Owner: "o"}) + if err != nil { + t.Fatal(err) + } + if err := failRemove.Delete(ctx, rec.ID, "x"); err != nil { + t.Fatalf("delete with failing git remove should fall back to os.RemoveAll: %v", err) + } + if _, err := os.Stat(rec.Path); !errors.Is(err, os.ErrNotExist) { + t.Fatal("worktree directory still present") + } +} + +func TestStoreCorruptionAndIO(t *testing.T) { + dir := t.TempDir() + st := &store{dir: dir} + id := "chk_0123456789ab" + if err := os.WriteFile(st.recordPath(id), []byte("{not json"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := st.get(id); err == nil || !strings.Contains(err.Error(), "corrupt record") { + t.Fatalf("corrupt get = %v", err) + } + if _, err := st.list(); err == nil { + t.Fatal("list must surface a corrupt record") + } + // Non-record files and directories under Dir are ignored by list. + if err := os.Remove(st.recordPath(id)); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "notes.json"), []byte("{}"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(dir, "runs"), 0o700); err != nil { + t.Fatal(err) + } + if list, err := st.list(); err != nil || len(list) != 0 { + t.Fatalf("list = %v %v", list, err) + } + if list, err := (&store{dir: filepath.Join(dir, "absent")}).list(); err != nil || len(list) != 0 { + t.Fatalf("list of absent dir = %v %v", list, err) + } + if err := st.remove("chk_ffffffffffff"); err != nil { + t.Fatalf("remove of absent record = %v", err) + } + // Writes into an unwritable dir fail loudly rather than silently. + ro := &store{dir: filepath.Join(dir, "absent")} + if err := ro.put(Record{ID: id}); err == nil { + t.Fatal("put into a missing dir must fail") + } + if err := ro.appendIndex(indexEntry{Op: "create"}); err == nil { + t.Fatal("appendIndex into a missing dir must fail") + } + if _, err := (&store{dir: filepath.Join(dir, "runs")}).list(); err != nil { + t.Fatal(err) + } + // A directory where the record file should be is a read error. + if err := os.MkdirAll(st.recordPath("chk_aaaaaaaaaaaa"), 0o700); err != nil { + t.Fatal(err) + } + if _, err := st.get("chk_aaaaaaaaaaaa"); err == nil || errors.Is(err, ErrNotFound) { + t.Fatalf("get of a directory = %v", err) + } +} + +func contains(list []string, want string) bool { + for _, s := range list { + if s == want { + return true + } + } + return false +} diff --git a/src/crux/checkout/store.go b/src/crux/checkout/store.go new file mode 100644 index 0000000000..fccce05656 --- /dev/null +++ b/src/crux/checkout/store.go @@ -0,0 +1,107 @@ +package checkout + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +// store is the record layout: Dir/.json per checkout and Dir/index.jsonl +// as the append-only create/delete history. +type store struct{ dir string } + +// indexEntry is one line of index.jsonl. +type indexEntry struct { + Op string `json:"op"` + ID string `json:"id"` + Owner string `json:"owner"` + Repo string `json:"repo"` + Commit string `json:"commit"` + At time.Time `json:"at"` + By string `json:"by,omitempty"` +} + +func (st *store) recordPath(id string) string { return filepath.Join(st.dir, id+".json") } + +func (st *store) put(rec Record) error { + data, err := json.Marshal(rec) + if err != nil { + return err + } + tmp := st.recordPath(rec.ID) + ".tmp" + if err := os.WriteFile(tmp, data, recordMode); err != nil { + return err + } + return os.Rename(tmp, st.recordPath(rec.ID)) +} + +func (st *store) get(id string) (Record, error) { + if !ValidID(id) { + return Record{}, fmt.Errorf("%w: %q is not a checkout id", ErrNotFound, id) + } + data, err := os.ReadFile(st.recordPath(id)) + if errors.Is(err, os.ErrNotExist) { + return Record{}, fmt.Errorf("%w: checkout %s", ErrNotFound, id) + } + if err != nil { + return Record{}, err + } + var rec Record + if err := json.Unmarshal(data, &rec); err != nil { + return Record{}, fmt.Errorf("checkout %s: corrupt record: %w", id, err) + } + return rec, nil +} + +func (st *store) remove(id string) error { + err := os.Remove(st.recordPath(id)) + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err +} + +func (st *store) list() ([]Record, error) { + entries, err := os.ReadDir(st.dir) + if errors.Is(err, os.ErrNotExist) { + return []Record{}, nil + } + if err != nil { + return nil, err + } + out := []Record{} + for _, e := range entries { + id, ok := strings.CutSuffix(e.Name(), ".json") + if !ok || e.IsDir() || !ValidID(id) { + continue + } + rec, err := st.get(id) + if err != nil { + return nil, err + } + out = append(out, rec) + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out, nil +} + +func (st *store) appendIndex(entry indexEntry) error { + data, err := json.Marshal(entry) + if err != nil { + return err + } + f, err := os.OpenFile(filepath.Join(st.dir, IndexName), os.O_APPEND|os.O_CREATE|os.O_WRONLY, recordMode) + if err != nil { + return err + } + if _, err := f.Write(append(data, '\n')); err != nil { + _ = f.Close() + return err + } + return f.Close() +} diff --git a/src/crux/cli/checkout.go b/src/crux/cli/checkout.go new file mode 100644 index 0000000000..459d416834 --- /dev/null +++ b/src/crux/cli/checkout.go @@ -0,0 +1,255 @@ +package cli + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + "strings" + "text/tabwriter" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/checkout" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/serve" +) + +// `crux checkout add|rm|list` is the pinned-checkout work service +// (hivecommons/crux#114 S1). It does not touch crux.Runtime: locally it +// drives checkout.Service on /checkouts as the operator; with +// --server it drives the /v1/checkouts routes as whoever the token is. + +const checkoutUsage = "usage: crux checkout add -repo URL -commit SHA [-owner O] [-ttl 72h] [-max-bytes N] [-no-branches] | rm ID | list [-owner O]" + +// checkoutClient is the one shape the local and remote paths share. +type checkoutClient interface { + CreateCheckout(ctx context.Context, in api.CheckoutRequest) (api.Checkout, error) + ListCheckouts(ctx context.Context, owner string) ([]api.Checkout, error) + DeleteCheckout(ctx context.Context, id string) error +} + +// localCheckouts adapts checkout.Service to checkoutClient for the +// in-process path, where the caller is the operator. +type localCheckouts struct{ svc *checkout.Service } + +func (l localCheckouts) CreateCheckout(ctx context.Context, in api.CheckoutRequest) (api.Checkout, error) { + req := checkout.Request{Repo: in.Repo, Commit: in.Commit, Owner: in.Owner, Branches: in.Branches, MaxBytes: in.MaxBytes} + if req.Owner == "" { + req.Owner = serve.DefaultOperatorName + } + if in.TTL != "" { + ttl, err := time.ParseDuration(in.TTL) + if err != nil { + return api.Checkout{}, fmt.Errorf("%w: ttl: %v", checkout.ErrBadRequest, err) + } + req.TTL = ttl + } + rec, err := l.svc.Create(ctx, req) + return checkoutRecordToAPI(rec), err +} + +func (l localCheckouts) ListCheckouts(_ context.Context, owner string) ([]api.Checkout, error) { + recs, err := l.svc.List(owner) + out := make([]api.Checkout, 0, len(recs)) + for _, rec := range recs { + out = append(out, checkoutRecordToAPI(rec)) + } + return out, err +} + +func (l localCheckouts) DeleteCheckout(ctx context.Context, id string) error { + return l.svc.Delete(ctx, id, serve.KindOperator+":"+serve.DefaultOperatorName) +} + +func checkoutRecordToAPI(rec checkout.Record) api.Checkout { + return api.Checkout{ID: rec.ID, Path: rec.Path, Repo: rec.Repo, Commit: rec.Commit, Owner: rec.Owner, Branches: rec.Branches, + MaxBytes: rec.MaxBytes, Bytes: rec.Bytes, TTL: rec.TTL, CreatedAt: rec.CreatedAt, ExpiresAt: rec.ExpiresAt} +} + +// newCheckoutService builds the service from the flag or environment +// allowlist; nil with no error means no allowlist was given, so the +// service stays off. +func newCheckoutService(allow, dir, workDir string) (*checkout.Service, error) { + if allow == "" { + allow = os.Getenv(config.CheckoutAllowlistEnvVar) + } + if strings.TrimSpace(allow) == "" { + return nil, nil + } + if dir == "" { + dir = os.Getenv(config.CheckoutDirEnvVar) + } + if dir == "" { + dir = checkout.DefaultDir(workDir) + } + var prefixes []string + for _, p := range strings.Split(allow, ",") { + if p = strings.TrimSpace(p); p != "" { + prefixes = append(prefixes, p) + } + } + return checkout.New(checkout.Config{Dir: dir, AllowedRepos: prefixes}) +} + +// cmdCheckout is the local `crux checkout`: the service on . +func (e *Env) cmdCheckout(ctx context.Context, opts globalOptions, args []string) int { + if len(args) == 0 { + fmt.Fprintln(e.Stderr, checkoutUsage) + return ExitUsage + } + fs := e.subFlags("checkout " + args[0]) + allow := fs.String("allow", "", "comma-separated repo URL prefixes this command may clone (default $"+config.CheckoutAllowlistEnvVar+")") + dir := fs.String("dir", "", "where checkouts live (default $"+config.CheckoutDirEnvVar+", then /"+checkout.DirName+")") + return e.checkout(ctx, func() (checkoutClient, error) { + svc, err := newCheckoutService(*allow, *dir, opts.workDir) + if err == nil && svc == nil { + // No allowlist: a service that refuses every clone but can + // still list and remove what exists. + svc, err = checkout.New(checkout.Config{Dir: checkoutDirFor(*dir, opts.workDir)}) + } + if err != nil { + return nil, err + } + return localCheckouts{svc}, nil + }, fs, args) +} + +func checkoutDirFor(dir, workDir string) string { + if dir == "" { + dir = os.Getenv(config.CheckoutDirEnvVar) + } + if dir == "" { + dir = checkout.DefaultDir(workDir) + } + return dir +} + +// remoteCheckout is `crux --server … checkout`. +func (e *Env) remoteCheckout(ctx context.Context, c *api.Client, args []string) int { + if len(args) == 0 { + fmt.Fprintln(e.Stderr, checkoutUsage) + return ExitUsage + } + return e.checkout(ctx, func() (checkoutClient, error) { return c, nil }, e.subFlags("checkout "+args[0]), args) +} + +// checkout dispatches the add/rm/list verb. fs may already carry the +// local-only flags; the verb's own flags are added here, and newClient runs +// after parsing so those flags are in effect. +func (e *Env) checkout(ctx context.Context, newClient func() (checkoutClient, error), fs *flag.FlagSet, args []string) int { + verb, rest := args[0], args[1:] + client := func() (checkoutClient, bool) { + c, err := newClient() + if err != nil { + fmt.Fprintln(e.Stderr, "crux: checkout:", err) + } + return c, err == nil + } + switch verb { + case "add": + var in api.CheckoutRequest + var noBranches bool + fs.StringVar(&in.Repo, "repo", "", "repository URL (must match the server's allowlist)") + fs.StringVar(&in.Commit, "commit", "", "full 40-hex commit SHA to pin") + fs.StringVar(&in.Owner, "owner", "", "owning run or driver (default: the caller)") + fs.StringVar(&in.TTL, "ttl", "", "duration after which a sweeper may reclaim it, e.g. 72h") + fs.Int64Var(&in.MaxBytes, "max-bytes", 0, "size cap for clone plus worktree (default 5 GiB); checked after cloning") + fs.BoolVar(&noBranches, "no-branches", false, "clone a single branch instead of all refs") + asJSON := fs.Bool("json", false, "emit the record as JSON") + if code, ok := parseSub(fs, rest); !ok { + return code + } + if in.Repo == "" || in.Commit == "" || fs.NArg() != 0 { + fmt.Fprintln(e.Stderr, checkoutUsage) + return ExitUsage + } + if noBranches { + f := false + in.Branches = &f + } + c, ok := client() + if !ok { + return ExitUsage + } + rec, err := c.CreateCheckout(ctx, in) + if err != nil { + return e.checkoutErr("add", err) + } + if *asJSON { + return e.writeJSON(rec) + } + fmt.Fprintf(e.Stdout, "%s\t%s\n", rec.ID, rec.Path) + return ExitOK + case "rm": + if code, ok := parseSub(fs, rest); !ok { + return code + } + if fs.NArg() != 1 { + fmt.Fprintln(e.Stderr, checkoutUsage) + return ExitUsage + } + c, ok := client() + if !ok { + return ExitUsage + } + if err := c.DeleteCheckout(ctx, fs.Arg(0)); err != nil { + return e.checkoutErr("rm", err) + } + return ExitOK + case "list": + owner := fs.String("owner", "", "only this owner's checkouts") + asJSON := fs.Bool("json", false, "emit a JSON array instead of a table") + if code, ok := parseSub(fs, rest); !ok { + return code + } + if fs.NArg() != 0 { + fmt.Fprintln(e.Stderr, checkoutUsage) + return ExitUsage + } + c, ok := client() + if !ok { + return ExitUsage + } + recs, err := c.ListCheckouts(ctx, *owner) + if err != nil { + return e.checkoutErr("list", err) + } + if *asJSON { + return e.writeJSON(recs) + } + tw := tabwriter.NewWriter(e.Stdout, 0, 0, 2, ' ', 0) + fmt.Fprintln(tw, "ID\tOWNER\tCOMMIT\tCREATED\tEXPIRES\tPATH") + for _, r := range recs { + expires := "-" + if r.ExpiresAt != nil { + expires = r.ExpiresAt.UTC().Format(time.RFC3339) + } + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\n", r.ID, r.Owner, r.Commit, r.CreatedAt.UTC().Format(time.RFC3339), expires, r.Path) + } + if err := tw.Flush(); err != nil { + fmt.Fprintln(e.Stderr, "crux:", err) + return ExitError + } + return ExitOK + default: + fmt.Fprintf(e.Stderr, "crux: checkout: unknown verb %q\n%s\n", verb, checkoutUsage) + return ExitUsage + } +} + +// checkoutErr prints a failure; malformed requests and refusals are usage +// errors, everything else a runtime failure. +func (e *Env) checkoutErr(verb string, err error) int { + fmt.Fprintf(e.Stderr, "crux: checkout %s: %v\n", verb, err) + var se *api.StatusError + if errors.Is(err, checkout.ErrBadRequest) || errors.Is(err, checkout.ErrForbidden) || + (errors.As(err, &se) && (se.Body.Code == api.CodeBadRequest || se.Body.Code == api.CodeForbidden)) { + return ExitUsage + } + if api.IsUnauthorized(err) { + fmt.Fprintf(e.Stderr, "crux: the server wants a bearer token; pass --token or set %s\n", config.ServeTokenEnvVar) + } + return ExitError +} diff --git a/src/crux/cli/checkout_test.go b/src/crux/cli/checkout_test.go new file mode 100644 index 0000000000..7e32fd3c8c --- /dev/null +++ b/src/crux/cli/checkout_test.go @@ -0,0 +1,209 @@ +package cli + +import ( + "encoding/json" + "net" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/checkout" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/serve" +) + +// checkoutFixtureRepo is a bare repo with one commit, as a file:// URL, +// plus the allowlist prefix that admits it. +func checkoutFixtureRepo(t *testing.T) (repo, sha, allow string) { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + work := filepath.Join(t.TempDir(), "work") + if err := os.MkdirAll(work, 0o700); err != nil { + t.Fatal(err) + } + git := func(args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = work + cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@x", "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@x") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + git("init", "-q") + if err := os.WriteFile(filepath.Join(work, "f"), []byte("f"), 0o600); err != nil { + t.Fatal(err) + } + git("add", "f") + git("commit", "-q", "-m", "c") + sha = git("rev-parse", "HEAD") + bare := filepath.Join(t.TempDir(), "fixture.git") + git("clone", "-q", "--bare", work, bare) + return "file://" + bare, sha, "file://" + filepath.Dir(bare) +} + +func TestCheckoutLocalAddListRm(t *testing.T) { + repo, sha, allow := checkoutFixtureRepo(t) + t.Setenv(config.CheckoutAllowlistEnvVar, "") + t.Setenv(config.CheckoutDirEnvVar, "") + h := newHarness(t) + dir := filepath.Join(t.TempDir(), "chk") + + if code := h.run("checkout", "add", "-allow", allow, "-dir", dir, "-repo", repo, "-commit", sha, "-ttl", "1h"); code != ExitOK { + t.Fatalf("add: %d %s", code, h.stderr) + } + fields := strings.Fields(h.stdout.String()) + if len(fields) != 2 || !strings.HasPrefix(fields[0], checkout.IDPrefix) || !strings.HasPrefix(fields[1], dir) { + t.Fatalf("add output = %q", h.stdout.String()) + } + id := fields[0] + if _, err := os.Stat(filepath.Join(fields[1], "f")); err != nil { + t.Fatalf("pinned file: %v", err) + } + + // Without an allowlist the local service still lists and removes. + h.stdout.Reset() + if code := h.run("checkout", "list", "-dir", dir); code != ExitOK { + t.Fatalf("list: %d %s", code, h.stderr) + } + if out := h.stdout.String(); !strings.Contains(out, "ID") || !strings.Contains(out, id) || !strings.Contains(out, serve.DefaultOperatorName) || !strings.Contains(out, sha) { + t.Fatalf("list output = %q", out) + } + h.stdout.Reset() + if code := h.run("checkout", "list", "-dir", dir, "-owner", "nobody", "-json"); code != ExitOK || strings.TrimSpace(h.stdout.String()) != "[]" { + t.Fatalf("list nobody: %d %q", code, h.stdout.String()) + } + h.stdout.Reset() + if code := h.run("checkout", "add", "-dir", dir, "-repo", repo, "-commit", sha); code != ExitUsage || !strings.Contains(h.stderr.String(), "allowlist is empty") { + t.Fatalf("add without allowlist: %d %s", code, h.stderr) + } + h.stderr.Reset() + if code := h.run("checkout", "add", "-allow", allow, "-dir", dir, "-repo", repo, "-commit", "main", "-json"); code != ExitUsage || !strings.Contains(h.stderr.String(), "40-hex") { + t.Fatalf("add branch: %d %s", code, h.stderr) + } + h.stderr.Reset() + if code := h.run("checkout", "rm", "-dir", dir, id); code != ExitOK { + t.Fatalf("rm: %d %s", code, h.stderr) + } + if code := h.run("checkout", "rm", "-dir", dir, id); code != ExitError || !strings.Contains(h.stderr.String(), "not found") { + t.Fatalf("rm again: %d %s", code, h.stderr) + } + + // The allowlist and dir also come from the environment, and the dir + // defaults under the work dir. + t.Setenv(config.CheckoutAllowlistEnvVar, allow) + work := t.TempDir() + h.stdout.Reset() + if code := h.env.Run(t.Context(), []string{"--workdir", work, "checkout", "add", "-repo", repo, "-commit", sha, "-no-branches", "-json"}); code != ExitOK { + t.Fatalf("add via env: %d %s", code, h.stderr) + } + var rec api.Checkout + if err := json.Unmarshal(h.stdout.Bytes(), &rec); err != nil || !strings.HasPrefix(rec.Path, checkout.DefaultDir(work)) || rec.Branches { + t.Fatalf("add via env = %s %v", h.stdout.String(), err) + } + if got := checkoutDirFor("", work); got != checkout.DefaultDir(work) { + t.Fatalf("checkoutDirFor = %q", got) + } + t.Setenv(config.CheckoutDirEnvVar, "/elsewhere") + if got := checkoutDirFor("", work); got != "/elsewhere" { + t.Fatalf("checkoutDirFor from env = %q", got) + } + if got := checkoutDirFor("/flag", work); got != "/flag" { + t.Fatalf("checkoutDirFor from flag = %q", got) + } +} + +func TestCheckoutUsageErrors(t *testing.T) { + t.Setenv(config.CheckoutAllowlistEnvVar, "") + h := newHarness(t) + cases := [][]string{ + {"checkout"}, + {"checkout", "bogus"}, + {"checkout", "add"}, + {"checkout", "add", "-repo", "x"}, + {"checkout", "add", "-repo", "x", "-commit", "y", "extra"}, + {"checkout", "rm"}, + {"checkout", "rm", "a", "b"}, + {"checkout", "list", "extra"}, + {"checkout", "list", "-nope"}, + } + for _, args := range cases { + h.stderr.Reset() + if code := h.run(args...); code != ExitUsage { + t.Errorf("%v: code %d, want usage; stderr %s", args, code, h.stderr) + } + } + h.stderr.Reset() + if code := h.run("checkout", "list", "-h"); code != ExitOK { + t.Errorf("list -h: %d", code) + } + if code := h.run("checkout", "add", "-allow", "ssh://bad", "-repo", "https://x/y", "-commit", strings.Repeat("a", 40)); code != ExitUsage || !strings.Contains(h.stderr.String(), "crux: checkout:") { + t.Errorf("bad allowlist: %d %s", code, h.stderr) + } + // The control socket does not forward checkout. + e := &Env{Stdout: h.stdout, Stderr: h.stderr} + h.stderr.Reset() + if code := e.dispatch(t.Context(), globalOptions{}, "checkout", []string{"list"}, true); code != ExitUsage || !strings.Contains(h.stderr.String(), "control socket") { + t.Errorf("remote dispatch: %d %s", code, h.stderr) + } +} + +func TestCheckoutRemote(t *testing.T) { + repo, sha, allow := checkoutFixtureRepo(t) + t.Setenv(config.ServerEnvVar, "") + t.Setenv(config.ServeTokenEnvVar, "") + svc, err := checkout.New(checkout.Config{Dir: filepath.Join(t.TempDir(), "chk"), AllowedRepos: []string{allow}}) + if err != nil { + t.Fatal(err) + } + s := serve.New(newFakeRuntime()) + s.Token, s.Checkouts = "tok", svc + path := filepath.Join(shortSocketDir(t), "api.sock") + ln, err := net.Listen("unix", path) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewUnstartedServer(s.Handler()) + srv.Listener = ln + srv.Start() + t.Cleanup(srv.Close) + + h := newHarness(t) + remote := func(args ...string) int { + return h.run(append([]string{"--server", path, "--token", "tok", "checkout"}, args...)...) + } + if code := remote(); code != ExitUsage { + t.Fatalf("no verb: %d", code) + } + if code := remote("add", "-repo", repo, "-commit", sha, "-owner", "atlas"); code != ExitOK { + t.Fatalf("add: %d %s", code, h.stderr) + } + id := strings.Fields(h.stdout.String())[0] + h.stdout.Reset() + if code := remote("list", "-owner", "atlas"); code != ExitOK || !strings.Contains(h.stdout.String(), id) { + t.Fatalf("list: %d %q %s", code, h.stdout.String(), h.stderr) + } + h.stderr.Reset() + if code := remote("add", "-repo", "https://github.com/o/r", "-commit", sha); code != ExitUsage || !strings.Contains(h.stderr.String(), "allowed prefixes") { + t.Fatalf("off allowlist: %d %s", code, h.stderr) + } + if code := remote("rm", id); code != ExitOK { + t.Fatalf("rm: %d %s", code, h.stderr) + } + h.stderr.Reset() + if code := remote("rm", id); code != ExitError || !strings.Contains(h.stderr.String(), "not_found") && !strings.Contains(h.stderr.String(), "not found") { + t.Fatalf("rm again: %d %s", code, h.stderr) + } + h.stderr.Reset() + if code := h.run("--server", path, "--token", "wrong", "checkout", "list"); code != ExitError || !strings.Contains(h.stderr.String(), "bearer token") { + t.Fatalf("wrong token: %d %s", code, h.stderr) + } +} diff --git a/src/crux/cli/cli.go b/src/crux/cli/cli.go index e1cf7eda33..b3879de2b5 100644 --- a/src/crux/cli/cli.go +++ b/src/crux/cli/cli.go @@ -196,6 +196,12 @@ func (e *Env) dispatch(ctx context.Context, opts globalOptions, name string, arg return ExitUsage } return cline.Run(args, os.Stdin, e.Stdout, e.Stderr) + case "checkout": + if remote { + fmt.Fprintln(e.Stderr, "crux: checkout is not available over the control socket") + return ExitUsage + } + return e.cmdCheckout(ctx, opts, args) case "ps", "logs", "kick", "stop": if !remote && !opts.local { if code, ok := e.forward(ctx, opts, append([]string{name}, args...)); ok { @@ -334,11 +340,16 @@ Commands (each maps onto crux.Runtime): cline-turn one bounded Cline CLI turn (stdin prompt); --model provider/model, --check for preflight +Work services (no runtime; hivecommons/crux#114): + checkout add|rm|list pinned checkouts under /checkouts + (add -repo URL -commit SHA; -allow PREFIXES or + $CRUX_CHECKOUT_ALLOWLIST gates which repos) + While 'crux run' is in the foreground it serves /crux.sock; ps, logs, kick and stop use that socket when present so they act on the live runtime. 'crux serve' is the API server other harnesses drive (crux/api, crux/serve). -With --server ADDR (or CRUX_SERVER set) run, ps, logs, kick and stop drive that -server over the API instead of an in-process runtime; --token / CRUX_TOKEN +With --server ADDR (or CRUX_SERVER set) run, ps, logs, kick, stop and checkout +drive that server over the API instead of an in-process runtime; --token / CRUX_TOKEN carries its bearer token, and 'logs -f' follows the agent's output. 'serve' itself always runs locally, whatever CRUX_SERVER says. diff --git a/src/crux/cli/remote.go b/src/crux/cli/remote.go index 1a2c7bbcae..20882c6a90 100644 --- a/src/crux/cli/remote.go +++ b/src/crux/cli/remote.go @@ -51,6 +51,8 @@ func (e *Env) remote(ctx context.Context, opts globalOptions, name string, args return e.remoteKick(ctx, c, args) case "stop": return e.remoteStop(ctx, c, args) + case "checkout": + return e.remoteCheckout(ctx, c, args) default: fmt.Fprintf(e.Stderr, "crux: %q is not available with --server; it runs where the runtime is\n", name) return ExitUsage diff --git a/src/crux/cli/serve.go b/src/crux/cli/serve.go index 9923fb5419..104848eccf 100644 --- a/src/crux/cli/serve.go +++ b/src/crux/cli/serve.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/hivecommons/hive/crux/api/callback" + "github.com/hivecommons/hive/crux/checkout" "github.com/hivecommons/hive/crux/config" "github.com/hivecommons/hive/crux/lanes" "github.com/hivecommons/hive/crux/serve" @@ -34,6 +35,9 @@ func (e *Env) cmdServe(ctx context.Context, opts globalOptions, args []string) i var hiveAddr string var laneConfig string var operatorName string + var checkoutAllow, checkoutDir string + fs.StringVar(&checkoutAllow, "checkout-allow", "", "comma-separated repo URL prefixes the pinned-checkout service may clone (default $"+config.CheckoutAllowlistEnvVar+"; none disables the service)") + fs.StringVar(&checkoutDir, "checkout-dir", "", "where pinned checkouts live (default $"+config.CheckoutDirEnvVar+", then /"+checkout.DirName+")") fs.StringVar(&operatorName, "operator-name", "", "name of the principal behind the control token on work-services routes (default $"+config.ServeOperatorNameEnvVar+", then "+serve.DefaultOperatorName+")") fs.StringVar(&laneConfig, "lanes-config", "", "opt-in durable lane configuration file (default $CRUX_LANES_CONFIG)") fs.StringVar(&hiveAddr, "hive-callback", "", "hive capability endpoint to dial (default $"+callback.SocketEnvVar+"; none when unset)") @@ -41,7 +45,7 @@ func (e *Env) cmdServe(ctx context.Context, opts globalOptions, args []string) i return code } if fs.NArg() != 0 { - fmt.Fprintln(e.Stderr, "usage: crux serve [-socket PATH] [-listen HOST:PORT] [-token T] [-hive-callback ADDR] [-lanes-config PATH] [-operator-name NAME]") + fmt.Fprintln(e.Stderr, "usage: crux serve [-socket PATH] [-listen HOST:PORT] [-token T] [-hive-callback ADDR] [-lanes-config PATH] [-operator-name NAME] [-checkout-allow PREFIXES] [-checkout-dir DIR]") return ExitUsage } if cfg.Token == "" { @@ -76,6 +80,12 @@ func (e *Env) cmdServe(ctx context.Context, opts globalOptions, args []string) i operatorName = os.Getenv(config.ServeOperatorNameEnvVar) } srv.OperatorName = operatorName + if checkouts, err := newCheckoutService(checkoutAllow, checkoutDir, opts.workDir); err != nil { + fmt.Fprintln(e.Stderr, "crux: checkouts:", err) + return ExitUsage + } else if checkouts != nil { + srv.Checkouts = checkouts + } srv.Logger = slog.New(slog.NewTextHandler(e.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})) if laneConfig == "" { laneConfig = os.Getenv("CRUX_LANES_CONFIG") diff --git a/src/crux/cli/verbs_test.go b/src/crux/cli/verbs_test.go index a32de5b68a..712e98ced8 100644 --- a/src/crux/cli/verbs_test.go +++ b/src/crux/cli/verbs_test.go @@ -15,7 +15,7 @@ import ( // nonRuntimeVerbs are dispatch cases that do not operate on a runtime: serve // exposes one, help prints usage, cline-turn executes one pane-side worker. -var nonRuntimeVerbs = map[string]bool{"serve": true, "help": true, "-h": true, "--help": true, "cline-turn": true} +var nonRuntimeVerbs = map[string]bool{"serve": true, "help": true, "-h": true, "--help": true, "cline-turn": true, "checkout": true} func TestVerbsHaveRuntimeMethods(t *testing.T) { rt := reflect.TypeOf((*crux.Runtime)(nil)).Elem() diff --git a/src/crux/config/serve.go b/src/crux/config/serve.go index 1e90a0fb96..063f93a9d0 100644 --- a/src/crux/config/serve.go +++ b/src/crux/config/serve.go @@ -29,6 +29,13 @@ const ( // for on the work-services routes (serve.CallerFrom); the server // defaults it when unset. ServeOperatorNameEnvVar = "CRUX_OPERATOR_NAME" + // CheckoutAllowlistEnvVar is the comma-separated list of repository URL + // prefixes the pinned-checkout work service may clone; unset or empty + // leaves the service off (every request is refused). + CheckoutAllowlistEnvVar = "CRUX_CHECKOUT_ALLOWLIST" + // CheckoutDirEnvVar overrides where pinned checkouts live (default + // /checkouts). + CheckoutDirEnvVar = "CRUX_CHECKOUT_DIR" // ServeSocketName is the socket file name under ServeSocketDir. ServeSocketName = "crux.sock" // ServeSocketDirName is the per-user directory under $XDG_RUNTIME_DIR or diff --git a/src/crux/serve/checkouts.go b/src/crux/serve/checkouts.go new file mode 100644 index 0000000000..52ea32e29d --- /dev/null +++ b/src/crux/serve/checkouts.go @@ -0,0 +1,166 @@ +package serve + +import ( + "errors" + "fmt" + "net/http" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/checkout" +) + +// Pinned checkouts (hivecommons/crux#114 S1) are the first work-services +// family: the routes accept project principals (authenticate admits them +// because registerCheckouts marks the prefix with workService), and +// ownership is enforced from the caller principal — a project may only +// create checkouts it owns, list its own, and delete its own; the operator +// may do anything. + +// registerCheckouts adds the checkout routes and marks them a work service. +func (s *Server) registerCheckouts(mux *http.ServeMux) { + mux.HandleFunc("POST "+api.PathCheckouts, s.handleCheckoutCreate) + mux.HandleFunc("GET "+api.PathCheckouts, s.handleCheckoutList) + mux.HandleFunc("GET "+api.PathCheckouts+"/{id}", s.handleCheckoutGet) + mux.HandleFunc("DELETE "+api.PathCheckouts+"/{id}", s.handleCheckoutDelete) + s.workService(api.PathCheckouts) +} + +// checkoutService answers 501 when the server has no checkout service. +func (s *Server) checkoutService(w http.ResponseWriter) (*checkout.Service, bool) { + if s.Checkouts == nil { + s.writeError(w, http.StatusNotImplemented, api.CodeNotSupported, "this server has no checkout service; start it with --checkout-allow") + return nil, false + } + return s.Checkouts, true +} + +func (s *Server) handleCheckoutCreate(w http.ResponseWriter, r *http.Request) { + svc, ok := s.checkoutService(w) + if !ok { + return + } + var in api.CheckoutRequest + if !s.readJSON(w, r, &in) { + return + } + caller, _ := CallerFrom(r.Context()) + owner := in.Owner + switch { + case owner == "": + owner = caller.Name + case !caller.IsOperator() && owner != caller.Name: + s.writeError(w, http.StatusForbidden, api.CodeForbidden, fmt.Sprintf("principal %s may only create checkouts owned by %q", caller, caller.Name)) + return + } + req := checkout.Request{Repo: in.Repo, Commit: in.Commit, Owner: owner, Branches: in.Branches, MaxBytes: in.MaxBytes} + if in.TTL != "" { + ttl, err := time.ParseDuration(in.TTL) + if err != nil { + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, "ttl: "+err.Error()) + return + } + req.TTL = ttl + } + rec, err := svc.Create(r.Context(), req) + if err != nil { + s.writeCheckoutError(w, err) + return + } + s.writeJSON(w, http.StatusCreated, checkoutToAPI(rec)) +} + +func (s *Server) handleCheckoutList(w http.ResponseWriter, r *http.Request) { + svc, ok := s.checkoutService(w) + if !ok { + return + } + caller, _ := CallerFrom(r.Context()) + owner := r.URL.Query().Get("owner") + if !caller.IsOperator() { + // A project sees only its own checkouts, whatever it asked for. + owner = caller.Name + } + recs, err := svc.List(owner) + if err != nil { + s.writeCheckoutError(w, err) + return + } + out := api.CheckoutList{Checkouts: make([]api.Checkout, 0, len(recs))} + for _, rec := range recs { + out.Checkouts = append(out.Checkouts, checkoutToAPI(rec)) + } + s.writeJSON(w, http.StatusOK, out) +} + +func (s *Server) handleCheckoutGet(w http.ResponseWriter, r *http.Request) { + svc, ok := s.checkoutService(w) + if !ok { + return + } + rec, ok := s.ownedCheckout(w, r, svc) + if !ok { + return + } + s.writeJSON(w, http.StatusOK, checkoutToAPI(rec)) +} + +func (s *Server) handleCheckoutDelete(w http.ResponseWriter, r *http.Request) { + svc, ok := s.checkoutService(w) + if !ok { + return + } + rec, ok := s.ownedCheckout(w, r, svc) + if !ok { + return + } + caller, _ := CallerFrom(r.Context()) + if err := svc.Delete(r.Context(), rec.ID, caller.String()); err != nil { + s.writeCheckoutError(w, err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +// ownedCheckout loads {id} and enforces that the caller owns it or is the +// operator; on failure the response has been written. +func (s *Server) ownedCheckout(w http.ResponseWriter, r *http.Request, svc *checkout.Service) (checkout.Record, bool) { + rec, err := svc.Get(r.PathValue("id")) + if err != nil { + s.writeCheckoutError(w, err) + return checkout.Record{}, false + } + caller, _ := CallerFrom(r.Context()) + if !checkout.CanDelete(rec, caller.Name, caller.IsOperator()) { + s.writeError(w, http.StatusForbidden, api.CodeForbidden, fmt.Sprintf("checkout %s is owned by %q, not by principal %s", rec.ID, rec.Owner, caller)) + return checkout.Record{}, false + } + return rec, true +} + +// writeCheckoutError maps the package's sentinels to typed codes. +func (s *Server) writeCheckoutError(w http.ResponseWriter, err error) { + status, code := http.StatusInternalServerError, api.CodeRuntime + switch { + case errors.Is(err, checkout.ErrBadRequest): + status, code = http.StatusBadRequest, api.CodeBadRequest + case errors.Is(err, checkout.ErrForbidden): + status, code = http.StatusForbidden, api.CodeForbidden + case errors.Is(err, checkout.ErrNotFound): + status, code = http.StatusNotFound, api.CodeNotFound + case errors.Is(err, checkout.ErrInUse): + status, code = http.StatusConflict, api.CodeConflict + case errors.Is(err, checkout.ErrMismatch): + status, code = http.StatusBadGateway, api.CodeUpstream + case errors.Is(err, checkout.ErrTooLarge): + status, code = http.StatusRequestEntityTooLarge, api.CodeTooLarge + case errors.Is(err, checkout.ErrUnsupported): + status, code = http.StatusNotImplemented, api.CodeUnsupported + } + s.writeError(w, status, code, err.Error()) +} + +func checkoutToAPI(rec checkout.Record) api.Checkout { + return api.Checkout{ID: rec.ID, Path: rec.Path, Repo: rec.Repo, Commit: rec.Commit, Owner: rec.Owner, Branches: rec.Branches, + MaxBytes: rec.MaxBytes, Bytes: rec.Bytes, TTL: rec.TTL, CreatedAt: rec.CreatedAt, ExpiresAt: rec.ExpiresAt} +} diff --git a/src/crux/serve/checkouts_test.go b/src/crux/serve/checkouts_test.go new file mode 100644 index 0000000000..3217eb7bbe --- /dev/null +++ b/src/crux/serve/checkouts_test.go @@ -0,0 +1,290 @@ +package serve + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/checkout" + "github.com/hivecommons/hive/crux/lanes" +) + +// checkoutFixtureRepo is a bare repo with one commit, as a file:// URL. +func checkoutFixtureRepo(t *testing.T) (repo, sha string) { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + work := filepath.Join(t.TempDir(), "work") + if err := os.MkdirAll(work, 0o700); err != nil { + t.Fatal(err) + } + git := func(args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = work + cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@x", "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@x") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + git("init", "-q") + if err := os.WriteFile(filepath.Join(work, "f"), []byte("f"), 0o600); err != nil { + t.Fatal(err) + } + git("add", "f") + git("commit", "-q", "-m", "c") + sha = git("rev-parse", "HEAD") + bare := filepath.Join(t.TempDir(), "fixture.git") + git("clone", "-q", "--bare", work, bare) + return "file://" + bare, sha +} + +type checkoutFixture struct { + t *testing.T + h http.Handler + s *Server + inUse map[string]bool + tokens map[string]string +} + +// newCheckoutFixture is a token-bearing server with a checkout service +// whose allowlist admits the fixture repo's directory, and two lane +// projects (atlas, beacon) besides the operator token. +func newCheckoutFixture(t *testing.T, repo string) *checkoutFixture { + t.Helper() + tokens := map[string]string{"atlas": strings.Repeat("a", 32), "beacon": strings.Repeat("b", 32), "operator": "agent-admin"} + cfg := lanes.Config{Principals: []lanes.Principal{{Project: "atlas", TokenEnv: "ATLAS"}, {Project: "beacon", TokenEnv: "BEACON"}}} + auth, err := lanes.Bind(cfg, func(env string) string { return tokens[strings.ToLower(env)] }, tokens["operator"]) + if err != nil { + t.Fatal(err) + } + f := &checkoutFixture{t: t, inUse: map[string]bool{}, tokens: tokens} + svc, err := checkout.New(checkout.Config{ + Dir: filepath.Join(t.TempDir(), "checkouts"), + AllowedRepos: []string{"file://" + filepath.Dir(strings.TrimPrefix(repo, "file://"))}, + InUse: func(id string) bool { return f.inUse[id] }, + }) + if err != nil { + t.Fatal(err) + } + f.s = New(newFakeRuntime()) + f.s.Token, f.s.LaneAuth, f.s.Checkouts = tokens["operator"], auth, svc + f.h = f.s.Handler() + return f +} + +// call sends method path as who ("operator", "atlas", "beacon" or "") and +// decodes a 2xx body into out, returning the status and any api.Error. +func (f *checkoutFixture) call(who, method, path string, body, out any) (int, api.Error) { + f.t.Helper() + var buf bytes.Buffer + if body != nil { + if err := json.NewEncoder(&buf).Encode(body); err != nil { + f.t.Fatal(err) + } + } + req := httptest.NewRequest(method, path, &buf) + if who != "" { + req.Header.Set("Authorization", "Bearer "+f.tokens[who]) + } + rec := httptest.NewRecorder() + f.h.ServeHTTP(rec, req) + var e api.Error + if rec.Code >= 300 { + _ = json.Unmarshal(rec.Body.Bytes(), &e) + } else if out != nil && rec.Code != http.StatusNoContent { + if err := json.Unmarshal(rec.Body.Bytes(), out); err != nil { + f.t.Fatalf("decode %s %s: %v: %s", method, path, err, rec.Body) + } + } + return rec.Code, e +} + +func TestCheckoutRoutesOwnershipAndListing(t *testing.T) { + repo, sha := checkoutFixtureRepo(t) + f := newCheckoutFixture(t, repo) + + // A project creates a checkout; the owner defaults to the project. + var mine api.Checkout + if code, e := f.call("atlas", http.MethodPost, api.PathCheckouts, api.CheckoutRequest{Repo: repo, Commit: sha, TTL: "72h"}, &mine); code != http.StatusCreated { + t.Fatalf("atlas create: %d %+v", code, e) + } + if mine.Owner != "atlas" || mine.Commit != sha || mine.Path == "" || mine.TTL != "72h0m0s" || mine.ExpiresAt == nil { + t.Fatalf("created = %+v", mine) + } + // A project may not name another owner; the operator may. + if code, e := f.call("atlas", http.MethodPost, api.PathCheckouts, api.CheckoutRequest{Repo: repo, Commit: sha, Owner: "beacon"}, nil); code != http.StatusForbidden || e.Code != api.CodeForbidden { + t.Fatalf("atlas create for beacon: %d %+v", code, e) + } + var theirs api.Checkout + if code, e := f.call("operator", http.MethodPost, api.PathCheckouts, api.CheckoutRequest{Repo: repo, Commit: sha, Owner: "beacon"}, &theirs); code != http.StatusCreated || theirs.Owner != "beacon" { + t.Fatalf("operator create for beacon: %d %+v %+v", code, e, theirs) + } + var ops api.Checkout + if code, _ := f.call("operator", http.MethodPost, api.PathCheckouts, api.CheckoutRequest{Repo: repo, Commit: sha}, &ops); code != http.StatusCreated || ops.Owner != DefaultOperatorName { + t.Fatalf("operator create for itself: %d %+v", code, ops) + } + + // Listing: the operator sees all and can filter; a project sees only + // its own whatever it asks for. + var list api.CheckoutList + if code, _ := f.call("operator", http.MethodGet, api.PathCheckouts, nil, &list); code != http.StatusOK || len(list.Checkouts) != 3 { + t.Fatalf("operator list: %d %d", code, len(list.Checkouts)) + } + if code, _ := f.call("operator", http.MethodGet, api.PathCheckouts+"?owner=beacon", nil, &list); code != http.StatusOK || len(list.Checkouts) != 1 || list.Checkouts[0].ID != theirs.ID { + t.Fatalf("operator list beacon: %d %+v", code, list) + } + if code, _ := f.call("atlas", http.MethodGet, api.PathCheckouts+"?owner=beacon", nil, &list); code != http.StatusOK || len(list.Checkouts) != 1 || list.Checkouts[0].ID != mine.ID { + t.Fatalf("atlas list: %d %+v", code, list) + } + + // Get: owner and operator yes, another project no. + var got api.Checkout + if code, _ := f.call("atlas", http.MethodGet, api.PathCheckouts+"/"+mine.ID, nil, &got); code != http.StatusOK || got.ID != mine.ID { + t.Fatalf("atlas get own: %d %+v", code, got) + } + if code, e := f.call("beacon", http.MethodGet, api.PathCheckouts+"/"+mine.ID, nil, nil); code != http.StatusForbidden || e.Code != api.CodeForbidden || !strings.Contains(e.Error, "owned by \"atlas\"") { + t.Fatalf("beacon get atlas's: %d %+v", code, e) + } + if code, _ := f.call("operator", http.MethodGet, api.PathCheckouts+"/"+mine.ID, nil, &got); code != http.StatusOK { + t.Fatalf("operator get: %d", code) + } + if code, e := f.call("operator", http.MethodGet, api.PathCheckouts+"/chk_000000000000", nil, nil); code != http.StatusNotFound || e.Code != api.CodeNotFound { + t.Fatalf("get unknown: %d %+v", code, e) + } + + // Delete: two drivers cannot delete each other's checkouts; live runs + // block deletion with 409; the owner and the operator can delete. + if code, e := f.call("beacon", http.MethodDelete, api.PathCheckouts+"/"+mine.ID, nil, nil); code != http.StatusForbidden || e.Code != api.CodeForbidden { + t.Fatalf("beacon delete atlas's: %d %+v", code, e) + } + f.inUse[mine.ID] = true + if code, e := f.call("atlas", http.MethodDelete, api.PathCheckouts+"/"+mine.ID, nil, nil); code != http.StatusConflict || e.Code != api.CodeConflict { + t.Fatalf("delete in use: %d %+v", code, e) + } + f.inUse[mine.ID] = false + if code, e := f.call("atlas", http.MethodDelete, api.PathCheckouts+"/"+mine.ID, nil, nil); code != http.StatusNoContent { + t.Fatalf("atlas delete own: %d %+v", code, e) + } + if code, e := f.call("operator", http.MethodDelete, api.PathCheckouts+"/"+theirs.ID, nil, nil); code != http.StatusNoContent { + t.Fatalf("operator delete beacon's: %d %+v", code, e) + } + if code, _ := f.call("operator", http.MethodGet, api.PathCheckouts, nil, &list); code != http.StatusOK || len(list.Checkouts) != 1 || list.Checkouts[0].ID != ops.ID { + t.Fatalf("list after deletes: %d %+v", code, list) + } + // Agent RPCs still refuse the project token. + if code, e := f.call("atlas", http.MethodGet, api.PathHealth, nil, nil); code != http.StatusUnauthorized || e.Code != api.CodeUnauthorized { + t.Fatalf("project token on health: %d %+v", code, e) + } + if code, _ := f.call("", http.MethodGet, api.PathCheckouts, nil, nil); code != http.StatusUnauthorized { + t.Fatalf("no token on checkouts: %d", code) + } +} + +func TestCheckoutRouteErrorMapping(t *testing.T) { + repo, sha := checkoutFixtureRepo(t) + f := newCheckoutFixture(t, repo) + cases := []struct { + name string + body any + code int + err string + msg string + }{ + {"short sha", api.CheckoutRequest{Repo: repo, Commit: sha[:7]}, http.StatusBadRequest, api.CodeBadRequest, "full 40-hex SHA"}, + {"branch", api.CheckoutRequest{Repo: repo, Commit: "main"}, http.StatusBadRequest, api.CodeBadRequest, "tag or branch"}, + {"bad ttl", api.CheckoutRequest{Repo: repo, Commit: sha, TTL: "soon"}, http.StatusBadRequest, api.CodeBadRequest, "ttl"}, + {"off allowlist", api.CheckoutRequest{Repo: "https://github.com/org/name.git", Commit: sha}, http.StatusForbidden, api.CodeForbidden, "allowed prefixes"}, + {"over cap", api.CheckoutRequest{Repo: repo, Commit: sha, MaxBytes: 1}, http.StatusRequestEntityTooLarge, api.CodeTooLarge, "after cloning"}, + {"unknown field", map[string]any{"repo": repo, "commit": sha, "branch": "main"}, http.StatusBadRequest, api.CodeBadRequest, "unknown field"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + code, e := f.call("operator", http.MethodPost, api.PathCheckouts, tc.body, nil) + if code != tc.code || e.Code != tc.err || !strings.Contains(e.Error, tc.msg) { + t.Fatalf("%d %+v, want %d %s containing %q", code, e, tc.code, tc.err, tc.msg) + } + }) + } + // The remaining sentinels map through writeCheckoutError directly. + for err, want := range map[error][2]any{ + checkout.ErrMismatch: {http.StatusBadGateway, api.CodeUpstream}, + checkout.ErrUnsupported: {http.StatusNotImplemented, api.CodeUnsupported}, + checkout.ErrGit: {http.StatusInternalServerError, api.CodeRuntime}, + errors.New("other"): {http.StatusInternalServerError, api.CodeRuntime}, + } { + rec := httptest.NewRecorder() + f.s.writeCheckoutError(rec, err) + var e api.Error + _ = json.Unmarshal(rec.Body.Bytes(), &e) + if rec.Code != want[0].(int) || e.Code != want[1].(string) { + t.Errorf("%v → %d %s, want %v", err, rec.Code, e.Code, want) + } + } +} + +func TestCheckoutRoutesWithoutService(t *testing.T) { + s := New(newFakeRuntime()) + s.Token = "t" + h := s.Handler() + for _, c := range []struct{ method, path string }{ + {http.MethodPost, api.PathCheckouts}, {http.MethodGet, api.PathCheckouts}, + {http.MethodGet, api.PathCheckouts + "/chk_000000000000"}, {http.MethodDelete, api.PathCheckouts + "/chk_000000000000"}, + } { + req := httptest.NewRequest(c.method, c.path, strings.NewReader("{}")) + req.Header.Set("Authorization", "Bearer t") + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + var e api.Error + _ = json.Unmarshal(rec.Body.Bytes(), &e) + if rec.Code != http.StatusNotImplemented || e.Code != api.CodeNotSupported { + t.Fatalf("%s %s without service: %d %+v", c.method, c.path, rec.Code, e) + } + } +} + +func TestCheckoutClientRoundTrip(t *testing.T) { + repo, sha := checkoutFixtureRepo(t) + f := newCheckoutFixture(t, repo) + srv := httptest.NewServer(f.h) + defer srv.Close() + c, err := api.NewClient(srv.URL, f.tokens["atlas"]) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + rec, err := c.CreateCheckout(ctx, api.CheckoutRequest{Repo: repo, Commit: sha}) + if err != nil || rec.Owner != "atlas" { + t.Fatalf("CreateCheckout = %+v %v", rec, err) + } + got, err := c.GetCheckout(ctx, rec.ID) + if err != nil || got.ID != rec.ID { + t.Fatalf("GetCheckout = %+v %v", got, err) + } + list, err := c.ListCheckouts(ctx, "atlas") + if err != nil || len(list) != 1 { + t.Fatalf("ListCheckouts = %v %v", list, err) + } + if err := c.DeleteCheckout(ctx, rec.ID); err != nil { + t.Fatal(err) + } + if err := c.DeleteCheckout(ctx, rec.ID); !api.IsNotFound(err) { + t.Fatalf("second delete = %v", err) + } + var se *api.StatusError + if _, err := c.CreateCheckout(ctx, api.CheckoutRequest{Repo: repo, Commit: "main"}); !errors.As(err, &se) || se.Body.Code != api.CodeBadRequest { + t.Fatalf("branch pin = %v", err) + } +} diff --git a/src/crux/serve/serve.go b/src/crux/serve/serve.go index 4e79f10452..eed2b429ca 100644 --- a/src/crux/serve/serve.go +++ b/src/crux/serve/serve.go @@ -35,6 +35,7 @@ import ( "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/checkout" "github.com/hivecommons/hive/crux/config" "github.com/hivecommons/hive/crux/lanes" ) @@ -104,6 +105,9 @@ type Server struct { // OperatorName names the principal behind the shared control token // (CallerFrom); empty means DefaultOperatorName. OperatorName string + // Checkouts, when set, serves the pinned-checkout work service + // (api.PathCheckouts); nil answers those routes 501. + Checkouts *checkout.Service now func() time.Time started time.Time @@ -161,6 +165,7 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("POST "+api.PathAgents+"/{name}/stop", s.handleStop) mux.HandleFunc("GET "+api.PathEvents, s.handleEvents) s.registerFamilies(mux) + s.registerCheckouts(mux) return s.laneHandler(s.authenticate(mux)) } diff --git a/src/docs/crux-serve.md b/src/docs/crux-serve.md index bb79e72980..f16325f2a9 100644 --- a/src/docs/crux-serve.md +++ b/src/docs/crux-serve.md @@ -34,6 +34,8 @@ runtime and every agent must arrive as a definition on `POST /v1/agents`. | `--listen HOST:PORT` | also serve on TCP. **Refused without a token.** | | `--token T`, `CRUX_TOKEN` | bearer token every request must carry. When set it gates the socket too. | | `--operator-name NAME`, `CRUX_OPERATOR_NAME` | the name the control token's caller is recorded under on work-services routes (see [Caller principals](#caller-principals)). Default `operator`. | +| `--checkout-allow PREFIX,…`, `CRUX_CHECKOUT_ALLOWLIST` | repository URL prefixes `POST /v1/checkouts` may clone; unset means no checkout service ([design](design/crux-work-services.md)). | +| `--checkout-dir PATH`, `CRUX_CHECKOUT_DIR` | where pinned checkouts live. Default `/checkouts`. | | `--lanes-config PATH`, `CRUX_LANES_CONFIG` | opt into [durable event/work lanes](crux-lanes.md) on this same server, backed by NATS JetStream. Lane project credentials are separate from agent-control credentials. | | `--hive-callback ADDR`, `CRUX_HIVE_SOCKET` (+ `CRUX_HIVE_TOKEN`) | dial hive's callback endpoint (`crux/api/callback`) for the capabilities hive owns — scoped GitHub tokens, PR creation, Linear credentials, text/canary scanners, conditions — and feed the runtime's seams from it instead of leaving them nil. Health reports it under `hive_callback`; an unreachable endpoint raises the `hive_callback_unavailable` condition. (#14246) When the runtime is a manager it also polls hive's host policy from the same endpoint; see [Host policy from hive](#host-policy-from-hive-14138-stage-4). | @@ -92,6 +94,7 @@ and [lane error codes and failure behavior](crux-lanes.md#bounds-and-failure-beh | `POST /v1/agents/{name}/kick` | `{"message":…}` | `Kicker.SendKick` | `{"agent","process"}` | | `POST /v1/agents/{name}/stop` | | `Lifecycle.Stop` | `{"agent","process"}` | | `GET /v1/events` | | watcher + `Kicker.SetKickObserver` + the `crux.Host` observers | `text/event-stream` of `Event` records | +| `POST /v1/checkouts`, `GET /v1/checkouts[?owner=]`, `GET`/`DELETE /v1/checkouts/{id}` | `{"repo","commit","owner"?,"branches"?,"max_bytes"?,"ttl"?}` | `checkout.Service` | Work service (S1): a worktree pinned at a full SHA, owned by the caller principal; `501` without `--checkout-allow`; [routes, codes and semantics](design/crux-work-services.md) | | `GET /v1/lanes` and lane/group subroutes | Publish/recovery/receipt JSON; [request shapes](crux-lanes.md#public-http-and-go-api) | `lanes.Broker` | Opt-in only; lane-project credential and capability ACLs; JSON or `204`; [full routes and responses](crux-lanes.md#public-http-and-go-api) | ### Runtime families (#14138 stage 3) @@ -284,6 +287,9 @@ crux logs -n 50 --buffer fixer crux logs -f fixer # follow the ring buffer over SSE crux kick fixer "look at the failing build on main" crux stop fixer +crux checkout add -repo https://github.com/hivecommons/hive -commit <40-hex sha> # S1 pinned checkout, owned by the caller +crux checkout list +crux checkout rm chk_0123456789ab ``` In remote mode nothing is built in this process: no runtime, no control diff --git a/src/docs/design/crux-work-services.md b/src/docs/design/crux-work-services.md new file mode 100644 index 0000000000..1530515246 --- /dev/null +++ b/src/docs/design/crux-work-services.md @@ -0,0 +1,160 @@ +# Crux work services: S1 pinned checkout (hivecommons/crux#114) + +Status: **S0 caller principals and S1 pinned checkout shipped in hive +`src/crux` (v5).** S2 contained run and S7 minimum are in flight in a +sibling change; everything else in the RFC is unscheduled. This records +S1 *as implemented*, including where it is weaker than the RFC text. + +The RFC asks for "a working copy of this repo at this commit, owned by +this caller, on this server" as the first work service, so that S2 runs +have something to run in. This is that: `crux serve` grows a +`checkout.Service` behind `/v1/checkouts`, and the `crux` CLI grows +`checkout add|rm|list`. + +## Surface + +### HTTP (`src/crux/api/checkouts.go`, `src/crux/serve/checkouts.go`) + +| Route | Body | Response | +|---|---|---| +| `POST /v1/checkouts` | `{"repo", "commit", "owner"?, "branches"?, "max_bytes"?, "ttl"?}` | `201 Checkout` | +| `GET /v1/checkouts[?owner=NAME]` | | `200 {"checkouts":[Checkout…]}` sorted by id | +| `GET /v1/checkouts/{id}` | | `200 Checkout` | +| `DELETE /v1/checkouts/{id}` | | `204` | + +`Checkout` is `{"id","path","repo","commit","owner","branches","max_bytes", +"bytes","ttl"?,"created_at","expires_at"?}`. Nothing in it is secret: the +record never carries environment, and repository URLs that embed +credentials are refused before anything is cloned. + +Error codes, all `api.Error` JSON: + +| Condition | Status | `code` | +|---|---|---| +| `commit` is not a full 40-hex SHA (a tag, branch or short SHA) | 400 | `bad_request` | +| bad `ttl`, bad `owner`, negative `max_bytes`, malformed URL, unknown JSON field | 400 | `bad_request` | +| `repo` matches no allowlist prefix; a project principal naming another `owner`; reading or deleting a checkout you do not own | 403 | `forbidden` (the message names the allowed prefixes or the owner) | +| unknown id | 404 | `not_found` | +| `DELETE` while runs use it (the `InUse` hook) | 409 | `conflict` | +| clone plus worktree measure more than `max_bytes` | 413 | `too_large` | +| the server has no checkout service; the size could not be measured | 501 | `not_supported` / `unsupported` | +| the worktree's `HEAD` is not the requested SHA | 502 | `upstream_error` (names both SHAs) | +| git failed | 500 | `runtime_error` | + +A server without `--checkout-allow` registers the routes and answers 501 +on all of them, so a client can tell "not here" from "not found". + +### CLI (`src/crux/cli/checkout.go`) + +``` +crux checkout add -repo URL -commit SHA [-owner NAME] [-ttl 72h] [-max-bytes N] [-no-branches] [-json] +crux checkout rm ID +crux checkout list [-owner NAME] [-json] +``` + +Local (`crux checkout …`) runs the service in-process on +`/checkouts` (`-dir`/`CRUX_CHECKOUT_DIR`) with `-allow`/ +`CRUX_CHECKOUT_ALLOWLIST` as its allowlist; without an allowlist it can +still list and remove but refuses every `add`. Remote (`crux --server … +checkout …`) drives the server's service through the API client. The +verb is not forwarded over the foreground control socket. Bad requests +and refusals exit with the usage status, everything else with the error +status. + +### Server flags (`crux serve`) + +| Flag / env | Meaning | +|---|---| +| `--checkout-allow PREFIX,…`, `CRUX_CHECKOUT_ALLOWLIST` | repository URL prefixes `POST /v1/checkouts` may clone. **Unset means no checkout service.** | +| `--checkout-dir PATH`, `CRUX_CHECKOUT_DIR` | where checkouts live. Default `/checkouts`. | + +## Semantics + +**Pin.** `commit` must be a full 40-hex SHA; tags and branches are refused +with `bad_request` because a moving ref is not a pin. After `git worktree +add --detach ` the service runs `git rev-parse HEAD` in the worktree +and refuses with 502 naming both SHAs if they differ. + +**Allowlist, deny by default.** The allowlist is a list of URL prefixes +(`https://github.com/hivecommons/`, `file:///srv/mirrors/`). Each entry +and each request URL must be `https://` or `file://`, have a host (or a +path, for `file`), and carry no userinfo, query or fragment; the request +URL must start with an entry. An empty allowlist refuses everything. This +is the shape of hive's `nousdeps` trust rules, ported without the +per-package specifics. Private repositories therefore work only through +whatever the server's own git credential helper provides; the API never +accepts a credential. + +**Ownership via S0.** `owner` is recorded on every checkout. The caller's +[principal](../crux-serve.md#caller-principals) decides what it may name: +a lane project principal may create only checkouts owned by itself and +sees only its own on `GET` (an `?owner=` filter is overridden); the +operator may name any owner (default: its own name) and sees everything. +`GET /v1/checkouts/{id}` and `DELETE` succeed for the owner and the +operator and answer 403 for anyone else. + +**Live runs block delete.** `checkout.Config.InUse func(id string) bool` +is the hook S2 wires to its run state; the default returns false. A true +answer turns `DELETE` into 409 `conflict`. Delete is otherwise +`git worktree remove --force` (falling back to `os.RemoveAll`), the record +file removed, and an index line appended naming who deleted it. + +**TTL is recorded, not enforced.** `ttl` is parsed as a Go duration and +stored with `expires_at`. There is no sweeper yet; `list -json` exposes +`expires_at` so one can be written against the index. + +**Byte cap, honestly.** Git cannot bound a clone before it happens; there +is no pre-flight size query across the transports we allow. The cap is +therefore *post-hoc*: the service clones (`--no-checkout`, plus +`--single-branch` when `branches` is false), adds the worktree, then walks +the regular files of the shared clone and the worktree and sums their +sizes. Over `max_bytes` (default 5 GiB) it removes the worktree and +answers 413 `too_large` with both numbers; if the measurement itself +fails it answers 501 `unsupported`. The bytes were already transferred +and the shared clone is kept (a later request for the same repo reuses +it). `--filter=blob:limit=` partial clones would make the clone itself +smaller but would also make the worktree need network at checkout time; +that trade is not taken here and is deferred. The record stores both +`max_bytes` and the measured `bytes`. + +**Bound on live worktrees.** `checkout.Config.MaxCheckouts` (default 64) +is the `sandbox.WorktreeManager` cap, counted as `.git` entries under the +runs directory. + +## Storage + +Under `` (`WorkDir()/checkouts` by default): + +``` +checkouts/.json one Record per checkout, 0600, written tmp+rename +checkouts/index.jsonl append-only {op, id, owner, repo, commit, at, by} +checkouts/src/-/ the shared bare-ish clone (--no-checkout), per owner and repo +checkouts/runs//pin-1/ the worktree handed back as `path` +``` + +Ids are `chk_` plus twelve hex characters. The `src` directory is keyed +by owner and a hash of the repository URL so two owners never share a +clone (a project cannot be made to pay for, or read, another's fetches). +Records follow the `sandbox_run.go` conventions: one file per entity plus +a `.jsonl` index. + +## Deliberately not in S1 + +- TTL sweeper (recorded, not enforced). +- `--filter=blob:limit=` partial clones; the cap is post-hoc only. +- A per-owner allowlist; one list per server. +- Wiring `InUse` to live runs: that is S2's side of the seam. +- `ssh://` and `http://` repositories; `git@` syntax. +- Anything about the hivecommons/crux repository or the publish path: + the service lives in hive `src/crux` and ships through the existing + mirror unchanged. + +## Package map + +| Path | Role | +|---|---| +| `src/crux/checkout/` | `Service`, `Config`, `Record`, typed sentinels (`ErrBadRequest`, `ErrForbidden`, `ErrNotFound`, `ErrInUse`, `ErrMismatch`, `ErrTooLarge`, `ErrUnsupported`, `ErrGit`); store and index | +| `src/crux/api/checkouts.go` | `PathCheckouts`, `CodeTooLarge`, `CodeUnsupported`, `CodeUpstream`, request/response types, `Client.{Create,List,Get,Delete}Checkout` | +| `src/crux/serve/checkouts.go` | routes, ownership checks, error mapping; registered from `Server.Handler` after the runtime families and marked a work service for S0 | +| `src/crux/serve/principal.go` | S0: `Principal`, `CallerFrom`, work-service gating | +| `src/crux/cli/checkout.go` | `crux checkout add|rm|list`, local and remote | From d967290f4e808f8a3cf29ec01fd8b91cabf21b7f Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Sat, 7 Nov 2026 12:22:14 -0500 Subject: [PATCH 15/17] =?UTF-8?q?=E2=9C=A8=20crux:=20let=20an=20in-hive=20?= =?UTF-8?q?Spek=20publish=20its=20RFC=20and=20children=20through=20the=20p?= =?UTF-8?q?roxy=20(#17017)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ crux: let an in-hive Spek publish its RFC and children through the proxy Allow the GitHub sub-issue and blocked-by dependency writes and the Linear issueRelationCreate/projectCreate/projectUpdate mutations at ISSUES_ONLY, add crux/spekmarker to parse the spek:publish provenance marker, and expose issueshape.SpekPublished as the already-specified signal. Refs hivecommons/spektacular#114 Signed-off-by: Andrew Anderson * 🌱 crux/proxy: move hard-deny rules to rules_deny.go to keep rules.go within its size ratchet Signed-off-by: Andrew Anderson --------- Signed-off-by: Andrew Anderson Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/added-crux-spek-publish-proxy.md | 1 + src/crux/issueshape/issueshape.go | 15 +++ src/crux/issueshape/issueshape_test.go | 22 ++++ src/crux/proxy/issue_shape_test.go | 19 +++ src/crux/proxy/linear_rules.go | 16 +++ src/crux/proxy/linear_rules_test.go | 50 +++++++ src/crux/proxy/rules.go | 105 ++------------- src/crux/proxy/rules_deny.go | 103 +++++++++++++++ src/crux/proxy/rules_test.go | 11 ++ src/crux/spekmarker/spekmarker.go | 131 +++++++++++++++++++ src/crux/spekmarker/spekmarker_test.go | 80 +++++++++++ src/docs/design/crux-tool-policy.md | 1 + src/docs/linear-agent.md | 8 +- 13 files changed, 466 insertions(+), 96 deletions(-) create mode 100644 changelog.d/added-crux-spek-publish-proxy.md create mode 100644 src/crux/proxy/rules_deny.go create mode 100644 src/crux/spekmarker/spekmarker.go create mode 100644 src/crux/spekmarker/spekmarker_test.go diff --git a/changelog.d/added-crux-spek-publish-proxy.md b/changelog.d/added-crux-spek-publish-proxy.md new file mode 100644 index 0000000000..0d350912a5 --- /dev/null +++ b/changelog.d/added-crux-spek-publish-proxy.md @@ -0,0 +1 @@ +- crux: the proxy lets an in-hive Spek publish its RFC and children — GitHub `POST …/issues/{n}/sub_issues` and `POST …/issues/{n}/dependencies/blocked_by` and Linear `issueRelationCreate`/`projectCreate`/`projectUpdate` are allowed at ISSUES_ONLY; a new `crux/spekmarker` package parses the `` marker and `issueshape.SpekPublished` exposes it as the "already specified" clarity-gate signal (spektacular#114). diff --git a/src/crux/issueshape/issueshape.go b/src/crux/issueshape/issueshape.go index dcec0abb70..e19660a887 100644 --- a/src/crux/issueshape/issueshape.go +++ b/src/crux/issueshape/issueshape.go @@ -12,6 +12,8 @@ package issueshape import ( "regexp" "strings" + + "github.com/hivecommons/hive/crux/spekmarker" ) var ( @@ -72,6 +74,19 @@ func UnsubstitutedTemplatePlaceholder(text string) (string, bool) { return "", false } +// SpekPublished reports whether body carries a valid Spek publish marker +// (``, see package +// spekmarker). Such a body was rendered from a finished spec by +// hivecommons/spektacular, so the clarity gate treats it as already specified +// — the same skip `approved-direction` grants — instead of parking it with +// `needs-spec` (hivecommons/spektacular#114). It is a clarity-scoring +// exemption only: the hard shape errors (UnsubstitutedTemplatePlaceholder, +// BodyHasMisEscapedNewlines) still apply to a marked body, because a marker +// pasted into an unfilled template proves nothing about the template. +func SpekPublished(body string) bool { + return spekmarker.Has(body) +} + // BodyHasMisEscapedNewlines reports whether body is a single physical line // whose markdown structure was encoded as literal backslash-n escape // sequences instead of real newlines (the classic `## X\n\n<...>\n\n## Y` diff --git a/src/crux/issueshape/issueshape_test.go b/src/crux/issueshape/issueshape_test.go index 37f47c2d0f..34dcd2584f 100644 --- a/src/crux/issueshape/issueshape_test.go +++ b/src/crux/issueshape/issueshape_test.go @@ -168,3 +168,25 @@ func TestStripAgentPrefixAndSameTitle(t *testing.T) { t.Fatal("different subjects must not match") } } + +func TestSpekPublished(t *testing.T) { + marked := "## Goal\n\nAdd a refresh endpoint.\n\n" + if !SpekPublished(marked) { + t.Fatal("body with a valid spek:publish marker not reported as published") + } + if SpekPublished("## Goal\n\nAdd a refresh endpoint.") { + t.Fatal("plain body reported as spek-published") + } + if SpekPublished("") { + t.Fatal("unsubstituted marker template reported as spek-published") + } + // The marker itself is not an unfilled template placeholder: its content + // carries `=` so the angle-bracket scan skips it. + if ph, ok := UnsubstitutedTemplatePlaceholder(marked); ok { + t.Fatalf("marker flagged as placeholder %q", ph) + } + // A marker does not excuse a template left unfilled around it. + if _, ok := UnsubstitutedTemplatePlaceholder("\n\n"); !ok { + t.Fatal("placeholder beside a marker must still be flagged") + } +} diff --git a/src/crux/proxy/issue_shape_test.go b/src/crux/proxy/issue_shape_test.go index 78b6114ca2..0199695b36 100644 --- a/src/crux/proxy/issue_shape_test.go +++ b/src/crux/proxy/issue_shape_test.go @@ -67,6 +67,25 @@ func TestEnforceIssueShapeRejectsMalformedAndPreservesBody(t *testing.T) { body: "{\"title\":\"[guide] generics example\",\"body\":\"## Finding\\n\\nUse the generic form:\\n\\n```go\\nvar x Foo\\n```\\n\\n## Recommendation\\n\\nKeep it.\"}", wantDeny: false, }, + { + // A Spek-published child (spektacular#114) carries a hidden + // provenance marker; the marker is not a template placeholder and + // the body is forwarded as written. + name: "spek:publish marker is accepted", + method: http.MethodPost, + path: "https://api.github.com/repos/org/repo/issues", + body: "{\"title\":\"Add refresh endpoint\",\"body\":\"## What\\n\\nAdd `POST /auth/refresh`.\\n\\n## Acceptance criteria\\n\\n- [ ] token rotates\\n\\n\"}", + wantDeny: false, + }, + { + // The marker does not excuse an unfilled template around it. + name: "spek:publish marker does not bypass placeholder check", + method: http.MethodPost, + path: "https://api.github.com/repos/org/repo/issues", + body: "{\"title\":\"Add refresh endpoint\",\"body\":\"\\n\\n\"}", + wantDeny: true, + errorSubstr: "unsubstituted template placeholder", + }, { name: "label sub-route is not shape-checked", method: http.MethodPost, diff --git a/src/crux/proxy/linear_rules.go b/src/crux/proxy/linear_rules.go index 21a1306a75..cb8dce203a 100644 --- a/src/crux/proxy/linear_rules.go +++ b/src/crux/proxy/linear_rules.go @@ -105,6 +105,17 @@ var linearReadRequirement = linearRequirement{MinMode: agent.ModeAdvisory} // issue. That is how a PR-capable agent reports what it produced, so they // sit at ModeIssuesAndPRs alongside PR creation. // +// - issueRelationCreate, projectCreate and projectUpdate are what an in-hive +// Spek needs to publish a finished spec as a parent plus children +// (hivecommons/spektacular#114): a Project (or a parent issue — that is +// plain issueCreate with parentId, already covered) for the RFC, child +// issueCreate per task, and `blockedBy` relations between children; the +// idempotent re-publish edits the parent in place with projectUpdate. They +// are the Linear analogues of GitHub's POST …/sub_issues and +// POST …/dependencies/blocked_by, so they sit at ModeIssuesOnly with no +// capability grant. Deliberately NOT included: issueRelationDelete, +// projectDelete, projectArchive — a published tree is only ever added to. +// // Mutations NOT listed are denied, including every destructive one // (issueDelete, commentDelete, projectDelete, teamDelete, issueArchive) and // every administrative one (organizationInviteCreate, apiKeyCreate, @@ -134,6 +145,11 @@ var linearMutations = map[string]linearRequirement{ "issueAddLabel": {MinMode: agent.ModeIssuesOnly}, "issueRemoveLabel": {MinMode: agent.ModeIssuesOnly}, + // Spek publish: RFC parent, children and their dependency edges. + "issueRelationCreate": {MinMode: agent.ModeIssuesOnly}, + "projectCreate": {MinMode: agent.ModeIssuesOnly}, + "projectUpdate": {MinMode: agent.ModeIssuesOnly}, + // Linking produced artifacts (PRs, builds) onto an issue. "attachmentCreate": {MinMode: agent.ModeIssuesAndPRs}, "attachmentLinkURL": {MinMode: agent.ModeIssuesAndPRs}, diff --git a/src/crux/proxy/linear_rules_test.go b/src/crux/proxy/linear_rules_test.go index 6f9374e007..44a00bd168 100644 --- a/src/crux/proxy/linear_rules_test.go +++ b/src/crux/proxy/linear_rules_test.go @@ -549,3 +549,53 @@ func TestLinearAgentSessionUpdateReachableAtEveryTier(t *testing.T) { t.Error("agentSessionUpdate + issueUpdate batch allowed at ADVISORY") } } + +// ───────────────────────────────────────────────────────────────────────────── +// Spek publish (hivecommons/spektacular#114): an in-hive Spek files an RFC +// parent, child issues and blockedBy edges through this gate. Each mutation +// it needs sits at ISSUES_ONLY with the other issue-artifact writes, and the +// destructive siblings stay absent from the allowlist. +// ───────────────────────────────────────────────────────────────────────────── + +func TestLinearSpekPublishMutationsRequireIssuesOnly(t *testing.T) { + queries := map[string]string{ + "issueCreate with parentId": `mutation { issueCreate(input: {teamId: "t", title: "child", parentId: "rfc"}) { success } }`, + "issueRelationCreate": `mutation { issueRelationCreate(input: {issueId: "a", relatedIssueId: "b", type: blocks}) { success } }`, + "projectCreate": `mutation { projectCreate(input: {teamIds: ["t"], name: "RFC: auth"}) { success } }`, + "projectUpdate": `mutation { projectUpdate(id: "p", input: {description: "v2"}) { success } }`, + } + for name, q := range queries { + t.Run(name, func(t *testing.T) { + if d := decide(agent.ModeAdvisory, q); d.Allowed { + t.Errorf("%s must be DENIED at ADVISORY", name) + } + if d := LinearGraphQLAllowed(agent.ModeAdvisory, agent.AgentCapabilities{Converse: true}, linearBody(q), false); d.Allowed { + t.Errorf("converse must NOT grant %s — publishing is artifact production", name) + } + for _, mode := range []agent.AgentMode{agent.ModeIssuesOnly, agent.ModeIssuesAndPRs, agent.ModeIssuesPRsMerge} { + if d := decide(mode, q); !d.Allowed { + t.Errorf("%s must be ALLOWED at %s, got reason %q", name, mode, d.Reason) + } + } + }) + } +} + +func TestLinearSpekPublishDestructiveSiblingsStayDenied(t *testing.T) { + for name, q := range map[string]string{ + "issueRelationDelete": `mutation { issueRelationDelete(id: "r") { success } }`, + "projectDelete": `mutation { projectDelete(id: "p") { success } }`, + "projectArchive": `mutation { projectArchive(id: "p") { success } }`, + } { + for _, mode := range allModes { + if d := decide(mode, q); d.Allowed { + t.Errorf("%s must be DENIED at %s", name, mode) + } + } + } + // A batch that creates a relation AND deletes one is denied as a whole. + const batch = `mutation { issueRelationCreate(input: {issueId: "a", relatedIssueId: "b", type: blocks}) { success } issueRelationDelete(id: "r") { success } }` + if d := decide(agent.ModeIssuesPRsMerge, batch); d.Allowed { + t.Error("batch mixing an allowed publish mutation with a delete must be denied") + } +} diff --git a/src/crux/proxy/rules.go b/src/crux/proxy/rules.go index 3f5665af45..a40e7eb844 100644 --- a/src/crux/proxy/rules.go +++ b/src/crux/proxy/rules.go @@ -203,6 +203,17 @@ var rules = []ProxyRule{ {PathPattern: regexp.MustCompile(`^(?:/api/v1)?/repos/[^/]+/[^/]+/issues/\d+/labels`), Method: "POST", MinMode: agent.ModeIssuesOnly}, {PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/labels`), Method: "POST", MinMode: agent.ModeIssuesOnly}, {PathPattern: regexp.MustCompile(`^(?:/api/v1)?/repos/[^/]+/[^/]+/issues/\d+/labels`), Method: "DELETE", MinMode: agent.ModeIssuesOnly}, + // Issue relations — the native sub-issue link (`hive-open-issue --parent`, + // POST …/issues/{n}/sub_issues) and the "blocked by" dependency + // (`hive-open-issue --blocked-by`, POST …/issues/{n}/dependencies/blocked_by). + // The hive's own relay files these from the owner UID and never crossed + // this table; an agent publishing a Spek RFC with children from inside a + // hive does (hivecommons/spektacular#114), and relating two issues it was + // already allowed to create is artifact production at the same tier. + // Exactly these two sub-routes: no PUT/PATCH reprioritisation, no DELETE + // unlink — a published tree is only ever added to. + {PathPattern: regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues/\d+/sub_issues$`), Method: "POST", MinMode: agent.ModeIssuesOnly}, + {PathPattern: regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues/\d+/dependencies/blocked_by$`), Method: "POST", MinMode: agent.ModeIssuesOnly}, // ── Read operations — ADVISORY and above ── // Catch-all: any GET/HEAD/OPTIONS on any path. {PathPattern: regexp.MustCompile(`.*`), Method: "GET", MinMode: agent.ModeAdvisory}, @@ -213,86 +224,6 @@ var rules = []ProxyRule{ // AllowedByMode returns true if the given HTTP method+path is permitted // for an agent running in the specified mode. Unknown operations are // denied by default. -// denyRule is a hard block that applies to EVERY agent mode, regardless of the -// mode-escalation rules. It exists for operations agents must route through a -// hive-mediated path instead of calling GitHub directly. -type denyRule struct { - PathPattern *regexp.Regexp - Method string - Msg string // agent-facing directive surfaced in the 403 body -} - -// denyRules are checked BEFORE the mode rules. Two cases are hard-denied for -// EVERY agent mode: -// -// 1. Direct PR creation: a POST /repos/*/pulls — whether from `gh pr create` -// or the GitHub MCP create_pull_request/create_pull_request_with_copilot -// tool — authors the PR as the Copilot login user, not the App bot. -// Blocking it here forces agents to use `hive-open-pr`, which the hive -// fulfills with the App token so the PR is authored by the App bot. This -// closes the MCP path the gh-wrapper redirect cannot see. -// -// 2. Direct PR merge: a PUT /repos/*/pulls/{n}/merge (H1, CWE-863). Permitting -// this at ModeIssuesPRsMerge on a UID-derived mode check ALONE let an -// injected merge-mode agent `gh api -X PUT .../merge` any reachable PR, -// bypassing the bound merge relay's fail-closed SHA pin + merge-eligible -// binding (see bindMergeAuthz). Hard-denying it forces ALL agent-initiated -// merges through `hive-merge`, which the hive fulfills as the App over REST -// with those bindings enforced. This mirrors the GraphQL mergePullRequest -// deny (GraphQLAllowed treats it as a mutation), so neither REST nor GraphQL -// is an agent-reachable merge bypass. -// -// In BOTH cases the hive's OWN call (CreatePR / MergePR) does NOT traverse this -// agent proxy — it originates from the hive process (owner UID), which the -// forced-egress iptables redirect exempts — so the relay is unaffected. -var denyRules = []denyRule{ - // ── GitHub ── - { - PathPattern: regexp.MustCompile(`^/repos/[^/]+/[^/]+/pulls$`), - Method: "POST", - Msg: "direct PR creation is disabled for agents — use `hive-open-pr --repo --head --title --body ` so the hive opens the PR as the App bot (never the login user). Do NOT use `gh pr create` or the GitHub MCP create_pull_request/create_pull_request_with_copilot.", - }, - { - PathPattern: regexp.MustCompile(`^/repos/[^/]+/[^/]+/pulls/\d+/merge$`), - Method: "PUT", - Msg: "direct PR merge is disabled for agents — use `hive-merge --repo --number --expect-sha ` so the hive merges as the App bot with the SHA-pin + merge-eligible binding enforced. Do NOT use `gh api -X PUT .../merge`, `gh pr merge`, or the GitHub MCP merge_pull_request.", - }, - - // ── GitLab ── - { - PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/merge_requests$`), - Method: "POST", - Msg: "direct MR creation is disabled for agents — use the request-file path so the hive opens the MR as the App/bot identity. Do NOT use direct GitLab API calls.", - }, - { - PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/merge_requests/\d+/merge$`), - Method: "PUT", - Msg: "direct MR merge is disabled for agents — use the request-file path so the hive merges as the App/bot identity with bindings enforced. Do NOT use direct GitLab API calls.", - }, - { - PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/merge_requests/\d+/merge$`), - Method: "POST", - Msg: "direct MR merge is disabled for agents — use the request-file path so the hive merges as the App/bot identity with bindings enforced. Do NOT use direct GitLab API calls.", - }, - - // ── Gitea / Forgejo ── - { - PathPattern: regexp.MustCompile(`^/api/v1/repos/[^/]+/[^/]+/pulls$`), - Method: "POST", - Msg: "direct PR creation is disabled for agents — use the request-file path so the hive opens the PR as the App/bot identity. Do NOT use direct Gitea API calls.", - }, - { - PathPattern: regexp.MustCompile(`^(?:/api/v1)?/repos/[^/]+/[^/]+/pulls/\d+/merge$`), - Method: "POST", - Msg: "direct PR merge is disabled for agents — use the request-file path so the hive merges as the App/bot identity with bindings enforced. Do NOT use direct Gitea API calls.", - }, - { - PathPattern: regexp.MustCompile(`^/api/v1/repos/[^/]+/[^/]+/pulls/\d+/merge$`), - Method: "PUT", - Msg: "direct PR merge is disabled for agents — use the request-file path so the hive merges as the App/bot identity with bindings enforced. Do NOT use direct Gitea API calls.", - }, -} - func AllowedByMode(mode agent.AgentMode, method, path string) bool { return AllowedByModeCaps(mode, agent.AgentCapabilities{}, method, path) } @@ -330,20 +261,6 @@ func AllowedByModeCaps(mode agent.AgentMode, caps agent.AgentCapabilities, metho return false } -// DeniedMessage returns the agent-facing directive for a request blocked by a -// hard-deny rule, and true if such a rule matched. It lets the proxy surface WHY -// a request was refused (e.g. "use hive-open-pr") instead of the generic ACMM -// message. Returns ("", false) when no deny rule matches — the caller then falls -// back to the normal mode-based block reason. -func DeniedMessage(method, path string) (string, bool) { - for _, r := range denyRules { - if r.Method == method && r.PathPattern.MatchString(path) { - return r.Msg, true - } - } - return "", false -} - var repoPathPrefix = regexp.MustCompile(`^(?:/api/v1)?/repos/([^/]+/[^/]+)`) // gitSmartHTTPEndpoints are the only things that may follow ".git/" for diff --git a/src/crux/proxy/rules_deny.go b/src/crux/proxy/rules_deny.go new file mode 100644 index 0000000000..6113d582a2 --- /dev/null +++ b/src/crux/proxy/rules_deny.go @@ -0,0 +1,103 @@ +package proxy + +import ( + "regexp" +) + +// Hard-deny rules: operations refused for EVERY agent mode because they must +// route through a hive-mediated relay. Split out of rules.go so the +// mode-escalation table stays within its size ratchet. + +// denyRule is a hard block that applies to EVERY agent mode, regardless of the +// mode-escalation rules. It exists for operations agents must route through a +// hive-mediated path instead of calling GitHub directly. +type denyRule struct { + PathPattern *regexp.Regexp + Method string + Msg string // agent-facing directive surfaced in the 403 body +} + +// denyRules are checked BEFORE the mode rules. Two cases are hard-denied for +// EVERY agent mode: +// +// 1. Direct PR creation: a POST /repos/*/pulls — whether from `gh pr create` +// or the GitHub MCP create_pull_request/create_pull_request_with_copilot +// tool — authors the PR as the Copilot login user, not the App bot. +// Blocking it here forces agents to use `hive-open-pr`, which the hive +// fulfills with the App token so the PR is authored by the App bot. This +// closes the MCP path the gh-wrapper redirect cannot see. +// +// 2. Direct PR merge: a PUT /repos/*/pulls/{n}/merge (H1, CWE-863). Permitting +// this at ModeIssuesPRsMerge on a UID-derived mode check ALONE let an +// injected merge-mode agent `gh api -X PUT .../merge` any reachable PR, +// bypassing the bound merge relay's fail-closed SHA pin + merge-eligible +// binding (see bindMergeAuthz). Hard-denying it forces ALL agent-initiated +// merges through `hive-merge`, which the hive fulfills as the App over REST +// with those bindings enforced. This mirrors the GraphQL mergePullRequest +// deny (GraphQLAllowed treats it as a mutation), so neither REST nor GraphQL +// is an agent-reachable merge bypass. +// +// In BOTH cases the hive's OWN call (CreatePR / MergePR) does NOT traverse this +// agent proxy — it originates from the hive process (owner UID), which the +// forced-egress iptables redirect exempts — so the relay is unaffected. +var denyRules = []denyRule{ + // ── GitHub ── + { + PathPattern: regexp.MustCompile(`^/repos/[^/]+/[^/]+/pulls$`), + Method: "POST", + Msg: "direct PR creation is disabled for agents — use `hive-open-pr --repo --head --title --body ` so the hive opens the PR as the App bot (never the login user). Do NOT use `gh pr create` or the GitHub MCP create_pull_request/create_pull_request_with_copilot.", + }, + { + PathPattern: regexp.MustCompile(`^/repos/[^/]+/[^/]+/pulls/\d+/merge$`), + Method: "PUT", + Msg: "direct PR merge is disabled for agents — use `hive-merge --repo --number --expect-sha ` so the hive merges as the App bot with the SHA-pin + merge-eligible binding enforced. Do NOT use `gh api -X PUT .../merge`, `gh pr merge`, or the GitHub MCP merge_pull_request.", + }, + + // ── GitLab ── + { + PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/merge_requests$`), + Method: "POST", + Msg: "direct MR creation is disabled for agents — use the request-file path so the hive opens the MR as the App/bot identity. Do NOT use direct GitLab API calls.", + }, + { + PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/merge_requests/\d+/merge$`), + Method: "PUT", + Msg: "direct MR merge is disabled for agents — use the request-file path so the hive merges as the App/bot identity with bindings enforced. Do NOT use direct GitLab API calls.", + }, + { + PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/merge_requests/\d+/merge$`), + Method: "POST", + Msg: "direct MR merge is disabled for agents — use the request-file path so the hive merges as the App/bot identity with bindings enforced. Do NOT use direct GitLab API calls.", + }, + + // ── Gitea / Forgejo ── + { + PathPattern: regexp.MustCompile(`^/api/v1/repos/[^/]+/[^/]+/pulls$`), + Method: "POST", + Msg: "direct PR creation is disabled for agents — use the request-file path so the hive opens the PR as the App/bot identity. Do NOT use direct Gitea API calls.", + }, + { + PathPattern: regexp.MustCompile(`^(?:/api/v1)?/repos/[^/]+/[^/]+/pulls/\d+/merge$`), + Method: "POST", + Msg: "direct PR merge is disabled for agents — use the request-file path so the hive merges as the App/bot identity with bindings enforced. Do NOT use direct Gitea API calls.", + }, + { + PathPattern: regexp.MustCompile(`^/api/v1/repos/[^/]+/[^/]+/pulls/\d+/merge$`), + Method: "PUT", + Msg: "direct PR merge is disabled for agents — use the request-file path so the hive merges as the App/bot identity with bindings enforced. Do NOT use direct Gitea API calls.", + }, +} + +// DeniedMessage returns the agent-facing directive for a request blocked by a +// hard-deny rule, and true if such a rule matched. It lets the proxy surface WHY +// a request was refused (e.g. "use hive-open-pr") instead of the generic ACMM +// message. Returns ("", false) when no deny rule matches — the caller then falls +// back to the normal mode-based block reason. +func DeniedMessage(method, path string) (string, bool) { + for _, r := range denyRules { + if r.Method == method && r.PathPattern.MatchString(path) { + return r.Msg, true + } + } + return "", false +} diff --git a/src/crux/proxy/rules_test.go b/src/crux/proxy/rules_test.go index 313c445e71..30b487ce2f 100644 --- a/src/crux/proxy/rules_test.go +++ b/src/crux/proxy/rules_test.go @@ -32,6 +32,17 @@ func TestAllowedByMode(t *testing.T) { {"issues-only allows PATCH issue", agent.ModeIssuesOnly, "PATCH", "/repos/org/repo/issues/42", true}, {"issues-only allows comment", agent.ModeIssuesOnly, "POST", "/repos/org/repo/issues/42/comments", true}, {"issues-only allows label", agent.ModeIssuesOnly, "POST", "/repos/org/repo/issues/42/labels", true}, + // Spek publishes an RFC with sub-issue children and "blocked by" edges + // through the proxy (spektacular#114): the same two writes + // hive-open-issue --parent / --blocked-by make, at the issue tier. + {"issues-only allows add sub-issue", agent.ModeIssuesOnly, "POST", "/repos/org/repo/issues/42/sub_issues", true}, + {"issues-only allows add blocked-by", agent.ModeIssuesOnly, "POST", "/repos/org/repo/issues/42/dependencies/blocked_by", true}, + {"advisory blocks add sub-issue", agent.ModeAdvisory, "POST", "/repos/org/repo/issues/42/sub_issues", false}, + {"advisory blocks add blocked-by", agent.ModeAdvisory, "POST", "/repos/org/repo/issues/42/dependencies/blocked_by", false}, + {"sub-issue unlink stays denied", agent.ModeIssuesPRsMerge, "DELETE", "/repos/org/repo/issues/42/sub_issue", false}, + {"sub-issue reprioritise stays denied", agent.ModeIssuesPRsMerge, "PATCH", "/repos/org/repo/issues/42/sub_issues/priority", false}, + {"blocked-by unlink stays denied", agent.ModeIssuesPRsMerge, "DELETE", "/repos/org/repo/issues/42/dependencies/blocked_by/7", false}, + {"blocking side is not a write route", agent.ModeIssuesPRsMerge, "POST", "/repos/org/repo/issues/42/dependencies/blocking", false}, {"issues-only blocks POST pulls", agent.ModeIssuesOnly, "POST", "/repos/org/repo/pulls", false}, {"issues-only blocks PATCH pull", agent.ModeIssuesOnly, "PATCH", "/repos/org/repo/pulls/42", false}, {"issues-only blocks PUT merge", agent.ModeIssuesOnly, "PUT", "/repos/org/repo/pulls/42/merge", false}, diff --git a/src/crux/spekmarker/spekmarker.go b/src/crux/spekmarker/spekmarker.go new file mode 100644 index 0000000000..d3c5699d36 --- /dev/null +++ b/src/crux/spekmarker/spekmarker.go @@ -0,0 +1,131 @@ +// Package spekmarker parses the hidden provenance marker Spek +// (hivecommons/spektacular) writes into every RFC and child issue it +// publishes into a work source: +// +// +// +// The marker is how a published item says "this body was produced from a +// finished spec" (hivecommons/spektacular#114). Crux reads it on the proxy's +// issue-create path so a Spek child filed from inside a hive is treated as +// already specified, and the hive's triage pass reads the same marker to keep +// the child on the direct-fix path instead of re-routing it into a new spec +// run. It is a stdlib-only Crux ring leaf; hive-side packages may re-export or +// copy it, Crux never imports them. +package spekmarker + +import ( + "regexp" + "strings" +) + +// Prefix is the marker's opening token. +const Prefix = "`) + // keyRE is the shape of a marker key: a lowercase identifier. + keyRE = regexp.MustCompile(`^[a-z][a-z0-9_]*$`) +) + +// Marker is one parsed `` comment. +type Marker struct { + // Name is the spek artifact name (`name=`). Required. + Name string + // ArtifactID is the spek's stable artifact id (`artifact_id=`). Required; + // it is what makes re-publish idempotent. + ArtifactID string + // Target is the work source the item was published to (`target=`): + // github, jira or linear. Required. + Target string + // Task is the plan task / member spec id a child carries (`task=`). Empty + // on the RFC itself. + Task string + // RFC is the URL of the parent RFC a child links back to (`rfc=`). + RFC string +} + +// Valid reports whether the marker carries every required field. +func (m Marker) Valid() bool { + return m.Name != "" && m.ArtifactID != "" && m.Target != "" +} + +// IsChild reports whether the marker belongs to a child issue (carries a +// task id) rather than the RFC. +func (m Marker) IsChild() bool { return m.Task != "" } + +// Parse returns the first valid spek:publish marker in text. ok is false +// when text carries no marker, or only markers missing a required field — +// a malformed marker is not evidence of a spec, so callers fall back to +// their default treatment of the body. +func Parse(text string) (Marker, bool) { + for _, match := range markerRE.FindAllStringSubmatch(text, -1) { + if len(match) != 2 { + continue + } + m, ok := parseFields(match[1]) + if ok && m.Valid() { + return m, true + } + } + return Marker{}, false +} + +// Has reports whether text carries a valid spek:publish marker. +func Has(text string) bool { + _, ok := Parse(text) + return ok +} + +// parseFields decodes the `key=value key=value` run inside a marker. Values +// run to the next whitespace; a value wrapped in double quotes may contain +// spaces. An unknown key is ignored so a newer Spek can add fields without +// an older Crux refusing the marker; a key with an empty value or a value +// that still looks like a template token (``) fails the parse. +func parseFields(s string) (Marker, bool) { + var m Marker + rest := strings.TrimSpace(s) + for rest != "" { + eq := strings.IndexByte(rest, '=') + if eq <= 0 { + return Marker{}, false + } + key := rest[:eq] + if !keyRE.MatchString(key) { + return Marker{}, false + } + rest = rest[eq+1:] + var val string + if strings.HasPrefix(rest, `"`) { + end := strings.IndexByte(rest[1:], '"') + if end < 0 { + return Marker{}, false + } + val = rest[1 : end+1] + rest = rest[end+2:] + } else { + end := strings.IndexAny(rest, " \t\r\n") + if end < 0 { + val, rest = rest, "" + } else { + val, rest = rest[:end], rest[end:] + } + } + rest = strings.TrimSpace(rest) + if val == "" || (strings.HasPrefix(val, "<") && strings.HasSuffix(val, ">")) { + return Marker{}, false + } + switch key { + case "name": + m.Name = val + case "artifact_id": + m.ArtifactID = val + case "target": + m.Target = strings.ToLower(val) + case "task": + m.Task = val + case "rfc": + m.RFC = val + } + } + return m, true +} diff --git a/src/crux/spekmarker/spekmarker_test.go b/src/crux/spekmarker/spekmarker_test.go new file mode 100644 index 0000000000..febb9198ed --- /dev/null +++ b/src/crux/spekmarker/spekmarker_test.go @@ -0,0 +1,80 @@ +package spekmarker + +import "testing" + +func TestParse(t *testing.T) { + tests := []struct { + name string + text string + want Marker + wantOK bool + }{ + { + name: "rfc marker with required fields", + text: "## Overview\n\n…\n\n", + want: Marker{Name: "auth-refresh", ArtifactID: "spk_01HX", Target: "github"}, + wantOK: true, + }, + { + name: "child marker with task and rfc", + text: "", + want: Marker{Name: "auth-refresh", ArtifactID: "spk_01HX", Target: "linear", Task: "T3", RFC: "https://github.com/o/r/issues/7"}, + wantOK: true, + }, + { + name: "quoted value with spaces and unknown key tolerated", + text: ``, + want: Marker{Name: "my spec", ArtifactID: "a1", Target: "jira"}, + wantOK: true, + }, + { + name: "loose whitespace around the comment delimiters", + text: "", + want: Marker{Name: "n", ArtifactID: "i", Target: "github"}, + wantOK: true, + }, + {name: "no marker", text: "## Goal\n\nJust a body.", wantOK: false}, + {name: "missing artifact_id", text: "", wantOK: false}, + {name: "missing target", text: "", wantOK: false}, + {name: "empty value", text: "", wantOK: false}, + { + name: "template still unsubstituted is not a marker", + text: "", + wantOK: false, + }, + {name: "bare word without key", text: "", wantOK: false}, + {name: "unterminated quote", text: ``, wantOK: false}, + {name: "other hive marker is not confused", text: "", wantOK: false}, + { + name: "first valid marker wins over an earlier malformed one", + text: "\n", + want: Marker{Name: "n", ArtifactID: "i", Target: "github"}, + wantOK: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := Parse(tt.text) + if ok != tt.wantOK { + t.Fatalf("Parse ok = %v, want %v (got %+v)", ok, tt.wantOK, got) + } + if ok && got != tt.want { + t.Fatalf("Parse = %+v, want %+v", got, tt.want) + } + if Has(tt.text) != tt.wantOK { + t.Fatalf("Has = %v, want %v", !tt.wantOK, tt.wantOK) + } + }) + } +} + +func TestMarkerIsChild(t *testing.T) { + rfc, _ := Parse("") + child, _ := Parse("") + if rfc.IsChild() { + t.Error("RFC marker without task reported as child") + } + if !child.IsChild() { + t.Error("marker with task not reported as child") + } +} diff --git a/src/docs/design/crux-tool-policy.md b/src/docs/design/crux-tool-policy.md index e2733ae340..9689ead397 100644 --- a/src/docs/design/crux-tool-policy.md +++ b/src/docs/design/crux-tool-policy.md @@ -118,6 +118,7 @@ and Linear tables gate. Defaults equal today's verdicts: | `issue.edit` | `PATCH …/issues/{n}` | (default bucket) | `ModeIssuesOnly` | | `issue.comment` | `POST …/issues/{n}/comments` | conversational fields (`converseMutations`) | `ModeIssuesOnly` or `converse` | | `label.add` | `POST …/issues/{n}/labels` | (default bucket) | `ModeIssuesOnly` | +| `issue.relate` | `POST …/issues/{n}/sub_issues`, `POST …/issues/{n}/dependencies/blocked_by` (Linear: `issueRelationCreate`, `projectCreate`, `projectUpdate`, `issueCreate` with `parentId`) | (default bucket) | `ModeIssuesOnly` — what an in-hive Spek publish needs to file an RFC with sub-issue children and "blocked by" edges ([spektacular#114](https://github.com/hivecommons/spektacular/issues/114)); unlink/delete/archive siblings stay denied | | `pr.create` | `POST …/pulls` | `createPullRequest` | **`relay(hive-open-pr)`** | | `pr.edit` | `PATCH …/pulls/{n}` | `markPullRequestReadyForReview`, `convertPullRequestToDraft` | `ModeIssuesAndPRs` | | `pr.review` | `POST …/pulls/{n}/reviews` | `addPullRequestReview`, `submitPullRequestReview` | `ModeIssuesAndPRs` or `converse` | diff --git a/src/docs/linear-agent.md b/src/docs/linear-agent.md index afca43183c..973383be83 100644 --- a/src/docs/linear-agent.md +++ b/src/docs/linear-agent.md @@ -224,8 +224,12 @@ Agent-side calls to `api.linear.app` go through the ACMM proxy (`pkg/proxy/linear_rules.go`, merged in #4522): a deny-by-default GraphQL mutation allowlist where `agentActivityCreate`/`agentSessionUpdate` are reachable at every tier (they carry the 10-second session invariant), -`issueUpdate`/`commentCreate` require ISSUES_ONLY, and anything unknown or -unparseable is denied. The control-plane client in `pkg/linearagent` (webhook +`issueUpdate`/`commentCreate` require ISSUES_ONLY, the mutations an in-hive +Spek publish makes (`issueCreate` with `parentId`, `issueRelationCreate` for +`blockedBy`, `projectCreate`/`projectUpdate` for the RFC parent) also sit at +ISSUES_ONLY, and anything unknown or unparseable is denied — including the +destructive siblings (`issueRelationDelete`, `projectDelete`, +`projectArchive`). The control-plane client in `pkg/linearagent` (webhook ack, identity query, token refresh) runs in the hive process itself, not through the agent proxy. From c9888226410407258bf42138430dcb909801deb6 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Sat, 7 Nov 2026 12:25:04 -0500 Subject: [PATCH 16/17] =?UTF-8?q?=E2=9C=A8=20triage:=20honour=20the=20Spek?= =?UTF-8?q?=20publish=20marker=20=E2=80=94=20children=20stay=20on=20the=20?= =?UTF-8?q?direct-fix=20path=20with=20the=20spek=20as=20context=20(#17019)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spek (hivecommons/spektacular#109, #114) publishes a finished spec as an RFC plus child issues into the work source, each body carrying . Hive now reads it (pkg/spekmarker): a marker-carrying item is never routed into a new spec run by runs.triage, is not scored by the clarity gate, and a child's kick line names the RFC and spek so the agent builds against the spek's acceptance criteria and writes Fixes #. A marker without task= is the RFC itself and is treated like epic for governor.claims.never_claim_labels and IsTrackerIssue, so Hive never claims the umbrella even in a repo without that label. Fixes #17018 Signed-off-by: Andrew Anderson --- changelog.d/added-spek-publish-marker.md | 1 + .../labels-and-control-signals/reference.md | 14 ++ src/docs/scale-envelope.md | 2 +- src/docs/spektacular.md | 49 +++++- src/docs/work-sources.md | 11 ++ src/pkg/claims/neverclaim.go | 23 ++- src/pkg/claims/neverclaim_test.go | 25 ++++ src/pkg/classify/triage.go | 11 ++ src/pkg/classify/triage_test.go | 9 ++ src/pkg/github/client.go | 6 +- src/pkg/github/client_test.go | 4 + src/pkg/github/issue_claim_gate.go | 20 ++- src/pkg/github/issue_claim_gate_test.go | 24 +++ src/pkg/github/issue_stage_facts.go | 2 +- src/pkg/github/issue_stage_labels.go | 7 +- src/pkg/github/issue_stage_labels_test.go | 1 + src/pkg/scheduler/clarity_gate.go | 8 +- src/pkg/scheduler/clarity_gate_test.go | 6 +- src/pkg/scheduler/kickmessage.go | 30 ++++ src/pkg/scheduler/lane_messages.go | 1 + src/pkg/scheduler/spek_context_test.go | 65 ++++++++ src/pkg/spekmarker/marker.go | 140 ++++++++++++++++++ src/pkg/spekmarker/marker_test.go | 101 +++++++++++++ 23 files changed, 546 insertions(+), 14 deletions(-) create mode 100644 changelog.d/added-spek-publish-marker.md create mode 100644 src/pkg/scheduler/spek_context_test.go create mode 100644 src/pkg/spekmarker/marker.go create mode 100644 src/pkg/spekmarker/marker_test.go diff --git a/changelog.d/added-spek-publish-marker.md b/changelog.d/added-spek-publish-marker.md new file mode 100644 index 0000000000..c311fb9f4b --- /dev/null +++ b/changelog.d/added-spek-publish-marker.md @@ -0,0 +1 @@ +- Hive honours the Spek publish marker `` (spektacular#109/#114): a Spek-published child stays on the direct-fix path under `runs.triage` (never a new `spec` run), skips the clarity score, and its kick line carries the RFC link and spek identity with the `Fixes #` instruction; the RFC itself (marker without `task=`) is treated like `epic` for `governor.claims.never_claim_labels` and `IsTrackerIssue`, so Hive never claims it even in a repo without that label. Parser in `pkg/spekmarker`; documented in `docs/spektacular.md` → Publish hand-off. diff --git a/src/docs/labels-and-control-signals/reference.md b/src/docs/labels-and-control-signals/reference.md index c53762a7a3..bff63274c7 100644 --- a/src/docs/labels-and-control-signals/reference.md +++ b/src/docs/labels-and-control-signals/reference.md @@ -272,6 +272,20 @@ With the gate on (default off), a human-filed, unclaimed issue whose clarity sco **Configuration and implementation.** `runs.triage.clarity_gate` (`src/pkg/scheduler/clarity_gate.go`, `src/pkg/classify/clarity.go`, `src/pkg/github/clarity_gate_resume.go`) +#### `` body marker (Spek publish hand-off) + +**Applies to:** Issues (GitHub, Jira, Linear bodies). **Kind:** Routing signal; never a hold. + +Spek writes `` into every RFC and child issue it publishes ([spektacular#114](https://github.com/hivecommons/spektacular/issues/114)). A marker **with** `task=` is a child: `runs.triage` keeps it on the direct-fix path (never a new `spec` run; `run/spec` still overrides), the clarity gate does not score it, and its kick line carries `specified by Spek: RFC (spek , artifact , task )` plus the `Fixes #` instruction. A marker **without** `task=` is the RFC itself: treated like `epic` for `governor.claims.never_claim_labels` — every automated claim producer refuses it with `never-claim label: spek:publish`, a stray claim is never staged 020/030 and `IsTrackerIssue` flags it — even when the repo has no `epic` label. A malformed marker is no marker; `rfc=` is dropped unless it is an http(s) URL. + +**Who applies it:** Spek (`spektacular publish`). + +**Checked by:** `pkg/spekmarker`; scheduler triage / clarity gate / kick assembly; the claim gate and stage derivation. **When:** Before kicks are built and on every claim. + +**Release or override.** Remove the marker from the body; `run/spec` sends a child into a spec run regardless. + +**Configuration and implementation.** None (`src/pkg/spekmarker/marker.go`, `src/pkg/classify/triage.go`, `src/pkg/scheduler/clarity_gate.go`, `src/pkg/scheduler/kickmessage.go`, `src/pkg/claims/neverclaim.go`, `src/pkg/github/issue_claim_gate.go`) + #### `stage/017-possible-existing` from the prior-art gate **Applies to:** Issues. **Kind:** Hard suppress. diff --git a/src/docs/scale-envelope.md b/src/docs/scale-envelope.md index 43fd9a3ce9..3e30779321 100644 --- a/src/docs/scale-envelope.md +++ b/src/docs/scale-envelope.md @@ -34,7 +34,7 @@ ones to: 2. **Truncation must preserve the prioritization signal, not cut an arbitrary slice.** This holds today and is easy to assume backwards: - Issues are sorted by actionable rank tier, then **oldest first** within each tier before any cap is applied - (`RankActionableIssues`, `src/pkg/github/client.go:5157-5167`, descending `AgeMinutes` within each tier). + (`RankActionableIssues`, `src/pkg/github/client.go:5161-5171`, descending `AgeMinutes` within each tier). - PRs are sorted by **review class then oldest-first within class** (`SortPullRequestsForReview`, `src/pkg/github/review_priority.go:334`; fixes → refactors/docs → tests). diff --git a/src/docs/spektacular.md b/src/docs/spektacular.md index 50f309a24e..2f98e63b39 100644 --- a/src/docs/spektacular.md +++ b/src/docs/spektacular.md @@ -81,6 +81,49 @@ contribute hub's cleanup tick. The dashboard never imports `pkg/spektacular` (its internal-import ratchet); `pkg/spektacular.NewHubRunner` takes the server through the `LeaseRegistry` interface. +## Publish hand-off + +Spek can also run the other way round: a finished spec or epic is +**published** as an RFC plus child issues into the hive's work source — GitHub +Issues, Jira or Linear — so Hive picks the children up from its normal queue +whether Spek ran standalone or in-hive +([spektacular#109](https://github.com/hivecommons/spektacular/issues/109), +[spektacular#114](https://github.com/hivecommons/spektacular/issues/114)). No +new input path is involved: the children are enumerated, staged and claimed +exactly like human-filed items. What Spek adds is one hidden marker in every +body it files: + +``` + +``` + +`name` and `artifact_id` identify the spek, `target` the work source. `task` +is the plan task / member spec id and is present on every **child**; the +**RFC** itself carries the marker without it. `rfc` is the RFC's URL. Values +are bare tokens (optionally quoted) separated by whitespace; a marker missing +`name`, `artifact_id` or a known `target` is malformed and is ignored as a +whole, and `rfc` is kept only when it parses as an http(s) URL — nothing in +the marker is echoed into a kick as free text. The parser is +`pkg/spekmarker`. + +Hive reads the marker in four places, none of which needs configuration: + +| Behaviour | Where | Effect of the marker | +|---|---|---| +| Triage | `runs.triage` (below) | A marker-carrying item is **never** routed into a new `spec` run, whatever `spec_labels` or complexity tier it carries; it takes the direct-fix path as `run/fix` does (signal `marker:spek:publish`). `run/spec` still overrides. | +| Clarity gate | `runs.triage.clarity_gate` | Not scored, the same way `approved-direction` is skipped: the item is already specified. | +| Kick context | kick issue list | A child's line gains `↳ specified by Spek: RFC (spek , artifact , task ) — build against the spek's acceptance criteria …` and the instruction that the PR body must say `Fixes #`. | +| Umbrella | `governor.claims.never_claim_labels`, `IsTrackerIssue` | A marker **without** `task=` is the RFC: it is treated like `epic` — refused by every automated claim producer (`never-claim label: spek:publish`), never staged 020/030 from a stray claim, and flagged `[TRACKER]` — even when the repo has no `epic` label. | + +Everything else in the handshake is unchanged and documented where it lives: +enumeration and the hard-suppress labels ([labels-and-control-signals.md](labels-and-control-signals.md)), +reporter trust (`issue_filter.reporter_trust`: publish as a trusted identity +or expect the `triage/accepted` step), the prior-art gate (expected to fire on +the RFC title, not the children), native sub-issue / blocked-by relations +([hive-open-issue.md](hive-open-issue.md)) and the Jira / Linear source +filters ([work-sources.md](work-sources.md)). The in-hive stage runner is not +involved in a publish; a published issue is ordinary work. + ## Triage The runs triage pass is separate from the Spek runner and is also OFF by @@ -89,6 +132,9 @@ incoming actionable GitHub issue before a direct-fix kick: - `run/spec` always admits a `spec` run. - `run/fix` always leaves the issue on the normal direct-fix path. +- An issue whose body carries the Spek publish marker (``, + see [Publish hand-off](#publish-hand-off)) likewise stays on the direct-fix + path: it is already specified. - Complex issues, or issues carrying `spec_labels` (default `kind/feature`, `Epic`, `architecture discussion`), admit a `spec` run by creating the first stage lease. The lease is keyed as the run-stage work source expects: the @@ -158,7 +204,8 @@ The sweep removes `needs-spec` and records the un-park on the stage engine, so the issue re-derives `stage/010-actionable` and re-enters the queue. Removing `needs-spec` by hand resumes it too. Either way the gate sees the marker and does not ask again, however the issue scores. Issues filed by Hive or an agent, -issues already claimed and issues carrying `approved-direction` are not scored. +issues already claimed, issues carrying `approved-direction` and Spek-published +issues (body marker ``) are not scored. ```yaml runs: diff --git a/src/docs/work-sources.md b/src/docs/work-sources.md index dc3c3fa8e8..30027bb5d4 100644 --- a/src/docs/work-sources.md +++ b/src/docs/work-sources.md @@ -80,6 +80,17 @@ GitHub repository. See [Forges: GitLab, Gitea and Forgejo](integrations/forges.md) for the operation list and what runs through the adapters today, and [Forge setup](forge-app-setup.md) for the reasons. +## Spek-published RFCs and children + +Spek (hivecommons/spektacular) can publish a finished spec as an RFC plus +child issues into any of the sources above. Nothing source-specific is needed: +the children are read like any other item, provided they pass the source's own +filters (`require_labels`, `hold_labels`, `teams[].repo` / `states`, +`project_keys` / `jql`). The hidden `` marker in each +body keeps a child on the direct-fix path, skips the clarity score, carries the +RFC link into the kick and marks the RFC itself as an umbrella Hive never +claims; see [spektacular.md → Publish hand-off](spektacular.md#publish-hand-off). + ## Run stages (`run_stages: true`)
diff --git a/src/pkg/claims/neverclaim.go b/src/pkg/claims/neverclaim.go index 571e80b983..d6eece2820 100644 --- a/src/pkg/claims/neverclaim.go +++ b/src/pkg/claims/neverclaim.go @@ -1,6 +1,10 @@ package claims -import "strings" +import ( + "strings" + + "github.com/hivecommons/hive/pkg/spekmarker" +) // Never-claim labels: an issue wearing one is not a unit of agent work, so no // automated claim producer — the relay `claim` request, the start-signal seam, @@ -38,3 +42,20 @@ func NeverClaimLabel(labels, never []string) string { } return "" } + +// SpekUmbrellaMarker is the never-claim "label" reported for an issue whose +// body carries the Spek publish marker with no `task=` — the RFC itself. +const SpekUmbrellaMarker = "spek:publish" + +// NeverClaimReason is NeverClaimLabel extended with the body: it returns the +// never-claim label the issue wears, else SpekUmbrellaMarker when body marks +// the issue as a Spek-published RFC, else "". +func NeverClaimReason(labels, never []string, body string) string { + if label := NeverClaimLabel(labels, never); label != "" { + return label + } + if spekmarker.IsUmbrella(body) { + return SpekUmbrellaMarker + } + return "" +} diff --git a/src/pkg/claims/neverclaim_test.go b/src/pkg/claims/neverclaim_test.go index 24a70f2daa..18ff9046ee 100644 --- a/src/pkg/claims/neverclaim_test.go +++ b/src/pkg/claims/neverclaim_test.go @@ -26,3 +26,28 @@ func TestNeverClaimLabel(t *testing.T) { }) } } + +func TestNeverClaimReason(t *testing.T) { + rfc := "" + child := "" + cases := []struct { + name string + labels []string + never []string + body string + want string + }{ + {"label wins over marker", []string{"Epic"}, nil, rfc, "Epic"}, + {"spek rfc without label", []string{"enhancement"}, nil, rfc, SpekUmbrellaMarker}, + {"spek rfc with gate disabled still umbrella", nil, []string{}, rfc, SpekUmbrellaMarker}, + {"spek child is work", nil, nil, child, ""}, + {"plain body", []string{"bug"}, nil, "no marker here", ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := NeverClaimReason(tc.labels, tc.never, tc.body); got != tc.want { + t.Fatalf("NeverClaimReason = %q, want %q", got, tc.want) + } + }) + } +} diff --git a/src/pkg/classify/triage.go b/src/pkg/classify/triage.go index 8bf8985292..c416831f56 100644 --- a/src/pkg/classify/triage.go +++ b/src/pkg/classify/triage.go @@ -7,8 +7,13 @@ import ( "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/issueshape" + "github.com/hivecommons/hive/pkg/spekmarker" ) +// SpekPublishedSignal is the TriageDecision signal for an item carrying the +// Spek publish marker (pkg/spekmarker): already specified, direct-fix path. +const SpekPublishedSignal = "marker:spek:publish" + type TriageVerdict string const ( @@ -31,6 +36,12 @@ func Triage(issue github.Issue, c Classification, cfg config.TriageConfig) Triag if hasLabel(labels, "run/fix") { return triageDecision(TriageFix, "run/fix override label", "label:run/fix") } + // A Spek-published item has already been specified: the RFC and its + // acceptance criteria exist, so it is never sent into another `spec` + // run, whatever labels or tier it carries (spektacular#114). + if m, ok := spekmarker.Parse(issue.Body); ok { + return triageDecision(TriageFix, "Spek-published item: already specified by spek "+m.Name, SpekPublishedSignal) + } body := strings.TrimSpace(issue.Body) if min := cfg.EffectiveMinBodyChars(); len(body) < min { return triageDecision(TriageClarify, fmt.Sprintf("issue body is shorter than %d characters", min), "body:short") diff --git a/src/pkg/classify/triage_test.go b/src/pkg/classify/triage_test.go index aa788a5051..518827cf80 100644 --- a/src/pkg/classify/triage_test.go +++ b/src/pkg/classify/triage_test.go @@ -7,6 +7,8 @@ import ( "github.com/hivecommons/hive/pkg/github" ) +const spekChildMarker = "" + func TestTriageRules(t *testing.T) { longBody := "This issue has enough detail to explain the requested behaviour, constraints, and expected outcome." cases := []struct { @@ -28,6 +30,13 @@ func TestTriageRules(t *testing.T) { {"simple fixes", github.Issue{Body: longBody}, Classification{Tier: TierSimple}, config.TriageConfig{}, TriageFix, "tier:Simple"}, {"medium fixes", github.Issue{Body: longBody}, Classification{Tier: TierMedium}, config.TriageConfig{}, TriageFix, "tier:Medium"}, {"custom spec label", github.Issue{Labels: []string{"needs-design"}, Body: longBody}, Classification{Tier: TierMedium}, config.TriageConfig{SpecLabels: []string{"needs-design"}}, TriageSpec, "label:needs-design"}, + // spektacular#114: a Spek-published child is already specified, so + // neither a spec label nor a Complex tier re-routes it into a spec run. + {"spek child beats spec label", github.Issue{Labels: []string{"kind/feature"}, Body: longBody + "\n" + spekChildMarker}, Classification{Tier: TierComplex}, config.TriageConfig{}, TriageFix, SpekPublishedSignal}, + {"spek child beats short body", github.Issue{Body: spekChildMarker}, Classification{Tier: TierMedium}, config.TriageConfig{}, TriageFix, SpekPublishedSignal}, + {"spek umbrella never specs", github.Issue{Labels: []string{"Epic"}, Body: longBody + "\n"}, Classification{Tier: TierComplex}, config.TriageConfig{}, TriageFix, SpekPublishedSignal}, + {"run/spec still overrides spek marker", github.Issue{Labels: []string{"run/spec"}, Body: spekChildMarker}, Classification{Tier: TierSimple}, config.TriageConfig{}, TriageSpec, "label:run/spec"}, + {"malformed spek marker is no marker", github.Issue{Labels: []string{"kind/feature"}, Body: longBody + "\n"}, Classification{Tier: TierMedium}, config.TriageConfig{}, TriageSpec, "label:kind/feature"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { diff --git a/src/pkg/github/client.go b/src/pkg/github/client.go index aa9725c2ac..b88e2a31b8 100644 --- a/src/pkg/github/client.go +++ b/src/pkg/github/client.go @@ -26,6 +26,7 @@ import ( "github.com/hivecommons/hive/pkg/ioscan" "github.com/hivecommons/hive/pkg/issueclaim" "github.com/hivecommons/hive/pkg/review" + "github.com/hivecommons/hive/pkg/spekmarker" "github.com/hivecommons/hive/pkg/stageengine" "github.com/hivecommons/hive/pkg/stagelabels" "github.com/hivecommons/hive/pkg/writeback" @@ -4081,7 +4082,10 @@ func IsTrackerIssue(title string, labels []string, body string) bool { if len(trackerTaskListRe.FindAllString(body, trackerTaskListMin)) >= trackerTaskListMin { return true } - return false + // A Spek-published RFC (spektacular#114) is the umbrella by + // construction: its publish marker carries no `task=`, and its children + // are the work. + return spekmarker.IsUmbrella(body) } func isTracker(title string, labels []string, body string) bool { diff --git a/src/pkg/github/client_test.go b/src/pkg/github/client_test.go index 0dc0abc038..160ae0adc5 100644 --- a/src/pkg/github/client_test.go +++ b/src/pkg/github/client_test.go @@ -871,6 +871,10 @@ func TestIsTrackerFromBodyTaskList(t *testing.T) { {"bare prose mentions are not a task list", prose, false}, {"a checklist without issue refs is not a tracker", checklistNoRefs, false}, {"empty body", "", false}, + // spektacular#114: the Spek-published RFC carries the publish marker + // with no task=; its children carry task= and stay claimable. + {"spek-published RFC is the umbrella", "RFC body\n", true}, + {"spek-published child is work", "child body\n", false}, } for _, tt := range tests { if got := isTracker("feature: something", nil, tt.body); got != tt.want { diff --git a/src/pkg/github/issue_claim_gate.go b/src/pkg/github/issue_claim_gate.go index dc2c0b29e9..21f083f8b6 100644 --- a/src/pkg/github/issue_claim_gate.go +++ b/src/pkg/github/issue_claim_gate.go @@ -70,13 +70,21 @@ func (c *Client) neverClaimLabelSet() []string { } // issueUmbrellaLabel returns the never-claim label labels carry under the -// set SetNeverClaimLabels last installed (package default until then), or "". -func issueUmbrellaLabel(labels []string) string { +// set SetNeverClaimLabels last installed (package default until then), +// claims.SpekUmbrellaMarker when spekUmbrella says the body marks a +// Spek-published RFC, or "". +func issueUmbrellaLabel(labels []string, spekUmbrella bool) string { var never []string if p := umbrellaLabels.Load(); p != nil { never = *p } - return claims.NeverClaimLabel(labels, never) + if label := claims.NeverClaimLabel(labels, never); label != "" { + return label + } + if spekUmbrella { + return claims.SpekUmbrellaMarker + } + return "" } // issueClaimParkLabel returns the park label that puts labels out of the @@ -112,7 +120,9 @@ func (c *Client) issueClaimParkLabel(labels []string) string { // agent escalation labels, the retired decision labels): a maintainer // holds it, so nobody starts on it until the un-park; // - an issue wearing a never-claim label (governor.claims.never_claim_labels, -// default `epic`): an umbrella issue is the list of work, not the work. +// default `epic`), or whose body carries the Spek publish marker with no +// `task=` (a Spek-published RFC, spektacular#114): an umbrella issue is +// the list of work, not the work. // // Lookup errors are returned so callers cannot mistake unknown labels for // permission to claim. Human claims do not use this gate. A producer about @@ -163,7 +173,7 @@ func (c *Client) issueClaimGate(ctx context.Context, repo string, number int) (i } if label := c.issueClaimParkLabel(labels); label != "" { res.reason = ClaimBlockReasonParkedPrefix + label - } else if label := claims.NeverClaimLabel(labels, c.neverClaimLabelSet()); label != "" { + } else if label := claims.NeverClaimReason(labels, c.neverClaimLabelSet(), issue.GetBody()); label != "" { res.reason = ClaimBlockReasonNeverClaimPrefix + label } return res, nil diff --git a/src/pkg/github/issue_claim_gate_test.go b/src/pkg/github/issue_claim_gate_test.go index ce76d33fe8..9f97b5bd8d 100644 --- a/src/pkg/github/issue_claim_gate_test.go +++ b/src/pkg/github/issue_claim_gate_test.go @@ -187,6 +187,30 @@ func TestIssueClaimBlockReasonNeverClaimLabelsConfigured(t *testing.T) { } } +// spektacular#114: a Spek-published RFC (publish marker, no task=) is an +// umbrella like `epic` even in a repo without that label; its children +// (task= set) are claimable. +func TestIssueClaimBlockReasonSpekUmbrella(t *testing.T) { + for _, tc := range []struct { + name, body, want string + }{ + {"rfc without epic label refused", "RFC\n", ClaimBlockReasonNeverClaimPrefix + "spek:publish"}, + {"child admitted", "child\n", ""}, + {"malformed marker admitted", "", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, `{"number":42,"labels":[{"name":"enhancement"}],"body":%q}`, tc.body) + })) + defer srv.Close() + got, err := issueTestClient(t, srv.URL).IssueClaimBlockReason(context.Background(), "o/r", 42) + if err != nil || got != tc.want { + t.Fatalf("reason = %q, %v; want %q", got, err, tc.want) + } + }) + } +} + // The `claim` relay request on an epic or a parked issue is refused before // any write, and reports no start signal. func TestIssueRequestWatcher_ClaimRefusesEpicAndParked(t *testing.T) { diff --git a/src/pkg/github/issue_stage_facts.go b/src/pkg/github/issue_stage_facts.go index 2fc9d2e34b..285f4bf7cd 100644 --- a/src/pkg/github/issue_stage_facts.go +++ b/src/pkg/github/issue_stage_facts.go @@ -101,7 +101,7 @@ func resolveIssueLabelFacts(in IssueStageInput, f *stageengine.Facts) { if issueHasLabel(in.Labels, HumanAckLabel) { is.ApprovedDirectionLabel = HumanAckLabel } - is.UmbrellaLabel = issueUmbrellaLabel(in.Labels) + is.UmbrellaLabel = issueUmbrellaLabel(in.Labels, in.SpekUmbrella) is.BlockedReason = "" if hasBlockedLabel(in.Labels) { is.BlockedReason = blockedStageReason(in) diff --git a/src/pkg/github/issue_stage_labels.go b/src/pkg/github/issue_stage_labels.go index 21b89cb1a7..178e2471a7 100644 --- a/src/pkg/github/issue_stage_labels.go +++ b/src/pkg/github/issue_stage_labels.go @@ -69,6 +69,7 @@ import ( "github.com/hivecommons/hive/pkg/authorkind" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/prstatus" + "github.com/hivecommons/hive/pkg/spekmarker" "github.com/hivecommons/hive/pkg/stageengine" "github.com/hivecommons/hive/pkg/stagelabels" ) @@ -181,6 +182,10 @@ type IssueStageInput struct { // DelegatesWork mirrors the watcher's unchecked issue-task predicate. // Unlike Children, it includes body-only delegation without ledger entries. DelegatesWork bool + // SpekUmbrella is set when the body carries the Spek publish marker with + // no `task=`: the issue is a Spek-published RFC (spektacular#114), an + // umbrella like `epic`, so a claim on it is never staged. + SpekUmbrella bool // Held is set for an issue the enumeration put in the Hold list // (Client.isHeld: the hive's hold-label matcher, including the // configured hive-pause/ label). IssueStageFor also reads the @@ -328,7 +333,7 @@ func IssueStageInputs(result *ActionableResult) []IssueStageInput { } var out []IssueStageInput for _, issue := range result.Issues.Items { - out = append(out, IssueStageInput{Repo: issue.Repo, Number: issue.Number, Labels: forgeIssueLabels(issue.Labels, issue.ClaimContext), ClaimKind: issue.ClaimKind, ClaimedBy: issue.ClaimedBy, PRStarted: issue.ClaimPRStarted, Claim: issue.ClaimContext, Children: issue.Children, AuthorKind: issueAuthorKind(issue), DelegatesWork: hasUnfinishedDelegatedWork(issue.Body), UpdatedAt: issue.UpdatedAt, Assignees: issue.Assignees}) + out = append(out, IssueStageInput{Repo: issue.Repo, Number: issue.Number, Labels: forgeIssueLabels(issue.Labels, issue.ClaimContext), ClaimKind: issue.ClaimKind, ClaimedBy: issue.ClaimedBy, PRStarted: issue.ClaimPRStarted, Claim: issue.ClaimContext, Children: issue.Children, AuthorKind: issueAuthorKind(issue), DelegatesWork: hasUnfinishedDelegatedWork(issue.Body), SpekUmbrella: spekmarker.IsUmbrella(issue.Body), UpdatedAt: issue.UpdatedAt, Assignees: issue.Assignees}) } // Held issues leave the pipeline at enumeration (Hold list, no // work-detail row). They still have to be walked so a stage label diff --git a/src/pkg/github/issue_stage_labels_test.go b/src/pkg/github/issue_stage_labels_test.go index e5e70fd009..5e2ba02b46 100644 --- a/src/pkg/github/issue_stage_labels_test.go +++ b/src/pkg/github/issue_stage_labels_test.go @@ -420,6 +420,7 @@ func TestIssueStageFor_TriageNeeded(t *testing.T) { {"hive-filed + hive/verified-open", IssueStageInput{AuthorKind: authorkind.Hive, Labels: []string{VerifiedOpenLabel}}, stagelabels.Actionable, ""}, {"hive-filed + approved-direction", IssueStageInput{AuthorKind: authorkind.Hive, Labels: []string{HumanAckLabel}}, stagelabels.Actionable, ""}, {"hive-filed epic without a maintainer label is 010 (umbrella, claim not staged)", IssueStageInput{AuthorKind: authorkind.Hive, Labels: []string{"Epic"}, Bucket: workBucketClaimed, ClaimKind: string(claims.KindAgent)}, stagelabels.Actionable, ""}, + {"spek-published RFC without an epic label is 010 (umbrella, claim not staged; spektacular#114)", IssueStageInput{AuthorKind: authorkind.Human, SpekUmbrella: true, Bucket: workBucketClaimed, ClaimKind: string(claims.KindAgent)}, stagelabels.Actionable, ""}, {"human-filed, no labels", IssueStageInput{AuthorKind: authorkind.Human}, stagelabels.Actionable, ""}, {"human-filed epic + verified-open (#12999)", IssueStageInput{AuthorKind: authorkind.Human, Labels: []string{"Epic", VerifiedOpenLabel}}, stagelabels.Actionable, ""}, {"unknown author is read as human", IssueStageInput{}, stagelabels.Actionable, ""}, diff --git a/src/pkg/scheduler/clarity_gate.go b/src/pkg/scheduler/clarity_gate.go index 9c8db4ce8f..46037faea7 100644 --- a/src/pkg/scheduler/clarity_gate.go +++ b/src/pkg/scheduler/clarity_gate.go @@ -11,6 +11,7 @@ import ( "github.com/hivecommons/hive/pkg/classify" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/spekmarker" ) // ClarityGateMarker tags the clarity gate's question comment. Its presence @@ -107,7 +108,9 @@ func (s *Scheduler) applyClarityGate(issues []github.Issue) []github.Issue { // clarityGated reports whether the gate judges issue at all: an unclaimed // GitHub issue a person filed, with no `approved-direction` from a -// maintainer. Hive- and agent-filed issues have nobody to answer questions. +// maintainer and no Spek publish marker (a Spek-published item is already +// specified; spektacular#114). Hive- and agent-filed issues have nobody to +// answer questions. func clarityGated(issue github.Issue) bool { if issue.Repo == "" || issue.Number <= 0 || !issue.AuthorIsHuman { return false @@ -118,6 +121,9 @@ func clarityGated(issue github.Issue) bool { if strings.TrimSpace(issue.ClaimedBy) != "" { return false } + if spekmarker.Has(issue.Body) { + return false + } return !issueHasLabel(issue, github.HumanAckLabel) } diff --git a/src/pkg/scheduler/clarity_gate_test.go b/src/pkg/scheduler/clarity_gate_test.go index f74534459e..afb74454d5 100644 --- a/src/pkg/scheduler/clarity_gate_test.go +++ b/src/pkg/scheduler/clarity_gate_test.go @@ -139,9 +139,11 @@ func TestClarityGateSkipsIssuesItDoesNotJudge(t *testing.T) { refs := kickedRefs(s, []github.Issue{ {Repo: "o/r", Number: 1, Title: "hive-filed", Body: "short"}, {Repo: "o/r", Number: 2, Title: "approved", Body: "short", AuthorIsHuman: true, Labels: []string{github.HumanAckLabel}}, + // spektacular#114: a Spek-published child is already specified. + {Repo: "o/r", Number: 3, Title: "spek child", Body: "", AuthorIsHuman: true}, }) - if !slices.Contains(refs, "o/r#1") || !slices.Contains(refs, "o/r#2") { - t.Fatalf("refs = %v, want both", refs) + if !slices.Contains(refs, "o/r#1") || !slices.Contains(refs, "o/r#2") || !slices.Contains(refs, "o/r#3") { + t.Fatalf("refs = %v, want all three", refs) } if len(forge.comments) != 0 || len(forge.labels) != 0 { t.Fatalf("gate acted on ungated issues: %v %v", forge.comments, forge.labels) diff --git a/src/pkg/scheduler/kickmessage.go b/src/pkg/scheduler/kickmessage.go index 1471f187c0..40e9ff6241 100644 --- a/src/pkg/scheduler/kickmessage.go +++ b/src/pkg/scheduler/kickmessage.go @@ -11,6 +11,7 @@ import ( "github.com/hivecommons/hive/pkg/decisionpolicy" "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/promptsrc" + "github.com/hivecommons/hive/pkg/spekmarker" "github.com/hivecommons/hive/pkg/stagelabels" "github.com/hivecommons/hive/pkg/upstreamwatch" ) @@ -70,6 +71,34 @@ func (s *Scheduler) upstreamPortLabels() []string { return labels } +// spekPublishedContext is the Spek hand-off block for an issue carrying the +// publish marker (pkg/spekmarker, spektacular#114): the RFC link and the spek +// name/artifact id, so the fixer builds against the spek's acceptance +// criteria and links its PR to this child with a closing keyword. Values are +// the parsed marker's own — the RFC URL only survives parsing as http(s) — +// never free text from the body. Empty for every other issue. +func spekPublishedContext(issue github.Issue) string { + m, ok := spekmarker.Parse(issue.Body) + if !ok { + return "" + } + var b strings.Builder + b.WriteString(" ↳ specified by Spek: ") + if m.RFC != "" { + b.WriteString("RFC " + m.RFC + " ") + } + fmt.Fprintf(&b, "(spek %s, artifact %s", m.Name, m.ArtifactID) + if m.Task != "" { + b.WriteString(", task " + m.Task) + } + b.WriteString(") — build against the spek's acceptance criteria for this slice; do not re-spec it") + if issue.Number > 0 { + fmt.Fprintf(&b, "; the PR body must say `Fixes #%d`", issue.Number) + } + b.WriteString("\n") + return b.String() +} + func hasAnyIssueLabel(labels []string, want []string) bool { for _, w := range want { if hasIssueLabel(labels, w) { @@ -142,6 +171,7 @@ func (s *Scheduler) formatIssueListWithPolicy(issues []github.Issue) (string, bo if diff := s.upstreamPortDiffURL(issue); diff != "" { b.WriteString(fmt.Sprintf(" ↳ upstream patch: %s — read it first, then port the change to this fork by hand; do not cherry-pick across repos\n", diff)) } + b.WriteString(spekPublishedContext(issue)) if hasIssueLabel(issue.Labels, stagelabels.LabelReauthored) { // The hive closed its own earlier PR for this issue after its // fixer gave up; the fresh PR is this kick's to open. diff --git a/src/pkg/scheduler/lane_messages.go b/src/pkg/scheduler/lane_messages.go index 74742c6551..2207065d11 100644 --- a/src/pkg/scheduler/lane_messages.go +++ b/src/pkg/scheduler/lane_messages.go @@ -49,6 +49,7 @@ func (s *Scheduler) buildScannerMessage(issues []github.Issue, actionable *githu tier, issue.ModelRec, strings.Join(issue.Labels, ","), title, tracker)) + b.WriteString(spekPublishedContext(issue)) shown++ } diff --git a/src/pkg/scheduler/spek_context_test.go b/src/pkg/scheduler/spek_context_test.go new file mode 100644 index 0000000000..47fe8e6793 --- /dev/null +++ b/src/pkg/scheduler/spek_context_test.go @@ -0,0 +1,65 @@ +package scheduler + +import ( + "log/slog" + "os" + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" +) + +// The Spek hand-off (spektacular#114): a Spek-published child kicked to a +// fixer carries the RFC link and spek identity in its kick context, so the +// agent builds against the spek's acceptance criteria and links its PR to +// the child with `Fixes #N`. + +func newSpekContextScheduler(t *testing.T) *Scheduler { + t.Helper() + cfg := &config.Config{Project: config.ProjectConfig{Org: "myorg", Repos: []string{"myorg/app"}}} + logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})) + return New(cfg, logger) +} + +func spekChildIssue(body string) github.Issue { + return github.Issue{Repo: "myorg/app", Number: 42, Title: "billing: retry failed charges", Body: body, URL: "https://github.com/myorg/app/issues/42"} +} + +func TestSpekPublishedContextInKickList(t *testing.T) { + s := newSpekContextScheduler(t) + issue := spekChildIssue("Retry failed charges.\n") + out, _ := s.formatIssueListWithPolicy([]github.Issue{issue}) + for _, want := range []string{ + "specified by Spek:", + "RFC https://github.com/myorg/app/issues/40", + "spek billing-retries, artifact spk_01, task t3", + "`Fixes #42`", + } { + if !strings.Contains(out, want) { + t.Errorf("kick list missing %q:\n%s", want, out) + } + } +} + +func TestSpekPublishedContextAbsentWithoutMarker(t *testing.T) { + s := newSpekContextScheduler(t) + for name, body := range map[string]string{ + "plain body": "an ordinary issue body", + "malformed": "", + } { + t.Run(name, func(t *testing.T) { + out, _ := s.formatIssueListWithPolicy([]github.Issue{spekChildIssue(body)}) + if strings.Contains(out, "specified by Spek") { + t.Errorf("unexpected Spek block:\n%s", out) + } + }) + } +} + +func TestSpekPublishedContextOmitsUntrustedRFC(t *testing.T) { + line := spekPublishedContext(spekChildIssue("")) + if line == "" || strings.Contains(line, "RFC ") || strings.Contains(line, "javascript") { + t.Errorf("untrusted rfc= must be dropped, block kept: %q", line) + } +} diff --git a/src/pkg/spekmarker/marker.go b/src/pkg/spekmarker/marker.go new file mode 100644 index 0000000000..55a5992df0 --- /dev/null +++ b/src/pkg/spekmarker/marker.go @@ -0,0 +1,140 @@ +// Package spekmarker reads the hidden publish marker Spek +// (hivecommons/spektacular) writes into every RFC and child issue it files +// into a work source (spektacular#109, #114): +// +// +// +// The marker is the hand-off contract between Spek and Hive. An item carrying +// it has already been specified, so Hive keeps it on the direct-fix path +// instead of routing it into a fresh `spec` run, skips the clarity score, and +// carries the RFC link and spek name into the kick so the agent builds against +// the spek's acceptance criteria. A marker WITHOUT `task=` names the RFC +// itself — the umbrella — which Hive treats like an `epic`: never claimed. +// +// Values are read the way pkg/upstreamwatch reads its `upstream-ref` marker: +// derived, never trusted. `rfc=` is echoed into a kick only when it parses as +// an http(s) URL; every other value is a bare token with no whitespace. +package spekmarker + +import ( + "net/url" + "strings" +) + +const ( + markerPrefix = "" +) + +// Targets Spek publishes into; every other `target=` is a malformed marker. +const ( + TargetGitHub = "github" + TargetJira = "jira" + TargetLinear = "linear" +) + +// Marker is one parsed `` marker. +type Marker struct { + // Name is the spek artifact's name (`name=`). + Name string + // ArtifactID is the spek artifact's stable id (`artifact_id=`), the key + // Spek re-publishes against. + ArtifactID string + // Target is the work source Spek published into (`target=`), lower-cased. + Target string + // Task is the plan task / member spec id of a child issue (`task=`); + // empty on the RFC itself. + Task string + // RFC is the RFC's URL (`rfc=`), kept only when it is a well-formed + // http(s) URL; empty otherwise. + RFC string +} + +// IsUmbrella reports whether the marker names the RFC itself rather than +// one of its children: it carries no `task=`. +func (m Marker) IsUmbrella() bool { return m.Task == "" } + +// IsChild reports whether the marker names a child work item (`task=` set). +func (m Marker) IsChild() bool { return m.Task != "" } + +// Parse returns the first well-formed publish marker in body. A marker +// missing `name=`, `artifact_id=` or a known `target=` is malformed and +// counts as no marker, so a body cannot half-opt into the hand-off. +func Parse(body string) (Marker, bool) { + rest := body + for { + start := strings.Index(rest, markerPrefix) + if start < 0 { + return Marker{}, false + } + rest = rest[start+len(markerPrefix):] + end := strings.Index(rest, markerSuffix) + if end < 0 { + return Marker{}, false + } + fields := rest[:end] + rest = rest[end+len(markerSuffix):] + // `` and the like share the prefix but are + // not the marker: the keyword must end at whitespace. + if fields != "" && !isSpace(fields[0]) { + continue + } + if m, ok := parseFields(fields); ok { + return m, true + } + } +} + +// Has reports whether body carries a well-formed publish marker. +func Has(body string) bool { + _, ok := Parse(body) + return ok +} + +// IsUmbrella reports whether body carries a publish marker with no `task=`: +// the item is the Spek-published RFC, the list of work rather than the work. +func IsUmbrella(body string) bool { + m, ok := Parse(body) + return ok && m.IsUmbrella() +} + +func parseFields(fields string) (Marker, bool) { + var m Marker + for _, tok := range strings.Fields(fields) { + key, val, ok := strings.Cut(tok, "=") + if !ok { + continue + } + val = strings.Trim(val, `"'`) + if val == "" { + continue + } + switch strings.ToLower(key) { + case "name": + m.Name = val + case "artifact_id": + m.ArtifactID = val + case "target": + m.Target = strings.ToLower(val) + case "task": + m.Task = val + case "rfc": + if u, err := url.Parse(val); err == nil && (u.Scheme == "https" || u.Scheme == "http") && u.Host != "" { + m.RFC = val + } + } + } + if m.Name == "" || m.ArtifactID == "" { + return Marker{}, false + } + switch m.Target { + case TargetGitHub, TargetJira, TargetLinear: + default: + return Marker{}, false + } + return m, true +} + +func isSpace(b byte) bool { + return b == ' ' || b == '\t' || b == '\n' || b == '\r' +} diff --git a/src/pkg/spekmarker/marker_test.go b/src/pkg/spekmarker/marker_test.go new file mode 100644 index 0000000000..96557377f6 --- /dev/null +++ b/src/pkg/spekmarker/marker_test.go @@ -0,0 +1,101 @@ +package spekmarker + +import "testing" + +const childMarker = `` + +func TestParseChildMarker(t *testing.T) { + body := "## What to build\n\nRetry failed charges.\n\n" + childMarker + "\n" + m, ok := Parse(body) + if !ok { + t.Fatalf("Parse: marker not found in %q", body) + } + want := Marker{Name: "billing-retries", ArtifactID: "spk_01HX", Target: TargetGitHub, Task: "t3", RFC: "https://github.com/acme/app/issues/41"} + if m != want { + t.Errorf("Parse = %+v, want %+v", m, want) + } + if !m.IsChild() || m.IsUmbrella() { + t.Errorf("child marker judged umbrella: %+v", m) + } + if !Has(body) || IsUmbrella(body) { + t.Errorf("Has=%v IsUmbrella=%v on a child body", Has(body), IsUmbrella(body)) + } +} + +func TestParseUmbrellaMarker(t *testing.T) { + body := "RFC body\n" + m, ok := Parse(body) + if !ok { + t.Fatal("Parse: umbrella marker not found") + } + if !m.IsUmbrella() || m.IsChild() { + t.Errorf("umbrella marker judged child: %+v", m) + } + if m.Target != TargetJira { + t.Errorf("target not lower-cased: %q", m.Target) + } + if !IsUmbrella(body) { + t.Error("IsUmbrella false on the RFC body") + } +} + +func TestParseTolerantSpelling(t *testing.T) { + body := "" + m, ok := Parse(body) + if !ok { + t.Fatal("Parse: tolerant spelling rejected") + } + if m.Name != "x" || m.ArtifactID != "a1" || m.Target != TargetLinear || m.Task != "s2" { + t.Errorf("Parse = %+v", m) + } +} + +func TestParseSkipsMalformedUsesLaterValid(t *testing.T) { + body := "\n" + childMarker + m, ok := Parse(body) + if !ok || m.Task != "t3" { + t.Errorf("Parse should skip the malformed marker and read the valid one: %+v ok=%v", m, ok) + } +} + +func TestParseRejectsInvalid(t *testing.T) { + for name, body := range map[string]string{ + "empty": "", + "no marker": "an ordinary issue body with only", + "unterminated": "", + "missing artifact": "", + "missing target": "", + "unknown target": "", + "keyword prefix": "", + "empty values": "", + } { + t.Run(name, func(t *testing.T) { + if m, ok := Parse(body); ok { + t.Errorf("Parse accepted %q: %+v", body, m) + } + if Has(body) || IsUmbrella(body) { + t.Errorf("Has/IsUmbrella true for %q", body) + } + }) + } +} + +func TestParseDropsUntrustedRFCURL(t *testing.T) { + for name, rfc := range map[string]string{ + "javascript": "javascript:alert(1)", + "relative": "/issues/41", + "no host": "https://", + "bare": "issues-41", + } { + t.Run(name, func(t *testing.T) { + m, ok := Parse("") + if !ok { + t.Fatal("marker with a bad rfc= must still parse") + } + if m.RFC != "" { + t.Errorf("untrusted rfc echoed: %q", m.RFC) + } + }) + } +} From 7284b56a473e101ce01fd43caf9b6227181cbcb5 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Sat, 7 Nov 2026 12:35:15 -0500 Subject: [PATCH 17/17] =?UTF-8?q?=F0=9F=90=9B=20fix(release):=20unblock=20?= =?UTF-8?q?tagged=20releases=20stalled=20by=20a=20release:none=20fragment?= =?UTF-8?q?=20(#17015)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit changelog.d/changed-12923-off-prompt-fixture.md (#12928, 2026-10-19) carried . compile-changelog.sh folds every fragment into ## Unreleased before derive-release-version.sh runs, so the marker was present on every decide job since v5.186.4 and each one skipped with 'explicit human override' — but the only thing that consumes a fragment is the release it was forbidding. ~1,600 commits shipped no tag. - delete the fragment (test-only change; should have been no-changelog) - check-changelog-fragment.sh / compile-changelog.sh reject release:none in a fragment; major|minor|patch bump markers remain legal - derive-release-version.sh emits a ::warning:: when a hand-placed none marker freezes releases, and fails the decide job once the freeze has outlived RELEASE_FREEZE_MAX_DAYS (default 7) since the newest tag - derive-release-version.sh: the Added-heading inference used grep -q under pipefail; on the 558 KiB compiled backlog printf took SIGPIPE and the pipeline failed => silent patch instead of minor. grep now reads to EOF. - docs: changelog.d/README.md, src/docs/releases.md - tests: test-compile-changelog.sh 5b/5c, test-derive-release-version.sh 3b (large body) and 5b (freeze age) Signed-off-by: Andrew Anderson Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .github/scripts/integration-governance.cjs | 2 +- changelog.d/README.md | 9 +++- .../changed-12923-off-prompt-fixture.md | 2 - .../fixed-release-none-fragment-stall.md | 1 + src/docs/releases.md | 15 ++++++- src/scripts/check-changelog-fragment.sh | 11 ++++- src/scripts/compile-changelog.sh | 19 ++++++-- src/scripts/derive-release-version.sh | 26 ++++++++++- src/scripts/test-compile-changelog.sh | 34 ++++++++++++++ src/scripts/test-derive-release-version.sh | 45 +++++++++++++++++++ 10 files changed, 154 insertions(+), 10 deletions(-) delete mode 100644 changelog.d/changed-12923-off-prompt-fixture.md create mode 100644 changelog.d/fixed-release-none-fragment-stall.md diff --git a/.github/scripts/integration-governance.cjs b/.github/scripts/integration-governance.cjs index 7344f0a980..f117381e0d 100644 --- a/.github/scripts/integration-governance.cjs +++ b/.github/scripts/integration-governance.cjs @@ -9,7 +9,7 @@ const MAX_COMMITS = 1000; const MAX_AUTHOR_BATCH = 50; // This literal is inside the enrolled .github closure. A caller-supplied // preparedBy.sourceSHA must never choose a new executable verifier tree. -const TRUSTED_SCRIPTS_TREE = '68aa6a3ea98f2ab257c8d492b39b8a661092786d'; +const TRUSTED_SCRIPTS_TREE = 'af7a0249a3df8e6197ff49031d4cb6c4e00581c6'; const SOURCE = 'https://github.com/kubernetes-sigs/kubebuilder-release-tools/tree/012269a88fa4c034a0acf1ba84c26b195c0dbab4'; const sha = value => crypto.createHash('sha256').update(value).digest('hex'); const git = (args, cwd = process.cwd()) => diff --git a/changelog.d/README.md b/changelog.d/README.md index 81f5c42cd4..01e1f28adf 100644 --- a/changelog.d/README.md +++ b/changelog.d/README.md @@ -36,10 +36,17 @@ deleting the fragments in the same release commit. need no fragment only if non-user-visible (routine refactors, dependency churn, or test-adjacent code like `testdata/` or non-`_test.go` test helpers) — put a `no-changelog` label on such PRs. -- The `` escape hatch +- The `` bump escape hatch (`src/docs/releases.md`) may ride in a fragment as its first line; it is compiled into `Unreleased` verbatim and honoured by `derive-release-version.sh` exactly as a hand-placed marker is. + **`` is rejected in fragments** (by + `check-changelog-fragment.sh` at PR time and `compile-changelog.sh` at + release time): a fragment persists until the release that consumes it, and + `none` is what forbids that release, so a single such fragment freezes + every later merge until a human deletes it — [#12928](https://github.com/hivecommons/hive/pull/12928) + stalled releases for three weeks this way. A non-user-visible change wants + no fragment and the `no-changelog` label instead. Example — a complete fragment, `changelog.d/fixed-1234-relay-timeout.md`: diff --git a/changelog.d/changed-12923-off-prompt-fixture.md b/changelog.d/changed-12923-off-prompt-fixture.md deleted file mode 100644 index 5d25addc33..0000000000 --- a/changelog.d/changed-12923-off-prompt-fixture.md +++ /dev/null @@ -1,2 +0,0 @@ - -- Keep the busy-visibility regression cases aligned by reusing the off-prompt pane fixture when checking the recovery threshold (#12923). diff --git a/changelog.d/fixed-release-none-fragment-stall.md b/changelog.d/fixed-release-none-fragment-stall.md new file mode 100644 index 0000000000..a2e8df80c2 --- /dev/null +++ b/changelog.d/fixed-release-none-fragment-stall.md @@ -0,0 +1 @@ +- Tagged releases resume: a `release: none` HTML-comment marker that rode in a `changelog.d/` fragment (#12928, merged 2026-10-19) was compiled into `## Unreleased` on every merge and made the release workflow's decide job skip every commit since v5.186.4 as an "explicit human override" — the fragment could only be consumed by the very release it vetoed. The fragment is removed; `check-changelog-fragment.sh` and `compile-changelog.sh` now reject a `none` marker in a fragment (bump markers stay allowed); and a hand-placed `none` freeze in CHANGELOG.md is surfaced as a workflow warning annotation and fails the decide job once it has outlived `RELEASE_FREEZE_MAX_DAYS` (default 7) instead of silently skipping forever. Also fixed on the way out: with the resulting 1,369-fragment backlog the bump inference's `grep -q` died of SIGPIPE under `pipefail` and would have labelled the catch-up release a patch instead of the minor its `### Added` section calls for. diff --git a/src/docs/releases.md b/src/docs/releases.md index 4b6d559e72..168d909e2e 100644 --- a/src/docs/releases.md +++ b/src/docs/releases.md @@ -134,7 +134,20 @@ decision: ``` suppresses a release for this merge even if `Unreleased` has content (the -entries stay queued for the next release that does fire). Or force a specific +entries stay queued for the next release that does fire). Note the +consequence under the fragment model: nothing but a release empties +`Unreleased`, so a `none` marker placed there (by hand, in `CHANGELOG.md`) +is a **release freeze** that lasts until a human removes it, not a one-merge +skip — the decide job emits a `::warning::` annotation on every run while it +is in force, and fails outright once the newest tag is older than +`RELEASE_FREEZE_MAX_DAYS` (default 7; `0` disables) so a forgotten freeze +turns red instead of looking like a healthy no-op. For the same reason the marker is **rejected in a +`changelog.d/` fragment** by `check-changelog-fragment.sh` and +`compile-changelog.sh`: a fragment lives until the release that consumes it, +so a `none` fragment can never be consumed and vetoes every later merge +(#12928 stalled v5 releases from 2026-10-19 to 2026-11-07 this way). A +change that should not be mentioned wants no fragment and the +`no-changelog` label. Or force a specific bump regardless of which headers are present: ```markdown diff --git a/src/scripts/check-changelog-fragment.sh b/src/scripts/check-changelog-fragment.sh index 42715385c2..f7843a3abd 100755 --- a/src/scripts/check-changelog-fragment.sh +++ b/src/scripts/check-changelog-fragment.sh @@ -71,10 +71,19 @@ while IFS= read -r f; do case "$first_line" in '- '*|'' marker) — it IS the entry; the compiler owns the ### headings" + echo "::error file=$f::a fragment must start with a '- ' entry bullet (or a '' marker) — it IS the entry; the compiler owns the ### headings" bad=1 ;; esac + # A fragment persists until the next release consumes it, and a + # `` marker is exactly what prevents that release — so + # one in a fragment vetoes every later merge, forever (#12928 stalled the + # release train for three weeks this way). Test-only/no-op changes want the + # `no-changelog` label and no fragment, not a none marker. + if grep -qE '' "$f"; then + echo "::error file=$f::'' is not allowed in a fragment: fragments accumulate until a release is cut, so this marker would block every release until a human deletes it. Delete the fragment (and label the PR no-changelog) if the change is not user-visible; only major|minor|patch markers may ride in a fragment." + bad=1 + fi done < "$work_dir"/fragments.txt if [ "$bad" -ne 0 ]; then exit 1 diff --git a/src/scripts/compile-changelog.sh b/src/scripts/compile-changelog.sh index 2efc45f7b5..da089b4a46 100755 --- a/src/scripts/compile-changelog.sh +++ b/src/scripts/compile-changelog.sh @@ -36,8 +36,11 @@ # CHANGELOG.md edits therefore keep working alongside fragments, which is # what the 7-day transition window relies on. # - A fragment's first non-blank line must start with `- ` (an entry bullet) -# or be a `` marker, so the escape hatch -# derive-release-version.sh honours can also ride in a fragment. +# or be a `` marker, so the bump escape +# hatch derive-release-version.sh honours can also ride in a fragment. +# `` is REJECTED in fragments: a fragment persists +# until the release that consumes it, and `none` is what prevents that +# release, so one such fragment silently freezes every later merge. # - Each fragment is deleted after a successful compile, so `git add -A # changelog.d` in the release commit records the consumption. # @@ -117,7 +120,17 @@ for f in "${fragments[@]}"; do echo "::error::${f}: fragment is empty" >&2 bad=1 elif ! [[ "$first_line" == "- "* || "$first_line" == "' marker); it is the entry itself, not a section with its own headings" >&2 + echo "::error::${f}: a fragment must start with a '- ' entry bullet (or a '' marker); it is the entry itself, not a section with its own headings" >&2 + bad=1 + fi + # `none` is self-deadlocking in a fragment: the fragment lives until the + # release that consumes it, and the marker is what forbids that release. + # #12928 landed one on 2026-10-19 and every decide job for the next three + # weeks (~1,600 commits) skipped on "explicit human override" while nobody + # had asked for a freeze. Fail loudly here so the stall shows up as a red + # decide job on the very next merge instead of a quiet skip. + if grep -qE '' "$f"; then + echo "::error::${f}: '' is not allowed in a fragment — it would veto every release until a human deletes it (see changelog.d/README.md). Delete the fragment instead; only major|minor|patch markers may ride in a fragment." >&2 bad=1 fi done diff --git a/src/scripts/derive-release-version.sh b/src/scripts/derive-release-version.sh index 17079e0e58..67d80047b2 100755 --- a/src/scripts/derive-release-version.sh +++ b/src/scripts/derive-release-version.sh @@ -115,7 +115,26 @@ if [[ ${#markers[@]} -eq 1 ]]; then fi if [[ "$override" == "none" ]]; then + # A ::warning:: rather than a plain echo: this skip is a human-placed + # freeze, and a freeze nobody remembers placing looks exactly like a + # healthy no-op run (#12928 held releases for three weeks that way). Once + # the freeze has outlived RELEASE_FREEZE_MAX_DAYS (default 7) since the + # newest tag, fail the decide job outright so the stall turns red. + echo "::warning::CHANGELOG.md Unreleased section carries ; releases are frozen until that marker is removed." echo "Unreleased section carries ; skipping this merge by explicit human override." + max_days="${RELEASE_FREEZE_MAX_DAYS:-7}" + newest_tag="$(git tag -l 'v[0-9]*.[0-9]*.[0-9]*' 2>/dev/null | sort -V | tail -1 || true)" + if [[ -n "$newest_tag" && "$max_days" -gt 0 ]]; then + tag_epoch="$(git log -1 --format=%ct "${newest_tag}^{commit}" 2>/dev/null || echo 0)" + if [[ "$tag_epoch" -gt 0 ]]; then + age_days=$(( ( $(date +%s) - tag_epoch ) / 86400 )) + if [[ "$age_days" -gt "$max_days" ]]; then + echo "::error:: has suppressed releases for ${age_days} days (newest tag ${newest_tag}; limit RELEASE_FREEZE_MAX_DAYS=${max_days}). Remove the marker from CHANGELOG.md's Unreleased section, or raise RELEASE_FREEZE_MAX_DAYS if the freeze is deliberate." >&2 + emit release false + exit 1 + fi + fi + fi emit release false exit 0 fi @@ -151,7 +170,12 @@ else # declared with the explicit `` marker. Crossing a # release line stays a human decision, which is what it already is # everywhere else in this repo. - if printf '%s\n' "$unreleased" | grep -qE '^### Added[[:space:]]*$'; then + # No `grep -q` here: under `set -o pipefail` a -q grep exits on the first + # match and the printf feeding it dies of SIGPIPE once the body outgrows + # the pipe buffer (~64 KiB), turning a found `### Added` into pipeline + # failure => a silent patch. Seen live on the 1,369-fragment backlog + # compiled after #12928. Let grep read to EOF instead. + if printf '%s\n' "$unreleased" | grep -E '^### Added[[:space:]]*$' >/dev/null; then bump=minor else bump=patch diff --git a/src/scripts/test-compile-changelog.sh b/src/scripts/test-compile-changelog.sh index c35d2131e0..80ac652ffb 100755 --- a/src/scripts/test-compile-changelog.sh +++ b/src/scripts/test-compile-changelog.sh @@ -187,6 +187,40 @@ else note_ok "a fragment that is not a '- ' entry bullet is rejected" fi +# --------------------------------------------------------------------------- +# Case 5b: `` in a fragment is rejected (#12928). The +# fragment survives until a release consumes it, and the marker forbids that +# release — so one fragment would freeze every later merge. A bump marker +# (`patch`) in a fragment stays legal and still compiles verbatim. +# --------------------------------------------------------------------------- +dir=$(new_case 5b) +printf -- '\n- a test-only tweak that should have been a no-changelog label.\n' > "$dir/changelog.d/changed-11-none.md" +printf -- '- a real fix ([#12](https://example.invalid/12)).\n' > "$dir/changelog.d/fixed-12-real.md" +if ( cd "$dir" && bash "$compile" CHANGELOG.md changelog.d ) > "$tmp/case5b.out" 2>&1; then + note_fail "a fragment carrying '' must be rejected" +else + if grep -q 'release: none' "$tmp/case5b.out" && [[ -f "$dir/changelog.d/fixed-12-real.md" ]]; then + note_ok "a fragment carrying '' is rejected loudly, with every fragment left on disk" + else + note_fail "rejection of a none-marker fragment must name the marker and leave fragments untouched: $(cat "$tmp/case5b.out")" + fi +fi + +dir=$(new_case 5c) +git init -q "$dir" +git -C "$dir" config user.email test@example.com +git -C "$dir" config user.name "Test" +git -C "$dir" -c commit.gpgsign=false commit -q --allow-empty -m init +git -C "$dir" tag v1.0.0 +printf -- '\n- an added thing that must only bump patch.\n' > "$dir/changelog.d/added-13-patch.md" +if ( cd "$dir" && bash "$compile" CHANGELOG.md changelog.d ) > "$tmp/case5c.out" 2>&1 \ + && grep -q '' "$dir/CHANGELOG.md" \ + && [[ "$(cd "$dir" && GITHUB_OUTPUT="" bash "$derive" CHANGELOG.md | grep -E '^(release|bump|version)=' | tr '\n' ' ')" == "release=true version=1.0.1 bump=patch " ]]; then + note_ok "a bump marker in a fragment still compiles and still forces the bump" +else + note_fail "a '' fragment must remain legal: $(cat "$tmp/case5c.out")" +fi + # --------------------------------------------------------------------------- # Case 6: fenced prose that mentions `## Unreleased` must not be treated as # the target section. This pins the #5918/#5939 bug class for both the diff --git a/src/scripts/test-derive-release-version.sh b/src/scripts/test-derive-release-version.sh index 65b6a4a36c..144e755249 100755 --- a/src/scripts/test-derive-release-version.sh +++ b/src/scripts/test-derive-release-version.sh @@ -74,6 +74,22 @@ else note_fail "expected release=true bump=minor version=1.3.0, got: $out" fi +# --------------------------------------------------------------------------- +# Case 3b: a very large Unreleased body (well past the ~64 KiB pipe buffer) +# with `### Added` near the top must still infer minor. A `grep -q` in the +# inference pipeline exits on first match, the feeding printf takes SIGPIPE, +# and pipefail turned that into "no Added => patch" on the 1,369-fragment +# backlog compiled after #12928. +# --------------------------------------------------------------------------- +big_body=$'## Unreleased\n\n### Added\n\n- first feature\n\n### Fixed\n\n' +big_body+="$(yes '- a long fixed entry line that pads the section well past the pipe buffer size so the test is meaningful.' | head -n 4000)"$'\n' +out=$(run "$big_body" v1.2.3) +if [[ "$(get "$out" release)" == "true" && "$(get "$out" version)" == "1.3.0" ]]; then + note_ok "### Added in a >64KiB Unreleased body still infers minor (no SIGPIPE/pipefail regression)" +else + note_fail "expected 1.3.0 from a large Added-carrying section, got: $(printf '%s' "$out" | tail -3)" +fi + # --------------------------------------------------------------------------- # Case 4: ### Security present is release-worthy but does not cut a new # release line by itself, so Added still controls the minor bump. @@ -96,6 +112,35 @@ else note_fail "expected release=false with release:none marker, got: $out" fi +# --------------------------------------------------------------------------- +# Case 5b: a release:none freeze that has outlived RELEASE_FREEZE_MAX_DAYS +# since the newest tag fails the decide job loudly (#12928: three silent +# weeks). Same marker on a fresh tag (case 5) stays a quiet release=false; +# RELEASE_FREEZE_MAX_DAYS=0 disables the age check. +# --------------------------------------------------------------------------- +old_epoch=$(( $(date +%s) - 30 * 86400 )) +GIT_AUTHOR_DATE="@${old_epoch} +0000" GIT_COMMITTER_DATE="@${old_epoch} +0000" \ + git -C "$repo" commit -q --allow-empty -m "old release point" +git -C "$repo" tag -d $(git -C "$repo" tag -l 'v*') >/dev/null 2>&1 || true +git -C "$repo" tag -f v2.0.0 >/dev/null 2>&1 +printf '%s' $'## Unreleased\n\n\n\n### Added\n\n- frozen\n' > "$repo/CHANGELOG.md" +if out=$( cd "$repo" && GITHUB_OUTPUT="" bash "$derive" CHANGELOG.md 2>&1 ); then + note_fail "a 30-day release:none freeze must fail the decide job, got exit 0: $out" +elif printf '%s' "$out" | grep -q '::error::.*release: none.*30 days'; then + note_ok "release:none freeze older than RELEASE_FREEZE_MAX_DAYS fails loudly with its age" +else + note_fail "expected a ::error:: naming the freeze age, got: $out" +fi +if out=$( cd "$repo" && GITHUB_OUTPUT="" RELEASE_FREEZE_MAX_DAYS=0 bash "$derive" CHANGELOG.md 2>&1 ) \ + && [[ "$(get "$out" release)" == "false" ]]; then + note_ok "RELEASE_FREEZE_MAX_DAYS=0 disables the freeze-age check" +else + note_fail "expected a quiet release=false with the age check disabled, got: $out" +fi +# Later cases tag the current commit; leave it dated now again. +git -C "$repo" commit -q --allow-empty -m "back to now" +git -C "$repo" tag -d $(git -C "$repo" tag -l 'v*') >/dev/null 2>&1 || true + # --------------------------------------------------------------------------- # Case 6: escape hatch forces a bump type regardless of headers present. # ---------------------------------------------------------------------------