diff --git a/.github/scripts/integration-governance.cjs b/.github/scripts/integration-governance.cjs index 0c160eebe9..3f61d7362a 100644 --- a/.github/scripts/integration-governance.cjs +++ b/.github/scripts/integration-governance.cjs @@ -9,7 +9,7 @@ const MAX_COMMITS = 1000; const MAX_AUTHOR_BATCH = 50; // This literal is inside the enrolled .github closure. A caller-supplied // preparedBy.sourceSHA must never choose a new executable verifier tree. -const TRUSTED_SCRIPTS_TREE = 'b3108c5a9ebe25702b15654d9f37a7236f081830'; +const TRUSTED_SCRIPTS_TREE = '267924f87d77a3fa335accce1b0c74d1a5cfdab8'; const SOURCE = 'https://github.com/kubernetes-sigs/kubebuilder-release-tools/tree/012269a88fa4c034a0acf1ba84c26b195c0dbab4'; const sha = value => crypto.createHash('sha256').update(value).digest('hex'); const git = (args, cwd = process.cwd()) => diff --git a/bin/hive-open-pr.sh b/bin/hive-open-pr.sh index f72624b307..d26774a7cb 100644 --- a/bin/hive-open-pr.sh +++ b/bin/hive-open-pr.sh @@ -86,6 +86,33 @@ commit_has_trailer() { grep -Fqx "$trailer" } +# warn_if_head_not_on_remote tells the agent, at request time, when the branch +# it is asking a PR for is not known to be on the remote (#16969). The watcher +# cannot open a PR for a branch GitHub does not have; it retries for 24h, so +# a push that never landed surfaces only as a fleet self-report hours later. +# Evidence is the checkout's own remote-tracking ref (written by a successful +# `git push`, no network) and, failing that, one `git ls-remote` of the exact +# ref. It never refuses: a hive-push-branch relay push may still be in flight, +# and the queued request lands on retry once the branch arrives. +warn_if_head_not_on_remote() { + head="$1" + [ -n "${HIVE_OPEN_PR_SKIP_REMOTE_CHECK:-}" ] && return 0 + git rev-parse --is-inside-work-tree >/dev/null 2>&1 || return 0 + remote="$(git config --get "branch.${head}.remote" 2>/dev/null || true)" + [ -n "$remote" ] || remote=origin + git remote get-url "$remote" >/dev/null 2>&1 || return 0 + if git rev-parse --verify --quiet "refs/remotes/${remote}/${head}" >/dev/null 2>&1; then + return 0 + fi + rc=0 + GIT_TERMINAL_PROMPT=0 git ls-remote --exit-code --heads "$remote" "refs/heads/${head}" >/dev/null 2>&1 || rc=$? + # 2 is ls-remote's definitive "no such ref"; anything else (auth, network) + # is not evidence either way, so stay quiet rather than cry wolf. + [ "$rc" -eq 2 ] || return 0 + echo "hive-open-pr: WARN: branch '${head}' is NOT on remote '${remote}' ($(git remote get-url "$remote" 2>/dev/null)) — the hive cannot open a PR for a branch GitHub does not have." >&2 + echo "hive-open-pr: WARN: push it first (git push -u ${remote} ${head}, or hive-push-branch --repo ${REPO:-}) and check the push's own output; the request below stays queued and succeeds on retry once the branch arrives." >&2 +} + check_issue_author_coauthors() { repo="$1" body="$2" @@ -218,6 +245,7 @@ if [ -z "$REPO" ] || [ -z "$HEAD" ] || [ -z "$TITLE" ]; then fi check_issue_author_coauthors "$REPO" "$BODY" +warn_if_head_not_on_remote "$HEAD" # Identify the requesting agent. Prefer the UID map (the watcher re-derives the # owner from the FILE's UID anyway, so this is informational + a nicer log line); diff --git a/bin/test_hive_open_pr.sh b/bin/test_hive_open_pr.sh index 8eeb4409fa..1ac8c78677 100755 --- a/bin/test_hive_open_pr.sh +++ b/bin/test_hive_open_pr.sh @@ -144,7 +144,7 @@ check "--body-file body reaches the request" \ reset_reqs check_exit "--body-file - reads stdin" 0 \ - bash -c 'printf "stdin body" | "$@"' _ env HIVE_AGENT=a3 HIVE_OPEN_PR_REQ_DIR="$REQ_DIR" \ + bash -c 'printf "stdin body" | "$@"' _ env HIVE_AGENT=a3 HIVE_OPEN_PR_REQ_DIR="$REQ_DIR" HIVE_OPEN_PR_SKIP_REMOTE_CHECK=1 \ bash "$MODIFIED_SCRIPT" --repo o/r --title T --head h --body-file - REQ="$(find_req a3)" check "stdin body reaches the request" "stdin body" "$(json_field "$REQ" body)" @@ -373,6 +373,36 @@ else fi reset_reqs +# --- Section 10: head-on-remote preflight (#16969) --- +echo "" +echo "--- head-on-remote preflight ---" + +BARE="$TMPDIR_ROOT/remote.git" +git init --quiet --bare "$BARE" +PUSHED_REPO="$TMPDIR_ROOT/pushed" +git clone --quiet "$BARE" "$PUSHED_REPO" 2>/dev/null +( cd "$PUSHED_REPO" && git checkout --quiet -b pushed/branch 2>/dev/null && touch f && git add f && + git -c user.name=t -c user.email=t@t -c commit.gpgsign=false commit --quiet -m seed && + git push --quiet -u origin pushed/branch 2>/dev/null && + git checkout --quiet -b local/only && touch g && git add g && + git -c user.name=t -c user.email=t@t -c commit.gpgsign=false commit --quiet -m local ) + +STDERR_OUT="$( (cd "$PUSHED_REPO" && env HIVE_AGENT=a10 HIVE_OPEN_PR_REQ_DIR="$REQ_DIR" bash "$MODIFIED_SCRIPT" --repo o/r --head pushed/branch --title T --body B) 2>&1 >/dev/null )" +check "pushed branch produces no warning" "" "$STDERR_OUT" +reset_reqs + +STDERR_OUT="$( (cd "$PUSHED_REPO" && env HIVE_AGENT=a10 HIVE_OPEN_PR_REQ_DIR="$REQ_DIR" bash "$MODIFIED_SCRIPT" --repo o/r --head local/only --title T --body B) 2>&1 >/dev/null )" +case "$STDERR_OUT" in + *"branch 'local/only' is NOT on remote 'origin'"*"hive-push-branch --repo o/r"*) check "unpushed branch warns and names branch, remote and the push step" ok ok;; + *) check "unpushed branch warns and names branch, remote and the push step" "warning naming local/only, origin and hive-push-branch" "$STDERR_OUT";; +esac +check_exit "unpushed branch still queues the request (watcher retries once it lands)" 0 test -n "$(find_req a10)" +reset_reqs + +STDERR_OUT="$( (cd "$PUSHED_REPO" && env HIVE_AGENT=a10 HIVE_OPEN_PR_REQ_DIR="$REQ_DIR" HIVE_OPEN_PR_SKIP_REMOTE_CHECK=1 bash "$MODIFIED_SCRIPT" --repo o/r --head local/only --title T --body B) 2>&1 >/dev/null )" +check "HIVE_OPEN_PR_SKIP_REMOTE_CHECK silences the preflight" "" "$STDERR_OUT" +reset_reqs + # --- Summary --- echo "" echo "=== Results: $PASS passed, $FAIL failed ===" diff --git a/changelog.d/README.md b/changelog.d/README.md index 81f5c42cd4..01e1f28adf 100644 --- a/changelog.d/README.md +++ b/changelog.d/README.md @@ -36,10 +36,17 @@ deleting the fragments in the same release commit. need no fragment only if non-user-visible (routine refactors, dependency churn, or test-adjacent code like `testdata/` or non-`_test.go` test helpers) — put a `no-changelog` label on such PRs. -- The `` escape hatch +- The `` bump escape hatch (`src/docs/releases.md`) may ride in a fragment as its first line; it is compiled into `Unreleased` verbatim and honoured by `derive-release-version.sh` exactly as a hand-placed marker is. + **`` is rejected in fragments** (by + `check-changelog-fragment.sh` at PR time and `compile-changelog.sh` at + release time): a fragment persists until the release that consumes it, and + `none` is what forbids that release, so a single such fragment freezes + every later merge until a human deletes it — [#12928](https://github.com/hivecommons/hive/pull/12928) + stalled releases for three weeks this way. A non-user-visible change wants + no fragment and the `no-changelog` label instead. Example — a complete fragment, `changelog.d/fixed-1234-relay-timeout.md`: diff --git a/changelog.d/added-crux-caller-principal.md b/changelog.d/added-crux-caller-principal.md new file mode 100644 index 0000000000..6956e0dbe3 --- /dev/null +++ b/changelog.d/added-crux-caller-principal.md @@ -0,0 +1 @@ +- crux: `serve` attaches a caller principal to every authenticated request (`serve.CallerFrom`): the control token is the operator (named by `--operator-name`/`CRUX_OPERATOR_NAME`), and lane project credentials are accepted as project principals on work-services routes only — agent RPCs still refuse them. Groundwork for the crux work services (hivecommons/crux#114 S0). diff --git a/changelog.d/added-crux-checkout-service.md b/changelog.d/added-crux-checkout-service.md new file mode 100644 index 0000000000..a04c81ce9a --- /dev/null +++ b/changelog.d/added-crux-checkout-service.md @@ -0,0 +1 @@ +- crux: pinned checkout service (hivecommons/crux#114 S1): `POST/GET/DELETE /v1/checkouts` on `crux serve` (enabled by `--checkout-allow`/`CRUX_CHECKOUT_ALLOWLIST`) and `crux checkout add|rm|list` provision a git worktree pinned at a full 40-hex SHA from an allowlisted repository, owned by the caller principal, with a recorded TTL and a post-hoc size cap. Records live under `/checkouts`; design notes in `docs/design/crux-work-services.md`. diff --git a/changelog.d/added-crux-spek-publish-proxy.md b/changelog.d/added-crux-spek-publish-proxy.md new file mode 100644 index 0000000000..0d350912a5 --- /dev/null +++ b/changelog.d/added-crux-spek-publish-proxy.md @@ -0,0 +1 @@ +- crux: the proxy lets an in-hive Spek publish its RFC and children — GitHub `POST …/issues/{n}/sub_issues` and `POST …/issues/{n}/dependencies/blocked_by` and Linear `issueRelationCreate`/`projectCreate`/`projectUpdate` are allowed at ISSUES_ONLY; a new `crux/spekmarker` package parses the `` marker and `issueshape.SpekPublished` exposes it as the "already specified" clarity-gate signal (spektacular#114). diff --git a/changelog.d/added-pin-only-human-merge-relay.md b/changelog.d/added-pin-only-human-merge-relay.md new file mode 100644 index 0000000000..5f3a2181d4 --- /dev/null +++ b/changelog.d/added-pin-only-human-merge-relay.md @@ -0,0 +1 @@ +- The `hive-merge` relay now honours the `auto_merge.human_merge_pin_only` exception, with the same checks as the automerge sweep plus an exact requested-head match; anything short of a proven pin-only change by a trusted author stays held (#16265). diff --git a/changelog.d/added-pin-only-human-merge-sweep.md b/changelog.d/added-pin-only-human-merge-sweep.md index 6f68c92ee2..53de8298b1 100644 --- a/changelog.d/added-pin-only-human-merge-sweep.md +++ b/changelog.d/added-pin-only-human-merge-sweep.md @@ -1 +1 @@ -- Enforce the `auto_merge.human_merge_pin_only` exception in the automerge sweep lanes and `/hive approve` (#16266): a PR by a listed trusted author whose complete diff is pin replacements on guarded paths is no longer held for a person. Any other changed file, including an unguarded changelog or doc, keeps the hold (#16326). Repos without an enabled entry are unchanged, and the `hive-merge` relay does not apply the exception yet (#16265). +- Enforce the `auto_merge.human_merge_pin_only` exception in the automerge sweep lanes and `/hive approve` (#16266): a PR by a listed trusted author whose complete diff is pin replacements on guarded paths is no longer held for a person. Any other changed file, including an unguarded changelog or doc, keeps the hold (#16326). Repos without an enabled entry are unchanged; the `hive-merge` relay enforces the same exception as of #16265. diff --git a/changelog.d/added-spek-publish-marker.md b/changelog.d/added-spek-publish-marker.md new file mode 100644 index 0000000000..c311fb9f4b --- /dev/null +++ b/changelog.d/added-spek-publish-marker.md @@ -0,0 +1 @@ +- Hive honours the Spek publish marker `` (spektacular#109/#114): a Spek-published child stays on the direct-fix path under `runs.triage` (never a new `spec` run), skips the clarity score, and its kick line carries the RFC link and spek identity with the `Fixes #` instruction; the RFC itself (marker without `task=`) is treated like `epic` for `governor.claims.never_claim_labels` and `IsTrackerIssue`, so Hive never claims it even in a repo without that label. Parser in `pkg/spekmarker`; documented in `docs/spektacular.md` → Publish hand-off. diff --git a/changelog.d/changed-12923-off-prompt-fixture.md b/changelog.d/changed-12923-off-prompt-fixture.md deleted file mode 100644 index 5d25addc33..0000000000 --- a/changelog.d/changed-12923-off-prompt-fixture.md +++ /dev/null @@ -1,2 +0,0 @@ - -- Keep the busy-visibility regression cases aligned by reusing the off-prompt pane fixture when checking the recovery threshold (#12923). diff --git a/changelog.d/fixed-16717-contributor-merge-eligibility.md b/changelog.d/fixed-16717-contributor-merge-eligibility.md new file mode 100644 index 0000000000..e1ca2d525b --- /dev/null +++ b/changelog.d/fixed-16717-contributor-merge-eligibility.md @@ -0,0 +1 @@ +- Contributor PRs with hive attribution now pass the merge relay's trust gate before being listed merge-eligible; trust refusals appear separately with their reason and maintainer unblock options ([#16717](https://github.com/hivecommons/hive/issues/16717)). diff --git a/changelog.d/fixed-17006.md b/changelog.d/fixed-17006.md new file mode 100644 index 0000000000..0930d8367c --- /dev/null +++ b/changelog.d/fixed-17006.md @@ -0,0 +1 @@ +- Agent panes that show hive's own login-pattern source or grep output (for example `file.go:NN: "Please run /login"`) are no longer misread as a login screen, so reviewers that read loginscan/watchdog code stop raising a false "needs login" banner; the watchdog also only checks the bottom of the pane for login prompts ([#17006](https://github.com/hivecommons/hive/issues/17006)). diff --git a/changelog.d/fixed-admitted-head-stale-red-recovery.md b/changelog.d/fixed-admitted-head-stale-red-recovery.md new file mode 100644 index 0000000000..44ed0fddb6 --- /dev/null +++ b/changelog.d/fixed-admitted-head-stale-red-recovery.md @@ -0,0 +1 @@ +- The merge sweep now re-runs cancelled or failed workflow jobs (or refreshes the branch) on an operator-approved contributor PR whose only blocker is a red check that is green on the base, instead of standing down from adoption and nudging the fork author to push a new head for a shared-CI flake they cannot re-run; cancelled runs now count as re-runnable in the adopted-PR recovery too. diff --git a/changelog.d/fixed-crux-proxy-canary-body-read.md b/changelog.d/fixed-crux-proxy-canary-body-read.md new file mode 100644 index 0000000000..1fbe4a7611 --- /dev/null +++ b/changelog.d/fixed-crux-proxy-canary-body-read.md @@ -0,0 +1 @@ +- The crux proxy's canary scan of GitHub write bodies now refuses a request whose body could not be fully read, matching the Linear gate and the oversized-body path. Before, a read error consumed the body and still let the request through, so it reached GitHub empty and unscanned. diff --git a/changelog.d/fixed-ioscan-classifier-settings-restart.md b/changelog.d/fixed-ioscan-classifier-settings-restart.md new file mode 100644 index 0000000000..904afb5037 --- /dev/null +++ b/changelog.d/fixed-ioscan-classifier-settings-restart.md @@ -0,0 +1 @@ +- Input Defense settings saved from the dashboard now actually unblock kicks at ACMM L5+ ([#16973](https://github.com/hivecommons/hive/issues/16973)). Saving ioscan fail mode "open" was stored as unset, which at L5+ falls back to the ACMM pack's fail-closed default, so the setting snapped back to "closed" and kicks carrying untrusted text stayed withheld. "open" is now stored as given, and every ioscan save in Settings → Security → Input Defense rebuilds the semantic classifier from the new backend and fail mode immediately, clearing the blocked-kick alert when it no longer applies. A runtime ACMM level change (for example L4 → L5 in the dashboard) rebuilds the classifier the same way, so the level's classifier requirement and fail mode apply without a restart. The alert and the Classifier backend help text no longer tell operators to restart for changes the dashboard now applies live, and the alert names the `ioscan.fail_mode` key the Security tab actually exposes. diff --git a/changelog.d/fixed-login-required-report-backend.md b/changelog.d/fixed-login-required-report-backend.md new file mode 100644 index 0000000000..afc6ba099c --- /dev/null +++ b/changelog.d/fixed-login-required-report-backend.md @@ -0,0 +1 @@ +- The `agent_login_required` selfreport now names the backend that lost its login: the login scanner's `login required detected` record carries `backend`, `login_hint` (the CLI's login command) and `credential_valid`, so the operator knows which provider to re-authenticate on which agent instead of being told only the agent name; `codex` gets a login hint (`codex login`) ([#16953](https://github.com/hivecommons/hive/issues/16953)). diff --git a/changelog.d/fixed-pr-request-head-unpushed-report.md b/changelog.d/fixed-pr-request-head-unpushed-report.md new file mode 100644 index 0000000000..1a08900aaa --- /dev/null +++ b/changelog.d/fixed-pr-request-head-unpushed-report.md @@ -0,0 +1 @@ +- The pr-request watcher now reports an unpushed head branch once per request instead of once per retry, states the exact ref and repository it checked and what the push relay recorded for that branch (queued, pushed, failed, or no record) rather than asserting a push-authentication failure, and logs when the request later succeeds on retry; `hive-open-pr` warns at request time when the head is not on the remote, naming the branch, remote and push step; fleet-report fingerprints no longer treat a possessive apostrophe as an opening quote, which truncated this report's scope to "the agentQ" ([#16969](https://github.com/hivecommons/hive/issues/16969)). diff --git a/changelog.d/fixed-proxy-tunnel-destination-policy.md b/changelog.d/fixed-proxy-tunnel-destination-policy.md new file mode 100644 index 0000000000..82de7bf416 --- /dev/null +++ b/changelog.d/fixed-proxy-tunnel-destination-policy.md @@ -0,0 +1 @@ +- The agent proxy now applies its plain-HTTP destination policy to uninspected CONNECT and transparent-TLS tunnels as well, so all three egress legs dial only globally routable addresses; `HIVE_PROXY_ALLOW_PRIVATE_FORWARD=true` lifts it for every leg at once diff --git a/changelog.d/fixed-release-none-fragment-stall.md b/changelog.d/fixed-release-none-fragment-stall.md new file mode 100644 index 0000000000..a2e8df80c2 --- /dev/null +++ b/changelog.d/fixed-release-none-fragment-stall.md @@ -0,0 +1 @@ +- Tagged releases resume: a `release: none` HTML-comment marker that rode in a `changelog.d/` fragment (#12928, merged 2026-10-19) was compiled into `## Unreleased` on every merge and made the release workflow's decide job skip every commit since v5.186.4 as an "explicit human override" — the fragment could only be consumed by the very release it vetoed. The fragment is removed; `check-changelog-fragment.sh` and `compile-changelog.sh` now reject a `none` marker in a fragment (bump markers stay allowed); and a hand-placed `none` freeze in CHANGELOG.md is surfaced as a workflow warning annotation and fails the decide job once it has outlived `RELEASE_FREEZE_MAX_DAYS` (default 7) instead of silently skipping forever. Also fixed on the way out: with the resulting 1,369-fragment backlog the bump inference's `grep -q` died of SIGPIPE under `pipefail` and would have labelled the catch-up release a patch instead of the minor its `### Added` section calls for. diff --git a/changelog.d/security-toolapprove-apply-patch-paths.md b/changelog.d/security-toolapprove-apply-patch-paths.md new file mode 100644 index 0000000000..65d2151779 --- /dev/null +++ b/changelog.d/security-toolapprove-apply-patch-paths.md @@ -0,0 +1 @@ +- Fixed the tool-approval guardrail-path check never seeing the target of a Copilot CLI `apply_patch` call ([#16979](https://github.com/hivecommons/hive/issues/16979)). The tool carries its paths in the patch text (`*** Update File:`, `*** Add File:`, `*** Delete File:`, `*** Move to:`) instead of a path argument, so `ToolRequest.GetFilePath` returned an empty string and a patch touching `.github/workflows/**`, `OWNERS`, `hive.yaml` or `policies/**` auto-approved at L5/L6. `GetFilePath` now falls back to the first path in the patch body, and the guardrail check now iterates the new `GetFilePaths` over every path the request names, so a patch that lists a harmless file first and a guardrail path later is still caught. diff --git a/src/cmd/hive/github_client_provider.go b/src/cmd/hive/github_client_provider.go index afdd6e5de3..018f628d7e 100644 --- a/src/cmd/hive/github_client_provider.go +++ b/src/cmd/hive/github_client_provider.go @@ -110,11 +110,15 @@ func (b *boot) armRequestRelays(client *github.Client) bool { // onACMMLevelChanged re-arms request relays when the level crosses the // self-authored auto-merge boundary. The relay generation re-reads the current // ACMM verdict when it starts, so this turns a runtime L5↔L6 change into the -// same safe hand-over used for GitHub App client rebuilds. +// same safe hand-over used for GitHub App client rebuilds. It also rebuilds +// the ioscan semantic classifier, whose enablement and fail mode follow the +// level (required and fail-closed at L5+), so an L4↔L5 change in the +// dashboard takes effect without a restart. func (b *boot) onACMMLevelChanged(prev, next int) { if b == nil || b.cfg == nil { return } + b.reinstallIoscanClassifier() before, after := b.cfg.AutoMerge.SelfAuthoredAutoMergeAllowed(&prev), b.cfg.AutoMerge.SelfAuthoredAutoMergeAllowed(&next) if before == after { return diff --git a/src/cmd/hive/ioscan_classifier_boot.go b/src/cmd/hive/ioscan_classifier_boot.go index db54b3ae61..739a424ae3 100644 --- a/src/cmd/hive/ioscan_classifier_boot.go +++ b/src/cmd/hive/ioscan_classifier_boot.go @@ -140,6 +140,55 @@ func installIoscanClassifier(sched ioscanClassifierSink, alerts ioscanClassifier } } +// reinstallIoscanClassifier re-runs installIoscanClassifier against the live +// config after an Input Defense save, so a new backend or fail mode applies +// without a restart (#16973). A run that installs nothing (classifier off, or +// fail-open with no runnable classifier) drops the classifier an earlier run +// installed, so a stale unavailable classifier cannot keep withholding text. +func reinstallIoscanClassifier(sched ioscanClassifierSink, alerts ioscanClassifierAlerter, cfg *config.Config, logger *slog.Logger, now func() time.Time) { + rec := &ioscanClassifierSetRecorder{ioscanClassifierSink: sched} + installIoscanClassifier(rec, alerts, cfg, logger, now) + if !rec.set { + sched.SetClassifier(nil, ioscan.Thresholds{}) + } +} + +// ioscanAlerts is the dashboard the ioscan classifier raises its blocked-kick +// alert on, as a nil interface (not a typed nil) when there is no dashboard. +func (b *boot) ioscanAlerts() ioscanClassifierAlerter { + if b == nil || b.dashSrv == nil { + return nil + } + return b.dashSrv +} + +// reinstallIoscanClassifier rebuilds the scheduler's semantic classifier +// from the live config: after an Input Defense save (#16973) and after a +// runtime ACMM level change, since the level decides whether the classifier +// is required and which fail mode applies. No-op before the scheduler exists. +func (b *boot) reinstallIoscanClassifier() { + if b == nil || b.sched == nil || b.cfg == nil { + return + } + logger := b.logger + if logger == nil { + logger = slog.Default() + } + reinstallIoscanClassifier(b.sched, b.ioscanAlerts(), b.cfg, logger, time.Now) +} + +// ioscanClassifierSetRecorder notes whether installIoscanClassifier installed +// a classifier. +type ioscanClassifierSetRecorder struct { + ioscanClassifierSink + set bool +} + +func (r *ioscanClassifierSetRecorder) SetClassifier(c ioscan.Classifier, t ioscan.Thresholds) { + r.set = true + r.ioscanClassifierSink.SetClassifier(c, t) +} + // ioscanClassifierFallbackBackend picks the subscription CLI the classifier // runs through when no backend is configured and no model endpoint is // resolved: the tool-less-capable backend most of the hive's agents run on, @@ -206,8 +255,10 @@ func ioscanClassifierBlockedAlertMessage(acmmLevel int, cause error) string { return fmt.Sprintf("ioscan semantic classifier is required at ACMM L%d but failed or is unavailable (%v). "+ "Kicks carrying untrusted text (issues, comments, PR bodies, reviews) can be withheld until this is fixed. "+ "Fix: configure a model endpoint (governor.trajectory or governor.litellm), set ioscan.classifier.backend to one of %s "+ - "(Settings → Security → Input Defense), or accept scanning without the classifier with ioscan.classifier.fail_mode: open. "+ - "Changes take effect after a restart.", acmmLevel, cause, strings.Join(ioscan.ToollessBackends(), ", ")) + "(Settings → Security → Input Defense), or accept scanning without the classifier by setting ioscan.fail_mode to open "+ + "(the Fail mode control in that tab; ioscan.classifier.fail_mode: open in hive.yaml overrides it for the classifier alone). "+ + "Input Defense changes saved in Settings → Security apply immediately; a model endpoint or hive.yaml edit takes effect after the hive restarts.", + acmmLevel, cause, strings.Join(ioscan.ToollessBackends(), ", ")) } func raiseIoscanClassifierAlert(alerts ioscanClassifierAlerter, acmmLevel int, cause error) { diff --git a/src/cmd/hive/ioscan_classifier_boot_test.go b/src/cmd/hive/ioscan_classifier_boot_test.go index 2a3d9c2890..482113352d 100644 --- a/src/cmd/hive/ioscan_classifier_boot_test.go +++ b/src/cmd/hive/ioscan_classifier_boot_test.go @@ -11,6 +11,7 @@ import ( "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/ioscan" + "github.com/hivecommons/hive/pkg/scheduler" ) type fakeClassifierSink struct { @@ -421,3 +422,92 @@ func TestInstallIoscanClassifier_RuntimeUnavailableAlert(t *testing.T) { t.Fatal("runtime alert remains after successful score") } } + +// A dashboard save re-runs the install against the live config (#16973): +// switching to fail-open drops the stale unavailable classifier and clears +// the alert, and naming a tool-less backend installs it, with no restart. +func TestReinstallIoscanClassifier_AppliesDashboardChanges(t *testing.T) { + sink := &fakeClassifierSink{} + alerts := &fakeClassifierAlerts{} + logger, _ := ioscanBootLogger() + cfg := ioscanBootConfigWithAgents(t, 6, "codex") + installIoscanClassifier(sink, alerts, cfg, logger, ioscanBootNow) + if _, ok := sink.classifier.(ioscan.UnavailableClassifier); !ok { + t.Fatalf("installed %T, want UnavailableClassifier", sink.classifier) + } + msg, ok := alerts.blocked() + if !ok || !strings.Contains(msg, "apply immediately") || strings.Contains(msg, "Changes take effect after a restart") { + t.Fatalf("blocked-kick alert does not say dashboard changes apply immediately: %q", msg) + } + // The Security tab exposes ioscan.fail_mode, not classifier.fail_mode; + // the remediation names the key the operator can actually set there. + if !strings.Contains(msg, "ioscan.fail_mode to open") { + t.Fatalf("blocked-kick alert does not name the dashboard-visible ioscan.fail_mode key: %q", msg) + } + + cfg.Ioscan.FailMode = "open" + reinstallIoscanClassifier(sink, alerts, cfg, logger, ioscanBootNow) + if sink.classifier != nil { + t.Fatalf("fail-open save kept %T installed", sink.classifier) + } + if _, ok := alerts.blocked(); ok { + t.Fatal("fail-open save left the blocked-kick alert raised") + } + + cfg.Ioscan.FailMode = "" + cfg.Ioscan.Classifier.Backend = "copilot" + reinstallIoscanClassifier(sink, alerts, cfg, logger, ioscanBootNow) + if _, ok := sink.classifier.(*ioscan.CachedClassifier); !ok { + t.Fatalf("backend save installed %T, want *ioscan.CachedClassifier", sink.classifier) + } + if _, ok := alerts.blocked(); ok { + t.Fatal("working backend left the blocked-kick alert raised") + } +} + +// A runtime ACMM level change rebuilds the classifier through the same hook +// the request relays use. With a backend that cannot run, L4 is fail-open +// (nothing installed) and L5 is fail-closed (the unavailable classifier is +// installed so untrusted text is withheld); raising L4->L5 in the dashboard +// must switch between them without a restart, and dropping back must undo +// it. The hook must tolerate a boot with no scheduler. +func TestOnACMMLevelChanged_ReinstallsIoscanClassifier(t *testing.T) { + t.Setenv(config.LiteLLMEndpointEnv, "") + cfg := ioscanBootConfig(4, "codex", "") + logger, _ := ioscanBootLogger() + b := &boot{cfg: cfg, logger: logger} + b.onACMMLevelChanged(4, 5) // no scheduler yet: must not panic + + b.sched = scheduler.New(cfg, logger) + installIoscanClassifier(b.sched, b.ioscanAlerts(), cfg, logger, ioscanBootNow) + if _, _, installed := b.sched.IoscanHooksForTest(); installed { + t.Fatal("refused backend installed a classifier at fail-open L4") + } + + l5 := 5 + cfg.ACMMLevel = &l5 + b.onACMMLevelChanged(4, 5) + if _, _, installed := b.sched.IoscanHooksForTest(); !installed { + t.Fatal("L4->L5 runtime change did not install the fail-closed unavailable classifier") + } + + l4 := 4 + cfg.ACMMLevel = &l4 + b.onACMMLevelChanged(5, 4) + if _, _, installed := b.sched.IoscanHooksForTest(); installed { + t.Fatal("L5->L4 runtime change left the unavailable classifier withholding text") + } +} + +// ioscanAlerts yields a nil interface, not a typed nil *dashboard.Server, +// when the boot has no dashboard, so raiseIoscanClassifierAlert's nil check +// holds. +func TestBootIoscanAlerts_NilWithoutDashboard(t *testing.T) { + var b *boot + if b.ioscanAlerts() != nil { + t.Fatal("nil boot yielded a non-nil alerter") + } + if (&boot{}).ioscanAlerts() != nil { + t.Fatal("boot without a dashboard yielded a non-nil alerter") + } +} diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index de1caccda2..8949c14ecb 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -1296,6 +1296,7 @@ func (b *boot) wireBootClosures() { ReInitFunc: func() { initAgentConfigDrivenSystems(b.cfg) }, + IoscanConfigApplied: b.reinstallIoscanClassifier, ReviewConfigApplied: func(rc config.ReviewConfig) { if b.ghClient == nil { return @@ -2717,11 +2718,7 @@ func (b *boot) bootDashboardWith(deps bootDashboardDeps) { } } }) - var ioscanAlerts ioscanClassifierAlerter - if b.dashSrv != nil { - ioscanAlerts = b.dashSrv - } - installIoscanClassifier(b.sched, ioscanAlerts, b.cfg, b.logger, time.Now) + installIoscanClassifier(b.sched, b.ioscanAlerts(), b.cfg, b.logger, time.Now) b.agentMgr.SetSandboxAuditCallback(func(agentName, action, detail string) { b.dashSrv.AuditLog(agentName, action, detail, agentName) if action == "sandbox_broker_rejected" { diff --git a/src/cmd/hive/merge_eligibility.go b/src/cmd/hive/merge_eligibility.go index 88c43d1fc6..b27a7e55cf 100644 --- a/src/cmd/hive/merge_eligibility.go +++ b/src/cmd/hive/merge_eligibility.go @@ -19,6 +19,7 @@ import ( "github.com/hivecommons/hive/pkg/intent" "github.com/hivecommons/hive/pkg/rebasemerge" "github.com/hivecommons/hive/pkg/review" + "github.com/hivecommons/hive/pkg/stageengine/prstage" "github.com/hivecommons/hive/pkg/stagelabels" ) @@ -493,13 +494,14 @@ func setMergeEligibleLaneGate(g github.MergeRequestPolicyGate) { // refuses must not be listed merge-eligible: hive-merge on it is a wasted // attempt every kick, and the contributor lane merges it itself when ready. // It reuses the relay's gate (automerge.Engine.RelayMergeRefusal) rather than -// restating the rule. Hive-authored PRs bypass the lane, and an unreadable -// gate leaves the PR listed — the relay still re-checks it at merge time. +// restating the rule. The gate itself owns author exemptions: hive attribution +// describes provenance, not contributor trust. An unreadable gate leaves the +// PR listed — the relay still re-checks it at merge time. func relayLaneRefusal(pr github.PullRequest, fullRepo string, logger *slog.Logger) string { mergeEligibleLaneGateMu.RLock() gate := mergeEligibleLaneGate mergeEligibleLaneGateMu.RUnlock() - if gate == nil || hiveAuthoredPR(pr) { + if gate == nil { return "" } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) @@ -515,6 +517,21 @@ func relayLaneRefusal(pr github.PullRequest, fullRepo string, logger *slog.Logge return refusal } +// relayRefusalNeedsMaintainer reports trust admission failures using the +// stage engine's existing classification. Soak, head age and pending review +// are temporary gates, not requests for a maintainer merge. +func relayRefusalNeedsMaintainer(refusal string) bool { + if !strings.HasPrefix(refusal, automerge.RelayMergeRefusalPrefix) { + return false + } + _, reason, ok := strings.Cut(refusal, ": ") + if !ok { + return false + } + reason, _, _ = strings.Cut(reason, " (") + return prstage.HumanMergeReason(reason) == prstage.HumanMergeReasonUntrustedAuthor +} + // mergeableJSON renders a tri-state mergeability verdict for the // merge-eligible.json marker, mapping the unknown zero value to an explicit // "unknown" rather than an empty string. @@ -1256,7 +1273,15 @@ func writeMergeEligible(actionable *github.ActionableResult, hold github.HoldRes prAgents := auditPRAgents(org, time.Now().Add(-auditPRAttributionWindow), "") + type maintainerMergePR struct { + Number int `json:"number"` + Repo string `json:"repo"` + Title string `json:"title"` + HeadSHA string `json:"head_sha,omitempty"` + Reason string `json:"reason"` + } var eligible []eligiblePR + var needsMaintainer []maintainerMergePR var failing []failingPR // escalated holds the escalated (needs-human) hive PRs that did NOT land // in the failing bucket — conflicted, green or pending ones @@ -1345,6 +1370,11 @@ func writeMergeEligible(actionable *github.ActionableResult, hold github.HoldRes if refusal := relayLaneRefusal(pr, fullRepo, logger); refusal != "" { bucket = mergeBucketSkip verdict = github.MergeVerdict{State: github.MergeVerdictOutstanding, Reason: refusal} + if relayRefusalNeedsMaintainer(refusal) { + needsMaintainer = append(needsMaintainer, maintainerMergePR{ + Number: pr.Number, Repo: fullRepo, Title: pr.Title, HeadSHA: pr.HeadSHA, Reason: refusal, + }) + } } } verdict = handMergeVerdict(verdict, prGates.repoAutoMergeOff) @@ -1515,8 +1545,9 @@ func writeMergeEligible(actionable *github.ActionableResult, hold github.HoldRes _ = os.MkdirAll("/var/run/hive-metrics", 0o755) payload := map[string]any{ - "generated_at": time.Now().UTC().Format(time.RFC3339), - "merge_eligible": eligible, + "generated_at": time.Now().UTC().Format(time.RFC3339), + "merge_eligible": eligible, + "needs_maintainer_merge": needsMaintainer, } data, err := json.Marshal(payload) if err != nil { diff --git a/src/cmd/hive/merge_eligible_lane_gate_test.go b/src/cmd/hive/merge_eligible_lane_gate_test.go index 95b7d3308c..44d76fff2c 100644 --- a/src/cmd/hive/merge_eligible_lane_gate_test.go +++ b/src/cmd/hive/merge_eligible_lane_gate_test.go @@ -3,11 +3,15 @@ package main import ( "context" "encoding/json" + "errors" "os" "path/filepath" + "strings" "testing" "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/github/automerge" + "github.com/hivecommons/hive/pkg/stageengine/prstage" ) // A PR the merge relay's contributor-lane gate refuses (a newcomer-tier @@ -23,9 +27,18 @@ func TestWriteMergeEligible_ExcludesPRsTheRelayLaneRefuses(t *testing.T) { ciFailingPath = origFail setMergeEligibleLaneGate(nil) }) - setMergeEligibleLaneGate(func(_ context.Context, _ string, number int, _ string) (string, error) { - if number == 15086 { - return "contributor-lane policy refuses the merge relay for @Karibhou's PR: contributor-lane:claim-tier-untrusted (author tier \"newcomer\", require_tiers merger, trusted)", nil + called := make(map[int]bool) + refusal := automerge.RelayMergeRefusalPrefix + " for @contributor's PR: " + prstage.ContributorLaneReasonNoClaim + ` (author tier "none", require_tiers merger, trusted)` + setMergeEligibleLaneGate(func(_ context.Context, repo string, number int, sha string) (string, error) { + called[number] = true + if repo != "hivecommons/hive" || sha != "cafe" { + t.Errorf("gate target = %s#%d@%s, want hivecommons/hive at cafe", repo, number, sha) + } + if number == 15086 || number == 15088 || number == 15089 { + return refusal, nil + } + if number == 15091 { + return automerge.RelayMergeRefusalPrefix + ": " + prstage.ContributorLaneReasonSoaking + " (10m remaining)", nil } return "", nil }) @@ -33,9 +46,19 @@ func TestWriteMergeEligible_ExcludesPRsTheRelayLaneRefuses(t *testing.T) { green := func(number int, author string) github.PullRequest { return github.PullRequest{Repo: "hive", Number: number, Author: author, CIStatus: "success", Mergeable: github.MergeableYes, HeadSHA: "cafe"} } + attributed := green(15088, "contributor") + attributed.HiveAttributed = true + agentAttributed := green(15089, "contributor") + agentAttributed.HiveAgent = "scanner" + app := green(15090, "hive[bot]") + app.AppAuthored = true actionable := &github.ActionableResult{PRs: github.PRResult{Items: []github.PullRequest{ green(15086, "Karibhou"), green(15087, "trusted-human"), + attributed, + agentAttributed, + app, + green(15091, "soaking-contributor"), }}} verdicts := writeMergeEligible(actionable, github.HoldResult{}, "hivecommons", nil, false, nil, false, nil, nil, nil, discardLogger()) @@ -47,14 +70,64 @@ func TestWriteMergeEligible_ExcludesPRsTheRelayLaneRefuses(t *testing.T) { Eligible []struct { Number int `json:"number"` } `json:"merge_eligible"` + NeedsMaintainer []struct { + Number int `json:"number"` + Repo string `json:"repo"` + HeadSHA string `json:"head_sha"` + Reason string `json:"reason"` + } `json:"needs_maintainer_merge"` } if err := json.Unmarshal(data, &out); err != nil { t.Fatal(err) } - if len(out.Eligible) != 1 || out.Eligible[0].Number != 15087 { - t.Fatalf("merge_eligible = %+v, want only #15087", out.Eligible) + if len(out.Eligible) != 2 || out.Eligible[0].Number != 15087 || out.Eligible[1].Number != 15090 { + t.Fatalf("merge_eligible = %+v, want trusted author #15087 and App #15090", out.Eligible) + } + if len(out.NeedsMaintainer) != 3 { + t.Fatalf("needs_maintainer_merge = %+v, want the three trust-refused PRs", out.NeedsMaintainer) + } + for i, number := range []int{15086, 15088, 15089} { + row := out.NeedsMaintainer[i] + if row.Number != number || row.Repo != "hivecommons/hive" || row.HeadSHA != "cafe" || row.Reason != refusal { + t.Errorf("maintainer row = %+v, want #%d with pinned head and relay refusal", row, number) + } + } + for _, pr := range actionable.PRs.Items { + if !called[pr.Number] { + t.Errorf("PR #%d bypassed the authoritative relay gate", pr.Number) + } + v := verdicts[github.MergeVerdictKey(pr)] + if pr.Number != 15087 && pr.Number != 15090 && (v.State == github.MergeVerdictEligible || !strings.Contains(v.Reason, automerge.RelayMergeRefusalPrefix)) { + t.Errorf("refused PR #%d verdict = %+v, want refusal", pr.Number, v) + } } - if v := verdicts[github.MergeVerdictKey(actionable.PRs.Items[0])]; v.State == github.MergeVerdictEligible { - t.Fatalf("refused PR verdict = %+v, want not eligible", v) +} + +func TestRelayLaneRefusalKeepsUnavailableEvidenceForRelayRecheck(t *testing.T) { + t.Cleanup(func() { setMergeEligibleLaneGate(nil) }) + setMergeEligibleLaneGate(func(context.Context, string, int, string) (string, error) { + return "", errors.New("evidence unavailable") + }) + if got := relayLaneRefusal(github.PullRequest{Number: 1}, "hivecommons/hive", discardLogger()); got != "" { + t.Fatalf("unreadable gate refusal = %q, want existing relay-recheck behavior", got) + } +} + +func TestRelayRefusalNeedsMaintainerOnlyForTrustAdmission(t *testing.T) { + for _, tc := range []struct { + reason string + want bool + }{ + {prstage.ContributorLaneReasonNoClaim, true}, + {prstage.ContributorLaneReasonClaimTier, true}, + {prstage.ContributorLaneReasonAuthorAssociation, true}, + {prstage.ContributorLaneReasonSoaking, false}, + {prstage.ContributorLaneReasonNoVerdict, false}, + {"pending: head pushed 1m ago", false}, + } { + refusal := automerge.RelayMergeRefusalPrefix + ": " + tc.reason + " (details)" + if got := relayRefusalNeedsMaintainer(refusal); got != tc.want { + t.Errorf("maintainer classification for %s = %v, want %v", tc.reason, got, tc.want) + } } } diff --git a/src/cmd/hive/notifywire.go b/src/cmd/hive/notifywire.go index cb6b6698fa..cda8a8d6a2 100644 --- a/src/cmd/hive/notifywire.go +++ b/src/cmd/hive/notifywire.go @@ -35,6 +35,13 @@ func syncAutoMergePolicyToGitHubClient(cfg *config.Config, ghClient *github.Clie ghClient.SetMergeRequestNoCIAllowedRepos(cfg.AutoMerge.NoCIOKSet()) // The merge-request relay refuses PRs touching these paths (#11039). ghClient.SetHumanMergePaths(cfg.AutoMerge.HumanMergePaths) + pinOnly := make(map[string]github.PinOnlySpec) + for repo := range cfg.AutoMerge.HumanMergePinOnly { + if ex, enabled := cfg.AutoMerge.HumanMergePinOnlyFor(repo); enabled { + pinOnly[repo] = github.PinOnlySpec{Enabled: true, Paths: ex.Paths, Kinds: ex.Kinds, TrustedAuthors: ex.TrustedAuthors} + } + } + ghClient.SetHumanMergePinOnly(pinOnly) ghClient.SetAutoMergeMinHeadAge(cfg.AutoMerge.EffectiveMinHeadAge()) return set, ok } diff --git a/src/crux/agent/agent_unit_test.go b/src/crux/agent/agent_unit_test.go index 02134ebcd3..55dc9f320f 100644 --- a/src/crux/agent/agent_unit_test.go +++ b/src/crux/agent/agent_unit_test.go @@ -738,3 +738,27 @@ func TestDefaultAgentModeByRole(t *testing.T) { } } } + +func TestPaneShowsLoginPromptIgnoresSourceText(t *testing.T) { + cases := []struct { + name string + line string + want bool + }{ + {"grep prefix directive", `src/crux/watchdog/classify.go:99: "Please run /login",`, false}, + {"quoted pattern", ` "Enter one-time code",`, false}, + {"alternation", `rg 'credentialValid|Not logged in|agent_login_required'`, false}, + {"table name", `var loginPromptPatterns = []string{ // Not logged in`, false}, + {"device flow", "Enter one-time code: ABCD-1234", true}, + {"login expired", "Login expired · Please run /login", true}, + {"not logged in", "Not logged in", true}, + {"device url", "Visit github.com/login/device", true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := paneShowsLoginPrompt([]string{tc.line}); got != tc.want { + t.Fatalf("paneShowsLoginPrompt(%q) = %v, want %v", tc.line, got, tc.want) + } + }) + } +} diff --git a/src/crux/agent/pane_classify.go b/src/crux/agent/pane_classify.go index 4865c13bc4..977e72abc4 100644 --- a/src/crux/agent/pane_classify.go +++ b/src/crux/agent/pane_classify.go @@ -632,11 +632,11 @@ func paneShowsLoginPrompt(lines []string) bool { if lineShowsUpstreamAuthorizationError(line) || paneShowsQuotaExhausted([]string{line}) { continue } - if lineHasLoginDirective(line) || ompLoginScreenLineRe.MatchString(line) { + if (lineHasLoginDirective(line) && !authHitIsSourceText(line, "/login")) || ompLoginScreenLineRe.MatchString(line) { return true } for _, pat := range loginPromptPatterns { - if strings.Contains(line, pat) { + if strings.Contains(line, pat) && !authHitIsSourceText(line, pat) { return true } } diff --git a/src/crux/agent/source_text.go b/src/crux/agent/source_text.go new file mode 100644 index 0000000000..9426732af2 --- /dev/null +++ b/src/crux/agent/source_text.go @@ -0,0 +1,56 @@ +package agent + +import ( + "regexp" + "strings" +) + +// grepPrefixRe matches the "path.go:NN:" prefix grep/rg put on a hit. +var grepPrefixRe = regexp.MustCompile("^\\s*[\\w./-]+\\.\\w+:\\d+[:-]") + +// patternTableNames are identifiers whose presence on a line means the pane is +// showing hive's own login-pattern source, not a login screen (#17006). +var patternTableNames = []string{"authScreenPatterns", "loginPromptPatterns", "classify.go"} + +// authHitIsSourceText reports whether every match of pat on line is source or +// grep output quoting the pattern (a path.go:NN: prefix, a pattern-table name, +// a string literal, or a |-joined alternation) rather than a CLI login screen. +func authHitIsSourceText(line, pat string) bool { + if grepPrefixRe.MatchString(line) { + return true + } + for _, name := range patternTableNames { + if strings.Contains(line, name) { + return true + } + } + for off := 0; ; { + i := strings.Index(line[off:], pat) + if i < 0 { + return true + } + start := off + i + end := start + len(pat) + if !hitIsQuoted(line, start, end) { + return false + } + off = end + } +} + +// hitIsQuoted reports whether the match at [start,end) sits inside a quoted +// literal or beside a regex alternation bar. +func hitIsQuoted(line string, start, end int) bool { + before, after := line[:start], line[end:] + if strings.Count(before, "\"")%2 == 1 || strings.Count(before, "`")%2 == 1 { + return true + } + b := strings.TrimRight(before, " ") + a := strings.TrimLeft(after, " ") + for _, q := range []string{"'", "\"", "`", "|"} { + if strings.HasSuffix(b, q) || strings.HasPrefix(a, q) { + return true + } + } + return false +} diff --git a/src/crux/api/checkouts.go b/src/crux/api/checkouts.go new file mode 100644 index 0000000000..c13c8a9df3 --- /dev/null +++ b/src/crux/api/checkouts.go @@ -0,0 +1,98 @@ +package api + +import ( + "context" + "net/url" + "time" +) + +// Pinned checkouts (hivecommons/crux#114 S1): "a working copy of this repo at +// this commit, owned by this caller, capped, deletable". Routes are a +// work-services family: lane project credentials are accepted here, with +// the project as the owner, and the control token acts as the operator. +const ( + // PathCheckouts is POST (create), GET (list, ?owner=) /v1/checkouts and + // GET/DELETE /v1/checkouts/{id}. + PathCheckouts = "/v1/checkouts" +) + +// Checkout error codes. +const ( + // CodeTooLarge is a checkout that exceeded max_bytes once measured; the + // clone has been removed (413). + CodeTooLarge = "too_large" + // CodeUnsupported is a requested limit the backend could not enforce or + // verify (501); the call was refused, never run with the limit dropped. + CodeUnsupported = "unsupported" + // CodeUpstream is a backend answer that contradicts the request, such as + // a worktree at a different commit than the one pinned (502). + CodeUpstream = "upstream_error" +) + +// CheckoutRequest is the body of POST /v1/checkouts. +type CheckoutRequest struct { + // Repo is the repository URL; it must match a configured allowlist prefix. + Repo string `json:"repo"` + // Commit is the full 40-hex SHA to pin; tags and branches are refused. + Commit string `json:"commit"` + // Owner identifies the owning run or driver. A project principal may + // only name itself (or leave it empty); the operator names anyone and + // defaults to its own name. + Owner string `json:"owner,omitempty"` + // Branches clones all refs when nil or true; false clones one branch. + Branches *bool `json:"branches,omitempty"` + // MaxBytes caps the clone plus worktree; zero means the server default. + MaxBytes int64 `json:"max_bytes,omitempty"` + // TTL is a Go duration ("72h") after which a sweeper may reclaim it. + TTL string `json:"ttl,omitempty"` +} + +// Checkout is one checkout record: the response of POST and GET. +type Checkout struct { + ID string `json:"id"` + Path string `json:"path"` + Repo string `json:"repo"` + Commit string `json:"commit"` + Owner string `json:"owner"` + Branches bool `json:"branches"` + MaxBytes int64 `json:"max_bytes"` + Bytes int64 `json:"bytes"` + TTL string `json:"ttl,omitempty"` + CreatedAt time.Time `json:"created_at"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` +} + +// CheckoutList is the response of GET /v1/checkouts. +type CheckoutList struct { + Checkouts []Checkout `json:"checkouts"` +} + +// CreateCheckout is POST /v1/checkouts. +func (c *Client) CreateCheckout(ctx context.Context, in CheckoutRequest) (Checkout, error) { + var out Checkout + err := c.do(ctx, "POST", PathCheckouts, in, &out) + return out, err +} + +// ListCheckouts is GET /v1/checkouts[?owner=]. +func (c *Client) ListCheckouts(ctx context.Context, owner string) ([]Checkout, error) { + path := PathCheckouts + if owner != "" { + path += "?owner=" + url.QueryEscape(owner) + } + var out CheckoutList + err := c.do(ctx, "GET", path, nil, &out) + return out.Checkouts, err +} + +// GetCheckout is GET /v1/checkouts/{id}. +func (c *Client) GetCheckout(ctx context.Context, id string) (Checkout, error) { + var out Checkout + err := c.do(ctx, "GET", PathCheckouts+"/"+url.PathEscape(id), nil, &out) + return out, err +} + +// DeleteCheckout is DELETE /v1/checkouts/{id}. +func (c *Client) DeleteCheckout(ctx context.Context, id string) error { + return c.do(ctx, "DELETE", PathCheckouts+"/"+url.PathEscape(id), nil, nil) +} diff --git a/src/crux/checkout/checkout.go b/src/crux/checkout/checkout.go new file mode 100644 index 0000000000..2ffaa3e8f0 --- /dev/null +++ b/src/crux/checkout/checkout.go @@ -0,0 +1,426 @@ +// Package checkout is the pinned-checkout work service (hivecommons/crux#114 +// S1): "a working copy of this repo at this commit, owned by this caller, +// capped, deletable". It builds on sandbox.WorktreeManager for the pinned +// worktree and keeps one JSON record per checkout plus an append-only +// index under Config.Dir, the way sandbox_run.go keeps run records. +// +// Trust is deny-by-default: a repository must match one of the configured +// URL prefixes (the shape of Hive's nousdeps fetch allowlist, as a prefix +// list) or it is refused before any clone is attempted. A commit must be a +// full 40-hex SHA so the pin cannot move under the caller. +// +// The size cap is honest, not pre-emptive: git cannot bound a clone before +// transferring it, so the clone and worktree are measured after they land +// and removed when they exceed the cap (ErrTooLarge). When they cannot be +// measured the checkout is removed and the call fails with ErrUnsupported +// rather than delivering an uncapped copy. +package checkout + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/url" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/hivecommons/hive/crux/sandbox" +) + +const ( + // DirName is the directory under the work dir that holds checkouts. + DirName = "checkouts" + // IndexName is the append-only create/delete history inside Dir. + IndexName = "index.jsonl" + // IDPrefix starts every checkout id. + IDPrefix = "chk_" + // DefaultMaxBytes is the size cap when a request names none: 5 GiB. + DefaultMaxBytes int64 = 5 << 30 + // DefaultMaxCheckouts bounds live worktrees under Dir. + DefaultMaxCheckouts = 64 + // MaxOwnerLen bounds the owner identifier. + MaxOwnerLen = 64 + // commitHexLen is the length of a full SHA-1 commit id. + commitHexLen = 40 + // idRandomBytes is the entropy behind an id (12 hex chars). + idRandomBytes = 6 + // sourceDirName holds the per-(repo, owner) clones worktrees pin from. + sourceDirName = "src" + // worktreeStage is the sandbox.WorktreeSpec stage name; a checkout has + // one generation, so the worktree lands at runs//pin-1. + worktreeStage = "pin" + // recordMode and dirMode keep records readable by the runtime's uid only. + recordMode = 0o600 + dirMode = 0o700 +) + +// Sentinel errors; Service methods wrap them so callers map them to typed +// API codes with errors.Is. +var ( + // ErrBadRequest is a malformed request (short SHA, bad URL, bad owner). + ErrBadRequest = errors.New("checkout: bad request") + // ErrForbidden is a repository off the allowlist or a caller that does + // not own the checkout. + ErrForbidden = errors.New("checkout: forbidden") + // ErrNotFound is an unknown checkout id. + ErrNotFound = errors.New("checkout: not found") + // ErrInUse is a delete while contained runs still use the checkout. + ErrInUse = errors.New("checkout: in use") + // ErrMismatch is a worktree whose HEAD is not the requested commit. + ErrMismatch = errors.New("checkout: commit mismatch") + // ErrTooLarge is a clone that exceeded max_bytes once measured. + ErrTooLarge = errors.New("checkout: exceeds size cap") + // ErrUnsupported is a cap the backend could not verify. + ErrUnsupported = errors.New("checkout: cannot enforce size cap") + // ErrGit is a git failure (clone, fetch, worktree). + ErrGit = errors.New("checkout: git") +) + +var ( + commitPattern = regexp.MustCompile(`^[0-9a-f]{40}$`) + ownerPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_.-]*$`) +) + +// Runner executes one command in dir and returns its combined output; the +// default runs it with os/exec. Tests substitute failures. +type Runner func(ctx context.Context, dir string, name string, args ...string) ([]byte, error) + +// Config is the administrator-owned configuration of the service. +type Config struct { + // Dir is where clones, worktrees and records live (DefaultDir). + Dir string + // AllowedRepos are URL prefixes a requested repo must start with. Empty + // denies every request. + AllowedRepos []string + // MaxCheckouts caps live worktrees; zero means DefaultMaxCheckouts. + MaxCheckouts int + // Git runs git; nil uses os/exec. + Git Runner + // Now is the clock; nil uses time.Now. + Now func() time.Time + // InUse reports whether contained runs still reference a checkout; the + // contained-run service wires it. Nil means never in use. + InUse func(id string) bool + // DiskUsage measures a tree in bytes; nil walks it. + DiskUsage func(path string) (int64, error) +} + +// DefaultDir is Dir under the runtime's work dir. +func DefaultDir(workDir string) string { return filepath.Join(workDir, DirName) } + +// Request is what a caller asks for. +type Request struct { + Repo string + Commit string + Owner string + Branches *bool + MaxBytes int64 + TTL time.Duration +} + +// Record is one checkout as stored under Dir/.json. It never carries +// environment or credentials: the repo URL is refused if it embeds any. +type Record struct { + ID string `json:"id"` + Path string `json:"path"` + Repo string `json:"repo"` + Commit string `json:"commit"` + Owner string `json:"owner"` + Branches bool `json:"branches"` + MaxBytes int64 `json:"max_bytes"` + Bytes int64 `json:"bytes"` + TTL string `json:"ttl,omitempty"` + CreatedAt time.Time `json:"created_at"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` + // Source is the shared clone the worktree pins from. + Source string `json:"source"` +} + +// Service creates, lists and deletes checkouts. +type Service struct { + cfg Config + trees sandbox.WorktreeManager + store *store +} + +// New validates cfg and returns a Service. Dir is created on first use. +func New(cfg Config) (*Service, error) { + if strings.TrimSpace(cfg.Dir) == "" { + return nil, fmt.Errorf("%w: dir is required", ErrBadRequest) + } + for i, p := range cfg.AllowedRepos { + p = strings.TrimSpace(p) + if _, err := parseRepo(p); err != nil { + return nil, fmt.Errorf("allowlist entry %q: %w", p, err) + } + cfg.AllowedRepos[i] = p + } + if cfg.MaxCheckouts <= 0 { + cfg.MaxCheckouts = DefaultMaxCheckouts + } + if cfg.Now == nil { + cfg.Now = time.Now + } + if cfg.Git == nil { + cfg.Git = execRunner + } + if cfg.DiskUsage == nil { + cfg.DiskUsage = diskUsage + } + return &Service{cfg: cfg, trees: sandbox.WorktreeManager{Runner: cfg.Git}, store: &store{dir: cfg.Dir}}, nil +} + +// InUse reports whether contained runs reference id (Config.InUse). +func (s *Service) InUse(id string) bool { + return s.cfg.InUse != nil && s.cfg.InUse(id) +} + +// Allowed reports whether repo is on the allowlist, naming the rule that +// admitted it or an ErrForbidden explaining why none did. +func (s *Service) Allowed(repo string) (rule string, err error) { + if len(s.cfg.AllowedRepos) == 0 { + return "", fmt.Errorf("%w: repository %q: the checkout allowlist is empty", ErrForbidden, repo) + } + for _, p := range s.cfg.AllowedRepos { + if strings.HasPrefix(repo, p) { + return p, nil + } + } + return "", fmt.Errorf("%w: repository %q matches none of the allowed prefixes %s", ErrForbidden, repo, strings.Join(s.cfg.AllowedRepos, ", ")) +} + +// Create provisions a checkout for req, recorded as owned by req.Owner. +func (s *Service) Create(ctx context.Context, req Request) (Record, error) { + rec, err := s.validate(req) + if err != nil { + return Record{}, err + } + if _, err := s.Allowed(rec.Repo); err != nil { + return Record{}, err + } + if err := os.MkdirAll(s.cfg.Dir, dirMode); err != nil { + return Record{}, err + } + rec.Source = s.sourceDir(rec.Repo, rec.Owner) + if err := s.ensureSource(ctx, rec); err != nil { + return Record{}, err + } + tree, err := s.trees.Add(ctx, sandbox.WorktreeSpec{ + SourceRepo: rec.Source, WorkspaceDir: s.cfg.Dir, RunKey: rec.ID, Stage: worktreeStage, Generation: 1, + TargetRef: rec.Commit, MaxWorktrees: s.cfg.MaxCheckouts, + }) + if err != nil { + return Record{}, fmt.Errorf("%w: %v", ErrGit, err) + } + rec.Path = tree.Path + if tree.SHA != rec.Commit { + _ = s.trees.Remove(ctx, rec.Source, rec.Path) + return Record{}, fmt.Errorf("%w: requested %s, worktree is at %s", ErrMismatch, rec.Commit, tree.SHA) + } + if err := s.enforceCap(&rec); err != nil { + _ = s.trees.Remove(ctx, rec.Source, rec.Path) + return Record{}, err + } + if err := s.store.put(rec); err != nil { + _ = s.trees.Remove(ctx, rec.Source, rec.Path) + return Record{}, err + } + return rec, s.store.appendIndex(indexEntry{Op: "create", ID: rec.ID, Owner: rec.Owner, Repo: rec.Repo, Commit: rec.Commit, At: rec.CreatedAt}) +} + +// Get returns the record for id. +func (s *Service) Get(id string) (Record, error) { return s.store.get(id) } + +// List returns every record, or only owner's when owner is non-empty, +// ordered by id. +func (s *Service) List(owner string) ([]Record, error) { + all, err := s.store.list() + if err != nil { + return nil, err + } + if owner == "" { + return all, nil + } + out := all[:0] + for _, r := range all { + if r.Owner == owner { + out = append(out, r) + } + } + return out, nil +} + +// Delete removes the worktree and the record. by is who asked, for the +// index; ownership is the caller's decision (see CanDelete). +func (s *Service) Delete(ctx context.Context, id, by string) error { + rec, err := s.store.get(id) + if err != nil { + return err + } + if s.InUse(id) { + return fmt.Errorf("%w: checkout %s has live contained runs", ErrInUse, id) + } + if err := s.trees.Remove(ctx, rec.Source, rec.Path); err != nil { + return fmt.Errorf("%w: remove worktree: %v", ErrGit, err) + } + if err := s.store.remove(id); err != nil { + return err + } + return s.store.appendIndex(indexEntry{Op: "delete", ID: id, Owner: rec.Owner, Repo: rec.Repo, Commit: rec.Commit, At: s.cfg.Now().UTC(), By: by}) +} + +// CanDelete is the ownership rule: the owner or an operator. +func CanDelete(rec Record, caller string, operator bool) bool { + return operator || (caller != "" && caller == rec.Owner) +} + +// validate checks req and returns the record it will produce. +func (s *Service) validate(req Request) (Record, error) { + repo := strings.TrimSpace(req.Repo) + if _, err := parseRepo(repo); err != nil { + return Record{}, err + } + commit := strings.ToLower(strings.TrimSpace(req.Commit)) + if !commitPattern.MatchString(commit) { + return Record{}, fmt.Errorf("%w: commit must be a full %d-hex SHA, not a tag or branch (got %q)", ErrBadRequest, commitHexLen, req.Commit) + } + owner := strings.TrimSpace(req.Owner) + if owner == "" || len(owner) > MaxOwnerLen || !ownerPattern.MatchString(owner) { + return Record{}, fmt.Errorf("%w: owner must be a plain identifier of at most %d characters (got %q)", ErrBadRequest, MaxOwnerLen, req.Owner) + } + if req.MaxBytes < 0 || req.TTL < 0 { + return Record{}, fmt.Errorf("%w: max_bytes and ttl must not be negative", ErrBadRequest) + } + id, err := newID() + if err != nil { + return Record{}, err + } + now := s.cfg.Now().UTC() + rec := Record{ID: id, Repo: repo, Commit: commit, Owner: owner, Branches: req.Branches == nil || *req.Branches, MaxBytes: req.MaxBytes, CreatedAt: now} + if rec.MaxBytes == 0 { + rec.MaxBytes = DefaultMaxBytes + } + if req.TTL > 0 { + rec.TTL = req.TTL.String() + exp := now.Add(req.TTL) + rec.ExpiresAt = &exp + } + return rec, nil +} + +// parseRepo accepts an https:// or file:// URL without credentials, query or +// fragment, so a token can never ride into a record. +func parseRepo(raw string) (*url.URL, error) { + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "https" && u.Scheme != "file") || u.User != nil || u.RawQuery != "" || u.Fragment != "" { + return nil, fmt.Errorf("%w: repo must be an https:// or file:// URL without credentials, query or fragment (got %q)", ErrBadRequest, raw) + } + if (u.Scheme == "https" && u.Host == "") || (u.Scheme == "file" && u.Path == "") { + return nil, fmt.Errorf("%w: repo %q names no host or path", ErrBadRequest, raw) + } + return u, nil +} + +// sourceDir is the shared clone for (repo, owner). +func (s *Service) sourceDir(repo, owner string) string { + sum := sha256.Sum256([]byte(repo)) + return filepath.Join(s.cfg.Dir, sourceDirName, owner+"-"+hex.EncodeToString(sum[:6])) +} + +// ensureSource clones rec.Repo into rec.Source once and makes sure +// rec.Commit is present, fetching it when a reused clone lacks it. +func (s *Service) ensureSource(ctx context.Context, rec Record) error { + if _, err := os.Stat(filepath.Join(rec.Source, ".git")); err != nil { + if err := os.MkdirAll(filepath.Dir(rec.Source), dirMode); err != nil { + return err + } + args := []string{"clone", "--no-checkout"} + if !rec.Branches { + args = append(args, "--single-branch") + } + args = append(args, "--", rec.Repo, rec.Source) + if out, err := s.cfg.Git(ctx, s.cfg.Dir, "git", args...); err != nil { + _ = os.RemoveAll(rec.Source) + return fmt.Errorf("%w: clone: %v: %s", ErrGit, err, strings.TrimSpace(string(out))) + } + } + if _, err := s.cfg.Git(ctx, rec.Source, "git", "cat-file", "-e", rec.Commit+"^{commit}"); err == nil { + return nil + } + if out, err := s.cfg.Git(ctx, rec.Source, "git", "fetch", "--quiet", "origin", rec.Commit); err != nil { + return fmt.Errorf("%w: commit %s is not in %s and could not be fetched: %s", ErrBadRequest, rec.Commit, rec.Repo, strings.TrimSpace(string(out))) + } + if _, err := s.cfg.Git(ctx, rec.Source, "git", "cat-file", "-e", rec.Commit+"^{commit}"); err != nil { + return fmt.Errorf("%w: commit %s is not in %s", ErrBadRequest, rec.Commit, rec.Repo) + } + return nil +} + +// enforceCap measures the clone and the worktree after the fact and refuses +// the checkout when they exceed rec.MaxBytes or cannot be measured. +func (s *Service) enforceCap(rec *Record) error { + src, err := s.cfg.DiskUsage(rec.Source) + if err != nil { + return fmt.Errorf("%w: measuring %s: %v", ErrUnsupported, rec.Source, err) + } + tree, err := s.cfg.DiskUsage(rec.Path) + if err != nil { + return fmt.Errorf("%w: measuring %s: %v", ErrUnsupported, rec.Path, err) + } + rec.Bytes = src + tree + if rec.Bytes > rec.MaxBytes { + return fmt.Errorf("%w: clone and worktree measure %d bytes, cap is %d (the cap is checked after cloning; git cannot bound a clone in advance)", ErrTooLarge, rec.Bytes, rec.MaxBytes) + } + return nil +} + +func newID() (string, error) { + b := make([]byte, idRandomBytes) + if _, err := rand.Read(b); err != nil { + return "", err + } + return IDPrefix + hex.EncodeToString(b), nil +} + +// ValidID reports whether id has the shape newID produces, so a path value +// can never escape Dir. +func ValidID(id string) bool { + rest, ok := strings.CutPrefix(id, IDPrefix) + if !ok || len(rest) != 2*idRandomBytes { + return false + } + _, err := hex.DecodeString(rest) + return err == nil +} + +func execRunner(ctx context.Context, dir string, name string, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Dir = dir + return cmd.CombinedOutput() +} + +// diskUsage sums regular file sizes under path. +func diskUsage(path string) (int64, error) { + var total int64 + err := filepath.WalkDir(path, func(_ string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.Type().IsRegular() { + info, err := d.Info() + if err != nil { + return err + } + total += info.Size() + } + return nil + }) + return total, err +} diff --git a/src/crux/checkout/checkout_test.go b/src/crux/checkout/checkout_test.go new file mode 100644 index 0000000000..95674d2fa5 --- /dev/null +++ b/src/crux/checkout/checkout_test.go @@ -0,0 +1,473 @@ +package checkout + +import ( + "context" + "encoding/json" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +// fixtureRepo creates a bare repository with two commits and returns its +// file:// URL and both SHAs (oldest first). +func fixtureRepo(t *testing.T) (repo string, shas []string) { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + work := filepath.Join(t.TempDir(), "work") + git := func(args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = work + cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@x", "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@x") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + if err := os.MkdirAll(work, 0o700); err != nil { + t.Fatal(err) + } + git("init", "-q") + for i, name := range []string{"one.txt", "two.txt"} { + if err := os.WriteFile(filepath.Join(work, name), []byte(name), 0o600); err != nil { + t.Fatal(err) + } + git("add", name) + git("commit", "-q", "-m", "c"+string(rune('0'+i))) + shas = append(shas, git("rev-parse", "HEAD")) + } + bare := filepath.Join(t.TempDir(), "fixture.git") + git("clone", "-q", "--bare", work, bare) + return "file://" + bare, shas +} + +func newTestService(t *testing.T, repo string, edit func(*Config)) *Service { + t.Helper() + cfg := Config{Dir: filepath.Join(t.TempDir(), "checkouts"), AllowedRepos: []string{"file://" + filepath.Dir(strings.TrimPrefix(repo, "file://"))}} + if edit != nil { + edit(&cfg) + } + svc, err := New(cfg) + if err != nil { + t.Fatal(err) + } + return svc +} + +func TestCreateListDeleteRoundTrip(t *testing.T) { + repo, shas := fixtureRepo(t) + now := time.Date(2026, 11, 7, 12, 0, 0, 0, time.UTC) + svc := newTestService(t, repo, func(c *Config) { c.Now = func() time.Time { return now } }) + ctx := context.Background() + + rec, err := svc.Create(ctx, Request{Repo: repo, Commit: shas[0], Owner: "nous-42", TTL: 72 * time.Hour}) + if err != nil { + t.Fatal(err) + } + if !ValidID(rec.ID) || rec.Commit != shas[0] || rec.Owner != "nous-42" || !rec.Branches || rec.MaxBytes != DefaultMaxBytes || rec.Bytes <= 0 { + t.Fatalf("record = %+v", rec) + } + if rec.TTL != "72h0m0s" || rec.ExpiresAt == nil || !rec.ExpiresAt.Equal(now.Add(72*time.Hour)) || !rec.CreatedAt.Equal(now) { + t.Fatalf("ttl/expiry = %q %v %v", rec.TTL, rec.ExpiresAt, rec.CreatedAt) + } + if _, err := os.Stat(filepath.Join(rec.Path, "one.txt")); err != nil { + t.Fatalf("worktree content: %v", err) + } + if _, err := os.Stat(filepath.Join(rec.Path, "two.txt")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("worktree is not pinned to the first commit: %v", err) + } + // Records are under Dir and the index has the create. + if _, err := os.Stat(filepath.Join(svc.cfg.Dir, rec.ID+".json")); err != nil { + t.Fatal(err) + } + got, err := svc.Get(rec.ID) + if err != nil || got.ID != rec.ID || got.Path != rec.Path || got.Source != rec.Source { + t.Fatalf("Get = %+v, %v", got, err) + } + + // A second checkout for the same (repo, owner) reuses the clone; the + // other commit is reachable without a fetch. + second, err := svc.Create(ctx, Request{Repo: repo, Commit: shas[1], Owner: "nous-42"}) + if err != nil { + t.Fatal(err) + } + if second.Source != rec.Source || second.Path == rec.Path { + t.Fatalf("second = %+v, first = %+v", second, rec) + } + // Another owner gets its own clone. + other, err := svc.Create(ctx, Request{Repo: repo, Commit: shas[1], Owner: "spotlights"}) + if err != nil { + t.Fatal(err) + } + if other.Source == rec.Source { + t.Fatal("owners must not share a clone") + } + + all, err := svc.List("") + if err != nil || len(all) != 3 { + t.Fatalf("List all = %d %v", len(all), err) + } + mine, err := svc.List("nous-42") + if err != nil || len(mine) != 2 { + t.Fatalf("List owner = %d %v", len(mine), err) + } + for i := 1; i < len(all); i++ { + if all[i-1].ID > all[i].ID { + t.Fatal("List is not ordered by id") + } + } + + if err := svc.Delete(ctx, rec.ID, "operator:operator"); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(rec.Path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("worktree still present: %v", err) + } + if _, err := svc.Get(rec.ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("Get after delete = %v", err) + } + if err := svc.Delete(ctx, rec.ID, "x"); !errors.Is(err, ErrNotFound) { + t.Fatalf("second delete = %v", err) + } + + index, err := os.ReadFile(filepath.Join(svc.cfg.Dir, IndexName)) + if err != nil { + t.Fatal(err) + } + lines := strings.Split(strings.TrimSpace(string(index)), "\n") + if len(lines) != 4 { + t.Fatalf("index has %d lines: %s", len(lines), index) + } + var last indexEntry + if err := json.Unmarshal([]byte(lines[3]), &last); err != nil { + t.Fatal(err) + } + if last.Op != "delete" || last.ID != rec.ID || last.By != "operator:operator" || last.Owner != "nous-42" { + t.Fatalf("last index entry = %+v", last) + } +} + +func TestCreateRefusals(t *testing.T) { + repo, shas := fixtureRepo(t) + svc := newTestService(t, repo, nil) + ctx := context.Background() + ok := Request{Repo: repo, Commit: shas[0], Owner: "o"} + cases := []struct { + name string + edit func(*Request) + want error + msg string + }{ + {"short sha", func(r *Request) { r.Commit = shas[0][:12] }, ErrBadRequest, "full 40-hex SHA"}, + {"branch", func(r *Request) { r.Commit = "main" }, ErrBadRequest, "not a tag or branch"}, + {"tag-like", func(r *Request) { r.Commit = "v1.0.0" }, ErrBadRequest, "full 40-hex SHA"}, + {"missing owner", func(r *Request) { r.Owner = "" }, ErrBadRequest, "owner"}, + {"bad owner", func(r *Request) { r.Owner = "Has Spaces" }, ErrBadRequest, "owner"}, + {"long owner", func(r *Request) { r.Owner = strings.Repeat("a", MaxOwnerLen+1) }, ErrBadRequest, "owner"}, + {"negative cap", func(r *Request) { r.MaxBytes = -1 }, ErrBadRequest, "max_bytes"}, + {"negative ttl", func(r *Request) { r.TTL = -time.Second }, ErrBadRequest, "ttl"}, + {"ssh url", func(r *Request) { r.Repo = "git@github.com:org/name.git" }, ErrBadRequest, "https:// or file://"}, + {"http url", func(r *Request) { r.Repo = "http://github.com/org/name.git" }, ErrBadRequest, "https:// or file://"}, + {"credentials in url", func(r *Request) { r.Repo = "https://user:tok@github.com/org/name.git" }, ErrBadRequest, "credentials"}, + {"query in url", func(r *Request) { r.Repo = "https://github.com/org/name.git?x=1" }, ErrBadRequest, "query"}, + {"no host", func(r *Request) { r.Repo = "https:///name.git" }, ErrBadRequest, "no host"}, + {"off allowlist", func(r *Request) { r.Repo = "https://github.com/org/name.git" }, ErrForbidden, "matches none of the allowed prefixes"}, + {"unknown commit", func(r *Request) { r.Commit = strings.Repeat("0", 40) }, ErrBadRequest, "could not be fetched"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + req := ok + tc.edit(&req) + _, err := svc.Create(ctx, req) + if !errors.Is(err, tc.want) || !strings.Contains(err.Error(), tc.msg) { + t.Fatalf("Create = %v, want %v containing %q", err, tc.want, tc.msg) + } + }) + } + // The forbidden error names the rule set that denied it, and an empty + // allowlist denies everything with its own message. + if _, err := svc.Allowed("https://evil.example/x.git"); err == nil || !strings.Contains(err.Error(), svc.cfg.AllowedRepos[0]) { + t.Fatalf("Allowed off-list = %v", err) + } + if rule, err := svc.Allowed(repo); err != nil || rule != svc.cfg.AllowedRepos[0] { + t.Fatalf("Allowed on-list = %q %v", rule, err) + } + empty := newTestService(t, repo, func(c *Config) { c.AllowedRepos = nil }) + if _, err := empty.Create(ctx, ok); !errors.Is(err, ErrForbidden) || !strings.Contains(err.Error(), "allowlist is empty") { + t.Fatalf("empty allowlist = %v", err) + } + // Nothing was written for refused requests. + if list, err := svc.List(""); err != nil || len(list) != 0 { + t.Fatalf("records after refusals = %v %v", list, err) + } +} + +func TestNewValidatesConfig(t *testing.T) { + if _, err := New(Config{}); !errors.Is(err, ErrBadRequest) { + t.Fatalf("New without dir = %v", err) + } + if _, err := New(Config{Dir: t.TempDir(), AllowedRepos: []string{"github.com/org/"}}); !errors.Is(err, ErrBadRequest) || !strings.Contains(err.Error(), "allowlist entry") { + t.Fatalf("New with bare prefix = %v", err) + } + svc, err := New(Config{Dir: t.TempDir(), AllowedRepos: []string{" https://github.com/org/ "}}) + if err != nil || svc.cfg.AllowedRepos[0] != "https://github.com/org/" || svc.cfg.MaxCheckouts != DefaultMaxCheckouts { + t.Fatalf("New = %+v %v", svc, err) + } + if svc.InUse("chk_000000000000") { + t.Fatal("nil InUse hook must report not in use") + } + if DefaultDir("/data/agents") != "/data/agents/checkouts" { + t.Fatal(DefaultDir("/data/agents")) + } + for id, want := range map[string]bool{"chk_0123456789ab": true, "chk_0123456789aB": true, "chk_0123456789": false, "run_0123456789ab": false, "chk_../../etc/pw": false, "": false} { + if ValidID(id) != want { + t.Errorf("ValidID(%q) = %v", id, !want) + } + } +} + +func TestSizeCapIsPostHocAndRefuses(t *testing.T) { + repo, shas := fixtureRepo(t) + svc := newTestService(t, repo, nil) + ctx := context.Background() + _, err := svc.Create(ctx, Request{Repo: repo, Commit: shas[0], Owner: "o", MaxBytes: 1}) + if !errors.Is(err, ErrTooLarge) || !strings.Contains(err.Error(), "cap is 1") || !strings.Contains(err.Error(), "after cloning") { + t.Fatalf("over cap = %v", err) + } + // The over-cap worktree is gone and nothing was recorded; the shared + // clone stays for the next request. + if list, _ := svc.List(""); len(list) != 0 { + t.Fatalf("records after refusal: %v", list) + } + if entries, _ := os.ReadDir(filepath.Join(svc.cfg.Dir, "runs")); len(entries) != 0 { + for _, e := range entries { + if sub, _ := os.ReadDir(filepath.Join(svc.cfg.Dir, "runs", e.Name())); len(sub) != 0 { + t.Fatalf("worktree left behind: %v", sub) + } + } + } + + // A cap that cannot be measured is refused as unsupported, not skipped. + broken := newTestService(t, repo, func(c *Config) { + c.DiskUsage = func(string) (int64, error) { return 0, errors.New("no du") } + }) + if _, err := broken.Create(ctx, Request{Repo: repo, Commit: shas[0], Owner: "o"}); !errors.Is(err, ErrUnsupported) { + t.Fatalf("unmeasurable = %v", err) + } + // Measuring only the worktree failing is the same refusal. + calls := 0 + broken = newTestService(t, repo, func(c *Config) { + c.DiskUsage = func(path string) (int64, error) { + calls++ + if calls == 2 { + return 0, errors.New("no du") + } + return 1, nil + } + }) + if _, err := broken.Create(ctx, Request{Repo: repo, Commit: shas[0], Owner: "o"}); !errors.Is(err, ErrUnsupported) { + t.Fatalf("unmeasurable worktree = %v", err) + } + if _, err := diskUsage(filepath.Join(t.TempDir(), "missing")); err == nil { + t.Fatal("diskUsage of a missing path must fail") + } +} + +func TestDeleteOwnershipAndInUse(t *testing.T) { + repo, shas := fixtureRepo(t) + inUse := map[string]bool{} + svc := newTestService(t, repo, func(c *Config) { c.InUse = func(id string) bool { return inUse[id] } }) + ctx := context.Background() + rec, err := svc.Create(ctx, Request{Repo: repo, Commit: shas[0], Owner: "atlas"}) + if err != nil { + t.Fatal(err) + } + if !CanDelete(rec, "atlas", false) || CanDelete(rec, "beacon", false) || !CanDelete(rec, "beacon", true) || CanDelete(rec, "", false) { + t.Fatal("ownership rule") + } + inUse[rec.ID] = true + if err := svc.Delete(ctx, rec.ID, "project:atlas"); !errors.Is(err, ErrInUse) || !svc.InUse(rec.ID) { + t.Fatalf("delete in use = %v", err) + } + inUse[rec.ID] = false + if err := svc.Delete(ctx, rec.ID, "project:atlas"); err != nil { + t.Fatal(err) + } + if _, err := svc.Get("../escape"); !errors.Is(err, ErrNotFound) { + t.Fatalf("Get with a path = %v", err) + } + if err := svc.Delete(ctx, "chk_000000000000", "x"); !errors.Is(err, ErrNotFound) { + t.Fatalf("Delete unknown = %v", err) + } +} + +func TestGitFailuresAreTyped(t *testing.T) { + repo, shas := fixtureRepo(t) + ctx := context.Background() + real := execRunner + // Clone fails: typed ErrGit and the half-made source removed. + failClone := newTestService(t, repo, func(c *Config) { + c.Git = func(ctx context.Context, dir, name string, args ...string) ([]byte, error) { + if args[0] == "clone" { + return []byte("fatal: nope"), errors.New("exit 128") + } + return real(ctx, dir, name, args...) + } + }) + _, err := failClone.Create(ctx, Request{Repo: repo, Commit: shas[0], Owner: "o"}) + if !errors.Is(err, ErrGit) || !strings.Contains(err.Error(), "fatal: nope") { + t.Fatalf("clone failure = %v", err) + } + if entries, _ := os.ReadDir(filepath.Join(failClone.cfg.Dir, sourceDirName)); len(entries) != 0 { + t.Fatalf("source left behind: %v", entries) + } + + // Worktree add fails. + failTree := newTestService(t, repo, func(c *Config) { + c.Git = func(ctx context.Context, dir, name string, args ...string) ([]byte, error) { + if args[0] == "worktree" { + return []byte("boom"), errors.New("exit 1") + } + return real(ctx, dir, name, args...) + } + }) + if _, err := failTree.Create(ctx, Request{Repo: repo, Commit: shas[0], Owner: "o"}); !errors.Is(err, ErrGit) || !strings.Contains(err.Error(), "boom") { + t.Fatalf("worktree failure = %v", err) + } + + // HEAD disagrees with the pin: 502-class mismatch naming both SHAs. + mismatch := newTestService(t, repo, func(c *Config) { + c.Git = func(ctx context.Context, dir, name string, args ...string) ([]byte, error) { + if args[0] == "rev-parse" { + return []byte(shas[1] + "\n"), nil + } + return real(ctx, dir, name, args...) + } + }) + _, err = mismatch.Create(ctx, Request{Repo: repo, Commit: shas[0], Owner: "o"}) + if !errors.Is(err, ErrMismatch) || !strings.Contains(err.Error(), shas[0]) || !strings.Contains(err.Error(), shas[1]) { + t.Fatalf("mismatch = %v", err) + } + + // A fetch that "succeeds" without delivering the commit is still a + // bad request, not a checkout at the wrong commit. + phantom := newTestService(t, repo, func(c *Config) { + c.Git = func(ctx context.Context, dir, name string, args ...string) ([]byte, error) { + if args[0] == "fetch" { + return nil, nil + } + return real(ctx, dir, name, args...) + } + }) + if _, err := phantom.Create(ctx, Request{Repo: repo, Commit: strings.Repeat("a", 40), Owner: "o"}); !errors.Is(err, ErrBadRequest) || !strings.Contains(err.Error(), "is not in") { + t.Fatalf("phantom fetch = %v", err) + } + + // Single-branch clones are requested when branches is false. + var cloneArgs []string + single := newTestService(t, repo, func(c *Config) { + c.Git = func(ctx context.Context, dir, name string, args ...string) ([]byte, error) { + if args[0] == "clone" { + cloneArgs = args + } + return real(ctx, dir, name, args...) + } + }) + no := false + rec, err := single.Create(ctx, Request{Repo: repo, Commit: shas[1], Owner: "o", Branches: &no}) + if err != nil || rec.Branches { + t.Fatalf("single branch = %+v %v", rec, err) + } + if !contains(cloneArgs, "--single-branch") { + t.Fatalf("clone args = %v", cloneArgs) + } + + // Removing a worktree whose git state is gone falls back to deleting + // the directory; a runner that fails on remove is reported. + failRemove := newTestService(t, repo, func(c *Config) { + c.Git = func(ctx context.Context, dir, name string, args ...string) ([]byte, error) { + if args[0] == "worktree" && args[1] == "remove" { + return nil, errors.New("locked") + } + return real(ctx, dir, name, args...) + } + }) + rec, err = failRemove.Create(ctx, Request{Repo: repo, Commit: shas[0], Owner: "o"}) + if err != nil { + t.Fatal(err) + } + if err := failRemove.Delete(ctx, rec.ID, "x"); err != nil { + t.Fatalf("delete with failing git remove should fall back to os.RemoveAll: %v", err) + } + if _, err := os.Stat(rec.Path); !errors.Is(err, os.ErrNotExist) { + t.Fatal("worktree directory still present") + } +} + +func TestStoreCorruptionAndIO(t *testing.T) { + dir := t.TempDir() + st := &store{dir: dir} + id := "chk_0123456789ab" + if err := os.WriteFile(st.recordPath(id), []byte("{not json"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := st.get(id); err == nil || !strings.Contains(err.Error(), "corrupt record") { + t.Fatalf("corrupt get = %v", err) + } + if _, err := st.list(); err == nil { + t.Fatal("list must surface a corrupt record") + } + // Non-record files and directories under Dir are ignored by list. + if err := os.Remove(st.recordPath(id)); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "notes.json"), []byte("{}"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(dir, "runs"), 0o700); err != nil { + t.Fatal(err) + } + if list, err := st.list(); err != nil || len(list) != 0 { + t.Fatalf("list = %v %v", list, err) + } + if list, err := (&store{dir: filepath.Join(dir, "absent")}).list(); err != nil || len(list) != 0 { + t.Fatalf("list of absent dir = %v %v", list, err) + } + if err := st.remove("chk_ffffffffffff"); err != nil { + t.Fatalf("remove of absent record = %v", err) + } + // Writes into an unwritable dir fail loudly rather than silently. + ro := &store{dir: filepath.Join(dir, "absent")} + if err := ro.put(Record{ID: id}); err == nil { + t.Fatal("put into a missing dir must fail") + } + if err := ro.appendIndex(indexEntry{Op: "create"}); err == nil { + t.Fatal("appendIndex into a missing dir must fail") + } + if _, err := (&store{dir: filepath.Join(dir, "runs")}).list(); err != nil { + t.Fatal(err) + } + // A directory where the record file should be is a read error. + if err := os.MkdirAll(st.recordPath("chk_aaaaaaaaaaaa"), 0o700); err != nil { + t.Fatal(err) + } + if _, err := st.get("chk_aaaaaaaaaaaa"); err == nil || errors.Is(err, ErrNotFound) { + t.Fatalf("get of a directory = %v", err) + } +} + +func contains(list []string, want string) bool { + for _, s := range list { + if s == want { + return true + } + } + return false +} diff --git a/src/crux/checkout/store.go b/src/crux/checkout/store.go new file mode 100644 index 0000000000..fccce05656 --- /dev/null +++ b/src/crux/checkout/store.go @@ -0,0 +1,107 @@ +package checkout + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +// store is the record layout: Dir/.json per checkout and Dir/index.jsonl +// as the append-only create/delete history. +type store struct{ dir string } + +// indexEntry is one line of index.jsonl. +type indexEntry struct { + Op string `json:"op"` + ID string `json:"id"` + Owner string `json:"owner"` + Repo string `json:"repo"` + Commit string `json:"commit"` + At time.Time `json:"at"` + By string `json:"by,omitempty"` +} + +func (st *store) recordPath(id string) string { return filepath.Join(st.dir, id+".json") } + +func (st *store) put(rec Record) error { + data, err := json.Marshal(rec) + if err != nil { + return err + } + tmp := st.recordPath(rec.ID) + ".tmp" + if err := os.WriteFile(tmp, data, recordMode); err != nil { + return err + } + return os.Rename(tmp, st.recordPath(rec.ID)) +} + +func (st *store) get(id string) (Record, error) { + if !ValidID(id) { + return Record{}, fmt.Errorf("%w: %q is not a checkout id", ErrNotFound, id) + } + data, err := os.ReadFile(st.recordPath(id)) + if errors.Is(err, os.ErrNotExist) { + return Record{}, fmt.Errorf("%w: checkout %s", ErrNotFound, id) + } + if err != nil { + return Record{}, err + } + var rec Record + if err := json.Unmarshal(data, &rec); err != nil { + return Record{}, fmt.Errorf("checkout %s: corrupt record: %w", id, err) + } + return rec, nil +} + +func (st *store) remove(id string) error { + err := os.Remove(st.recordPath(id)) + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err +} + +func (st *store) list() ([]Record, error) { + entries, err := os.ReadDir(st.dir) + if errors.Is(err, os.ErrNotExist) { + return []Record{}, nil + } + if err != nil { + return nil, err + } + out := []Record{} + for _, e := range entries { + id, ok := strings.CutSuffix(e.Name(), ".json") + if !ok || e.IsDir() || !ValidID(id) { + continue + } + rec, err := st.get(id) + if err != nil { + return nil, err + } + out = append(out, rec) + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out, nil +} + +func (st *store) appendIndex(entry indexEntry) error { + data, err := json.Marshal(entry) + if err != nil { + return err + } + f, err := os.OpenFile(filepath.Join(st.dir, IndexName), os.O_APPEND|os.O_CREATE|os.O_WRONLY, recordMode) + if err != nil { + return err + } + if _, err := f.Write(append(data, '\n')); err != nil { + _ = f.Close() + return err + } + return f.Close() +} diff --git a/src/crux/cli/checkout.go b/src/crux/cli/checkout.go new file mode 100644 index 0000000000..459d416834 --- /dev/null +++ b/src/crux/cli/checkout.go @@ -0,0 +1,255 @@ +package cli + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + "strings" + "text/tabwriter" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/checkout" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/serve" +) + +// `crux checkout add|rm|list` is the pinned-checkout work service +// (hivecommons/crux#114 S1). It does not touch crux.Runtime: locally it +// drives checkout.Service on /checkouts as the operator; with +// --server it drives the /v1/checkouts routes as whoever the token is. + +const checkoutUsage = "usage: crux checkout add -repo URL -commit SHA [-owner O] [-ttl 72h] [-max-bytes N] [-no-branches] | rm ID | list [-owner O]" + +// checkoutClient is the one shape the local and remote paths share. +type checkoutClient interface { + CreateCheckout(ctx context.Context, in api.CheckoutRequest) (api.Checkout, error) + ListCheckouts(ctx context.Context, owner string) ([]api.Checkout, error) + DeleteCheckout(ctx context.Context, id string) error +} + +// localCheckouts adapts checkout.Service to checkoutClient for the +// in-process path, where the caller is the operator. +type localCheckouts struct{ svc *checkout.Service } + +func (l localCheckouts) CreateCheckout(ctx context.Context, in api.CheckoutRequest) (api.Checkout, error) { + req := checkout.Request{Repo: in.Repo, Commit: in.Commit, Owner: in.Owner, Branches: in.Branches, MaxBytes: in.MaxBytes} + if req.Owner == "" { + req.Owner = serve.DefaultOperatorName + } + if in.TTL != "" { + ttl, err := time.ParseDuration(in.TTL) + if err != nil { + return api.Checkout{}, fmt.Errorf("%w: ttl: %v", checkout.ErrBadRequest, err) + } + req.TTL = ttl + } + rec, err := l.svc.Create(ctx, req) + return checkoutRecordToAPI(rec), err +} + +func (l localCheckouts) ListCheckouts(_ context.Context, owner string) ([]api.Checkout, error) { + recs, err := l.svc.List(owner) + out := make([]api.Checkout, 0, len(recs)) + for _, rec := range recs { + out = append(out, checkoutRecordToAPI(rec)) + } + return out, err +} + +func (l localCheckouts) DeleteCheckout(ctx context.Context, id string) error { + return l.svc.Delete(ctx, id, serve.KindOperator+":"+serve.DefaultOperatorName) +} + +func checkoutRecordToAPI(rec checkout.Record) api.Checkout { + return api.Checkout{ID: rec.ID, Path: rec.Path, Repo: rec.Repo, Commit: rec.Commit, Owner: rec.Owner, Branches: rec.Branches, + MaxBytes: rec.MaxBytes, Bytes: rec.Bytes, TTL: rec.TTL, CreatedAt: rec.CreatedAt, ExpiresAt: rec.ExpiresAt} +} + +// newCheckoutService builds the service from the flag or environment +// allowlist; nil with no error means no allowlist was given, so the +// service stays off. +func newCheckoutService(allow, dir, workDir string) (*checkout.Service, error) { + if allow == "" { + allow = os.Getenv(config.CheckoutAllowlistEnvVar) + } + if strings.TrimSpace(allow) == "" { + return nil, nil + } + if dir == "" { + dir = os.Getenv(config.CheckoutDirEnvVar) + } + if dir == "" { + dir = checkout.DefaultDir(workDir) + } + var prefixes []string + for _, p := range strings.Split(allow, ",") { + if p = strings.TrimSpace(p); p != "" { + prefixes = append(prefixes, p) + } + } + return checkout.New(checkout.Config{Dir: dir, AllowedRepos: prefixes}) +} + +// cmdCheckout is the local `crux checkout`: the service on . +func (e *Env) cmdCheckout(ctx context.Context, opts globalOptions, args []string) int { + if len(args) == 0 { + fmt.Fprintln(e.Stderr, checkoutUsage) + return ExitUsage + } + fs := e.subFlags("checkout " + args[0]) + allow := fs.String("allow", "", "comma-separated repo URL prefixes this command may clone (default $"+config.CheckoutAllowlistEnvVar+")") + dir := fs.String("dir", "", "where checkouts live (default $"+config.CheckoutDirEnvVar+", then /"+checkout.DirName+")") + return e.checkout(ctx, func() (checkoutClient, error) { + svc, err := newCheckoutService(*allow, *dir, opts.workDir) + if err == nil && svc == nil { + // No allowlist: a service that refuses every clone but can + // still list and remove what exists. + svc, err = checkout.New(checkout.Config{Dir: checkoutDirFor(*dir, opts.workDir)}) + } + if err != nil { + return nil, err + } + return localCheckouts{svc}, nil + }, fs, args) +} + +func checkoutDirFor(dir, workDir string) string { + if dir == "" { + dir = os.Getenv(config.CheckoutDirEnvVar) + } + if dir == "" { + dir = checkout.DefaultDir(workDir) + } + return dir +} + +// remoteCheckout is `crux --server … checkout`. +func (e *Env) remoteCheckout(ctx context.Context, c *api.Client, args []string) int { + if len(args) == 0 { + fmt.Fprintln(e.Stderr, checkoutUsage) + return ExitUsage + } + return e.checkout(ctx, func() (checkoutClient, error) { return c, nil }, e.subFlags("checkout "+args[0]), args) +} + +// checkout dispatches the add/rm/list verb. fs may already carry the +// local-only flags; the verb's own flags are added here, and newClient runs +// after parsing so those flags are in effect. +func (e *Env) checkout(ctx context.Context, newClient func() (checkoutClient, error), fs *flag.FlagSet, args []string) int { + verb, rest := args[0], args[1:] + client := func() (checkoutClient, bool) { + c, err := newClient() + if err != nil { + fmt.Fprintln(e.Stderr, "crux: checkout:", err) + } + return c, err == nil + } + switch verb { + case "add": + var in api.CheckoutRequest + var noBranches bool + fs.StringVar(&in.Repo, "repo", "", "repository URL (must match the server's allowlist)") + fs.StringVar(&in.Commit, "commit", "", "full 40-hex commit SHA to pin") + fs.StringVar(&in.Owner, "owner", "", "owning run or driver (default: the caller)") + fs.StringVar(&in.TTL, "ttl", "", "duration after which a sweeper may reclaim it, e.g. 72h") + fs.Int64Var(&in.MaxBytes, "max-bytes", 0, "size cap for clone plus worktree (default 5 GiB); checked after cloning") + fs.BoolVar(&noBranches, "no-branches", false, "clone a single branch instead of all refs") + asJSON := fs.Bool("json", false, "emit the record as JSON") + if code, ok := parseSub(fs, rest); !ok { + return code + } + if in.Repo == "" || in.Commit == "" || fs.NArg() != 0 { + fmt.Fprintln(e.Stderr, checkoutUsage) + return ExitUsage + } + if noBranches { + f := false + in.Branches = &f + } + c, ok := client() + if !ok { + return ExitUsage + } + rec, err := c.CreateCheckout(ctx, in) + if err != nil { + return e.checkoutErr("add", err) + } + if *asJSON { + return e.writeJSON(rec) + } + fmt.Fprintf(e.Stdout, "%s\t%s\n", rec.ID, rec.Path) + return ExitOK + case "rm": + if code, ok := parseSub(fs, rest); !ok { + return code + } + if fs.NArg() != 1 { + fmt.Fprintln(e.Stderr, checkoutUsage) + return ExitUsage + } + c, ok := client() + if !ok { + return ExitUsage + } + if err := c.DeleteCheckout(ctx, fs.Arg(0)); err != nil { + return e.checkoutErr("rm", err) + } + return ExitOK + case "list": + owner := fs.String("owner", "", "only this owner's checkouts") + asJSON := fs.Bool("json", false, "emit a JSON array instead of a table") + if code, ok := parseSub(fs, rest); !ok { + return code + } + if fs.NArg() != 0 { + fmt.Fprintln(e.Stderr, checkoutUsage) + return ExitUsage + } + c, ok := client() + if !ok { + return ExitUsage + } + recs, err := c.ListCheckouts(ctx, *owner) + if err != nil { + return e.checkoutErr("list", err) + } + if *asJSON { + return e.writeJSON(recs) + } + tw := tabwriter.NewWriter(e.Stdout, 0, 0, 2, ' ', 0) + fmt.Fprintln(tw, "ID\tOWNER\tCOMMIT\tCREATED\tEXPIRES\tPATH") + for _, r := range recs { + expires := "-" + if r.ExpiresAt != nil { + expires = r.ExpiresAt.UTC().Format(time.RFC3339) + } + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\n", r.ID, r.Owner, r.Commit, r.CreatedAt.UTC().Format(time.RFC3339), expires, r.Path) + } + if err := tw.Flush(); err != nil { + fmt.Fprintln(e.Stderr, "crux:", err) + return ExitError + } + return ExitOK + default: + fmt.Fprintf(e.Stderr, "crux: checkout: unknown verb %q\n%s\n", verb, checkoutUsage) + return ExitUsage + } +} + +// checkoutErr prints a failure; malformed requests and refusals are usage +// errors, everything else a runtime failure. +func (e *Env) checkoutErr(verb string, err error) int { + fmt.Fprintf(e.Stderr, "crux: checkout %s: %v\n", verb, err) + var se *api.StatusError + if errors.Is(err, checkout.ErrBadRequest) || errors.Is(err, checkout.ErrForbidden) || + (errors.As(err, &se) && (se.Body.Code == api.CodeBadRequest || se.Body.Code == api.CodeForbidden)) { + return ExitUsage + } + if api.IsUnauthorized(err) { + fmt.Fprintf(e.Stderr, "crux: the server wants a bearer token; pass --token or set %s\n", config.ServeTokenEnvVar) + } + return ExitError +} diff --git a/src/crux/cli/checkout_test.go b/src/crux/cli/checkout_test.go new file mode 100644 index 0000000000..7e32fd3c8c --- /dev/null +++ b/src/crux/cli/checkout_test.go @@ -0,0 +1,209 @@ +package cli + +import ( + "encoding/json" + "net" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/checkout" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/serve" +) + +// checkoutFixtureRepo is a bare repo with one commit, as a file:// URL, +// plus the allowlist prefix that admits it. +func checkoutFixtureRepo(t *testing.T) (repo, sha, allow string) { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + work := filepath.Join(t.TempDir(), "work") + if err := os.MkdirAll(work, 0o700); err != nil { + t.Fatal(err) + } + git := func(args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = work + cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@x", "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@x") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + git("init", "-q") + if err := os.WriteFile(filepath.Join(work, "f"), []byte("f"), 0o600); err != nil { + t.Fatal(err) + } + git("add", "f") + git("commit", "-q", "-m", "c") + sha = git("rev-parse", "HEAD") + bare := filepath.Join(t.TempDir(), "fixture.git") + git("clone", "-q", "--bare", work, bare) + return "file://" + bare, sha, "file://" + filepath.Dir(bare) +} + +func TestCheckoutLocalAddListRm(t *testing.T) { + repo, sha, allow := checkoutFixtureRepo(t) + t.Setenv(config.CheckoutAllowlistEnvVar, "") + t.Setenv(config.CheckoutDirEnvVar, "") + h := newHarness(t) + dir := filepath.Join(t.TempDir(), "chk") + + if code := h.run("checkout", "add", "-allow", allow, "-dir", dir, "-repo", repo, "-commit", sha, "-ttl", "1h"); code != ExitOK { + t.Fatalf("add: %d %s", code, h.stderr) + } + fields := strings.Fields(h.stdout.String()) + if len(fields) != 2 || !strings.HasPrefix(fields[0], checkout.IDPrefix) || !strings.HasPrefix(fields[1], dir) { + t.Fatalf("add output = %q", h.stdout.String()) + } + id := fields[0] + if _, err := os.Stat(filepath.Join(fields[1], "f")); err != nil { + t.Fatalf("pinned file: %v", err) + } + + // Without an allowlist the local service still lists and removes. + h.stdout.Reset() + if code := h.run("checkout", "list", "-dir", dir); code != ExitOK { + t.Fatalf("list: %d %s", code, h.stderr) + } + if out := h.stdout.String(); !strings.Contains(out, "ID") || !strings.Contains(out, id) || !strings.Contains(out, serve.DefaultOperatorName) || !strings.Contains(out, sha) { + t.Fatalf("list output = %q", out) + } + h.stdout.Reset() + if code := h.run("checkout", "list", "-dir", dir, "-owner", "nobody", "-json"); code != ExitOK || strings.TrimSpace(h.stdout.String()) != "[]" { + t.Fatalf("list nobody: %d %q", code, h.stdout.String()) + } + h.stdout.Reset() + if code := h.run("checkout", "add", "-dir", dir, "-repo", repo, "-commit", sha); code != ExitUsage || !strings.Contains(h.stderr.String(), "allowlist is empty") { + t.Fatalf("add without allowlist: %d %s", code, h.stderr) + } + h.stderr.Reset() + if code := h.run("checkout", "add", "-allow", allow, "-dir", dir, "-repo", repo, "-commit", "main", "-json"); code != ExitUsage || !strings.Contains(h.stderr.String(), "40-hex") { + t.Fatalf("add branch: %d %s", code, h.stderr) + } + h.stderr.Reset() + if code := h.run("checkout", "rm", "-dir", dir, id); code != ExitOK { + t.Fatalf("rm: %d %s", code, h.stderr) + } + if code := h.run("checkout", "rm", "-dir", dir, id); code != ExitError || !strings.Contains(h.stderr.String(), "not found") { + t.Fatalf("rm again: %d %s", code, h.stderr) + } + + // The allowlist and dir also come from the environment, and the dir + // defaults under the work dir. + t.Setenv(config.CheckoutAllowlistEnvVar, allow) + work := t.TempDir() + h.stdout.Reset() + if code := h.env.Run(t.Context(), []string{"--workdir", work, "checkout", "add", "-repo", repo, "-commit", sha, "-no-branches", "-json"}); code != ExitOK { + t.Fatalf("add via env: %d %s", code, h.stderr) + } + var rec api.Checkout + if err := json.Unmarshal(h.stdout.Bytes(), &rec); err != nil || !strings.HasPrefix(rec.Path, checkout.DefaultDir(work)) || rec.Branches { + t.Fatalf("add via env = %s %v", h.stdout.String(), err) + } + if got := checkoutDirFor("", work); got != checkout.DefaultDir(work) { + t.Fatalf("checkoutDirFor = %q", got) + } + t.Setenv(config.CheckoutDirEnvVar, "/elsewhere") + if got := checkoutDirFor("", work); got != "/elsewhere" { + t.Fatalf("checkoutDirFor from env = %q", got) + } + if got := checkoutDirFor("/flag", work); got != "/flag" { + t.Fatalf("checkoutDirFor from flag = %q", got) + } +} + +func TestCheckoutUsageErrors(t *testing.T) { + t.Setenv(config.CheckoutAllowlistEnvVar, "") + h := newHarness(t) + cases := [][]string{ + {"checkout"}, + {"checkout", "bogus"}, + {"checkout", "add"}, + {"checkout", "add", "-repo", "x"}, + {"checkout", "add", "-repo", "x", "-commit", "y", "extra"}, + {"checkout", "rm"}, + {"checkout", "rm", "a", "b"}, + {"checkout", "list", "extra"}, + {"checkout", "list", "-nope"}, + } + for _, args := range cases { + h.stderr.Reset() + if code := h.run(args...); code != ExitUsage { + t.Errorf("%v: code %d, want usage; stderr %s", args, code, h.stderr) + } + } + h.stderr.Reset() + if code := h.run("checkout", "list", "-h"); code != ExitOK { + t.Errorf("list -h: %d", code) + } + if code := h.run("checkout", "add", "-allow", "ssh://bad", "-repo", "https://x/y", "-commit", strings.Repeat("a", 40)); code != ExitUsage || !strings.Contains(h.stderr.String(), "crux: checkout:") { + t.Errorf("bad allowlist: %d %s", code, h.stderr) + } + // The control socket does not forward checkout. + e := &Env{Stdout: h.stdout, Stderr: h.stderr} + h.stderr.Reset() + if code := e.dispatch(t.Context(), globalOptions{}, "checkout", []string{"list"}, true); code != ExitUsage || !strings.Contains(h.stderr.String(), "control socket") { + t.Errorf("remote dispatch: %d %s", code, h.stderr) + } +} + +func TestCheckoutRemote(t *testing.T) { + repo, sha, allow := checkoutFixtureRepo(t) + t.Setenv(config.ServerEnvVar, "") + t.Setenv(config.ServeTokenEnvVar, "") + svc, err := checkout.New(checkout.Config{Dir: filepath.Join(t.TempDir(), "chk"), AllowedRepos: []string{allow}}) + if err != nil { + t.Fatal(err) + } + s := serve.New(newFakeRuntime()) + s.Token, s.Checkouts = "tok", svc + path := filepath.Join(shortSocketDir(t), "api.sock") + ln, err := net.Listen("unix", path) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewUnstartedServer(s.Handler()) + srv.Listener = ln + srv.Start() + t.Cleanup(srv.Close) + + h := newHarness(t) + remote := func(args ...string) int { + return h.run(append([]string{"--server", path, "--token", "tok", "checkout"}, args...)...) + } + if code := remote(); code != ExitUsage { + t.Fatalf("no verb: %d", code) + } + if code := remote("add", "-repo", repo, "-commit", sha, "-owner", "atlas"); code != ExitOK { + t.Fatalf("add: %d %s", code, h.stderr) + } + id := strings.Fields(h.stdout.String())[0] + h.stdout.Reset() + if code := remote("list", "-owner", "atlas"); code != ExitOK || !strings.Contains(h.stdout.String(), id) { + t.Fatalf("list: %d %q %s", code, h.stdout.String(), h.stderr) + } + h.stderr.Reset() + if code := remote("add", "-repo", "https://github.com/o/r", "-commit", sha); code != ExitUsage || !strings.Contains(h.stderr.String(), "allowed prefixes") { + t.Fatalf("off allowlist: %d %s", code, h.stderr) + } + if code := remote("rm", id); code != ExitOK { + t.Fatalf("rm: %d %s", code, h.stderr) + } + h.stderr.Reset() + if code := remote("rm", id); code != ExitError || !strings.Contains(h.stderr.String(), "not_found") && !strings.Contains(h.stderr.String(), "not found") { + t.Fatalf("rm again: %d %s", code, h.stderr) + } + h.stderr.Reset() + if code := h.run("--server", path, "--token", "wrong", "checkout", "list"); code != ExitError || !strings.Contains(h.stderr.String(), "bearer token") { + t.Fatalf("wrong token: %d %s", code, h.stderr) + } +} diff --git a/src/crux/cli/cli.go b/src/crux/cli/cli.go index e1cf7eda33..b3879de2b5 100644 --- a/src/crux/cli/cli.go +++ b/src/crux/cli/cli.go @@ -196,6 +196,12 @@ func (e *Env) dispatch(ctx context.Context, opts globalOptions, name string, arg return ExitUsage } return cline.Run(args, os.Stdin, e.Stdout, e.Stderr) + case "checkout": + if remote { + fmt.Fprintln(e.Stderr, "crux: checkout is not available over the control socket") + return ExitUsage + } + return e.cmdCheckout(ctx, opts, args) case "ps", "logs", "kick", "stop": if !remote && !opts.local { if code, ok := e.forward(ctx, opts, append([]string{name}, args...)); ok { @@ -334,11 +340,16 @@ Commands (each maps onto crux.Runtime): cline-turn one bounded Cline CLI turn (stdin prompt); --model provider/model, --check for preflight +Work services (no runtime; hivecommons/crux#114): + checkout add|rm|list pinned checkouts under /checkouts + (add -repo URL -commit SHA; -allow PREFIXES or + $CRUX_CHECKOUT_ALLOWLIST gates which repos) + While 'crux run' is in the foreground it serves /crux.sock; ps, logs, kick and stop use that socket when present so they act on the live runtime. 'crux serve' is the API server other harnesses drive (crux/api, crux/serve). -With --server ADDR (or CRUX_SERVER set) run, ps, logs, kick and stop drive that -server over the API instead of an in-process runtime; --token / CRUX_TOKEN +With --server ADDR (or CRUX_SERVER set) run, ps, logs, kick, stop and checkout +drive that server over the API instead of an in-process runtime; --token / CRUX_TOKEN carries its bearer token, and 'logs -f' follows the agent's output. 'serve' itself always runs locally, whatever CRUX_SERVER says. diff --git a/src/crux/cli/remote.go b/src/crux/cli/remote.go index 1a2c7bbcae..20882c6a90 100644 --- a/src/crux/cli/remote.go +++ b/src/crux/cli/remote.go @@ -51,6 +51,8 @@ func (e *Env) remote(ctx context.Context, opts globalOptions, name string, args return e.remoteKick(ctx, c, args) case "stop": return e.remoteStop(ctx, c, args) + case "checkout": + return e.remoteCheckout(ctx, c, args) default: fmt.Fprintf(e.Stderr, "crux: %q is not available with --server; it runs where the runtime is\n", name) return ExitUsage diff --git a/src/crux/cli/serve.go b/src/crux/cli/serve.go index 5549ed9496..104848eccf 100644 --- a/src/crux/cli/serve.go +++ b/src/crux/cli/serve.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/hivecommons/hive/crux/api/callback" + "github.com/hivecommons/hive/crux/checkout" "github.com/hivecommons/hive/crux/config" "github.com/hivecommons/hive/crux/lanes" "github.com/hivecommons/hive/crux/serve" @@ -33,13 +34,18 @@ func (e *Env) cmdServe(ctx context.Context, opts globalOptions, args []string) i fs.StringVar(&cfg.Token, "token", "", "bearer token every request must carry (default $"+config.ServeTokenEnvVar+")") var hiveAddr string var laneConfig string + var operatorName string + var checkoutAllow, checkoutDir string + fs.StringVar(&checkoutAllow, "checkout-allow", "", "comma-separated repo URL prefixes the pinned-checkout service may clone (default $"+config.CheckoutAllowlistEnvVar+"; none disables the service)") + fs.StringVar(&checkoutDir, "checkout-dir", "", "where pinned checkouts live (default $"+config.CheckoutDirEnvVar+", then /"+checkout.DirName+")") + fs.StringVar(&operatorName, "operator-name", "", "name of the principal behind the control token on work-services routes (default $"+config.ServeOperatorNameEnvVar+", then "+serve.DefaultOperatorName+")") fs.StringVar(&laneConfig, "lanes-config", "", "opt-in durable lane configuration file (default $CRUX_LANES_CONFIG)") fs.StringVar(&hiveAddr, "hive-callback", "", "hive capability endpoint to dial (default $"+callback.SocketEnvVar+"; none when unset)") if code, ok := parseSub(fs, args); !ok { return code } if fs.NArg() != 0 { - fmt.Fprintln(e.Stderr, "usage: crux serve [-socket PATH] [-listen HOST:PORT] [-token T] [-hive-callback ADDR] [-lanes-config PATH]") + fmt.Fprintln(e.Stderr, "usage: crux serve [-socket PATH] [-listen HOST:PORT] [-token T] [-hive-callback ADDR] [-lanes-config PATH] [-operator-name NAME] [-checkout-allow PREFIXES] [-checkout-dir DIR]") return ExitUsage } if cfg.Token == "" { @@ -70,6 +76,16 @@ func (e *Env) cmdServe(ctx context.Context, opts globalOptions, args []string) i srv := serve.New(rt) srv.Token = cfg.Token + if operatorName == "" { + operatorName = os.Getenv(config.ServeOperatorNameEnvVar) + } + srv.OperatorName = operatorName + if checkouts, err := newCheckoutService(checkoutAllow, checkoutDir, opts.workDir); err != nil { + fmt.Fprintln(e.Stderr, "crux: checkouts:", err) + return ExitUsage + } else if checkouts != nil { + srv.Checkouts = checkouts + } srv.Logger = slog.New(slog.NewTextHandler(e.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})) if laneConfig == "" { laneConfig = os.Getenv("CRUX_LANES_CONFIG") diff --git a/src/crux/cli/verbs_test.go b/src/crux/cli/verbs_test.go index a32de5b68a..712e98ced8 100644 --- a/src/crux/cli/verbs_test.go +++ b/src/crux/cli/verbs_test.go @@ -15,7 +15,7 @@ import ( // nonRuntimeVerbs are dispatch cases that do not operate on a runtime: serve // exposes one, help prints usage, cline-turn executes one pane-side worker. -var nonRuntimeVerbs = map[string]bool{"serve": true, "help": true, "-h": true, "--help": true, "cline-turn": true} +var nonRuntimeVerbs = map[string]bool{"serve": true, "help": true, "-h": true, "--help": true, "cline-turn": true, "checkout": true} func TestVerbsHaveRuntimeMethods(t *testing.T) { rt := reflect.TypeOf((*crux.Runtime)(nil)).Elem() diff --git a/src/crux/config/serve.go b/src/crux/config/serve.go index 9971fa19e6..063f93a9d0 100644 --- a/src/crux/config/serve.go +++ b/src/crux/config/serve.go @@ -25,6 +25,17 @@ const ( // ServeTokenEnvVar is the bearer token the server requires and the client // presents. A TCP listener refuses to start without one. ServeTokenEnvVar = "CRUX_TOKEN" + // ServeOperatorNameEnvVar names the principal the control token stands + // for on the work-services routes (serve.CallerFrom); the server + // defaults it when unset. + ServeOperatorNameEnvVar = "CRUX_OPERATOR_NAME" + // CheckoutAllowlistEnvVar is the comma-separated list of repository URL + // prefixes the pinned-checkout work service may clone; unset or empty + // leaves the service off (every request is refused). + CheckoutAllowlistEnvVar = "CRUX_CHECKOUT_ALLOWLIST" + // CheckoutDirEnvVar overrides where pinned checkouts live (default + // /checkouts). + CheckoutDirEnvVar = "CRUX_CHECKOUT_DIR" // ServeSocketName is the socket file name under ServeSocketDir. ServeSocketName = "crux.sock" // ServeSocketDirName is the per-user directory under $XDG_RUNTIME_DIR or diff --git a/src/crux/issueshape/issueshape.go b/src/crux/issueshape/issueshape.go index dcec0abb70..e19660a887 100644 --- a/src/crux/issueshape/issueshape.go +++ b/src/crux/issueshape/issueshape.go @@ -12,6 +12,8 @@ package issueshape import ( "regexp" "strings" + + "github.com/hivecommons/hive/crux/spekmarker" ) var ( @@ -72,6 +74,19 @@ func UnsubstitutedTemplatePlaceholder(text string) (string, bool) { return "", false } +// SpekPublished reports whether body carries a valid Spek publish marker +// (``, see package +// spekmarker). Such a body was rendered from a finished spec by +// hivecommons/spektacular, so the clarity gate treats it as already specified +// — the same skip `approved-direction` grants — instead of parking it with +// `needs-spec` (hivecommons/spektacular#114). It is a clarity-scoring +// exemption only: the hard shape errors (UnsubstitutedTemplatePlaceholder, +// BodyHasMisEscapedNewlines) still apply to a marked body, because a marker +// pasted into an unfilled template proves nothing about the template. +func SpekPublished(body string) bool { + return spekmarker.Has(body) +} + // BodyHasMisEscapedNewlines reports whether body is a single physical line // whose markdown structure was encoded as literal backslash-n escape // sequences instead of real newlines (the classic `## X\n\n<...>\n\n## Y` diff --git a/src/crux/issueshape/issueshape_test.go b/src/crux/issueshape/issueshape_test.go index 37f47c2d0f..34dcd2584f 100644 --- a/src/crux/issueshape/issueshape_test.go +++ b/src/crux/issueshape/issueshape_test.go @@ -168,3 +168,25 @@ func TestStripAgentPrefixAndSameTitle(t *testing.T) { t.Fatal("different subjects must not match") } } + +func TestSpekPublished(t *testing.T) { + marked := "## Goal\n\nAdd a refresh endpoint.\n\n" + if !SpekPublished(marked) { + t.Fatal("body with a valid spek:publish marker not reported as published") + } + if SpekPublished("## Goal\n\nAdd a refresh endpoint.") { + t.Fatal("plain body reported as spek-published") + } + if SpekPublished("") { + t.Fatal("unsubstituted marker template reported as spek-published") + } + // The marker itself is not an unfilled template placeholder: its content + // carries `=` so the angle-bracket scan skips it. + if ph, ok := UnsubstitutedTemplatePlaceholder(marked); ok { + t.Fatalf("marker flagged as placeholder %q", ph) + } + // A marker does not excuse a template left unfilled around it. + if _, ok := UnsubstitutedTemplatePlaceholder("\n\n"); !ok { + t.Fatal("placeholder beside a marker must still be flagged") + } +} diff --git a/src/crux/proxy/canary_test.go b/src/crux/proxy/canary_test.go index 98909d297c..f60da4d5b4 100644 --- a/src/crux/proxy/canary_test.go +++ b/src/crux/proxy/canary_test.go @@ -1,6 +1,7 @@ package proxy import ( + "errors" "io" "log/slog" "net/http" @@ -140,6 +141,46 @@ func TestInspectCanaryEgressBlocksOversizedBodies(t *testing.T) { } } +// brokenBody fails partway through the read, the way a client that drops the +// connection mid-request does. +type brokenBody struct { + io.Reader + closed bool +} + +func (b *brokenBody) Read(p []byte) (int, error) { + n, err := b.Reader.Read(p) + if err == io.EOF { + return n, errors.New("unexpected EOF") + } + return n, err +} + +func (b *brokenBody) Close() error { b.closed = true; return nil } + +func TestInspectCanaryEgressDeniesUnreadableBodies(t *testing.T) { + r := newFakeCanaryScanner() + p := &GitHubProxy{logger: slog.Default()} + p.SetCanaryScanner(true, false, r, nil) + body := &brokenBody{Reader: strings.NewReader(`{"body":"partial`)} + req, _ := http.NewRequest(http.MethodPost, "https://api.github.com/repos/org/repo/issues/1/comments", nil) + req.Body = body + reason, deny, detected := p.inspectCanaryEgress("scanner", req) + if !detected || !deny || !strings.Contains(reason, "unreadable") { + t.Fatalf("got detected=%v deny=%v reason=%q, want fail-closed block", detected, deny, reason) + } + if len(r.seen) != 0 { + t.Fatalf("unreadable body was handed to the scanner") + } + if !body.closed { + t.Fatalf("original body was not closed") + } + rest, err := io.ReadAll(req.Body) + if err != nil || string(rest) != `{"body":"partial` { + t.Fatalf("request body was not restored: %q, %v", rest, err) + } +} + func TestInspectCanaryEgressCoversDirectPullCreatePath(t *testing.T) { r := newFakeCanaryScanner() tok := r.plant("quality") diff --git a/src/crux/proxy/github_proxy.go b/src/crux/proxy/github_proxy.go index 12d0b36a18..aa131f4065 100644 --- a/src/crux/proxy/github_proxy.go +++ b/src/crux/proxy/github_proxy.go @@ -198,9 +198,10 @@ type GitHubProxy struct { // allowPrivateForward snapshots HIVE_PROXY_ALLOW_PRIVATE_FORWARD at // construction (a boot-time deployment choice like proxyAdvisoryOK). The - // zero value keeps the plain-HTTP forward guard ON: non-routable - // destinations (loopback, link-local/metadata, RFC 1918, CGNAT, multicast) - // are refused on the plain path. See plain_forward_guard.go. + // zero value keeps the destination guard ON: non-routable destinations + // (loopback, link-local/metadata, RFC 1918, CGNAT, multicast) are refused + // on the plain-HTTP forward path and on uninspected CONNECT/transparent + // tunnels. See plain_forward_guard.go. allowPrivateForward bool // injectGHAuth snapshots config.ProxyInjectGHAuth() at construction (#1861, @@ -365,7 +366,7 @@ func newGitHubProxyWithCA(caCert tls.Certificate, caX509 *x509.Certificate, logg // dial the private estate on an agent's behalf. allowPrivateForward := plainForwardAllowPrivate() if allowPrivateForward { - logger.Warn("proxy: plain-HTTP forwards to non-routable destinations are ALLOWED — agents can reach loopback, link-local (cloud metadata) and private-network services through this proxy", + logger.Warn("proxy: plain-HTTP forwards and uninspected tunnels to non-routable destinations are ALLOWED — agents can reach loopback, link-local (cloud metadata) and private-network services through this proxy", "env", plainForwardAllowPrivateEnv) } diff --git a/src/crux/proxy/github_proxy_connect.go b/src/crux/proxy/github_proxy_connect.go index a9c2e23f0a..ab26876bae 100644 --- a/src/crux/proxy/github_proxy_connect.go +++ b/src/crux/proxy/github_proxy_connect.go @@ -168,6 +168,12 @@ func (p *GitHubProxy) handleTransparentTLS(conn net.Conn, peeked []byte) { // so the forced-egress redirect exempts this proxy-originated dial. upstream, err := p.dialTunnelUpstream(host) if err != nil { + var denied *plainForwardDenied + if errors.As(err, &denied) && denied.reason != plainForwardReasonLoopback { + p.logger.Warn("proxy: transparent tunnel refused — destination is not globally routable", + "agent", agentName, "host", host, "target", denied.target, "class", denied.reason) + p.countViolation(agentName) + } return } defer func() { _ = upstream.Close() }() @@ -268,12 +274,15 @@ func (p *GitHubProxy) attributeTransparentConn(conn net.Conn) string { // dialTunnelUpstream opens the raw TCP leg of a non-inspected transparent // tunnel. The socket is SO_MARKed so the forced-egress redirect exempts this -// proxy-originated dial. tunnelDial, when set, overrides it (tests). +// proxy-originated dial, and the destination is guarded like every other +// uninspected egress leg (see plain_forward_guard.go): the host here comes +// from the client's SNI, which names whatever the client chose. tunnelDial, +// when set, overrides it (tests). func (p *GitHubProxy) dialTunnelUpstream(host string) (net.Conn, error) { if p.tunnelDial != nil { return p.tunnelDial(host) } - return markDialer(transparentProxyTimeout).Dial("tcp", host+":443") + return p.tunnelDialer(transparentProxyTimeout).Dial("tcp", host+":443") } const tlsClientHelloMaxSize = 4096 @@ -598,9 +607,17 @@ func (p *GitHubProxy) logClientRead(msg string, err error, attrs ...any) { func (p *GitHubProxy) tunnelDirect(conn net.Conn, r *http.Request) { const tunnelDialTimeout = 10 * time.Second // SO_MARK the socket so the forced-egress redirect exempts this - // proxy-originated tunnel dial (CONNECT targets include :443). - upstream, err := markDialer(tunnelDialTimeout).Dial("tcp", r.Host) + // proxy-originated tunnel dial (CONNECT targets include :443). The dialer + // is destination-guarded like the plain forward path: a CONNECT names an + // arbitrary host:port, and without the guard it was a raw TCP tunnel from + // the proxy's uid to the cloud metadata service or the cluster estate. + upstream, err := p.tunnelDialer(tunnelDialTimeout).Dial("tcp", r.Host) if err != nil { + var denied *plainForwardDenied + if errors.As(err, &denied) { + p.refuseTunnel(conn, r, p.identifyAgentFromConn(conn, r), denied) + return + } p.logger.Warn("proxy: CONNECT dial failed", "host", r.Host, "error", err) _, _ = fmt.Fprintf(conn, "HTTP/1.1 502 Bad Gateway\r\n\r\nconnection failed\n") return @@ -612,6 +629,24 @@ func (p *GitHubProxy) tunnelDirect(conn net.Conn, r *http.Request) { relayTunnel(conn, upstream) } +// refuseTunnel answers a CONNECT whose destination the guard barred, with the +// same loopback-vs-estate split as refusePlainForward: loopback is a client +// that ignored NO_PROXY and is logged at Debug without a violation; every +// other class is counted against the caller. +func (p *GitHubProxy) refuseTunnel(conn net.Conn, r *http.Request, agentName string, denied *plainForwardDenied) { + if denied.reason == plainForwardReasonLoopback { + p.logger.Debug("proxy: CONNECT to loopback refused (clients must bypass the proxy for loopback; NO_PROXY)", + "agent", agentName, "host", r.Host, "target", denied.target) + } else { + p.logger.Warn("proxy: CONNECT refused — destination is not globally routable", + "agent", agentName, "host", r.Host, "target", denied.target, "class", denied.reason) + p.countViolation(agentName) + } + _, _ = fmt.Fprintf(conn, "HTTP/1.1 403 Forbidden\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n"+ + "the hive proxy does not tunnel to %s addresses (%s); set %s=true to allow it\n", + denied.reason, denied.target, plainForwardAllowPrivateEnv) +} + // forwardPlainDirect handles non-CONNECT (plain HTTP) requests on a raw // connection. // diff --git a/src/crux/proxy/github_proxy_policy.go b/src/crux/proxy/github_proxy_policy.go index b18a136dea..b61b8d03ab 100644 --- a/src/crux/proxy/github_proxy_policy.go +++ b/src/crux/proxy/github_proxy_policy.go @@ -267,7 +267,13 @@ func (p *GitHubProxy) inspectCanaryEgress(agentName string, req *http.Request) ( _ = req.Body.Close() } if err != nil { - return "", false, false + // Fail closed, as enforceLinear does: a body we could not fully read + // is a body we could not scan, and the original body is already + // consumed, so letting the request through would forward it empty. + req.Body = io.NopCloser(bytes.NewReader(body)) + req.ContentLength = int64(len(body)) + p.logger.Warn("ioscan canary scan: request body read failed — denying", "agent", agentName, "path", req.URL.Path) + return "ioscan canary scan body unreadable", true, true } if len(body) > maxGitHubWriteBodyScan { return "ioscan canary scan body too large", true, true diff --git a/src/crux/proxy/issue_shape_test.go b/src/crux/proxy/issue_shape_test.go index 78b6114ca2..0199695b36 100644 --- a/src/crux/proxy/issue_shape_test.go +++ b/src/crux/proxy/issue_shape_test.go @@ -67,6 +67,25 @@ func TestEnforceIssueShapeRejectsMalformedAndPreservesBody(t *testing.T) { body: "{\"title\":\"[guide] generics example\",\"body\":\"## Finding\\n\\nUse the generic form:\\n\\n```go\\nvar x Foo\\n```\\n\\n## Recommendation\\n\\nKeep it.\"}", wantDeny: false, }, + { + // A Spek-published child (spektacular#114) carries a hidden + // provenance marker; the marker is not a template placeholder and + // the body is forwarded as written. + name: "spek:publish marker is accepted", + method: http.MethodPost, + path: "https://api.github.com/repos/org/repo/issues", + body: "{\"title\":\"Add refresh endpoint\",\"body\":\"## What\\n\\nAdd `POST /auth/refresh`.\\n\\n## Acceptance criteria\\n\\n- [ ] token rotates\\n\\n\"}", + wantDeny: false, + }, + { + // The marker does not excuse an unfilled template around it. + name: "spek:publish marker does not bypass placeholder check", + method: http.MethodPost, + path: "https://api.github.com/repos/org/repo/issues", + body: "{\"title\":\"Add refresh endpoint\",\"body\":\"\\n\\n\"}", + wantDeny: true, + errorSubstr: "unsubstituted template placeholder", + }, { name: "label sub-route is not shape-checked", method: http.MethodPost, diff --git a/src/crux/proxy/linear_rules.go b/src/crux/proxy/linear_rules.go index 21a1306a75..cb8dce203a 100644 --- a/src/crux/proxy/linear_rules.go +++ b/src/crux/proxy/linear_rules.go @@ -105,6 +105,17 @@ var linearReadRequirement = linearRequirement{MinMode: agent.ModeAdvisory} // issue. That is how a PR-capable agent reports what it produced, so they // sit at ModeIssuesAndPRs alongside PR creation. // +// - issueRelationCreate, projectCreate and projectUpdate are what an in-hive +// Spek needs to publish a finished spec as a parent plus children +// (hivecommons/spektacular#114): a Project (or a parent issue — that is +// plain issueCreate with parentId, already covered) for the RFC, child +// issueCreate per task, and `blockedBy` relations between children; the +// idempotent re-publish edits the parent in place with projectUpdate. They +// are the Linear analogues of GitHub's POST …/sub_issues and +// POST …/dependencies/blocked_by, so they sit at ModeIssuesOnly with no +// capability grant. Deliberately NOT included: issueRelationDelete, +// projectDelete, projectArchive — a published tree is only ever added to. +// // Mutations NOT listed are denied, including every destructive one // (issueDelete, commentDelete, projectDelete, teamDelete, issueArchive) and // every administrative one (organizationInviteCreate, apiKeyCreate, @@ -134,6 +145,11 @@ var linearMutations = map[string]linearRequirement{ "issueAddLabel": {MinMode: agent.ModeIssuesOnly}, "issueRemoveLabel": {MinMode: agent.ModeIssuesOnly}, + // Spek publish: RFC parent, children and their dependency edges. + "issueRelationCreate": {MinMode: agent.ModeIssuesOnly}, + "projectCreate": {MinMode: agent.ModeIssuesOnly}, + "projectUpdate": {MinMode: agent.ModeIssuesOnly}, + // Linking produced artifacts (PRs, builds) onto an issue. "attachmentCreate": {MinMode: agent.ModeIssuesAndPRs}, "attachmentLinkURL": {MinMode: agent.ModeIssuesAndPRs}, diff --git a/src/crux/proxy/linear_rules_test.go b/src/crux/proxy/linear_rules_test.go index 6f9374e007..44a00bd168 100644 --- a/src/crux/proxy/linear_rules_test.go +++ b/src/crux/proxy/linear_rules_test.go @@ -549,3 +549,53 @@ func TestLinearAgentSessionUpdateReachableAtEveryTier(t *testing.T) { t.Error("agentSessionUpdate + issueUpdate batch allowed at ADVISORY") } } + +// ───────────────────────────────────────────────────────────────────────────── +// Spek publish (hivecommons/spektacular#114): an in-hive Spek files an RFC +// parent, child issues and blockedBy edges through this gate. Each mutation +// it needs sits at ISSUES_ONLY with the other issue-artifact writes, and the +// destructive siblings stay absent from the allowlist. +// ───────────────────────────────────────────────────────────────────────────── + +func TestLinearSpekPublishMutationsRequireIssuesOnly(t *testing.T) { + queries := map[string]string{ + "issueCreate with parentId": `mutation { issueCreate(input: {teamId: "t", title: "child", parentId: "rfc"}) { success } }`, + "issueRelationCreate": `mutation { issueRelationCreate(input: {issueId: "a", relatedIssueId: "b", type: blocks}) { success } }`, + "projectCreate": `mutation { projectCreate(input: {teamIds: ["t"], name: "RFC: auth"}) { success } }`, + "projectUpdate": `mutation { projectUpdate(id: "p", input: {description: "v2"}) { success } }`, + } + for name, q := range queries { + t.Run(name, func(t *testing.T) { + if d := decide(agent.ModeAdvisory, q); d.Allowed { + t.Errorf("%s must be DENIED at ADVISORY", name) + } + if d := LinearGraphQLAllowed(agent.ModeAdvisory, agent.AgentCapabilities{Converse: true}, linearBody(q), false); d.Allowed { + t.Errorf("converse must NOT grant %s — publishing is artifact production", name) + } + for _, mode := range []agent.AgentMode{agent.ModeIssuesOnly, agent.ModeIssuesAndPRs, agent.ModeIssuesPRsMerge} { + if d := decide(mode, q); !d.Allowed { + t.Errorf("%s must be ALLOWED at %s, got reason %q", name, mode, d.Reason) + } + } + }) + } +} + +func TestLinearSpekPublishDestructiveSiblingsStayDenied(t *testing.T) { + for name, q := range map[string]string{ + "issueRelationDelete": `mutation { issueRelationDelete(id: "r") { success } }`, + "projectDelete": `mutation { projectDelete(id: "p") { success } }`, + "projectArchive": `mutation { projectArchive(id: "p") { success } }`, + } { + for _, mode := range allModes { + if d := decide(mode, q); d.Allowed { + t.Errorf("%s must be DENIED at %s", name, mode) + } + } + } + // A batch that creates a relation AND deletes one is denied as a whole. + const batch = `mutation { issueRelationCreate(input: {issueId: "a", relatedIssueId: "b", type: blocks}) { success } issueRelationDelete(id: "r") { success } }` + if d := decide(agent.ModeIssuesPRsMerge, batch); d.Allowed { + t.Error("batch mixing an allowed publish mutation with a delete must be denied") + } +} diff --git a/src/crux/proxy/plain_forward_guard.go b/src/crux/proxy/plain_forward_guard.go index f2628b4af9..1ef80f0ef7 100644 --- a/src/crux/proxy/plain_forward_guard.go +++ b/src/crux/proxy/plain_forward_guard.go @@ -9,17 +9,22 @@ import ( "os" "strings" "syscall" + "time" ) -// The plain-HTTP (non-CONNECT) forward path is the one egress path through +// The plain-HTTP (non-CONNECT) forward path and the uninspected tunnel paths +// (a CONNECT to a host the proxy neither inspects nor intercepts, and the +// transparent-TLS passthrough for such a host) are the egress paths through // this proxy that carried NO destination policy at all: handleConn hands every -// non-CONNECT request to forwardPlainDirect, which RoundTripped it unchanged. -// An agent that sets a proxy URL by hand — or simply asks its CLI for -// `http://` — therefore had the proxy open a connection on its -// behalf, from the PROXY's uid, to any address the agent named. The forced -// egress gate does not cover it: the iptables rule only redirects :443, so -// plain HTTP is reachable only through this path, and the proxy's own uid is -// exempted from the redirect precisely so its upstream dials work. +// non-CONNECT request to forwardPlainDirect, which RoundTripped it unchanged, +// and tunnelDirect dialled whatever host:port the CONNECT line named. An agent +// that sets a proxy URL by hand — or simply asks its CLI for +// `http://`, or sends `CONNECT 169.254.169.254:80` — therefore had +// the proxy open a connection on its behalf, from the PROXY's uid, to any +// address the agent named. The forced egress gate does not cover it: the +// iptables rule only redirects :443, so plain HTTP is reachable only through +// this path, and the proxy's own uid is exempted from the redirect precisely +// so its upstream dials work. // // The interesting targets are not on the public internet, where the agent's // own egress would have reached anyway. They are the addresses only the pod @@ -29,7 +34,8 @@ import ( // (node credentials on most clouds), and the cluster-internal RFC1918 estate // (the Kubernetes API, other spokes, internal registries). // -// So the plain forward path dials only globally routable addresses. The check +// So the plain forward and uninspected tunnel paths dial only globally +// routable addresses. The check // runs in the dialer's Control hook rather than on the request's host string: // Control is called AFTER name resolution with the concrete remote address, so // a hostname that resolves to 127.0.0.1 or 169.254.169.254 — a public DNS @@ -132,6 +138,36 @@ var ( unguardedPlainForwardTransport = newPlainForwardTransport(false) ) +// guardedMarkDialer is markDialer with the destination check layered into the +// Control hook, so the address refused is the resolved one the kernel is about +// to connect to. It is the single dialer both the plain forward transport and +// the uninspected tunnel paths build on; keep the policy here so the two +// paths cannot drift apart. +func guardedMarkDialer(timeout time.Duration) *net.Dialer { + dialer := markDialer(timeout) + mark := dialer.Control + dialer.Control = func(network, address string, c syscall.RawConn) error { + if reason := plainForwardDenyReason(address); reason != "" { + return &plainForwardDenied{target: address, reason: reason} + } + return mark(network, address, c) + } + return dialer +} + +// tunnelDialer returns the dialer an uninspected tunnel (CONNECT passthrough +// or transparent-TLS passthrough) opens its upstream leg with: guarded unless +// the deployment lifted the guard with HIVE_PROXY_ALLOW_PRIVATE_FORWARD, the +// same boot-time choice the plain forward path honours. The zero value of +// allowPrivateForward is guarded, so a struct-literal GitHubProxy is +// protected without opting in. +func (p *GitHubProxy) tunnelDialer(timeout time.Duration) *net.Dialer { + if p.allowPrivateForward { + return markDialer(timeout) + } + return guardedMarkDialer(timeout) +} + // newPlainForwardTransport clones the standard transport (keeping its // connection pooling and timeouts), drops its ProxyFromEnvironment so an // HTTP_PROXY/HTTPS_PROXY on the proxy process is deliberately ignored — with @@ -146,13 +182,7 @@ func newPlainForwardTransport(guard bool) *http.Transport { transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { dialer := markDialer(upstreamDialTimeout) if guard { - mark := dialer.Control - dialer.Control = func(network, address string, c syscall.RawConn) error { - if reason := plainForwardDenyReason(address); reason != "" { - return &plainForwardDenied{target: address, reason: reason} - } - return mark(network, address, c) - } + dialer = guardedMarkDialer(upstreamDialTimeout) } return dialer.DialContext(ctx, network, address) } diff --git a/src/crux/proxy/proxy_deep2_test.go b/src/crux/proxy/proxy_deep2_test.go index ebdcf8e5f0..657b884275 100644 --- a/src/crux/proxy/proxy_deep2_test.go +++ b/src/crux/proxy/proxy_deep2_test.go @@ -321,6 +321,7 @@ func TestProxyHTTPGraphQLMutationAllowed(t *testing.T) { func TestHandleConnectDirectAnthropicNoRoute(t *testing.T) { p := newTestProxy() + p.allowPrivateForward = true // the tunnel target under test is a loopback listener p.inference = newInferenceRouter() // No route set for any agent @@ -641,6 +642,7 @@ func TestProxyHTTPUpstreamWriteError(t *testing.T) { func TestTunnelDirectSuccess(t *testing.T) { p := newTestProxy() + p.allowPrivateForward = true // the tunnel target under test is a loopback listener // Create a local TCP server ln, err := net.Listen("tcp", "127.0.0.1:0") diff --git a/src/crux/proxy/proxy_deep_test.go b/src/crux/proxy/proxy_deep_test.go index d678aa0d79..09c3dc0dac 100644 --- a/src/crux/proxy/proxy_deep_test.go +++ b/src/crux/proxy/proxy_deep_test.go @@ -738,6 +738,7 @@ func TestUpdateMaxContextLenStaleGuard(t *testing.T) { func TestHandleConnCONNECTNonGitHub(t *testing.T) { p := newTestProxy() + p.allowPrivateForward = true // the tunnel target under test is a loopback listener // Start a mock TCP server to tunnel to ln, err := net.Listen("tcp", "127.0.0.1:0") @@ -957,6 +958,7 @@ func TestExtractTextFromContentInvalid(t *testing.T) { func TestHandleConnectDirectNonGitHub(t *testing.T) { p := newTestProxy() + p.allowPrivateForward = true // the tunnel target under test is a loopback listener p.inference = newInferenceRouter() // Create a local TCP server to tunnel to diff --git a/src/crux/proxy/rules.go b/src/crux/proxy/rules.go index 3f5665af45..a40e7eb844 100644 --- a/src/crux/proxy/rules.go +++ b/src/crux/proxy/rules.go @@ -203,6 +203,17 @@ var rules = []ProxyRule{ {PathPattern: regexp.MustCompile(`^(?:/api/v1)?/repos/[^/]+/[^/]+/issues/\d+/labels`), Method: "POST", MinMode: agent.ModeIssuesOnly}, {PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/labels`), Method: "POST", MinMode: agent.ModeIssuesOnly}, {PathPattern: regexp.MustCompile(`^(?:/api/v1)?/repos/[^/]+/[^/]+/issues/\d+/labels`), Method: "DELETE", MinMode: agent.ModeIssuesOnly}, + // Issue relations — the native sub-issue link (`hive-open-issue --parent`, + // POST …/issues/{n}/sub_issues) and the "blocked by" dependency + // (`hive-open-issue --blocked-by`, POST …/issues/{n}/dependencies/blocked_by). + // The hive's own relay files these from the owner UID and never crossed + // this table; an agent publishing a Spek RFC with children from inside a + // hive does (hivecommons/spektacular#114), and relating two issues it was + // already allowed to create is artifact production at the same tier. + // Exactly these two sub-routes: no PUT/PATCH reprioritisation, no DELETE + // unlink — a published tree is only ever added to. + {PathPattern: regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues/\d+/sub_issues$`), Method: "POST", MinMode: agent.ModeIssuesOnly}, + {PathPattern: regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues/\d+/dependencies/blocked_by$`), Method: "POST", MinMode: agent.ModeIssuesOnly}, // ── Read operations — ADVISORY and above ── // Catch-all: any GET/HEAD/OPTIONS on any path. {PathPattern: regexp.MustCompile(`.*`), Method: "GET", MinMode: agent.ModeAdvisory}, @@ -213,86 +224,6 @@ var rules = []ProxyRule{ // AllowedByMode returns true if the given HTTP method+path is permitted // for an agent running in the specified mode. Unknown operations are // denied by default. -// denyRule is a hard block that applies to EVERY agent mode, regardless of the -// mode-escalation rules. It exists for operations agents must route through a -// hive-mediated path instead of calling GitHub directly. -type denyRule struct { - PathPattern *regexp.Regexp - Method string - Msg string // agent-facing directive surfaced in the 403 body -} - -// denyRules are checked BEFORE the mode rules. Two cases are hard-denied for -// EVERY agent mode: -// -// 1. Direct PR creation: a POST /repos/*/pulls — whether from `gh pr create` -// or the GitHub MCP create_pull_request/create_pull_request_with_copilot -// tool — authors the PR as the Copilot login user, not the App bot. -// Blocking it here forces agents to use `hive-open-pr`, which the hive -// fulfills with the App token so the PR is authored by the App bot. This -// closes the MCP path the gh-wrapper redirect cannot see. -// -// 2. Direct PR merge: a PUT /repos/*/pulls/{n}/merge (H1, CWE-863). Permitting -// this at ModeIssuesPRsMerge on a UID-derived mode check ALONE let an -// injected merge-mode agent `gh api -X PUT .../merge` any reachable PR, -// bypassing the bound merge relay's fail-closed SHA pin + merge-eligible -// binding (see bindMergeAuthz). Hard-denying it forces ALL agent-initiated -// merges through `hive-merge`, which the hive fulfills as the App over REST -// with those bindings enforced. This mirrors the GraphQL mergePullRequest -// deny (GraphQLAllowed treats it as a mutation), so neither REST nor GraphQL -// is an agent-reachable merge bypass. -// -// In BOTH cases the hive's OWN call (CreatePR / MergePR) does NOT traverse this -// agent proxy — it originates from the hive process (owner UID), which the -// forced-egress iptables redirect exempts — so the relay is unaffected. -var denyRules = []denyRule{ - // ── GitHub ── - { - PathPattern: regexp.MustCompile(`^/repos/[^/]+/[^/]+/pulls$`), - Method: "POST", - Msg: "direct PR creation is disabled for agents — use `hive-open-pr --repo --head --title --body ` so the hive opens the PR as the App bot (never the login user). Do NOT use `gh pr create` or the GitHub MCP create_pull_request/create_pull_request_with_copilot.", - }, - { - PathPattern: regexp.MustCompile(`^/repos/[^/]+/[^/]+/pulls/\d+/merge$`), - Method: "PUT", - Msg: "direct PR merge is disabled for agents — use `hive-merge --repo --number --expect-sha ` so the hive merges as the App bot with the SHA-pin + merge-eligible binding enforced. Do NOT use `gh api -X PUT .../merge`, `gh pr merge`, or the GitHub MCP merge_pull_request.", - }, - - // ── GitLab ── - { - PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/merge_requests$`), - Method: "POST", - Msg: "direct MR creation is disabled for agents — use the request-file path so the hive opens the MR as the App/bot identity. Do NOT use direct GitLab API calls.", - }, - { - PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/merge_requests/\d+/merge$`), - Method: "PUT", - Msg: "direct MR merge is disabled for agents — use the request-file path so the hive merges as the App/bot identity with bindings enforced. Do NOT use direct GitLab API calls.", - }, - { - PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/merge_requests/\d+/merge$`), - Method: "POST", - Msg: "direct MR merge is disabled for agents — use the request-file path so the hive merges as the App/bot identity with bindings enforced. Do NOT use direct GitLab API calls.", - }, - - // ── Gitea / Forgejo ── - { - PathPattern: regexp.MustCompile(`^/api/v1/repos/[^/]+/[^/]+/pulls$`), - Method: "POST", - Msg: "direct PR creation is disabled for agents — use the request-file path so the hive opens the PR as the App/bot identity. Do NOT use direct Gitea API calls.", - }, - { - PathPattern: regexp.MustCompile(`^(?:/api/v1)?/repos/[^/]+/[^/]+/pulls/\d+/merge$`), - Method: "POST", - Msg: "direct PR merge is disabled for agents — use the request-file path so the hive merges as the App/bot identity with bindings enforced. Do NOT use direct Gitea API calls.", - }, - { - PathPattern: regexp.MustCompile(`^/api/v1/repos/[^/]+/[^/]+/pulls/\d+/merge$`), - Method: "PUT", - Msg: "direct PR merge is disabled for agents — use the request-file path so the hive merges as the App/bot identity with bindings enforced. Do NOT use direct Gitea API calls.", - }, -} - func AllowedByMode(mode agent.AgentMode, method, path string) bool { return AllowedByModeCaps(mode, agent.AgentCapabilities{}, method, path) } @@ -330,20 +261,6 @@ func AllowedByModeCaps(mode agent.AgentMode, caps agent.AgentCapabilities, metho return false } -// DeniedMessage returns the agent-facing directive for a request blocked by a -// hard-deny rule, and true if such a rule matched. It lets the proxy surface WHY -// a request was refused (e.g. "use hive-open-pr") instead of the generic ACMM -// message. Returns ("", false) when no deny rule matches — the caller then falls -// back to the normal mode-based block reason. -func DeniedMessage(method, path string) (string, bool) { - for _, r := range denyRules { - if r.Method == method && r.PathPattern.MatchString(path) { - return r.Msg, true - } - } - return "", false -} - var repoPathPrefix = regexp.MustCompile(`^(?:/api/v1)?/repos/([^/]+/[^/]+)`) // gitSmartHTTPEndpoints are the only things that may follow ".git/" for diff --git a/src/crux/proxy/rules_deny.go b/src/crux/proxy/rules_deny.go new file mode 100644 index 0000000000..6113d582a2 --- /dev/null +++ b/src/crux/proxy/rules_deny.go @@ -0,0 +1,103 @@ +package proxy + +import ( + "regexp" +) + +// Hard-deny rules: operations refused for EVERY agent mode because they must +// route through a hive-mediated relay. Split out of rules.go so the +// mode-escalation table stays within its size ratchet. + +// denyRule is a hard block that applies to EVERY agent mode, regardless of the +// mode-escalation rules. It exists for operations agents must route through a +// hive-mediated path instead of calling GitHub directly. +type denyRule struct { + PathPattern *regexp.Regexp + Method string + Msg string // agent-facing directive surfaced in the 403 body +} + +// denyRules are checked BEFORE the mode rules. Two cases are hard-denied for +// EVERY agent mode: +// +// 1. Direct PR creation: a POST /repos/*/pulls — whether from `gh pr create` +// or the GitHub MCP create_pull_request/create_pull_request_with_copilot +// tool — authors the PR as the Copilot login user, not the App bot. +// Blocking it here forces agents to use `hive-open-pr`, which the hive +// fulfills with the App token so the PR is authored by the App bot. This +// closes the MCP path the gh-wrapper redirect cannot see. +// +// 2. Direct PR merge: a PUT /repos/*/pulls/{n}/merge (H1, CWE-863). Permitting +// this at ModeIssuesPRsMerge on a UID-derived mode check ALONE let an +// injected merge-mode agent `gh api -X PUT .../merge` any reachable PR, +// bypassing the bound merge relay's fail-closed SHA pin + merge-eligible +// binding (see bindMergeAuthz). Hard-denying it forces ALL agent-initiated +// merges through `hive-merge`, which the hive fulfills as the App over REST +// with those bindings enforced. This mirrors the GraphQL mergePullRequest +// deny (GraphQLAllowed treats it as a mutation), so neither REST nor GraphQL +// is an agent-reachable merge bypass. +// +// In BOTH cases the hive's OWN call (CreatePR / MergePR) does NOT traverse this +// agent proxy — it originates from the hive process (owner UID), which the +// forced-egress iptables redirect exempts — so the relay is unaffected. +var denyRules = []denyRule{ + // ── GitHub ── + { + PathPattern: regexp.MustCompile(`^/repos/[^/]+/[^/]+/pulls$`), + Method: "POST", + Msg: "direct PR creation is disabled for agents — use `hive-open-pr --repo --head --title --body ` so the hive opens the PR as the App bot (never the login user). Do NOT use `gh pr create` or the GitHub MCP create_pull_request/create_pull_request_with_copilot.", + }, + { + PathPattern: regexp.MustCompile(`^/repos/[^/]+/[^/]+/pulls/\d+/merge$`), + Method: "PUT", + Msg: "direct PR merge is disabled for agents — use `hive-merge --repo --number --expect-sha ` so the hive merges as the App bot with the SHA-pin + merge-eligible binding enforced. Do NOT use `gh api -X PUT .../merge`, `gh pr merge`, or the GitHub MCP merge_pull_request.", + }, + + // ── GitLab ── + { + PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/merge_requests$`), + Method: "POST", + Msg: "direct MR creation is disabled for agents — use the request-file path so the hive opens the MR as the App/bot identity. Do NOT use direct GitLab API calls.", + }, + { + PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/merge_requests/\d+/merge$`), + Method: "PUT", + Msg: "direct MR merge is disabled for agents — use the request-file path so the hive merges as the App/bot identity with bindings enforced. Do NOT use direct GitLab API calls.", + }, + { + PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/merge_requests/\d+/merge$`), + Method: "POST", + Msg: "direct MR merge is disabled for agents — use the request-file path so the hive merges as the App/bot identity with bindings enforced. Do NOT use direct GitLab API calls.", + }, + + // ── Gitea / Forgejo ── + { + PathPattern: regexp.MustCompile(`^/api/v1/repos/[^/]+/[^/]+/pulls$`), + Method: "POST", + Msg: "direct PR creation is disabled for agents — use the request-file path so the hive opens the PR as the App/bot identity. Do NOT use direct Gitea API calls.", + }, + { + PathPattern: regexp.MustCompile(`^(?:/api/v1)?/repos/[^/]+/[^/]+/pulls/\d+/merge$`), + Method: "POST", + Msg: "direct PR merge is disabled for agents — use the request-file path so the hive merges as the App/bot identity with bindings enforced. Do NOT use direct Gitea API calls.", + }, + { + PathPattern: regexp.MustCompile(`^/api/v1/repos/[^/]+/[^/]+/pulls/\d+/merge$`), + Method: "PUT", + Msg: "direct PR merge is disabled for agents — use the request-file path so the hive merges as the App/bot identity with bindings enforced. Do NOT use direct Gitea API calls.", + }, +} + +// DeniedMessage returns the agent-facing directive for a request blocked by a +// hard-deny rule, and true if such a rule matched. It lets the proxy surface WHY +// a request was refused (e.g. "use hive-open-pr") instead of the generic ACMM +// message. Returns ("", false) when no deny rule matches — the caller then falls +// back to the normal mode-based block reason. +func DeniedMessage(method, path string) (string, bool) { + for _, r := range denyRules { + if r.Method == method && r.PathPattern.MatchString(path) { + return r.Msg, true + } + } + return "", false +} diff --git a/src/crux/proxy/rules_test.go b/src/crux/proxy/rules_test.go index 313c445e71..30b487ce2f 100644 --- a/src/crux/proxy/rules_test.go +++ b/src/crux/proxy/rules_test.go @@ -32,6 +32,17 @@ func TestAllowedByMode(t *testing.T) { {"issues-only allows PATCH issue", agent.ModeIssuesOnly, "PATCH", "/repos/org/repo/issues/42", true}, {"issues-only allows comment", agent.ModeIssuesOnly, "POST", "/repos/org/repo/issues/42/comments", true}, {"issues-only allows label", agent.ModeIssuesOnly, "POST", "/repos/org/repo/issues/42/labels", true}, + // Spek publishes an RFC with sub-issue children and "blocked by" edges + // through the proxy (spektacular#114): the same two writes + // hive-open-issue --parent / --blocked-by make, at the issue tier. + {"issues-only allows add sub-issue", agent.ModeIssuesOnly, "POST", "/repos/org/repo/issues/42/sub_issues", true}, + {"issues-only allows add blocked-by", agent.ModeIssuesOnly, "POST", "/repos/org/repo/issues/42/dependencies/blocked_by", true}, + {"advisory blocks add sub-issue", agent.ModeAdvisory, "POST", "/repos/org/repo/issues/42/sub_issues", false}, + {"advisory blocks add blocked-by", agent.ModeAdvisory, "POST", "/repos/org/repo/issues/42/dependencies/blocked_by", false}, + {"sub-issue unlink stays denied", agent.ModeIssuesPRsMerge, "DELETE", "/repos/org/repo/issues/42/sub_issue", false}, + {"sub-issue reprioritise stays denied", agent.ModeIssuesPRsMerge, "PATCH", "/repos/org/repo/issues/42/sub_issues/priority", false}, + {"blocked-by unlink stays denied", agent.ModeIssuesPRsMerge, "DELETE", "/repos/org/repo/issues/42/dependencies/blocked_by/7", false}, + {"blocking side is not a write route", agent.ModeIssuesPRsMerge, "POST", "/repos/org/repo/issues/42/dependencies/blocking", false}, {"issues-only blocks POST pulls", agent.ModeIssuesOnly, "POST", "/repos/org/repo/pulls", false}, {"issues-only blocks PATCH pull", agent.ModeIssuesOnly, "PATCH", "/repos/org/repo/pulls/42", false}, {"issues-only blocks PUT merge", agent.ModeIssuesOnly, "PUT", "/repos/org/repo/pulls/42/merge", false}, diff --git a/src/crux/proxy/tunnel_destination_guard_test.go b/src/crux/proxy/tunnel_destination_guard_test.go new file mode 100644 index 0000000000..68763283cb --- /dev/null +++ b/src/crux/proxy/tunnel_destination_guard_test.go @@ -0,0 +1,149 @@ +package proxy + +import ( + "bufio" + "errors" + "io" + "net" + "net/http" + "strings" + "testing" + "time" +) + +// connectRoundTrip drives handleConnectDirect over a pipe for a CONNECT to +// target and returns the response the agent-side client would see. +func connectRoundTrip(t *testing.T, p *GitHubProxy, target string) *http.Response { + t.Helper() + req, err := http.NewRequest(http.MethodConnect, "http://"+target, nil) + if err != nil { + t.Fatalf("new request: %v", err) + } + req.Host = target + req.Header.Set("Proxy-Authorization", "hive scanner") + + clientConn, proxyClient := net.Pipe() + t.Cleanup(func() { _ = clientConn.Close() }) + + go func() { + p.handleConnectDirect(proxyClient, req) + _ = proxyClient.Close() + }() + + _ = clientConn.SetReadDeadline(time.Now().Add(10 * time.Second)) + resp, err := http.ReadResponse(bufio.NewReader(clientConn), req) + if err != nil { + t.Fatalf("read response: %v", err) + } + t.Cleanup(func() { _ = resp.Body.Close() }) + return resp +} + +// TestTunnelDirectRefusesNonRoutableDestination pins that the uninspected +// CONNECT path applies the same destination policy as the plain-HTTP forward: +// a CONNECT to the cloud metadata address or the RFC 1918 estate is answered +// 403 and counted against the caller, not tunnelled from the proxy's uid. +func TestTunnelDirectRefusesNonRoutableDestination(t *testing.T) { + p := newTestProxy() + p.proxyAdvisoryOK = true // no UID map in tests; accept the header identity + + for _, target := range []string{"169.254.169.254:80", "10.255.255.1:6443", "100.64.0.1:443"} { + resp := connectRoundTrip(t, p, target) + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("CONNECT %s: status = %d, want 403", target, resp.StatusCode) + } + body, _ := io.ReadAll(resp.Body) + if !strings.Contains(string(body), plainForwardAllowPrivateEnv) { + t.Errorf("CONNECT %s: body %q should name the override env var", target, body) + } + } + if got := p.violations["scanner"]; got != 3 { + t.Errorf("violations[scanner] = %d, want 3 — each estate-bound CONNECT must be attributed and counted", got) + } +} + +// TestTunnelDirectLoopbackRefusalIsNotAViolation mirrors the plain path: a +// loopback CONNECT is a client that ignored NO_PROXY, refused but not held +// against the caller. +func TestTunnelDirectLoopbackRefusalIsNotAViolation(t *testing.T) { + p := newTestProxy() + p.proxyAdvisoryOK = true + + if resp := connectRoundTrip(t, p, "127.0.0.1:9"); resp.StatusCode != http.StatusForbidden { + t.Fatalf("status = %d, want 403 for a loopback CONNECT", resp.StatusCode) + } + if got := p.violations["scanner"]; got != 0 { + t.Errorf("violations[scanner] = %d, want 0 — loopback is a misconfiguration, not a violation", got) + } +} + +// TestTunnelDirectAllowPrivateForwardLiftsGuard pins that the same boot-time +// override that lifts the plain-forward guard lifts the tunnel guard, so a +// deployment that opted in keeps working as before. +func TestTunnelDirectAllowPrivateForwardLiftsGuard(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer func() { _ = ln.Close() }() + go func() { + conn, err := ln.Accept() + if err != nil { + return + } + _ = conn.Close() + }() + + p := newTestProxy() + p.allowPrivateForward = true + + if resp := connectRoundTrip(t, p, ln.Addr().String()); resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200 with the guard lifted", resp.StatusCode) + } +} + +// TestDialTunnelUpstreamRefusesNonRoutableSNI covers the transparent-TLS +// passthrough leg: the host there comes from the client's SNI, so an SNI that +// names a private address must be refused on the same evidence. +func TestDialTunnelUpstreamRefusesNonRoutableSNI(t *testing.T) { + p := newTestProxy() + + _, err := p.dialTunnelUpstream("10.255.255.1") + var denied *plainForwardDenied + if !errors.As(err, &denied) { + t.Fatalf("err = %v, want *plainForwardDenied", err) + } + if denied.reason != plainForwardReasonPrivate { + t.Errorf("reason = %q, want %q", denied.reason, plainForwardReasonPrivate) + } +} + +// TestHandleTransparentTLSRefusalCountsEstateNotLoopback pins the handler-level +// split in handleTransparentTLS: a refused estate-bound SNI is counted against +// the caller, a refused loopback SNI is not. Over net.Pipe attribution yields +// "", so the counter lands under that key. +func TestHandleTransparentTLSRefusalCountsEstateNotLoopback(t *testing.T) { + for _, tc := range []struct { + sni string + want int + }{{"10.255.255.1", 1}, {"127.0.0.1", 0}} { + p := newTestProxy() + clientHello := buildClientHello(tc.sni) + clientConn, proxyConn := net.Pipe() + done := make(chan struct{}) + go func() { + defer close(done) + p.handleTransparentTLS(proxyConn, clientHello[:1]) + _ = proxyConn.Close() + }() + _, _ = clientConn.Write(clientHello[1:]) + <-done + _ = clientConn.Close() + p.mu.Lock() + got := p.violations[""] + p.mu.Unlock() + if got != tc.want { + t.Errorf("SNI %s: violations[\"\"] = %d, want %d", tc.sni, got, tc.want) + } + } +} diff --git a/src/crux/serve/checkouts.go b/src/crux/serve/checkouts.go new file mode 100644 index 0000000000..52ea32e29d --- /dev/null +++ b/src/crux/serve/checkouts.go @@ -0,0 +1,166 @@ +package serve + +import ( + "errors" + "fmt" + "net/http" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/checkout" +) + +// Pinned checkouts (hivecommons/crux#114 S1) are the first work-services +// family: the routes accept project principals (authenticate admits them +// because registerCheckouts marks the prefix with workService), and +// ownership is enforced from the caller principal — a project may only +// create checkouts it owns, list its own, and delete its own; the operator +// may do anything. + +// registerCheckouts adds the checkout routes and marks them a work service. +func (s *Server) registerCheckouts(mux *http.ServeMux) { + mux.HandleFunc("POST "+api.PathCheckouts, s.handleCheckoutCreate) + mux.HandleFunc("GET "+api.PathCheckouts, s.handleCheckoutList) + mux.HandleFunc("GET "+api.PathCheckouts+"/{id}", s.handleCheckoutGet) + mux.HandleFunc("DELETE "+api.PathCheckouts+"/{id}", s.handleCheckoutDelete) + s.workService(api.PathCheckouts) +} + +// checkoutService answers 501 when the server has no checkout service. +func (s *Server) checkoutService(w http.ResponseWriter) (*checkout.Service, bool) { + if s.Checkouts == nil { + s.writeError(w, http.StatusNotImplemented, api.CodeNotSupported, "this server has no checkout service; start it with --checkout-allow") + return nil, false + } + return s.Checkouts, true +} + +func (s *Server) handleCheckoutCreate(w http.ResponseWriter, r *http.Request) { + svc, ok := s.checkoutService(w) + if !ok { + return + } + var in api.CheckoutRequest + if !s.readJSON(w, r, &in) { + return + } + caller, _ := CallerFrom(r.Context()) + owner := in.Owner + switch { + case owner == "": + owner = caller.Name + case !caller.IsOperator() && owner != caller.Name: + s.writeError(w, http.StatusForbidden, api.CodeForbidden, fmt.Sprintf("principal %s may only create checkouts owned by %q", caller, caller.Name)) + return + } + req := checkout.Request{Repo: in.Repo, Commit: in.Commit, Owner: owner, Branches: in.Branches, MaxBytes: in.MaxBytes} + if in.TTL != "" { + ttl, err := time.ParseDuration(in.TTL) + if err != nil { + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, "ttl: "+err.Error()) + return + } + req.TTL = ttl + } + rec, err := svc.Create(r.Context(), req) + if err != nil { + s.writeCheckoutError(w, err) + return + } + s.writeJSON(w, http.StatusCreated, checkoutToAPI(rec)) +} + +func (s *Server) handleCheckoutList(w http.ResponseWriter, r *http.Request) { + svc, ok := s.checkoutService(w) + if !ok { + return + } + caller, _ := CallerFrom(r.Context()) + owner := r.URL.Query().Get("owner") + if !caller.IsOperator() { + // A project sees only its own checkouts, whatever it asked for. + owner = caller.Name + } + recs, err := svc.List(owner) + if err != nil { + s.writeCheckoutError(w, err) + return + } + out := api.CheckoutList{Checkouts: make([]api.Checkout, 0, len(recs))} + for _, rec := range recs { + out.Checkouts = append(out.Checkouts, checkoutToAPI(rec)) + } + s.writeJSON(w, http.StatusOK, out) +} + +func (s *Server) handleCheckoutGet(w http.ResponseWriter, r *http.Request) { + svc, ok := s.checkoutService(w) + if !ok { + return + } + rec, ok := s.ownedCheckout(w, r, svc) + if !ok { + return + } + s.writeJSON(w, http.StatusOK, checkoutToAPI(rec)) +} + +func (s *Server) handleCheckoutDelete(w http.ResponseWriter, r *http.Request) { + svc, ok := s.checkoutService(w) + if !ok { + return + } + rec, ok := s.ownedCheckout(w, r, svc) + if !ok { + return + } + caller, _ := CallerFrom(r.Context()) + if err := svc.Delete(r.Context(), rec.ID, caller.String()); err != nil { + s.writeCheckoutError(w, err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +// ownedCheckout loads {id} and enforces that the caller owns it or is the +// operator; on failure the response has been written. +func (s *Server) ownedCheckout(w http.ResponseWriter, r *http.Request, svc *checkout.Service) (checkout.Record, bool) { + rec, err := svc.Get(r.PathValue("id")) + if err != nil { + s.writeCheckoutError(w, err) + return checkout.Record{}, false + } + caller, _ := CallerFrom(r.Context()) + if !checkout.CanDelete(rec, caller.Name, caller.IsOperator()) { + s.writeError(w, http.StatusForbidden, api.CodeForbidden, fmt.Sprintf("checkout %s is owned by %q, not by principal %s", rec.ID, rec.Owner, caller)) + return checkout.Record{}, false + } + return rec, true +} + +// writeCheckoutError maps the package's sentinels to typed codes. +func (s *Server) writeCheckoutError(w http.ResponseWriter, err error) { + status, code := http.StatusInternalServerError, api.CodeRuntime + switch { + case errors.Is(err, checkout.ErrBadRequest): + status, code = http.StatusBadRequest, api.CodeBadRequest + case errors.Is(err, checkout.ErrForbidden): + status, code = http.StatusForbidden, api.CodeForbidden + case errors.Is(err, checkout.ErrNotFound): + status, code = http.StatusNotFound, api.CodeNotFound + case errors.Is(err, checkout.ErrInUse): + status, code = http.StatusConflict, api.CodeConflict + case errors.Is(err, checkout.ErrMismatch): + status, code = http.StatusBadGateway, api.CodeUpstream + case errors.Is(err, checkout.ErrTooLarge): + status, code = http.StatusRequestEntityTooLarge, api.CodeTooLarge + case errors.Is(err, checkout.ErrUnsupported): + status, code = http.StatusNotImplemented, api.CodeUnsupported + } + s.writeError(w, status, code, err.Error()) +} + +func checkoutToAPI(rec checkout.Record) api.Checkout { + return api.Checkout{ID: rec.ID, Path: rec.Path, Repo: rec.Repo, Commit: rec.Commit, Owner: rec.Owner, Branches: rec.Branches, + MaxBytes: rec.MaxBytes, Bytes: rec.Bytes, TTL: rec.TTL, CreatedAt: rec.CreatedAt, ExpiresAt: rec.ExpiresAt} +} diff --git a/src/crux/serve/checkouts_test.go b/src/crux/serve/checkouts_test.go new file mode 100644 index 0000000000..3217eb7bbe --- /dev/null +++ b/src/crux/serve/checkouts_test.go @@ -0,0 +1,290 @@ +package serve + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/checkout" + "github.com/hivecommons/hive/crux/lanes" +) + +// checkoutFixtureRepo is a bare repo with one commit, as a file:// URL. +func checkoutFixtureRepo(t *testing.T) (repo, sha string) { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + work := filepath.Join(t.TempDir(), "work") + if err := os.MkdirAll(work, 0o700); err != nil { + t.Fatal(err) + } + git := func(args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = work + cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@x", "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@x") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + git("init", "-q") + if err := os.WriteFile(filepath.Join(work, "f"), []byte("f"), 0o600); err != nil { + t.Fatal(err) + } + git("add", "f") + git("commit", "-q", "-m", "c") + sha = git("rev-parse", "HEAD") + bare := filepath.Join(t.TempDir(), "fixture.git") + git("clone", "-q", "--bare", work, bare) + return "file://" + bare, sha +} + +type checkoutFixture struct { + t *testing.T + h http.Handler + s *Server + inUse map[string]bool + tokens map[string]string +} + +// newCheckoutFixture is a token-bearing server with a checkout service +// whose allowlist admits the fixture repo's directory, and two lane +// projects (atlas, beacon) besides the operator token. +func newCheckoutFixture(t *testing.T, repo string) *checkoutFixture { + t.Helper() + tokens := map[string]string{"atlas": strings.Repeat("a", 32), "beacon": strings.Repeat("b", 32), "operator": "agent-admin"} + cfg := lanes.Config{Principals: []lanes.Principal{{Project: "atlas", TokenEnv: "ATLAS"}, {Project: "beacon", TokenEnv: "BEACON"}}} + auth, err := lanes.Bind(cfg, func(env string) string { return tokens[strings.ToLower(env)] }, tokens["operator"]) + if err != nil { + t.Fatal(err) + } + f := &checkoutFixture{t: t, inUse: map[string]bool{}, tokens: tokens} + svc, err := checkout.New(checkout.Config{ + Dir: filepath.Join(t.TempDir(), "checkouts"), + AllowedRepos: []string{"file://" + filepath.Dir(strings.TrimPrefix(repo, "file://"))}, + InUse: func(id string) bool { return f.inUse[id] }, + }) + if err != nil { + t.Fatal(err) + } + f.s = New(newFakeRuntime()) + f.s.Token, f.s.LaneAuth, f.s.Checkouts = tokens["operator"], auth, svc + f.h = f.s.Handler() + return f +} + +// call sends method path as who ("operator", "atlas", "beacon" or "") and +// decodes a 2xx body into out, returning the status and any api.Error. +func (f *checkoutFixture) call(who, method, path string, body, out any) (int, api.Error) { + f.t.Helper() + var buf bytes.Buffer + if body != nil { + if err := json.NewEncoder(&buf).Encode(body); err != nil { + f.t.Fatal(err) + } + } + req := httptest.NewRequest(method, path, &buf) + if who != "" { + req.Header.Set("Authorization", "Bearer "+f.tokens[who]) + } + rec := httptest.NewRecorder() + f.h.ServeHTTP(rec, req) + var e api.Error + if rec.Code >= 300 { + _ = json.Unmarshal(rec.Body.Bytes(), &e) + } else if out != nil && rec.Code != http.StatusNoContent { + if err := json.Unmarshal(rec.Body.Bytes(), out); err != nil { + f.t.Fatalf("decode %s %s: %v: %s", method, path, err, rec.Body) + } + } + return rec.Code, e +} + +func TestCheckoutRoutesOwnershipAndListing(t *testing.T) { + repo, sha := checkoutFixtureRepo(t) + f := newCheckoutFixture(t, repo) + + // A project creates a checkout; the owner defaults to the project. + var mine api.Checkout + if code, e := f.call("atlas", http.MethodPost, api.PathCheckouts, api.CheckoutRequest{Repo: repo, Commit: sha, TTL: "72h"}, &mine); code != http.StatusCreated { + t.Fatalf("atlas create: %d %+v", code, e) + } + if mine.Owner != "atlas" || mine.Commit != sha || mine.Path == "" || mine.TTL != "72h0m0s" || mine.ExpiresAt == nil { + t.Fatalf("created = %+v", mine) + } + // A project may not name another owner; the operator may. + if code, e := f.call("atlas", http.MethodPost, api.PathCheckouts, api.CheckoutRequest{Repo: repo, Commit: sha, Owner: "beacon"}, nil); code != http.StatusForbidden || e.Code != api.CodeForbidden { + t.Fatalf("atlas create for beacon: %d %+v", code, e) + } + var theirs api.Checkout + if code, e := f.call("operator", http.MethodPost, api.PathCheckouts, api.CheckoutRequest{Repo: repo, Commit: sha, Owner: "beacon"}, &theirs); code != http.StatusCreated || theirs.Owner != "beacon" { + t.Fatalf("operator create for beacon: %d %+v %+v", code, e, theirs) + } + var ops api.Checkout + if code, _ := f.call("operator", http.MethodPost, api.PathCheckouts, api.CheckoutRequest{Repo: repo, Commit: sha}, &ops); code != http.StatusCreated || ops.Owner != DefaultOperatorName { + t.Fatalf("operator create for itself: %d %+v", code, ops) + } + + // Listing: the operator sees all and can filter; a project sees only + // its own whatever it asks for. + var list api.CheckoutList + if code, _ := f.call("operator", http.MethodGet, api.PathCheckouts, nil, &list); code != http.StatusOK || len(list.Checkouts) != 3 { + t.Fatalf("operator list: %d %d", code, len(list.Checkouts)) + } + if code, _ := f.call("operator", http.MethodGet, api.PathCheckouts+"?owner=beacon", nil, &list); code != http.StatusOK || len(list.Checkouts) != 1 || list.Checkouts[0].ID != theirs.ID { + t.Fatalf("operator list beacon: %d %+v", code, list) + } + if code, _ := f.call("atlas", http.MethodGet, api.PathCheckouts+"?owner=beacon", nil, &list); code != http.StatusOK || len(list.Checkouts) != 1 || list.Checkouts[0].ID != mine.ID { + t.Fatalf("atlas list: %d %+v", code, list) + } + + // Get: owner and operator yes, another project no. + var got api.Checkout + if code, _ := f.call("atlas", http.MethodGet, api.PathCheckouts+"/"+mine.ID, nil, &got); code != http.StatusOK || got.ID != mine.ID { + t.Fatalf("atlas get own: %d %+v", code, got) + } + if code, e := f.call("beacon", http.MethodGet, api.PathCheckouts+"/"+mine.ID, nil, nil); code != http.StatusForbidden || e.Code != api.CodeForbidden || !strings.Contains(e.Error, "owned by \"atlas\"") { + t.Fatalf("beacon get atlas's: %d %+v", code, e) + } + if code, _ := f.call("operator", http.MethodGet, api.PathCheckouts+"/"+mine.ID, nil, &got); code != http.StatusOK { + t.Fatalf("operator get: %d", code) + } + if code, e := f.call("operator", http.MethodGet, api.PathCheckouts+"/chk_000000000000", nil, nil); code != http.StatusNotFound || e.Code != api.CodeNotFound { + t.Fatalf("get unknown: %d %+v", code, e) + } + + // Delete: two drivers cannot delete each other's checkouts; live runs + // block deletion with 409; the owner and the operator can delete. + if code, e := f.call("beacon", http.MethodDelete, api.PathCheckouts+"/"+mine.ID, nil, nil); code != http.StatusForbidden || e.Code != api.CodeForbidden { + t.Fatalf("beacon delete atlas's: %d %+v", code, e) + } + f.inUse[mine.ID] = true + if code, e := f.call("atlas", http.MethodDelete, api.PathCheckouts+"/"+mine.ID, nil, nil); code != http.StatusConflict || e.Code != api.CodeConflict { + t.Fatalf("delete in use: %d %+v", code, e) + } + f.inUse[mine.ID] = false + if code, e := f.call("atlas", http.MethodDelete, api.PathCheckouts+"/"+mine.ID, nil, nil); code != http.StatusNoContent { + t.Fatalf("atlas delete own: %d %+v", code, e) + } + if code, e := f.call("operator", http.MethodDelete, api.PathCheckouts+"/"+theirs.ID, nil, nil); code != http.StatusNoContent { + t.Fatalf("operator delete beacon's: %d %+v", code, e) + } + if code, _ := f.call("operator", http.MethodGet, api.PathCheckouts, nil, &list); code != http.StatusOK || len(list.Checkouts) != 1 || list.Checkouts[0].ID != ops.ID { + t.Fatalf("list after deletes: %d %+v", code, list) + } + // Agent RPCs still refuse the project token. + if code, e := f.call("atlas", http.MethodGet, api.PathHealth, nil, nil); code != http.StatusUnauthorized || e.Code != api.CodeUnauthorized { + t.Fatalf("project token on health: %d %+v", code, e) + } + if code, _ := f.call("", http.MethodGet, api.PathCheckouts, nil, nil); code != http.StatusUnauthorized { + t.Fatalf("no token on checkouts: %d", code) + } +} + +func TestCheckoutRouteErrorMapping(t *testing.T) { + repo, sha := checkoutFixtureRepo(t) + f := newCheckoutFixture(t, repo) + cases := []struct { + name string + body any + code int + err string + msg string + }{ + {"short sha", api.CheckoutRequest{Repo: repo, Commit: sha[:7]}, http.StatusBadRequest, api.CodeBadRequest, "full 40-hex SHA"}, + {"branch", api.CheckoutRequest{Repo: repo, Commit: "main"}, http.StatusBadRequest, api.CodeBadRequest, "tag or branch"}, + {"bad ttl", api.CheckoutRequest{Repo: repo, Commit: sha, TTL: "soon"}, http.StatusBadRequest, api.CodeBadRequest, "ttl"}, + {"off allowlist", api.CheckoutRequest{Repo: "https://github.com/org/name.git", Commit: sha}, http.StatusForbidden, api.CodeForbidden, "allowed prefixes"}, + {"over cap", api.CheckoutRequest{Repo: repo, Commit: sha, MaxBytes: 1}, http.StatusRequestEntityTooLarge, api.CodeTooLarge, "after cloning"}, + {"unknown field", map[string]any{"repo": repo, "commit": sha, "branch": "main"}, http.StatusBadRequest, api.CodeBadRequest, "unknown field"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + code, e := f.call("operator", http.MethodPost, api.PathCheckouts, tc.body, nil) + if code != tc.code || e.Code != tc.err || !strings.Contains(e.Error, tc.msg) { + t.Fatalf("%d %+v, want %d %s containing %q", code, e, tc.code, tc.err, tc.msg) + } + }) + } + // The remaining sentinels map through writeCheckoutError directly. + for err, want := range map[error][2]any{ + checkout.ErrMismatch: {http.StatusBadGateway, api.CodeUpstream}, + checkout.ErrUnsupported: {http.StatusNotImplemented, api.CodeUnsupported}, + checkout.ErrGit: {http.StatusInternalServerError, api.CodeRuntime}, + errors.New("other"): {http.StatusInternalServerError, api.CodeRuntime}, + } { + rec := httptest.NewRecorder() + f.s.writeCheckoutError(rec, err) + var e api.Error + _ = json.Unmarshal(rec.Body.Bytes(), &e) + if rec.Code != want[0].(int) || e.Code != want[1].(string) { + t.Errorf("%v → %d %s, want %v", err, rec.Code, e.Code, want) + } + } +} + +func TestCheckoutRoutesWithoutService(t *testing.T) { + s := New(newFakeRuntime()) + s.Token = "t" + h := s.Handler() + for _, c := range []struct{ method, path string }{ + {http.MethodPost, api.PathCheckouts}, {http.MethodGet, api.PathCheckouts}, + {http.MethodGet, api.PathCheckouts + "/chk_000000000000"}, {http.MethodDelete, api.PathCheckouts + "/chk_000000000000"}, + } { + req := httptest.NewRequest(c.method, c.path, strings.NewReader("{}")) + req.Header.Set("Authorization", "Bearer t") + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + var e api.Error + _ = json.Unmarshal(rec.Body.Bytes(), &e) + if rec.Code != http.StatusNotImplemented || e.Code != api.CodeNotSupported { + t.Fatalf("%s %s without service: %d %+v", c.method, c.path, rec.Code, e) + } + } +} + +func TestCheckoutClientRoundTrip(t *testing.T) { + repo, sha := checkoutFixtureRepo(t) + f := newCheckoutFixture(t, repo) + srv := httptest.NewServer(f.h) + defer srv.Close() + c, err := api.NewClient(srv.URL, f.tokens["atlas"]) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + rec, err := c.CreateCheckout(ctx, api.CheckoutRequest{Repo: repo, Commit: sha}) + if err != nil || rec.Owner != "atlas" { + t.Fatalf("CreateCheckout = %+v %v", rec, err) + } + got, err := c.GetCheckout(ctx, rec.ID) + if err != nil || got.ID != rec.ID { + t.Fatalf("GetCheckout = %+v %v", got, err) + } + list, err := c.ListCheckouts(ctx, "atlas") + if err != nil || len(list) != 1 { + t.Fatalf("ListCheckouts = %v %v", list, err) + } + if err := c.DeleteCheckout(ctx, rec.ID); err != nil { + t.Fatal(err) + } + if err := c.DeleteCheckout(ctx, rec.ID); !api.IsNotFound(err) { + t.Fatalf("second delete = %v", err) + } + var se *api.StatusError + if _, err := c.CreateCheckout(ctx, api.CheckoutRequest{Repo: repo, Commit: "main"}); !errors.As(err, &se) || se.Body.Code != api.CodeBadRequest { + t.Fatalf("branch pin = %v", err) + } +} diff --git a/src/crux/serve/principal.go b/src/crux/serve/principal.go new file mode 100644 index 0000000000..fd3f53f2da --- /dev/null +++ b/src/crux/serve/principal.go @@ -0,0 +1,85 @@ +package serve + +import ( + "context" + "net/http" + "strings" +) + +// Caller identity on serve routes (hivecommons/crux#114 S0). Every request +// that passes authenticate carries a Principal in its context: the shared +// control token (or the unauthenticated unix socket of a token-less +// server) is the operator; a lane project credential (lanes.Authenticator) +// is a project principal, accepted only on the routes a work-service +// family registered through workService. Agent RPCs keep refusing project +// credentials exactly as before — the work services are the only surface +// where a non-operator caller exists, because they own records per caller +// (S1 checkouts) and gate answers by caller (S3). + +const ( + // KindOperator is the principal behind the shared control token. It may + // act on every record regardless of owner. + KindOperator = "operator" + // KindProject is a lane project principal (lanes.Config Principals); its + // Name is the project id. + KindProject = "project" + // DefaultOperatorName is Principal.Name for the operator when the + // server was given none. + DefaultOperatorName = "operator" +) + +// Principal is who made an authenticated request. +type Principal struct { + // Kind is KindOperator or KindProject. + Kind string `json:"kind"` + // Name is the operator name or the lane project id. + Name string `json:"name"` +} + +// IsOperator reports whether p holds the shared control credential. +func (p Principal) IsOperator() bool { return p.Kind == KindOperator } + +// String renders kind:name for logs and error messages. +func (p Principal) String() string { return p.Kind + ":" + p.Name } + +// principalKey carries the Principal through the request context. +type principalKey struct{} + +// CallerFrom returns the Principal authenticate attached to ctx. ok is +// false on a context that did not pass through the server's auth gate. +func CallerFrom(ctx context.Context) (p Principal, ok bool) { + p, ok = ctx.Value(principalKey{}).(Principal) + return p, ok +} + +// withCaller returns r with p as its caller. +func withCaller(r *http.Request, p Principal) *http.Request { + return r.WithContext(context.WithValue(r.Context(), principalKey{}, p)) +} + +// operator is the principal for the shared control credential. +func (s *Server) operator() Principal { + name := s.OperatorName + if name == "" { + name = DefaultOperatorName + } + return Principal{Kind: KindOperator, Name: name} +} + +// workService marks every route under prefix (the path itself and its +// subpaths) as a work-services route: project principals are accepted there +// and nowhere else. Handler calls it for each family before the mux is +// wrapped in authenticate. +func (s *Server) workService(prefix string) { + s.workServices = append(s.workServices, prefix) +} + +// isWorkService reports whether path belongs to a work-services family. +func (s *Server) isWorkService(path string) bool { + for _, prefix := range s.workServices { + if path == prefix || strings.HasPrefix(path, prefix+"/") { + return true + } + } + return false +} diff --git a/src/crux/serve/principal_test.go b/src/crux/serve/principal_test.go new file mode 100644 index 0000000000..6a060a1680 --- /dev/null +++ b/src/crux/serve/principal_test.go @@ -0,0 +1,132 @@ +package serve + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/lanes" +) + +const testProjectToken = "pppppppppppppppppppppppppppppppp" + +// principalFixture is a token-bearing server with one lane project +// ("atlas") and a work-services route at /v1/whoami that echoes the caller. +func principalFixture(t *testing.T) (*Server, http.Handler) { + t.Helper() + cfg := lanes.Config{Principals: []lanes.Principal{{Project: "atlas", TokenEnv: "ATLAS"}}} + auth, err := lanes.Bind(cfg, func(string) string { return testProjectToken }, "agent-admin") + if err != nil { + t.Fatal(err) + } + s := New(newFakeRuntime()) + s.Token, s.LaneAuth = "agent-admin", auth + mux := http.NewServeMux() + mux.HandleFunc("GET "+api.PathHealth, s.handleHealth) + mux.HandleFunc("GET /v1/whoami", func(w http.ResponseWriter, r *http.Request) { + p, ok := CallerFrom(r.Context()) + if !ok { + s.writeError(w, http.StatusInternalServerError, api.CodeRuntime, "no caller") + return + } + s.writeJSON(w, http.StatusOK, p) + }) + mux.HandleFunc("GET /v1/whoami/{id}", func(w http.ResponseWriter, r *http.Request) { + p, _ := CallerFrom(r.Context()) + s.writeJSON(w, http.StatusOK, p) + }) + s.workService("/v1/whoami") + return s, s.authenticate(mux) +} + +func callWith(h http.Handler, path, token string) (*httptest.ResponseRecorder, Principal, api.Error) { + req := httptest.NewRequest(http.MethodGet, path, nil) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + var p Principal + var e api.Error + if rec.Code == http.StatusOK { + _ = json.Unmarshal(rec.Body.Bytes(), &p) + } else { + _ = json.Unmarshal(rec.Body.Bytes(), &e) + } + return rec, p, e +} + +func TestPrincipalControlTokenIsOperator(t *testing.T) { + s, h := principalFixture(t) + rec, p, _ := callWith(h, "/v1/whoami", "agent-admin") + if rec.Code != http.StatusOK || p != (Principal{Kind: KindOperator, Name: DefaultOperatorName}) { + t.Fatalf("control token: %d %+v", rec.Code, p) + } + if !p.IsOperator() || p.String() != "operator:operator" { + t.Fatalf("operator predicate/string: %v %q", p.IsOperator(), p) + } + s.OperatorName = "hive-prod" + if _, p, _ = callWith(h, "/v1/whoami", "agent-admin"); p.Name != "hive-prod" { + t.Fatalf("configured operator name: %+v", p) + } + // The operator is also the caller on agent RPCs, which keep working. + if rec, _, _ := callWith(h, api.PathHealth, "agent-admin"); rec.Code != http.StatusOK { + t.Fatalf("health with control token: %d", rec.Code) + } +} + +func TestPrincipalProjectTokenOnlyOnWorkServices(t *testing.T) { + _, h := principalFixture(t) + for _, path := range []string{"/v1/whoami", "/v1/whoami/chk_1"} { + rec, p, _ := callWith(h, path, testProjectToken) + if rec.Code != http.StatusOK || p != (Principal{Kind: KindProject, Name: "atlas"}) { + t.Fatalf("project token on %s: %d %+v %s", path, rec.Code, p, rec.Body) + } + if p.IsOperator() { + t.Fatal("project principal must not be the operator") + } + } + // A path that merely shares the prefix string is not a work service. + if rec, _, e := callWith(h, "/v1/whoamix", testProjectToken); rec.Code != http.StatusUnauthorized || e.Code != api.CodeUnauthorized { + t.Fatalf("prefix lookalike: %d %+v", rec.Code, e) + } + // Agent RPCs refuse project credentials exactly as before. + rec, _, e := callWith(h, api.PathHealth, testProjectToken) + if rec.Code != http.StatusUnauthorized || e.Code != api.CodeUnauthorized || !strings.Contains(e.Error, "project credentials cannot access agent RPCs") { + t.Fatalf("project token on agent RPC: %d %+v", rec.Code, e) + } +} + +func TestPrincipalMissingOrWrongTokenUnchanged(t *testing.T) { + _, h := principalFixture(t) + for _, token := range []string{"", "wrong"} { + for _, path := range []string{"/v1/whoami", api.PathHealth} { + rec, _, e := callWith(h, path, token) + if rec.Code != http.StatusUnauthorized || e.Code != api.CodeUnauthorized || rec.Header().Get("WWW-Authenticate") == "" { + t.Fatalf("token %q on %s: %d %+v", token, path, rec.Code, e) + } + } + } +} + +func TestPrincipalTokenlessUnixSocketIsOperator(t *testing.T) { + s := New(newFakeRuntime()) + var got Principal + var ok bool + h := s.authenticate(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got, ok = CallerFrom(r.Context()) + })) + req := httptest.NewRequest(http.MethodGet, api.PathHealth, nil) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req.WithContext(context.WithValue(req.Context(), networkKey, networkUnix))) + if rec.Code != http.StatusOK || !ok || got != s.operator() { + t.Fatalf("token-less unix request: %d %v %+v", rec.Code, ok, got) + } + if _, ok := CallerFrom(context.Background()); ok { + t.Fatal("a context outside the auth gate has no caller") + } +} diff --git a/src/crux/serve/serve.go b/src/crux/serve/serve.go index 91d7856aec..c020ef905d 100644 --- a/src/crux/serve/serve.go +++ b/src/crux/serve/serve.go @@ -35,6 +35,7 @@ import ( "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/checkout" "github.com/hivecommons/hive/crux/config" "github.com/hivecommons/hive/crux/lanes" ) @@ -101,6 +102,12 @@ type Server struct { // lane surface. They confer no agent-control permissions. Lanes *lanes.Broker LaneAuth *lanes.Authenticator + // OperatorName names the principal behind the shared control token + // (CallerFrom); empty means DefaultOperatorName. + OperatorName string + // Checkouts, when set, serves the pinned-checkout work service + // (api.PathCheckouts); nil answers those routes 501. + Checkouts *checkout.Service now func() time.Time started time.Time @@ -109,6 +116,9 @@ type Server struct { subscribers map[chan api.Event]struct{} lastStates map[string]string background []string + // workServices lists the route prefixes project principals may call; + // see workService. + workServices []string } // New returns a Server for rt with the defaults filled in. @@ -155,6 +165,7 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("POST "+api.PathAgents+"/{name}/stop", s.handleStop) mux.HandleFunc("GET "+api.PathEvents, s.handleEvents) s.registerFamilies(mux) + s.registerCheckouts(mux) return s.laneHandler(s.authenticate(mux)) } @@ -166,13 +177,21 @@ var agentNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]*$`) // authenticate is the fail-closed gate: when the server has no token, only // a request Serve tagged as arriving on the unix socket is allowed, so a // Handler mounted elsewhere (any TCP listener, tagged or not) is refused; -// when the server has a token, every request must present it. +// when the server has a token, every request must present it. Whoever +// passes is attached to the request as a Principal (CallerFrom): the +// control token is the operator, and a lane project credential is a +// project principal on work-services routes only — on agent RPCs it is +// refused as before. func (s *Server) authenticate(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { network, _ := r.Context().Value(networkKey).(string) token, _ := strings.CutPrefix(r.Header.Get("Authorization"), bearerPrefix) - if _, projectToken := s.LaneAuth.Project(token); projectToken { - s.writeError(w, http.StatusUnauthorized, api.CodeUnauthorized, "project credentials cannot access agent RPCs") + if project, projectToken := s.LaneAuth.Project(token); projectToken { + if !s.isWorkService(r.URL.Path) { + s.writeError(w, http.StatusUnauthorized, api.CodeUnauthorized, "project credentials cannot access agent RPCs") + return + } + next.ServeHTTP(w, withCaller(r, Principal{Kind: KindProject, Name: project})) return } if s.Token == "" { @@ -180,7 +199,7 @@ func (s *Server) authenticate(next http.Handler) http.Handler { s.writeError(w, http.StatusUnauthorized, api.CodeUnauthorized, "this server has no bearer token; only unix-socket requests are accepted") return } - next.ServeHTTP(w, r) + next.ServeHTTP(w, withCaller(r, s.operator())) return } got, ok := strings.CutPrefix(r.Header.Get("Authorization"), bearerPrefix) @@ -189,7 +208,7 @@ func (s *Server) authenticate(next http.Handler) http.Handler { s.writeError(w, http.StatusUnauthorized, api.CodeUnauthorized, "missing or invalid bearer token") return } - next.ServeHTTP(w, r) + next.ServeHTTP(w, withCaller(r, s.operator())) }) } diff --git a/src/crux/spekmarker/spekmarker.go b/src/crux/spekmarker/spekmarker.go new file mode 100644 index 0000000000..d3c5699d36 --- /dev/null +++ b/src/crux/spekmarker/spekmarker.go @@ -0,0 +1,131 @@ +// Package spekmarker parses the hidden provenance marker Spek +// (hivecommons/spektacular) writes into every RFC and child issue it +// publishes into a work source: +// +// +// +// The marker is how a published item says "this body was produced from a +// finished spec" (hivecommons/spektacular#114). Crux reads it on the proxy's +// issue-create path so a Spek child filed from inside a hive is treated as +// already specified, and the hive's triage pass reads the same marker to keep +// the child on the direct-fix path instead of re-routing it into a new spec +// run. It is a stdlib-only Crux ring leaf; hive-side packages may re-export or +// copy it, Crux never imports them. +package spekmarker + +import ( + "regexp" + "strings" +) + +// Prefix is the marker's opening token. +const Prefix = "`) + // keyRE is the shape of a marker key: a lowercase identifier. + keyRE = regexp.MustCompile(`^[a-z][a-z0-9_]*$`) +) + +// Marker is one parsed `` comment. +type Marker struct { + // Name is the spek artifact name (`name=`). Required. + Name string + // ArtifactID is the spek's stable artifact id (`artifact_id=`). Required; + // it is what makes re-publish idempotent. + ArtifactID string + // Target is the work source the item was published to (`target=`): + // github, jira or linear. Required. + Target string + // Task is the plan task / member spec id a child carries (`task=`). Empty + // on the RFC itself. + Task string + // RFC is the URL of the parent RFC a child links back to (`rfc=`). + RFC string +} + +// Valid reports whether the marker carries every required field. +func (m Marker) Valid() bool { + return m.Name != "" && m.ArtifactID != "" && m.Target != "" +} + +// IsChild reports whether the marker belongs to a child issue (carries a +// task id) rather than the RFC. +func (m Marker) IsChild() bool { return m.Task != "" } + +// Parse returns the first valid spek:publish marker in text. ok is false +// when text carries no marker, or only markers missing a required field — +// a malformed marker is not evidence of a spec, so callers fall back to +// their default treatment of the body. +func Parse(text string) (Marker, bool) { + for _, match := range markerRE.FindAllStringSubmatch(text, -1) { + if len(match) != 2 { + continue + } + m, ok := parseFields(match[1]) + if ok && m.Valid() { + return m, true + } + } + return Marker{}, false +} + +// Has reports whether text carries a valid spek:publish marker. +func Has(text string) bool { + _, ok := Parse(text) + return ok +} + +// parseFields decodes the `key=value key=value` run inside a marker. Values +// run to the next whitespace; a value wrapped in double quotes may contain +// spaces. An unknown key is ignored so a newer Spek can add fields without +// an older Crux refusing the marker; a key with an empty value or a value +// that still looks like a template token (``) fails the parse. +func parseFields(s string) (Marker, bool) { + var m Marker + rest := strings.TrimSpace(s) + for rest != "" { + eq := strings.IndexByte(rest, '=') + if eq <= 0 { + return Marker{}, false + } + key := rest[:eq] + if !keyRE.MatchString(key) { + return Marker{}, false + } + rest = rest[eq+1:] + var val string + if strings.HasPrefix(rest, `"`) { + end := strings.IndexByte(rest[1:], '"') + if end < 0 { + return Marker{}, false + } + val = rest[1 : end+1] + rest = rest[end+2:] + } else { + end := strings.IndexAny(rest, " \t\r\n") + if end < 0 { + val, rest = rest, "" + } else { + val, rest = rest[:end], rest[end:] + } + } + rest = strings.TrimSpace(rest) + if val == "" || (strings.HasPrefix(val, "<") && strings.HasSuffix(val, ">")) { + return Marker{}, false + } + switch key { + case "name": + m.Name = val + case "artifact_id": + m.ArtifactID = val + case "target": + m.Target = strings.ToLower(val) + case "task": + m.Task = val + case "rfc": + m.RFC = val + } + } + return m, true +} diff --git a/src/crux/spekmarker/spekmarker_test.go b/src/crux/spekmarker/spekmarker_test.go new file mode 100644 index 0000000000..febb9198ed --- /dev/null +++ b/src/crux/spekmarker/spekmarker_test.go @@ -0,0 +1,80 @@ +package spekmarker + +import "testing" + +func TestParse(t *testing.T) { + tests := []struct { + name string + text string + want Marker + wantOK bool + }{ + { + name: "rfc marker with required fields", + text: "## Overview\n\n…\n\n", + want: Marker{Name: "auth-refresh", ArtifactID: "spk_01HX", Target: "github"}, + wantOK: true, + }, + { + name: "child marker with task and rfc", + text: "", + want: Marker{Name: "auth-refresh", ArtifactID: "spk_01HX", Target: "linear", Task: "T3", RFC: "https://github.com/o/r/issues/7"}, + wantOK: true, + }, + { + name: "quoted value with spaces and unknown key tolerated", + text: ``, + want: Marker{Name: "my spec", ArtifactID: "a1", Target: "jira"}, + wantOK: true, + }, + { + name: "loose whitespace around the comment delimiters", + text: "", + want: Marker{Name: "n", ArtifactID: "i", Target: "github"}, + wantOK: true, + }, + {name: "no marker", text: "## Goal\n\nJust a body.", wantOK: false}, + {name: "missing artifact_id", text: "", wantOK: false}, + {name: "missing target", text: "", wantOK: false}, + {name: "empty value", text: "", wantOK: false}, + { + name: "template still unsubstituted is not a marker", + text: "", + wantOK: false, + }, + {name: "bare word without key", text: "", wantOK: false}, + {name: "unterminated quote", text: ``, wantOK: false}, + {name: "other hive marker is not confused", text: "", wantOK: false}, + { + name: "first valid marker wins over an earlier malformed one", + text: "\n", + want: Marker{Name: "n", ArtifactID: "i", Target: "github"}, + wantOK: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := Parse(tt.text) + if ok != tt.wantOK { + t.Fatalf("Parse ok = %v, want %v (got %+v)", ok, tt.wantOK, got) + } + if ok && got != tt.want { + t.Fatalf("Parse = %+v, want %+v", got, tt.want) + } + if Has(tt.text) != tt.wantOK { + t.Fatalf("Has = %v, want %v", !tt.wantOK, tt.wantOK) + } + }) + } +} + +func TestMarkerIsChild(t *testing.T) { + rfc, _ := Parse("") + child, _ := Parse("") + if rfc.IsChild() { + t.Error("RFC marker without task reported as child") + } + if !child.IsChild() { + t.Error("marker with task not reported as child") + } +} diff --git a/src/crux/toolapprove/security.go b/src/crux/toolapprove/security.go index 01d71107da..f52b45094e 100644 --- a/src/crux/toolapprove/security.go +++ b/src/crux/toolapprove/security.go @@ -179,10 +179,13 @@ func (s *DefaultSecurityScanner) Scan(ctx context.Context, req ToolRequest) (Sca // str_replace_editor, an agent's own name for it) must not reach the // guardrail paths unseen. The gate fails closed — a read tool IsReadOnly // does not spell is denied on these paths until it is added there. + // Every path is checked, not just the first: an apply_patch body can + // name a harmless file first and OWNERS second. if req.IsSideEffectful() { - filePath := req.GetFilePath() - if filePath != "" && s.touchesGuardrailPath(filePath) { - violations = append(violations, fmt.Sprintf("tampering with guardrail-critical path: %q", filePath)) + for _, filePath := range req.GetFilePaths() { + if s.touchesGuardrailPath(filePath) { + violations = append(violations, fmt.Sprintf("tampering with guardrail-critical path: %q", filePath)) + } } } diff --git a/src/crux/toolapprove/types.go b/src/crux/toolapprove/types.go index 29b6bb4eb8..62a6ff88c0 100644 --- a/src/crux/toolapprove/types.go +++ b/src/crux/toolapprove/types.go @@ -66,10 +66,75 @@ func (r ToolRequest) GetFilePath() string { } } } + if paths := r.patchPaths(); len(paths) > 0 { + return paths[0] + } } return "" } +// GetFilePaths returns every file path this request targets: the single +// path GetFilePath finds, plus every path named in an apply_patch body. +func (r ToolRequest) GetFilePaths() []string { + var paths []string + seen := map[string]bool{} + add := func(p string) { + if p != "" && !seen[p] { + seen[p] = true + paths = append(paths, p) + } + } + if r.Target != "" { + add(r.Target) + } else if r.Arguments != nil { + for _, k := range []string{"file_path", "filePath", "path", "target", "filename", "file"} { + if s, ok := r.Arguments[k].(string); ok { + add(s) + break + } + } + } + for _, p := range r.patchPaths() { + add(p) + } + return paths +} + +var patchPathHeaders = []string{"*** Update File:", "*** Add File:", "*** Delete File:", "*** Move to:"} + +// patchPaths parses the target paths out of an apply_patch body, which +// carries them in its "*** Update File: " style headers rather than in +// a path argument. +func (r ToolRequest) patchPaths() []string { + if r.Arguments == nil { + return nil + } + isPatchTool := strings.EqualFold(strings.TrimSpace(r.Tool), "apply_patch") + var paths []string + seen := map[string]bool{} + for _, k := range []string{"input", "patch"} { + body, ok := r.Arguments[k].(string) + if !ok || (!isPatchTool && !strings.HasPrefix(strings.TrimSpace(body), "*** Begin Patch")) { + continue + } + for _, line := range strings.Split(body, "\n") { + line = strings.TrimSpace(line) + for _, h := range patchPathHeaders { + if !strings.HasPrefix(line, h) { + continue + } + p := strings.TrimSpace(strings.TrimPrefix(line, h)) + if p != "" && !seen[p] { + seen[p] = true + paths = append(paths, p) + } + break + } + } + } + return paths +} + // GetContent returns the text content or body payload associated with this request. func (r ToolRequest) GetContent() string { if r.Content != "" { diff --git a/src/crux/toolapprove/types_apply_patch_test.go b/src/crux/toolapprove/types_apply_patch_test.go new file mode 100644 index 0000000000..24280b8d74 --- /dev/null +++ b/src/crux/toolapprove/types_apply_patch_test.go @@ -0,0 +1,68 @@ +package toolapprove + +import ( + "context" + "reflect" + "strings" + "testing" +) + +func TestToolRequestApplyPatchPaths(t *testing.T) { + multi := "*** Begin Patch\n*** Update File: src/a.go\n@@\n-x\n+y\n*** Add File: OWNERS \n+a\n*** Delete File: policies/p.yaml\n*** Update File: src/a.go\n*** Move to: hive.yaml\n\n*** End Patch" + tests := []struct { + name string + req ToolRequest + wantFirst string + wantAll []string + }{ + {"input key", ToolRequest{Tool: "apply_patch", Arguments: map[string]any{"input": multi}}, "src/a.go", []string{"src/a.go", "OWNERS", "policies/p.yaml", "hive.yaml"}}, + {"patch key", ToolRequest{Tool: "apply_patch", Arguments: map[string]any{"patch": "*** Begin Patch\n*** Update File: .github/workflows/ci.yml\n*** End Patch"}}, ".github/workflows/ci.yml", []string{".github/workflows/ci.yml"}}, + {"unknown tool with patch text", ToolRequest{Tool: "editor", Arguments: map[string]any{"input": "*** Begin Patch\n*** Add File: OWNERS\n*** End Patch"}}, "OWNERS", []string{"OWNERS"}}, + {"path argument wins", ToolRequest{Tool: "apply_patch", Arguments: map[string]any{"path": "x.go", "input": multi}}, "x.go", []string{"x.go", "src/a.go", "OWNERS", "policies/p.yaml", "hive.yaml"}}, + {"non-patch text ignored", ToolRequest{Tool: "write", Arguments: map[string]any{"input": "*** Update File: OWNERS"}}, "", nil}, + {"no arguments", ToolRequest{Tool: "apply_patch"}, "", nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.req.GetFilePath(); got != tt.wantFirst { + t.Errorf("GetFilePath() = %q, want %q", got, tt.wantFirst) + } + if got := tt.req.GetFilePaths(); !reflect.DeepEqual(got, tt.wantAll) { + t.Errorf("GetFilePaths() = %v, want %v", got, tt.wantAll) + } + }) + } +} + +// TestSecurityScanner_ApplyPatchGuardrailAnyPosition drives Scan with a +// multi-file apply_patch so a guardrail path that is not the first header +// still fails the scan (#16979). +func TestSecurityScanner_ApplyPatchGuardrailAnyPosition(t *testing.T) { + scanner := NewDefaultSecurityScanner() + tests := []struct { + name string + patch string + wantPassed bool + wantPath string + }{ + {"guardrail path second", "*** Begin Patch\n*** Update File: src/a.go\n@@\n-x\n+y\n*** Add File: OWNERS\n+a\n*** End Patch", false, `"OWNERS"`}, + {"guardrail path last via move", "*** Begin Patch\n*** Update File: src/a.go\n*** Update File: src/b.go\n*** Move to: policies/p.yaml\n*** End Patch", false, `"policies/p.yaml"`}, + {"guardrail path first", "*** Begin Patch\n*** Update File: hive.yaml\n*** Update File: src/a.go\n*** End Patch", false, `"hive.yaml"`}, + {"no guardrail paths", "*** Begin Patch\n*** Update File: src/a.go\n*** Add File: src/b.go\n*** End Patch", true, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := ToolRequest{Tool: "apply_patch", Arguments: map[string]any{"input": tt.patch}} + res, err := scanner.Scan(context.Background(), req) + if err != nil { + t.Fatalf("scanner error: %v", err) + } + if res.Passed != tt.wantPassed { + t.Fatalf("Passed = %v, want %v (violations: %v)", res.Passed, tt.wantPassed, res.Violations) + } + if tt.wantPath != "" && !strings.Contains(res.Details, tt.wantPath) { + t.Errorf("Details %q does not name %s", res.Details, tt.wantPath) + } + }) + } +} diff --git a/src/crux/watchdog/classify.go b/src/crux/watchdog/classify.go index 916d8886a0..565a104aff 100644 --- a/src/crux/watchdog/classify.go +++ b/src/crux/watchdog/classify.go @@ -132,6 +132,27 @@ func paneMatchesAny(pane string, patterns []string) bool { return false } +// authScreenTailLines bounds where a real login prompt is looked for: it sits +// at the bottom of the pane, while source/grep output scrolls through above. +const authScreenTailLines = 15 + +// paneShowsAuthScreen reports whether the tail of the pane carries an auth +// pattern outside source/grep output (#17006). +func paneShowsAuthScreen(pane string) bool { + lines := strings.Split(strings.TrimRight(pane, " \t\r\n"), "\n") + if len(lines) > authScreenTailLines { + lines = lines[len(lines)-authScreenTailLines:] + } + for _, line := range lines { + for _, p := range authScreenPatterns { + if strings.Contains(line, p) && !authHitIsSourceText(line, p) { + return true + } + } + } + return false +} + // looksLikeShellPrompt reports whether the last non-empty line of the pane is // a bare shell prompt (short line ending in a prompt character). func looksLikeShellPrompt(pane string) bool { @@ -197,7 +218,7 @@ func Classify(obs Observation, now time.Time, s Settings) Classification { } return Classification{ClassNoSession, "tmux session missing"} } - if obs.ShowsLoginPrompt || paneMatchesAny(obs.Pane, authScreenPatterns) { + if obs.ShowsLoginPrompt || paneShowsAuthScreen(obs.Pane) { // Boot grace covers login chrome too. Interactive CLIs paint a login // screen DURING their startup auth handshake even when the seeded // credential is about to be accepted -- the pane poller already diff --git a/src/crux/watchdog/classify_test.go b/src/crux/watchdog/classify_test.go index 6ab3e8e6fe..f7eea61d59 100644 --- a/src/crux/watchdog/classify_test.go +++ b/src/crux/watchdog/classify_test.go @@ -200,3 +200,29 @@ func TestLooksLikeShellPromptSkipsTrailingBlanks(t *testing.T) { t.Fatal("empty pane is not a shell prompt") } } + +func TestPaneShowsAuthScreenIgnoresSourceText(t *testing.T) { + filler := strings.Repeat("working\n", authScreenTailLines) + cases := []struct { + name string + pane string + want bool + }{ + {"grep prefix", "classify.go:99: \"Please run /login\",", false}, + {"quoted literal", " \"Login expired\",", false}, + {"backtick literal", "see `Sign in to continue` here", false}, + {"regex alternation", "rg 'credentialValid|Login expired|Run /login'", false}, + {"table name", "var authScreenPatterns = []string{ Login expired", false}, + {"login expired chrome", "● Login expired · Please run /login", true}, + {"sign in", "Sign in to continue", true}, + {"scrolled out of tail", "Sign in to continue\n" + filler, false}, + {"real prompt under old output", filler + "Login expired · Please run /login\n", true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := paneShowsAuthScreen(tc.pane); got != tc.want { + t.Fatalf("paneShowsAuthScreen() = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/src/crux/watchdog/source_text.go b/src/crux/watchdog/source_text.go new file mode 100644 index 0000000000..a82c9e1084 --- /dev/null +++ b/src/crux/watchdog/source_text.go @@ -0,0 +1,56 @@ +package watchdog + +import ( + "regexp" + "strings" +) + +// grepPrefixRe matches the "path.go:NN:" prefix grep/rg put on a hit. +var grepPrefixRe = regexp.MustCompile("^\\s*[\\w./-]+\\.\\w+:\\d+[:-]") + +// patternTableNames are identifiers whose presence on a line means the pane is +// showing hive's own login-pattern source, not a login screen (#17006). +var patternTableNames = []string{"authScreenPatterns", "loginPromptPatterns", "classify.go"} + +// authHitIsSourceText reports whether every match of pat on line is source or +// grep output quoting the pattern (a path.go:NN: prefix, a pattern-table name, +// a string literal, or a |-joined alternation) rather than a CLI login screen. +func authHitIsSourceText(line, pat string) bool { + if grepPrefixRe.MatchString(line) { + return true + } + for _, name := range patternTableNames { + if strings.Contains(line, name) { + return true + } + } + for off := 0; ; { + i := strings.Index(line[off:], pat) + if i < 0 { + return true + } + start := off + i + end := start + len(pat) + if !hitIsQuoted(line, start, end) { + return false + } + off = end + } +} + +// hitIsQuoted reports whether the match at [start,end) sits inside a quoted +// literal or beside a regex alternation bar. +func hitIsQuoted(line string, start, end int) bool { + before, after := line[:start], line[end:] + if strings.Count(before, "\"")%2 == 1 || strings.Count(before, "`")%2 == 1 { + return true + } + b := strings.TrimRight(before, " ") + a := strings.TrimLeft(after, " ") + for _, q := range []string{"'", "\"", "`", "|"} { + if strings.HasSuffix(b, q) || strings.HasPrefix(a, q) { + return true + } + } + return false +} diff --git a/src/docs/acmm-advisor.md b/src/docs/acmm-advisor.md index 02806082e3..cb2015baa7 100644 --- a/src/docs/acmm-advisor.md +++ b/src/docs/acmm-advisor.md @@ -18,7 +18,7 @@ The advisor evaluates a fixed set of six signals (`Signals` struct, `src/pkg/acm | `HoldCount` | Count of open PRs still carrying a `hold` label awaiting human review. | | `HasQualityAgent` | Whether an active quality/coverage agent is present in the config. | -Levels run L1 (Assisted) through L6 (Autonomous), matching the level pack files and the dashboard's 1–6 bound (`src/pkg/acmmadvisor/acmmadvisor.go:34-42`). The advisor never proposes a target more than one level above the current level — advancement is always incremental (`levelStep = 1`, same file). +Levels run L1 (Assisted) through L6 (Autonomous), matching the level [pack](acmm-policy-matrix.md#what-is-a-pack) files and the dashboard's 1–6 bound (`src/pkg/acmmadvisor/acmmadvisor.go:34-42`). The advisor never proposes a target more than one level above the current level — advancement is always incremental (`levelStep = 1`, same file). ### Where the live signals come from diff --git a/src/docs/acmm-policy-matrix.md b/src/docs/acmm-policy-matrix.md index 9b1c71edfc..00753cea08 100644 --- a/src/docs/acmm-policy-matrix.md +++ b/src/docs/acmm-policy-matrix.md @@ -16,6 +16,20 @@ issues. The **KB** is the knowledge base of reusable facts; an **RFC** is a request for comments on a proposed design. For a first deployment's trust choices, start with [Securing your hive](securing-your-hive.md). +## What is a pack? + +An ACMM **pack** is the bundle of settings Hive applies for one ACMM level. +Hive ships six, `level-1.yaml` through `level-6.yaml` +(`src/pkg/config/packs/`, embedded in the binary), one per level. Each pack +defines that level's agent roster, each agent's policy mode and kick template, +the governor cadences, and the merge policy. Setting `acmm_level` (or +`PUT /api/packs/level`) selects the pack, and applying it reconciles the +hive's agents to match; see [Changing a hive's ACMM level](#changing-a-hives-acmm-level). +Your explicit `backend:`, `model:`, and `enabled: false` settings win over +pack values. "Pack" is a Hive term for implementing a level; the ACMM paper +describes levels, not packs. For the full reconcile rules, see +[ACMM levels: agent rosters as packs](agent-configuration.md#acmm-levels-agent-rosters-as-packs). + ## Repository E2E detection The codebase evaluator's `acmm:prereq-e2e` prerequisite accepts the existing diff --git a/src/docs/agent-configuration.md b/src/docs/agent-configuration.md index 88d601eaba..49d77ec494 100644 --- a/src/docs/agent-configuration.md +++ b/src/docs/agent-configuration.md @@ -880,7 +880,7 @@ agents: ## ACMM levels: agent rosters as packs -You don't have to design a roster. Hive ships six **ACMM packs** (`level-1.yaml` … `level-6.yaml`, embedded in the binary and forkable) that pair a curated agent roster with governor cadences and a merge policy: +You don't have to design a roster. Hive ships six **ACMM packs** (see [What is a pack?](acmm-policy-matrix.md#what-is-a-pack); `level-1.yaml` … `level-6.yaml`, embedded in the binary and forkable) that pair a curated agent roster with governor cadences and a merge policy: | Level | Name | Posture | |---|---|---| diff --git a/src/docs/crux-serve.md b/src/docs/crux-serve.md index 06067bedd4..e5d7a33d77 100644 --- a/src/docs/crux-serve.md +++ b/src/docs/crux-serve.md @@ -33,6 +33,9 @@ runtime and every agent must arrive as a definition on `POST /v1/agents`. | `--socket PATH`, `CRUX_SOCKET` | unix socket to serve on. Default `$XDG_RUNTIME_DIR/crux/crux.sock`, else `/crux-/crux.sock`. The directory is created `0700`, the socket `0600`, and it is never reachable at that path with wider permissions (it is bound under a temporary name, chmod'ed, then renamed). A socket somebody already answers on is refused; a stale one is replaced. | | `--listen HOST:PORT` | also serve on TCP. **Refused without a token.** | | `--token T`, `CRUX_TOKEN` | bearer token every request must carry. When set it gates the socket too. | +| `--operator-name NAME`, `CRUX_OPERATOR_NAME` | the name the control token's caller is recorded under on work-services routes (see [Caller principals](#caller-principals)). Default `operator`. | +| `--checkout-allow PREFIX,…`, `CRUX_CHECKOUT_ALLOWLIST` | repository URL prefixes `POST /v1/checkouts` may clone; unset means no checkout service ([design](design/crux-work-services.md)). | +| `--checkout-dir PATH`, `CRUX_CHECKOUT_DIR` | where pinned checkouts live. Default `/checkouts`. | | `--lanes-config PATH`, `CRUX_LANES_CONFIG` | opt into [durable event/work lanes](crux-lanes.md) on this same server, backed by NATS JetStream. Lane project credentials are separate from agent-control credentials. | | `--hive-callback ADDR`, `CRUX_HIVE_SOCKET` (+ `CRUX_HIVE_TOKEN`) | dial hive's callback endpoint (`crux/api/callback`) for the capabilities hive owns — scoped GitHub tokens, PR creation, Linear credentials, text/canary scanners, conditions — and feed the runtime's seams from it instead of leaving them nil. Health reports it under `hive_callback`; an unreachable endpoint raises the `hive_callback_unavailable` condition. (#14246) When the runtime is a manager it also polls hive's host policy from the same endpoint; see [Host policy from hive](#host-policy-from-hive-14138-stage-4). | @@ -55,6 +58,23 @@ runtime and every agent must arrive as a definition on `POST /v1/agents`. world-readable `/crux-` is an error, not a trust. - Request bodies are capped at 1 MiB and unknown JSON fields are rejected. +### Caller principals + +Every request that passes the auth gate carries a *principal* in its +context, readable by a route handler with `serve.CallerFrom(ctx)` +(hivecommons/crux#114, S0). It is the identity the work services record and +enforce ownership by; nothing on the agent-RPC routes reads it. + +| Credential presented | Principal | Where accepted | +|---|---|---| +| the control token (`--token`/`CRUX_TOKEN`), or no token on a token-less server's own unix socket | `{"kind":"operator","name":"<--operator-name>"}` | every route | +| a lane project credential (`--lanes-config` principals) | `{"kind":"project","name":""}` | work-services routes only (families registered as such, e.g. `/v1/checkouts`); on every agent RPC it is still `401 unauthorized`, as before | +| anything else | — | `401 unauthorized` | + +The operator may act on every record regardless of who owns it; a project +principal only on its own. Lane routes (`/v1/lanes…`) keep their separate +project-only gate and are not affected. + ## The API All routes are under `/v1`. Bodies and responses are JSON; agent-control errors are @@ -74,6 +94,7 @@ and [lane error codes and failure behavior](crux-lanes.md#bounds-and-failure-beh | `POST /v1/agents/{name}/kick` | `{"message":…}` | `Kicker.SendKick` | `{"agent","process"}` | | `POST /v1/agents/{name}/stop` | | `Lifecycle.Stop` | `{"agent","process"}` | | `GET /v1/events` | | watcher + `Kicker.SetKickObserver` + the `crux.Host` observers | `text/event-stream` of `Event` records | +| `POST /v1/checkouts`, `GET /v1/checkouts[?owner=]`, `GET`/`DELETE /v1/checkouts/{id}` | `{"repo","commit","owner"?,"branches"?,"max_bytes"?,"ttl"?}` | `checkout.Service` | Work service (S1): a worktree pinned at a full SHA, owned by the caller principal; `501` without `--checkout-allow`; [routes, codes and semantics](design/crux-work-services.md) | | `GET /v1/lanes` and lane/group subroutes | Publish/recovery/receipt JSON; [request shapes](crux-lanes.md#public-http-and-go-api) | `lanes.Broker` | Opt-in only; lane-project credential and capability ACLs; JSON or `204`; [full routes and responses](crux-lanes.md#public-http-and-go-api) | ### Runtime families (#14138 stage 3) @@ -266,6 +287,9 @@ crux logs -n 50 --buffer fixer crux logs -f fixer # follow the ring buffer over SSE crux kick fixer "look at the failing build on main" crux stop fixer +crux checkout add -repo https://github.com/hivecommons/hive -commit <40-hex sha> # S1 pinned checkout, owned by the caller +crux checkout list +crux checkout rm chk_0123456789ab ``` In remote mode nothing is built in this process: no runtime, no control diff --git a/src/docs/design/crux-tool-policy.md b/src/docs/design/crux-tool-policy.md index e2733ae340..9689ead397 100644 --- a/src/docs/design/crux-tool-policy.md +++ b/src/docs/design/crux-tool-policy.md @@ -118,6 +118,7 @@ and Linear tables gate. Defaults equal today's verdicts: | `issue.edit` | `PATCH …/issues/{n}` | (default bucket) | `ModeIssuesOnly` | | `issue.comment` | `POST …/issues/{n}/comments` | conversational fields (`converseMutations`) | `ModeIssuesOnly` or `converse` | | `label.add` | `POST …/issues/{n}/labels` | (default bucket) | `ModeIssuesOnly` | +| `issue.relate` | `POST …/issues/{n}/sub_issues`, `POST …/issues/{n}/dependencies/blocked_by` (Linear: `issueRelationCreate`, `projectCreate`, `projectUpdate`, `issueCreate` with `parentId`) | (default bucket) | `ModeIssuesOnly` — what an in-hive Spek publish needs to file an RFC with sub-issue children and "blocked by" edges ([spektacular#114](https://github.com/hivecommons/spektacular/issues/114)); unlink/delete/archive siblings stay denied | | `pr.create` | `POST …/pulls` | `createPullRequest` | **`relay(hive-open-pr)`** | | `pr.edit` | `PATCH …/pulls/{n}` | `markPullRequestReadyForReview`, `convertPullRequestToDraft` | `ModeIssuesAndPRs` | | `pr.review` | `POST …/pulls/{n}/reviews` | `addPullRequestReview`, `submitPullRequestReview` | `ModeIssuesAndPRs` or `converse` | diff --git a/src/docs/design/crux-work-services.md b/src/docs/design/crux-work-services.md new file mode 100644 index 0000000000..1530515246 --- /dev/null +++ b/src/docs/design/crux-work-services.md @@ -0,0 +1,160 @@ +# Crux work services: S1 pinned checkout (hivecommons/crux#114) + +Status: **S0 caller principals and S1 pinned checkout shipped in hive +`src/crux` (v5).** S2 contained run and S7 minimum are in flight in a +sibling change; everything else in the RFC is unscheduled. This records +S1 *as implemented*, including where it is weaker than the RFC text. + +The RFC asks for "a working copy of this repo at this commit, owned by +this caller, on this server" as the first work service, so that S2 runs +have something to run in. This is that: `crux serve` grows a +`checkout.Service` behind `/v1/checkouts`, and the `crux` CLI grows +`checkout add|rm|list`. + +## Surface + +### HTTP (`src/crux/api/checkouts.go`, `src/crux/serve/checkouts.go`) + +| Route | Body | Response | +|---|---|---| +| `POST /v1/checkouts` | `{"repo", "commit", "owner"?, "branches"?, "max_bytes"?, "ttl"?}` | `201 Checkout` | +| `GET /v1/checkouts[?owner=NAME]` | | `200 {"checkouts":[Checkout…]}` sorted by id | +| `GET /v1/checkouts/{id}` | | `200 Checkout` | +| `DELETE /v1/checkouts/{id}` | | `204` | + +`Checkout` is `{"id","path","repo","commit","owner","branches","max_bytes", +"bytes","ttl"?,"created_at","expires_at"?}`. Nothing in it is secret: the +record never carries environment, and repository URLs that embed +credentials are refused before anything is cloned. + +Error codes, all `api.Error` JSON: + +| Condition | Status | `code` | +|---|---|---| +| `commit` is not a full 40-hex SHA (a tag, branch or short SHA) | 400 | `bad_request` | +| bad `ttl`, bad `owner`, negative `max_bytes`, malformed URL, unknown JSON field | 400 | `bad_request` | +| `repo` matches no allowlist prefix; a project principal naming another `owner`; reading or deleting a checkout you do not own | 403 | `forbidden` (the message names the allowed prefixes or the owner) | +| unknown id | 404 | `not_found` | +| `DELETE` while runs use it (the `InUse` hook) | 409 | `conflict` | +| clone plus worktree measure more than `max_bytes` | 413 | `too_large` | +| the server has no checkout service; the size could not be measured | 501 | `not_supported` / `unsupported` | +| the worktree's `HEAD` is not the requested SHA | 502 | `upstream_error` (names both SHAs) | +| git failed | 500 | `runtime_error` | + +A server without `--checkout-allow` registers the routes and answers 501 +on all of them, so a client can tell "not here" from "not found". + +### CLI (`src/crux/cli/checkout.go`) + +``` +crux checkout add -repo URL -commit SHA [-owner NAME] [-ttl 72h] [-max-bytes N] [-no-branches] [-json] +crux checkout rm ID +crux checkout list [-owner NAME] [-json] +``` + +Local (`crux checkout …`) runs the service in-process on +`/checkouts` (`-dir`/`CRUX_CHECKOUT_DIR`) with `-allow`/ +`CRUX_CHECKOUT_ALLOWLIST` as its allowlist; without an allowlist it can +still list and remove but refuses every `add`. Remote (`crux --server … +checkout …`) drives the server's service through the API client. The +verb is not forwarded over the foreground control socket. Bad requests +and refusals exit with the usage status, everything else with the error +status. + +### Server flags (`crux serve`) + +| Flag / env | Meaning | +|---|---| +| `--checkout-allow PREFIX,…`, `CRUX_CHECKOUT_ALLOWLIST` | repository URL prefixes `POST /v1/checkouts` may clone. **Unset means no checkout service.** | +| `--checkout-dir PATH`, `CRUX_CHECKOUT_DIR` | where checkouts live. Default `/checkouts`. | + +## Semantics + +**Pin.** `commit` must be a full 40-hex SHA; tags and branches are refused +with `bad_request` because a moving ref is not a pin. After `git worktree +add --detach ` the service runs `git rev-parse HEAD` in the worktree +and refuses with 502 naming both SHAs if they differ. + +**Allowlist, deny by default.** The allowlist is a list of URL prefixes +(`https://github.com/hivecommons/`, `file:///srv/mirrors/`). Each entry +and each request URL must be `https://` or `file://`, have a host (or a +path, for `file`), and carry no userinfo, query or fragment; the request +URL must start with an entry. An empty allowlist refuses everything. This +is the shape of hive's `nousdeps` trust rules, ported without the +per-package specifics. Private repositories therefore work only through +whatever the server's own git credential helper provides; the API never +accepts a credential. + +**Ownership via S0.** `owner` is recorded on every checkout. The caller's +[principal](../crux-serve.md#caller-principals) decides what it may name: +a lane project principal may create only checkouts owned by itself and +sees only its own on `GET` (an `?owner=` filter is overridden); the +operator may name any owner (default: its own name) and sees everything. +`GET /v1/checkouts/{id}` and `DELETE` succeed for the owner and the +operator and answer 403 for anyone else. + +**Live runs block delete.** `checkout.Config.InUse func(id string) bool` +is the hook S2 wires to its run state; the default returns false. A true +answer turns `DELETE` into 409 `conflict`. Delete is otherwise +`git worktree remove --force` (falling back to `os.RemoveAll`), the record +file removed, and an index line appended naming who deleted it. + +**TTL is recorded, not enforced.** `ttl` is parsed as a Go duration and +stored with `expires_at`. There is no sweeper yet; `list -json` exposes +`expires_at` so one can be written against the index. + +**Byte cap, honestly.** Git cannot bound a clone before it happens; there +is no pre-flight size query across the transports we allow. The cap is +therefore *post-hoc*: the service clones (`--no-checkout`, plus +`--single-branch` when `branches` is false), adds the worktree, then walks +the regular files of the shared clone and the worktree and sums their +sizes. Over `max_bytes` (default 5 GiB) it removes the worktree and +answers 413 `too_large` with both numbers; if the measurement itself +fails it answers 501 `unsupported`. The bytes were already transferred +and the shared clone is kept (a later request for the same repo reuses +it). `--filter=blob:limit=` partial clones would make the clone itself +smaller but would also make the worktree need network at checkout time; +that trade is not taken here and is deferred. The record stores both +`max_bytes` and the measured `bytes`. + +**Bound on live worktrees.** `checkout.Config.MaxCheckouts` (default 64) +is the `sandbox.WorktreeManager` cap, counted as `.git` entries under the +runs directory. + +## Storage + +Under `` (`WorkDir()/checkouts` by default): + +``` +checkouts/.json one Record per checkout, 0600, written tmp+rename +checkouts/index.jsonl append-only {op, id, owner, repo, commit, at, by} +checkouts/src/-/ the shared bare-ish clone (--no-checkout), per owner and repo +checkouts/runs//pin-1/ the worktree handed back as `path` +``` + +Ids are `chk_` plus twelve hex characters. The `src` directory is keyed +by owner and a hash of the repository URL so two owners never share a +clone (a project cannot be made to pay for, or read, another's fetches). +Records follow the `sandbox_run.go` conventions: one file per entity plus +a `.jsonl` index. + +## Deliberately not in S1 + +- TTL sweeper (recorded, not enforced). +- `--filter=blob:limit=` partial clones; the cap is post-hoc only. +- A per-owner allowlist; one list per server. +- Wiring `InUse` to live runs: that is S2's side of the seam. +- `ssh://` and `http://` repositories; `git@` syntax. +- Anything about the hivecommons/crux repository or the publish path: + the service lives in hive `src/crux` and ships through the existing + mirror unchanged. + +## Package map + +| Path | Role | +|---|---| +| `src/crux/checkout/` | `Service`, `Config`, `Record`, typed sentinels (`ErrBadRequest`, `ErrForbidden`, `ErrNotFound`, `ErrInUse`, `ErrMismatch`, `ErrTooLarge`, `ErrUnsupported`, `ErrGit`); store and index | +| `src/crux/api/checkouts.go` | `PathCheckouts`, `CodeTooLarge`, `CodeUnsupported`, `CodeUpstream`, request/response types, `Client.{Create,List,Get,Delete}Checkout` | +| `src/crux/serve/checkouts.go` | routes, ownership checks, error mapping; registered from `Server.Handler` after the runtime families and marked a work service for S0 | +| `src/crux/serve/principal.go` | S0: `Principal`, `CallerFrom`, work-service gating | +| `src/crux/cli/checkout.go` | `crux checkout add|rm|list`, local and remote | diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index 1c94ba7386..7f889fe0ac 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -145,8 +145,8 @@ title is `[selfreport] : `. | `effect_ledger_op_stuck` | the mutation boundary refused a retry because the journaled external effect (issue create, PR open, …) has been `Unknown`/`Planned` for ≥30 min — the caller keeps replaying the same logical op and it never resolves. Key is the op's logical ID; message carries kind, status and age, diagnostics the attempt count, repo and claim target. Clears when the op is applied, replayed as already-applied, or reconciled | medium | | `agent_dir_setup_failed` | the harness failed to set up the per-agent sandbox directory structure — create XDG directories, restore or tighten directory permissions on interactive/inference home directories. Log-fed: keywords include `failed to create per-agent xdg dir`, `failed to restore per-agent dir mode`, `failed to restore interactive home mode`, `failed to tighten per-agent dir mode`, `failed to tighten interactive home mode`, `failed to restore inference home mode`, `failed to tighten inference home mode` (`crux/agent/manager.go`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `agent_pane_timeout` | the agent-readiness wait timed out and the harness dumped the tmux pane (`prompt timeout — dumping pane` / `pane content at timeout`, `crux/agent/manager.go`), and the startup kick dropped because the CLI never became ready or reached its input prompt (`startup kick dropped: …`, `crux/agent/manager_kick.go`, the same event as the pane dump). Log-fed: same ≥5-in-10-min repeat threshold and 20-min idle clear as the catch-all below; pane excerpts are scrubbed and bounded before storage | low | -| `agent_login_required` | the login scanner saw the agent CLI asking for `/login` (`login required detected`, `pkg/loginscan`) — the agent's auth lapsed and it is paused until an operator re-authenticates. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | -| `pr_request_head_unpushed` | the pr-request watcher found the PR head branch missing on the remote (`head branch is not on the remote`) — the agent's push did not authenticate or never ran. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | +| `agent_login_required` | the login scanner saw the agent CLI asking for `/login` (`login required detected`, `pkg/loginscan`) — the agent's auth lapsed and it is paused until an operator re-authenticates. The record carries `backend` (which provider to re-authenticate), `login_hint` (the CLI's login command) and `credential_valid`; re-login in the agent's tmux pane, then Resume. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | +| `pr_request_head_unpushed` | the pr-request watcher found the PR head branch missing on the remote (`head branch is not on the remote`): the agent's push failed, never ran, or went to another ref — the line's `checked` and `push_outcome` attributes say which. Logged once per request (retries stay at INFO), so the repeat threshold counts distinct unpushed requests, not one request's retries ([#16969](https://github.com/hivecommons/hive/issues/16969)). Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `bead_fact_ingest_failed` | the knowledge system failed to ingest a bead fact from an agent (`failed to ingest bead fact`, `pkg/knowledge`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `feedback_issue_refresh_failed` | the dashboard's feedback-issue watcher failed to refresh or update a feedback issue (`feedback: issue refresh failed`, `pkg/dashboard`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `github_app_repo_not_covered` | the GitHub App recheck (`state=repo-not-covered`) found the installation does not include every configured repo — an operator configuration condition (tick the repos in the App's repository access), not a hive defect. Log-fed: same ≥5-in-10-min repeat gate and 20-minute idle clear as `agent_pane_timeout` | low | diff --git a/src/docs/hive-merge.md b/src/docs/hive-merge.md index 8c4d1cb100..caac5ad845 100644 --- a/src/docs/hive-merge.md +++ b/src/docs/hive-merge.md @@ -172,13 +172,12 @@ can edit the map from **Settings → Features → Auto-merge → Per-repository policy**, one comma-separated pattern list per repo in `project.repos`. See [operator-reference.md](operator-reference.md). -**Pin-only policy (enforced by the automerge sweep and `/approve`; relay pending).** +**Pin-only policy (enforced by the automerge sweep, `/approve`, and relay).** `auto_merge.human_merge_pin_only` provides the schema and conservative diff classifier for #16242 / #16262. **The automerge sweep lanes and `/approve` -enforce it (#16266); the `hive-merge` relay does not yet** (#16265) and still -requires the existing human-merge path policy. The native merge queue is entered -only after a lane's human-merge-path gate, so queued PRs inherit the exception. A sweep lane +(#16266) and the `hive-merge` relay (#16265) enforce it.** The native merge +queue is entered only after a lane's human-merge-path gate, so queued PRs inherit the exception. A sweep lane does not hold a PR on its human-merge-path matches when the repo has an enabled entry, the PR author is in `trusted_authors` (case-insensitive login match), GitHub's reported changed-file count is present and equals the fetched list, @@ -238,8 +237,10 @@ This classifier proves **syntax only**, not authenticated identity, exact-head binding, file-list completeness, semantic interpretation or review of upstream references. The sweep supplies the author (GitHub's PR author login), the verified-commit provenance of the head, file-list completeness and the -upstream anchoring of each action SHA itself (#16399); the relay integration -remains #16265; +upstream anchoring of each action SHA itself (#16399). The relay applies +the same checks through `Client.SetHumanMergePinOnly` (#16265) and additionally +requires a nonempty requested head SHA matching the PR head before and after +fetching files; attribution and safe hold release are tracked in #16269. None of the schema or classifier APIs grants permission to merge or clears a hold. Native required checks and producer bindings, freshness, current reviews, substantive @@ -247,9 +248,9 @@ objections, unrelated holds and configured soak must remain independent gates. Operators stage the key in the controller's configured `hive.yaml` seed, not in the image repository (reload behaviour is documented in the operator reference -linked above). On an image that contains #16266, reloading the seed enables -the sweep and `/approve` exception at once; the `hive-merge` relay still -ignores the key until #16265 lands, so relay merges keep holding matching PRs. +linked above). On an image that contains #16266 and #16265, reloading the seed +enables the sweep, `/approve`, and `hive-merge` relay exception at once. Non-GitHub +forge relays retain the existing fail-closed human-merge-path policy. There is **no minimum deployable version enabling this lane yet**. Utah rollout and current-head positive/negative deployment evidence remain tracked by @@ -259,8 +260,8 @@ projectbluefin/utah#1082 and #16242; projectbluefin/utah#1068 is not pre-authori `auto_merge.human_merge_pin_only`; see the [operator reference](operator-reference/README.md#human-merge-pin-only-rollout) for its configuration and deployment prerequisites. The schema/classifier -(#16262), relay admission (#16265), and sweep admission (#16266, landed) are -separate changes: this attribution change alone does **not** enable the exception. +(#16262), relay admission (#16265), and sweep admission (#16266) are +separate, landed changes: this attribution change alone does **not** enable the exception. New path-hold comments retain `` and append `` only when Hive observes that exact head without a pre-existing hold. Legacy comments and diff --git a/src/docs/hive-open-pr.md b/src/docs/hive-open-pr.md index 5d4b16f572..0c848508aa 100644 --- a/src/docs/hive-open-pr.md +++ b/src/docs/hive-open-pr.md @@ -380,26 +380,37 @@ recognise it. The branch is not implicated at all. Fix the App installation — see [GitHub App setup](github-app-setup.md). -**2. The repository is visible but the head ref is genuinely absent.** This is -the push-authentication case, and the error says so: +**2. The repository is visible but the head ref is genuinely absent.** The +branch is not where the watcher looked, and the error says what it checked: -> branch `` was never pushed to `/` — the commits exist only in the agent's working copy. This is almost always a PUSH AUTHENTICATION failure, not a branch-creation problem +> branch `` is not on `/` — the commits exist only in the agent's working copy, or were pushed under another name or to another remote -It names the two causes to check, in order: +It names the three things to check, in order: -1. **The git credential helper is not reachable from the agent's UID.** +1. **The push's own outcome.** A `hive-push-branch` relay push leaves + `.result.json` under `/var/run/hive-metrics/push-requests`; a + direct `git push` printed its result in the agent's pane. The watcher's + log line (below) already carries the relay's record as `push_outcome`. +2. **The git credential helper is not reachable from the agent's UID.** `su -s /bin/sh hive- -c 'git config --get-regexp credential'` must list `/usr/local/bin/git-credential-hive.sh`. It is wired system-wide in `/etc/gitconfig` precisely because agents do not share the dev user's `$HOME`. -2. **The agent's scoped token file is absent or unreadable by that UID.** The +3. **The agent's scoped token file is absent or unreadable by that UID.** The path is in `$HIVE_AGENT_TOKEN_CACHE`. Check readability only — never print its contents. This stays a **retryable** error rather than a policy rejection: once the agent pushes the branch, the same request becomes valid and succeeds on retry. That -is exactly what the bounded retry is for, so fix the credential problem and let -the queued request land — you do not need to re-issue it. +is exactly what the bounded retry is for, so fix the push problem and let the +queued request land — you do not need to re-issue it. + +`hive-open-pr` itself warns at request time when the head is not known to be +on the remote ([#16969](https://github.com/hivecommons/hive/issues/16969)): it +checks the checkout's remote-tracking ref and, failing that, one +`git ls-remote` of the exact ref. The warning names the branch, the remote and +the push step; the request is still written, because a relay push may be in +flight. Set `HIVE_OPEN_PR_SKIP_REMOTE_CHECK=1` to silence it. **3. The head ref exists, so the 404 was about something else.** Usually the base: @@ -421,8 +432,16 @@ since the agent's session ended healthy: kubectl -n hive logs deploy/hive | grep 'pr-request watcher' ``` -Look for `head branch is not on the remote — the agent's push did not -authenticate`, which carries `repo`, `head`, `agent`, and the full `diagnosis`. +Look for `head branch is not on the remote`, which carries `repo`, `head`, +`checked` (the exact ref and repository the watcher asked GitHub for), +`push_outcome` (what the push relay recorded for that branch — queued, pushed +as ``, failed with the git error, or no record because the agent pushed +directly), `attempt`, `agent`, and the full `diagnosis`. It is logged **once +per request**: retries of the same request log `head branch still not on the +remote, retrying` at INFO, and the eventual success logs `request succeeded +on retry` ([#16969](https://github.com/hivecommons/hive/issues/16969) — a +single unpushed branch used to re-log at ERROR every retry and file a fleet +self-report before the branch arrived). For the broader symptom — an agent that completed a session with no branch and no PR, and how to tell a credential-helper failure from a mid-task token diff --git a/src/docs/ioscan.md b/src/docs/ioscan.md index e5b79979f8..d873e1b1fb 100644 --- a/src/docs/ioscan.md +++ b/src/docs/ioscan.md @@ -154,8 +154,13 @@ agents all run on a subscription CLI such as Copilot has no such endpoint; with `classifier.backend` left empty, boot then runs the classifier through the tool-less CLI most of the hive's agents run on (the most common of `claude`/`copilot`/`omp` across `agents.*.backend`, ties by name), so such a hive -needs no extra setting. The classifier itself is built at boot, so changing -its backend needs a restart. +needs no extra setting. The classifier is built at boot and rebuilt whenever +an ioscan setting is saved in Settings → Security → Input Defense or the ACMM +level changes at runtime, so a new backend or fail mode saved there, or an +L4↔L5 level change, applies immediately; a model endpoint change or a direct +`hive.yaml` edit takes effect after a restart. The Security tab exposes +`ioscan.fail_mode` (which the classifier inherits), not +`classifier.fail_mode`; the alert names that dashboard key. **What still passes.** The classifier has a budget of 64 calls per scheduler pass (8 with a CLI backend, see below). Text beyond that budget is passed on with a `budget_exhausted` skip in @@ -230,7 +235,9 @@ and `omp`. `daily_quota_exhausted` skip in the audit log, in either fail mode, just as for an exhausted per-pass budget, and the scheduler logs one WARN per UTC day. The count starts again at 00:00 UTC. Cache hits do not count. -* The backend is chosen at boot; changing it takes effect after a restart. +* The backend is chosen at boot and again on every Input Defense save or + runtime ACMM level change in the dashboard; a direct `hive.yaml` edit takes + effect after a restart. **Cost.** One call per untrusted text per pass, cached by content, with an 8,000-character cap per call. At the default model that is on the order of diff --git a/src/docs/labels-and-control-signals/reference.md b/src/docs/labels-and-control-signals/reference.md index 774e0a6e03..bf13833c37 100644 --- a/src/docs/labels-and-control-signals/reference.md +++ b/src/docs/labels-and-control-signals/reference.md @@ -348,6 +348,20 @@ With the gate on (default off), a human-filed, unclaimed issue whose clarity sco **Configuration and implementation.** `runs.triage.clarity_gate` (`src/pkg/scheduler/clarity_gate.go`, `src/pkg/classify/clarity.go`, `src/pkg/github/clarity_gate_resume.go`) +#### `` body marker (Spek publish hand-off) + +**Applies to:** Issues (GitHub, Jira, Linear bodies). **Kind:** Routing signal; never a hold. + +Spek writes `` into every RFC and child issue it publishes ([spektacular#114](https://github.com/hivecommons/spektacular/issues/114)). A marker **with** `task=` is a child: `runs.triage` keeps it on the direct-fix path (never a new `spec` run; `run/spec` still overrides), the clarity gate does not score it, and its kick line carries `specified by Spek: RFC (spek , artifact , task )` plus the `Fixes #` instruction. A marker **without** `task=` is the RFC itself: treated like `epic` for `governor.claims.never_claim_labels` — every automated claim producer refuses it with `never-claim label: spek:publish`, a stray claim is never staged 020/030 and `IsTrackerIssue` flags it — even when the repo has no `epic` label. A malformed marker is no marker; `rfc=` is dropped unless it is an http(s) URL. + +**Who applies it:** Spek (`spektacular publish`). + +**Checked by:** `pkg/spekmarker`; scheduler triage / clarity gate / kick assembly; the claim gate and stage derivation. **When:** Before kicks are built and on every claim. + +**Release or override.** Remove the marker from the body; `run/spec` sends a child into a spec run regardless. + +**Configuration and implementation.** None (`src/pkg/spekmarker/marker.go`, `src/pkg/classify/triage.go`, `src/pkg/scheduler/clarity_gate.go`, `src/pkg/scheduler/kickmessage.go`, `src/pkg/claims/neverclaim.go`, `src/pkg/github/issue_claim_gate.go`) + #### `stage/017-possible-existing` from the prior-art gate **Applies to:** Issues. **Kind:** Hard suppress. diff --git a/src/docs/linear-agent.md b/src/docs/linear-agent.md index afca43183c..973383be83 100644 --- a/src/docs/linear-agent.md +++ b/src/docs/linear-agent.md @@ -224,8 +224,12 @@ Agent-side calls to `api.linear.app` go through the ACMM proxy (`pkg/proxy/linear_rules.go`, merged in #4522): a deny-by-default GraphQL mutation allowlist where `agentActivityCreate`/`agentSessionUpdate` are reachable at every tier (they carry the 10-second session invariant), -`issueUpdate`/`commentCreate` require ISSUES_ONLY, and anything unknown or -unparseable is denied. The control-plane client in `pkg/linearagent` (webhook +`issueUpdate`/`commentCreate` require ISSUES_ONLY, the mutations an in-hive +Spek publish makes (`issueCreate` with `parentId`, `issueRelationCreate` for +`blockedBy`, `projectCreate`/`projectUpdate` for the RFC parent) also sit at +ISSUES_ONLY, and anything unknown or unparseable is denied — including the +destructive siblings (`issueRelationDelete`, `projectDelete`, +`projectArchive`). The control-plane client in `pkg/linearagent` (webhook ack, identity query, token refresh) runs in the hive process itself, not through the agent proxy. diff --git a/src/docs/operator-reference/README.md b/src/docs/operator-reference/README.md index 03b53ab188..624956b249 100644 --- a/src/docs/operator-reference/README.md +++ b/src/docs/operator-reference/README.md @@ -270,7 +270,7 @@ Go callers keep reading `cfg.Agents`, `cfg.Governor.Gateways` and so on through | `config.refuse_gate_regression` | `false` | Safety-gate regression guard ([#16438](https://github.com/hivecommons/hive/issues/16438)). At boot the hive compares the effective config against any previous runtime overlay still on the PVC (`/data/hive.yaml.runtime`, an operator's `hive.yaml.runtime.bak`, `/data/hive.yaml.dashboard`) for `review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos` and `writeback.per_repo` modes. A gate the overlay carried that the effective config lacks or loosens — typically a boot from the ConfigMap seed after the overlay was moved aside — is logged once per gate as `WARN safety gate dropped: (was ) — effective config has `, listed in `/api/status` under `config.gate_regressions`, and shown as a banner at the top of Settings. `true` makes it fatal: the hive exits (code 21) with the gates named instead of running with weaker gates. Set it in the overlay itself; the ConfigMap seed is the file the regression usually comes from. Legacy `/data/hive.yaml.bak` is not consulted — it is frozen at the rename and would warn forever about gates removed on purpose. | | `auto_merge.allow_unprotected_base` | Deprecated no-op. | Accepted so older configs keep loading. `hive-merge` now merges into any protected or unprotected branch the App can write, subject to the CI-evidence gate and GitHub's own merge rules. | | `auto_merge.no_ci_ok` | Empty by default. | Explicit repo list whose zero-CI merge-request verdict may pass; failing or pending CI evidence is still enforced. | -| `auto_merge.human_merge_pin_only` | Unset (off); enforced by the automerge sweep lanes and `/approve` (#16266), **not yet by the `hive-merge` relay** (#16265). | Per-repo schema for `enabled`, `paths`, `kinds` (`action-sha`, `image-digest`, `version-pin`) and `trusted_authors`. Enabled entries require all three lists. The sweep does not place a human-merge-path hold on a PR whose author is in `trusted_authors`, whose complete diff (every changed file, guarded or not) is pin replacements, whose every commit is authored by a `trusted_authors` login with a GitHub-verified signature, and whose every new action SHA resolves in its upstream action repository (the `# vN` tag points at it, or it is on the default branch; #16399); it never clears an existing hold. See [hive-merge.md](../hive-merge.md) for grammar, restrictions and outstanding integration/deployment requirements (#16242). | +| `auto_merge.human_merge_pin_only` | Unset (off); enforced by the automerge sweep lanes and `/approve` (#16266), and by the `hive-merge` relay (#16265), which additionally requires the requested head SHA to match the PR head before and after fetching files. | Per-repo schema for `enabled`, `paths`, `kinds` (`action-sha`, `image-digest`, `version-pin`) and `trusted_authors`. Enabled entries require all three lists. The sweep does not place a human-merge-path hold on a PR whose author is in `trusted_authors`, whose complete diff (every changed file, guarded or not) is pin replacements, whose every commit is authored by a `trusted_authors` login with a GitHub-verified signature, and whose every new action SHA resolves in its upstream action repository (the `# vN` tag points at it, or it is on the default branch; #16399); it never clears an existing hold. See [hive-merge.md](../hive-merge.md) for grammar, restrictions and outstanding integration/deployment requirements (#16242). | | `auto_merge.human_merge_paths` | Unset (off). | Per-repo map of `owner/repo` to glob patterns (same syntax as `intent.guardrail_path_patterns`) for paths a person must merge. Applies to every merge lane (App self-merge, trusted-author, label-queued, merge-request relay) whatever the intent tier; a matching PR is held with one comment, and an incomplete changed-file list fails closed. Details in the sweep key table below. | | `auto_merge.human_merge_title_prefixes` | `["⚠️", "⚠"]` | Title prefixes (after trimming) that mark a PR as a breaking change a person must merge. Every auto-merge lane and the merge train skip the PR; the tracker parks it at 090 *Ready for human merge · human merge*. Set to `[]` to disable. | | `auto_merge.human_merge_labels` | `["needs-owner-merge"]` | Labels (case-insensitive) that mark a PR a person must merge; same effect as the title prefixes. Set to `[]` to disable. | @@ -513,7 +513,7 @@ advanced into the merge path. | `auto_merge.no_ci_ok` | unset (refuse) | **Merge-request watcher key, not a sweep key.** Per-repo opt-in that downgrades only the "unverified" CI verdict (zero statuses, check runs, and workflow runs) to green, for adopted repos with no CI by design. Red and pending verdicts are never downgraded (#6281). See [hive-merge.md](../hive-merge.md). | | `auto_merge.queue_on_approve_and_unhold` | empty (off) | Per-repo opt-in, ACMM L5 only (#13629). When one trusted merger both submits an Approve review on an agent PR's current head and removes its level `hold`, the queued sweep queues the PR through the same path as the dashboard's queue-auto-merge button. Nothing else counts as consent; the self-merge floor (L6), head-SHA re-check, hold guard and Sentinel are unchanged. See [hive-merge.md](../hive-merge.md#one-human-step-at-l5-approve-and-unhold). | | `auto_merge.squash_only` | unset (no repo) | Repo list (`owner/repo`) whose branch policy forbids merge commits. Elsewhere, a `role/trusted` PR (or one whose current head a trusted merger approved) whose head mixes the contributor's commits with App-linked, App-signed ones (the hive's own in-place adoption fixes), every commit self-signed, is labelled `merge-method/merge` and merged with a merge commit instead of parking at `stage/085` as `adoption-stalled:hive-signoff-attribution`; a listed repo keeps squash-only policy and re-homes an in-place adoption to an App-authored superseder instead (#13975). | -| `auto_merge.human_merge_paths` | unset | Per-repo map of `owner/repo` to glob patterns (same syntax as `intent.guardrail_path_patterns`) for paths a person must merge, e.g. `Danathar/goodreads-mcp: [".claude/settings.json", ".claude/hooks/**"]`. Every App merge path honors it: the automerge sweep lanes (App self-merge, trusted-author, label-queued) and the agent merge-request relay refuse a PR that touches a listed path, add `hold`, and post one `` comment naming the path(s); a person must merge it. The sweep lanes skip that hold only for a PR admitted by `auto_merge.human_merge_pin_only` (#16266); the relay does not apply that exception yet. If the changed-file list cannot be fetched completely for a configured repo, the merge is withheld (#11038, #11039). | +| `auto_merge.human_merge_paths` | unset | Per-repo map of `owner/repo` to glob patterns (same syntax as `intent.guardrail_path_patterns`) for paths a person must merge, e.g. `Danathar/goodreads-mcp: [".claude/settings.json", ".claude/hooks/**"]`. Every App merge path honors it: the automerge sweep lanes (App self-merge, trusted-author, label-queued) and the agent merge-request relay refuse a PR that touches a listed path, add `hold`, and post one `` comment naming the path(s); a person must merge it. The sweep lanes and GitHub relay skip that hold only for a PR admitted by `auto_merge.human_merge_pin_only` (#16266, #16265); the relay also requires the exact requested head before and after fetching files. If the changed-file list cannot be fetched completely for a configured repo, the merge is withheld (#11038, #11039). | | `auto_merge.human_merge_title_prefixes` | `["⚠️", "⚠"]` | PR-title prefixes that mean "breaking change — owner merges" (U+26A0 with or without the VS16 selector by default, matched after trimming leading whitespace). One shared predicate, `automerge.RequiresHumanMerge(pr)`, runs where each lane computes eligibility — App self-merge, label-queued, trusted-author and contributor lanes, plus the refetch before a merge — so a matching PR is never merged by hive and never fronts a merge train, however green and however approved. The PR parks at stage 090 *Ready for human merge* with the `human merge` chip and reason *breaking change — owner merges*; the owner merges it with **Approve & merge as maintainer** or on GitHub. Hive posts one writeback-gated comment saying it will wait for a human. Set to `[]` to turn the title rule off. Added after a ⚠️-titled PR was auto-merged by the trusted-author lane while its owner was still deciding whether to ship it. | | `auto_merge.human_merge_labels` | `["needs-owner-merge"]` | Labels (case-insensitive, trimmed) that carry the same "a person merges this" meaning as the title prefixes above; either match is enough. Set to `[]` to turn the label rule off. | | `auto_merge.require_current_base` | unset | Repo list (`owner/repo`) with an explicit **on** override for the current-base merge gate (Settings → Repos → "Merge only on current base"). On a gated repo the App self-merge, trusted-author and contributor lanes only merge a head that contains the current base tip. A green head tested against an older base is updated with GitHub update-branch (a merge commit; no author commit is rewritten) and merges once CI is green on the new head, so a semantic conflict with work merged since turns that PR red instead of breaking the base ([#13634](https://github.com/hivecommons/hive/issues/13634), [#15335](https://github.com/hivecommons/hive/issues/15335)). Costs one compare call per merge candidate. Listing a repo here is also **strict**: a behind head the App cannot update (fork without maintainer edits, branch protection or a missing `contents: write`) is parked (`fork-not-editable` / `update-branch`) rather than merged as tested. Wins over `skip_current_base` and over the default. A dependency bot's branch (`renovate/*` head ref, or a Renovate, Dependabot or Mergeraptor author) is never updated: on a listed repo, or when GitHub itself reports it `behind`, it holds as `behind` until the bot rebases it; a repo gated only by the default merges it as tested (projectbluefin/utah#1097). On v6, a base that refused a direct merge as merge-queue-only (`merge-queue-required`) in the last 24 hours is not updated: GitHub's queue tests base plus PR itself; the PR stays at `100-merge-queue` for a person to enqueue (projectbluefin/finpilot#599). | @@ -558,8 +558,8 @@ hosted spoke changes survive restarts without editing `hive.yaml`. ### Human-merge pin-only rollout -**Availability:** the automerge sweep lanes and `/approve` enforce the -exception (#16266); the relay does not yet. Minimum +**Availability:** the automerge sweep lanes and `/approve` (#16266) and the +GitHub relay (#16265) enforce the exception. Minimum supported version: the first deployed v5 release containing the schema/classifier (#16262), relay (#16265), sweep (#16266), and hold-release admission integration (#16269). No released version is established here; do not assume an existing @@ -568,7 +568,7 @@ digest/version and live positive/negative evidence in projectbluefin/utah#1082 before enabling that rollout. The additive configuration (schema #16262; enforced by the automerge sweep lanes -and `/approve` per #16266, relay pending #16265) is: +and `/approve` per #16266, and the GitHub relay per #16265) is: ```yaml auto_merge: diff --git a/src/docs/releases.md b/src/docs/releases.md index 5becc1aeeb..975d707f12 100644 --- a/src/docs/releases.md +++ b/src/docs/releases.md @@ -145,7 +145,20 @@ decision: ``` suppresses a release for this merge even if `Unreleased` has content (the -entries stay queued for the next release that does fire). Or force a specific +entries stay queued for the next release that does fire). Note the +consequence under the fragment model: nothing but a release empties +`Unreleased`, so a `none` marker placed there (by hand, in `CHANGELOG.md`) +is a **release freeze** that lasts until a human removes it, not a one-merge +skip — the decide job emits a `::warning::` annotation on every run while it +is in force, and fails outright once the newest tag is older than +`RELEASE_FREEZE_MAX_DAYS` (default 7; `0` disables) so a forgotten freeze +turns red instead of looking like a healthy no-op. For the same reason the marker is **rejected in a +`changelog.d/` fragment** by `check-changelog-fragment.sh` and +`compile-changelog.sh`: a fragment lives until the release that consumes it, +so a `none` fragment can never be consumed and vetoes every later merge +(#12928 stalled v5 releases from 2026-10-19 to 2026-11-07 this way). A +change that should not be mentioned wants no fragment and the +`no-changelog` label. Or force a specific bump regardless of which headers are present: ```markdown diff --git a/src/docs/scale-envelope.md b/src/docs/scale-envelope.md index e9c6e6c1a0..04625e5cba 100644 --- a/src/docs/scale-envelope.md +++ b/src/docs/scale-envelope.md @@ -34,7 +34,7 @@ ones to: 2. **Truncation must preserve the prioritization signal, not cut an arbitrary slice.** This holds today and is easy to assume backwards: - Issues are sorted by actionable rank tier, then **oldest first** within each tier before any cap is applied - (`RankActionableIssues`, `src/pkg/github/client.go:5394-5404`, descending `AgeMinutes` within each tier). + (`RankActionableIssues`, `src/pkg/github/client.go:5401-5411`, descending `AgeMinutes` within each tier). - PRs are sorted by **review class then oldest-first within class** (`SortPullRequestsForReview`, `src/pkg/github/review_priority.go:334`; fixes → refactors/docs → tests). diff --git a/src/docs/spektacular.md b/src/docs/spektacular.md index fd6c9d9e8f..f127efea6a 100644 --- a/src/docs/spektacular.md +++ b/src/docs/spektacular.md @@ -360,6 +360,49 @@ plus comments. Gitea/GitLab do not currently have `work_source` adapters in Hive (they are SCM/forge integrations only), so Spektacular design write-back does not target them until those adapters exist. +## Publish hand-off + +Spek can also run the other way round: a finished spec or epic is +**published** as an RFC plus child issues into the hive's work source — GitHub +Issues, Jira or Linear — so Hive picks the children up from its normal queue +whether Spek ran standalone or in-hive +([spektacular#109](https://github.com/hivecommons/spektacular/issues/109), +[spektacular#114](https://github.com/hivecommons/spektacular/issues/114)). No +new input path is involved: the children are enumerated, staged and claimed +exactly like human-filed items. What Spek adds is one hidden marker in every +body it files: + +``` + +``` + +`name` and `artifact_id` identify the spek, `target` the work source. `task` +is the plan task / member spec id and is present on every **child**; the +**RFC** itself carries the marker without it. `rfc` is the RFC's URL. Values +are bare tokens (optionally quoted) separated by whitespace; a marker missing +`name`, `artifact_id` or a known `target` is malformed and is ignored as a +whole, and `rfc` is kept only when it parses as an http(s) URL — nothing in +the marker is echoed into a kick as free text. The parser is +`pkg/spekmarker`. + +Hive reads the marker in four places, none of which needs configuration: + +| Behaviour | Where | Effect of the marker | +|---|---|---| +| Triage | `runs.triage` (below) | A marker-carrying item is **never** routed into a new `spec` run, whatever `spec_labels` or complexity tier it carries; it takes the direct-fix path as `run/fix` does (signal `marker:spek:publish`). `run/spec` still overrides. | +| Clarity gate | `runs.triage.clarity_gate` | Not scored, the same way `approved-direction` is skipped: the item is already specified. | +| Kick context | kick issue list | A child's line gains `↳ specified by Spek: RFC (spek , artifact , task ) — build against the spek's acceptance criteria …` and the instruction that the PR body must say `Fixes #`. | +| Umbrella | `governor.claims.never_claim_labels`, `IsTrackerIssue` | A marker **without** `task=` is the RFC: it is treated like `epic` — refused by every automated claim producer (`never-claim label: spek:publish`), never staged 020/030 from a stray claim, and flagged `[TRACKER]` — even when the repo has no `epic` label. | + +Everything else in the handshake is unchanged and documented where it lives: +enumeration and the hard-suppress labels ([labels-and-control-signals.md](labels-and-control-signals.md)), +reporter trust (`issue_filter.reporter_trust`: publish as a trusted identity +or expect the `triage/accepted` step), the prior-art gate (expected to fire on +the RFC title, not the children), native sub-issue / blocked-by relations +([hive-open-issue.md](hive-open-issue.md)) and the Jira / Linear source +filters ([work-sources.md](work-sources.md)). The in-hive stage runner is not +involved in a publish; a published issue is ordinary work. + ## Triage The runs triage pass is separate from the Spek runner and is also OFF by @@ -368,6 +411,9 @@ incoming actionable GitHub issue before a direct-fix kick: - `run/spec` always admits a `spec` run. - `run/fix` always leaves the issue on the normal direct-fix path. +- An issue whose body carries the Spek publish marker (``, + see [Publish hand-off](#publish-hand-off)) likewise stays on the direct-fix + path: it is already specified. - Complex issues, or issues carrying `spec_labels` (default `kind/feature`, `Epic`, `architecture discussion`), admit a `spec` run by creating the first stage lease. The lease is keyed as the run-stage work source expects: the @@ -440,7 +486,8 @@ The sweep removes `needs-spec` and records the un-park on the stage engine, so the issue re-derives `stage/010-actionable` and re-enters the queue. Removing `needs-spec` by hand resumes it too. Either way the gate sees the marker and does not ask again, however the issue scores. Issues filed by Hive or an agent, -issues already claimed and issues carrying `approved-direction` are not scored. +issues already claimed, issues carrying `approved-direction` and Spek-published +issues (body marker ``) are not scored. ```yaml runs: diff --git a/src/docs/work-sources.md b/src/docs/work-sources.md index 86ea1f9988..76e7cbbcaa 100644 --- a/src/docs/work-sources.md +++ b/src/docs/work-sources.md @@ -108,6 +108,17 @@ GitHub repository. See [Forges: GitLab, Gitea and Forgejo](integrations/forges.md) for the operation list and what runs through the adapters today, and [Forge setup](forge-app-setup.md) for the reasons. +## Spek-published RFCs and children + +Spek (hivecommons/spektacular) can publish a finished spec as an RFC plus +child issues into any of the sources above. Nothing source-specific is needed: +the children are read like any other item, provided they pass the source's own +filters (`require_labels`, `hold_labels`, `teams[].repo` / `states`, +`project_keys` / `jql`). The hidden `` marker in each +body keeps a child on the direct-fix path, skips the clarity score, carries the +RFC link into the kick and marks the RFC itself as an umbrella Hive never +claims; see [spektacular.md → Publish hand-off](spektacular.md#publish-hand-off). + ## Run stages (`run_stages: true`)
diff --git a/src/pkg/claims/neverclaim.go b/src/pkg/claims/neverclaim.go index 571e80b983..d6eece2820 100644 --- a/src/pkg/claims/neverclaim.go +++ b/src/pkg/claims/neverclaim.go @@ -1,6 +1,10 @@ package claims -import "strings" +import ( + "strings" + + "github.com/hivecommons/hive/pkg/spekmarker" +) // Never-claim labels: an issue wearing one is not a unit of agent work, so no // automated claim producer — the relay `claim` request, the start-signal seam, @@ -38,3 +42,20 @@ func NeverClaimLabel(labels, never []string) string { } return "" } + +// SpekUmbrellaMarker is the never-claim "label" reported for an issue whose +// body carries the Spek publish marker with no `task=` — the RFC itself. +const SpekUmbrellaMarker = "spek:publish" + +// NeverClaimReason is NeverClaimLabel extended with the body: it returns the +// never-claim label the issue wears, else SpekUmbrellaMarker when body marks +// the issue as a Spek-published RFC, else "". +func NeverClaimReason(labels, never []string, body string) string { + if label := NeverClaimLabel(labels, never); label != "" { + return label + } + if spekmarker.IsUmbrella(body) { + return SpekUmbrellaMarker + } + return "" +} diff --git a/src/pkg/claims/neverclaim_test.go b/src/pkg/claims/neverclaim_test.go index 24a70f2daa..18ff9046ee 100644 --- a/src/pkg/claims/neverclaim_test.go +++ b/src/pkg/claims/neverclaim_test.go @@ -26,3 +26,28 @@ func TestNeverClaimLabel(t *testing.T) { }) } } + +func TestNeverClaimReason(t *testing.T) { + rfc := "" + child := "" + cases := []struct { + name string + labels []string + never []string + body string + want string + }{ + {"label wins over marker", []string{"Epic"}, nil, rfc, "Epic"}, + {"spek rfc without label", []string{"enhancement"}, nil, rfc, SpekUmbrellaMarker}, + {"spek rfc with gate disabled still umbrella", nil, []string{}, rfc, SpekUmbrellaMarker}, + {"spek child is work", nil, nil, child, ""}, + {"plain body", []string{"bug"}, nil, "no marker here", ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := NeverClaimReason(tc.labels, tc.never, tc.body); got != tc.want { + t.Fatalf("NeverClaimReason = %q, want %q", got, tc.want) + } + }) + } +} diff --git a/src/pkg/classify/triage.go b/src/pkg/classify/triage.go index 7701ae9055..fb26c62146 100644 --- a/src/pkg/classify/triage.go +++ b/src/pkg/classify/triage.go @@ -8,8 +8,13 @@ import ( "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/issueshape" + "github.com/hivecommons/hive/pkg/spekmarker" ) +// SpekPublishedSignal is the TriageDecision signal for an item carrying the +// Spek publish marker (pkg/spekmarker): already specified, direct-fix path. +const SpekPublishedSignal = "marker:spek:publish" + type TriageVerdict string const ( @@ -45,6 +50,12 @@ func keywordTriage(issue github.Issue, c Classification, cfg config.TriageConfig if hasLabel(labels, "run/fix") { return triageDecision(TriageFix, "run/fix override label", "label:run/fix") } + // A Spek-published item has already been specified: the RFC and its + // acceptance criteria exist, so it is never sent into another `spec` + // run, whatever labels or tier it carries (spektacular#114). + if m, ok := spekmarker.Parse(issue.Body); ok { + return triageDecision(TriageFix, "Spek-published item: already specified by spek "+m.Name, SpekPublishedSignal) + } body := strings.TrimSpace(issue.Body) if min := cfg.EffectiveMinBodyChars(); len(body) < min { return triageDecision(TriageClarify, fmt.Sprintf("issue body is shorter than %d characters", min), "body:short") diff --git a/src/pkg/classify/triage_test.go b/src/pkg/classify/triage_test.go index 0dabd496a5..70e8b50cbd 100644 --- a/src/pkg/classify/triage_test.go +++ b/src/pkg/classify/triage_test.go @@ -8,6 +8,8 @@ import ( "github.com/hivecommons/hive/pkg/github" ) +const spekChildMarker = "" + func TestTriageRules(t *testing.T) { longBody := "This issue has enough detail to explain the requested behaviour, constraints, and expected outcome." cases := []struct { @@ -29,6 +31,13 @@ func TestTriageRules(t *testing.T) { {"simple fixes", github.Issue{Body: longBody}, Classification{Tier: TierSimple}, config.TriageConfig{}, TriageFix, "tier:Simple"}, {"medium fixes", github.Issue{Body: longBody}, Classification{Tier: TierMedium}, config.TriageConfig{}, TriageFix, "tier:Medium"}, {"custom spec label", github.Issue{Labels: []string{"needs-design"}, Body: longBody}, Classification{Tier: TierMedium}, config.TriageConfig{SpecLabels: []string{"needs-design"}}, TriageSpec, "label:needs-design"}, + // spektacular#114: a Spek-published child is already specified, so + // neither a spec label nor a Complex tier re-routes it into a spec run. + {"spek child beats spec label", github.Issue{Labels: []string{"kind/feature"}, Body: longBody + "\n" + spekChildMarker}, Classification{Tier: TierComplex}, config.TriageConfig{}, TriageFix, SpekPublishedSignal}, + {"spek child beats short body", github.Issue{Body: spekChildMarker}, Classification{Tier: TierMedium}, config.TriageConfig{}, TriageFix, SpekPublishedSignal}, + {"spek umbrella never specs", github.Issue{Labels: []string{"Epic"}, Body: longBody + "\n"}, Classification{Tier: TierComplex}, config.TriageConfig{}, TriageFix, SpekPublishedSignal}, + {"run/spec still overrides spek marker", github.Issue{Labels: []string{"run/spec"}, Body: spekChildMarker}, Classification{Tier: TierSimple}, config.TriageConfig{}, TriageSpec, "label:run/spec"}, + {"malformed spek marker is no marker", github.Issue{Labels: []string{"kind/feature"}, Body: longBody + "\n"}, Classification{Tier: TierMedium}, config.TriageConfig{}, TriageSpec, "label:kind/feature"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { diff --git a/src/pkg/config/automerge_pin_only.go b/src/pkg/config/automerge_pin_only.go index 8a2088d3f6..11cd07ce39 100644 --- a/src/pkg/config/automerge_pin_only.go +++ b/src/pkg/config/automerge_pin_only.go @@ -13,8 +13,8 @@ import ( // and commit provenance: every commit behind the head must have a trusted // author login, a committer that is GitHub itself (web-flow) or a trusted // login, and a GitHub-verified signature (#16399). It does not prove upstream -// review (separate gates handle that). The hive-merge relay does not yet apply -// it (#16265). +// review (separate gates handle that). The hive-merge relay applies the same +// policy through Client.SetHumanMergePinOnly (#16265). type PinOnlyException struct { Enabled bool `yaml:"enabled" json:"enabled"` Paths []string `yaml:"paths" json:"paths"` diff --git a/src/pkg/config/review_config.go b/src/pkg/config/review_config.go index 82966f2f9a..4cea1d7979 100644 --- a/src/pkg/config/review_config.go +++ b/src/pkg/config/review_config.go @@ -562,8 +562,8 @@ type AutoMergeConfig struct { HumanMergePaths map[string][]string `yaml:"human_merge_paths,omitempty" json:"human_merge_paths,omitempty"` // HumanMergePinOnly holds the opt-in pin-only exception, enforced by the - // automerge sweep and /approve gates via humanMergePinOnlyExempt (#16266); - // the hive-merge relay does not apply it yet (#16265). + // automerge sweep and /approve gates (#16266), and the hive-merge relay + // (#16265). HumanMergePinOnly map[string]PinOnlyException `yaml:"human_merge_pin_only,omitempty" json:"human_merge_pin_only,omitempty"` // Unset uses the default owner-merge rule; explicit empty lists disable // their half of the title/label rule. diff --git a/src/pkg/dashboard/api_governor_security.go b/src/pkg/dashboard/api_governor_security.go index f5a8c0564d..a9af31ec78 100644 --- a/src/pkg/dashboard/api_governor_security.go +++ b/src/pkg/dashboard/api_governor_security.go @@ -171,6 +171,9 @@ func (s *Server) handleGovernorSecurity(w http.ResponseWriter, r *http.Request) // ACMM level", which is closed at L5-L6, so storing open as empty made // the dashboard unable to choose open on those levels. if body.IoscanFailMode != nil { + // "open" is stored as given: an empty fail_mode follows the ACMM + // pack, which is closed at L5+, so the operator's choice would be + // silently lost there (#16973). cfg.Ioscan.FailMode = strings.ToLower(strings.TrimSpace(*body.IoscanFailMode)) } if body.IoscanClassifierFailMode != nil { @@ -300,6 +303,11 @@ func (s *Server) handleGovernorSecurity(w http.ResponseWriter, r *http.Request) if err := s.saveConfig(); err != nil { s.logger.Error("failed to persist config after security update", "error", err) } + ioscanChanged := body.IoscanEnabled != nil || body.IoscanFailMode != nil || body.IoscanClassifierBackend != nil || + body.IoscanClassifierCLIBudgetPerEval != nil || body.IoscanClassifierCLIDailyQuota != nil + if ioscanChanged && s.deps.IoscanConfigApplied != nil { + s.deps.IoscanConfigApplied() + } s.auditFromRequest(r, "config_governor_security", auditDetail("section", "security"), "") s.refreshAndPersist() okResponse(w, map[string]string{"status": "updated"}) diff --git a/src/pkg/dashboard/api_governor_security_ioscan_test.go b/src/pkg/dashboard/api_governor_security_ioscan_test.go index 98b6d04d9d..0514fb03ac 100644 --- a/src/pkg/dashboard/api_governor_security_ioscan_test.go +++ b/src/pkg/dashboard/api_governor_security_ioscan_test.go @@ -151,3 +151,48 @@ func TestGovernorSecurityRejectsInvalidClassifierBackendValues(t *testing.T) { }) } } + +// At L5 the ACMM pack defaults ioscan to fail-closed, so a dashboard save of +// "open" must be stored as given and reported back as open, and every ioscan +// save must reach IoscanConfigApplied so it applies without a restart +// (#16973). +func TestGovernorSecurityIoscanFailModeOpenSticksAtL5AndApplies(t *testing.T) { + s, deps := apiServer(t) + level := 5 + deps.Config.ACMMLevel = &level + applied := 0 + deps.IoscanConfigApplied = func() { applied++ } + + sec := decodeJSON(t, doGet(s, "/api/config/governor"))["security"].(map[string]any) + if sec["ioscanFailMode"] != "closed" { + t.Fatalf("L5 default fail mode = %v, want closed", sec["ioscanFailMode"]) + } + + rec := doPut(s, "/api/config/governor/security", map[string]any{"ioscanFailMode": "open"}) + if rec.Code != http.StatusOK { + t.Fatalf("PUT security = %d, want 200: %s", rec.Code, rec.Body.String()) + } + if deps.Config.Ioscan.FailMode != "open" { + t.Fatalf("stored fail mode = %q, want open", deps.Config.Ioscan.FailMode) + } + if deps.Config.Ioscan.ClassifierFailClosedAtLevel(level) { + t.Fatal("classifier still fail-closed after saving fail mode open") + } + sec = decodeJSON(t, doGet(s, "/api/config/governor"))["security"].(map[string]any) + if sec["ioscanFailMode"] != "open" { + t.Fatalf("fail mode reported back as %v after saving open", sec["ioscanFailMode"]) + } + if applied != 1 { + t.Fatalf("IoscanConfigApplied calls after fail mode save = %d, want 1", applied) + } + + rec = doPut(s, "/api/config/governor/security", map[string]any{"ioscanClassifierBackend": "copilot"}) + if rec.Code != http.StatusOK || applied != 2 { + t.Fatalf("backend save: code=%d applied=%d, want 200 and 2", rec.Code, applied) + } + + rec = doPut(s, "/api/config/governor/security", map[string]any{"reviewFanOut": true}) + if rec.Code != http.StatusOK || applied != 2 { + t.Fatalf("non-ioscan save: code=%d applied=%d, want 200 and 2", rec.Code, applied) + } +} diff --git a/src/pkg/dashboard/deps.go b/src/pkg/dashboard/deps.go index f309c2c239..2e726e5d17 100644 --- a/src/pkg/dashboard/deps.go +++ b/src/pkg/dashboard/deps.go @@ -169,6 +169,11 @@ type Dependencies struct { // OnACMMLevelChanged is called after an in-process runtime level change // has been persisted/applied. Nil means no live component needs to react. OnACMMLevelChanged func(prev, next int) + // IoscanConfigApplied is called after PUT /api/config/governor/security + // has changed an ioscan setting, so the owner of the scheduler can rebuild + // the semantic classifier from the new values without a restart. Nil means + // nothing caches. + IoscanConfigApplied func() // ReviewConfigApplied is called after PUT /api/config/review has mutated // Config.Review, so the owner of any component that caches review // settings at boot (the GitHub client's revise allowlist and perspective diff --git a/src/pkg/dashboard/static-src/js/config-watchdog-security.js b/src/pkg/dashboard/static-src/js/config-watchdog-security.js index 4435bc4de0..1c8e4e1c36 100644 --- a/src/pkg/dashboard/static-src/js/config-watchdog-security.js +++ b/src/pkg/dashboard/static-src/js/config-watchdog-security.js @@ -865,7 +865,7 @@
- +
- +