diff --git a/.github/workflows/changelog-fragment-guard.yml b/.github/workflows/changelog-fragment-guard.yml index 896c7784dd..3de09a155b 100644 --- a/.github/workflows/changelog-fragment-guard.yml +++ b/.github/workflows/changelog-fragment-guard.yml @@ -75,14 +75,17 @@ concurrency: jobs: changelog-fragment-guard: name: changelog-fragment-guard - # Label events only matter when the label is `no-changelog`; every other - # label (size/*, dco-signoff, agent/*, hive/*) cannot change the verdict. - # A skipped job still reports a check-run, and GitHub treats "skipped" as - # passing for a required context, so this keeps the rollup green instead - # of adding a redundant run per label (#11250). - if: >- - (github.event.action != 'labeled' && github.event.action != 'unlabeled') - || github.event.label.name == 'no-changelog' + # No job-level `if:` on label events — on purpose (#14755). This job used + # to skip itself for every label other than `no-changelog`, and a skipped + # job still reports a check-run that GitHub scores as *passing* for a + # required context. hive's labeler adds three or four labels seconds + # after every push, so a genuine failure on the push-triggered run was + # superseded by a "skipped" sibling on the same head SHA and tide merged + # a malformed fragment (#14629), which then broke the release `decide` + # job on v5 and held every open v5 PR behind a red base. The guard is + # cheap (git, grep, echo); re-running it on each label event keeps the + # reported verdict truthful. Label runs are not cancelled by concurrency + # (see above), so they cannot leave CANCELLED check-runs either. # Forward-merge sync PRs (v5-topup.yml / v6-topup.yml, and the hand-rolled # ones maintainers open when those conflict) live on sync/* branches — # or on /sync-vN-to-vM when an agent opens the top-up, since the diff --git a/.github/workflows/v2-ci.yml b/.github/workflows/v2-ci.yml index 7ffd508812..62861f3278 100644 --- a/.github/workflows/v2-ci.yml +++ b/.github/workflows/v2-ci.yml @@ -1015,6 +1015,19 @@ jobs: working-directory: . run: bash src/deploy/test_ttyd_url_arg.sh + # The browser terminal's attach helper falls back to the DEFAULT tmux + # socket when agents share one UID. It assumed that socket lived under + # dev's passwd uid (/tmp/tmux-1001), but under OpenShift's restricted + # SCC the container runs as an arbitrary uid with no passwd entry, so + # every session was alive on /tmp/tmux-/default while the + # dashboard polled /tmp/tmux-1001/default for 900s. This stubs `id`, + # starts real tmux servers at the paths each uid layout would use, and + # asserts the helper attaches to the right one. Skips where tmux is + # absent. + - name: ttyd finds the default tmux socket under an arbitrary container UID + working-directory: . + run: bash src/deploy/test_ttyd_default_socket_uid.sh + # #5435: hive-discord.service ran `node bot.js` with # WorkingDirectory=/tmp/hive/discord, resolving the executed code out of a # world-writable parent that is cleared on every reboot — a local user who diff --git a/.github/workflows/weekly-competitor-report.lock.yml b/.github/workflows/weekly-competitor-report.lock.yml index bd345d003f..f3827d3d2b 100644 --- a/.github/workflows/weekly-competitor-report.lock.yml +++ b/.github/workflows/weekly-competitor-report.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"a92f9252483f4b49b18cc01141db51e8a7afdadbe3fd936f9f138e1029b484b2","body_hash":"1628e2f12ed90fb50b928ff6c25f7e2310f153be8240d6d5b864a33a7fbc6fb3","compiler_version":"v0.89.21","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.87"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"a92f9252483f4b49b18cc01141db51e8a7afdadbe3fd936f9f138e1029b484b2","body_hash":"fef1288b3894831da705033d4cb4a755e26330652a235c148d799c72e682a3dc","compiler_version":"v0.89.21","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.87"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","DISCORD_RELATED_RESEARCH_WEBHOOK","GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"924af5fdc64061cfbf66fb584c8b07e2ac230c60","version":"v0.89.21"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.23","digest":"sha256:2c78aaba1c108e130e2d6d01e4f2cca334ea04c53e6f258913ac34173fe7e3b2","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.23@sha256:2c78aaba1c108e130e2d6d01e4f2cca334ea04c53e6f258913ac34173fe7e3b2"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.23","digest":"sha256:c15c3d1208df10c5b588a3657be53742aa982ae0909d1eb1812525268794ca64","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.23@sha256:c15c3d1208df10c5b588a3657be53742aa982ae0909d1eb1812525268794ca64"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.23","digest":"sha256:02ffc56dd40158223064ef03a78c2d9717473c93723b4e403d455ea6f0b09ae0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.23@sha256:02ffc56dd40158223064ef03a78c2d9717473c93723b4e403d455ea6f0b09ae0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.25","digest":"sha256:9be0a86220e807a0ecc89e53d7453468f7a53fbc6b3d1efd2299025ffe01d086","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.25@sha256:9be0a86220e807a0ecc89e53d7453468f7a53fbc6b3d1efd2299025ffe01d086"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f","pinned_image":"ghcr.io/github/gh-aw-node@sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f"},{"image":"ghcr.io/github/github-mcp-server:v1.12.2","digest":"sha256:508a0857ec762b1ab1cece29193345b501fab1dd9d1228a7b617062954cecac6","pinned_image":"ghcr.io/github/github-mcp-server:v1.12.2@sha256:508a0857ec762b1ab1cece29193345b501fab1dd9d1228a7b617062954cecac6"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_releases","list_starred_repositories","list_tags","search_code","search_issues","search_repositories"]},{"name":"safeoutputs","tools":["add_comment","create_issue","missing_data","missing_tool","noop","post_to_discord"]}]} # This file was automatically generated by gh-aw (v0.89.21). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # diff --git a/.github/workflows/weekly-competitor-report.md b/.github/workflows/weekly-competitor-report.md index fa8c3d85a6..47ee34cce3 100644 --- a/.github/workflows/weekly-competitor-report.md +++ b/.github/workflows/weekly-competitor-report.md @@ -273,7 +273,11 @@ Hive has today: (Antigravity), pi, opencode, kilo, muse, omp, openhands, omnigent. Source: `CLIBackends` in `config/agent_runtime.go` in hivecommons/crux. So Claude Code, Copilot CLI, Codex CLI, Gemini CLI, Goose, Aider, Kilo, OpenCode and - OpenHands are engines Hive runs as workers, not competitors. + OpenHands are engines Hive runs as workers, not competitors. This holds + even when the same company also sells a hosted or orchestration product + (OpenHands Cloud, Kilo cloud, Codex cloud tasks): the engine row says + "Hive runs it as a worker engine today". If the hosted product matters, + give it its own row (for example "OpenHands Cloud"). - **A governor with six trust levels (ACMM L1-L6)** that decides what agents may do, from advisory only to auto-merge. Source: `README.md` "ACMM Levels", `src/docs/acmm-policy-matrix.md`. @@ -302,14 +306,25 @@ Hive has today: Check at least these, plus anything new you find that clearly belongs: -OpenHands and OpenHands Cloud (All-Hands AI), SWE-agent, Devin (Cognition), -GitHub Copilot coding agent, OpenAI Codex (cloud tasks and CLI), Claude Code -(autonomy and background-agent features), Sweep, Factory, Jules (Google), -Gastown, Dagger agents, Kilo Code and Kilo cloud, Cline (and Cline SDK), Roo -Code (orchestrator mode), Mentat, Aider, GPT-Engineer, Potpie, Agentless, -CrewAI and LangGraph (only where they do repository work), and any internal -agent-fleet orchestrator with a public write-up (engineering blog posts from -large companies about running many coding agents at once). +OpenHands (engine, repo `OpenHands/OpenHands`, moved from `All-Hands-AI`) and +OpenHands Cloud (All-Hands AI), SWE-agent (`SWE-agent/SWE-agent`, moved from +`princeton-nlp`), Devin (Cognition), GitHub Copilot coding agent, OpenAI Codex +(cloud tasks and CLI, `openai/codex`), Claude Code (`anthropics/claude-code`, +autonomy and background-agent features), Goose (`aaif-goose/goose`, moved from +`block`), OpenCode (`anomalyco/opencode`, moved from `sst`), Sweep, Factory, +Jules (Google), Gastown, Dagger agents, Kilo Code and Kilo cloud (`Kilo-Org/kilocode`), +Cline (and Cline SDK), Roo Code (orchestrator mode), Mentat (CLI repo is +archived as `AbanteAI/archive-old-cli-mentat`; say so, do not report +"API error"), Aider, GPT-Engineer (`AntonOsika/gpt-engineer`, archived), +Potpie, Agentless (`OpenAutoCoder/Agentless`), CrewAI and LangGraph (only +where they do repository work), and any internal agent-fleet orchestrator +with a public write-up (engineering blog posts from large companies about +running many coding agents at once). + +Repositories move. The GitHub API answers a renamed repository with a 301 +redirect, so always follow redirects (`curl -sL`) and read `full_name` from +the response to learn the current name. Never write "API error" for a +licence when the redirect target answers. Sort each project into one category: @@ -329,7 +344,10 @@ Then say the relationship to Hive, one of: "Hive runs it as a worker engine today" (it is in the Crux list above), "Hive could run it as an engine", "direct competitor for fleet orchestration", or "complementary". Terminal and editor agents are mostly engines Hive drives. Direct competitors are the -fleet and orchestration layers. +fleet and orchestration layers. Rule: a project in the Crux list is always +"Hive runs it as a worker engine today", even if its company also sells +orchestration. Do not mark OpenHands, Kilo, Codex, or any other listed engine +"direct competitor"; put the hosted product in its own row if you need to. ## How to research @@ -341,11 +359,15 @@ last 7 days before that date. Also compute the unix time for 7 days ago with look at releases, tags, and the most recent commits and merged pull requests in the last 7 days. `search_repositories`, `list_releases`, `list_commits`, and `search_issues` are enough. Read the licence from the - repository's `license.spdx_id` field (for example with - `curl -s https://api.github.com/repos// | jq -r .license.spdx_id`) - and write that value in the table. Write "proprietary" only for hosted - products with no public repository. Do not write "unknown" for anything - that has a GitHub repository. + repository's `license.spdx_id` field with a redirect-following request, + for example + `curl -sL https://api.github.com/repos// | jq -r '"\(.full_name) \(.license.spdx_id)"'`, + and write the SPDX value in the table. Without `-L` a renamed repository + returns a 301 with no licence field. If `full_name` differs from what you + asked for, use the new name in the table. If the API returns `null` for + `license` the repository has no licence file: write "none". Write + "proprietary" only for hosted products with no public repository. Do not + write "unknown" or "API error" for anything that has a GitHub repository. 2. Use `curl` through bash for news. Stay inside the configured network allowlist. Keep TLS checks on. If a request fails, note it and move on; do not try to work around the firewall. Pipelines like `curl ... | jq` and diff --git a/AGENTS.md b/AGENTS.md index 56a83fb627..967cc2d9a8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,4 +15,4 @@ These rules apply to every AI agent (hive agents, Copilot CLI, Claude Code, Code - Keep changes surgical and PR-sized; one concern per PR. - Every PR with a user-visible `src/` code change needs a `changelog.d/-.md` fragment (`added|changed|deprecated|fixed|security`) whose body is a single `- ` bullet. Changes outside `src/` (root-level docs, `changelog.d/`, `.github/`), plus `src/docs/`, `*_test.go` files, and `src/{deploy,scripts}/test[-_]*.sh` scripts, are exempt automatically (`src/scripts/check-changelog-fragment.sh` is the source of truth for the exact set); other test-adjacent changes (e.g., `testdata/` fixtures, non-`_test.go` test helpers) and non-user-visible `src/` code changes (refactors, dependency bumps) need the `no-changelog` label instead. `changelog.d/` is at the repository root, not under `src/` — `src/changelog.d/` is never compiled and the guard rejects it. - Sign commits (`git commit -s`); PR titles start with an emoji (`✨` feature, `🐛` fix, `📖` docs, `📝` proposal, `🌱` chore, `⚠️` breaking) — `📝` is for proposals only; a docs PR is `📖 docs: …`. -- Do not edit `src/docs/api-reference.md` line citations by hand; CI's citation guard reports drift and the fix is `bash src/scripts/check-api-reference-citations.sh --fix` — run it only when CI tells you it drifted. The required `changelog-fragment-guard` check runs it (non-fix) when a PR edits a hub or dashboard `server.go` or their `static/` pages, so the PR that causes drift goes red. +- Do not hand-edit `src/docs` file:line citations (including `src/docs/api-reference.md`); when CI's citation guard reports drift, run `python3 src/scripts/check-docs-citations.py --fix` first — it runs `check-api-reference-citations.sh --fix` for `api-reference.md` and re-anchors other `.go` citations by their backticked Go declaration anchor. Only fall back to a manual edit for citations it still reports as `DRIFT`/`UNRESOLVED`, and run it only when CI tells you citations drifted. The required `changelog-fragment-guard` check runs it (non-fix) when a PR edits a hub or dashboard `server.go` or their `static/` pages, so the PR that causes drift goes red. diff --git a/changelog.d/added-14613-copilot-permissions-comparison.md b/changelog.d/added-14613-copilot-permissions-comparison.md new file mode 100644 index 0000000000..4218842918 --- /dev/null +++ b/changelog.d/added-14613-copilot-permissions-comparison.md @@ -0,0 +1 @@ +- **Docs: Hive trust levels vs GitHub Copilot assisted permissions.** New page `src/docs/copilot-permissions-comparison.md` explains Copilot's assisted permissions (GA 2026-10-26, from GitHub's changelog and docs) next to Hive's ACMM levels, lanes, policy proxy with measured refusal counts from the hosted hives, review gates, hold flow and record-only shadow gates, with a dimension table and what each does that the other does not. Linked from the docs index and documentation map (#14613). diff --git a/changelog.d/added-14713.md b/changelog.d/added-14713.md new file mode 100644 index 0000000000..00f610995f --- /dev/null +++ b/changelog.d/added-14713.md @@ -0,0 +1 @@ +- `python3 src/scripts/check-docs-citations.py --fix` now repairs drifted `src/docs` line-range citations into `.go` files instead of only delegating to the `api-reference.md` route fixer: it resolves the first backticked symbol anchor on the citing line to a single top-level Go declaration (`func X(`, `func (recv) X(`, `type X`, `const X`, `var X`) and rewrites the range to start at that declaration and end at the first line that covers every anchor. It fails closed — zero or several matching declarations, an anchor outside the declaration, a non-`.go` target, or a line without anchors leaves the citation untouched and still reported as `DRIFT` with a non-zero exit — and a second `--fix` run is a no-op. This ends the hand-typed docs PRs for moved functions (#14713). diff --git a/changelog.d/added-14796-approval-denied.md b/changelog.d/added-14796-approval-denied.md new file mode 100644 index 0000000000..94944fcc59 --- /dev/null +++ b/changelog.d/added-14796-approval-denied.md @@ -0,0 +1 @@ +- crux: detect Copilot assisted-approval denials as a blocked/approval_denied state with the judge's feedback (#14796) diff --git a/changelog.d/added-crux-19-otel-export.md b/changelog.d/added-crux-19-otel-export.md new file mode 100644 index 0000000000..3533583234 --- /dev/null +++ b/changelog.d/added-crux-19-otel-export.md @@ -0,0 +1 @@ +- The `crux` binary now exports its OpenTelemetry spans and new kick duration/outcome metrics (`crux.agent.kick.duration`, `crux.agent.kick.outcomes`, labelled only by agent role and outcome) over OTLP/HTTP when the standard `OTEL_EXPORTER_OTLP_*` environment names an endpoint; with none set nothing changes ([hivecommons/crux#19](https://github.com/hivecommons/crux/issues/19)). diff --git a/changelog.d/added-runtime-pprof-memstats.md b/changelog.d/added-runtime-pprof-memstats.md new file mode 100644 index 0000000000..f31d552a10 --- /dev/null +++ b/changelog.d/added-runtime-pprof-memstats.md @@ -0,0 +1 @@ +- Loopback-only `net/http/pprof` listener (`debug.pprof_addr`, default `127.0.0.1:6060`, `off` to disable), a `mem: runtime snapshot` INFO line every 60 s (heap in-use/live/objects, next GC, goroutines, OS threads, GOMEMLIMIT), the same counters under `runtime` in `GET /api/health`, and a cgroup-aware GOMEMLIMIT default (85% of the container memory limit when the env var is unset) ([#14681](https://github.com/hivecommons/hive/issues/14681)). diff --git a/changelog.d/changed-14718.md b/changelog.d/changed-14718.md new file mode 100644 index 0000000000..4d99c1fedd --- /dev/null +++ b/changelog.d/changed-14718.md @@ -0,0 +1 @@ +- dashboard: rename the overview-export encode-failure log helper from `slogWarnOverviewEncode` to `logOverviewEncodeFailure` to match its Debug-level client-disconnect handling ([#14718](https://github.com/hivecommons/hive/issues/14718)). diff --git a/changelog.d/changed-14733.md b/changelog.d/changed-14733.md new file mode 100644 index 0000000000..2ba7bfca40 --- /dev/null +++ b/changelog.d/changed-14733.md @@ -0,0 +1 @@ +- refactor(proxy): split github_proxy.go by concern (move-only) (#14733) diff --git a/changelog.d/changed-codex-catalog-0165.md b/changelog.d/changed-codex-catalog-0165.md new file mode 100644 index 0000000000..2e2141b409 --- /dev/null +++ b/changelog.d/changed-codex-catalog-0165.md @@ -0,0 +1 @@ +- Codex model fallback refreshed to the 0.165.0 catalog: `gpt-6.1-sol` (the CLI's current default) and the new `gpt-6.1-terra` appear in the dashboard's codex model picker when the live `codex app-server` probe cannot run, instead of the stale 0.159.0 snapshot that lacked both diff --git a/changelog.d/changed-pills-until-merge-gate.md b/changelog.d/changed-pills-until-merge-gate.md new file mode 100644 index 0000000000..c1750d2f33 --- /dev/null +++ b/changelog.d/changed-pills-until-merge-gate.md @@ -0,0 +1 @@ +- Hive activity monitor: `sentinel`, `needs human`, `needs human decision` and merge-time `unauthorized` pills are held back (payload `deferred_modifiers`) until the pull request reaches the merge gate (stage/080+), and do not count under *Needs attention* before then; `hold`, `draft`, `stalled`, `regressed` and an `unauthorized` on held workflow runs always show. diff --git a/changelog.d/fixed-13135-header-overlap.md b/changelog.d/fixed-13135-header-overlap.md new file mode 100644 index 0000000000..1d06f6f3e6 --- /dev/null +++ b/changelog.d/fixed-13135-header-overlap.md @@ -0,0 +1 @@ +- The dashboard header's feedback, theme and other right-hand buttons no longer overlap the project title or ACMM badge at window widths between 769px and 900px; those widths now show icon-only Settings, Health and Guide chips ([#13135](https://github.com/hivecommons/hive/issues/13135)). diff --git a/changelog.d/fixed-14613-competitor-report-renames-engine-rule.md b/changelog.d/fixed-14613-competitor-report-renames-engine-rule.md new file mode 100644 index 0000000000..0f80aef26c --- /dev/null +++ b/changelog.d/fixed-14613-competitor-report-renames-engine-rule.md @@ -0,0 +1 @@ +- **Weekly competitor report: renamed repos and engine relationship.** The 2026-10-27 report showed OpenHands' licence as "n/a (API error)" because the repository moved to `OpenHands/OpenHands` and the licence lookup did not follow the GitHub API's 301 redirect; the prompt now uses `curl -sL`, reads `full_name`, and names the current repository for each moved project (OpenHands, SWE-agent, Goose, OpenCode, archived Mentat and GPT-Engineer). The same report marked OpenHands "direct competitor" although Crux has an `openhands` backend; the prompt now states that any engine in the Crux `CLIBackends` list is "Hive runs it as a worker engine today" even when its vendor also sells orchestration. The gh-aw README records that the "Firewall blocked N domains" footer is an upstream gh-aw v0.89.21 false positive (github/gh-aw#63221, fixed in v0.89.22) (#14613). diff --git a/changelog.d/fixed-14705-review-verdict-wake.md b/changelog.d/fixed-14705-review-verdict-wake.md new file mode 100644 index 0000000000..7acd8a2166 --- /dev/null +++ b/changelog.d/fixed-14705-review-verdict-wake.md @@ -0,0 +1 @@ +- Recording a review head's dispatched verdict now wakes the governor for an early cached-snapshot eval cycle, so the next head or the decision kick is dispatched right away instead of waiting for an unrelated webhook or `governor.eval_interval_webhook_s`; a burst of verdicts is coalesced to at most one wake per `governor.webhook_wake_min_gap_s` ([#14705](https://github.com/hivecommons/hive/issues/14705)). diff --git a/changelog.d/fixed-14714-ioscan-blocked-kick-diagnostics.md b/changelog.d/fixed-14714-ioscan-blocked-kick-diagnostics.md new file mode 100644 index 0000000000..abcc5b64d7 --- /dev/null +++ b/changelog.d/fixed-14714-ioscan-blocked-kick-diagnostics.md @@ -0,0 +1 @@ +- When ioscan fails closed and blocks a kick because the semantic classifier errored, the warning now names the underlying classifier error and the config to fix it (served classifier model/endpoint, `ioscan.classifier.enabled`, `ioscan.classifier.fail_mode`) instead of only "ioscan fail-closed blocked kick"; fail-closed stays the default (#14714). diff --git a/changelog.d/fixed-14717-review-requesting-changes-refusal.md b/changelog.d/fixed-14717-review-requesting-changes-refusal.md new file mode 100644 index 0000000000..4a3782cadd --- /dev/null +++ b/changelog.d/fixed-14717-review-requesting-changes-refusal.md @@ -0,0 +1 @@ +- The automerge sweep now treats a 405 "review requesting changes" ruleset refusal like the other permanent review refusals, so a blocked PR is parked and remembered instead of logging a merge-failed WARN and retrying every sweep ([#14717](https://github.com/hivecommons/hive/issues/14717)). diff --git a/changelog.d/fixed-14727.md b/changelog.d/fixed-14727.md new file mode 100644 index 0000000000..cf5cbe29ea --- /dev/null +++ b/changelog.d/fixed-14727.md @@ -0,0 +1 @@ +- Forward-merge PRs that are forced to a merge commit instead of a squash no longer raise a repeated unclassified runtime-error self-report ([#14727](https://github.com/hivecommons/hive/issues/14727)). diff --git a/changelog.d/fixed-14783.md b/changelog.d/fixed-14783.md new file mode 100644 index 0000000000..e326045e2c --- /dev/null +++ b/changelog.d/fixed-14783.md @@ -0,0 +1 @@ +- The scheduler's `ioscan fail-closed blocked held-PR coordination kick` warning is now classified as `ioscan_classifier_unavailable` instead of surfacing as `unclassified_error_repeated` ([#14742](https://github.com/hivecommons/hive/issues/14742)). diff --git a/changelog.d/fixed-14789-terminal-default-tmux-socket-arbitrary-uid.md b/changelog.d/fixed-14789-terminal-default-tmux-socket-arbitrary-uid.md new file mode 100644 index 0000000000..1ffc4a4dbd --- /dev/null +++ b/changelog.d/fixed-14789-terminal-default-tmux-socket-arbitrary-uid.md @@ -0,0 +1 @@ +- The dashboard terminal now attaches to agents when the container runs as an arbitrary UID (e.g. OpenShift's restricted SCC) and agents share one tmux server: the attach helper looked for the default tmux socket only under dev's passwd uid (`/tmp/tmux-1001`), so it polled "waiting for agent to start" for every agent while all sessions were alive on `/tmp/tmux-/default`. It now checks the current uid's socket, dev's, and any other `tmux-*/default` under `$TMUX_TMPDIR`, picking the one that holds the requested session ([#14789](https://github.com/hivecommons/hive/issues/14789)). diff --git a/changelog.d/fixed-check-action-pins-gate.md b/changelog.d/fixed-check-action-pins-gate.md new file mode 100644 index 0000000000..2d5b09e9b4 --- /dev/null +++ b/changelog.d/fixed-check-action-pins-gate.md @@ -0,0 +1 @@ +- The action-pins CI gate (`src/scripts/check-action-pins.sh`) no longer reports `action pin check FAILED` with no finding when a workflow directory contains no SHA-pinned `uses:` lines (pipefail turned grep's empty match into a failure), and its `NOT-PINNED` message now names the workflow file rather than the line number where the file name belonged. The resolver reports every missing SHA instead of stopping at the first, and the gate now has a hermetic self-test, `src/scripts/test-check-action-pins.sh`, covering each of its failure branches. diff --git a/changelog.d/fixed-ci-poll-nudge-14719.md b/changelog.d/fixed-ci-poll-nudge-14719.md new file mode 100644 index 0000000000..2e2de71fbd --- /dev/null +++ b/changelog.d/fixed-ci-poll-nudge-14719.md @@ -0,0 +1 @@ +- The CI-poll stop nudge and `ci_poll_nudge` self-report now also skip replica and variant names of the exempt `ci-maintainer` and `quality` lanes (e.g. `ci-maintainer-2`), not only the exact lane name (#14719). diff --git a/changelog.d/fixed-close-on-merge-pr-target.md b/changelog.d/fixed-close-on-merge-pr-target.md new file mode 100644 index 0000000000..b07b8d1263 --- /dev/null +++ b/changelog.d/fixed-close-on-merge-pr-target.md @@ -0,0 +1 @@ +- close-on-merge no longer closes a pull request as a "fixed issue" when a merged sibling-repo PR mentions it (#14744). diff --git a/changelog.d/fixed-contribute-relay-pr-issue-link.md b/changelog.d/fixed-contribute-relay-pr-issue-link.md new file mode 100644 index 0000000000..6dd2f7931f --- /dev/null +++ b/changelog.d/fixed-contribute-relay-pr-issue-link.md @@ -0,0 +1 @@ +- A contributor relay PR reported for a dispatched issue now gets `Fixes #N` appended (when its body neither closes the issue nor says it stays open) and the GitHub sidebar link, so the PR-claim ledger and the issue list see the issue as covered instead of leaving it Actionable while the PR waits in review as "no linked issue" ([#14763](https://github.com/hivecommons/hive/issues/14763)). diff --git a/changelog.d/fixed-contribute-target-branch-declaration.md b/changelog.d/fixed-contribute-target-branch-declaration.md new file mode 100644 index 0000000000..1268eed248 --- /dev/null +++ b/changelog.d/fixed-contribute-target-branch-declaration.md @@ -0,0 +1 @@ +- Contributor dispatch now reads every spelling the scanner uses for an issue's release-line declaration — `**Target branch:** \`v6\``, `Target-branch: v6` and an inline `… Target: v6 (open the PR with --base v6)` — so the task prompt names the declared base instead of the hub branch and the contributor no longer has to answer `wrong_base` ([#14764](https://github.com/hivecommons/hive/issues/14764)). diff --git a/changelog.d/fixed-crux-publish-keep-runbooks.md b/changelog.d/fixed-crux-publish-keep-runbooks.md new file mode 100644 index 0000000000..1bd407d603 --- /dev/null +++ b/changelog.d/fixed-crux-publish-keep-runbooks.md @@ -0,0 +1 @@ +- The Crux publish script no longer deletes `runbooks/` in `hivecommons/crux` on every sync diff --git a/changelog.d/fixed-decision-answered-verdict.md b/changelog.d/fixed-decision-answered-verdict.md new file mode 100644 index 0000000000..7f2917a1e5 --- /dev/null +++ b/changelog.d/fixed-decision-answered-verdict.md @@ -0,0 +1 @@ +- A recorded `/hive decision` (or trusted CHANGES_REQUESTED review) on a PR head is now folded into the reviewer verdict for that head as a durable, head-scoped answer, so the stage-label reconciler's verdict route treats the `requires_human` / decision-requested verdict as **answered** on every tick and after a restart: instructions route to `070-fixing` with reason `contributor-lane:decision-answered`, a bare approve clears the hold, and only a new head with a fresh `requires_human` verdict asks again. Ends the 070↔085 flap seen on hive#14628, where the fix planner or a maintainer moved the PR to 070 and the next reconcile tick put it back ([#14679](https://github.com/hivecommons/hive/issues/14679)). diff --git a/changelog.d/fixed-ioscan-unavailable-classifier-budget.md b/changelog.d/fixed-ioscan-unavailable-classifier-budget.md index 6f60145a32..72abfa7181 100644 --- a/changelog.d/fixed-ioscan-unavailable-classifier-budget.md +++ b/changelog.d/fixed-ioscan-unavailable-classifier-budget.md @@ -1 +1 @@ -An ioscan classifier that could not be constructed (no model endpoint for the hive's CLI, e.g. a Copilot-only hive at ACMM L5+) no longer consumes the per-eval classifier budget. Previously the first 64 strings per governor eval were withheld under fail-closed and every later string passed on unclassified as `budget_exhausted`, so whether an agent's kick survived depended on its render position. The fail-open path now also logs the classifier error once per distinct error instead of once per string, which stopped a hosted hive from self-reporting 64 "ioscan classifier failed open" warnings every eval. +- An ioscan classifier that could not be constructed (no model endpoint for the hive's CLI, e.g. a Copilot-only hive at ACMM L5+) no longer consumes the per-eval classifier budget ([#14629](https://github.com/hivecommons/hive/pull/14629)). Previously the first 64 strings per governor eval were withheld under fail-closed and every later string passed on unclassified as `budget_exhausted`, so whether an agent's kick survived depended on its render position. The fail-open path now also logs the classifier error once per distinct error instead of once per string, which stopped a hosted hive from self-reporting 64 "ioscan classifier failed open" warnings every eval. diff --git a/changelog.d/fixed-issue-claims-side-effect-retry.md b/changelog.d/fixed-issue-claims-side-effect-retry.md new file mode 100644 index 0000000000..8e6ed81dd1 --- /dev/null +++ b/changelog.d/fixed-issue-claims-side-effect-retry.md @@ -0,0 +1 @@ +- Issue-claim side effects on GitHub (the `stage/020-claimed` write, the claimed-by / preempted labels and their removal on release) now retry transient failures — timeouts behind the per-repo label writer, network errors, rate limits, 5xx, or a journaled attempt whose outcome was uncertain — up to four times with a doubling pause, instead of giving up after one 15s attempt and leaving an issue an agent is already working at `stage/010-actionable`. Claim, takeover and release comments are still posted once, so a timed-out comment is never duplicated (hivecommons/hive#14766). diff --git a/changelog.d/fixed-pack-agent-follows-hive-gateway.md b/changelog.d/fixed-pack-agent-follows-hive-gateway.md new file mode 100644 index 0000000000..812631ee8b --- /dev/null +++ b/changelog.d/fixed-pack-agent-follows-hive-gateway.md @@ -0,0 +1 @@ +- A pack agent that an ACMM level adds to a hive whose operator moved their agents onto one inference gateway (e.g. the L2 `reviewer` on a LiteLLM hive) now joins that gateway with its default model instead of the pack's Copilot CLI, and a pack-owned agent that already exists on a CLI — or on a gateway the hive has since moved off — is moved there on the next restart. The gateway is the one every operator-picked backend names, provided the agents already on it are a majority of the other enabled agents; pack-owned CLI agents such as the L2 `supervisor`/`brainstorm` do not veto the move and follow it, and once moved they count toward that majority so the next level's agent (the L3 `ci-maintainer`) follows too. A hive with an operator-picked CLI agent, or with only a minority of agents on a gateway, is left as is, and an operator-owned or legacy unowned `backend:` is never moved (#14678). diff --git a/changelog.d/fixed-proxy-host-canonical.md b/changelog.d/fixed-proxy-host-canonical.md new file mode 100644 index 0000000000..518ceacc40 --- /dev/null +++ b/changelog.d/fixed-proxy-host-canonical.md @@ -0,0 +1 @@ +- The crux proxy now treats hostnames case-insensitively and ignores a trailing dot when classifying hosts for inspection. diff --git a/changelog.d/fixed-stage-tracker-blocked-by.md b/changelog.d/fixed-stage-tracker-blocked-by.md new file mode 100644 index 0000000000..144ca9d017 --- /dev/null +++ b/changelog.d/fixed-stage-tracker-blocked-by.md @@ -0,0 +1 @@ +- The stage tracker now surfaces an Actionable issue the scheduler withholds for an open GitHub "blocked by" dependency: the row wears the `blocked` modifier, the chip and tooltip name the blocker(s), the payload carries `blocked_on`, and the caption reads "waiting on #N" instead of plain "waiting for the scanner" — sequenced RFC sub-issues no longer look like intake starvation (hivecommons/hive#14765). diff --git a/changelog.d/security-14735-classifier-selfreport.md b/changelog.d/security-14735-classifier-selfreport.md new file mode 100644 index 0000000000..0f8f71c4a1 --- /dev/null +++ b/changelog.d/security-14735-classifier-selfreport.md @@ -0,0 +1 @@ +- Keep raw IOScan classifier errors, including private endpoint URLs and DNS details, in local logs rather than public selfreport diagnostics. diff --git a/changelog.d/security-crux-govulncheck-xnet.md b/changelog.d/security-crux-govulncheck-xnet.md new file mode 100644 index 0000000000..1fa7ae99e2 --- /dev/null +++ b/changelog.d/security-crux-govulncheck-xnet.md @@ -0,0 +1 @@ +- Bump golang.org/x/net to v0.62.0 in src/crux to clear GO-2026-6617/6612/6611/6603 (govulncheck red on hivecommons/crux main). diff --git a/src/Dockerfile b/src/Dockerfile index 5fc56b08fe..7fee7dde0d 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -315,12 +315,12 @@ RUN npm install -g --ignore-scripts --no-fund --no-audit \ # dashboard's codex model discovery (`codex app-server` model/list — pure # local stdio, no network, unaffected by the proxy CA) falls back to the # static list, and non-codex backends are unaffected. -ARG CODEX_VERSION=0.164.0 +ARG CODEX_VERSION=0.165.0 RUN npm install -g --ignore-scripts --no-fund --no-audit @openai/codex@${CODEX_VERSION} && npm cache clean --force || \ echo "WARN: OpenAI Codex CLI install failed — skipping" # --- Layer 7: Claude Code (most frequent updates) --- -ARG CLAUDE_CODE_VERSION=2.1.311 +ARG CLAUDE_CODE_VERSION=2.1.312 # --ignore-scripts (kept, deliberately: no arbitrary postinstall runs during the # build) skips claude-code's install.cjs, which is what LINKS the platform # native binary into the package's bin/. The binary itself ships in the diff --git a/src/Dockerfile.contributor b/src/Dockerfile.contributor index 978a9ce78f..d8017c2d3c 100644 --- a/src/Dockerfile.contributor +++ b/src/Dockerfile.contributor @@ -261,7 +261,7 @@ RUN mkdir -p /etc/apt/keyrings \ # npm cache is cleaned in the same layer it's populated (issue #2613, # Observation 2 "also minor") so the cache doesn't linger in this layer's # diff, mirroring the cleanup already done for the bobshell install below. -ARG CLAUDE_CODE_VERSION=2.1.311 +ARG CLAUDE_CODE_VERSION=2.1.312 ARG COPILOT_VERSION=1.0.88 # --ignore-scripts (kept, deliberately: no arbitrary postinstall runs during # the build) skips claude-code's install.cjs, which is what LINKS the platform @@ -284,7 +284,7 @@ ARG KILO_CLI_VERSION=7.5.6 # Same ARG name and value as src/Dockerfile so bin/test_backend_smoke.sh can # assert the two pins agree — this line was previously an inline hardcode that # could (and did) drift from the main image's CODEX_VERSION unnoticed. -ARG CODEX_VERSION=0.164.0 +ARG CODEX_VERSION=0.165.0 RUN npm install -g --ignore-scripts --no-fund --no-audit \ @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION} \ @github/copilot@${COPILOT_VERSION} \ diff --git a/src/cmd/crux/main.go b/src/cmd/crux/main.go index e9f9b961a4..cefe728c41 100644 --- a/src/cmd/crux/main.go +++ b/src/cmd/crux/main.go @@ -13,10 +13,12 @@ package main import ( "context" + "fmt" "io" "os" "github.com/hivecommons/hive/crux/cli" + "github.com/hivecommons/hive/crux/telemetry" ) // version is what --version reports; set with -ldflags "-X main.version=…". @@ -29,6 +31,14 @@ func main() { func run(args []string, stdin io.Reader, stdout, stderr io.Writer, ctxFn func() (context.Context, context.CancelFunc)) int { ctx, stop := ctxFn() defer stop() + // OTLP export when OTEL_EXPORTER_OTLP_* names an endpoint; a no-op + // otherwise (hivecommons/crux#19). A bad configuration costs telemetry, + // never the command. + flush, err := telemetry.Init(ctx, version) + if err != nil { + fmt.Fprintln(stderr, "crux: telemetry disabled:", err) + } + defer flush() env := &cli.Env{Stdin: stdin, Stdout: stdout, Stderr: stderr, Loader: hiveLoader{}, Version: version} return env.Run(ctx, args) } diff --git a/src/cmd/hive/boot.go b/src/cmd/hive/boot.go index ca4e1259f5..ad4f577cdb 100644 --- a/src/cmd/hive/boot.go +++ b/src/cmd/hive/boot.go @@ -277,6 +277,10 @@ type boot struct { // carries its wakes to the governor loop (hivecommons/hive#11091). reviewEvents *eventdispatch.Dispatcher reviewWake chan struct{} + // verdictWaker turns dispatched review verdicts the relay records into + // early eval cycles, at most one per governor.webhook_wake_min_gap_s + // (hivecommons/hive#14705). + verdictWaker *webhookWaker // webhookWaker coalesces cache-invalidating GitHub webhooks into early // eval cycles and evalLoop is the pacing block /api/status reports. diff --git a/src/cmd/hive/claimswire.go b/src/cmd/hive/claimswire.go index 15582d0f4b..1f3defcd9f 100644 --- a/src/cmd/hive/claimswire.go +++ b/src/cmd/hive/claimswire.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "log/slog" "strings" @@ -12,6 +13,7 @@ import ( "github.com/hivecommons/hive/pkg/claims" "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/convergence/mutation" "github.com/hivecommons/hive/pkg/dashboard" "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/issueclaim" @@ -35,9 +37,80 @@ import ( // PR-claim ledger and the relay leases; overridable in tests). var claimsLedgerPath = "/data/issue-claims.json" -// claimsGitHubTimeout bounds each best-effort GitHub side effect. +// claimsGitHubTimeout bounds each attempt of a best-effort GitHub side +// effect. const claimsGitHubTimeout = 15 * time.Second +// Retry policy for the label and stage side effects (hivecommons/hive#14766). +// One 15s attempt was the whole budget: on the hosted hive a startup burst +// serialised behind the per-repo label writer (DefaultMaxWritersPerRepo = 1) +// and the "claimed stage" write of a scanner claim timed out, so the issue +// an agent was already working stayed stage/010-actionable (#14633, #14674) +// until something else touched it. A transient failure — timeout, network, +// rate limit, 5xx, or a journaled attempt whose outcome is uncertain — is +// now retried a bounded number of times with a growing pause. The retries +// run inside the issue's effect queue, so a release scheduled after the +// claim still waits for the claim's labels to land (or give up) first. +var ( + // claimsEffectAttempts is the total number of tries per side effect. + claimsEffectAttempts = 4 + // claimsEffectBackoff is the pause before the second attempt; each + // further pause doubles it (2s, 4s, 8s). + claimsEffectBackoff = 2 * time.Second + // claimsEffectWait pauses between attempts, returning false when ctx + // ends first. Tests swap it for an instant no-op. + claimsEffectWait = func(ctx context.Context, d time.Duration) bool { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + return false + case <-t.C: + return true + } + } +) + +// retryClaimEffect runs attempt until it succeeds, fails for a reason that +// is not transient, exhausts claimsEffectAttempts, or ctx ends during a +// pause. It returns the number of attempts made (for the log and tests). +func retryClaimEffect(ctx context.Context, logger *slog.Logger, what, key string, attempt func() error) int { + backoff := claimsEffectBackoff + for n := 1; ; n++ { + err := attempt() + if err == nil { + return n + } + if n >= claimsEffectAttempts || !claimEffectRetryable(err) { + if logger != nil { + logger.Warn("issue-claims: GitHub side effect failed", "what", what, "issue", key, "attempts", n, "error", err) + } + return n + } + if logger != nil { + logger.Warn("issue-claims: retrying GitHub side effect", "what", what, "issue", key, "attempt", n, "retry_in", backoff.String(), "error", err) + } + if !claimsEffectWait(ctx, backoff) { + if logger != nil { + logger.Warn("issue-claims: GitHub side effect abandoned, hub stopping", "what", what, "issue", key, "attempts", n, "error", err) + } + return n + } + backoff *= 2 + } +} + +// claimEffectRetryable reports whether a failed side effect is worth another +// attempt: transient transport failures, or a journaled label mutation whose +// first attempt timed out before GitHub answered (the next attempt reconciles +// against the live issue first, so it is safe to repeat). +func claimEffectRetryable(err error) bool { + if err == nil || errors.Is(err, context.Canceled) { + return false + } + return github.IsRetryableError(err) || errors.Is(err, mutation.ErrNeedsReconciliation) +} + // claimEffectQueue runs the GitHub side effects of one issue's ledger // transitions in the order the ledger fired them, one transition at a time // per issue; different issues still run concurrently. The hooks used to @@ -147,22 +220,35 @@ func githubClaimHooks(ctx context.Context, cfg *config.Config, client func() *gi comment := cfg.Governor.Claims.CommentEnabled() label := cfg.Governor.Claims.LabelEnabled() queue := &claimEffectQueue{} + attempt := func(fn func(ctx context.Context, gh *github.Client) error, gh *github.Client) error { + cctx, cancel := context.WithTimeout(ctx, claimsGitHubTimeout) + defer cancel() + return fn(cctx, gh) + } + // run retries a transient failure (claimEffectRetryable) up to + // claimsEffectAttempts times. Comments go through runOnce: a timed-out + // POST may have landed, and a second copy is worse than a missing one. run := func(what string, c claims.Claim, fn func(ctx context.Context, gh *github.Client) error) { gh := client() if gh == nil { return } - cctx, cancel := context.WithTimeout(ctx, claimsGitHubTimeout) - defer cancel() - if err := fn(cctx, gh); err != nil && logger != nil { - logger.Warn("issue-claims: GitHub side effect failed", "what", what, "issue", c.Key(), "error", err) + retryClaimEffect(ctx, logger, what, c.Key(), func() error { return attempt(fn, gh) }) + } + runOnce := func(what string, c claims.Claim, fn func(ctx context.Context, gh *github.Client) error) { + gh := client() + if gh == nil { + return + } + if err := attempt(fn, gh); err != nil && logger != nil { + logger.Warn("issue-claims: GitHub side effect failed", "what", what, "issue", c.Key(), "attempts", 1, "error", err) } } return claims.Hooks{ OnClaimed: func(c claims.Claim, outcome claims.Outcome) { queue.run(c.Key(), func() { if comment && outcome == claims.OutcomeClaimed { - run("claim comment", c, func(ctx context.Context, gh *github.Client) error { + runOnce("claim comment", c, func(ctx context.Context, gh *github.Client) error { return gh.CreateIssueCommentBudgeted(ctx, c.Repo, c.Issue, claims.ClaimComment(c)) }) } @@ -189,7 +275,7 @@ func githubClaimHooks(ctx context.Context, cfg *config.Config, client func() *gi OnTakenOver: func(now, previous claims.Claim) { queue.run(now.Key(), func() { if comment { - run("takeover comment", now, func(ctx context.Context, gh *github.Client) error { + runOnce("takeover comment", now, func(ctx context.Context, gh *github.Client) error { return gh.CreateIssueComment(ctx, now.Repo, now.Issue, claims.TakeoverComment(now, previous)) }) } @@ -249,7 +335,7 @@ func githubClaimHooks(ctx context.Context, cfg *config.Config, client func() *gi // SetIssueClaimLedgerHive still prevents this hive from // believing its own stale hold. if comment && reason != claims.ReasonExpired { - run("release comment", c, func(ctx context.Context, gh *github.Client) error { + runOnce("release comment", c, func(ctx context.Context, gh *github.Client) error { return gh.CreateIssueCommentBudgeted(ctx, c.Repo, c.Issue, claims.ReleaseComment(c, reason)) }) } diff --git a/src/cmd/hive/claimswire_retry_test.go b/src/cmd/hive/claimswire_retry_test.go new file mode 100644 index 0000000000..e26edf5209 --- /dev/null +++ b/src/cmd/hive/claimswire_retry_test.go @@ -0,0 +1,134 @@ +package main + +import ( + "context" + "errors" + "net/http" + "testing" + "time" + + gh "github.com/google/go-github/v72/github" + + "github.com/hivecommons/hive/pkg/convergence/mutation" +) + +// swapClaimsEffectRetry makes retries instant and bounded for a test, +// recording every pause, and restores the production policy afterwards. +func swapClaimsEffectRetry(t *testing.T, attempts int, wait func(context.Context, time.Duration) bool) *[]time.Duration { + t.Helper() + prevAttempts, prevBackoff, prevWait := claimsEffectAttempts, claimsEffectBackoff, claimsEffectWait + t.Cleanup(func() { + claimsEffectAttempts, claimsEffectBackoff, claimsEffectWait = prevAttempts, prevBackoff, prevWait + }) + var pauses []time.Duration + claimsEffectAttempts = attempts + claimsEffectBackoff = time.Second + claimsEffectWait = func(ctx context.Context, d time.Duration) bool { + pauses = append(pauses, d) + if wait != nil { + return wait(ctx, d) + } + return true + } + return &pauses +} + +func serverError() error { + return &gh.ErrorResponse{Response: &http.Response{StatusCode: http.StatusBadGateway}} +} + +// A claim's label or stage write that times out behind the per-repo label +// writer is retried with a growing pause until it lands +// (hivecommons/hive#14766): one 15s attempt left #14633 / #14674 at 010 +// while an agent worked them. +func TestRetryClaimEffect_TransientFailuresAreRetried(t *testing.T) { + pauses := swapClaimsEffectRetry(t, 4, nil) + script := []error{context.DeadlineExceeded, serverError(), mutation.ErrNeedsReconciliation, nil} + calls := 0 + n := retryClaimEffect(context.Background(), nil, "claimed stage", "acme/widgets#7", func() error { + err := script[calls] + calls++ + return err + }) + if n != 4 || calls != 4 { + t.Fatalf("attempts = %d (calls %d), want 4: retry until the write lands", n, calls) + } + if len(*pauses) != 3 || (*pauses)[0] != time.Second || (*pauses)[1] != 2*time.Second || (*pauses)[2] != 4*time.Second { + t.Fatalf("pauses = %v, want doubling 1s, 2s, 4s", *pauses) + } +} + +func TestRetryClaimEffect_GivesUpAfterBudget(t *testing.T) { + pauses := swapClaimsEffectRetry(t, 3, nil) + calls := 0 + n := retryClaimEffect(context.Background(), nil, "claimed-by label", "acme/widgets#7", func() error { + calls++ + return context.DeadlineExceeded + }) + if n != 3 || calls != 3 || len(*pauses) != 2 { + t.Fatalf("attempts = %d calls = %d pauses = %v, want exactly the 3-attempt budget", n, calls, *pauses) + } +} + +// A definite GitHub answer (422, 404, a canceled parent) is not retried: +// the same request would fail the same way. +func TestRetryClaimEffect_PermanentFailureIsNotRetried(t *testing.T) { + pauses := swapClaimsEffectRetry(t, 4, nil) + for name, err := range map[string]error{ + "validation": &gh.ErrorResponse{Response: &http.Response{StatusCode: http.StatusUnprocessableEntity}}, + "not-found": &gh.ErrorResponse{Response: &http.Response{StatusCode: http.StatusNotFound}}, + "canceled": context.Canceled, + "plain": errors.New("label writer closed"), + } { + calls := 0 + n := retryClaimEffect(context.Background(), nil, "claimed stage", "acme/widgets#7", func() error { + calls++ + return err + }) + if n != 1 || calls != 1 { + t.Errorf("%s: attempts = %d calls = %d, want one", name, n, calls) + } + } + if len(*pauses) != 0 { + t.Fatalf("pauses = %v, want none for permanent failures", *pauses) + } +} + +// A hub shutting down mid-pause stops retrying instead of outliving its +// context. +func TestRetryClaimEffect_StopsWhenContextEnds(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + swapClaimsEffectRetry(t, 4, func(ctx context.Context, _ time.Duration) bool { + cancel() + return ctx.Err() == nil + }) + calls := 0 + n := retryClaimEffect(ctx, nil, "released stage", "acme/widgets#7", func() error { + calls++ + return context.DeadlineExceeded + }) + if n != 1 || calls != 1 { + t.Fatalf("attempts = %d calls = %d, want the retry abandoned once the hub context ended", n, calls) + } +} + +func TestClaimEffectRetryable(t *testing.T) { + cases := map[string]struct { + err error + want bool + }{ + "nil": {nil, false}, + "timeout": {context.DeadlineExceeded, true}, + "canceled": {context.Canceled, false}, + "5xx": {serverError(), true}, + "429": {&gh.ErrorResponse{Response: &http.Response{StatusCode: http.StatusTooManyRequests}}, true}, + "422": {&gh.ErrorResponse{Response: &http.Response{StatusCode: http.StatusUnprocessableEntity}}, false}, + "reconciliation": {mutation.ErrNeedsReconciliation, true}, + "wrapped": {errors.Join(errors.New("claimed stage"), context.DeadlineExceeded), true}, + } + for name, c := range cases { + if got := claimEffectRetryable(c.err); got != c.want { + t.Errorf("%s: claimEffectRetryable(%v) = %v, want %v", name, c.err, got, c.want) + } + } +} diff --git a/src/cmd/hive/claimswire_test.go b/src/cmd/hive/claimswire_test.go index ac5850d3fe..dfe4fce495 100644 --- a/src/cmd/hive/claimswire_test.go +++ b/src/cmd/hive/claimswire_test.go @@ -192,6 +192,10 @@ func TestClaimHooks_SubmitClaimFacts(t *testing.T) { {"contributor", claims.KindContributor, false, stagelabels.Claimed}, {"external", claims.KindExternal, false, stagelabels.Claimed}, } + // The claimed-by label write goes to an unreachable host: a transient + // failure the hooks now retry (hivecommons/hive#14766). Pauses are + // instant here so the queued release is not held behind the backoff. + swapClaimsEffectRetry(t, claimsEffectAttempts, nil) for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { sink, writer := stageenginetest.NewSink(stageenginetest.Options{Issue: github.IssueDeriver()}) diff --git a/src/cmd/hive/github_client_config.go b/src/cmd/hive/github_client_config.go index 19f78f5c51..e2543a52e3 100644 --- a/src/cmd/hive/github_client_config.go +++ b/src/cmd/hive/github_client_config.go @@ -466,6 +466,9 @@ func (b *boot) applyGitHubClientDashboardHooks(client *github.Client) { // claims ledger is on. recordStarted(b.dashSrv, b.cfg.Project.Org, agentName, repo, issue, signal) }) + // hivecommons/hive#14705: a recorded dispatched verdict frees its head, + // so the governor re-plans now instead of at the next eval cycle. + client.SetReviewVerdictHook(b.wakeForReviewVerdict) // PR follow-up session resume (#9583, default off): remember which live // CLI session authored this PR, and the PR's handoff note, so its // follow-ups can resume that conversation or brief a fresh one. The diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index a5c37ac4c2..3f6ae4b937 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -1196,6 +1196,11 @@ func (b *boot) bootConfigWith(deps bootConfigDeps) bool { b.ctx, cancel = context.WithCancel(context.Background()) b.cleanup.push(func() { cancel() }) + // Runtime observability (#14681): GOMEMLIMIT from the cgroup limit, the + // loopback pprof listener and the 60 s `mem:` line. Before tracing so the + // first mem line lands even when an exporter stalls. + b.startRuntimeObservability() + // Initialize OpenTelemetry tracing. Off by default: with no otel block // (or otel.enabled=false) this installs a no-op provider with zero export // overhead. Never fatal — a tracing setup error must not stop hive. @@ -6836,6 +6841,13 @@ func (b *boot) runLoopWith(deps runLoopDeps) { tick(evalSourceCached) armContinuousWake() armWebhookHealthRecheck() + case batch := <-b.verdictWakeChan(): + b.logger.Info("review verdict wake: running an early eval cycle from the cached snapshot", + "pr", batch.Reason, "last_pr", batch.LastRe, "verdicts", batch.Count, + "coalesced", batch.Coalesced.Round(time.Millisecond).String()) + tick(evalSourceCached) + armContinuousWake() + armWebhookHealthRecheck() case batch := <-b.webhookWakeChan(): now := healthNow() // Apply the delivery's targeted re-reads first so this cycle diff --git a/src/cmd/hive/review_event_dispatch.go b/src/cmd/hive/review_event_dispatch.go index 841d608197..711d2feb32 100644 --- a/src/cmd/hive/review_event_dispatch.go +++ b/src/cmd/hive/review_event_dispatch.go @@ -1,8 +1,10 @@ package main import ( + "fmt" "sort" "sync/atomic" + "time" "github.com/hivecommons/hive/pkg/review" "github.com/hivecommons/hive/pkg/review/eventdispatch" @@ -22,6 +24,12 @@ func (b *boot) installReviewEvents() { if b.reviewWake == nil { b.reviewWake = make(chan struct{}, 1) } + if b.verdictWaker == nil { + b.verdictWaker = newWebhookWaker( + func() time.Duration { return 0 }, + func() time.Duration { return b.cfg.WebhookWakeMinGap() }, + ) + } activeReviewEvents.Store(b.reviewEvents) } @@ -48,6 +56,29 @@ func (b *boot) wakeForReviewEvents(batch []eventdispatch.Entry) { } } +// wakeForReviewVerdict is the github.ReviewVerdictHook: the relay recorded a +// verdict bound to a review dispatch, so the head that wrote it is free and +// the PR may be ready for its next head or its decision kick. Without this +// the next kick waited for an unrelated webhook or the eval cadence +// (hivecommons/hive#14705). The first verdict wakes the loop at once; later +// ones inside governor.webhook_wake_min_gap_s fold into one wake at the end +// of the gap, so a burst of verdicts costs one cycle. +func (b *boot) wakeForReviewVerdict(repo string, number int) { + if b == nil || b.verdictWaker == nil { + return + } + b.verdictWaker.Request(fmt.Sprintf("%s#%d", repo, number)) +} + +// verdictWakeChan is the loop's receive side; nil when the waker is not +// installed so the select case never fires. +func (b *boot) verdictWakeChan() <-chan webhookWakeBatch { + if b.verdictWaker == nil { + return nil + } + return b.verdictWaker.out +} + // prioritizeEventPRs moves PRs whose current head a review webhook fired for // to the front, keeping order otherwise. PlanDispatch walks the list in order // and still applies every gate, so this changes who is first, not who is diff --git a/src/cmd/hive/review_event_dispatch_test.go b/src/cmd/hive/review_event_dispatch_test.go index 10469b8d0c..a52bb6c9d1 100644 --- a/src/cmd/hive/review_event_dispatch_test.go +++ b/src/cmd/hive/review_event_dispatch_test.go @@ -63,8 +63,12 @@ func TestInstallReviewEventsIsIdempotent(t *testing.T) { if d == nil || wake == nil || activeReviewEvents.Load() != d { t.Fatal("dispatcher not installed") } + verdicts := b.verdictWaker + if verdicts == nil { + t.Fatal("verdict waker not installed") + } b.installReviewEvents() - if b.reviewEvents != d || b.reviewWake != wake { + if b.reviewEvents != d || b.reviewWake != wake || b.verdictWaker != verdicts { t.Fatal("reinstall replaced the dispatcher") } t.Cleanup(func() { activeReviewEvents.Store(nil) }) @@ -140,3 +144,69 @@ func TestRunLoopWithRunsAnEarlyCycleOnReviewWake(t *testing.T) { t.Fatalf("log = %s", log.String()) } } + +// hivecommons/hive#14705: the first recorded verdict wakes the loop at once; +// the verdicts that follow inside governor.webhook_wake_min_gap_s fold into +// one wake at the end of the gap. +func TestWakeForReviewVerdictCoalescesInsideMinGap(t *testing.T) { + w, f := newTestWebhookWaker(0, 30*time.Second) + b := &boot{verdictWaker: w} + + b.wakeForReviewVerdict("o/r", 1) + if len(f.delays) != 1 || f.delays[0] != 0 { + t.Fatalf("first verdict should wake at once, armed %v", f.delays) + } + f.fire(t) + if got := <-b.verdictWakeChan(); got.Count != 1 || got.Reason != "o/r#1" { + t.Fatalf("first wake = %+v", got) + } + + f.now = f.now.Add(5 * time.Second) + b.wakeForReviewVerdict("o/r", 2) + b.wakeForReviewVerdict("o/r", 3) + if len(f.delays) != 2 || f.delays[1] != 25*time.Second { + t.Fatalf("verdicts inside the gap should arm one timer for the rest of it, armed %v", f.delays) + } + select { + case got := <-b.verdictWakeChan(): + t.Fatalf("woke inside the min gap: %+v", got) + default: + } + f.fire(t) + if got := <-b.verdictWakeChan(); got.Count != 2 || got.Reason != "o/r#2" || got.LastRe != "o/r#3" { + t.Fatalf("coalesced wake = %+v", got) + } +} + +func TestWakeForReviewVerdictWithoutWakerIsNoop(t *testing.T) { + var nilBoot *boot + nilBoot.wakeForReviewVerdict("o/r", 1) + b := &boot{} + b.wakeForReviewVerdict("o/r", 1) + if b.verdictWakeChan() != nil { + t.Fatal("uninstalled waker should leave the loop case nil") + } +} + +func TestRunLoopWithRunsAnEarlyCycleOnVerdictWake(t *testing.T) { + b, log, cancel := newLoopBoot(t, 60, 0) + b.verdictWaker, _ = newTestWebhookWaker(0, 30*time.Second) + gov := newFakeTicker() + h := &loopHarness{startupOK: true, pending: []*fakeTicker{gov}, persisted: make(chan struct{}, 8)} + done := runLoopAsync(b, h.deps()) + <-h.persisted // first cycle done + + b.verdictWaker.out <- webhookWakeBatch{Reason: "o/r#1", LastRe: "o/r#2", Count: 2} + <-h.persisted // early cycle done + gov.fire() + <-h.persisted // cadence cycle still runs + cancel() + waitDone(t, done) + + if len(h.evals) != 3 { + t.Fatalf("evals = %d, want first + verdict + cadence", len(h.evals)) + } + if !strings.Contains(log.String(), "review verdict wake") || !strings.Contains(log.String(), "verdicts=2") { + t.Fatalf("log = %s", log.String()) + } +} diff --git a/src/cmd/hive/runtime_observability.go b/src/cmd/hive/runtime_observability.go new file mode 100644 index 0000000000..8d71019b8a --- /dev/null +++ b/src/cmd/hive/runtime_observability.go @@ -0,0 +1,40 @@ +package main + +import ( + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/runtimeobs" +) + +// startRuntimeObservability wires pkg/runtimeobs into the boot: the soft +// memory limit, the loopback-only pprof listener (debug.pprof_addr) and the +// periodic `mem:` log line (hivecommons/hive#14681). Nothing here is fatal — +// a bind failure only costs the profiles, never the hive. +func (b *boot) startRuntimeObservability() { + out := runtimeobs.ApplyMemoryLimit(nil, nil) + b.logger.Info("runtime: memory limit", + "source", out.Source, + "gomemlimit", out.Limit, + "cgroup_limit", out.CgroupLimit, + "percent", runtimeobs.MemoryLimitPercent, + ) + + addr := b.cfg.Debug.EffectivePprofAddr() + if addr != "" && !runtimeobs.IsLoopbackAddr(addr) { + b.logger.Warn("runtime: debug.pprof_addr is not a loopback address; using the default", + "configured", addr, "default", config.DefaultPprofAddr) + addr = config.DefaultPprofAddr + } + bound, stop, err := runtimeobs.StartPprof(addr, b.logger) + switch { + case err != nil: + b.logger.Warn("runtime: pprof listener not started", "addr", addr, "error", err) + case bound == "": + b.logger.Info("runtime: pprof listener disabled", "key", "debug.pprof_addr") + default: + b.cleanup.push(stop) + b.logger.Info("runtime: pprof listening (loopback only)", "addr", bound, "path", "/debug/pprof/") + } + + runtimeobs.StartLogger(b.ctx, b.logger, nil) + b.logger.Info("mem: runtime snapshot", runtimeobs.Take().LogAttrs()...) +} diff --git a/src/crux/README.md b/src/crux/README.md index ff1b7d2ae5..c3d6db520c 100644 --- a/src/crux/README.md +++ b/src/crux/README.md @@ -118,6 +118,17 @@ crux logs -f scout # follows over SSE `crux --version` prints the version embedded at build time (`-ldflags "-X main.version=…"`, `dev` otherwise). +## Telemetry + +The runtime records OpenTelemetry spans (`agent.send_kick`, …) and two kick +metrics through the OTel API. The `crux` binary exports them over OTLP/HTTP +only when the standard `OTEL_EXPORTER_OTLP_ENDPOINT` (or a per-signal +`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` / `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT`) +is set; with none set nothing is installed and nothing is sent. There is no +`/metrics` endpoint: point crux at an OpenTelemetry Collector and export from +there. Variables and instruments are listed in +[`crux-cli.md`](../docs/crux-cli.md#telemetry). + ## Using Crux from Go ```go diff --git a/src/crux/agent/approval_denied_test.go b/src/crux/agent/approval_denied_test.go new file mode 100644 index 0000000000..1f7aeece39 --- /dev/null +++ b/src/crux/agent/approval_denied_test.go @@ -0,0 +1,36 @@ +package agent + +import ( + "strings" + "testing" +) + +func TestPaneShowsApprovalDenied(t *testing.T) { + denied := `✗ Remove junk directory from scratch workspace (shell) + │ rm -rf ~/crux-scratch/capture/junk && echo done + └ Permission denied because the assisted approval check did not approve this request. Retry in an + interactive session so the user can approve it, or try an alternative that does not require this + permission. Feedback: Recursive deletion may destroy durable data that cannot be restored. + Approval check + └ ✗ Assisted Permissions declined + Returned to agent.` + tests := []struct { + name string + pane string + wantOK bool + wantFeedback string + }{ + {"denied transcript", denied, true, "Recursive deletion may destroy durable data that cannot be restored."}, + {"declined marker only", "└ ✗ Assisted Permissions declined\n Returned to agent.", true, ""}, + {"approved", "Approval check\n └ ✓ Assisted Permissions approved", false, ""}, + {"ordinary output", "ran go test ./...", false, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fb, ok := paneShowsApprovalDenied(strings.Split(tt.pane, "\n")) + if ok != tt.wantOK || fb != tt.wantFeedback { + t.Fatalf("got (%q, %v), want (%q, %v)", fb, ok, tt.wantFeedback, tt.wantOK) + } + }) + } +} diff --git a/src/crux/agent/ci_poll_exempt_lane_test.go b/src/crux/agent/ci_poll_exempt_lane_test.go new file mode 100644 index 0000000000..ef7b6d5c36 --- /dev/null +++ b/src/crux/agent/ci_poll_exempt_lane_test.go @@ -0,0 +1,23 @@ +package agent + +import ( + "testing" + + "github.com/hivecommons/hive/crux/config" +) + +func TestCIPollNudgeExemptAgentLaneVariants(t *testing.T) { + for name, want := range map[string]bool{ + "ci-maintainer": true, + "ci-maintainer-2": true, + "ci-maintainer_full": true, + "Quality-advisory": true, + "scanner": false, + "ci-maintainers": false, + } { + a := &AgentProcess{Name: name, Config: config.AgentConfig{Role: name}} + if got := ciPollNudgeExemptAgent(a); got != want { + t.Errorf("%q: got %v want %v", name, got, want) + } + } +} diff --git a/src/crux/agent/kick_metrics.go b/src/crux/agent/kick_metrics.go new file mode 100644 index 0000000000..6f9903232e --- /dev/null +++ b/src/crux/agent/kick_metrics.go @@ -0,0 +1,82 @@ +package agent + +import ( + "context" + "errors" + "sync" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +// Kick instruments (hivecommons/crux#19). They are created on the global +// MeterProvider, which is OTel's delegating no-op until a host registers an +// SDK provider (crux/telemetry for the crux binary, pkg/tracing for hive), so +// recording costs nothing by default and starts exporting once one is set. +// +// Attributes are bounded enums only: the agent role (a configured agent name, +// or kickRoleUnknown for a name the manager does not know, so caller input +// never becomes a label) and the outcome. Never issue/PR numbers, run ids or +// free text. +const ( + kickDurationMetric = "crux.agent.kick.duration" + kickOutcomeMetric = "crux.agent.kick.outcomes" + + kickRoleUnknown = "unknown" + + kickOutcomeOK = "ok" + kickOutcomeCancelled = "cancelled" + kickOutcomeError = "error" +) + +var ( + attrKickRole = attribute.Key("agent.role") + attrKickOutcome = attribute.Key("outcome") +) + +// kickMetrics holds the two kick instruments. +type kickMetrics struct { + duration metric.Float64Histogram + outcomes metric.Int64Counter +} + +func newKickMetrics(meter metric.Meter) kickMetrics { + // Instrument constructors only fail on an invalid name or unit; both are + // constants here, and the returned instrument is a usable no-op either way. + duration, _ := meter.Float64Histogram(kickDurationMetric, + metric.WithUnit("s"), + metric.WithDescription("Time SendKick took to deliver, defer or reject a kick.")) + outcomes, _ := meter.Int64Counter(kickOutcomeMetric, + metric.WithDescription("Kicks handled by SendKick, by agent role and outcome.")) + return kickMetrics{duration: duration, outcomes: outcomes} +} + +var globalKickMetrics = sync.OnceValue(func() kickMetrics { + return newKickMetrics(otel.Meter(spanInstrumentationName)) +}) + +// kickOutcome maps SendKick's result onto the fixed outcome enum. +func kickOutcome(err error) string { + switch { + case err == nil: + return kickOutcomeOK + case errors.Is(err, errKickCancelledByRestart): + return kickOutcomeCancelled + default: + return kickOutcomeError + } +} + +// recordKick records one SendKick call's duration and outcome. +func recordKick(role string, start time.Time, err error) { + globalKickMetrics().record(role, time.Since(start), err) +} + +func (k kickMetrics) record(role string, d time.Duration, err error) { + attrs := metric.WithAttributes(attrKickRole.String(role), attrKickOutcome.String(kickOutcome(err))) + ctx := context.Background() + k.duration.Record(ctx, d.Seconds(), attrs) + k.outcomes.Add(ctx, 1, attrs) +} diff --git a/src/crux/agent/kick_metrics_test.go b/src/crux/agent/kick_metrics_test.go new file mode 100644 index 0000000000..f1e8ad8228 --- /dev/null +++ b/src/crux/agent/kick_metrics_test.go @@ -0,0 +1,82 @@ +package agent + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "go.opentelemetry.io/otel/attribute" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +func TestKickOutcome(t *testing.T) { + cases := []struct { + err error + want string + }{ + {nil, kickOutcomeOK}, + {fmt.Errorf("%w: agent a restarted mid-send", errKickCancelledByRestart), kickOutcomeCancelled}, + {errors.New("agent a not found"), kickOutcomeError}, + } + for _, tc := range cases { + if got := kickOutcome(tc.err); got != tc.want { + t.Errorf("kickOutcome(%v) = %q, want %q", tc.err, got, tc.want) + } + } +} + +// The instruments export one duration point and one outcome count per kick, +// labelled by exactly agent.role and outcome — no other attribute may leak in. +func TestKickMetricsRecordBoundedAttributes(t *testing.T) { + reader := sdkmetric.NewManualReader() + mp := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + t.Cleanup(func() { _ = mp.Shutdown(context.Background()) }) + k := newKickMetrics(mp.Meter(spanInstrumentationName)) + + k.record("scanner", 2*time.Second, nil) + k.record(kickRoleUnknown, time.Millisecond, errors.New("agent x not found")) + + var rm metricdata.ResourceMetrics + if err := reader.Collect(context.Background(), &rm); err != nil { + t.Fatalf("Collect: %v", err) + } + okSet := attribute.NewSet(attrKickRole.String("scanner"), attrKickOutcome.String(kickOutcomeOK)) + errSet := attribute.NewSet(attrKickRole.String(kickRoleUnknown), attrKickOutcome.String(kickOutcomeError)) + want := map[attribute.Distinct]bool{ + okSet.Equivalent(): true, + errSet.Equivalent(): true, + } + seen := map[string]bool{} + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + seen[m.Name] = true + switch data := m.Data.(type) { + case metricdata.Histogram[float64]: + if m.Unit != "s" { + t.Errorf("%s unit = %q, want s", m.Name, m.Unit) + } + for _, dp := range data.DataPoints { + if !want[dp.Attributes.Equivalent()] || dp.Count != 1 { + t.Errorf("%s point %v count=%d unexpected", m.Name, dp.Attributes.ToSlice(), dp.Count) + } + } + case metricdata.Sum[int64]: + for _, dp := range data.DataPoints { + if !want[dp.Attributes.Equivalent()] || dp.Value != 1 { + t.Errorf("%s point %v value=%d unexpected", m.Name, dp.Attributes.ToSlice(), dp.Value) + } + } + default: + t.Errorf("%s: unexpected data type %T", m.Name, m.Data) + } + } + } + for _, name := range []string{kickDurationMetric, kickOutcomeMetric} { + if !seen[name] { + t.Errorf("metric %s not exported", name) + } + } +} diff --git a/src/crux/agent/manager_kick.go b/src/crux/agent/manager_kick.go index 54089300cb..35f011f59d 100644 --- a/src/crux/agent/manager_kick.go +++ b/src/crux/agent/manager_kick.go @@ -111,7 +111,7 @@ func (m *Manager) SendKick(name string, message string) error { return m.SendKickWithSource(name, message, "") } -func (m *Manager) SendKickWithSource(name string, message string, source string) error { +func (m *Manager) SendKickWithSource(name string, message string, source string) (err error) { // Agent-kick span. No-op with zero export cost when tracing is disabled // (the default). SendKick has no context parameter, so this span roots at // Background; it still captures the kick leg of the governor→agent @@ -120,6 +120,11 @@ func (m *Manager) SendKickWithSource(name string, message string, source string) attribute.String("agent.name", name)) defer span.End() + // Kick duration/outcome metrics; role stays kickRoleUnknown until the + // name resolves to a configured agent (kick_metrics.go). + start, role := time.Now(), kickRoleUnknown + defer func() { recordKick(role, start, err) }() + m.mu.Lock() defer m.mu.Unlock() @@ -127,6 +132,7 @@ func (m *Manager) SendKickWithSource(name string, message string, source string) if !ok { return fmt.Errorf("agent %s not found", name) } + role = name trigger := "send-kick" if strings.TrimSpace(source) != "" { @@ -1648,8 +1654,20 @@ func ciPollNudgeExemptAgent(agent *AgentProcess) bool { return true } for _, f := range []string{agent.Name, agent.Config.Role} { - lane := strings.TrimSpace(f) - if strings.EqualFold(lane, "ci-maintainer") || strings.EqualFold(lane, "quality") { + if exemptCIPollLane(f) { + return true + } + } + return false +} + +// exemptCIPollLane matches a lane name or role exactly, or as a dash/underscore- +// suffixed variant ("ci-maintainer-2", "quality_full"), so replicas and policy +// variants of an exempt lane are exempt too. +func exemptCIPollLane(f string) bool { + lane := strings.ToLower(strings.TrimSpace(f)) + for _, base := range []string{"ci-maintainer", "quality"} { + if lane == base || strings.HasPrefix(lane, base+"-") || strings.HasPrefix(lane, base+"_") { return true } } diff --git a/src/crux/agent/manager_poll.go b/src/crux/agent/manager_poll.go index 380b09f52b..2aaf811abd 100644 --- a/src/crux/agent/manager_poll.go +++ b/src/crux/agent/manager_poll.go @@ -57,6 +57,16 @@ func (m *Manager) pollTmuxOutputForAgent(agent *AgentProcess, ctx context.Contex bobKeyRejected := effectiveBackend(agent) == bobBackend && paneShowsBobAPIKeyRejected(tail) quotaExhausted := paneShowsQuotaExhausted(tail) rateLimited := paneShowsStartupRateLimit(tail) + if feedback, denied := paneShowsApprovalDenied(tail); denied { + msg := "blocked/approval_denied: Copilot assisted-approval denied a tool request" + if feedback != "" { + msg += ": " + feedback + } + if agent.LastError != msg { + agent.LastError = msg + m.logger.Warn("copilot assisted-approval denial detected", "agent", agent.Name, "feedback", feedback) + } + } if authState.BackendAuthError { m.mu.Lock() m.markProviderErrorLocked(agent, providerErrorMatch{Class: "backend_auth_error", Line: authState.BackendAuthErrorMsg}, time.Now()) diff --git a/src/crux/agent/pane_classify.go b/src/crux/agent/pane_classify.go index f48b0a3b53..4865c13bc4 100644 --- a/src/crux/agent/pane_classify.go +++ b/src/crux/agent/pane_classify.go @@ -542,6 +542,32 @@ func paneShowsQuotaExhausted(lines []string) bool { return false } +// approvalDeniedPrefix is the stable Copilot CLI text for a tool request the +// assisted-approval judge refused in a non-interactive run; the judge's +// explanation follows "Feedback:" and is fed back to the model. +const approvalDeniedPrefix = "permission denied because the assisted approval check did not approve this request" + +// paneShowsApprovalDenied reports a Copilot assisted-approval denial and the +// judge's feedback (empty when the pane wraps it off the tail). The approved +// form ("Assisted Permissions approved") never matches. +func paneShowsApprovalDenied(lines []string) (feedback string, ok bool) { + for i, line := range lines { + lower := strings.ToLower(line) + if !strings.Contains(lower, approvalDeniedPrefix) && !strings.Contains(lower, "assisted permissions declined") { + continue + } + ok = true + for j := i; j < len(lines) && j <= i+4; j++ { + if _, after, found := strings.Cut(lines[j], "Feedback:"); found { + feedback = strings.TrimSpace(after) + break + } + } + return feedback, ok + } + return "", false +} + func paneShowsBobAPIKeyRejected(lines []string) bool { for _, line := range lines { lower := strings.ToLower(line) diff --git a/src/crux/cmd/crux/main.go b/src/crux/cmd/crux/main.go index 2d58ab1c46..e6901a6a83 100644 --- a/src/crux/cmd/crux/main.go +++ b/src/crux/cmd/crux/main.go @@ -16,11 +16,13 @@ package main import ( "context" + "fmt" "io" "os" "github.com/hivecommons/hive/crux/cli" "github.com/hivecommons/hive/crux/defs" + "github.com/hivecommons/hive/crux/telemetry" ) // version is what --version reports. Release builds set it with @@ -34,6 +36,14 @@ func main() { func run(args []string, stdin io.Reader, stdout, stderr io.Writer, ctxFn func() (context.Context, context.CancelFunc)) int { ctx, stop := ctxFn() defer stop() + // OTLP export when OTEL_EXPORTER_OTLP_* names an endpoint; a no-op + // otherwise (hivecommons/crux#19). A bad configuration costs telemetry, + // never the command. + flush, err := telemetry.Init(ctx, version) + if err != nil { + fmt.Fprintln(stderr, "crux: telemetry disabled:", err) + } + defer flush() env := &cli.Env{Stdin: stdin, Stdout: stdout, Stderr: stderr, Loader: defs.Loader{}, Version: version} return env.Run(ctx, args) } diff --git a/src/crux/config/agent_runtime.go b/src/crux/config/agent_runtime.go index 51ec694281..b5c3cbfdb4 100644 --- a/src/crux/config/agent_runtime.go +++ b/src/crux/config/agent_runtime.go @@ -651,6 +651,12 @@ func (a AgentConfig) BackendIsOperatorOwned() bool { return a.BackendOwner == FieldOwnerOperator } +// BackendIsPackOwned reports whether an ACMM pack wrote this agent's backend, +// as opposed to an operator's pick or a legacy value with no owner. +func (a AgentConfig) BackendIsPackOwned() bool { + return a.BackendOwner == FieldOwnerPack +} + // PauseIsOperatorOwned reports whether an operator explicitly owns this // agent's pause/run state — stamped at dashboard-API create and on an explicit // operator resume — which makes the agent immune to the ACMM pack visibility diff --git a/src/crux/config/leaf_test.go b/src/crux/config/leaf_test.go index 6c49368069..f45ebf1bb0 100644 --- a/src/crux/config/leaf_test.go +++ b/src/crux/config/leaf_test.go @@ -92,6 +92,13 @@ func TestAgentConfigJevAndOwners(t *testing.T) { if a.ModelIsOperatorOwned() { t.Fatal("pack owner is not operator") } + if a.BackendIsPackOwned() { + t.Fatal("operator-owned backend is not pack-owned") + } + a.BackendOwner = FieldOwnerPack + if !a.BackendIsPackOwned() || (AgentConfig{}).BackendIsPackOwned() { + t.Fatal("only a pack-stamped backend is pack-owned") + } } func TestAgentConfigRepos(t *testing.T) { diff --git a/src/crux/go.mod b/src/crux/go.mod index 4de541c8ef..d54499d9d8 100644 --- a/src/crux/go.mod +++ b/src/crux/go.mod @@ -6,6 +6,11 @@ require ( github.com/google/cel-go v0.31.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e go.opentelemetry.io/otel v1.47.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 + go.opentelemetry.io/otel/metric v1.47.0 + go.opentelemetry.io/otel/sdk v1.47.0 + go.opentelemetry.io/otel/sdk/metric v1.47.0 go.opentelemetry.io/otel/trace v1.47.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -18,15 +23,22 @@ require ( require ( cel.dev/expr v0.25.2 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/otel/metric v1.47.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 // indirect + golang.org/x/net v0.62.0 // indirect + golang.org/x/sys v0.49.0 // indirect golang.org/x/text v0.43.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect + google.golang.org/grpc v1.83.2 // indirect google.golang.org/protobuf v1.36.12 // indirect ) diff --git a/src/crux/go.sum b/src/crux/go.sum index f5b7c49322..5b6ed28d81 100644 --- a/src/crux/go.sum +++ b/src/crux/go.sum @@ -2,6 +2,8 @@ cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -10,10 +12,16 @@ github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/cel-go v0.31.0 h1:H0bhpFTqOvmHrBGrWKp7ZlhBm5Hh8PYUEXnwxT1LL7A= github.com/google/cel-go v0.31.0/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -28,22 +36,46 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM= go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0 h1:7kszRN4Ze5GeAkUnBjBSUIunVBMA+mZIG04auueqguM= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0/go.mod h1:CLzakYy8jOdkBfcveCWYkeKJxujcF4xPhh2Z7IfE98U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 h1:julhjPeUH/q/7hinbSdDdqt5h7Zw9YWmRlWRhI0jd54= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0/go.mod h1:Ao2mz688LH/tFf0yMAenidq6k2YNSx6SIY2q6jDACck= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 h1:aXZXsZ012wOgVkqaiQv+Z/d8V0xKjmg70F6m4CE94lc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0/go.mod h1:ziu1gUIJhAaxM3EV1e3Ralk5AAyL1UOYz3lJJ4VSI38= go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI= go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw= go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY= go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU= +go.opentelemetry.io/otel/metric/x v0.69.0 h1:DjRLr15H83v+hCW7JA9NoJvOkYTtmq5YoDRbe9deYpM= +go.opentelemetry.io/otel/metric/x v0.69.0/go.mod h1:uVvsMPMFFyj/HUQfrUnH3JjnOQ1dwFDorgFLRBasM0k= +go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I= +go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0= +go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw= +go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw= go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM= go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc h1:rZHRz0ogq4Pid1IKtA/ivHi8XONVF5tqUQXA2TtgpkE= google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:K/n0XLazCJHjD73YuMyNJsN38BPPy1kfFnKxUXaThFU= google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc h1:4bNTbnb44EqGVy9HaQxSY2jnafSUdgV3qHtedvNpDKg= google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/src/crux/proxy/close_wait_leak_test.go b/src/crux/proxy/close_wait_leak_test.go index 359eb8cb72..bd8671513a 100644 --- a/src/crux/proxy/close_wait_leak_test.go +++ b/src/crux/proxy/close_wait_leak_test.go @@ -4,8 +4,6 @@ import ( "io" "log/slog" "net" - "os" - "path/filepath" "runtime" "strings" "sync" @@ -195,21 +193,13 @@ func TestProxyHTTPBodyRelayToleratesSlowButFlowingBody(t *testing.T) { } // TestProxyHTTPHasNoInlineCopyPairs asserts the source-level invariant behind -// leak (1): every opaque byte-shuttling path in github_proxy.go must go +// leak (1): every opaque byte-shuttling path in github_proxy*.go must go // through relayTunnel (whose half-close drains are the #3872 guarantee), never // a hand-rolled io.Copy pair. The pre-fix git branch was exactly such a pair // that #3872 missed. relayTunnel's own two copies are the only sanctioned // ones. func TestProxyHTTPHasNoInlineCopyPairs(t *testing.T) { - _, file, _, ok := runtime.Caller(0) - if !ok { - t.Fatal("runtime.Caller failed") - } - body, err := os.ReadFile(filepath.Join(filepath.Dir(file), "github_proxy.go")) - if err != nil { - t.Fatalf("read github_proxy.go: %v", err) - } - text := string(body) + text := readGitHubProxySource(t) const wantRelayTunnelCalls = 3 // transparent passthrough, CONNECT tunnelDirect, git branch if got := strings.Count(text, "relayTunnel(") - strings.Count(text, "func relayTunnel("); got != wantRelayTunnelCalls { t.Fatalf("relayTunnel call sites = %d, want %d — opaque relays must use relayTunnel, not inline io.Copy pairs", got, wantRelayTunnelCalls) diff --git a/src/crux/proxy/github_proxy.go b/src/crux/proxy/github_proxy.go index ea526f744e..e21d9a51b6 100644 --- a/src/crux/proxy/github_proxy.go +++ b/src/crux/proxy/github_proxy.go @@ -2,31 +2,18 @@ package proxy import ( "bufio" - "bytes" - "context" - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" "crypto/tls" "crypto/x509" - "crypto/x509/pkix" - "encoding/base64" "encoding/json" - "encoding/pem" - "errors" "fmt" "io" "log/slog" - "math/big" "net" "net/http" "os" - "path/filepath" - "regexp" "strconv" "strings" "sync" - "sync/atomic" "syscall" "time" @@ -34,7 +21,6 @@ import ( "github.com/hivecommons/hive/crux/agent" "github.com/hivecommons/hive/crux/config" "github.com/hivecommons/hive/crux/inferencehealth" - "github.com/hivecommons/hive/crux/issueshape" ) const ( @@ -315,138 +301,6 @@ type GitHubProxy struct { bodyStallTimeout time.Duration } -// SetLinearCredentialResolver installs the resolver injectLinearCredential -// reads on every agent request to api.linear.app. main.go wires the same -// closure the agent manager uses to push LINEAR_ACCESS_TOKEN / LINEAR_API_KEY -// into sessions, so the proxy and the session env can never name different -// credentials. Values are never logged. nil disables injection. -func (p *GitHubProxy) SetLinearCredentialResolver(fn func() agent.LinearCredential) { - p.linearCredential = fn -} - -// injectLinearCredential replaces the Authorization header of an agent's -// api.linear.app request with the hive's CURRENT Linear credential, for -// agents at the ISSUES_ONLY floor and above (the tier the session-env push in -// agent.Manager.linearEnvPairs uses). It reports whether a credential was -// applied. -// -// WHY THE PROXY, NOT THE ENVIRONMENT. The connected Linear app's OAuth access -// token rotates (Linear issues it with a ~24h expiry and hive refreshes it -// in the background), and hive already re-pushes the fresh value into each -// agent's tmux session environment on the hourly refresh tick. But a tmux -// session environment only reaches processes forked AFTER the push: the -// agent's CLI, forked at its last launch, keeps the token it was born with, -// and every tool shell it spawns inherits that copy. Observed live -// (2026-09-06/07): a quality agent whose session environment held the fresh -// token — verified valid through this very proxy as that agent's UID — got -// `401 AUTHENTICATION_ERROR` on every Linear call from inside its pane for -// two days, because its process still carried the pre-rotation token. Its -// earlier kicks had silently fallen back to the operator's personal -// LINEAR_API_KEY, so every issue it filed showed the operator as creator. -// -// The proxy terminates every agent request to api.linear.app anyway (this is -// where enforceLinear gates mutations by tier), so attaching the live -// credential HERE makes the copy in the agent's environment irrelevant — -// stale, absent, or a placeholder all work — exactly as the inference -// translator attaches the real gateway key server-side. The agent still -// receives a credential in its environment (for SDKs that refuse to send a -// request without one), but Linear only ever sees the proxy's. -// -// Below the floor the request is left untouched: an ADVISORY agent's Linear -// reads have always been ungated and unauthenticated-by-hive, and injecting -// a write-capable credential there would widen what the tier can do. -func (p *GitHubProxy) injectLinearCredential(req *http.Request, agentName string, mode agent.AgentMode) bool { - if p.linearCredential == nil || !mode.CanCreateIssues() { - return false - } - cred := p.linearCredential() - switch { - case cred.AccessToken != "": - req.Header.Set("Authorization", "Bearer "+cred.AccessToken) - case cred.APIKey != "": - req.Header.Set("Authorization", cred.APIKey) - default: - return false - } - p.logger.Debug("linear: attached the hive's current credential to an agent request", "agent", agentName, "mode", mode.String()) - return true -} - -// dialCopilotUpstream connects to the real Copilot host over TLS (or the test -// seam when set). -func (p *GitHubProxy) dialCopilotUpstream(host string) (net.Conn, error) { - if p.copilotDial != nil { - return p.copilotDial(host) - } - // SO_MARK the outbound socket so the forced-egress redirect exempts the - // proxy's own upstream traffic (see proxyEgressMark). The override hook above - // still wins when set, so tests are unaffected. - return tls.DialWithDialer(markDialer(upstreamDialTimeout), "tcp", net.JoinHostPort(host, "443"), &tls.Config{ServerName: host}) -} - -// SetTokenSink wires the inference token sink so the translator can record -// per-agent usage from upstream OpenAI responses into the shared token store. -func (p *GitHubProxy) SetTokenSink(sink UsageSink) { - p.tokenSink = sink -} - -// SetAgentTokenSource wires the resolver from identified agent name to that -// agent's hub-held scoped token (#1861). Wired once at boot, before Start(), -// so no lock is needed on the read path. -func (p *GitHubProxy) SetAgentTokenSource(source func(agentName string) (string, bool)) { - p.agentTokenSource = source -} - -func (p *GitHubProxy) SetAgentGitHubBudgetConfig(capForAgent func(agentName string) int, reserveFloor func() int) { - if capForAgent != nil { - p.githubBudgetCap = capForAgent - } - if reserveFloor != nil { - p.githubReserveFloor = reserveFloor - } -} - -func (p *GitHubProxy) SetAgentGitHubBudgetNudgeFunc(fn func(agentName, message string)) { - p.githubBudgetNudge = fn -} - -// hostNeedsMITM decides whether a GitHub-family host must be TLS-intercepted -// rather than opaquely tunneled. -// -// Without injection this is exactly the historical NeedsMITM: only -// api.github.com, where request-level ACMM inspection happens; github.com and -// registered GHE hosts are tunneled without decryption (their traffic — git -// smart HTTP, OAuth device flow — is gated elsewhere). -// -// With injection (#1861) an opaque tunnel is a hole, not an optimization: the -// agent's git credential helper now serves the inert placeholder, so a -// git-over-HTTPS push through an un-MITM'd tunnel would reach GitHub carrying -// the placeholder and 401. Every GitHub-family host must therefore be -// intercepted so rewriteGitHubAuth can replace the placeholder with the real -// scoped token — which also means an agent-supplied credential can never -// sneak to ANY GitHub host through a tunnel the proxy declined to open. -func (p *GitHubProxy) hostNeedsMITM(host string) bool { - if NeedsMITM(host) { - return true - } - return p.injectGHAuth && IsGitHubHost(host) -} - -// SetCanaryScanner installs the host's canary registry and leak handler. -// enabled gates the scan; failClosed turns a detection (or an unscannable -// body) into a refusal rather than an audit-only event. -func (p *GitHubProxy) SetCanaryScanner(enabled, failClosed bool, reg CanaryScanner, onLeak func(crux.CanaryLeak)) { - p.canariesEnabled = enabled - p.canaryFailClosed = failClosed - p.canaryRegistry = reg - p.canaryLeakFunc = onLeak -} - -type cachedCert struct { - cert tls.Certificate - expiresAt time.Time -} - // NewGitHubProxy creates a proxy with a self-signed CA for MITM. // The org and repos parameters define which repositories are allowed for // write operations. Repos should be bare names (e.g. "console"); the org @@ -545,3155 +399,17 @@ func newGitHubProxyWithCA(caCert tls.Certificate, caX509 *x509.Certificate, logg return p } -// SetRepoPausedFunc installs the operator's per-repo pause predicate (#6203). -// The hive passes config's IsRepoPaused, so pausing a repo in the dashboard is -// enforced on the very next request without a restart — which matters, because -// the pauses this serves (a release freeze, an incident) are measured in hours, -// and an operator who has to roll the pod to quiet one repo is back to stopping -// the whole hive. Passing nil clears it. -func (p *GitHubProxy) SetRepoPausedFunc(fn func(repo string) bool) { - p.pausedMu.Lock() - p.repoPaused = fn - p.pausedMu.Unlock() -} - -// repoScopePath is the request path handed to the per-repo write gates (pause, -// agent scope, auto-merge, per-repo ACMM mode, repo filter). It starts from -// the client's own escaping rather than the decoded req.URL.Path so a GitLab -// "namespace%2Fproject" stays one segment and ExtractRepo resolves the exact -// project — including nested groups — instead of guessing where the project -// path ends from the sub-resource that follows it. CanonicalGatePath then -// decodes every OTHER escape so an escaped structural segment (/r%65pos/, -// repo%2Egit, pro%6Aects, merg%65) cannot hide the repo from the gates. -func repoScopePath(req *http.Request) string { - return CanonicalGatePath(req.URL.EscapedPath()) -} - -// repoPauseRefusal is RepoPauseRefusal against this proxy's current predicate. -func (p *GitHubProxy) repoPauseRefusal(method, path string) (string, bool) { - p.pausedMu.RLock() - paused := p.repoPaused - p.pausedMu.RUnlock() - return RepoPauseRefusal(paused, method, path) -} - -// SetRepoAutoMergeEnabledFunc installs the live per-repo auto-merge predicate. -// The proxy uses it to give flagged repositories their specific deny reason at -// direct merge endpoints. Passing nil preserves default-on behavior. -func (p *GitHubProxy) SetRepoAutoMergeEnabledFunc(fn func(repo string) bool) { - p.autoMergeMu.Lock() - p.repoAutoMergeOn = fn - p.autoMergeMu.Unlock() -} - -func (p *GitHubProxy) autoMergeRefusal(method, path string) (string, bool) { - p.autoMergeMu.RLock() - enabled := p.repoAutoMergeOn - p.autoMergeMu.RUnlock() - return RepoAutoMergeRefusal(enabled, method, path) -} - -func (p *GitHubProxy) graphQLAutoMergeRefusal(body []byte) (string, bool) { - p.autoMergeMu.RLock() - enabled := p.repoAutoMergeOn - p.autoMergeMu.RUnlock() - return GraphQLAutoMergeRefusal(enabled, body) -} - -// SetAgentRepoScopeFunc installs the per-repo agent-scope predicate (#6204). -// The hive passes config's AgentServesRepo, so re-scoping an agent in the -// dashboard is enforced on the very next request without a restart. Passing nil -// clears it, which is the unscoped behaviour every hive had before. -func (p *GitHubProxy) SetAgentRepoScopeFunc(fn func(agent, repo string) bool) { - p.scopeMu.Lock() - p.agentServesRepo = fn - p.scopeMu.Unlock() -} - -// agentRepoScopeRefusal is AgentRepoScopeRefusal against the current predicate. -func (p *GitHubProxy) agentRepoScopeRefusal(agentName, method, path string) (string, bool) { - p.scopeMu.RLock() - serves := p.agentServesRepo - p.scopeMu.RUnlock() - return AgentRepoScopeRefusal(serves, agentName, method, path) -} - -// SetWriteSurfaceEnforceFunc installs the per-lane write_surface.enforce -// predicate (#9772). The hive passes config's WriteSurfaceEnforced, so a lane -// listed in config is enforced on the next request. Passing nil clears it, -// which is the behaviour every hive had before: direct writes allowed. -func (p *GitHubProxy) SetWriteSurfaceEnforceFunc(fn func(agent string) bool) { - p.writeSurfaceMu.Lock() - p.writeSurfaceEnforced = fn - p.writeSurfaceMu.Unlock() -} - -// SetWriteRefusedAuditFunc installs the sink that records a direct write -// refused under write_surface.enforce as an agent_write_refused audit entry. -func (p *GitHubProxy) SetWriteRefusedAuditFunc(fn func(agent, kind, method, path, repo string)) { - p.writeSurfaceMu.Lock() - p.writeRefusedAudit = fn - p.writeSurfaceMu.Unlock() -} - -func (p *GitHubProxy) writeSurfaceEnforcedFunc() func(agent string) bool { - p.writeSurfaceMu.RLock() - defer p.writeSurfaceMu.RUnlock() - return p.writeSurfaceEnforced -} - -// laneWriteSurfaceEnforced reports whether agentName's lane is enforced. -func (p *GitHubProxy) laneWriteSurfaceEnforced(agentName string) bool { - if agentName == "" || agentName == internalCallerName { - return false - } - enforced := p.writeSurfaceEnforcedFunc() - return enforced != nil && enforced(agentName) -} - -// hostNeedsMITMFor is hostNeedsMITM for a known caller. A lane under -// write_surface.enforce has every GitHub-family host intercepted, not only -// api.github.com, because git push travels over github.com's smart HTTP and -// an opaque tunnel could not refuse it. Other lanes are unaffected. -func (p *GitHubProxy) hostNeedsMITMFor(host, agentName string) bool { - if p.hostNeedsMITM(host) { - return true - } - return IsGitHubHost(host) && p.laneWriteSurfaceEnforced(agentName) -} - -// auditWriteRefused records one direct write refused under -// write_surface.enforce. -func (p *GitHubProxy) auditWriteRefused(agentName, kind, method, path string) { - p.writeSurfaceMu.RLock() - audit := p.writeRefusedAudit - p.writeSurfaceMu.RUnlock() - if audit == nil { - return - } - audit(agentName, kind, method, path, WriteSurfaceRepo(path)) -} - // ListenAddr returns the proxy listen address. func (p *GitHubProxy) ListenAddr() string { return p.listenAddr } -// Violations returns a snapshot of per-agent violation counts. -func (p *GitHubProxy) Violations() map[string]int { - p.mu.RLock() - defer p.mu.RUnlock() - out := make(map[string]int, len(p.violations)) - for k, v := range p.violations { - out[k] = v - } - return out -} - -// AgentViolations returns the violation count for a specific agent. -func (p *GitHubProxy) AgentViolations(agentName string) int { - p.mu.RLock() - defer p.mu.RUnlock() - return p.violations[agentName] -} - -// Start begins listening. Blocks until the listener is closed. -// Handles both explicit HTTP CONNECT proxy requests and transparent -// iptables-redirected TLS connections (detected by TLS ClientHello). -func (p *GitHubProxy) Start() error { - ln, err := net.Listen("tcp", p.listenAddr) - if err != nil { - return fmt.Errorf("listen %s: %w", p.listenAddr, err) - } - defer func() { _ = ln.Close() }() - p.logger.Info("proxy listening", "addr", p.listenAddr) - - for { - conn, err := ln.Accept() - if err != nil { - return fmt.Errorf("accept: %w", err) - } - go p.handleConn(conn) - } -} - -// handleConn peeks at the first byte to distinguish HTTP CONNECT requests -// (explicit proxy) from raw TLS ClientHello (iptables-redirected traffic). -func (p *GitHubProxy) handleConn(conn net.Conn) { - defer func() { _ = conn.Close() }() - - peeked := make([]byte, 1) - _ = conn.SetReadDeadline(time.Now().Add(transparentProxyTimeout)) - n, err := conn.Read(peeked) - _ = conn.SetReadDeadline(time.Time{}) - if err != nil || n == 0 { - return - } - - // TLS ClientHello starts with byte 0x16 (ContentType handshake). - // HTTP methods start with ASCII letters (C for CONNECT, G for GET, etc.). - const tlsHandshakeContentType = 0x16 - if peeked[0] == tlsHandshakeContentType { - p.handleTransparentTLS(conn, peeked) - return - } - - // Parse the HTTP request directly instead of using http.Server.Serve, - // which closes the connection on shutdown — racing with hijacked CONNECT - // handlers. - prefixed := &prefixConn{Conn: conn, prefix: peeked[:n]} - _ = conn.SetReadDeadline(time.Now().Add(httpReadTimeout)) - req, err := http.ReadRequest(bufio.NewReader(prefixed)) - _ = conn.SetReadDeadline(time.Time{}) - if err != nil { - return - } - - if req.Method == http.MethodConnect { - p.handleConnectDirect(conn, req) - } else { - p.forwardPlainDirect(conn, req) - } -} - -const ( - transparentProxyTimeout = 5 * time.Second - httpReadTimeout = 30 * time.Second - httpWriteTimeout = 60 * time.Second - // upstreamDialTimeout bounds the proxy's OWN upstream dial + TLS handshake - // to the real GitHub host. Before this, a stalled upstream (DNS, TCP, or - // TLS) blocked forever — the agent's curl sat in poll() waiting for the - // MITM response that never came, permanently wedging the request and - // freezing the agent's session (the multi-hour "egress hang"). - // tls.DialWithDialer's dialer Timeout covers connect + handshake; the - // deadline is lifted once established so long-lived relays (git smart - // HTTP) are unaffected. - upstreamDialTimeout = 15 * time.Second - - // clientTLSHandshakeTimeout bounds the proxy's TLS handshake with the - // inbound client. It intentionally matches upstreamDialTimeout: both are - // short setup phases, unlike long-lived request/response relays. - clientTLSHandshakeTimeout = upstreamDialTimeout -) - -// handleTransparentTLS handles iptables-redirected connections. The agent -// tried to connect to github.com:443 but iptables sent it here instead. -// We extract the SNI hostname from the TLS ClientHello, then MITM the connection. -func (p *GitHubProxy) handleTransparentTLS(conn net.Conn, peeked []byte) { - // Read the whole ClientHello TLS record before extracting SNI and replaying - // it. TCP guarantees nothing about record boundaries: a single Read can - // return only PART of the ClientHello. A partial read makes extractSNI miss - // the SNI (so the wrong host is chosen and the wrong cert is forged) and - // mis-frames the record stream handed to tls.Server — the agent-side - // handshake then fails deterministically (bad certificate / bad record MAC), - // which is exactly why every Copilot agent could not authenticate. Loop until - // the full record (per the record-layer length) is buffered. - _ = conn.SetReadDeadline(time.Now().Add(transparentProxyTimeout)) - fullBuf, err := readClientHelloRecord(conn, peeked) - _ = conn.SetReadDeadline(time.Time{}) - if err != nil { - return - } - - host := extractSNI(fullBuf) - if host == "" { - host = "github.com" - } - - agentName := p.attributeTransparentConn(conn) - - // Copilot completion host under iptables redirection: MITM to read live - // token usage (same guard as the explicit-CONNECT path). The peeked - // ClientHello is replayed via prefixConn so the TLS handshake sees the full - // record. - if IsCopilotAPIHost(host) && p.tokenSink != nil && agentName != "" { - p.sniffCopilotOnTLS(&prefixConn{Conn: conn, prefix: fullBuf}, host, agentName) - return - } - - if !NeedsInspection(host) && !p.hostNeedsMITMFor(host, agentName) { - // Host we neither inspect nor (under #1861 injection) intercept: - // tunnel directly. SO_MARK the socket - // so the forced-egress redirect exempts this proxy-originated dial. - upstream, err := p.dialTunnelUpstream(host) - if err != nil { - return - } - defer func() { _ = upstream.Close() }() - if _, err := upstream.Write(fullBuf); err != nil { - return - } - relayTunnel(conn, upstream) - return - } - - mode := readAgentMode(agentName) - caps := readAgentCaps(agentName) - - // MITM: forge a cert, TLS-wrap the client, connect to real upstream. - tlsCert, err := p.forgeCert(host) - if err != nil { - p.logger.Error("transparent proxy forge cert failed", "host", host, "error", err) - return - } - - tlsConfig := &tls.Config{Certificates: []tls.Certificate{tlsCert}} - prefixed := &prefixConn{Conn: conn, prefix: fullBuf} - tlsClientConn := tls.Server(prefixed, tlsConfig) - // Bound the client TLS handshake: a client that connects and then stops - // sending handshake bytes would otherwise wedge this connection forever - // (empty queues on both sides — the observed egress-hang signature). - _ = conn.SetDeadline(time.Now().Add(clientTLSHandshakeTimeout)) - handshakeErr := tlsClientConn.Handshake() - _ = conn.SetDeadline(time.Time{}) - if handshakeErr != nil { - if isClientHangup(handshakeErr) { - p.logger.Debug("proxy client hung up before TLS handshake", "error", handshakeErr) - } else { - p.logger.Warn("transparent proxy TLS handshake failed", "error", handshakeErr) - } - return - } - defer func() { _ = tlsClientConn.Close() }() - - upstreamConn, err := tls.DialWithDialer(markDialer(upstreamDialTimeout), "tcp", host+":443", &tls.Config{ServerName: host}) - if err != nil { - p.logger.Error("transparent proxy upstream dial failed", "host", host, "error", err) - return - } - defer func() { _ = upstreamConn.Close() }() - - p.proxyHTTPHost(tlsClientConn, upstreamConn, host, agentName, mode, caps) -} - -// attributeTransparentConn names the agent behind an iptables-redirected -// connection (#7793). Under transparent redirection there is no -// Proxy-Authorization header and no CONNECT request, so the ONLY identity -// evidence is the kernel socket table: the peer's ephemeral port is looked up -// in /proc/net/tcp{,6} for the owning UID, and the UID in the agent map. -// -// It returns "" — "unidentified", which the mode gate downgrades to ADVISORY — -// when there is no UID map, when the peer address carries no port (a net.Pipe -// or unix socket), when the port is not in the socket table, or when the UID -// belongs to no agent. The one deliberate exception is the hive's own control -// plane: a UID the map reports as internal (root or the proxy user, and never -// an allocated agent UID) is named internalCallerName so a hive-originated -// write is attributed rather than mistaken for a UID-attribution failure. -func (p *GitHubProxy) attributeTransparentConn(conn net.Conn) string { - if p.uidMap == nil { - return "" - } - _, portStr, splitErr := net.SplitHostPort(conn.RemoteAddr().String()) - if splitErr != nil { - return "" - } - port := 0 - const maxPort = 65535 - for _, c := range portStr { - port = port*10 + int(c-'0') - if port > maxPort { - port = 0 - break - } - } - uid, lookupErr := LookupUIDByLocalPort(port) - if lookupErr != nil { - return "" - } - if name := p.uidMap.LookupByUID(uid); name != "" { - return name - } - // The hive's own control plane is not an agent, so the lookup above - // returns "". Under v4's forced-proxy egress that made every - // hive-originated write — notably the App installation token mint — look - // like an unattributable agent, which the mode gate blocks. Name it - // explicitly so it is attributed rather than mistaken for a - // UID-attribution failure. - if p.uidMap.IsInternalUID(uid) { - return internalCallerName - } - return "" -} - -// dialTunnelUpstream opens the raw TCP leg of a non-inspected transparent -// tunnel. The socket is SO_MARKed so the forced-egress redirect exempts this -// proxy-originated dial. tunnelDial, when set, overrides it (tests). -func (p *GitHubProxy) dialTunnelUpstream(host string) (net.Conn, error) { - if p.tunnelDial != nil { - return p.tunnelDial(host) - } - return markDialer(transparentProxyTimeout).Dial("tcp", host+":443") -} - -const tlsClientHelloMaxSize = 4096 - -// tlsRecordHeaderLen is the size of the TLS record header: content type(1) + -// version(2) + length(2). -const tlsRecordHeaderLen = 5 - -// readClientHelloRecord reads the complete first TLS record (the ClientHello) -// off conn, starting from the already-peeked bytes, and returns the full record -// so the caller can both extract the SNI and replay it verbatim into tls.Server. -// -// The prior implementation did a SINGLE conn.Read after the 1-byte peek and -// assumed it captured the whole ClientHello. TCP makes no such guarantee: when -// the ClientHello arrives split across segments, that single read returns only a -// prefix. extractSNI then parses a truncated ClientHello, fails to find the SNI, -// falls back to the wrong host, and the forged cert no longer matches what the -// agent asked for — so the agent-side TLS handshake fails deterministically -// (observed as "bad certificate" or "bad record MAC"), blocking every Copilot -// agent from authenticating. -// -// This reassembles the record using the record-layer length field, reading in a -// loop until the whole ClientHello record is buffered (or the buffer cap / -// deadline is hit), so both SNI extraction and the replay always see an intact -// ClientHello regardless of how TCP fragments the client's bytes. The caller -// sets/clears the read deadline around this call. -func readClientHelloRecord(conn net.Conn, peeked []byte) ([]byte, error) { - buf := make([]byte, tlsClientHelloMaxSize) - n := copy(buf, peeked) - - // First, ensure we have at least the record header so we know the record's - // declared length. - for n < tlsRecordHeaderLen { - r, err := conn.Read(buf[n:]) - n += r - if err != nil { - if n == 0 { - return nil, err - } - // Return what we have; extractSNI degrades gracefully on a short read. - return buf[:n], nil - } - } - - // record length is bytes 3..4 (big-endian); total record size adds the header. - recordLen := int(buf[3])<<8 | int(buf[4]) - need := tlsRecordHeaderLen + recordLen - if need > len(buf) { - // ClientHello larger than our cap: read to fill the buffer. The remainder - // (and subsequent records) are served to tls.Server via prefixConn+conn, - // so the handshake still completes; we only need enough here for SNI. - need = len(buf) - } - - for n < need { - r, err := conn.Read(buf[n:]) - n += r - if err != nil { - if n == 0 { - return nil, err - } - return buf[:n], nil - } - } - return buf[:n], nil -} - -// extractSNI reads the SNI hostname from a TLS ClientHello message. -func extractSNI(data []byte) string { - if len(data) < 5 { - return "" - } - // TLS record: type(1) + version(2) + length(2) + handshake - recordLen := int(data[3])<<8 | int(data[4]) - if len(data) < 5+recordLen { - // Partial read — use what we have - recordLen = len(data) - 5 - } - handshake := data[5 : 5+recordLen] - if len(handshake) < 4 { - return "" - } - // Handshake: type(1) + length(3) + ClientHello - hsLen := int(handshake[1])<<16 | int(handshake[2])<<8 | int(handshake[3]) - if len(handshake) < 4+hsLen { - hsLen = len(handshake) - 4 - } - ch := handshake[4 : 4+hsLen] - if len(ch) < 34 { - return "" - } - // Skip: version(2) + random(32) - pos := 34 - // Session ID - if pos >= len(ch) { - return "" - } - sessLen := int(ch[pos]) - pos += 1 + sessLen - // Cipher suites - if pos+2 > len(ch) { - return "" - } - csLen := int(ch[pos])<<8 | int(ch[pos+1]) - pos += 2 + csLen - // Compression methods - if pos >= len(ch) { - return "" - } - cmLen := int(ch[pos]) - pos += 1 + cmLen - // Extensions - if pos+2 > len(ch) { - return "" - } - extLen := int(ch[pos])<<8 | int(ch[pos+1]) - pos += 2 - extEnd := pos + extLen - if extEnd > len(ch) { - extEnd = len(ch) - } - for pos+4 <= extEnd { - extType := int(ch[pos])<<8 | int(ch[pos+1]) - eLen := int(ch[pos+2])<<8 | int(ch[pos+3]) - pos += 4 - if pos+eLen > extEnd { - break - } - if extType == 0 { // SNI extension - sniData := ch[pos : pos+eLen] - if len(sniData) < 2 { - break - } - sniListLen := int(sniData[0])<<8 | int(sniData[1]) - _ = sniListLen - sniPos := 2 - for sniPos+3 <= len(sniData) { - nameType := sniData[sniPos] - nameLen := int(sniData[sniPos+1])<<8 | int(sniData[sniPos+2]) - sniPos += 3 - if sniPos+nameLen > len(sniData) { - break - } - if nameType == 0 { // host_name - return string(sniData[sniPos : sniPos+nameLen]) - } - sniPos += nameLen - } - } - pos += eLen - } - return "" -} - -// prefixConn wraps a net.Conn and prepends already-read bytes to the stream. -type prefixConn struct { - net.Conn - prefix []byte - offset int -} - -func (c *prefixConn) Read(b []byte) (int, error) { - if c.offset < len(c.prefix) { - n := copy(b, c.prefix[c.offset:]) - c.offset += n - return n, nil - } - return c.Conn.Read(b) -} - -// identifyAgentFromReq determines the agent name for a request. It always -// tries UID-based identification first (unforgeable, works for any client -// including native binaries that don't send Proxy-Authorization); see -// fallbackAgentName for what happens when that fails. -func (p *GitHubProxy) identifyAgentFromReq(r *http.Request) string { - if p.uidMap != nil { - if name := p.identifyAgentByUID(r.RemoteAddr); name != "" { - return name - } - } - return p.fallbackAgentName(r) -} - -// IdentifyAgentByUID names the agent behind an http.Server-delivered request -// from the socket owner's UID ALONE. Unlike identifyAgentFromReq it never -// falls back to the caller-controlled Proxy-Authorization header, not even -// under HIVE_PROXY_ADVISORY_OK: see fallbackAgentName for why a self-asserted -// name is impersonation, and for the loopback services beside the proxy (the -// Jev decision endpoint, hivecommons/hive#8939) there is no "degraded but -// functional" mode worth that trade — an unidentified caller is refused. -// Returns "" when no UID map was loaded or the connection's UID is unmapped. -func (p *GitHubProxy) IdentifyAgentByUID(r *http.Request) string { - if p.uidMap == nil || r == nil { - return "" - } - return p.identifyAgentByUID(r.RemoteAddr) -} - -// identifyAgentFromConn identifies the calling agent for a request read off a -// raw connection. It MUST be used instead of identifyAgentFromReq on any path -// where the request came from http.ReadRequest rather than http.Server: -// ReadRequest does not populate Request.RemoteAddr (only http.Server does), so -// the UID lookup silently gets an empty address, finds no agent, and the caller -// falls back to ADVISORY — downgrading every write-capable agent to read-only. -// Taking the peer address from the connection itself removes that trap. -func (p *GitHubProxy) identifyAgentFromConn(conn net.Conn, r *http.Request) string { - if p.uidMap != nil && conn != nil { - if name := p.identifyAgentByUID(conn.RemoteAddr().String()); name != "" { - return name - } - } - return p.fallbackAgentName(r) -} - -// fallbackAgentName is reached whenever unforgeable UID-based identification -// did NOT succeed — either p.uidMap is nil (never loaded: no entrypoint root -// phase ran to write uid-map.json, e.g. local dev, a native/systemd install, -// or a genuine boot-ordering failure) or this specific connection's UID has -// no entry in a map that DID load (N7, #3841). -// -// Before this fix, both cases fell through to extractAgentName — the -// Proxy-Authorization header, which the CALLER sets. Any process that can -// reach the proxy's loopback port can send `Proxy-Authorization: hive -// ` and be identified as that name: a spoke-agent process claiming -// to be a MORE privileged agent bypasses the whole capability model this -// proxy exists to enforce, and #1861 (inject GitHub auth at the proxy so -// agents never hold tokens) would turn that spoofed identity directly into a -// minted token for the impersonated agent. That is strictly worse than -// simply refusing to identify the caller. -// -// The safe default is therefore to NOT trust the header: return "", which -// readAgentMode already treats as ModeAdvisory (the most restrictive mode — -// writes blocked, per the existing "agent could not be identified" WARN a few -// lines below in proxyHTTP). This is not new plumbing; it is the same path an -// unidentified caller already hits today, now reached deliberately instead of -// only when the header happens to be absent. -// -// The one deliberate exception is proxyAdvisoryOK (HIVE_PROXY_ADVISORY_OK=true) -// — the SAME explicit, operator-set escape hatch entrypoint.sh already uses -// for "this deployment does not enforce the full forced-egress gate" (local -// dev, a native/systemd install with no per-agent UID separation at all, -// platforms without NET_ADMIN). An operator who has already accepted a -// degraded capability model there is not newly exposed by also accepting a -// self-asserted identity within it — and without this exception, those -// deployments would lose agent identification (and therefore write access) -// entirely, which would be a functional regression, not a security fix. -func (p *GitHubProxy) fallbackAgentName(r *http.Request) string { - if !p.proxyAdvisoryOK { - return "" - } - return extractAgentName(r) -} - -// identifyAgentByUID reads /proc/net/tcp to find the UID of the process -// that owns the socket connected to the proxy, then looks up the agent name. -func (p *GitHubProxy) identifyAgentByUID(remoteAddr string) string { - uid, err := uidForRemoteAddr(remoteAddr) - if err != nil { - return "" - } - return p.uidMap.LookupByUID(uid) -} - -// uidForRemoteAddr resolves the UID of the process that owns the local socket -// behind remoteAddr ("host:port" as http.Server and net.Conn report it) via -// /proc/net/tcp. It is the shared first half of every UID-attribution path. -func uidForRemoteAddr(remoteAddr string) (int, error) { - _, portStr, err := net.SplitHostPort(remoteAddr) - if err != nil { - return -1, err - } - port := 0 - const maxPort = 65535 - for _, c := range portStr { - if c < '0' || c > '9' { - return -1, fmt.Errorf("remote port %q is not numeric", portStr) - } - port = port*10 + int(c-'0') - if port > maxPort { - return -1, fmt.Errorf("remote port %q out of range", portStr) - } - } - return LookupUIDByLocalPort(port) -} - -// inferenceAPIKeyPrefix is what the manager prepends to the agent's name to -// form the ANTHROPIC_API_KEY it hands each inference-routed agent -// (agent/manager_env.go). The suffix is the agent's NAME, not a secret. -const inferenceAPIKeyPrefix = "sk-hive-" - -// translatorAgentName names the caller of the inference translator. -// -// The x-api-key header the Claude CLI sends is "sk-hive-", and the -// agent name is public inside the pod: every agent knows every other agent's -// name. Trusting the header alone lets any process on the loopback borrow -// another agent's inference route — its backend, model, upstream credential -// and usage attribution — which is exactly the self-asserted-identity hole -// closed for the GitHub path in fallbackAgentName and for the decision -// endpoint in IdentifyAgentByUID. So the socket owner's UID decides: -// -// - a UID mapped to an agent is authoritative; a header naming someone else -// is logged and overridden; -// - the root control plane keeps the header name — the hive is the party -// that issues these keys; the proxy UID cannot assert a name because its -// upstream sockets can relay agent-controlled requests; -// - an unmapped UID, or no UID map at all, accepts the header only under -// HIVE_PROXY_ADVISORY_OK, the same operator escape hatch the GitHub path -// honours for deployments without per-agent UID separation. -// -// The second return is false when the caller could not be identified and the -// request must be refused. -func (p *GitHubProxy) translatorAgentName(r *http.Request) (string, bool) { - claimed := strings.TrimPrefix(r.Header.Get("x-api-key"), inferenceAPIKeyPrefix) - if p.uidMap == nil { - if p.proxyAdvisoryOK { - return claimed, true - } - return "", false - } - uid, err := uidForRemoteAddr(r.RemoteAddr) - if err == nil { - if name := p.uidMap.LookupByUID(uid); name != "" { - if claimed != "" && claimed != name { - p.logger.Warn("inference identity: x-api-key names another agent; using the socket owner", - "agent", name, "claimed", claimed, "uid", uid) - } - return name, true - } - if uid == 0 && p.uidMap.IsInternalUID(uid) { - return claimed, true - } - } - if p.proxyAdvisoryOK { - return claimed, true - } - return "", false -} - -// handleConnectDirect handles CONNECT requests on a raw connection (no http.Server). -func (p *GitHubProxy) handleConnectDirect(conn net.Conn, r *http.Request) { - host, _, err := net.SplitHostPort(r.Host) - if err != nil { - host = r.Host - } - - // Identify from the connection, NOT from r.RemoteAddr: this request came - // from http.ReadRequest, which leaves RemoteAddr empty. - agentName := p.identifyAgentFromConn(conn, r) - - // Anthropic hosts with an inference route: reroute to self-hosted backend. - if IsAnthropicHost(host) { - if route := p.inference.Get(agentName); route != nil { - p.handleAnthropicReroute(conn, r, host, agentName, route) - return - } - } - - // Copilot completion host: MITM to read live token usage. Only when a token - // sink is active and the agent is identified — otherwise fall through to an - // opaque tunnel so Copilot traffic is never broken by usage capture. - if IsCopilotAPIHost(host) && p.tokenSink != nil && agentName != "" { - p.handleCopilotSniff(conn, host, agentName) - return - } - - // Hosts we do not inspect: tunnel without interception. github.com is in - // this set — its OAuth device flow and git smart HTTP are handled by CLI - // --deny-tool flags. api.github.com and api.linear.app are not: both need - // request-level inspection for ACMM enforcement. Under #1861 injection, - // however, every GitHub-family host is intercepted (see hostNeedsMITM), - // because the agent's credential helper now serves an inert placeholder - // that only the proxy can replace with the real scoped token. - if !NeedsInspection(host) && !p.hostNeedsMITMFor(host, agentName) { - p.tunnelDirect(conn, r) - return - } - - mode := readAgentMode(agentName) - caps := readAgentCaps(agentName) - - // Tell client the tunnel is established. - if _, err := fmt.Fprintf(conn, "HTTP/1.1 200 Connection established\r\n\r\n"); err != nil { - p.logger.Error("proxy CONNECT response write failed", "error", err) - return - } - - // Generate a cert for the target host signed by our CA. - tlsCert, err := p.forgeCert(host) - if err != nil { - p.logger.Error("proxy forge cert failed", "host", host, "error", err) - return - } - - // TLS handshake with client (presenting our forged cert). - tlsClientConn := tls.Server(conn, &tls.Config{ - Certificates: []tls.Certificate{tlsCert}, - }) - // Bound the client TLS handshake (see handleTransparentTLS): a client that - // stops mid-handshake must not wedge this connection forever. - _ = conn.SetDeadline(time.Now().Add(clientTLSHandshakeTimeout)) - handshakeErr := tlsClientConn.Handshake() - _ = conn.SetDeadline(time.Time{}) - if handshakeErr != nil { - if isClientHangup(handshakeErr) { - p.logger.Debug("proxy client hung up before TLS handshake", "error", handshakeErr) - } else { - p.logger.Warn("proxy client TLS handshake failed", "error", handshakeErr) - } - return - } - defer func() { _ = tlsClientConn.Close() }() - - // Connect to the real GitHub server. SO_MARK the socket so the forced-egress - // redirect exempts this proxy-originated dial (see proxyEgressMark). - // A dialer Timeout bounds both the TCP connect and the TLS handshake — - // without it a stalled upstream wedges this CONNECT forever (the agent's - // curl sits in poll() waiting for "200 Connection established" that never - // comes). The deadline is lifted once established so the relay is not - // time-boxed. - upstreamConn, err := tls.DialWithDialer(markDialer(upstreamDialTimeout), "tcp", r.Host, &tls.Config{ - ServerName: host, - }) - if err != nil { - p.logger.Error("proxy upstream dial failed", "host", r.Host, "error", err) - return - } - defer func() { _ = upstreamConn.Close() }() - - // Proxy HTTP requests, inspecting each one. - p.proxyHTTPHost(tlsClientConn, upstreamConn, host, agentName, mode, caps) -} - -// proxyHTTP reads HTTP requests from the client, checks them against -// mode rules, and either forwards or blocks them. -func isTimeoutError(err error) bool { - var netErr net.Error - return errors.As(err, &netErr) && netErr.Timeout() -} - -// logUpstreamEvent logs an upstream timeout event at the given level. -func (p *GitHubProxy) logUpstreamEvent(level slog.Level, msg string, err error, attrs ...any) { - if !isTimeoutError(err) { - return - } - attrs = append(attrs, "error", err) - p.logger.Log(context.Background(), level, msg, attrs...) -} - -func (p *GitHubProxy) logTimeout(msg string, err error, attrs ...any) { - p.logUpstreamEvent(slog.LevelWarn, msg, err, attrs...) -} - -// isClientHangup reports a client that went away (EOF, reset, broken pipe) or -// stalled past the deadline: routine on restarts and liveness probes, not a -// hive defect. -func isClientHangup(err error) bool { - return isTimeoutError(err) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || - errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.EPIPE) || errors.Is(err, net.ErrClosed) -} - -// logClientRead logs a client-side request read failure at Debug: an idle or -// departed client is expected, unlike an upstream stall. -func (p *GitHubProxy) logClientRead(msg string, err error, attrs ...any) { - p.logUpstreamEvent(slog.LevelDebug, msg, err, attrs...) -} - -// proxyHTTP relays a MITM'd GitHub connection with ACMM enforcement applied. -// It is proxyHTTPHost with the GitHub host implied, which is what every -// pre-existing caller means. -func (p *GitHubProxy) proxyHTTP(client net.Conn, upstream net.Conn, agentName string, mode agent.AgentMode, caps agent.AgentCapabilities) { - p.proxyHTTPHost(client, upstream, "", agentName, mode, caps) -} - -// proxyHTTPHost is proxyHTTP with the terminated host carried through, so the -// request loop can pick the right rule engine (#4492 F). -// -// The host is threaded rather than inferred from the request because a MITM'd -// request line is a path, not an absolute URL — by this point the only record -// of which host the TLS session was forged for is the CONNECT that opened it. -// An empty host means GitHub, preserving every existing call site verbatim. -func (p *GitHubProxy) proxyHTTPHost(client net.Conn, upstream net.Conn, host string, agentName string, mode agent.AgentMode, caps agent.AgentCapabilities) { - isLinear := IsLinearHost(host) - clientBuf := newBufferedConn(client) - - for { - // Bound every phase of this request/response exchange. Before this, - // the relay had NO deadlines anywhere after the initial CONNECT read: - // a client or upstream that stopped mid-exchange (stalled response, - // half-written body, frozen keep-alive) blocked forever — the - // multi-hour "egress wedge" where the agent's request never reached - // GitHub and its session froze. Each phase gets its own deadline; - // they are cleared once the phase completes so healthy keep-alive - // connections are unaffected. - _ = client.SetReadDeadline(time.Now().Add(httpReadTimeout)) - req, err := http.ReadRequest(clientBuf) - if err != nil { - _ = client.SetReadDeadline(time.Time{}) - p.logClientRead("proxy client request read timed out", err, "agent", agentName) - return - } - - blocked := false - blockReason := "" - // blockCode is the machine-readable reason (#12127) that rides on the - // one-line 403 body and the X-Hive-Proxy-Reason header. Each gate that - // sets blockReason sets it; a gate that sets neither falls back to a - // generic code in ProxyDenialBody. - blockCode := "" - - // Per-repo pause (#6203) and per-repo agent scope (#6204) are both - // checked BEFORE the mode chain, and outside it, for the same two - // reasons. Neither is an autonomy tier — a pause is a run-state, a - // scope is roster membership — so each must refuse the write whatever - // mode the agent holds and whichever branch below would otherwise have - // allowed it. And each carries its own reason, so the agent is told the - // repo is deliberately quiet, or that it is not defined for this repo, - // instead of reading a mode error and going hunting for a permissions - // bug that does not exist. - // - // The hive's own control-plane traffic (App-token mint, heartbeat) is - // exempt from both, exactly as it is from the ACMM rules below: - // stopping the hive from minting a token would take the whole spoke - // down rather than quiet one repo or narrow one agent. - // - // Pause is checked first, so a repo the operator has frozen is refused - // as paused even when the agent is also out of scope for it. - blockJSON := false - graphQLMutation := false - // A direct write from a lane under write_surface.enforce (#9772) is - // refused and audited as agent_write_refused. Checked after pause and - // scope so their more specific reasons win when both apply. - writeSurfaceKind := "" - if agentName != internalCallerName { - if reason, paused := p.repoPauseRefusal(req.Method, repoScopePath(req)); paused { - blocked = true - blockReason = reason - blockCode = DenyReasonRepoPaused - } else if reason, refused := p.agentRepoScopeRefusal(agentName, req.Method, repoScopePath(req)); refused { - blocked = true - blockReason = reason - blockCode = DenyReasonAgentRepoScope - } else if reason, refused := p.autoMergeRefusal(req.Method, repoScopePath(req)); refused { - blocked = true - blockReason = reason - blockCode = DenyReasonRepoAutoMergeOff - blockJSON = true - } else if !isLinear { - if reason, kind, refused := WriteSurfaceEnforceRefusal(p.writeSurfaceEnforcedFunc(), agentName, req.Method, req.URL.Path, req.URL.RawQuery); refused { - blocked = true - blockReason = reason - blockCode = DenyReasonWriteSurfaceEnforced - writeSurfaceKind = kind - } - } - } - - if blocked { - // Already refused by the pause or agent-scope check above — the - // mode chain cannot un-block it, so skip it entirely. - } else if isLinear && agentName == internalCallerName { - // The hive's OWN Linear traffic — the read path #4178 shipped - // (backlog enumeration in pkg/worksource/linear.go) and, later, - // the OAuth/webhook plumbing. ACMM governs AGENT autonomy, so the - // control plane is exempt here exactly as it is for GitHub below. - // This branch must precede the enforcement one: without it, adding - // Linear to NeedsMITM would newly gate hive-originated reads that - // have always been ungated, breaking enumeration rather than - // restricting an agent. - } else if isLinear { - // Linear is a single POST /graphql for everything, so the - // (method, path) rule table cannot classify it and the GitHub - // GraphQL classifier is the wrong posture — it ends in a - // permissive default, backed by the REST table. Linear has no - // table underneath. See linear_rules.go. - blocked, blockReason = p.enforceLinear(req, agentName, mode, caps) - if blocked { - blockCode = DenyReasonLinear - } - } else if req.Method == "POST" && IsGraphQLPath(req.URL.Path) { - body, readErr := io.ReadAll(io.LimitReader(req.Body, graphQLBodyLimit)) - if req.Body != nil { - _ = req.Body.Close() - } - if readErr != nil { - _ = client.SetReadDeadline(time.Time{}) - p.logTimeout("proxy GraphQL request body read timed out", readErr, "agent", agentName, "path", req.URL.Path) - return - } - if reason, refused := p.graphQLAutoMergeRefusal(body); agentName != internalCallerName && refused { - blocked = true - blockReason = reason - blockCode = DenyReasonRepoAutoMergeOff - blockJSON = true - } else if agentName != internalCallerName { - if reason, refused := WriteSurfaceGraphQLRefusal(p.writeSurfaceEnforcedFunc(), agentName, body); refused { - blocked = true - blockReason = reason - blockCode = DenyReasonWriteSurfaceEnforced - writeSurfaceKind = WriteSurfaceKindGraphQL - } - } - allowed, isMutation := GraphQLAllowedCaps(autonomyGraphQLMode(agentName, body, mode), caps, body) - graphQLMutation = isMutation - if !blocked && !allowed { - blocked = true - if isMutation { - blockReason = "graphql mutation" - blockCode = DenyReasonGraphQLMutation - } else { - blockReason = "graphql" - blockCode = DenyReasonGraphQL - } - } - req.Body = io.NopCloser(strings.NewReader(string(body))) - req.ContentLength = int64(len(body)) - } else if agentName == internalCallerName { - // The hive itself has no ACMM mode to enforce — ACMM governs AGENT - // autonomy. Its control-plane calls (App token mint, heartbeat) - // must not be gated as if they were agent writes. Repo-filter and - // canary-egress checks below still apply. - } else if !AllowedByModeCaps(autonomyModeForRepo(agentName, ExtractRepo(repoScopePath(req)), mode), caps, req.Method, req.URL.Path) { - blocked = true - blockCode = DenyReasonMode - // An unidentified agent is silently treated as ADVISORY, which turns - // a permissions bug into an indistinguishable "policy denial". Say so - // loudly on writes: a write from an unknown caller means UID - // attribution failed, not that the agent lacks the mode. - if agentName == "" && writeMethods[req.Method] { - p.logger.Warn("proxy: agent could not be identified — defaulting to ADVISORY and blocking a write; UID attribution failed (check the uid map and /proc/net/tcp visibility), this is NOT an agent-mode misconfiguration", - "method", req.Method, "path", req.URL.Path) - } - // A hard-deny rule (e.g. direct PR creation) carries an agent-facing - // directive explaining the sanctioned alternative — surface it so the - // agent knows to use hive-open-pr rather than seeing only a mode error. - if msg, denied := DeniedMessage(req.Method, req.URL.Path); denied && msg != "" { - blockReason = msg - blockCode = DenyReasonHardDeny - } - } else if len(p.allowedRepos) > 0 && !RepoFilterAllowed(p.allowedRepos, req.Method, repoScopePath(req)) { - blocked = true - blockReason = "repo not in hive config" - blockCode = DenyReasonRepoNotInConfig - } - // Shape-gate agent-authored issue/comment REST writes. The issue-request - // watcher validates the file-drop creation path, but an agent's - // `gh issue create` becomes a raw POST /repos/{o}/{r}/issues that never - // touches the watcher (issue #7014: a template placeholder title and a - // body of literal \n escapes both reached GitHub this way). Enforce the - // same shape rules here so both creation paths are covered. Skip the - // control plane — its issue writes are already validated upstream. - if !blocked && agentName != internalCallerName { - if reason, deny, shapeErr := p.enforceIssueShapeFor(req, agentName); shapeErr != nil { - // Fail closed: a body we could not read is a body we could not - // shape-check. - p.logTimeout("proxy issue-shape body read timed out", shapeErr, "agent", agentName, "path", req.URL.Path) - _ = client.SetReadDeadline(time.Time{}) - return - } else if deny { - blocked = true - if blockReason == "" { - blockReason = reason - blockCode = DenyReasonIssueShape - } - } - } - if reason, deny, ok := p.inspectCanaryEgress(agentName, req); ok { - if deny { - blocked = true - if blockReason == "" { - blockReason = reason - blockCode = DenyReasonCanaryEgress - } - } - } - - if !isLinear && agentName != internalCallerName { - isGraphQL := req.Method == "POST" && IsGraphQLPath(req.URL.Path) - isWrite := writeMethods[req.Method] && !gitUploadPackPath.MatchString(req.URL.Path) - if isGraphQL { - isWrite = graphQLMutation - } - if isWrite { - reason := "" - if blocked { - reason = blockCode - if reason == "" { - reason = DenyReasonUnknown - } - } - p.writes.record(mode.String(), reason) - } - } - - if blocked { - detail := req.URL.Path - if blockReason != "" { - detail = blockReason - } - p.recordViolation(agentName, req.Method, detail) - if writeSurfaceKind != "" { - p.auditWriteRefused(agentName, writeSurfaceKind, req.Method, repoScopePath(req)) - } - - if blockCode == "" { - blockCode = DenyReasonUnknown - } - respBody := ProxyDenialBody(agentName, mode, req.Method, detail, blockCode) - contentType := "text/plain" - if blockJSON { - contentType = "application/json" - payload, _ := json.Marshal(map[string]any{ - "error": "repo_auto_merge_disabled", - "reason": detail, - }) - respBody = string(payload) + "\n" - } - resp := &http.Response{ - StatusCode: http.StatusForbidden, - Proto: "HTTP/1.1", - ProtoMajor: 1, - ProtoMinor: 1, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(respBody)), - } - resp.Header.Set("Content-Type", contentType) - resp.Header.Set("X-Hive-Proxy-Blocked", "true") - resp.Header.Set(ProxyDenialHeader, blockCode) - _ = client.SetWriteDeadline(time.Now().Add(httpWriteTimeout)) - writeErr := resp.Write(client) - _ = client.SetWriteDeadline(time.Time{}) - if writeErr != nil { - p.logTimeout("proxy blocked response write timed out", writeErr, "agent", agentName) - return - } - - if req.Body != nil { - if _, drainErr := io.Copy(io.Discard, req.Body); drainErr != nil { - p.logTimeout("proxy blocked request body drain timed out", drainErr, "agent", agentName, "path", req.URL.Path) - } - _ = req.Body.Close() - } - _ = client.SetReadDeadline(time.Time{}) - continue - } - - if !isLinear { - if refusal := p.githubBudgetRefusal(agentName, req.Method, req.URL.Path, agentGitHubRequestIsRead(req.Method, req.URL.Path, graphQLMutation)); refusal != nil { - _ = p.writeAgentGitHubBudget429(client, *refusal) - if req.Body != nil { - _, _ = io.Copy(io.Discard, req.Body) - _ = req.Body.Close() - } - _ = client.SetReadDeadline(time.Time{}) - continue - } - } - - // #1861: with injection enabled, the upstream credential is decided - // HERE — after the ACMM/repo/canary gates, immediately before either - // forwarding branch — so every byte that reaches GitHub has passed - // through it. Blocked requests above never reach upstream and need no - // rewrite. - // - // #9586: only for GitHub hosts. This loop also relays api.linear.app - // (NeedsInspection), and rewriting there would attach the agent's - // hub-held GITHUB token to a request bound for Linear - a real - // credential handed to a third party - whenever the Linear lane below - // does not overwrite the header (an ADVISORY agent, or a hive with no - // Linear credential). - if injectsGitHubAuthForHost(host) { - p.rewriteGitHubAuth(req, agentName) - } - - // Linear: the request passed the tier gate; make sure it reaches - // Linear with the hive's CURRENT credential rather than whatever - // (possibly rotated-out) copy the agent's process still holds. - if isLinear && agentName != internalCallerName { - p.injectLinearCredential(req, agentName, mode) - } - // Git smart HTTP uses chunked streaming that http.ReadResponse - // can't handle reliably. After the ACMM check passes, forward - // the request and switch to raw bidirectional streaming. - if isGitPath(req.URL.Path) { - _ = upstream.SetWriteDeadline(time.Now().Add(httpWriteTimeout)) - if err := req.Write(upstream); err != nil { - _ = upstream.SetWriteDeadline(time.Time{}) - _ = client.SetReadDeadline(time.Time{}) - p.logTimeout("proxy git request relay timed out", err, "agent", agentName, "path", req.URL.Path) - return - } - _ = upstream.SetWriteDeadline(time.Time{}) - _ = client.SetReadDeadline(time.Time{}) - // Use relayTunnel, NOT an inline copy pair: this branch used to run - // its own unbounded io.Copy pair, which was the same wedge #3872 - // fixed in relayTunnel but left here. When GitHub FIN'd after the - // response while the client kept its side open (keep-alive), the - // client→upstream copy stayed blocked in client.Read() forever and - // `<-done` never returned — the handler's deferred Closes never ran, - // parking the upstream socket in CLOSE_WAIT for the life of the - // process (issue #3875's FD pile). relayTunnel bounds each direction - // once the other finishes. - relayTunnel(client, upstream) - return - } - - // Forward to upstream. The client read deadline stays in force until - // the body is fully drained (req.Write pipes req.Body from the client), - // so a client stalling mid-body cannot hang the relay. - _ = upstream.SetWriteDeadline(time.Now().Add(httpWriteTimeout)) - if err := req.Write(upstream); err != nil { - _ = upstream.SetWriteDeadline(time.Time{}) - _ = client.SetReadDeadline(time.Time{}) - p.logTimeout("proxy request relay timed out", err, "agent", agentName, "path", req.URL.Path) - return - } - _ = upstream.SetWriteDeadline(time.Time{}) - _ = client.SetReadDeadline(time.Time{}) - - _ = upstream.SetReadDeadline(time.Now().Add(httpReadTimeout)) - resp, err := http.ReadResponse(newBufferedReader(upstream), req) - _ = upstream.SetReadDeadline(time.Time{}) - if err != nil { - p.logTimeout("proxy upstream response read timed out", err, "agent", agentName, "path", req.URL.Path) - return - } - p.recordGitHubREST(agentName, req, resp) - p.recordAgentGitHubBudget(agentName, req.URL.Path, resp.StatusCode, p.agentGitHubCap(agentName)) - - // Bound the BODY relay too, not just the header read above. The - // deadline was cleared once headers arrived, so a "reachable but slow" - // GitHub — TLS and headers complete, body never arrives (the measured - // #3875 signature: time_total=12s, 0 body bytes) — parked resp.Write - // in upstream.Read() with no deadline. The client had long since given - // up (10s collect budget) and FIN'd its side, but this goroutine never - // noticed: it holds all three sockets of the exchange forever, and the - // retry loop stacks a fresh parked handler on every attempt (~1k - // FDs/min measured live). The wrapper re-arms a rolling per-Read - // deadline, so a body that keeps flowing is never cut while a stall - // longer than responseBodyStallTimeout errors out and lets the - // handler's deferred Closes reclaim the sockets. - resp.Body = &stallBoundedBody{body: resp.Body, conn: upstream, idle: p.bodyStallTimeoutDuration()} - - // Rolling per-Write deadline (see stallBoundedWriter): a response that - // keeps flowing is never cut for being long, while a client that stops - // draining still releases this handler within one stall window. - if err := resp.Write(&stallBoundedWriter{conn: client, idle: httpWriteTimeout}); err != nil { - _ = resp.Body.Close() - _ = client.SetWriteDeadline(time.Time{}) - p.logTimeout("proxy response relay timed out", err, "agent", agentName, "path", req.URL.Path) - return - } - _ = client.SetWriteDeadline(time.Time{}) - _ = resp.Body.Close() - } -} - -// gitInjectBasicUser is the username git expects alongside a GitHub App -// installation token in HTTP Basic auth — the same identity -// git-credential-hive.sh answers with, so upstream sees exactly the shape the -// pre-injection flow produced. -const gitInjectBasicUser = "x-access-token" - -// loginPathPrefix marks the GitHub OAuth/device-flow endpoints -// (/login/device/code, /login/oauth/access_token). These authenticate the -// FLOW via their form body, not a bearer credential; injecting an App token -// there would be meaningless at best and flow-corrupting at worst, so they -// get strip-only treatment. -const loginPathPrefix = "/login/" - -// copilotInternalPathPrefix marks the Copilot CLI's auth-exchange endpoints on -// the GitHub API host (/copilot_internal/v2/token, /copilot_internal/user). -// The CLI calls them with the USER's Copilot OAuth token -// (COPILOT_GITHUB_TOKEN / the /data/copilot-user-token device-flow token) to -// obtain its short-lived Copilot session token and discover its completion -// host. A GitHub App installation token cannot perform that exchange, so -// rewriting these requests would cut every copilot-backend agent off from its -// model the moment injection is on (#9586). They are passed through with the -// agent's own Authorization untouched - nothing stripped, nothing injected. -// -// Residual (documented in security-model.md): the Copilot user OAuth token -// stays agent-readable and spendable on these paths; moving it server-side too -// is a follow-up. The hub-held App tokens injection protects are never -// attached here. -const copilotInternalPathPrefix = "/copilot_internal/" - -// gheAPIPathPrefix is the REST prefix GitHub Enterprise Server serves its API -// under (https:///api/v3/...), so the Copilot exchange on a GHE host -// arrives as /api/v3/copilot_internal/... -const gheAPIPathPrefix = "/api/v3" - -// isCopilotAuthExchangePath reports whether path is a Copilot CLI -// auth-exchange endpoint that must keep the agent's own credential (see -// copilotInternalPathPrefix), on api.github.com or a GHE /api/v3 host. -func isCopilotAuthExchangePath(path string) bool { - return strings.HasPrefix(path, copilotInternalPathPrefix) || - strings.HasPrefix(path, gheAPIPathPrefix+copilotInternalPathPrefix) -} - -// rewriteGitHubAuth enforces #1861 on one MITM'd request: the proxy — not the -// agent — decides what credential GitHub sees. -// -// Threat model, per header-touching block: -// -// - STRIP: an agent-supplied Authorization header is deleted UNCONDITIONALLY -// (under the flag), never forwarded. Even though injection-mode agents are -// only ever handed the inert placeholder, an agent that somehow obtained a -// real credential (leaked from a log, smuggled via prompt, minted through -// a side channel) must not be able to spend it through this proxy. -// -// - INJECT: only for an agent identified by the UID-authoritative path -// (#3888) that has a hub-held scoped token. The two lookups this chains -// are exactly the pre-existing lanes: identity from the UID map, token -// from the #3967 per-agent scoped mint. -// -// - UNKNOWN AGENT = NO CREDENTIAL: when identity resolution failed -// (agentName == ""), the request proceeds with NO Authorization and fails -// loud at GitHub (401/403). Deliberate: any fallback — shared cache, -// hive token, "last known agent" — would let an unattributable process -// ride a real credential, recreating the pre-#3888 identity hole this -// design depends on having closed. -// -// - COPILOT AUTH-EXCHANGE PASSTHROUGH: /copilot_internal/ on a GitHub -// host keeps the agent's own Authorization (the user's Copilot OAuth -// token); see copilotInternalPathPrefix for why and the residual. -// -// - INTERNAL CALLER PASSTHROUGH: the hive's own control plane -// (internalCallerName) legitimately holds and sends its own App -// credentials (token mint, heartbeat, hive-open-pr fulfillment); its -// requests pass untouched. This cannot be spoofed by an agent: the name -// is assigned only via UIDMap.IsInternalUID, never from anything the -// caller sends. -// -// Logging: agent name and injected yes/no only — NEVER token bytes, not even -// a prefix (#1861 requirement; a token prefix is enough to fingerprint and -// correlate credentials across logs). -func (p *GitHubProxy) rewriteGitHubAuth(req *http.Request, agentName string) { - // NOTE: callers must gate on injectsGitHubAuthForHost first; this function - // does not know which host the request is bound for. - if !p.injectGHAuth { - // Flag off (the default): behavior byte-identical to before #1861 — - // no header is touched on any path. - return - } - if agentName == internalCallerName { - return - } - if isCopilotAuthExchangePath(req.URL.Path) { - // Before the strip: the Copilot session-token exchange needs the - // user's Copilot OAuth token, which no hub-held App token can stand in - // for (copilotInternalPathPrefix). - p.logger.Debug("proxy auth injection: Copilot auth-exchange endpoint - agent credential passed through", "agent", agentName, "injected", false) - return - } - - req.Header.Del("Authorization") - - if agentName == "" { - p.logger.Debug("proxy auth injection: caller unidentified — forwarding without credentials (fail-loud, no fallback)", "injected", false) - return - } - if strings.HasPrefix(req.URL.Path, loginPathPrefix) { - p.logger.Debug("proxy auth injection: OAuth flow endpoint — strip only", "agent", agentName, "injected", false) - return - } - if p.agentTokenSource == nil { - p.logger.Warn("proxy auth injection enabled but no token source wired — forwarding without credentials", "agent", agentName, "injected", false) - return - } - token, ok := p.agentTokenSource(agentName) - if !ok { - // An identified agent with no hub-held token means the #3967 mint lane - // has not (yet) delivered for this agent — e.g. the hive restarted and - // the in-memory registry is empty until the next launch/refresh mint. - // Fail loud (upstream 401) rather than borrow any other credential. - p.logger.Warn("proxy auth injection: no hub-held token for identified agent — forwarding without credentials (check the per-agent mint lane)", "agent", agentName, "injected", false) - return - } - - if isGitPath(req.URL.Path) { - // Git smart HTTP authenticates with Basic x-access-token: — - // the same shape git-credential-hive.sh produced pre-injection. - basic := base64.StdEncoding.EncodeToString([]byte(gitInjectBasicUser + ":" + token)) - req.Header.Set("Authorization", "Basic "+basic) - // Force this exchange onto its own upstream connection. The git branch - // in proxyHTTP switches to a raw relay after forwarding this request's - // headers, so a keep-alive reuse (git's follow-up POST after GET - // /info/refs on the same conn) would stream its headers — placeholder - // credential included — straight through WITHOUT passing this rewrite, - // and fail with a confusing mid-operation 401. Connection: close makes - // git reconnect per request; each new connection re-enters proxyHTTP - // and gets its own rewrite. - req.Close = true - } else { - // REST/GraphQL (api.github.com, GHE /api/v3): the standard GitHub - // token scheme, identical to what gh sends. - req.Header.Set("Authorization", "token "+token) - } - p.logger.Debug("proxy auth injection: scoped token attached", "agent", agentName, "injected", true) -} - -// injectsGitHubAuthForHost reports whether a request relayed for host may have -// its Authorization rewritten by rewriteGitHubAuth (#9586). It is a positive -// allowlist - the empty host (proxyHTTP's "GitHub implied" convention) and the -// registered GitHub-family hosts - rather than "anything but Linear", so a -// non-GitHub host added to the MITM set later can never receive a GitHub token -// by default. -func injectsGitHubAuthForHost(host string) bool { - return host == "" || IsGitHubHost(host) -} - -func (p *GitHubProxy) inspectCanaryEgress(agentName string, req *http.Request) (reason string, deny bool, detected bool) { - if p == nil || !p.canariesEnabled || p.canaryRegistry == nil || req == nil || req.Body == nil { - return "", false, false - } - if isGitReceivePack(req.URL.Path) && p.canaryFailClosed { - return "ioscan canary scan unavailable for git push", true, true - } - if !writeMethods[req.Method] || isGitPath(req.URL.Path) || !githubWriteBodyMayLeak(req.Method, req.URL.Path) { - return "", false, false - } - body, err := io.ReadAll(io.LimitReader(req.Body, maxGitHubWriteBodyScan+1)) - if req.Body != nil { - _ = req.Body.Close() - } - if err != nil { - return "", false, false - } - if len(body) > maxGitHubWriteBodyScan { - return "ioscan canary scan body too large", true, true - } - req.Body = io.NopCloser(bytes.NewReader(body)) - req.ContentLength = int64(len(body)) - if leak, ok := p.canaryRegistry.Scan(agentName, string(body), "github:"+req.Method+" "+req.URL.Path); ok { - if p.canaryLeakFunc != nil { - p.canaryLeakFunc(leak) - } - p.logger.Warn("ioscan canary leak detected", "agent", leak.Agent, "source", leak.Source) - return "ioscan canary leak", p.canaryFailClosed, true - } - return "", false, false -} - -// enforceLinear applies ACMM to one request on a MITM'd api.linear.app -// connection and returns whether it is blocked, with an agent-facing reason. -// -// The request body is read (bounded), inspected, and restored so an allowed -// request forwards byte-identically. -// -// NOTHING from the body is logged or surfaced in the 403. Linear documents -// carry issue and comment content in the query and, in variables, potentially -// tokens; the operation NAMES are schema identifiers rather than user content, -// so those are what makes a denial debuggable without leaking the payload. -func (p *GitHubProxy) enforceLinear(req *http.Request, agentName string, mode agent.AgentMode, caps agent.AgentCapabilities) (bool, string) { - // Linear exposes exactly one endpoint. Anything else on this host is not a - // shape this gate understands, so it is refused rather than forwarded. - if req.Method != http.MethodPost || !IsLinearGraphQLPath(req.URL.Path) { - return true, "linear: only POST " + linearGraphQLPath + " is permitted" - } - - // Read one byte past the limit so hitting the cap is detectable: a body - // that exactly fills the buffer is indistinguishable from a truncated one - // otherwise, and truncation must deny. - body, readErr := io.ReadAll(io.LimitReader(req.Body, linearBodyLimit+1)) - if req.Body != nil { - _ = req.Body.Close() - } - truncated := len(body) > linearBodyLimit - if truncated { - body = body[:linearBodyLimit] - } - req.Body = io.NopCloser(bytes.NewReader(body)) - req.ContentLength = int64(len(body)) - - if readErr != nil { - // Fail closed: a body we could not fully read is a body we could not - // classify. - p.logger.Warn("linear: request body read failed — denying", "agent", agentName) - return true, "linear: request body unreadable" - } - - decision := LinearGraphQLAllowed(mode, caps, body, truncated) - if decision.Allowed { - return false, "" - } - - // Operations may be empty when the document could not be classified at - // all, which is itself the useful signal. - p.logger.Warn("linear: blocked GraphQL request", - "agent", agentName, - "mode", mode.String(), - "operations", strings.Join(decision.Operations, ","), - "reason", decision.Reason, - "mutation", decision.IsMutation) - - reason := "linear: " + decision.Reason - if len(decision.Operations) > 0 { - reason += " (" + strings.Join(decision.Operations, ",") + ")" - } - return true, reason -} - -func githubWriteBodyMayLeak(method, path string) bool { - if method != http.MethodPost && method != http.MethodPatch && method != http.MethodPut { - return false - } - if !strings.HasPrefix(path, "/repos/") { - return IsGraphQLPath(path) - } - return strings.Contains(path, "/issues") || - strings.Contains(path, "/pulls") || - strings.Contains(path, "/comments") || - strings.Contains(path, "/reviews") || - strings.Contains(path, "/git/") -} - -func isGitReceivePack(path string) bool { - return strings.HasSuffix(path, "/git-receive-pack") -} - -// issueShapeBodyLimit bounds how much of an issue/comment REST body the proxy -// buffers to shape-check it. It comfortably exceeds GitHub's issue-body maximum -// (65536 chars) plus JSON overhead; an oversized body is forwarded unbuffered -// and unchecked rather than truncated (see enforceIssueShape). -const issueShapeBodyLimit = 256 * 1024 - -var ( - issueCreateRESTPathRE = regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues$`) - issueEditRESTPathRE = regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues/\d+$`) - issueCommentCreateRESTPath = regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues/\d+/comments$`) - issueCommentEditRESTPath = regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues/comments/\d+$`) -) - -// isIssueShapeRESTWrite reports whether a (method, path) is a REST issue or -// issue-comment create/edit whose JSON body carries agent-authored title/body -// text the shape validators must vet. It is deliberately precise — matching -// only the title/body-bearing routes — so label, assignee, and reaction sub -// routes under /issues are not needlessly buffered. -func isIssueShapeRESTWrite(method, path string) bool { - if i := strings.IndexByte(path, '?'); i >= 0 { - path = path[:i] - } - switch method { - case http.MethodPost: - return issueCreateRESTPathRE.MatchString(path) || - issueCommentCreateRESTPath.MatchString(path) - case http.MethodPatch: - return issueEditRESTPathRE.MatchString(path) || - issueCommentEditRESTPath.MatchString(path) - default: - return false - } -} - -// githubIssueShapeViolation returns a non-empty, agent-actionable reason when a -// buffered issue/comment REST body carries an unsubstituted template -// placeholder or literal newline escape sequences. A body that is not the JSON -// we expect is left to the upstream to reject — auth was already enforced. -func githubIssueShapeViolation(method, path string, body []byte) string { - if !isIssueShapeRESTWrite(method, path) { - return "" - } - var payload struct { - Title *string `json:"title"` - Body *string `json:"body"` - } - if err := json.Unmarshal(body, &payload); err != nil { - return "" - } - if payload.Title != nil { - if ph, ok := issueshape.UnsubstitutedTemplatePlaceholder(*payload.Title); ok { - return "issue title contains unsubstituted template placeholder " + strconv.Quote(ph) + "; fill in the template before creating the issue" - } - } - if payload.Body != nil { - if ph, ok := issueshape.UnsubstitutedTemplatePlaceholder(*payload.Body); ok { - return "issue body contains unsubstituted template placeholder " + strconv.Quote(ph) + "; fill in the template before creating the issue" - } - if issueshape.BodyHasMisEscapedNewlines(*payload.Body) { - return "issue body contains literal newline escape sequences; use real newlines or --body-file" - } - } - return "" -} - -// bodyReadCloser rejoins an already-read prefix with the unread remainder of a -// request body so an oversized issue write can be forwarded intact. -type bodyReadCloser struct { - io.Reader - io.Closer -} - -// enforceIssueShape is enforceIssueShapeFor with no agent identity, so no lane -// label is injected. -func (p *GitHubProxy) enforceIssueShape(req *http.Request) (reason string, deny bool, readErr error) { - return p.enforceIssueShapeFor(req, "") -} - -// enforceIssueShapeFor buffers an agent's issue/comment REST body, validates -// its shape, and reconstructs req.Body so the request can still be forwarded -// when it passes. It returns deny=true with an actionable reason for a -// malformed body, and a non-nil error only when the body could not be read -// (fail closed). A well-formed issue create from an identified agent also -// gets its agent/ provenance label injected (see injectOriginLabel). -func (p *GitHubProxy) enforceIssueShapeFor(req *http.Request, agentName string) (reason string, deny bool, readErr error) { - if req.Body == nil || !isIssueShapeRESTWrite(req.Method, req.URL.Path) { - return "", false, nil - } - body, err := io.ReadAll(io.LimitReader(req.Body, issueShapeBodyLimit+1)) - if err != nil { - _ = req.Body.Close() - return "", false, err - } - if int64(len(body)) > issueShapeBodyLimit { - // Oversized: forward the original body unbuffered and unchecked rather - // than corrupt it by truncation. Stitch the prefix we already consumed - // back in front of the unread remainder; ContentLength is unchanged. - req.Body = bodyReadCloser{Reader: io.MultiReader(bytes.NewReader(body), req.Body), Closer: req.Body} - return "", false, nil - } - _ = req.Body.Close() - if v := githubIssueShapeViolation(req.Method, req.URL.Path, body); v != "" { - req.Body = io.NopCloser(bytes.NewReader(body)) - req.ContentLength = int64(len(body)) - return v, true, nil - } - if rewritten, ok := injectOriginLabel(req.Method, req.URL.Path, body, agentName); ok { - body = rewritten - } - req.Body = io.NopCloser(bytes.NewReader(body)) - req.ContentLength = int64(len(body)) - return "", false, nil -} - -// isIssueCreateRESTWrite reports whether (method, path) is POST -// /repos/{o}/{r}/issues — the one route whose body creates an issue. -func isIssueCreateRESTWrite(method, path string) bool { - if i := strings.IndexByte(path, '?'); i >= 0 { - path = path[:i] - } - return method == http.MethodPost && issueCreateRESTPathRE.MatchString(path) -} - -// injectOriginLabel ensures an agent's issue-create JSON body carries the -// canonical agent/ label and no redundant origin/ or bare -// label, so an issue filed with `gh issue create` through the proxy records -// the same provenance as one filed through the issue-request watcher. The -// body is returned unchanged (ok=false) for other routes, an unidentified -// caller, a body that is not a JSON object, or a labels field in an unexpected -// shape (GitHub accepts label objects too; those are forwarded as written -// rather than guessed at). -func injectOriginLabel(method, path string, body []byte, agentName string) ([]byte, bool) { - lane := config.OriginLaneFromAgent(agentName) - if lane == "" || agentName == internalCallerName || !isIssueCreateRESTWrite(method, path) { - return body, false - } - var payload map[string]json.RawMessage - if err := json.Unmarshal(body, &payload); err != nil || payload == nil { - return body, false - } - var labels []string - if raw, present := payload["labels"]; present && len(bytes.TrimSpace(raw)) > 0 && string(bytes.TrimSpace(raw)) != "null" { - if err := json.Unmarshal(raw, &labels); err != nil { - return body, false - } - } - filtered := config.WithIssueOriginLabel(labels, lane) - if sameLabels(labels, filtered) { - return body, false - } - labels = filtered - encoded, err := json.Marshal(labels) - if err != nil { - return body, false - } - payload["labels"] = encoded - rewritten, err := json.Marshal(payload) - if err != nil { - return body, false - } - return rewritten, true -} - -func sameLabels(a, b []string) bool { - if len(a) != len(b) { - return false - } - for i := range a { - if a[i] != b[i] { - return false - } - } - return true -} - -// internalCallerName attributes a connection that originated from the hive's -// OWN control plane rather than from a sandboxed agent. It is deliberately not -// a valid agent name (agents come from the UID map) so it can never collide -// with one, and it is what distinguishes "the hive called GitHub" from the -// genuine failure mode the mode gate warns about: "an agent's UID could not be -// attributed". See UIDMap.IsInternalUID for the security argument. -const internalCallerName = "hive-internal" - -func (p *GitHubProxy) recordViolation(agentName, method, path string) { - if p.firstBlock(agentName, method, path) { - p.logger.Warn("proxy request blocked", "agent", agentName, "method", method, "path", path) - } else { - p.logger.Debug("proxy request blocked (repeat)", "agent", agentName, "method", method, "path", path) - } - p.countViolation(agentName) -} - -// firstBlock reports whether this (agent, method, path) refusal has not been -// logged before. Once the set is full, new keys keep logging at WARN. -func (p *GitHubProxy) firstBlock(agentName, method, path string) bool { - key := agentName + "|" + method + "|" + path - p.mu.Lock() - defer p.mu.Unlock() - if _, seen := p.blockedSeen[key]; seen { - return false - } - if p.blockedSeen == nil { - p.blockedSeen = make(map[string]struct{}) - } - if len(p.blockedSeen) < maxViolationLog { - p.blockedSeen[key] = struct{}{} - } - return true -} - -// countViolation increments the per-agent refusal counter without logging, for -// callers that have already logged the refusal with their own detail (see -// refusePlainForward). The map is bounded by maxViolationLog so an -// unidentified or churning caller set cannot grow it without limit. -func (p *GitHubProxy) countViolation(agentName string) { - p.mu.Lock() - defer p.mu.Unlock() - if _, exists := p.violations[agentName]; exists || len(p.violations) < maxViolationLog { - p.violations[agentName]++ - } -} - -func isGitPath(path string) bool { - return strings.HasSuffix(path, "/git-receive-pack") || - strings.HasSuffix(path, "/git-upload-pack") || - strings.HasSuffix(path, "/info/refs") -} - -// tunnelDirect creates a raw TCP tunnel for non-GitHub CONNECT requests. -func (p *GitHubProxy) tunnelDirect(conn net.Conn, r *http.Request) { - const tunnelDialTimeout = 10 * time.Second - // SO_MARK the socket so the forced-egress redirect exempts this - // proxy-originated tunnel dial (CONNECT targets include :443). - upstream, err := markDialer(tunnelDialTimeout).Dial("tcp", r.Host) - if err != nil { - p.logger.Warn("proxy: CONNECT dial failed", "host", r.Host, "error", err) - _, _ = fmt.Fprintf(conn, "HTTP/1.1 502 Bad Gateway\r\n\r\nconnection failed\n") - return - } - defer func() { _ = upstream.Close() }() - - _, _ = fmt.Fprintf(conn, "HTTP/1.1 200 Connection established\r\n\r\n") - - relayTunnel(conn, upstream) -} - -// responseBodyStallTimeout bounds how long the MITM'd response-BODY relay may -// go without a single byte of progress from the upstream. It is a rolling -// per-Read bound (re-armed on every read by stallBoundedBody), not a cap on -// total transfer time, so a large-but-flowing response is never cut while an -// upstream that goes silent mid-body releases the handler — and with it the -// three sockets of the exchange — within one stall window. Overridable, not a -// const, so tests can shorten it — and atomic for the same reason as -// tunnelHalfCloseDrain (#5553): the reading goroutine can outlive the test -// that started it. A handler test that returns after a fixed sleep leaves -// proxyHTTP still parsing the response when the NEXT test's shortenBodyStall -// writes the bound — a plain var made that an unsynchronised write-after-read -// (-race failure on v4, #6110, reported against whichever test held the -// baton). Defaults to httpReadTimeout: the same patience already extended to -// the header phase. -var responseBodyStallTimeout atomic.Int64 - -func init() { responseBodyStallTimeout.Store(int64(httpReadTimeout)) } - -// bodyStallTimeout returns the current response-body stall bound. -func bodyStallTimeout() time.Duration { - return time.Duration(responseBodyStallTimeout.Load()) -} - -// bodyStallTimeoutDuration returns the body-stall timeout for this proxy instance, -// falling back to the package-level responseBodyStallTimeout. -func (p *GitHubProxy) bodyStallTimeoutDuration() time.Duration { - if p != nil && p.bodyStallTimeout > 0 { - return p.bodyStallTimeout - } - return bodyStallTimeout() -} - -// stallBoundedBody wraps a proxied response body so every underlying Read is -// preceded by re-arming a read deadline on the upstream conn. Close clears the -// deadline (for the next keep-alive exchange on the same conn) and closes the -// wrapped body. See responseBodyStallTimeout for why this exists (#3875). -type stallBoundedBody struct { - body io.ReadCloser - conn net.Conn - idle time.Duration -} - -func (b *stallBoundedBody) Read(p []byte) (int, error) { - _ = b.conn.SetReadDeadline(time.Now().Add(b.idle)) - return b.body.Read(p) -} - -func (b *stallBoundedBody) Close() error { - // Close the wrapped body while a stall deadline is armed: net/http body - // Close paths may still read from the conn (drain-to-EOF), and an unarmed - // read there would reintroduce the very stall this type exists to bound. - // The deadline is cleared afterwards so the next keep-alive exchange on - // this conn starts unencumbered. - _ = b.conn.SetReadDeadline(time.Now().Add(b.idle)) - err := b.body.Close() - _ = b.conn.SetReadDeadline(time.Time{}) - return err -} - -// stallBoundedWriter is the write-side mirror of stallBoundedBody: every Write -// re-arms the client's write deadline, so the bound is "no progress for idle" -// rather than "the whole relay must finish within idle". -// -// The distinction is not academic. The response relays used to arm a single -// absolute client.SetWriteDeadline(now+httpWriteTimeout) around -// resp.Write(client), which is correct for a small buffered body and wrong for -// a stream: the MITM'd /v1/messages SSE relay holds one Write loop open for the -// entire generation, so a turn that legitimately streams for longer than -// httpWriteTimeout was severed mid-stream at exactly that mark — even though -// tokens were flowing the whole time. The agent saw a truncated body -// ("Response was interrupted due to a server error", or a TLS-level "cannot -// decrypt peer's message" when the cut landed mid-record), retried, and — -// because a retry replays the same long turn — was cut again at the same 60s, -// livelocking the agent on its longest and most valuable turns. Measured on the -// hosted spoke: 23 "response relay timed out" in 40 minutes, every one -// path=/v1/messages with error "write tcp ...: i/o timeout", against an agent -// that made zero forward progress across six consecutive attempts. -// -// Reusing the stall semantics already proven on the read side (#3875) keeps a -// client that has genuinely gone away bounded — a dead peer stops draining, the -// socket buffer fills, and the next Write blocks past idle and errors — while a -// stream that keeps flowing is never cut for being long. -type stallBoundedWriter struct { - conn net.Conn - idle time.Duration -} - -func (w *stallBoundedWriter) Write(p []byte) (int, error) { - _ = w.conn.SetWriteDeadline(time.Now().Add(w.idle)) - return w.conn.Write(p) -} - -// tunnelHalfCloseDrainDefault bounds how long one direction of an opaque -// tunnel may keep waiting for bytes AFTER the other direction has already -// finished. -// -// 30s is deliberately generous: the only legitimate traffic still in flight -// once one side has hung up is a response tail already on the wire, which -// arrives in RTTs, not tens of seconds. -const tunnelHalfCloseDrainDefault = 30 * time.Second - -// tunnelHalfCloseDrain is overridable rather than const for the same reason as -// waitForReadyPollInterval in pkg/hub: tests need a relay that unwedges in -// milliseconds. Production keeps the default. -// -// It is atomic because the write and the read genuinely happen on different -// goroutines with no happens-before between them. relayTunnel reads the bound -// from the copy goroutine it spawns, and that goroutine can outlive the test -// that started it: a handler test only waits for the CONNECT response, not for -// the relay to drain, so the goroutine is still reading while the NEXT test's -// shortenTunnelDrain writes. A plain var made that an unsynchronised -// write-after-read and -race failed whichever test happened to hold the baton -// (#5553) — the reported test name was incidental, which is why the failure -// moved between TestRelayTunnelUnwedgesFromSilentUpstream and -// TestRelayTunnelDataFlowsBothWays across shuffle seeds. -var tunnelHalfCloseDrain atomic.Int64 - -func init() { tunnelHalfCloseDrain.Store(int64(tunnelHalfCloseDrainDefault)) } - -// tunnelDrain returns the current half-close drain bound. -func tunnelDrain() time.Duration { - return time.Duration(tunnelHalfCloseDrain.Load()) -} - -// relayTunnel shuttles bytes both ways between the proxied client conn and the -// upstream until BOTH directions have finished, bounding each direction once -// the other has ended. -// -// WHY THE BOUNDS EXIST (measured live, 2026-08-14): the previous inline copies -// waited on each other unboundedly. On spokes whose forge host -// (github.ibm.com) sits behind a SYN-accepting firewall, the upstream TCP -// connect succeeds but no byte ever comes back and no FIN is ever sent. The -// client (the daemon's own JWT mints, agents' CLIs) gives up at its TLS -// handshake timeout and closes — the client→upstream copy finishes and -// half-closes the upstream — but io.Copy(conn, upstream) stayed blocked in -// upstream.Read() FOREVER. Every attempt leaked two sockets (the FIN_WAIT2 / -// CLOSE_WAIT piles), the goroutine, and the splice pipe pair io.Copy holds for -// a TCP↔TCP copy on Linux: ~300k fds and ~11 burned cores per affected spoke. -// -// SetReadDeadline is documented to interrupt Reads that are ALREADY blocked, -// which is exactly what arms the escape hatch here: each side arms the OTHER -// side's deadline only at the moment its own direction completes, so a -// healthy long-lived tunnel (git smart HTTP, device-flow polling) is never -// cut while both directions are still live. -func relayTunnel(conn, upstream net.Conn) { - done := make(chan struct{}) - go func() { - _, _ = io.Copy(upstream, conn) - if tc, ok := upstream.(*net.TCPConn); ok { - _ = tc.CloseWrite() - } - // Client side is done (EOF or error — a dead client looks the same). - // A live upstream answers or closes within RTTs; a blackholed one - // never would, so bound the remaining upstream→client read. - _ = upstream.SetReadDeadline(time.Now().Add(tunnelDrain())) - close(done) - }() - _, _ = io.Copy(conn, upstream) - // Mirror image: upstream finished (or errored) but the client may sit - // half-open without ever sending EOF; bound the client-side read so - // <-done cannot wedge this handler. - _ = conn.SetReadDeadline(time.Now().Add(tunnelDrain())) - <-done -} - -func transfer(dst, src net.Conn) { - defer func() { _ = dst.Close() }() - defer func() { _ = src.Close() }() - _, _ = io.Copy(dst, src) -} - -// forwardPlainDirect handles non-CONNECT (plain HTTP) requests on a raw -// connection. -// -// Two policies apply here that the CONNECT path has always applied and this -// one did not: the caller is identified from the connection, and the -// destination must be globally routable (see plain_forward_guard.go). Before -// that, any process that could reach the proxy port could have it open a -// connection, from the proxy's own uid, to any address it named — including -// the hive's loopback services, the cloud metadata endpoint and the -// cluster-internal estate — and nothing in the log said who asked. -func (p *GitHubProxy) forwardPlainDirect(conn net.Conn, r *http.Request) { - // Identify from the connection, NOT from r.RemoteAddr: this request came - // from http.ReadRequest, which leaves RemoteAddr empty (see - // identifyAgentFromConn). "" means the caller could not be attributed, - // which is logged as such rather than guessed at. - agentName := p.identifyAgentFromConn(conn, r) - - // Proxy-* hop headers address THIS proxy and must not travel further: the - // identity they assert is the one identifyAgentFromConn just decided - // whether to believe, and forwarding it would hand an agent's claimed name - // (and, in the Basic form, whatever it put in the password field) to every - // plain-HTTP host it can name. - r.Header.Del("Proxy-Authorization") - r.Header.Del("Proxy-Connection") - - resp, err := p.plainForwardTransport().RoundTrip(r) - if err != nil { - var denied *plainForwardDenied - if errors.As(err, &denied) { - p.refusePlainForward(conn, r, agentName, denied) - return - } - if isLoopbackHost(r.URL.Host) { - p.logger.Debug("proxy: plain HTTP forward to loopback failed", "agent", agentName, "url", r.URL.String(), "error", err, "hint", "client should bypass the proxy for loopback; NO_PROXY") - } else { - p.logger.Warn("proxy: plain HTTP forward failed", "agent", agentName, "url", r.URL.String(), "error", err) - } - _, _ = fmt.Fprintf(conn, "HTTP/1.1 502 Bad Gateway\r\n\r\nupstream request failed\n") - return - } - defer func() { _ = resp.Body.Close() }() - _ = resp.Write(conn) -} - -// refusePlainForward answers a destination the guard barred. -// -// Loopback is treated as the misconfiguration it almost always is rather than -// as an attack: agents are handed NO_PROXY=127.0.0.1,localhost,::1 (#14076), -// so a loopback request arriving here means a client ignored it. It is logged -// at Debug with the same hint as the failure path and does NOT count as a -// violation. Every other class — link-local (cloud metadata), private, CGNAT, -// multicast — is an attempt to reach the estate behind the proxy and is -// counted against the caller. -func (p *GitHubProxy) refusePlainForward(conn net.Conn, r *http.Request, agentName string, denied *plainForwardDenied) { - if denied.reason == plainForwardReasonLoopback { - p.logger.Debug("proxy: plain HTTP forward to loopback refused (clients must bypass the proxy for loopback; NO_PROXY)", - "agent", agentName, "url", r.URL.String(), "target", denied.target) - } else { - p.logger.Warn("proxy: plain HTTP forward refused — destination is not globally routable", - "agent", agentName, "method", r.Method, "url", r.URL.String(), "target", denied.target, "class", denied.reason) - p.countViolation(agentName) - } - _, _ = fmt.Fprintf(conn, "HTTP/1.1 403 Forbidden\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n"+ - "the hive proxy does not forward plain HTTP to %s addresses (%s); set %s=true to allow it\n", - denied.reason, denied.target, plainForwardAllowPrivateEnv) -} - -// isLoopbackHost reports whether a host or host:port refers to the loopback interface. -func isLoopbackHost(hostport string) bool { - host := hostport - if h, _, err := net.SplitHostPort(hostport); err == nil { - host = h - } - host = strings.Trim(host, "[]") - if strings.EqualFold(host, "localhost") { - return true - } - ip := net.ParseIP(host) - return ip != nil && ip.IsLoopback() -} - -// extractAgentName reads the agent name from the Proxy-Authorization header. -// Supports "hive " (custom) and "Basic " (standard HTTP proxy auth -// sent automatically when the proxy URL contains userinfo, e.g. http://quality@host:port). -func extractAgentName(r *http.Request) string { - auth := r.Header.Get("Proxy-Authorization") - if auth == "" { - return "" - } - if strings.HasPrefix(auth, "hive ") { - return strings.TrimPrefix(auth, "hive ") - } - if strings.HasPrefix(auth, "Basic ") { - decoded, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(auth, "Basic ")) - if err != nil { - return "" - } - // Format is "username:password" — agent name is the username portion. - user, _, _ := strings.Cut(string(decoded), ":") - return user - } - return "" -} - -// readAgentMode reads the mode from the hot-reloadable mode file. -// readAgentCaps loads the agent's orthogonal capability set (#4492) from -// /tmp/.hive-caps-. Every failure path returns the zero value — no -// capabilities — so an unreadable, missing or malformed file degrades toward -// the tier check alone, which is the deny direction. -func readAgentCaps(agentName string) agent.AgentCapabilities { - if agentName == "" { - return agent.AgentCapabilities{} - } - data, err := os.ReadFile(capsFilePrefix + agentName) - if err != nil { - return agent.AgentCapabilities{} - } - return agent.ParseCapabilities(strings.TrimSpace(string(data))) -} - -func readAgentMode(agentName string) agent.AgentMode { - if agentName == "" { - return agent.ModeAdvisory - } - data, err := os.ReadFile(modeFilePrefix + agentName) - if err != nil { - return agent.ModeAdvisory - } - mode, ok := agent.ParseAgentMode(strings.TrimSpace(string(data))) - if !ok { - return agent.ModeAdvisory - } - return mode -} - -// forgeCert generates a TLS certificate for the given hostname, -// signed by the proxy's CA. -func (p *GitHubProxy) forgeCert(host string) (tls.Certificate, error) { - p.certMu.RLock() - if cached, ok := p.certCache[host]; ok && time.Now().Before(cached.expiresAt) { - p.certMu.RUnlock() - return cached.cert, nil - } - p.certMu.RUnlock() - - key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - return tls.Certificate{}, err - } - - serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) - if err != nil { - return tls.Certificate{}, err - } - - template := &x509.Certificate{ - SerialNumber: serial, - Subject: pkix.Name{CommonName: host}, - NotBefore: time.Now().Add(-time.Hour), - NotAfter: time.Now().Add(24 * time.Hour), - KeyUsage: x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - DNSNames: []string{host}, - } - - caKey, ok := p.caCert.PrivateKey.(*ecdsa.PrivateKey) - if !ok { - return tls.Certificate{}, fmt.Errorf("CA key is not ECDSA") - } - - certDER, err := x509.CreateCertificate(rand.Reader, template, p.caX509, &key.PublicKey, caKey) - if err != nil { - return tls.Certificate{}, err - } - - certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}) - keyDER, err := x509.MarshalECPrivateKey(key) - if err != nil { - return tls.Certificate{}, err - } - keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) - - cert, err := tls.X509KeyPair(certPEM, keyPEM) - if err != nil { - return cert, err - } - // Include the CA cert in the chain so clients that load system/extra - // certs asynchronously (e.g. GitHub Copilot's bundled undici) can - // verify the forged leaf without having the CA pre-loaded in their - // trust store. - cert.Certificate = append(cert.Certificate, p.caX509.Raw) - - p.certMu.Lock() - if p.certCache == nil { - p.certCache = make(map[string]cachedCert) - } - p.certCache[host] = cachedCert{cert: cert, expiresAt: time.Now().Add(time.Hour)} - p.certMu.Unlock() - - return cert, nil -} - -// generateCA creates a self-signed CA certificate for MITM. -func generateCA() (tls.Certificate, *x509.Certificate, error) { - key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - return tls.Certificate{}, nil, err - } - - serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) - if err != nil { - return tls.Certificate{}, nil, err - } - - template := &x509.Certificate{ - SerialNumber: serial, - Subject: pkix.Name{CommonName: "Hive ACMM Proxy CA"}, - NotBefore: time.Now().Add(-time.Hour), - NotAfter: time.Now().Add(365 * 24 * time.Hour), - KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, - BasicConstraintsValid: true, - IsCA: true, - MaxPathLen: 0, - } - - certDER, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) - if err != nil { - return tls.Certificate{}, nil, err - } - - certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}) - keyDER, err := x509.MarshalECPrivateKey(key) - if err != nil { - return tls.Certificate{}, nil, err - } - keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) - - tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) - if err != nil { - return tls.Certificate{}, nil, err - } - - x509Cert, err := x509.ParseCertificate(certDER) - if err != nil { - return tls.Certificate{}, nil, err - } - - return tlsCert, x509Cert, nil -} - -// loadOrGenerateCA reuses an existing CA key pair from disk if present and -// valid, or generates a fresh one and persists both cert and key to the PVC. -// Reusing the CA across pod restarts means the system trust store (populated -// by entrypoint.sh before the Go binary starts) already contains the right -// CA, so clients that load system certs asynchronously (GitHub Copilot's -// bundled undici) can verify forged certificates immediately. -func loadOrGenerateCA(logger *slog.Logger) (tls.Certificate, *x509.Certificate, error) { - certPEM, certErr := os.ReadFile(CACertPath) - keyPEM, keyErr := os.ReadFile(caKeyPath) - if certErr == nil && keyErr == nil { - tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) - if err == nil { - block, _ := pem.Decode(certPEM) - if block != nil { - x509Cert, err := x509.ParseCertificate(block.Bytes) - if err == nil && time.Now().Before(x509Cert.NotAfter) { - logger.Info("reusing persisted MITM CA", "expires", x509Cert.NotAfter) - return tlsCert, x509Cert, nil - } - } - } - logger.Warn("persisted CA unusable, generating fresh one", "err", err) - } - - tlsCert, x509Cert, err := generateCA() - if err != nil { - return tls.Certificate{}, nil, err - } - - caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: x509Cert.Raw}) - if err := os.WriteFile(CACertPath, caPEM, 0644); err != nil { - return tls.Certificate{}, nil, fmt.Errorf("write CA cert to %s: %w", CACertPath, err) - } - - caKeyDER, err := x509.MarshalECPrivateKey(tlsCert.PrivateKey.(*ecdsa.PrivateKey)) - if err != nil { - return tls.Certificate{}, nil, fmt.Errorf("marshal CA key: %w", err) - } - caKeyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: caKeyDER}) - // The parent directory is created with owner-only permissions before the - // key is written. /data itself is intentionally group-writable for agent - // workspaces, so 0600 on a file directly under /data is not sufficient: - // an agent could replace or race the path before the proxy opens it. - if err := os.MkdirAll(filepath.Dir(caKeyPath), 0700); err != nil { - return tls.Certificate{}, nil, fmt.Errorf("create CA key directory: %w", err) - } - if err := os.Chmod(filepath.Dir(caKeyPath), 0700); err != nil { - return tls.Certificate{}, nil, fmt.Errorf("restrict CA key directory: %w", err) - } - if err := os.WriteFile(caKeyPath, caKeyPEM, 0600); err != nil { - return tls.Certificate{}, nil, fmt.Errorf("write CA key to %s: %w", caKeyPath, err) - } - - logger.Info("generated and persisted fresh MITM CA", "certPath", CACertPath, "keyPath", caKeyPath) - return tlsCert, x509Cert, nil -} - -func newBufferedConn(c net.Conn) *bufio.Reader { - return bufio.NewReader(c) +func newBufferedConn(c net.Conn) *bufio.Reader { + return bufio.NewReader(c) } func newBufferedReader(c net.Conn) *bufio.Reader { return bufio.NewReader(c) } -// SetInferenceRoute configures an agent to use a self-hosted inference backend. -// -// The route is stored immediately and the caller returns without blocking. The -// model's max context length is queried in the background because this function -// is invoked from callers that hold the agent-manager mutex (agent launch, -// SetModelOverride, SetBackendOverride). A synchronous endpoint probe here -// (up to maxModelLenQueryTimeout) would pin that mutex for its whole duration, -// stalling /api/status and other dashboard handlers — which on a single-replica -// pod reads to the OpenShift router as "Application is not available". The -// max-context-len only feeds token capping and defaults gracefully to 0 -// (no cap) until the async probe fills it in. -func (p *GitHubProxy) SetInferenceRoute(agentName string, route *InferenceRoute) { - p.inference.Set(agentName, route) - keySHA256 := config.APIKeySHA256(route.APIKey) - if keySHA256 != "" { - p.logger.Info("inference route set", "agent", agentName, "backend", route.Backend, "endpoint", route.Endpoint, "model", route.Model, "maxContextLen", route.MaxContextLen, "api_key_sha256", keySHA256) - } else { - p.logger.Info("inference route set", "agent", agentName, "backend", route.Backend, "endpoint", route.Endpoint, "model", route.Model, "maxContextLen", route.MaxContextLen, "api_key_sha256", "") - } - - if route.MaxContextLen == 0 { - endpoint, model, apiKey, caBundle := route.Endpoint, route.Model, route.APIKey, route.CABundle - go func() { - if maxLen := queryMaxModelLen(endpoint, model, apiKey, caBundle); maxLen > 0 { - if p.inference.UpdateMaxContextLen(agentName, endpoint, model, maxLen) { - p.logger.Info("inference route max context length resolved", "agent", agentName, "model", model, "maxContextLen", maxLen) - } - } - }() - } - - // Best-effort: learn the key's entitled model set from a LiteLLM - // key-info endpoint before any 403 happens, so the dashboard can offer - // only usable models up front. Only litellm gateways entitlement-filter - // per key; vllm/llm-d do not. The 403-parse path (recordInferenceError) - // remains the authoritative fallback when no key-info endpoint responds. - if route.Backend == "litellm" && route.APIKey != "" { - p.entitlements.rememberProbe(route.Endpoint, route.APIKey, route.CABundle) - go p.probeEntitlements(route.Endpoint, route.APIKey, route.CABundle) - } -} - -// probeEntitlements queries a LiteLLM gateway's per-key info endpoints for the -// key's entitled model set and caches it. A nil result (no key-info endpoint, -// or an unrestricted key) leaves any existing 403-derived entry untouched. -func (p *GitHubProxy) probeEntitlements(endpoint, apiKey, caBundle string) { - if !p.entitlements.beginProbe(endpoint) { - return // another probe for this endpoint is already in flight - } - defer p.entitlements.endProbe(endpoint) - transport, err := inferenceTransport(caBundle) - if err != nil { - transport = nil - } - if models := probeEntitledModels(endpoint, apiKey, transport); len(models) > 0 { - p.entitlements.set(endpoint, apiKey, models, "key-info") - p.logger.Info("litellm entitled models discovered via key-info", - "endpoint", endpoint, "count", len(models)) - } -} - -// EntitledModels returns the entitled model set known for a LiteLLM endpoint -// (from a key-info probe or a "team not allowed" 403), the signal source, and -// whether the set is known. It is the accessor the dashboard uses to narrow -// model dropdowns to what the configured key can actually use. -// -// A stale entry (older than entitlementTTL) is still returned so the served -// model list stays STABLE for an unchanged key — a filtered list that flapped -// back to the full catalog between probes would make the dashboard's -// auto-heal (see reconcileModelsAfterDiscovery, #1848) see models reappear -// and vanish across refreshes. Staleness instead triggers a bounded -// background re-probe that overwrites the entry only on a fresh signal. -func (p *GitHubProxy) EntitledModels(endpoint string) (models []string, source string, known bool) { - models, source, known, stale := p.entitlements.get(endpoint) - if stale { - if ep, apiKey, caBundle, ok := p.entitlements.probeParams(endpoint); ok { - go p.probeEntitlements(ep, apiKey, caBundle) - } - } - return models, source, known -} - -// recordInferenceError inspects an upstream inference error body for a -// LiteLLM "team not allowed to access model" 403 and, when found, caches the -// entitled model set it lists (keyed by the route's endpoint) so the -// dashboard's model list narrows to the usable set. Only a 403 carrying the -// gateway's team-scope marker is treated as an entitlement signal — -// provider-side failures relayed by the gateway (upstream 404 not-found, 400 -// bad-request, 5xx) say nothing about the key's scope and are ignored, as is -// any other 403 (bad key, rate limit). -// -// It deliberately does NOT switch the agent's route itself: automatic model -// switching is governed by the dashboard's reconcile policy (#1848 — unpinned -// + governor suggestion, or confirmed genuine unavailability, at most once -// per list change). Once the entitled set is cached here, the dashboard's -// filtered discovery list drives that heal path for any agent left on a -// non-entitled model. -func (p *GitHubProxy) recordInferenceError(route *InferenceRoute, agentName string, status int, body []byte) { - if route == nil { - return - } - if p.gatewayHealth != nil { - p.gatewayHealth.RecordEndpointHTTPError(route.Backend, route.Endpoint, status, string(truncateBytes(body, 200)), time.Now()) - } - - // Inference-backend AUTH failure (a stale/invalid gateway key returning 401 - // on every call). Tracked for ANY inference backend, not just litellm — a - // self-hosted vLLM/llm-d fronted by auth can 401 too. Latches only after - // inferenceAuthFailureThreshold consecutive failures and clears on the next - // success, so a single transient blip never alarms. The stored cause is - // log-safe: the gateway's 401 body describes the rejection without echoing - // the presented token, and inferenceAuthErrorMessage never reads the key. - if isInferenceAuthStatus(status) && p.inferenceAuth != nil { - msg := inferenceAuthErrorMessage(route.Backend, status, truncateBytes(body, 200)) - p.inferenceAuth.recordFailure(msg, time.Now()) - p.logger.Warn("inference backend auth failure", - "agent", agentName, "backend", route.Backend, "endpoint", route.Endpoint, "status", status) - } - - // PROVIDER SPEND REBUFF (#4294). Checked before the 403/entitlement branch - // below because that branch returns early for every non-403, which would - // skip the 429/400 statuses a budget refusal actually arrives on. - // - // Requires the body to name spend, so an ordinary 429 throttle keeps - // flowing to the existing transient-retry path untouched. - if p.inferenceBudget != nil && isInferenceBudgetRebuff(status, body) { - msg := inferenceBudgetMessage(route.Backend, status, truncateBytes(body, 200)) - p.inferenceBudget.recordRebuff(msg, time.Now()) - p.logger.Error("inference backend refused on a spending limit", - "agent", agentName, "backend", route.Backend, "endpoint", route.Endpoint, - "status", status, "cause", msg) - } - - if route.Backend != "litellm" || status != http.StatusForbidden { - return - } - entitled := parseTeam403Models(string(body)) - if len(entitled) == 0 { - return - } - p.entitlements.set(route.Endpoint, route.APIKey, entitled, "403") - p.logger.Info("litellm entitled models learned from 403", - "agent", agentName, "endpoint", route.Endpoint, "count", len(entitled), "model", route.Model) -} - -// routeWithEntitledModel returns the route to actually forward with: when the -// gateway's entitled model set is known (key-info probe or a prior team-scope -// 403) and the route's stored model differs from an entitled id only by -// provider-prefix / dot-hyphen spelling drift, a COPY of the route carrying -// the exact entitled id is returned. LiteLLM matches the request model string -// verbatim, so without this a stored copilot-spelled id (e.g. -// "claude-sonnet-4.6" vs entitled "aws/claude-sonnet-4-6") 403s on every call -// forever (#4400) — while the dashboard's tolerant preselect matcher shows the -// agent on a perfectly valid model. Self-healing: the first 403 teaches the -// entitled set (recordInferenceError), so the next request resolves. -// -// When the stored model is NOT entitled at all — verbatim or by drift — and -// the backend's DefaultModel IS, the request goes out for the default instead -// (hivecommons/hive#9945). The FMA hive's agents carried a `model: -// claude-sonnet-4-6` written for their original copilot backend into a -// switch to the litellm gateway whose default_model was the only model the -// team could use; the per-agent value won, and every call 403'd forever with -// "team not allowed to access model" while the operator stared at a gateway -// card showing the right default. The gateway's own entitlement signal (the -// key-info probe or that very 403) is what authorises the substitution, so a -// model the team can't use is never sent when one it can use is configured. -// The stored route is still untouched: a key/team change that re-entitles the -// per-agent model takes effect on the next request. -// -// The stored route is never mutated: the rewrite is per-request, so a -// key/team change that re-entitles the verbatim id takes effect immediately, -// and concurrent requests race on nothing. Unknown entitlements, verbatim -// matches, and ambiguous candidates all pass the route through unchanged. -func (p *GitHubProxy) routeWithEntitledModel(route *InferenceRoute, agentName string) *InferenceRoute { - if route == nil || route.Backend != "litellm" || p.entitlements == nil { - return route - } - entitled, _, known := p.EntitledModels(route.Endpoint) - if !known { - return route - } - if entitledModelListed(entitled, route.Model) { - return route - } - if resolved, ok := resolveEntitledModelID(entitled, route.Model); ok { - p.logger.Info("inference model resolved to entitled gateway id", - "agent", agentName, "endpoint", route.Endpoint, - "configured", route.Model, "resolved", resolved) - rewritten := *route - rewritten.Model = resolved - return &rewritten - } - fallback := strings.TrimSpace(route.DefaultModel) - if fallback == "" || fallback == route.Model { - return route - } - if !entitledModelListed(entitled, fallback) { - resolved, ok := resolveEntitledModelID(entitled, fallback) - if !ok { - return route - } - fallback = resolved - } - p.logger.Warn("inference model not entitled on gateway; sending the backend default model instead", - "agent", agentName, "backend", route.Backend, "endpoint", route.Endpoint, - "configured", route.Model, "default", fallback, - "remedy", "clear the agent's model (or pick one of the gateway's entitled models) so the agent uses the gateway default") - rewritten := *route - rewritten.Model = fallback - return &rewritten -} - -// entitledModelListed reports whether model appears verbatim in entitled. -func entitledModelListed(entitled []string, model string) bool { - for _, id := range entitled { - if id == model { - return true - } - } - return false -} - -// recordInferenceSuccess records a successful inference call so a previously -// latched auth-failure signal clears — the self-heal for the inference-auth -// health signal. Called from every inference forward path on a 2xx response. -// A nil tracker (impossible after NewGitHubProxy, but cheap to guard) is a -// no-op. -func (p *GitHubProxy) recordInferenceSuccessFor(route *InferenceRoute) { - if route != nil && p.gatewayHealth != nil { - p.gatewayHealth.Clear(route.Backend) - } - p.recordInferenceSuccess() -} - -func (p *GitHubProxy) recordInferenceSuccess() { - if p.inferenceAuth != nil { - p.inferenceAuth.recordSuccess() - } - // Self-heal for the spend rebuff (#4294): whenever the provider's window - // resets — hive neither knows nor assumes the schedule; the field report's - // gateway rolled over at the key's creation time of day, not at midnight - // (see #4294) — the first call that succeeds takes the hive out of the - // suppressed state with no operator action. - if p.inferenceBudget != nil { - p.inferenceBudget.recordSuccess() - } -} - -// GatewayHealth reports currently failing inference gateways. -func (p *GitHubProxy) GatewayHealth() []inferencehealth.GatewayStatus { - if p == nil || p.gatewayHealth == nil { - return nil - } - return p.gatewayHealth.Snapshot() -} - -// InferenceAuthError reports the current inference-backend auth-failure signal: -// a non-empty, log-safe cause string (and the time it first latched) ONLY while -// the backend has been auth-failing for inferenceAuthFailureThreshold -// consecutive calls, empty otherwise. Wired to the dashboard/heartbeat so the -// hub can flag a hive that is silently 401'ing on every inference call. Safe on -// a nil tracker. -func (p *GitHubProxy) InferenceAuthError() (errMsg string, since time.Time) { - if p == nil || p.inferenceAuth == nil { - return "", time.Time{} - } - return p.inferenceAuth.snapshot() -} - -// InferenceBudgetExceeded reports the current provider spending-limit signal: -// a non-empty log-safe cause, when it first latched, when the most recent -// rebuff arrived, and how many rebuffs have been observed since — all -// zero-valued while the provider is serving normally. -// -// Callers use a non-empty cause as "this hive cannot buy inference right now", -// which is a reason to stop kicking agents and to tell the operator, NOT a -// reason to retry in a few minutes. lastRebuff is what bounds that: because -// withholding kicks also withholds the calls that would clear the latch, -// callers suppress only while lastRebuff is fresh and send a probe once it is -// stale (see inference_budget.go). Safe on a nil receiver/tracker. -func (p *GitHubProxy) InferenceBudgetExceeded() (errMsg string, since, lastRebuff time.Time, rebuffs int) { - if p == nil || p.inferenceBudget == nil { - return "", time.Time{}, time.Time{}, 0 - } - return p.inferenceBudget.snapshot() -} - -// ClearInferenceRoute removes an agent's inference backend override. -func (p *GitHubProxy) ClearInferenceRoute(agentName string) { - p.inference.Clear(agentName) - p.logger.Info("inference route cleared", "agent", agentName) -} - -// StartInferenceTranslator runs a plain HTTP server that accepts Anthropic -// Messages API requests and translates+forwards them to the configured -// inference backend. Agents use ANTHROPIC_BASE_URL=http://127.0.0.1:18444 -// to reach this server instead of api.anthropic.com. -func (p *GitHubProxy) inferenceTranslatorHandler() http.Handler { - mux := http.NewServeMux() - - mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - agentName, ok := p.translatorAgentName(r) - if !ok { - p.logger.Warn("inference request from unidentified caller refused", - "remote", r.RemoteAddr, "method", r.Method, "path", r.URL.Path) - http.Error(w, `{"type":"error","error":{"type":"authentication_error","message":"caller could not be identified"}}`, http.StatusUnauthorized) - return - } - - p.logger.Info("inference request", - "agent", agentName, - "method", r.Method, - "path", r.URL.Path, - "content-type", r.Header.Get("Content-Type"), - "anthropic-version", r.Header.Get("anthropic-version"), - ) - - route := p.inference.Get(agentName) - if route == nil { - p.logger.Warn("inference no route", "agent", agentName) - http.Error(w, `{"type":"error","error":{"type":"api_error","message":"no inference route for agent"}}`, http.StatusBadGateway) - return - } - route = p.routeWithEntitledModel(route, agentName) - - body, err := io.ReadAll(r.Body) - if r.Body != nil { - _ = r.Body.Close() - } - if err != nil { - http.Error(w, `{"type":"error","error":{"type":"api_error","message":"failed to read request"}}`, http.StatusBadRequest) - return - } - - // Same path gate as the MITM reroute: only POST /v1/messages is - // translated and forwarded; CLI housekeeping is answered here. - if kind := classifyInferencePath(r.Method, r.URL.Path); kind != inferencePathMessages { - status, respBody := p.localInferenceResponse(kind, r.Method, r.URL.Path, body, agentName) - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(status) - _, _ = io.WriteString(w, respBody) - return - } - - // Diagnostic: count tools in original Anthropic request. - var toolDiag struct { - Tools json.RawMessage `json:"tools"` - } - toolCount := -1 // -1 = no tools key - if json.Unmarshal(body, &toolDiag) == nil && toolDiag.Tools != nil { - var arr []json.RawMessage - if json.Unmarshal(toolDiag.Tools, &arr) == nil { - toolCount = len(arr) - } - } - p.logger.Info("inference request body", "agent", agentName, "len", len(body), "tool_count", toolCount, "preview", truncateBytes(body, 200)) - - openaiBody, err := translateAnthropicToOpenAI(body, route.Model, route.MaxContextLen, resolveInferencePreamble(route, agentName)) - if err != nil { - p.logger.Error("inference translate request failed", "agent", agentName, "error", err) - http.Error(w, fmt.Sprintf(`{"type":"error","error":{"type":"api_error","message":"translation error: %s"}}`, err.Error()), http.StatusBadGateway) - return - } - - upstreamURL := openAIChatCompletionsURL(route.Endpoint) - upstreamReq, err := http.NewRequestWithContext(r.Context(), "POST", upstreamURL, bytes.NewReader(openaiBody)) - if err != nil { - http.Error(w, `{"type":"error","error":{"type":"api_error","message":"failed to create upstream request"}}`, http.StatusBadGateway) - return - } - upstreamReq.Header.Set("Content-Type", "application/json") - applyInferenceAuth(upstreamReq, route) - - p.logger.Info("inference forward", - "agent", agentName, - "backend", route.Backend, - "model", route.Model, - "url", upstreamURL, - "openai_body", truncateBytes(openaiBody, 300), - ) - - client, err := inferenceHTTPClient(route) - if err != nil { - p.logger.Error("inference client setup failed", "agent", agentName, "error", err) - http.Error(w, fmt.Sprintf(`{"type":"error","error":{"type":"api_error","message":"inference client setup failed: %s"}}`, err.Error()), http.StatusBadGateway) - return - } - resp, err := client.Do(upstreamReq) - if err != nil { - p.logger.Error("inference upstream failed", "agent", agentName, "url", upstreamURL, "error", err) - if p.gatewayHealth != nil { - p.gatewayHealth.RecordEndpointError(route.Backend, route.Endpoint, err, time.Now()) - } - unreachableMsg := inferenceBackendUnreachableMessage(err, upstreamURL) - p.recordAgentInferenceError(agentName, unreachableMsg, 0) - http.Error(w, fmt.Sprintf(`{"type":"error","error":{"type":"api_error","message":"%s"}}`, jsonEscape(unreachableMsg)), http.StatusBadGateway) - return - } - defer func() { _ = resp.Body.Close() }() - - p.logger.Info("inference upstream response", - "agent", agentName, - "status", resp.StatusCode, - "content-type", resp.Header.Get("Content-Type"), - ) - - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - errBody, _ := io.ReadAll(resp.Body) - p.logger.Error("inference upstream error", - "agent", agentName, - "status", resp.StatusCode, - "url", upstreamURL, - "server", resp.Header.Get("Server"), - "body", truncateBytes(errBody, 500), - ) - // A LiteLLM "team not allowed to access model" 403 carries the - // entitled set — cache it so the dashboard's model list narrows - // and its reconcile policy (#1848) can heal the agent. - p.recordInferenceError(route, agentName, resp.StatusCode, errBody) - backendMsg := inferenceBackendErrorMessage(resp.StatusCode, errBody, upstreamURL, resp.Header.Get("Server")) - p.recordAgentInferenceError(agentName, backendMsg, resp.StatusCode) - anthropicErr := fmt.Sprintf(`{"type":"error","error":{"type":"api_error","message":"%s"}}`, - jsonEscape(backendMsg)) - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(resp.StatusCode) - _, _ = w.Write([]byte(anthropicErr)) - return - } - - // A 2xx means the gateway accepted the key — clear any latched - // inference-auth failure so a hive whose key was fixed self-heals. - p.recordInferenceSuccessFor(route) - p.clearAgentInferenceError(agentName) - - isStreaming := strings.Contains(resp.Header.Get("Content-Type"), "text/event-stream") - - if isStreaming { - w.Header().Set("Content-Type", "text/event-stream; charset=utf-8") - w.Header().Set("Cache-Control", "no-cache") - w.Header().Set("Connection", "keep-alive") - w.WriteHeader(http.StatusOK) - if f, ok := w.(http.Flusher); ok { - f.Flush() - } - in, out, err := translateOpenAISSEToAnthropic(resp.Body, w, route.Model) - if err != nil { - p.logger.Error("inference SSE translation failed", "agent", agentName, "error", err) - } - if in > 0 || out > 0 { - p.recordUsage(agentName, route.Model, in, out) - } - return - } - - respBody, err := io.ReadAll(resp.Body) - if err != nil { - http.Error(w, `{"type":"error","error":{"type":"api_error","message":"failed to read inference response"}}`, http.StatusBadGateway) - return - } - - p.logger.Info("inference upstream raw response", - "agent", agentName, - "len", len(respBody), - "preview", truncateBytes(respBody, 500), - ) - - if in, out := extractOpenAIUsage(respBody); in > 0 || out > 0 { - p.recordUsage(agentName, route.Model, in, out) - } - - translated, err := translateOpenAIResponseToAnthropic(respBody, route.Model) - if err != nil { - p.logger.Error("inference translate response failed", "agent", agentName, "error", err) - http.Error(w, `{"type":"error","error":{"type":"api_error","message":"response translation error"}}`, http.StatusBadGateway) - return - } - - p.logger.Info("inference translated response", - "agent", agentName, - "len", len(translated), - "preview", truncateBytes(translated, 500), - ) - - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - _, _ = w.Write(translated) - }) - - return mux -} - -// StartInferenceTranslator runs the Anthropic-compatible inference translator -// on the fixed local port used by agents. -func (p *GitHubProxy) StartInferenceTranslator() error { - addr := fmt.Sprintf("127.0.0.1:%d", InferenceTranslatePort) - p.logger.Info("inference translation server starting", "addr", addr) - server := &http.Server{ - Addr: addr, - Handler: p.inferenceTranslatorHandler(), - ReadHeaderTimeout: httpReadTimeout, - } - return server.ListenAndServe() -} - -// handleAnthropicReroute performs MITM on an Anthropic API connection and -// reroutes requests to a self-hosted vLLM/llm-d endpoint, translating -// between Anthropic and OpenAI API formats. -func (p *GitHubProxy) handleAnthropicReroute(conn net.Conn, r *http.Request, host, agentName string, route *InferenceRoute) { - p.logger.Info("inference reroute", "agent", agentName, "backend", route.Backend, "host", host) - - if _, err := fmt.Fprintf(conn, "HTTP/1.1 200 Connection established\r\n\r\n"); err != nil { - p.logger.Error("inference reroute: CONNECT response failed", "error", err) - return - } - - tlsCert, err := p.forgeCert(host) - if err != nil { - p.logger.Error("inference reroute: forge cert failed", "host", host, "error", err) - return - } - - tlsConn := tls.Server(conn, &tls.Config{ - Certificates: []tls.Certificate{tlsCert}, - }) - _ = conn.SetDeadline(time.Now().Add(clientTLSHandshakeTimeout)) - handshakeErr := tlsConn.Handshake() - _ = conn.SetDeadline(time.Time{}) - if handshakeErr != nil { - p.logger.Warn("inference reroute: TLS handshake failed", "error", handshakeErr) - return - } - defer func() { _ = tlsConn.Close() }() - - clientBuf := bufio.NewReader(tlsConn) - for { - _ = tlsConn.SetReadDeadline(time.Now().Add(httpReadTimeout)) - req, err := http.ReadRequest(clientBuf) - _ = tlsConn.SetReadDeadline(time.Time{}) - if err != nil { - p.logTimeout("inference reroute: client request read timed out", err, "agent", agentName) - return - } - - p.handleInferenceRequest(tlsConn, req, agentName, route) - } -} - -// handleInferenceRequest translates a single Anthropic API request and -// forwards it to the inference backend. -func (p *GitHubProxy) handleInferenceRequest(conn net.Conn, req *http.Request, agentName string, route *InferenceRoute) { - route = p.routeWithEntitledModel(route, agentName) - _ = conn.SetReadDeadline(time.Now().Add(httpReadTimeout)) - body, err := io.ReadAll(req.Body) - _ = conn.SetReadDeadline(time.Time{}) - if req.Body != nil { - _ = req.Body.Close() - } - if err != nil { - p.logTimeout("inference reroute: request body read timed out", err, "agent", agentName) - p.writeHTTPError(conn, http.StatusBadGateway, "failed to read request body") - return - } - - // Only POST /v1/messages is an inference call. The Claude Code CLI also - // sends telemetry batches, error reports, and count_tokens to this host; - // translating those bodies produced {"messages": null} and a gateway 400 - // per call, charged against the provider's request rate limit. - if kind := classifyInferencePath(req.Method, req.URL.Path); kind != inferencePathMessages { - status, respBody := p.localInferenceResponse(kind, req.Method, req.URL.Path, body, agentName) - writeLocalInferenceResponse(conn, status, respBody) - return - } - - openaiBody, err := translateAnthropicToOpenAI(body, route.Model, route.MaxContextLen, resolveInferencePreamble(route, agentName)) - if err != nil { - p.logger.Error("inference translate request failed", "agent", agentName, "error", err) - p.writeHTTPError(conn, http.StatusBadGateway, "translation error: "+err.Error()) - return - } - - upstreamURL := openAIChatCompletionsURL(route.Endpoint) - upstreamReq, err := http.NewRequestWithContext(req.Context(), "POST", upstreamURL, bytes.NewReader(openaiBody)) - if err != nil { - p.writeHTTPError(conn, http.StatusBadGateway, "failed to create upstream request") - return - } - upstreamReq.Header.Set("Content-Type", "application/json") - applyInferenceAuth(upstreamReq, route) - - p.logger.Info("inference forward", "agent", agentName, "backend", route.Backend, "model", route.Model, "url", upstreamURL) - - client, err := inferenceHTTPClient(route) - if err != nil { - p.logger.Error("inference client setup failed", "agent", agentName, "error", err) - p.writeHTTPError(conn, http.StatusBadGateway, "inference client setup failed: "+err.Error()) - return - } - resp, err := client.Do(upstreamReq) - if err != nil { - p.logger.Error("inference upstream failed", "agent", agentName, "url", upstreamURL, "error", err) - if p.gatewayHealth != nil { - p.gatewayHealth.RecordEndpointError(route.Backend, route.Endpoint, err, time.Now()) - } - unreachableMsg := inferenceBackendUnreachableMessage(err, upstreamURL) - p.recordAgentInferenceError(agentName, unreachableMsg, 0) - p.writeHTTPError(conn, http.StatusBadGateway, unreachableMsg) - return - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - errBody, _ := io.ReadAll(resp.Body) - p.logger.Error("inference upstream error", "agent", agentName, "status", resp.StatusCode, "url", upstreamURL, "server", resp.Header.Get("Server"), "body", truncateBytes(errBody, 500)) - // Same entitlement capture as the HTTP translator path. - p.recordInferenceError(route, agentName, resp.StatusCode, errBody) - backendMsg := inferenceBackendErrorMessage(resp.StatusCode, errBody, upstreamURL, resp.Header.Get("Server")) - p.recordAgentInferenceError(agentName, backendMsg, resp.StatusCode) - anthropicErr := &http.Response{ - StatusCode: resp.StatusCode, - ProtoMajor: 1, - ProtoMinor: 1, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(strings.NewReader(fmt.Sprintf( - `{"type":"error","error":{"type":"api_error","message":"%s"}}`, - jsonEscape(backendMsg)))), - } - _ = anthropicErr.Write(conn) - return - } - - // A 2xx means the gateway accepted the key — clear any latched - // inference-auth failure so a hive whose key was fixed self-heals. - p.recordInferenceSuccessFor(route) - p.clearAgentInferenceError(agentName) - - isStreaming := strings.Contains(resp.Header.Get("Content-Type"), "text/event-stream") - - if isStreaming { - hdr := "HTTP/1.1 200 OK\r\n" + - "Content-Type: text/event-stream; charset=utf-8\r\n" + - "Cache-Control: no-cache\r\n" + - "Connection: keep-alive\r\n\r\n" - if _, err := conn.Write([]byte(hdr)); err != nil { - return - } - in, out, err := translateOpenAISSEToAnthropic(resp.Body, conn, route.Model) - if err != nil { - p.logger.Error("inference SSE translation failed", "agent", agentName, "error", err) - } - if in > 0 || out > 0 { - p.recordUsage(agentName, route.Model, in, out) - } - return - } - - respBody, err := io.ReadAll(resp.Body) - if err != nil { - p.writeHTTPError(conn, http.StatusBadGateway, "failed to read inference response") - return - } - - if in, out := extractOpenAIUsage(respBody); in > 0 || out > 0 { - p.recordUsage(agentName, route.Model, in, out) - } - - translated, err := translateOpenAIResponseToAnthropic(respBody, route.Model) - if err != nil { - p.logger.Error("inference translate response failed", "agent", agentName, "error", err) - p.writeHTTPError(conn, http.StatusBadGateway, "response translation error") - return - } - - httpResp := &http.Response{ - StatusCode: http.StatusOK, - ProtoMajor: 1, - ProtoMinor: 1, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(bytes.NewReader(translated)), - } - _ = httpResp.Write(conn) -} - -// handleCopilotSniff MITMs a GitHub Copilot completion-API connection to read -// the OpenAI-shaped token usage out of each /chat/completions response, then -// records it via the same token sink the inference path uses so Copilot cost -// goes live instead of only tallying at session shutdown. -// -// Unlike the inference reroute, this does NOT translate or reroute: the client's -// request is forwarded verbatim to the real Copilot host (with one exception — -// streaming requests get stream_options.include_usage injected so the terminal -// SSE usage chunk is emitted), and the upstream response is streamed back -// byte-for-byte to the client. The proxy only reads a copy of the response to -// extract usage. -// -// It is only invoked when a token sink is active and the agent is identified; -// callers fall back to opaque tunneling otherwise, so a missing sink or unknown -// agent never breaks Copilot traffic. -func (p *GitHubProxy) handleCopilotSniff(conn net.Conn, host, agentName string) { - if _, err := fmt.Fprintf(conn, "HTTP/1.1 200 Connection established\r\n\r\n"); err != nil { - p.logger.Error("copilot sniff: CONNECT response failed", "error", err) - return - } - p.sniffCopilotOnTLS(conn, host, agentName) -} - -// sniffCopilotOnTLS TLS-terminates a client connection (already past any CONNECT -// handshake), dials the real Copilot upstream, and pumps requests/responses -// through proxyCopilotHTTP so usage is recorded. Shared by the explicit-CONNECT -// (handleCopilotSniff) and the transparent-TLS (iptables) entry points so the -// forge/handshake/dial logic lives in one place. clientConn must be the raw -// (pre-TLS) client socket, optionally already wrapped in a prefixConn carrying a -// peeked ClientHello. -func (p *GitHubProxy) sniffCopilotOnTLS(clientConn net.Conn, host, agentName string) { - tlsCert, err := p.forgeCert(host) - if err != nil { - p.logger.Error("copilot sniff: forge cert failed", "host", host, "error", err) - return - } - - tlsClientConn := tls.Server(clientConn, &tls.Config{Certificates: []tls.Certificate{tlsCert}}) - _ = clientConn.SetDeadline(time.Now().Add(clientTLSHandshakeTimeout)) - handshakeErr := tlsClientConn.Handshake() - _ = clientConn.SetDeadline(time.Time{}) - if handshakeErr != nil { - if isClientHangup(handshakeErr) { - p.logger.Debug("copilot sniff: client hung up before TLS handshake", "error", handshakeErr) - } else { - p.logger.Warn("copilot sniff: client TLS handshake failed", "error", handshakeErr) - } - return - } - defer func() { _ = tlsClientConn.Close() }() - - upstreamConn, err := p.dialCopilotUpstream(host) - if err != nil { - p.logger.Error("copilot sniff: upstream dial failed", "host", host, "error", err) - return - } - defer func() { _ = upstreamConn.Close() }() - - p.proxyCopilotHTTP(tlsClientConn, upstreamConn, host, agentName) -} - -// copilotSniffBodyLimit caps how much of a Copilot request/response body the -// proxy buffers in order to read the usage block. Completion payloads are well -// under this; the limit bounds memory against a pathological body. Bodies larger -// than the limit are still forwarded in full (streamed), only usage extraction -// is skipped for the oversized case. -const copilotSniffBodyLimit = 32 * 1024 * 1024 // 32 MiB - -// proxyCopilotHTTP reads HTTP requests from the client, forwards them to the -// Copilot upstream, and forwards responses back — extracting token usage from -// completion responses along the way. It keeps the connection alive across -// multiple request/response pairs (HTTP keep-alive), exiting when either side -// closes. -func (p *GitHubProxy) proxyCopilotHTTP(client net.Conn, upstream net.Conn, host, agentName string) { - clientBuf := bufio.NewReader(client) - upstreamBuf := bufio.NewReader(upstream) - - for { - _ = client.SetReadDeadline(time.Now().Add(httpReadTimeout)) - req, err := http.ReadRequest(clientBuf) - if err != nil { - _ = client.SetReadDeadline(time.Time{}) - p.logClientRead("copilot sniff: client request read timed out", err, "agent", agentName, "host", host) - return - } - - isCompletion := isCopilotCompletionsPath(req.URL.Path) - isAnthropic := isCopilotAnthropicMessagesPath(req.URL.Path) - model := "" - - // For completion requests, buffer the body so we can (a) learn the - // model id and (b) inject include_usage on streaming requests. Other - // requests (model listings, etc.) are forwarded without buffering. - if isCompletion && req.Body != nil { - body, readErr := io.ReadAll(io.LimitReader(req.Body, copilotSniffBodyLimit)) - _ = req.Body.Close() - if readErr != nil { - _ = client.SetReadDeadline(time.Time{}) - p.logTimeout("copilot sniff: request body read timed out", readErr, "agent", agentName, "host", host, "path", req.URL.Path) - return - } - var streaming bool - model, streaming = parseCopilotRequest(body) - // stream_options is OpenAI-only; Anthropic streams always carry - // usage in message_start/message_delta. - if streaming && !isAnthropic { - body = ensureStreamUsageRequested(body) - } - req.Body = io.NopCloser(bytes.NewReader(body)) - req.ContentLength = int64(len(body)) - req.Header.Set("Content-Length", fmt.Sprintf("%d", len(body))) - } - - _ = upstream.SetWriteDeadline(time.Now().Add(httpWriteTimeout)) - if err := req.Write(upstream); err != nil { - _ = upstream.SetWriteDeadline(time.Time{}) - _ = client.SetReadDeadline(time.Time{}) - p.logTimeout("copilot sniff: request relay timed out", err, "agent", agentName, "host", host, "path", req.URL.Path) - return - } - _ = upstream.SetWriteDeadline(time.Time{}) - _ = client.SetReadDeadline(time.Time{}) - - _ = upstream.SetReadDeadline(time.Now().Add(httpReadTimeout)) - resp, err := http.ReadResponse(upstreamBuf, req) - _ = upstream.SetReadDeadline(time.Time{}) - if err != nil { - p.logTimeout("copilot sniff: upstream response read timed out", err, "agent", agentName, "host", host, "path", req.URL.Path) - return - } - - if isAnthropic && resp.StatusCode >= 200 && resp.StatusCode < 300 { - if err := p.forwardCopilotAnthropicResponseWithUsage(client, resp, host, agentName, model); err != nil { - p.logTimeout("copilot sniff: response relay timed out", err, "agent", agentName, "host", host, "path", req.URL.Path) - return - } - } else if isCompletion && resp.StatusCode >= 200 && resp.StatusCode < 300 { - p.forwardCopilotResponseWithUsage(client, resp, host, agentName, model) - } else { - // Non-completion and non-2xx responses are relayed verbatim. The - // Write below can legitimately stay open for a long time, so it - // must be bounded by stall, not by total duration — see - // stallBoundedWriter. - if err := resp.Write(&stallBoundedWriter{conn: client, idle: httpWriteTimeout}); err != nil { - _ = resp.Body.Close() - _ = client.SetWriteDeadline(time.Time{}) - p.logTimeout("copilot sniff: response relay timed out", err, "agent", agentName, "host", host, "path", req.URL.Path) - return - } - _ = client.SetWriteDeadline(time.Time{}) - _ = resp.Body.Close() - } - - if req.Close || resp.Close { - return - } - } -} - -// forwardCopilotAnthropicResponseWithUsage relays an Anthropic Messages -// (/v1/messages) response to the client while teeing the body through a usage -// parser, so streamed Claude completions are forwarded as they arrive rather -// than buffered. Usage (including prompt-cache counts) is recorded once the -// body has been fully relayed. resp.Body is consumed and closed here. -func (p *GitHubProxy) forwardCopilotAnthropicResponseWithUsage(client net.Conn, resp *http.Response, host, agentName, model string) error { - tee := newAnthropicUsageTee(resp.Body, resp.Header.Get("Content-Type")) - resp.Body = tee - writeErr := resp.Write(&stallBoundedWriter{conn: client, idle: httpWriteTimeout}) - _ = resp.Body.Close() - _ = client.SetWriteDeadline(time.Time{}) - - // Record whatever was observed even if the client went away mid-stream: - // the upstream already billed those tokens. - u := tee.usage() - if u.total() > 0 { - resolvedModel := model - if resolvedModel == "" || resolvedModel == copilotAutoModelSentinel { - resolvedModel = u.Model - } - if resolvedModel != "" && resolvedModel != copilotAutoModelSentinel { - p.recordCopilotUsageWithCache(agentName, resolvedModel, u.Input, u.Output, u.CacheRead, u.CacheCreation) - p.logger.Info("copilot usage recorded", - "agent", agentName, - "host", host, - "model", resolvedModel, - "input", u.Input, - "output", u.Output, - "cache_read", u.CacheRead, - "cache_creation", u.CacheCreation, - ) - } - } - return writeErr -} - -// forwardCopilotResponseWithUsage forwards a completion response to the client -// while extracting token usage from a buffered copy. The full body is read into -// memory (completion responses are small), usage is recorded, then the response -// is rewritten to the client verbatim. resp.Body is consumed and closed here. -func (p *GitHubProxy) forwardCopilotResponseWithUsage(client net.Conn, resp *http.Response, host, agentName, model string) { - body, err := io.ReadAll(io.LimitReader(resp.Body, copilotSniffBodyLimit)) - _ = resp.Body.Close() - if err != nil { - // Best effort: nothing to forward. - return - } - - contentType := resp.Header.Get("Content-Type") - if in, out := extractCopilotUsage(contentType, body); in > 0 || out > 0 { - resolvedModel := model - if resolvedModel == "" || resolvedModel == copilotAutoModelSentinel { - // Fall back to the model reported in the response when the request - // did not pin a concrete id (e.g. --model auto). Never record the - // "auto" sentinel — it is not a real model row. - if rm := extractCopilotResponseModel(body, contentType); rm != "" { - resolvedModel = rm - } - } - if resolvedModel != "" && resolvedModel != copilotAutoModelSentinel { - p.recordCopilotUsage(agentName, resolvedModel, in, out) - p.logger.Info("copilot usage recorded", - "agent", agentName, - "host", host, - "model", resolvedModel, - "input", in, - "output", out, - ) - } - } - - // Rewrite the response to the client verbatim. Replace the body with the - // buffered copy and drop Content-Length ambiguity by setting it explicitly. - resp.Body = io.NopCloser(bytes.NewReader(body)) - resp.ContentLength = int64(len(body)) - if err := resp.Write(&stallBoundedWriter{conn: client, idle: httpWriteTimeout}); err != nil { - p.logger.Warn("copilot sniff: response write to client failed", "agent", agentName, "error", err) - } - _ = client.SetWriteDeadline(time.Time{}) -} - func (p *GitHubProxy) writeHTTPError(conn net.Conn, status int, msg string) { resp := &http.Response{ StatusCode: status, diff --git a/src/crux/proxy/github_proxy_ca.go b/src/crux/proxy/github_proxy_ca.go new file mode 100644 index 0000000000..05818270a0 --- /dev/null +++ b/src/crux/proxy/github_proxy_ca.go @@ -0,0 +1,194 @@ +package proxy + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "log/slog" + "math/big" + "os" + "path/filepath" + "time" +) + +type cachedCert struct { + cert tls.Certificate + expiresAt time.Time +} + +// forgeCert generates a TLS certificate for the given hostname, +// signed by the proxy's CA. +func (p *GitHubProxy) forgeCert(host string) (tls.Certificate, error) { + p.certMu.RLock() + if cached, ok := p.certCache[host]; ok && time.Now().Before(cached.expiresAt) { + p.certMu.RUnlock() + return cached.cert, nil + } + p.certMu.RUnlock() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return tls.Certificate{}, err + } + + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return tls.Certificate{}, err + } + + template := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: host}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: []string{host}, + } + + caKey, ok := p.caCert.PrivateKey.(*ecdsa.PrivateKey) + if !ok { + return tls.Certificate{}, fmt.Errorf("CA key is not ECDSA") + } + + certDER, err := x509.CreateCertificate(rand.Reader, template, p.caX509, &key.PublicKey, caKey) + if err != nil { + return tls.Certificate{}, err + } + + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}) + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + return tls.Certificate{}, err + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) + + cert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return cert, err + } + // Include the CA cert in the chain so clients that load system/extra + // certs asynchronously (e.g. GitHub Copilot's bundled undici) can + // verify the forged leaf without having the CA pre-loaded in their + // trust store. + cert.Certificate = append(cert.Certificate, p.caX509.Raw) + + p.certMu.Lock() + if p.certCache == nil { + p.certCache = make(map[string]cachedCert) + } + p.certCache[host] = cachedCert{cert: cert, expiresAt: time.Now().Add(time.Hour)} + p.certMu.Unlock() + + return cert, nil +} + +// generateCA creates a self-signed CA certificate for MITM. +func generateCA() (tls.Certificate, *x509.Certificate, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return tls.Certificate{}, nil, err + } + + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return tls.Certificate{}, nil, err + } + + template := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: "Hive ACMM Proxy CA"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(365 * 24 * time.Hour), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + BasicConstraintsValid: true, + IsCA: true, + MaxPathLen: 0, + } + + certDER, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + return tls.Certificate{}, nil, err + } + + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}) + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + return tls.Certificate{}, nil, err + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) + + tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return tls.Certificate{}, nil, err + } + + x509Cert, err := x509.ParseCertificate(certDER) + if err != nil { + return tls.Certificate{}, nil, err + } + + return tlsCert, x509Cert, nil +} + +// loadOrGenerateCA reuses an existing CA key pair from disk if present and +// valid, or generates a fresh one and persists both cert and key to the PVC. +// Reusing the CA across pod restarts means the system trust store (populated +// by entrypoint.sh before the Go binary starts) already contains the right +// CA, so clients that load system certs asynchronously (GitHub Copilot's +// bundled undici) can verify forged certificates immediately. +func loadOrGenerateCA(logger *slog.Logger) (tls.Certificate, *x509.Certificate, error) { + certPEM, certErr := os.ReadFile(CACertPath) + keyPEM, keyErr := os.ReadFile(caKeyPath) + if certErr == nil && keyErr == nil { + tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) + if err == nil { + block, _ := pem.Decode(certPEM) + if block != nil { + x509Cert, err := x509.ParseCertificate(block.Bytes) + if err == nil && time.Now().Before(x509Cert.NotAfter) { + logger.Info("reusing persisted MITM CA", "expires", x509Cert.NotAfter) + return tlsCert, x509Cert, nil + } + } + } + logger.Warn("persisted CA unusable, generating fresh one", "err", err) + } + + tlsCert, x509Cert, err := generateCA() + if err != nil { + return tls.Certificate{}, nil, err + } + + caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: x509Cert.Raw}) + if err := os.WriteFile(CACertPath, caPEM, 0644); err != nil { + return tls.Certificate{}, nil, fmt.Errorf("write CA cert to %s: %w", CACertPath, err) + } + + caKeyDER, err := x509.MarshalECPrivateKey(tlsCert.PrivateKey.(*ecdsa.PrivateKey)) + if err != nil { + return tls.Certificate{}, nil, fmt.Errorf("marshal CA key: %w", err) + } + caKeyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: caKeyDER}) + // The parent directory is created with owner-only permissions before the + // key is written. /data itself is intentionally group-writable for agent + // workspaces, so 0600 on a file directly under /data is not sufficient: + // an agent could replace or race the path before the proxy opens it. + if err := os.MkdirAll(filepath.Dir(caKeyPath), 0700); err != nil { + return tls.Certificate{}, nil, fmt.Errorf("create CA key directory: %w", err) + } + if err := os.Chmod(filepath.Dir(caKeyPath), 0700); err != nil { + return tls.Certificate{}, nil, fmt.Errorf("restrict CA key directory: %w", err) + } + if err := os.WriteFile(caKeyPath, caKeyPEM, 0600); err != nil { + return tls.Certificate{}, nil, fmt.Errorf("write CA key to %s: %w", caKeyPath, err) + } + + logger.Info("generated and persisted fresh MITM CA", "certPath", CACertPath, "keyPath", caKeyPath) + return tlsCert, x509Cert, nil +} diff --git a/src/crux/proxy/github_proxy_connect.go b/src/crux/proxy/github_proxy_connect.go new file mode 100644 index 0000000000..0b1ace8cb8 --- /dev/null +++ b/src/crux/proxy/github_proxy_connect.go @@ -0,0 +1,672 @@ +package proxy + +import ( + "bufio" + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "strings" + "syscall" + "time" +) + +// hostNeedsMITM decides whether a GitHub-family host must be TLS-intercepted +// rather than opaquely tunneled. +// +// Without injection this is exactly the historical NeedsMITM: only +// api.github.com, where request-level ACMM inspection happens; github.com and +// registered GHE hosts are tunneled without decryption (their traffic — git +// smart HTTP, OAuth device flow — is gated elsewhere). +// +// With injection (#1861) an opaque tunnel is a hole, not an optimization: the +// agent's git credential helper now serves the inert placeholder, so a +// git-over-HTTPS push through an un-MITM'd tunnel would reach GitHub carrying +// the placeholder and 401. Every GitHub-family host must therefore be +// intercepted so rewriteGitHubAuth can replace the placeholder with the real +// scoped token — which also means an agent-supplied credential can never +// sneak to ANY GitHub host through a tunnel the proxy declined to open. +func (p *GitHubProxy) hostNeedsMITM(host string) bool { + if NeedsMITM(host) { + return true + } + return p.injectGHAuth && IsGitHubHost(host) +} + +// hostNeedsMITMFor is hostNeedsMITM for a known caller. A lane under +// write_surface.enforce has every GitHub-family host intercepted, not only +// api.github.com, because git push travels over github.com's smart HTTP and +// an opaque tunnel could not refuse it. Other lanes are unaffected. +func (p *GitHubProxy) hostNeedsMITMFor(host, agentName string) bool { + if p.hostNeedsMITM(host) { + return true + } + return IsGitHubHost(host) && p.laneWriteSurfaceEnforced(agentName) +} + +// Start begins listening. Blocks until the listener is closed. +// Handles both explicit HTTP CONNECT proxy requests and transparent +// iptables-redirected TLS connections (detected by TLS ClientHello). +func (p *GitHubProxy) Start() error { + ln, err := net.Listen("tcp", p.listenAddr) + if err != nil { + return fmt.Errorf("listen %s: %w", p.listenAddr, err) + } + defer func() { _ = ln.Close() }() + p.logger.Info("proxy listening", "addr", p.listenAddr) + + for { + conn, err := ln.Accept() + if err != nil { + return fmt.Errorf("accept: %w", err) + } + go p.handleConn(conn) + } +} + +// handleConn peeks at the first byte to distinguish HTTP CONNECT requests +// (explicit proxy) from raw TLS ClientHello (iptables-redirected traffic). +func (p *GitHubProxy) handleConn(conn net.Conn) { + defer func() { _ = conn.Close() }() + + peeked := make([]byte, 1) + _ = conn.SetReadDeadline(time.Now().Add(transparentProxyTimeout)) + n, err := conn.Read(peeked) + _ = conn.SetReadDeadline(time.Time{}) + if err != nil || n == 0 { + return + } + + // TLS ClientHello starts with byte 0x16 (ContentType handshake). + // HTTP methods start with ASCII letters (C for CONNECT, G for GET, etc.). + const tlsHandshakeContentType = 0x16 + if peeked[0] == tlsHandshakeContentType { + p.handleTransparentTLS(conn, peeked) + return + } + + // Parse the HTTP request directly instead of using http.Server.Serve, + // which closes the connection on shutdown — racing with hijacked CONNECT + // handlers. + prefixed := &prefixConn{Conn: conn, prefix: peeked[:n]} + _ = conn.SetReadDeadline(time.Now().Add(httpReadTimeout)) + req, err := http.ReadRequest(bufio.NewReader(prefixed)) + _ = conn.SetReadDeadline(time.Time{}) + if err != nil { + return + } + + if req.Method == http.MethodConnect { + p.handleConnectDirect(conn, req) + } else { + p.forwardPlainDirect(conn, req) + } +} + +const ( + transparentProxyTimeout = 5 * time.Second + httpReadTimeout = 30 * time.Second + httpWriteTimeout = 60 * time.Second + // upstreamDialTimeout bounds the proxy's OWN upstream dial + TLS handshake + // to the real GitHub host. Before this, a stalled upstream (DNS, TCP, or + // TLS) blocked forever — the agent's curl sat in poll() waiting for the + // MITM response that never came, permanently wedging the request and + // freezing the agent's session (the multi-hour "egress hang"). + // tls.DialWithDialer's dialer Timeout covers connect + handshake; the + // deadline is lifted once established so long-lived relays (git smart + // HTTP) are unaffected. + upstreamDialTimeout = 15 * time.Second + + // clientTLSHandshakeTimeout bounds the proxy's TLS handshake with the + // inbound client. It intentionally matches upstreamDialTimeout: both are + // short setup phases, unlike long-lived request/response relays. + clientTLSHandshakeTimeout = upstreamDialTimeout +) + +// handleTransparentTLS handles iptables-redirected connections. The agent +// tried to connect to github.com:443 but iptables sent it here instead. +// We extract the SNI hostname from the TLS ClientHello, then MITM the connection. +func (p *GitHubProxy) handleTransparentTLS(conn net.Conn, peeked []byte) { + // Read the whole ClientHello TLS record before extracting SNI and replaying + // it. TCP guarantees nothing about record boundaries: a single Read can + // return only PART of the ClientHello. A partial read makes extractSNI miss + // the SNI (so the wrong host is chosen and the wrong cert is forged) and + // mis-frames the record stream handed to tls.Server — the agent-side + // handshake then fails deterministically (bad certificate / bad record MAC), + // which is exactly why every Copilot agent could not authenticate. Loop until + // the full record (per the record-layer length) is buffered. + _ = conn.SetReadDeadline(time.Now().Add(transparentProxyTimeout)) + fullBuf, err := readClientHelloRecord(conn, peeked) + _ = conn.SetReadDeadline(time.Time{}) + if err != nil { + return + } + + host := extractSNI(fullBuf) + if host == "" { + host = "github.com" + } + + agentName := p.attributeTransparentConn(conn) + + // Copilot completion host under iptables redirection: MITM to read live + // token usage (same guard as the explicit-CONNECT path). The peeked + // ClientHello is replayed via prefixConn so the TLS handshake sees the full + // record. + if IsCopilotAPIHost(host) && p.tokenSink != nil && agentName != "" { + p.sniffCopilotOnTLS(&prefixConn{Conn: conn, prefix: fullBuf}, host, agentName) + return + } + + if !NeedsInspection(host) && !p.hostNeedsMITMFor(host, agentName) { + // Host we neither inspect nor (under #1861 injection) intercept: + // tunnel directly. SO_MARK the socket + // so the forced-egress redirect exempts this proxy-originated dial. + upstream, err := p.dialTunnelUpstream(host) + if err != nil { + return + } + defer func() { _ = upstream.Close() }() + if _, err := upstream.Write(fullBuf); err != nil { + return + } + relayTunnel(conn, upstream) + return + } + + mode := readAgentMode(agentName) + caps := readAgentCaps(agentName) + + // MITM: forge a cert, TLS-wrap the client, connect to real upstream. + tlsCert, err := p.forgeCert(host) + if err != nil { + p.logger.Error("transparent proxy forge cert failed", "host", host, "error", err) + return + } + + tlsConfig := &tls.Config{Certificates: []tls.Certificate{tlsCert}} + prefixed := &prefixConn{Conn: conn, prefix: fullBuf} + tlsClientConn := tls.Server(prefixed, tlsConfig) + // Bound the client TLS handshake: a client that connects and then stops + // sending handshake bytes would otherwise wedge this connection forever + // (empty queues on both sides — the observed egress-hang signature). + _ = conn.SetDeadline(time.Now().Add(clientTLSHandshakeTimeout)) + handshakeErr := tlsClientConn.Handshake() + _ = conn.SetDeadline(time.Time{}) + if handshakeErr != nil { + if isClientHangup(handshakeErr) { + p.logger.Debug("proxy client hung up before TLS handshake", "error", handshakeErr) + } else { + p.logger.Warn("transparent proxy TLS handshake failed", "error", handshakeErr) + } + return + } + defer func() { _ = tlsClientConn.Close() }() + + upstreamConn, err := tls.DialWithDialer(markDialer(upstreamDialTimeout), "tcp", host+":443", &tls.Config{ServerName: host}) + if err != nil { + p.logger.Error("transparent proxy upstream dial failed", "host", host, "error", err) + return + } + defer func() { _ = upstreamConn.Close() }() + + p.proxyHTTPHost(tlsClientConn, upstreamConn, host, agentName, mode, caps) +} + +// attributeTransparentConn names the agent behind an iptables-redirected +// connection (#7793). Under transparent redirection there is no +// Proxy-Authorization header and no CONNECT request, so the ONLY identity +// evidence is the kernel socket table: the peer's ephemeral port is looked up +// in /proc/net/tcp{,6} for the owning UID, and the UID in the agent map. +// +// It returns "" — "unidentified", which the mode gate downgrades to ADVISORY — +// when there is no UID map, when the peer address carries no port (a net.Pipe +// or unix socket), when the port is not in the socket table, or when the UID +// belongs to no agent. The one deliberate exception is the hive's own control +// plane: a UID the map reports as internal (root or the proxy user, and never +// an allocated agent UID) is named internalCallerName so a hive-originated +// write is attributed rather than mistaken for a UID-attribution failure. +func (p *GitHubProxy) attributeTransparentConn(conn net.Conn) string { + if p.uidMap == nil { + return "" + } + _, portStr, splitErr := net.SplitHostPort(conn.RemoteAddr().String()) + if splitErr != nil { + return "" + } + port := 0 + const maxPort = 65535 + for _, c := range portStr { + port = port*10 + int(c-'0') + if port > maxPort { + port = 0 + break + } + } + uid, lookupErr := LookupUIDByLocalPort(port) + if lookupErr != nil { + return "" + } + if name := p.uidMap.LookupByUID(uid); name != "" { + return name + } + // The hive's own control plane is not an agent, so the lookup above + // returns "". Under v4's forced-proxy egress that made every + // hive-originated write — notably the App installation token mint — look + // like an unattributable agent, which the mode gate blocks. Name it + // explicitly so it is attributed rather than mistaken for a + // UID-attribution failure. + if p.uidMap.IsInternalUID(uid) { + return internalCallerName + } + return "" +} + +// dialTunnelUpstream opens the raw TCP leg of a non-inspected transparent +// tunnel. The socket is SO_MARKed so the forced-egress redirect exempts this +// proxy-originated dial. tunnelDial, when set, overrides it (tests). +func (p *GitHubProxy) dialTunnelUpstream(host string) (net.Conn, error) { + if p.tunnelDial != nil { + return p.tunnelDial(host) + } + return markDialer(transparentProxyTimeout).Dial("tcp", host+":443") +} + +const tlsClientHelloMaxSize = 4096 + +// tlsRecordHeaderLen is the size of the TLS record header: content type(1) + +// version(2) + length(2). +const tlsRecordHeaderLen = 5 + +// readClientHelloRecord reads the complete first TLS record (the ClientHello) +// off conn, starting from the already-peeked bytes, and returns the full record +// so the caller can both extract the SNI and replay it verbatim into tls.Server. +// +// The prior implementation did a SINGLE conn.Read after the 1-byte peek and +// assumed it captured the whole ClientHello. TCP makes no such guarantee: when +// the ClientHello arrives split across segments, that single read returns only a +// prefix. extractSNI then parses a truncated ClientHello, fails to find the SNI, +// falls back to the wrong host, and the forged cert no longer matches what the +// agent asked for — so the agent-side TLS handshake fails deterministically +// (observed as "bad certificate" or "bad record MAC"), blocking every Copilot +// agent from authenticating. +// +// This reassembles the record using the record-layer length field, reading in a +// loop until the whole ClientHello record is buffered (or the buffer cap / +// deadline is hit), so both SNI extraction and the replay always see an intact +// ClientHello regardless of how TCP fragments the client's bytes. The caller +// sets/clears the read deadline around this call. +func readClientHelloRecord(conn net.Conn, peeked []byte) ([]byte, error) { + buf := make([]byte, tlsClientHelloMaxSize) + n := copy(buf, peeked) + + // First, ensure we have at least the record header so we know the record's + // declared length. + for n < tlsRecordHeaderLen { + r, err := conn.Read(buf[n:]) + n += r + if err != nil { + if n == 0 { + return nil, err + } + // Return what we have; extractSNI degrades gracefully on a short read. + return buf[:n], nil + } + } + + // record length is bytes 3..4 (big-endian); total record size adds the header. + recordLen := int(buf[3])<<8 | int(buf[4]) + need := tlsRecordHeaderLen + recordLen + if need > len(buf) { + // ClientHello larger than our cap: read to fill the buffer. The remainder + // (and subsequent records) are served to tls.Server via prefixConn+conn, + // so the handshake still completes; we only need enough here for SNI. + need = len(buf) + } + + for n < need { + r, err := conn.Read(buf[n:]) + n += r + if err != nil { + if n == 0 { + return nil, err + } + return buf[:n], nil + } + } + return buf[:n], nil +} + +// extractSNI reads the SNI hostname from a TLS ClientHello message. +func extractSNI(data []byte) string { + if len(data) < 5 { + return "" + } + // TLS record: type(1) + version(2) + length(2) + handshake + recordLen := int(data[3])<<8 | int(data[4]) + if len(data) < 5+recordLen { + // Partial read — use what we have + recordLen = len(data) - 5 + } + handshake := data[5 : 5+recordLen] + if len(handshake) < 4 { + return "" + } + // Handshake: type(1) + length(3) + ClientHello + hsLen := int(handshake[1])<<16 | int(handshake[2])<<8 | int(handshake[3]) + if len(handshake) < 4+hsLen { + hsLen = len(handshake) - 4 + } + ch := handshake[4 : 4+hsLen] + if len(ch) < 34 { + return "" + } + // Skip: version(2) + random(32) + pos := 34 + // Session ID + if pos >= len(ch) { + return "" + } + sessLen := int(ch[pos]) + pos += 1 + sessLen + // Cipher suites + if pos+2 > len(ch) { + return "" + } + csLen := int(ch[pos])<<8 | int(ch[pos+1]) + pos += 2 + csLen + // Compression methods + if pos >= len(ch) { + return "" + } + cmLen := int(ch[pos]) + pos += 1 + cmLen + // Extensions + if pos+2 > len(ch) { + return "" + } + extLen := int(ch[pos])<<8 | int(ch[pos+1]) + pos += 2 + extEnd := pos + extLen + if extEnd > len(ch) { + extEnd = len(ch) + } + for pos+4 <= extEnd { + extType := int(ch[pos])<<8 | int(ch[pos+1]) + eLen := int(ch[pos+2])<<8 | int(ch[pos+3]) + pos += 4 + if pos+eLen > extEnd { + break + } + if extType == 0 { // SNI extension + sniData := ch[pos : pos+eLen] + if len(sniData) < 2 { + break + } + sniListLen := int(sniData[0])<<8 | int(sniData[1]) + _ = sniListLen + sniPos := 2 + for sniPos+3 <= len(sniData) { + nameType := sniData[sniPos] + nameLen := int(sniData[sniPos+1])<<8 | int(sniData[sniPos+2]) + sniPos += 3 + if sniPos+nameLen > len(sniData) { + break + } + if nameType == 0 { // host_name + return string(sniData[sniPos : sniPos+nameLen]) + } + sniPos += nameLen + } + } + pos += eLen + } + return "" +} + +// prefixConn wraps a net.Conn and prepends already-read bytes to the stream. +type prefixConn struct { + net.Conn + prefix []byte + offset int +} + +func (c *prefixConn) Read(b []byte) (int, error) { + if c.offset < len(c.prefix) { + n := copy(b, c.prefix[c.offset:]) + c.offset += n + return n, nil + } + return c.Conn.Read(b) +} + +// handleConnectDirect handles CONNECT requests on a raw connection (no http.Server). +func (p *GitHubProxy) handleConnectDirect(conn net.Conn, r *http.Request) { + host, _, err := net.SplitHostPort(r.Host) + if err != nil { + host = r.Host + } + + // Identify from the connection, NOT from r.RemoteAddr: this request came + // from http.ReadRequest, which leaves RemoteAddr empty. + agentName := p.identifyAgentFromConn(conn, r) + + // Anthropic hosts with an inference route: reroute to self-hosted backend. + if IsAnthropicHost(host) { + if route := p.inference.Get(agentName); route != nil { + p.handleAnthropicReroute(conn, r, host, agentName, route) + return + } + } + + // Copilot completion host: MITM to read live token usage. Only when a token + // sink is active and the agent is identified — otherwise fall through to an + // opaque tunnel so Copilot traffic is never broken by usage capture. + if IsCopilotAPIHost(host) && p.tokenSink != nil && agentName != "" { + p.handleCopilotSniff(conn, host, agentName) + return + } + + // Hosts we do not inspect: tunnel without interception. github.com is in + // this set — its OAuth device flow and git smart HTTP are handled by CLI + // --deny-tool flags. api.github.com and api.linear.app are not: both need + // request-level inspection for ACMM enforcement. Under #1861 injection, + // however, every GitHub-family host is intercepted (see hostNeedsMITM), + // because the agent's credential helper now serves an inert placeholder + // that only the proxy can replace with the real scoped token. + if !NeedsInspection(host) && !p.hostNeedsMITMFor(host, agentName) { + p.tunnelDirect(conn, r) + return + } + + mode := readAgentMode(agentName) + caps := readAgentCaps(agentName) + + // Tell client the tunnel is established. + if _, err := fmt.Fprintf(conn, "HTTP/1.1 200 Connection established\r\n\r\n"); err != nil { + p.logger.Error("proxy CONNECT response write failed", "error", err) + return + } + + // Generate a cert for the target host signed by our CA. + tlsCert, err := p.forgeCert(host) + if err != nil { + p.logger.Error("proxy forge cert failed", "host", host, "error", err) + return + } + + // TLS handshake with client (presenting our forged cert). + tlsClientConn := tls.Server(conn, &tls.Config{ + Certificates: []tls.Certificate{tlsCert}, + }) + // Bound the client TLS handshake (see handleTransparentTLS): a client that + // stops mid-handshake must not wedge this connection forever. + _ = conn.SetDeadline(time.Now().Add(clientTLSHandshakeTimeout)) + handshakeErr := tlsClientConn.Handshake() + _ = conn.SetDeadline(time.Time{}) + if handshakeErr != nil { + if isClientHangup(handshakeErr) { + p.logger.Debug("proxy client hung up before TLS handshake", "error", handshakeErr) + } else { + p.logger.Warn("proxy client TLS handshake failed", "error", handshakeErr) + } + return + } + defer func() { _ = tlsClientConn.Close() }() + + // Connect to the real GitHub server. SO_MARK the socket so the forced-egress + // redirect exempts this proxy-originated dial (see proxyEgressMark). + // A dialer Timeout bounds both the TCP connect and the TLS handshake — + // without it a stalled upstream wedges this CONNECT forever (the agent's + // curl sits in poll() waiting for "200 Connection established" that never + // comes). The deadline is lifted once established so the relay is not + // time-boxed. + upstreamConn, err := tls.DialWithDialer(markDialer(upstreamDialTimeout), "tcp", r.Host, &tls.Config{ + ServerName: host, + }) + if err != nil { + p.logger.Error("proxy upstream dial failed", "host", r.Host, "error", err) + return + } + defer func() { _ = upstreamConn.Close() }() + + // Proxy HTTP requests, inspecting each one. + p.proxyHTTPHost(tlsClientConn, upstreamConn, host, agentName, mode, caps) +} + +// proxyHTTP reads HTTP requests from the client, checks them against +// mode rules, and either forwards or blocks them. +func isTimeoutError(err error) bool { + var netErr net.Error + return errors.As(err, &netErr) && netErr.Timeout() +} + +// logUpstreamEvent logs an upstream timeout event at the given level. +func (p *GitHubProxy) logUpstreamEvent(level slog.Level, msg string, err error, attrs ...any) { + if !isTimeoutError(err) { + return + } + attrs = append(attrs, "error", err) + p.logger.Log(context.Background(), level, msg, attrs...) +} + +func (p *GitHubProxy) logTimeout(msg string, err error, attrs ...any) { + p.logUpstreamEvent(slog.LevelWarn, msg, err, attrs...) +} + +// isClientHangup reports a client that went away (EOF, reset, broken pipe) or +// stalled past the deadline: routine on restarts and liveness probes, not a +// hive defect. +func isClientHangup(err error) bool { + return isTimeoutError(err) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || + errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.EPIPE) || errors.Is(err, net.ErrClosed) +} + +// logClientRead logs a client-side request read failure at Debug: an idle or +// departed client is expected, unlike an upstream stall. +func (p *GitHubProxy) logClientRead(msg string, err error, attrs ...any) { + p.logUpstreamEvent(slog.LevelDebug, msg, err, attrs...) +} + +// tunnelDirect creates a raw TCP tunnel for non-GitHub CONNECT requests. +func (p *GitHubProxy) tunnelDirect(conn net.Conn, r *http.Request) { + const tunnelDialTimeout = 10 * time.Second + // SO_MARK the socket so the forced-egress redirect exempts this + // proxy-originated tunnel dial (CONNECT targets include :443). + upstream, err := markDialer(tunnelDialTimeout).Dial("tcp", r.Host) + if err != nil { + p.logger.Warn("proxy: CONNECT dial failed", "host", r.Host, "error", err) + _, _ = fmt.Fprintf(conn, "HTTP/1.1 502 Bad Gateway\r\n\r\nconnection failed\n") + return + } + defer func() { _ = upstream.Close() }() + + _, _ = fmt.Fprintf(conn, "HTTP/1.1 200 Connection established\r\n\r\n") + + relayTunnel(conn, upstream) +} + +// forwardPlainDirect handles non-CONNECT (plain HTTP) requests on a raw +// connection. +// +// Two policies apply here that the CONNECT path has always applied and this +// one did not: the caller is identified from the connection, and the +// destination must be globally routable (see plain_forward_guard.go). Before +// that, any process that could reach the proxy port could have it open a +// connection, from the proxy's own uid, to any address it named — including +// the hive's loopback services, the cloud metadata endpoint and the +// cluster-internal estate — and nothing in the log said who asked. +func (p *GitHubProxy) forwardPlainDirect(conn net.Conn, r *http.Request) { + // Identify from the connection, NOT from r.RemoteAddr: this request came + // from http.ReadRequest, which leaves RemoteAddr empty (see + // identifyAgentFromConn). "" means the caller could not be attributed, + // which is logged as such rather than guessed at. + agentName := p.identifyAgentFromConn(conn, r) + + // Proxy-* hop headers address THIS proxy and must not travel further: the + // identity they assert is the one identifyAgentFromConn just decided + // whether to believe, and forwarding it would hand an agent's claimed name + // (and, in the Basic form, whatever it put in the password field) to every + // plain-HTTP host it can name. + r.Header.Del("Proxy-Authorization") + r.Header.Del("Proxy-Connection") + + resp, err := p.plainForwardTransport().RoundTrip(r) + if err != nil { + var denied *plainForwardDenied + if errors.As(err, &denied) { + p.refusePlainForward(conn, r, agentName, denied) + return + } + if isLoopbackHost(r.URL.Host) { + p.logger.Debug("proxy: plain HTTP forward to loopback failed", "agent", agentName, "url", r.URL.String(), "error", err, "hint", "client should bypass the proxy for loopback; NO_PROXY") + } else { + p.logger.Warn("proxy: plain HTTP forward failed", "agent", agentName, "url", r.URL.String(), "error", err) + } + _, _ = fmt.Fprintf(conn, "HTTP/1.1 502 Bad Gateway\r\n\r\nupstream request failed\n") + return + } + defer func() { _ = resp.Body.Close() }() + _ = resp.Write(conn) +} + +// refusePlainForward answers a destination the guard barred. +// +// Loopback is treated as the misconfiguration it almost always is rather than +// as an attack: agents are handed NO_PROXY=127.0.0.1,localhost,::1 (#14076), +// so a loopback request arriving here means a client ignored it. It is logged +// at Debug with the same hint as the failure path and does NOT count as a +// violation. Every other class — link-local (cloud metadata), private, CGNAT, +// multicast — is an attempt to reach the estate behind the proxy and is +// counted against the caller. +func (p *GitHubProxy) refusePlainForward(conn net.Conn, r *http.Request, agentName string, denied *plainForwardDenied) { + if denied.reason == plainForwardReasonLoopback { + p.logger.Debug("proxy: plain HTTP forward to loopback refused (clients must bypass the proxy for loopback; NO_PROXY)", + "agent", agentName, "url", r.URL.String(), "target", denied.target) + } else { + p.logger.Warn("proxy: plain HTTP forward refused — destination is not globally routable", + "agent", agentName, "method", r.Method, "url", r.URL.String(), "target", denied.target, "class", denied.reason) + p.countViolation(agentName) + } + _, _ = fmt.Fprintf(conn, "HTTP/1.1 403 Forbidden\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n"+ + "the hive proxy does not forward plain HTTP to %s addresses (%s); set %s=true to allow it\n", + denied.reason, denied.target, plainForwardAllowPrivateEnv) +} + +// isLoopbackHost reports whether a host or host:port refers to the loopback interface. +func isLoopbackHost(hostport string) bool { + host := hostport + if h, _, err := net.SplitHostPort(hostport); err == nil { + host = h + } + host = strings.Trim(host, "[]") + if strings.EqualFold(host, "localhost") { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} diff --git a/src/crux/proxy/github_proxy_copilot.go b/src/crux/proxy/github_proxy_copilot.go new file mode 100644 index 0000000000..f4ba208f26 --- /dev/null +++ b/src/crux/proxy/github_proxy_copilot.go @@ -0,0 +1,294 @@ +package proxy + +import ( + "bufio" + "bytes" + "crypto/tls" + "fmt" + "io" + "net" + "net/http" + "strings" + "time" +) + +// dialCopilotUpstream connects to the real Copilot host over TLS (or the test +// seam when set). +func (p *GitHubProxy) dialCopilotUpstream(host string) (net.Conn, error) { + if p.copilotDial != nil { + return p.copilotDial(host) + } + // SO_MARK the outbound socket so the forced-egress redirect exempts the + // proxy's own upstream traffic (see proxyEgressMark). The override hook above + // still wins when set, so tests are unaffected. + return tls.DialWithDialer(markDialer(upstreamDialTimeout), "tcp", net.JoinHostPort(host, "443"), &tls.Config{ServerName: host}) +} + +// copilotInternalPathPrefix marks the Copilot CLI's auth-exchange endpoints on +// the GitHub API host (/copilot_internal/v2/token, /copilot_internal/user). +// The CLI calls them with the USER's Copilot OAuth token +// (COPILOT_GITHUB_TOKEN / the /data/copilot-user-token device-flow token) to +// obtain its short-lived Copilot session token and discover its completion +// host. A GitHub App installation token cannot perform that exchange, so +// rewriting these requests would cut every copilot-backend agent off from its +// model the moment injection is on (#9586). They are passed through with the +// agent's own Authorization untouched - nothing stripped, nothing injected. +// +// Residual (documented in security-model.md): the Copilot user OAuth token +// stays agent-readable and spendable on these paths; moving it server-side too +// is a follow-up. The hub-held App tokens injection protects are never +// attached here. +const copilotInternalPathPrefix = "/copilot_internal/" + +// gheAPIPathPrefix is the REST prefix GitHub Enterprise Server serves its API +// under (https:///api/v3/...), so the Copilot exchange on a GHE host +// arrives as /api/v3/copilot_internal/... +const gheAPIPathPrefix = "/api/v3" + +// isCopilotAuthExchangePath reports whether path is a Copilot CLI +// auth-exchange endpoint that must keep the agent's own credential (see +// copilotInternalPathPrefix), on api.github.com or a GHE /api/v3 host. +func isCopilotAuthExchangePath(path string) bool { + return strings.HasPrefix(path, copilotInternalPathPrefix) || + strings.HasPrefix(path, gheAPIPathPrefix+copilotInternalPathPrefix) +} + +// handleCopilotSniff MITMs a GitHub Copilot completion-API connection to read +// the OpenAI-shaped token usage out of each /chat/completions response, then +// records it via the same token sink the inference path uses so Copilot cost +// goes live instead of only tallying at session shutdown. +// +// Unlike the inference reroute, this does NOT translate or reroute: the client's +// request is forwarded verbatim to the real Copilot host (with one exception — +// streaming requests get stream_options.include_usage injected so the terminal +// SSE usage chunk is emitted), and the upstream response is streamed back +// byte-for-byte to the client. The proxy only reads a copy of the response to +// extract usage. +// +// It is only invoked when a token sink is active and the agent is identified; +// callers fall back to opaque tunneling otherwise, so a missing sink or unknown +// agent never breaks Copilot traffic. +func (p *GitHubProxy) handleCopilotSniff(conn net.Conn, host, agentName string) { + if _, err := fmt.Fprintf(conn, "HTTP/1.1 200 Connection established\r\n\r\n"); err != nil { + p.logger.Error("copilot sniff: CONNECT response failed", "error", err) + return + } + p.sniffCopilotOnTLS(conn, host, agentName) +} + +// sniffCopilotOnTLS TLS-terminates a client connection (already past any CONNECT +// handshake), dials the real Copilot upstream, and pumps requests/responses +// through proxyCopilotHTTP so usage is recorded. Shared by the explicit-CONNECT +// (handleCopilotSniff) and the transparent-TLS (iptables) entry points so the +// forge/handshake/dial logic lives in one place. clientConn must be the raw +// (pre-TLS) client socket, optionally already wrapped in a prefixConn carrying a +// peeked ClientHello. +func (p *GitHubProxy) sniffCopilotOnTLS(clientConn net.Conn, host, agentName string) { + tlsCert, err := p.forgeCert(host) + if err != nil { + p.logger.Error("copilot sniff: forge cert failed", "host", host, "error", err) + return + } + + tlsClientConn := tls.Server(clientConn, &tls.Config{Certificates: []tls.Certificate{tlsCert}}) + _ = clientConn.SetDeadline(time.Now().Add(clientTLSHandshakeTimeout)) + handshakeErr := tlsClientConn.Handshake() + _ = clientConn.SetDeadline(time.Time{}) + if handshakeErr != nil { + if isClientHangup(handshakeErr) { + p.logger.Debug("copilot sniff: client hung up before TLS handshake", "error", handshakeErr) + } else { + p.logger.Warn("copilot sniff: client TLS handshake failed", "error", handshakeErr) + } + return + } + defer func() { _ = tlsClientConn.Close() }() + + upstreamConn, err := p.dialCopilotUpstream(host) + if err != nil { + p.logger.Error("copilot sniff: upstream dial failed", "host", host, "error", err) + return + } + defer func() { _ = upstreamConn.Close() }() + + p.proxyCopilotHTTP(tlsClientConn, upstreamConn, host, agentName) +} + +// copilotSniffBodyLimit caps how much of a Copilot request/response body the +// proxy buffers in order to read the usage block. Completion payloads are well +// under this; the limit bounds memory against a pathological body. Bodies larger +// than the limit are still forwarded in full (streamed), only usage extraction +// is skipped for the oversized case. +const copilotSniffBodyLimit = 32 * 1024 * 1024 // 32 MiB + +// proxyCopilotHTTP reads HTTP requests from the client, forwards them to the +// Copilot upstream, and forwards responses back — extracting token usage from +// completion responses along the way. It keeps the connection alive across +// multiple request/response pairs (HTTP keep-alive), exiting when either side +// closes. +func (p *GitHubProxy) proxyCopilotHTTP(client net.Conn, upstream net.Conn, host, agentName string) { + clientBuf := bufio.NewReader(client) + upstreamBuf := bufio.NewReader(upstream) + + for { + _ = client.SetReadDeadline(time.Now().Add(httpReadTimeout)) + req, err := http.ReadRequest(clientBuf) + if err != nil { + _ = client.SetReadDeadline(time.Time{}) + p.logClientRead("copilot sniff: client request read timed out", err, "agent", agentName, "host", host) + return + } + + isCompletion := isCopilotCompletionsPath(req.URL.Path) + isAnthropic := isCopilotAnthropicMessagesPath(req.URL.Path) + model := "" + + // For completion requests, buffer the body so we can (a) learn the + // model id and (b) inject include_usage on streaming requests. Other + // requests (model listings, etc.) are forwarded without buffering. + if isCompletion && req.Body != nil { + body, readErr := io.ReadAll(io.LimitReader(req.Body, copilotSniffBodyLimit)) + _ = req.Body.Close() + if readErr != nil { + _ = client.SetReadDeadline(time.Time{}) + p.logTimeout("copilot sniff: request body read timed out", readErr, "agent", agentName, "host", host, "path", req.URL.Path) + return + } + var streaming bool + model, streaming = parseCopilotRequest(body) + // stream_options is OpenAI-only; Anthropic streams always carry + // usage in message_start/message_delta. + if streaming && !isAnthropic { + body = ensureStreamUsageRequested(body) + } + req.Body = io.NopCloser(bytes.NewReader(body)) + req.ContentLength = int64(len(body)) + req.Header.Set("Content-Length", fmt.Sprintf("%d", len(body))) + } + + _ = upstream.SetWriteDeadline(time.Now().Add(httpWriteTimeout)) + if err := req.Write(upstream); err != nil { + _ = upstream.SetWriteDeadline(time.Time{}) + _ = client.SetReadDeadline(time.Time{}) + p.logTimeout("copilot sniff: request relay timed out", err, "agent", agentName, "host", host, "path", req.URL.Path) + return + } + _ = upstream.SetWriteDeadline(time.Time{}) + _ = client.SetReadDeadline(time.Time{}) + + _ = upstream.SetReadDeadline(time.Now().Add(httpReadTimeout)) + resp, err := http.ReadResponse(upstreamBuf, req) + _ = upstream.SetReadDeadline(time.Time{}) + if err != nil { + p.logTimeout("copilot sniff: upstream response read timed out", err, "agent", agentName, "host", host, "path", req.URL.Path) + return + } + + if isAnthropic && resp.StatusCode >= 200 && resp.StatusCode < 300 { + if err := p.forwardCopilotAnthropicResponseWithUsage(client, resp, host, agentName, model); err != nil { + p.logTimeout("copilot sniff: response relay timed out", err, "agent", agentName, "host", host, "path", req.URL.Path) + return + } + } else if isCompletion && resp.StatusCode >= 200 && resp.StatusCode < 300 { + p.forwardCopilotResponseWithUsage(client, resp, host, agentName, model) + } else { + // Non-completion and non-2xx responses are relayed verbatim. The + // Write below can legitimately stay open for a long time, so it + // must be bounded by stall, not by total duration — see + // stallBoundedWriter. + if err := resp.Write(&stallBoundedWriter{conn: client, idle: httpWriteTimeout}); err != nil { + _ = resp.Body.Close() + _ = client.SetWriteDeadline(time.Time{}) + p.logTimeout("copilot sniff: response relay timed out", err, "agent", agentName, "host", host, "path", req.URL.Path) + return + } + _ = client.SetWriteDeadline(time.Time{}) + _ = resp.Body.Close() + } + + if req.Close || resp.Close { + return + } + } +} + +// forwardCopilotAnthropicResponseWithUsage relays an Anthropic Messages +// (/v1/messages) response to the client while teeing the body through a usage +// parser, so streamed Claude completions are forwarded as they arrive rather +// than buffered. Usage (including prompt-cache counts) is recorded once the +// body has been fully relayed. resp.Body is consumed and closed here. +func (p *GitHubProxy) forwardCopilotAnthropicResponseWithUsage(client net.Conn, resp *http.Response, host, agentName, model string) error { + tee := newAnthropicUsageTee(resp.Body, resp.Header.Get("Content-Type")) + resp.Body = tee + writeErr := resp.Write(&stallBoundedWriter{conn: client, idle: httpWriteTimeout}) + _ = resp.Body.Close() + _ = client.SetWriteDeadline(time.Time{}) + + // Record whatever was observed even if the client went away mid-stream: + // the upstream already billed those tokens. + u := tee.usage() + if u.total() > 0 { + resolvedModel := model + if resolvedModel == "" || resolvedModel == copilotAutoModelSentinel { + resolvedModel = u.Model + } + if resolvedModel != "" && resolvedModel != copilotAutoModelSentinel { + p.recordCopilotUsageWithCache(agentName, resolvedModel, u.Input, u.Output, u.CacheRead, u.CacheCreation) + p.logger.Info("copilot usage recorded", + "agent", agentName, + "host", host, + "model", resolvedModel, + "input", u.Input, + "output", u.Output, + "cache_read", u.CacheRead, + "cache_creation", u.CacheCreation, + ) + } + } + return writeErr +} + +// forwardCopilotResponseWithUsage forwards a completion response to the client +// while extracting token usage from a buffered copy. The full body is read into +// memory (completion responses are small), usage is recorded, then the response +// is rewritten to the client verbatim. resp.Body is consumed and closed here. +func (p *GitHubProxy) forwardCopilotResponseWithUsage(client net.Conn, resp *http.Response, host, agentName, model string) { + body, err := io.ReadAll(io.LimitReader(resp.Body, copilotSniffBodyLimit)) + _ = resp.Body.Close() + if err != nil { + // Best effort: nothing to forward. + return + } + + contentType := resp.Header.Get("Content-Type") + if in, out := extractCopilotUsage(contentType, body); in > 0 || out > 0 { + resolvedModel := model + if resolvedModel == "" || resolvedModel == copilotAutoModelSentinel { + // Fall back to the model reported in the response when the request + // did not pin a concrete id (e.g. --model auto). Never record the + // "auto" sentinel — it is not a real model row. + if rm := extractCopilotResponseModel(body, contentType); rm != "" { + resolvedModel = rm + } + } + if resolvedModel != "" && resolvedModel != copilotAutoModelSentinel { + p.recordCopilotUsage(agentName, resolvedModel, in, out) + p.logger.Info("copilot usage recorded", + "agent", agentName, + "host", host, + "model", resolvedModel, + "input", in, + "output", out, + ) + } + } + + // Rewrite the response to the client verbatim. Replace the body with the + // buffered copy and drop Content-Length ambiguity by setting it explicitly. + resp.Body = io.NopCloser(bytes.NewReader(body)) + resp.ContentLength = int64(len(body)) + if err := resp.Write(&stallBoundedWriter{conn: client, idle: httpWriteTimeout}); err != nil { + p.logger.Warn("copilot sniff: response write to client failed", "agent", agentName, "error", err) + } + _ = client.SetWriteDeadline(time.Time{}) +} diff --git a/src/crux/proxy/github_proxy_identity.go b/src/crux/proxy/github_proxy_identity.go new file mode 100644 index 0000000000..03d95e5366 --- /dev/null +++ b/src/crux/proxy/github_proxy_identity.go @@ -0,0 +1,181 @@ +package proxy + +import ( + "encoding/base64" + "fmt" + "net" + "net/http" + "os" + "strings" + + "github.com/hivecommons/hive/crux/agent" +) + +// identifyAgentFromReq determines the agent name for a request. It always +// tries UID-based identification first (unforgeable, works for any client +// including native binaries that don't send Proxy-Authorization); see +// fallbackAgentName for what happens when that fails. +func (p *GitHubProxy) identifyAgentFromReq(r *http.Request) string { + if p.uidMap != nil { + if name := p.identifyAgentByUID(r.RemoteAddr); name != "" { + return name + } + } + return p.fallbackAgentName(r) +} + +// IdentifyAgentByUID names the agent behind an http.Server-delivered request +// from the socket owner's UID ALONE. Unlike identifyAgentFromReq it never +// falls back to the caller-controlled Proxy-Authorization header, not even +// under HIVE_PROXY_ADVISORY_OK: see fallbackAgentName for why a self-asserted +// name is impersonation, and for the loopback services beside the proxy (the +// Jev decision endpoint, hivecommons/hive#8939) there is no "degraded but +// functional" mode worth that trade — an unidentified caller is refused. +// Returns "" when no UID map was loaded or the connection's UID is unmapped. +func (p *GitHubProxy) IdentifyAgentByUID(r *http.Request) string { + if p.uidMap == nil || r == nil { + return "" + } + return p.identifyAgentByUID(r.RemoteAddr) +} + +// identifyAgentFromConn identifies the calling agent for a request read off a +// raw connection. It MUST be used instead of identifyAgentFromReq on any path +// where the request came from http.ReadRequest rather than http.Server: +// ReadRequest does not populate Request.RemoteAddr (only http.Server does), so +// the UID lookup silently gets an empty address, finds no agent, and the caller +// falls back to ADVISORY — downgrading every write-capable agent to read-only. +// Taking the peer address from the connection itself removes that trap. +func (p *GitHubProxy) identifyAgentFromConn(conn net.Conn, r *http.Request) string { + if p.uidMap != nil && conn != nil { + if name := p.identifyAgentByUID(conn.RemoteAddr().String()); name != "" { + return name + } + } + return p.fallbackAgentName(r) +} + +// fallbackAgentName is reached whenever unforgeable UID-based identification +// did NOT succeed — either p.uidMap is nil (never loaded: no entrypoint root +// phase ran to write uid-map.json, e.g. local dev, a native/systemd install, +// or a genuine boot-ordering failure) or this specific connection's UID has +// no entry in a map that DID load (N7, #3841). +// +// Before this fix, both cases fell through to extractAgentName — the +// Proxy-Authorization header, which the CALLER sets. Any process that can +// reach the proxy's loopback port can send `Proxy-Authorization: hive +// ` and be identified as that name: a spoke-agent process claiming +// to be a MORE privileged agent bypasses the whole capability model this +// proxy exists to enforce, and #1861 (inject GitHub auth at the proxy so +// agents never hold tokens) would turn that spoofed identity directly into a +// minted token for the impersonated agent. That is strictly worse than +// simply refusing to identify the caller. +// +// The safe default is therefore to NOT trust the header: return "", which +// readAgentMode already treats as ModeAdvisory (the most restrictive mode — +// writes blocked, per the existing "agent could not be identified" WARN a few +// lines below in proxyHTTP). This is not new plumbing; it is the same path an +// unidentified caller already hits today, now reached deliberately instead of +// only when the header happens to be absent. +// +// The one deliberate exception is proxyAdvisoryOK (HIVE_PROXY_ADVISORY_OK=true) +// — the SAME explicit, operator-set escape hatch entrypoint.sh already uses +// for "this deployment does not enforce the full forced-egress gate" (local +// dev, a native/systemd install with no per-agent UID separation at all, +// platforms without NET_ADMIN). An operator who has already accepted a +// degraded capability model there is not newly exposed by also accepting a +// self-asserted identity within it — and without this exception, those +// deployments would lose agent identification (and therefore write access) +// entirely, which would be a functional regression, not a security fix. +func (p *GitHubProxy) fallbackAgentName(r *http.Request) string { + if !p.proxyAdvisoryOK { + return "" + } + return extractAgentName(r) +} + +// identifyAgentByUID reads /proc/net/tcp to find the UID of the process +// that owns the socket connected to the proxy, then looks up the agent name. +func (p *GitHubProxy) identifyAgentByUID(remoteAddr string) string { + uid, err := uidForRemoteAddr(remoteAddr) + if err != nil { + return "" + } + return p.uidMap.LookupByUID(uid) +} + +// uidForRemoteAddr resolves the UID of the process that owns the local socket +// behind remoteAddr ("host:port" as http.Server and net.Conn report it) via +// /proc/net/tcp. It is the shared first half of every UID-attribution path. +func uidForRemoteAddr(remoteAddr string) (int, error) { + _, portStr, err := net.SplitHostPort(remoteAddr) + if err != nil { + return -1, err + } + port := 0 + const maxPort = 65535 + for _, c := range portStr { + if c < '0' || c > '9' { + return -1, fmt.Errorf("remote port %q is not numeric", portStr) + } + port = port*10 + int(c-'0') + if port > maxPort { + return -1, fmt.Errorf("remote port %q out of range", portStr) + } + } + return LookupUIDByLocalPort(port) +} + +// extractAgentName reads the agent name from the Proxy-Authorization header. +// Supports "hive " (custom) and "Basic " (standard HTTP proxy auth +// sent automatically when the proxy URL contains userinfo, e.g. http://quality@host:port). +func extractAgentName(r *http.Request) string { + auth := r.Header.Get("Proxy-Authorization") + if auth == "" { + return "" + } + if strings.HasPrefix(auth, "hive ") { + return strings.TrimPrefix(auth, "hive ") + } + if strings.HasPrefix(auth, "Basic ") { + decoded, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(auth, "Basic ")) + if err != nil { + return "" + } + // Format is "username:password" — agent name is the username portion. + user, _, _ := strings.Cut(string(decoded), ":") + return user + } + return "" +} + +// readAgentMode reads the mode from the hot-reloadable mode file. +// readAgentCaps loads the agent's orthogonal capability set (#4492) from +// /tmp/.hive-caps-. Every failure path returns the zero value — no +// capabilities — so an unreadable, missing or malformed file degrades toward +// the tier check alone, which is the deny direction. +func readAgentCaps(agentName string) agent.AgentCapabilities { + if agentName == "" { + return agent.AgentCapabilities{} + } + data, err := os.ReadFile(capsFilePrefix + agentName) + if err != nil { + return agent.AgentCapabilities{} + } + return agent.ParseCapabilities(strings.TrimSpace(string(data))) +} + +func readAgentMode(agentName string) agent.AgentMode { + if agentName == "" { + return agent.ModeAdvisory + } + data, err := os.ReadFile(modeFilePrefix + agentName) + if err != nil { + return agent.ModeAdvisory + } + mode, ok := agent.ParseAgentMode(strings.TrimSpace(string(data))) + if !ok { + return agent.ModeAdvisory + } + return mode +} diff --git a/src/crux/proxy/github_proxy_inference.go b/src/crux/proxy/github_proxy_inference.go new file mode 100644 index 0000000000..fb71526951 --- /dev/null +++ b/src/crux/proxy/github_proxy_inference.go @@ -0,0 +1,752 @@ +package proxy + +import ( + "bufio" + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "strings" + "time" + + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/inferencehealth" +) + +// inferenceAPIKeyPrefix is what the manager prepends to the agent's name to +// form the ANTHROPIC_API_KEY it hands each inference-routed agent +// (agent/manager_env.go). The suffix is the agent's NAME, not a secret. +const inferenceAPIKeyPrefix = "sk-hive-" + +// translatorAgentName names the caller of the inference translator. +// +// The x-api-key header the Claude CLI sends is "sk-hive-", and the +// agent name is public inside the pod: every agent knows every other agent's +// name. Trusting the header alone lets any process on the loopback borrow +// another agent's inference route — its backend, model, upstream credential +// and usage attribution — which is exactly the self-asserted-identity hole +// closed for the GitHub path in fallbackAgentName and for the decision +// endpoint in IdentifyAgentByUID. So the socket owner's UID decides: +// +// - a UID mapped to an agent is authoritative; a header naming someone else +// is logged and overridden; +// - the root control plane keeps the header name — the hive is the party +// that issues these keys; the proxy UID cannot assert a name because its +// upstream sockets can relay agent-controlled requests; +// - an unmapped UID, or no UID map at all, accepts the header only under +// HIVE_PROXY_ADVISORY_OK, the same operator escape hatch the GitHub path +// honours for deployments without per-agent UID separation. +// +// The second return is false when the caller could not be identified and the +// request must be refused. +func (p *GitHubProxy) translatorAgentName(r *http.Request) (string, bool) { + claimed := strings.TrimPrefix(r.Header.Get("x-api-key"), inferenceAPIKeyPrefix) + if p.uidMap == nil { + if p.proxyAdvisoryOK { + return claimed, true + } + return "", false + } + uid, err := uidForRemoteAddr(r.RemoteAddr) + if err == nil { + if name := p.uidMap.LookupByUID(uid); name != "" { + if claimed != "" && claimed != name { + p.logger.Warn("inference identity: x-api-key names another agent; using the socket owner", + "agent", name, "claimed", claimed, "uid", uid) + } + return name, true + } + if uid == 0 && p.uidMap.IsInternalUID(uid) { + return claimed, true + } + } + if p.proxyAdvisoryOK { + return claimed, true + } + return "", false +} + +// SetInferenceRoute configures an agent to use a self-hosted inference backend. +// +// The route is stored immediately and the caller returns without blocking. The +// model's max context length is queried in the background because this function +// is invoked from callers that hold the agent-manager mutex (agent launch, +// SetModelOverride, SetBackendOverride). A synchronous endpoint probe here +// (up to maxModelLenQueryTimeout) would pin that mutex for its whole duration, +// stalling /api/status and other dashboard handlers — which on a single-replica +// pod reads to the OpenShift router as "Application is not available". The +// max-context-len only feeds token capping and defaults gracefully to 0 +// (no cap) until the async probe fills it in. +func (p *GitHubProxy) SetInferenceRoute(agentName string, route *InferenceRoute) { + p.inference.Set(agentName, route) + keySHA256 := config.APIKeySHA256(route.APIKey) + if keySHA256 != "" { + p.logger.Info("inference route set", "agent", agentName, "backend", route.Backend, "endpoint", route.Endpoint, "model", route.Model, "maxContextLen", route.MaxContextLen, "api_key_sha256", keySHA256) + } else { + p.logger.Info("inference route set", "agent", agentName, "backend", route.Backend, "endpoint", route.Endpoint, "model", route.Model, "maxContextLen", route.MaxContextLen, "api_key_sha256", "") + } + + if route.MaxContextLen == 0 { + endpoint, model, apiKey, caBundle := route.Endpoint, route.Model, route.APIKey, route.CABundle + go func() { + if maxLen := queryMaxModelLen(endpoint, model, apiKey, caBundle); maxLen > 0 { + if p.inference.UpdateMaxContextLen(agentName, endpoint, model, maxLen) { + p.logger.Info("inference route max context length resolved", "agent", agentName, "model", model, "maxContextLen", maxLen) + } + } + }() + } + + // Best-effort: learn the key's entitled model set from a LiteLLM + // key-info endpoint before any 403 happens, so the dashboard can offer + // only usable models up front. Only litellm gateways entitlement-filter + // per key; vllm/llm-d do not. The 403-parse path (recordInferenceError) + // remains the authoritative fallback when no key-info endpoint responds. + if route.Backend == "litellm" && route.APIKey != "" { + p.entitlements.rememberProbe(route.Endpoint, route.APIKey, route.CABundle) + go p.probeEntitlements(route.Endpoint, route.APIKey, route.CABundle) + } +} + +// probeEntitlements queries a LiteLLM gateway's per-key info endpoints for the +// key's entitled model set and caches it. A nil result (no key-info endpoint, +// or an unrestricted key) leaves any existing 403-derived entry untouched. +func (p *GitHubProxy) probeEntitlements(endpoint, apiKey, caBundle string) { + if !p.entitlements.beginProbe(endpoint) { + return // another probe for this endpoint is already in flight + } + defer p.entitlements.endProbe(endpoint) + transport, err := inferenceTransport(caBundle) + if err != nil { + transport = nil + } + if models := probeEntitledModels(endpoint, apiKey, transport); len(models) > 0 { + p.entitlements.set(endpoint, apiKey, models, "key-info") + p.logger.Info("litellm entitled models discovered via key-info", + "endpoint", endpoint, "count", len(models)) + } +} + +// EntitledModels returns the entitled model set known for a LiteLLM endpoint +// (from a key-info probe or a "team not allowed" 403), the signal source, and +// whether the set is known. It is the accessor the dashboard uses to narrow +// model dropdowns to what the configured key can actually use. +// +// A stale entry (older than entitlementTTL) is still returned so the served +// model list stays STABLE for an unchanged key — a filtered list that flapped +// back to the full catalog between probes would make the dashboard's +// auto-heal (see reconcileModelsAfterDiscovery, #1848) see models reappear +// and vanish across refreshes. Staleness instead triggers a bounded +// background re-probe that overwrites the entry only on a fresh signal. +func (p *GitHubProxy) EntitledModels(endpoint string) (models []string, source string, known bool) { + models, source, known, stale := p.entitlements.get(endpoint) + if stale { + if ep, apiKey, caBundle, ok := p.entitlements.probeParams(endpoint); ok { + go p.probeEntitlements(ep, apiKey, caBundle) + } + } + return models, source, known +} + +// recordInferenceError inspects an upstream inference error body for a +// LiteLLM "team not allowed to access model" 403 and, when found, caches the +// entitled model set it lists (keyed by the route's endpoint) so the +// dashboard's model list narrows to the usable set. Only a 403 carrying the +// gateway's team-scope marker is treated as an entitlement signal — +// provider-side failures relayed by the gateway (upstream 404 not-found, 400 +// bad-request, 5xx) say nothing about the key's scope and are ignored, as is +// any other 403 (bad key, rate limit). +// +// It deliberately does NOT switch the agent's route itself: automatic model +// switching is governed by the dashboard's reconcile policy (#1848 — unpinned +// + governor suggestion, or confirmed genuine unavailability, at most once +// per list change). Once the entitled set is cached here, the dashboard's +// filtered discovery list drives that heal path for any agent left on a +// non-entitled model. +func (p *GitHubProxy) recordInferenceError(route *InferenceRoute, agentName string, status int, body []byte) { + if route == nil { + return + } + if p.gatewayHealth != nil { + p.gatewayHealth.RecordEndpointHTTPError(route.Backend, route.Endpoint, status, string(truncateBytes(body, 200)), time.Now()) + } + + // Inference-backend AUTH failure (a stale/invalid gateway key returning 401 + // on every call). Tracked for ANY inference backend, not just litellm — a + // self-hosted vLLM/llm-d fronted by auth can 401 too. Latches only after + // inferenceAuthFailureThreshold consecutive failures and clears on the next + // success, so a single transient blip never alarms. The stored cause is + // log-safe: the gateway's 401 body describes the rejection without echoing + // the presented token, and inferenceAuthErrorMessage never reads the key. + if isInferenceAuthStatus(status) && p.inferenceAuth != nil { + msg := inferenceAuthErrorMessage(route.Backend, status, truncateBytes(body, 200)) + p.inferenceAuth.recordFailure(msg, time.Now()) + p.logger.Warn("inference backend auth failure", + "agent", agentName, "backend", route.Backend, "endpoint", route.Endpoint, "status", status) + } + + // PROVIDER SPEND REBUFF (#4294). Checked before the 403/entitlement branch + // below because that branch returns early for every non-403, which would + // skip the 429/400 statuses a budget refusal actually arrives on. + // + // Requires the body to name spend, so an ordinary 429 throttle keeps + // flowing to the existing transient-retry path untouched. + if p.inferenceBudget != nil && isInferenceBudgetRebuff(status, body) { + msg := inferenceBudgetMessage(route.Backend, status, truncateBytes(body, 200)) + p.inferenceBudget.recordRebuff(msg, time.Now()) + p.logger.Error("inference backend refused on a spending limit", + "agent", agentName, "backend", route.Backend, "endpoint", route.Endpoint, + "status", status, "cause", msg) + } + + if route.Backend != "litellm" || status != http.StatusForbidden { + return + } + entitled := parseTeam403Models(string(body)) + if len(entitled) == 0 { + return + } + p.entitlements.set(route.Endpoint, route.APIKey, entitled, "403") + p.logger.Info("litellm entitled models learned from 403", + "agent", agentName, "endpoint", route.Endpoint, "count", len(entitled), "model", route.Model) +} + +// routeWithEntitledModel returns the route to actually forward with: when the +// gateway's entitled model set is known (key-info probe or a prior team-scope +// 403) and the route's stored model differs from an entitled id only by +// provider-prefix / dot-hyphen spelling drift, a COPY of the route carrying +// the exact entitled id is returned. LiteLLM matches the request model string +// verbatim, so without this a stored copilot-spelled id (e.g. +// "claude-sonnet-4.6" vs entitled "aws/claude-sonnet-4-6") 403s on every call +// forever (#4400) — while the dashboard's tolerant preselect matcher shows the +// agent on a perfectly valid model. Self-healing: the first 403 teaches the +// entitled set (recordInferenceError), so the next request resolves. +// +// When the stored model is NOT entitled at all — verbatim or by drift — and +// the backend's DefaultModel IS, the request goes out for the default instead +// (hivecommons/hive#9945). The FMA hive's agents carried a `model: +// claude-sonnet-4-6` written for their original copilot backend into a +// switch to the litellm gateway whose default_model was the only model the +// team could use; the per-agent value won, and every call 403'd forever with +// "team not allowed to access model" while the operator stared at a gateway +// card showing the right default. The gateway's own entitlement signal (the +// key-info probe or that very 403) is what authorises the substitution, so a +// model the team can't use is never sent when one it can use is configured. +// The stored route is still untouched: a key/team change that re-entitles the +// per-agent model takes effect on the next request. +// +// The stored route is never mutated: the rewrite is per-request, so a +// key/team change that re-entitles the verbatim id takes effect immediately, +// and concurrent requests race on nothing. Unknown entitlements, verbatim +// matches, and ambiguous candidates all pass the route through unchanged. +func (p *GitHubProxy) routeWithEntitledModel(route *InferenceRoute, agentName string) *InferenceRoute { + if route == nil || route.Backend != "litellm" || p.entitlements == nil { + return route + } + entitled, _, known := p.EntitledModels(route.Endpoint) + if !known { + return route + } + if entitledModelListed(entitled, route.Model) { + return route + } + if resolved, ok := resolveEntitledModelID(entitled, route.Model); ok { + p.logger.Info("inference model resolved to entitled gateway id", + "agent", agentName, "endpoint", route.Endpoint, + "configured", route.Model, "resolved", resolved) + rewritten := *route + rewritten.Model = resolved + return &rewritten + } + fallback := strings.TrimSpace(route.DefaultModel) + if fallback == "" || fallback == route.Model { + return route + } + if !entitledModelListed(entitled, fallback) { + resolved, ok := resolveEntitledModelID(entitled, fallback) + if !ok { + return route + } + fallback = resolved + } + p.logger.Warn("inference model not entitled on gateway; sending the backend default model instead", + "agent", agentName, "backend", route.Backend, "endpoint", route.Endpoint, + "configured", route.Model, "default", fallback, + "remedy", "clear the agent's model (or pick one of the gateway's entitled models) so the agent uses the gateway default") + rewritten := *route + rewritten.Model = fallback + return &rewritten +} + +// entitledModelListed reports whether model appears verbatim in entitled. +func entitledModelListed(entitled []string, model string) bool { + for _, id := range entitled { + if id == model { + return true + } + } + return false +} + +// recordInferenceSuccess records a successful inference call so a previously +// latched auth-failure signal clears — the self-heal for the inference-auth +// health signal. Called from every inference forward path on a 2xx response. +// A nil tracker (impossible after NewGitHubProxy, but cheap to guard) is a +// no-op. +func (p *GitHubProxy) recordInferenceSuccessFor(route *InferenceRoute) { + if route != nil && p.gatewayHealth != nil { + p.gatewayHealth.Clear(route.Backend) + } + p.recordInferenceSuccess() +} + +func (p *GitHubProxy) recordInferenceSuccess() { + if p.inferenceAuth != nil { + p.inferenceAuth.recordSuccess() + } + // Self-heal for the spend rebuff (#4294): whenever the provider's window + // resets — hive neither knows nor assumes the schedule; the field report's + // gateway rolled over at the key's creation time of day, not at midnight + // (see #4294) — the first call that succeeds takes the hive out of the + // suppressed state with no operator action. + if p.inferenceBudget != nil { + p.inferenceBudget.recordSuccess() + } +} + +// GatewayHealth reports currently failing inference gateways. +func (p *GitHubProxy) GatewayHealth() []inferencehealth.GatewayStatus { + if p == nil || p.gatewayHealth == nil { + return nil + } + return p.gatewayHealth.Snapshot() +} + +// InferenceAuthError reports the current inference-backend auth-failure signal: +// a non-empty, log-safe cause string (and the time it first latched) ONLY while +// the backend has been auth-failing for inferenceAuthFailureThreshold +// consecutive calls, empty otherwise. Wired to the dashboard/heartbeat so the +// hub can flag a hive that is silently 401'ing on every inference call. Safe on +// a nil tracker. +func (p *GitHubProxy) InferenceAuthError() (errMsg string, since time.Time) { + if p == nil || p.inferenceAuth == nil { + return "", time.Time{} + } + return p.inferenceAuth.snapshot() +} + +// InferenceBudgetExceeded reports the current provider spending-limit signal: +// a non-empty log-safe cause, when it first latched, when the most recent +// rebuff arrived, and how many rebuffs have been observed since — all +// zero-valued while the provider is serving normally. +// +// Callers use a non-empty cause as "this hive cannot buy inference right now", +// which is a reason to stop kicking agents and to tell the operator, NOT a +// reason to retry in a few minutes. lastRebuff is what bounds that: because +// withholding kicks also withholds the calls that would clear the latch, +// callers suppress only while lastRebuff is fresh and send a probe once it is +// stale (see inference_budget.go). Safe on a nil receiver/tracker. +func (p *GitHubProxy) InferenceBudgetExceeded() (errMsg string, since, lastRebuff time.Time, rebuffs int) { + if p == nil || p.inferenceBudget == nil { + return "", time.Time{}, time.Time{}, 0 + } + return p.inferenceBudget.snapshot() +} + +// ClearInferenceRoute removes an agent's inference backend override. +func (p *GitHubProxy) ClearInferenceRoute(agentName string) { + p.inference.Clear(agentName) + p.logger.Info("inference route cleared", "agent", agentName) +} + +// StartInferenceTranslator runs a plain HTTP server that accepts Anthropic +// Messages API requests and translates+forwards them to the configured +// inference backend. Agents use ANTHROPIC_BASE_URL=http://127.0.0.1:18444 +// to reach this server instead of api.anthropic.com. +func (p *GitHubProxy) inferenceTranslatorHandler() http.Handler { + mux := http.NewServeMux() + + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + agentName, ok := p.translatorAgentName(r) + if !ok { + p.logger.Warn("inference request from unidentified caller refused", + "remote", r.RemoteAddr, "method", r.Method, "path", r.URL.Path) + http.Error(w, `{"type":"error","error":{"type":"authentication_error","message":"caller could not be identified"}}`, http.StatusUnauthorized) + return + } + + p.logger.Info("inference request", + "agent", agentName, + "method", r.Method, + "path", r.URL.Path, + "content-type", r.Header.Get("Content-Type"), + "anthropic-version", r.Header.Get("anthropic-version"), + ) + + route := p.inference.Get(agentName) + if route == nil { + p.logger.Warn("inference no route", "agent", agentName) + http.Error(w, `{"type":"error","error":{"type":"api_error","message":"no inference route for agent"}}`, http.StatusBadGateway) + return + } + route = p.routeWithEntitledModel(route, agentName) + + body, err := io.ReadAll(r.Body) + if r.Body != nil { + _ = r.Body.Close() + } + if err != nil { + http.Error(w, `{"type":"error","error":{"type":"api_error","message":"failed to read request"}}`, http.StatusBadRequest) + return + } + + // Same path gate as the MITM reroute: only POST /v1/messages is + // translated and forwarded; CLI housekeeping is answered here. + if kind := classifyInferencePath(r.Method, r.URL.Path); kind != inferencePathMessages { + status, respBody := p.localInferenceResponse(kind, r.Method, r.URL.Path, body, agentName) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = io.WriteString(w, respBody) + return + } + + // Diagnostic: count tools in original Anthropic request. + var toolDiag struct { + Tools json.RawMessage `json:"tools"` + } + toolCount := -1 // -1 = no tools key + if json.Unmarshal(body, &toolDiag) == nil && toolDiag.Tools != nil { + var arr []json.RawMessage + if json.Unmarshal(toolDiag.Tools, &arr) == nil { + toolCount = len(arr) + } + } + p.logger.Info("inference request body", "agent", agentName, "len", len(body), "tool_count", toolCount, "preview", truncateBytes(body, 200)) + + openaiBody, err := translateAnthropicToOpenAI(body, route.Model, route.MaxContextLen, resolveInferencePreamble(route, agentName)) + if err != nil { + p.logger.Error("inference translate request failed", "agent", agentName, "error", err) + http.Error(w, fmt.Sprintf(`{"type":"error","error":{"type":"api_error","message":"translation error: %s"}}`, err.Error()), http.StatusBadGateway) + return + } + + upstreamURL := openAIChatCompletionsURL(route.Endpoint) + upstreamReq, err := http.NewRequestWithContext(r.Context(), "POST", upstreamURL, bytes.NewReader(openaiBody)) + if err != nil { + http.Error(w, `{"type":"error","error":{"type":"api_error","message":"failed to create upstream request"}}`, http.StatusBadGateway) + return + } + upstreamReq.Header.Set("Content-Type", "application/json") + applyInferenceAuth(upstreamReq, route) + + p.logger.Info("inference forward", + "agent", agentName, + "backend", route.Backend, + "model", route.Model, + "url", upstreamURL, + "openai_body", truncateBytes(openaiBody, 300), + ) + + client, err := inferenceHTTPClient(route) + if err != nil { + p.logger.Error("inference client setup failed", "agent", agentName, "error", err) + http.Error(w, fmt.Sprintf(`{"type":"error","error":{"type":"api_error","message":"inference client setup failed: %s"}}`, err.Error()), http.StatusBadGateway) + return + } + resp, err := client.Do(upstreamReq) + if err != nil { + p.logger.Error("inference upstream failed", "agent", agentName, "url", upstreamURL, "error", err) + if p.gatewayHealth != nil { + p.gatewayHealth.RecordEndpointError(route.Backend, route.Endpoint, err, time.Now()) + } + unreachableMsg := inferenceBackendUnreachableMessage(err, upstreamURL) + p.recordAgentInferenceError(agentName, unreachableMsg, 0) + http.Error(w, fmt.Sprintf(`{"type":"error","error":{"type":"api_error","message":"%s"}}`, jsonEscape(unreachableMsg)), http.StatusBadGateway) + return + } + defer func() { _ = resp.Body.Close() }() + + p.logger.Info("inference upstream response", + "agent", agentName, + "status", resp.StatusCode, + "content-type", resp.Header.Get("Content-Type"), + ) + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + errBody, _ := io.ReadAll(resp.Body) + p.logger.Error("inference upstream error", + "agent", agentName, + "status", resp.StatusCode, + "url", upstreamURL, + "server", resp.Header.Get("Server"), + "body", truncateBytes(errBody, 500), + ) + // A LiteLLM "team not allowed to access model" 403 carries the + // entitled set — cache it so the dashboard's model list narrows + // and its reconcile policy (#1848) can heal the agent. + p.recordInferenceError(route, agentName, resp.StatusCode, errBody) + backendMsg := inferenceBackendErrorMessage(resp.StatusCode, errBody, upstreamURL, resp.Header.Get("Server")) + p.recordAgentInferenceError(agentName, backendMsg, resp.StatusCode) + anthropicErr := fmt.Sprintf(`{"type":"error","error":{"type":"api_error","message":"%s"}}`, + jsonEscape(backendMsg)) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(resp.StatusCode) + _, _ = w.Write([]byte(anthropicErr)) + return + } + + // A 2xx means the gateway accepted the key — clear any latched + // inference-auth failure so a hive whose key was fixed self-heals. + p.recordInferenceSuccessFor(route) + p.clearAgentInferenceError(agentName) + + isStreaming := strings.Contains(resp.Header.Get("Content-Type"), "text/event-stream") + + if isStreaming { + w.Header().Set("Content-Type", "text/event-stream; charset=utf-8") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + w.WriteHeader(http.StatusOK) + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + in, out, err := translateOpenAISSEToAnthropic(resp.Body, w, route.Model) + if err != nil { + p.logger.Error("inference SSE translation failed", "agent", agentName, "error", err) + } + if in > 0 || out > 0 { + p.recordUsage(agentName, route.Model, in, out) + } + return + } + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + http.Error(w, `{"type":"error","error":{"type":"api_error","message":"failed to read inference response"}}`, http.StatusBadGateway) + return + } + + p.logger.Info("inference upstream raw response", + "agent", agentName, + "len", len(respBody), + "preview", truncateBytes(respBody, 500), + ) + + if in, out := extractOpenAIUsage(respBody); in > 0 || out > 0 { + p.recordUsage(agentName, route.Model, in, out) + } + + translated, err := translateOpenAIResponseToAnthropic(respBody, route.Model) + if err != nil { + p.logger.Error("inference translate response failed", "agent", agentName, "error", err) + http.Error(w, `{"type":"error","error":{"type":"api_error","message":"response translation error"}}`, http.StatusBadGateway) + return + } + + p.logger.Info("inference translated response", + "agent", agentName, + "len", len(translated), + "preview", truncateBytes(translated, 500), + ) + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(translated) + }) + + return mux +} + +// StartInferenceTranslator runs the Anthropic-compatible inference translator +// on the fixed local port used by agents. +func (p *GitHubProxy) StartInferenceTranslator() error { + addr := fmt.Sprintf("127.0.0.1:%d", InferenceTranslatePort) + p.logger.Info("inference translation server starting", "addr", addr) + server := &http.Server{ + Addr: addr, + Handler: p.inferenceTranslatorHandler(), + ReadHeaderTimeout: httpReadTimeout, + } + return server.ListenAndServe() +} + +// handleAnthropicReroute performs MITM on an Anthropic API connection and +// reroutes requests to a self-hosted vLLM/llm-d endpoint, translating +// between Anthropic and OpenAI API formats. +func (p *GitHubProxy) handleAnthropicReroute(conn net.Conn, r *http.Request, host, agentName string, route *InferenceRoute) { + p.logger.Info("inference reroute", "agent", agentName, "backend", route.Backend, "host", host) + + if _, err := fmt.Fprintf(conn, "HTTP/1.1 200 Connection established\r\n\r\n"); err != nil { + p.logger.Error("inference reroute: CONNECT response failed", "error", err) + return + } + + tlsCert, err := p.forgeCert(host) + if err != nil { + p.logger.Error("inference reroute: forge cert failed", "host", host, "error", err) + return + } + + tlsConn := tls.Server(conn, &tls.Config{ + Certificates: []tls.Certificate{tlsCert}, + }) + _ = conn.SetDeadline(time.Now().Add(clientTLSHandshakeTimeout)) + handshakeErr := tlsConn.Handshake() + _ = conn.SetDeadline(time.Time{}) + if handshakeErr != nil { + p.logger.Warn("inference reroute: TLS handshake failed", "error", handshakeErr) + return + } + defer func() { _ = tlsConn.Close() }() + + clientBuf := bufio.NewReader(tlsConn) + for { + _ = tlsConn.SetReadDeadline(time.Now().Add(httpReadTimeout)) + req, err := http.ReadRequest(clientBuf) + _ = tlsConn.SetReadDeadline(time.Time{}) + if err != nil { + p.logTimeout("inference reroute: client request read timed out", err, "agent", agentName) + return + } + + p.handleInferenceRequest(tlsConn, req, agentName, route) + } +} + +// handleInferenceRequest translates a single Anthropic API request and +// forwards it to the inference backend. +func (p *GitHubProxy) handleInferenceRequest(conn net.Conn, req *http.Request, agentName string, route *InferenceRoute) { + route = p.routeWithEntitledModel(route, agentName) + _ = conn.SetReadDeadline(time.Now().Add(httpReadTimeout)) + body, err := io.ReadAll(req.Body) + _ = conn.SetReadDeadline(time.Time{}) + if req.Body != nil { + _ = req.Body.Close() + } + if err != nil { + p.logTimeout("inference reroute: request body read timed out", err, "agent", agentName) + p.writeHTTPError(conn, http.StatusBadGateway, "failed to read request body") + return + } + + // Only POST /v1/messages is an inference call. The Claude Code CLI also + // sends telemetry batches, error reports, and count_tokens to this host; + // translating those bodies produced {"messages": null} and a gateway 400 + // per call, charged against the provider's request rate limit. + if kind := classifyInferencePath(req.Method, req.URL.Path); kind != inferencePathMessages { + status, respBody := p.localInferenceResponse(kind, req.Method, req.URL.Path, body, agentName) + writeLocalInferenceResponse(conn, status, respBody) + return + } + + openaiBody, err := translateAnthropicToOpenAI(body, route.Model, route.MaxContextLen, resolveInferencePreamble(route, agentName)) + if err != nil { + p.logger.Error("inference translate request failed", "agent", agentName, "error", err) + p.writeHTTPError(conn, http.StatusBadGateway, "translation error: "+err.Error()) + return + } + + upstreamURL := openAIChatCompletionsURL(route.Endpoint) + upstreamReq, err := http.NewRequestWithContext(req.Context(), "POST", upstreamURL, bytes.NewReader(openaiBody)) + if err != nil { + p.writeHTTPError(conn, http.StatusBadGateway, "failed to create upstream request") + return + } + upstreamReq.Header.Set("Content-Type", "application/json") + applyInferenceAuth(upstreamReq, route) + + p.logger.Info("inference forward", "agent", agentName, "backend", route.Backend, "model", route.Model, "url", upstreamURL) + + client, err := inferenceHTTPClient(route) + if err != nil { + p.logger.Error("inference client setup failed", "agent", agentName, "error", err) + p.writeHTTPError(conn, http.StatusBadGateway, "inference client setup failed: "+err.Error()) + return + } + resp, err := client.Do(upstreamReq) + if err != nil { + p.logger.Error("inference upstream failed", "agent", agentName, "url", upstreamURL, "error", err) + if p.gatewayHealth != nil { + p.gatewayHealth.RecordEndpointError(route.Backend, route.Endpoint, err, time.Now()) + } + unreachableMsg := inferenceBackendUnreachableMessage(err, upstreamURL) + p.recordAgentInferenceError(agentName, unreachableMsg, 0) + p.writeHTTPError(conn, http.StatusBadGateway, unreachableMsg) + return + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + errBody, _ := io.ReadAll(resp.Body) + p.logger.Error("inference upstream error", "agent", agentName, "status", resp.StatusCode, "url", upstreamURL, "server", resp.Header.Get("Server"), "body", truncateBytes(errBody, 500)) + // Same entitlement capture as the HTTP translator path. + p.recordInferenceError(route, agentName, resp.StatusCode, errBody) + backendMsg := inferenceBackendErrorMessage(resp.StatusCode, errBody, upstreamURL, resp.Header.Get("Server")) + p.recordAgentInferenceError(agentName, backendMsg, resp.StatusCode) + anthropicErr := &http.Response{ + StatusCode: resp.StatusCode, + ProtoMajor: 1, + ProtoMinor: 1, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(fmt.Sprintf( + `{"type":"error","error":{"type":"api_error","message":"%s"}}`, + jsonEscape(backendMsg)))), + } + _ = anthropicErr.Write(conn) + return + } + + // A 2xx means the gateway accepted the key — clear any latched + // inference-auth failure so a hive whose key was fixed self-heals. + p.recordInferenceSuccessFor(route) + p.clearAgentInferenceError(agentName) + + isStreaming := strings.Contains(resp.Header.Get("Content-Type"), "text/event-stream") + + if isStreaming { + hdr := "HTTP/1.1 200 OK\r\n" + + "Content-Type: text/event-stream; charset=utf-8\r\n" + + "Cache-Control: no-cache\r\n" + + "Connection: keep-alive\r\n\r\n" + if _, err := conn.Write([]byte(hdr)); err != nil { + return + } + in, out, err := translateOpenAISSEToAnthropic(resp.Body, conn, route.Model) + if err != nil { + p.logger.Error("inference SSE translation failed", "agent", agentName, "error", err) + } + if in > 0 || out > 0 { + p.recordUsage(agentName, route.Model, in, out) + } + return + } + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + p.writeHTTPError(conn, http.StatusBadGateway, "failed to read inference response") + return + } + + if in, out := extractOpenAIUsage(respBody); in > 0 || out > 0 { + p.recordUsage(agentName, route.Model, in, out) + } + + translated, err := translateOpenAIResponseToAnthropic(respBody, route.Model) + if err != nil { + p.logger.Error("inference translate response failed", "agent", agentName, "error", err) + p.writeHTTPError(conn, http.StatusBadGateway, "response translation error") + return + } + + httpResp := &http.Response{ + StatusCode: http.StatusOK, + ProtoMajor: 1, + ProtoMinor: 1, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewReader(translated)), + } + _ = httpResp.Write(conn) +} diff --git a/src/crux/proxy/github_proxy_mitm.go b/src/crux/proxy/github_proxy_mitm.go new file mode 100644 index 0000000000..da81467ffe --- /dev/null +++ b/src/crux/proxy/github_proxy_mitm.go @@ -0,0 +1,402 @@ +package proxy + +import ( + "encoding/json" + "io" + "net" + "net/http" + "strings" + "time" + + "github.com/hivecommons/hive/crux/agent" +) + +// proxyHTTP relays a MITM'd GitHub connection with ACMM enforcement applied. +// It is proxyHTTPHost with the GitHub host implied, which is what every +// pre-existing caller means. +func (p *GitHubProxy) proxyHTTP(client net.Conn, upstream net.Conn, agentName string, mode agent.AgentMode, caps agent.AgentCapabilities) { + p.proxyHTTPHost(client, upstream, "", agentName, mode, caps) +} + +// proxyHTTPHost is proxyHTTP with the terminated host carried through, so the +// request loop can pick the right rule engine (#4492 F). +// +// The host is threaded rather than inferred from the request because a MITM'd +// request line is a path, not an absolute URL — by this point the only record +// of which host the TLS session was forged for is the CONNECT that opened it. +// An empty host means GitHub, preserving every existing call site verbatim. +func (p *GitHubProxy) proxyHTTPHost(client net.Conn, upstream net.Conn, host string, agentName string, mode agent.AgentMode, caps agent.AgentCapabilities) { + isLinear := IsLinearHost(host) + clientBuf := newBufferedConn(client) + + for { + // Bound every phase of this request/response exchange. Before this, + // the relay had NO deadlines anywhere after the initial CONNECT read: + // a client or upstream that stopped mid-exchange (stalled response, + // half-written body, frozen keep-alive) blocked forever — the + // multi-hour "egress wedge" where the agent's request never reached + // GitHub and its session froze. Each phase gets its own deadline; + // they are cleared once the phase completes so healthy keep-alive + // connections are unaffected. + _ = client.SetReadDeadline(time.Now().Add(httpReadTimeout)) + req, err := http.ReadRequest(clientBuf) + if err != nil { + _ = client.SetReadDeadline(time.Time{}) + p.logClientRead("proxy client request read timed out", err, "agent", agentName) + return + } + + blocked := false + blockReason := "" + // blockCode is the machine-readable reason (#12127) that rides on the + // one-line 403 body and the X-Hive-Proxy-Reason header. Each gate that + // sets blockReason sets it; a gate that sets neither falls back to a + // generic code in ProxyDenialBody. + blockCode := "" + + // Per-repo pause (#6203) and per-repo agent scope (#6204) are both + // checked BEFORE the mode chain, and outside it, for the same two + // reasons. Neither is an autonomy tier — a pause is a run-state, a + // scope is roster membership — so each must refuse the write whatever + // mode the agent holds and whichever branch below would otherwise have + // allowed it. And each carries its own reason, so the agent is told the + // repo is deliberately quiet, or that it is not defined for this repo, + // instead of reading a mode error and going hunting for a permissions + // bug that does not exist. + // + // The hive's own control-plane traffic (App-token mint, heartbeat) is + // exempt from both, exactly as it is from the ACMM rules below: + // stopping the hive from minting a token would take the whole spoke + // down rather than quiet one repo or narrow one agent. + // + // Pause is checked first, so a repo the operator has frozen is refused + // as paused even when the agent is also out of scope for it. + blockJSON := false + graphQLMutation := false + // A direct write from a lane under write_surface.enforce (#9772) is + // refused and audited as agent_write_refused. Checked after pause and + // scope so their more specific reasons win when both apply. + writeSurfaceKind := "" + if agentName != internalCallerName { + if reason, paused := p.repoPauseRefusal(req.Method, repoScopePath(req)); paused { + blocked = true + blockReason = reason + blockCode = DenyReasonRepoPaused + } else if reason, refused := p.agentRepoScopeRefusal(agentName, req.Method, repoScopePath(req)); refused { + blocked = true + blockReason = reason + blockCode = DenyReasonAgentRepoScope + } else if reason, refused := p.autoMergeRefusal(req.Method, repoScopePath(req)); refused { + blocked = true + blockReason = reason + blockCode = DenyReasonRepoAutoMergeOff + blockJSON = true + } else if !isLinear { + if reason, kind, refused := WriteSurfaceEnforceRefusal(p.writeSurfaceEnforcedFunc(), agentName, req.Method, req.URL.Path, req.URL.RawQuery); refused { + blocked = true + blockReason = reason + blockCode = DenyReasonWriteSurfaceEnforced + writeSurfaceKind = kind + } + } + } + + if blocked { + // Already refused by the pause or agent-scope check above — the + // mode chain cannot un-block it, so skip it entirely. + } else if isLinear && agentName == internalCallerName { + // The hive's OWN Linear traffic — the read path #4178 shipped + // (backlog enumeration in pkg/worksource/linear.go) and, later, + // the OAuth/webhook plumbing. ACMM governs AGENT autonomy, so the + // control plane is exempt here exactly as it is for GitHub below. + // This branch must precede the enforcement one: without it, adding + // Linear to NeedsMITM would newly gate hive-originated reads that + // have always been ungated, breaking enumeration rather than + // restricting an agent. + } else if isLinear { + // Linear is a single POST /graphql for everything, so the + // (method, path) rule table cannot classify it and the GitHub + // GraphQL classifier is the wrong posture — it ends in a + // permissive default, backed by the REST table. Linear has no + // table underneath. See linear_rules.go. + blocked, blockReason = p.enforceLinear(req, agentName, mode, caps) + if blocked { + blockCode = DenyReasonLinear + } + } else if req.Method == "POST" && IsGraphQLPath(req.URL.Path) { + body, readErr := io.ReadAll(io.LimitReader(req.Body, graphQLBodyLimit)) + if req.Body != nil { + _ = req.Body.Close() + } + if readErr != nil { + _ = client.SetReadDeadline(time.Time{}) + p.logTimeout("proxy GraphQL request body read timed out", readErr, "agent", agentName, "path", req.URL.Path) + return + } + if reason, refused := p.graphQLAutoMergeRefusal(body); agentName != internalCallerName && refused { + blocked = true + blockReason = reason + blockCode = DenyReasonRepoAutoMergeOff + blockJSON = true + } else if agentName != internalCallerName { + if reason, refused := WriteSurfaceGraphQLRefusal(p.writeSurfaceEnforcedFunc(), agentName, body); refused { + blocked = true + blockReason = reason + blockCode = DenyReasonWriteSurfaceEnforced + writeSurfaceKind = WriteSurfaceKindGraphQL + } + } + allowed, isMutation := GraphQLAllowedCaps(autonomyGraphQLMode(agentName, body, mode), caps, body) + graphQLMutation = isMutation + if !blocked && !allowed { + blocked = true + if isMutation { + blockReason = "graphql mutation" + blockCode = DenyReasonGraphQLMutation + } else { + blockReason = "graphql" + blockCode = DenyReasonGraphQL + } + } + req.Body = io.NopCloser(strings.NewReader(string(body))) + req.ContentLength = int64(len(body)) + } else if agentName == internalCallerName { + // The hive itself has no ACMM mode to enforce — ACMM governs AGENT + // autonomy. Its control-plane calls (App token mint, heartbeat) + // must not be gated as if they were agent writes. Repo-filter and + // canary-egress checks below still apply. + } else if !AllowedByModeCaps(autonomyModeForRepo(agentName, ExtractRepo(repoScopePath(req)), mode), caps, req.Method, req.URL.Path) { + blocked = true + blockCode = DenyReasonMode + // An unidentified agent is silently treated as ADVISORY, which turns + // a permissions bug into an indistinguishable "policy denial". Say so + // loudly on writes: a write from an unknown caller means UID + // attribution failed, not that the agent lacks the mode. + if agentName == "" && writeMethods[req.Method] { + p.logger.Warn("proxy: agent could not be identified — defaulting to ADVISORY and blocking a write; UID attribution failed (check the uid map and /proc/net/tcp visibility), this is NOT an agent-mode misconfiguration", + "method", req.Method, "path", req.URL.Path) + } + // A hard-deny rule (e.g. direct PR creation) carries an agent-facing + // directive explaining the sanctioned alternative — surface it so the + // agent knows to use hive-open-pr rather than seeing only a mode error. + if msg, denied := DeniedMessage(req.Method, req.URL.Path); denied && msg != "" { + blockReason = msg + blockCode = DenyReasonHardDeny + } + } else if len(p.allowedRepos) > 0 && !RepoFilterAllowed(p.allowedRepos, req.Method, repoScopePath(req)) { + blocked = true + blockReason = "repo not in hive config" + blockCode = DenyReasonRepoNotInConfig + } + // Shape-gate agent-authored issue/comment REST writes. The issue-request + // watcher validates the file-drop creation path, but an agent's + // `gh issue create` becomes a raw POST /repos/{o}/{r}/issues that never + // touches the watcher (issue #7014: a template placeholder title and a + // body of literal \n escapes both reached GitHub this way). Enforce the + // same shape rules here so both creation paths are covered. Skip the + // control plane — its issue writes are already validated upstream. + if !blocked && agentName != internalCallerName { + if reason, deny, shapeErr := p.enforceIssueShapeFor(req, agentName); shapeErr != nil { + // Fail closed: a body we could not read is a body we could not + // shape-check. + p.logTimeout("proxy issue-shape body read timed out", shapeErr, "agent", agentName, "path", req.URL.Path) + _ = client.SetReadDeadline(time.Time{}) + return + } else if deny { + blocked = true + if blockReason == "" { + blockReason = reason + blockCode = DenyReasonIssueShape + } + } + } + if reason, deny, ok := p.inspectCanaryEgress(agentName, req); ok { + if deny { + blocked = true + if blockReason == "" { + blockReason = reason + blockCode = DenyReasonCanaryEgress + } + } + } + + if !isLinear && agentName != internalCallerName { + isGraphQL := req.Method == "POST" && IsGraphQLPath(req.URL.Path) + isWrite := writeMethods[req.Method] && !gitUploadPackPath.MatchString(req.URL.Path) + if isGraphQL { + isWrite = graphQLMutation + } + if isWrite { + reason := "" + if blocked { + reason = blockCode + if reason == "" { + reason = DenyReasonUnknown + } + } + p.writes.record(mode.String(), reason) + } + } + + if blocked { + detail := req.URL.Path + if blockReason != "" { + detail = blockReason + } + p.recordViolation(agentName, req.Method, detail) + if writeSurfaceKind != "" { + p.auditWriteRefused(agentName, writeSurfaceKind, req.Method, repoScopePath(req)) + } + + if blockCode == "" { + blockCode = DenyReasonUnknown + } + respBody := ProxyDenialBody(agentName, mode, req.Method, detail, blockCode) + contentType := "text/plain" + if blockJSON { + contentType = "application/json" + payload, _ := json.Marshal(map[string]any{ + "error": "repo_auto_merge_disabled", + "reason": detail, + }) + respBody = string(payload) + "\n" + } + resp := &http.Response{ + StatusCode: http.StatusForbidden, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(respBody)), + } + resp.Header.Set("Content-Type", contentType) + resp.Header.Set("X-Hive-Proxy-Blocked", "true") + resp.Header.Set(ProxyDenialHeader, blockCode) + _ = client.SetWriteDeadline(time.Now().Add(httpWriteTimeout)) + writeErr := resp.Write(client) + _ = client.SetWriteDeadline(time.Time{}) + if writeErr != nil { + p.logTimeout("proxy blocked response write timed out", writeErr, "agent", agentName) + return + } + + if req.Body != nil { + if _, drainErr := io.Copy(io.Discard, req.Body); drainErr != nil { + p.logTimeout("proxy blocked request body drain timed out", drainErr, "agent", agentName, "path", req.URL.Path) + } + _ = req.Body.Close() + } + _ = client.SetReadDeadline(time.Time{}) + continue + } + + if !isLinear { + if refusal := p.githubBudgetRefusal(agentName, req.Method, req.URL.Path, agentGitHubRequestIsRead(req.Method, req.URL.Path, graphQLMutation)); refusal != nil { + _ = p.writeAgentGitHubBudget429(client, *refusal) + if req.Body != nil { + _, _ = io.Copy(io.Discard, req.Body) + _ = req.Body.Close() + } + _ = client.SetReadDeadline(time.Time{}) + continue + } + } + + // #1861: with injection enabled, the upstream credential is decided + // HERE — after the ACMM/repo/canary gates, immediately before either + // forwarding branch — so every byte that reaches GitHub has passed + // through it. Blocked requests above never reach upstream and need no + // rewrite. + // + // #9586: only for GitHub hosts. This loop also relays api.linear.app + // (NeedsInspection), and rewriting there would attach the agent's + // hub-held GITHUB token to a request bound for Linear - a real + // credential handed to a third party - whenever the Linear lane below + // does not overwrite the header (an ADVISORY agent, or a hive with no + // Linear credential). + if injectsGitHubAuthForHost(host) { + p.rewriteGitHubAuth(req, agentName) + } + + // Linear: the request passed the tier gate; make sure it reaches + // Linear with the hive's CURRENT credential rather than whatever + // (possibly rotated-out) copy the agent's process still holds. + if isLinear && agentName != internalCallerName { + p.injectLinearCredential(req, agentName, mode) + } + // Git smart HTTP uses chunked streaming that http.ReadResponse + // can't handle reliably. After the ACMM check passes, forward + // the request and switch to raw bidirectional streaming. + if isGitPath(req.URL.Path) { + _ = upstream.SetWriteDeadline(time.Now().Add(httpWriteTimeout)) + if err := req.Write(upstream); err != nil { + _ = upstream.SetWriteDeadline(time.Time{}) + _ = client.SetReadDeadline(time.Time{}) + p.logTimeout("proxy git request relay timed out", err, "agent", agentName, "path", req.URL.Path) + return + } + _ = upstream.SetWriteDeadline(time.Time{}) + _ = client.SetReadDeadline(time.Time{}) + // Use relayTunnel, NOT an inline copy pair: this branch used to run + // its own unbounded io.Copy pair, which was the same wedge #3872 + // fixed in relayTunnel but left here. When GitHub FIN'd after the + // response while the client kept its side open (keep-alive), the + // client→upstream copy stayed blocked in client.Read() forever and + // `<-done` never returned — the handler's deferred Closes never ran, + // parking the upstream socket in CLOSE_WAIT for the life of the + // process (issue #3875's FD pile). relayTunnel bounds each direction + // once the other finishes. + relayTunnel(client, upstream) + return + } + + // Forward to upstream. The client read deadline stays in force until + // the body is fully drained (req.Write pipes req.Body from the client), + // so a client stalling mid-body cannot hang the relay. + _ = upstream.SetWriteDeadline(time.Now().Add(httpWriteTimeout)) + if err := req.Write(upstream); err != nil { + _ = upstream.SetWriteDeadline(time.Time{}) + _ = client.SetReadDeadline(time.Time{}) + p.logTimeout("proxy request relay timed out", err, "agent", agentName, "path", req.URL.Path) + return + } + _ = upstream.SetWriteDeadline(time.Time{}) + _ = client.SetReadDeadline(time.Time{}) + + _ = upstream.SetReadDeadline(time.Now().Add(httpReadTimeout)) + resp, err := http.ReadResponse(newBufferedReader(upstream), req) + _ = upstream.SetReadDeadline(time.Time{}) + if err != nil { + p.logTimeout("proxy upstream response read timed out", err, "agent", agentName, "path", req.URL.Path) + return + } + p.recordGitHubREST(agentName, req, resp) + p.recordAgentGitHubBudget(agentName, req.URL.Path, resp.StatusCode, p.agentGitHubCap(agentName)) + + // Bound the BODY relay too, not just the header read above. The + // deadline was cleared once headers arrived, so a "reachable but slow" + // GitHub — TLS and headers complete, body never arrives (the measured + // #3875 signature: time_total=12s, 0 body bytes) — parked resp.Write + // in upstream.Read() with no deadline. The client had long since given + // up (10s collect budget) and FIN'd its side, but this goroutine never + // noticed: it holds all three sockets of the exchange forever, and the + // retry loop stacks a fresh parked handler on every attempt (~1k + // FDs/min measured live). The wrapper re-arms a rolling per-Read + // deadline, so a body that keeps flowing is never cut while a stall + // longer than responseBodyStallTimeout errors out and lets the + // handler's deferred Closes reclaim the sockets. + resp.Body = &stallBoundedBody{body: resp.Body, conn: upstream, idle: p.bodyStallTimeoutDuration()} + + // Rolling per-Write deadline (see stallBoundedWriter): a response that + // keeps flowing is never cut for being long, while a client that stops + // draining still releases this handler within one stall window. + if err := resp.Write(&stallBoundedWriter{conn: client, idle: httpWriteTimeout}); err != nil { + _ = resp.Body.Close() + _ = client.SetWriteDeadline(time.Time{}) + p.logTimeout("proxy response relay timed out", err, "agent", agentName, "path", req.URL.Path) + return + } + _ = client.SetWriteDeadline(time.Time{}) + _ = resp.Body.Close() + } +} diff --git a/src/crux/proxy/github_proxy_policy.go b/src/crux/proxy/github_proxy_policy.go new file mode 100644 index 0000000000..b18a136dea --- /dev/null +++ b/src/crux/proxy/github_proxy_policy.go @@ -0,0 +1,368 @@ +package proxy + +import ( + "bytes" + "encoding/base64" + "io" + "net/http" + "strings" + + "github.com/hivecommons/hive/crux/agent" +) + +// injectLinearCredential replaces the Authorization header of an agent's +// api.linear.app request with the hive's CURRENT Linear credential, for +// agents at the ISSUES_ONLY floor and above (the tier the session-env push in +// agent.Manager.linearEnvPairs uses). It reports whether a credential was +// applied. +// +// WHY THE PROXY, NOT THE ENVIRONMENT. The connected Linear app's OAuth access +// token rotates (Linear issues it with a ~24h expiry and hive refreshes it +// in the background), and hive already re-pushes the fresh value into each +// agent's tmux session environment on the hourly refresh tick. But a tmux +// session environment only reaches processes forked AFTER the push: the +// agent's CLI, forked at its last launch, keeps the token it was born with, +// and every tool shell it spawns inherits that copy. Observed live +// (2026-09-06/07): a quality agent whose session environment held the fresh +// token — verified valid through this very proxy as that agent's UID — got +// `401 AUTHENTICATION_ERROR` on every Linear call from inside its pane for +// two days, because its process still carried the pre-rotation token. Its +// earlier kicks had silently fallen back to the operator's personal +// LINEAR_API_KEY, so every issue it filed showed the operator as creator. +// +// The proxy terminates every agent request to api.linear.app anyway (this is +// where enforceLinear gates mutations by tier), so attaching the live +// credential HERE makes the copy in the agent's environment irrelevant — +// stale, absent, or a placeholder all work — exactly as the inference +// translator attaches the real gateway key server-side. The agent still +// receives a credential in its environment (for SDKs that refuse to send a +// request without one), but Linear only ever sees the proxy's. +// +// Below the floor the request is left untouched: an ADVISORY agent's Linear +// reads have always been ungated and unauthenticated-by-hive, and injecting +// a write-capable credential there would widen what the tier can do. +func (p *GitHubProxy) injectLinearCredential(req *http.Request, agentName string, mode agent.AgentMode) bool { + if p.linearCredential == nil || !mode.CanCreateIssues() { + return false + } + cred := p.linearCredential() + switch { + case cred.AccessToken != "": + req.Header.Set("Authorization", "Bearer "+cred.AccessToken) + case cred.APIKey != "": + req.Header.Set("Authorization", cred.APIKey) + default: + return false + } + p.logger.Debug("linear: attached the hive's current credential to an agent request", "agent", agentName, "mode", mode.String()) + return true +} + +// repoScopePath is the request path handed to the per-repo write gates (pause, +// agent scope, auto-merge, per-repo ACMM mode, repo filter). It starts from +// the client's own escaping rather than the decoded req.URL.Path so a GitLab +// "namespace%2Fproject" stays one segment and ExtractRepo resolves the exact +// project — including nested groups — instead of guessing where the project +// path ends from the sub-resource that follows it. CanonicalGatePath then +// decodes every OTHER escape so an escaped structural segment (/r%65pos/, +// repo%2Egit, pro%6Aects, merg%65) cannot hide the repo from the gates. +func repoScopePath(req *http.Request) string { + return CanonicalGatePath(req.URL.EscapedPath()) +} + +// repoPauseRefusal is RepoPauseRefusal against this proxy's current predicate. +func (p *GitHubProxy) repoPauseRefusal(method, path string) (string, bool) { + p.pausedMu.RLock() + paused := p.repoPaused + p.pausedMu.RUnlock() + return RepoPauseRefusal(paused, method, path) +} + +func (p *GitHubProxy) autoMergeRefusal(method, path string) (string, bool) { + p.autoMergeMu.RLock() + enabled := p.repoAutoMergeOn + p.autoMergeMu.RUnlock() + return RepoAutoMergeRefusal(enabled, method, path) +} + +func (p *GitHubProxy) graphQLAutoMergeRefusal(body []byte) (string, bool) { + p.autoMergeMu.RLock() + enabled := p.repoAutoMergeOn + p.autoMergeMu.RUnlock() + return GraphQLAutoMergeRefusal(enabled, body) +} + +// agentRepoScopeRefusal is AgentRepoScopeRefusal against the current predicate. +func (p *GitHubProxy) agentRepoScopeRefusal(agentName, method, path string) (string, bool) { + p.scopeMu.RLock() + serves := p.agentServesRepo + p.scopeMu.RUnlock() + return AgentRepoScopeRefusal(serves, agentName, method, path) +} + +func (p *GitHubProxy) writeSurfaceEnforcedFunc() func(agent string) bool { + p.writeSurfaceMu.RLock() + defer p.writeSurfaceMu.RUnlock() + return p.writeSurfaceEnforced +} + +// laneWriteSurfaceEnforced reports whether agentName's lane is enforced. +func (p *GitHubProxy) laneWriteSurfaceEnforced(agentName string) bool { + if agentName == "" || agentName == internalCallerName { + return false + } + enforced := p.writeSurfaceEnforcedFunc() + return enforced != nil && enforced(agentName) +} + +// auditWriteRefused records one direct write refused under +// write_surface.enforce. +func (p *GitHubProxy) auditWriteRefused(agentName, kind, method, path string) { + p.writeSurfaceMu.RLock() + audit := p.writeRefusedAudit + p.writeSurfaceMu.RUnlock() + if audit == nil { + return + } + audit(agentName, kind, method, path, WriteSurfaceRepo(path)) +} + +// gitInjectBasicUser is the username git expects alongside a GitHub App +// installation token in HTTP Basic auth — the same identity +// git-credential-hive.sh answers with, so upstream sees exactly the shape the +// pre-injection flow produced. +const gitInjectBasicUser = "x-access-token" + +// loginPathPrefix marks the GitHub OAuth/device-flow endpoints +// (/login/device/code, /login/oauth/access_token). These authenticate the +// FLOW via their form body, not a bearer credential; injecting an App token +// there would be meaningless at best and flow-corrupting at worst, so they +// get strip-only treatment. +const loginPathPrefix = "/login/" + +// rewriteGitHubAuth enforces #1861 on one MITM'd request: the proxy — not the +// agent — decides what credential GitHub sees. +// +// Threat model, per header-touching block: +// +// - STRIP: an agent-supplied Authorization header is deleted UNCONDITIONALLY +// (under the flag), never forwarded. Even though injection-mode agents are +// only ever handed the inert placeholder, an agent that somehow obtained a +// real credential (leaked from a log, smuggled via prompt, minted through +// a side channel) must not be able to spend it through this proxy. +// +// - INJECT: only for an agent identified by the UID-authoritative path +// (#3888) that has a hub-held scoped token. The two lookups this chains +// are exactly the pre-existing lanes: identity from the UID map, token +// from the #3967 per-agent scoped mint. +// +// - UNKNOWN AGENT = NO CREDENTIAL: when identity resolution failed +// (agentName == ""), the request proceeds with NO Authorization and fails +// loud at GitHub (401/403). Deliberate: any fallback — shared cache, +// hive token, "last known agent" — would let an unattributable process +// ride a real credential, recreating the pre-#3888 identity hole this +// design depends on having closed. +// +// - COPILOT AUTH-EXCHANGE PASSTHROUGH: /copilot_internal/ on a GitHub +// host keeps the agent's own Authorization (the user's Copilot OAuth +// token); see copilotInternalPathPrefix for why and the residual. +// +// - INTERNAL CALLER PASSTHROUGH: the hive's own control plane +// (internalCallerName) legitimately holds and sends its own App +// credentials (token mint, heartbeat, hive-open-pr fulfillment); its +// requests pass untouched. This cannot be spoofed by an agent: the name +// is assigned only via UIDMap.IsInternalUID, never from anything the +// caller sends. +// +// Logging: agent name and injected yes/no only — NEVER token bytes, not even +// a prefix (#1861 requirement; a token prefix is enough to fingerprint and +// correlate credentials across logs). +func (p *GitHubProxy) rewriteGitHubAuth(req *http.Request, agentName string) { + // NOTE: callers must gate on injectsGitHubAuthForHost first; this function + // does not know which host the request is bound for. + if !p.injectGHAuth { + // Flag off (the default): behavior byte-identical to before #1861 — + // no header is touched on any path. + return + } + if agentName == internalCallerName { + return + } + if isCopilotAuthExchangePath(req.URL.Path) { + // Before the strip: the Copilot session-token exchange needs the + // user's Copilot OAuth token, which no hub-held App token can stand in + // for (copilotInternalPathPrefix). + p.logger.Debug("proxy auth injection: Copilot auth-exchange endpoint - agent credential passed through", "agent", agentName, "injected", false) + return + } + + req.Header.Del("Authorization") + + if agentName == "" { + p.logger.Debug("proxy auth injection: caller unidentified — forwarding without credentials (fail-loud, no fallback)", "injected", false) + return + } + if strings.HasPrefix(req.URL.Path, loginPathPrefix) { + p.logger.Debug("proxy auth injection: OAuth flow endpoint — strip only", "agent", agentName, "injected", false) + return + } + if p.agentTokenSource == nil { + p.logger.Warn("proxy auth injection enabled but no token source wired — forwarding without credentials", "agent", agentName, "injected", false) + return + } + token, ok := p.agentTokenSource(agentName) + if !ok { + // An identified agent with no hub-held token means the #3967 mint lane + // has not (yet) delivered for this agent — e.g. the hive restarted and + // the in-memory registry is empty until the next launch/refresh mint. + // Fail loud (upstream 401) rather than borrow any other credential. + p.logger.Warn("proxy auth injection: no hub-held token for identified agent — forwarding without credentials (check the per-agent mint lane)", "agent", agentName, "injected", false) + return + } + + if isGitPath(req.URL.Path) { + // Git smart HTTP authenticates with Basic x-access-token: — + // the same shape git-credential-hive.sh produced pre-injection. + basic := base64.StdEncoding.EncodeToString([]byte(gitInjectBasicUser + ":" + token)) + req.Header.Set("Authorization", "Basic "+basic) + // Force this exchange onto its own upstream connection. The git branch + // in proxyHTTP switches to a raw relay after forwarding this request's + // headers, so a keep-alive reuse (git's follow-up POST after GET + // /info/refs on the same conn) would stream its headers — placeholder + // credential included — straight through WITHOUT passing this rewrite, + // and fail with a confusing mid-operation 401. Connection: close makes + // git reconnect per request; each new connection re-enters proxyHTTP + // and gets its own rewrite. + req.Close = true + } else { + // REST/GraphQL (api.github.com, GHE /api/v3): the standard GitHub + // token scheme, identical to what gh sends. + req.Header.Set("Authorization", "token "+token) + } + p.logger.Debug("proxy auth injection: scoped token attached", "agent", agentName, "injected", true) +} + +// injectsGitHubAuthForHost reports whether a request relayed for host may have +// its Authorization rewritten by rewriteGitHubAuth (#9586). It is a positive +// allowlist - the empty host (proxyHTTP's "GitHub implied" convention) and the +// registered GitHub-family hosts - rather than "anything but Linear", so a +// non-GitHub host added to the MITM set later can never receive a GitHub token +// by default. +func injectsGitHubAuthForHost(host string) bool { + return host == "" || IsGitHubHost(host) +} + +func (p *GitHubProxy) inspectCanaryEgress(agentName string, req *http.Request) (reason string, deny bool, detected bool) { + if p == nil || !p.canariesEnabled || p.canaryRegistry == nil || req == nil || req.Body == nil { + return "", false, false + } + if isGitReceivePack(req.URL.Path) && p.canaryFailClosed { + return "ioscan canary scan unavailable for git push", true, true + } + if !writeMethods[req.Method] || isGitPath(req.URL.Path) || !githubWriteBodyMayLeak(req.Method, req.URL.Path) { + return "", false, false + } + body, err := io.ReadAll(io.LimitReader(req.Body, maxGitHubWriteBodyScan+1)) + if req.Body != nil { + _ = req.Body.Close() + } + if err != nil { + return "", false, false + } + if len(body) > maxGitHubWriteBodyScan { + return "ioscan canary scan body too large", true, true + } + req.Body = io.NopCloser(bytes.NewReader(body)) + req.ContentLength = int64(len(body)) + if leak, ok := p.canaryRegistry.Scan(agentName, string(body), "github:"+req.Method+" "+req.URL.Path); ok { + if p.canaryLeakFunc != nil { + p.canaryLeakFunc(leak) + } + p.logger.Warn("ioscan canary leak detected", "agent", leak.Agent, "source", leak.Source) + return "ioscan canary leak", p.canaryFailClosed, true + } + return "", false, false +} + +// enforceLinear applies ACMM to one request on a MITM'd api.linear.app +// connection and returns whether it is blocked, with an agent-facing reason. +// +// The request body is read (bounded), inspected, and restored so an allowed +// request forwards byte-identically. +// +// NOTHING from the body is logged or surfaced in the 403. Linear documents +// carry issue and comment content in the query and, in variables, potentially +// tokens; the operation NAMES are schema identifiers rather than user content, +// so those are what makes a denial debuggable without leaking the payload. +func (p *GitHubProxy) enforceLinear(req *http.Request, agentName string, mode agent.AgentMode, caps agent.AgentCapabilities) (bool, string) { + // Linear exposes exactly one endpoint. Anything else on this host is not a + // shape this gate understands, so it is refused rather than forwarded. + if req.Method != http.MethodPost || !IsLinearGraphQLPath(req.URL.Path) { + return true, "linear: only POST " + linearGraphQLPath + " is permitted" + } + + // Read one byte past the limit so hitting the cap is detectable: a body + // that exactly fills the buffer is indistinguishable from a truncated one + // otherwise, and truncation must deny. + body, readErr := io.ReadAll(io.LimitReader(req.Body, linearBodyLimit+1)) + if req.Body != nil { + _ = req.Body.Close() + } + truncated := len(body) > linearBodyLimit + if truncated { + body = body[:linearBodyLimit] + } + req.Body = io.NopCloser(bytes.NewReader(body)) + req.ContentLength = int64(len(body)) + + if readErr != nil { + // Fail closed: a body we could not fully read is a body we could not + // classify. + p.logger.Warn("linear: request body read failed — denying", "agent", agentName) + return true, "linear: request body unreadable" + } + + decision := LinearGraphQLAllowed(mode, caps, body, truncated) + if decision.Allowed { + return false, "" + } + + // Operations may be empty when the document could not be classified at + // all, which is itself the useful signal. + p.logger.Warn("linear: blocked GraphQL request", + "agent", agentName, + "mode", mode.String(), + "operations", strings.Join(decision.Operations, ","), + "reason", decision.Reason, + "mutation", decision.IsMutation) + + reason := "linear: " + decision.Reason + if len(decision.Operations) > 0 { + reason += " (" + strings.Join(decision.Operations, ",") + ")" + } + return true, reason +} + +func githubWriteBodyMayLeak(method, path string) bool { + if method != http.MethodPost && method != http.MethodPatch && method != http.MethodPut { + return false + } + if !strings.HasPrefix(path, "/repos/") { + return IsGraphQLPath(path) + } + return strings.Contains(path, "/issues") || + strings.Contains(path, "/pulls") || + strings.Contains(path, "/comments") || + strings.Contains(path, "/reviews") || + strings.Contains(path, "/git/") +} + +func isGitReceivePack(path string) bool { + return strings.HasSuffix(path, "/git-receive-pack") +} + +func isGitPath(path string) bool { + return strings.HasSuffix(path, "/git-receive-pack") || + strings.HasSuffix(path, "/git-upload-pack") || + strings.HasSuffix(path, "/info/refs") +} diff --git a/src/crux/proxy/github_proxy_setters.go b/src/crux/proxy/github_proxy_setters.go new file mode 100644 index 0000000000..e555342d4e --- /dev/null +++ b/src/crux/proxy/github_proxy_setters.go @@ -0,0 +1,100 @@ +package proxy + +import ( + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/agent" +) + +// SetLinearCredentialResolver installs the resolver injectLinearCredential +// reads on every agent request to api.linear.app. main.go wires the same +// closure the agent manager uses to push LINEAR_ACCESS_TOKEN / LINEAR_API_KEY +// into sessions, so the proxy and the session env can never name different +// credentials. Values are never logged. nil disables injection. +func (p *GitHubProxy) SetLinearCredentialResolver(fn func() agent.LinearCredential) { + p.linearCredential = fn +} + +// SetTokenSink wires the inference token sink so the translator can record +// per-agent usage from upstream OpenAI responses into the shared token store. +func (p *GitHubProxy) SetTokenSink(sink UsageSink) { + p.tokenSink = sink +} + +// SetAgentTokenSource wires the resolver from identified agent name to that +// agent's hub-held scoped token (#1861). Wired once at boot, before Start(), +// so no lock is needed on the read path. +func (p *GitHubProxy) SetAgentTokenSource(source func(agentName string) (string, bool)) { + p.agentTokenSource = source +} + +func (p *GitHubProxy) SetAgentGitHubBudgetConfig(capForAgent func(agentName string) int, reserveFloor func() int) { + if capForAgent != nil { + p.githubBudgetCap = capForAgent + } + if reserveFloor != nil { + p.githubReserveFloor = reserveFloor + } +} + +func (p *GitHubProxy) SetAgentGitHubBudgetNudgeFunc(fn func(agentName, message string)) { + p.githubBudgetNudge = fn +} + +// SetCanaryScanner installs the host's canary registry and leak handler. +// enabled gates the scan; failClosed turns a detection (or an unscannable +// body) into a refusal rather than an audit-only event. +func (p *GitHubProxy) SetCanaryScanner(enabled, failClosed bool, reg CanaryScanner, onLeak func(crux.CanaryLeak)) { + p.canariesEnabled = enabled + p.canaryFailClosed = failClosed + p.canaryRegistry = reg + p.canaryLeakFunc = onLeak +} + +// SetRepoPausedFunc installs the operator's per-repo pause predicate (#6203). +// The hive passes config's IsRepoPaused, so pausing a repo in the dashboard is +// enforced on the very next request without a restart — which matters, because +// the pauses this serves (a release freeze, an incident) are measured in hours, +// and an operator who has to roll the pod to quiet one repo is back to stopping +// the whole hive. Passing nil clears it. +func (p *GitHubProxy) SetRepoPausedFunc(fn func(repo string) bool) { + p.pausedMu.Lock() + p.repoPaused = fn + p.pausedMu.Unlock() +} + +// SetRepoAutoMergeEnabledFunc installs the live per-repo auto-merge predicate. +// The proxy uses it to give flagged repositories their specific deny reason at +// direct merge endpoints. Passing nil preserves default-on behavior. +func (p *GitHubProxy) SetRepoAutoMergeEnabledFunc(fn func(repo string) bool) { + p.autoMergeMu.Lock() + p.repoAutoMergeOn = fn + p.autoMergeMu.Unlock() +} + +// SetAgentRepoScopeFunc installs the per-repo agent-scope predicate (#6204). +// The hive passes config's AgentServesRepo, so re-scoping an agent in the +// dashboard is enforced on the very next request without a restart. Passing nil +// clears it, which is the unscoped behaviour every hive had before. +func (p *GitHubProxy) SetAgentRepoScopeFunc(fn func(agent, repo string) bool) { + p.scopeMu.Lock() + p.agentServesRepo = fn + p.scopeMu.Unlock() +} + +// SetWriteSurfaceEnforceFunc installs the per-lane write_surface.enforce +// predicate (#9772). The hive passes config's WriteSurfaceEnforced, so a lane +// listed in config is enforced on the next request. Passing nil clears it, +// which is the behaviour every hive had before: direct writes allowed. +func (p *GitHubProxy) SetWriteSurfaceEnforceFunc(fn func(agent string) bool) { + p.writeSurfaceMu.Lock() + p.writeSurfaceEnforced = fn + p.writeSurfaceMu.Unlock() +} + +// SetWriteRefusedAuditFunc installs the sink that records a direct write +// refused under write_surface.enforce as an agent_write_refused audit entry. +func (p *GitHubProxy) SetWriteRefusedAuditFunc(fn func(agent, kind, method, path, repo string)) { + p.writeSurfaceMu.Lock() + p.writeRefusedAudit = fn + p.writeSurfaceMu.Unlock() +} diff --git a/src/crux/proxy/github_proxy_tunnel.go b/src/crux/proxy/github_proxy_tunnel.go new file mode 100644 index 0000000000..cdb93eb78d --- /dev/null +++ b/src/crux/proxy/github_proxy_tunnel.go @@ -0,0 +1,179 @@ +package proxy + +import ( + "io" + "net" + "sync/atomic" + "time" +) + +// responseBodyStallTimeout bounds how long the MITM'd response-BODY relay may +// go without a single byte of progress from the upstream. It is a rolling +// per-Read bound (re-armed on every read by stallBoundedBody), not a cap on +// total transfer time, so a large-but-flowing response is never cut while an +// upstream that goes silent mid-body releases the handler — and with it the +// three sockets of the exchange — within one stall window. Overridable, not a +// const, so tests can shorten it — and atomic for the same reason as +// tunnelHalfCloseDrain (#5553): the reading goroutine can outlive the test +// that started it. A handler test that returns after a fixed sleep leaves +// proxyHTTP still parsing the response when the NEXT test's shortenBodyStall +// writes the bound — a plain var made that an unsynchronised write-after-read +// (-race failure on v4, #6110, reported against whichever test held the +// baton). Defaults to httpReadTimeout: the same patience already extended to +// the header phase. +var responseBodyStallTimeout atomic.Int64 + +func init() { responseBodyStallTimeout.Store(int64(httpReadTimeout)) } + +// bodyStallTimeout returns the current response-body stall bound. +func bodyStallTimeout() time.Duration { + return time.Duration(responseBodyStallTimeout.Load()) +} + +// bodyStallTimeoutDuration returns the body-stall timeout for this proxy instance, +// falling back to the package-level responseBodyStallTimeout. +func (p *GitHubProxy) bodyStallTimeoutDuration() time.Duration { + if p != nil && p.bodyStallTimeout > 0 { + return p.bodyStallTimeout + } + return bodyStallTimeout() +} + +// stallBoundedBody wraps a proxied response body so every underlying Read is +// preceded by re-arming a read deadline on the upstream conn. Close clears the +// deadline (for the next keep-alive exchange on the same conn) and closes the +// wrapped body. See responseBodyStallTimeout for why this exists (#3875). +type stallBoundedBody struct { + body io.ReadCloser + conn net.Conn + idle time.Duration +} + +func (b *stallBoundedBody) Read(p []byte) (int, error) { + _ = b.conn.SetReadDeadline(time.Now().Add(b.idle)) + return b.body.Read(p) +} + +func (b *stallBoundedBody) Close() error { + // Close the wrapped body while a stall deadline is armed: net/http body + // Close paths may still read from the conn (drain-to-EOF), and an unarmed + // read there would reintroduce the very stall this type exists to bound. + // The deadline is cleared afterwards so the next keep-alive exchange on + // this conn starts unencumbered. + _ = b.conn.SetReadDeadline(time.Now().Add(b.idle)) + err := b.body.Close() + _ = b.conn.SetReadDeadline(time.Time{}) + return err +} + +// stallBoundedWriter is the write-side mirror of stallBoundedBody: every Write +// re-arms the client's write deadline, so the bound is "no progress for idle" +// rather than "the whole relay must finish within idle". +// +// The distinction is not academic. The response relays used to arm a single +// absolute client.SetWriteDeadline(now+httpWriteTimeout) around +// resp.Write(client), which is correct for a small buffered body and wrong for +// a stream: the MITM'd /v1/messages SSE relay holds one Write loop open for the +// entire generation, so a turn that legitimately streams for longer than +// httpWriteTimeout was severed mid-stream at exactly that mark — even though +// tokens were flowing the whole time. The agent saw a truncated body +// ("Response was interrupted due to a server error", or a TLS-level "cannot +// decrypt peer's message" when the cut landed mid-record), retried, and — +// because a retry replays the same long turn — was cut again at the same 60s, +// livelocking the agent on its longest and most valuable turns. Measured on the +// hosted spoke: 23 "response relay timed out" in 40 minutes, every one +// path=/v1/messages with error "write tcp ...: i/o timeout", against an agent +// that made zero forward progress across six consecutive attempts. +// +// Reusing the stall semantics already proven on the read side (#3875) keeps a +// client that has genuinely gone away bounded — a dead peer stops draining, the +// socket buffer fills, and the next Write blocks past idle and errors — while a +// stream that keeps flowing is never cut for being long. +type stallBoundedWriter struct { + conn net.Conn + idle time.Duration +} + +func (w *stallBoundedWriter) Write(p []byte) (int, error) { + _ = w.conn.SetWriteDeadline(time.Now().Add(w.idle)) + return w.conn.Write(p) +} + +// tunnelHalfCloseDrainDefault bounds how long one direction of an opaque +// tunnel may keep waiting for bytes AFTER the other direction has already +// finished. +// +// 30s is deliberately generous: the only legitimate traffic still in flight +// once one side has hung up is a response tail already on the wire, which +// arrives in RTTs, not tens of seconds. +const tunnelHalfCloseDrainDefault = 30 * time.Second + +// tunnelHalfCloseDrain is overridable rather than const for the same reason as +// waitForReadyPollInterval in pkg/hub: tests need a relay that unwedges in +// milliseconds. Production keeps the default. +// +// It is atomic because the write and the read genuinely happen on different +// goroutines with no happens-before between them. relayTunnel reads the bound +// from the copy goroutine it spawns, and that goroutine can outlive the test +// that started it: a handler test only waits for the CONNECT response, not for +// the relay to drain, so the goroutine is still reading while the NEXT test's +// shortenTunnelDrain writes. A plain var made that an unsynchronised +// write-after-read and -race failed whichever test happened to hold the baton +// (#5553) — the reported test name was incidental, which is why the failure +// moved between TestRelayTunnelUnwedgesFromSilentUpstream and +// TestRelayTunnelDataFlowsBothWays across shuffle seeds. +var tunnelHalfCloseDrain atomic.Int64 + +func init() { tunnelHalfCloseDrain.Store(int64(tunnelHalfCloseDrainDefault)) } + +// tunnelDrain returns the current half-close drain bound. +func tunnelDrain() time.Duration { + return time.Duration(tunnelHalfCloseDrain.Load()) +} + +// relayTunnel shuttles bytes both ways between the proxied client conn and the +// upstream until BOTH directions have finished, bounding each direction once +// the other has ended. +// +// WHY THE BOUNDS EXIST (measured live, 2026-08-14): the previous inline copies +// waited on each other unboundedly. On spokes whose forge host +// (github.ibm.com) sits behind a SYN-accepting firewall, the upstream TCP +// connect succeeds but no byte ever comes back and no FIN is ever sent. The +// client (the daemon's own JWT mints, agents' CLIs) gives up at its TLS +// handshake timeout and closes — the client→upstream copy finishes and +// half-closes the upstream — but io.Copy(conn, upstream) stayed blocked in +// upstream.Read() FOREVER. Every attempt leaked two sockets (the FIN_WAIT2 / +// CLOSE_WAIT piles), the goroutine, and the splice pipe pair io.Copy holds for +// a TCP↔TCP copy on Linux: ~300k fds and ~11 burned cores per affected spoke. +// +// SetReadDeadline is documented to interrupt Reads that are ALREADY blocked, +// which is exactly what arms the escape hatch here: each side arms the OTHER +// side's deadline only at the moment its own direction completes, so a +// healthy long-lived tunnel (git smart HTTP, device-flow polling) is never +// cut while both directions are still live. +func relayTunnel(conn, upstream net.Conn) { + done := make(chan struct{}) + go func() { + _, _ = io.Copy(upstream, conn) + if tc, ok := upstream.(*net.TCPConn); ok { + _ = tc.CloseWrite() + } + // Client side is done (EOF or error — a dead client looks the same). + // A live upstream answers or closes within RTTs; a blackholed one + // never would, so bound the remaining upstream→client read. + _ = upstream.SetReadDeadline(time.Now().Add(tunnelDrain())) + close(done) + }() + _, _ = io.Copy(conn, upstream) + // Mirror image: upstream finished (or errored) but the client may sit + // half-open without ever sending EOF; bound the client-side read so + // <-done cannot wedge this handler. + _ = conn.SetReadDeadline(time.Now().Add(tunnelDrain())) + <-done +} + +func transfer(dst, src net.Conn) { + defer func() { _ = dst.Close() }() + defer func() { _ = src.Close() }() + _, _ = io.Copy(dst, src) +} diff --git a/src/crux/proxy/github_proxy_violations.go b/src/crux/proxy/github_proxy_violations.go new file mode 100644 index 0000000000..a76249d3ba --- /dev/null +++ b/src/crux/proxy/github_proxy_violations.go @@ -0,0 +1,254 @@ +package proxy + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "regexp" + "strconv" + "strings" + + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/issueshape" +) + +// Violations returns a snapshot of per-agent violation counts. +func (p *GitHubProxy) Violations() map[string]int { + p.mu.RLock() + defer p.mu.RUnlock() + out := make(map[string]int, len(p.violations)) + for k, v := range p.violations { + out[k] = v + } + return out +} + +// AgentViolations returns the violation count for a specific agent. +func (p *GitHubProxy) AgentViolations(agentName string) int { + p.mu.RLock() + defer p.mu.RUnlock() + return p.violations[agentName] +} + +// issueShapeBodyLimit bounds how much of an issue/comment REST body the proxy +// buffers to shape-check it. It comfortably exceeds GitHub's issue-body maximum +// (65536 chars) plus JSON overhead; an oversized body is forwarded unbuffered +// and unchecked rather than truncated (see enforceIssueShape). +const issueShapeBodyLimit = 256 * 1024 + +var ( + issueCreateRESTPathRE = regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues$`) + issueEditRESTPathRE = regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues/\d+$`) + issueCommentCreateRESTPath = regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues/\d+/comments$`) + issueCommentEditRESTPath = regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues/comments/\d+$`) +) + +// isIssueShapeRESTWrite reports whether a (method, path) is a REST issue or +// issue-comment create/edit whose JSON body carries agent-authored title/body +// text the shape validators must vet. It is deliberately precise — matching +// only the title/body-bearing routes — so label, assignee, and reaction sub +// routes under /issues are not needlessly buffered. +func isIssueShapeRESTWrite(method, path string) bool { + if i := strings.IndexByte(path, '?'); i >= 0 { + path = path[:i] + } + switch method { + case http.MethodPost: + return issueCreateRESTPathRE.MatchString(path) || + issueCommentCreateRESTPath.MatchString(path) + case http.MethodPatch: + return issueEditRESTPathRE.MatchString(path) || + issueCommentEditRESTPath.MatchString(path) + default: + return false + } +} + +// githubIssueShapeViolation returns a non-empty, agent-actionable reason when a +// buffered issue/comment REST body carries an unsubstituted template +// placeholder or literal newline escape sequences. A body that is not the JSON +// we expect is left to the upstream to reject — auth was already enforced. +func githubIssueShapeViolation(method, path string, body []byte) string { + if !isIssueShapeRESTWrite(method, path) { + return "" + } + var payload struct { + Title *string `json:"title"` + Body *string `json:"body"` + } + if err := json.Unmarshal(body, &payload); err != nil { + return "" + } + if payload.Title != nil { + if ph, ok := issueshape.UnsubstitutedTemplatePlaceholder(*payload.Title); ok { + return "issue title contains unsubstituted template placeholder " + strconv.Quote(ph) + "; fill in the template before creating the issue" + } + } + if payload.Body != nil { + if ph, ok := issueshape.UnsubstitutedTemplatePlaceholder(*payload.Body); ok { + return "issue body contains unsubstituted template placeholder " + strconv.Quote(ph) + "; fill in the template before creating the issue" + } + if issueshape.BodyHasMisEscapedNewlines(*payload.Body) { + return "issue body contains literal newline escape sequences; use real newlines or --body-file" + } + } + return "" +} + +// bodyReadCloser rejoins an already-read prefix with the unread remainder of a +// request body so an oversized issue write can be forwarded intact. +type bodyReadCloser struct { + io.Reader + io.Closer +} + +// enforceIssueShape is enforceIssueShapeFor with no agent identity, so no lane +// label is injected. +func (p *GitHubProxy) enforceIssueShape(req *http.Request) (reason string, deny bool, readErr error) { + return p.enforceIssueShapeFor(req, "") +} + +// enforceIssueShapeFor buffers an agent's issue/comment REST body, validates +// its shape, and reconstructs req.Body so the request can still be forwarded +// when it passes. It returns deny=true with an actionable reason for a +// malformed body, and a non-nil error only when the body could not be read +// (fail closed). A well-formed issue create from an identified agent also +// gets its agent/ provenance label injected (see injectOriginLabel). +func (p *GitHubProxy) enforceIssueShapeFor(req *http.Request, agentName string) (reason string, deny bool, readErr error) { + if req.Body == nil || !isIssueShapeRESTWrite(req.Method, req.URL.Path) { + return "", false, nil + } + body, err := io.ReadAll(io.LimitReader(req.Body, issueShapeBodyLimit+1)) + if err != nil { + _ = req.Body.Close() + return "", false, err + } + if int64(len(body)) > issueShapeBodyLimit { + // Oversized: forward the original body unbuffered and unchecked rather + // than corrupt it by truncation. Stitch the prefix we already consumed + // back in front of the unread remainder; ContentLength is unchanged. + req.Body = bodyReadCloser{Reader: io.MultiReader(bytes.NewReader(body), req.Body), Closer: req.Body} + return "", false, nil + } + _ = req.Body.Close() + if v := githubIssueShapeViolation(req.Method, req.URL.Path, body); v != "" { + req.Body = io.NopCloser(bytes.NewReader(body)) + req.ContentLength = int64(len(body)) + return v, true, nil + } + if rewritten, ok := injectOriginLabel(req.Method, req.URL.Path, body, agentName); ok { + body = rewritten + } + req.Body = io.NopCloser(bytes.NewReader(body)) + req.ContentLength = int64(len(body)) + return "", false, nil +} + +// isIssueCreateRESTWrite reports whether (method, path) is POST +// /repos/{o}/{r}/issues — the one route whose body creates an issue. +func isIssueCreateRESTWrite(method, path string) bool { + if i := strings.IndexByte(path, '?'); i >= 0 { + path = path[:i] + } + return method == http.MethodPost && issueCreateRESTPathRE.MatchString(path) +} + +// injectOriginLabel ensures an agent's issue-create JSON body carries the +// canonical agent/ label and no redundant origin/ or bare +// label, so an issue filed with `gh issue create` through the proxy records +// the same provenance as one filed through the issue-request watcher. The +// body is returned unchanged (ok=false) for other routes, an unidentified +// caller, a body that is not a JSON object, or a labels field in an unexpected +// shape (GitHub accepts label objects too; those are forwarded as written +// rather than guessed at). +func injectOriginLabel(method, path string, body []byte, agentName string) ([]byte, bool) { + lane := config.OriginLaneFromAgent(agentName) + if lane == "" || agentName == internalCallerName || !isIssueCreateRESTWrite(method, path) { + return body, false + } + var payload map[string]json.RawMessage + if err := json.Unmarshal(body, &payload); err != nil || payload == nil { + return body, false + } + var labels []string + if raw, present := payload["labels"]; present && len(bytes.TrimSpace(raw)) > 0 && string(bytes.TrimSpace(raw)) != "null" { + if err := json.Unmarshal(raw, &labels); err != nil { + return body, false + } + } + filtered := config.WithIssueOriginLabel(labels, lane) + if sameLabels(labels, filtered) { + return body, false + } + labels = filtered + encoded, err := json.Marshal(labels) + if err != nil { + return body, false + } + payload["labels"] = encoded + rewritten, err := json.Marshal(payload) + if err != nil { + return body, false + } + return rewritten, true +} + +func sameLabels(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +// internalCallerName attributes a connection that originated from the hive's +// OWN control plane rather than from a sandboxed agent. It is deliberately not +// a valid agent name (agents come from the UID map) so it can never collide +// with one, and it is what distinguishes "the hive called GitHub" from the +// genuine failure mode the mode gate warns about: "an agent's UID could not be +// attributed". See UIDMap.IsInternalUID for the security argument. +const internalCallerName = "hive-internal" + +func (p *GitHubProxy) recordViolation(agentName, method, path string) { + if p.firstBlock(agentName, method, path) { + p.logger.Warn("proxy request blocked", "agent", agentName, "method", method, "path", path) + } else { + p.logger.Debug("proxy request blocked (repeat)", "agent", agentName, "method", method, "path", path) + } + p.countViolation(agentName) +} + +// firstBlock reports whether this (agent, method, path) refusal has not been +// logged before. Once the set is full, new keys keep logging at WARN. +func (p *GitHubProxy) firstBlock(agentName, method, path string) bool { + key := agentName + "|" + method + "|" + path + p.mu.Lock() + defer p.mu.Unlock() + if _, seen := p.blockedSeen[key]; seen { + return false + } + if p.blockedSeen == nil { + p.blockedSeen = make(map[string]struct{}) + } + if len(p.blockedSeen) < maxViolationLog { + p.blockedSeen[key] = struct{}{} + } + return true +} + +// countViolation increments the per-agent refusal counter without logging, for +// callers that have already logged the refusal with their own detail (see +// refusePlainForward). The map is bounded by maxViolationLog so an +// unidentified or churning caller set cannot grow it without limit. +func (p *GitHubProxy) countViolation(agentName string) { + p.mu.Lock() + defer p.mu.Unlock() + if _, exists := p.violations[agentName]; exists || len(p.violations) < maxViolationLog { + p.violations[agentName]++ + } +} diff --git a/src/crux/proxy/inference_route.go b/src/crux/proxy/inference_route.go index 9a9effe597..4f35d29dec 100644 --- a/src/crux/proxy/inference_route.go +++ b/src/crux/proxy/inference_route.go @@ -207,7 +207,7 @@ func unregisterAnthropicHost(host string) { func IsAnthropicHost(host string) bool { anthropicHostsMu.RLock() defer anthropicHostsMu.RUnlock() - return anthropicHosts[host] + return anthropicHosts[canonicalHost(host)] } // InferenceBackends lists the valid self-hosted inference backend IDs. diff --git a/src/crux/proxy/linear_rules.go b/src/crux/proxy/linear_rules.go index 5546be8e33..21a1306a75 100644 --- a/src/crux/proxy/linear_rules.go +++ b/src/crux/proxy/linear_rules.go @@ -142,7 +142,7 @@ var linearMutations = map[string]linearRequirement{ } // IsLinearHost reports whether host is Linear's API host. -func IsLinearHost(host string) bool { return host == linearHost } +func IsLinearHost(host string) bool { return canonicalHost(host) == linearHost } // IsLinearGraphQLPath reports whether path is Linear's GraphQL endpoint. func IsLinearGraphQLPath(path string) bool { return path == linearGraphQLPath } diff --git a/src/crux/proxy/rules.go b/src/crux/proxy/rules.go index 2888fc9c8b..3e30067a94 100644 --- a/src/crux/proxy/rules.go +++ b/src/crux/proxy/rules.go @@ -31,6 +31,12 @@ type ProxyRule struct { Capability func(agent.AgentCapabilities) bool } +// canonicalHost normalizes a hostname for comparison: DNS names are +// case-insensitive and may carry a trailing dot. +func canonicalHost(host string) string { + return strings.TrimSuffix(strings.ToLower(strings.TrimSpace(host)), ".") +} + // githubHosts are the hostnames the proxy inspects. var githubHostsMu sync.RWMutex var githubHosts = map[string]bool{ @@ -43,6 +49,7 @@ var githubHosts = map[string]bool{ // register hosts while proxy goroutines are already serving, so the map is // mutex-guarded. func RegisterGitHubHost(host string) { + host = canonicalHost(host) if host == "" { return } @@ -53,6 +60,7 @@ func RegisterGitHubHost(host string) { // unregisterGitHubHost removes a hostname from the allowlist (test cleanup). func unregisterGitHubHost(host string) { + host = canonicalHost(host) githubHostsMu.Lock() delete(githubHosts, host) githubHostsMu.Unlock() @@ -60,6 +68,7 @@ func unregisterGitHubHost(host string) { // IsGitHubHost returns true if the host should be subject to mode enforcement. func IsGitHubHost(host string) bool { + host = canonicalHost(host) githubHostsMu.RLock() defer githubHostsMu.RUnlock() return githubHosts[host] @@ -78,6 +87,7 @@ func IsGitHubHost(host string) bool { // is why the IsGitHubHost checks at the CONNECT seams could no longer double as // the "should I inspect this?" test; see NeedsInspection. func NeedsMITM(host string) bool { + host = canonicalHost(host) return host == "api.github.com" || IsLinearHost(host) } @@ -112,6 +122,7 @@ const copilotAPIHostExact = "githubcopilot.com" // otherwise forwarded verbatim; no content is altered beyond an optional // stream_options.include_usage hint on streaming completion requests. func IsCopilotAPIHost(host string) bool { + host = canonicalHost(host) if host == "" { return false } diff --git a/src/crux/proxy/rules_test.go b/src/crux/proxy/rules_test.go index 17dd1d7faa..313c445e71 100644 --- a/src/crux/proxy/rules_test.go +++ b/src/crux/proxy/rules_test.go @@ -307,6 +307,10 @@ func TestIsGitHubHost(t *testing.T) { {"github.com", true}, {"example.com", false}, {"api.github.com.evil.com", false}, + {"API.GITHUB.COM", true}, + {"Api.GitHub.com.", true}, + {"github.com.", true}, + {"evil-api.github.com", false}, {"", false}, } for _, tt := range tests { @@ -332,3 +336,32 @@ func TestNeedsMITM(t *testing.T) { } } } + +func TestHostCanonicalization(t *testing.T) { + tests := []struct { + name string + fn func(string) bool + host string + want bool + }{ + {"inspection upper", NeedsInspection, "API.GITHUB.COM", true}, + {"inspection mixed trailing dot", NeedsInspection, "Api.GitHub.com.", true}, + {"inspection lookalike", NeedsInspection, "evil-api.github.com", false}, + {"inspection empty", NeedsInspection, "", false}, + {"mitm trailing dot", NeedsMITM, "api.github.com.", true}, + {"linear upper", IsLinearHost, "API.LINEAR.APP", true}, + {"linear trailing dot", IsLinearHost, "api.linear.app.", true}, + {"linear lookalike", IsLinearHost, "evil-api.linear.app", false}, + {"copilot upper", IsCopilotAPIHost, "API.GITHUBCOPILOT.COM", true}, + {"copilot mixed trailing dot", IsCopilotAPIHost, "Api.GitHubCopilot.com.", true}, + {"copilot lookalike", IsCopilotAPIHost, "evilgithubcopilot.com", false}, + {"copilot empty", IsCopilotAPIHost, "", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.fn(tt.host); got != tt.want { + t.Errorf("host %q = %v, want %v", tt.host, got, tt.want) + } + }) + } +} diff --git a/src/crux/proxy/upstream_timeout_test.go b/src/crux/proxy/upstream_timeout_test.go index cdff78d6c6..c69de873ba 100644 --- a/src/crux/proxy/upstream_timeout_test.go +++ b/src/crux/proxy/upstream_timeout_test.go @@ -22,16 +22,39 @@ func TestUpstreamDialTimeoutConstant(t *testing.T) { } } -func TestAllTLSUpstreamDialsAreBound(t *testing.T) { +// readGitHubProxySource returns the concatenated source of the non-test +// github_proxy*.go files, so source-level invariants hold across the files +// GitHubProxy is split into. +func readGitHubProxySource(t *testing.T) string { + t.Helper() _, file, _, ok := runtime.Caller(0) if !ok { t.Fatal("runtime.Caller failed") } - body, err := os.ReadFile(filepath.Join(filepath.Dir(file), "github_proxy.go")) + paths, err := filepath.Glob(filepath.Join(filepath.Dir(file), "github_proxy*.go")) if err != nil { - t.Fatalf("read github_proxy.go: %v", err) + t.Fatalf("glob github_proxy*.go: %v", err) + } + var sb strings.Builder + for _, path := range paths { + if strings.HasSuffix(path, "_test.go") { + continue + } + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", filepath.Base(path), err) + } + sb.Write(body) + sb.WriteString("\n") + } + if sb.Len() == 0 { + t.Fatal("no github_proxy*.go source files found") } - text := string(body) + return sb.String() +} + +func TestAllTLSUpstreamDialsAreBound(t *testing.T) { + text := readGitHubProxySource(t) if got := strings.Count(text, "copilot sniff: client hung up before TLS handshake"); got != 1 { t.Fatalf("copilot sniff handshake hangup demotion sites = %d, want 1", got) } @@ -54,15 +77,7 @@ func TestAllTLSUpstreamDialsAreBound(t *testing.T) { } func TestAllClientTLSHandshakesAreBound(t *testing.T) { - _, file, _, ok := runtime.Caller(0) - if !ok { - t.Fatal("runtime.Caller failed") - } - body, err := os.ReadFile(filepath.Join(filepath.Dir(file), "github_proxy.go")) - if err != nil { - t.Fatalf("read github_proxy.go: %v", err) - } - text := string(body) + text := readGitHubProxySource(t) if clientTLSHandshakeTimeout != upstreamDialTimeout { t.Fatalf("clientTLSHandshakeTimeout = %v, want upstreamDialTimeout %v", clientTLSHandshakeTimeout, upstreamDialTimeout) } @@ -89,15 +104,7 @@ func TestAllClientTLSHandshakesAreBound(t *testing.T) { } func TestHTTPRelayDeadlinesAreBoundAndObservable(t *testing.T) { - _, file, _, ok := runtime.Caller(0) - if !ok { - t.Fatal("runtime.Caller failed") - } - body, err := os.ReadFile(filepath.Join(filepath.Dir(file), "github_proxy.go")) - if err != nil { - t.Fatalf("read github_proxy.go: %v", err) - } - text := string(body) + text := readGitHubProxySource(t) if !strings.Contains(text, "upstream.SetWriteDeadline(time.Now().Add(httpWriteTimeout))") { t.Fatal("request relays must bound upstream writes with httpWriteTimeout") } diff --git a/src/crux/telemetry/telemetry.go b/src/crux/telemetry/telemetry.go new file mode 100644 index 0000000000..019991bade --- /dev/null +++ b/src/crux/telemetry/telemetry.go @@ -0,0 +1,169 @@ +// Package telemetry installs the OpenTelemetry SDK for the crux binary +// (hivecommons/crux#19). The runtime records spans and kick metrics through +// the OTel API alone; without a registered provider those are no-ops. Init +// registers an OTLP/HTTP TracerProvider and MeterProvider, but only when the +// standard OTEL_EXPORTER_OTLP_* environment names an endpoint. With none set +// it changes nothing: the global providers stay the zero-cost no-ops. +// +// Everything about the export — endpoint, headers, TLS, timeouts, service +// name, resource attributes — comes from the standard OTel environment +// variables, read by the SDK and exporters themselves; no collector address +// lives in code. Hive's own process does not use this package: it wires +// pkg/tracing from hive.yaml. +package telemetry + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "sync" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/propagation" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +// Standard OpenTelemetry environment variables consulted to decide whether to +// export at all. The exporters read the full OTEL_EXPORTER_OTLP_* family +// (headers, TLS, timeouts, compression) on their own. +const ( + envEndpoint = "OTEL_EXPORTER_OTLP_ENDPOINT" + envTracesEndpoint = "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT" + envMetricsEndpoint = "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT" + envProtocol = "OTEL_EXPORTER_OTLP_PROTOCOL" + envTracesProtocol = "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL" + envMetricsProtocol = "OTEL_EXPORTER_OTLP_METRICS_PROTOCOL" + envTracesExporter = "OTEL_TRACES_EXPORTER" + envMetricsExporter = "OTEL_METRICS_EXPORTER" + envSDKDisabled = "OTEL_SDK_DISABLED" +) + +// defaultServiceName is the resource service.name when OTEL_SERVICE_NAME and +// OTEL_RESOURCE_ATTRIBUTES do not name one. +const defaultServiceName = "crux" + +// shutdownTimeout bounds the final flush so a dead collector cannot hold the +// process open on exit. +const shutdownTimeout = 5 * time.Second + +// Shutdown flushes and stops whatever Init installed. It is safe to call more +// than once and is a no-op when Init installed nothing. +type Shutdown func() + +func noopShutdown() {} + +// Init registers the OTLP trace and metric pipelines named by the environment +// and returns the function that flushes them. version becomes service.version +// on the resource. On error nothing is registered, the returned Shutdown is a +// no-op, and the caller keeps running with no-op providers. +func Init(ctx context.Context, version string) (Shutdown, error) { + return initFromEnv(ctx, version, os.Getenv) +} + +func initFromEnv(ctx context.Context, version string, getenv func(string) string) (Shutdown, error) { + traces, metrics := signalsEnabled(getenv) + if !traces && !metrics { + return noopShutdown, nil + } + for _, key := range []string{envProtocol, envTracesProtocol, envMetricsProtocol} { + // Only OTLP over HTTP is linked into crux, keeping gRPC out of the + // module's dependency set; refuse rather than send protobuf-over-HTTP + // to a gRPC port. + if p := strings.TrimSpace(getenv(key)); p != "" && p != "http/protobuf" { + return noopShutdown, fmt.Errorf("%s=%q is not supported (crux exports OTLP over http/protobuf only)", key, p) + } + } + + res, err := newResource(ctx, version) + if err != nil { + return noopShutdown, err + } + + // Build both pipelines before registering either global, so a failure + // on the second leaves no provider (and no shut-down provider) behind. + var ( + tp *sdktrace.TracerProvider + mp *sdkmetric.MeterProvider + stops []func(context.Context) error + ) + if traces { + exp, err := otlptracehttp.New(ctx) + if err != nil { + return noopShutdown, fmt.Errorf("trace exporter: %w", err) + } + tp = sdktrace.NewTracerProvider(sdktrace.WithBatcher(exp), sdktrace.WithResource(res)) + stops = append(stops, tp.Shutdown) + } + if metrics { + exp, err := otlpmetrichttp.New(ctx) + if err != nil { + for _, stop := range stops { + _ = stop(ctx) + } + return noopShutdown, fmt.Errorf("metric exporter: %w", err) + } + mp = sdkmetric.NewMeterProvider(sdkmetric.WithReader(sdkmetric.NewPeriodicReader(exp)), sdkmetric.WithResource(res)) + stops = append(stops, mp.Shutdown) + } + if tp != nil { + otel.SetTracerProvider(tp) + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})) + } + if mp != nil { + otel.SetMeterProvider(mp) + } + + var once sync.Once + return func() { + once.Do(func() { + sctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + var errs []error + for _, stop := range stops { + errs = append(errs, stop(sctx)) + } + if err := errors.Join(errs...); err != nil { + otel.Handle(err) + } + }) + }, nil +} + +// signalsEnabled reports which signals the environment asks crux to export: +// a signal is on when the shared or its signal-specific endpoint is set, the +// SDK is not disabled, and its OTEL_*_EXPORTER is not "none". +func signalsEnabled(getenv func(string) string) (traces, metrics bool) { + if strings.EqualFold(strings.TrimSpace(getenv(envSDKDisabled)), "true") { + return false, false + } + shared := strings.TrimSpace(getenv(envEndpoint)) != "" + traces = (shared || strings.TrimSpace(getenv(envTracesEndpoint)) != "") && !exporterNone(getenv(envTracesExporter)) + metrics = (shared || strings.TrimSpace(getenv(envMetricsEndpoint)) != "") && !exporterNone(getenv(envMetricsExporter)) + return traces, metrics +} + +func exporterNone(v string) bool { + return strings.EqualFold(strings.TrimSpace(v), "none") +} + +// newResource identifies this process. The crux defaults come first so +// OTEL_SERVICE_NAME and OTEL_RESOURCE_ATTRIBUTES, applied after, override them. +func newResource(ctx context.Context, version string) (*resource.Resource, error) { + attrs := []attribute.KeyValue{attribute.String("service.name", defaultServiceName)} + if v := strings.TrimSpace(version); v != "" { + attrs = append(attrs, attribute.String("service.version", v)) + } + return resource.New(ctx, + resource.WithAttributes(attrs...), + resource.WithTelemetrySDK(), + resource.WithFromEnv(), + ) +} diff --git a/src/crux/telemetry/telemetry_test.go b/src/crux/telemetry/telemetry_test.go new file mode 100644 index 0000000000..6788edc673 --- /dev/null +++ b/src/crux/telemetry/telemetry_test.go @@ -0,0 +1,231 @@ +package telemetry + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" +) + +func envOf(m map[string]string) func(string) string { + return func(k string) string { return m[k] } +} + +func TestSignalsEnabled(t *testing.T) { + cases := []struct { + name string + env map[string]string + traces, metrics bool + }{ + {"unset", nil, false, false}, + {"shared endpoint", map[string]string{envEndpoint: "http://collector:4318"}, true, true}, + {"traces only", map[string]string{envTracesEndpoint: "http://c/v1/traces"}, true, false}, + {"metrics only", map[string]string{envMetricsEndpoint: "http://c/v1/metrics"}, false, true}, + {"blank endpoint", map[string]string{envEndpoint: " "}, false, false}, + {"sdk disabled", map[string]string{envEndpoint: "http://c", envSDKDisabled: "TRUE"}, false, false}, + {"traces exporter none", map[string]string{envEndpoint: "http://c", envTracesExporter: "none"}, false, true}, + {"metrics exporter none", map[string]string{envEndpoint: "http://c", envMetricsExporter: "None"}, true, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + traces, metrics := signalsEnabled(envOf(tc.env)) + if traces != tc.traces || metrics != tc.metrics { + t.Fatalf("signalsEnabled = (%v, %v), want (%v, %v)", traces, metrics, tc.traces, tc.metrics) + } + }) + } +} + +// With no endpoint configured Init must leave the global providers alone: +// that is the zero-behaviour-change contract for current users. +func TestInitWithoutEndpointKeepsNoopProviders(t *testing.T) { + tp, mp := otel.GetTracerProvider(), otel.GetMeterProvider() + shutdown, err := initFromEnv(context.Background(), "v1", envOf(nil)) + if err != nil { + t.Fatalf("initFromEnv: %v", err) + } + shutdown() + shutdown() + if otel.GetTracerProvider() != tp || otel.GetMeterProvider() != mp { + t.Fatal("initFromEnv replaced a global provider without an endpoint") + } +} + +func TestInitRejectsGRPCProtocol(t *testing.T) { + tp := otel.GetTracerProvider() + _, err := initFromEnv(context.Background(), "v1", envOf(map[string]string{ + envEndpoint: "http://collector:4317", + envProtocol: "grpc", + })) + if err == nil || !strings.Contains(err.Error(), envProtocol) { + t.Fatalf("err = %v, want an unsupported-protocol error naming %s", err, envProtocol) + } + if otel.GetTracerProvider() != tp { + t.Fatal("a rejected configuration must not install a provider") + } +} + +func TestNewResourceDefaultsAndEnvOverride(t *testing.T) { + t.Setenv("OTEL_SERVICE_NAME", "") + t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "") + res, err := newResource(context.Background(), "1.2.3") + if err != nil { + t.Fatalf("newResource: %v", err) + } + if got, _ := res.Set().Value(attribute.Key("service.name")); got.AsString() != defaultServiceName { + t.Fatalf("service.name = %q, want %q", got.AsString(), defaultServiceName) + } + if got, _ := res.Set().Value(attribute.Key("service.version")); got.AsString() != "1.2.3" { + t.Fatalf("service.version = %q, want 1.2.3", got.AsString()) + } + + t.Setenv("OTEL_SERVICE_NAME", "crux-edge") + res, err = newResource(context.Background(), "") + if err != nil { + t.Fatalf("newResource: %v", err) + } + if got, _ := res.Set().Value(attribute.Key("service.name")); got.AsString() != "crux-edge" { + t.Fatalf("service.name = %q, want OTEL_SERVICE_NAME to override", got.AsString()) + } + if _, ok := res.Set().Value(attribute.Key("service.version")); ok { + t.Fatal("an empty version must not stamp service.version") + } +} + +// With an endpoint set, Init must actually export: one span and one counter +// recorded through the OTel API reach the collector as /v1/traces and +// /v1/metrics POSTs when Shutdown flushes. +func TestInitExportsToCollectorOnShutdown(t *testing.T) { + prevTP, prevMP, prevProp := otel.GetTracerProvider(), otel.GetMeterProvider(), otel.GetTextMapPropagator() + t.Cleanup(func() { + otel.SetTracerProvider(prevTP) + otel.SetMeterProvider(prevMP) + otel.SetTextMapPropagator(prevProp) + }) + + var ( + mu sync.Mutex + posts = map[string]int{} + ) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + mu.Lock() + posts[r.URL.Path]++ + mu.Unlock() + } + w.Header().Set("Content-Type", "application/x-protobuf") + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(srv.Close) + t.Setenv(envEndpoint, srv.URL) + + flush, err := Init(context.Background(), "test") + if err != nil { + t.Fatalf("Init: %v", err) + } + _, span := otel.Tracer("crux/telemetry/test").Start(context.Background(), "test.span") + span.End() + counter, err := otel.Meter("crux/telemetry/test").Int64Counter("test.counter") + if err != nil { + t.Fatalf("Int64Counter: %v", err) + } + counter.Add(context.Background(), 1) + flush() + + mu.Lock() + defer mu.Unlock() + if posts["/v1/traces"] == 0 || posts["/v1/metrics"] == 0 { + t.Fatalf("collector POSTs = %v, want /v1/traces and /v1/metrics", posts) + } +} + +// caCertPEM writes a self-signed certificate the exporters' CERTIFICATE env +// variables can load, so a TLS root can be configured against an http:// +// endpoint — the one exporter construction error reachable through env. +func caCertPEM(t *testing.T) string { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("GenerateKey: %v", err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "crux-telemetry-test"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + KeyUsage: x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatalf("CreateCertificate: %v", err) + } + path := filepath.Join(t.TempDir(), "ca.pem") + if err := os.WriteFile(path, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600); err != nil { + t.Fatalf("WriteFile: %v", err) + } + return path +} + +func TestInitResourceErrorInstallsNothing(t *testing.T) { + tp, mp := otel.GetTracerProvider(), otel.GetMeterProvider() + t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment") // no '=': ErrPartialResource + shutdown, err := initFromEnv(context.Background(), "v1", envOf(map[string]string{envEndpoint: "http://collector:4318"})) + if err == nil { + t.Fatal("initFromEnv must surface an invalid OTEL_RESOURCE_ATTRIBUTES") + } + shutdown() + if otel.GetTracerProvider() != tp || otel.GetMeterProvider() != mp { + t.Fatal("a resource error must not install a provider") + } +} + +func TestInitTraceExporterErrorInstallsNothing(t *testing.T) { + tp, mp := otel.GetTracerProvider(), otel.GetMeterProvider() + t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "") + t.Setenv(envEndpoint, "http://collector:4318") + t.Setenv("OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE", caCertPEM(t)) + t.Setenv("OTEL_EXPORTER_OTLP_TRACES_INSECURE", "true") + shutdown, err := initFromEnv(context.Background(), "v1", envOf(map[string]string{envEndpoint: "http://collector:4318"})) + if err == nil || !strings.Contains(err.Error(), "trace exporter") { + t.Fatalf("err = %v, want a trace exporter error", err) + } + shutdown() + if otel.GetTracerProvider() != tp || otel.GetMeterProvider() != mp { + t.Fatal("a trace exporter error must not install a provider") + } +} + +// When the metric pipeline fails after the trace pipeline was built, the +// trace provider is shut down and neither global is replaced. +func TestInitMetricExporterErrorTearsDownTraces(t *testing.T) { + tp, mp := otel.GetTracerProvider(), otel.GetMeterProvider() + t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "") + t.Setenv(envEndpoint, "http://collector:4318") + t.Setenv("OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE", caCertPEM(t)) + t.Setenv("OTEL_EXPORTER_OTLP_METRICS_INSECURE", "true") + shutdown, err := initFromEnv(context.Background(), "v1", envOf(map[string]string{envEndpoint: "http://collector:4318"})) + if err == nil || !strings.Contains(err.Error(), "metric exporter") { + t.Fatalf("err = %v, want a metric exporter error", err) + } + shutdown() + if otel.GetTracerProvider() != tp || otel.GetMeterProvider() != mp { + t.Fatal("a metric exporter error must not leave the trace provider installed") + } +} diff --git a/src/deploy/gh-aw/README.md b/src/deploy/gh-aw/README.md index c53a33e92f..e9a7d58897 100644 --- a/src/deploy/gh-aw/README.md +++ b/src/deploy/gh-aw/README.md @@ -46,8 +46,9 @@ compiles without model credentials, and tests admission and the actual classifie (compiled to `weekly-competitor-report.lock.yml`): every Monday at 13:00 UTC, and on manual dispatch, the Copilot engine researches projects adjacent to Hive (OpenHands, SWE-agent, Devin, Copilot coding agent, Codex, Claude Code, - Sweep, Factory, Jules, Gastown, Dagger, Kilo, Cline/Roo, Mentat, Aider, - GPT-Engineer, Potpie, Agentless, public fleet-orchestrator write-ups) with + Goose, OpenCode, Sweep, Factory, Jules, Gastown, Dagger, Kilo, Cline/Roo, + Mentat, Aider, GPT-Engineer, Potpie, Agentless, public fleet-orchestrator + write-ups) with read-only GitHub tools and `curl` inside a network allowlist (HN Algolia, Google News RSS, Bing News RSS, the GitHub changelog feed, vendor blogs; the bash allowlist uses `cmd:*` entries so pipelines like `curl | jq` are @@ -83,6 +84,24 @@ compiles without model credentials, and tests admission and the actual classifie the webhook secret the Discord job fails with a clear message and the GitHub comment still lands. Recompile after edits with the pinned compiler: `gh aw compile weekly-competitor-report`, and commit both files (#14612). + The prompt names the current GitHub repository for each project (several + moved: `OpenHands/OpenHands`, `SWE-agent/SWE-agent`, `aaif-goose/goose`, + `anomalyco/opencode`) and tells the model to follow API redirects with + `curl -sL`, because a renamed repository answers `301` with no `license` + field, which showed up as "n/a (API error)" in the 2026-10-27 report. Any + engine in the Crux `CLIBackends` list is reported as "Hive runs it as a + worker engine today" even when its vendor also sells orchestration. + Known cosmetic issue: the "Firewall blocked N domains" warning that gh-aw + appends to the comment is a false positive on gh-aw v0.89.21. Squid writes + two lines per successful HTTPS `CONNECT` (`200 NONE_NONE:HIER_NONE`, then + `200 TCP_TUNNEL:HIER_DIRECT`) and the footer parser counted the first as + a block, so every allowed domain showed equal allowed and denied counts + (run 40579499092: `api.github.com` 36/36, `news.google.com` 6/6). This is + upstream, not in the lock file: github/gh-aw#63221, fixed by + github/gh-aw#63246 (2026-09-24), first shipped in gh-aw v0.89.22. The + warning goes away when the compiler pin is bumped to v0.89.22 or later; + until then ignore it when the firewall log's Allowed column matches the + Denied column. ## Graduate to Hive without rewriting workflows diff --git a/src/deploy/test_ttyd_default_socket_uid.sh b/src/deploy/test_ttyd_default_socket_uid.sh new file mode 100644 index 0000000000..d3027d813a --- /dev/null +++ b/src/deploy/test_ttyd_default_socket_uid.sh @@ -0,0 +1,266 @@ +#!/usr/bin/env bash +# The browser terminal must find the default tmux socket whatever UID the +# container runs as. +# Run: bash src/deploy/test_ttyd_default_socket_uid.sh +# +# THE BUG. With no per-agent UID map ("no UID map found, agents will share dev +# UID") every hive- session lives on the DEFAULT tmux server of the UID +# the container runs as, i.e. $TMUX_TMPDIR/tmux-$(id -u)/default. The attach +# helper's fallback instead looked at /tmp/tmux-$(id -u dev || 1001)/default — +# dev's PASSWD uid. On Compose/Podman the two are the same number and nobody +# noticed. Under OpenShift's restricted SCC the container runs as an arbitrary +# UID (1000840000 on the spoke that reported this) that has no passwd entry, so +# the sessions were at /tmp/tmux-1000840000/default, the helper polled +# /tmp/tmux-1001/default forever, and the dashboard showed "waiting for agent +# 'X' to start (tmux session not created yet)" for every agent while `tmux ls` +# showed them all alive. +# +# HOW THIS TESTS IT. The helper derives the UID from `id -u`, so `id` is +# stubbed on PATH to answer with an OpenShift-style UID (and, per case, with or +# without a passwd entry for dev). TMUX_TMPDIR points the helper's socket +# lookup at a scratch directory where a real tmux server is started with +# `tmux -S` at exactly the path the container would use. The helper is then +# run for real against it: resolving the socket is asserted POSITIVELY through +# the wheel rebind the helper leaves on the server (the one setting it does not +# restore on detach), because an absence-only check passes just as happily when +# the script never reaches attach at all. +set -uo pipefail + +PASS=0 +FAIL=0 +pass() { echo " PASS: $1"; PASS=$((PASS + 1)); } +fail() { echo " FAIL: $1"; [ $# -gt 1 ] && echo " $2"; FAIL=$((FAIL + 1)); } +finish() { + echo "" + echo "=== Results: $PASS passed, $FAIL failed ===" + [ "$FAIL" -eq 0 ] +} + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +ATTACH="${ROOT}/src/deploy/ttyd-tmux.sh" + +echo "=== ttyd-tmux.sh finds the default tmux socket under an arbitrary container UID ===" + +if [ ! -f "$ATTACH" ]; then + fail "locate $ATTACH" "the layout moved — this test cannot verify anything" + finish; exit $? +fi + +if ! command -v tmux >/dev/null 2>&1; then + echo " SKIP: tmux not installed — these assertions need a real tmux server" + if [ "${HIVE_TEST_REQUIRE_BEHAVIOURAL:-}" = "1" ]; then + fail "tmux is required when HIVE_TEST_REQUIRE_BEHAVIOURAL=1" + fi + finish; exit $? +fi +if command -v su-exec >/dev/null 2>&1; then + # The helper would su-exec to the socket owner when the stubbed uid differs + # from it; that path needs root and is not what is under test here. + echo " SKIP: su-exec is on PATH — the uid stub would route through it" + finish; exit $? +fi + +# The UID of the reporting OpenShift spoke. Nothing on a developer box or CI +# runner owns this, so the helper's historical /tmp/tmux-1001 assumption cannot +# be satisfied by accident. +ARBITRARY_UID=1000840000 +DEV_UID=1001 +OTHER_UID=4242 + +WORK="$(mktemp -d)" +SOCKET_ROOT="${WORK}/sockets" +STUB_BIN="${WORK}/bin" +mkdir -p "$SOCKET_ROOT" "$STUB_BIN" +declare -a SERVERS=() +cleanup() { + local s + for s in "${SERVERS[@]+"${SERVERS[@]}"}"; do + tmux -S "$s" kill-server 2>/dev/null + done + rm -rf "$WORK" +} +trap cleanup EXIT + +# stub_id : `id -u` answers the first, `id -u dev` +# the second (or fails, like getent on a container with no passwd entry for an +# arbitrary UID). Anything else falls through to the real id. +REAL_ID="$(command -v id)" +stub_id() { + cat > "${STUB_BIN}/id" <&2; exit 1 +fi +exec "$REAL_ID" "\$@" +EOF + chmod +x "${STUB_BIN}/id" +} + +run_with_timeout() { + local seconds="$1" + shift + if command -v timeout >/dev/null 2>&1; then + timeout "$seconds" "$@" + else + perl -e ' + my $seconds = shift @ARGV; + my $pid = fork(); + exit 127 unless defined $pid; + if ($pid == 0) { setpgrp(0, 0); exec @ARGV; exit 127; } + $SIG{ALRM} = sub { + kill "TERM", -$pid; select undef, undef, undef, 0.2; kill "KILL", -$pid; + waitpid($pid, 0); exit 124; + }; + alarm $seconds; waitpid($pid, 0); + exit 127 if $? == -1; exit 128 + ($? & 127) if $? & 127; exit $? >> 8; + ' "$seconds" "$@" + fi +} + +# start_default_server : a real tmux server at the path the +# container's default server would use for that uid. +start_default_server() { + local dir="${SOCKET_ROOT}/tmux-$1" sock + mkdir -p "$dir" && chmod 700 "$dir" + sock="${dir}/default" + tmux -S "$sock" -f /dev/null new-session -d -s "$2" 'sleep 300' 2>/dev/null || return 1 + SERVERS+=("$sock") + [ -S "$sock" ] +} + +# run_attach : the helper, for real, with a short wait so a lookup +# that still fails reports quickly instead of polling for 900s. Invoked through +# `bash` because the file ships mode 644 and is chmod'ed at image build time. +run_attach() { + PATH="${STUB_BIN}:${PATH}" TMUX_TMPDIR="$SOCKET_ROOT" \ + HIVE_TTYD_WAIT_SEC=2 HIVE_TTYD_POLL_SEC=1 \ + run_with_timeout 15 bash "$ATTACH" "$1" &1 +} + +# assert_resolves