From f5cc08b48fe3dc3c29fad163a061deac90b19338 Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:39:50 +0100 Subject: [PATCH 01/38] =?UTF-8?q?=F0=9F=90=9B=20fix(review):=20wake=20the?= =?UTF-8?q?=20governor=20when=20a=20dispatched=20review=20verdict=20is=20r?= =?UTF-8?q?ecorded=20(#14706)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(review): wake the governor when a dispatched review verdict is recorded Review dispatch was planned only on a governor eval cycle, and recording a head's verdict did not start one, so on a quiet repo every reviewer sat idle for up to governor.eval_interval_webhook_s (900s) between kicks. The relay now calls a ReviewVerdictHook (Client.SetReviewVerdictHook) once per request whose verdicts bind a review dispatch; advisory and refused verdicts do not call it. The hive wires it to a verdict waker whose batches runLoop turns into a cached-snapshot eval cycle. The first verdict wakes at once; later ones inside governor.webhook_wake_min_gap_s fold into one wake at the end of the gap. Hive-Run: hivecommons/hive#14705 Hive-Plan: hivecommons/hive#14705 Proposed fix Hive-Spec: review-verdict-wake#14705 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HRH6kngWRcKiKJefZzZeqz Signed-off-by: Karibhou * πŸ“– docs: re-anchor RankActionableIssues citation in scale-envelope.md The verdict-hook addition to client.go shifted RankActionableIssues down 10 lines, so the citation guard reported drift. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Karibhou * πŸ“– docs: say the review verdict wake ignores webhook_wake_debounce_s Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HRH6kn4BsouThkrpz1TTCU Signed-off-by: Karibhou Co-authored-by: gregoryhunt <31053269+gregoryhunt@users.noreply.github.com> --- .../fixed-14705-review-verdict-wake.md | 1 + src/cmd/hive/boot.go | 4 + src/cmd/hive/github_client_config.go | 3 + src/cmd/hive/main.go | 7 ++ src/cmd/hive/review_event_dispatch.go | 31 ++++++ src/cmd/hive/review_event_dispatch_test.go | 72 +++++++++++++- src/docs/operator-reference.md | 11 +++ src/docs/scale-envelope.md | 2 +- src/docs/webhooks.md | 2 +- src/pkg/github/client.go | 3 + src/pkg/github/review_verdict_hook.go | 33 +++++++ src/pkg/github/review_verdict_hook_test.go | 94 +++++++++++++++++++ src/pkg/github/review_verdict_relay.go | 28 ++++-- 13 files changed, 279 insertions(+), 12 deletions(-) create mode 100644 changelog.d/fixed-14705-review-verdict-wake.md create mode 100644 src/pkg/github/review_verdict_hook.go create mode 100644 src/pkg/github/review_verdict_hook_test.go diff --git a/changelog.d/fixed-14705-review-verdict-wake.md b/changelog.d/fixed-14705-review-verdict-wake.md new file mode 100644 index 0000000000..7acd8a2166 --- /dev/null +++ b/changelog.d/fixed-14705-review-verdict-wake.md @@ -0,0 +1 @@ +- Recording a review head's dispatched verdict now wakes the governor for an early cached-snapshot eval cycle, so the next head or the decision kick is dispatched right away instead of waiting for an unrelated webhook or `governor.eval_interval_webhook_s`; a burst of verdicts is coalesced to at most one wake per `governor.webhook_wake_min_gap_s` ([#14705](https://github.com/hivecommons/hive/issues/14705)). diff --git a/src/cmd/hive/boot.go b/src/cmd/hive/boot.go index e1728bbdf6..2110d29067 100644 --- a/src/cmd/hive/boot.go +++ b/src/cmd/hive/boot.go @@ -259,6 +259,10 @@ type boot struct { // carries its wakes to the governor loop (hivecommons/hive#11091). reviewEvents *eventdispatch.Dispatcher reviewWake chan struct{} + // verdictWaker turns dispatched review verdicts the relay records into + // early eval cycles, at most one per governor.webhook_wake_min_gap_s + // (hivecommons/hive#14705). + verdictWaker *webhookWaker // webhookWaker coalesces cache-invalidating GitHub webhooks into early // eval cycles and evalLoop is the pacing block /api/status reports. diff --git a/src/cmd/hive/github_client_config.go b/src/cmd/hive/github_client_config.go index a1a58ac668..4c690fc89f 100644 --- a/src/cmd/hive/github_client_config.go +++ b/src/cmd/hive/github_client_config.go @@ -387,6 +387,9 @@ func (b *boot) applyGitHubClientDashboardHooks(client *github.Client) { // claims ledger is on. recordStarted(b.dashSrv, b.cfg.Project.Org, agentName, repo, issue, signal) }) + // hivecommons/hive#14705: a recorded dispatched verdict frees its head, + // so the governor re-plans now instead of at the next eval cycle. + client.SetReviewVerdictHook(b.wakeForReviewVerdict) // PR follow-up session resume (#9583, default off): remember which live // CLI session authored this PR, and the PR's handoff note, so its // follow-ups can resume that conversation or brief a fresh one. The diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 38e81c4143..96fcfec70f 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -6371,6 +6371,13 @@ func (b *boot) runLoopWith(deps runLoopDeps) { tick(evalSourceCached) armContinuousWake() armWebhookHealthRecheck() + case batch := <-b.verdictWakeChan(): + b.logger.Info("review verdict wake: running an early eval cycle from the cached snapshot", + "pr", batch.Reason, "last_pr", batch.LastRe, "verdicts", batch.Count, + "coalesced", batch.Coalesced.Round(time.Millisecond).String()) + tick(evalSourceCached) + armContinuousWake() + armWebhookHealthRecheck() case batch := <-b.webhookWakeChan(): now := healthNow() // Apply the delivery's targeted re-reads first so this cycle diff --git a/src/cmd/hive/review_event_dispatch.go b/src/cmd/hive/review_event_dispatch.go index 841d608197..711d2feb32 100644 --- a/src/cmd/hive/review_event_dispatch.go +++ b/src/cmd/hive/review_event_dispatch.go @@ -1,8 +1,10 @@ package main import ( + "fmt" "sort" "sync/atomic" + "time" "github.com/hivecommons/hive/pkg/review" "github.com/hivecommons/hive/pkg/review/eventdispatch" @@ -22,6 +24,12 @@ func (b *boot) installReviewEvents() { if b.reviewWake == nil { b.reviewWake = make(chan struct{}, 1) } + if b.verdictWaker == nil { + b.verdictWaker = newWebhookWaker( + func() time.Duration { return 0 }, + func() time.Duration { return b.cfg.WebhookWakeMinGap() }, + ) + } activeReviewEvents.Store(b.reviewEvents) } @@ -48,6 +56,29 @@ func (b *boot) wakeForReviewEvents(batch []eventdispatch.Entry) { } } +// wakeForReviewVerdict is the github.ReviewVerdictHook: the relay recorded a +// verdict bound to a review dispatch, so the head that wrote it is free and +// the PR may be ready for its next head or its decision kick. Without this +// the next kick waited for an unrelated webhook or the eval cadence +// (hivecommons/hive#14705). The first verdict wakes the loop at once; later +// ones inside governor.webhook_wake_min_gap_s fold into one wake at the end +// of the gap, so a burst of verdicts costs one cycle. +func (b *boot) wakeForReviewVerdict(repo string, number int) { + if b == nil || b.verdictWaker == nil { + return + } + b.verdictWaker.Request(fmt.Sprintf("%s#%d", repo, number)) +} + +// verdictWakeChan is the loop's receive side; nil when the waker is not +// installed so the select case never fires. +func (b *boot) verdictWakeChan() <-chan webhookWakeBatch { + if b.verdictWaker == nil { + return nil + } + return b.verdictWaker.out +} + // prioritizeEventPRs moves PRs whose current head a review webhook fired for // to the front, keeping order otherwise. PlanDispatch walks the list in order // and still applies every gate, so this changes who is first, not who is diff --git a/src/cmd/hive/review_event_dispatch_test.go b/src/cmd/hive/review_event_dispatch_test.go index 10469b8d0c..a52bb6c9d1 100644 --- a/src/cmd/hive/review_event_dispatch_test.go +++ b/src/cmd/hive/review_event_dispatch_test.go @@ -63,8 +63,12 @@ func TestInstallReviewEventsIsIdempotent(t *testing.T) { if d == nil || wake == nil || activeReviewEvents.Load() != d { t.Fatal("dispatcher not installed") } + verdicts := b.verdictWaker + if verdicts == nil { + t.Fatal("verdict waker not installed") + } b.installReviewEvents() - if b.reviewEvents != d || b.reviewWake != wake { + if b.reviewEvents != d || b.reviewWake != wake || b.verdictWaker != verdicts { t.Fatal("reinstall replaced the dispatcher") } t.Cleanup(func() { activeReviewEvents.Store(nil) }) @@ -140,3 +144,69 @@ func TestRunLoopWithRunsAnEarlyCycleOnReviewWake(t *testing.T) { t.Fatalf("log = %s", log.String()) } } + +// hivecommons/hive#14705: the first recorded verdict wakes the loop at once; +// the verdicts that follow inside governor.webhook_wake_min_gap_s fold into +// one wake at the end of the gap. +func TestWakeForReviewVerdictCoalescesInsideMinGap(t *testing.T) { + w, f := newTestWebhookWaker(0, 30*time.Second) + b := &boot{verdictWaker: w} + + b.wakeForReviewVerdict("o/r", 1) + if len(f.delays) != 1 || f.delays[0] != 0 { + t.Fatalf("first verdict should wake at once, armed %v", f.delays) + } + f.fire(t) + if got := <-b.verdictWakeChan(); got.Count != 1 || got.Reason != "o/r#1" { + t.Fatalf("first wake = %+v", got) + } + + f.now = f.now.Add(5 * time.Second) + b.wakeForReviewVerdict("o/r", 2) + b.wakeForReviewVerdict("o/r", 3) + if len(f.delays) != 2 || f.delays[1] != 25*time.Second { + t.Fatalf("verdicts inside the gap should arm one timer for the rest of it, armed %v", f.delays) + } + select { + case got := <-b.verdictWakeChan(): + t.Fatalf("woke inside the min gap: %+v", got) + default: + } + f.fire(t) + if got := <-b.verdictWakeChan(); got.Count != 2 || got.Reason != "o/r#2" || got.LastRe != "o/r#3" { + t.Fatalf("coalesced wake = %+v", got) + } +} + +func TestWakeForReviewVerdictWithoutWakerIsNoop(t *testing.T) { + var nilBoot *boot + nilBoot.wakeForReviewVerdict("o/r", 1) + b := &boot{} + b.wakeForReviewVerdict("o/r", 1) + if b.verdictWakeChan() != nil { + t.Fatal("uninstalled waker should leave the loop case nil") + } +} + +func TestRunLoopWithRunsAnEarlyCycleOnVerdictWake(t *testing.T) { + b, log, cancel := newLoopBoot(t, 60, 0) + b.verdictWaker, _ = newTestWebhookWaker(0, 30*time.Second) + gov := newFakeTicker() + h := &loopHarness{startupOK: true, pending: []*fakeTicker{gov}, persisted: make(chan struct{}, 8)} + done := runLoopAsync(b, h.deps()) + <-h.persisted // first cycle done + + b.verdictWaker.out <- webhookWakeBatch{Reason: "o/r#1", LastRe: "o/r#2", Count: 2} + <-h.persisted // early cycle done + gov.fire() + <-h.persisted // cadence cycle still runs + cancel() + waitDone(t, done) + + if len(h.evals) != 3 { + t.Fatalf("evals = %d, want first + verdict + cadence", len(h.evals)) + } + if !strings.Contains(log.String(), "review verdict wake") || !strings.Contains(log.String(), "verdicts=2") { + t.Fatalf("log = %s", log.String()) + } +} diff --git a/src/docs/operator-reference.md b/src/docs/operator-reference.md index 7970cbcb2a..f5da073492 100644 --- a/src/docs/operator-reference.md +++ b/src/docs/operator-reference.md @@ -1220,6 +1220,17 @@ v5 and v6 spokes) tracking one repo all stay on webhook mode. `review_requested` deliveries also queue an early review dispatch when `review.event_driven` is on; see [Event-driven dispatch](review-swarm.md#event-driven-dispatch). +- **A recorded review verdict also wakes the governor loop.** When the + relay records a verdict bound to a review dispatch, the head that wrote it + is free and the PR may be ready for its next head or its decision kick, so + the loop runs a cached-snapshot cycle now instead of waiting for an + unrelated delivery or `governor.eval_interval_webhook_s`. Advisory verdicts + do not wake it. The first verdict wakes at once; later ones inside + `governor.webhook_wake_min_gap_s` fold into one wake at the end of the gap. + `governor.webhook_wake_debounce_s` does not apply to this wake (its + debounce is fixed at 0, so a verdict is never held back), and the spoke logs `review verdict wake: running an early eval cycle from + the cached snapshot` + ([#14705](https://github.com/hivecommons/hive/issues/14705)). - Every status publish (cadence or wake) invalidates the `/api/stage-tracker` cache and sends a `stage-tracker` event on `/api/events`, so the dashboard's activity monitor refetches at once instead of waiting out diff --git a/src/docs/scale-envelope.md b/src/docs/scale-envelope.md index 4074105726..92e0db2f76 100644 --- a/src/docs/scale-envelope.md +++ b/src/docs/scale-envelope.md @@ -34,7 +34,7 @@ ones to: 2. **Truncation must preserve the prioritization signal, not cut an arbitrary slice.** This holds today and is easy to assume backwards: - Issues are sorted by actionable rank tier, then **oldest first** within each tier before any cap is applied - (`RankActionableIssues`, `src/pkg/github/client.go:4676-4693`, descending `AgeMinutes`). + (`RankActionableIssues`, `src/pkg/github/client.go:4679-4696`, descending `AgeMinutes`). - PRs are sorted by **review class then oldest-first within class** (`SortPullRequestsForReview`, `src/pkg/github/review_priority.go:334`; fixes β†’ refactors/docs β†’ tests). diff --git a/src/docs/webhooks.md b/src/docs/webhooks.md index 3529558f1e..1a42425872 100644 --- a/src/docs/webhooks.md +++ b/src/docs/webhooks.md @@ -59,7 +59,7 @@ These are the acceptance criteria of #13373 and the tests that hold them. | `governor.eval_interval_webhook_s` | `900` | the base tick in webhook mode; `0` disables webhook mode (always poll) | | `governor.eval_interval_max_s` | `1800` | cap on either tick after the API-budget stretch | | `governor.webhook_wake_debounce_s` | `10` | a delivery and its check fan-out coalesce into one wake cycle | -| `governor.webhook_wake_min_gap_s` | `30` | minimum spacing between webhook-triggered cycles | +| `governor.webhook_wake_min_gap_s` | `30` | minimum spacing between webhook-triggered cycles, and between the cycles recorded review verdicts trigger | | `forgestate.reconcile_interval_s` | `900` | ForgeState's own full-list interval per repo, shortened while drift is high | | `forgestate.freshness_bound_s` | `60` | maximum item age before merge/close/claim/kick re-reads it | | `forgestate.drift_threshold` | `0.05` | fraction of items a reconcile may find wrong before the interval shortens | diff --git a/src/pkg/github/client.go b/src/pkg/github/client.go index ea7f9c15a1..a125ad2993 100644 --- a/src/pkg/github/client.go +++ b/src/pkg/github/client.go @@ -304,6 +304,9 @@ type Client struct { // agentStartHook is told when an agent's own relay request shows it // working an issue (#10527). See agent_start_hook.go. agentStartHook atomic.Pointer[AgentStartHook] + // reviewVerdictHook is told when the relay records a verdict bound to a + // review dispatch (review_verdict_hook.go). + reviewVerdictHook atomic.Pointer[ReviewVerdictHook] // mergeAuthz gates merge requests from the merge-request watcher against the // per-agent ACMM merge-policy (CanMerge) + forge-resistance AND the merge // TARGET (pinned SHA + governor merge-eligible membership; see diff --git a/src/pkg/github/review_verdict_hook.go b/src/pkg/github/review_verdict_hook.go new file mode 100644 index 0000000000..724f2076a4 --- /dev/null +++ b/src/pkg/github/review_verdict_hook.go @@ -0,0 +1,33 @@ +package github + +// ReviewVerdictHook is told that the relay recorded at least one verdict +// bound to a review dispatch for PR number in repo. The head that wrote it +// is free again, and the PR may now be ready for its next head or its +// decision kick, so the hive uses it to wake the governor loop rather than +// leave the reviewers idle until the next eval cycle +// (hivecommons/hive#14705). +type ReviewVerdictHook func(repo string, number int) + +// SetReviewVerdictHook installs (or with nil, removes) the verdict hook. +// Safe to call before or after the watchers start. +func (c *Client) SetReviewVerdictHook(fn ReviewVerdictHook) { + if c == nil { + return + } + if fn == nil { + c.reviewVerdictHook.Store(nil) + return + } + c.reviewVerdictHook.Store(&fn) +} + +// notifyReviewVerdict reports a recorded dispatched verdict to the installed +// hook, if any. +func (c *Client) notifyReviewVerdict(repo string, number int) { + if c == nil { + return + } + if hook := c.reviewVerdictHook.Load(); hook != nil && *hook != nil { + (*hook)(repo, number) + } +} diff --git a/src/pkg/github/review_verdict_hook_test.go b/src/pkg/github/review_verdict_hook_test.go new file mode 100644 index 0000000000..b3cae147bb --- /dev/null +++ b/src/pkg/github/review_verdict_hook_test.go @@ -0,0 +1,94 @@ +package github + +import ( + "context" + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/review" +) + +type verdictHookCall struct { + repo string + number int +} + +func recordingVerdictHook(c *Client) *[]verdictHookCall { + var calls []verdictHookCall + c.SetReviewVerdictHook(func(repo string, number int) { + calls = append(calls, verdictHookCall{repo, number}) + }) + return &calls +} + +// A recorded dispatched verdict frees its head, so the hook fires: once per +// request, even when a combined review answered several perspectives +// (hivecommons/hive#14705). +func TestRecordReviewVerdictCallsHookOnceForDispatchedVerdicts(t *testing.T) { + c := &Client{logger: testLogger()} + calls := recordingVerdictHook(c) + withVerdictDispatchState(t, review.DispatchState{Pending: []review.PendingReview{ + pendingVerdict("o/r", 7, "correctness", "reviewer"), + pendingVerdict("o/r", 7, "security", "reviewer"), + }}) + raw := "[" + strings.Join([]string{ + validVerdictJSON(t, "o/r", 7, "correctness", "approve"), + validVerdictJSON(t, "o/r", 7, "security", "approve"), + }, ",") + "]" + + c.recordReviewVerdict(context.Background(), ReviewRequest{Repo: "o/r", Number: 7, Agent: "reviewer", Report: raw}, t.TempDir()) + + if len(*calls) != 1 || (*calls)[0] != (verdictHookCall{"o/r", 7}) { + t.Fatalf("hook calls = %+v, want one for o/r#7", *calls) + } +} + +// An advisory verdict binds no dispatch, so no head was freed and nothing +// wakes. +func TestRecordReviewVerdictDoesNotCallHookForAdvisoryVerdict(t *testing.T) { + c := &Client{logger: testLogger()} + calls := recordingVerdictHook(c) + withVerdictDispatchState(t, review.DispatchState{Pending: []review.PendingReview{ + pendingVerdict("o/r", 2, review.PerspectiveSecurity, "reviewer"), + }}) + raw := validVerdictJSONWithHead(t, "o/r", 1, "security", "approve", "abc123") + + got := c.recordReviewVerdict(context.Background(), ReviewRequest{Repo: "o/r", Number: 1, Agent: "reviewer", Report: raw}, t.TempDir()) + + if len(got) != 1 { + t.Fatalf("advisory verdict was not recorded: %+v", got) + } + if len(*calls) != 0 { + t.Fatalf("advisory verdict called the hook: %+v", *calls) + } +} + +// A refused verdict records nothing and wakes nothing. +func TestRecordReviewVerdictDoesNotCallHookForRefusedVerdict(t *testing.T) { + c := &Client{logger: testLogger()} + calls := recordingVerdictHook(c) + withVerdictDispatchState(t, review.DispatchState{}) + raw := validVerdictJSON(t, "o/r", 1, "correctness", "approve") + + got := c.recordReviewVerdict(context.Background(), ReviewRequest{Repo: "o/r", Number: 1, Agent: "reviewer", Report: raw}, t.TempDir()) + + if len(got) != 0 { + t.Fatalf("refused verdict was recorded: %+v", got) + } + if len(*calls) != 0 { + t.Fatalf("refused verdict called the hook: %+v", *calls) + } +} + +func TestSetReviewVerdictHookNilClears(t *testing.T) { + c := &Client{logger: testLogger()} + calls := recordingVerdictHook(c) + c.SetReviewVerdictHook(nil) + c.notifyReviewVerdict("o/r", 1) + if len(*calls) != 0 { + t.Fatalf("cleared hook still called: %+v", *calls) + } + var nilClient *Client + nilClient.SetReviewVerdictHook(func(string, int) {}) + nilClient.notifyReviewVerdict("o/r", 1) +} diff --git a/src/pkg/github/review_verdict_relay.go b/src/pkg/github/review_verdict_relay.go index b186ce7dbf..f04c724e96 100644 --- a/src/pkg/github/review_verdict_relay.go +++ b/src/pkg/github/review_verdict_relay.go @@ -92,6 +92,7 @@ func (c *Client) recordReviewVerdict(ctx context.Context, req ReviewRequest, dir prSnapshot := c.reviewedPRSnapshotFunc(ctx, req) prHead := func() string { return prSnapshot().Head } var recorded []review.PerspectiveReport + wake := false for _, report := range reports { if decisionRequested { report.DecisionRequested = true @@ -100,10 +101,18 @@ func (c *Client) recordReviewVerdict(ctx context.Context, req ReviewRequest, dir report.Verdict = review.VerdictRequiresHuman } report.PermissionsTighteningOnly = tightening - if written, ok := c.writeOneVerdict(ctx, req, report, dir, prHead, prSnapshot); ok { + if written, ok, dispatched := c.writeOneVerdict(ctx, req, report, dir, prHead, prSnapshot); ok { recorded = append(recorded, written) + wake = wake || dispatched } } + // One wake per request, however many perspectives a combined review + // answered: the head that wrote them is free, and the PR may be ready + // for its next head or its decision kick. An advisory verdict freed no + // dispatch slot, so it wakes nothing (hivecommons/hive#14705). + if wake { + c.notifyReviewVerdict(req.Repo, req.Number) + } return recorded } @@ -339,8 +348,9 @@ func maintainerCallQuestion(body string) string { // will find it. One file per perspective, which is what the collector has // always read β€” a combined review changes how verdicts arrive, not how they // are stored, so nothing downstream needs to know the difference. It returns -// the report as stored and whether it was. -func (c *Client) writeOneVerdict(ctx context.Context, req ReviewRequest, report review.PerspectiveReport, dir string, prHead func() string, prSnapshot func() reviewedPRSnapshot) (review.PerspectiveReport, bool) { +// the report as stored, whether it was, and whether it was bound to a review +// dispatch rather than admitted as advisory. +func (c *Client) writeOneVerdict(ctx context.Context, req ReviewRequest, report review.PerspectiveReport, dir string, prHead func() string, prSnapshot func() reviewedPRSnapshot) (review.PerspectiveReport, bool, bool) { // The verdict must be about the PR that was actually reviewed. Without this // an agent authorized to comment on one PR could record a binding // requires_human/reject verdict against any other PR in the fleet. @@ -349,7 +359,7 @@ func (c *Client) writeOneVerdict(ctx context.Context, req ReviewRequest, report slog.String("reviewed", fmt.Sprintf("%s#%d", req.Repo, req.Number)), slog.String("claimed", fmt.Sprintf("%s#%d", report.Repo, report.Number)), slog.String("agent", req.Agent)) - return report, false + return report, false, false } // Bind to a real dispatch record. A combined review answers one kick with // several verdicts, so each element is checked on its own: the dispatch is @@ -362,7 +372,7 @@ func (c *Client) writeOneVerdict(ctx context.Context, req ReviewRequest, report slog.String("perspective", string(report.Perspective)), slog.String("agent", req.Agent), slog.String("reason", auth.Reason)) - return report, false + return report, false, false } // A dispatched verdict is stored under the head its dispatch record names, // whatever the reviewer wrote: the match above already proved the two @@ -429,7 +439,7 @@ func (c *Client) writeOneVerdict(ctx context.Context, req ReviewRequest, report if err != nil { c.logger.Warn("review-request watcher: could not serialize verdict", slog.String("perspective", string(report.Perspective)), slog.String("error", err.Error())) - return report, false + return report, false, false } // One file per perspective, and per review head when the perspective was @@ -454,20 +464,20 @@ func (c *Client) writeOneVerdict(ctx context.Context, req ReviewRequest, report if err := os.WriteFile(tmp, raw, 0o644); err != nil { c.logger.Warn("review-request watcher: could not write verdict", slog.String("path", path), slog.String("error", err.Error())) - return report, false + return report, false, false } if err := os.Rename(tmp, path); err != nil { _ = os.Remove(tmp) c.logger.Warn("review-request watcher: could not commit verdict", slog.String("path", path), slog.String("error", err.Error())) - return report, false + return report, false, false } c.logger.Info("review-request watcher: verdict recorded", slog.String("repo", report.Repo), slog.Int("number", report.Number), slog.String("perspective", string(report.Perspective)), slog.String("verdict", string(report.Verdict)), slog.String("path", path)) - return report, true + return report, true, auth.Dispatched } // verdictAuthorization is what admitting a verdict establishes about it: From 0b92a781f74cb6244c882a80e50ed68acc5c33c6 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:04:15 -0400 Subject: [PATCH 02/38] =?UTF-8?q?=F0=9F=90=9B=20fix(forward-merge):=20trea?= =?UTF-8?q?t=20the=20squash-refusal=20guard=20as=20expected,=20not=20a=20r?= =?UTF-8?q?untime=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-14727.md | 1 + src/pkg/fleetreport/fleetreport.go | 5 +++++ src/pkg/fleetreport/runtime_test.go | 9 +++++++++ 3 files changed, 15 insertions(+) create mode 100644 changelog.d/fixed-14727.md diff --git a/changelog.d/fixed-14727.md b/changelog.d/fixed-14727.md new file mode 100644 index 0000000000..47b3ad3b97 --- /dev/null +++ b/changelog.d/fixed-14727.md @@ -0,0 +1 @@ +Forward-merge PRs that are forced to a merge commit instead of a squash no longer raise a repeated unclassified runtime-error self-report ([#14727](https://github.com/hivecommons/hive/issues/14727)). diff --git a/src/pkg/fleetreport/fleetreport.go b/src/pkg/fleetreport/fleetreport.go index 5f42ef8839..f039e1dbe6 100644 --- a/src/pkg/fleetreport/fleetreport.go +++ b/src/pkg/fleetreport/fleetreport.go @@ -767,6 +767,11 @@ var benignSubstrings = []string{ // output, not a repeated runtime defect (hivecommons/hive#14377). The // enforce-mode denial (effects.ErrDenied) stays reportable. "shadow fence would have denied external mutation", + // The forward-merge guard (src/pkg/github/pullrequest.go) warns each time + // it overrides a squash request on a forward-merge PR with a merge commit. + // That is the merge-safety policy working as designed, re-logged per + // merge attempt, not a repeated runtime defect (hivecommons/hive#14727). + "forward-merge guard: refusing to squash", } // IsBenignMessage reports whether a raw log message is noise the generic diff --git a/src/pkg/fleetreport/runtime_test.go b/src/pkg/fleetreport/runtime_test.go index b2100d14ce..37255fb418 100644 --- a/src/pkg/fleetreport/runtime_test.go +++ b/src/pkg/fleetreport/runtime_test.go @@ -797,6 +797,15 @@ func TestIsBenignMessagePolicySeedDivergence(t *testing.T) { } } +func TestIsBenignMessageForwardMergeGuard(t *testing.T) { + if !IsBenignMessage("forward-merge guard: refusing to squash, forcing merge commit") { + t.Error("forward-merge squash refusal not benign") + } + if IsBenignMessage("forward-merge guard: head-history check failed") { + t.Error("head-history check failure wrongly benign") + } +} + func TestIsBenignMessageCopilotModelFallback(t *testing.T) { const msg = "copilot CLI substituted the configured model" if !IsBenignMessage(msg) { From f0f399c2bcca1f35aa741d057df14b8ce2f566bd Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:05:00 +0000 Subject: [PATCH 03/38] changelog: fragment for #14736 Generated by changelog-fragment-guard from the PR title (#5675). Edit the wording freely, or replace this file with the no-changelog label if the change is not user-visible. Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-14736.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/fixed-14736.md diff --git a/changelog.d/fixed-14736.md b/changelog.d/fixed-14736.md new file mode 100644 index 0000000000..a4bd03bfaf --- /dev/null +++ b/changelog.d/fixed-14736.md @@ -0,0 +1 @@ +- fix(forward-merge): treat the squash-refusal guard as expected, not a runtime error (#14736) From 648ca07ca7dd70bef5e99f6eb40be194b74a7b22 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:14:28 +0000 Subject: [PATCH 04/38] =?UTF-8?q?=F0=9F=90=9B=20fix(deps):=20bump=20golang?= =?UTF-8?q?.org/x/net=20to=20v0.62.0=20to=20clear=20GO-2026-6860=20(#14731?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(deps): bump golang.org/x/net to v0.62.0 for GO-2026-6860 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * fix(deps): regenerate NOTICE for golang.org/x/net v0.62.0 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- NOTICE | 4 ++-- changelog.d/security-govulncheck-xnet.md | 1 + src/go.mod | 4 ++-- src/go.sum | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 changelog.d/security-govulncheck-xnet.md diff --git a/NOTICE b/NOTICE index 175ad41d70..dd6097af1a 100644 --- a/NOTICE +++ b/NOTICE @@ -4820,9 +4820,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ----------------------------------------------------------------------------- Package: golang.org/x/net -Version: v0.61.0 +Version: v0.62.0 License: BSD-3-Clause -Source: https://cs.opensource.google/go/x/net/+/v0.61.0:LICENSE +Source: https://cs.opensource.google/go/x/net/+/v0.62.0:LICENSE Copyright 2009 The Go Authors. diff --git a/changelog.d/security-govulncheck-xnet.md b/changelog.d/security-govulncheck-xnet.md new file mode 100644 index 0000000000..45b5a2ce36 --- /dev/null +++ b/changelog.d/security-govulncheck-xnet.md @@ -0,0 +1 @@ +- Bump golang.org/x/net to v0.62.0 to clear GO-2026-6860 (govulncheck). diff --git a/src/go.mod b/src/go.mod index d91689a7d1..a787500400 100644 --- a/src/go.mod +++ b/src/go.mod @@ -5,6 +5,7 @@ go 1.26.10 require ( github.com/charmbracelet/bubbletea v1.3.10 github.com/charmbracelet/lipgloss v1.1.0 + github.com/charmbracelet/x/ansi v0.10.1 github.com/charmbracelet/x/exp/teatest v0.0.0-20260823001701-96af6d2cb5f6 github.com/charmbracelet/x/term v0.2.2 github.com/fsnotify/fsnotify v1.10.1 @@ -34,7 +35,6 @@ require ( github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymanbagabas/go-udiff v0.3.1 // indirect github.com/charmbracelet/colorprofile v0.3.2 // indirect - github.com/charmbracelet/x/ansi v0.10.1 // indirect github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect @@ -67,7 +67,7 @@ require ( go.uber.org/automaxprocs v1.6.0 go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 // indirect - golang.org/x/net v0.61.0 + golang.org/x/net v0.62.0 golang.org/x/sys v0.49.0 golang.org/x/text v0.43.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect diff --git a/src/go.sum b/src/go.sum index ac522ab39e..7214b2e5f6 100644 --- a/src/go.sum +++ b/src/go.sum @@ -135,8 +135,8 @@ go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= -golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw= -golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From 0c1061d9a9c73c7973fe2de9a22b3083fd694182 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:15:10 +0000 Subject: [PATCH 05/38] =?UTF-8?q?=E2=9C=A8=20feature(crux):=20export=20OTe?= =?UTF-8?q?l=20spans=20and=20kick=20metrics=20over=20OTLP=20when=20configu?= =?UTF-8?q?red=20(#14618)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ feature(crux): export OTel spans and kick metrics over OTLP when configured The Crux runtime opened spans through the OpenTelemetry API, but the crux binary never registered an SDK provider, so every span was discarded and there were no metrics at all (hivecommons/crux#19). - crux/telemetry: install an OTLP/HTTP TracerProvider and MeterProvider at startup only when OTEL_EXPORTER_OTLP_ENDPOINT (or a per-signal endpoint) is set; honour OTEL_SDK_DISABLED, OTEL_*_EXPORTER=none and the resource env vars. Unset keeps the no-op providers. A non-http/protobuf protocol disables export with a warning instead of failing the command. - crux/agent: crux.agent.kick.duration (histogram, s) and crux.agent.kick.outcomes (counter), labelled only by agent.role (a configured agent name, else "unknown") and outcome (ok|cancelled|error). - Both crux mains call telemetry.Init and flush on exit (5s bound). - No /metrics endpoint and no Prometheus dependency. - Document the env vars and instruments; changelog fragment. Hive-Run: hivecommons/crux#19 Hive-Plan: maintainer decision on hivecommons/crux#19 (OTLP env-gated providers, kick metrics, no /metrics) Hive-Spec: hivecommons/crux#19#recommendation Signed-off-by: Karibhou * πŸ› fix(crux): register OTel globals only after both pipelines build Also test the OTLP export path against a fake collector and document the bounded five-second exit flush when the collector is unreachable. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HRH6kixWVJF9x9b1JF77JQ Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * πŸ› fix(crux): bind attribute sets before calling pointer method Equivalent in kick metrics test Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * test(crux/telemetry): cover the resource and exporter error branches The coverage gate holds crux/telemetry at 90%; the error paths Init takes when the resource or an OTLP exporter cannot be built were untested. Each is reached through the standard environment alone: an OTEL_RESOURCE_ATTRIBUTES pair with no value, and a CA certificate configured against an insecure endpoint for either signal. The metric-side case also checks that the already-built trace provider is torn down and no global is replaced. Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: Karibhou Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Karibhou Co-authored-by: Claude Opus 5.5 Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- changelog.d/added-crux-19-otel-export.md | 1 + src/cmd/crux/main.go | 10 + src/crux/README.md | 11 ++ src/crux/agent/kick_metrics.go | 82 ++++++++ src/crux/agent/kick_metrics_test.go | 82 ++++++++ src/crux/agent/manager_kick.go | 8 +- src/crux/cmd/crux/main.go | 10 + src/crux/go.mod | 14 +- src/crux/go.sum | 32 ++++ src/crux/telemetry/telemetry.go | 169 +++++++++++++++++ src/crux/telemetry/telemetry_test.go | 231 +++++++++++++++++++++++ src/docs/crux-cli.md | 40 ++++ 12 files changed, 688 insertions(+), 2 deletions(-) create mode 100644 changelog.d/added-crux-19-otel-export.md create mode 100644 src/crux/agent/kick_metrics.go create mode 100644 src/crux/agent/kick_metrics_test.go create mode 100644 src/crux/telemetry/telemetry.go create mode 100644 src/crux/telemetry/telemetry_test.go diff --git a/changelog.d/added-crux-19-otel-export.md b/changelog.d/added-crux-19-otel-export.md new file mode 100644 index 0000000000..3533583234 --- /dev/null +++ b/changelog.d/added-crux-19-otel-export.md @@ -0,0 +1 @@ +- The `crux` binary now exports its OpenTelemetry spans and new kick duration/outcome metrics (`crux.agent.kick.duration`, `crux.agent.kick.outcomes`, labelled only by agent role and outcome) over OTLP/HTTP when the standard `OTEL_EXPORTER_OTLP_*` environment names an endpoint; with none set nothing changes ([hivecommons/crux#19](https://github.com/hivecommons/crux/issues/19)). diff --git a/src/cmd/crux/main.go b/src/cmd/crux/main.go index e9f9b961a4..cefe728c41 100644 --- a/src/cmd/crux/main.go +++ b/src/cmd/crux/main.go @@ -13,10 +13,12 @@ package main import ( "context" + "fmt" "io" "os" "github.com/hivecommons/hive/crux/cli" + "github.com/hivecommons/hive/crux/telemetry" ) // version is what --version reports; set with -ldflags "-X main.version=…". @@ -29,6 +31,14 @@ func main() { func run(args []string, stdin io.Reader, stdout, stderr io.Writer, ctxFn func() (context.Context, context.CancelFunc)) int { ctx, stop := ctxFn() defer stop() + // OTLP export when OTEL_EXPORTER_OTLP_* names an endpoint; a no-op + // otherwise (hivecommons/crux#19). A bad configuration costs telemetry, + // never the command. + flush, err := telemetry.Init(ctx, version) + if err != nil { + fmt.Fprintln(stderr, "crux: telemetry disabled:", err) + } + defer flush() env := &cli.Env{Stdin: stdin, Stdout: stdout, Stderr: stderr, Loader: hiveLoader{}, Version: version} return env.Run(ctx, args) } diff --git a/src/crux/README.md b/src/crux/README.md index ff1b7d2ae5..c3d6db520c 100644 --- a/src/crux/README.md +++ b/src/crux/README.md @@ -118,6 +118,17 @@ crux logs -f scout # follows over SSE `crux --version` prints the version embedded at build time (`-ldflags "-X main.version=…"`, `dev` otherwise). +## Telemetry + +The runtime records OpenTelemetry spans (`agent.send_kick`, …) and two kick +metrics through the OTel API. The `crux` binary exports them over OTLP/HTTP +only when the standard `OTEL_EXPORTER_OTLP_ENDPOINT` (or a per-signal +`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` / `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT`) +is set; with none set nothing is installed and nothing is sent. There is no +`/metrics` endpoint: point crux at an OpenTelemetry Collector and export from +there. Variables and instruments are listed in +[`crux-cli.md`](../docs/crux-cli.md#telemetry). + ## Using Crux from Go ```go diff --git a/src/crux/agent/kick_metrics.go b/src/crux/agent/kick_metrics.go new file mode 100644 index 0000000000..6f9903232e --- /dev/null +++ b/src/crux/agent/kick_metrics.go @@ -0,0 +1,82 @@ +package agent + +import ( + "context" + "errors" + "sync" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +// Kick instruments (hivecommons/crux#19). They are created on the global +// MeterProvider, which is OTel's delegating no-op until a host registers an +// SDK provider (crux/telemetry for the crux binary, pkg/tracing for hive), so +// recording costs nothing by default and starts exporting once one is set. +// +// Attributes are bounded enums only: the agent role (a configured agent name, +// or kickRoleUnknown for a name the manager does not know, so caller input +// never becomes a label) and the outcome. Never issue/PR numbers, run ids or +// free text. +const ( + kickDurationMetric = "crux.agent.kick.duration" + kickOutcomeMetric = "crux.agent.kick.outcomes" + + kickRoleUnknown = "unknown" + + kickOutcomeOK = "ok" + kickOutcomeCancelled = "cancelled" + kickOutcomeError = "error" +) + +var ( + attrKickRole = attribute.Key("agent.role") + attrKickOutcome = attribute.Key("outcome") +) + +// kickMetrics holds the two kick instruments. +type kickMetrics struct { + duration metric.Float64Histogram + outcomes metric.Int64Counter +} + +func newKickMetrics(meter metric.Meter) kickMetrics { + // Instrument constructors only fail on an invalid name or unit; both are + // constants here, and the returned instrument is a usable no-op either way. + duration, _ := meter.Float64Histogram(kickDurationMetric, + metric.WithUnit("s"), + metric.WithDescription("Time SendKick took to deliver, defer or reject a kick.")) + outcomes, _ := meter.Int64Counter(kickOutcomeMetric, + metric.WithDescription("Kicks handled by SendKick, by agent role and outcome.")) + return kickMetrics{duration: duration, outcomes: outcomes} +} + +var globalKickMetrics = sync.OnceValue(func() kickMetrics { + return newKickMetrics(otel.Meter(spanInstrumentationName)) +}) + +// kickOutcome maps SendKick's result onto the fixed outcome enum. +func kickOutcome(err error) string { + switch { + case err == nil: + return kickOutcomeOK + case errors.Is(err, errKickCancelledByRestart): + return kickOutcomeCancelled + default: + return kickOutcomeError + } +} + +// recordKick records one SendKick call's duration and outcome. +func recordKick(role string, start time.Time, err error) { + globalKickMetrics().record(role, time.Since(start), err) +} + +func (k kickMetrics) record(role string, d time.Duration, err error) { + attrs := metric.WithAttributes(attrKickRole.String(role), attrKickOutcome.String(kickOutcome(err))) + ctx := context.Background() + k.duration.Record(ctx, d.Seconds(), attrs) + k.outcomes.Add(ctx, 1, attrs) +} diff --git a/src/crux/agent/kick_metrics_test.go b/src/crux/agent/kick_metrics_test.go new file mode 100644 index 0000000000..f1e8ad8228 --- /dev/null +++ b/src/crux/agent/kick_metrics_test.go @@ -0,0 +1,82 @@ +package agent + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "go.opentelemetry.io/otel/attribute" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +func TestKickOutcome(t *testing.T) { + cases := []struct { + err error + want string + }{ + {nil, kickOutcomeOK}, + {fmt.Errorf("%w: agent a restarted mid-send", errKickCancelledByRestart), kickOutcomeCancelled}, + {errors.New("agent a not found"), kickOutcomeError}, + } + for _, tc := range cases { + if got := kickOutcome(tc.err); got != tc.want { + t.Errorf("kickOutcome(%v) = %q, want %q", tc.err, got, tc.want) + } + } +} + +// The instruments export one duration point and one outcome count per kick, +// labelled by exactly agent.role and outcome β€” no other attribute may leak in. +func TestKickMetricsRecordBoundedAttributes(t *testing.T) { + reader := sdkmetric.NewManualReader() + mp := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + t.Cleanup(func() { _ = mp.Shutdown(context.Background()) }) + k := newKickMetrics(mp.Meter(spanInstrumentationName)) + + k.record("scanner", 2*time.Second, nil) + k.record(kickRoleUnknown, time.Millisecond, errors.New("agent x not found")) + + var rm metricdata.ResourceMetrics + if err := reader.Collect(context.Background(), &rm); err != nil { + t.Fatalf("Collect: %v", err) + } + okSet := attribute.NewSet(attrKickRole.String("scanner"), attrKickOutcome.String(kickOutcomeOK)) + errSet := attribute.NewSet(attrKickRole.String(kickRoleUnknown), attrKickOutcome.String(kickOutcomeError)) + want := map[attribute.Distinct]bool{ + okSet.Equivalent(): true, + errSet.Equivalent(): true, + } + seen := map[string]bool{} + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + seen[m.Name] = true + switch data := m.Data.(type) { + case metricdata.Histogram[float64]: + if m.Unit != "s" { + t.Errorf("%s unit = %q, want s", m.Name, m.Unit) + } + for _, dp := range data.DataPoints { + if !want[dp.Attributes.Equivalent()] || dp.Count != 1 { + t.Errorf("%s point %v count=%d unexpected", m.Name, dp.Attributes.ToSlice(), dp.Count) + } + } + case metricdata.Sum[int64]: + for _, dp := range data.DataPoints { + if !want[dp.Attributes.Equivalent()] || dp.Value != 1 { + t.Errorf("%s point %v value=%d unexpected", m.Name, dp.Attributes.ToSlice(), dp.Value) + } + } + default: + t.Errorf("%s: unexpected data type %T", m.Name, m.Data) + } + } + } + for _, name := range []string{kickDurationMetric, kickOutcomeMetric} { + if !seen[name] { + t.Errorf("metric %s not exported", name) + } + } +} diff --git a/src/crux/agent/manager_kick.go b/src/crux/agent/manager_kick.go index 553fa1a02f..d6c6619672 100644 --- a/src/crux/agent/manager_kick.go +++ b/src/crux/agent/manager_kick.go @@ -100,7 +100,7 @@ func notRunningReason(agent *AgentProcess) string { } } -func (m *Manager) SendKick(name string, message string) error { +func (m *Manager) SendKick(name string, message string) (err error) { // Agent-kick span. No-op with zero export cost when tracing is disabled // (the default). SendKick has no context parameter, so this span roots at // Background; it still captures the kick leg of the governorβ†’agent @@ -109,6 +109,11 @@ func (m *Manager) SendKick(name string, message string) error { attribute.String("agent.name", name)) defer span.End() + // Kick duration/outcome metrics; role stays kickRoleUnknown until the + // name resolves to a configured agent (kick_metrics.go). + start, role := time.Now(), kickRoleUnknown + defer func() { recordKick(role, start, err) }() + m.mu.Lock() defer m.mu.Unlock() @@ -116,6 +121,7 @@ func (m *Manager) SendKick(name string, message string) error { if !ok { return fmt.Errorf("agent %s not found", name) } + role = name if m.agentSandboxEnabledLocked(agent) { return m.startSandboxKickLocked(agent, message) diff --git a/src/crux/cmd/crux/main.go b/src/crux/cmd/crux/main.go index 2d58ab1c46..e6901a6a83 100644 --- a/src/crux/cmd/crux/main.go +++ b/src/crux/cmd/crux/main.go @@ -16,11 +16,13 @@ package main import ( "context" + "fmt" "io" "os" "github.com/hivecommons/hive/crux/cli" "github.com/hivecommons/hive/crux/defs" + "github.com/hivecommons/hive/crux/telemetry" ) // version is what --version reports. Release builds set it with @@ -34,6 +36,14 @@ func main() { func run(args []string, stdin io.Reader, stdout, stderr io.Writer, ctxFn func() (context.Context, context.CancelFunc)) int { ctx, stop := ctxFn() defer stop() + // OTLP export when OTEL_EXPORTER_OTLP_* names an endpoint; a no-op + // otherwise (hivecommons/crux#19). A bad configuration costs telemetry, + // never the command. + flush, err := telemetry.Init(ctx, version) + if err != nil { + fmt.Fprintln(stderr, "crux: telemetry disabled:", err) + } + defer flush() env := &cli.Env{Stdin: stdin, Stdout: stdout, Stderr: stderr, Loader: defs.Loader{}, Version: version} return env.Run(ctx, args) } diff --git a/src/crux/go.mod b/src/crux/go.mod index 4de541c8ef..5685f185cf 100644 --- a/src/crux/go.mod +++ b/src/crux/go.mod @@ -6,6 +6,11 @@ require ( github.com/google/cel-go v0.31.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e go.opentelemetry.io/otel v1.47.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 + go.opentelemetry.io/otel/metric v1.47.0 + go.opentelemetry.io/otel/sdk v1.47.0 + go.opentelemetry.io/otel/sdk/metric v1.47.0 go.opentelemetry.io/otel/trace v1.47.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -18,15 +23,22 @@ require ( require ( cel.dev/expr v0.25.2 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/otel/metric v1.47.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 // indirect + golang.org/x/net v0.59.0 // indirect + golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.43.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect + google.golang.org/grpc v1.83.2 // indirect google.golang.org/protobuf v1.36.12 // indirect ) diff --git a/src/crux/go.sum b/src/crux/go.sum index f5b7c49322..1c26caea7e 100644 --- a/src/crux/go.sum +++ b/src/crux/go.sum @@ -2,6 +2,8 @@ cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -10,10 +12,16 @@ github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/cel-go v0.31.0 h1:H0bhpFTqOvmHrBGrWKp7ZlhBm5Hh8PYUEXnwxT1LL7A= github.com/google/cel-go v0.31.0/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -28,22 +36,46 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM= go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0 h1:7kszRN4Ze5GeAkUnBjBSUIunVBMA+mZIG04auueqguM= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0/go.mod h1:CLzakYy8jOdkBfcveCWYkeKJxujcF4xPhh2Z7IfE98U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 h1:julhjPeUH/q/7hinbSdDdqt5h7Zw9YWmRlWRhI0jd54= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0/go.mod h1:Ao2mz688LH/tFf0yMAenidq6k2YNSx6SIY2q6jDACck= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 h1:aXZXsZ012wOgVkqaiQv+Z/d8V0xKjmg70F6m4CE94lc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0/go.mod h1:ziu1gUIJhAaxM3EV1e3Ralk5AAyL1UOYz3lJJ4VSI38= go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI= go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw= go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY= go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU= +go.opentelemetry.io/otel/metric/x v0.69.0 h1:DjRLr15H83v+hCW7JA9NoJvOkYTtmq5YoDRbe9deYpM= +go.opentelemetry.io/otel/metric/x v0.69.0/go.mod h1:uVvsMPMFFyj/HUQfrUnH3JjnOQ1dwFDorgFLRBasM0k= +go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I= +go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0= +go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw= +go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw= go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM= go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= +golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= +golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc h1:rZHRz0ogq4Pid1IKtA/ivHi8XONVF5tqUQXA2TtgpkE= google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:K/n0XLazCJHjD73YuMyNJsN38BPPy1kfFnKxUXaThFU= google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc h1:4bNTbnb44EqGVy9HaQxSY2jnafSUdgV3qHtedvNpDKg= google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/src/crux/telemetry/telemetry.go b/src/crux/telemetry/telemetry.go new file mode 100644 index 0000000000..019991bade --- /dev/null +++ b/src/crux/telemetry/telemetry.go @@ -0,0 +1,169 @@ +// Package telemetry installs the OpenTelemetry SDK for the crux binary +// (hivecommons/crux#19). The runtime records spans and kick metrics through +// the OTel API alone; without a registered provider those are no-ops. Init +// registers an OTLP/HTTP TracerProvider and MeterProvider, but only when the +// standard OTEL_EXPORTER_OTLP_* environment names an endpoint. With none set +// it changes nothing: the global providers stay the zero-cost no-ops. +// +// Everything about the export β€” endpoint, headers, TLS, timeouts, service +// name, resource attributes β€” comes from the standard OTel environment +// variables, read by the SDK and exporters themselves; no collector address +// lives in code. Hive's own process does not use this package: it wires +// pkg/tracing from hive.yaml. +package telemetry + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "sync" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/propagation" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +// Standard OpenTelemetry environment variables consulted to decide whether to +// export at all. The exporters read the full OTEL_EXPORTER_OTLP_* family +// (headers, TLS, timeouts, compression) on their own. +const ( + envEndpoint = "OTEL_EXPORTER_OTLP_ENDPOINT" + envTracesEndpoint = "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT" + envMetricsEndpoint = "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT" + envProtocol = "OTEL_EXPORTER_OTLP_PROTOCOL" + envTracesProtocol = "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL" + envMetricsProtocol = "OTEL_EXPORTER_OTLP_METRICS_PROTOCOL" + envTracesExporter = "OTEL_TRACES_EXPORTER" + envMetricsExporter = "OTEL_METRICS_EXPORTER" + envSDKDisabled = "OTEL_SDK_DISABLED" +) + +// defaultServiceName is the resource service.name when OTEL_SERVICE_NAME and +// OTEL_RESOURCE_ATTRIBUTES do not name one. +const defaultServiceName = "crux" + +// shutdownTimeout bounds the final flush so a dead collector cannot hold the +// process open on exit. +const shutdownTimeout = 5 * time.Second + +// Shutdown flushes and stops whatever Init installed. It is safe to call more +// than once and is a no-op when Init installed nothing. +type Shutdown func() + +func noopShutdown() {} + +// Init registers the OTLP trace and metric pipelines named by the environment +// and returns the function that flushes them. version becomes service.version +// on the resource. On error nothing is registered, the returned Shutdown is a +// no-op, and the caller keeps running with no-op providers. +func Init(ctx context.Context, version string) (Shutdown, error) { + return initFromEnv(ctx, version, os.Getenv) +} + +func initFromEnv(ctx context.Context, version string, getenv func(string) string) (Shutdown, error) { + traces, metrics := signalsEnabled(getenv) + if !traces && !metrics { + return noopShutdown, nil + } + for _, key := range []string{envProtocol, envTracesProtocol, envMetricsProtocol} { + // Only OTLP over HTTP is linked into crux, keeping gRPC out of the + // module's dependency set; refuse rather than send protobuf-over-HTTP + // to a gRPC port. + if p := strings.TrimSpace(getenv(key)); p != "" && p != "http/protobuf" { + return noopShutdown, fmt.Errorf("%s=%q is not supported (crux exports OTLP over http/protobuf only)", key, p) + } + } + + res, err := newResource(ctx, version) + if err != nil { + return noopShutdown, err + } + + // Build both pipelines before registering either global, so a failure + // on the second leaves no provider (and no shut-down provider) behind. + var ( + tp *sdktrace.TracerProvider + mp *sdkmetric.MeterProvider + stops []func(context.Context) error + ) + if traces { + exp, err := otlptracehttp.New(ctx) + if err != nil { + return noopShutdown, fmt.Errorf("trace exporter: %w", err) + } + tp = sdktrace.NewTracerProvider(sdktrace.WithBatcher(exp), sdktrace.WithResource(res)) + stops = append(stops, tp.Shutdown) + } + if metrics { + exp, err := otlpmetrichttp.New(ctx) + if err != nil { + for _, stop := range stops { + _ = stop(ctx) + } + return noopShutdown, fmt.Errorf("metric exporter: %w", err) + } + mp = sdkmetric.NewMeterProvider(sdkmetric.WithReader(sdkmetric.NewPeriodicReader(exp)), sdkmetric.WithResource(res)) + stops = append(stops, mp.Shutdown) + } + if tp != nil { + otel.SetTracerProvider(tp) + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})) + } + if mp != nil { + otel.SetMeterProvider(mp) + } + + var once sync.Once + return func() { + once.Do(func() { + sctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + var errs []error + for _, stop := range stops { + errs = append(errs, stop(sctx)) + } + if err := errors.Join(errs...); err != nil { + otel.Handle(err) + } + }) + }, nil +} + +// signalsEnabled reports which signals the environment asks crux to export: +// a signal is on when the shared or its signal-specific endpoint is set, the +// SDK is not disabled, and its OTEL_*_EXPORTER is not "none". +func signalsEnabled(getenv func(string) string) (traces, metrics bool) { + if strings.EqualFold(strings.TrimSpace(getenv(envSDKDisabled)), "true") { + return false, false + } + shared := strings.TrimSpace(getenv(envEndpoint)) != "" + traces = (shared || strings.TrimSpace(getenv(envTracesEndpoint)) != "") && !exporterNone(getenv(envTracesExporter)) + metrics = (shared || strings.TrimSpace(getenv(envMetricsEndpoint)) != "") && !exporterNone(getenv(envMetricsExporter)) + return traces, metrics +} + +func exporterNone(v string) bool { + return strings.EqualFold(strings.TrimSpace(v), "none") +} + +// newResource identifies this process. The crux defaults come first so +// OTEL_SERVICE_NAME and OTEL_RESOURCE_ATTRIBUTES, applied after, override them. +func newResource(ctx context.Context, version string) (*resource.Resource, error) { + attrs := []attribute.KeyValue{attribute.String("service.name", defaultServiceName)} + if v := strings.TrimSpace(version); v != "" { + attrs = append(attrs, attribute.String("service.version", v)) + } + return resource.New(ctx, + resource.WithAttributes(attrs...), + resource.WithTelemetrySDK(), + resource.WithFromEnv(), + ) +} diff --git a/src/crux/telemetry/telemetry_test.go b/src/crux/telemetry/telemetry_test.go new file mode 100644 index 0000000000..6788edc673 --- /dev/null +++ b/src/crux/telemetry/telemetry_test.go @@ -0,0 +1,231 @@ +package telemetry + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" +) + +func envOf(m map[string]string) func(string) string { + return func(k string) string { return m[k] } +} + +func TestSignalsEnabled(t *testing.T) { + cases := []struct { + name string + env map[string]string + traces, metrics bool + }{ + {"unset", nil, false, false}, + {"shared endpoint", map[string]string{envEndpoint: "http://collector:4318"}, true, true}, + {"traces only", map[string]string{envTracesEndpoint: "http://c/v1/traces"}, true, false}, + {"metrics only", map[string]string{envMetricsEndpoint: "http://c/v1/metrics"}, false, true}, + {"blank endpoint", map[string]string{envEndpoint: " "}, false, false}, + {"sdk disabled", map[string]string{envEndpoint: "http://c", envSDKDisabled: "TRUE"}, false, false}, + {"traces exporter none", map[string]string{envEndpoint: "http://c", envTracesExporter: "none"}, false, true}, + {"metrics exporter none", map[string]string{envEndpoint: "http://c", envMetricsExporter: "None"}, true, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + traces, metrics := signalsEnabled(envOf(tc.env)) + if traces != tc.traces || metrics != tc.metrics { + t.Fatalf("signalsEnabled = (%v, %v), want (%v, %v)", traces, metrics, tc.traces, tc.metrics) + } + }) + } +} + +// With no endpoint configured Init must leave the global providers alone: +// that is the zero-behaviour-change contract for current users. +func TestInitWithoutEndpointKeepsNoopProviders(t *testing.T) { + tp, mp := otel.GetTracerProvider(), otel.GetMeterProvider() + shutdown, err := initFromEnv(context.Background(), "v1", envOf(nil)) + if err != nil { + t.Fatalf("initFromEnv: %v", err) + } + shutdown() + shutdown() + if otel.GetTracerProvider() != tp || otel.GetMeterProvider() != mp { + t.Fatal("initFromEnv replaced a global provider without an endpoint") + } +} + +func TestInitRejectsGRPCProtocol(t *testing.T) { + tp := otel.GetTracerProvider() + _, err := initFromEnv(context.Background(), "v1", envOf(map[string]string{ + envEndpoint: "http://collector:4317", + envProtocol: "grpc", + })) + if err == nil || !strings.Contains(err.Error(), envProtocol) { + t.Fatalf("err = %v, want an unsupported-protocol error naming %s", err, envProtocol) + } + if otel.GetTracerProvider() != tp { + t.Fatal("a rejected configuration must not install a provider") + } +} + +func TestNewResourceDefaultsAndEnvOverride(t *testing.T) { + t.Setenv("OTEL_SERVICE_NAME", "") + t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "") + res, err := newResource(context.Background(), "1.2.3") + if err != nil { + t.Fatalf("newResource: %v", err) + } + if got, _ := res.Set().Value(attribute.Key("service.name")); got.AsString() != defaultServiceName { + t.Fatalf("service.name = %q, want %q", got.AsString(), defaultServiceName) + } + if got, _ := res.Set().Value(attribute.Key("service.version")); got.AsString() != "1.2.3" { + t.Fatalf("service.version = %q, want 1.2.3", got.AsString()) + } + + t.Setenv("OTEL_SERVICE_NAME", "crux-edge") + res, err = newResource(context.Background(), "") + if err != nil { + t.Fatalf("newResource: %v", err) + } + if got, _ := res.Set().Value(attribute.Key("service.name")); got.AsString() != "crux-edge" { + t.Fatalf("service.name = %q, want OTEL_SERVICE_NAME to override", got.AsString()) + } + if _, ok := res.Set().Value(attribute.Key("service.version")); ok { + t.Fatal("an empty version must not stamp service.version") + } +} + +// With an endpoint set, Init must actually export: one span and one counter +// recorded through the OTel API reach the collector as /v1/traces and +// /v1/metrics POSTs when Shutdown flushes. +func TestInitExportsToCollectorOnShutdown(t *testing.T) { + prevTP, prevMP, prevProp := otel.GetTracerProvider(), otel.GetMeterProvider(), otel.GetTextMapPropagator() + t.Cleanup(func() { + otel.SetTracerProvider(prevTP) + otel.SetMeterProvider(prevMP) + otel.SetTextMapPropagator(prevProp) + }) + + var ( + mu sync.Mutex + posts = map[string]int{} + ) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + mu.Lock() + posts[r.URL.Path]++ + mu.Unlock() + } + w.Header().Set("Content-Type", "application/x-protobuf") + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(srv.Close) + t.Setenv(envEndpoint, srv.URL) + + flush, err := Init(context.Background(), "test") + if err != nil { + t.Fatalf("Init: %v", err) + } + _, span := otel.Tracer("crux/telemetry/test").Start(context.Background(), "test.span") + span.End() + counter, err := otel.Meter("crux/telemetry/test").Int64Counter("test.counter") + if err != nil { + t.Fatalf("Int64Counter: %v", err) + } + counter.Add(context.Background(), 1) + flush() + + mu.Lock() + defer mu.Unlock() + if posts["/v1/traces"] == 0 || posts["/v1/metrics"] == 0 { + t.Fatalf("collector POSTs = %v, want /v1/traces and /v1/metrics", posts) + } +} + +// caCertPEM writes a self-signed certificate the exporters' CERTIFICATE env +// variables can load, so a TLS root can be configured against an http:// +// endpoint β€” the one exporter construction error reachable through env. +func caCertPEM(t *testing.T) string { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("GenerateKey: %v", err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "crux-telemetry-test"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + KeyUsage: x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatalf("CreateCertificate: %v", err) + } + path := filepath.Join(t.TempDir(), "ca.pem") + if err := os.WriteFile(path, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600); err != nil { + t.Fatalf("WriteFile: %v", err) + } + return path +} + +func TestInitResourceErrorInstallsNothing(t *testing.T) { + tp, mp := otel.GetTracerProvider(), otel.GetMeterProvider() + t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment") // no '=': ErrPartialResource + shutdown, err := initFromEnv(context.Background(), "v1", envOf(map[string]string{envEndpoint: "http://collector:4318"})) + if err == nil { + t.Fatal("initFromEnv must surface an invalid OTEL_RESOURCE_ATTRIBUTES") + } + shutdown() + if otel.GetTracerProvider() != tp || otel.GetMeterProvider() != mp { + t.Fatal("a resource error must not install a provider") + } +} + +func TestInitTraceExporterErrorInstallsNothing(t *testing.T) { + tp, mp := otel.GetTracerProvider(), otel.GetMeterProvider() + t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "") + t.Setenv(envEndpoint, "http://collector:4318") + t.Setenv("OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE", caCertPEM(t)) + t.Setenv("OTEL_EXPORTER_OTLP_TRACES_INSECURE", "true") + shutdown, err := initFromEnv(context.Background(), "v1", envOf(map[string]string{envEndpoint: "http://collector:4318"})) + if err == nil || !strings.Contains(err.Error(), "trace exporter") { + t.Fatalf("err = %v, want a trace exporter error", err) + } + shutdown() + if otel.GetTracerProvider() != tp || otel.GetMeterProvider() != mp { + t.Fatal("a trace exporter error must not install a provider") + } +} + +// When the metric pipeline fails after the trace pipeline was built, the +// trace provider is shut down and neither global is replaced. +func TestInitMetricExporterErrorTearsDownTraces(t *testing.T) { + tp, mp := otel.GetTracerProvider(), otel.GetMeterProvider() + t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "") + t.Setenv(envEndpoint, "http://collector:4318") + t.Setenv("OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE", caCertPEM(t)) + t.Setenv("OTEL_EXPORTER_OTLP_METRICS_INSECURE", "true") + shutdown, err := initFromEnv(context.Background(), "v1", envOf(map[string]string{envEndpoint: "http://collector:4318"})) + if err == nil || !strings.Contains(err.Error(), "metric exporter") { + t.Fatalf("err = %v, want a metric exporter error", err) + } + shutdown() + if otel.GetTracerProvider() != tp || otel.GetMeterProvider() != mp { + t.Fatal("a metric exporter error must not leave the trace provider installed") + } +} diff --git a/src/docs/crux-cli.md b/src/docs/crux-cli.md index 08b4f805e0..2f466d71e2 100644 --- a/src/docs/crux-cli.md +++ b/src/docs/crux-cli.md @@ -140,6 +140,46 @@ file nobody answers on (a `run` that died without cleanup) is reported and treated as absent. `--detach` leaves an agent running in tmux with no supervisor; use it when something else will own the session. +## Telemetry + +The runtime opens OpenTelemetry spans (for example `agent.send_kick`) and +records two kick instruments through the OTel API +([hivecommons/crux#19](https://github.com/hivecommons/crux/issues/19)): + +| Instrument | Type | Attributes | +|---|---|---| +| `crux.agent.kick.duration` | histogram, seconds | `agent.role`, `outcome` | +| `crux.agent.kick.outcomes` | counter | `agent.role`, `outcome` | + +`agent.role` is the configured agent name (`unknown` when the kicked name is +not a configured agent); `outcome` is `ok`, `cancelled` (the agent restarted +mid-kick) or `error`. No issue or PR numbers, run ids or free text are ever +attributes. + +Export is off by default. Both `crux` builds install an OTLP/HTTP trace and +metric pipeline at startup only when the standard OpenTelemetry environment +names an endpoint, and leave the no-op providers in place otherwise: + +| Variable | Effect | +|---|---| +| `OTEL_EXPORTER_OTLP_ENDPOINT` | base collector URL for both signals (e.g. `http://collector:4318`) | +| `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` / `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` | enable and address one signal only | +| `OTEL_TRACES_EXPORTER=none` / `OTEL_METRICS_EXPORTER=none` | turn one signal off | +| `OTEL_SDK_DISABLED=true` | turn both off | +| `OTEL_EXPORTER_OTLP_HEADERS`, `…_CERTIFICATE`, `…_TIMEOUT`, `…_COMPRESSION` | read by the exporters as usual | +| `OTEL_SERVICE_NAME`, `OTEL_RESOURCE_ATTRIBUTES` | override the resource (`service.name` defaults to `crux`; `service.version` is the build version) | + +Only the `http/protobuf` protocol is built in: setting +`OTEL_EXPORTER_OTLP_PROTOCOL` (or a per-signal variant) to anything else +makes crux print `crux: telemetry disabled: …` and run without export rather +than fail the command. There is no `/metrics` endpoint and no Prometheus +dependency; scrape or forward from an OpenTelemetry Collector instead. Spans +and metrics are flushed on exit, bounded to five seconds: with an endpoint set +and the collector unreachable, each `crux` invocation can take up to that long +to exit after its work is done β€” that is the flush timing out, not a hang. +Unset the endpoint (or set `OTEL_SDK_DISABLED=true`) to avoid it. Inside Hive the +same instruments flow through Hive's own `tracing:` configuration instead. + ## What this is not The CLI reaches `*agent.Manager` only to construct it From 4d8bd731214d60250c9ff9f35d733b4d630f3c1a Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:30:26 +0000 Subject: [PATCH 06/38] =?UTF-8?q?=F0=9F=90=9B=20fix(proxy):=20canonicalize?= =?UTF-8?q?=20hostnames=20before=20host-class=20lookups=20(#14737)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * πŸ› fix(proxy): canonicalize hostnames before host-class lookups DNS names are case-insensitive and may end in a dot; normalize inside the host predicates. Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * docs: refresh rules.go citation shifted by host canonicalisation Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-proxy-host-canonical.md | 1 + src/crux/proxy/inference_route.go | 2 +- src/crux/proxy/linear_rules.go | 2 +- src/crux/proxy/rules.go | 11 ++++++++ src/crux/proxy/rules_test.go | 33 ++++++++++++++++++++++ src/docs/design/github-mention-triggers.md | 2 +- 6 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 changelog.d/fixed-proxy-host-canonical.md diff --git a/changelog.d/fixed-proxy-host-canonical.md b/changelog.d/fixed-proxy-host-canonical.md new file mode 100644 index 0000000000..518ceacc40 --- /dev/null +++ b/changelog.d/fixed-proxy-host-canonical.md @@ -0,0 +1 @@ +- The crux proxy now treats hostnames case-insensitively and ignores a trailing dot when classifying hosts for inspection. diff --git a/src/crux/proxy/inference_route.go b/src/crux/proxy/inference_route.go index 9a9effe597..4f35d29dec 100644 --- a/src/crux/proxy/inference_route.go +++ b/src/crux/proxy/inference_route.go @@ -207,7 +207,7 @@ func unregisterAnthropicHost(host string) { func IsAnthropicHost(host string) bool { anthropicHostsMu.RLock() defer anthropicHostsMu.RUnlock() - return anthropicHosts[host] + return anthropicHosts[canonicalHost(host)] } // InferenceBackends lists the valid self-hosted inference backend IDs. diff --git a/src/crux/proxy/linear_rules.go b/src/crux/proxy/linear_rules.go index 5546be8e33..21a1306a75 100644 --- a/src/crux/proxy/linear_rules.go +++ b/src/crux/proxy/linear_rules.go @@ -142,7 +142,7 @@ var linearMutations = map[string]linearRequirement{ } // IsLinearHost reports whether host is Linear's API host. -func IsLinearHost(host string) bool { return host == linearHost } +func IsLinearHost(host string) bool { return canonicalHost(host) == linearHost } // IsLinearGraphQLPath reports whether path is Linear's GraphQL endpoint. func IsLinearGraphQLPath(path string) bool { return path == linearGraphQLPath } diff --git a/src/crux/proxy/rules.go b/src/crux/proxy/rules.go index 2888fc9c8b..3e30067a94 100644 --- a/src/crux/proxy/rules.go +++ b/src/crux/proxy/rules.go @@ -31,6 +31,12 @@ type ProxyRule struct { Capability func(agent.AgentCapabilities) bool } +// canonicalHost normalizes a hostname for comparison: DNS names are +// case-insensitive and may carry a trailing dot. +func canonicalHost(host string) string { + return strings.TrimSuffix(strings.ToLower(strings.TrimSpace(host)), ".") +} + // githubHosts are the hostnames the proxy inspects. var githubHostsMu sync.RWMutex var githubHosts = map[string]bool{ @@ -43,6 +49,7 @@ var githubHosts = map[string]bool{ // register hosts while proxy goroutines are already serving, so the map is // mutex-guarded. func RegisterGitHubHost(host string) { + host = canonicalHost(host) if host == "" { return } @@ -53,6 +60,7 @@ func RegisterGitHubHost(host string) { // unregisterGitHubHost removes a hostname from the allowlist (test cleanup). func unregisterGitHubHost(host string) { + host = canonicalHost(host) githubHostsMu.Lock() delete(githubHosts, host) githubHostsMu.Unlock() @@ -60,6 +68,7 @@ func unregisterGitHubHost(host string) { // IsGitHubHost returns true if the host should be subject to mode enforcement. func IsGitHubHost(host string) bool { + host = canonicalHost(host) githubHostsMu.RLock() defer githubHostsMu.RUnlock() return githubHosts[host] @@ -78,6 +87,7 @@ func IsGitHubHost(host string) bool { // is why the IsGitHubHost checks at the CONNECT seams could no longer double as // the "should I inspect this?" test; see NeedsInspection. func NeedsMITM(host string) bool { + host = canonicalHost(host) return host == "api.github.com" || IsLinearHost(host) } @@ -112,6 +122,7 @@ const copilotAPIHostExact = "githubcopilot.com" // otherwise forwarded verbatim; no content is altered beyond an optional // stream_options.include_usage hint on streaming completion requests. func IsCopilotAPIHost(host string) bool { + host = canonicalHost(host) if host == "" { return false } diff --git a/src/crux/proxy/rules_test.go b/src/crux/proxy/rules_test.go index 17dd1d7faa..313c445e71 100644 --- a/src/crux/proxy/rules_test.go +++ b/src/crux/proxy/rules_test.go @@ -307,6 +307,10 @@ func TestIsGitHubHost(t *testing.T) { {"github.com", true}, {"example.com", false}, {"api.github.com.evil.com", false}, + {"API.GITHUB.COM", true}, + {"Api.GitHub.com.", true}, + {"github.com.", true}, + {"evil-api.github.com", false}, {"", false}, } for _, tt := range tests { @@ -332,3 +336,32 @@ func TestNeedsMITM(t *testing.T) { } } } + +func TestHostCanonicalization(t *testing.T) { + tests := []struct { + name string + fn func(string) bool + host string + want bool + }{ + {"inspection upper", NeedsInspection, "API.GITHUB.COM", true}, + {"inspection mixed trailing dot", NeedsInspection, "Api.GitHub.com.", true}, + {"inspection lookalike", NeedsInspection, "evil-api.github.com", false}, + {"inspection empty", NeedsInspection, "", false}, + {"mitm trailing dot", NeedsMITM, "api.github.com.", true}, + {"linear upper", IsLinearHost, "API.LINEAR.APP", true}, + {"linear trailing dot", IsLinearHost, "api.linear.app.", true}, + {"linear lookalike", IsLinearHost, "evil-api.linear.app", false}, + {"copilot upper", IsCopilotAPIHost, "API.GITHUBCOPILOT.COM", true}, + {"copilot mixed trailing dot", IsCopilotAPIHost, "Api.GitHubCopilot.com.", true}, + {"copilot lookalike", IsCopilotAPIHost, "evilgithubcopilot.com", false}, + {"copilot empty", IsCopilotAPIHost, "", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.fn(tt.host); got != tt.want { + t.Errorf("host %q = %v, want %v", tt.host, got, tt.want) + } + }) + } +} diff --git a/src/docs/design/github-mention-triggers.md b/src/docs/design/github-mention-triggers.md index 56a705a2da..2d90d46cfe 100644 --- a/src/docs/design/github-mention-triggers.md +++ b/src/docs/design/github-mention-triggers.md @@ -345,7 +345,7 @@ the decision follows each one, with the code on `v6` that now carries it. - `src/pkg/linearagent/responder.go` β€” ack-first, resolve, kick, track. - `src/pkg/linearagent/oauth.go:56` β€” `app:mentionable`. - `src/crux/agent/capabilities.go:32` β€” `Converse`, documented for mentions. -- `src/crux/proxy/rules.go:160,189` β€” where `Converse` is enforced. +- `src/crux/proxy/rules.go:171,200` β€” where `Converse` is enforced. - `src/pkg/config/config.go:1716` β€” `linear.session_agent` resolution rule. - `src/pkg/config/config.go:5538` β€” the removed declarative channel types (#5591). - `src/pkg/config/review_bots.go:33` β€” `classification.review_bots`, the loop From 1c7d524e955c1e8fedf84be5cf795d288726ba79 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:46:17 -0400 Subject: [PATCH 07/38] changelog: single bullet fragment for #14727 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-14727.md | 2 +- changelog.d/fixed-14736.md | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 changelog.d/fixed-14736.md diff --git a/changelog.d/fixed-14727.md b/changelog.d/fixed-14727.md index 47b3ad3b97..cf5cbe29ea 100644 --- a/changelog.d/fixed-14727.md +++ b/changelog.d/fixed-14727.md @@ -1 +1 @@ -Forward-merge PRs that are forced to a merge commit instead of a squash no longer raise a repeated unclassified runtime-error self-report ([#14727](https://github.com/hivecommons/hive/issues/14727)). +- Forward-merge PRs that are forced to a merge commit instead of a squash no longer raise a repeated unclassified runtime-error self-report ([#14727](https://github.com/hivecommons/hive/issues/14727)). diff --git a/changelog.d/fixed-14736.md b/changelog.d/fixed-14736.md deleted file mode 100644 index a4bd03bfaf..0000000000 --- a/changelog.d/fixed-14736.md +++ /dev/null @@ -1 +0,0 @@ -- fix(forward-merge): treat the squash-refusal guard as expected, not a runtime error (#14736) From 525a4ead6a9bf27dbe43b3d86743ef85b177ed0d Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 00:02:11 +0000 Subject: [PATCH 08/38] fix(agent): exempt ci-maintainer/quality lane variants from CI-poll nudge (#14732) Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-ci-poll-nudge-14719.md | 1 + src/crux/agent/ci_poll_exempt_lane_test.go | 23 ++++++++++++++++++++++ src/crux/agent/manager_kick.go | 16 +++++++++++++-- 3 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 changelog.d/fixed-ci-poll-nudge-14719.md create mode 100644 src/crux/agent/ci_poll_exempt_lane_test.go diff --git a/changelog.d/fixed-ci-poll-nudge-14719.md b/changelog.d/fixed-ci-poll-nudge-14719.md new file mode 100644 index 0000000000..2e2de71fbd --- /dev/null +++ b/changelog.d/fixed-ci-poll-nudge-14719.md @@ -0,0 +1 @@ +- The CI-poll stop nudge and `ci_poll_nudge` self-report now also skip replica and variant names of the exempt `ci-maintainer` and `quality` lanes (e.g. `ci-maintainer-2`), not only the exact lane name (#14719). diff --git a/src/crux/agent/ci_poll_exempt_lane_test.go b/src/crux/agent/ci_poll_exempt_lane_test.go new file mode 100644 index 0000000000..ef7b6d5c36 --- /dev/null +++ b/src/crux/agent/ci_poll_exempt_lane_test.go @@ -0,0 +1,23 @@ +package agent + +import ( + "testing" + + "github.com/hivecommons/hive/crux/config" +) + +func TestCIPollNudgeExemptAgentLaneVariants(t *testing.T) { + for name, want := range map[string]bool{ + "ci-maintainer": true, + "ci-maintainer-2": true, + "ci-maintainer_full": true, + "Quality-advisory": true, + "scanner": false, + "ci-maintainers": false, + } { + a := &AgentProcess{Name: name, Config: config.AgentConfig{Role: name}} + if got := ciPollNudgeExemptAgent(a); got != want { + t.Errorf("%q: got %v want %v", name, got, want) + } + } +} diff --git a/src/crux/agent/manager_kick.go b/src/crux/agent/manager_kick.go index d6c6619672..909a9339dd 100644 --- a/src/crux/agent/manager_kick.go +++ b/src/crux/agent/manager_kick.go @@ -1487,8 +1487,20 @@ func ciPollNudgeExemptAgent(agent *AgentProcess) bool { return true } for _, f := range []string{agent.Name, agent.Config.Role} { - lane := strings.TrimSpace(f) - if strings.EqualFold(lane, "ci-maintainer") || strings.EqualFold(lane, "quality") { + if exemptCIPollLane(f) { + return true + } + } + return false +} + +// exemptCIPollLane matches a lane name or role exactly, or as a dash/underscore- +// suffixed variant ("ci-maintainer-2", "quality_full"), so replicas and policy +// variants of an exempt lane are exempt too. +func exemptCIPollLane(f string) bool { + lane := strings.ToLower(strings.TrimSpace(f)) + for _, base := range []string{"ci-maintainer", "quality"} { + if lane == base || strings.HasPrefix(lane, base+"-") || strings.HasPrefix(lane, base+"_") { return true } } From 1997bc97c426f189d061f740f89fed334715b901 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 00:02:36 +0000 Subject: [PATCH 09/38] =?UTF-8?q?=F0=9F=90=9B=20fix(crux):=20keep=20runboo?= =?UTF-8?q?ks/=20when=20publishing=20to=20hivecommons/crux=20(#14723)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * πŸ› fix(crux): keep runbooks/ when publishing to hivecommons/crux Signed-off-by: operations <280983584+hivecommons-hive[bot]@users.noreply.github.com> * πŸ› fix(deps): bump golang.org/x/net to v0.62.0 Signed-off-by: operations <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: operations <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: operations <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-crux-publish-keep-runbooks.md | 1 + src/scripts/crux-publish.sh | 4 +- src/scripts/test-crux-publish-keep.sh | 38 +++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 changelog.d/fixed-crux-publish-keep-runbooks.md create mode 100755 src/scripts/test-crux-publish-keep.sh diff --git a/changelog.d/fixed-crux-publish-keep-runbooks.md b/changelog.d/fixed-crux-publish-keep-runbooks.md new file mode 100644 index 0000000000..1bd407d603 --- /dev/null +++ b/changelog.d/fixed-crux-publish-keep-runbooks.md @@ -0,0 +1 @@ +- The Crux publish script no longer deletes `runbooks/` in `hivecommons/crux` on every sync diff --git a/src/scripts/crux-publish.sh b/src/scripts/crux-publish.sh index d92b851bc8..dae2867091 100755 --- a/src/scripts/crux-publish.sh +++ b/src/scripts/crux-publish.sh @@ -8,7 +8,7 @@ # `go install github.com/hivecommons/crux/cmd/crux@latest` and # `brew install hivecommons/hive/crux` work without pulling hive. Nothing is # edited in hivecommons/crux by hand except its own build scaffolding -# (.github/, .goreleaser.yaml, Dockerfile, README.md, LICENSE). +# (.github/, .goreleaser.yaml, Dockerfile, README.md, LICENSE, runbooks/). # # This script is the whole mechanism (.github/workflows/crux-sync.yml calls # it; it also runs by hand for a first/emergency sync): @@ -43,7 +43,7 @@ OLD_MODULE="github.com/hivecommons/hive/crux" NEW_MODULE="github.com/hivecommons/crux" # Hand-maintained in hivecommons/crux; never touched by a sync. -KEEP=(.git .github .goreleaser.yaml Dockerfile Dockerfile.goreleaser LICENSE README.md .gitignore .golangci.yml .dockerignore) +KEEP=(.git .github .goreleaser.yaml Dockerfile Dockerfile.goreleaser LICENSE README.md .gitignore .golangci.yml .dockerignore runbooks) [ -f "$SRC/go.mod" ] || { echo "::error::$SRC/go.mod not found" >&2; exit 1; } [ -d "$DEST" ] || { echo "::error::$DEST is not a directory" >&2; exit 1; } diff --git a/src/scripts/test-crux-publish-keep.sh b/src/scripts/test-crux-publish-keep.sh new file mode 100755 index 0000000000..b688726061 --- /dev/null +++ b/src/scripts/test-crux-publish-keep.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# test-crux-publish-keep.sh β€” proves crux-publish.sh leaves the hand-maintained +# scaffolding in hivecommons/crux (notably runbooks/) alone while clearing +# stale synced files. A sync once deleted runbooks/ because it was not in KEEP. +# +# Usage: src/scripts/test-crux-publish-keep.sh +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +fail=0 +pass() { echo " ok: $*"; } +bad() { echo " FAIL: $*"; fail=1; } + +command -v rsync >/dev/null || { echo " skip: rsync not installed"; exit 0; } + +HIVE="$TMP/hive"; DEST="$TMP/dest" +mkdir -p "$HIVE/src/crux" "$DEST/runbooks" "$DEST/stale" +git -C "$HIVE" init -q +git -C "$HIVE" -c user.name=t -c user.email=t@example.com commit -q --allow-empty -m init +printf 'module github.com/hivecommons/hive/crux\n\ngo 1.22\n' > "$HIVE/src/crux/go.mod" +echo '# module' > "$HIVE/src/crux/README.md" +mkdir -p "$HIVE/src/crux/docs" +echo x > "$DEST/runbooks/release-rollback.md" +echo x > "$DEST/stale/old.txt" +echo x > "$DEST/README.md" + +if ! out="$(CRUX_SKIP_BUILD=1 bash "$HERE/crux-publish.sh" "$HIVE" "$DEST" 2>&1)"; then + bad "crux-publish.sh failed"; echo "$out" | sed 's/^/ | /' +fi + +[[ -f "$DEST/runbooks/release-rollback.md" ]] && pass "runbooks/ survives a sync" || bad "runbooks/ was deleted" +[[ -f "$DEST/README.md" ]] && pass "README.md survives a sync" || bad "README.md was deleted" +[[ ! -e "$DEST/stale" ]] && pass "stale synced entries are cleared" || bad "stale/ was not cleared" + +exit $fail From 0fd981b288fba50c456303a21472fe41492b565f Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 00:03:50 +0000 Subject: [PATCH 10/38] =?UTF-8?q?=F0=9F=90=9B=20fix(ioscan):=20name=20clas?= =?UTF-8?q?sifier=20error=20and=20fix=20in=20blocked-kick=20warning=20(#14?= =?UTF-8?q?722)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * πŸ› fix(ioscan): name classifier error and fix in blocked-kick warning Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * fix(ioscan): clear stale classifier error once the classifier recovers lastClassifierErr was only ever written, so after one classifier error every later blocked kick, including one withheld for a genuine injection detection with a healthy classifier, carried a hint to disable the classifier. Clear it on the success path. The fail-open branch of classifierErrored is left alone: the classifier did error there, so the value is not stale, and fail-open does not withhold kicks. Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...d-14714-ioscan-blocked-kick-diagnostics.md | 1 + src/pkg/scheduler/ioscan_enforce.go | 20 +++++++++- src/pkg/scheduler/ioscan_enforce_test.go | 37 +++++++++++++++++++ src/pkg/scheduler/scheduler.go | 2 + src/pkg/scheduler/templates.go | 2 +- 5 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 changelog.d/fixed-14714-ioscan-blocked-kick-diagnostics.md diff --git a/changelog.d/fixed-14714-ioscan-blocked-kick-diagnostics.md b/changelog.d/fixed-14714-ioscan-blocked-kick-diagnostics.md new file mode 100644 index 0000000000..abcc5b64d7 --- /dev/null +++ b/changelog.d/fixed-14714-ioscan-blocked-kick-diagnostics.md @@ -0,0 +1 @@ +- When ioscan fails closed and blocks a kick because the semantic classifier errored, the warning now names the underlying classifier error and the config to fix it (served classifier model/endpoint, `ioscan.classifier.enabled`, `ioscan.classifier.fail_mode`) instead of only "ioscan fail-closed blocked kick"; fail-closed stays the default (#14714). diff --git a/src/pkg/scheduler/ioscan_enforce.go b/src/pkg/scheduler/ioscan_enforce.go index 18691ef9c1..88075e78ab 100644 --- a/src/pkg/scheduler/ioscan_enforce.go +++ b/src/pkg/scheduler/ioscan_enforce.go @@ -260,6 +260,7 @@ func (s *Scheduler) enforceTextFrom(text string, src ioscan.Source) (string, ios return s.classifierErrored(sanitized, v, err) } s.classifierUnavailable(nil) + s.lastClassifierErr.Store("") action := ioscan.ActionForScore(score, thresholds, s.classifierFailClosed()) s.recordClassifierResult(score, action) switch action { @@ -279,6 +280,18 @@ func (s *Scheduler) enforceTextFrom(text string, src ioscan.Source) (string, ios return sanitized, v } +const classifierFixHint = "check the ioscan classifier model/endpoint is served by the inference gateway (governor.trajectory/litellm, ioscan.classifier.model), or set ioscan.classifier.enabled: false or ioscan.classifier.fail_mode: open" + +// warnBlockedKick logs that fail-closed withheld a kick, naming the latest +// classifier error and the config knobs so operators can act on it. +func (s *Scheduler) warnBlockedKick(agentName string) { + args := []any{"agent", agentName} + if e, _ := s.lastClassifierErr.Load().(string); e != "" { + args = append(args, "classifier_error", e, "fix", classifierFixHint) + } + s.warnOnce("blocked-kick:"+agentName, "ioscan fail-closed blocked kick", args...) +} + // classifierErrored applies the fail mode when the classifier could not score // sanitized: fail-closed withholds the text, fail-open passes it on. Both // branches log at most once per distinct error so a dead backend does not @@ -287,13 +300,16 @@ func (s *Scheduler) classifierErrored(sanitized string, v ioscan.Verdict, err er if s.classifierFailClosed() { s.recordClassifierSkip("error_fail_closed") s.classifierUnavailable(err) + s.lastClassifierErr.Store(err.Error()) if s.logger != nil { if strings.Contains(err.Error(), "no model endpoint resolved") { s.warnOnce("classifier-no-endpoint", "ioscan classifier failed closed; text withheld", "error", err, - "fix", "configure governor.trajectory/litellm for the ioscan model endpoint, or set ioscan.classifier.enabled: false or ioscan.classifier.fail_mode: open") + "fix", classifierFixHint) } else { - s.warnOnce("classifier-error:"+err.Error(), "ioscan classifier failed closed; text withheld", "error", err) + s.warnOnce("classifier-error:"+err.Error(), "ioscan classifier failed closed; text withheld", + "error", err, + "fix", classifierFixHint) } } cv := ioscan.UnavailableVerdict(err.Error()) diff --git a/src/pkg/scheduler/ioscan_enforce_test.go b/src/pkg/scheduler/ioscan_enforce_test.go index b2f57baa28..e2e8d306fc 100644 --- a/src/pkg/scheduler/ioscan_enforce_test.go +++ b/src/pkg/scheduler/ioscan_enforce_test.go @@ -2,6 +2,7 @@ package scheduler import ( "context" + "errors" "fmt" "io" "log/slog" @@ -465,3 +466,39 @@ func TestUnavailableClassifierFailOpenWarnsOnce(t *testing.T) { t.Fatalf("WARN count = %d, want 1; log:\n%s", n, buf.String()) } } + +func TestBlockedKickWarnNamesClassifierError(t *testing.T) { + var buf strings.Builder + s := newSchedulerWithIoscanFailMode(true, "closed") + s.logger = slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + s.SetClassifier(&fakeInjectionClassifier{err: errors.New("model not found: ioscan-judge")}, ioscan.Thresholds{Warn: 0.5, Block: 0.8}) + + s.enforceIssueTextVerdict("some issue title") + buf.Reset() + s.warnBlockedKick("reviewer") + for _, want := range []string{"ioscan fail-closed blocked kick", "model not found: ioscan-judge", "ioscan.classifier.fail_mode"} { + if !strings.Contains(buf.String(), want) { + t.Fatalf("blocked-kick log missing %q: %s", want, buf.String()) + } + } +} + +func TestBlockedKickWarnOmitsClassifierErrorAfterRecovery(t *testing.T) { + var buf strings.Builder + s := newSchedulerWithIoscanFailMode(true, "closed") + s.logger = slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + fc := &fakeInjectionClassifier{err: errors.New("model not found: ioscan-judge")} + s.SetClassifier(fc, ioscan.Thresholds{Warn: 0.5, Block: 0.8}) + + s.enforceIssueTextVerdict("some issue title") + fc.err = nil + s.enforceIssueTextVerdict("some issue title") + buf.Reset() + s.warnBlockedKick("reviewer") + if !strings.Contains(buf.String(), "ioscan fail-closed blocked kick") { + t.Fatalf("blocked-kick warning missing: %s", buf.String()) + } + if strings.Contains(buf.String(), "classifier_error") { + t.Fatalf("stale classifier error reported after recovery: %s", buf.String()) + } +} diff --git a/src/pkg/scheduler/scheduler.go b/src/pkg/scheduler/scheduler.go index d63d52d353..b01154702f 100644 --- a/src/pkg/scheduler/scheduler.go +++ b/src/pkg/scheduler/scheduler.go @@ -3,6 +3,7 @@ package scheduler import ( "log/slog" "sync" + "sync/atomic" "time" "github.com/hivecommons/hive/pkg/config" @@ -47,6 +48,7 @@ type Scheduler struct { deferredKicks map[string]func(string) firstScanDeferred map[string]bool ioscanWarned sync.Map + lastClassifierErr atomic.Value // string: latest fail-closed classifier error mu sync.RWMutex } diff --git a/src/pkg/scheduler/templates.go b/src/pkg/scheduler/templates.go index fa95ff687c..d5422987e3 100644 --- a/src/pkg/scheduler/templates.go +++ b/src/pkg/scheduler/templates.go @@ -334,7 +334,7 @@ func (s *Scheduler) substituteTemplateWithVars(template string, actionable *gith issueList, issueFailClosed := s.formatIssueListWithPolicy(agentIssuesForList) prList, prFailClosed := s.formatPRListWithPolicyForAgent(actionable, baseName) if issueFailClosed || prFailClosed { - s.warnOnce("blocked-kick:"+agentName, "ioscan fail-closed blocked kick", "agent", agentName) + s.warnBlockedKick(agentName) return "", true } From 5d3addfa789ff2815efed539bd32b99c7894d5d6 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 00:04:08 +0000 Subject: [PATCH 11/38] =?UTF-8?q?=F0=9F=90=9B=20fix(automerge):=20classify?= =?UTF-8?q?=20review-requesting-changes=20merge=20refusal=20(#14720)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-14717-review-requesting-changes-refusal.md | 1 + src/pkg/github/automerge/automerge_merge_refusal.go | 5 +++-- src/pkg/github/automerge/automerge_merge_refusal_test.go | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 changelog.d/fixed-14717-review-requesting-changes-refusal.md diff --git a/changelog.d/fixed-14717-review-requesting-changes-refusal.md b/changelog.d/fixed-14717-review-requesting-changes-refusal.md new file mode 100644 index 0000000000..4a3782cadd --- /dev/null +++ b/changelog.d/fixed-14717-review-requesting-changes-refusal.md @@ -0,0 +1 @@ +- The automerge sweep now treats a 405 "review requesting changes" ruleset refusal like the other permanent review refusals, so a blocked PR is parked and remembered instead of logging a merge-failed WARN and retrying every sweep ([#14717](https://github.com/hivecommons/hive/issues/14717)). diff --git a/src/pkg/github/automerge/automerge_merge_refusal.go b/src/pkg/github/automerge/automerge_merge_refusal.go index c3a69203d9..d80a5e489e 100644 --- a/src/pkg/github/automerge/automerge_merge_refusal.go +++ b/src/pkg/github/automerge/automerge_merge_refusal.go @@ -38,8 +38,9 @@ const mergeRefusalRecheck = 30 * time.Minute const mergeRefusalPruneAt = 1024 // "At least 1 approving review is required by reviewers with write access.", -// and "At least 2 approving reviews are required ...". -var mergeRefusedReviewRE = regexp.MustCompile(`(?i)\d+ approving reviews? (?:is|are) required`) +// "At least 2 approving reviews are required ...", and the ruleset form +// "1 review requesting changes by reviewers with write access.". +var mergeRefusedReviewRE = regexp.MustCompile(`(?i)\d+ approving reviews? (?:is|are) required|\d+ reviews? requesting changes`) // "Merge commits are not allowed on this repository." and the squash and // rebase forms. diff --git a/src/pkg/github/automerge/automerge_merge_refusal_test.go b/src/pkg/github/automerge/automerge_merge_refusal_test.go index ee1bfcba5b..c2f7524b7c 100644 --- a/src/pkg/github/automerge/automerge_merge_refusal_test.go +++ b/src/pkg/github/automerge/automerge_merge_refusal_test.go @@ -37,6 +37,7 @@ func TestMergeRefusalReason(t *testing.T) { {name: "one approving review", err: resp(http.StatusMethodNotAllowed, reviewRequiredTestMessage), want: autoMergeReasonMergeRefusedReview}, {name: "two approving reviews", err: resp(http.StatusMethodNotAllowed, "At least 2 approving reviews are required by reviewers with write access."), want: autoMergeReasonMergeRefusedReview}, {name: "ruleset approving review", err: resp(http.StatusMethodNotAllowed, "Repository rule violations found\n\nAt least 1 approving review is required by reviewers with write access."), want: autoMergeReasonMergeRefusedReview}, + {name: "ruleset review requesting changes", err: resp(http.StatusMethodNotAllowed, "Repository rule violations found\n\n1 review requesting changes by reviewers with write access.\n\n"), want: autoMergeReasonMergeRefusedReview}, {name: "approving review wrapped not-applied", err: effects.NotApplied(resp(http.StatusMethodNotAllowed, reviewRequiredTestMessage)), want: autoMergeReasonMergeRefusedReview}, {name: "merge commits not allowed", err: resp(http.StatusMethodNotAllowed, mergeCommitsTestMessage), want: autoMergeReasonMergeRefusedMethod}, {name: "squash not allowed", err: resp(http.StatusMethodNotAllowed, "Squash merges are not allowed on this repository."), want: autoMergeReasonMergeRefusedMethod}, From 3dce48b30d6a8faf136c0963a52523834b7db772 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 00:18:35 +0000 Subject: [PATCH 12/38] =?UTF-8?q?=F0=9F=8C=B1=20refactor(proxy):=20split?= =?UTF-8?q?=20github=5Fproxy.go=20by=20concern=20(move-only)=20(#14733)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(proxy): split github_proxy.go by concern (move-only) Move-only split of src/crux/proxy/github_proxy.go into per-concern files inside package proxy. No signature, behaviour or exported-name changes. Source-scanning tests now read all non-test github_proxy*.go files. Refs hivecommons/crux#23 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * changelog: fragment for #14733 Generated by changelog-fragment-guard from the PR title (#5675). Edit the wording freely, or replace this file with the no-changelog label if the change is not user-visible. Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> * docs: re-home proxy citations to the split github_proxy_*.go files Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Signed-off-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- changelog.d/changed-14733.md | 1 + src/crux/proxy/close_wait_leak_test.go | 14 +- src/crux/proxy/github_proxy.go | 3288 +-------------------- src/crux/proxy/github_proxy_ca.go | 194 ++ src/crux/proxy/github_proxy_connect.go | 672 +++++ src/crux/proxy/github_proxy_copilot.go | 294 ++ src/crux/proxy/github_proxy_identity.go | 181 ++ src/crux/proxy/github_proxy_inference.go | 752 +++++ src/crux/proxy/github_proxy_mitm.go | 402 +++ src/crux/proxy/github_proxy_policy.go | 368 +++ src/crux/proxy/github_proxy_setters.go | 100 + src/crux/proxy/github_proxy_tunnel.go | 179 ++ src/crux/proxy/github_proxy_violations.go | 254 ++ src/crux/proxy/upstream_timeout_test.go | 51 +- src/docs/design/README.md | 2 +- src/docs/design/copilot-cost-capture.md | 6 +- src/docs/fleet-report.md | 6 +- 17 files changed, 3437 insertions(+), 3327 deletions(-) create mode 100644 changelog.d/changed-14733.md create mode 100644 src/crux/proxy/github_proxy_ca.go create mode 100644 src/crux/proxy/github_proxy_connect.go create mode 100644 src/crux/proxy/github_proxy_copilot.go create mode 100644 src/crux/proxy/github_proxy_identity.go create mode 100644 src/crux/proxy/github_proxy_inference.go create mode 100644 src/crux/proxy/github_proxy_mitm.go create mode 100644 src/crux/proxy/github_proxy_policy.go create mode 100644 src/crux/proxy/github_proxy_setters.go create mode 100644 src/crux/proxy/github_proxy_tunnel.go create mode 100644 src/crux/proxy/github_proxy_violations.go diff --git a/changelog.d/changed-14733.md b/changelog.d/changed-14733.md new file mode 100644 index 0000000000..2ba7bfca40 --- /dev/null +++ b/changelog.d/changed-14733.md @@ -0,0 +1 @@ +- refactor(proxy): split github_proxy.go by concern (move-only) (#14733) diff --git a/src/crux/proxy/close_wait_leak_test.go b/src/crux/proxy/close_wait_leak_test.go index 359eb8cb72..bd8671513a 100644 --- a/src/crux/proxy/close_wait_leak_test.go +++ b/src/crux/proxy/close_wait_leak_test.go @@ -4,8 +4,6 @@ import ( "io" "log/slog" "net" - "os" - "path/filepath" "runtime" "strings" "sync" @@ -195,21 +193,13 @@ func TestProxyHTTPBodyRelayToleratesSlowButFlowingBody(t *testing.T) { } // TestProxyHTTPHasNoInlineCopyPairs asserts the source-level invariant behind -// leak (1): every opaque byte-shuttling path in github_proxy.go must go +// leak (1): every opaque byte-shuttling path in github_proxy*.go must go // through relayTunnel (whose half-close drains are the #3872 guarantee), never // a hand-rolled io.Copy pair. The pre-fix git branch was exactly such a pair // that #3872 missed. relayTunnel's own two copies are the only sanctioned // ones. func TestProxyHTTPHasNoInlineCopyPairs(t *testing.T) { - _, file, _, ok := runtime.Caller(0) - if !ok { - t.Fatal("runtime.Caller failed") - } - body, err := os.ReadFile(filepath.Join(filepath.Dir(file), "github_proxy.go")) - if err != nil { - t.Fatalf("read github_proxy.go: %v", err) - } - text := string(body) + text := readGitHubProxySource(t) const wantRelayTunnelCalls = 3 // transparent passthrough, CONNECT tunnelDirect, git branch if got := strings.Count(text, "relayTunnel(") - strings.Count(text, "func relayTunnel("); got != wantRelayTunnelCalls { t.Fatalf("relayTunnel call sites = %d, want %d β€” opaque relays must use relayTunnel, not inline io.Copy pairs", got, wantRelayTunnelCalls) diff --git a/src/crux/proxy/github_proxy.go b/src/crux/proxy/github_proxy.go index 67a25a2b03..e21d9a51b6 100644 --- a/src/crux/proxy/github_proxy.go +++ b/src/crux/proxy/github_proxy.go @@ -2,31 +2,18 @@ package proxy import ( "bufio" - "bytes" - "context" - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" "crypto/tls" "crypto/x509" - "crypto/x509/pkix" - "encoding/base64" "encoding/json" - "encoding/pem" - "errors" "fmt" "io" "log/slog" - "math/big" "net" "net/http" "os" - "path/filepath" - "regexp" "strconv" "strings" "sync" - "sync/atomic" "syscall" "time" @@ -34,7 +21,6 @@ import ( "github.com/hivecommons/hive/crux/agent" "github.com/hivecommons/hive/crux/config" "github.com/hivecommons/hive/crux/inferencehealth" - "github.com/hivecommons/hive/crux/issueshape" ) const ( @@ -315,138 +301,6 @@ type GitHubProxy struct { bodyStallTimeout time.Duration } -// SetLinearCredentialResolver installs the resolver injectLinearCredential -// reads on every agent request to api.linear.app. main.go wires the same -// closure the agent manager uses to push LINEAR_ACCESS_TOKEN / LINEAR_API_KEY -// into sessions, so the proxy and the session env can never name different -// credentials. Values are never logged. nil disables injection. -func (p *GitHubProxy) SetLinearCredentialResolver(fn func() agent.LinearCredential) { - p.linearCredential = fn -} - -// injectLinearCredential replaces the Authorization header of an agent's -// api.linear.app request with the hive's CURRENT Linear credential, for -// agents at the ISSUES_ONLY floor and above (the tier the session-env push in -// agent.Manager.linearEnvPairs uses). It reports whether a credential was -// applied. -// -// WHY THE PROXY, NOT THE ENVIRONMENT. The connected Linear app's OAuth access -// token rotates (Linear issues it with a ~24h expiry and hive refreshes it -// in the background), and hive already re-pushes the fresh value into each -// agent's tmux session environment on the hourly refresh tick. But a tmux -// session environment only reaches processes forked AFTER the push: the -// agent's CLI, forked at its last launch, keeps the token it was born with, -// and every tool shell it spawns inherits that copy. Observed live -// (2026-09-06/07): a quality agent whose session environment held the fresh -// token β€” verified valid through this very proxy as that agent's UID β€” got -// `401 AUTHENTICATION_ERROR` on every Linear call from inside its pane for -// two days, because its process still carried the pre-rotation token. Its -// earlier kicks had silently fallen back to the operator's personal -// LINEAR_API_KEY, so every issue it filed showed the operator as creator. -// -// The proxy terminates every agent request to api.linear.app anyway (this is -// where enforceLinear gates mutations by tier), so attaching the live -// credential HERE makes the copy in the agent's environment irrelevant β€” -// stale, absent, or a placeholder all work β€” exactly as the inference -// translator attaches the real gateway key server-side. The agent still -// receives a credential in its environment (for SDKs that refuse to send a -// request without one), but Linear only ever sees the proxy's. -// -// Below the floor the request is left untouched: an ADVISORY agent's Linear -// reads have always been ungated and unauthenticated-by-hive, and injecting -// a write-capable credential there would widen what the tier can do. -func (p *GitHubProxy) injectLinearCredential(req *http.Request, agentName string, mode agent.AgentMode) bool { - if p.linearCredential == nil || !mode.CanCreateIssues() { - return false - } - cred := p.linearCredential() - switch { - case cred.AccessToken != "": - req.Header.Set("Authorization", "Bearer "+cred.AccessToken) - case cred.APIKey != "": - req.Header.Set("Authorization", cred.APIKey) - default: - return false - } - p.logger.Debug("linear: attached the hive's current credential to an agent request", "agent", agentName, "mode", mode.String()) - return true -} - -// dialCopilotUpstream connects to the real Copilot host over TLS (or the test -// seam when set). -func (p *GitHubProxy) dialCopilotUpstream(host string) (net.Conn, error) { - if p.copilotDial != nil { - return p.copilotDial(host) - } - // SO_MARK the outbound socket so the forced-egress redirect exempts the - // proxy's own upstream traffic (see proxyEgressMark). The override hook above - // still wins when set, so tests are unaffected. - return tls.DialWithDialer(markDialer(upstreamDialTimeout), "tcp", net.JoinHostPort(host, "443"), &tls.Config{ServerName: host}) -} - -// SetTokenSink wires the inference token sink so the translator can record -// per-agent usage from upstream OpenAI responses into the shared token store. -func (p *GitHubProxy) SetTokenSink(sink UsageSink) { - p.tokenSink = sink -} - -// SetAgentTokenSource wires the resolver from identified agent name to that -// agent's hub-held scoped token (#1861). Wired once at boot, before Start(), -// so no lock is needed on the read path. -func (p *GitHubProxy) SetAgentTokenSource(source func(agentName string) (string, bool)) { - p.agentTokenSource = source -} - -func (p *GitHubProxy) SetAgentGitHubBudgetConfig(capForAgent func(agentName string) int, reserveFloor func() int) { - if capForAgent != nil { - p.githubBudgetCap = capForAgent - } - if reserveFloor != nil { - p.githubReserveFloor = reserveFloor - } -} - -func (p *GitHubProxy) SetAgentGitHubBudgetNudgeFunc(fn func(agentName, message string)) { - p.githubBudgetNudge = fn -} - -// hostNeedsMITM decides whether a GitHub-family host must be TLS-intercepted -// rather than opaquely tunneled. -// -// Without injection this is exactly the historical NeedsMITM: only -// api.github.com, where request-level ACMM inspection happens; github.com and -// registered GHE hosts are tunneled without decryption (their traffic β€” git -// smart HTTP, OAuth device flow β€” is gated elsewhere). -// -// With injection (#1861) an opaque tunnel is a hole, not an optimization: the -// agent's git credential helper now serves the inert placeholder, so a -// git-over-HTTPS push through an un-MITM'd tunnel would reach GitHub carrying -// the placeholder and 401. Every GitHub-family host must therefore be -// intercepted so rewriteGitHubAuth can replace the placeholder with the real -// scoped token β€” which also means an agent-supplied credential can never -// sneak to ANY GitHub host through a tunnel the proxy declined to open. -func (p *GitHubProxy) hostNeedsMITM(host string) bool { - if NeedsMITM(host) { - return true - } - return p.injectGHAuth && IsGitHubHost(host) -} - -// SetCanaryScanner installs the host's canary registry and leak handler. -// enabled gates the scan; failClosed turns a detection (or an unscannable -// body) into a refusal rather than an audit-only event. -func (p *GitHubProxy) SetCanaryScanner(enabled, failClosed bool, reg CanaryScanner, onLeak func(crux.CanaryLeak)) { - p.canariesEnabled = enabled - p.canaryFailClosed = failClosed - p.canaryRegistry = reg - p.canaryLeakFunc = onLeak -} - -type cachedCert struct { - cert tls.Certificate - expiresAt time.Time -} - // NewGitHubProxy creates a proxy with a self-signed CA for MITM. // The org and repos parameters define which repositories are allowed for // write operations. Repos should be bare names (e.g. "console"); the org @@ -545,3155 +399,17 @@ func newGitHubProxyWithCA(caCert tls.Certificate, caX509 *x509.Certificate, logg return p } -// SetRepoPausedFunc installs the operator's per-repo pause predicate (#6203). -// The hive passes config's IsRepoPaused, so pausing a repo in the dashboard is -// enforced on the very next request without a restart β€” which matters, because -// the pauses this serves (a release freeze, an incident) are measured in hours, -// and an operator who has to roll the pod to quiet one repo is back to stopping -// the whole hive. Passing nil clears it. -func (p *GitHubProxy) SetRepoPausedFunc(fn func(repo string) bool) { - p.pausedMu.Lock() - p.repoPaused = fn - p.pausedMu.Unlock() -} - -// repoScopePath is the request path handed to the per-repo write gates (pause, -// agent scope, auto-merge, per-repo ACMM mode, repo filter). It starts from -// the client's own escaping rather than the decoded req.URL.Path so a GitLab -// "namespace%2Fproject" stays one segment and ExtractRepo resolves the exact -// project β€” including nested groups β€” instead of guessing where the project -// path ends from the sub-resource that follows it. CanonicalGatePath then -// decodes every OTHER escape so an escaped structural segment (/r%65pos/, -// repo%2Egit, pro%6Aects, merg%65) cannot hide the repo from the gates. -func repoScopePath(req *http.Request) string { - return CanonicalGatePath(req.URL.EscapedPath()) -} - -// repoPauseRefusal is RepoPauseRefusal against this proxy's current predicate. -func (p *GitHubProxy) repoPauseRefusal(method, path string) (string, bool) { - p.pausedMu.RLock() - paused := p.repoPaused - p.pausedMu.RUnlock() - return RepoPauseRefusal(paused, method, path) -} - -// SetRepoAutoMergeEnabledFunc installs the live per-repo auto-merge predicate. -// The proxy uses it to give flagged repositories their specific deny reason at -// direct merge endpoints. Passing nil preserves default-on behavior. -func (p *GitHubProxy) SetRepoAutoMergeEnabledFunc(fn func(repo string) bool) { - p.autoMergeMu.Lock() - p.repoAutoMergeOn = fn - p.autoMergeMu.Unlock() -} - -func (p *GitHubProxy) autoMergeRefusal(method, path string) (string, bool) { - p.autoMergeMu.RLock() - enabled := p.repoAutoMergeOn - p.autoMergeMu.RUnlock() - return RepoAutoMergeRefusal(enabled, method, path) -} - -func (p *GitHubProxy) graphQLAutoMergeRefusal(body []byte) (string, bool) { - p.autoMergeMu.RLock() - enabled := p.repoAutoMergeOn - p.autoMergeMu.RUnlock() - return GraphQLAutoMergeRefusal(enabled, body) -} - -// SetAgentRepoScopeFunc installs the per-repo agent-scope predicate (#6204). -// The hive passes config's AgentServesRepo, so re-scoping an agent in the -// dashboard is enforced on the very next request without a restart. Passing nil -// clears it, which is the unscoped behaviour every hive had before. -func (p *GitHubProxy) SetAgentRepoScopeFunc(fn func(agent, repo string) bool) { - p.scopeMu.Lock() - p.agentServesRepo = fn - p.scopeMu.Unlock() -} - -// agentRepoScopeRefusal is AgentRepoScopeRefusal against the current predicate. -func (p *GitHubProxy) agentRepoScopeRefusal(agentName, method, path string) (string, bool) { - p.scopeMu.RLock() - serves := p.agentServesRepo - p.scopeMu.RUnlock() - return AgentRepoScopeRefusal(serves, agentName, method, path) -} - -// SetWriteSurfaceEnforceFunc installs the per-lane write_surface.enforce -// predicate (#9772). The hive passes config's WriteSurfaceEnforced, so a lane -// listed in config is enforced on the next request. Passing nil clears it, -// which is the behaviour every hive had before: direct writes allowed. -func (p *GitHubProxy) SetWriteSurfaceEnforceFunc(fn func(agent string) bool) { - p.writeSurfaceMu.Lock() - p.writeSurfaceEnforced = fn - p.writeSurfaceMu.Unlock() -} - -// SetWriteRefusedAuditFunc installs the sink that records a direct write -// refused under write_surface.enforce as an agent_write_refused audit entry. -func (p *GitHubProxy) SetWriteRefusedAuditFunc(fn func(agent, kind, method, path, repo string)) { - p.writeSurfaceMu.Lock() - p.writeRefusedAudit = fn - p.writeSurfaceMu.Unlock() -} - -func (p *GitHubProxy) writeSurfaceEnforcedFunc() func(agent string) bool { - p.writeSurfaceMu.RLock() - defer p.writeSurfaceMu.RUnlock() - return p.writeSurfaceEnforced -} - -// laneWriteSurfaceEnforced reports whether agentName's lane is enforced. -func (p *GitHubProxy) laneWriteSurfaceEnforced(agentName string) bool { - if agentName == "" || agentName == internalCallerName { - return false - } - enforced := p.writeSurfaceEnforcedFunc() - return enforced != nil && enforced(agentName) -} - -// hostNeedsMITMFor is hostNeedsMITM for a known caller. A lane under -// write_surface.enforce has every GitHub-family host intercepted, not only -// api.github.com, because git push travels over github.com's smart HTTP and -// an opaque tunnel could not refuse it. Other lanes are unaffected. -func (p *GitHubProxy) hostNeedsMITMFor(host, agentName string) bool { - if p.hostNeedsMITM(host) { - return true - } - return IsGitHubHost(host) && p.laneWriteSurfaceEnforced(agentName) -} - -// auditWriteRefused records one direct write refused under -// write_surface.enforce. -func (p *GitHubProxy) auditWriteRefused(agentName, kind, method, path string) { - p.writeSurfaceMu.RLock() - audit := p.writeRefusedAudit - p.writeSurfaceMu.RUnlock() - if audit == nil { - return - } - audit(agentName, kind, method, path, WriteSurfaceRepo(path)) -} - // ListenAddr returns the proxy listen address. func (p *GitHubProxy) ListenAddr() string { return p.listenAddr } -// Violations returns a snapshot of per-agent violation counts. -func (p *GitHubProxy) Violations() map[string]int { - p.mu.RLock() - defer p.mu.RUnlock() - out := make(map[string]int, len(p.violations)) - for k, v := range p.violations { - out[k] = v - } - return out -} - -// AgentViolations returns the violation count for a specific agent. -func (p *GitHubProxy) AgentViolations(agentName string) int { - p.mu.RLock() - defer p.mu.RUnlock() - return p.violations[agentName] -} - -// Start begins listening. Blocks until the listener is closed. -// Handles both explicit HTTP CONNECT proxy requests and transparent -// iptables-redirected TLS connections (detected by TLS ClientHello). -func (p *GitHubProxy) Start() error { - ln, err := net.Listen("tcp", p.listenAddr) - if err != nil { - return fmt.Errorf("listen %s: %w", p.listenAddr, err) - } - defer func() { _ = ln.Close() }() - p.logger.Info("proxy listening", "addr", p.listenAddr) - - for { - conn, err := ln.Accept() - if err != nil { - return fmt.Errorf("accept: %w", err) - } - go p.handleConn(conn) - } -} - -// handleConn peeks at the first byte to distinguish HTTP CONNECT requests -// (explicit proxy) from raw TLS ClientHello (iptables-redirected traffic). -func (p *GitHubProxy) handleConn(conn net.Conn) { - defer func() { _ = conn.Close() }() - - peeked := make([]byte, 1) - _ = conn.SetReadDeadline(time.Now().Add(transparentProxyTimeout)) - n, err := conn.Read(peeked) - _ = conn.SetReadDeadline(time.Time{}) - if err != nil || n == 0 { - return - } - - // TLS ClientHello starts with byte 0x16 (ContentType handshake). - // HTTP methods start with ASCII letters (C for CONNECT, G for GET, etc.). - const tlsHandshakeContentType = 0x16 - if peeked[0] == tlsHandshakeContentType { - p.handleTransparentTLS(conn, peeked) - return - } - - // Parse the HTTP request directly instead of using http.Server.Serve, - // which closes the connection on shutdown β€” racing with hijacked CONNECT - // handlers. - prefixed := &prefixConn{Conn: conn, prefix: peeked[:n]} - _ = conn.SetReadDeadline(time.Now().Add(httpReadTimeout)) - req, err := http.ReadRequest(bufio.NewReader(prefixed)) - _ = conn.SetReadDeadline(time.Time{}) - if err != nil { - return - } - - if req.Method == http.MethodConnect { - p.handleConnectDirect(conn, req) - } else { - p.forwardPlainDirect(conn, req) - } -} - -const ( - transparentProxyTimeout = 5 * time.Second - httpReadTimeout = 30 * time.Second - httpWriteTimeout = 60 * time.Second - // upstreamDialTimeout bounds the proxy's OWN upstream dial + TLS handshake - // to the real GitHub host. Before this, a stalled upstream (DNS, TCP, or - // TLS) blocked forever β€” the agent's curl sat in poll() waiting for the - // MITM response that never came, permanently wedging the request and - // freezing the agent's session (the multi-hour "egress hang"). - // tls.DialWithDialer's dialer Timeout covers connect + handshake; the - // deadline is lifted once established so long-lived relays (git smart - // HTTP) are unaffected. - upstreamDialTimeout = 15 * time.Second - - // clientTLSHandshakeTimeout bounds the proxy's TLS handshake with the - // inbound client. It intentionally matches upstreamDialTimeout: both are - // short setup phases, unlike long-lived request/response relays. - clientTLSHandshakeTimeout = upstreamDialTimeout -) - -// handleTransparentTLS handles iptables-redirected connections. The agent -// tried to connect to github.com:443 but iptables sent it here instead. -// We extract the SNI hostname from the TLS ClientHello, then MITM the connection. -func (p *GitHubProxy) handleTransparentTLS(conn net.Conn, peeked []byte) { - // Read the whole ClientHello TLS record before extracting SNI and replaying - // it. TCP guarantees nothing about record boundaries: a single Read can - // return only PART of the ClientHello. A partial read makes extractSNI miss - // the SNI (so the wrong host is chosen and the wrong cert is forged) and - // mis-frames the record stream handed to tls.Server β€” the agent-side - // handshake then fails deterministically (bad certificate / bad record MAC), - // which is exactly why every Copilot agent could not authenticate. Loop until - // the full record (per the record-layer length) is buffered. - _ = conn.SetReadDeadline(time.Now().Add(transparentProxyTimeout)) - fullBuf, err := readClientHelloRecord(conn, peeked) - _ = conn.SetReadDeadline(time.Time{}) - if err != nil { - return - } - - host := extractSNI(fullBuf) - if host == "" { - host = "github.com" - } - - agentName := p.attributeTransparentConn(conn) - - // Copilot completion host under iptables redirection: MITM to read live - // token usage (same guard as the explicit-CONNECT path). The peeked - // ClientHello is replayed via prefixConn so the TLS handshake sees the full - // record. - if IsCopilotAPIHost(host) && p.tokenSink != nil && agentName != "" { - p.sniffCopilotOnTLS(&prefixConn{Conn: conn, prefix: fullBuf}, host, agentName) - return - } - - if !NeedsInspection(host) && !p.hostNeedsMITMFor(host, agentName) { - // Host we neither inspect nor (under #1861 injection) intercept: - // tunnel directly. SO_MARK the socket - // so the forced-egress redirect exempts this proxy-originated dial. - upstream, err := p.dialTunnelUpstream(host) - if err != nil { - return - } - defer func() { _ = upstream.Close() }() - if _, err := upstream.Write(fullBuf); err != nil { - return - } - relayTunnel(conn, upstream) - return - } - - mode := readAgentMode(agentName) - caps := readAgentCaps(agentName) - - // MITM: forge a cert, TLS-wrap the client, connect to real upstream. - tlsCert, err := p.forgeCert(host) - if err != nil { - p.logger.Error("transparent proxy forge cert failed", "host", host, "error", err) - return - } - - tlsConfig := &tls.Config{Certificates: []tls.Certificate{tlsCert}} - prefixed := &prefixConn{Conn: conn, prefix: fullBuf} - tlsClientConn := tls.Server(prefixed, tlsConfig) - // Bound the client TLS handshake: a client that connects and then stops - // sending handshake bytes would otherwise wedge this connection forever - // (empty queues on both sides β€” the observed egress-hang signature). - _ = conn.SetDeadline(time.Now().Add(clientTLSHandshakeTimeout)) - handshakeErr := tlsClientConn.Handshake() - _ = conn.SetDeadline(time.Time{}) - if handshakeErr != nil { - if isClientHangup(handshakeErr) { - p.logger.Debug("proxy client hung up before TLS handshake", "error", handshakeErr) - } else { - p.logger.Warn("transparent proxy TLS handshake failed", "error", handshakeErr) - } - return - } - defer func() { _ = tlsClientConn.Close() }() - - upstreamConn, err := tls.DialWithDialer(markDialer(upstreamDialTimeout), "tcp", host+":443", &tls.Config{ServerName: host}) - if err != nil { - p.logger.Error("transparent proxy upstream dial failed", "host", host, "error", err) - return - } - defer func() { _ = upstreamConn.Close() }() - - p.proxyHTTPHost(tlsClientConn, upstreamConn, host, agentName, mode, caps) -} - -// attributeTransparentConn names the agent behind an iptables-redirected -// connection (#7793). Under transparent redirection there is no -// Proxy-Authorization header and no CONNECT request, so the ONLY identity -// evidence is the kernel socket table: the peer's ephemeral port is looked up -// in /proc/net/tcp{,6} for the owning UID, and the UID in the agent map. -// -// It returns "" β€” "unidentified", which the mode gate downgrades to ADVISORY β€” -// when there is no UID map, when the peer address carries no port (a net.Pipe -// or unix socket), when the port is not in the socket table, or when the UID -// belongs to no agent. The one deliberate exception is the hive's own control -// plane: a UID the map reports as internal (root or the proxy user, and never -// an allocated agent UID) is named internalCallerName so a hive-originated -// write is attributed rather than mistaken for a UID-attribution failure. -func (p *GitHubProxy) attributeTransparentConn(conn net.Conn) string { - if p.uidMap == nil { - return "" - } - _, portStr, splitErr := net.SplitHostPort(conn.RemoteAddr().String()) - if splitErr != nil { - return "" - } - port := 0 - const maxPort = 65535 - for _, c := range portStr { - port = port*10 + int(c-'0') - if port > maxPort { - port = 0 - break - } - } - uid, lookupErr := LookupUIDByLocalPort(port) - if lookupErr != nil { - return "" - } - if name := p.uidMap.LookupByUID(uid); name != "" { - return name - } - // The hive's own control plane is not an agent, so the lookup above - // returns "". Under v4's forced-proxy egress that made every - // hive-originated write β€” notably the App installation token mint β€” look - // like an unattributable agent, which the mode gate blocks. Name it - // explicitly so it is attributed rather than mistaken for a - // UID-attribution failure. - if p.uidMap.IsInternalUID(uid) { - return internalCallerName - } - return "" -} - -// dialTunnelUpstream opens the raw TCP leg of a non-inspected transparent -// tunnel. The socket is SO_MARKed so the forced-egress redirect exempts this -// proxy-originated dial. tunnelDial, when set, overrides it (tests). -func (p *GitHubProxy) dialTunnelUpstream(host string) (net.Conn, error) { - if p.tunnelDial != nil { - return p.tunnelDial(host) - } - return markDialer(transparentProxyTimeout).Dial("tcp", host+":443") -} - -const tlsClientHelloMaxSize = 4096 - -// tlsRecordHeaderLen is the size of the TLS record header: content type(1) + -// version(2) + length(2). -const tlsRecordHeaderLen = 5 - -// readClientHelloRecord reads the complete first TLS record (the ClientHello) -// off conn, starting from the already-peeked bytes, and returns the full record -// so the caller can both extract the SNI and replay it verbatim into tls.Server. -// -// The prior implementation did a SINGLE conn.Read after the 1-byte peek and -// assumed it captured the whole ClientHello. TCP makes no such guarantee: when -// the ClientHello arrives split across segments, that single read returns only a -// prefix. extractSNI then parses a truncated ClientHello, fails to find the SNI, -// falls back to the wrong host, and the forged cert no longer matches what the -// agent asked for β€” so the agent-side TLS handshake fails deterministically -// (observed as "bad certificate" or "bad record MAC"), blocking every Copilot -// agent from authenticating. -// -// This reassembles the record using the record-layer length field, reading in a -// loop until the whole ClientHello record is buffered (or the buffer cap / -// deadline is hit), so both SNI extraction and the replay always see an intact -// ClientHello regardless of how TCP fragments the client's bytes. The caller -// sets/clears the read deadline around this call. -func readClientHelloRecord(conn net.Conn, peeked []byte) ([]byte, error) { - buf := make([]byte, tlsClientHelloMaxSize) - n := copy(buf, peeked) - - // First, ensure we have at least the record header so we know the record's - // declared length. - for n < tlsRecordHeaderLen { - r, err := conn.Read(buf[n:]) - n += r - if err != nil { - if n == 0 { - return nil, err - } - // Return what we have; extractSNI degrades gracefully on a short read. - return buf[:n], nil - } - } - - // record length is bytes 3..4 (big-endian); total record size adds the header. - recordLen := int(buf[3])<<8 | int(buf[4]) - need := tlsRecordHeaderLen + recordLen - if need > len(buf) { - // ClientHello larger than our cap: read to fill the buffer. The remainder - // (and subsequent records) are served to tls.Server via prefixConn+conn, - // so the handshake still completes; we only need enough here for SNI. - need = len(buf) - } - - for n < need { - r, err := conn.Read(buf[n:]) - n += r - if err != nil { - if n == 0 { - return nil, err - } - return buf[:n], nil - } - } - return buf[:n], nil -} - -// extractSNI reads the SNI hostname from a TLS ClientHello message. -func extractSNI(data []byte) string { - if len(data) < 5 { - return "" - } - // TLS record: type(1) + version(2) + length(2) + handshake - recordLen := int(data[3])<<8 | int(data[4]) - if len(data) < 5+recordLen { - // Partial read β€” use what we have - recordLen = len(data) - 5 - } - handshake := data[5 : 5+recordLen] - if len(handshake) < 4 { - return "" - } - // Handshake: type(1) + length(3) + ClientHello - hsLen := int(handshake[1])<<16 | int(handshake[2])<<8 | int(handshake[3]) - if len(handshake) < 4+hsLen { - hsLen = len(handshake) - 4 - } - ch := handshake[4 : 4+hsLen] - if len(ch) < 34 { - return "" - } - // Skip: version(2) + random(32) - pos := 34 - // Session ID - if pos >= len(ch) { - return "" - } - sessLen := int(ch[pos]) - pos += 1 + sessLen - // Cipher suites - if pos+2 > len(ch) { - return "" - } - csLen := int(ch[pos])<<8 | int(ch[pos+1]) - pos += 2 + csLen - // Compression methods - if pos >= len(ch) { - return "" - } - cmLen := int(ch[pos]) - pos += 1 + cmLen - // Extensions - if pos+2 > len(ch) { - return "" - } - extLen := int(ch[pos])<<8 | int(ch[pos+1]) - pos += 2 - extEnd := pos + extLen - if extEnd > len(ch) { - extEnd = len(ch) - } - for pos+4 <= extEnd { - extType := int(ch[pos])<<8 | int(ch[pos+1]) - eLen := int(ch[pos+2])<<8 | int(ch[pos+3]) - pos += 4 - if pos+eLen > extEnd { - break - } - if extType == 0 { // SNI extension - sniData := ch[pos : pos+eLen] - if len(sniData) < 2 { - break - } - sniListLen := int(sniData[0])<<8 | int(sniData[1]) - _ = sniListLen - sniPos := 2 - for sniPos+3 <= len(sniData) { - nameType := sniData[sniPos] - nameLen := int(sniData[sniPos+1])<<8 | int(sniData[sniPos+2]) - sniPos += 3 - if sniPos+nameLen > len(sniData) { - break - } - if nameType == 0 { // host_name - return string(sniData[sniPos : sniPos+nameLen]) - } - sniPos += nameLen - } - } - pos += eLen - } - return "" -} - -// prefixConn wraps a net.Conn and prepends already-read bytes to the stream. -type prefixConn struct { - net.Conn - prefix []byte - offset int -} - -func (c *prefixConn) Read(b []byte) (int, error) { - if c.offset < len(c.prefix) { - n := copy(b, c.prefix[c.offset:]) - c.offset += n - return n, nil - } - return c.Conn.Read(b) -} - -// identifyAgentFromReq determines the agent name for a request. It always -// tries UID-based identification first (unforgeable, works for any client -// including native binaries that don't send Proxy-Authorization); see -// fallbackAgentName for what happens when that fails. -func (p *GitHubProxy) identifyAgentFromReq(r *http.Request) string { - if p.uidMap != nil { - if name := p.identifyAgentByUID(r.RemoteAddr); name != "" { - return name - } - } - return p.fallbackAgentName(r) -} - -// IdentifyAgentByUID names the agent behind an http.Server-delivered request -// from the socket owner's UID ALONE. Unlike identifyAgentFromReq it never -// falls back to the caller-controlled Proxy-Authorization header, not even -// under HIVE_PROXY_ADVISORY_OK: see fallbackAgentName for why a self-asserted -// name is impersonation, and for the loopback services beside the proxy (the -// Jev decision endpoint, hivecommons/hive#8939) there is no "degraded but -// functional" mode worth that trade β€” an unidentified caller is refused. -// Returns "" when no UID map was loaded or the connection's UID is unmapped. -func (p *GitHubProxy) IdentifyAgentByUID(r *http.Request) string { - if p.uidMap == nil || r == nil { - return "" - } - return p.identifyAgentByUID(r.RemoteAddr) -} - -// identifyAgentFromConn identifies the calling agent for a request read off a -// raw connection. It MUST be used instead of identifyAgentFromReq on any path -// where the request came from http.ReadRequest rather than http.Server: -// ReadRequest does not populate Request.RemoteAddr (only http.Server does), so -// the UID lookup silently gets an empty address, finds no agent, and the caller -// falls back to ADVISORY β€” downgrading every write-capable agent to read-only. -// Taking the peer address from the connection itself removes that trap. -func (p *GitHubProxy) identifyAgentFromConn(conn net.Conn, r *http.Request) string { - if p.uidMap != nil && conn != nil { - if name := p.identifyAgentByUID(conn.RemoteAddr().String()); name != "" { - return name - } - } - return p.fallbackAgentName(r) -} - -// fallbackAgentName is reached whenever unforgeable UID-based identification -// did NOT succeed β€” either p.uidMap is nil (never loaded: no entrypoint root -// phase ran to write uid-map.json, e.g. local dev, a native/systemd install, -// or a genuine boot-ordering failure) or this specific connection's UID has -// no entry in a map that DID load (N7, #3841). -// -// Before this fix, both cases fell through to extractAgentName β€” the -// Proxy-Authorization header, which the CALLER sets. Any process that can -// reach the proxy's loopback port can send `Proxy-Authorization: hive -// ` and be identified as that name: a spoke-agent process claiming -// to be a MORE privileged agent bypasses the whole capability model this -// proxy exists to enforce, and #1861 (inject GitHub auth at the proxy so -// agents never hold tokens) would turn that spoofed identity directly into a -// minted token for the impersonated agent. That is strictly worse than -// simply refusing to identify the caller. -// -// The safe default is therefore to NOT trust the header: return "", which -// readAgentMode already treats as ModeAdvisory (the most restrictive mode β€” -// writes blocked, per the existing "agent could not be identified" WARN a few -// lines below in proxyHTTP). This is not new plumbing; it is the same path an -// unidentified caller already hits today, now reached deliberately instead of -// only when the header happens to be absent. -// -// The one deliberate exception is proxyAdvisoryOK (HIVE_PROXY_ADVISORY_OK=true) -// β€” the SAME explicit, operator-set escape hatch entrypoint.sh already uses -// for "this deployment does not enforce the full forced-egress gate" (local -// dev, a native/systemd install with no per-agent UID separation at all, -// platforms without NET_ADMIN). An operator who has already accepted a -// degraded capability model there is not newly exposed by also accepting a -// self-asserted identity within it β€” and without this exception, those -// deployments would lose agent identification (and therefore write access) -// entirely, which would be a functional regression, not a security fix. -func (p *GitHubProxy) fallbackAgentName(r *http.Request) string { - if !p.proxyAdvisoryOK { - return "" - } - return extractAgentName(r) -} - -// identifyAgentByUID reads /proc/net/tcp to find the UID of the process -// that owns the socket connected to the proxy, then looks up the agent name. -func (p *GitHubProxy) identifyAgentByUID(remoteAddr string) string { - uid, err := uidForRemoteAddr(remoteAddr) - if err != nil { - return "" - } - return p.uidMap.LookupByUID(uid) -} - -// uidForRemoteAddr resolves the UID of the process that owns the local socket -// behind remoteAddr ("host:port" as http.Server and net.Conn report it) via -// /proc/net/tcp. It is the shared first half of every UID-attribution path. -func uidForRemoteAddr(remoteAddr string) (int, error) { - _, portStr, err := net.SplitHostPort(remoteAddr) - if err != nil { - return -1, err - } - port := 0 - const maxPort = 65535 - for _, c := range portStr { - if c < '0' || c > '9' { - return -1, fmt.Errorf("remote port %q is not numeric", portStr) - } - port = port*10 + int(c-'0') - if port > maxPort { - return -1, fmt.Errorf("remote port %q out of range", portStr) - } - } - return LookupUIDByLocalPort(port) -} - -// inferenceAPIKeyPrefix is what the manager prepends to the agent's name to -// form the ANTHROPIC_API_KEY it hands each inference-routed agent -// (agent/manager_env.go). The suffix is the agent's NAME, not a secret. -const inferenceAPIKeyPrefix = "sk-hive-" - -// translatorAgentName names the caller of the inference translator. -// -// The x-api-key header the Claude CLI sends is "sk-hive-", and the -// agent name is public inside the pod: every agent knows every other agent's -// name. Trusting the header alone lets any process on the loopback borrow -// another agent's inference route β€” its backend, model, upstream credential -// and usage attribution β€” which is exactly the self-asserted-identity hole -// closed for the GitHub path in fallbackAgentName and for the decision -// endpoint in IdentifyAgentByUID. So the socket owner's UID decides: -// -// - a UID mapped to an agent is authoritative; a header naming someone else -// is logged and overridden; -// - the root control plane keeps the header name β€” the hive is the party -// that issues these keys; the proxy UID cannot assert a name because its -// upstream sockets can relay agent-controlled requests; -// - an unmapped UID, or no UID map at all, accepts the header only under -// HIVE_PROXY_ADVISORY_OK, the same operator escape hatch the GitHub path -// honours for deployments without per-agent UID separation. -// -// The second return is false when the caller could not be identified and the -// request must be refused. -func (p *GitHubProxy) translatorAgentName(r *http.Request) (string, bool) { - claimed := strings.TrimPrefix(r.Header.Get("x-api-key"), inferenceAPIKeyPrefix) - if p.uidMap == nil { - if p.proxyAdvisoryOK { - return claimed, true - } - return "", false - } - uid, err := uidForRemoteAddr(r.RemoteAddr) - if err == nil { - if name := p.uidMap.LookupByUID(uid); name != "" { - if claimed != "" && claimed != name { - p.logger.Warn("inference identity: x-api-key names another agent; using the socket owner", - "agent", name, "claimed", claimed, "uid", uid) - } - return name, true - } - if uid == 0 && p.uidMap.IsInternalUID(uid) { - return claimed, true - } - } - if p.proxyAdvisoryOK { - return claimed, true - } - return "", false -} - -// handleConnectDirect handles CONNECT requests on a raw connection (no http.Server). -func (p *GitHubProxy) handleConnectDirect(conn net.Conn, r *http.Request) { - host, _, err := net.SplitHostPort(r.Host) - if err != nil { - host = r.Host - } - - // Identify from the connection, NOT from r.RemoteAddr: this request came - // from http.ReadRequest, which leaves RemoteAddr empty. - agentName := p.identifyAgentFromConn(conn, r) - - // Anthropic hosts with an inference route: reroute to self-hosted backend. - if IsAnthropicHost(host) { - if route := p.inference.Get(agentName); route != nil { - p.handleAnthropicReroute(conn, r, host, agentName, route) - return - } - } - - // Copilot completion host: MITM to read live token usage. Only when a token - // sink is active and the agent is identified β€” otherwise fall through to an - // opaque tunnel so Copilot traffic is never broken by usage capture. - if IsCopilotAPIHost(host) && p.tokenSink != nil && agentName != "" { - p.handleCopilotSniff(conn, host, agentName) - return - } - - // Hosts we do not inspect: tunnel without interception. github.com is in - // this set β€” its OAuth device flow and git smart HTTP are handled by CLI - // --deny-tool flags. api.github.com and api.linear.app are not: both need - // request-level inspection for ACMM enforcement. Under #1861 injection, - // however, every GitHub-family host is intercepted (see hostNeedsMITM), - // because the agent's credential helper now serves an inert placeholder - // that only the proxy can replace with the real scoped token. - if !NeedsInspection(host) && !p.hostNeedsMITMFor(host, agentName) { - p.tunnelDirect(conn, r) - return - } - - mode := readAgentMode(agentName) - caps := readAgentCaps(agentName) - - // Tell client the tunnel is established. - if _, err := fmt.Fprintf(conn, "HTTP/1.1 200 Connection established\r\n\r\n"); err != nil { - p.logger.Error("proxy CONNECT response write failed", "error", err) - return - } - - // Generate a cert for the target host signed by our CA. - tlsCert, err := p.forgeCert(host) - if err != nil { - p.logger.Error("proxy forge cert failed", "host", host, "error", err) - return - } - - // TLS handshake with client (presenting our forged cert). - tlsClientConn := tls.Server(conn, &tls.Config{ - Certificates: []tls.Certificate{tlsCert}, - }) - // Bound the client TLS handshake (see handleTransparentTLS): a client that - // stops mid-handshake must not wedge this connection forever. - _ = conn.SetDeadline(time.Now().Add(clientTLSHandshakeTimeout)) - handshakeErr := tlsClientConn.Handshake() - _ = conn.SetDeadline(time.Time{}) - if handshakeErr != nil { - if isClientHangup(handshakeErr) { - p.logger.Debug("proxy client hung up before TLS handshake", "error", handshakeErr) - } else { - p.logger.Warn("proxy client TLS handshake failed", "error", handshakeErr) - } - return - } - defer func() { _ = tlsClientConn.Close() }() - - // Connect to the real GitHub server. SO_MARK the socket so the forced-egress - // redirect exempts this proxy-originated dial (see proxyEgressMark). - // A dialer Timeout bounds both the TCP connect and the TLS handshake β€” - // without it a stalled upstream wedges this CONNECT forever (the agent's - // curl sits in poll() waiting for "200 Connection established" that never - // comes). The deadline is lifted once established so the relay is not - // time-boxed. - upstreamConn, err := tls.DialWithDialer(markDialer(upstreamDialTimeout), "tcp", r.Host, &tls.Config{ - ServerName: host, - }) - if err != nil { - p.logger.Error("proxy upstream dial failed", "host", r.Host, "error", err) - return - } - defer func() { _ = upstreamConn.Close() }() - - // Proxy HTTP requests, inspecting each one. - p.proxyHTTPHost(tlsClientConn, upstreamConn, host, agentName, mode, caps) -} - -// proxyHTTP reads HTTP requests from the client, checks them against -// mode rules, and either forwards or blocks them. -func isTimeoutError(err error) bool { - var netErr net.Error - return errors.As(err, &netErr) && netErr.Timeout() -} - -// logUpstreamEvent logs an upstream timeout event at the given level. -func (p *GitHubProxy) logUpstreamEvent(level slog.Level, msg string, err error, attrs ...any) { - if !isTimeoutError(err) { - return - } - attrs = append(attrs, "error", err) - p.logger.Log(context.Background(), level, msg, attrs...) -} - -func (p *GitHubProxy) logTimeout(msg string, err error, attrs ...any) { - p.logUpstreamEvent(slog.LevelWarn, msg, err, attrs...) -} - -// isClientHangup reports a client that went away (EOF, reset, broken pipe) or -// stalled past the deadline: routine on restarts and liveness probes, not a -// hive defect. -func isClientHangup(err error) bool { - return isTimeoutError(err) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || - errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.EPIPE) || errors.Is(err, net.ErrClosed) -} - -// logClientRead logs a client-side request read failure at Debug: an idle or -// departed client is expected, unlike an upstream stall. -func (p *GitHubProxy) logClientRead(msg string, err error, attrs ...any) { - p.logUpstreamEvent(slog.LevelDebug, msg, err, attrs...) -} - -// proxyHTTP relays a MITM'd GitHub connection with ACMM enforcement applied. -// It is proxyHTTPHost with the GitHub host implied, which is what every -// pre-existing caller means. -func (p *GitHubProxy) proxyHTTP(client net.Conn, upstream net.Conn, agentName string, mode agent.AgentMode, caps agent.AgentCapabilities) { - p.proxyHTTPHost(client, upstream, "", agentName, mode, caps) -} - -// proxyHTTPHost is proxyHTTP with the terminated host carried through, so the -// request loop can pick the right rule engine (#4492 F). -// -// The host is threaded rather than inferred from the request because a MITM'd -// request line is a path, not an absolute URL β€” by this point the only record -// of which host the TLS session was forged for is the CONNECT that opened it. -// An empty host means GitHub, preserving every existing call site verbatim. -func (p *GitHubProxy) proxyHTTPHost(client net.Conn, upstream net.Conn, host string, agentName string, mode agent.AgentMode, caps agent.AgentCapabilities) { - isLinear := IsLinearHost(host) - clientBuf := newBufferedConn(client) - - for { - // Bound every phase of this request/response exchange. Before this, - // the relay had NO deadlines anywhere after the initial CONNECT read: - // a client or upstream that stopped mid-exchange (stalled response, - // half-written body, frozen keep-alive) blocked forever β€” the - // multi-hour "egress wedge" where the agent's request never reached - // GitHub and its session froze. Each phase gets its own deadline; - // they are cleared once the phase completes so healthy keep-alive - // connections are unaffected. - _ = client.SetReadDeadline(time.Now().Add(httpReadTimeout)) - req, err := http.ReadRequest(clientBuf) - if err != nil { - _ = client.SetReadDeadline(time.Time{}) - p.logClientRead("proxy client request read timed out", err, "agent", agentName) - return - } - - blocked := false - blockReason := "" - // blockCode is the machine-readable reason (#12127) that rides on the - // one-line 403 body and the X-Hive-Proxy-Reason header. Each gate that - // sets blockReason sets it; a gate that sets neither falls back to a - // generic code in ProxyDenialBody. - blockCode := "" - - // Per-repo pause (#6203) and per-repo agent scope (#6204) are both - // checked BEFORE the mode chain, and outside it, for the same two - // reasons. Neither is an autonomy tier β€” a pause is a run-state, a - // scope is roster membership β€” so each must refuse the write whatever - // mode the agent holds and whichever branch below would otherwise have - // allowed it. And each carries its own reason, so the agent is told the - // repo is deliberately quiet, or that it is not defined for this repo, - // instead of reading a mode error and going hunting for a permissions - // bug that does not exist. - // - // The hive's own control-plane traffic (App-token mint, heartbeat) is - // exempt from both, exactly as it is from the ACMM rules below: - // stopping the hive from minting a token would take the whole spoke - // down rather than quiet one repo or narrow one agent. - // - // Pause is checked first, so a repo the operator has frozen is refused - // as paused even when the agent is also out of scope for it. - blockJSON := false - graphQLMutation := false - // A direct write from a lane under write_surface.enforce (#9772) is - // refused and audited as agent_write_refused. Checked after pause and - // scope so their more specific reasons win when both apply. - writeSurfaceKind := "" - if agentName != internalCallerName { - if reason, paused := p.repoPauseRefusal(req.Method, repoScopePath(req)); paused { - blocked = true - blockReason = reason - blockCode = DenyReasonRepoPaused - } else if reason, refused := p.agentRepoScopeRefusal(agentName, req.Method, repoScopePath(req)); refused { - blocked = true - blockReason = reason - blockCode = DenyReasonAgentRepoScope - } else if reason, refused := p.autoMergeRefusal(req.Method, repoScopePath(req)); refused { - blocked = true - blockReason = reason - blockCode = DenyReasonRepoAutoMergeOff - blockJSON = true - } else if !isLinear { - if reason, kind, refused := WriteSurfaceEnforceRefusal(p.writeSurfaceEnforcedFunc(), agentName, req.Method, req.URL.Path, req.URL.RawQuery); refused { - blocked = true - blockReason = reason - blockCode = DenyReasonWriteSurfaceEnforced - writeSurfaceKind = kind - } - } - } - - if blocked { - // Already refused by the pause or agent-scope check above β€” the - // mode chain cannot un-block it, so skip it entirely. - } else if isLinear && agentName == internalCallerName { - // The hive's OWN Linear traffic β€” the read path #4178 shipped - // (backlog enumeration in pkg/worksource/linear.go) and, later, - // the OAuth/webhook plumbing. ACMM governs AGENT autonomy, so the - // control plane is exempt here exactly as it is for GitHub below. - // This branch must precede the enforcement one: without it, adding - // Linear to NeedsMITM would newly gate hive-originated reads that - // have always been ungated, breaking enumeration rather than - // restricting an agent. - } else if isLinear { - // Linear is a single POST /graphql for everything, so the - // (method, path) rule table cannot classify it and the GitHub - // GraphQL classifier is the wrong posture β€” it ends in a - // permissive default, backed by the REST table. Linear has no - // table underneath. See linear_rules.go. - blocked, blockReason = p.enforceLinear(req, agentName, mode, caps) - if blocked { - blockCode = DenyReasonLinear - } - } else if req.Method == "POST" && IsGraphQLPath(req.URL.Path) { - body, readErr := io.ReadAll(io.LimitReader(req.Body, graphQLBodyLimit)) - if req.Body != nil { - _ = req.Body.Close() - } - if readErr != nil { - _ = client.SetReadDeadline(time.Time{}) - p.logTimeout("proxy GraphQL request body read timed out", readErr, "agent", agentName, "path", req.URL.Path) - return - } - if reason, refused := p.graphQLAutoMergeRefusal(body); agentName != internalCallerName && refused { - blocked = true - blockReason = reason - blockCode = DenyReasonRepoAutoMergeOff - blockJSON = true - } else if agentName != internalCallerName { - if reason, refused := WriteSurfaceGraphQLRefusal(p.writeSurfaceEnforcedFunc(), agentName, body); refused { - blocked = true - blockReason = reason - blockCode = DenyReasonWriteSurfaceEnforced - writeSurfaceKind = WriteSurfaceKindGraphQL - } - } - allowed, isMutation := GraphQLAllowedCaps(autonomyGraphQLMode(agentName, body, mode), caps, body) - graphQLMutation = isMutation - if !blocked && !allowed { - blocked = true - if isMutation { - blockReason = "graphql mutation" - blockCode = DenyReasonGraphQLMutation - } else { - blockReason = "graphql" - blockCode = DenyReasonGraphQL - } - } - req.Body = io.NopCloser(strings.NewReader(string(body))) - req.ContentLength = int64(len(body)) - } else if agentName == internalCallerName { - // The hive itself has no ACMM mode to enforce β€” ACMM governs AGENT - // autonomy. Its control-plane calls (App token mint, heartbeat) - // must not be gated as if they were agent writes. Repo-filter and - // canary-egress checks below still apply. - } else if !AllowedByModeCaps(autonomyModeForRepo(agentName, ExtractRepo(repoScopePath(req)), mode), caps, req.Method, req.URL.Path) { - blocked = true - blockCode = DenyReasonMode - // An unidentified agent is silently treated as ADVISORY, which turns - // a permissions bug into an indistinguishable "policy denial". Say so - // loudly on writes: a write from an unknown caller means UID - // attribution failed, not that the agent lacks the mode. - if agentName == "" && writeMethods[req.Method] { - p.logger.Warn("proxy: agent could not be identified β€” defaulting to ADVISORY and blocking a write; UID attribution failed (check the uid map and /proc/net/tcp visibility), this is NOT an agent-mode misconfiguration", - "method", req.Method, "path", req.URL.Path) - } - // A hard-deny rule (e.g. direct PR creation) carries an agent-facing - // directive explaining the sanctioned alternative β€” surface it so the - // agent knows to use hive-open-pr rather than seeing only a mode error. - if msg, denied := DeniedMessage(req.Method, req.URL.Path); denied && msg != "" { - blockReason = msg - blockCode = DenyReasonHardDeny - } - } else if len(p.allowedRepos) > 0 && !RepoFilterAllowed(p.allowedRepos, req.Method, repoScopePath(req)) { - blocked = true - blockReason = "repo not in hive config" - blockCode = DenyReasonRepoNotInConfig - } - // Shape-gate agent-authored issue/comment REST writes. The issue-request - // watcher validates the file-drop creation path, but an agent's - // `gh issue create` becomes a raw POST /repos/{o}/{r}/issues that never - // touches the watcher (issue #7014: a template placeholder title and a - // body of literal \n escapes both reached GitHub this way). Enforce the - // same shape rules here so both creation paths are covered. Skip the - // control plane β€” its issue writes are already validated upstream. - if !blocked && agentName != internalCallerName { - if reason, deny, shapeErr := p.enforceIssueShapeFor(req, agentName); shapeErr != nil { - // Fail closed: a body we could not read is a body we could not - // shape-check. - p.logTimeout("proxy issue-shape body read timed out", shapeErr, "agent", agentName, "path", req.URL.Path) - _ = client.SetReadDeadline(time.Time{}) - return - } else if deny { - blocked = true - if blockReason == "" { - blockReason = reason - blockCode = DenyReasonIssueShape - } - } - } - if reason, deny, ok := p.inspectCanaryEgress(agentName, req); ok { - if deny { - blocked = true - if blockReason == "" { - blockReason = reason - blockCode = DenyReasonCanaryEgress - } - } - } - - if !isLinear && agentName != internalCallerName { - isGraphQL := req.Method == "POST" && IsGraphQLPath(req.URL.Path) - isWrite := writeMethods[req.Method] && !gitUploadPackPath.MatchString(req.URL.Path) - if isGraphQL { - isWrite = graphQLMutation - } - if isWrite { - reason := "" - if blocked { - reason = blockCode - if reason == "" { - reason = DenyReasonUnknown - } - } - p.writes.record(mode.String(), reason) - } - } - - if blocked { - detail := req.URL.Path - if blockReason != "" { - detail = blockReason - } - p.recordViolation(agentName, req.Method, detail) - if writeSurfaceKind != "" { - p.auditWriteRefused(agentName, writeSurfaceKind, req.Method, repoScopePath(req)) - } - - if blockCode == "" { - blockCode = DenyReasonUnknown - } - respBody := ProxyDenialBody(agentName, mode, req.Method, detail, blockCode) - contentType := "text/plain" - if blockJSON { - contentType = "application/json" - payload, _ := json.Marshal(map[string]any{ - "error": "repo_auto_merge_disabled", - "reason": detail, - }) - respBody = string(payload) + "\n" - } - resp := &http.Response{ - StatusCode: http.StatusForbidden, - Proto: "HTTP/1.1", - ProtoMajor: 1, - ProtoMinor: 1, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(respBody)), - } - resp.Header.Set("Content-Type", contentType) - resp.Header.Set("X-Hive-Proxy-Blocked", "true") - resp.Header.Set(ProxyDenialHeader, blockCode) - _ = client.SetWriteDeadline(time.Now().Add(httpWriteTimeout)) - writeErr := resp.Write(client) - _ = client.SetWriteDeadline(time.Time{}) - if writeErr != nil { - p.logTimeout("proxy blocked response write timed out", writeErr, "agent", agentName) - return - } - - if req.Body != nil { - if _, drainErr := io.Copy(io.Discard, req.Body); drainErr != nil { - p.logTimeout("proxy blocked request body drain timed out", drainErr, "agent", agentName, "path", req.URL.Path) - } - _ = req.Body.Close() - } - _ = client.SetReadDeadline(time.Time{}) - continue - } - - if !isLinear { - if refusal := p.githubBudgetRefusal(agentName, req.Method, req.URL.Path, agentGitHubRequestIsRead(req.Method, req.URL.Path, graphQLMutation)); refusal != nil { - _ = p.writeAgentGitHubBudget429(client, *refusal) - if req.Body != nil { - _, _ = io.Copy(io.Discard, req.Body) - _ = req.Body.Close() - } - _ = client.SetReadDeadline(time.Time{}) - continue - } - } - - // #1861: with injection enabled, the upstream credential is decided - // HERE β€” after the ACMM/repo/canary gates, immediately before either - // forwarding branch β€” so every byte that reaches GitHub has passed - // through it. Blocked requests above never reach upstream and need no - // rewrite. - // - // #9586: only for GitHub hosts. This loop also relays api.linear.app - // (NeedsInspection), and rewriting there would attach the agent's - // hub-held GITHUB token to a request bound for Linear - a real - // credential handed to a third party - whenever the Linear lane below - // does not overwrite the header (an ADVISORY agent, or a hive with no - // Linear credential). - if injectsGitHubAuthForHost(host) { - p.rewriteGitHubAuth(req, agentName) - } - - // Linear: the request passed the tier gate; make sure it reaches - // Linear with the hive's CURRENT credential rather than whatever - // (possibly rotated-out) copy the agent's process still holds. - if isLinear && agentName != internalCallerName { - p.injectLinearCredential(req, agentName, mode) - } - // Git smart HTTP uses chunked streaming that http.ReadResponse - // can't handle reliably. After the ACMM check passes, forward - // the request and switch to raw bidirectional streaming. - if isGitPath(req.URL.Path) { - _ = upstream.SetWriteDeadline(time.Now().Add(httpWriteTimeout)) - if err := req.Write(upstream); err != nil { - _ = upstream.SetWriteDeadline(time.Time{}) - _ = client.SetReadDeadline(time.Time{}) - p.logTimeout("proxy git request relay timed out", err, "agent", agentName, "path", req.URL.Path) - return - } - _ = upstream.SetWriteDeadline(time.Time{}) - _ = client.SetReadDeadline(time.Time{}) - // Use relayTunnel, NOT an inline copy pair: this branch used to run - // its own unbounded io.Copy pair, which was the same wedge #3872 - // fixed in relayTunnel but left here. When GitHub FIN'd after the - // response while the client kept its side open (keep-alive), the - // clientβ†’upstream copy stayed blocked in client.Read() forever and - // `<-done` never returned β€” the handler's deferred Closes never ran, - // parking the upstream socket in CLOSE_WAIT for the life of the - // process (issue #3875's FD pile). relayTunnel bounds each direction - // once the other finishes. - relayTunnel(client, upstream) - return - } - - // Forward to upstream. The client read deadline stays in force until - // the body is fully drained (req.Write pipes req.Body from the client), - // so a client stalling mid-body cannot hang the relay. - _ = upstream.SetWriteDeadline(time.Now().Add(httpWriteTimeout)) - if err := req.Write(upstream); err != nil { - _ = upstream.SetWriteDeadline(time.Time{}) - _ = client.SetReadDeadline(time.Time{}) - p.logTimeout("proxy request relay timed out", err, "agent", agentName, "path", req.URL.Path) - return - } - _ = upstream.SetWriteDeadline(time.Time{}) - _ = client.SetReadDeadline(time.Time{}) - - _ = upstream.SetReadDeadline(time.Now().Add(httpReadTimeout)) - resp, err := http.ReadResponse(newBufferedReader(upstream), req) - _ = upstream.SetReadDeadline(time.Time{}) - if err != nil { - p.logTimeout("proxy upstream response read timed out", err, "agent", agentName, "path", req.URL.Path) - return - } - p.recordGitHubREST(agentName, req, resp) - p.recordAgentGitHubBudget(agentName, req.URL.Path, resp.StatusCode, p.agentGitHubCap(agentName)) - - // Bound the BODY relay too, not just the header read above. The - // deadline was cleared once headers arrived, so a "reachable but slow" - // GitHub β€” TLS and headers complete, body never arrives (the measured - // #3875 signature: time_total=12s, 0 body bytes) β€” parked resp.Write - // in upstream.Read() with no deadline. The client had long since given - // up (10s collect budget) and FIN'd its side, but this goroutine never - // noticed: it holds all three sockets of the exchange forever, and the - // retry loop stacks a fresh parked handler on every attempt (~1k - // FDs/min measured live). The wrapper re-arms a rolling per-Read - // deadline, so a body that keeps flowing is never cut while a stall - // longer than responseBodyStallTimeout errors out and lets the - // handler's deferred Closes reclaim the sockets. - resp.Body = &stallBoundedBody{body: resp.Body, conn: upstream, idle: p.bodyStallTimeoutDuration()} - - // Rolling per-Write deadline (see stallBoundedWriter): a response that - // keeps flowing is never cut for being long, while a client that stops - // draining still releases this handler within one stall window. - if err := resp.Write(&stallBoundedWriter{conn: client, idle: httpWriteTimeout}); err != nil { - _ = resp.Body.Close() - _ = client.SetWriteDeadline(time.Time{}) - p.logTimeout("proxy response relay timed out", err, "agent", agentName, "path", req.URL.Path) - return - } - _ = client.SetWriteDeadline(time.Time{}) - _ = resp.Body.Close() - } -} - -// gitInjectBasicUser is the username git expects alongside a GitHub App -// installation token in HTTP Basic auth β€” the same identity -// git-credential-hive.sh answers with, so upstream sees exactly the shape the -// pre-injection flow produced. -const gitInjectBasicUser = "x-access-token" - -// loginPathPrefix marks the GitHub OAuth/device-flow endpoints -// (/login/device/code, /login/oauth/access_token). These authenticate the -// FLOW via their form body, not a bearer credential; injecting an App token -// there would be meaningless at best and flow-corrupting at worst, so they -// get strip-only treatment. -const loginPathPrefix = "/login/" - -// copilotInternalPathPrefix marks the Copilot CLI's auth-exchange endpoints on -// the GitHub API host (/copilot_internal/v2/token, /copilot_internal/user). -// The CLI calls them with the USER's Copilot OAuth token -// (COPILOT_GITHUB_TOKEN / the /data/copilot-user-token device-flow token) to -// obtain its short-lived Copilot session token and discover its completion -// host. A GitHub App installation token cannot perform that exchange, so -// rewriting these requests would cut every copilot-backend agent off from its -// model the moment injection is on (#9586). They are passed through with the -// agent's own Authorization untouched - nothing stripped, nothing injected. -// -// Residual (documented in security-model.md): the Copilot user OAuth token -// stays agent-readable and spendable on these paths; moving it server-side too -// is a follow-up. The hub-held App tokens injection protects are never -// attached here. -const copilotInternalPathPrefix = "/copilot_internal/" - -// gheAPIPathPrefix is the REST prefix GitHub Enterprise Server serves its API -// under (https:///api/v3/...), so the Copilot exchange on a GHE host -// arrives as /api/v3/copilot_internal/... -const gheAPIPathPrefix = "/api/v3" - -// isCopilotAuthExchangePath reports whether path is a Copilot CLI -// auth-exchange endpoint that must keep the agent's own credential (see -// copilotInternalPathPrefix), on api.github.com or a GHE /api/v3 host. -func isCopilotAuthExchangePath(path string) bool { - return strings.HasPrefix(path, copilotInternalPathPrefix) || - strings.HasPrefix(path, gheAPIPathPrefix+copilotInternalPathPrefix) -} - -// rewriteGitHubAuth enforces #1861 on one MITM'd request: the proxy β€” not the -// agent β€” decides what credential GitHub sees. -// -// Threat model, per header-touching block: -// -// - STRIP: an agent-supplied Authorization header is deleted UNCONDITIONALLY -// (under the flag), never forwarded. Even though injection-mode agents are -// only ever handed the inert placeholder, an agent that somehow obtained a -// real credential (leaked from a log, smuggled via prompt, minted through -// a side channel) must not be able to spend it through this proxy. -// -// - INJECT: only for an agent identified by the UID-authoritative path -// (#3888) that has a hub-held scoped token. The two lookups this chains -// are exactly the pre-existing lanes: identity from the UID map, token -// from the #3967 per-agent scoped mint. -// -// - UNKNOWN AGENT = NO CREDENTIAL: when identity resolution failed -// (agentName == ""), the request proceeds with NO Authorization and fails -// loud at GitHub (401/403). Deliberate: any fallback β€” shared cache, -// hive token, "last known agent" β€” would let an unattributable process -// ride a real credential, recreating the pre-#3888 identity hole this -// design depends on having closed. -// -// - COPILOT AUTH-EXCHANGE PASSTHROUGH: /copilot_internal/ on a GitHub -// host keeps the agent's own Authorization (the user's Copilot OAuth -// token); see copilotInternalPathPrefix for why and the residual. -// -// - INTERNAL CALLER PASSTHROUGH: the hive's own control plane -// (internalCallerName) legitimately holds and sends its own App -// credentials (token mint, heartbeat, hive-open-pr fulfillment); its -// requests pass untouched. This cannot be spoofed by an agent: the name -// is assigned only via UIDMap.IsInternalUID, never from anything the -// caller sends. -// -// Logging: agent name and injected yes/no only β€” NEVER token bytes, not even -// a prefix (#1861 requirement; a token prefix is enough to fingerprint and -// correlate credentials across logs). -func (p *GitHubProxy) rewriteGitHubAuth(req *http.Request, agentName string) { - // NOTE: callers must gate on injectsGitHubAuthForHost first; this function - // does not know which host the request is bound for. - if !p.injectGHAuth { - // Flag off (the default): behavior byte-identical to before #1861 β€” - // no header is touched on any path. - return - } - if agentName == internalCallerName { - return - } - if isCopilotAuthExchangePath(req.URL.Path) { - // Before the strip: the Copilot session-token exchange needs the - // user's Copilot OAuth token, which no hub-held App token can stand in - // for (copilotInternalPathPrefix). - p.logger.Debug("proxy auth injection: Copilot auth-exchange endpoint - agent credential passed through", "agent", agentName, "injected", false) - return - } - - req.Header.Del("Authorization") - - if agentName == "" { - p.logger.Debug("proxy auth injection: caller unidentified β€” forwarding without credentials (fail-loud, no fallback)", "injected", false) - return - } - if strings.HasPrefix(req.URL.Path, loginPathPrefix) { - p.logger.Debug("proxy auth injection: OAuth flow endpoint β€” strip only", "agent", agentName, "injected", false) - return - } - if p.agentTokenSource == nil { - p.logger.Warn("proxy auth injection enabled but no token source wired β€” forwarding without credentials", "agent", agentName, "injected", false) - return - } - token, ok := p.agentTokenSource(agentName) - if !ok { - // An identified agent with no hub-held token means the #3967 mint lane - // has not (yet) delivered for this agent β€” e.g. the hive restarted and - // the in-memory registry is empty until the next launch/refresh mint. - // Fail loud (upstream 401) rather than borrow any other credential. - p.logger.Warn("proxy auth injection: no hub-held token for identified agent β€” forwarding without credentials (check the per-agent mint lane)", "agent", agentName, "injected", false) - return - } - - if isGitPath(req.URL.Path) { - // Git smart HTTP authenticates with Basic x-access-token: β€” - // the same shape git-credential-hive.sh produced pre-injection. - basic := base64.StdEncoding.EncodeToString([]byte(gitInjectBasicUser + ":" + token)) - req.Header.Set("Authorization", "Basic "+basic) - // Force this exchange onto its own upstream connection. The git branch - // in proxyHTTP switches to a raw relay after forwarding this request's - // headers, so a keep-alive reuse (git's follow-up POST after GET - // /info/refs on the same conn) would stream its headers β€” placeholder - // credential included β€” straight through WITHOUT passing this rewrite, - // and fail with a confusing mid-operation 401. Connection: close makes - // git reconnect per request; each new connection re-enters proxyHTTP - // and gets its own rewrite. - req.Close = true - } else { - // REST/GraphQL (api.github.com, GHE /api/v3): the standard GitHub - // token scheme, identical to what gh sends. - req.Header.Set("Authorization", "token "+token) - } - p.logger.Debug("proxy auth injection: scoped token attached", "agent", agentName, "injected", true) -} - -// injectsGitHubAuthForHost reports whether a request relayed for host may have -// its Authorization rewritten by rewriteGitHubAuth (#9586). It is a positive -// allowlist - the empty host (proxyHTTP's "GitHub implied" convention) and the -// registered GitHub-family hosts - rather than "anything but Linear", so a -// non-GitHub host added to the MITM set later can never receive a GitHub token -// by default. -func injectsGitHubAuthForHost(host string) bool { - return host == "" || IsGitHubHost(host) -} - -func (p *GitHubProxy) inspectCanaryEgress(agentName string, req *http.Request) (reason string, deny bool, detected bool) { - if p == nil || !p.canariesEnabled || p.canaryRegistry == nil || req == nil || req.Body == nil { - return "", false, false - } - if isGitReceivePack(req.URL.Path) && p.canaryFailClosed { - return "ioscan canary scan unavailable for git push", true, true - } - if !writeMethods[req.Method] || isGitPath(req.URL.Path) || !githubWriteBodyMayLeak(req.Method, req.URL.Path) { - return "", false, false - } - body, err := io.ReadAll(io.LimitReader(req.Body, maxGitHubWriteBodyScan+1)) - if req.Body != nil { - _ = req.Body.Close() - } - if err != nil { - return "", false, false - } - if len(body) > maxGitHubWriteBodyScan { - return "ioscan canary scan body too large", true, true - } - req.Body = io.NopCloser(bytes.NewReader(body)) - req.ContentLength = int64(len(body)) - if leak, ok := p.canaryRegistry.Scan(agentName, string(body), "github:"+req.Method+" "+req.URL.Path); ok { - if p.canaryLeakFunc != nil { - p.canaryLeakFunc(leak) - } - p.logger.Warn("ioscan canary leak detected", "agent", leak.Agent, "source", leak.Source) - return "ioscan canary leak", p.canaryFailClosed, true - } - return "", false, false -} - -// enforceLinear applies ACMM to one request on a MITM'd api.linear.app -// connection and returns whether it is blocked, with an agent-facing reason. -// -// The request body is read (bounded), inspected, and restored so an allowed -// request forwards byte-identically. -// -// NOTHING from the body is logged or surfaced in the 403. Linear documents -// carry issue and comment content in the query and, in variables, potentially -// tokens; the operation NAMES are schema identifiers rather than user content, -// so those are what makes a denial debuggable without leaking the payload. -func (p *GitHubProxy) enforceLinear(req *http.Request, agentName string, mode agent.AgentMode, caps agent.AgentCapabilities) (bool, string) { - // Linear exposes exactly one endpoint. Anything else on this host is not a - // shape this gate understands, so it is refused rather than forwarded. - if req.Method != http.MethodPost || !IsLinearGraphQLPath(req.URL.Path) { - return true, "linear: only POST " + linearGraphQLPath + " is permitted" - } - - // Read one byte past the limit so hitting the cap is detectable: a body - // that exactly fills the buffer is indistinguishable from a truncated one - // otherwise, and truncation must deny. - body, readErr := io.ReadAll(io.LimitReader(req.Body, linearBodyLimit+1)) - if req.Body != nil { - _ = req.Body.Close() - } - truncated := len(body) > linearBodyLimit - if truncated { - body = body[:linearBodyLimit] - } - req.Body = io.NopCloser(bytes.NewReader(body)) - req.ContentLength = int64(len(body)) - - if readErr != nil { - // Fail closed: a body we could not fully read is a body we could not - // classify. - p.logger.Warn("linear: request body read failed β€” denying", "agent", agentName) - return true, "linear: request body unreadable" - } - - decision := LinearGraphQLAllowed(mode, caps, body, truncated) - if decision.Allowed { - return false, "" - } - - // Operations may be empty when the document could not be classified at - // all, which is itself the useful signal. - p.logger.Warn("linear: blocked GraphQL request", - "agent", agentName, - "mode", mode.String(), - "operations", strings.Join(decision.Operations, ","), - "reason", decision.Reason, - "mutation", decision.IsMutation) - - reason := "linear: " + decision.Reason - if len(decision.Operations) > 0 { - reason += " (" + strings.Join(decision.Operations, ",") + ")" - } - return true, reason -} - -func githubWriteBodyMayLeak(method, path string) bool { - if method != http.MethodPost && method != http.MethodPatch && method != http.MethodPut { - return false - } - if !strings.HasPrefix(path, "/repos/") { - return IsGraphQLPath(path) - } - return strings.Contains(path, "/issues") || - strings.Contains(path, "/pulls") || - strings.Contains(path, "/comments") || - strings.Contains(path, "/reviews") || - strings.Contains(path, "/git/") -} - -func isGitReceivePack(path string) bool { - return strings.HasSuffix(path, "/git-receive-pack") -} - -// issueShapeBodyLimit bounds how much of an issue/comment REST body the proxy -// buffers to shape-check it. It comfortably exceeds GitHub's issue-body maximum -// (65536 chars) plus JSON overhead; an oversized body is forwarded unbuffered -// and unchecked rather than truncated (see enforceIssueShape). -const issueShapeBodyLimit = 256 * 1024 - -var ( - issueCreateRESTPathRE = regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues$`) - issueEditRESTPathRE = regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues/\d+$`) - issueCommentCreateRESTPath = regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues/\d+/comments$`) - issueCommentEditRESTPath = regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues/comments/\d+$`) -) - -// isIssueShapeRESTWrite reports whether a (method, path) is a REST issue or -// issue-comment create/edit whose JSON body carries agent-authored title/body -// text the shape validators must vet. It is deliberately precise β€” matching -// only the title/body-bearing routes β€” so label, assignee, and reaction sub -// routes under /issues are not needlessly buffered. -func isIssueShapeRESTWrite(method, path string) bool { - if i := strings.IndexByte(path, '?'); i >= 0 { - path = path[:i] - } - switch method { - case http.MethodPost: - return issueCreateRESTPathRE.MatchString(path) || - issueCommentCreateRESTPath.MatchString(path) - case http.MethodPatch: - return issueEditRESTPathRE.MatchString(path) || - issueCommentEditRESTPath.MatchString(path) - default: - return false - } -} - -// githubIssueShapeViolation returns a non-empty, agent-actionable reason when a -// buffered issue/comment REST body carries an unsubstituted template -// placeholder or literal newline escape sequences. A body that is not the JSON -// we expect is left to the upstream to reject β€” auth was already enforced. -func githubIssueShapeViolation(method, path string, body []byte) string { - if !isIssueShapeRESTWrite(method, path) { - return "" - } - var payload struct { - Title *string `json:"title"` - Body *string `json:"body"` - } - if err := json.Unmarshal(body, &payload); err != nil { - return "" - } - if payload.Title != nil { - if ph, ok := issueshape.UnsubstitutedTemplatePlaceholder(*payload.Title); ok { - return "issue title contains unsubstituted template placeholder " + strconv.Quote(ph) + "; fill in the template before creating the issue" - } - } - if payload.Body != nil { - if ph, ok := issueshape.UnsubstitutedTemplatePlaceholder(*payload.Body); ok { - return "issue body contains unsubstituted template placeholder " + strconv.Quote(ph) + "; fill in the template before creating the issue" - } - if issueshape.BodyHasMisEscapedNewlines(*payload.Body) { - return "issue body contains literal newline escape sequences; use real newlines or --body-file" - } - } - return "" -} - -// bodyReadCloser rejoins an already-read prefix with the unread remainder of a -// request body so an oversized issue write can be forwarded intact. -type bodyReadCloser struct { - io.Reader - io.Closer -} - -// enforceIssueShape is enforceIssueShapeFor with no agent identity, so no lane -// label is injected. -func (p *GitHubProxy) enforceIssueShape(req *http.Request) (reason string, deny bool, readErr error) { - return p.enforceIssueShapeFor(req, "") -} - -// enforceIssueShapeFor buffers an agent's issue/comment REST body, validates -// its shape, and reconstructs req.Body so the request can still be forwarded -// when it passes. It returns deny=true with an actionable reason for a -// malformed body, and a non-nil error only when the body could not be read -// (fail closed). A well-formed issue create from an identified agent also -// gets its agent/ provenance label injected (see injectOriginLabel). -func (p *GitHubProxy) enforceIssueShapeFor(req *http.Request, agentName string) (reason string, deny bool, readErr error) { - if req.Body == nil || !isIssueShapeRESTWrite(req.Method, req.URL.Path) { - return "", false, nil - } - body, err := io.ReadAll(io.LimitReader(req.Body, issueShapeBodyLimit+1)) - if err != nil { - _ = req.Body.Close() - return "", false, err - } - if int64(len(body)) > issueShapeBodyLimit { - // Oversized: forward the original body unbuffered and unchecked rather - // than corrupt it by truncation. Stitch the prefix we already consumed - // back in front of the unread remainder; ContentLength is unchanged. - req.Body = bodyReadCloser{Reader: io.MultiReader(bytes.NewReader(body), req.Body), Closer: req.Body} - return "", false, nil - } - _ = req.Body.Close() - if v := githubIssueShapeViolation(req.Method, req.URL.Path, body); v != "" { - req.Body = io.NopCloser(bytes.NewReader(body)) - req.ContentLength = int64(len(body)) - return v, true, nil - } - if rewritten, ok := injectOriginLabel(req.Method, req.URL.Path, body, agentName); ok { - body = rewritten - } - req.Body = io.NopCloser(bytes.NewReader(body)) - req.ContentLength = int64(len(body)) - return "", false, nil -} - -// isIssueCreateRESTWrite reports whether (method, path) is POST -// /repos/{o}/{r}/issues β€” the one route whose body creates an issue. -func isIssueCreateRESTWrite(method, path string) bool { - if i := strings.IndexByte(path, '?'); i >= 0 { - path = path[:i] - } - return method == http.MethodPost && issueCreateRESTPathRE.MatchString(path) -} - -// injectOriginLabel ensures an agent's issue-create JSON body carries the -// canonical agent/ label and no redundant origin/ or bare -// label, so an issue filed with `gh issue create` through the proxy records -// the same provenance as one filed through the issue-request watcher. The -// body is returned unchanged (ok=false) for other routes, an unidentified -// caller, a body that is not a JSON object, or a labels field in an unexpected -// shape (GitHub accepts label objects too; those are forwarded as written -// rather than guessed at). -func injectOriginLabel(method, path string, body []byte, agentName string) ([]byte, bool) { - lane := config.OriginLaneFromAgent(agentName) - if lane == "" || agentName == internalCallerName || !isIssueCreateRESTWrite(method, path) { - return body, false - } - var payload map[string]json.RawMessage - if err := json.Unmarshal(body, &payload); err != nil || payload == nil { - return body, false - } - var labels []string - if raw, present := payload["labels"]; present && len(bytes.TrimSpace(raw)) > 0 && string(bytes.TrimSpace(raw)) != "null" { - if err := json.Unmarshal(raw, &labels); err != nil { - return body, false - } - } - filtered := config.WithIssueOriginLabel(labels, lane) - if sameLabels(labels, filtered) { - return body, false - } - labels = filtered - encoded, err := json.Marshal(labels) - if err != nil { - return body, false - } - payload["labels"] = encoded - rewritten, err := json.Marshal(payload) - if err != nil { - return body, false - } - return rewritten, true -} - -func sameLabels(a, b []string) bool { - if len(a) != len(b) { - return false - } - for i := range a { - if a[i] != b[i] { - return false - } - } - return true -} - -// internalCallerName attributes a connection that originated from the hive's -// OWN control plane rather than from a sandboxed agent. It is deliberately not -// a valid agent name (agents come from the UID map) so it can never collide -// with one, and it is what distinguishes "the hive called GitHub" from the -// genuine failure mode the mode gate warns about: "an agent's UID could not be -// attributed". See UIDMap.IsInternalUID for the security argument. -const internalCallerName = "hive-internal" - -func (p *GitHubProxy) recordViolation(agentName, method, path string) { - if p.firstBlock(agentName, method, path) { - p.logger.Warn("proxy request blocked", "agent", agentName, "method", method, "path", path) - } else { - p.logger.Debug("proxy request blocked (repeat)", "agent", agentName, "method", method, "path", path) - } - p.countViolation(agentName) -} - -// firstBlock reports whether this (agent, method, path) refusal has not been -// logged before. Once the set is full, new keys keep logging at WARN. -func (p *GitHubProxy) firstBlock(agentName, method, path string) bool { - key := agentName + "|" + method + "|" + path - p.mu.Lock() - defer p.mu.Unlock() - if _, seen := p.blockedSeen[key]; seen { - return false - } - if p.blockedSeen == nil { - p.blockedSeen = make(map[string]struct{}) - } - if len(p.blockedSeen) < maxViolationLog { - p.blockedSeen[key] = struct{}{} - } - return true -} - -// countViolation increments the per-agent refusal counter without logging, for -// callers that have already logged the refusal with their own detail (see -// refusePlainForward). The map is bounded by maxViolationLog so an -// unidentified or churning caller set cannot grow it without limit. -func (p *GitHubProxy) countViolation(agentName string) { - p.mu.Lock() - defer p.mu.Unlock() - if _, exists := p.violations[agentName]; exists || len(p.violations) < maxViolationLog { - p.violations[agentName]++ - } -} - -func isGitPath(path string) bool { - return strings.HasSuffix(path, "/git-receive-pack") || - strings.HasSuffix(path, "/git-upload-pack") || - strings.HasSuffix(path, "/info/refs") -} - -// tunnelDirect creates a raw TCP tunnel for non-GitHub CONNECT requests. -func (p *GitHubProxy) tunnelDirect(conn net.Conn, r *http.Request) { - const tunnelDialTimeout = 10 * time.Second - // SO_MARK the socket so the forced-egress redirect exempts this - // proxy-originated tunnel dial (CONNECT targets include :443). - upstream, err := markDialer(tunnelDialTimeout).Dial("tcp", r.Host) - if err != nil { - p.logger.Warn("proxy: CONNECT dial failed", "host", r.Host, "error", err) - _, _ = fmt.Fprintf(conn, "HTTP/1.1 502 Bad Gateway\r\n\r\nconnection failed\n") - return - } - defer func() { _ = upstream.Close() }() - - _, _ = fmt.Fprintf(conn, "HTTP/1.1 200 Connection established\r\n\r\n") - - relayTunnel(conn, upstream) -} - -// responseBodyStallTimeout bounds how long the MITM'd response-BODY relay may -// go without a single byte of progress from the upstream. It is a rolling -// per-Read bound (re-armed on every read by stallBoundedBody), not a cap on -// total transfer time, so a large-but-flowing response is never cut while an -// upstream that goes silent mid-body releases the handler β€” and with it the -// three sockets of the exchange β€” within one stall window. Overridable, not a -// const, so tests can shorten it β€” and atomic for the same reason as -// tunnelHalfCloseDrain (#5553): the reading goroutine can outlive the test -// that started it. A handler test that returns after a fixed sleep leaves -// proxyHTTP still parsing the response when the NEXT test's shortenBodyStall -// writes the bound β€” a plain var made that an unsynchronised write-after-read -// (-race failure on v4, #6110, reported against whichever test held the -// baton). Defaults to httpReadTimeout: the same patience already extended to -// the header phase. -var responseBodyStallTimeout atomic.Int64 - -func init() { responseBodyStallTimeout.Store(int64(httpReadTimeout)) } - -// bodyStallTimeout returns the current response-body stall bound. -func bodyStallTimeout() time.Duration { - return time.Duration(responseBodyStallTimeout.Load()) -} - -// bodyStallTimeoutDuration returns the body-stall timeout for this proxy instance, -// falling back to the package-level responseBodyStallTimeout. -func (p *GitHubProxy) bodyStallTimeoutDuration() time.Duration { - if p != nil && p.bodyStallTimeout > 0 { - return p.bodyStallTimeout - } - return bodyStallTimeout() -} - -// stallBoundedBody wraps a proxied response body so every underlying Read is -// preceded by re-arming a read deadline on the upstream conn. Close clears the -// deadline (for the next keep-alive exchange on the same conn) and closes the -// wrapped body. See responseBodyStallTimeout for why this exists (#3875). -type stallBoundedBody struct { - body io.ReadCloser - conn net.Conn - idle time.Duration -} - -func (b *stallBoundedBody) Read(p []byte) (int, error) { - _ = b.conn.SetReadDeadline(time.Now().Add(b.idle)) - return b.body.Read(p) -} - -func (b *stallBoundedBody) Close() error { - // Close the wrapped body while a stall deadline is armed: net/http body - // Close paths may still read from the conn (drain-to-EOF), and an unarmed - // read there would reintroduce the very stall this type exists to bound. - // The deadline is cleared afterwards so the next keep-alive exchange on - // this conn starts unencumbered. - _ = b.conn.SetReadDeadline(time.Now().Add(b.idle)) - err := b.body.Close() - _ = b.conn.SetReadDeadline(time.Time{}) - return err -} - -// stallBoundedWriter is the write-side mirror of stallBoundedBody: every Write -// re-arms the client's write deadline, so the bound is "no progress for idle" -// rather than "the whole relay must finish within idle". -// -// The distinction is not academic. The response relays used to arm a single -// absolute client.SetWriteDeadline(now+httpWriteTimeout) around -// resp.Write(client), which is correct for a small buffered body and wrong for -// a stream: the MITM'd /v1/messages SSE relay holds one Write loop open for the -// entire generation, so a turn that legitimately streams for longer than -// httpWriteTimeout was severed mid-stream at exactly that mark β€” even though -// tokens were flowing the whole time. The agent saw a truncated body -// ("Response was interrupted due to a server error", or a TLS-level "cannot -// decrypt peer's message" when the cut landed mid-record), retried, and β€” -// because a retry replays the same long turn β€” was cut again at the same 60s, -// livelocking the agent on its longest and most valuable turns. Measured on the -// hosted spoke: 23 "response relay timed out" in 40 minutes, every one -// path=/v1/messages with error "write tcp ...: i/o timeout", against an agent -// that made zero forward progress across six consecutive attempts. -// -// Reusing the stall semantics already proven on the read side (#3875) keeps a -// client that has genuinely gone away bounded β€” a dead peer stops draining, the -// socket buffer fills, and the next Write blocks past idle and errors β€” while a -// stream that keeps flowing is never cut for being long. -type stallBoundedWriter struct { - conn net.Conn - idle time.Duration -} - -func (w *stallBoundedWriter) Write(p []byte) (int, error) { - _ = w.conn.SetWriteDeadline(time.Now().Add(w.idle)) - return w.conn.Write(p) -} - -// tunnelHalfCloseDrainDefault bounds how long one direction of an opaque -// tunnel may keep waiting for bytes AFTER the other direction has already -// finished. -// -// 30s is deliberately generous: the only legitimate traffic still in flight -// once one side has hung up is a response tail already on the wire, which -// arrives in RTTs, not tens of seconds. -const tunnelHalfCloseDrainDefault = 30 * time.Second - -// tunnelHalfCloseDrain is overridable rather than const for the same reason as -// waitForReadyPollInterval in pkg/hub: tests need a relay that unwedges in -// milliseconds. Production keeps the default. -// -// It is atomic because the write and the read genuinely happen on different -// goroutines with no happens-before between them. relayTunnel reads the bound -// from the copy goroutine it spawns, and that goroutine can outlive the test -// that started it: a handler test only waits for the CONNECT response, not for -// the relay to drain, so the goroutine is still reading while the NEXT test's -// shortenTunnelDrain writes. A plain var made that an unsynchronised -// write-after-read and -race failed whichever test happened to hold the baton -// (#5553) β€” the reported test name was incidental, which is why the failure -// moved between TestRelayTunnelUnwedgesFromSilentUpstream and -// TestRelayTunnelDataFlowsBothWays across shuffle seeds. -var tunnelHalfCloseDrain atomic.Int64 - -func init() { tunnelHalfCloseDrain.Store(int64(tunnelHalfCloseDrainDefault)) } - -// tunnelDrain returns the current half-close drain bound. -func tunnelDrain() time.Duration { - return time.Duration(tunnelHalfCloseDrain.Load()) -} - -// relayTunnel shuttles bytes both ways between the proxied client conn and the -// upstream until BOTH directions have finished, bounding each direction once -// the other has ended. -// -// WHY THE BOUNDS EXIST (measured live, 2026-08-14): the previous inline copies -// waited on each other unboundedly. On spokes whose forge host -// (github.ibm.com) sits behind a SYN-accepting firewall, the upstream TCP -// connect succeeds but no byte ever comes back and no FIN is ever sent. The -// client (the daemon's own JWT mints, agents' CLIs) gives up at its TLS -// handshake timeout and closes β€” the clientβ†’upstream copy finishes and -// half-closes the upstream β€” but io.Copy(conn, upstream) stayed blocked in -// upstream.Read() FOREVER. Every attempt leaked two sockets (the FIN_WAIT2 / -// CLOSE_WAIT piles), the goroutine, and the splice pipe pair io.Copy holds for -// a TCP↔TCP copy on Linux: ~300k fds and ~11 burned cores per affected spoke. -// -// SetReadDeadline is documented to interrupt Reads that are ALREADY blocked, -// which is exactly what arms the escape hatch here: each side arms the OTHER -// side's deadline only at the moment its own direction completes, so a -// healthy long-lived tunnel (git smart HTTP, device-flow polling) is never -// cut while both directions are still live. -func relayTunnel(conn, upstream net.Conn) { - done := make(chan struct{}) - go func() { - _, _ = io.Copy(upstream, conn) - if tc, ok := upstream.(*net.TCPConn); ok { - _ = tc.CloseWrite() - } - // Client side is done (EOF or error β€” a dead client looks the same). - // A live upstream answers or closes within RTTs; a blackholed one - // never would, so bound the remaining upstreamβ†’client read. - _ = upstream.SetReadDeadline(time.Now().Add(tunnelDrain())) - close(done) - }() - _, _ = io.Copy(conn, upstream) - // Mirror image: upstream finished (or errored) but the client may sit - // half-open without ever sending EOF; bound the client-side read so - // <-done cannot wedge this handler. - _ = conn.SetReadDeadline(time.Now().Add(tunnelDrain())) - <-done -} - -func transfer(dst, src net.Conn) { - defer func() { _ = dst.Close() }() - defer func() { _ = src.Close() }() - _, _ = io.Copy(dst, src) -} - -// forwardPlainDirect handles non-CONNECT (plain HTTP) requests on a raw -// connection. -// -// Two policies apply here that the CONNECT path has always applied and this -// one did not: the caller is identified from the connection, and the -// destination must be globally routable (see plain_forward_guard.go). Before -// that, any process that could reach the proxy port could have it open a -// connection, from the proxy's own uid, to any address it named β€” including -// the hive's loopback services, the cloud metadata endpoint and the -// cluster-internal estate β€” and nothing in the log said who asked. -func (p *GitHubProxy) forwardPlainDirect(conn net.Conn, r *http.Request) { - // Identify from the connection, NOT from r.RemoteAddr: this request came - // from http.ReadRequest, which leaves RemoteAddr empty (see - // identifyAgentFromConn). "" means the caller could not be attributed, - // which is logged as such rather than guessed at. - agentName := p.identifyAgentFromConn(conn, r) - - // Proxy-* hop headers address THIS proxy and must not travel further: the - // identity they assert is the one identifyAgentFromConn just decided - // whether to believe, and forwarding it would hand an agent's claimed name - // (and, in the Basic form, whatever it put in the password field) to every - // plain-HTTP host it can name. - r.Header.Del("Proxy-Authorization") - r.Header.Del("Proxy-Connection") - - resp, err := p.plainForwardTransport().RoundTrip(r) - if err != nil { - var denied *plainForwardDenied - if errors.As(err, &denied) { - p.refusePlainForward(conn, r, agentName, denied) - return - } - if isLoopbackHost(r.URL.Host) { - p.logger.Debug("proxy: plain HTTP forward to loopback failed", "agent", agentName, "url", r.URL.String(), "error", err, "hint", "client should bypass the proxy for loopback; NO_PROXY") - } else { - p.logger.Warn("proxy: plain HTTP forward failed", "agent", agentName, "url", r.URL.String(), "error", err) - } - _, _ = fmt.Fprintf(conn, "HTTP/1.1 502 Bad Gateway\r\n\r\nupstream request failed\n") - return - } - defer func() { _ = resp.Body.Close() }() - _ = resp.Write(conn) -} - -// refusePlainForward answers a destination the guard barred. -// -// Loopback is treated as the misconfiguration it almost always is rather than -// as an attack: agents are handed NO_PROXY=127.0.0.1,localhost,::1 (#14076), -// so a loopback request arriving here means a client ignored it. It is logged -// at Debug with the same hint as the failure path and does NOT count as a -// violation. Every other class β€” link-local (cloud metadata), private, CGNAT, -// multicast β€” is an attempt to reach the estate behind the proxy and is -// counted against the caller. -func (p *GitHubProxy) refusePlainForward(conn net.Conn, r *http.Request, agentName string, denied *plainForwardDenied) { - if denied.reason == plainForwardReasonLoopback { - p.logger.Debug("proxy: plain HTTP forward to loopback refused (clients must bypass the proxy for loopback; NO_PROXY)", - "agent", agentName, "url", r.URL.String(), "target", denied.target) - } else { - p.logger.Warn("proxy: plain HTTP forward refused β€” destination is not globally routable", - "agent", agentName, "method", r.Method, "url", r.URL.String(), "target", denied.target, "class", denied.reason) - p.countViolation(agentName) - } - _, _ = fmt.Fprintf(conn, "HTTP/1.1 403 Forbidden\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n"+ - "the hive proxy does not forward plain HTTP to %s addresses (%s); set %s=true to allow it\n", - denied.reason, denied.target, plainForwardAllowPrivateEnv) -} - -// isLoopbackHost reports whether a host or host:port refers to the loopback interface. -func isLoopbackHost(hostport string) bool { - host := hostport - if h, _, err := net.SplitHostPort(hostport); err == nil { - host = h - } - host = strings.Trim(host, "[]") - if strings.EqualFold(host, "localhost") { - return true - } - ip := net.ParseIP(host) - return ip != nil && ip.IsLoopback() -} - -// extractAgentName reads the agent name from the Proxy-Authorization header. -// Supports "hive " (custom) and "Basic " (standard HTTP proxy auth -// sent automatically when the proxy URL contains userinfo, e.g. http://quality@host:port). -func extractAgentName(r *http.Request) string { - auth := r.Header.Get("Proxy-Authorization") - if auth == "" { - return "" - } - if strings.HasPrefix(auth, "hive ") { - return strings.TrimPrefix(auth, "hive ") - } - if strings.HasPrefix(auth, "Basic ") { - decoded, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(auth, "Basic ")) - if err != nil { - return "" - } - // Format is "username:password" β€” agent name is the username portion. - user, _, _ := strings.Cut(string(decoded), ":") - return user - } - return "" -} - -// readAgentMode reads the mode from the hot-reloadable mode file. -// readAgentCaps loads the agent's orthogonal capability set (#4492) from -// /tmp/.hive-caps-. Every failure path returns the zero value β€” no -// capabilities β€” so an unreadable, missing or malformed file degrades toward -// the tier check alone, which is the deny direction. -func readAgentCaps(agentName string) agent.AgentCapabilities { - if agentName == "" { - return agent.AgentCapabilities{} - } - data, err := os.ReadFile(capsFilePrefix + agentName) - if err != nil { - return agent.AgentCapabilities{} - } - return agent.ParseCapabilities(strings.TrimSpace(string(data))) -} - -func readAgentMode(agentName string) agent.AgentMode { - if agentName == "" { - return agent.ModeAdvisory - } - data, err := os.ReadFile(modeFilePrefix + agentName) - if err != nil { - return agent.ModeAdvisory - } - mode, ok := agent.ParseAgentMode(strings.TrimSpace(string(data))) - if !ok { - return agent.ModeAdvisory - } - return mode -} - -// forgeCert generates a TLS certificate for the given hostname, -// signed by the proxy's CA. -func (p *GitHubProxy) forgeCert(host string) (tls.Certificate, error) { - p.certMu.RLock() - if cached, ok := p.certCache[host]; ok && time.Now().Before(cached.expiresAt) { - p.certMu.RUnlock() - return cached.cert, nil - } - p.certMu.RUnlock() - - key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - return tls.Certificate{}, err - } - - serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) - if err != nil { - return tls.Certificate{}, err - } - - template := &x509.Certificate{ - SerialNumber: serial, - Subject: pkix.Name{CommonName: host}, - NotBefore: time.Now().Add(-time.Hour), - NotAfter: time.Now().Add(24 * time.Hour), - KeyUsage: x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - DNSNames: []string{host}, - } - - caKey, ok := p.caCert.PrivateKey.(*ecdsa.PrivateKey) - if !ok { - return tls.Certificate{}, fmt.Errorf("CA key is not ECDSA") - } - - certDER, err := x509.CreateCertificate(rand.Reader, template, p.caX509, &key.PublicKey, caKey) - if err != nil { - return tls.Certificate{}, err - } - - certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}) - keyDER, err := x509.MarshalECPrivateKey(key) - if err != nil { - return tls.Certificate{}, err - } - keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) - - cert, err := tls.X509KeyPair(certPEM, keyPEM) - if err != nil { - return cert, err - } - // Include the CA cert in the chain so clients that load system/extra - // certs asynchronously (e.g. GitHub Copilot's bundled undici) can - // verify the forged leaf without having the CA pre-loaded in their - // trust store. - cert.Certificate = append(cert.Certificate, p.caX509.Raw) - - p.certMu.Lock() - if p.certCache == nil { - p.certCache = make(map[string]cachedCert) - } - p.certCache[host] = cachedCert{cert: cert, expiresAt: time.Now().Add(time.Hour)} - p.certMu.Unlock() - - return cert, nil -} - -// generateCA creates a self-signed CA certificate for MITM. -func generateCA() (tls.Certificate, *x509.Certificate, error) { - key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - return tls.Certificate{}, nil, err - } - - serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) - if err != nil { - return tls.Certificate{}, nil, err - } - - template := &x509.Certificate{ - SerialNumber: serial, - Subject: pkix.Name{CommonName: "Hive ACMM Proxy CA"}, - NotBefore: time.Now().Add(-time.Hour), - NotAfter: time.Now().Add(365 * 24 * time.Hour), - KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, - BasicConstraintsValid: true, - IsCA: true, - MaxPathLen: 0, - } - - certDER, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) - if err != nil { - return tls.Certificate{}, nil, err - } - - certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}) - keyDER, err := x509.MarshalECPrivateKey(key) - if err != nil { - return tls.Certificate{}, nil, err - } - keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) - - tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) - if err != nil { - return tls.Certificate{}, nil, err - } - - x509Cert, err := x509.ParseCertificate(certDER) - if err != nil { - return tls.Certificate{}, nil, err - } - - return tlsCert, x509Cert, nil -} - -// loadOrGenerateCA reuses an existing CA key pair from disk if present and -// valid, or generates a fresh one and persists both cert and key to the PVC. -// Reusing the CA across pod restarts means the system trust store (populated -// by entrypoint.sh before the Go binary starts) already contains the right -// CA, so clients that load system certs asynchronously (GitHub Copilot's -// bundled undici) can verify forged certificates immediately. -func loadOrGenerateCA(logger *slog.Logger) (tls.Certificate, *x509.Certificate, error) { - certPEM, certErr := os.ReadFile(CACertPath) - keyPEM, keyErr := os.ReadFile(caKeyPath) - if certErr == nil && keyErr == nil { - tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) - if err == nil { - block, _ := pem.Decode(certPEM) - if block != nil { - x509Cert, err := x509.ParseCertificate(block.Bytes) - if err == nil && time.Now().Before(x509Cert.NotAfter) { - logger.Info("reusing persisted MITM CA", "expires", x509Cert.NotAfter) - return tlsCert, x509Cert, nil - } - } - } - logger.Warn("persisted CA unusable, generating fresh one", "err", err) - } - - tlsCert, x509Cert, err := generateCA() - if err != nil { - return tls.Certificate{}, nil, err - } - - caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: x509Cert.Raw}) - if err := os.WriteFile(CACertPath, caPEM, 0644); err != nil { - return tls.Certificate{}, nil, fmt.Errorf("write CA cert to %s: %w", CACertPath, err) - } - - caKeyDER, err := x509.MarshalECPrivateKey(tlsCert.PrivateKey.(*ecdsa.PrivateKey)) - if err != nil { - return tls.Certificate{}, nil, fmt.Errorf("marshal CA key: %w", err) - } - caKeyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: caKeyDER}) - // The parent directory is created with owner-only permissions before the - // key is written. /data itself is intentionally group-writable for agent - // workspaces, so 0600 on a file directly under /data is not sufficient: - // an agent could replace or race the path before the proxy opens it. - if err := os.MkdirAll(filepath.Dir(caKeyPath), 0700); err != nil { - return tls.Certificate{}, nil, fmt.Errorf("create CA key directory: %w", err) - } - if err := os.Chmod(filepath.Dir(caKeyPath), 0700); err != nil { - return tls.Certificate{}, nil, fmt.Errorf("restrict CA key directory: %w", err) - } - if err := os.WriteFile(caKeyPath, caKeyPEM, 0600); err != nil { - return tls.Certificate{}, nil, fmt.Errorf("write CA key to %s: %w", caKeyPath, err) - } - - logger.Info("generated and persisted fresh MITM CA", "certPath", CACertPath, "keyPath", caKeyPath) - return tlsCert, x509Cert, nil -} - -func newBufferedConn(c net.Conn) *bufio.Reader { - return bufio.NewReader(c) +func newBufferedConn(c net.Conn) *bufio.Reader { + return bufio.NewReader(c) } func newBufferedReader(c net.Conn) *bufio.Reader { return bufio.NewReader(c) } -// SetInferenceRoute configures an agent to use a self-hosted inference backend. -// -// The route is stored immediately and the caller returns without blocking. The -// model's max context length is queried in the background because this function -// is invoked from callers that hold the agent-manager mutex (agent launch, -// SetModelOverride, SetBackendOverride). A synchronous endpoint probe here -// (up to maxModelLenQueryTimeout) would pin that mutex for its whole duration, -// stalling /api/status and other dashboard handlers β€” which on a single-replica -// pod reads to the OpenShift router as "Application is not available". The -// max-context-len only feeds token capping and defaults gracefully to 0 -// (no cap) until the async probe fills it in. -func (p *GitHubProxy) SetInferenceRoute(agentName string, route *InferenceRoute) { - p.inference.Set(agentName, route) - keySHA256 := config.APIKeySHA256(route.APIKey) - if keySHA256 != "" { - p.logger.Info("inference route set", "agent", agentName, "backend", route.Backend, "endpoint", route.Endpoint, "model", route.Model, "maxContextLen", route.MaxContextLen, "api_key_sha256", keySHA256) - } else { - p.logger.Info("inference route set", "agent", agentName, "backend", route.Backend, "endpoint", route.Endpoint, "model", route.Model, "maxContextLen", route.MaxContextLen, "api_key_sha256", "") - } - - if route.MaxContextLen == 0 { - endpoint, model, apiKey, caBundle := route.Endpoint, route.Model, route.APIKey, route.CABundle - go func() { - if maxLen := queryMaxModelLen(endpoint, model, apiKey, caBundle); maxLen > 0 { - if p.inference.UpdateMaxContextLen(agentName, endpoint, model, maxLen) { - p.logger.Info("inference route max context length resolved", "agent", agentName, "model", model, "maxContextLen", maxLen) - } - } - }() - } - - // Best-effort: learn the key's entitled model set from a LiteLLM - // key-info endpoint before any 403 happens, so the dashboard can offer - // only usable models up front. Only litellm gateways entitlement-filter - // per key; vllm/llm-d do not. The 403-parse path (recordInferenceError) - // remains the authoritative fallback when no key-info endpoint responds. - if route.Backend == "litellm" && route.APIKey != "" { - p.entitlements.rememberProbe(route.Endpoint, route.APIKey, route.CABundle) - go p.probeEntitlements(route.Endpoint, route.APIKey, route.CABundle) - } -} - -// probeEntitlements queries a LiteLLM gateway's per-key info endpoints for the -// key's entitled model set and caches it. A nil result (no key-info endpoint, -// or an unrestricted key) leaves any existing 403-derived entry untouched. -func (p *GitHubProxy) probeEntitlements(endpoint, apiKey, caBundle string) { - if !p.entitlements.beginProbe(endpoint) { - return // another probe for this endpoint is already in flight - } - defer p.entitlements.endProbe(endpoint) - transport, err := inferenceTransport(caBundle) - if err != nil { - transport = nil - } - if models := probeEntitledModels(endpoint, apiKey, transport); len(models) > 0 { - p.entitlements.set(endpoint, apiKey, models, "key-info") - p.logger.Info("litellm entitled models discovered via key-info", - "endpoint", endpoint, "count", len(models)) - } -} - -// EntitledModels returns the entitled model set known for a LiteLLM endpoint -// (from a key-info probe or a "team not allowed" 403), the signal source, and -// whether the set is known. It is the accessor the dashboard uses to narrow -// model dropdowns to what the configured key can actually use. -// -// A stale entry (older than entitlementTTL) is still returned so the served -// model list stays STABLE for an unchanged key β€” a filtered list that flapped -// back to the full catalog between probes would make the dashboard's -// auto-heal (see reconcileModelsAfterDiscovery, #1848) see models reappear -// and vanish across refreshes. Staleness instead triggers a bounded -// background re-probe that overwrites the entry only on a fresh signal. -func (p *GitHubProxy) EntitledModels(endpoint string) (models []string, source string, known bool) { - models, source, known, stale := p.entitlements.get(endpoint) - if stale { - if ep, apiKey, caBundle, ok := p.entitlements.probeParams(endpoint); ok { - go p.probeEntitlements(ep, apiKey, caBundle) - } - } - return models, source, known -} - -// recordInferenceError inspects an upstream inference error body for a -// LiteLLM "team not allowed to access model" 403 and, when found, caches the -// entitled model set it lists (keyed by the route's endpoint) so the -// dashboard's model list narrows to the usable set. Only a 403 carrying the -// gateway's team-scope marker is treated as an entitlement signal β€” -// provider-side failures relayed by the gateway (upstream 404 not-found, 400 -// bad-request, 5xx) say nothing about the key's scope and are ignored, as is -// any other 403 (bad key, rate limit). -// -// It deliberately does NOT switch the agent's route itself: automatic model -// switching is governed by the dashboard's reconcile policy (#1848 β€” unpinned -// + governor suggestion, or confirmed genuine unavailability, at most once -// per list change). Once the entitled set is cached here, the dashboard's -// filtered discovery list drives that heal path for any agent left on a -// non-entitled model. -func (p *GitHubProxy) recordInferenceError(route *InferenceRoute, agentName string, status int, body []byte) { - if route == nil { - return - } - if p.gatewayHealth != nil { - p.gatewayHealth.RecordEndpointHTTPError(route.Backend, route.Endpoint, status, string(truncateBytes(body, 200)), time.Now()) - } - - // Inference-backend AUTH failure (a stale/invalid gateway key returning 401 - // on every call). Tracked for ANY inference backend, not just litellm β€” a - // self-hosted vLLM/llm-d fronted by auth can 401 too. Latches only after - // inferenceAuthFailureThreshold consecutive failures and clears on the next - // success, so a single transient blip never alarms. The stored cause is - // log-safe: the gateway's 401 body describes the rejection without echoing - // the presented token, and inferenceAuthErrorMessage never reads the key. - if isInferenceAuthStatus(status) && p.inferenceAuth != nil { - msg := inferenceAuthErrorMessage(route.Backend, status, truncateBytes(body, 200)) - p.inferenceAuth.recordFailure(msg, time.Now()) - p.logger.Warn("inference backend auth failure", - "agent", agentName, "backend", route.Backend, "endpoint", route.Endpoint, "status", status) - } - - // PROVIDER SPEND REBUFF (#4294). Checked before the 403/entitlement branch - // below because that branch returns early for every non-403, which would - // skip the 429/400 statuses a budget refusal actually arrives on. - // - // Requires the body to name spend, so an ordinary 429 throttle keeps - // flowing to the existing transient-retry path untouched. - if p.inferenceBudget != nil && isInferenceBudgetRebuff(status, body) { - msg := inferenceBudgetMessage(route.Backend, status, truncateBytes(body, 200)) - p.inferenceBudget.recordRebuff(msg, time.Now()) - p.logger.Error("inference backend refused on a spending limit", - "agent", agentName, "backend", route.Backend, "endpoint", route.Endpoint, - "status", status, "cause", msg) - } - - if route.Backend != "litellm" || status != http.StatusForbidden { - return - } - entitled := parseTeam403Models(string(body)) - if len(entitled) == 0 { - return - } - p.entitlements.set(route.Endpoint, route.APIKey, entitled, "403") - p.logger.Info("litellm entitled models learned from 403", - "agent", agentName, "endpoint", route.Endpoint, "count", len(entitled), "model", route.Model) -} - -// routeWithEntitledModel returns the route to actually forward with: when the -// gateway's entitled model set is known (key-info probe or a prior team-scope -// 403) and the route's stored model differs from an entitled id only by -// provider-prefix / dot-hyphen spelling drift, a COPY of the route carrying -// the exact entitled id is returned. LiteLLM matches the request model string -// verbatim, so without this a stored copilot-spelled id (e.g. -// "claude-sonnet-4.6" vs entitled "aws/claude-sonnet-4-6") 403s on every call -// forever (#4400) β€” while the dashboard's tolerant preselect matcher shows the -// agent on a perfectly valid model. Self-healing: the first 403 teaches the -// entitled set (recordInferenceError), so the next request resolves. -// -// When the stored model is NOT entitled at all β€” verbatim or by drift β€” and -// the backend's DefaultModel IS, the request goes out for the default instead -// (hivecommons/hive#9945). The FMA hive's agents carried a `model: -// claude-sonnet-4-6` written for their original copilot backend into a -// switch to the litellm gateway whose default_model was the only model the -// team could use; the per-agent value won, and every call 403'd forever with -// "team not allowed to access model" while the operator stared at a gateway -// card showing the right default. The gateway's own entitlement signal (the -// key-info probe or that very 403) is what authorises the substitution, so a -// model the team can't use is never sent when one it can use is configured. -// The stored route is still untouched: a key/team change that re-entitles the -// per-agent model takes effect on the next request. -// -// The stored route is never mutated: the rewrite is per-request, so a -// key/team change that re-entitles the verbatim id takes effect immediately, -// and concurrent requests race on nothing. Unknown entitlements, verbatim -// matches, and ambiguous candidates all pass the route through unchanged. -func (p *GitHubProxy) routeWithEntitledModel(route *InferenceRoute, agentName string) *InferenceRoute { - if route == nil || route.Backend != "litellm" || p.entitlements == nil { - return route - } - entitled, _, known := p.EntitledModels(route.Endpoint) - if !known { - return route - } - if entitledModelListed(entitled, route.Model) { - return route - } - if resolved, ok := resolveEntitledModelID(entitled, route.Model); ok { - p.logger.Info("inference model resolved to entitled gateway id", - "agent", agentName, "endpoint", route.Endpoint, - "configured", route.Model, "resolved", resolved) - rewritten := *route - rewritten.Model = resolved - return &rewritten - } - fallback := strings.TrimSpace(route.DefaultModel) - if fallback == "" || fallback == route.Model { - return route - } - if !entitledModelListed(entitled, fallback) { - resolved, ok := resolveEntitledModelID(entitled, fallback) - if !ok { - return route - } - fallback = resolved - } - p.logger.Warn("inference model not entitled on gateway; sending the backend default model instead", - "agent", agentName, "backend", route.Backend, "endpoint", route.Endpoint, - "configured", route.Model, "default", fallback, - "remedy", "clear the agent's model (or pick one of the gateway's entitled models) so the agent uses the gateway default") - rewritten := *route - rewritten.Model = fallback - return &rewritten -} - -// entitledModelListed reports whether model appears verbatim in entitled. -func entitledModelListed(entitled []string, model string) bool { - for _, id := range entitled { - if id == model { - return true - } - } - return false -} - -// recordInferenceSuccess records a successful inference call so a previously -// latched auth-failure signal clears β€” the self-heal for the inference-auth -// health signal. Called from every inference forward path on a 2xx response. -// A nil tracker (impossible after NewGitHubProxy, but cheap to guard) is a -// no-op. -func (p *GitHubProxy) recordInferenceSuccessFor(route *InferenceRoute) { - if route != nil && p.gatewayHealth != nil { - p.gatewayHealth.Clear(route.Backend) - } - p.recordInferenceSuccess() -} - -func (p *GitHubProxy) recordInferenceSuccess() { - if p.inferenceAuth != nil { - p.inferenceAuth.recordSuccess() - } - // Self-heal for the spend rebuff (#4294): whenever the provider's window - // resets β€” hive neither knows nor assumes the schedule; the field report's - // gateway rolled over at the key's creation time of day, not at midnight - // (see #4294) β€” the first call that succeeds takes the hive out of the - // suppressed state with no operator action. - if p.inferenceBudget != nil { - p.inferenceBudget.recordSuccess() - } -} - -// GatewayHealth reports currently failing inference gateways. -func (p *GitHubProxy) GatewayHealth() []inferencehealth.GatewayStatus { - if p == nil || p.gatewayHealth == nil { - return nil - } - return p.gatewayHealth.Snapshot() -} - -// InferenceAuthError reports the current inference-backend auth-failure signal: -// a non-empty, log-safe cause string (and the time it first latched) ONLY while -// the backend has been auth-failing for inferenceAuthFailureThreshold -// consecutive calls, empty otherwise. Wired to the dashboard/heartbeat so the -// hub can flag a hive that is silently 401'ing on every inference call. Safe on -// a nil tracker. -func (p *GitHubProxy) InferenceAuthError() (errMsg string, since time.Time) { - if p == nil || p.inferenceAuth == nil { - return "", time.Time{} - } - return p.inferenceAuth.snapshot() -} - -// InferenceBudgetExceeded reports the current provider spending-limit signal: -// a non-empty log-safe cause, when it first latched, when the most recent -// rebuff arrived, and how many rebuffs have been observed since β€” all -// zero-valued while the provider is serving normally. -// -// Callers use a non-empty cause as "this hive cannot buy inference right now", -// which is a reason to stop kicking agents and to tell the operator, NOT a -// reason to retry in a few minutes. lastRebuff is what bounds that: because -// withholding kicks also withholds the calls that would clear the latch, -// callers suppress only while lastRebuff is fresh and send a probe once it is -// stale (see inference_budget.go). Safe on a nil receiver/tracker. -func (p *GitHubProxy) InferenceBudgetExceeded() (errMsg string, since, lastRebuff time.Time, rebuffs int) { - if p == nil || p.inferenceBudget == nil { - return "", time.Time{}, time.Time{}, 0 - } - return p.inferenceBudget.snapshot() -} - -// ClearInferenceRoute removes an agent's inference backend override. -func (p *GitHubProxy) ClearInferenceRoute(agentName string) { - p.inference.Clear(agentName) - p.logger.Info("inference route cleared", "agent", agentName) -} - -// StartInferenceTranslator runs a plain HTTP server that accepts Anthropic -// Messages API requests and translates+forwards them to the configured -// inference backend. Agents use ANTHROPIC_BASE_URL=http://127.0.0.1:18444 -// to reach this server instead of api.anthropic.com. -func (p *GitHubProxy) inferenceTranslatorHandler() http.Handler { - mux := http.NewServeMux() - - mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - agentName, ok := p.translatorAgentName(r) - if !ok { - p.logger.Warn("inference request from unidentified caller refused", - "remote", r.RemoteAddr, "method", r.Method, "path", r.URL.Path) - http.Error(w, `{"type":"error","error":{"type":"authentication_error","message":"caller could not be identified"}}`, http.StatusUnauthorized) - return - } - - p.logger.Info("inference request", - "agent", agentName, - "method", r.Method, - "path", r.URL.Path, - "content-type", r.Header.Get("Content-Type"), - "anthropic-version", r.Header.Get("anthropic-version"), - ) - - route := p.inference.Get(agentName) - if route == nil { - p.logger.Warn("inference no route", "agent", agentName) - http.Error(w, `{"type":"error","error":{"type":"api_error","message":"no inference route for agent"}}`, http.StatusBadGateway) - return - } - route = p.routeWithEntitledModel(route, agentName) - - body, err := io.ReadAll(r.Body) - if r.Body != nil { - _ = r.Body.Close() - } - if err != nil { - http.Error(w, `{"type":"error","error":{"type":"api_error","message":"failed to read request"}}`, http.StatusBadRequest) - return - } - - // Same path gate as the MITM reroute: only POST /v1/messages is - // translated and forwarded; CLI housekeeping is answered here. - if kind := classifyInferencePath(r.Method, r.URL.Path); kind != inferencePathMessages { - status, respBody := p.localInferenceResponse(kind, r.Method, r.URL.Path, body, agentName) - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(status) - _, _ = io.WriteString(w, respBody) - return - } - - // Diagnostic: count tools in original Anthropic request. - var toolDiag struct { - Tools json.RawMessage `json:"tools"` - } - toolCount := -1 // -1 = no tools key - if json.Unmarshal(body, &toolDiag) == nil && toolDiag.Tools != nil { - var arr []json.RawMessage - if json.Unmarshal(toolDiag.Tools, &arr) == nil { - toolCount = len(arr) - } - } - p.logger.Info("inference request body", "agent", agentName, "len", len(body), "tool_count", toolCount, "preview", truncateBytes(body, 200)) - - openaiBody, err := translateAnthropicToOpenAI(body, route.Model, route.MaxContextLen, resolveInferencePreamble(route, agentName)) - if err != nil { - p.logger.Error("inference translate request failed", "agent", agentName, "error", err) - http.Error(w, fmt.Sprintf(`{"type":"error","error":{"type":"api_error","message":"translation error: %s"}}`, err.Error()), http.StatusBadGateway) - return - } - - upstreamURL := strings.TrimRight(route.Endpoint, "/") + "/v1/chat/completions" - upstreamReq, err := http.NewRequestWithContext(r.Context(), "POST", upstreamURL, bytes.NewReader(openaiBody)) - if err != nil { - http.Error(w, `{"type":"error","error":{"type":"api_error","message":"failed to create upstream request"}}`, http.StatusBadGateway) - return - } - upstreamReq.Header.Set("Content-Type", "application/json") - applyInferenceAuth(upstreamReq, route) - - p.logger.Info("inference forward", - "agent", agentName, - "backend", route.Backend, - "model", route.Model, - "url", upstreamURL, - "openai_body", truncateBytes(openaiBody, 300), - ) - - client, err := inferenceHTTPClient(route) - if err != nil { - p.logger.Error("inference client setup failed", "agent", agentName, "error", err) - http.Error(w, fmt.Sprintf(`{"type":"error","error":{"type":"api_error","message":"inference client setup failed: %s"}}`, err.Error()), http.StatusBadGateway) - return - } - resp, err := client.Do(upstreamReq) - if err != nil { - p.logger.Error("inference upstream failed", "agent", agentName, "url", upstreamURL, "error", err) - if p.gatewayHealth != nil { - p.gatewayHealth.RecordEndpointError(route.Backend, route.Endpoint, err, time.Now()) - } - unreachableMsg := inferenceBackendUnreachableMessage(err, upstreamURL) - p.recordAgentInferenceError(agentName, unreachableMsg, 0) - http.Error(w, fmt.Sprintf(`{"type":"error","error":{"type":"api_error","message":"%s"}}`, jsonEscape(unreachableMsg)), http.StatusBadGateway) - return - } - defer func() { _ = resp.Body.Close() }() - - p.logger.Info("inference upstream response", - "agent", agentName, - "status", resp.StatusCode, - "content-type", resp.Header.Get("Content-Type"), - ) - - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - errBody, _ := io.ReadAll(resp.Body) - p.logger.Error("inference upstream error", - "agent", agentName, - "status", resp.StatusCode, - "url", upstreamURL, - "server", resp.Header.Get("Server"), - "body", truncateBytes(errBody, 500), - ) - // A LiteLLM "team not allowed to access model" 403 carries the - // entitled set β€” cache it so the dashboard's model list narrows - // and its reconcile policy (#1848) can heal the agent. - p.recordInferenceError(route, agentName, resp.StatusCode, errBody) - backendMsg := inferenceBackendErrorMessage(resp.StatusCode, errBody, upstreamURL, resp.Header.Get("Server")) - p.recordAgentInferenceError(agentName, backendMsg, resp.StatusCode) - anthropicErr := fmt.Sprintf(`{"type":"error","error":{"type":"api_error","message":"%s"}}`, - jsonEscape(backendMsg)) - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(resp.StatusCode) - _, _ = w.Write([]byte(anthropicErr)) - return - } - - // A 2xx means the gateway accepted the key β€” clear any latched - // inference-auth failure so a hive whose key was fixed self-heals. - p.recordInferenceSuccessFor(route) - p.clearAgentInferenceError(agentName) - - isStreaming := strings.Contains(resp.Header.Get("Content-Type"), "text/event-stream") - - if isStreaming { - w.Header().Set("Content-Type", "text/event-stream; charset=utf-8") - w.Header().Set("Cache-Control", "no-cache") - w.Header().Set("Connection", "keep-alive") - w.WriteHeader(http.StatusOK) - if f, ok := w.(http.Flusher); ok { - f.Flush() - } - in, out, err := translateOpenAISSEToAnthropic(resp.Body, w, route.Model) - if err != nil { - p.logger.Error("inference SSE translation failed", "agent", agentName, "error", err) - } - if in > 0 || out > 0 { - p.recordUsage(agentName, route.Model, in, out) - } - return - } - - respBody, err := io.ReadAll(resp.Body) - if err != nil { - http.Error(w, `{"type":"error","error":{"type":"api_error","message":"failed to read inference response"}}`, http.StatusBadGateway) - return - } - - p.logger.Info("inference upstream raw response", - "agent", agentName, - "len", len(respBody), - "preview", truncateBytes(respBody, 500), - ) - - if in, out := extractOpenAIUsage(respBody); in > 0 || out > 0 { - p.recordUsage(agentName, route.Model, in, out) - } - - translated, err := translateOpenAIResponseToAnthropic(respBody, route.Model) - if err != nil { - p.logger.Error("inference translate response failed", "agent", agentName, "error", err) - http.Error(w, `{"type":"error","error":{"type":"api_error","message":"response translation error"}}`, http.StatusBadGateway) - return - } - - p.logger.Info("inference translated response", - "agent", agentName, - "len", len(translated), - "preview", truncateBytes(translated, 500), - ) - - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - _, _ = w.Write(translated) - }) - - return mux -} - -// StartInferenceTranslator runs the Anthropic-compatible inference translator -// on the fixed local port used by agents. -func (p *GitHubProxy) StartInferenceTranslator() error { - addr := fmt.Sprintf("127.0.0.1:%d", InferenceTranslatePort) - p.logger.Info("inference translation server starting", "addr", addr) - server := &http.Server{ - Addr: addr, - Handler: p.inferenceTranslatorHandler(), - ReadHeaderTimeout: httpReadTimeout, - } - return server.ListenAndServe() -} - -// handleAnthropicReroute performs MITM on an Anthropic API connection and -// reroutes requests to a self-hosted vLLM/llm-d endpoint, translating -// between Anthropic and OpenAI API formats. -func (p *GitHubProxy) handleAnthropicReroute(conn net.Conn, r *http.Request, host, agentName string, route *InferenceRoute) { - p.logger.Info("inference reroute", "agent", agentName, "backend", route.Backend, "host", host) - - if _, err := fmt.Fprintf(conn, "HTTP/1.1 200 Connection established\r\n\r\n"); err != nil { - p.logger.Error("inference reroute: CONNECT response failed", "error", err) - return - } - - tlsCert, err := p.forgeCert(host) - if err != nil { - p.logger.Error("inference reroute: forge cert failed", "host", host, "error", err) - return - } - - tlsConn := tls.Server(conn, &tls.Config{ - Certificates: []tls.Certificate{tlsCert}, - }) - _ = conn.SetDeadline(time.Now().Add(clientTLSHandshakeTimeout)) - handshakeErr := tlsConn.Handshake() - _ = conn.SetDeadline(time.Time{}) - if handshakeErr != nil { - p.logger.Warn("inference reroute: TLS handshake failed", "error", handshakeErr) - return - } - defer func() { _ = tlsConn.Close() }() - - clientBuf := bufio.NewReader(tlsConn) - for { - _ = tlsConn.SetReadDeadline(time.Now().Add(httpReadTimeout)) - req, err := http.ReadRequest(clientBuf) - _ = tlsConn.SetReadDeadline(time.Time{}) - if err != nil { - p.logTimeout("inference reroute: client request read timed out", err, "agent", agentName) - return - } - - p.handleInferenceRequest(tlsConn, req, agentName, route) - } -} - -// handleInferenceRequest translates a single Anthropic API request and -// forwards it to the inference backend. -func (p *GitHubProxy) handleInferenceRequest(conn net.Conn, req *http.Request, agentName string, route *InferenceRoute) { - route = p.routeWithEntitledModel(route, agentName) - _ = conn.SetReadDeadline(time.Now().Add(httpReadTimeout)) - body, err := io.ReadAll(req.Body) - _ = conn.SetReadDeadline(time.Time{}) - if req.Body != nil { - _ = req.Body.Close() - } - if err != nil { - p.logTimeout("inference reroute: request body read timed out", err, "agent", agentName) - p.writeHTTPError(conn, http.StatusBadGateway, "failed to read request body") - return - } - - // Only POST /v1/messages is an inference call. The Claude Code CLI also - // sends telemetry batches, error reports, and count_tokens to this host; - // translating those bodies produced {"messages": null} and a gateway 400 - // per call, charged against the provider's request rate limit. - if kind := classifyInferencePath(req.Method, req.URL.Path); kind != inferencePathMessages { - status, respBody := p.localInferenceResponse(kind, req.Method, req.URL.Path, body, agentName) - writeLocalInferenceResponse(conn, status, respBody) - return - } - - openaiBody, err := translateAnthropicToOpenAI(body, route.Model, route.MaxContextLen, resolveInferencePreamble(route, agentName)) - if err != nil { - p.logger.Error("inference translate request failed", "agent", agentName, "error", err) - p.writeHTTPError(conn, http.StatusBadGateway, "translation error: "+err.Error()) - return - } - - upstreamURL := strings.TrimRight(route.Endpoint, "/") + "/v1/chat/completions" - upstreamReq, err := http.NewRequestWithContext(req.Context(), "POST", upstreamURL, bytes.NewReader(openaiBody)) - if err != nil { - p.writeHTTPError(conn, http.StatusBadGateway, "failed to create upstream request") - return - } - upstreamReq.Header.Set("Content-Type", "application/json") - applyInferenceAuth(upstreamReq, route) - - p.logger.Info("inference forward", "agent", agentName, "backend", route.Backend, "model", route.Model, "url", upstreamURL) - - client, err := inferenceHTTPClient(route) - if err != nil { - p.logger.Error("inference client setup failed", "agent", agentName, "error", err) - p.writeHTTPError(conn, http.StatusBadGateway, "inference client setup failed: "+err.Error()) - return - } - resp, err := client.Do(upstreamReq) - if err != nil { - p.logger.Error("inference upstream failed", "agent", agentName, "url", upstreamURL, "error", err) - if p.gatewayHealth != nil { - p.gatewayHealth.RecordEndpointError(route.Backend, route.Endpoint, err, time.Now()) - } - unreachableMsg := inferenceBackendUnreachableMessage(err, upstreamURL) - p.recordAgentInferenceError(agentName, unreachableMsg, 0) - p.writeHTTPError(conn, http.StatusBadGateway, unreachableMsg) - return - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - errBody, _ := io.ReadAll(resp.Body) - p.logger.Error("inference upstream error", "agent", agentName, "status", resp.StatusCode, "url", upstreamURL, "server", resp.Header.Get("Server"), "body", truncateBytes(errBody, 500)) - // Same entitlement capture as the HTTP translator path. - p.recordInferenceError(route, agentName, resp.StatusCode, errBody) - backendMsg := inferenceBackendErrorMessage(resp.StatusCode, errBody, upstreamURL, resp.Header.Get("Server")) - p.recordAgentInferenceError(agentName, backendMsg, resp.StatusCode) - anthropicErr := &http.Response{ - StatusCode: resp.StatusCode, - ProtoMajor: 1, - ProtoMinor: 1, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(strings.NewReader(fmt.Sprintf( - `{"type":"error","error":{"type":"api_error","message":"%s"}}`, - jsonEscape(backendMsg)))), - } - _ = anthropicErr.Write(conn) - return - } - - // A 2xx means the gateway accepted the key β€” clear any latched - // inference-auth failure so a hive whose key was fixed self-heals. - p.recordInferenceSuccessFor(route) - p.clearAgentInferenceError(agentName) - - isStreaming := strings.Contains(resp.Header.Get("Content-Type"), "text/event-stream") - - if isStreaming { - hdr := "HTTP/1.1 200 OK\r\n" + - "Content-Type: text/event-stream; charset=utf-8\r\n" + - "Cache-Control: no-cache\r\n" + - "Connection: keep-alive\r\n\r\n" - if _, err := conn.Write([]byte(hdr)); err != nil { - return - } - in, out, err := translateOpenAISSEToAnthropic(resp.Body, conn, route.Model) - if err != nil { - p.logger.Error("inference SSE translation failed", "agent", agentName, "error", err) - } - if in > 0 || out > 0 { - p.recordUsage(agentName, route.Model, in, out) - } - return - } - - respBody, err := io.ReadAll(resp.Body) - if err != nil { - p.writeHTTPError(conn, http.StatusBadGateway, "failed to read inference response") - return - } - - if in, out := extractOpenAIUsage(respBody); in > 0 || out > 0 { - p.recordUsage(agentName, route.Model, in, out) - } - - translated, err := translateOpenAIResponseToAnthropic(respBody, route.Model) - if err != nil { - p.logger.Error("inference translate response failed", "agent", agentName, "error", err) - p.writeHTTPError(conn, http.StatusBadGateway, "response translation error") - return - } - - httpResp := &http.Response{ - StatusCode: http.StatusOK, - ProtoMajor: 1, - ProtoMinor: 1, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(bytes.NewReader(translated)), - } - _ = httpResp.Write(conn) -} - -// handleCopilotSniff MITMs a GitHub Copilot completion-API connection to read -// the OpenAI-shaped token usage out of each /chat/completions response, then -// records it via the same token sink the inference path uses so Copilot cost -// goes live instead of only tallying at session shutdown. -// -// Unlike the inference reroute, this does NOT translate or reroute: the client's -// request is forwarded verbatim to the real Copilot host (with one exception β€” -// streaming requests get stream_options.include_usage injected so the terminal -// SSE usage chunk is emitted), and the upstream response is streamed back -// byte-for-byte to the client. The proxy only reads a copy of the response to -// extract usage. -// -// It is only invoked when a token sink is active and the agent is identified; -// callers fall back to opaque tunneling otherwise, so a missing sink or unknown -// agent never breaks Copilot traffic. -func (p *GitHubProxy) handleCopilotSniff(conn net.Conn, host, agentName string) { - if _, err := fmt.Fprintf(conn, "HTTP/1.1 200 Connection established\r\n\r\n"); err != nil { - p.logger.Error("copilot sniff: CONNECT response failed", "error", err) - return - } - p.sniffCopilotOnTLS(conn, host, agentName) -} - -// sniffCopilotOnTLS TLS-terminates a client connection (already past any CONNECT -// handshake), dials the real Copilot upstream, and pumps requests/responses -// through proxyCopilotHTTP so usage is recorded. Shared by the explicit-CONNECT -// (handleCopilotSniff) and the transparent-TLS (iptables) entry points so the -// forge/handshake/dial logic lives in one place. clientConn must be the raw -// (pre-TLS) client socket, optionally already wrapped in a prefixConn carrying a -// peeked ClientHello. -func (p *GitHubProxy) sniffCopilotOnTLS(clientConn net.Conn, host, agentName string) { - tlsCert, err := p.forgeCert(host) - if err != nil { - p.logger.Error("copilot sniff: forge cert failed", "host", host, "error", err) - return - } - - tlsClientConn := tls.Server(clientConn, &tls.Config{Certificates: []tls.Certificate{tlsCert}}) - _ = clientConn.SetDeadline(time.Now().Add(clientTLSHandshakeTimeout)) - handshakeErr := tlsClientConn.Handshake() - _ = clientConn.SetDeadline(time.Time{}) - if handshakeErr != nil { - if isClientHangup(handshakeErr) { - p.logger.Debug("copilot sniff: client hung up before TLS handshake", "error", handshakeErr) - } else { - p.logger.Warn("copilot sniff: client TLS handshake failed", "error", handshakeErr) - } - return - } - defer func() { _ = tlsClientConn.Close() }() - - upstreamConn, err := p.dialCopilotUpstream(host) - if err != nil { - p.logger.Error("copilot sniff: upstream dial failed", "host", host, "error", err) - return - } - defer func() { _ = upstreamConn.Close() }() - - p.proxyCopilotHTTP(tlsClientConn, upstreamConn, host, agentName) -} - -// copilotSniffBodyLimit caps how much of a Copilot request/response body the -// proxy buffers in order to read the usage block. Completion payloads are well -// under this; the limit bounds memory against a pathological body. Bodies larger -// than the limit are still forwarded in full (streamed), only usage extraction -// is skipped for the oversized case. -const copilotSniffBodyLimit = 32 * 1024 * 1024 // 32 MiB - -// proxyCopilotHTTP reads HTTP requests from the client, forwards them to the -// Copilot upstream, and forwards responses back β€” extracting token usage from -// completion responses along the way. It keeps the connection alive across -// multiple request/response pairs (HTTP keep-alive), exiting when either side -// closes. -func (p *GitHubProxy) proxyCopilotHTTP(client net.Conn, upstream net.Conn, host, agentName string) { - clientBuf := bufio.NewReader(client) - upstreamBuf := bufio.NewReader(upstream) - - for { - _ = client.SetReadDeadline(time.Now().Add(httpReadTimeout)) - req, err := http.ReadRequest(clientBuf) - if err != nil { - _ = client.SetReadDeadline(time.Time{}) - p.logClientRead("copilot sniff: client request read timed out", err, "agent", agentName, "host", host) - return - } - - isCompletion := isCopilotCompletionsPath(req.URL.Path) - isAnthropic := isCopilotAnthropicMessagesPath(req.URL.Path) - model := "" - - // For completion requests, buffer the body so we can (a) learn the - // model id and (b) inject include_usage on streaming requests. Other - // requests (model listings, etc.) are forwarded without buffering. - if isCompletion && req.Body != nil { - body, readErr := io.ReadAll(io.LimitReader(req.Body, copilotSniffBodyLimit)) - _ = req.Body.Close() - if readErr != nil { - _ = client.SetReadDeadline(time.Time{}) - p.logTimeout("copilot sniff: request body read timed out", readErr, "agent", agentName, "host", host, "path", req.URL.Path) - return - } - var streaming bool - model, streaming = parseCopilotRequest(body) - // stream_options is OpenAI-only; Anthropic streams always carry - // usage in message_start/message_delta. - if streaming && !isAnthropic { - body = ensureStreamUsageRequested(body) - } - req.Body = io.NopCloser(bytes.NewReader(body)) - req.ContentLength = int64(len(body)) - req.Header.Set("Content-Length", fmt.Sprintf("%d", len(body))) - } - - _ = upstream.SetWriteDeadline(time.Now().Add(httpWriteTimeout)) - if err := req.Write(upstream); err != nil { - _ = upstream.SetWriteDeadline(time.Time{}) - _ = client.SetReadDeadline(time.Time{}) - p.logTimeout("copilot sniff: request relay timed out", err, "agent", agentName, "host", host, "path", req.URL.Path) - return - } - _ = upstream.SetWriteDeadline(time.Time{}) - _ = client.SetReadDeadline(time.Time{}) - - _ = upstream.SetReadDeadline(time.Now().Add(httpReadTimeout)) - resp, err := http.ReadResponse(upstreamBuf, req) - _ = upstream.SetReadDeadline(time.Time{}) - if err != nil { - p.logTimeout("copilot sniff: upstream response read timed out", err, "agent", agentName, "host", host, "path", req.URL.Path) - return - } - - if isAnthropic && resp.StatusCode >= 200 && resp.StatusCode < 300 { - if err := p.forwardCopilotAnthropicResponseWithUsage(client, resp, host, agentName, model); err != nil { - p.logTimeout("copilot sniff: response relay timed out", err, "agent", agentName, "host", host, "path", req.URL.Path) - return - } - } else if isCompletion && resp.StatusCode >= 200 && resp.StatusCode < 300 { - p.forwardCopilotResponseWithUsage(client, resp, host, agentName, model) - } else { - // Non-completion and non-2xx responses are relayed verbatim. The - // Write below can legitimately stay open for a long time, so it - // must be bounded by stall, not by total duration β€” see - // stallBoundedWriter. - if err := resp.Write(&stallBoundedWriter{conn: client, idle: httpWriteTimeout}); err != nil { - _ = resp.Body.Close() - _ = client.SetWriteDeadline(time.Time{}) - p.logTimeout("copilot sniff: response relay timed out", err, "agent", agentName, "host", host, "path", req.URL.Path) - return - } - _ = client.SetWriteDeadline(time.Time{}) - _ = resp.Body.Close() - } - - if req.Close || resp.Close { - return - } - } -} - -// forwardCopilotAnthropicResponseWithUsage relays an Anthropic Messages -// (/v1/messages) response to the client while teeing the body through a usage -// parser, so streamed Claude completions are forwarded as they arrive rather -// than buffered. Usage (including prompt-cache counts) is recorded once the -// body has been fully relayed. resp.Body is consumed and closed here. -func (p *GitHubProxy) forwardCopilotAnthropicResponseWithUsage(client net.Conn, resp *http.Response, host, agentName, model string) error { - tee := newAnthropicUsageTee(resp.Body, resp.Header.Get("Content-Type")) - resp.Body = tee - writeErr := resp.Write(&stallBoundedWriter{conn: client, idle: httpWriteTimeout}) - _ = resp.Body.Close() - _ = client.SetWriteDeadline(time.Time{}) - - // Record whatever was observed even if the client went away mid-stream: - // the upstream already billed those tokens. - u := tee.usage() - if u.total() > 0 { - resolvedModel := model - if resolvedModel == "" || resolvedModel == copilotAutoModelSentinel { - resolvedModel = u.Model - } - if resolvedModel != "" && resolvedModel != copilotAutoModelSentinel { - p.recordCopilotUsageWithCache(agentName, resolvedModel, u.Input, u.Output, u.CacheRead, u.CacheCreation) - p.logger.Info("copilot usage recorded", - "agent", agentName, - "host", host, - "model", resolvedModel, - "input", u.Input, - "output", u.Output, - "cache_read", u.CacheRead, - "cache_creation", u.CacheCreation, - ) - } - } - return writeErr -} - -// forwardCopilotResponseWithUsage forwards a completion response to the client -// while extracting token usage from a buffered copy. The full body is read into -// memory (completion responses are small), usage is recorded, then the response -// is rewritten to the client verbatim. resp.Body is consumed and closed here. -func (p *GitHubProxy) forwardCopilotResponseWithUsage(client net.Conn, resp *http.Response, host, agentName, model string) { - body, err := io.ReadAll(io.LimitReader(resp.Body, copilotSniffBodyLimit)) - _ = resp.Body.Close() - if err != nil { - // Best effort: nothing to forward. - return - } - - contentType := resp.Header.Get("Content-Type") - if in, out := extractCopilotUsage(contentType, body); in > 0 || out > 0 { - resolvedModel := model - if resolvedModel == "" || resolvedModel == copilotAutoModelSentinel { - // Fall back to the model reported in the response when the request - // did not pin a concrete id (e.g. --model auto). Never record the - // "auto" sentinel β€” it is not a real model row. - if rm := extractCopilotResponseModel(body, contentType); rm != "" { - resolvedModel = rm - } - } - if resolvedModel != "" && resolvedModel != copilotAutoModelSentinel { - p.recordCopilotUsage(agentName, resolvedModel, in, out) - p.logger.Info("copilot usage recorded", - "agent", agentName, - "host", host, - "model", resolvedModel, - "input", in, - "output", out, - ) - } - } - - // Rewrite the response to the client verbatim. Replace the body with the - // buffered copy and drop Content-Length ambiguity by setting it explicitly. - resp.Body = io.NopCloser(bytes.NewReader(body)) - resp.ContentLength = int64(len(body)) - if err := resp.Write(&stallBoundedWriter{conn: client, idle: httpWriteTimeout}); err != nil { - p.logger.Warn("copilot sniff: response write to client failed", "agent", agentName, "error", err) - } - _ = client.SetWriteDeadline(time.Time{}) -} - func (p *GitHubProxy) writeHTTPError(conn net.Conn, status int, msg string) { resp := &http.Response{ StatusCode: status, diff --git a/src/crux/proxy/github_proxy_ca.go b/src/crux/proxy/github_proxy_ca.go new file mode 100644 index 0000000000..05818270a0 --- /dev/null +++ b/src/crux/proxy/github_proxy_ca.go @@ -0,0 +1,194 @@ +package proxy + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "log/slog" + "math/big" + "os" + "path/filepath" + "time" +) + +type cachedCert struct { + cert tls.Certificate + expiresAt time.Time +} + +// forgeCert generates a TLS certificate for the given hostname, +// signed by the proxy's CA. +func (p *GitHubProxy) forgeCert(host string) (tls.Certificate, error) { + p.certMu.RLock() + if cached, ok := p.certCache[host]; ok && time.Now().Before(cached.expiresAt) { + p.certMu.RUnlock() + return cached.cert, nil + } + p.certMu.RUnlock() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return tls.Certificate{}, err + } + + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return tls.Certificate{}, err + } + + template := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: host}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: []string{host}, + } + + caKey, ok := p.caCert.PrivateKey.(*ecdsa.PrivateKey) + if !ok { + return tls.Certificate{}, fmt.Errorf("CA key is not ECDSA") + } + + certDER, err := x509.CreateCertificate(rand.Reader, template, p.caX509, &key.PublicKey, caKey) + if err != nil { + return tls.Certificate{}, err + } + + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}) + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + return tls.Certificate{}, err + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) + + cert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return cert, err + } + // Include the CA cert in the chain so clients that load system/extra + // certs asynchronously (e.g. GitHub Copilot's bundled undici) can + // verify the forged leaf without having the CA pre-loaded in their + // trust store. + cert.Certificate = append(cert.Certificate, p.caX509.Raw) + + p.certMu.Lock() + if p.certCache == nil { + p.certCache = make(map[string]cachedCert) + } + p.certCache[host] = cachedCert{cert: cert, expiresAt: time.Now().Add(time.Hour)} + p.certMu.Unlock() + + return cert, nil +} + +// generateCA creates a self-signed CA certificate for MITM. +func generateCA() (tls.Certificate, *x509.Certificate, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return tls.Certificate{}, nil, err + } + + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return tls.Certificate{}, nil, err + } + + template := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: "Hive ACMM Proxy CA"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(365 * 24 * time.Hour), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + BasicConstraintsValid: true, + IsCA: true, + MaxPathLen: 0, + } + + certDER, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + return tls.Certificate{}, nil, err + } + + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}) + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + return tls.Certificate{}, nil, err + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) + + tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return tls.Certificate{}, nil, err + } + + x509Cert, err := x509.ParseCertificate(certDER) + if err != nil { + return tls.Certificate{}, nil, err + } + + return tlsCert, x509Cert, nil +} + +// loadOrGenerateCA reuses an existing CA key pair from disk if present and +// valid, or generates a fresh one and persists both cert and key to the PVC. +// Reusing the CA across pod restarts means the system trust store (populated +// by entrypoint.sh before the Go binary starts) already contains the right +// CA, so clients that load system certs asynchronously (GitHub Copilot's +// bundled undici) can verify forged certificates immediately. +func loadOrGenerateCA(logger *slog.Logger) (tls.Certificate, *x509.Certificate, error) { + certPEM, certErr := os.ReadFile(CACertPath) + keyPEM, keyErr := os.ReadFile(caKeyPath) + if certErr == nil && keyErr == nil { + tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) + if err == nil { + block, _ := pem.Decode(certPEM) + if block != nil { + x509Cert, err := x509.ParseCertificate(block.Bytes) + if err == nil && time.Now().Before(x509Cert.NotAfter) { + logger.Info("reusing persisted MITM CA", "expires", x509Cert.NotAfter) + return tlsCert, x509Cert, nil + } + } + } + logger.Warn("persisted CA unusable, generating fresh one", "err", err) + } + + tlsCert, x509Cert, err := generateCA() + if err != nil { + return tls.Certificate{}, nil, err + } + + caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: x509Cert.Raw}) + if err := os.WriteFile(CACertPath, caPEM, 0644); err != nil { + return tls.Certificate{}, nil, fmt.Errorf("write CA cert to %s: %w", CACertPath, err) + } + + caKeyDER, err := x509.MarshalECPrivateKey(tlsCert.PrivateKey.(*ecdsa.PrivateKey)) + if err != nil { + return tls.Certificate{}, nil, fmt.Errorf("marshal CA key: %w", err) + } + caKeyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: caKeyDER}) + // The parent directory is created with owner-only permissions before the + // key is written. /data itself is intentionally group-writable for agent + // workspaces, so 0600 on a file directly under /data is not sufficient: + // an agent could replace or race the path before the proxy opens it. + if err := os.MkdirAll(filepath.Dir(caKeyPath), 0700); err != nil { + return tls.Certificate{}, nil, fmt.Errorf("create CA key directory: %w", err) + } + if err := os.Chmod(filepath.Dir(caKeyPath), 0700); err != nil { + return tls.Certificate{}, nil, fmt.Errorf("restrict CA key directory: %w", err) + } + if err := os.WriteFile(caKeyPath, caKeyPEM, 0600); err != nil { + return tls.Certificate{}, nil, fmt.Errorf("write CA key to %s: %w", caKeyPath, err) + } + + logger.Info("generated and persisted fresh MITM CA", "certPath", CACertPath, "keyPath", caKeyPath) + return tlsCert, x509Cert, nil +} diff --git a/src/crux/proxy/github_proxy_connect.go b/src/crux/proxy/github_proxy_connect.go new file mode 100644 index 0000000000..0b1ace8cb8 --- /dev/null +++ b/src/crux/proxy/github_proxy_connect.go @@ -0,0 +1,672 @@ +package proxy + +import ( + "bufio" + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "strings" + "syscall" + "time" +) + +// hostNeedsMITM decides whether a GitHub-family host must be TLS-intercepted +// rather than opaquely tunneled. +// +// Without injection this is exactly the historical NeedsMITM: only +// api.github.com, where request-level ACMM inspection happens; github.com and +// registered GHE hosts are tunneled without decryption (their traffic β€” git +// smart HTTP, OAuth device flow β€” is gated elsewhere). +// +// With injection (#1861) an opaque tunnel is a hole, not an optimization: the +// agent's git credential helper now serves the inert placeholder, so a +// git-over-HTTPS push through an un-MITM'd tunnel would reach GitHub carrying +// the placeholder and 401. Every GitHub-family host must therefore be +// intercepted so rewriteGitHubAuth can replace the placeholder with the real +// scoped token β€” which also means an agent-supplied credential can never +// sneak to ANY GitHub host through a tunnel the proxy declined to open. +func (p *GitHubProxy) hostNeedsMITM(host string) bool { + if NeedsMITM(host) { + return true + } + return p.injectGHAuth && IsGitHubHost(host) +} + +// hostNeedsMITMFor is hostNeedsMITM for a known caller. A lane under +// write_surface.enforce has every GitHub-family host intercepted, not only +// api.github.com, because git push travels over github.com's smart HTTP and +// an opaque tunnel could not refuse it. Other lanes are unaffected. +func (p *GitHubProxy) hostNeedsMITMFor(host, agentName string) bool { + if p.hostNeedsMITM(host) { + return true + } + return IsGitHubHost(host) && p.laneWriteSurfaceEnforced(agentName) +} + +// Start begins listening. Blocks until the listener is closed. +// Handles both explicit HTTP CONNECT proxy requests and transparent +// iptables-redirected TLS connections (detected by TLS ClientHello). +func (p *GitHubProxy) Start() error { + ln, err := net.Listen("tcp", p.listenAddr) + if err != nil { + return fmt.Errorf("listen %s: %w", p.listenAddr, err) + } + defer func() { _ = ln.Close() }() + p.logger.Info("proxy listening", "addr", p.listenAddr) + + for { + conn, err := ln.Accept() + if err != nil { + return fmt.Errorf("accept: %w", err) + } + go p.handleConn(conn) + } +} + +// handleConn peeks at the first byte to distinguish HTTP CONNECT requests +// (explicit proxy) from raw TLS ClientHello (iptables-redirected traffic). +func (p *GitHubProxy) handleConn(conn net.Conn) { + defer func() { _ = conn.Close() }() + + peeked := make([]byte, 1) + _ = conn.SetReadDeadline(time.Now().Add(transparentProxyTimeout)) + n, err := conn.Read(peeked) + _ = conn.SetReadDeadline(time.Time{}) + if err != nil || n == 0 { + return + } + + // TLS ClientHello starts with byte 0x16 (ContentType handshake). + // HTTP methods start with ASCII letters (C for CONNECT, G for GET, etc.). + const tlsHandshakeContentType = 0x16 + if peeked[0] == tlsHandshakeContentType { + p.handleTransparentTLS(conn, peeked) + return + } + + // Parse the HTTP request directly instead of using http.Server.Serve, + // which closes the connection on shutdown β€” racing with hijacked CONNECT + // handlers. + prefixed := &prefixConn{Conn: conn, prefix: peeked[:n]} + _ = conn.SetReadDeadline(time.Now().Add(httpReadTimeout)) + req, err := http.ReadRequest(bufio.NewReader(prefixed)) + _ = conn.SetReadDeadline(time.Time{}) + if err != nil { + return + } + + if req.Method == http.MethodConnect { + p.handleConnectDirect(conn, req) + } else { + p.forwardPlainDirect(conn, req) + } +} + +const ( + transparentProxyTimeout = 5 * time.Second + httpReadTimeout = 30 * time.Second + httpWriteTimeout = 60 * time.Second + // upstreamDialTimeout bounds the proxy's OWN upstream dial + TLS handshake + // to the real GitHub host. Before this, a stalled upstream (DNS, TCP, or + // TLS) blocked forever β€” the agent's curl sat in poll() waiting for the + // MITM response that never came, permanently wedging the request and + // freezing the agent's session (the multi-hour "egress hang"). + // tls.DialWithDialer's dialer Timeout covers connect + handshake; the + // deadline is lifted once established so long-lived relays (git smart + // HTTP) are unaffected. + upstreamDialTimeout = 15 * time.Second + + // clientTLSHandshakeTimeout bounds the proxy's TLS handshake with the + // inbound client. It intentionally matches upstreamDialTimeout: both are + // short setup phases, unlike long-lived request/response relays. + clientTLSHandshakeTimeout = upstreamDialTimeout +) + +// handleTransparentTLS handles iptables-redirected connections. The agent +// tried to connect to github.com:443 but iptables sent it here instead. +// We extract the SNI hostname from the TLS ClientHello, then MITM the connection. +func (p *GitHubProxy) handleTransparentTLS(conn net.Conn, peeked []byte) { + // Read the whole ClientHello TLS record before extracting SNI and replaying + // it. TCP guarantees nothing about record boundaries: a single Read can + // return only PART of the ClientHello. A partial read makes extractSNI miss + // the SNI (so the wrong host is chosen and the wrong cert is forged) and + // mis-frames the record stream handed to tls.Server β€” the agent-side + // handshake then fails deterministically (bad certificate / bad record MAC), + // which is exactly why every Copilot agent could not authenticate. Loop until + // the full record (per the record-layer length) is buffered. + _ = conn.SetReadDeadline(time.Now().Add(transparentProxyTimeout)) + fullBuf, err := readClientHelloRecord(conn, peeked) + _ = conn.SetReadDeadline(time.Time{}) + if err != nil { + return + } + + host := extractSNI(fullBuf) + if host == "" { + host = "github.com" + } + + agentName := p.attributeTransparentConn(conn) + + // Copilot completion host under iptables redirection: MITM to read live + // token usage (same guard as the explicit-CONNECT path). The peeked + // ClientHello is replayed via prefixConn so the TLS handshake sees the full + // record. + if IsCopilotAPIHost(host) && p.tokenSink != nil && agentName != "" { + p.sniffCopilotOnTLS(&prefixConn{Conn: conn, prefix: fullBuf}, host, agentName) + return + } + + if !NeedsInspection(host) && !p.hostNeedsMITMFor(host, agentName) { + // Host we neither inspect nor (under #1861 injection) intercept: + // tunnel directly. SO_MARK the socket + // so the forced-egress redirect exempts this proxy-originated dial. + upstream, err := p.dialTunnelUpstream(host) + if err != nil { + return + } + defer func() { _ = upstream.Close() }() + if _, err := upstream.Write(fullBuf); err != nil { + return + } + relayTunnel(conn, upstream) + return + } + + mode := readAgentMode(agentName) + caps := readAgentCaps(agentName) + + // MITM: forge a cert, TLS-wrap the client, connect to real upstream. + tlsCert, err := p.forgeCert(host) + if err != nil { + p.logger.Error("transparent proxy forge cert failed", "host", host, "error", err) + return + } + + tlsConfig := &tls.Config{Certificates: []tls.Certificate{tlsCert}} + prefixed := &prefixConn{Conn: conn, prefix: fullBuf} + tlsClientConn := tls.Server(prefixed, tlsConfig) + // Bound the client TLS handshake: a client that connects and then stops + // sending handshake bytes would otherwise wedge this connection forever + // (empty queues on both sides β€” the observed egress-hang signature). + _ = conn.SetDeadline(time.Now().Add(clientTLSHandshakeTimeout)) + handshakeErr := tlsClientConn.Handshake() + _ = conn.SetDeadline(time.Time{}) + if handshakeErr != nil { + if isClientHangup(handshakeErr) { + p.logger.Debug("proxy client hung up before TLS handshake", "error", handshakeErr) + } else { + p.logger.Warn("transparent proxy TLS handshake failed", "error", handshakeErr) + } + return + } + defer func() { _ = tlsClientConn.Close() }() + + upstreamConn, err := tls.DialWithDialer(markDialer(upstreamDialTimeout), "tcp", host+":443", &tls.Config{ServerName: host}) + if err != nil { + p.logger.Error("transparent proxy upstream dial failed", "host", host, "error", err) + return + } + defer func() { _ = upstreamConn.Close() }() + + p.proxyHTTPHost(tlsClientConn, upstreamConn, host, agentName, mode, caps) +} + +// attributeTransparentConn names the agent behind an iptables-redirected +// connection (#7793). Under transparent redirection there is no +// Proxy-Authorization header and no CONNECT request, so the ONLY identity +// evidence is the kernel socket table: the peer's ephemeral port is looked up +// in /proc/net/tcp{,6} for the owning UID, and the UID in the agent map. +// +// It returns "" β€” "unidentified", which the mode gate downgrades to ADVISORY β€” +// when there is no UID map, when the peer address carries no port (a net.Pipe +// or unix socket), when the port is not in the socket table, or when the UID +// belongs to no agent. The one deliberate exception is the hive's own control +// plane: a UID the map reports as internal (root or the proxy user, and never +// an allocated agent UID) is named internalCallerName so a hive-originated +// write is attributed rather than mistaken for a UID-attribution failure. +func (p *GitHubProxy) attributeTransparentConn(conn net.Conn) string { + if p.uidMap == nil { + return "" + } + _, portStr, splitErr := net.SplitHostPort(conn.RemoteAddr().String()) + if splitErr != nil { + return "" + } + port := 0 + const maxPort = 65535 + for _, c := range portStr { + port = port*10 + int(c-'0') + if port > maxPort { + port = 0 + break + } + } + uid, lookupErr := LookupUIDByLocalPort(port) + if lookupErr != nil { + return "" + } + if name := p.uidMap.LookupByUID(uid); name != "" { + return name + } + // The hive's own control plane is not an agent, so the lookup above + // returns "". Under v4's forced-proxy egress that made every + // hive-originated write β€” notably the App installation token mint β€” look + // like an unattributable agent, which the mode gate blocks. Name it + // explicitly so it is attributed rather than mistaken for a + // UID-attribution failure. + if p.uidMap.IsInternalUID(uid) { + return internalCallerName + } + return "" +} + +// dialTunnelUpstream opens the raw TCP leg of a non-inspected transparent +// tunnel. The socket is SO_MARKed so the forced-egress redirect exempts this +// proxy-originated dial. tunnelDial, when set, overrides it (tests). +func (p *GitHubProxy) dialTunnelUpstream(host string) (net.Conn, error) { + if p.tunnelDial != nil { + return p.tunnelDial(host) + } + return markDialer(transparentProxyTimeout).Dial("tcp", host+":443") +} + +const tlsClientHelloMaxSize = 4096 + +// tlsRecordHeaderLen is the size of the TLS record header: content type(1) + +// version(2) + length(2). +const tlsRecordHeaderLen = 5 + +// readClientHelloRecord reads the complete first TLS record (the ClientHello) +// off conn, starting from the already-peeked bytes, and returns the full record +// so the caller can both extract the SNI and replay it verbatim into tls.Server. +// +// The prior implementation did a SINGLE conn.Read after the 1-byte peek and +// assumed it captured the whole ClientHello. TCP makes no such guarantee: when +// the ClientHello arrives split across segments, that single read returns only a +// prefix. extractSNI then parses a truncated ClientHello, fails to find the SNI, +// falls back to the wrong host, and the forged cert no longer matches what the +// agent asked for β€” so the agent-side TLS handshake fails deterministically +// (observed as "bad certificate" or "bad record MAC"), blocking every Copilot +// agent from authenticating. +// +// This reassembles the record using the record-layer length field, reading in a +// loop until the whole ClientHello record is buffered (or the buffer cap / +// deadline is hit), so both SNI extraction and the replay always see an intact +// ClientHello regardless of how TCP fragments the client's bytes. The caller +// sets/clears the read deadline around this call. +func readClientHelloRecord(conn net.Conn, peeked []byte) ([]byte, error) { + buf := make([]byte, tlsClientHelloMaxSize) + n := copy(buf, peeked) + + // First, ensure we have at least the record header so we know the record's + // declared length. + for n < tlsRecordHeaderLen { + r, err := conn.Read(buf[n:]) + n += r + if err != nil { + if n == 0 { + return nil, err + } + // Return what we have; extractSNI degrades gracefully on a short read. + return buf[:n], nil + } + } + + // record length is bytes 3..4 (big-endian); total record size adds the header. + recordLen := int(buf[3])<<8 | int(buf[4]) + need := tlsRecordHeaderLen + recordLen + if need > len(buf) { + // ClientHello larger than our cap: read to fill the buffer. The remainder + // (and subsequent records) are served to tls.Server via prefixConn+conn, + // so the handshake still completes; we only need enough here for SNI. + need = len(buf) + } + + for n < need { + r, err := conn.Read(buf[n:]) + n += r + if err != nil { + if n == 0 { + return nil, err + } + return buf[:n], nil + } + } + return buf[:n], nil +} + +// extractSNI reads the SNI hostname from a TLS ClientHello message. +func extractSNI(data []byte) string { + if len(data) < 5 { + return "" + } + // TLS record: type(1) + version(2) + length(2) + handshake + recordLen := int(data[3])<<8 | int(data[4]) + if len(data) < 5+recordLen { + // Partial read β€” use what we have + recordLen = len(data) - 5 + } + handshake := data[5 : 5+recordLen] + if len(handshake) < 4 { + return "" + } + // Handshake: type(1) + length(3) + ClientHello + hsLen := int(handshake[1])<<16 | int(handshake[2])<<8 | int(handshake[3]) + if len(handshake) < 4+hsLen { + hsLen = len(handshake) - 4 + } + ch := handshake[4 : 4+hsLen] + if len(ch) < 34 { + return "" + } + // Skip: version(2) + random(32) + pos := 34 + // Session ID + if pos >= len(ch) { + return "" + } + sessLen := int(ch[pos]) + pos += 1 + sessLen + // Cipher suites + if pos+2 > len(ch) { + return "" + } + csLen := int(ch[pos])<<8 | int(ch[pos+1]) + pos += 2 + csLen + // Compression methods + if pos >= len(ch) { + return "" + } + cmLen := int(ch[pos]) + pos += 1 + cmLen + // Extensions + if pos+2 > len(ch) { + return "" + } + extLen := int(ch[pos])<<8 | int(ch[pos+1]) + pos += 2 + extEnd := pos + extLen + if extEnd > len(ch) { + extEnd = len(ch) + } + for pos+4 <= extEnd { + extType := int(ch[pos])<<8 | int(ch[pos+1]) + eLen := int(ch[pos+2])<<8 | int(ch[pos+3]) + pos += 4 + if pos+eLen > extEnd { + break + } + if extType == 0 { // SNI extension + sniData := ch[pos : pos+eLen] + if len(sniData) < 2 { + break + } + sniListLen := int(sniData[0])<<8 | int(sniData[1]) + _ = sniListLen + sniPos := 2 + for sniPos+3 <= len(sniData) { + nameType := sniData[sniPos] + nameLen := int(sniData[sniPos+1])<<8 | int(sniData[sniPos+2]) + sniPos += 3 + if sniPos+nameLen > len(sniData) { + break + } + if nameType == 0 { // host_name + return string(sniData[sniPos : sniPos+nameLen]) + } + sniPos += nameLen + } + } + pos += eLen + } + return "" +} + +// prefixConn wraps a net.Conn and prepends already-read bytes to the stream. +type prefixConn struct { + net.Conn + prefix []byte + offset int +} + +func (c *prefixConn) Read(b []byte) (int, error) { + if c.offset < len(c.prefix) { + n := copy(b, c.prefix[c.offset:]) + c.offset += n + return n, nil + } + return c.Conn.Read(b) +} + +// handleConnectDirect handles CONNECT requests on a raw connection (no http.Server). +func (p *GitHubProxy) handleConnectDirect(conn net.Conn, r *http.Request) { + host, _, err := net.SplitHostPort(r.Host) + if err != nil { + host = r.Host + } + + // Identify from the connection, NOT from r.RemoteAddr: this request came + // from http.ReadRequest, which leaves RemoteAddr empty. + agentName := p.identifyAgentFromConn(conn, r) + + // Anthropic hosts with an inference route: reroute to self-hosted backend. + if IsAnthropicHost(host) { + if route := p.inference.Get(agentName); route != nil { + p.handleAnthropicReroute(conn, r, host, agentName, route) + return + } + } + + // Copilot completion host: MITM to read live token usage. Only when a token + // sink is active and the agent is identified β€” otherwise fall through to an + // opaque tunnel so Copilot traffic is never broken by usage capture. + if IsCopilotAPIHost(host) && p.tokenSink != nil && agentName != "" { + p.handleCopilotSniff(conn, host, agentName) + return + } + + // Hosts we do not inspect: tunnel without interception. github.com is in + // this set β€” its OAuth device flow and git smart HTTP are handled by CLI + // --deny-tool flags. api.github.com and api.linear.app are not: both need + // request-level inspection for ACMM enforcement. Under #1861 injection, + // however, every GitHub-family host is intercepted (see hostNeedsMITM), + // because the agent's credential helper now serves an inert placeholder + // that only the proxy can replace with the real scoped token. + if !NeedsInspection(host) && !p.hostNeedsMITMFor(host, agentName) { + p.tunnelDirect(conn, r) + return + } + + mode := readAgentMode(agentName) + caps := readAgentCaps(agentName) + + // Tell client the tunnel is established. + if _, err := fmt.Fprintf(conn, "HTTP/1.1 200 Connection established\r\n\r\n"); err != nil { + p.logger.Error("proxy CONNECT response write failed", "error", err) + return + } + + // Generate a cert for the target host signed by our CA. + tlsCert, err := p.forgeCert(host) + if err != nil { + p.logger.Error("proxy forge cert failed", "host", host, "error", err) + return + } + + // TLS handshake with client (presenting our forged cert). + tlsClientConn := tls.Server(conn, &tls.Config{ + Certificates: []tls.Certificate{tlsCert}, + }) + // Bound the client TLS handshake (see handleTransparentTLS): a client that + // stops mid-handshake must not wedge this connection forever. + _ = conn.SetDeadline(time.Now().Add(clientTLSHandshakeTimeout)) + handshakeErr := tlsClientConn.Handshake() + _ = conn.SetDeadline(time.Time{}) + if handshakeErr != nil { + if isClientHangup(handshakeErr) { + p.logger.Debug("proxy client hung up before TLS handshake", "error", handshakeErr) + } else { + p.logger.Warn("proxy client TLS handshake failed", "error", handshakeErr) + } + return + } + defer func() { _ = tlsClientConn.Close() }() + + // Connect to the real GitHub server. SO_MARK the socket so the forced-egress + // redirect exempts this proxy-originated dial (see proxyEgressMark). + // A dialer Timeout bounds both the TCP connect and the TLS handshake β€” + // without it a stalled upstream wedges this CONNECT forever (the agent's + // curl sits in poll() waiting for "200 Connection established" that never + // comes). The deadline is lifted once established so the relay is not + // time-boxed. + upstreamConn, err := tls.DialWithDialer(markDialer(upstreamDialTimeout), "tcp", r.Host, &tls.Config{ + ServerName: host, + }) + if err != nil { + p.logger.Error("proxy upstream dial failed", "host", r.Host, "error", err) + return + } + defer func() { _ = upstreamConn.Close() }() + + // Proxy HTTP requests, inspecting each one. + p.proxyHTTPHost(tlsClientConn, upstreamConn, host, agentName, mode, caps) +} + +// proxyHTTP reads HTTP requests from the client, checks them against +// mode rules, and either forwards or blocks them. +func isTimeoutError(err error) bool { + var netErr net.Error + return errors.As(err, &netErr) && netErr.Timeout() +} + +// logUpstreamEvent logs an upstream timeout event at the given level. +func (p *GitHubProxy) logUpstreamEvent(level slog.Level, msg string, err error, attrs ...any) { + if !isTimeoutError(err) { + return + } + attrs = append(attrs, "error", err) + p.logger.Log(context.Background(), level, msg, attrs...) +} + +func (p *GitHubProxy) logTimeout(msg string, err error, attrs ...any) { + p.logUpstreamEvent(slog.LevelWarn, msg, err, attrs...) +} + +// isClientHangup reports a client that went away (EOF, reset, broken pipe) or +// stalled past the deadline: routine on restarts and liveness probes, not a +// hive defect. +func isClientHangup(err error) bool { + return isTimeoutError(err) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || + errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.EPIPE) || errors.Is(err, net.ErrClosed) +} + +// logClientRead logs a client-side request read failure at Debug: an idle or +// departed client is expected, unlike an upstream stall. +func (p *GitHubProxy) logClientRead(msg string, err error, attrs ...any) { + p.logUpstreamEvent(slog.LevelDebug, msg, err, attrs...) +} + +// tunnelDirect creates a raw TCP tunnel for non-GitHub CONNECT requests. +func (p *GitHubProxy) tunnelDirect(conn net.Conn, r *http.Request) { + const tunnelDialTimeout = 10 * time.Second + // SO_MARK the socket so the forced-egress redirect exempts this + // proxy-originated tunnel dial (CONNECT targets include :443). + upstream, err := markDialer(tunnelDialTimeout).Dial("tcp", r.Host) + if err != nil { + p.logger.Warn("proxy: CONNECT dial failed", "host", r.Host, "error", err) + _, _ = fmt.Fprintf(conn, "HTTP/1.1 502 Bad Gateway\r\n\r\nconnection failed\n") + return + } + defer func() { _ = upstream.Close() }() + + _, _ = fmt.Fprintf(conn, "HTTP/1.1 200 Connection established\r\n\r\n") + + relayTunnel(conn, upstream) +} + +// forwardPlainDirect handles non-CONNECT (plain HTTP) requests on a raw +// connection. +// +// Two policies apply here that the CONNECT path has always applied and this +// one did not: the caller is identified from the connection, and the +// destination must be globally routable (see plain_forward_guard.go). Before +// that, any process that could reach the proxy port could have it open a +// connection, from the proxy's own uid, to any address it named β€” including +// the hive's loopback services, the cloud metadata endpoint and the +// cluster-internal estate β€” and nothing in the log said who asked. +func (p *GitHubProxy) forwardPlainDirect(conn net.Conn, r *http.Request) { + // Identify from the connection, NOT from r.RemoteAddr: this request came + // from http.ReadRequest, which leaves RemoteAddr empty (see + // identifyAgentFromConn). "" means the caller could not be attributed, + // which is logged as such rather than guessed at. + agentName := p.identifyAgentFromConn(conn, r) + + // Proxy-* hop headers address THIS proxy and must not travel further: the + // identity they assert is the one identifyAgentFromConn just decided + // whether to believe, and forwarding it would hand an agent's claimed name + // (and, in the Basic form, whatever it put in the password field) to every + // plain-HTTP host it can name. + r.Header.Del("Proxy-Authorization") + r.Header.Del("Proxy-Connection") + + resp, err := p.plainForwardTransport().RoundTrip(r) + if err != nil { + var denied *plainForwardDenied + if errors.As(err, &denied) { + p.refusePlainForward(conn, r, agentName, denied) + return + } + if isLoopbackHost(r.URL.Host) { + p.logger.Debug("proxy: plain HTTP forward to loopback failed", "agent", agentName, "url", r.URL.String(), "error", err, "hint", "client should bypass the proxy for loopback; NO_PROXY") + } else { + p.logger.Warn("proxy: plain HTTP forward failed", "agent", agentName, "url", r.URL.String(), "error", err) + } + _, _ = fmt.Fprintf(conn, "HTTP/1.1 502 Bad Gateway\r\n\r\nupstream request failed\n") + return + } + defer func() { _ = resp.Body.Close() }() + _ = resp.Write(conn) +} + +// refusePlainForward answers a destination the guard barred. +// +// Loopback is treated as the misconfiguration it almost always is rather than +// as an attack: agents are handed NO_PROXY=127.0.0.1,localhost,::1 (#14076), +// so a loopback request arriving here means a client ignored it. It is logged +// at Debug with the same hint as the failure path and does NOT count as a +// violation. Every other class β€” link-local (cloud metadata), private, CGNAT, +// multicast β€” is an attempt to reach the estate behind the proxy and is +// counted against the caller. +func (p *GitHubProxy) refusePlainForward(conn net.Conn, r *http.Request, agentName string, denied *plainForwardDenied) { + if denied.reason == plainForwardReasonLoopback { + p.logger.Debug("proxy: plain HTTP forward to loopback refused (clients must bypass the proxy for loopback; NO_PROXY)", + "agent", agentName, "url", r.URL.String(), "target", denied.target) + } else { + p.logger.Warn("proxy: plain HTTP forward refused β€” destination is not globally routable", + "agent", agentName, "method", r.Method, "url", r.URL.String(), "target", denied.target, "class", denied.reason) + p.countViolation(agentName) + } + _, _ = fmt.Fprintf(conn, "HTTP/1.1 403 Forbidden\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n"+ + "the hive proxy does not forward plain HTTP to %s addresses (%s); set %s=true to allow it\n", + denied.reason, denied.target, plainForwardAllowPrivateEnv) +} + +// isLoopbackHost reports whether a host or host:port refers to the loopback interface. +func isLoopbackHost(hostport string) bool { + host := hostport + if h, _, err := net.SplitHostPort(hostport); err == nil { + host = h + } + host = strings.Trim(host, "[]") + if strings.EqualFold(host, "localhost") { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} diff --git a/src/crux/proxy/github_proxy_copilot.go b/src/crux/proxy/github_proxy_copilot.go new file mode 100644 index 0000000000..f4ba208f26 --- /dev/null +++ b/src/crux/proxy/github_proxy_copilot.go @@ -0,0 +1,294 @@ +package proxy + +import ( + "bufio" + "bytes" + "crypto/tls" + "fmt" + "io" + "net" + "net/http" + "strings" + "time" +) + +// dialCopilotUpstream connects to the real Copilot host over TLS (or the test +// seam when set). +func (p *GitHubProxy) dialCopilotUpstream(host string) (net.Conn, error) { + if p.copilotDial != nil { + return p.copilotDial(host) + } + // SO_MARK the outbound socket so the forced-egress redirect exempts the + // proxy's own upstream traffic (see proxyEgressMark). The override hook above + // still wins when set, so tests are unaffected. + return tls.DialWithDialer(markDialer(upstreamDialTimeout), "tcp", net.JoinHostPort(host, "443"), &tls.Config{ServerName: host}) +} + +// copilotInternalPathPrefix marks the Copilot CLI's auth-exchange endpoints on +// the GitHub API host (/copilot_internal/v2/token, /copilot_internal/user). +// The CLI calls them with the USER's Copilot OAuth token +// (COPILOT_GITHUB_TOKEN / the /data/copilot-user-token device-flow token) to +// obtain its short-lived Copilot session token and discover its completion +// host. A GitHub App installation token cannot perform that exchange, so +// rewriting these requests would cut every copilot-backend agent off from its +// model the moment injection is on (#9586). They are passed through with the +// agent's own Authorization untouched - nothing stripped, nothing injected. +// +// Residual (documented in security-model.md): the Copilot user OAuth token +// stays agent-readable and spendable on these paths; moving it server-side too +// is a follow-up. The hub-held App tokens injection protects are never +// attached here. +const copilotInternalPathPrefix = "/copilot_internal/" + +// gheAPIPathPrefix is the REST prefix GitHub Enterprise Server serves its API +// under (https:///api/v3/...), so the Copilot exchange on a GHE host +// arrives as /api/v3/copilot_internal/... +const gheAPIPathPrefix = "/api/v3" + +// isCopilotAuthExchangePath reports whether path is a Copilot CLI +// auth-exchange endpoint that must keep the agent's own credential (see +// copilotInternalPathPrefix), on api.github.com or a GHE /api/v3 host. +func isCopilotAuthExchangePath(path string) bool { + return strings.HasPrefix(path, copilotInternalPathPrefix) || + strings.HasPrefix(path, gheAPIPathPrefix+copilotInternalPathPrefix) +} + +// handleCopilotSniff MITMs a GitHub Copilot completion-API connection to read +// the OpenAI-shaped token usage out of each /chat/completions response, then +// records it via the same token sink the inference path uses so Copilot cost +// goes live instead of only tallying at session shutdown. +// +// Unlike the inference reroute, this does NOT translate or reroute: the client's +// request is forwarded verbatim to the real Copilot host (with one exception β€” +// streaming requests get stream_options.include_usage injected so the terminal +// SSE usage chunk is emitted), and the upstream response is streamed back +// byte-for-byte to the client. The proxy only reads a copy of the response to +// extract usage. +// +// It is only invoked when a token sink is active and the agent is identified; +// callers fall back to opaque tunneling otherwise, so a missing sink or unknown +// agent never breaks Copilot traffic. +func (p *GitHubProxy) handleCopilotSniff(conn net.Conn, host, agentName string) { + if _, err := fmt.Fprintf(conn, "HTTP/1.1 200 Connection established\r\n\r\n"); err != nil { + p.logger.Error("copilot sniff: CONNECT response failed", "error", err) + return + } + p.sniffCopilotOnTLS(conn, host, agentName) +} + +// sniffCopilotOnTLS TLS-terminates a client connection (already past any CONNECT +// handshake), dials the real Copilot upstream, and pumps requests/responses +// through proxyCopilotHTTP so usage is recorded. Shared by the explicit-CONNECT +// (handleCopilotSniff) and the transparent-TLS (iptables) entry points so the +// forge/handshake/dial logic lives in one place. clientConn must be the raw +// (pre-TLS) client socket, optionally already wrapped in a prefixConn carrying a +// peeked ClientHello. +func (p *GitHubProxy) sniffCopilotOnTLS(clientConn net.Conn, host, agentName string) { + tlsCert, err := p.forgeCert(host) + if err != nil { + p.logger.Error("copilot sniff: forge cert failed", "host", host, "error", err) + return + } + + tlsClientConn := tls.Server(clientConn, &tls.Config{Certificates: []tls.Certificate{tlsCert}}) + _ = clientConn.SetDeadline(time.Now().Add(clientTLSHandshakeTimeout)) + handshakeErr := tlsClientConn.Handshake() + _ = clientConn.SetDeadline(time.Time{}) + if handshakeErr != nil { + if isClientHangup(handshakeErr) { + p.logger.Debug("copilot sniff: client hung up before TLS handshake", "error", handshakeErr) + } else { + p.logger.Warn("copilot sniff: client TLS handshake failed", "error", handshakeErr) + } + return + } + defer func() { _ = tlsClientConn.Close() }() + + upstreamConn, err := p.dialCopilotUpstream(host) + if err != nil { + p.logger.Error("copilot sniff: upstream dial failed", "host", host, "error", err) + return + } + defer func() { _ = upstreamConn.Close() }() + + p.proxyCopilotHTTP(tlsClientConn, upstreamConn, host, agentName) +} + +// copilotSniffBodyLimit caps how much of a Copilot request/response body the +// proxy buffers in order to read the usage block. Completion payloads are well +// under this; the limit bounds memory against a pathological body. Bodies larger +// than the limit are still forwarded in full (streamed), only usage extraction +// is skipped for the oversized case. +const copilotSniffBodyLimit = 32 * 1024 * 1024 // 32 MiB + +// proxyCopilotHTTP reads HTTP requests from the client, forwards them to the +// Copilot upstream, and forwards responses back β€” extracting token usage from +// completion responses along the way. It keeps the connection alive across +// multiple request/response pairs (HTTP keep-alive), exiting when either side +// closes. +func (p *GitHubProxy) proxyCopilotHTTP(client net.Conn, upstream net.Conn, host, agentName string) { + clientBuf := bufio.NewReader(client) + upstreamBuf := bufio.NewReader(upstream) + + for { + _ = client.SetReadDeadline(time.Now().Add(httpReadTimeout)) + req, err := http.ReadRequest(clientBuf) + if err != nil { + _ = client.SetReadDeadline(time.Time{}) + p.logClientRead("copilot sniff: client request read timed out", err, "agent", agentName, "host", host) + return + } + + isCompletion := isCopilotCompletionsPath(req.URL.Path) + isAnthropic := isCopilotAnthropicMessagesPath(req.URL.Path) + model := "" + + // For completion requests, buffer the body so we can (a) learn the + // model id and (b) inject include_usage on streaming requests. Other + // requests (model listings, etc.) are forwarded without buffering. + if isCompletion && req.Body != nil { + body, readErr := io.ReadAll(io.LimitReader(req.Body, copilotSniffBodyLimit)) + _ = req.Body.Close() + if readErr != nil { + _ = client.SetReadDeadline(time.Time{}) + p.logTimeout("copilot sniff: request body read timed out", readErr, "agent", agentName, "host", host, "path", req.URL.Path) + return + } + var streaming bool + model, streaming = parseCopilotRequest(body) + // stream_options is OpenAI-only; Anthropic streams always carry + // usage in message_start/message_delta. + if streaming && !isAnthropic { + body = ensureStreamUsageRequested(body) + } + req.Body = io.NopCloser(bytes.NewReader(body)) + req.ContentLength = int64(len(body)) + req.Header.Set("Content-Length", fmt.Sprintf("%d", len(body))) + } + + _ = upstream.SetWriteDeadline(time.Now().Add(httpWriteTimeout)) + if err := req.Write(upstream); err != nil { + _ = upstream.SetWriteDeadline(time.Time{}) + _ = client.SetReadDeadline(time.Time{}) + p.logTimeout("copilot sniff: request relay timed out", err, "agent", agentName, "host", host, "path", req.URL.Path) + return + } + _ = upstream.SetWriteDeadline(time.Time{}) + _ = client.SetReadDeadline(time.Time{}) + + _ = upstream.SetReadDeadline(time.Now().Add(httpReadTimeout)) + resp, err := http.ReadResponse(upstreamBuf, req) + _ = upstream.SetReadDeadline(time.Time{}) + if err != nil { + p.logTimeout("copilot sniff: upstream response read timed out", err, "agent", agentName, "host", host, "path", req.URL.Path) + return + } + + if isAnthropic && resp.StatusCode >= 200 && resp.StatusCode < 300 { + if err := p.forwardCopilotAnthropicResponseWithUsage(client, resp, host, agentName, model); err != nil { + p.logTimeout("copilot sniff: response relay timed out", err, "agent", agentName, "host", host, "path", req.URL.Path) + return + } + } else if isCompletion && resp.StatusCode >= 200 && resp.StatusCode < 300 { + p.forwardCopilotResponseWithUsage(client, resp, host, agentName, model) + } else { + // Non-completion and non-2xx responses are relayed verbatim. The + // Write below can legitimately stay open for a long time, so it + // must be bounded by stall, not by total duration β€” see + // stallBoundedWriter. + if err := resp.Write(&stallBoundedWriter{conn: client, idle: httpWriteTimeout}); err != nil { + _ = resp.Body.Close() + _ = client.SetWriteDeadline(time.Time{}) + p.logTimeout("copilot sniff: response relay timed out", err, "agent", agentName, "host", host, "path", req.URL.Path) + return + } + _ = client.SetWriteDeadline(time.Time{}) + _ = resp.Body.Close() + } + + if req.Close || resp.Close { + return + } + } +} + +// forwardCopilotAnthropicResponseWithUsage relays an Anthropic Messages +// (/v1/messages) response to the client while teeing the body through a usage +// parser, so streamed Claude completions are forwarded as they arrive rather +// than buffered. Usage (including prompt-cache counts) is recorded once the +// body has been fully relayed. resp.Body is consumed and closed here. +func (p *GitHubProxy) forwardCopilotAnthropicResponseWithUsage(client net.Conn, resp *http.Response, host, agentName, model string) error { + tee := newAnthropicUsageTee(resp.Body, resp.Header.Get("Content-Type")) + resp.Body = tee + writeErr := resp.Write(&stallBoundedWriter{conn: client, idle: httpWriteTimeout}) + _ = resp.Body.Close() + _ = client.SetWriteDeadline(time.Time{}) + + // Record whatever was observed even if the client went away mid-stream: + // the upstream already billed those tokens. + u := tee.usage() + if u.total() > 0 { + resolvedModel := model + if resolvedModel == "" || resolvedModel == copilotAutoModelSentinel { + resolvedModel = u.Model + } + if resolvedModel != "" && resolvedModel != copilotAutoModelSentinel { + p.recordCopilotUsageWithCache(agentName, resolvedModel, u.Input, u.Output, u.CacheRead, u.CacheCreation) + p.logger.Info("copilot usage recorded", + "agent", agentName, + "host", host, + "model", resolvedModel, + "input", u.Input, + "output", u.Output, + "cache_read", u.CacheRead, + "cache_creation", u.CacheCreation, + ) + } + } + return writeErr +} + +// forwardCopilotResponseWithUsage forwards a completion response to the client +// while extracting token usage from a buffered copy. The full body is read into +// memory (completion responses are small), usage is recorded, then the response +// is rewritten to the client verbatim. resp.Body is consumed and closed here. +func (p *GitHubProxy) forwardCopilotResponseWithUsage(client net.Conn, resp *http.Response, host, agentName, model string) { + body, err := io.ReadAll(io.LimitReader(resp.Body, copilotSniffBodyLimit)) + _ = resp.Body.Close() + if err != nil { + // Best effort: nothing to forward. + return + } + + contentType := resp.Header.Get("Content-Type") + if in, out := extractCopilotUsage(contentType, body); in > 0 || out > 0 { + resolvedModel := model + if resolvedModel == "" || resolvedModel == copilotAutoModelSentinel { + // Fall back to the model reported in the response when the request + // did not pin a concrete id (e.g. --model auto). Never record the + // "auto" sentinel β€” it is not a real model row. + if rm := extractCopilotResponseModel(body, contentType); rm != "" { + resolvedModel = rm + } + } + if resolvedModel != "" && resolvedModel != copilotAutoModelSentinel { + p.recordCopilotUsage(agentName, resolvedModel, in, out) + p.logger.Info("copilot usage recorded", + "agent", agentName, + "host", host, + "model", resolvedModel, + "input", in, + "output", out, + ) + } + } + + // Rewrite the response to the client verbatim. Replace the body with the + // buffered copy and drop Content-Length ambiguity by setting it explicitly. + resp.Body = io.NopCloser(bytes.NewReader(body)) + resp.ContentLength = int64(len(body)) + if err := resp.Write(&stallBoundedWriter{conn: client, idle: httpWriteTimeout}); err != nil { + p.logger.Warn("copilot sniff: response write to client failed", "agent", agentName, "error", err) + } + _ = client.SetWriteDeadline(time.Time{}) +} diff --git a/src/crux/proxy/github_proxy_identity.go b/src/crux/proxy/github_proxy_identity.go new file mode 100644 index 0000000000..03d95e5366 --- /dev/null +++ b/src/crux/proxy/github_proxy_identity.go @@ -0,0 +1,181 @@ +package proxy + +import ( + "encoding/base64" + "fmt" + "net" + "net/http" + "os" + "strings" + + "github.com/hivecommons/hive/crux/agent" +) + +// identifyAgentFromReq determines the agent name for a request. It always +// tries UID-based identification first (unforgeable, works for any client +// including native binaries that don't send Proxy-Authorization); see +// fallbackAgentName for what happens when that fails. +func (p *GitHubProxy) identifyAgentFromReq(r *http.Request) string { + if p.uidMap != nil { + if name := p.identifyAgentByUID(r.RemoteAddr); name != "" { + return name + } + } + return p.fallbackAgentName(r) +} + +// IdentifyAgentByUID names the agent behind an http.Server-delivered request +// from the socket owner's UID ALONE. Unlike identifyAgentFromReq it never +// falls back to the caller-controlled Proxy-Authorization header, not even +// under HIVE_PROXY_ADVISORY_OK: see fallbackAgentName for why a self-asserted +// name is impersonation, and for the loopback services beside the proxy (the +// Jev decision endpoint, hivecommons/hive#8939) there is no "degraded but +// functional" mode worth that trade β€” an unidentified caller is refused. +// Returns "" when no UID map was loaded or the connection's UID is unmapped. +func (p *GitHubProxy) IdentifyAgentByUID(r *http.Request) string { + if p.uidMap == nil || r == nil { + return "" + } + return p.identifyAgentByUID(r.RemoteAddr) +} + +// identifyAgentFromConn identifies the calling agent for a request read off a +// raw connection. It MUST be used instead of identifyAgentFromReq on any path +// where the request came from http.ReadRequest rather than http.Server: +// ReadRequest does not populate Request.RemoteAddr (only http.Server does), so +// the UID lookup silently gets an empty address, finds no agent, and the caller +// falls back to ADVISORY β€” downgrading every write-capable agent to read-only. +// Taking the peer address from the connection itself removes that trap. +func (p *GitHubProxy) identifyAgentFromConn(conn net.Conn, r *http.Request) string { + if p.uidMap != nil && conn != nil { + if name := p.identifyAgentByUID(conn.RemoteAddr().String()); name != "" { + return name + } + } + return p.fallbackAgentName(r) +} + +// fallbackAgentName is reached whenever unforgeable UID-based identification +// did NOT succeed β€” either p.uidMap is nil (never loaded: no entrypoint root +// phase ran to write uid-map.json, e.g. local dev, a native/systemd install, +// or a genuine boot-ordering failure) or this specific connection's UID has +// no entry in a map that DID load (N7, #3841). +// +// Before this fix, both cases fell through to extractAgentName β€” the +// Proxy-Authorization header, which the CALLER sets. Any process that can +// reach the proxy's loopback port can send `Proxy-Authorization: hive +// ` and be identified as that name: a spoke-agent process claiming +// to be a MORE privileged agent bypasses the whole capability model this +// proxy exists to enforce, and #1861 (inject GitHub auth at the proxy so +// agents never hold tokens) would turn that spoofed identity directly into a +// minted token for the impersonated agent. That is strictly worse than +// simply refusing to identify the caller. +// +// The safe default is therefore to NOT trust the header: return "", which +// readAgentMode already treats as ModeAdvisory (the most restrictive mode β€” +// writes blocked, per the existing "agent could not be identified" WARN a few +// lines below in proxyHTTP). This is not new plumbing; it is the same path an +// unidentified caller already hits today, now reached deliberately instead of +// only when the header happens to be absent. +// +// The one deliberate exception is proxyAdvisoryOK (HIVE_PROXY_ADVISORY_OK=true) +// β€” the SAME explicit, operator-set escape hatch entrypoint.sh already uses +// for "this deployment does not enforce the full forced-egress gate" (local +// dev, a native/systemd install with no per-agent UID separation at all, +// platforms without NET_ADMIN). An operator who has already accepted a +// degraded capability model there is not newly exposed by also accepting a +// self-asserted identity within it β€” and without this exception, those +// deployments would lose agent identification (and therefore write access) +// entirely, which would be a functional regression, not a security fix. +func (p *GitHubProxy) fallbackAgentName(r *http.Request) string { + if !p.proxyAdvisoryOK { + return "" + } + return extractAgentName(r) +} + +// identifyAgentByUID reads /proc/net/tcp to find the UID of the process +// that owns the socket connected to the proxy, then looks up the agent name. +func (p *GitHubProxy) identifyAgentByUID(remoteAddr string) string { + uid, err := uidForRemoteAddr(remoteAddr) + if err != nil { + return "" + } + return p.uidMap.LookupByUID(uid) +} + +// uidForRemoteAddr resolves the UID of the process that owns the local socket +// behind remoteAddr ("host:port" as http.Server and net.Conn report it) via +// /proc/net/tcp. It is the shared first half of every UID-attribution path. +func uidForRemoteAddr(remoteAddr string) (int, error) { + _, portStr, err := net.SplitHostPort(remoteAddr) + if err != nil { + return -1, err + } + port := 0 + const maxPort = 65535 + for _, c := range portStr { + if c < '0' || c > '9' { + return -1, fmt.Errorf("remote port %q is not numeric", portStr) + } + port = port*10 + int(c-'0') + if port > maxPort { + return -1, fmt.Errorf("remote port %q out of range", portStr) + } + } + return LookupUIDByLocalPort(port) +} + +// extractAgentName reads the agent name from the Proxy-Authorization header. +// Supports "hive " (custom) and "Basic " (standard HTTP proxy auth +// sent automatically when the proxy URL contains userinfo, e.g. http://quality@host:port). +func extractAgentName(r *http.Request) string { + auth := r.Header.Get("Proxy-Authorization") + if auth == "" { + return "" + } + if strings.HasPrefix(auth, "hive ") { + return strings.TrimPrefix(auth, "hive ") + } + if strings.HasPrefix(auth, "Basic ") { + decoded, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(auth, "Basic ")) + if err != nil { + return "" + } + // Format is "username:password" β€” agent name is the username portion. + user, _, _ := strings.Cut(string(decoded), ":") + return user + } + return "" +} + +// readAgentMode reads the mode from the hot-reloadable mode file. +// readAgentCaps loads the agent's orthogonal capability set (#4492) from +// /tmp/.hive-caps-. Every failure path returns the zero value β€” no +// capabilities β€” so an unreadable, missing or malformed file degrades toward +// the tier check alone, which is the deny direction. +func readAgentCaps(agentName string) agent.AgentCapabilities { + if agentName == "" { + return agent.AgentCapabilities{} + } + data, err := os.ReadFile(capsFilePrefix + agentName) + if err != nil { + return agent.AgentCapabilities{} + } + return agent.ParseCapabilities(strings.TrimSpace(string(data))) +} + +func readAgentMode(agentName string) agent.AgentMode { + if agentName == "" { + return agent.ModeAdvisory + } + data, err := os.ReadFile(modeFilePrefix + agentName) + if err != nil { + return agent.ModeAdvisory + } + mode, ok := agent.ParseAgentMode(strings.TrimSpace(string(data))) + if !ok { + return agent.ModeAdvisory + } + return mode +} diff --git a/src/crux/proxy/github_proxy_inference.go b/src/crux/proxy/github_proxy_inference.go new file mode 100644 index 0000000000..85803f15eb --- /dev/null +++ b/src/crux/proxy/github_proxy_inference.go @@ -0,0 +1,752 @@ +package proxy + +import ( + "bufio" + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "strings" + "time" + + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/inferencehealth" +) + +// inferenceAPIKeyPrefix is what the manager prepends to the agent's name to +// form the ANTHROPIC_API_KEY it hands each inference-routed agent +// (agent/manager_env.go). The suffix is the agent's NAME, not a secret. +const inferenceAPIKeyPrefix = "sk-hive-" + +// translatorAgentName names the caller of the inference translator. +// +// The x-api-key header the Claude CLI sends is "sk-hive-", and the +// agent name is public inside the pod: every agent knows every other agent's +// name. Trusting the header alone lets any process on the loopback borrow +// another agent's inference route β€” its backend, model, upstream credential +// and usage attribution β€” which is exactly the self-asserted-identity hole +// closed for the GitHub path in fallbackAgentName and for the decision +// endpoint in IdentifyAgentByUID. So the socket owner's UID decides: +// +// - a UID mapped to an agent is authoritative; a header naming someone else +// is logged and overridden; +// - the root control plane keeps the header name β€” the hive is the party +// that issues these keys; the proxy UID cannot assert a name because its +// upstream sockets can relay agent-controlled requests; +// - an unmapped UID, or no UID map at all, accepts the header only under +// HIVE_PROXY_ADVISORY_OK, the same operator escape hatch the GitHub path +// honours for deployments without per-agent UID separation. +// +// The second return is false when the caller could not be identified and the +// request must be refused. +func (p *GitHubProxy) translatorAgentName(r *http.Request) (string, bool) { + claimed := strings.TrimPrefix(r.Header.Get("x-api-key"), inferenceAPIKeyPrefix) + if p.uidMap == nil { + if p.proxyAdvisoryOK { + return claimed, true + } + return "", false + } + uid, err := uidForRemoteAddr(r.RemoteAddr) + if err == nil { + if name := p.uidMap.LookupByUID(uid); name != "" { + if claimed != "" && claimed != name { + p.logger.Warn("inference identity: x-api-key names another agent; using the socket owner", + "agent", name, "claimed", claimed, "uid", uid) + } + return name, true + } + if uid == 0 && p.uidMap.IsInternalUID(uid) { + return claimed, true + } + } + if p.proxyAdvisoryOK { + return claimed, true + } + return "", false +} + +// SetInferenceRoute configures an agent to use a self-hosted inference backend. +// +// The route is stored immediately and the caller returns without blocking. The +// model's max context length is queried in the background because this function +// is invoked from callers that hold the agent-manager mutex (agent launch, +// SetModelOverride, SetBackendOverride). A synchronous endpoint probe here +// (up to maxModelLenQueryTimeout) would pin that mutex for its whole duration, +// stalling /api/status and other dashboard handlers β€” which on a single-replica +// pod reads to the OpenShift router as "Application is not available". The +// max-context-len only feeds token capping and defaults gracefully to 0 +// (no cap) until the async probe fills it in. +func (p *GitHubProxy) SetInferenceRoute(agentName string, route *InferenceRoute) { + p.inference.Set(agentName, route) + keySHA256 := config.APIKeySHA256(route.APIKey) + if keySHA256 != "" { + p.logger.Info("inference route set", "agent", agentName, "backend", route.Backend, "endpoint", route.Endpoint, "model", route.Model, "maxContextLen", route.MaxContextLen, "api_key_sha256", keySHA256) + } else { + p.logger.Info("inference route set", "agent", agentName, "backend", route.Backend, "endpoint", route.Endpoint, "model", route.Model, "maxContextLen", route.MaxContextLen, "api_key_sha256", "") + } + + if route.MaxContextLen == 0 { + endpoint, model, apiKey, caBundle := route.Endpoint, route.Model, route.APIKey, route.CABundle + go func() { + if maxLen := queryMaxModelLen(endpoint, model, apiKey, caBundle); maxLen > 0 { + if p.inference.UpdateMaxContextLen(agentName, endpoint, model, maxLen) { + p.logger.Info("inference route max context length resolved", "agent", agentName, "model", model, "maxContextLen", maxLen) + } + } + }() + } + + // Best-effort: learn the key's entitled model set from a LiteLLM + // key-info endpoint before any 403 happens, so the dashboard can offer + // only usable models up front. Only litellm gateways entitlement-filter + // per key; vllm/llm-d do not. The 403-parse path (recordInferenceError) + // remains the authoritative fallback when no key-info endpoint responds. + if route.Backend == "litellm" && route.APIKey != "" { + p.entitlements.rememberProbe(route.Endpoint, route.APIKey, route.CABundle) + go p.probeEntitlements(route.Endpoint, route.APIKey, route.CABundle) + } +} + +// probeEntitlements queries a LiteLLM gateway's per-key info endpoints for the +// key's entitled model set and caches it. A nil result (no key-info endpoint, +// or an unrestricted key) leaves any existing 403-derived entry untouched. +func (p *GitHubProxy) probeEntitlements(endpoint, apiKey, caBundle string) { + if !p.entitlements.beginProbe(endpoint) { + return // another probe for this endpoint is already in flight + } + defer p.entitlements.endProbe(endpoint) + transport, err := inferenceTransport(caBundle) + if err != nil { + transport = nil + } + if models := probeEntitledModels(endpoint, apiKey, transport); len(models) > 0 { + p.entitlements.set(endpoint, apiKey, models, "key-info") + p.logger.Info("litellm entitled models discovered via key-info", + "endpoint", endpoint, "count", len(models)) + } +} + +// EntitledModels returns the entitled model set known for a LiteLLM endpoint +// (from a key-info probe or a "team not allowed" 403), the signal source, and +// whether the set is known. It is the accessor the dashboard uses to narrow +// model dropdowns to what the configured key can actually use. +// +// A stale entry (older than entitlementTTL) is still returned so the served +// model list stays STABLE for an unchanged key β€” a filtered list that flapped +// back to the full catalog between probes would make the dashboard's +// auto-heal (see reconcileModelsAfterDiscovery, #1848) see models reappear +// and vanish across refreshes. Staleness instead triggers a bounded +// background re-probe that overwrites the entry only on a fresh signal. +func (p *GitHubProxy) EntitledModels(endpoint string) (models []string, source string, known bool) { + models, source, known, stale := p.entitlements.get(endpoint) + if stale { + if ep, apiKey, caBundle, ok := p.entitlements.probeParams(endpoint); ok { + go p.probeEntitlements(ep, apiKey, caBundle) + } + } + return models, source, known +} + +// recordInferenceError inspects an upstream inference error body for a +// LiteLLM "team not allowed to access model" 403 and, when found, caches the +// entitled model set it lists (keyed by the route's endpoint) so the +// dashboard's model list narrows to the usable set. Only a 403 carrying the +// gateway's team-scope marker is treated as an entitlement signal β€” +// provider-side failures relayed by the gateway (upstream 404 not-found, 400 +// bad-request, 5xx) say nothing about the key's scope and are ignored, as is +// any other 403 (bad key, rate limit). +// +// It deliberately does NOT switch the agent's route itself: automatic model +// switching is governed by the dashboard's reconcile policy (#1848 β€” unpinned +// + governor suggestion, or confirmed genuine unavailability, at most once +// per list change). Once the entitled set is cached here, the dashboard's +// filtered discovery list drives that heal path for any agent left on a +// non-entitled model. +func (p *GitHubProxy) recordInferenceError(route *InferenceRoute, agentName string, status int, body []byte) { + if route == nil { + return + } + if p.gatewayHealth != nil { + p.gatewayHealth.RecordEndpointHTTPError(route.Backend, route.Endpoint, status, string(truncateBytes(body, 200)), time.Now()) + } + + // Inference-backend AUTH failure (a stale/invalid gateway key returning 401 + // on every call). Tracked for ANY inference backend, not just litellm β€” a + // self-hosted vLLM/llm-d fronted by auth can 401 too. Latches only after + // inferenceAuthFailureThreshold consecutive failures and clears on the next + // success, so a single transient blip never alarms. The stored cause is + // log-safe: the gateway's 401 body describes the rejection without echoing + // the presented token, and inferenceAuthErrorMessage never reads the key. + if isInferenceAuthStatus(status) && p.inferenceAuth != nil { + msg := inferenceAuthErrorMessage(route.Backend, status, truncateBytes(body, 200)) + p.inferenceAuth.recordFailure(msg, time.Now()) + p.logger.Warn("inference backend auth failure", + "agent", agentName, "backend", route.Backend, "endpoint", route.Endpoint, "status", status) + } + + // PROVIDER SPEND REBUFF (#4294). Checked before the 403/entitlement branch + // below because that branch returns early for every non-403, which would + // skip the 429/400 statuses a budget refusal actually arrives on. + // + // Requires the body to name spend, so an ordinary 429 throttle keeps + // flowing to the existing transient-retry path untouched. + if p.inferenceBudget != nil && isInferenceBudgetRebuff(status, body) { + msg := inferenceBudgetMessage(route.Backend, status, truncateBytes(body, 200)) + p.inferenceBudget.recordRebuff(msg, time.Now()) + p.logger.Error("inference backend refused on a spending limit", + "agent", agentName, "backend", route.Backend, "endpoint", route.Endpoint, + "status", status, "cause", msg) + } + + if route.Backend != "litellm" || status != http.StatusForbidden { + return + } + entitled := parseTeam403Models(string(body)) + if len(entitled) == 0 { + return + } + p.entitlements.set(route.Endpoint, route.APIKey, entitled, "403") + p.logger.Info("litellm entitled models learned from 403", + "agent", agentName, "endpoint", route.Endpoint, "count", len(entitled), "model", route.Model) +} + +// routeWithEntitledModel returns the route to actually forward with: when the +// gateway's entitled model set is known (key-info probe or a prior team-scope +// 403) and the route's stored model differs from an entitled id only by +// provider-prefix / dot-hyphen spelling drift, a COPY of the route carrying +// the exact entitled id is returned. LiteLLM matches the request model string +// verbatim, so without this a stored copilot-spelled id (e.g. +// "claude-sonnet-4.6" vs entitled "aws/claude-sonnet-4-6") 403s on every call +// forever (#4400) β€” while the dashboard's tolerant preselect matcher shows the +// agent on a perfectly valid model. Self-healing: the first 403 teaches the +// entitled set (recordInferenceError), so the next request resolves. +// +// When the stored model is NOT entitled at all β€” verbatim or by drift β€” and +// the backend's DefaultModel IS, the request goes out for the default instead +// (hivecommons/hive#9945). The FMA hive's agents carried a `model: +// claude-sonnet-4-6` written for their original copilot backend into a +// switch to the litellm gateway whose default_model was the only model the +// team could use; the per-agent value won, and every call 403'd forever with +// "team not allowed to access model" while the operator stared at a gateway +// card showing the right default. The gateway's own entitlement signal (the +// key-info probe or that very 403) is what authorises the substitution, so a +// model the team can't use is never sent when one it can use is configured. +// The stored route is still untouched: a key/team change that re-entitles the +// per-agent model takes effect on the next request. +// +// The stored route is never mutated: the rewrite is per-request, so a +// key/team change that re-entitles the verbatim id takes effect immediately, +// and concurrent requests race on nothing. Unknown entitlements, verbatim +// matches, and ambiguous candidates all pass the route through unchanged. +func (p *GitHubProxy) routeWithEntitledModel(route *InferenceRoute, agentName string) *InferenceRoute { + if route == nil || route.Backend != "litellm" || p.entitlements == nil { + return route + } + entitled, _, known := p.EntitledModels(route.Endpoint) + if !known { + return route + } + if entitledModelListed(entitled, route.Model) { + return route + } + if resolved, ok := resolveEntitledModelID(entitled, route.Model); ok { + p.logger.Info("inference model resolved to entitled gateway id", + "agent", agentName, "endpoint", route.Endpoint, + "configured", route.Model, "resolved", resolved) + rewritten := *route + rewritten.Model = resolved + return &rewritten + } + fallback := strings.TrimSpace(route.DefaultModel) + if fallback == "" || fallback == route.Model { + return route + } + if !entitledModelListed(entitled, fallback) { + resolved, ok := resolveEntitledModelID(entitled, fallback) + if !ok { + return route + } + fallback = resolved + } + p.logger.Warn("inference model not entitled on gateway; sending the backend default model instead", + "agent", agentName, "backend", route.Backend, "endpoint", route.Endpoint, + "configured", route.Model, "default", fallback, + "remedy", "clear the agent's model (or pick one of the gateway's entitled models) so the agent uses the gateway default") + rewritten := *route + rewritten.Model = fallback + return &rewritten +} + +// entitledModelListed reports whether model appears verbatim in entitled. +func entitledModelListed(entitled []string, model string) bool { + for _, id := range entitled { + if id == model { + return true + } + } + return false +} + +// recordInferenceSuccess records a successful inference call so a previously +// latched auth-failure signal clears β€” the self-heal for the inference-auth +// health signal. Called from every inference forward path on a 2xx response. +// A nil tracker (impossible after NewGitHubProxy, but cheap to guard) is a +// no-op. +func (p *GitHubProxy) recordInferenceSuccessFor(route *InferenceRoute) { + if route != nil && p.gatewayHealth != nil { + p.gatewayHealth.Clear(route.Backend) + } + p.recordInferenceSuccess() +} + +func (p *GitHubProxy) recordInferenceSuccess() { + if p.inferenceAuth != nil { + p.inferenceAuth.recordSuccess() + } + // Self-heal for the spend rebuff (#4294): whenever the provider's window + // resets β€” hive neither knows nor assumes the schedule; the field report's + // gateway rolled over at the key's creation time of day, not at midnight + // (see #4294) β€” the first call that succeeds takes the hive out of the + // suppressed state with no operator action. + if p.inferenceBudget != nil { + p.inferenceBudget.recordSuccess() + } +} + +// GatewayHealth reports currently failing inference gateways. +func (p *GitHubProxy) GatewayHealth() []inferencehealth.GatewayStatus { + if p == nil || p.gatewayHealth == nil { + return nil + } + return p.gatewayHealth.Snapshot() +} + +// InferenceAuthError reports the current inference-backend auth-failure signal: +// a non-empty, log-safe cause string (and the time it first latched) ONLY while +// the backend has been auth-failing for inferenceAuthFailureThreshold +// consecutive calls, empty otherwise. Wired to the dashboard/heartbeat so the +// hub can flag a hive that is silently 401'ing on every inference call. Safe on +// a nil tracker. +func (p *GitHubProxy) InferenceAuthError() (errMsg string, since time.Time) { + if p == nil || p.inferenceAuth == nil { + return "", time.Time{} + } + return p.inferenceAuth.snapshot() +} + +// InferenceBudgetExceeded reports the current provider spending-limit signal: +// a non-empty log-safe cause, when it first latched, when the most recent +// rebuff arrived, and how many rebuffs have been observed since β€” all +// zero-valued while the provider is serving normally. +// +// Callers use a non-empty cause as "this hive cannot buy inference right now", +// which is a reason to stop kicking agents and to tell the operator, NOT a +// reason to retry in a few minutes. lastRebuff is what bounds that: because +// withholding kicks also withholds the calls that would clear the latch, +// callers suppress only while lastRebuff is fresh and send a probe once it is +// stale (see inference_budget.go). Safe on a nil receiver/tracker. +func (p *GitHubProxy) InferenceBudgetExceeded() (errMsg string, since, lastRebuff time.Time, rebuffs int) { + if p == nil || p.inferenceBudget == nil { + return "", time.Time{}, time.Time{}, 0 + } + return p.inferenceBudget.snapshot() +} + +// ClearInferenceRoute removes an agent's inference backend override. +func (p *GitHubProxy) ClearInferenceRoute(agentName string) { + p.inference.Clear(agentName) + p.logger.Info("inference route cleared", "agent", agentName) +} + +// StartInferenceTranslator runs a plain HTTP server that accepts Anthropic +// Messages API requests and translates+forwards them to the configured +// inference backend. Agents use ANTHROPIC_BASE_URL=http://127.0.0.1:18444 +// to reach this server instead of api.anthropic.com. +func (p *GitHubProxy) inferenceTranslatorHandler() http.Handler { + mux := http.NewServeMux() + + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + agentName, ok := p.translatorAgentName(r) + if !ok { + p.logger.Warn("inference request from unidentified caller refused", + "remote", r.RemoteAddr, "method", r.Method, "path", r.URL.Path) + http.Error(w, `{"type":"error","error":{"type":"authentication_error","message":"caller could not be identified"}}`, http.StatusUnauthorized) + return + } + + p.logger.Info("inference request", + "agent", agentName, + "method", r.Method, + "path", r.URL.Path, + "content-type", r.Header.Get("Content-Type"), + "anthropic-version", r.Header.Get("anthropic-version"), + ) + + route := p.inference.Get(agentName) + if route == nil { + p.logger.Warn("inference no route", "agent", agentName) + http.Error(w, `{"type":"error","error":{"type":"api_error","message":"no inference route for agent"}}`, http.StatusBadGateway) + return + } + route = p.routeWithEntitledModel(route, agentName) + + body, err := io.ReadAll(r.Body) + if r.Body != nil { + _ = r.Body.Close() + } + if err != nil { + http.Error(w, `{"type":"error","error":{"type":"api_error","message":"failed to read request"}}`, http.StatusBadRequest) + return + } + + // Same path gate as the MITM reroute: only POST /v1/messages is + // translated and forwarded; CLI housekeeping is answered here. + if kind := classifyInferencePath(r.Method, r.URL.Path); kind != inferencePathMessages { + status, respBody := p.localInferenceResponse(kind, r.Method, r.URL.Path, body, agentName) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = io.WriteString(w, respBody) + return + } + + // Diagnostic: count tools in original Anthropic request. + var toolDiag struct { + Tools json.RawMessage `json:"tools"` + } + toolCount := -1 // -1 = no tools key + if json.Unmarshal(body, &toolDiag) == nil && toolDiag.Tools != nil { + var arr []json.RawMessage + if json.Unmarshal(toolDiag.Tools, &arr) == nil { + toolCount = len(arr) + } + } + p.logger.Info("inference request body", "agent", agentName, "len", len(body), "tool_count", toolCount, "preview", truncateBytes(body, 200)) + + openaiBody, err := translateAnthropicToOpenAI(body, route.Model, route.MaxContextLen, resolveInferencePreamble(route, agentName)) + if err != nil { + p.logger.Error("inference translate request failed", "agent", agentName, "error", err) + http.Error(w, fmt.Sprintf(`{"type":"error","error":{"type":"api_error","message":"translation error: %s"}}`, err.Error()), http.StatusBadGateway) + return + } + + upstreamURL := strings.TrimRight(route.Endpoint, "/") + "/v1/chat/completions" + upstreamReq, err := http.NewRequestWithContext(r.Context(), "POST", upstreamURL, bytes.NewReader(openaiBody)) + if err != nil { + http.Error(w, `{"type":"error","error":{"type":"api_error","message":"failed to create upstream request"}}`, http.StatusBadGateway) + return + } + upstreamReq.Header.Set("Content-Type", "application/json") + applyInferenceAuth(upstreamReq, route) + + p.logger.Info("inference forward", + "agent", agentName, + "backend", route.Backend, + "model", route.Model, + "url", upstreamURL, + "openai_body", truncateBytes(openaiBody, 300), + ) + + client, err := inferenceHTTPClient(route) + if err != nil { + p.logger.Error("inference client setup failed", "agent", agentName, "error", err) + http.Error(w, fmt.Sprintf(`{"type":"error","error":{"type":"api_error","message":"inference client setup failed: %s"}}`, err.Error()), http.StatusBadGateway) + return + } + resp, err := client.Do(upstreamReq) + if err != nil { + p.logger.Error("inference upstream failed", "agent", agentName, "url", upstreamURL, "error", err) + if p.gatewayHealth != nil { + p.gatewayHealth.RecordEndpointError(route.Backend, route.Endpoint, err, time.Now()) + } + unreachableMsg := inferenceBackendUnreachableMessage(err, upstreamURL) + p.recordAgentInferenceError(agentName, unreachableMsg, 0) + http.Error(w, fmt.Sprintf(`{"type":"error","error":{"type":"api_error","message":"%s"}}`, jsonEscape(unreachableMsg)), http.StatusBadGateway) + return + } + defer func() { _ = resp.Body.Close() }() + + p.logger.Info("inference upstream response", + "agent", agentName, + "status", resp.StatusCode, + "content-type", resp.Header.Get("Content-Type"), + ) + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + errBody, _ := io.ReadAll(resp.Body) + p.logger.Error("inference upstream error", + "agent", agentName, + "status", resp.StatusCode, + "url", upstreamURL, + "server", resp.Header.Get("Server"), + "body", truncateBytes(errBody, 500), + ) + // A LiteLLM "team not allowed to access model" 403 carries the + // entitled set β€” cache it so the dashboard's model list narrows + // and its reconcile policy (#1848) can heal the agent. + p.recordInferenceError(route, agentName, resp.StatusCode, errBody) + backendMsg := inferenceBackendErrorMessage(resp.StatusCode, errBody, upstreamURL, resp.Header.Get("Server")) + p.recordAgentInferenceError(agentName, backendMsg, resp.StatusCode) + anthropicErr := fmt.Sprintf(`{"type":"error","error":{"type":"api_error","message":"%s"}}`, + jsonEscape(backendMsg)) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(resp.StatusCode) + _, _ = w.Write([]byte(anthropicErr)) + return + } + + // A 2xx means the gateway accepted the key β€” clear any latched + // inference-auth failure so a hive whose key was fixed self-heals. + p.recordInferenceSuccessFor(route) + p.clearAgentInferenceError(agentName) + + isStreaming := strings.Contains(resp.Header.Get("Content-Type"), "text/event-stream") + + if isStreaming { + w.Header().Set("Content-Type", "text/event-stream; charset=utf-8") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + w.WriteHeader(http.StatusOK) + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + in, out, err := translateOpenAISSEToAnthropic(resp.Body, w, route.Model) + if err != nil { + p.logger.Error("inference SSE translation failed", "agent", agentName, "error", err) + } + if in > 0 || out > 0 { + p.recordUsage(agentName, route.Model, in, out) + } + return + } + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + http.Error(w, `{"type":"error","error":{"type":"api_error","message":"failed to read inference response"}}`, http.StatusBadGateway) + return + } + + p.logger.Info("inference upstream raw response", + "agent", agentName, + "len", len(respBody), + "preview", truncateBytes(respBody, 500), + ) + + if in, out := extractOpenAIUsage(respBody); in > 0 || out > 0 { + p.recordUsage(agentName, route.Model, in, out) + } + + translated, err := translateOpenAIResponseToAnthropic(respBody, route.Model) + if err != nil { + p.logger.Error("inference translate response failed", "agent", agentName, "error", err) + http.Error(w, `{"type":"error","error":{"type":"api_error","message":"response translation error"}}`, http.StatusBadGateway) + return + } + + p.logger.Info("inference translated response", + "agent", agentName, + "len", len(translated), + "preview", truncateBytes(translated, 500), + ) + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(translated) + }) + + return mux +} + +// StartInferenceTranslator runs the Anthropic-compatible inference translator +// on the fixed local port used by agents. +func (p *GitHubProxy) StartInferenceTranslator() error { + addr := fmt.Sprintf("127.0.0.1:%d", InferenceTranslatePort) + p.logger.Info("inference translation server starting", "addr", addr) + server := &http.Server{ + Addr: addr, + Handler: p.inferenceTranslatorHandler(), + ReadHeaderTimeout: httpReadTimeout, + } + return server.ListenAndServe() +} + +// handleAnthropicReroute performs MITM on an Anthropic API connection and +// reroutes requests to a self-hosted vLLM/llm-d endpoint, translating +// between Anthropic and OpenAI API formats. +func (p *GitHubProxy) handleAnthropicReroute(conn net.Conn, r *http.Request, host, agentName string, route *InferenceRoute) { + p.logger.Info("inference reroute", "agent", agentName, "backend", route.Backend, "host", host) + + if _, err := fmt.Fprintf(conn, "HTTP/1.1 200 Connection established\r\n\r\n"); err != nil { + p.logger.Error("inference reroute: CONNECT response failed", "error", err) + return + } + + tlsCert, err := p.forgeCert(host) + if err != nil { + p.logger.Error("inference reroute: forge cert failed", "host", host, "error", err) + return + } + + tlsConn := tls.Server(conn, &tls.Config{ + Certificates: []tls.Certificate{tlsCert}, + }) + _ = conn.SetDeadline(time.Now().Add(clientTLSHandshakeTimeout)) + handshakeErr := tlsConn.Handshake() + _ = conn.SetDeadline(time.Time{}) + if handshakeErr != nil { + p.logger.Warn("inference reroute: TLS handshake failed", "error", handshakeErr) + return + } + defer func() { _ = tlsConn.Close() }() + + clientBuf := bufio.NewReader(tlsConn) + for { + _ = tlsConn.SetReadDeadline(time.Now().Add(httpReadTimeout)) + req, err := http.ReadRequest(clientBuf) + _ = tlsConn.SetReadDeadline(time.Time{}) + if err != nil { + p.logTimeout("inference reroute: client request read timed out", err, "agent", agentName) + return + } + + p.handleInferenceRequest(tlsConn, req, agentName, route) + } +} + +// handleInferenceRequest translates a single Anthropic API request and +// forwards it to the inference backend. +func (p *GitHubProxy) handleInferenceRequest(conn net.Conn, req *http.Request, agentName string, route *InferenceRoute) { + route = p.routeWithEntitledModel(route, agentName) + _ = conn.SetReadDeadline(time.Now().Add(httpReadTimeout)) + body, err := io.ReadAll(req.Body) + _ = conn.SetReadDeadline(time.Time{}) + if req.Body != nil { + _ = req.Body.Close() + } + if err != nil { + p.logTimeout("inference reroute: request body read timed out", err, "agent", agentName) + p.writeHTTPError(conn, http.StatusBadGateway, "failed to read request body") + return + } + + // Only POST /v1/messages is an inference call. The Claude Code CLI also + // sends telemetry batches, error reports, and count_tokens to this host; + // translating those bodies produced {"messages": null} and a gateway 400 + // per call, charged against the provider's request rate limit. + if kind := classifyInferencePath(req.Method, req.URL.Path); kind != inferencePathMessages { + status, respBody := p.localInferenceResponse(kind, req.Method, req.URL.Path, body, agentName) + writeLocalInferenceResponse(conn, status, respBody) + return + } + + openaiBody, err := translateAnthropicToOpenAI(body, route.Model, route.MaxContextLen, resolveInferencePreamble(route, agentName)) + if err != nil { + p.logger.Error("inference translate request failed", "agent", agentName, "error", err) + p.writeHTTPError(conn, http.StatusBadGateway, "translation error: "+err.Error()) + return + } + + upstreamURL := strings.TrimRight(route.Endpoint, "/") + "/v1/chat/completions" + upstreamReq, err := http.NewRequestWithContext(req.Context(), "POST", upstreamURL, bytes.NewReader(openaiBody)) + if err != nil { + p.writeHTTPError(conn, http.StatusBadGateway, "failed to create upstream request") + return + } + upstreamReq.Header.Set("Content-Type", "application/json") + applyInferenceAuth(upstreamReq, route) + + p.logger.Info("inference forward", "agent", agentName, "backend", route.Backend, "model", route.Model, "url", upstreamURL) + + client, err := inferenceHTTPClient(route) + if err != nil { + p.logger.Error("inference client setup failed", "agent", agentName, "error", err) + p.writeHTTPError(conn, http.StatusBadGateway, "inference client setup failed: "+err.Error()) + return + } + resp, err := client.Do(upstreamReq) + if err != nil { + p.logger.Error("inference upstream failed", "agent", agentName, "url", upstreamURL, "error", err) + if p.gatewayHealth != nil { + p.gatewayHealth.RecordEndpointError(route.Backend, route.Endpoint, err, time.Now()) + } + unreachableMsg := inferenceBackendUnreachableMessage(err, upstreamURL) + p.recordAgentInferenceError(agentName, unreachableMsg, 0) + p.writeHTTPError(conn, http.StatusBadGateway, unreachableMsg) + return + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + errBody, _ := io.ReadAll(resp.Body) + p.logger.Error("inference upstream error", "agent", agentName, "status", resp.StatusCode, "url", upstreamURL, "server", resp.Header.Get("Server"), "body", truncateBytes(errBody, 500)) + // Same entitlement capture as the HTTP translator path. + p.recordInferenceError(route, agentName, resp.StatusCode, errBody) + backendMsg := inferenceBackendErrorMessage(resp.StatusCode, errBody, upstreamURL, resp.Header.Get("Server")) + p.recordAgentInferenceError(agentName, backendMsg, resp.StatusCode) + anthropicErr := &http.Response{ + StatusCode: resp.StatusCode, + ProtoMajor: 1, + ProtoMinor: 1, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(fmt.Sprintf( + `{"type":"error","error":{"type":"api_error","message":"%s"}}`, + jsonEscape(backendMsg)))), + } + _ = anthropicErr.Write(conn) + return + } + + // A 2xx means the gateway accepted the key β€” clear any latched + // inference-auth failure so a hive whose key was fixed self-heals. + p.recordInferenceSuccessFor(route) + p.clearAgentInferenceError(agentName) + + isStreaming := strings.Contains(resp.Header.Get("Content-Type"), "text/event-stream") + + if isStreaming { + hdr := "HTTP/1.1 200 OK\r\n" + + "Content-Type: text/event-stream; charset=utf-8\r\n" + + "Cache-Control: no-cache\r\n" + + "Connection: keep-alive\r\n\r\n" + if _, err := conn.Write([]byte(hdr)); err != nil { + return + } + in, out, err := translateOpenAISSEToAnthropic(resp.Body, conn, route.Model) + if err != nil { + p.logger.Error("inference SSE translation failed", "agent", agentName, "error", err) + } + if in > 0 || out > 0 { + p.recordUsage(agentName, route.Model, in, out) + } + return + } + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + p.writeHTTPError(conn, http.StatusBadGateway, "failed to read inference response") + return + } + + if in, out := extractOpenAIUsage(respBody); in > 0 || out > 0 { + p.recordUsage(agentName, route.Model, in, out) + } + + translated, err := translateOpenAIResponseToAnthropic(respBody, route.Model) + if err != nil { + p.logger.Error("inference translate response failed", "agent", agentName, "error", err) + p.writeHTTPError(conn, http.StatusBadGateway, "response translation error") + return + } + + httpResp := &http.Response{ + StatusCode: http.StatusOK, + ProtoMajor: 1, + ProtoMinor: 1, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewReader(translated)), + } + _ = httpResp.Write(conn) +} diff --git a/src/crux/proxy/github_proxy_mitm.go b/src/crux/proxy/github_proxy_mitm.go new file mode 100644 index 0000000000..da81467ffe --- /dev/null +++ b/src/crux/proxy/github_proxy_mitm.go @@ -0,0 +1,402 @@ +package proxy + +import ( + "encoding/json" + "io" + "net" + "net/http" + "strings" + "time" + + "github.com/hivecommons/hive/crux/agent" +) + +// proxyHTTP relays a MITM'd GitHub connection with ACMM enforcement applied. +// It is proxyHTTPHost with the GitHub host implied, which is what every +// pre-existing caller means. +func (p *GitHubProxy) proxyHTTP(client net.Conn, upstream net.Conn, agentName string, mode agent.AgentMode, caps agent.AgentCapabilities) { + p.proxyHTTPHost(client, upstream, "", agentName, mode, caps) +} + +// proxyHTTPHost is proxyHTTP with the terminated host carried through, so the +// request loop can pick the right rule engine (#4492 F). +// +// The host is threaded rather than inferred from the request because a MITM'd +// request line is a path, not an absolute URL β€” by this point the only record +// of which host the TLS session was forged for is the CONNECT that opened it. +// An empty host means GitHub, preserving every existing call site verbatim. +func (p *GitHubProxy) proxyHTTPHost(client net.Conn, upstream net.Conn, host string, agentName string, mode agent.AgentMode, caps agent.AgentCapabilities) { + isLinear := IsLinearHost(host) + clientBuf := newBufferedConn(client) + + for { + // Bound every phase of this request/response exchange. Before this, + // the relay had NO deadlines anywhere after the initial CONNECT read: + // a client or upstream that stopped mid-exchange (stalled response, + // half-written body, frozen keep-alive) blocked forever β€” the + // multi-hour "egress wedge" where the agent's request never reached + // GitHub and its session froze. Each phase gets its own deadline; + // they are cleared once the phase completes so healthy keep-alive + // connections are unaffected. + _ = client.SetReadDeadline(time.Now().Add(httpReadTimeout)) + req, err := http.ReadRequest(clientBuf) + if err != nil { + _ = client.SetReadDeadline(time.Time{}) + p.logClientRead("proxy client request read timed out", err, "agent", agentName) + return + } + + blocked := false + blockReason := "" + // blockCode is the machine-readable reason (#12127) that rides on the + // one-line 403 body and the X-Hive-Proxy-Reason header. Each gate that + // sets blockReason sets it; a gate that sets neither falls back to a + // generic code in ProxyDenialBody. + blockCode := "" + + // Per-repo pause (#6203) and per-repo agent scope (#6204) are both + // checked BEFORE the mode chain, and outside it, for the same two + // reasons. Neither is an autonomy tier β€” a pause is a run-state, a + // scope is roster membership β€” so each must refuse the write whatever + // mode the agent holds and whichever branch below would otherwise have + // allowed it. And each carries its own reason, so the agent is told the + // repo is deliberately quiet, or that it is not defined for this repo, + // instead of reading a mode error and going hunting for a permissions + // bug that does not exist. + // + // The hive's own control-plane traffic (App-token mint, heartbeat) is + // exempt from both, exactly as it is from the ACMM rules below: + // stopping the hive from minting a token would take the whole spoke + // down rather than quiet one repo or narrow one agent. + // + // Pause is checked first, so a repo the operator has frozen is refused + // as paused even when the agent is also out of scope for it. + blockJSON := false + graphQLMutation := false + // A direct write from a lane under write_surface.enforce (#9772) is + // refused and audited as agent_write_refused. Checked after pause and + // scope so their more specific reasons win when both apply. + writeSurfaceKind := "" + if agentName != internalCallerName { + if reason, paused := p.repoPauseRefusal(req.Method, repoScopePath(req)); paused { + blocked = true + blockReason = reason + blockCode = DenyReasonRepoPaused + } else if reason, refused := p.agentRepoScopeRefusal(agentName, req.Method, repoScopePath(req)); refused { + blocked = true + blockReason = reason + blockCode = DenyReasonAgentRepoScope + } else if reason, refused := p.autoMergeRefusal(req.Method, repoScopePath(req)); refused { + blocked = true + blockReason = reason + blockCode = DenyReasonRepoAutoMergeOff + blockJSON = true + } else if !isLinear { + if reason, kind, refused := WriteSurfaceEnforceRefusal(p.writeSurfaceEnforcedFunc(), agentName, req.Method, req.URL.Path, req.URL.RawQuery); refused { + blocked = true + blockReason = reason + blockCode = DenyReasonWriteSurfaceEnforced + writeSurfaceKind = kind + } + } + } + + if blocked { + // Already refused by the pause or agent-scope check above β€” the + // mode chain cannot un-block it, so skip it entirely. + } else if isLinear && agentName == internalCallerName { + // The hive's OWN Linear traffic β€” the read path #4178 shipped + // (backlog enumeration in pkg/worksource/linear.go) and, later, + // the OAuth/webhook plumbing. ACMM governs AGENT autonomy, so the + // control plane is exempt here exactly as it is for GitHub below. + // This branch must precede the enforcement one: without it, adding + // Linear to NeedsMITM would newly gate hive-originated reads that + // have always been ungated, breaking enumeration rather than + // restricting an agent. + } else if isLinear { + // Linear is a single POST /graphql for everything, so the + // (method, path) rule table cannot classify it and the GitHub + // GraphQL classifier is the wrong posture β€” it ends in a + // permissive default, backed by the REST table. Linear has no + // table underneath. See linear_rules.go. + blocked, blockReason = p.enforceLinear(req, agentName, mode, caps) + if blocked { + blockCode = DenyReasonLinear + } + } else if req.Method == "POST" && IsGraphQLPath(req.URL.Path) { + body, readErr := io.ReadAll(io.LimitReader(req.Body, graphQLBodyLimit)) + if req.Body != nil { + _ = req.Body.Close() + } + if readErr != nil { + _ = client.SetReadDeadline(time.Time{}) + p.logTimeout("proxy GraphQL request body read timed out", readErr, "agent", agentName, "path", req.URL.Path) + return + } + if reason, refused := p.graphQLAutoMergeRefusal(body); agentName != internalCallerName && refused { + blocked = true + blockReason = reason + blockCode = DenyReasonRepoAutoMergeOff + blockJSON = true + } else if agentName != internalCallerName { + if reason, refused := WriteSurfaceGraphQLRefusal(p.writeSurfaceEnforcedFunc(), agentName, body); refused { + blocked = true + blockReason = reason + blockCode = DenyReasonWriteSurfaceEnforced + writeSurfaceKind = WriteSurfaceKindGraphQL + } + } + allowed, isMutation := GraphQLAllowedCaps(autonomyGraphQLMode(agentName, body, mode), caps, body) + graphQLMutation = isMutation + if !blocked && !allowed { + blocked = true + if isMutation { + blockReason = "graphql mutation" + blockCode = DenyReasonGraphQLMutation + } else { + blockReason = "graphql" + blockCode = DenyReasonGraphQL + } + } + req.Body = io.NopCloser(strings.NewReader(string(body))) + req.ContentLength = int64(len(body)) + } else if agentName == internalCallerName { + // The hive itself has no ACMM mode to enforce β€” ACMM governs AGENT + // autonomy. Its control-plane calls (App token mint, heartbeat) + // must not be gated as if they were agent writes. Repo-filter and + // canary-egress checks below still apply. + } else if !AllowedByModeCaps(autonomyModeForRepo(agentName, ExtractRepo(repoScopePath(req)), mode), caps, req.Method, req.URL.Path) { + blocked = true + blockCode = DenyReasonMode + // An unidentified agent is silently treated as ADVISORY, which turns + // a permissions bug into an indistinguishable "policy denial". Say so + // loudly on writes: a write from an unknown caller means UID + // attribution failed, not that the agent lacks the mode. + if agentName == "" && writeMethods[req.Method] { + p.logger.Warn("proxy: agent could not be identified β€” defaulting to ADVISORY and blocking a write; UID attribution failed (check the uid map and /proc/net/tcp visibility), this is NOT an agent-mode misconfiguration", + "method", req.Method, "path", req.URL.Path) + } + // A hard-deny rule (e.g. direct PR creation) carries an agent-facing + // directive explaining the sanctioned alternative β€” surface it so the + // agent knows to use hive-open-pr rather than seeing only a mode error. + if msg, denied := DeniedMessage(req.Method, req.URL.Path); denied && msg != "" { + blockReason = msg + blockCode = DenyReasonHardDeny + } + } else if len(p.allowedRepos) > 0 && !RepoFilterAllowed(p.allowedRepos, req.Method, repoScopePath(req)) { + blocked = true + blockReason = "repo not in hive config" + blockCode = DenyReasonRepoNotInConfig + } + // Shape-gate agent-authored issue/comment REST writes. The issue-request + // watcher validates the file-drop creation path, but an agent's + // `gh issue create` becomes a raw POST /repos/{o}/{r}/issues that never + // touches the watcher (issue #7014: a template placeholder title and a + // body of literal \n escapes both reached GitHub this way). Enforce the + // same shape rules here so both creation paths are covered. Skip the + // control plane β€” its issue writes are already validated upstream. + if !blocked && agentName != internalCallerName { + if reason, deny, shapeErr := p.enforceIssueShapeFor(req, agentName); shapeErr != nil { + // Fail closed: a body we could not read is a body we could not + // shape-check. + p.logTimeout("proxy issue-shape body read timed out", shapeErr, "agent", agentName, "path", req.URL.Path) + _ = client.SetReadDeadline(time.Time{}) + return + } else if deny { + blocked = true + if blockReason == "" { + blockReason = reason + blockCode = DenyReasonIssueShape + } + } + } + if reason, deny, ok := p.inspectCanaryEgress(agentName, req); ok { + if deny { + blocked = true + if blockReason == "" { + blockReason = reason + blockCode = DenyReasonCanaryEgress + } + } + } + + if !isLinear && agentName != internalCallerName { + isGraphQL := req.Method == "POST" && IsGraphQLPath(req.URL.Path) + isWrite := writeMethods[req.Method] && !gitUploadPackPath.MatchString(req.URL.Path) + if isGraphQL { + isWrite = graphQLMutation + } + if isWrite { + reason := "" + if blocked { + reason = blockCode + if reason == "" { + reason = DenyReasonUnknown + } + } + p.writes.record(mode.String(), reason) + } + } + + if blocked { + detail := req.URL.Path + if blockReason != "" { + detail = blockReason + } + p.recordViolation(agentName, req.Method, detail) + if writeSurfaceKind != "" { + p.auditWriteRefused(agentName, writeSurfaceKind, req.Method, repoScopePath(req)) + } + + if blockCode == "" { + blockCode = DenyReasonUnknown + } + respBody := ProxyDenialBody(agentName, mode, req.Method, detail, blockCode) + contentType := "text/plain" + if blockJSON { + contentType = "application/json" + payload, _ := json.Marshal(map[string]any{ + "error": "repo_auto_merge_disabled", + "reason": detail, + }) + respBody = string(payload) + "\n" + } + resp := &http.Response{ + StatusCode: http.StatusForbidden, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(respBody)), + } + resp.Header.Set("Content-Type", contentType) + resp.Header.Set("X-Hive-Proxy-Blocked", "true") + resp.Header.Set(ProxyDenialHeader, blockCode) + _ = client.SetWriteDeadline(time.Now().Add(httpWriteTimeout)) + writeErr := resp.Write(client) + _ = client.SetWriteDeadline(time.Time{}) + if writeErr != nil { + p.logTimeout("proxy blocked response write timed out", writeErr, "agent", agentName) + return + } + + if req.Body != nil { + if _, drainErr := io.Copy(io.Discard, req.Body); drainErr != nil { + p.logTimeout("proxy blocked request body drain timed out", drainErr, "agent", agentName, "path", req.URL.Path) + } + _ = req.Body.Close() + } + _ = client.SetReadDeadline(time.Time{}) + continue + } + + if !isLinear { + if refusal := p.githubBudgetRefusal(agentName, req.Method, req.URL.Path, agentGitHubRequestIsRead(req.Method, req.URL.Path, graphQLMutation)); refusal != nil { + _ = p.writeAgentGitHubBudget429(client, *refusal) + if req.Body != nil { + _, _ = io.Copy(io.Discard, req.Body) + _ = req.Body.Close() + } + _ = client.SetReadDeadline(time.Time{}) + continue + } + } + + // #1861: with injection enabled, the upstream credential is decided + // HERE β€” after the ACMM/repo/canary gates, immediately before either + // forwarding branch β€” so every byte that reaches GitHub has passed + // through it. Blocked requests above never reach upstream and need no + // rewrite. + // + // #9586: only for GitHub hosts. This loop also relays api.linear.app + // (NeedsInspection), and rewriting there would attach the agent's + // hub-held GITHUB token to a request bound for Linear - a real + // credential handed to a third party - whenever the Linear lane below + // does not overwrite the header (an ADVISORY agent, or a hive with no + // Linear credential). + if injectsGitHubAuthForHost(host) { + p.rewriteGitHubAuth(req, agentName) + } + + // Linear: the request passed the tier gate; make sure it reaches + // Linear with the hive's CURRENT credential rather than whatever + // (possibly rotated-out) copy the agent's process still holds. + if isLinear && agentName != internalCallerName { + p.injectLinearCredential(req, agentName, mode) + } + // Git smart HTTP uses chunked streaming that http.ReadResponse + // can't handle reliably. After the ACMM check passes, forward + // the request and switch to raw bidirectional streaming. + if isGitPath(req.URL.Path) { + _ = upstream.SetWriteDeadline(time.Now().Add(httpWriteTimeout)) + if err := req.Write(upstream); err != nil { + _ = upstream.SetWriteDeadline(time.Time{}) + _ = client.SetReadDeadline(time.Time{}) + p.logTimeout("proxy git request relay timed out", err, "agent", agentName, "path", req.URL.Path) + return + } + _ = upstream.SetWriteDeadline(time.Time{}) + _ = client.SetReadDeadline(time.Time{}) + // Use relayTunnel, NOT an inline copy pair: this branch used to run + // its own unbounded io.Copy pair, which was the same wedge #3872 + // fixed in relayTunnel but left here. When GitHub FIN'd after the + // response while the client kept its side open (keep-alive), the + // clientβ†’upstream copy stayed blocked in client.Read() forever and + // `<-done` never returned β€” the handler's deferred Closes never ran, + // parking the upstream socket in CLOSE_WAIT for the life of the + // process (issue #3875's FD pile). relayTunnel bounds each direction + // once the other finishes. + relayTunnel(client, upstream) + return + } + + // Forward to upstream. The client read deadline stays in force until + // the body is fully drained (req.Write pipes req.Body from the client), + // so a client stalling mid-body cannot hang the relay. + _ = upstream.SetWriteDeadline(time.Now().Add(httpWriteTimeout)) + if err := req.Write(upstream); err != nil { + _ = upstream.SetWriteDeadline(time.Time{}) + _ = client.SetReadDeadline(time.Time{}) + p.logTimeout("proxy request relay timed out", err, "agent", agentName, "path", req.URL.Path) + return + } + _ = upstream.SetWriteDeadline(time.Time{}) + _ = client.SetReadDeadline(time.Time{}) + + _ = upstream.SetReadDeadline(time.Now().Add(httpReadTimeout)) + resp, err := http.ReadResponse(newBufferedReader(upstream), req) + _ = upstream.SetReadDeadline(time.Time{}) + if err != nil { + p.logTimeout("proxy upstream response read timed out", err, "agent", agentName, "path", req.URL.Path) + return + } + p.recordGitHubREST(agentName, req, resp) + p.recordAgentGitHubBudget(agentName, req.URL.Path, resp.StatusCode, p.agentGitHubCap(agentName)) + + // Bound the BODY relay too, not just the header read above. The + // deadline was cleared once headers arrived, so a "reachable but slow" + // GitHub β€” TLS and headers complete, body never arrives (the measured + // #3875 signature: time_total=12s, 0 body bytes) β€” parked resp.Write + // in upstream.Read() with no deadline. The client had long since given + // up (10s collect budget) and FIN'd its side, but this goroutine never + // noticed: it holds all three sockets of the exchange forever, and the + // retry loop stacks a fresh parked handler on every attempt (~1k + // FDs/min measured live). The wrapper re-arms a rolling per-Read + // deadline, so a body that keeps flowing is never cut while a stall + // longer than responseBodyStallTimeout errors out and lets the + // handler's deferred Closes reclaim the sockets. + resp.Body = &stallBoundedBody{body: resp.Body, conn: upstream, idle: p.bodyStallTimeoutDuration()} + + // Rolling per-Write deadline (see stallBoundedWriter): a response that + // keeps flowing is never cut for being long, while a client that stops + // draining still releases this handler within one stall window. + if err := resp.Write(&stallBoundedWriter{conn: client, idle: httpWriteTimeout}); err != nil { + _ = resp.Body.Close() + _ = client.SetWriteDeadline(time.Time{}) + p.logTimeout("proxy response relay timed out", err, "agent", agentName, "path", req.URL.Path) + return + } + _ = client.SetWriteDeadline(time.Time{}) + _ = resp.Body.Close() + } +} diff --git a/src/crux/proxy/github_proxy_policy.go b/src/crux/proxy/github_proxy_policy.go new file mode 100644 index 0000000000..b18a136dea --- /dev/null +++ b/src/crux/proxy/github_proxy_policy.go @@ -0,0 +1,368 @@ +package proxy + +import ( + "bytes" + "encoding/base64" + "io" + "net/http" + "strings" + + "github.com/hivecommons/hive/crux/agent" +) + +// injectLinearCredential replaces the Authorization header of an agent's +// api.linear.app request with the hive's CURRENT Linear credential, for +// agents at the ISSUES_ONLY floor and above (the tier the session-env push in +// agent.Manager.linearEnvPairs uses). It reports whether a credential was +// applied. +// +// WHY THE PROXY, NOT THE ENVIRONMENT. The connected Linear app's OAuth access +// token rotates (Linear issues it with a ~24h expiry and hive refreshes it +// in the background), and hive already re-pushes the fresh value into each +// agent's tmux session environment on the hourly refresh tick. But a tmux +// session environment only reaches processes forked AFTER the push: the +// agent's CLI, forked at its last launch, keeps the token it was born with, +// and every tool shell it spawns inherits that copy. Observed live +// (2026-09-06/07): a quality agent whose session environment held the fresh +// token β€” verified valid through this very proxy as that agent's UID β€” got +// `401 AUTHENTICATION_ERROR` on every Linear call from inside its pane for +// two days, because its process still carried the pre-rotation token. Its +// earlier kicks had silently fallen back to the operator's personal +// LINEAR_API_KEY, so every issue it filed showed the operator as creator. +// +// The proxy terminates every agent request to api.linear.app anyway (this is +// where enforceLinear gates mutations by tier), so attaching the live +// credential HERE makes the copy in the agent's environment irrelevant β€” +// stale, absent, or a placeholder all work β€” exactly as the inference +// translator attaches the real gateway key server-side. The agent still +// receives a credential in its environment (for SDKs that refuse to send a +// request without one), but Linear only ever sees the proxy's. +// +// Below the floor the request is left untouched: an ADVISORY agent's Linear +// reads have always been ungated and unauthenticated-by-hive, and injecting +// a write-capable credential there would widen what the tier can do. +func (p *GitHubProxy) injectLinearCredential(req *http.Request, agentName string, mode agent.AgentMode) bool { + if p.linearCredential == nil || !mode.CanCreateIssues() { + return false + } + cred := p.linearCredential() + switch { + case cred.AccessToken != "": + req.Header.Set("Authorization", "Bearer "+cred.AccessToken) + case cred.APIKey != "": + req.Header.Set("Authorization", cred.APIKey) + default: + return false + } + p.logger.Debug("linear: attached the hive's current credential to an agent request", "agent", agentName, "mode", mode.String()) + return true +} + +// repoScopePath is the request path handed to the per-repo write gates (pause, +// agent scope, auto-merge, per-repo ACMM mode, repo filter). It starts from +// the client's own escaping rather than the decoded req.URL.Path so a GitLab +// "namespace%2Fproject" stays one segment and ExtractRepo resolves the exact +// project β€” including nested groups β€” instead of guessing where the project +// path ends from the sub-resource that follows it. CanonicalGatePath then +// decodes every OTHER escape so an escaped structural segment (/r%65pos/, +// repo%2Egit, pro%6Aects, merg%65) cannot hide the repo from the gates. +func repoScopePath(req *http.Request) string { + return CanonicalGatePath(req.URL.EscapedPath()) +} + +// repoPauseRefusal is RepoPauseRefusal against this proxy's current predicate. +func (p *GitHubProxy) repoPauseRefusal(method, path string) (string, bool) { + p.pausedMu.RLock() + paused := p.repoPaused + p.pausedMu.RUnlock() + return RepoPauseRefusal(paused, method, path) +} + +func (p *GitHubProxy) autoMergeRefusal(method, path string) (string, bool) { + p.autoMergeMu.RLock() + enabled := p.repoAutoMergeOn + p.autoMergeMu.RUnlock() + return RepoAutoMergeRefusal(enabled, method, path) +} + +func (p *GitHubProxy) graphQLAutoMergeRefusal(body []byte) (string, bool) { + p.autoMergeMu.RLock() + enabled := p.repoAutoMergeOn + p.autoMergeMu.RUnlock() + return GraphQLAutoMergeRefusal(enabled, body) +} + +// agentRepoScopeRefusal is AgentRepoScopeRefusal against the current predicate. +func (p *GitHubProxy) agentRepoScopeRefusal(agentName, method, path string) (string, bool) { + p.scopeMu.RLock() + serves := p.agentServesRepo + p.scopeMu.RUnlock() + return AgentRepoScopeRefusal(serves, agentName, method, path) +} + +func (p *GitHubProxy) writeSurfaceEnforcedFunc() func(agent string) bool { + p.writeSurfaceMu.RLock() + defer p.writeSurfaceMu.RUnlock() + return p.writeSurfaceEnforced +} + +// laneWriteSurfaceEnforced reports whether agentName's lane is enforced. +func (p *GitHubProxy) laneWriteSurfaceEnforced(agentName string) bool { + if agentName == "" || agentName == internalCallerName { + return false + } + enforced := p.writeSurfaceEnforcedFunc() + return enforced != nil && enforced(agentName) +} + +// auditWriteRefused records one direct write refused under +// write_surface.enforce. +func (p *GitHubProxy) auditWriteRefused(agentName, kind, method, path string) { + p.writeSurfaceMu.RLock() + audit := p.writeRefusedAudit + p.writeSurfaceMu.RUnlock() + if audit == nil { + return + } + audit(agentName, kind, method, path, WriteSurfaceRepo(path)) +} + +// gitInjectBasicUser is the username git expects alongside a GitHub App +// installation token in HTTP Basic auth β€” the same identity +// git-credential-hive.sh answers with, so upstream sees exactly the shape the +// pre-injection flow produced. +const gitInjectBasicUser = "x-access-token" + +// loginPathPrefix marks the GitHub OAuth/device-flow endpoints +// (/login/device/code, /login/oauth/access_token). These authenticate the +// FLOW via their form body, not a bearer credential; injecting an App token +// there would be meaningless at best and flow-corrupting at worst, so they +// get strip-only treatment. +const loginPathPrefix = "/login/" + +// rewriteGitHubAuth enforces #1861 on one MITM'd request: the proxy β€” not the +// agent β€” decides what credential GitHub sees. +// +// Threat model, per header-touching block: +// +// - STRIP: an agent-supplied Authorization header is deleted UNCONDITIONALLY +// (under the flag), never forwarded. Even though injection-mode agents are +// only ever handed the inert placeholder, an agent that somehow obtained a +// real credential (leaked from a log, smuggled via prompt, minted through +// a side channel) must not be able to spend it through this proxy. +// +// - INJECT: only for an agent identified by the UID-authoritative path +// (#3888) that has a hub-held scoped token. The two lookups this chains +// are exactly the pre-existing lanes: identity from the UID map, token +// from the #3967 per-agent scoped mint. +// +// - UNKNOWN AGENT = NO CREDENTIAL: when identity resolution failed +// (agentName == ""), the request proceeds with NO Authorization and fails +// loud at GitHub (401/403). Deliberate: any fallback β€” shared cache, +// hive token, "last known agent" β€” would let an unattributable process +// ride a real credential, recreating the pre-#3888 identity hole this +// design depends on having closed. +// +// - COPILOT AUTH-EXCHANGE PASSTHROUGH: /copilot_internal/ on a GitHub +// host keeps the agent's own Authorization (the user's Copilot OAuth +// token); see copilotInternalPathPrefix for why and the residual. +// +// - INTERNAL CALLER PASSTHROUGH: the hive's own control plane +// (internalCallerName) legitimately holds and sends its own App +// credentials (token mint, heartbeat, hive-open-pr fulfillment); its +// requests pass untouched. This cannot be spoofed by an agent: the name +// is assigned only via UIDMap.IsInternalUID, never from anything the +// caller sends. +// +// Logging: agent name and injected yes/no only β€” NEVER token bytes, not even +// a prefix (#1861 requirement; a token prefix is enough to fingerprint and +// correlate credentials across logs). +func (p *GitHubProxy) rewriteGitHubAuth(req *http.Request, agentName string) { + // NOTE: callers must gate on injectsGitHubAuthForHost first; this function + // does not know which host the request is bound for. + if !p.injectGHAuth { + // Flag off (the default): behavior byte-identical to before #1861 β€” + // no header is touched on any path. + return + } + if agentName == internalCallerName { + return + } + if isCopilotAuthExchangePath(req.URL.Path) { + // Before the strip: the Copilot session-token exchange needs the + // user's Copilot OAuth token, which no hub-held App token can stand in + // for (copilotInternalPathPrefix). + p.logger.Debug("proxy auth injection: Copilot auth-exchange endpoint - agent credential passed through", "agent", agentName, "injected", false) + return + } + + req.Header.Del("Authorization") + + if agentName == "" { + p.logger.Debug("proxy auth injection: caller unidentified β€” forwarding without credentials (fail-loud, no fallback)", "injected", false) + return + } + if strings.HasPrefix(req.URL.Path, loginPathPrefix) { + p.logger.Debug("proxy auth injection: OAuth flow endpoint β€” strip only", "agent", agentName, "injected", false) + return + } + if p.agentTokenSource == nil { + p.logger.Warn("proxy auth injection enabled but no token source wired β€” forwarding without credentials", "agent", agentName, "injected", false) + return + } + token, ok := p.agentTokenSource(agentName) + if !ok { + // An identified agent with no hub-held token means the #3967 mint lane + // has not (yet) delivered for this agent β€” e.g. the hive restarted and + // the in-memory registry is empty until the next launch/refresh mint. + // Fail loud (upstream 401) rather than borrow any other credential. + p.logger.Warn("proxy auth injection: no hub-held token for identified agent β€” forwarding without credentials (check the per-agent mint lane)", "agent", agentName, "injected", false) + return + } + + if isGitPath(req.URL.Path) { + // Git smart HTTP authenticates with Basic x-access-token: β€” + // the same shape git-credential-hive.sh produced pre-injection. + basic := base64.StdEncoding.EncodeToString([]byte(gitInjectBasicUser + ":" + token)) + req.Header.Set("Authorization", "Basic "+basic) + // Force this exchange onto its own upstream connection. The git branch + // in proxyHTTP switches to a raw relay after forwarding this request's + // headers, so a keep-alive reuse (git's follow-up POST after GET + // /info/refs on the same conn) would stream its headers β€” placeholder + // credential included β€” straight through WITHOUT passing this rewrite, + // and fail with a confusing mid-operation 401. Connection: close makes + // git reconnect per request; each new connection re-enters proxyHTTP + // and gets its own rewrite. + req.Close = true + } else { + // REST/GraphQL (api.github.com, GHE /api/v3): the standard GitHub + // token scheme, identical to what gh sends. + req.Header.Set("Authorization", "token "+token) + } + p.logger.Debug("proxy auth injection: scoped token attached", "agent", agentName, "injected", true) +} + +// injectsGitHubAuthForHost reports whether a request relayed for host may have +// its Authorization rewritten by rewriteGitHubAuth (#9586). It is a positive +// allowlist - the empty host (proxyHTTP's "GitHub implied" convention) and the +// registered GitHub-family hosts - rather than "anything but Linear", so a +// non-GitHub host added to the MITM set later can never receive a GitHub token +// by default. +func injectsGitHubAuthForHost(host string) bool { + return host == "" || IsGitHubHost(host) +} + +func (p *GitHubProxy) inspectCanaryEgress(agentName string, req *http.Request) (reason string, deny bool, detected bool) { + if p == nil || !p.canariesEnabled || p.canaryRegistry == nil || req == nil || req.Body == nil { + return "", false, false + } + if isGitReceivePack(req.URL.Path) && p.canaryFailClosed { + return "ioscan canary scan unavailable for git push", true, true + } + if !writeMethods[req.Method] || isGitPath(req.URL.Path) || !githubWriteBodyMayLeak(req.Method, req.URL.Path) { + return "", false, false + } + body, err := io.ReadAll(io.LimitReader(req.Body, maxGitHubWriteBodyScan+1)) + if req.Body != nil { + _ = req.Body.Close() + } + if err != nil { + return "", false, false + } + if len(body) > maxGitHubWriteBodyScan { + return "ioscan canary scan body too large", true, true + } + req.Body = io.NopCloser(bytes.NewReader(body)) + req.ContentLength = int64(len(body)) + if leak, ok := p.canaryRegistry.Scan(agentName, string(body), "github:"+req.Method+" "+req.URL.Path); ok { + if p.canaryLeakFunc != nil { + p.canaryLeakFunc(leak) + } + p.logger.Warn("ioscan canary leak detected", "agent", leak.Agent, "source", leak.Source) + return "ioscan canary leak", p.canaryFailClosed, true + } + return "", false, false +} + +// enforceLinear applies ACMM to one request on a MITM'd api.linear.app +// connection and returns whether it is blocked, with an agent-facing reason. +// +// The request body is read (bounded), inspected, and restored so an allowed +// request forwards byte-identically. +// +// NOTHING from the body is logged or surfaced in the 403. Linear documents +// carry issue and comment content in the query and, in variables, potentially +// tokens; the operation NAMES are schema identifiers rather than user content, +// so those are what makes a denial debuggable without leaking the payload. +func (p *GitHubProxy) enforceLinear(req *http.Request, agentName string, mode agent.AgentMode, caps agent.AgentCapabilities) (bool, string) { + // Linear exposes exactly one endpoint. Anything else on this host is not a + // shape this gate understands, so it is refused rather than forwarded. + if req.Method != http.MethodPost || !IsLinearGraphQLPath(req.URL.Path) { + return true, "linear: only POST " + linearGraphQLPath + " is permitted" + } + + // Read one byte past the limit so hitting the cap is detectable: a body + // that exactly fills the buffer is indistinguishable from a truncated one + // otherwise, and truncation must deny. + body, readErr := io.ReadAll(io.LimitReader(req.Body, linearBodyLimit+1)) + if req.Body != nil { + _ = req.Body.Close() + } + truncated := len(body) > linearBodyLimit + if truncated { + body = body[:linearBodyLimit] + } + req.Body = io.NopCloser(bytes.NewReader(body)) + req.ContentLength = int64(len(body)) + + if readErr != nil { + // Fail closed: a body we could not fully read is a body we could not + // classify. + p.logger.Warn("linear: request body read failed β€” denying", "agent", agentName) + return true, "linear: request body unreadable" + } + + decision := LinearGraphQLAllowed(mode, caps, body, truncated) + if decision.Allowed { + return false, "" + } + + // Operations may be empty when the document could not be classified at + // all, which is itself the useful signal. + p.logger.Warn("linear: blocked GraphQL request", + "agent", agentName, + "mode", mode.String(), + "operations", strings.Join(decision.Operations, ","), + "reason", decision.Reason, + "mutation", decision.IsMutation) + + reason := "linear: " + decision.Reason + if len(decision.Operations) > 0 { + reason += " (" + strings.Join(decision.Operations, ",") + ")" + } + return true, reason +} + +func githubWriteBodyMayLeak(method, path string) bool { + if method != http.MethodPost && method != http.MethodPatch && method != http.MethodPut { + return false + } + if !strings.HasPrefix(path, "/repos/") { + return IsGraphQLPath(path) + } + return strings.Contains(path, "/issues") || + strings.Contains(path, "/pulls") || + strings.Contains(path, "/comments") || + strings.Contains(path, "/reviews") || + strings.Contains(path, "/git/") +} + +func isGitReceivePack(path string) bool { + return strings.HasSuffix(path, "/git-receive-pack") +} + +func isGitPath(path string) bool { + return strings.HasSuffix(path, "/git-receive-pack") || + strings.HasSuffix(path, "/git-upload-pack") || + strings.HasSuffix(path, "/info/refs") +} diff --git a/src/crux/proxy/github_proxy_setters.go b/src/crux/proxy/github_proxy_setters.go new file mode 100644 index 0000000000..e555342d4e --- /dev/null +++ b/src/crux/proxy/github_proxy_setters.go @@ -0,0 +1,100 @@ +package proxy + +import ( + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/agent" +) + +// SetLinearCredentialResolver installs the resolver injectLinearCredential +// reads on every agent request to api.linear.app. main.go wires the same +// closure the agent manager uses to push LINEAR_ACCESS_TOKEN / LINEAR_API_KEY +// into sessions, so the proxy and the session env can never name different +// credentials. Values are never logged. nil disables injection. +func (p *GitHubProxy) SetLinearCredentialResolver(fn func() agent.LinearCredential) { + p.linearCredential = fn +} + +// SetTokenSink wires the inference token sink so the translator can record +// per-agent usage from upstream OpenAI responses into the shared token store. +func (p *GitHubProxy) SetTokenSink(sink UsageSink) { + p.tokenSink = sink +} + +// SetAgentTokenSource wires the resolver from identified agent name to that +// agent's hub-held scoped token (#1861). Wired once at boot, before Start(), +// so no lock is needed on the read path. +func (p *GitHubProxy) SetAgentTokenSource(source func(agentName string) (string, bool)) { + p.agentTokenSource = source +} + +func (p *GitHubProxy) SetAgentGitHubBudgetConfig(capForAgent func(agentName string) int, reserveFloor func() int) { + if capForAgent != nil { + p.githubBudgetCap = capForAgent + } + if reserveFloor != nil { + p.githubReserveFloor = reserveFloor + } +} + +func (p *GitHubProxy) SetAgentGitHubBudgetNudgeFunc(fn func(agentName, message string)) { + p.githubBudgetNudge = fn +} + +// SetCanaryScanner installs the host's canary registry and leak handler. +// enabled gates the scan; failClosed turns a detection (or an unscannable +// body) into a refusal rather than an audit-only event. +func (p *GitHubProxy) SetCanaryScanner(enabled, failClosed bool, reg CanaryScanner, onLeak func(crux.CanaryLeak)) { + p.canariesEnabled = enabled + p.canaryFailClosed = failClosed + p.canaryRegistry = reg + p.canaryLeakFunc = onLeak +} + +// SetRepoPausedFunc installs the operator's per-repo pause predicate (#6203). +// The hive passes config's IsRepoPaused, so pausing a repo in the dashboard is +// enforced on the very next request without a restart β€” which matters, because +// the pauses this serves (a release freeze, an incident) are measured in hours, +// and an operator who has to roll the pod to quiet one repo is back to stopping +// the whole hive. Passing nil clears it. +func (p *GitHubProxy) SetRepoPausedFunc(fn func(repo string) bool) { + p.pausedMu.Lock() + p.repoPaused = fn + p.pausedMu.Unlock() +} + +// SetRepoAutoMergeEnabledFunc installs the live per-repo auto-merge predicate. +// The proxy uses it to give flagged repositories their specific deny reason at +// direct merge endpoints. Passing nil preserves default-on behavior. +func (p *GitHubProxy) SetRepoAutoMergeEnabledFunc(fn func(repo string) bool) { + p.autoMergeMu.Lock() + p.repoAutoMergeOn = fn + p.autoMergeMu.Unlock() +} + +// SetAgentRepoScopeFunc installs the per-repo agent-scope predicate (#6204). +// The hive passes config's AgentServesRepo, so re-scoping an agent in the +// dashboard is enforced on the very next request without a restart. Passing nil +// clears it, which is the unscoped behaviour every hive had before. +func (p *GitHubProxy) SetAgentRepoScopeFunc(fn func(agent, repo string) bool) { + p.scopeMu.Lock() + p.agentServesRepo = fn + p.scopeMu.Unlock() +} + +// SetWriteSurfaceEnforceFunc installs the per-lane write_surface.enforce +// predicate (#9772). The hive passes config's WriteSurfaceEnforced, so a lane +// listed in config is enforced on the next request. Passing nil clears it, +// which is the behaviour every hive had before: direct writes allowed. +func (p *GitHubProxy) SetWriteSurfaceEnforceFunc(fn func(agent string) bool) { + p.writeSurfaceMu.Lock() + p.writeSurfaceEnforced = fn + p.writeSurfaceMu.Unlock() +} + +// SetWriteRefusedAuditFunc installs the sink that records a direct write +// refused under write_surface.enforce as an agent_write_refused audit entry. +func (p *GitHubProxy) SetWriteRefusedAuditFunc(fn func(agent, kind, method, path, repo string)) { + p.writeSurfaceMu.Lock() + p.writeRefusedAudit = fn + p.writeSurfaceMu.Unlock() +} diff --git a/src/crux/proxy/github_proxy_tunnel.go b/src/crux/proxy/github_proxy_tunnel.go new file mode 100644 index 0000000000..cdb93eb78d --- /dev/null +++ b/src/crux/proxy/github_proxy_tunnel.go @@ -0,0 +1,179 @@ +package proxy + +import ( + "io" + "net" + "sync/atomic" + "time" +) + +// responseBodyStallTimeout bounds how long the MITM'd response-BODY relay may +// go without a single byte of progress from the upstream. It is a rolling +// per-Read bound (re-armed on every read by stallBoundedBody), not a cap on +// total transfer time, so a large-but-flowing response is never cut while an +// upstream that goes silent mid-body releases the handler β€” and with it the +// three sockets of the exchange β€” within one stall window. Overridable, not a +// const, so tests can shorten it β€” and atomic for the same reason as +// tunnelHalfCloseDrain (#5553): the reading goroutine can outlive the test +// that started it. A handler test that returns after a fixed sleep leaves +// proxyHTTP still parsing the response when the NEXT test's shortenBodyStall +// writes the bound β€” a plain var made that an unsynchronised write-after-read +// (-race failure on v4, #6110, reported against whichever test held the +// baton). Defaults to httpReadTimeout: the same patience already extended to +// the header phase. +var responseBodyStallTimeout atomic.Int64 + +func init() { responseBodyStallTimeout.Store(int64(httpReadTimeout)) } + +// bodyStallTimeout returns the current response-body stall bound. +func bodyStallTimeout() time.Duration { + return time.Duration(responseBodyStallTimeout.Load()) +} + +// bodyStallTimeoutDuration returns the body-stall timeout for this proxy instance, +// falling back to the package-level responseBodyStallTimeout. +func (p *GitHubProxy) bodyStallTimeoutDuration() time.Duration { + if p != nil && p.bodyStallTimeout > 0 { + return p.bodyStallTimeout + } + return bodyStallTimeout() +} + +// stallBoundedBody wraps a proxied response body so every underlying Read is +// preceded by re-arming a read deadline on the upstream conn. Close clears the +// deadline (for the next keep-alive exchange on the same conn) and closes the +// wrapped body. See responseBodyStallTimeout for why this exists (#3875). +type stallBoundedBody struct { + body io.ReadCloser + conn net.Conn + idle time.Duration +} + +func (b *stallBoundedBody) Read(p []byte) (int, error) { + _ = b.conn.SetReadDeadline(time.Now().Add(b.idle)) + return b.body.Read(p) +} + +func (b *stallBoundedBody) Close() error { + // Close the wrapped body while a stall deadline is armed: net/http body + // Close paths may still read from the conn (drain-to-EOF), and an unarmed + // read there would reintroduce the very stall this type exists to bound. + // The deadline is cleared afterwards so the next keep-alive exchange on + // this conn starts unencumbered. + _ = b.conn.SetReadDeadline(time.Now().Add(b.idle)) + err := b.body.Close() + _ = b.conn.SetReadDeadline(time.Time{}) + return err +} + +// stallBoundedWriter is the write-side mirror of stallBoundedBody: every Write +// re-arms the client's write deadline, so the bound is "no progress for idle" +// rather than "the whole relay must finish within idle". +// +// The distinction is not academic. The response relays used to arm a single +// absolute client.SetWriteDeadline(now+httpWriteTimeout) around +// resp.Write(client), which is correct for a small buffered body and wrong for +// a stream: the MITM'd /v1/messages SSE relay holds one Write loop open for the +// entire generation, so a turn that legitimately streams for longer than +// httpWriteTimeout was severed mid-stream at exactly that mark β€” even though +// tokens were flowing the whole time. The agent saw a truncated body +// ("Response was interrupted due to a server error", or a TLS-level "cannot +// decrypt peer's message" when the cut landed mid-record), retried, and β€” +// because a retry replays the same long turn β€” was cut again at the same 60s, +// livelocking the agent on its longest and most valuable turns. Measured on the +// hosted spoke: 23 "response relay timed out" in 40 minutes, every one +// path=/v1/messages with error "write tcp ...: i/o timeout", against an agent +// that made zero forward progress across six consecutive attempts. +// +// Reusing the stall semantics already proven on the read side (#3875) keeps a +// client that has genuinely gone away bounded β€” a dead peer stops draining, the +// socket buffer fills, and the next Write blocks past idle and errors β€” while a +// stream that keeps flowing is never cut for being long. +type stallBoundedWriter struct { + conn net.Conn + idle time.Duration +} + +func (w *stallBoundedWriter) Write(p []byte) (int, error) { + _ = w.conn.SetWriteDeadline(time.Now().Add(w.idle)) + return w.conn.Write(p) +} + +// tunnelHalfCloseDrainDefault bounds how long one direction of an opaque +// tunnel may keep waiting for bytes AFTER the other direction has already +// finished. +// +// 30s is deliberately generous: the only legitimate traffic still in flight +// once one side has hung up is a response tail already on the wire, which +// arrives in RTTs, not tens of seconds. +const tunnelHalfCloseDrainDefault = 30 * time.Second + +// tunnelHalfCloseDrain is overridable rather than const for the same reason as +// waitForReadyPollInterval in pkg/hub: tests need a relay that unwedges in +// milliseconds. Production keeps the default. +// +// It is atomic because the write and the read genuinely happen on different +// goroutines with no happens-before between them. relayTunnel reads the bound +// from the copy goroutine it spawns, and that goroutine can outlive the test +// that started it: a handler test only waits for the CONNECT response, not for +// the relay to drain, so the goroutine is still reading while the NEXT test's +// shortenTunnelDrain writes. A plain var made that an unsynchronised +// write-after-read and -race failed whichever test happened to hold the baton +// (#5553) β€” the reported test name was incidental, which is why the failure +// moved between TestRelayTunnelUnwedgesFromSilentUpstream and +// TestRelayTunnelDataFlowsBothWays across shuffle seeds. +var tunnelHalfCloseDrain atomic.Int64 + +func init() { tunnelHalfCloseDrain.Store(int64(tunnelHalfCloseDrainDefault)) } + +// tunnelDrain returns the current half-close drain bound. +func tunnelDrain() time.Duration { + return time.Duration(tunnelHalfCloseDrain.Load()) +} + +// relayTunnel shuttles bytes both ways between the proxied client conn and the +// upstream until BOTH directions have finished, bounding each direction once +// the other has ended. +// +// WHY THE BOUNDS EXIST (measured live, 2026-08-14): the previous inline copies +// waited on each other unboundedly. On spokes whose forge host +// (github.ibm.com) sits behind a SYN-accepting firewall, the upstream TCP +// connect succeeds but no byte ever comes back and no FIN is ever sent. The +// client (the daemon's own JWT mints, agents' CLIs) gives up at its TLS +// handshake timeout and closes β€” the clientβ†’upstream copy finishes and +// half-closes the upstream β€” but io.Copy(conn, upstream) stayed blocked in +// upstream.Read() FOREVER. Every attempt leaked two sockets (the FIN_WAIT2 / +// CLOSE_WAIT piles), the goroutine, and the splice pipe pair io.Copy holds for +// a TCP↔TCP copy on Linux: ~300k fds and ~11 burned cores per affected spoke. +// +// SetReadDeadline is documented to interrupt Reads that are ALREADY blocked, +// which is exactly what arms the escape hatch here: each side arms the OTHER +// side's deadline only at the moment its own direction completes, so a +// healthy long-lived tunnel (git smart HTTP, device-flow polling) is never +// cut while both directions are still live. +func relayTunnel(conn, upstream net.Conn) { + done := make(chan struct{}) + go func() { + _, _ = io.Copy(upstream, conn) + if tc, ok := upstream.(*net.TCPConn); ok { + _ = tc.CloseWrite() + } + // Client side is done (EOF or error β€” a dead client looks the same). + // A live upstream answers or closes within RTTs; a blackholed one + // never would, so bound the remaining upstreamβ†’client read. + _ = upstream.SetReadDeadline(time.Now().Add(tunnelDrain())) + close(done) + }() + _, _ = io.Copy(conn, upstream) + // Mirror image: upstream finished (or errored) but the client may sit + // half-open without ever sending EOF; bound the client-side read so + // <-done cannot wedge this handler. + _ = conn.SetReadDeadline(time.Now().Add(tunnelDrain())) + <-done +} + +func transfer(dst, src net.Conn) { + defer func() { _ = dst.Close() }() + defer func() { _ = src.Close() }() + _, _ = io.Copy(dst, src) +} diff --git a/src/crux/proxy/github_proxy_violations.go b/src/crux/proxy/github_proxy_violations.go new file mode 100644 index 0000000000..a76249d3ba --- /dev/null +++ b/src/crux/proxy/github_proxy_violations.go @@ -0,0 +1,254 @@ +package proxy + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "regexp" + "strconv" + "strings" + + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/issueshape" +) + +// Violations returns a snapshot of per-agent violation counts. +func (p *GitHubProxy) Violations() map[string]int { + p.mu.RLock() + defer p.mu.RUnlock() + out := make(map[string]int, len(p.violations)) + for k, v := range p.violations { + out[k] = v + } + return out +} + +// AgentViolations returns the violation count for a specific agent. +func (p *GitHubProxy) AgentViolations(agentName string) int { + p.mu.RLock() + defer p.mu.RUnlock() + return p.violations[agentName] +} + +// issueShapeBodyLimit bounds how much of an issue/comment REST body the proxy +// buffers to shape-check it. It comfortably exceeds GitHub's issue-body maximum +// (65536 chars) plus JSON overhead; an oversized body is forwarded unbuffered +// and unchecked rather than truncated (see enforceIssueShape). +const issueShapeBodyLimit = 256 * 1024 + +var ( + issueCreateRESTPathRE = regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues$`) + issueEditRESTPathRE = regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues/\d+$`) + issueCommentCreateRESTPath = regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues/\d+/comments$`) + issueCommentEditRESTPath = regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues/comments/\d+$`) +) + +// isIssueShapeRESTWrite reports whether a (method, path) is a REST issue or +// issue-comment create/edit whose JSON body carries agent-authored title/body +// text the shape validators must vet. It is deliberately precise β€” matching +// only the title/body-bearing routes β€” so label, assignee, and reaction sub +// routes under /issues are not needlessly buffered. +func isIssueShapeRESTWrite(method, path string) bool { + if i := strings.IndexByte(path, '?'); i >= 0 { + path = path[:i] + } + switch method { + case http.MethodPost: + return issueCreateRESTPathRE.MatchString(path) || + issueCommentCreateRESTPath.MatchString(path) + case http.MethodPatch: + return issueEditRESTPathRE.MatchString(path) || + issueCommentEditRESTPath.MatchString(path) + default: + return false + } +} + +// githubIssueShapeViolation returns a non-empty, agent-actionable reason when a +// buffered issue/comment REST body carries an unsubstituted template +// placeholder or literal newline escape sequences. A body that is not the JSON +// we expect is left to the upstream to reject β€” auth was already enforced. +func githubIssueShapeViolation(method, path string, body []byte) string { + if !isIssueShapeRESTWrite(method, path) { + return "" + } + var payload struct { + Title *string `json:"title"` + Body *string `json:"body"` + } + if err := json.Unmarshal(body, &payload); err != nil { + return "" + } + if payload.Title != nil { + if ph, ok := issueshape.UnsubstitutedTemplatePlaceholder(*payload.Title); ok { + return "issue title contains unsubstituted template placeholder " + strconv.Quote(ph) + "; fill in the template before creating the issue" + } + } + if payload.Body != nil { + if ph, ok := issueshape.UnsubstitutedTemplatePlaceholder(*payload.Body); ok { + return "issue body contains unsubstituted template placeholder " + strconv.Quote(ph) + "; fill in the template before creating the issue" + } + if issueshape.BodyHasMisEscapedNewlines(*payload.Body) { + return "issue body contains literal newline escape sequences; use real newlines or --body-file" + } + } + return "" +} + +// bodyReadCloser rejoins an already-read prefix with the unread remainder of a +// request body so an oversized issue write can be forwarded intact. +type bodyReadCloser struct { + io.Reader + io.Closer +} + +// enforceIssueShape is enforceIssueShapeFor with no agent identity, so no lane +// label is injected. +func (p *GitHubProxy) enforceIssueShape(req *http.Request) (reason string, deny bool, readErr error) { + return p.enforceIssueShapeFor(req, "") +} + +// enforceIssueShapeFor buffers an agent's issue/comment REST body, validates +// its shape, and reconstructs req.Body so the request can still be forwarded +// when it passes. It returns deny=true with an actionable reason for a +// malformed body, and a non-nil error only when the body could not be read +// (fail closed). A well-formed issue create from an identified agent also +// gets its agent/ provenance label injected (see injectOriginLabel). +func (p *GitHubProxy) enforceIssueShapeFor(req *http.Request, agentName string) (reason string, deny bool, readErr error) { + if req.Body == nil || !isIssueShapeRESTWrite(req.Method, req.URL.Path) { + return "", false, nil + } + body, err := io.ReadAll(io.LimitReader(req.Body, issueShapeBodyLimit+1)) + if err != nil { + _ = req.Body.Close() + return "", false, err + } + if int64(len(body)) > issueShapeBodyLimit { + // Oversized: forward the original body unbuffered and unchecked rather + // than corrupt it by truncation. Stitch the prefix we already consumed + // back in front of the unread remainder; ContentLength is unchanged. + req.Body = bodyReadCloser{Reader: io.MultiReader(bytes.NewReader(body), req.Body), Closer: req.Body} + return "", false, nil + } + _ = req.Body.Close() + if v := githubIssueShapeViolation(req.Method, req.URL.Path, body); v != "" { + req.Body = io.NopCloser(bytes.NewReader(body)) + req.ContentLength = int64(len(body)) + return v, true, nil + } + if rewritten, ok := injectOriginLabel(req.Method, req.URL.Path, body, agentName); ok { + body = rewritten + } + req.Body = io.NopCloser(bytes.NewReader(body)) + req.ContentLength = int64(len(body)) + return "", false, nil +} + +// isIssueCreateRESTWrite reports whether (method, path) is POST +// /repos/{o}/{r}/issues β€” the one route whose body creates an issue. +func isIssueCreateRESTWrite(method, path string) bool { + if i := strings.IndexByte(path, '?'); i >= 0 { + path = path[:i] + } + return method == http.MethodPost && issueCreateRESTPathRE.MatchString(path) +} + +// injectOriginLabel ensures an agent's issue-create JSON body carries the +// canonical agent/ label and no redundant origin/ or bare +// label, so an issue filed with `gh issue create` through the proxy records +// the same provenance as one filed through the issue-request watcher. The +// body is returned unchanged (ok=false) for other routes, an unidentified +// caller, a body that is not a JSON object, or a labels field in an unexpected +// shape (GitHub accepts label objects too; those are forwarded as written +// rather than guessed at). +func injectOriginLabel(method, path string, body []byte, agentName string) ([]byte, bool) { + lane := config.OriginLaneFromAgent(agentName) + if lane == "" || agentName == internalCallerName || !isIssueCreateRESTWrite(method, path) { + return body, false + } + var payload map[string]json.RawMessage + if err := json.Unmarshal(body, &payload); err != nil || payload == nil { + return body, false + } + var labels []string + if raw, present := payload["labels"]; present && len(bytes.TrimSpace(raw)) > 0 && string(bytes.TrimSpace(raw)) != "null" { + if err := json.Unmarshal(raw, &labels); err != nil { + return body, false + } + } + filtered := config.WithIssueOriginLabel(labels, lane) + if sameLabels(labels, filtered) { + return body, false + } + labels = filtered + encoded, err := json.Marshal(labels) + if err != nil { + return body, false + } + payload["labels"] = encoded + rewritten, err := json.Marshal(payload) + if err != nil { + return body, false + } + return rewritten, true +} + +func sameLabels(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +// internalCallerName attributes a connection that originated from the hive's +// OWN control plane rather than from a sandboxed agent. It is deliberately not +// a valid agent name (agents come from the UID map) so it can never collide +// with one, and it is what distinguishes "the hive called GitHub" from the +// genuine failure mode the mode gate warns about: "an agent's UID could not be +// attributed". See UIDMap.IsInternalUID for the security argument. +const internalCallerName = "hive-internal" + +func (p *GitHubProxy) recordViolation(agentName, method, path string) { + if p.firstBlock(agentName, method, path) { + p.logger.Warn("proxy request blocked", "agent", agentName, "method", method, "path", path) + } else { + p.logger.Debug("proxy request blocked (repeat)", "agent", agentName, "method", method, "path", path) + } + p.countViolation(agentName) +} + +// firstBlock reports whether this (agent, method, path) refusal has not been +// logged before. Once the set is full, new keys keep logging at WARN. +func (p *GitHubProxy) firstBlock(agentName, method, path string) bool { + key := agentName + "|" + method + "|" + path + p.mu.Lock() + defer p.mu.Unlock() + if _, seen := p.blockedSeen[key]; seen { + return false + } + if p.blockedSeen == nil { + p.blockedSeen = make(map[string]struct{}) + } + if len(p.blockedSeen) < maxViolationLog { + p.blockedSeen[key] = struct{}{} + } + return true +} + +// countViolation increments the per-agent refusal counter without logging, for +// callers that have already logged the refusal with their own detail (see +// refusePlainForward). The map is bounded by maxViolationLog so an +// unidentified or churning caller set cannot grow it without limit. +func (p *GitHubProxy) countViolation(agentName string) { + p.mu.Lock() + defer p.mu.Unlock() + if _, exists := p.violations[agentName]; exists || len(p.violations) < maxViolationLog { + p.violations[agentName]++ + } +} diff --git a/src/crux/proxy/upstream_timeout_test.go b/src/crux/proxy/upstream_timeout_test.go index cdff78d6c6..c69de873ba 100644 --- a/src/crux/proxy/upstream_timeout_test.go +++ b/src/crux/proxy/upstream_timeout_test.go @@ -22,16 +22,39 @@ func TestUpstreamDialTimeoutConstant(t *testing.T) { } } -func TestAllTLSUpstreamDialsAreBound(t *testing.T) { +// readGitHubProxySource returns the concatenated source of the non-test +// github_proxy*.go files, so source-level invariants hold across the files +// GitHubProxy is split into. +func readGitHubProxySource(t *testing.T) string { + t.Helper() _, file, _, ok := runtime.Caller(0) if !ok { t.Fatal("runtime.Caller failed") } - body, err := os.ReadFile(filepath.Join(filepath.Dir(file), "github_proxy.go")) + paths, err := filepath.Glob(filepath.Join(filepath.Dir(file), "github_proxy*.go")) if err != nil { - t.Fatalf("read github_proxy.go: %v", err) + t.Fatalf("glob github_proxy*.go: %v", err) + } + var sb strings.Builder + for _, path := range paths { + if strings.HasSuffix(path, "_test.go") { + continue + } + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", filepath.Base(path), err) + } + sb.Write(body) + sb.WriteString("\n") + } + if sb.Len() == 0 { + t.Fatal("no github_proxy*.go source files found") } - text := string(body) + return sb.String() +} + +func TestAllTLSUpstreamDialsAreBound(t *testing.T) { + text := readGitHubProxySource(t) if got := strings.Count(text, "copilot sniff: client hung up before TLS handshake"); got != 1 { t.Fatalf("copilot sniff handshake hangup demotion sites = %d, want 1", got) } @@ -54,15 +77,7 @@ func TestAllTLSUpstreamDialsAreBound(t *testing.T) { } func TestAllClientTLSHandshakesAreBound(t *testing.T) { - _, file, _, ok := runtime.Caller(0) - if !ok { - t.Fatal("runtime.Caller failed") - } - body, err := os.ReadFile(filepath.Join(filepath.Dir(file), "github_proxy.go")) - if err != nil { - t.Fatalf("read github_proxy.go: %v", err) - } - text := string(body) + text := readGitHubProxySource(t) if clientTLSHandshakeTimeout != upstreamDialTimeout { t.Fatalf("clientTLSHandshakeTimeout = %v, want upstreamDialTimeout %v", clientTLSHandshakeTimeout, upstreamDialTimeout) } @@ -89,15 +104,7 @@ func TestAllClientTLSHandshakesAreBound(t *testing.T) { } func TestHTTPRelayDeadlinesAreBoundAndObservable(t *testing.T) { - _, file, _, ok := runtime.Caller(0) - if !ok { - t.Fatal("runtime.Caller failed") - } - body, err := os.ReadFile(filepath.Join(filepath.Dir(file), "github_proxy.go")) - if err != nil { - t.Fatalf("read github_proxy.go: %v", err) - } - text := string(body) + text := readGitHubProxySource(t) if !strings.Contains(text, "upstream.SetWriteDeadline(time.Now().Add(httpWriteTimeout))") { t.Fatal("request relays must bound upstream writes with httpWriteTimeout") } diff --git a/src/docs/design/README.md b/src/docs/design/README.md index b8eab3f093..c40c789f33 100644 --- a/src/docs/design/README.md +++ b/src/docs/design/README.md @@ -168,7 +168,7 @@ that status is the thing to check before treating a page as current behaviour: whether Copilot token usage can be captured per request with repo context and timestamps at the proxy. Its answer is a recommendation **against** building phase 4 as scoped: per-request usage is available and already captured - (`src/crux/proxy/github_proxy.go:2662`), but per-request repo context is not + (`src/crux/proxy/github_proxy_copilot.go:236`), but per-request repo context is not available at the proxy in any form, and the nearest substitute β€” a "last `api.github.com` repo seen" from `ExtractRepo` β€” is an inference from read traffic, weaker than the audited `repo=` events phase 3 already joins against. diff --git a/src/docs/design/copilot-cost-capture.md b/src/docs/design/copilot-cost-capture.md index bdaa6b0bde..77d8241f52 100644 --- a/src/docs/design/copilot-cost-capture.md +++ b/src/docs/design/copilot-cost-capture.md @@ -42,9 +42,9 @@ The path, end to end: (`src/crux/proxy/rules.go:112`) matches `githubcopilot.com` and any `.githubcopilot.com` subdomain, covering the enterprise per-account hosts. Two call seams dispatch into the sniff path β€” the explicit-CONNECT seam - (`src/crux/proxy/github_proxy.go:877`) and the iptables-transparent seam - (`:461`), both guarded identically by `p.tokenSink != nil && agentName != ""`. -- `proxyCopilotHTTP` (`src/crux/proxy/github_proxy.go:2559`) runs a keep-alive + (`src/crux/proxy/github_proxy_connect.go:469`) and the iptables-transparent seam + (`:160`), both guarded identically by `p.tokenSink != nil && agentName != ""`. +- `proxyCopilotHTTP` (`src/crux/proxy/github_proxy_copilot.go:129`) runs a keep-alive HTTP request/response loop over the terminated TLS connection. It identifies completion requests via `isCopilotCompletionsPath` (`src/crux/proxy/copilot_usage.go:40`, suffix `/chat/completions`), buffers the diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index 1a03802b67..33fabcf85f 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -150,11 +150,11 @@ title is `[selfreport] : `. | `github_fetch_transient` | a non-critical GitHub fetch failed and the feature degrades gracefully (`channel commit date: fetch failed β€” the row will render without a timestamp`, `pkg/hub/channel_commit_date.go`) β€” typically a rate limit or transient API error, not a hive defect. Reported only when persistent: log-fed, same β‰₯5-in-10-min repeat threshold and idle clear as `agent_pane_timeout`. The sibling `SHA poll: commit message fetch non-200` has its own class, `sha_poll_commit_message_unavailable` | low | | `label_provisioning_failed` | provisioning the issue stage labels on a repo failed (`issue stage labels: provisioning failed`, `pkg/github/issue_stage_labels.go`) β€” typically the App lacking label permission on that repo or a transient API error; diagnostics carry the repo and scrubbed error. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | | `agent_cli_restart_loop` | the agent CLI crashed (bare shell) or sat on a consent screen and the harness restarted it before the kick (`agent CLI crashed or stuck on consent screen, restarting before kick`, `crux/agent/manager_kick.go`, `kick_async.go`), repeated β‰₯5 times in 10 min β€” a restart loop, not transient noise. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | -| `proxy_connect_dial_failed` | the hive proxy could not dial the upstream host of a CONNECT request (`proxy: CONNECT dial failed`, `crux/proxy/github_proxy.go`) β€” typically DNS or network unreachability of an agent-requested host, repeated β‰₯5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | -| `proxy_tls_handshake_failed` | the hive proxy's TLS handshake with an upstream host failed (`proxy client tls handshake failed`, `transparent proxy tls handshake failed`, `copilot sniff: client tls handshake failed`, `crux/proxy/github_proxy.go`) β€” typically a certificate validation failure or encryption mismatch on an agent-requested host, repeated β‰₯5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | +| `proxy_connect_dial_failed` | the hive proxy could not dial the upstream host of a CONNECT request (`proxy: CONNECT dial failed`, `crux/proxy/github_proxy_connect.go`) β€” typically DNS or network unreachability of an agent-requested host, repeated β‰₯5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | +| `proxy_tls_handshake_failed` | the hive proxy's TLS handshake with an upstream host failed (`proxy client tls handshake failed`, `transparent proxy tls handshake failed`, `copilot sniff: client tls handshake failed`, `crux/proxy/github_proxy_connect.go`, `crux/proxy/github_proxy_copilot.go`) β€” typically a certificate validation failure or encryption mismatch on an agent-requested host, repeated β‰₯5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `review_evidence_write_failed` | the github-client failed to write a review evidence bundle to persistent storage (`review evidence: could not write bundle`, `pkg/github`), repeated β‰₯5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `selfreport_cleared_comment_failed` | the fleet report publisher failed to post the "condition cleared" comment on a self-report issue (`fleet report: cleared comment failed`, `pkg/github`), repeated β‰₯5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | -| `inference_upstream_error` | a failed or non-2xx response from the configured inference backend (LiteLLM, vLLM/llm-d) on the Anthropic-to-OpenAI translator or MITM reroute path (`inference upstream error` / `inference upstream failed`, `crux/proxy/github_proxy.go`), repeated β‰₯5 times in 10 min β€” covers transport failures and statuses other than the dedicated 401/403 auth signal. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | +| `inference_upstream_error` | a failed or non-2xx response from the configured inference backend (LiteLLM, vLLM/llm-d) on the Anthropic-to-OpenAI translator or MITM reroute path (`inference upstream error` / `inference upstream failed`, `crux/proxy/github_proxy_inference.go`), repeated β‰₯5 times in 10 min β€” covers transport failures and statuses other than the dedicated 401/403 auth signal. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `sha_poll_commit_message_unavailable` | the SHA poller's commit-message fetch got a non-200 from GitHub (typically a 403 rate limit or token scope; `SHA poll: commit message fetch non-200`, `pkg/hub/saas_sha_poller.go`). The poller backs off per SHA and keeps serving the SHA without a message. The WARN itself is deduped to once per hour per (SHA, status) (`warnGitHubFetchOnce`), so one stuck commit logs one line and never trips this class; it is reported only when β‰₯5 distinct SHA/status pairs fail within 10 min (a broad outage such as an exhausted rate limit or a revoked token). Its main job is keeping the residual WARN out of `unclassified_error_repeated`. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | | `tmux_session_check_failed` | the agent's tmux session `has-session` check failed (`tmux has-session failed`, `crux/agent/manager.go`) β€” typically the session was lost or the tmux server is unhealthy, repeated β‰₯5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `transcript_scan_slow` | a Copilot transcript activity scan took longer than its interval (`transcript activity scan outlived its interval`, `crux/agent/busy_visibility.go`) β€” agent liveness reads may lag. Log-fed: same β‰₯5-in-10-min repeat threshold and 20-minute idle clear as `agent_pane_timeout`. Operator diagnostics: check the agent's transcript/session volume; it is bounded by concurrency, so repeated hits indicate slow disk or very large session directories | low | From 64a7bcdca64d7e6a05b0adf42b73f001efbd98cb Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 00:42:37 +0000 Subject: [PATCH 13/38] =?UTF-8?q?=F0=9F=90=9B=20fix(scripts):=20action-pin?= =?UTF-8?q?s=20gate=20fails=20silently=20on=20a=20tree=20with=20no=20pins;?= =?UTF-8?q?=20add=20self-test=20(#14730)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * πŸ› fix(scripts): action-pins gate fails silently on a tree with no pins; add self-test check-action-pins.sh only ever ran against this repository's own, correct workflows, so none of its failure branches had executed anywhere. Writing a hermetic self-test for it (src/scripts/test-check-action-pins.sh, fake `gh`, synthetic workflow directories) exposed two defects: - a directory with no SHA-pinned `uses:` made grep exit 1, and pipefail fed that straight into `fail=1`: "action pin check FAILED" with no finding printed above it; - NOT-PINNED printed the line number where the file name belongs (`grep -rhn` drops the name), so the message never said which workflow. The resolver now collects pins first and iterates, reporting every missing SHA instead of stopping at the first; the shape check uses `grep -rHn` so the message names the file. The self-test covers NOT-PINNED (tag, branch, short and upper-case hex), MISSING-SHA, reusable-workflow owner/repo extraction, pin de-duplication, the offline (no gh) path, EMPTY-EXPRESSION, BRACKETED-EMAIL with the numeric-noreply exclusion, combined failures and the default directory. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: quality <280983584+hivecommons-hive[bot]@users.noreply.github.com> * πŸ”§ chore(deps): bump golang.org/x/net to v0.62.0 (GO-2026-6860) Aligns go.mod/go.sum with v5 so govulncheck passes on this branch. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: quality <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: quality <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: quality <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- changelog.d/fixed-check-action-pins-gate.md | 1 + src/scripts/check-action-pins.sh | 19 +- src/scripts/test-check-action-pins.sh | 405 ++++++++++++++++++++ 3 files changed, 420 insertions(+), 5 deletions(-) create mode 100644 changelog.d/fixed-check-action-pins-gate.md create mode 100755 src/scripts/test-check-action-pins.sh diff --git a/changelog.d/fixed-check-action-pins-gate.md b/changelog.d/fixed-check-action-pins-gate.md new file mode 100644 index 0000000000..2d5b09e9b4 --- /dev/null +++ b/changelog.d/fixed-check-action-pins-gate.md @@ -0,0 +1 @@ +- The action-pins CI gate (`src/scripts/check-action-pins.sh`) no longer reports `action pin check FAILED` with no finding when a workflow directory contains no SHA-pinned `uses:` lines (pipefail turned grep's empty match into a failure), and its `NOT-PINNED` message now names the workflow file rather than the line number where the file name belonged. The resolver reports every missing SHA instead of stopping at the first, and the gate now has a hermetic self-test, `src/scripts/test-check-action-pins.sh`, covering each of its failure branches. diff --git a/src/scripts/check-action-pins.sh b/src/scripts/check-action-pins.sh index 30fcb7f217..e60dd905a0 100755 --- a/src/scripts/check-action-pins.sh +++ b/src/scripts/check-action-pins.sh @@ -30,7 +30,9 @@ while IFS= read -r line; do [0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]) ;; *) echo "NOT-PINNED $file: $ref (use a 40-hex commit SHA)"; fail=1; continue ;; esac -done < <(grep -rhnE '^\s*(-\s*)?uses:\s*[a-zA-Z0-9._-]+/' "$DIR" 2>/dev/null | grep -v '^\s*#') +# -H keeps the file name in front of the line number even for a single-file +# DIR, so `${line%%:*}` names the workflow and not its line number. +done < <(grep -rHnE '^\s*(-\s*)?uses:\s*[a-zA-Z0-9._-]+/' "$DIR" 2>/dev/null | grep -v '^\s*#') # Resolve each unique pin. Skipped without a token so the script stays usable # offline; CI always has one. @@ -38,15 +40,22 @@ if command -v gh >/dev/null 2>&1; then # Extract owner/repo from the `uses:` VALUE, taking only the first two path # segments: a reusable workflow is `owner/repo/.github/workflows/x.yml@SHA`, # so naively splitting on the last '/' asks the API about a path, not a repo. - grep -rhoE 'uses:[[:space:]]*[a-zA-Z0-9._-]+/[a-zA-Z0-9._/-]+@[0-9a-f]{40}' "$DIR" 2>/dev/null \ - | sed -E 's/^uses:[[:space:]]*//' | sort -u | while read -r ref; do + # + # Collected first, then iterated: when the pipeline's status fed `|| fail=1` + # directly, a tree with no SHA-pinned `uses:` at all made grep exit 1 and + # pipefail turned that into a FAILED verdict with no finding printed. + # Nothing to resolve is a pass, not a failure. + pins="$(grep -rhoE 'uses:[[:space:]]*[a-zA-Z0-9._-]+/[a-zA-Z0-9._/-]+@[0-9a-f]{40}' "$DIR" 2>/dev/null \ + | sed -E 's/^uses:[[:space:]]*//' | sort -u)" + while read -r ref; do + [ -z "$ref" ] && continue sha="${ref#*@}"; path="${ref%@*}" repo="$(printf '%s' "$path" | cut -d/ -f1,2)" if ! gh api "repos/$repo/commits/$sha" --jq '.sha' >/dev/null 2>&1; then echo "MISSING-SHA $repo@$sha does not exist in that repository" - exit 1 + fail=1 fi - done || fail=1 + done <<<"$pins" fi # An EMPTY workflow-expression delimiter pair anywhere in a workflow file is a diff --git a/src/scripts/test-check-action-pins.sh b/src/scripts/test-check-action-pins.sh new file mode 100755 index 0000000000..c796f1384b --- /dev/null +++ b/src/scripts/test-check-action-pins.sh @@ -0,0 +1,405 @@ +#!/usr/bin/env bash +# Exercises check-action-pins.sh with a hermetic `gh` test double: no network +# and no real workflow files are used. Each case writes a synthetic workflow +# directory, runs the check against it with a fake `gh` first on PATH that +# records every API call and can be told which SHA to report missing, then +# asserts the exit status and the verdict printed. +# +# The check's job is to fail closed. In CI it only ever sees this repository's +# own (correct) workflows, so every failure branch (NOT-PINNED, MISSING-SHA, +# EMPTY-EXPRESSION, BRACKETED-EMAIL), the numeric-noreply exclusion and the +# reusable-workflow owner/repo extraction would otherwise run for the first +# time on the day they matter. This test runs them on every change to the +# script. +# +# Usage: bash src/scripts/test-check-action-pins.sh (from the repository root) +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +SCRIPT="${ROOT}/src/scripts/check-action-pins.sh" +TMP="$(mktemp -d "${TMPDIR:-/tmp}/check-action-pins-test.XXXXXX")" +trap 'rm -rf "$TMP"' EXIT + +failures=0 +pass() { printf ' ok: %s\n' "$*"; } +bad() { + printf ' FAIL: %s\n' "$*" + failures=$((failures + 1)) +} +# Here-strings, not pipes, so a grep that stops reading early cannot turn +# pipefail into a false FAIL. +assert_contains() { + local haystack="$1" needle="$2" msg="$3" + if grep -qF -- "$needle" <<<"$haystack"; then pass "$msg"; else bad "$msg (missing: $needle)"; fi +} +assert_not_contains() { + local haystack="$1" needle="$2" msg="$3" + if grep -qF -- "$needle" <<<"$haystack"; then bad "$msg (unexpected: $needle)"; else pass "$msg"; fi +} +assert_exit() { + local want="$1" got="$2" msg="$3" + if [ "$want" = "$got" ]; then pass "$msg (exit $got)"; else bad "$msg (want exit $want, got $got)"; fi +} +assert_eq() { + local want="$1" got="$2" msg="$3" + if [ "$want" = "$got" ]; then pass "$msg ($got)"; else bad "$msg (want $want, got $got)"; fi +} + +# Fake `gh`: only `gh api repos///commits/ --jq .sha` is +# ever called by the check. Every call is appended to $FAKE_GH_LOG; a call +# whose SHA equals $FAKE_GH_MISSING fails like the real API does for a commit +# that does not exist in the repository. +BIN="$TMP/bin" +mkdir -p "$BIN" +cat >"$BIN/gh" <<'SH' +#!/usr/bin/env bash +printf '%s\n' "$*" >>"${FAKE_GH_LOG:?}" +case "$2" in + *"/commits/${FAKE_GH_MISSING:-no-such-sha}") exit 1 ;; +esac +printf '%s\n' "${2##*/}" +exit 0 +SH +chmod +x "$BIN/gh" + +# A PATH with every external tool the check uses except gh, so the "no token, +# stay usable offline" branch can be exercised without uninstalling anything. +NOGH="$TMP/nogh" +mkdir -p "$NOGH" +for tool in grep sed sort cut head; do + ln -s "$(command -v "$tool")" "$NOGH/$tool" +done + +SHA_A=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +SHA_B=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +SHA_C=cccccccccccccccccccccccccccccccccccccccc + +# workflow DIR NAME <"$1/$2"; } + +# run_check CASE [env VAR=VALUE ...] -- ARGS... +# Runs the check with the fake gh first on PATH. Leaves stdout+stderr in +# $TMP/.out, the exit status in RC, the output in OUT and the fake gh's +# call log in GH_CALLS. +run_check() { + local name="$1"; shift + local envs=() + while [ "$#" -gt 0 ] && [ "$1" != "--" ]; do envs+=("$1"); shift; done + shift + : >"$TMP/$name.gh" + ( + env FAKE_GH_LOG="$TMP/$name.gh" "${envs[@]}" PATH="$BIN:$PATH" bash "$SCRIPT" "$@" + ) >"$TMP/$name.out" 2>&1 + RC=$? + OUT="$(cat "$TMP/$name.out")" + GH_CALLS="$(cat "$TMP/$name.gh")" +} + +echo "case: every action pinned to a resolvable SHA" +D="$TMP/green" +workflow "$D" ci.yml <"$BIN/gh" <>"\${FAKE_GH_LOG:?}" +case "\$2" in + */commits/\${FAKE_GH_MISSING:-no-such-sha}|*/commits/$FAKE_SECOND_MISSING) exit 1 ;; +esac +printf '%s\n' "\${2##*/}" +exit 0 +SH +run_check missing-two FAKE_GH_MISSING="$SHA_C" -- "$D" +assert_exit 1 "$RC" "two fabricated SHAs fail" +assert_contains "$OUT" "MISSING-SHA anchore/sbom-action@$SHA_C" "first missing SHA reported" +assert_contains "$OUT" "MISSING-SHA actions/checkout@$SHA_A" "second missing SHA reported too" +assert_contains "$GH_CALLS" "api repos/actions/setup-go/commits/$SHA_B" "resolution continues past a missing SHA" +assert_eq 3 "$(grep -c . <<<"$GH_CALLS")" "every unique pin is still resolved" +# Restore the single-miss double for the remaining cases. +cat >"$BIN/gh" <<'SH' +#!/usr/bin/env bash +printf '%s\n' "$*" >>"${FAKE_GH_LOG:?}" +case "$2" in + *"/commits/${FAKE_GH_MISSING:-no-such-sha}") exit 1 ;; +esac +printf '%s\n' "${2##*/}" +exit 0 +SH + +echo "case: a directory with no third-party actions passes with gh present" +D="$TMP/nopins" +workflow "$D" ci.yml <"$TMP/nogh.gh" +OUT="$( env FAKE_GH_LOG="$TMP/nogh.gh" PATH="$NOGH" "$(command -v bash)" "$SCRIPT" "$D" 2>&1 )" +RC=$? +assert_exit 0 "$RC" "a pinned workflow passes without gh" +assert_contains "$OUT" "action pin check OK" "prints the pass verdict offline" +assert_eq 0 "$(grep -c . <"$TMP/nogh.gh")" "no API calls attempted without gh" +workflow "$D" bad.yml <&1 )" +RC=$? +assert_exit 1 "$RC" "the shape check still fails without gh" +assert_contains "$OUT" "NOT-PINNED" "NOT-PINNED reported offline" + +echo "case: an empty expression delimiter pair" +D="$TMP/emptyexpr" +workflow "$D" ci.yml <"$TMP/default.gh" +OUT="$( cd "$D" && env FAKE_GH_LOG="$TMP/default.gh" PATH="$BIN:$PATH" bash "$SCRIPT" 2>&1 )" +RC=$? +assert_exit 1 "$RC" "the default directory is scanned when no argument is given" +assert_contains "$OUT" "NOT-PINNED .github/workflows/ci.yml: actions/checkout@v4" "reports the path relative to the default directory" + +echo +if [ "$failures" -eq 0 ]; then + echo "check-action-pins tests: all passed" + exit 0 +fi +echo "check-action-pins tests: $failures failure(s)" +exit 1 From 22b9d5f55e96ec9aafd4b82024016b0f6249e318 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 27 Oct 2026 20:49:32 -0400 Subject: [PATCH 14/38] =?UTF-8?q?=F0=9F=93=96=20Compare=20Hive=20trust=20l?= =?UTF-8?q?evels=20with=20Copilot=20assisted=20permissions=20(#14760)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andrew Anderson --- ...ed-14613-copilot-permissions-comparison.md | 1 + src/docs/README.md | 1 + src/docs/copilot-permissions-comparison.md | 132 ++++++++++++++++++ src/docs/documentation-map.md | 1 + 4 files changed, 135 insertions(+) create mode 100644 changelog.d/added-14613-copilot-permissions-comparison.md create mode 100644 src/docs/copilot-permissions-comparison.md diff --git a/changelog.d/added-14613-copilot-permissions-comparison.md b/changelog.d/added-14613-copilot-permissions-comparison.md new file mode 100644 index 0000000000..4218842918 --- /dev/null +++ b/changelog.d/added-14613-copilot-permissions-comparison.md @@ -0,0 +1 @@ +- **Docs: Hive trust levels vs GitHub Copilot assisted permissions.** New page `src/docs/copilot-permissions-comparison.md` explains Copilot's assisted permissions (GA 2026-10-26, from GitHub's changelog and docs) next to Hive's ACMM levels, lanes, policy proxy with measured refusal counts from the hosted hives, review gates, hold flow and record-only shadow gates, with a dimension table and what each does that the other does not. Linked from the docs index and documentation map (#14613). diff --git a/src/docs/README.md b/src/docs/README.md index a095ebf1bd..7252426143 100644 --- a/src/docs/README.md +++ b/src/docs/README.md @@ -50,6 +50,7 @@ Start with [Zero to Automation: Getting Started with Hive](getting-started.md). or assignment helpers. - [Release channels](release-channels.md) β€” what `stable`, `candidate`, and `edge` each mean and what to expect from them. - [Security model](security-model.md) β€” agent safety control inventory, threats, enforcement modes, residual gaps and operator audit checklist. +- [Hive trust levels vs Copilot assisted permissions](copilot-permissions-comparison.md) β€” what GitHub's per-action permission mode (GA 2026-10-26) does, what Hive's levels, lanes, proxy and gates do, and where each covers ground the other does not. - Security docs rule: `SECURITY.md` and every `src/docs/security*.md` / `ioscan*.md` page, plus `landscape.md`, must score Flesch-Kincaid grade 11 or lower (CI: `src/scripts/check-security-docs-readability.sh`). - Security docs rot rule: the same pages carry a `Last reviewed: YYYY-MM-DD` line in their first 15 lines, no older than 120 days, and may not say an item is "tracked in" / "in progress" / "planned" against an issue that is already closed (CI: `src/scripts/check-security-docs-rot.sh`). - [Data collection and telemetry](telemetry.md) diff --git a/src/docs/copilot-permissions-comparison.md b/src/docs/copilot-permissions-comparison.md new file mode 100644 index 0000000000..99fdf739e6 --- /dev/null +++ b/src/docs/copilot-permissions-comparison.md @@ -0,0 +1,132 @@ +# Hive trust levels vs GitHub Copilot assisted permissions + +Last reviewed: 2026-10-27 + +GitHub made Copilot "assisted permissions" generally available on 2026-10-26. +This page says what that feature is, what Hive does in the same space, and +where each does something the other does not. It is a comparison, not a +ranking. The two tools answer different questions. + +## What Copilot assisted permissions is + +Source: the GitHub changelog entry of 2026-10-26, +, +and the docs page it links, +. + +- It is a permission mode for one Copilot session in Copilot CLI, the Copilot + app, and VS Code. The three modes are manual approval, assisted, and + unconditional approval (`--allow-all` / `--yolo`). +- When Copilot wants to run a shell command, touch a file path, or open a URL, + a second language model looks at the request. If that model is confident the + action is safe, the action runs with no prompt. If not, the user is asked. +- In non-interactive runs (`copilot -p ... --assisted-approval`, or + `COPILOT_ASSISTED_APPROVAL=true` in CI), there is nobody to ask, so anything + the review model will not clear is denied and the reason is handed back to + Copilot so it can try another route. +- Existing allow and deny rules run first. Enterprise deny rules still apply. + Enterprise admins can block the unconditional mode while leaving assisted + on, disable assisted on its own, and set the default mode for new sessions. +- GitHub's own docs say it "is not a substitute for sandboxing or access + restrictions" and that the review model "can make mistakes and approve + dangerous or unintended actions". + +In short: it decides, per action, inside one developer's session, whether to +show a prompt. It does not know about pull requests, merges, or repositories. + +## What Hive does + +Hive runs fleets of agents against repositories without a person at the +keyboard. Its controls are about what reaches GitHub, not about prompts. + +- **Trust levels (ACMM L1-L6).** The level decides which agents run and in + which policy mode. Advisory agents write nothing to GitHub. Measured agents + may open issues. Hold-gated agents may open PRs, and every PR carries a + `hold` label a human must remove. Full agents may merge on green CI. L1 and + L2 are advisory, L3 and L4 add issues and hold-gated PRs, L5 opens PRs under + `hold`, and only L6 merges. See [ACMM policy matrix](acmm-policy-matrix.md). +- **Lanes.** Each piece of work is assigned to a lane, the agent that owns it + (scanner, reviewer, ci-maintainer, and so on). GitHub writes are allowlisted + per lane in `write_surface.allowlist`, and a lane can be listed in + `write_surface.enforce`. See [GitHub write surface](github-write-surface.md). +- **The policy proxy.** Every agent's GitHub traffic passes through a + man-in-the-middle proxy (`pkg/proxy`). For an enforced lane it refuses every + REST `POST`, `PUT`, `PATCH`, `DELETE`, every GraphQL mutation, and `git push`, + returns `403`, and writes an `agent_write_refused` audit entry. Reads and + the hive's own relays keep working. The proxy also applies repo pause, agent + repo scope, the auto-merge policy, and the ACMM mode gate. The refusals are + counted. Read on 2026-10-27 from `GET /api/evidence/write-counters` on the + three hosted hives (hivecommons, kubestellar, projectbluefin), since the + counter started on 2026-10-12: 538 direct agent write attempts, 123 refused, + all with reason `mode_denied`. Hub admins see the same counters pooled in + `GET /api/hub/evidence` (`write_counters[]` with `block_rate`). +- **Review gates.** `bin/merge-gate.sh` is deterministic: required CI green, + not a draft, no `hold`, and an approved review for community PRs. The + reviewer swarm adds verdicts after the change; `review.require_approval` + makes a same-head approval required at L6. See [review swarm](review-swarm.md) + and [running at level 6](running-at-level-6.md). +- **Hold and approval.** Below L6 every agent PR gets `hold`; a human removes + it. `needs-human` parks a PR for a person. At L6 a `reject` or + `changes_requested` verdict still needs a fix or a human merge. See + [labels and control signals](labels-and-control-signals.md). +- **Shadow gates.** `governor.review_gate.mode: shadow` (the default) and + `intent.shadow` evaluate a gate and record what it would have held, and never + block. The record is in each evidence bundle under `shadow`. The study that + would turn those records into an accuracy figure is planned and not yet + running ([ablation plan](research/ablation-plan-2026-10.md)). Enforcement from + a shadow result is a config switch, not a separate feature. +- **Evidence.** Every mediated merge gets an evidence card; outcomes are + published with n and 95% intervals. See [evidence](evidence.md) and + [benchmarks](benchmarks/README.md). + +## Comparison + +| Dimension | Copilot assisted permissions | Hive | +|---|---|---| +| Unit of control | One action (command, path, URL) in one session | One GitHub write, one PR, one repository, one lane | +| Who decides | A second model, then the user | Deterministic config: level, lane allowlist, proxy rules, CI, labels | +| Where it runs | Inside the Copilot client | Outside the agent, in the proxy and governor; the agent cannot turn it off | +| Levels | Three modes per session (manual, assisted, allow-all) | Six repository levels, four policy modes per agent | +| Enterprise controls | Block allow-all, disable assisted, set default mode | Hub-managed level, repo pause, per-lane enforce, auto-merge toggle per repo | +| What it protects | The developer's machine and the paths and URLs Copilot touches | The repository's issues, PRs, branches, and merges | +| Unattended runs | Denies what the model will not clear; explanation goes back to the agent | Refuses at the proxy with `403` and a reason; audit entry written | +| Merge authority | None; Copilot has no view of merges | Only at L6, only on green CI, only in eligible lanes | +| Human approval step | A prompt per uncleared action | `hold` label and `needs-human` per PR; batch review | +| Measurement | None published | Refusal counters per tier, evidence cards, shadow-gate records | +| Model risk | The review model can approve a dangerous action (GitHub's own caveat) | Review verdicts come from models, but every merge still needs deterministic gates | +| Scope of engines | Copilot only | 15 CLIs through Crux, including Copilot CLI | + +## What Copilot does that Hive does not + +- Judge a shell command, file path, or URL before it runs. Hive sandboxes the + agent and limits network egress, but it does not score individual commands. +- Reduce prompts for a developer sitting at a terminal. Hive has no interactive + prompt step; its agents run unattended. +- Enterprise-managed defaults that apply to every developer's local session. +- Ship inside the editor and CLI people already use, with no deployment. + +## What Hive does that Copilot does not + +- Tie autonomy to repository maturity with a level a team can raise or lower. +- Refuse a GitHub write at the network layer so an agent cannot talk its way + past the rule; the proxy is not a model and does not take explanations. +- Separate "may open a PR" from "may merge", and gate the merge on CI and + review, not on a session mode. +- Put a `hold` on every agent PR below L6 so humans review in batches. +- Count refusals and publish merge outcomes with intervals. +- Run many agents from many vendors under one policy. + +## How they fit together + +Copilot CLI is one of the engines Hive drives (`copilot` in Crux +`CLIBackends`). An operator can run Copilot CLI inside Hive with +`--assisted-approval` for the command-level check, and still have every GitHub +write go through Hive's proxy and gates. The two layers do not overlap: one +watches commands, the other watches the repository. + +## What this page does not claim + +Hive has not measured how often Copilot's review model approves a dangerous +action, and GitHub does not publish that figure. The 123 refusals above are +counts from three hives over 15 days, not a rate for anyone else's fleet. +Neither tool stops a determined person with repository write access. diff --git a/src/docs/documentation-map.md b/src/docs/documentation-map.md index e44b10e686..2386617a46 100644 --- a/src/docs/documentation-map.md +++ b/src/docs/documentation-map.md @@ -216,6 +216,7 @@ Some documents describe planned or design-only work rather than live features. T - [Securing your hive: a first-time operator's guide](securing-your-hive.md) β€” decision-oriented walkthrough of ACMM level, reporter trust, and repo scope: starting postures, the #9758/#9762 real-world walk-through, Q&A, a pre-L6 checklist, and a glossary. - [Security model β€” agent safety inventory](security-model.md) β€” threat/control/verification table; runtime, context, credential, code-admission, contributor, detection, platform and human-control layers; residual gaps and running-hive audit steps. +- [Hive trust levels vs Copilot assisted permissions](copilot-permissions-comparison.md) β€” GitHub Copilot's assisted permissions (GA 2026-10-26) explained from the changelog and docs, next to Hive's ACMM levels, lanes, policy proxy with measured refusal counts, review gates, hold flow and record-only shadow gates; a dimension-by-dimension table and what each side does that the other does not. - [Compliance controls](compliance.md) β€” the control registry: framework profiles (SOC 2 Type II, FedRAMP Moderate subset, ISO 27001 Annex A) mapping controls to Hive settings, `GET /api/compliance/status` with meets/deviates/off/not_covered per control, the generated per-framework mapping tables, posture checks, exports and attestations as designed, and the gaps Hive does not cover. Not a certification. - [Security threat model](security-threat-model.md) β€” actors, boundaries, layered defenses, known gaps, and reporting. - [OSS Scanner threat model](../../.oss-scanner/threat_model.md) β€” the scanner-facing companion kept at the repository root with [its Dockerfile](../../.oss-scanner/Dockerfile): where untrusted input enters, which packages matter most, how to exercise a hive offline, and the severity rubric used to triage findings from [Anthropic's OSS Scanner](https://red.anthropic.com/oss-scanner/). From b096432fc9a0c2cbe0635a51fe91f928c1510468 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 27 Oct 2026 20:50:05 -0400 Subject: [PATCH 15/38] =?UTF-8?q?=F0=9F=90=9B=20changelog:=20fix=20malform?= =?UTF-8?q?ed=20ioscan=20fragment=20so=20the=20release=20decide=20job=20pa?= =?UTF-8?q?sses=20(#14756)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andy Anderson --- changelog.d/fixed-ioscan-unavailable-classifier-budget.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/fixed-ioscan-unavailable-classifier-budget.md b/changelog.d/fixed-ioscan-unavailable-classifier-budget.md index 6f60145a32..72abfa7181 100644 --- a/changelog.d/fixed-ioscan-unavailable-classifier-budget.md +++ b/changelog.d/fixed-ioscan-unavailable-classifier-budget.md @@ -1 +1 @@ -An ioscan classifier that could not be constructed (no model endpoint for the hive's CLI, e.g. a Copilot-only hive at ACMM L5+) no longer consumes the per-eval classifier budget. Previously the first 64 strings per governor eval were withheld under fail-closed and every later string passed on unclassified as `budget_exhausted`, so whether an agent's kick survived depended on its render position. The fail-open path now also logs the classifier error once per distinct error instead of once per string, which stopped a hosted hive from self-reporting 64 "ioscan classifier failed open" warnings every eval. +- An ioscan classifier that could not be constructed (no model endpoint for the hive's CLI, e.g. a Copilot-only hive at ACMM L5+) no longer consumes the per-eval classifier budget ([#14629](https://github.com/hivecommons/hive/pull/14629)). Previously the first 64 strings per governor eval were withheld under fail-closed and every later string passed on unclassified as `budget_exhausted`, so whether an agent's kick survived depended on its render position. The fail-open path now also logs the classifier error once per distinct error instead of once per string, which stopped a hosted hive from self-reporting 64 "ioscan classifier failed open" warnings every eval. From d6aafbbaadbeb9975592870e9612afa450972a6d Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Wed, 28 Oct 2026 01:54:19 +0100 Subject: [PATCH 16/38] =?UTF-8?q?=F0=9F=90=9B=20fix:=20keep=20the=20header?= =?UTF-8?q?'s=20right=20chips=20from=20overlapping=20the=20title=20at=2076?= =?UTF-8?q?9-900px=20(#14656)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The #13173 fix pinned .oc-topbar-right to max-content only from 901px, so at 769-900px the cluster shrank to 0, its nowrap flex-end chips overflowed leftwards and the feedback/theme buttons painted over the project title. Extend the guard down to 769px (the phone block takes over below) and drop the Settings/Health/Guide labels in that band so the drawer toggle and title keep room. Fixes #13135 Hive-Run: hivecommons/hive#13135 Hive-Plan: dashboard-header-overlap Hive-Spec: dashboard-navbar#13135 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HRH6kizfUBgiWsEGGur6Rr Signed-off-by: Karibhou Co-authored-by: MikeSpreitzer <14296719+MikeSpreitzer@users.noreply.github.com> --- changelog.d/fixed-13135-header-overlap.md | 1 + .../navbar_header_overlap_13135_test.go | 26 +++++++++++++++++++ src/pkg/dashboard/static/index.html | 8 ++++-- 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 changelog.d/fixed-13135-header-overlap.md create mode 100644 src/pkg/dashboard/navbar_header_overlap_13135_test.go diff --git a/changelog.d/fixed-13135-header-overlap.md b/changelog.d/fixed-13135-header-overlap.md new file mode 100644 index 0000000000..1d06f6f3e6 --- /dev/null +++ b/changelog.d/fixed-13135-header-overlap.md @@ -0,0 +1 @@ +- The dashboard header's feedback, theme and other right-hand buttons no longer overlap the project title or ACMM badge at window widths between 769px and 900px; those widths now show icon-only Settings, Health and Guide chips ([#13135](https://github.com/hivecommons/hive/issues/13135)). diff --git a/src/pkg/dashboard/navbar_header_overlap_13135_test.go b/src/pkg/dashboard/navbar_header_overlap_13135_test.go new file mode 100644 index 0000000000..a949f58df6 --- /dev/null +++ b/src/pkg/dashboard/navbar_header_overlap_13135_test.go @@ -0,0 +1,26 @@ +package dashboard + +import ( + "strings" + "testing" +) + +// TestNavbarRightClusterCannotSpillOverTitle guards #13135: between the +// phone block (≀768px) and 1400px the right chip cluster must keep its +// max-content width, otherwise its flex-end overflow paints the feedback and +// theme buttons over the ACMM badge or project title. +func TestNavbarRightClusterCannotSpillOverTitle(t *testing.T) { + html := indexHTML(t) + for _, want := range []string{ + "@media (min-width: 769px) and (max-width: 1400px) {\n .oc-topbar-right { min-width: max-content; }", + "@media (min-width: 769px) and (max-width: 900px) {\n #acmm-badge { display: none; }", + "#welcome-topbar-btn .nav-chip__label { display: none; }\n }", + } { + if !strings.Contains(html, want) { + t.Fatalf("navbar #13135 overlap guard missing %q", want) + } + } + if strings.Contains(html, "@media (min-width: 901px) and (max-width: 1400px) {\n .oc-topbar-right") { + t.Fatal("right cluster min-width guard must also cover 769-900px, where the dashboard was still overlapping") + } +} diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 80f8c20a32..dee1a4c419 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -557,13 +557,17 @@ .oc-topbar-right .widget-dl { margin-left: var(--sp-0); margin-right: var(--sp-0); } } - /* Right chip cluster keeps its full width so it cannot shrink and spill over the left ACMM badge (#13135); the left/center tracks absorb the squeeze. */ - @media (min-width: 901px) and (max-width: 1400px) { + /* Right chip cluster keeps its full width so it cannot shrink and spill over the left ACMM badge or project title (#13135); the left/center tracks absorb the squeeze. Starts at 769px because the ≀768px block clips the cluster instead. */ + @media (min-width: 769px) and (max-width: 1400px) { .oc-topbar-right { min-width: max-content; } } + /* Icon-only chips here so the full-width right cluster still leaves room for the drawer toggle and title (#13135). */ @media (min-width: 769px) and (max-width: 900px) { #acmm-badge { display: none; } + .oc-health-badge .nav-chip__label, + #oc-settings-btn .nav-chip__label, + #welcome-topbar-btn .nav-chip__label { display: none; } } /* Light main content area */ From ba31984c58f5c5ac7d90433dae3620db223c5e14 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 00:57:43 +0000 Subject: [PATCH 17/38] fix(github): skip close-on-merge when the claimed target is a pull request (#14745) Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-close-on-merge-pr-target.md | 1 + src/pkg/github/close_on_merge.go | 9 ++++++ src/pkg/github/close_on_merge_test.go | 32 +++++++++++++++++++ 3 files changed, 42 insertions(+) create mode 100644 changelog.d/fixed-close-on-merge-pr-target.md diff --git a/changelog.d/fixed-close-on-merge-pr-target.md b/changelog.d/fixed-close-on-merge-pr-target.md new file mode 100644 index 0000000000..b07b8d1263 --- /dev/null +++ b/changelog.d/fixed-close-on-merge-pr-target.md @@ -0,0 +1 @@ +- close-on-merge no longer closes a pull request as a "fixed issue" when a merged sibling-repo PR mentions it (#14744). diff --git a/src/pkg/github/close_on_merge.go b/src/pkg/github/close_on_merge.go index dbedf5f20a..97509637ad 100644 --- a/src/pkg/github/close_on_merge.go +++ b/src/pkg/github/close_on_merge.go @@ -538,6 +538,15 @@ func (c *Client) closeIssueForMergedPRClaimWithIssue(ctx context.Context, issueR return res } } + if issue.IsPullRequest() { + // The Issues API also returns pull requests; a cross-repo mention can + // claim one. A verdict, not a failed step, so INFO rather than WARN. + res.Action, res.Reason = CloseOnMergeSkipped, "target_is_pull_request" + if opts.Logger != nil { + opts.Logger.Info("close-on-merge: claimed target is a pull request, not an issue; skipping", "repo", issueRepo, "number", issueNumber, "pr", pr.GetNumber()) + } + return res + } if !strings.EqualFold(issue.GetState(), "open") { res.Action, res.Reason = CloseOnMergeAlreadyDone, "issue_not_open" return res diff --git a/src/pkg/github/close_on_merge_test.go b/src/pkg/github/close_on_merge_test.go index a883b5893e..a4429eff83 100644 --- a/src/pkg/github/close_on_merge_test.go +++ b/src/pkg/github/close_on_merge_test.go @@ -512,3 +512,35 @@ func TestCloseOnMergeReopenDecision(t *testing.T) { }) } } + +func TestCloseOnMergeSkipsClaimedPullRequestTarget(t *testing.T) { + defer resetPRDetailCacheForTest(nil, 0)() + var writes []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writes = append(writes, r.Method+" "+r.URL.Path) + w.WriteHeader(http.StatusInternalServerError) + return + } + if r.URL.Path == "/repos/o/r/issues/14730" { + _ = json.NewEncoder(w).Encode(map[string]any{ + "number": 14730, + "state": "open", + "pull_request": map[string]any{"url": "https://api.github.com/repos/o/r/pulls/14730"}, + }) + return + } + http.NotFound(w, r) + })) + defer srv.Close() + c := newTestClient(t, srv, "o", []string{"r"}) + pr := mergedTestPR(25, "hive[bot]", "fix", "the fix is made in o/r#14730") + opts := CloseOnMergeOptions{Identity: HiveIdentity{AppLogin: "hive[bot]"}} + res := c.closeIssueForMergedPRClaim(context.Background(), "o/r", 14730, pr, true, opts) + if res.Action != CloseOnMergeSkipped || res.Reason != "target_is_pull_request" { + t.Fatalf("result = %+v, want target_is_pull_request skip", res) + } + if len(writes) != 0 { + t.Fatalf("unexpected write requests: %v", writes) + } +} From df119058d5575d6ebe097570b20d54e0545513f2 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 00:57:58 +0000 Subject: [PATCH 18/38] =?UTF-8?q?=F0=9F=8C=B1=20dashboard:=20rename=20slog?= =?UTF-8?q?WarnOverviewEncode=20to=20logOverviewEncodeFailure=20(#14740)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/changed-14718.md | 1 + src/pkg/dashboard/overview_export.go | 6 +++--- src/pkg/dashboard/overview_export_14696_test.go | 4 ++-- src/pkg/dashboard/stage_tracker.go | 6 +++--- 4 files changed, 9 insertions(+), 8 deletions(-) create mode 100644 changelog.d/changed-14718.md diff --git a/changelog.d/changed-14718.md b/changelog.d/changed-14718.md new file mode 100644 index 0000000000..4d99c1fedd --- /dev/null +++ b/changelog.d/changed-14718.md @@ -0,0 +1 @@ +- dashboard: rename the overview-export encode-failure log helper from `slogWarnOverviewEncode` to `logOverviewEncodeFailure` to match its Debug-level client-disconnect handling ([#14718](https://github.com/hivecommons/hive/issues/14718)). diff --git a/src/pkg/dashboard/overview_export.go b/src/pkg/dashboard/overview_export.go index ccbe9f369b..0d0ddd722e 100644 --- a/src/pkg/dashboard/overview_export.go +++ b/src/pkg/dashboard/overview_export.go @@ -83,18 +83,18 @@ func (s *Server) handleOverviewExport(w http.ResponseWriter, r *http.Request, ki w.Header().Set("Content-Type", overviewJSONType) resp := overviewJSONResponse{Kind: kind, GeneratedAt: now, HiveID: status.HiveID, StaleDays: normalizeIssueBandsConfig(cfg).StaleDays, Bands: bands, Rows: rows} if err := json.NewEncoder(w).Encode(resp); err != nil { - slogWarnOverviewEncode(err) + logOverviewEncodeFailure(err) } return } w.Header().Set("Content-Type", overviewCSVType) w.Header().Set("Content-Disposition", `inline; filename="`+overviewFilename(status.HiveID, kind, filters, bands, now)+`"`) if _, err := w.Write([]byte(overviewCSV(rows, overviewCSVColumns(kind)))); err != nil { - slogWarnOverviewEncode(err) + logOverviewEncodeFailure(err) } } -func slogWarnOverviewEncode(err error) { +func logOverviewEncodeFailure(err error) { if err != nil { logEncodeFailure("overview export encode failed", err) } diff --git a/src/pkg/dashboard/overview_export_14696_test.go b/src/pkg/dashboard/overview_export_14696_test.go index 76a6537e7a..e83b79f389 100644 --- a/src/pkg/dashboard/overview_export_14696_test.go +++ b/src/pkg/dashboard/overview_export_14696_test.go @@ -17,13 +17,13 @@ func TestOverviewEncodeLogLevels14696(t *testing.T) { slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))) defer slog.SetDefault(prev) - slogWarnOverviewEncode(&net.OpError{Op: "write", Err: os.NewSyscallError("write", syscall.EPIPE)}) + logOverviewEncodeFailure(&net.OpError{Op: "write", Err: os.NewSyscallError("write", syscall.EPIPE)}) if strings.Contains(buf.String(), "level=WARN") { t.Errorf("client disconnect must not log WARN, got:\n%s", buf.String()) } buf.Reset() - slogWarnOverviewEncode(errors.New("boom")) + logOverviewEncodeFailure(errors.New("boom")) if !strings.Contains(buf.String(), "level=WARN") { t.Errorf("genuine encode failure should log WARN, got:\n%s", buf.String()) } diff --git a/src/pkg/dashboard/stage_tracker.go b/src/pkg/dashboard/stage_tracker.go index 281077acf6..d752730a70 100644 --- a/src/pkg/dashboard/stage_tracker.go +++ b/src/pkg/dashboard/stage_tracker.go @@ -1685,12 +1685,12 @@ func (s *Server) encodeStageTracker(status *StatusPayload, now time.Time, ttl ti hashResp := resp hashResp.GeneratedAt = time.Time{} if err := json.NewEncoder(&hashBuf).Encode(hashResp); err != nil { - slogWarnOverviewEncode(err) + logOverviewEncodeFailure(err) return stageTrackerCache{}, false } var buf bytes.Buffer if err := json.NewEncoder(&buf).Encode(resp); err != nil { - slogWarnOverviewEncode(err) + logOverviewEncodeFailure(err) return stageTrackerCache{}, false } return stageTrackerCache{body: buf.Bytes(), etag: webstatic.ETagFor(hashBuf.Bytes()), generatedAt: now}, true @@ -1767,7 +1767,7 @@ func (s *Server) handleStageTracker(w http.ResponseWriter, r *http.Request) { body = bytes.Replace(body, stageTrackerSourceCache, stageTrackerSourceStale, 1) } if _, err := w.Write(body); err != nil { - slogWarnOverviewEncode(err) + logOverviewEncodeFailure(err) } } From edaeb7e880f245fea3197abada26d1d7a2f8c65a Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 27 Oct 2026 21:02:25 -0400 Subject: [PATCH 19/38] =?UTF-8?q?=F0=9F=8C=B1=20dashboard:=20refresh=20cod?= =?UTF-8?q?ex=20static=20model=20catalog=20to=200.165.0=20(#14770)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Captured live from `codex app-server` model/list against the 0.165.0 binary (same exchange cli_models_codex.go performs). Diff against the 0.159.0 snapshot: adds gpt-6.1-sol (visible, isDefault, present since at least 0.160.0) and gpt-6.1-terra (visible, new in 0.165.0). Hidden ids unchanged. index.html mirror and the #9804 parity test updated. Signed-off-by: Andrew Anderson --- changelog.d/changed-codex-catalog-0165.md | 1 + .../dashboard/cli_model_lists_9804_test.go | 4 ++-- src/pkg/dashboard/cli_models.go | 20 +++++++++++-------- src/pkg/dashboard/static/index.html | 15 ++++++++------ 4 files changed, 24 insertions(+), 16 deletions(-) create mode 100644 changelog.d/changed-codex-catalog-0165.md diff --git a/changelog.d/changed-codex-catalog-0165.md b/changelog.d/changed-codex-catalog-0165.md new file mode 100644 index 0000000000..2e2141b409 --- /dev/null +++ b/changelog.d/changed-codex-catalog-0165.md @@ -0,0 +1 @@ +- Codex model fallback refreshed to the 0.165.0 catalog: `gpt-6.1-sol` (the CLI's current default) and the new `gpt-6.1-terra` appear in the dashboard's codex model picker when the live `codex app-server` probe cannot run, instead of the stale 0.159.0 snapshot that lacked both diff --git a/src/pkg/dashboard/cli_model_lists_9804_test.go b/src/pkg/dashboard/cli_model_lists_9804_test.go index d07daee00d..b794c37bf9 100644 --- a/src/pkg/dashboard/cli_model_lists_9804_test.go +++ b/src/pkg/dashboard/cli_model_lists_9804_test.go @@ -42,10 +42,10 @@ func TestIssue9804ModelListRefresh(t *testing.T) { mustAbsent: []string{"gemini-3-pro-preview", "gemini-2.5-pro"}, }, { - name: "codex fallback matches pinned 0.159.0 snapshot", + name: "codex fallback matches the 0.165.0 snapshot", models: codexStaticModels, htmlConst: "CODEX_CLI_MODELS", - mustHave: []string{"gpt-6-astra", "gpt-6-sol", "gpt-6-luna"}, + mustHave: []string{"gpt-6.1-sol", "gpt-6.1-terra", "gpt-6-astra", "gpt-6-sol", "gpt-6-luna"}, mustAbsent: []string{"gpt-5.4"}, }, { diff --git a/src/pkg/dashboard/cli_models.go b/src/pkg/dashboard/cli_models.go index 0799c91159..81f4c7c868 100644 --- a/src/pkg/dashboard/cli_models.go +++ b/src/pkg/dashboard/cli_models.go @@ -406,23 +406,27 @@ var bobStaticModels = []string{bobAutoModel} // models β€” Claude ids are rejected with a ChatGPT account ("model is not // supported when using Codex with a ChatGPT account"), so codex must never // fall through to the copilot list. The catalog is baked into the CLI binary -// (per-CLI-version, not per-account); this snapshot matches codex 0.159.0 -// (captured 2026-09-30 from a live `codex app-server` model/list exchange -// against the pinned 0.159.0 binary, #9804): visible ids in picker order -// (astra is the default; GPT-6 Sol and Luna arrived in 0.156.1, #8417), then -// the hidden ids β€” hidden entries are still valid --model values (see -// orderCodexServedModels). gpt-5.2 and gpt-5.4-mini were retired from the -// catalog in 0.155.0; gpt-5.4 was retired between 0.156.1 and 0.159.0. +// (per-CLI-version, not per-account); this snapshot matches codex 0.165.0 +// (captured 2026-10-27 from a live `codex app-server` model/list exchange +// against the 0.165.0 binary; the 0.159.0 capture was #9804): visible ids in +// picker order (GPT-6.1 Sol is the default and has been since at least +// 0.160.0; GPT-6.1 Terra arrived in 0.165.0; GPT-6 Sol and Luna arrived in +// 0.156.1, #8417), then the hidden ids β€” hidden entries are still valid +// --model values (see orderCodexServedModels). gpt-5.2 and gpt-5.4-mini were +// retired from the catalog in 0.155.0; gpt-5.4 was retired between 0.156.1 +// and 0.159.0. // Keep in sync with CODEX_CLI_MODELS in static/index.html. var codexStaticModels = []string{ + "gpt-6.1-sol", "gpt-6-astra", "gpt-6-sol", "gpt-6-luna", "gpt-5.6-sol", + "gpt-6.1-terra", "gpt-5.6-terra", "gpt-5.6-luna", "gpt-5.5", - // Hidden in the 0.159.0 picker but still valid --model values. + // Hidden in the 0.165.0 picker but still valid --model values. "gpt-daybreak-blue-latest", "gpt-daybreak-red-latest", "codex-auto-review", diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index dee1a4c419..50eebe1f94 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -24055,18 +24055,21 @@

All labels

// using Codex with a ChatGPT account"), so codex must NOT fall through to // the copilot list. The server discovers the live catalog via // `codex app-server` model/list (per-CLI-version, baked into the binary); - // this is only the pre-discovery fallback. Snapshot of codex 0.159.0 - // (live model/list capture, 2026-09-30, #9804): visible ids in picker - // order (astra is the default; GPT-6 Sol and Luna arrived in 0.156.1, - // #8417), then hidden ids β€” hidden entries are still valid --model - // values. gpt-5.2 and gpt-5.4-mini were retired in 0.155.0; gpt-5.4 was - // retired between 0.156.1 and 0.159.0. + // this is only the pre-discovery fallback. Snapshot of codex 0.165.0 + // (live model/list capture, 2026-10-27; the 0.159.0 capture was #9804): + // visible ids in picker order (GPT-6.1 Sol is the default; GPT-6.1 Terra + // arrived in 0.165.0; GPT-6 Sol and Luna arrived in 0.156.1, #8417), then + // hidden ids β€” hidden entries are still valid --model values. gpt-5.2 and + // gpt-5.4-mini were retired in 0.155.0; gpt-5.4 was retired between + // 0.156.1 and 0.159.0. // Keep in sync with codexStaticModels in cli_models.go. const CODEX_CLI_MODELS = [ + { value: 'gpt-6.1-sol', label: 'GPT-6.1 Sol' }, { value: 'gpt-6-astra', label: 'GPT-6 Astra' }, { value: 'gpt-6-sol', label: 'GPT-6 Sol' }, { value: 'gpt-6-luna', label: 'GPT-6 Luna' }, { value: 'gpt-5.6-sol', label: 'GPT-5.6 Sol' }, + { value: 'gpt-6.1-terra', label: 'GPT-6.1 Terra' }, { value: 'gpt-5.6-terra', label: 'GPT-5.6 Terra' }, { value: 'gpt-5.6-luna', label: 'GPT-5.6 Luna' }, { value: 'gpt-5.5', label: 'GPT-5.5' }, From 23fdfd6b4ca22074840950f736c6fd5453399bbd Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 27 Oct 2026 21:02:49 -0400 Subject: [PATCH 20/38] =?UTF-8?q?=F0=9F=90=9B=20contribute:=20parse=20ever?= =?UTF-8?q?y=20target-branch=20declaration=20spelling=20(#14768)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scanner's newer issue templates write the release line as '**Target branch:** `v6`' or inline after a sentence ('Target: v6 (open the PR with --base v6)'). targetBranchDeclarationPattern only matched a line beginning 'target: vN', so every relay dispatch of those issues defaulted to the hub branch and came back wrong_base, leaving the issue at 010 with no visible reason. Fixes #14764 Signed-off-by: Andy Anderson --- ...ed-contribute-target-branch-declaration.md | 1 + .../contribute_task_base_branch_test.go | 29 +++++++++++++++++++ src/pkg/dashboard/contribute_task_prompt.go | 21 ++++++++++---- 3 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 changelog.d/fixed-contribute-target-branch-declaration.md diff --git a/changelog.d/fixed-contribute-target-branch-declaration.md b/changelog.d/fixed-contribute-target-branch-declaration.md new file mode 100644 index 0000000000..1268eed248 --- /dev/null +++ b/changelog.d/fixed-contribute-target-branch-declaration.md @@ -0,0 +1 @@ +- Contributor dispatch now reads every spelling the scanner uses for an issue's release-line declaration β€” `**Target branch:** \`v6\``, `Target-branch: v6` and an inline `… Target: v6 (open the PR with --base v6)` β€” so the task prompt names the declared base instead of the hub branch and the contributor no longer has to answer `wrong_base` ([#14764](https://github.com/hivecommons/hive/issues/14764)). diff --git a/src/pkg/dashboard/contribute_task_base_branch_test.go b/src/pkg/dashboard/contribute_task_base_branch_test.go index 262c032433..9a06e42723 100644 --- a/src/pkg/dashboard/contribute_task_base_branch_test.go +++ b/src/pkg/dashboard/contribute_task_base_branch_test.go @@ -385,6 +385,35 @@ func TestTaskBaseBranch_DeclaredIssueTarget(t *testing.T) { } } +// hivecommons/hive#14764: the scanner's newer templates write the declaration +// as "**Target branch:** `v6`" or inline after a sentence ("… Depends on +// #14558. Target: v6 (open the PR with `--base v6`)."). Neither matched, so +// every relay dispatch of those issues defaulted to the hub branch and came +// back wrong_base. +func TestReleaseLineFromBody_TemplateSpellings(t *testing.T) { + cases := map[string]string{ + "Context.\n\n**Target branch:** `v6`\n\nDetails.": "v6", + "**Target branch**: v7": "v7", + "Target-branch: v5": "v5", + "target branch: `v6`.": "v6", + "Part of #14550. Depends on #14558. Target: v6 (open the PR with `--base v6`).": "v6", + "Scope is narrow; target: v6 β€” nothing else.": "v6", + "**Target: v6.** Running build `abc123`.": "v6", + "the target: v6 is discussed": "", + "Target branch: main": "", + "Retarget: v6 later": "", + } + for body, want := range cases { + if got := releaseLineFromBody(body); got != want { + t.Errorf("releaseLineFromBody(%q) = %q, want %q", body, got, want) + } + } + body := "Part of #14550. Target: v6 (open the PR with `--base v6`)." + if got := taskBaseBranchForIssue("plain title", body, "", "hivecommons/hive", "v5"); got != "v6" { + t.Fatalf("inline body target = %q, want v6", got) + } +} + func TestBuildTaskPrompt_DeclaredBodyTargetAndWrongBaseContract(t *testing.T) { base := taskBaseBranchForIssue("scanner finding", "**Target: v6.**", "", "hivecommons/hive", "v5") prompt := buildTaskPromptBody("hivecommons/hive", "hivecommons/hive#13146", "scanner finding", "", base) diff --git a/src/pkg/dashboard/contribute_task_prompt.go b/src/pkg/dashboard/contribute_task_prompt.go index 1c45216606..ba27aebc3d 100644 --- a/src/pkg/dashboard/contribute_task_prompt.go +++ b/src/pkg/dashboard/contribute_task_prompt.go @@ -205,11 +205,22 @@ func sourceLabel(sourceType string) string { return sourceType } -var targetBranchDeclarationPattern = regexp.MustCompile(`(?im)^[ \t]*(?:\*\*)?target:[ \t]*(v[0-9]+)\.?(?:\*\*)?(?:[ \t]|$)`) - -// releaseLineFromBody extracts the scanner's explicit issue-body declaration, -// including the observed "**Target: v6.**" spelling. Only release-line names -// are accepted; prose containing "target" is not a routing directive. +// targetBranchDeclarationPattern matches every spelling the scanner's issue +// templates have used for the release-line declaration: +// +// **Target: v6.** +// **Target branch:** `v6` +// Target-branch: v6 +// … Depends on #14558. Target: v6 (open the PR with `--base v6`). +// +// A declaration starts a line or follows a sentence boundary; the value may +// be backticked. Prose that merely mentions "the target: v6" is not a +// routing directive (hivecommons/hive#14764). +var targetBranchDeclarationPattern = regexp.MustCompile("(?im)(?:^[ \t]*|[.;)][ \t]+)(?:\\*\\*)?target(?:[ \t-]branch)?(?:\\*\\*)?:(?:\\*\\*)?[ \t]*`?(v[0-9]+)`?\\.?(?:\\*\\*)?(?:[ \t]|$|\\()") + +// releaseLineFromBody extracts the scanner's explicit issue-body declaration +// (see targetBranchDeclarationPattern). Only release-line names are accepted; +// prose containing "target" is not a routing directive. func releaseLineFromBody(body string) string { match := targetBranchDeclarationPattern.FindStringSubmatch(body) if len(match) != 2 { From b835abd084fc0b05c33c8329f9bab802d628c738 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 27 Oct 2026 21:03:05 -0400 Subject: [PATCH 21/38] =?UTF-8?q?=F0=9F=90=9B=20ci:=20never=20report=20the?= =?UTF-8?q?=20fragment=20guard=20as=20skipped=20on=20label=20events=20(#14?= =?UTF-8?q?767)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andy Anderson --- .../workflows/changelog-fragment-guard.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/changelog-fragment-guard.yml b/.github/workflows/changelog-fragment-guard.yml index 896c7784dd..3de09a155b 100644 --- a/.github/workflows/changelog-fragment-guard.yml +++ b/.github/workflows/changelog-fragment-guard.yml @@ -75,14 +75,17 @@ concurrency: jobs: changelog-fragment-guard: name: changelog-fragment-guard - # Label events only matter when the label is `no-changelog`; every other - # label (size/*, dco-signoff, agent/*, hive/*) cannot change the verdict. - # A skipped job still reports a check-run, and GitHub treats "skipped" as - # passing for a required context, so this keeps the rollup green instead - # of adding a redundant run per label (#11250). - if: >- - (github.event.action != 'labeled' && github.event.action != 'unlabeled') - || github.event.label.name == 'no-changelog' + # No job-level `if:` on label events β€” on purpose (#14755). This job used + # to skip itself for every label other than `no-changelog`, and a skipped + # job still reports a check-run that GitHub scores as *passing* for a + # required context. hive's labeler adds three or four labels seconds + # after every push, so a genuine failure on the push-triggered run was + # superseded by a "skipped" sibling on the same head SHA and tide merged + # a malformed fragment (#14629), which then broke the release `decide` + # job on v5 and held every open v5 PR behind a red base. The guard is + # cheap (git, grep, echo); re-running it on each label event keeps the + # reported verdict truthful. Label runs are not cancelled by concurrency + # (see above), so they cannot leave CANCELLED check-runs either. # Forward-merge sync PRs (v5-topup.yml / v6-topup.yml, and the hand-rolled # ones maintainers open when those conflict) live on sync/* branches β€” # or on /sync-vN-to-vM when an agent opens the top-up, since the From 4e9346be96d137242d4f52f5026c5dc0113e21a3 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 27 Oct 2026 21:10:46 -0400 Subject: [PATCH 22/38] =?UTF-8?q?=F0=9F=90=9B=20Follow=20repo=20renames=20?= =?UTF-8?q?and=20fix=20engine=20relationship=20rule=20in=20the=20weekly=20?= =?UTF-8?q?competitor=20report=20(#14757)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andrew Anderson Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../weekly-competitor-report.lock.yml | 2 +- .github/workflows/weekly-competitor-report.md | 52 +++++++++++++------ ...3-competitor-report-renames-engine-rule.md | 1 + src/deploy/gh-aw/README.md | 23 +++++++- 4 files changed, 60 insertions(+), 18 deletions(-) create mode 100644 changelog.d/fixed-14613-competitor-report-renames-engine-rule.md diff --git a/.github/workflows/weekly-competitor-report.lock.yml b/.github/workflows/weekly-competitor-report.lock.yml index bd345d003f..f3827d3d2b 100644 --- a/.github/workflows/weekly-competitor-report.lock.yml +++ b/.github/workflows/weekly-competitor-report.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"a92f9252483f4b49b18cc01141db51e8a7afdadbe3fd936f9f138e1029b484b2","body_hash":"1628e2f12ed90fb50b928ff6c25f7e2310f153be8240d6d5b864a33a7fbc6fb3","compiler_version":"v0.89.21","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.87"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"a92f9252483f4b49b18cc01141db51e8a7afdadbe3fd936f9f138e1029b484b2","body_hash":"fef1288b3894831da705033d4cb4a755e26330652a235c148d799c72e682a3dc","compiler_version":"v0.89.21","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.87"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","DISCORD_RELATED_RESEARCH_WEBHOOK","GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"924af5fdc64061cfbf66fb584c8b07e2ac230c60","version":"v0.89.21"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.23","digest":"sha256:2c78aaba1c108e130e2d6d01e4f2cca334ea04c53e6f258913ac34173fe7e3b2","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.23@sha256:2c78aaba1c108e130e2d6d01e4f2cca334ea04c53e6f258913ac34173fe7e3b2"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.23","digest":"sha256:c15c3d1208df10c5b588a3657be53742aa982ae0909d1eb1812525268794ca64","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.23@sha256:c15c3d1208df10c5b588a3657be53742aa982ae0909d1eb1812525268794ca64"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.23","digest":"sha256:02ffc56dd40158223064ef03a78c2d9717473c93723b4e403d455ea6f0b09ae0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.23@sha256:02ffc56dd40158223064ef03a78c2d9717473c93723b4e403d455ea6f0b09ae0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.25","digest":"sha256:9be0a86220e807a0ecc89e53d7453468f7a53fbc6b3d1efd2299025ffe01d086","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.25@sha256:9be0a86220e807a0ecc89e53d7453468f7a53fbc6b3d1efd2299025ffe01d086"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f","pinned_image":"ghcr.io/github/gh-aw-node@sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f"},{"image":"ghcr.io/github/github-mcp-server:v1.12.2","digest":"sha256:508a0857ec762b1ab1cece29193345b501fab1dd9d1228a7b617062954cecac6","pinned_image":"ghcr.io/github/github-mcp-server:v1.12.2@sha256:508a0857ec762b1ab1cece29193345b501fab1dd9d1228a7b617062954cecac6"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_releases","list_starred_repositories","list_tags","search_code","search_issues","search_repositories"]},{"name":"safeoutputs","tools":["add_comment","create_issue","missing_data","missing_tool","noop","post_to_discord"]}]} # This file was automatically generated by gh-aw (v0.89.21). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # diff --git a/.github/workflows/weekly-competitor-report.md b/.github/workflows/weekly-competitor-report.md index fa8c3d85a6..47ee34cce3 100644 --- a/.github/workflows/weekly-competitor-report.md +++ b/.github/workflows/weekly-competitor-report.md @@ -273,7 +273,11 @@ Hive has today: (Antigravity), pi, opencode, kilo, muse, omp, openhands, omnigent. Source: `CLIBackends` in `config/agent_runtime.go` in hivecommons/crux. So Claude Code, Copilot CLI, Codex CLI, Gemini CLI, Goose, Aider, Kilo, OpenCode and - OpenHands are engines Hive runs as workers, not competitors. + OpenHands are engines Hive runs as workers, not competitors. This holds + even when the same company also sells a hosted or orchestration product + (OpenHands Cloud, Kilo cloud, Codex cloud tasks): the engine row says + "Hive runs it as a worker engine today". If the hosted product matters, + give it its own row (for example "OpenHands Cloud"). - **A governor with six trust levels (ACMM L1-L6)** that decides what agents may do, from advisory only to auto-merge. Source: `README.md` "ACMM Levels", `src/docs/acmm-policy-matrix.md`. @@ -302,14 +306,25 @@ Hive has today: Check at least these, plus anything new you find that clearly belongs: -OpenHands and OpenHands Cloud (All-Hands AI), SWE-agent, Devin (Cognition), -GitHub Copilot coding agent, OpenAI Codex (cloud tasks and CLI), Claude Code -(autonomy and background-agent features), Sweep, Factory, Jules (Google), -Gastown, Dagger agents, Kilo Code and Kilo cloud, Cline (and Cline SDK), Roo -Code (orchestrator mode), Mentat, Aider, GPT-Engineer, Potpie, Agentless, -CrewAI and LangGraph (only where they do repository work), and any internal -agent-fleet orchestrator with a public write-up (engineering blog posts from -large companies about running many coding agents at once). +OpenHands (engine, repo `OpenHands/OpenHands`, moved from `All-Hands-AI`) and +OpenHands Cloud (All-Hands AI), SWE-agent (`SWE-agent/SWE-agent`, moved from +`princeton-nlp`), Devin (Cognition), GitHub Copilot coding agent, OpenAI Codex +(cloud tasks and CLI, `openai/codex`), Claude Code (`anthropics/claude-code`, +autonomy and background-agent features), Goose (`aaif-goose/goose`, moved from +`block`), OpenCode (`anomalyco/opencode`, moved from `sst`), Sweep, Factory, +Jules (Google), Gastown, Dagger agents, Kilo Code and Kilo cloud (`Kilo-Org/kilocode`), +Cline (and Cline SDK), Roo Code (orchestrator mode), Mentat (CLI repo is +archived as `AbanteAI/archive-old-cli-mentat`; say so, do not report +"API error"), Aider, GPT-Engineer (`AntonOsika/gpt-engineer`, archived), +Potpie, Agentless (`OpenAutoCoder/Agentless`), CrewAI and LangGraph (only +where they do repository work), and any internal agent-fleet orchestrator +with a public write-up (engineering blog posts from large companies about +running many coding agents at once). + +Repositories move. The GitHub API answers a renamed repository with a 301 +redirect, so always follow redirects (`curl -sL`) and read `full_name` from +the response to learn the current name. Never write "API error" for a +licence when the redirect target answers. Sort each project into one category: @@ -329,7 +344,10 @@ Then say the relationship to Hive, one of: "Hive runs it as a worker engine today" (it is in the Crux list above), "Hive could run it as an engine", "direct competitor for fleet orchestration", or "complementary". Terminal and editor agents are mostly engines Hive drives. Direct competitors are the -fleet and orchestration layers. +fleet and orchestration layers. Rule: a project in the Crux list is always +"Hive runs it as a worker engine today", even if its company also sells +orchestration. Do not mark OpenHands, Kilo, Codex, or any other listed engine +"direct competitor"; put the hosted product in its own row if you need to. ## How to research @@ -341,11 +359,15 @@ last 7 days before that date. Also compute the unix time for 7 days ago with look at releases, tags, and the most recent commits and merged pull requests in the last 7 days. `search_repositories`, `list_releases`, `list_commits`, and `search_issues` are enough. Read the licence from the - repository's `license.spdx_id` field (for example with - `curl -s https://api.github.com/repos// | jq -r .license.spdx_id`) - and write that value in the table. Write "proprietary" only for hosted - products with no public repository. Do not write "unknown" for anything - that has a GitHub repository. + repository's `license.spdx_id` field with a redirect-following request, + for example + `curl -sL https://api.github.com/repos// | jq -r '"\(.full_name) \(.license.spdx_id)"'`, + and write the SPDX value in the table. Without `-L` a renamed repository + returns a 301 with no licence field. If `full_name` differs from what you + asked for, use the new name in the table. If the API returns `null` for + `license` the repository has no licence file: write "none". Write + "proprietary" only for hosted products with no public repository. Do not + write "unknown" or "API error" for anything that has a GitHub repository. 2. Use `curl` through bash for news. Stay inside the configured network allowlist. Keep TLS checks on. If a request fails, note it and move on; do not try to work around the firewall. Pipelines like `curl ... | jq` and diff --git a/changelog.d/fixed-14613-competitor-report-renames-engine-rule.md b/changelog.d/fixed-14613-competitor-report-renames-engine-rule.md new file mode 100644 index 0000000000..0f80aef26c --- /dev/null +++ b/changelog.d/fixed-14613-competitor-report-renames-engine-rule.md @@ -0,0 +1 @@ +- **Weekly competitor report: renamed repos and engine relationship.** The 2026-10-27 report showed OpenHands' licence as "n/a (API error)" because the repository moved to `OpenHands/OpenHands` and the licence lookup did not follow the GitHub API's 301 redirect; the prompt now uses `curl -sL`, reads `full_name`, and names the current repository for each moved project (OpenHands, SWE-agent, Goose, OpenCode, archived Mentat and GPT-Engineer). The same report marked OpenHands "direct competitor" although Crux has an `openhands` backend; the prompt now states that any engine in the Crux `CLIBackends` list is "Hive runs it as a worker engine today" even when its vendor also sells orchestration. The gh-aw README records that the "Firewall blocked N domains" footer is an upstream gh-aw v0.89.21 false positive (github/gh-aw#63221, fixed in v0.89.22) (#14613). diff --git a/src/deploy/gh-aw/README.md b/src/deploy/gh-aw/README.md index c53a33e92f..e9a7d58897 100644 --- a/src/deploy/gh-aw/README.md +++ b/src/deploy/gh-aw/README.md @@ -46,8 +46,9 @@ compiles without model credentials, and tests admission and the actual classifie (compiled to `weekly-competitor-report.lock.yml`): every Monday at 13:00 UTC, and on manual dispatch, the Copilot engine researches projects adjacent to Hive (OpenHands, SWE-agent, Devin, Copilot coding agent, Codex, Claude Code, - Sweep, Factory, Jules, Gastown, Dagger, Kilo, Cline/Roo, Mentat, Aider, - GPT-Engineer, Potpie, Agentless, public fleet-orchestrator write-ups) with + Goose, OpenCode, Sweep, Factory, Jules, Gastown, Dagger, Kilo, Cline/Roo, + Mentat, Aider, GPT-Engineer, Potpie, Agentless, public fleet-orchestrator + write-ups) with read-only GitHub tools and `curl` inside a network allowlist (HN Algolia, Google News RSS, Bing News RSS, the GitHub changelog feed, vendor blogs; the bash allowlist uses `cmd:*` entries so pipelines like `curl | jq` are @@ -83,6 +84,24 @@ compiles without model credentials, and tests admission and the actual classifie the webhook secret the Discord job fails with a clear message and the GitHub comment still lands. Recompile after edits with the pinned compiler: `gh aw compile weekly-competitor-report`, and commit both files (#14612). + The prompt names the current GitHub repository for each project (several + moved: `OpenHands/OpenHands`, `SWE-agent/SWE-agent`, `aaif-goose/goose`, + `anomalyco/opencode`) and tells the model to follow API redirects with + `curl -sL`, because a renamed repository answers `301` with no `license` + field, which showed up as "n/a (API error)" in the 2026-10-27 report. Any + engine in the Crux `CLIBackends` list is reported as "Hive runs it as a + worker engine today" even when its vendor also sells orchestration. + Known cosmetic issue: the "Firewall blocked N domains" warning that gh-aw + appends to the comment is a false positive on gh-aw v0.89.21. Squid writes + two lines per successful HTTPS `CONNECT` (`200 NONE_NONE:HIER_NONE`, then + `200 TCP_TUNNEL:HIER_DIRECT`) and the footer parser counted the first as + a block, so every allowed domain showed equal allowed and denied counts + (run 40579499092: `api.github.com` 36/36, `news.google.com` 6/6). This is + upstream, not in the lock file: github/gh-aw#63221, fixed by + github/gh-aw#63246 (2026-09-24), first shipped in gh-aw v0.89.22. The + warning goes away when the compiler pin is bumped to v0.89.22 or later; + until then ignore it when the firewall log's Allowed column matches the + Denied column. ## Graduate to Hive without rewriting workflows From 7606a2cf933a3bf6ca29d91d32e9c71a5e8ffd7b Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 27 Oct 2026 21:12:46 -0400 Subject: [PATCH 23/38] =?UTF-8?q?=F0=9F=8C=B1=20images:=20bump=20claude=20?= =?UTF-8?q?2.1.311=20->=202.1.312=20(#14773)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated pin bump (#8419). Digests recomputed from the downloaded release artifacts by src/scripts/cli-pin-bump.sh: VERSION=2.1.312 Signed-off-by: hive-release[bot] Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: hive-release[bot] --- src/Dockerfile | 2 +- src/Dockerfile.contributor | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Dockerfile b/src/Dockerfile index 2484e218c7..26682629d7 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -319,7 +319,7 @@ RUN npm install -g --ignore-scripts --no-fund --no-audit @openai/codex@${CODEX_V echo "WARN: OpenAI Codex CLI install failed β€” skipping" # --- Layer 7: Claude Code (most frequent updates) --- -ARG CLAUDE_CODE_VERSION=2.1.311 +ARG CLAUDE_CODE_VERSION=2.1.312 # --ignore-scripts (kept, deliberately: no arbitrary postinstall runs during the # build) skips claude-code's install.cjs, which is what LINKS the platform # native binary into the package's bin/. The binary itself ships in the diff --git a/src/Dockerfile.contributor b/src/Dockerfile.contributor index 85791a7167..ff3b9c960e 100644 --- a/src/Dockerfile.contributor +++ b/src/Dockerfile.contributor @@ -237,7 +237,7 @@ RUN mkdir -p /etc/apt/keyrings \ # npm cache is cleaned in the same layer it's populated (issue #2613, # Observation 2 "also minor") so the cache doesn't linger in this layer's # diff, mirroring the cleanup already done for the bobshell install below. -ARG CLAUDE_CODE_VERSION=2.1.311 +ARG CLAUDE_CODE_VERSION=2.1.312 ARG COPILOT_VERSION=1.0.88 # --ignore-scripts (kept, deliberately: no arbitrary postinstall runs during # the build) skips claude-code's install.cjs, which is what LINKS the platform From 9de515f774ad391a9049977186f3728e84a4f5bf Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 27 Oct 2026 21:13:09 -0400 Subject: [PATCH 24/38] fix(dashboard): surface GitHub blocked-by dependencies on Actionable rows (#14772) An issue the scheduler withholds for an open "blocked by" dependency rendered as plain Actionable ("waiting for the scanner"), so sequenced RFC sub-issues read as intake starvation. The row now carries the blocked modifier, blocked_on keys, and a "waiting on #N" caption; the chip names the blocker. Signed-off-by: Andy Anderson --- changelog.d/fixed-stage-tracker-blocked-by.md | 1 + src/pkg/dashboard/stage_tracker.go | 53 +++++++++- .../stage_tracker_blocked_by_test.go | 96 +++++++++++++++++++ src/pkg/dashboard/static/index.html | 5 +- 4 files changed, 152 insertions(+), 3 deletions(-) create mode 100644 changelog.d/fixed-stage-tracker-blocked-by.md create mode 100644 src/pkg/dashboard/stage_tracker_blocked_by_test.go diff --git a/changelog.d/fixed-stage-tracker-blocked-by.md b/changelog.d/fixed-stage-tracker-blocked-by.md new file mode 100644 index 0000000000..144ca9d017 --- /dev/null +++ b/changelog.d/fixed-stage-tracker-blocked-by.md @@ -0,0 +1 @@ +- The stage tracker now surfaces an Actionable issue the scheduler withholds for an open GitHub "blocked by" dependency: the row wears the `blocked` modifier, the chip and tooltip name the blocker(s), the payload carries `blocked_on`, and the caption reads "waiting on #N" instead of plain "waiting for the scanner" β€” sequenced RFC sub-issues no longer look like intake starvation (hivecommons/hive#14765). diff --git a/src/pkg/dashboard/stage_tracker.go b/src/pkg/dashboard/stage_tracker.go index d752730a70..00cb6088dc 100644 --- a/src/pkg/dashboard/stage_tracker.go +++ b/src/pkg/dashboard/stage_tracker.go @@ -325,7 +325,13 @@ type StageTrackerItem struct { // issue at 020 or later (015 is not a gate; the reconciler removes the // label on its next pass, hivecommons/hive#14586). Rendered as a muted // operator tooltip, never as the stage word. - StaleLabels []string `json:"stale_labels,omitempty"` + StaleLabels []string `json:"stale_labels,omitempty"` + // BlockedOn lists the open GitHub "blocked by" dependencies ("repo#N") + // of an issue the scheduler withholds from every kick until they close + // (scheduler.partitionBlockedIssues). Without it a sequenced sub-issue + // read as plain Actionable nobody was picking up (hivecommons/hive#14765); + // the row wears the `blocked` modifier and the caption names the wait. + BlockedOn []string `json:"blocked_on,omitempty"` NudgeCount int `json:"nudge_count,omitempty"` LastNudgeAt *time.Time `json:"last_nudge_at,omitempty"` AdoptedBy int `json:"adopted_by,omitempty"` @@ -674,6 +680,10 @@ func (s *Server) stageTrackerSentinel(verifier *github.ReviewEvidenceVerifier, r } return alert, findings, reason } + +// stageTrackerModifierOrder is the modifier precedence the payload carries. +var stageTrackerModifierOrder = []string{modStalled, modRegressed, modHold, modUnauthorized, modNeedsHuman, modNeedsDecision, modNeedsTriage, modPossibleDupe, modNeedsRebase, modBlocked, modDraft} + func stageTrackerModifiers(labels []string, hiveID string, rec stagelabels.Record, hasRec bool, overrides map[string]time.Duration, now time.Time, draft bool) []string { set := map[string]bool{} for _, l := range labels { @@ -718,7 +728,7 @@ func stageTrackerModifiers(labels []string, hiveID string, rec stagelabels.Recor set[modDraft] = true } out := make([]string, 0, len(set)) - for _, m := range []string{modStalled, modRegressed, modHold, modUnauthorized, modNeedsHuman, modNeedsDecision, modNeedsTriage, modPossibleDupe, modNeedsRebase, modBlocked, modDraft} { + for _, m := range stageTrackerModifierOrder { if set[m] { out = append(out, m) } @@ -726,6 +736,44 @@ func stageTrackerModifiers(labels []string, hiveID string, rec stagelabels.Recor return out } +// stageTrackerBlockedBy marks an open issue the scheduler is deliberately +// holding back for unresolved GitHub "blocked by" dependencies +// (hivecommons/hive#14765). The kick list already withholds such issues +// ("Blocked by open dependencies … do NOT start these"), but the tracker +// showed them as plain Actionable with reason `actionable`, so sequenced +// sub-issues of an RFC read as intake starvation. The row gets the same +// `blocked` modifier the `blocked` label yields, the blocker keys, and a +// caption clause naming what it waits on. Only the pre-claim stages +// qualify: once a claim or PR exists the sequencing no longer gates it. +func stageTrackerBlockedBy(it *StageTrackerItem, blockers []string) { + if it == nil || len(blockers) == 0 || it.Stage > trackerNumberPossiblyExists { + return + } + it.BlockedOn = append([]string(nil), blockers...) + if !slices.Contains(it.Modifiers, modBlocked) { + it.Modifiers = append(it.Modifiers, modBlocked) + slices.SortFunc(it.Modifiers, func(a, b string) int { + return slices.Index(stageTrackerModifierOrder, a) - slices.Index(stageTrackerModifierOrder, b) + }) + } + it.Caption = joinClause(it.Caption, "waiting on "+stageTrackerBlockerList(it.Repo, blockers)) + it.Actor = "its blockers" +} + +// stageTrackerBlockerList renders blocker keys ("repo#N") for a caption, +// shortening a same-repo blocker to "#N". +func stageTrackerBlockerList(repo string, blockers []string) string { + parts := make([]string, 0, len(blockers)) + for _, b := range blockers { + b = strings.TrimSpace(b) + if r, n, ok := strings.Cut(b, "#"); ok && (r == "" || strings.EqualFold(r, repo) || strings.HasSuffix(strings.ToLower(repo), "/"+strings.ToLower(r))) { + b = "#" + n + } + parts = append(parts, b) + } + return strings.Join(parts, ", ") +} + // noVerdictClause is the bare no-verdict clause; stageTrackerCaption swaps // in "awaiting review of head " when the item's head is known. const noVerdictClause = "no review verdict yet" @@ -1317,6 +1365,7 @@ func (s *Server) buildStageTracker(status *StatusPayload, store *stagelabels.Sto stage, rec, ok = stagelabels.None, stagelabels.Record{}, false } finish(&it, issue.Labels, stage, rec, ok, false) + stageTrackerBlockedBy(&it, issue.OpenBlockers()) if it.Stage == trackerNumberClaimed { it.ClaimantKind = s.stageTrackerClaimantKind(repoName, issue.Number) } diff --git a/src/pkg/dashboard/stage_tracker_blocked_by_test.go b/src/pkg/dashboard/stage_tracker_blocked_by_test.go new file mode 100644 index 0000000000..51618fb677 --- /dev/null +++ b/src/pkg/dashboard/stage_tracker_blocked_by_test.go @@ -0,0 +1,96 @@ +package dashboard + +import ( + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/github" +) + +// An Actionable issue the scheduler withholds for an open GitHub "blocked +// by" dependency must say so: `blocked` modifier, the blocker keys, and a +// caption naming the wait (hivecommons/hive#14765). Once the dependency +// resolves, or the issue is past the gate (claimed, covered by a PR), the +// row is untouched. +func TestStageTrackerBlockedByDependencyIsSurfaced(t *testing.T) { + now := time.Date(2026, 10, 23, 9, 0, 0, 0, time.UTC) + s := newTestServer() + + status := minimalPayload() + status.Repos = []FrontendRepo{{ + Name: "hive", Full: "hivecommons/hive", + ActionableIssues: []any{ + github.Issue{Number: 14635, Title: "search: index", Author: "pat", AuthorIsHuman: true, + Labels: []string{"stage/010-actionable"}, + DependsOn: []github.IssueDependency{{Key: "hive#14633"}, {Key: "hivecommons/crux#19"}, {Key: "hive#14600", Resolved: true}}}, + github.Issue{Number: 14636, Title: "search: ready", Author: "pat", AuthorIsHuman: true, + Labels: []string{"stage/010-actionable"}, + DependsOn: []github.IssueDependency{{Key: "hive#14600", Resolved: true}}}, + github.Issue{Number: 14637, Title: "search: claimed", Author: "pat", AuthorIsHuman: true, + Labels: []string{"stage/020-claimed"}, + DependsOn: []github.IssueDependency{{Key: "hive#14633"}}}, + github.Issue{Number: 14638, Title: "search: labelled and linked", Author: "pat", AuthorIsHuman: true, + Labels: []string{"stage/010-actionable", "blocked"}, + DependsOn: []github.IssueDependency{{Key: "hive#14633"}}}, + }, + }} + s.UpdateStatus(status) + resp := s.buildStageTracker(s.status, s.stageStore, now) + + blocked := trackerItem(t, resp, "issue", 14635) + if blocked.Stage != 10 || !hasMod(blocked, modBlocked) { + t.Errorf("#14635 = stage %d mods %v, want 010 + blocked", blocked.Stage, blocked.Modifiers) + } + if got, want := strings.Join(blocked.BlockedOn, ","), "hive#14633,hivecommons/crux#19"; got != want { + t.Errorf("#14635 blocked_on = %q, want %q (resolved blocker dropped, sorted)", got, want) + } + if !strings.HasSuffix(blocked.Caption, "waiting on #14633, hivecommons/crux#19") { + t.Errorf("#14635 caption = %q, want it to end naming the blockers", blocked.Caption) + } + if blocked.Actor != "its blockers" { + t.Errorf("#14635 actor = %q, want its blockers", blocked.Actor) + } + + ready := trackerItem(t, resp, "issue", 14636) + if hasMod(ready, modBlocked) || ready.BlockedOn != nil || strings.Contains(ready.Caption, "waiting on") { + t.Errorf("#14636 with only resolved deps = mods %v blocked_on %v caption %q, want plain Actionable", ready.Modifiers, ready.BlockedOn, ready.Caption) + } + + claimed := trackerItem(t, resp, "issue", 14637) + if claimed.Stage != 20 || hasMod(claimed, modBlocked) || claimed.BlockedOn != nil { + t.Errorf("#14637 past the gate = stage %d mods %v blocked_on %v, want 020 untouched", claimed.Stage, claimed.Modifiers, claimed.BlockedOn) + } + + labelled := trackerItem(t, resp, "issue", 14638) + if n := countMod(labelled, modBlocked); n != 1 { + t.Errorf("#14638 with label and dependency carries blocked Γ—%d, want once: %v", n, labelled.Modifiers) + } +} + +func TestStageTrackerBlockerList(t *testing.T) { + cases := []struct { + repo string + in []string + want string + }{ + {"hive", []string{"hive#1"}, "#1"}, + {"hivecommons/hive", []string{"hive#1", "hivecommons/hive#2"}, "#1, #2"}, + {"hive", []string{"hivecommons/crux#19", "#3"}, "hivecommons/crux#19, #3"}, + } + for _, c := range cases { + if got := stageTrackerBlockerList(c.repo, c.in); got != c.want { + t.Errorf("stageTrackerBlockerList(%q, %v) = %q, want %q", c.repo, c.in, got, c.want) + } + } +} + +func countMod(it StageTrackerItem, m string) int { + n := 0 + for _, x := range it.Modifiers { + if x == m { + n++ + } + } + return n +} diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 50eebe1f94..df56424e2d 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -14282,6 +14282,8 @@

Where to get help

const handled = stageTrackerRegressionHandled(item); if (handled && mods.length > 1) mods.push(mods.shift()); let text = handled && mods[0] === 'regressed' ? STAGE_TRACKER_HANDLED_TEXT : (STAGE_TRACKER_MOD_TEXT[mods[0]] || mods[0].replace(/-/g, ' ')); + // A row the scheduler withholds for open GitHub "blocked by" dependencies names them (hivecommons/hive#14765). + if (mods[0] === 'blocked' && Array.isArray(item.blocked_on) && item.blocked_on.length) text = stageTrackerBlockedTip(item); // A blocked row with a hive:recheck-after deadline ("blocked until ") names when the sweep re-checks it. const recheck = mods[0] === 'blocked' ? /^blocked until (\S+)/i.exec(String(item.bucket_reason || '')) : null; const recheckMs = recheck ? Date.parse(recheck[1]) : NaN; @@ -14289,7 +14291,8 @@

Where to get help

// The regressed pill (lead or folded into "+1") explains itself on hover. const regressed = stageTrackerRegressionText(item, Date.now()); // An unauthorized pill names the missing App permission and the human step. - const tip = (mods.indexOf('unauthorized') >= 0 && String(item.unauthorized || '').trim()) || regressed; + const tip = (mods.indexOf('unauthorized') >= 0 && String(item.unauthorized || '').trim()) || regressed + || (mods.indexOf('blocked') >= 0 && Array.isArray(item.blocked_on) && item.blocked_on.length ? 'Withheld from the scanner until ' + item.blocked_on.join(', ') + (item.blocked_on.length > 1 ? ' close' : ' closes') : ''); const cls = 'ham-mod-' + mods[0] + (handled && mods[0] === 'regressed' ? ' ham-mod-handled' : ''); return sentinel + `${esc(text)}${mods.length > 1 ? ' +' + (mods.length - 1) : ''}`; } From 4fc7d11ade4705115222e760bb4441070379740d Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 27 Oct 2026 21:25:56 -0400 Subject: [PATCH 25/38] =?UTF-8?q?=F0=9F=90=9B=20contribute:=20link=20a=20r?= =?UTF-8?q?eported=20PR=20to=20the=20issue=20it=20was=20dispatched=20for?= =?UTF-8?q?=20(#14771)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The relay's task prompt only asks the contributor model for `Closes #N`. hivecommons/hive#14701 shipped #14633 with an empty body; the PR-claim ledger derives claims from PR text and branch names only, so the issue stayed at stage/010-actionable for hours while the PR waited in review as "no linked issue" and every lane was free to re-claim it. On a verified task_complete for a GitHub-issue task the hub now appends `Fixes #N` to the contributor's own open PR (unless the body already closes the issue or says it stays open) and ensures the GitHub sidebar link so release-line PRs show in the issue list too. Detached and best-effort like the attribution reconcile. Fixes #14763 Signed-off-by: Andy Anderson --- .../fixed-contribute-relay-pr-issue-link.md | 1 + src/pkg/dashboard/contribute_ws.go | 57 ++++++ src/pkg/github/pr_contributor_link.go | 124 ++++++++++++ src/pkg/github/pr_contributor_link_test.go | 187 ++++++++++++++++++ 4 files changed, 369 insertions(+) create mode 100644 changelog.d/fixed-contribute-relay-pr-issue-link.md create mode 100644 src/pkg/github/pr_contributor_link.go create mode 100644 src/pkg/github/pr_contributor_link_test.go diff --git a/changelog.d/fixed-contribute-relay-pr-issue-link.md b/changelog.d/fixed-contribute-relay-pr-issue-link.md new file mode 100644 index 0000000000..6dd2f7931f --- /dev/null +++ b/changelog.d/fixed-contribute-relay-pr-issue-link.md @@ -0,0 +1 @@ +- A contributor relay PR reported for a dispatched issue now gets `Fixes #N` appended (when its body neither closes the issue nor says it stays open) and the GitHub sidebar link, so the PR-claim ledger and the issue list see the issue as covered instead of leaving it Actionable while the PR waits in review as "no linked issue" ([#14763](https://github.com/hivecommons/hive/issues/14763)). diff --git a/src/pkg/dashboard/contribute_ws.go b/src/pkg/dashboard/contribute_ws.go index c6804da8ff..b4127838b3 100644 --- a/src/pkg/dashboard/contribute_ws.go +++ b/src/pkg/dashboard/contribute_ws.go @@ -1656,6 +1656,56 @@ func (h *ContributeWSHub) reconcilePRAttribution(prURL string, contributor *Cont } } +// linkReportedPRToIssue appends `Fixes #N` to a verified contributor PR that +// does not yet close the issue it was dispatched for, and ensures the GitHub +// sidebar link (hivecommons/hive#14763). The relay's task prompt only asks the +// model for the trailer; hivecommons/hive#14701 shipped #14633 with an empty +// body, the PR-claim ledger (PR text and branch names only) never saw the +// issue as covered, and it sat at stage/010-actionable for hours while the PR +// waited in review as "no linked issue". +// +// Only GitHub-issue tasks in the task's own repo qualify: a fix-pr task or an +// external work item has no issue to close. The PR was verified moments ago as +// open, in this repo and authored by username, and LinkContributorPRToIssue +// re-checks the author before editing; it never edits a stranger's PR or a +// body that says the issue stays open. Best-effort and detached like +// reconcilePRAttribution: it gates nothing. +func (h *ContributeWSHub) linkReportedPRToIssue(task *WSTaskAssign, prURL, username string) { + if task == nil || task.Kind != "issue" || task.Number <= 0 || prURL == "" || username == "" { + return + } + if task.SourceType != "" && task.SourceType != "github" { + return + } + if h.server == nil || h.server.deps == nil || h.server.deps.GHClient == nil { + return + } + ref, err := ghpkg.ParsePRURL(prURL) + if err != nil || !strings.EqualFold(ref.FullName(), task.Repo) { + return + } + ctx := h.server.deps.Ctx + if ctx == nil { + ctx = context.Background() + } + ctx, cancel := context.WithTimeout(ctx, prAttributionReconcileTimeout) + defer cancel() + res, err := h.server.deps.GHClient.LinkContributorPRToIssue(ctx, task.Repo, ref.Number, task.Number, username) + if err != nil { + h.logger.Warn("[contribute-ws] linking reported PR to dispatched issue failed", + "repo", task.Repo, "issue", task.Number, "pr_url", prURL, "username", username, "error", err.Error()) + return + } + if res.EditedBody || res.SidebarLinked { + h.logger.Info("[contribute-ws] linked reported PR to dispatched issue", + "repo", task.Repo, "issue", task.Number, "pr_url", prURL, "username", username, + "edited_body", res.EditedBody, "sidebar_linked", res.SidebarLinked, "base", res.Base) + return + } + h.logger.Info("[contribute-ws] reported PR already linked to dispatched issue, or left alone", + "repo", task.Repo, "issue", task.Number, "pr_url", prURL, "username", username, "skipped", res.Skipped) +} + // abandonReason is the synthetic reason text written to the run log for a task // that ended with no terminal report. Free text like a client-reported reason, // but hub-authored, and phrased to say what the hub OBSERVED rather than to @@ -3918,6 +3968,13 @@ func (s *wsSession) handleTaskComplete(msg WSMessage) { // push a perfectly healthy contributor to `Stale` in the // fleet view for the sake of a PR-body edit. go h.reconcilePRAttribution(msg.PRURL, s.contributor) + // Same detachment, same reasoning: make the PR↔issue link + // durable so the claim ledger and the issue list see the + // dispatched issue as covered (hivecommons/hive#14763). + if completedTask != nil { + taskCopy := *completedTask + go h.linkReportedPRToIssue(&taskCopy, msg.PRURL, s.contributor.profile.GitHubUsername) + } // A MERGED fix retires the finding it addresses, so the // digest stops carrying work that is already done. Gated // on prDetail.Merged, not on verification alone: a PR diff --git a/src/pkg/github/pr_contributor_link.go b/src/pkg/github/pr_contributor_link.go new file mode 100644 index 0000000000..8e10bb1ce5 --- /dev/null +++ b/src/pkg/github/pr_contributor_link.go @@ -0,0 +1,124 @@ +package github + +import ( + "context" + "fmt" + "strings" + + gh "github.com/google/go-github/v72/github" +) + +// ContributorPRLinkResult reports how LinkContributorPRToIssue made a relay +// contributor's PR visible as the implementation of the issue it was +// dispatched for. +type ContributorPRLinkResult struct { + // EditedBody is true when the hub appended `Fixes #N` to the PR body. + EditedBody bool + // SidebarLinked is true when this call added the GitHub closing reference + // (Development sidebar) for the pair. + SidebarLinked bool + // Skipped names why the body was left alone: "pr-not-open", + // "already-closing" (a closing keyword for the issue is already there), + // "partial-reference" (the body says the issue stays open), + // "author-mismatch" (the PR is not the verified contributor's). Empty + // when the body was edited. + Skipped string + // Base is the PR's base branch as observed by the lookup. + Base string +} + +// LinkContributorPRToIssue makes the link between a relay-dispatched issue +// and the PR the contributor reported for it durable (hivecommons/hive#14763). +// +// The relay's task prompt only ASKS the contributor's model for `Closes #N`; +// hosted hives saw PRs (hivecommons/hive#14701 for #14633) arrive with an +// empty body, so the PR-claim ledger β€” which derives claims from PR text and +// branch names only β€” never saw the issue as covered. The issue stayed at +// stage/010-actionable for hours while its implementation sat in review as +// "no linked issue", and every lane was free to re-claim it. +// +// The caller has already verified the PR against GitHub (open, same base +// repo, authored by the contributor it dispatched to): this method only +// edits a PR whose author is that contributor login, never a stranger's. +// It appends `Fixes #N` when the body carries no closing keyword for the +// issue and does not say the issue stays open (`Refs`/`Part of`/leave-open +// wording β€” those already reach the ledger as reference claims). Whatever +// happened to the body, it then ensures the GitHub sidebar link so a +// non-default-base PR is visible in the issue list too. Best-effort: a +// failure must never affect the completion outcome. +func (c *Client) LinkContributorPRToIssue(ctx context.Context, repo string, prNumber, issueNumber int, contributorLogin string) (ContributorPRLinkResult, error) { + if c == nil || c.client == nil { + return ContributorPRLinkResult{}, ErrNoGitHubClient + } + owner, name := c.splitRepo(repo) + if owner == "" || name == "" || prNumber <= 0 || issueNumber <= 0 { + return ContributorPRLinkResult{}, fmt.Errorf("LinkContributorPRToIssue: repo, pr and issue are required") + } + contributorLogin = strings.TrimSpace(contributorLogin) + if contributorLogin == "" { + return ContributorPRLinkResult{Skipped: "author-mismatch"}, nil + } + + var out struct { + Repository struct { + PullRequest struct { + State string `json:"state"` + BaseRefName string `json:"baseRefName"` + Body string `json:"body"` + Author struct { + Login string `json:"login"` + } `json:"author"` + } `json:"pullRequest"` + } `json:"repository"` + } + query := `query($owner:String!, $name:String!, $pr:Int!) { + repository(owner:$owner, name:$name) { + pullRequest(number:$pr) { + state + baseRefName + body + author { login } + } + } +}` + vars := map[string]any{"owner": owner, "name": name, "pr": prNumber} + if err := c.graphQL(WithRESTCaller(ctx, "hive:contributor_pr_link"), query, vars, &out); err != nil { + return ContributorPRLinkResult{}, fmt.Errorf("LinkContributorPRToIssue %s#%d->#%d lookup: %w", repo, issueNumber, prNumber, err) + } + pr := out.Repository.PullRequest + res := ContributorPRLinkResult{Base: strings.TrimSpace(pr.BaseRefName)} + fullRepo := owner + "/" + name + switch { + case !strings.EqualFold(pr.State, "OPEN"): + res.Skipped = "pr-not-open" + return res, nil + case !strings.EqualFold(strings.TrimSpace(pr.Author.Login), contributorLogin): + res.Skipped = "author-mismatch" + return res, nil + } + if _, ok := FindClosingKeywordReference(pr.Body, fullRepo, fullRepo, issueNumber); ok { + res.Skipped = "already-closing" + } else if bodySaysIssueStaysOpen(pr.Body, fullRepo, issueNumber) { + res.Skipped = "partial-reference" + } else { + body := strings.TrimRight(pr.Body, " \t\r\n") + if body != "" { + body += "\n\n" + } + body += fmt.Sprintf("Fixes #%d", issueNumber) + if _, _, err := c.client.PullRequests.Edit(WithRESTCaller(ctx, "hive:contributor_pr_link"), owner, name, prNumber, &gh.PullRequest{Body: gh.Ptr(body)}); err != nil { + return res, fmt.Errorf("LinkContributorPRToIssue %s#%d->#%d edit body: %w", repo, issueNumber, prNumber, err) + } + res.EditedBody = true + } + if res.Skipped == "partial-reference" { + // A deliberately partial PR must not become the issue's closer. + return res, nil + } + link, err := c.EnsureClosingIssueLink(ctx, fullRepo, prNumber, issueNumber) + if err != nil { + return res, err + } + res.SidebarLinked = link.Linked + return res, nil +} diff --git a/src/pkg/github/pr_contributor_link_test.go b/src/pkg/github/pr_contributor_link_test.go new file mode 100644 index 0000000000..5034109bd3 --- /dev/null +++ b/src/pkg/github/pr_contributor_link_test.go @@ -0,0 +1,187 @@ +package github + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" +) + +// contributorPRLinkFixture answers the PR lookup LinkContributorPRToIssue +// issues, the body PATCH it may send, and the EnsureClosingIssueLink lookup + +// addCloseIssueReferences mutation that follow. +type contributorPRLinkFixture struct { + body string + base string + author string + state string + patches atomic.Int32 + patchedBody string + mutations atomic.Int32 +} + +func (f *contributorPRLinkFixture) handler(t *testing.T) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.Method == http.MethodPatch && r.URL.Path == "/repos/hivecommons/hive/pulls/1" { + var req struct { + Body string `json:"body"` + } + _ = json.NewDecoder(r.Body).Decode(&req) + f.patches.Add(1) + f.patchedBody = req.Body + f.body = req.Body + _ = json.NewEncoder(w).Encode(map[string]any{"number": 1, "body": req.Body}) + return + } + if r.Method != http.MethodPost || !strings.HasSuffix(r.URL.Path, "/graphql") { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + raw, _ := io.ReadAll(r.Body) + var req struct { + Query string `json:"query"` + } + _ = json.Unmarshal(raw, &req) + state := f.state + if state == "" { + state = "OPEN" + } + switch { + case strings.Contains(req.Query, "addCloseIssueReferences"): + f.mutations.Add(1) + _ = json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{ + "addCloseIssueReferences": map[string]any{"issue": map[string]any{"number": 7}}, + }}) + case strings.Contains(req.Query, "closingIssuesReferences"): + _ = json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{ + "repository": map[string]any{ + "pullRequest": map[string]any{ + "id": "PR_1", + "state": state, + "baseRefName": f.base, + "closingIssuesReferences": map[string]any{"nodes": []map[string]any{}}, + }, + "issue": map[string]any{"id": "I_7", "state": "OPEN"}, + }, + }}) + default: + _ = json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{ + "repository": map[string]any{ + "pullRequest": map[string]any{ + "state": state, + "baseRefName": f.base, + "body": f.body, + "author": map[string]any{"login": f.author}, + }, + }, + }}) + } + } +} + +func newContributorPRLinkClient(t *testing.T, f *contributorPRLinkFixture) *Client { + t.Helper() + srv := httptest.NewServer(f.handler(t)) + t.Cleanup(srv.Close) + return NewClientForTest(srv.URL, "hivecommons", []string{"hive"}, testLogger()) +} + +// hivecommons/hive#14763: a contributor PR reported for a dispatched issue +// with no closing keyword (hivecommons/hive#14701 arrived with an empty body) +// gets `Fixes #N` appended and the sidebar link, even on a release-line base. +func TestLinkContributorPRToIssueAppendsFixesAndSidebarLink(t *testing.T) { + f := &contributorPRLinkFixture{body: "", base: "v6", author: "karibhou"} + c := newContributorPRLinkClient(t, f) + got, err := c.LinkContributorPRToIssue(context.Background(), "hivecommons/hive", 1, 7, "Karibhou") + if err != nil { + t.Fatalf("LinkContributorPRToIssue: %v", err) + } + if !got.EditedBody || got.Skipped != "" || f.patches.Load() != 1 { + t.Fatalf("result=%+v patches=%d, want one body edit", got, f.patches.Load()) + } + if f.patchedBody != "Fixes #7" { + t.Fatalf("patched body = %q, want bare Fixes trailer on an empty body", f.patchedBody) + } + if !got.SidebarLinked || f.mutations.Load() != 1 { + t.Fatalf("result=%+v mutations=%d, want the sidebar link too", got, f.mutations.Load()) + } + if got.Base != "v6" { + t.Fatalf("base = %q, want v6", got.Base) + } +} + +func TestLinkContributorPRToIssueAppendsAfterExistingBody(t *testing.T) { + f := &contributorPRLinkFixture{body: "## Summary\n\nAdds the PKCE cookie.\n\n", base: "v5", author: "karibhou"} + c := newContributorPRLinkClient(t, f) + got, err := c.LinkContributorPRToIssue(context.Background(), "hivecommons/hive", 1, 7, "karibhou") + if err != nil { + t.Fatalf("LinkContributorPRToIssue: %v", err) + } + if !got.EditedBody || f.patchedBody != "## Summary\n\nAdds the PKCE cookie.\n\nFixes #7" { + t.Fatalf("result=%+v patched body = %q", got, f.patchedBody) + } +} + +func TestLinkContributorPRToIssueLeavesClosingAndPartialBodiesAlone(t *testing.T) { + for _, tc := range []struct { + name string + body string + wantSkipped string + wantSidebar bool + }{ + {"already-closing", "Closes #7", "already-closing", true}, + {"already-closing-full-ref", "Fixes hivecommons/hive#7", "already-closing", true}, + {"part-of", "Part of #7 β€” first slice only.", "partial-reference", false}, + {"refs", "Refs #7", "partial-reference", false}, + {"stays-open", "Groundwork; leave #7 open.", "partial-reference", false}, + } { + t.Run(tc.name, func(t *testing.T) { + f := &contributorPRLinkFixture{body: tc.body, base: "v6", author: "karibhou"} + c := newContributorPRLinkClient(t, f) + got, err := c.LinkContributorPRToIssue(context.Background(), "hivecommons/hive", 1, 7, "karibhou") + if err != nil { + t.Fatalf("LinkContributorPRToIssue: %v", err) + } + if got.EditedBody || f.patches.Load() != 0 || got.Skipped != tc.wantSkipped { + t.Fatalf("result=%+v patches=%d, want skipped=%q and no edit", got, f.patches.Load(), tc.wantSkipped) + } + if got.SidebarLinked != tc.wantSidebar || (f.mutations.Load() == 1) != tc.wantSidebar { + t.Fatalf("result=%+v mutations=%d, want sidebar=%v", got, f.mutations.Load(), tc.wantSidebar) + } + }) + } +} + +func TestLinkContributorPRToIssueNeverEditsAnotherAuthorsPR(t *testing.T) { + f := &contributorPRLinkFixture{body: "", base: "v6", author: "someone-else"} + c := newContributorPRLinkClient(t, f) + got, err := c.LinkContributorPRToIssue(context.Background(), "hivecommons/hive", 1, 7, "karibhou") + if err != nil { + t.Fatalf("LinkContributorPRToIssue: %v", err) + } + if got.EditedBody || got.Skipped != "author-mismatch" || f.patches.Load() != 0 || f.mutations.Load() != 0 { + t.Fatalf("result=%+v patches=%d mutations=%d, want untouched", got, f.patches.Load(), f.mutations.Load()) + } + if _, err := c.LinkContributorPRToIssue(context.Background(), "hivecommons/hive", 1, 7, ""); err != nil { + t.Fatalf("empty login: %v", err) + } + if f.patches.Load() != 0 { + t.Fatalf("empty login must never edit") + } +} + +func TestLinkContributorPRToIssueSkipsClosedPR(t *testing.T) { + f := &contributorPRLinkFixture{body: "", base: "v6", author: "karibhou", state: "MERGED"} + c := newContributorPRLinkClient(t, f) + got, err := c.LinkContributorPRToIssue(context.Background(), "hivecommons/hive", 1, 7, "karibhou") + if err != nil { + t.Fatalf("LinkContributorPRToIssue: %v", err) + } + if got.EditedBody || got.Skipped != "pr-not-open" || f.patches.Load() != 0 || f.mutations.Load() != 0 { + t.Fatalf("result=%+v, want closed PR left alone", got) + } +} From f30028e7f5b5b6472c0e73253293572b0eba3910 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 27 Oct 2026 21:35:47 -0400 Subject: [PATCH 26/38] =?UTF-8?q?=F0=9F=8C=B1=20images:=20bump=20codex=200?= =?UTF-8?q?.164.0=20->=200.165.0=20(#14769)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated pin bump (#8419). Digests recomputed from the downloaded release artifacts by src/scripts/cli-pin-bump.sh: VERSION=0.165.0 Signed-off-by: hive-release[bot] Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: hive-release[bot] --- src/Dockerfile | 2 +- src/Dockerfile.contributor | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Dockerfile b/src/Dockerfile index 26682629d7..3ccbf67a69 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -314,7 +314,7 @@ RUN npm install -g --ignore-scripts --no-fund --no-audit \ # dashboard's codex model discovery (`codex app-server` model/list β€” pure # local stdio, no network, unaffected by the proxy CA) falls back to the # static list, and non-codex backends are unaffected. -ARG CODEX_VERSION=0.164.0 +ARG CODEX_VERSION=0.165.0 RUN npm install -g --ignore-scripts --no-fund --no-audit @openai/codex@${CODEX_VERSION} && npm cache clean --force || \ echo "WARN: OpenAI Codex CLI install failed β€” skipping" diff --git a/src/Dockerfile.contributor b/src/Dockerfile.contributor index ff3b9c960e..f3066ae536 100644 --- a/src/Dockerfile.contributor +++ b/src/Dockerfile.contributor @@ -260,7 +260,7 @@ ARG KILO_CLI_VERSION=7.5.6 # Same ARG name and value as src/Dockerfile so bin/test_backend_smoke.sh can # assert the two pins agree β€” this line was previously an inline hardcode that # could (and did) drift from the main image's CODEX_VERSION unnoticed. -ARG CODEX_VERSION=0.164.0 +ARG CODEX_VERSION=0.165.0 RUN npm install -g --ignore-scripts --no-fund --no-audit \ @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION} \ @github/copilot@${COPILOT_VERSION} \ From ebaf1c02212ac1e2034826adcb8f22dabd4f29e9 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 27 Oct 2026 21:36:08 -0400 Subject: [PATCH 27/38] =?UTF-8?q?=F0=9F=8C=B1=20runtime:=20loopback=20ppro?= =?UTF-8?q?f,=20periodic=20memstats=20and=20cgroup-aware=20GOMEMLIMIT=20(#?= =?UTF-8?q?14682)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🌱 runtime: loopback pprof, periodic memstats and cgroup-aware GOMEMLIMIT Adds pkg/runtimeobs: a loopback-only net/http/pprof listener (debug.pprof_addr, default 127.0.0.1:6060, 'off' disables), a 60 s 'mem: runtime snapshot' INFO line (heap in-use/live/objects, next GC, goroutines, OS threads, GOMEMLIMIT) read via runtime/metrics, the same snapshot under 'runtime' in GET /api/health, and a GOMEMLIMIT default of 85% of the cgroup memory limit when the env var is unset. Refs #14681 Signed-off-by: Andy Anderson * runtime: satisfy CI ratchets for pprof/memstats - health body test decodes the new runtime object - settings parity allowlist: debug.pprof_addr is an operator knob - dashboard import ratchet: allow pkg/runtimeobs - agent runtime golden: Config gained the debug block - runtimeobs logger test waits on the record instead of sleeping - api-reference citations re-pinned after the server.go import shift Signed-off-by: Andy Anderson * changelog: fragment for #14682 Generated by changelog-fragment-guard from the PR title (#5675). Edit the wording freely, or replace this file with the no-changelog label if the change is not user-visible. Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> * runtimeobs: cover the default logger, disabled/occupied pprof and no-/proc paths Signed-off-by: Andy Anderson * changelog: single bulleted fragment for the runtime diagnostics Signed-off-by: Andy Anderson * docs: re-pin api-reference citations after the server.go import shift Signed-off-by: Andy Anderson --------- Signed-off-by: Andy Anderson Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- changelog.d/added-runtime-pprof-memstats.md | 1 + src/cmd/hive/main.go | 5 + src/cmd/hive/runtime_observability.go | 40 +++ src/docs/api-reference.md | 32 +- src/docs/operator-reference.md | 21 ++ .../testutil/dashboard_import_allowlist.txt | 1 + src/pkg/config/config.go | 36 +++ .../config/testdata/agent_runtime_golden.json | 1 + src/pkg/dashboard/server.go | 3 +- src/pkg/dashboard/server_test.go | 9 +- .../dashboard/settings_parity_allowlist.txt | 1 + src/pkg/runtimeobs/runtimeobs.go | 284 ++++++++++++++++++ src/pkg/runtimeobs/runtimeobs_test.go | 251 ++++++++++++++++ 13 files changed, 667 insertions(+), 18 deletions(-) create mode 100644 changelog.d/added-runtime-pprof-memstats.md create mode 100644 src/cmd/hive/runtime_observability.go create mode 100644 src/pkg/runtimeobs/runtimeobs.go create mode 100644 src/pkg/runtimeobs/runtimeobs_test.go diff --git a/changelog.d/added-runtime-pprof-memstats.md b/changelog.d/added-runtime-pprof-memstats.md new file mode 100644 index 0000000000..f31d552a10 --- /dev/null +++ b/changelog.d/added-runtime-pprof-memstats.md @@ -0,0 +1 @@ +- Loopback-only `net/http/pprof` listener (`debug.pprof_addr`, default `127.0.0.1:6060`, `off` to disable), a `mem: runtime snapshot` INFO line every 60 s (heap in-use/live/objects, next GC, goroutines, OS threads, GOMEMLIMIT), the same counters under `runtime` in `GET /api/health`, and a cgroup-aware GOMEMLIMIT default (85% of the container memory limit when the env var is unset) ([#14681](https://github.com/hivecommons/hive/issues/14681)). diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 96fcfec70f..2c34a3bb90 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -1168,6 +1168,11 @@ func (b *boot) bootConfigWith(deps bootConfigDeps) bool { b.ctx, cancel = context.WithCancel(context.Background()) b.cleanup.push(func() { cancel() }) + // Runtime observability (#14681): GOMEMLIMIT from the cgroup limit, the + // loopback pprof listener and the 60 s `mem:` line. Before tracing so the + // first mem line lands even when an exporter stalls. + b.startRuntimeObservability() + // Initialize OpenTelemetry tracing. Off by default: with no otel block // (or otel.enabled=false) this installs a no-op provider with zero export // overhead. Never fatal β€” a tracing setup error must not stop hive. diff --git a/src/cmd/hive/runtime_observability.go b/src/cmd/hive/runtime_observability.go new file mode 100644 index 0000000000..8d71019b8a --- /dev/null +++ b/src/cmd/hive/runtime_observability.go @@ -0,0 +1,40 @@ +package main + +import ( + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/runtimeobs" +) + +// startRuntimeObservability wires pkg/runtimeobs into the boot: the soft +// memory limit, the loopback-only pprof listener (debug.pprof_addr) and the +// periodic `mem:` log line (hivecommons/hive#14681). Nothing here is fatal β€” +// a bind failure only costs the profiles, never the hive. +func (b *boot) startRuntimeObservability() { + out := runtimeobs.ApplyMemoryLimit(nil, nil) + b.logger.Info("runtime: memory limit", + "source", out.Source, + "gomemlimit", out.Limit, + "cgroup_limit", out.CgroupLimit, + "percent", runtimeobs.MemoryLimitPercent, + ) + + addr := b.cfg.Debug.EffectivePprofAddr() + if addr != "" && !runtimeobs.IsLoopbackAddr(addr) { + b.logger.Warn("runtime: debug.pprof_addr is not a loopback address; using the default", + "configured", addr, "default", config.DefaultPprofAddr) + addr = config.DefaultPprofAddr + } + bound, stop, err := runtimeobs.StartPprof(addr, b.logger) + switch { + case err != nil: + b.logger.Warn("runtime: pprof listener not started", "addr", addr, "error", err) + case bound == "": + b.logger.Info("runtime: pprof listener disabled", "key", "debug.pprof_addr") + default: + b.cleanup.push(stop) + b.logger.Info("runtime: pprof listening (loopback only)", "addr", bound, "path", "/debug/pprof/") + } + + runtimeobs.StartLogger(b.ctx, b.logger, nil) + b.logger.Info("mem: runtime snapshot", runtimeobs.Take().LogAttrs()...) +} diff --git a/src/docs/api-reference.md b/src/docs/api-reference.md index 08c74f9c68..ce798b8162 100644 --- a/src/docs/api-reference.md +++ b/src/docs/api-reference.md @@ -13,17 +13,17 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` β€” the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` β€” one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` β€” `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` β€” the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)) | `pkg/dashboard/api.go:42` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=&to=` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` β€” the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:43` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:64` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1505` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1506` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1507` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1513` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` β€” the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced}` is the governor strip's `running N Β· $X this hour` figure; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` β€” issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures β€” the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376) | `pkg/dashboard/server.go:1518` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1519` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1520` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1521` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1522` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1523` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1524` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1506` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1507` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1508` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1514` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` β€” the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced}` is the governor strip's `running N Β· $X this hour` figure; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` β€” issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures β€” the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376) | `pkg/dashboard/server.go:1519` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1520` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1521` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1522` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1523` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1524` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1525` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | @@ -66,7 +66,7 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:44` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](branding.md)) | `pkg/dashboard/server.go:1585` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](branding.md)) | `pkg/dashboard/server.go:1586` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:77` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:78` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:79` | @@ -94,9 +94,9 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1525` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1526` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1527` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1526` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1527` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1528` | ## Configuration @@ -593,7 +593,7 @@ always resolved server-side from the validated token. | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:275` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:276` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:277` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1532` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1533` | ## Hub SaaS diff --git a/src/docs/operator-reference.md b/src/docs/operator-reference.md index f5da073492..7e5753bcd1 100644 --- a/src/docs/operator-reference.md +++ b/src/docs/operator-reference.md @@ -1466,6 +1466,27 @@ means the App/key/installation are healthy, but the hive still names the old repository owner after an org transfer; update the configured project org rather than trying to add the moved repository back to the old installation. +## Runtime diagnostics (`debug`, `GOMEMLIMIT`, `mem:` log line) + +Added for [#14681](https://github.com/hivecommons/hive/issues/14681) (hive +process RSS growing hundreds of MiB/min with no pprof to name the culprit). + +| Name | Source | Default | Purpose | +|---|---|---|---| +| `debug.pprof_addr` | `hive.yaml` | `127.0.0.1:6060` | `net/http/pprof` listener. **Loopback only**: a non-loopback host (`0.0.0.0`, pod IP) is rejected at startup with a warning and the default is used instead. `off` disables the listener. Profiles are under `/debug/pprof/`; `/debug/runtime` prints the same counters as the `mem:` log line. | +| `GOMEMLIMIT` | env | 85% of the cgroup memory limit | Go soft memory limit. When the env var is unset the hive reads `/sys/fs/cgroup/memory.max` (v2) or `memory/memory.limit_in_bytes` (v1) and sets `MemoryLimitPercent` (85%) of it via `debug.SetMemoryLimit`; an unlimited cgroup leaves the runtime default. Set the env var to override. Logged at startup as `runtime: memory limit` with `source=env|cgroup|none`. | +| `mem: runtime snapshot` | log (INFO, every 60 s) | always on | `heap_inuse`, `heap_live`, `heap_objects`, `total_sys`, `next_gc`, `gc_cycles`, `goroutines`, `threads` (OS threads from `/proc/self/status`), `gomemlimit`. Read from `runtime/metrics`, so no stop-the-world. | + +The same snapshot is embedded in `GET /api/health` as `runtime` so the +dashboard health probe and `kubectl exec … curl` see goroutine count and heap +without the pprof port. Taking profiles on a pod: + +```bash +kubectl -n port-forward deploy/hive 6060:6060 & +go tool pprof -top http://127.0.0.1:6060/debug/pprof/heap # inuse_space +curl -s 'http://127.0.0.1:6060/debug/pprof/goroutine?debug=1' | head # parked goroutines +``` + ## `cmd/hive` flags and environment ### Startup diff --git a/src/internal/testutil/dashboard_import_allowlist.txt b/src/internal/testutil/dashboard_import_allowlist.txt index bd257e6647..ac334ce4b8 100644 --- a/src/internal/testutil/dashboard_import_allowlist.txt +++ b/src/internal/testutil/dashboard_import_allowlist.txt @@ -54,6 +54,7 @@ github.com/hivecommons/hive/pkg/resolve github.com/hivecommons/hive/pkg/retro github.com/hivecommons/hive/pkg/review github.com/hivecommons/hive/pkg/rotation +github.com/hivecommons/hive/pkg/runtimeobs github.com/hivecommons/hive/pkg/scheduler github.com/hivecommons/hive/pkg/spokebackup github.com/hivecommons/hive/pkg/stageengine diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index 050b06a3fe..be4dfde5c1 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -98,6 +98,9 @@ type Config struct { // operator-facing block; Tracing is retained as a legacy alias. OTel OTelConfig `yaml:"otel,omitempty" json:"otel,omitempty"` Tracing OTelConfig `yaml:"tracing,omitempty"` + // Debug holds process-level diagnostics knobs (pprof listener). See + // docs/operator-reference.md "Runtime diagnostics". + Debug DebugConfig `yaml:"debug,omitempty" json:"debug,omitempty"` // Triggers is an additive list of CEL-based declarative agent triggers. // Default empty β†’ existing label/governor triggering is unchanged. Triggers []TriggerRule `yaml:"triggers,omitempty" json:"triggers,omitempty"` @@ -350,6 +353,39 @@ func (o OTelConfig) ServiceNameOrDefault() string { return DefaultOTelServiceName } +const ( + // DefaultPprofAddr is where the net/http/pprof listener binds when + // debug.pprof_addr is unset. Loopback only: the profiles leak source paths + // and heap contents, so they are reached via `kubectl port-forward` or an + // in-container curl, never over the pod network. + DefaultPprofAddr = "127.0.0.1:6060" + // PprofAddrOff is the debug.pprof_addr sentinel that disables the listener. + PprofAddrOff = "off" +) + +// DebugConfig is the optional `debug:` block for process-level diagnostics +// (hivecommons/hive#14681). +type DebugConfig struct { + // PprofAddr is the host:port the net/http/pprof listener binds to. Empty + // means DefaultPprofAddr; "off" disables it. The host must be a loopback + // address β€” any other host is rejected at startup and falls back to the + // default with a logged warning, so a typo can never expose profiles. + PprofAddr string `yaml:"pprof_addr,omitempty" json:"pprof_addr,omitempty"` +} + +// EffectivePprofAddr resolves PprofAddr to the address to bind, or "" when +// the listener is disabled. +func (d DebugConfig) EffectivePprofAddr() string { + addr := strings.TrimSpace(d.PprofAddr) + switch { + case addr == "": + return DefaultPprofAddr + case strings.EqualFold(addr, PprofAddrOff): + return "" + } + return addr +} + // IsZero reports whether the block contains no operator-provided settings. func (o OTelConfig) IsZero() bool { return !o.Enabled && o.Endpoint == "" && len(o.Headers) == 0 && o.ServiceName == "" && !o.Insecure && o.SampleRatio == 0 diff --git a/src/pkg/config/testdata/agent_runtime_golden.json b/src/pkg/config/testdata/agent_runtime_golden.json index 09dd9530d7..3d31e6a2aa 100644 --- a/src/pkg/config/testdata/agent_runtime_golden.json +++ b/src/pkg/config/testdata/agent_runtime_golden.json @@ -615,6 +615,7 @@ }, "otel": {}, "Tracing": {}, + "debug": {}, "tool_approval": { "enabled": true, "rules": [ diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index d25d2b77e6..5d26805530 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -34,6 +34,7 @@ import ( spoke "github.com/hivecommons/hive/pkg/hub/spoke" "github.com/hivecommons/hive/pkg/planning" "github.com/hivecommons/hive/pkg/review/autoscale" + "github.com/hivecommons/hive/pkg/runtimeobs" "github.com/hivecommons/hive/pkg/stageengine" "github.com/hivecommons/hive/pkg/stagelabels" "github.com/hivecommons/hive/pkg/timeline" @@ -3085,7 +3086,7 @@ func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { jsonResponse(w, map[string]string{"status": "starting"}) return } - body := map[string]any{"status": "ok"} + body := map[string]any{"status": "ok", "runtime": runtimeobs.Take()} if s.deps != nil && s.deps.CruxRuntimeHealth != nil { rt := s.deps.CruxRuntimeHealth() body["crux_runtime"] = rt diff --git a/src/pkg/dashboard/server_test.go b/src/pkg/dashboard/server_test.go index afd2ca0d11..522f7bfa06 100644 --- a/src/pkg/dashboard/server_test.go +++ b/src/pkg/dashboard/server_test.go @@ -117,13 +117,20 @@ func TestHandleHealth_Body(t *testing.T) { rec := httptest.NewRecorder() s.handleHealth(rec, httptest.NewRequest(http.MethodGet, "/api/health", nil)) - var payload map[string]string + var payload map[string]any if err := json.NewDecoder(rec.Body).Decode(&payload); err != nil { t.Fatalf("decode error: %v", err) } if payload["status"] != "ok" { t.Errorf(`expected {"status":"ok"}, got %v`, payload) } + rt, ok := payload["runtime"].(map[string]any) + if !ok { + t.Fatalf("expected runtime object in health body, got %v", payload["runtime"]) + } + if g, _ := rt["goroutines"].(float64); g < 1 { + t.Errorf("runtime.goroutines = %v, want >= 1", rt["goroutines"]) + } } func TestHandleStatus_NilStatus_StatusCode(t *testing.T) { diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index 06bd0ed9b4..be7e8cce68 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -160,6 +160,7 @@ compliance.frameworks[] # TODO #11854 contribute.token_idle_max # TODO #11854 contribute.token_max_age # TODO #11854 convergence.mutation.max_writers_per_target # boundary capacity tuning; yaml-only (#14645) +debug.pprof_addr # loopback-only pprof bind address, operator/debug knob, not a Settings control (#14681) deployment.runtime # set via HIVE_DEPLOYMENT_RUNTIME env var or auto-detected, not a Settings control (#14072) duplicate_sweep.bot_authors[] # TODO #11854 evidence.complexity.enabled # TODO #11854 diff --git a/src/pkg/runtimeobs/runtimeobs.go b/src/pkg/runtimeobs/runtimeobs.go new file mode 100644 index 0000000000..a9e62ff927 --- /dev/null +++ b/src/pkg/runtimeobs/runtimeobs.go @@ -0,0 +1,284 @@ +// Package runtimeobs gives the hive process the three things it lacked when +// hivecommons/hive#14681 (RSS +360 MiB/min, OOM at 24Gi) had to be diagnosed +// from /proc alone: a loopback net/http/pprof listener, a periodic memory / +// goroutine / thread log line, and a cgroup-aware GOMEMLIMIT default so the +// GC pushes back before the kernel does. +package runtimeobs + +import ( + "bytes" + "context" + "errors" + "fmt" + "log/slog" + "net" + "net/http" + "net/http/pprof" + "os" + "runtime" + "runtime/debug" + "runtime/metrics" + "strconv" + "strings" + "time" +) + +const ( + // LogInterval is how often StartLogger emits the `mem:` line. + LogInterval = 60 * time.Second + + // MemoryLimitPercent is the share of the cgroup memory limit handed to + // the Go runtime as GOMEMLIMIT when the env var is unset. The remaining + // 15% covers the non-Go RSS in the same cgroup (agent CLIs run in the + // same container) and runtime overhead outside the heap. + MemoryLimitPercent = 85 + + // cgroupV2MemoryMax and cgroupV1MemoryLimit are the files the container + // memory limit is read from, in order of preference. "max" (v2) or the + // kernel's "no limit" sentinel (v1) leave GOMEMLIMIT untouched. + cgroupV2MemoryMax = "/sys/fs/cgroup/memory.max" + cgroupV1MemoryLimit = "/sys/fs/cgroup/memory/memory.limit_in_bytes" + + // cgroupV1NoLimitFloor: v1 reports an unlimited cgroup as a page-rounded + // value just under 2^63; anything at or above this is "no limit". + cgroupV1NoLimitFloor = 1 << 62 + + // memoryLimitEnv is the Go runtime's own variable; when set the runtime + // has already applied it and this package only reports it. + memoryLimitEnv = "GOMEMLIMIT" + + // pprofReadHeaderTimeout bounds a slow client on the debug listener; the + // handlers themselves stream for as long as the profile asks (seconds=). + pprofReadHeaderTimeout = 10 * time.Second +) + +// procSelfStatus is where the OS thread count comes from on Linux; a +// variable so tests can point it at a missing file. +var procSelfStatus = "/proc/self/status" + +// Snapshot is one reading of the process-level runtime counters. Every field +// comes from runtime/metrics (no stop-the-world) except Threads, which is +// read from /proc on Linux and is 0 where that is unavailable. +type Snapshot struct { + HeapInuse uint64 `json:"heap_inuse_bytes"` + HeapLive uint64 `json:"heap_live_bytes"` + HeapObjects uint64 `json:"heap_objects"` + NextGC uint64 `json:"next_gc_bytes"` + TotalSys uint64 `json:"total_sys_bytes"` + NumGC uint64 `json:"gc_cycles"` + Goroutines int `json:"goroutines"` + Threads int `json:"threads"` + MemLimit int64 `json:"gomemlimit_bytes"` +} + +var sampleNames = []string{ + "/memory/classes/heap/objects:bytes", + "/gc/heap/live:bytes", + "/gc/heap/objects:objects", + "/gc/heap/goal:bytes", + "/memory/classes/total:bytes", + "/gc/cycles/total:gc-cycles", +} + +// Take reads a Snapshot. +func Take() Snapshot { + samples := make([]metrics.Sample, len(sampleNames)) + for i, n := range sampleNames { + samples[i].Name = n + } + metrics.Read(samples) + u := func(i int) uint64 { + if samples[i].Value.Kind() == metrics.KindUint64 { + return samples[i].Value.Uint64() + } + return 0 + } + return Snapshot{ + HeapInuse: u(0), + HeapLive: u(1), + HeapObjects: u(2), + NextGC: u(3), + TotalSys: u(4), + NumGC: u(5), + Goroutines: runtime.NumGoroutine(), + Threads: threadCount(), + MemLimit: debug.SetMemoryLimit(-1), + } +} + +// LogAttrs renders the snapshot as slog attributes. +func (s Snapshot) LogAttrs() []any { + return []any{ + "heap_inuse", s.HeapInuse, + "heap_live", s.HeapLive, + "heap_objects", s.HeapObjects, + "total_sys", s.TotalSys, + "next_gc", s.NextGC, + "gc_cycles", s.NumGC, + "goroutines", s.Goroutines, + "threads", s.Threads, + "gomemlimit", s.MemLimit, + } +} + +// threadCount parses Threads: from /proc/self/status; 0 when unavailable. +func threadCount() int { + data, err := os.ReadFile(procSelfStatus) + if err != nil { + return 0 + } + return ParseThreads(data) +} + +// ParseThreads extracts the Threads: field from a /proc//status body. +func ParseThreads(status []byte) int { + for _, line := range bytes.Split(status, []byte("\n")) { + if !bytes.HasPrefix(line, []byte("Threads:")) { + continue + } + n, err := strconv.Atoi(strings.TrimSpace(string(line[len("Threads:"):]))) + if err != nil { + return 0 + } + return n + } + return 0 +} + +// StartLogger emits one `mem:` INFO line per tick until ctx is done. tick +// is injectable so tests drive it without sleeping; nil means a real ticker +// at LogInterval. +func StartLogger(ctx context.Context, logger *slog.Logger, tick <-chan time.Time) { + if logger == nil { + logger = slog.Default() + } + var stop func() + if tick == nil { + t := time.NewTicker(LogInterval) + tick, stop = t.C, t.Stop + } + go func() { + if stop != nil { + defer stop() + } + for { + select { + case <-ctx.Done(): + return + case <-tick: + logger.Info("mem: runtime snapshot", Take().LogAttrs()...) + } + } + }() +} + +// ErrNotLoopback is returned by StartPprof for a non-loopback bind host. +var ErrNotLoopback = errors.New("pprof listener address must be loopback (127.0.0.0/8, ::1, or localhost)") + +// IsLoopbackAddr reports whether host:port names a loopback host. +func IsLoopbackAddr(addr string) bool { + host, _, err := net.SplitHostPort(addr) + if err != nil { + return false + } + if strings.EqualFold(host, "localhost") { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + +// StartPprof serves net/http/pprof on addr, which must be a loopback address. +// It returns the bound address and a stop func. An empty addr is a no-op. +func StartPprof(addr string, logger *slog.Logger) (bound string, stop func(), err error) { + if addr == "" { + return "", func() {}, nil + } + if !IsLoopbackAddr(addr) { + return "", nil, fmt.Errorf("%w: %q", ErrNotLoopback, addr) + } + ln, err := net.Listen("tcp", addr) + if err != nil { + return "", nil, err + } + mux := http.NewServeMux() + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + mux.HandleFunc("/debug/runtime", func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + attrs := Take().LogAttrs() + for i := 0; i+1 < len(attrs); i += 2 { + fmt.Fprintf(w, "%v=%v\n", attrs[i], attrs[i+1]) + } + }) + srv := &http.Server{Handler: mux, ReadHeaderTimeout: pprofReadHeaderTimeout} + go func() { + if serveErr := srv.Serve(ln); serveErr != nil && !errors.Is(serveErr, http.ErrServerClosed) && logger != nil { + logger.Warn("pprof listener stopped", "addr", addr, "error", serveErr) + } + }() + return ln.Addr().String(), func() { _ = srv.Close() }, nil +} + +// MemoryLimitOutcome says what ApplyMemoryLimit did, for the startup log. +type MemoryLimitOutcome struct { + // Source is "env" (GOMEMLIMIT already set), "cgroup" (derived), or + // "none" (no limit found; runtime default of math.MaxInt64 kept). + Source string + // CgroupLimit is the raw cgroup limit in bytes when one was found. + CgroupLimit int64 + // Limit is the effective GOMEMLIMIT in bytes. + Limit int64 +} + +// ApplyMemoryLimit sets the Go soft memory limit to MemoryLimitPercent of +// the cgroup memory limit unless GOMEMLIMIT is already set (the runtime has +// then applied it) or no finite cgroup limit exists. getenv and readFile are +// injectable for tests; nil means os.Getenv / os.ReadFile. +func ApplyMemoryLimit(getenv func(string) string, readFile func(string) ([]byte, error)) MemoryLimitOutcome { + if getenv == nil { + getenv = os.Getenv + } + if readFile == nil { + readFile = os.ReadFile + } + if strings.TrimSpace(getenv(memoryLimitEnv)) != "" { + return MemoryLimitOutcome{Source: "env", Limit: debug.SetMemoryLimit(-1)} + } + cg, ok := CgroupMemoryLimit(readFile) + if !ok { + return MemoryLimitOutcome{Source: "none", Limit: debug.SetMemoryLimit(-1)} + } + limit := DeriveMemoryLimit(cg) + debug.SetMemoryLimit(limit) + return MemoryLimitOutcome{Source: "cgroup", CgroupLimit: cg, Limit: limit} +} + +// DeriveMemoryLimit is MemoryLimitPercent of the cgroup limit. +func DeriveMemoryLimit(cgroupLimit int64) int64 { + return cgroupLimit / 100 * MemoryLimitPercent +} + +// CgroupMemoryLimit reads the container memory limit from cgroup v2, then +// v1. ok is false when neither file exists or the cgroup is unlimited. +func CgroupMemoryLimit(readFile func(string) ([]byte, error)) (limit int64, ok bool) { + for _, path := range []string{cgroupV2MemoryMax, cgroupV1MemoryLimit} { + data, err := readFile(path) + if err != nil { + continue + } + raw := strings.TrimSpace(string(data)) + if raw == "max" || raw == "" { + return 0, false + } + n, err := strconv.ParseInt(raw, 10, 64) + if err != nil || n <= 0 || n >= cgroupV1NoLimitFloor { + return 0, false + } + return n, true + } + return 0, false +} diff --git a/src/pkg/runtimeobs/runtimeobs_test.go b/src/pkg/runtimeobs/runtimeobs_test.go new file mode 100644 index 0000000000..c30e42ae1d --- /dev/null +++ b/src/pkg/runtimeobs/runtimeobs_test.go @@ -0,0 +1,251 @@ +package runtimeobs + +import ( + "bytes" + "context" + "errors" + "io" + "log/slog" + "net" + "net/http" + "path/filepath" + "runtime" + "runtime/debug" + "strings" + "testing" + "time" +) + +func TestParseThreads(t *testing.T) { + status := []byte("Name:\thive\nState:\tS (sleeping)\nThreads:\t156\nSigQ:\t0/1\n") + if got := ParseThreads(status); got != 156 { + t.Fatalf("ParseThreads = %d, want 156", got) + } + if got := ParseThreads([]byte("Name:\thive\n")); got != 0 { + t.Fatalf("ParseThreads without field = %d, want 0", got) + } +} + +func TestTakeIsPopulated(t *testing.T) { + s := Take() + if s.Goroutines < 1 { + t.Fatalf("Goroutines = %d, want >= 1", s.Goroutines) + } + if s.TotalSys == 0 || s.NextGC == 0 { + t.Fatalf("runtime/metrics samples missing: %+v", s) + } +} + +func TestStartLoggerEmitsPerTick(t *testing.T) { + var buf bytes.Buffer + written := make(chan struct{}, 8) + logger := slog.New(&signallingHandler{Handler: slog.NewTextHandler(&buf, nil), done: written}) + tick := make(chan time.Time) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + StartLogger(ctx, logger, tick) + for i := 0; i < 3; i++ { + tick <- time.Time{} + // Each tick is handled synchronously; wait for its record before the + // next so the buffer is complete when the goroutine is stopped. + select { + case <-written: + case <-time.After(5 * time.Second): + t.Fatalf("tick %d: no log record written", i) + } + } + cancel() + out := buf.String() + if n := strings.Count(out, "mem: runtime snapshot"); n != 3 { + t.Fatalf("got %d mem lines, want 3:\n%s", n, out) + } + for _, key := range []string{"heap_inuse=", "heap_objects=", "goroutines=", "threads=", "next_gc="} { + if !strings.Contains(out, key) { + t.Errorf("mem line lacks %s:\n%s", key, out) + } + } +} + +func TestIsLoopbackAddr(t *testing.T) { + for addr, want := range map[string]bool{ + "127.0.0.1:6060": true, + "localhost:6060": true, + "[::1]:6060": true, + "127.9.9.9:1": true, + "0.0.0.0:6060": false, + ":6060": false, + "10.0.0.1:6060": false, + "127.0.0.1": false, + } { + if got := IsLoopbackAddr(addr); got != want { + t.Errorf("IsLoopbackAddr(%q) = %v, want %v", addr, got, want) + } + } +} + +func TestStartPprofRejectsNonLoopback(t *testing.T) { + if _, _, err := StartPprof("0.0.0.0:0", nil); !errors.Is(err, ErrNotLoopback) { + t.Fatalf("err = %v, want ErrNotLoopback", err) + } + if _, stop, err := StartPprof("", nil); err != nil || stop == nil { + t.Fatalf("empty addr: err=%v stop-nil=%v", err, stop == nil) + } +} + +func TestStartPprofServesIndexAndRuntime(t *testing.T) { + bound, stop, err := StartPprof("127.0.0.1:0", nil) + if err != nil { + t.Fatal(err) + } + defer stop() + for _, path := range []string{"/debug/pprof/", "/debug/pprof/goroutine?debug=1", "/debug/runtime"} { + resp, err := http.Get("http://" + bound + path) + if err != nil { + t.Fatalf("GET %s: %v", path, err) + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s: status %d", path, resp.StatusCode) + } + if path == "/debug/runtime" && !strings.Contains(string(body), "goroutines=") { + t.Fatalf("/debug/runtime body: %s", body) + } + } +} + +func TestCgroupMemoryLimit(t *testing.T) { + files := func(m map[string]string) func(string) ([]byte, error) { + return func(p string) ([]byte, error) { + if v, ok := m[p]; ok { + return []byte(v), nil + } + return nil, errors.New("missing") + } + } + cases := []struct { + name string + files map[string]string + want int64 + ok bool + }{ + {"v2 limit", map[string]string{cgroupV2MemoryMax: "42949672960\n"}, 42949672960, true}, + {"v2 max", map[string]string{cgroupV2MemoryMax: "max\n"}, 0, false}, + {"v1 limit", map[string]string{cgroupV1MemoryLimit: "25769803776\n"}, 25769803776, true}, + {"v1 unlimited", map[string]string{cgroupV1MemoryLimit: "9223372036854771712\n"}, 0, false}, + {"none", map[string]string{}, 0, false}, + {"garbage", map[string]string{cgroupV2MemoryMax: "lots\n"}, 0, false}, + } + for _, tc := range cases { + got, ok := CgroupMemoryLimit(files(tc.files)) + if got != tc.want || ok != tc.ok { + t.Errorf("%s: got (%d,%v) want (%d,%v)", tc.name, got, ok, tc.want, tc.ok) + } + } +} + +func TestApplyMemoryLimit(t *testing.T) { + prev := debug.SetMemoryLimit(-1) + t.Cleanup(func() { debug.SetMemoryLimit(prev) }) + + const cg = int64(40 << 30) + readV2 := func(p string) ([]byte, error) { + if p == cgroupV2MemoryMax { + return []byte("42949672960"), nil + } + return nil, errors.New("missing") + } + out := ApplyMemoryLimit(func(string) string { return "" }, readV2) + want := DeriveMemoryLimit(cg) + if out.Source != "cgroup" || out.Limit != want || debug.SetMemoryLimit(-1) != want { + t.Fatalf("cgroup path: %+v (runtime limit %d, want %d)", out, debug.SetMemoryLimit(-1), want) + } + if want >= cg || want < cg*80/100 { + t.Fatalf("derived limit %d is not ~%d%% of %d", want, MemoryLimitPercent, cg) + } + + debug.SetMemoryLimit(prev) + out = ApplyMemoryLimit(func(string) string { return "16GiB" }, readV2) + if out.Source != "env" || debug.SetMemoryLimit(-1) != prev { + t.Fatalf("env path must not touch the runtime limit: %+v", out) + } + + out = ApplyMemoryLimit(func(string) string { return "" }, func(string) ([]byte, error) { return nil, errors.New("missing") }) + if out.Source != "none" || debug.SetMemoryLimit(-1) != prev { + t.Fatalf("no-cgroup path must not touch the runtime limit: %+v", out) + } +} + +// signallingHandler sends on done after every record so tests can wait on the +// observable write instead of sleeping. +type signallingHandler struct { + slog.Handler + done chan<- struct{} +} + +func (h *signallingHandler) Handle(ctx context.Context, r slog.Record) error { + err := h.Handler.Handle(ctx, r) + h.done <- struct{}{} + return err +} + +func TestStartLoggerDefaultsAndStopsOnCancel(t *testing.T) { + before := runtime.NumGoroutine() + ctx, cancel := context.WithCancel(context.Background()) + StartLogger(ctx, nil, nil) + cancel() + deadline := time.After(5 * time.Second) + for runtime.NumGoroutine() > before { + select { + case <-deadline: + t.Fatalf("logger goroutine did not stop: %d goroutines, baseline %d", runtime.NumGoroutine(), before) + default: + runtime.Gosched() + } + } +} + +func TestStartPprofEmptyAddrIsDisabled(t *testing.T) { + bound, stop, err := StartPprof("", nil) + if err != nil || bound != "" || stop == nil { + t.Fatalf("StartPprof(\"\") = %q, %v, stop nil=%v", bound, err, stop == nil) + } + stop() +} + +func TestStartPprofReportsListenError(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Skip("no loopback listener:", err) + } + defer ln.Close() + if _, _, err := StartPprof(ln.Addr().String(), nil); err == nil { + t.Fatal("StartPprof on an occupied port must fail") + } +} + +func TestThreadCountWithoutProcStatus(t *testing.T) { + saved := procSelfStatus + procSelfStatus = filepath.Join(t.TempDir(), "missing") + t.Cleanup(func() { procSelfStatus = saved }) + if got := threadCount(); got != 0 { + t.Fatalf("threadCount without /proc = %d, want 0", got) + } + if s := Take(); s.Threads != 0 { + t.Fatalf("Take().Threads without /proc = %d, want 0", s.Threads) + } +} + +func TestApplyMemoryLimitUsesProcessDefaults(t *testing.T) { + saved := debug.SetMemoryLimit(-1) + t.Cleanup(func() { debug.SetMemoryLimit(saved) }) + out := ApplyMemoryLimit(nil, nil) + switch out.Source { + case "env", "none", "cgroup": + default: + t.Fatalf("unexpected source %q: %+v", out.Source, out) + } + if out.Limit <= 0 { + t.Fatalf("effective limit must be positive: %+v", out) + } +} From 350d7de05b742d80cf18e5e1cff530cb2307f135 Mon Sep 17 00:00:00 2001 From: thatguywob Date: Tue, 27 Oct 2026 19:37:40 -0600 Subject: [PATCH 28/38] =?UTF-8?q?=F0=9F=90=9B=20fix:=20withhold=20classifi?= =?UTF-8?q?er=20errors=20from=20public=20selfreports=20(#14774)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep raw classifier transport errors in local logs while retaining the warning and remediation context in fleet diagnostics. Filter both error fields, including nested and bound attributes, before rendering the public diagnostic excerpt. Hive-Run: hivecommons/hive#14735 Hive-Plan: classifier-selfreport-error-boundary Hive-Spec: issue-14735#private-classifier-errors Signed-off-by: thatguywob Co-authored-by: thatguywob --- .../security-14735-classifier-selfreport.md | 1 + src/pkg/fleetreport/slogfeed/slogfeed.go | 25 +++++++-- src/pkg/fleetreport/slogfeed/slogfeed_test.go | 51 +++++++++++++++++++ 3 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 changelog.d/security-14735-classifier-selfreport.md diff --git a/changelog.d/security-14735-classifier-selfreport.md b/changelog.d/security-14735-classifier-selfreport.md new file mode 100644 index 0000000000..0f8f71c4a1 --- /dev/null +++ b/changelog.d/security-14735-classifier-selfreport.md @@ -0,0 +1 @@ +- Keep raw IOScan classifier errors, including private endpoint URLs and DNS details, in local logs rather than public selfreport diagnostics. diff --git a/src/pkg/fleetreport/slogfeed/slogfeed.go b/src/pkg/fleetreport/slogfeed/slogfeed.go index 9b12ac85d9..d261261cff 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed.go @@ -141,16 +141,35 @@ func expectedKickHold(r slog.Record) bool { return held } -// diagnostics renders "level attr=value …" from the bound and record -// attributes, bounded. It runs only for WARN+ records that passed the -// filters, so the allocation is paid on the rare path. +// Classifier errors stay in local logs only: transport errors can expose the +// private endpoint in both the URL and the nested DNS/dial error. +func redactClassifierError(a slog.Attr) slog.Attr { + if a.Key == "error" || a.Key == "classifier_error" { + return slog.String(a.Key, "[withheld: see local logs]") + } + a.Value = a.Value.Resolve() + if a.Value.Kind() == slog.KindGroup { + attrs := append([]slog.Attr(nil), a.Value.Group()...) + for i := range attrs { + attrs[i] = redactClassifierError(attrs[i]) + } + a.Value = slog.GroupValue(attrs...) + } + return a +} + +// diagnostics renders bounded attributes for WARN+ records that passed the filters. func diagnostics(r slog.Record, bound []slog.Attr) string { + classifier := fleetreport.LogFedClass(r.Message) == fleetreport.ClassIOScanClassifierUnavail var b strings.Builder b.WriteString(r.Level.String()) write := func(a slog.Attr) bool { if b.Len() >= maxDiagnosticsBytes { return false } + if classifier { + a = redactClassifierError(a) + } b.WriteByte(' ') b.WriteString(a.Key) b.WriteByte('=') diff --git a/src/pkg/fleetreport/slogfeed/slogfeed_test.go b/src/pkg/fleetreport/slogfeed/slogfeed_test.go index baee48a47e..6ba2adc1b6 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed_test.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed_test.go @@ -3,7 +3,9 @@ package slogfeed import ( "bytes" "context" + "errors" "log/slog" + "net/url" "strings" "testing" "time" @@ -466,3 +468,52 @@ func TestHandlerSkipsContributorTierLimits(t *testing.T) { } } } + +func TestClassifierErrorsStayInLocalLogs(t *testing.T) { + const host = "private-inference.internal" + err := &url.Error{Op: "Post", URL: "https://" + host + "/v1/chat/completions", Err: errors.New("dial tcp: lookup " + host + ": no such host")} + for _, msg := range []string{ + "ioscan classifier failed closed; text withheld", + "ioscan classifier failed open; text passes on unclassified", + "ioscan fail-closed blocked kick", + } { + for _, key := range []string{"error", "classifier_error"} { + for _, placement := range []string{"record", "bound", "group"} { + t.Run(msg+"/"+key+"/"+placement, func(t *testing.T) { + now := time.Date(2026, 10, 28, 0, 0, 0, 0, time.UTC) + tr := fleetreport.NewRuntimeTracker(func() time.Time { return now }) + var out bytes.Buffer + logger := slog.New(NewWithObserver(slog.NewJSONHandler(&out, nil), tr.Observe)) + attr := slog.Any(key, err) + if key == "classifier_error" { + attr = slog.String(key, err.Error()) + } + args := []any{"agent", "reviewer", "fix", "check ioscan.classifier.model"} + switch placement { + case "record": + args = append(args, attr) + case "bound": + logger = logger.With(attr) + case "group": + logger = logger.With(slog.Group("classifier", slog.Group("transport", attr))) + } + logger.Warn(msg, args...) + if !strings.Contains(out.String(), host) || !strings.Contains(out.String(), "no such host") { + t.Fatalf("local log lost the original error: %s", out.String()) + } + ev := tr.Evidence() + if len(ev) != 1 || ev[0].Class != fleetreport.ClassIOScanClassifierUnavail { + t.Fatalf("classifier warning lost its report: %+v", ev) + } + report := fleetreport.BuildRuntimeReport(fleetreport.Observation{}, ev[0], "fp", "inst", "hive", "v5", "commit", now) + if strings.Contains(report.Body, host) || strings.Contains(report.Body, "no such host") || strings.Contains(report.Body, "/v1/chat/completions") { + t.Fatalf("public report leaked classifier error: %s", report.Body) + } + if !strings.Contains(report.Body, msg) || !strings.Contains(report.Body, "agent=reviewer") || !strings.Contains(report.Body, "fix=check ioscan.classifier.model") { + t.Fatalf("public report lost actionable context: %s", report.Body) + } + }) + } + } + } +} From 0efc11e249e909b3a5784b986be60d5d6d8c6e2a Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 27 Oct 2026 21:48:38 -0400 Subject: [PATCH 29/38] =?UTF-8?q?=F0=9F=90=9B=20deps:=20bump=20golang.org/?= =?UTF-8?q?x/net=20to=20clear=20govulncheck=20finding=20(#14782)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andy Anderson --- changelog.d/security-crux-govulncheck-xnet.md | 1 + src/crux/go.mod | 4 ++-- src/crux/go.sum | 8 ++++---- 3 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 changelog.d/security-crux-govulncheck-xnet.md diff --git a/changelog.d/security-crux-govulncheck-xnet.md b/changelog.d/security-crux-govulncheck-xnet.md new file mode 100644 index 0000000000..1fa7ae99e2 --- /dev/null +++ b/changelog.d/security-crux-govulncheck-xnet.md @@ -0,0 +1 @@ +- Bump golang.org/x/net to v0.62.0 in src/crux to clear GO-2026-6617/6612/6611/6603 (govulncheck red on hivecommons/crux main). diff --git a/src/crux/go.mod b/src/crux/go.mod index 5685f185cf..d54499d9d8 100644 --- a/src/crux/go.mod +++ b/src/crux/go.mod @@ -34,8 +34,8 @@ require ( go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 // indirect - golang.org/x/net v0.59.0 // indirect - golang.org/x/sys v0.48.0 // indirect + golang.org/x/net v0.62.0 // indirect + golang.org/x/sys v0.49.0 // indirect golang.org/x/text v0.43.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect diff --git a/src/crux/go.sum b/src/crux/go.sum index 1c26caea7e..5b6ed28d81 100644 --- a/src/crux/go.sum +++ b/src/crux/go.sum @@ -62,10 +62,10 @@ go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= -golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= -golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= -golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= -golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= From 49afb3d6d1af21585f8de4b832c940051b7384ef Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 27 Oct 2026 21:52:11 -0400 Subject: [PATCH 30/38] fix(terminal): find default tmux socket under arbitrary container UID (#14791) With no per-agent UID map every hive- session lives on the default tmux server of the UID the container runs as. ttyd-tmux.sh's fallback assumed that was dev's passwd uid (/tmp/tmux-1001/default). Under OpenShift's restricted SCC the container runs as an arbitrary uid with no passwd entry, so the helper polled the wrong socket for 900s while every session was alive. Walk a deduplicated candidate list instead: the current uid's default socket, dev's, then any ${TMUX_TMPDIR:-/tmp}/tmux-*/default, and use the first that is a socket and holds the requested session. The available-sessions hint lists sessions from every default server found and no longer reports the socket name 'default' as a session. Adds test_ttyd_default_socket_uid.sh (id stubbed, real tmux servers at each layout's path) and wires it into v2-ci. Fixes #14789 Signed-off-by: Andrew Anderson --- .github/workflows/v2-ci.yml | 13 + ...minal-default-tmux-socket-arbitrary-uid.md | 1 + src/deploy/test_ttyd_default_socket_uid.sh | 266 ++++++++++++++++++ src/deploy/ttyd-tmux.sh | 81 ++++-- 4 files changed, 341 insertions(+), 20 deletions(-) create mode 100644 changelog.d/fixed-14789-terminal-default-tmux-socket-arbitrary-uid.md create mode 100644 src/deploy/test_ttyd_default_socket_uid.sh diff --git a/.github/workflows/v2-ci.yml b/.github/workflows/v2-ci.yml index b6f69f5a70..458052a823 100644 --- a/.github/workflows/v2-ci.yml +++ b/.github/workflows/v2-ci.yml @@ -996,6 +996,19 @@ jobs: working-directory: . run: bash src/deploy/test_ttyd_url_arg.sh + # The browser terminal's attach helper falls back to the DEFAULT tmux + # socket when agents share one UID. It assumed that socket lived under + # dev's passwd uid (/tmp/tmux-1001), but under OpenShift's restricted + # SCC the container runs as an arbitrary uid with no passwd entry, so + # every session was alive on /tmp/tmux-/default while the + # dashboard polled /tmp/tmux-1001/default for 900s. This stubs `id`, + # starts real tmux servers at the paths each uid layout would use, and + # asserts the helper attaches to the right one. Skips where tmux is + # absent. + - name: ttyd finds the default tmux socket under an arbitrary container UID + working-directory: . + run: bash src/deploy/test_ttyd_default_socket_uid.sh + # #5435: hive-discord.service ran `node bot.js` with # WorkingDirectory=/tmp/hive/discord, resolving the executed code out of a # world-writable parent that is cleared on every reboot β€” a local user who diff --git a/changelog.d/fixed-14789-terminal-default-tmux-socket-arbitrary-uid.md b/changelog.d/fixed-14789-terminal-default-tmux-socket-arbitrary-uid.md new file mode 100644 index 0000000000..1ffc4a4dbd --- /dev/null +++ b/changelog.d/fixed-14789-terminal-default-tmux-socket-arbitrary-uid.md @@ -0,0 +1 @@ +- The dashboard terminal now attaches to agents when the container runs as an arbitrary UID (e.g. OpenShift's restricted SCC) and agents share one tmux server: the attach helper looked for the default tmux socket only under dev's passwd uid (`/tmp/tmux-1001`), so it polled "waiting for agent to start" for every agent while all sessions were alive on `/tmp/tmux-/default`. It now checks the current uid's socket, dev's, and any other `tmux-*/default` under `$TMUX_TMPDIR`, picking the one that holds the requested session ([#14789](https://github.com/hivecommons/hive/issues/14789)). diff --git a/src/deploy/test_ttyd_default_socket_uid.sh b/src/deploy/test_ttyd_default_socket_uid.sh new file mode 100644 index 0000000000..d3027d813a --- /dev/null +++ b/src/deploy/test_ttyd_default_socket_uid.sh @@ -0,0 +1,266 @@ +#!/usr/bin/env bash +# The browser terminal must find the default tmux socket whatever UID the +# container runs as. +# Run: bash src/deploy/test_ttyd_default_socket_uid.sh +# +# THE BUG. With no per-agent UID map ("no UID map found, agents will share dev +# UID") every hive- session lives on the DEFAULT tmux server of the UID +# the container runs as, i.e. $TMUX_TMPDIR/tmux-$(id -u)/default. The attach +# helper's fallback instead looked at /tmp/tmux-$(id -u dev || 1001)/default β€” +# dev's PASSWD uid. On Compose/Podman the two are the same number and nobody +# noticed. Under OpenShift's restricted SCC the container runs as an arbitrary +# UID (1000840000 on the spoke that reported this) that has no passwd entry, so +# the sessions were at /tmp/tmux-1000840000/default, the helper polled +# /tmp/tmux-1001/default forever, and the dashboard showed "waiting for agent +# 'X' to start (tmux session not created yet)" for every agent while `tmux ls` +# showed them all alive. +# +# HOW THIS TESTS IT. The helper derives the UID from `id -u`, so `id` is +# stubbed on PATH to answer with an OpenShift-style UID (and, per case, with or +# without a passwd entry for dev). TMUX_TMPDIR points the helper's socket +# lookup at a scratch directory where a real tmux server is started with +# `tmux -S` at exactly the path the container would use. The helper is then +# run for real against it: resolving the socket is asserted POSITIVELY through +# the wheel rebind the helper leaves on the server (the one setting it does not +# restore on detach), because an absence-only check passes just as happily when +# the script never reaches attach at all. +set -uo pipefail + +PASS=0 +FAIL=0 +pass() { echo " PASS: $1"; PASS=$((PASS + 1)); } +fail() { echo " FAIL: $1"; [ $# -gt 1 ] && echo " $2"; FAIL=$((FAIL + 1)); } +finish() { + echo "" + echo "=== Results: $PASS passed, $FAIL failed ===" + [ "$FAIL" -eq 0 ] +} + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +ATTACH="${ROOT}/src/deploy/ttyd-tmux.sh" + +echo "=== ttyd-tmux.sh finds the default tmux socket under an arbitrary container UID ===" + +if [ ! -f "$ATTACH" ]; then + fail "locate $ATTACH" "the layout moved β€” this test cannot verify anything" + finish; exit $? +fi + +if ! command -v tmux >/dev/null 2>&1; then + echo " SKIP: tmux not installed β€” these assertions need a real tmux server" + if [ "${HIVE_TEST_REQUIRE_BEHAVIOURAL:-}" = "1" ]; then + fail "tmux is required when HIVE_TEST_REQUIRE_BEHAVIOURAL=1" + fi + finish; exit $? +fi +if command -v su-exec >/dev/null 2>&1; then + # The helper would su-exec to the socket owner when the stubbed uid differs + # from it; that path needs root and is not what is under test here. + echo " SKIP: su-exec is on PATH β€” the uid stub would route through it" + finish; exit $? +fi + +# The UID of the reporting OpenShift spoke. Nothing on a developer box or CI +# runner owns this, so the helper's historical /tmp/tmux-1001 assumption cannot +# be satisfied by accident. +ARBITRARY_UID=1000840000 +DEV_UID=1001 +OTHER_UID=4242 + +WORK="$(mktemp -d)" +SOCKET_ROOT="${WORK}/sockets" +STUB_BIN="${WORK}/bin" +mkdir -p "$SOCKET_ROOT" "$STUB_BIN" +declare -a SERVERS=() +cleanup() { + local s + for s in "${SERVERS[@]+"${SERVERS[@]}"}"; do + tmux -S "$s" kill-server 2>/dev/null + done + rm -rf "$WORK" +} +trap cleanup EXIT + +# stub_id : `id -u` answers the first, `id -u dev` +# the second (or fails, like getent on a container with no passwd entry for an +# arbitrary UID). Anything else falls through to the real id. +REAL_ID="$(command -v id)" +stub_id() { + cat > "${STUB_BIN}/id" <&2; exit 1 +fi +exec "$REAL_ID" "\$@" +EOF + chmod +x "${STUB_BIN}/id" +} + +run_with_timeout() { + local seconds="$1" + shift + if command -v timeout >/dev/null 2>&1; then + timeout "$seconds" "$@" + else + perl -e ' + my $seconds = shift @ARGV; + my $pid = fork(); + exit 127 unless defined $pid; + if ($pid == 0) { setpgrp(0, 0); exec @ARGV; exit 127; } + $SIG{ALRM} = sub { + kill "TERM", -$pid; select undef, undef, undef, 0.2; kill "KILL", -$pid; + waitpid($pid, 0); exit 124; + }; + alarm $seconds; waitpid($pid, 0); + exit 127 if $? == -1; exit 128 + ($? & 127) if $? & 127; exit $? >> 8; + ' "$seconds" "$@" + fi +} + +# start_default_server : a real tmux server at the path the +# container's default server would use for that uid. +start_default_server() { + local dir="${SOCKET_ROOT}/tmux-$1" sock + mkdir -p "$dir" && chmod 700 "$dir" + sock="${dir}/default" + tmux -S "$sock" -f /dev/null new-session -d -s "$2" 'sleep 300' 2>/dev/null || return 1 + SERVERS+=("$sock") + [ -S "$sock" ] +} + +# run_attach : the helper, for real, with a short wait so a lookup +# that still fails reports quickly instead of polling for 900s. Invoked through +# `bash` because the file ships mode 644 and is chmod'ed at image build time. +run_attach() { + PATH="${STUB_BIN}:${PATH}" TMUX_TMPDIR="$SOCKET_ROOT" \ + HIVE_TTYD_WAIT_SEC=2 HIVE_TTYD_POLL_SEC=1 \ + run_with_timeout 15 bash "$ATTACH" "$1" &1 +} + +# assert_resolves