Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-cacheable-since-reads.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- The hive's routine "everything since a time" GitHub reads can now be answered with free `304`s. The un-park sweep's comment reads, the closed-issue stage sweep, the review backlog split, the shared-CI rerun sweep and the same-component check built their `since=` from the current clock, so the URL changed on every pass and the ETag cache (keyed on the full URL) never matched. On a hosted hive with 20 repos on 2026-11-07, untagged issue-comment lists were answered `304` only 51% of the time and issue lists only 28%, against 83% to 91% for the pull-request lists, while the hive needed about 8,250 calls an hour against a 7,000 limit. The `since` sent to GitHub is now rounded down to the hour; each sweep still filters on its exact cutoff, so nothing is missed. The calls the periodic sweeps, the governor cycle, the state cache, the metrics collector, the request watchers, the mention poller and dashboard requests make are now named in `/api/gh-rate-limits` (`hive:<sweep>`) instead of all reading as `hive`.
2 changes: 1 addition & 1 deletion src/cmd/hive/forgestate_boot.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ func (b *boot) applyGitHubClientForgeState(client *github.Client) {
ctx = context.Background()
}
go func() {
if err := store.Run(ctx); err != nil && !errors.Is(err, context.Canceled) {
if err := store.Run(github.WithRESTCaller(ctx, "hive:forgestate")); err != nil && !errors.Is(err, context.Canceled) {
b.logger.Error("forgestate: reconcile loop stopped", "error", err)
}
}()
Expand Down
19 changes: 11 additions & 8 deletions src/cmd/hive/main_eval_cycle.go
Original file line number Diff line number Diff line change
Expand Up @@ -550,6 +550,9 @@ func runEvalCycle(
attribute.String("hive.id", cfg.HiveID),
attribute.Int(tracing.AttrHiveACMMLevel, inferACMMLevel(cfg)))
defer span.End()
// GitHub calls this cycle makes are "hive:eval_cycle" in
// /api/gh-rate-limits unless a step below names itself more narrowly.
ctx = github.WithRESTCaller(ctx, "hive:eval_cycle")

// A hive running without GitHub credentials (placeholder app_id, or a real
// App whose key could not be read) has nothing to enumerate. Return before
Expand Down Expand Up @@ -611,7 +614,7 @@ func runEvalCycle(
"stale", actionable.Stale())
} else {
enumCtx, enumSpan := tracing.StartSpan(ctx, "governor.enumerate_actionable")
fresh, err := ghClient.EnumerateActionable(enumCtx)
fresh, err := ghClient.EnumerateActionable(github.WithRESTCaller(enumCtx, "hive:enumerate_actionable"))
enumSpan.End()
var ok bool
actionable, ok = actionableAfterGitHubEnumerate(cfg, fresh, err, logger)
Expand Down Expand Up @@ -691,12 +694,12 @@ func runEvalCycle(
// another. Backed by a PVC ledger so it survives those restarts, and fails
// closed (keeps the last known claims) when the GitHub API is unavailable.
if !budgetDecision.SkipOptional {
applyDuplicatePRGuard(ctx, cfg, ghClient, actionable, logger)
applyDuplicatePRGuard(github.WithRESTCaller(ctx, "hive:duplicate_pr_guard"), cfg, ghClient, actionable, logger)
}
// Every cycle, cached ones included: heal issue stage records the
// claim ledger says an open PR covers but the store still holds below
// 040 (a claim release written over the cover between reconcile passes).
reconcileLedgerCoverRecords(ctx, cfg, ghClient, actionable, logger)
reconcileLedgerCoverRecords(github.WithRESTCaller(ctx, "hive:ledger_cover"), cfg, ghClient, actionable, logger)

// PR review queue (#9590): stamp each PR's rank, priority and reasons
// onto the snapshot so last-actionable.json carries them next to
Expand Down Expand Up @@ -731,7 +734,7 @@ func runEvalCycle(

escalatedPRs := map[string]bool{}
if !budgetDecision.SkipUnstick {
escalatedPRs = runEscalationSweep(ctx, cfg, governorForge(cfg, ghClient, logger), actionable, notifier, dashSrv, logger)
escalatedPRs = runEscalationSweep(github.WithRESTCaller(ctx, "hive:escalation_sweep"), cfg, governorForge(cfg, ghClient, logger), actionable, notifier, dashSrv, logger)
runDependencyBotConflictSweep(ctx, cfg, governorForge(cfg, ghClient, logger), ghClient, actionable, logger)
}
runWaitEscalationSweep(cfg, dashSrv, logger)
Expand All @@ -758,7 +761,7 @@ func runEvalCycle(
}
return cfg.SelfAuthorizationHoldEnabledForRepoAtLevel(repo, level)
}
holdDriftPRs := enforceHoldGuard(ctx, cfg, ghClient, governorForge(cfg, ghClient, logger), actionable, logger, selfAuthorizationHoldEnabled)
holdDriftPRs := enforceHoldGuard(github.WithRESTCaller(ctx, "hive:hold_guard"), cfg, ghClient, governorForge(cfg, ghClient, logger), actionable, logger, selfAuthorizationHoldEnabled)

// The per-PR verdicts come back so the dashboard's PR pills can be
// painted from the sweep's own classification rather than a looser
Expand All @@ -785,7 +788,7 @@ func runEvalCycle(
// PR this hive opened back into the CLI session that authored it, while
// that session is still live. Anything it cannot resume stays on the
// fix-before-new path above, unchanged.
routePRFollowUps(ctx, cfg, ghClient, actionable, escalatedPRs, agentMgr, logger)
routePRFollowUps(github.WithRESTCaller(ctx, "hive:pr_follow_ups"), cfg, ghClient, actionable, escalatedPRs, agentMgr, logger)

// Stuck-PR reaper (backstop): DELIVER a targeted FIX-BEFORE-NEW kick for any
// hive-authored PR that is red on a required check AND stale (its red head
Expand Down Expand Up @@ -860,8 +863,8 @@ func runEvalCycle(
"skipped", shaResult.Skipped,
)
}
runHardSuppressReconcile(ctx, cfg, ghClient, logger)
runParkRecheck(ctx, cfg, ghClient, logger)
runHardSuppressReconcile(github.WithRESTCaller(ctx, "hive:hard_suppress_reconcile"), cfg, ghClient, logger)
runParkRecheck(github.WithRESTCaller(ctx, "hive:park_recheck"), cfg, ghClient, logger)
}

// Refresh budget spend from lifetime token totals before Evaluate so
Expand Down
10 changes: 9 additions & 1 deletion src/cmd/hive/sweep_workers.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,14 @@ type sweepJob struct {
run func(context.Context, *github.Client, *config.Config)
}

// runTagged runs the job with its GitHub REST calls attributed to
// "hive:<name>" in /api/gh-rate-limits. Calls a sweep already tags more
// narrowly keep their own name; before this, every untagged call from every
// sweep was reported as plain "hive".
func (j sweepJob) runTagged(ctx context.Context, gh *github.Client, cfg *config.Config) {
j.run(github.WithRESTCaller(ctx, "hive:"+j.name), gh, cfg)
}

// sweepStartupSpread is the window the non-essential sweeps' first passes are
// spread across after a start. All workers start in the same second; on a
// cold ETag cache their first passes each listed every repo at once, all
Expand Down Expand Up @@ -214,7 +222,7 @@ func (b *boot) startSweepWorkers() func() {
if skipped {
return
}
job.run(ctx, gh, cfg)
job.runTagged(ctx, gh, cfg)
if b.dashSrv != nil {
b.dashSrv.RecordSweepCadence(job.name, started, job.interval(cfg), true, false)
}
Expand Down
33 changes: 33 additions & 0 deletions src/cmd/hive/sweep_workers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package main

import (
"context"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"time"
Expand Down Expand Up @@ -249,3 +251,34 @@ func TestSweepFirstDelaysSpreadNonEssential(t *testing.T) {
prev = d
}
}

// A sweep worker's GitHub calls are attributed to "hive:<job name>" in REST
// accounting (/api/gh-rate-limits), not the undifferentiated "hive" that hid
// ~4,400 calls an hour on the knuckle hive. The timeline read stands in for
// any untagged call a sweep makes.
func TestSweepJobRunTaggedAttributesRESTCalls(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet || r.URL.Path != "/repos/testorg/widget/issues/1/timeline" {
t.Errorf("unexpected operation: %s %s", r.Method, r.URL.Path)
w.WriteHeader(http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`[]`))
}))
defer srv.Close()
ghClient := github.NewClientForTest(srv.URL, "testorg", []string{"widget"}, sweepTestLogger(nil))
job := sweepJob{name: "p34_caller_probe", run: func(ctx context.Context, gh *github.Client, _ *config.Config) {
if _, err := gh.IssueTimelineSince(ctx, "widget", 1, time.Time{}); err != nil {
t.Errorf("timeline read: %v", err)
}
}}
job.runTagged(context.Background(), ghClient, &config.Config{})

for _, c := range github.RESTTopConsumers(1000) {
if c.Caller == "hive:p34_caller_probe" && c.Endpoint == "/repos/{owner}/{repo}/issues/{number}/timeline" && c.Requests >= 1 {
return
}
}
t.Fatalf("REST accounting has no hive:p34_caller_probe timeline row: %+v", github.RESTTopConsumers(1000))
}
4 changes: 2 additions & 2 deletions src/docs/design/agent-turn-model.md
Original file line number Diff line number Diff line change
Expand Up @@ -139,9 +139,9 @@ Kick *timing* lives in `pkg/governor`; kick *text* is built in `pkg/scheduler`.
`deps.newTicker(initialEvalInterval)` at `src/cmd/hive/main.go:5772`, loop at
`src/cmd/hive/main.go:6112`, evaluation via `deps.runEval` at
`src/cmd/hive/main.go:5956` (`runEvalCycle`, `src/cmd/hive/main_eval_cycle.go:471`),
message assembly via `sched.BuildKickMessagesWithSkips` at `src/cmd/hive/main_eval_cycle.go:964`
message assembly via `sched.BuildKickMessagesWithSkips` at `src/cmd/hive/main_eval_cycle.go:1094`
(`src/pkg/scheduler/kickmessage.go:261`), and delivery via
`agentMgr.SendKick` wired into `dispatchAgentKicks` at `src/cmd/hive/main_eval_cycle.go:1127`.
`agentMgr.SendKick` wired into `dispatchAgentKicks` at `src/cmd/hive/main_eval_cycle.go:1580`.

This matters for the RFC: the scheduler is already **stateless with respect to
turns**. It does not hold a continuation, does not await turn *N* before
Expand Down
4 changes: 4 additions & 0 deletions src/docs/operator-reference/sweeps-and-operations.md
Original file line number Diff line number Diff line change
Expand Up @@ -907,6 +907,10 @@ Mode exits use hysteresis: Hive leaves conserve/critical only after quota rises

**Historical PR accounting waits for a normal budget.** The worker that reads each recently merged or closed PR's merger and reviews (for the throughput and review counts) runs only in `normal` mode. In `conserve` and `critical` it makes no GitHub calls; its queue is kept and drained once the budget is back to `normal`. The hive logs `historical PR enrichment paused for GitHub API budget` once when it pauses and `historical PR enrichment resumed` once when it starts again. This matters most after a restart: the 72-hour settle replay queues every recent merged or closed PR again, at about two calls each.

**Sweeps that read "everything since a time" round that time down to the hour.** A free `304` answer needs the request URL to match the one the hive fetched before, query string included. The un-park sweep's comment reads (comments from the last seven days), the closed-issue stage sweep and the review backlog split (issues closed since the last pass), the shared-CI rerun sweep (incidents closed in its lookback) and the same-component check when an agent files an issue all used to build that time from the current clock, so the URL changed on every pass and every pass was charged in full. Now the time sent to GitHub is rounded down to the hour, so every pass in the same hour asks for the same URL. The window only gets wider, by less than an hour, and each sweep still filters on its exact cutoff, so nothing is missed and no old command is answered again.

**`/api/gh-rate-limits` names the sweep behind each call.** In `top_consumers`, the calls a periodic sweep makes are listed as `hive:<sweep name>` (for example `hive:issue_unpark` or `hive:closed_issue_stage`), the governor cycle's calls as `hive:eval_cycle` (its issue listing as `hive:enumerate_actionable`), the state cache's as `hive:forgestate`, the dashboard metrics as `hive:metrics_collector`, and the calls made while serving a dashboard request as `hive:dashboard_api`. A call that a sweep already tags more narrowly keeps that name. Plain `hive` is now only what none of these cover.

- Recovery from a provider spending limit is automatic, via a probe. Withholding kicks also withholds the inference calls that would reveal the provider is serving again, so the hive suppresses only while the last refusal is recent and then lets a single kick through to test the gateway; the probe re-arms suppression the moment it is released, so at most one probe run flies per interval. A still-clipped key refuses the probe and suppression resumes for another interval; once the provider's window resets the probe succeeds, normal kicking resumes with no operator action, and a one-time recovery notification is sent (the entering notification is likewise sent once per clip, not once per cycle). Tune with `governor.provider_budget.probe_interval_s` (default 1800 — 30 minutes):

```yaml
Expand Down
1 change: 1 addition & 0 deletions src/pkg/dashboard/metrics_collector.go
Original file line number Diff line number Diff line change
Expand Up @@ -322,6 +322,7 @@ func (mc *MetricsCollector) collect(ctx context.Context) {
// Dashboard metrics are housekeeping against the pacer: they stand down
// for the governor's enumeration while a caution window is engaged.
ctx = ghpkg.WithPacingClass(ctx, ghpkg.PacingLow)
ctx = ghpkg.WithRESTCaller(ctx, "hive:metrics_collector")
metrics := make(map[string]any)

outreach := mc.collectOutreach(ctx)
Expand Down
11 changes: 10 additions & 1 deletion src/pkg/dashboard/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -2838,7 +2838,16 @@ async function poll(interval,flowId){
</script></body></html>`

func (s *Server) Handler() http.Handler {
return s.withRequestLog(s.noIndex(s.authenticatedMux()))
return s.withRequestLog(s.noIndex(withDashboardRESTCaller(s.authenticatedMux())))
}

// withDashboardRESTCaller attributes the GitHub REST calls a dashboard request
// makes to "hive:dashboard_api" in /api/gh-rate-limits instead of plain
// "hive". A handler that tags its calls more narrowly keeps its own name.
func withDashboardRESTCaller(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
next.ServeHTTP(w, r.WithContext(github.WithRESTCaller(r.Context(), "hive:dashboard_api")))
})
}

func (s *Server) roleEnforcement(next http.Handler) http.Handler {
Expand Down
170 changes: 170 additions & 0 deletions src/pkg/github/cacheable_since_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,170 @@
package github

import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"sync"
"testing"
"time"

gh "github.com/google/go-github/v72/github"
)

// These tests pin the ETag fix for moving since= cutoffs: a sweep that built
// its since from time.Now() changed the URL on every pass, so the ETag cache
// (keyed on the full URL) never matched and every pass was charged. Rounded
// down to the hour, every pass in that hour asks for the same URL and the
// repeat gets a free 304.

func TestCacheableSince(t *testing.T) {
if got := cacheableSince(time.Time{}); !got.IsZero() {
t.Fatalf("zero since = %v, want zero (no filter)", got)
}
east := time.FixedZone("UTC+5:30", 5*3600+1800)
cases := []struct {
in, want time.Time
}{
{time.Date(2026, 11, 7, 19, 42, 17, 500, time.UTC), time.Date(2026, 11, 7, 19, 0, 0, 0, time.UTC)},
{time.Date(2026, 11, 7, 19, 0, 0, 0, time.UTC), time.Date(2026, 11, 7, 19, 0, 0, 0, time.UTC)},
{time.Date(2026, 11, 7, 19, 59, 59, 999, time.UTC), time.Date(2026, 11, 7, 19, 0, 0, 0, time.UTC)},
// A half-hour zone still rounds to the UTC hour, so two hives in
// different zones ask for the same URL.
{time.Date(2026, 11, 8, 1, 12, 0, 0, east), time.Date(2026, 11, 7, 19, 0, 0, 0, time.UTC)},
}
for _, tc := range cases {
got := cacheableSince(tc.in)
if !got.Equal(tc.want) || got.Location() != time.UTC {
t.Errorf("cacheableSince(%v) = %v, want %v in UTC", tc.in, got, tc.want)
}
if got.After(tc.in) || tc.in.Sub(got) >= time.Hour {
t.Errorf("cacheableSince(%v) = %v: must be earlier by less than an hour", tc.in, got)
}
}
}

// etagListServer serves fixed JSON bodies by path with an ETag, and 304 when
// the client presents it. It records the since= and status of every request
// to the paths it watches.
type etagListServer struct {
t *testing.T
bodies map[string]any
watch string

mu sync.Mutex
seen []etagListRequest
}

type etagListRequest struct {
since string
status int
}

func (s *etagListServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
body, ok := s.bodies[r.URL.Path]
if r.Method != http.MethodGet || !ok {
s.t.Errorf("unexpected operation: %s %s", r.Method, r.URL.String())
w.WriteHeader(http.StatusNotFound)
return
}
etag := `W/"` + r.URL.Path + `"`
status := http.StatusOK
if r.Header.Get("If-None-Match") == etag {
status = http.StatusNotModified
}
if r.URL.Path == s.watch {
s.mu.Lock()
s.seen = append(s.seen, etagListRequest{since: r.URL.Query().Get("since"), status: status})
s.mu.Unlock()
}
w.Header().Set("ETag", etag)
if status == http.StatusNotModified {
w.WriteHeader(status)
return
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
_ = json.NewEncoder(w).Encode(body)
}

func (s *etagListServer) requests() []etagListRequest {
s.mu.Lock()
defer s.mu.Unlock()
return append([]etagListRequest(nil), s.seen...)
}

// assertCacheablePasses checks three passes: the second (same hour) asks for
// the first's exact URL and is answered 304; the third (next hour) moves
// since forward to that hour.
func assertCacheablePasses(t *testing.T, got []etagListRequest, sameHour, nextHour string) {
t.Helper()
if len(got) != 3 {
t.Fatalf("watched list requests = %+v, want one per pass (3)", got)
}
if got[0].since != sameHour || got[0].status != http.StatusOK {
t.Errorf("pass 1 = %+v, want since=%s answered 200", got[0], sameHour)
}
if got[1].since != got[0].since || got[1].status != http.StatusNotModified {
t.Errorf("pass 2 in the same hour = %+v, want since=%s (identical URL) answered 304", got[1], got[0].since)
}
if got[2].since != nextHour || got[2].status != http.StatusOK {
t.Errorf("pass 3 after the hour = %+v, want since=%s answered 200", got[2], nextHour)
}
}

func TestSweepIssueUnparkCommands_CommentListSinceIsCacheable(t *testing.T) {
t0 := time.Date(2026, 11, 7, 19, 5, 13, 0, time.UTC)
srv := &etagListServer{t: t, watch: "/repos/org/repo/issues/7/comments", bodies: map[string]any{
"/repos/org/repo/issues": []*gh.Issue{{
Number: gh.Ptr(7),
Comments: gh.Ptr(1),
UpdatedAt: &gh.Timestamp{Time: t0.Add(-time.Hour)},
PullRequestLinks: &gh.PullRequestLinks{URL: gh.Ptr("pr")},
}},
"/repos/org/repo/issues/7/comments": []*gh.IssueComment{{
ID: gh.Ptr(int64(1)),
User: &gh.User{Login: gh.Ptr("owner")},
Body: gh.Ptr("looks good"),
CreatedAt: &gh.Timestamp{Time: t0.Add(-2 * time.Hour)},
}},
}}
ts := httptest.NewServer(srv)
defer ts.Close()
client := newTestClient(t, ts, "org", []string{"repo"})
client.SetAppBotLogin("hive[bot]")

for _, now := range []time.Time{t0, t0.Add(40 * time.Minute), t0.Add(time.Hour)} {
opts := IssueUnparkSweepOptions{
Now: func() time.Time { return now },
PRApproveReply: func(context.Context, string, int, *gh.IssueComment) (string, error) {
t.Fatal("no /hive approve command was posted")
return "", nil
},
}
if _, err := client.SweepIssueUnparkCommands(context.Background(), opts); err != nil {
t.Fatalf("sweep at %v: %v", now, err)
}
}
assertCacheablePasses(t, srv.requests(), "2026-10-31T19:00:00Z", "2026-10-31T20:00:00Z")
}

func TestSweepClosedIssueStageLabels_ListSinceIsCacheable(t *testing.T) {
srv := &etagListServer{t: t, watch: "/repos/org/repo/issues", bodies: map[string]any{
"/repos/org/repo/issues": []*gh.Issue{},
}}
ts := httptest.NewServer(srv)
defer ts.Close()
client := newTestClient(t, ts, "org", []string{"repo"})

// The scheduler passes lastRun-10m, which moves on every 10-minute pass.
for _, since := range []time.Time{
time.Date(2026, 11, 7, 19, 5, 13, 0, time.UTC),
time.Date(2026, 11, 7, 19, 15, 13, 0, time.UTC),
time.Date(2026, 11, 7, 20, 5, 13, 0, time.UTC),
} {
if _, err := client.SweepClosedIssueStageLabels(context.Background(), ClosedIssueStageSweepOptions{Since: since}); err != nil {
t.Fatalf("sweep since %v: %v", since, err)
}
}
assertCacheablePasses(t, srv.requests(), "2026-11-07T19:00:00Z", "2026-11-07T20:00:00Z")
}
Loading
Loading