diff --git a/changelog.d/fixed-cacheable-since-reads.md b/changelog.d/fixed-cacheable-since-reads.md new file mode 100644 index 0000000000..bf63509c7d --- /dev/null +++ b/changelog.d/fixed-cacheable-since-reads.md @@ -0,0 +1 @@ +- The hive's routine "everything since a time" GitHub reads can now be answered with free `304`s. The un-park sweep's comment reads, the closed-issue stage sweep, the review backlog split, the shared-CI rerun sweep and the same-component check built their `since=` from the current clock, so the URL changed on every pass and the ETag cache (keyed on the full URL) never matched. On a hosted hive with 20 repos on 2026-11-07, untagged issue-comment lists were answered `304` only 51% of the time and issue lists only 28%, against 83% to 91% for the pull-request lists, while the hive needed about 8,250 calls an hour against a 7,000 limit. The `since` sent to GitHub is now rounded down to the hour; each sweep still filters on its exact cutoff, so nothing is missed. The calls the periodic sweeps, the governor cycle, the state cache, the metrics collector, the request watchers, the mention poller and dashboard requests make are now named in `/api/gh-rate-limits` (`hive:`) instead of all reading as `hive`. diff --git a/src/cmd/hive/forgestate_boot.go b/src/cmd/hive/forgestate_boot.go index cc8aaeb599..ebe693b043 100644 --- a/src/cmd/hive/forgestate_boot.go +++ b/src/cmd/hive/forgestate_boot.go @@ -53,7 +53,7 @@ func (b *boot) applyGitHubClientForgeState(client *github.Client) { ctx = context.Background() } go func() { - if err := store.Run(ctx); err != nil && !errors.Is(err, context.Canceled) { + if err := store.Run(github.WithRESTCaller(ctx, "hive:forgestate")); err != nil && !errors.Is(err, context.Canceled) { b.logger.Error("forgestate: reconcile loop stopped", "error", err) } }() diff --git a/src/cmd/hive/main_eval_cycle.go b/src/cmd/hive/main_eval_cycle.go index 5c1e3540b3..7e1276c957 100644 --- a/src/cmd/hive/main_eval_cycle.go +++ b/src/cmd/hive/main_eval_cycle.go @@ -550,6 +550,9 @@ func runEvalCycle( attribute.String("hive.id", cfg.HiveID), attribute.Int(tracing.AttrHiveACMMLevel, inferACMMLevel(cfg))) defer span.End() + // GitHub calls this cycle makes are "hive:eval_cycle" in + // /api/gh-rate-limits unless a step below names itself more narrowly. + ctx = github.WithRESTCaller(ctx, "hive:eval_cycle") // A hive running without GitHub credentials (placeholder app_id, or a real // App whose key could not be read) has nothing to enumerate. Return before @@ -611,7 +614,7 @@ func runEvalCycle( "stale", actionable.Stale()) } else { enumCtx, enumSpan := tracing.StartSpan(ctx, "governor.enumerate_actionable") - fresh, err := ghClient.EnumerateActionable(enumCtx) + fresh, err := ghClient.EnumerateActionable(github.WithRESTCaller(enumCtx, "hive:enumerate_actionable")) enumSpan.End() var ok bool actionable, ok = actionableAfterGitHubEnumerate(cfg, fresh, err, logger) @@ -691,12 +694,12 @@ func runEvalCycle( // another. Backed by a PVC ledger so it survives those restarts, and fails // closed (keeps the last known claims) when the GitHub API is unavailable. if !budgetDecision.SkipOptional { - applyDuplicatePRGuard(ctx, cfg, ghClient, actionable, logger) + applyDuplicatePRGuard(github.WithRESTCaller(ctx, "hive:duplicate_pr_guard"), cfg, ghClient, actionable, logger) } // Every cycle, cached ones included: heal issue stage records the // claim ledger says an open PR covers but the store still holds below // 040 (a claim release written over the cover between reconcile passes). - reconcileLedgerCoverRecords(ctx, cfg, ghClient, actionable, logger) + reconcileLedgerCoverRecords(github.WithRESTCaller(ctx, "hive:ledger_cover"), cfg, ghClient, actionable, logger) // PR review queue (#9590): stamp each PR's rank, priority and reasons // onto the snapshot so last-actionable.json carries them next to @@ -731,7 +734,7 @@ func runEvalCycle( escalatedPRs := map[string]bool{} if !budgetDecision.SkipUnstick { - escalatedPRs = runEscalationSweep(ctx, cfg, governorForge(cfg, ghClient, logger), actionable, notifier, dashSrv, logger) + escalatedPRs = runEscalationSweep(github.WithRESTCaller(ctx, "hive:escalation_sweep"), cfg, governorForge(cfg, ghClient, logger), actionable, notifier, dashSrv, logger) runDependencyBotConflictSweep(ctx, cfg, governorForge(cfg, ghClient, logger), ghClient, actionable, logger) } runWaitEscalationSweep(cfg, dashSrv, logger) @@ -758,7 +761,7 @@ func runEvalCycle( } return cfg.SelfAuthorizationHoldEnabledForRepoAtLevel(repo, level) } - holdDriftPRs := enforceHoldGuard(ctx, cfg, ghClient, governorForge(cfg, ghClient, logger), actionable, logger, selfAuthorizationHoldEnabled) + holdDriftPRs := enforceHoldGuard(github.WithRESTCaller(ctx, "hive:hold_guard"), cfg, ghClient, governorForge(cfg, ghClient, logger), actionable, logger, selfAuthorizationHoldEnabled) // The per-PR verdicts come back so the dashboard's PR pills can be // painted from the sweep's own classification rather than a looser @@ -785,7 +788,7 @@ func runEvalCycle( // PR this hive opened back into the CLI session that authored it, while // that session is still live. Anything it cannot resume stays on the // fix-before-new path above, unchanged. - routePRFollowUps(ctx, cfg, ghClient, actionable, escalatedPRs, agentMgr, logger) + routePRFollowUps(github.WithRESTCaller(ctx, "hive:pr_follow_ups"), cfg, ghClient, actionable, escalatedPRs, agentMgr, logger) // Stuck-PR reaper (backstop): DELIVER a targeted FIX-BEFORE-NEW kick for any // hive-authored PR that is red on a required check AND stale (its red head @@ -860,8 +863,8 @@ func runEvalCycle( "skipped", shaResult.Skipped, ) } - runHardSuppressReconcile(ctx, cfg, ghClient, logger) - runParkRecheck(ctx, cfg, ghClient, logger) + runHardSuppressReconcile(github.WithRESTCaller(ctx, "hive:hard_suppress_reconcile"), cfg, ghClient, logger) + runParkRecheck(github.WithRESTCaller(ctx, "hive:park_recheck"), cfg, ghClient, logger) } // Refresh budget spend from lifetime token totals before Evaluate so diff --git a/src/cmd/hive/sweep_workers.go b/src/cmd/hive/sweep_workers.go index c6027f41dc..2928a06633 100644 --- a/src/cmd/hive/sweep_workers.go +++ b/src/cmd/hive/sweep_workers.go @@ -42,6 +42,14 @@ type sweepJob struct { run func(context.Context, *github.Client, *config.Config) } +// runTagged runs the job with its GitHub REST calls attributed to +// "hive:" in /api/gh-rate-limits. Calls a sweep already tags more +// narrowly keep their own name; before this, every untagged call from every +// sweep was reported as plain "hive". +func (j sweepJob) runTagged(ctx context.Context, gh *github.Client, cfg *config.Config) { + j.run(github.WithRESTCaller(ctx, "hive:"+j.name), gh, cfg) +} + // sweepStartupSpread is the window the non-essential sweeps' first passes are // spread across after a start. All workers start in the same second; on a // cold ETag cache their first passes each listed every repo at once, all @@ -214,7 +222,7 @@ func (b *boot) startSweepWorkers() func() { if skipped { return } - job.run(ctx, gh, cfg) + job.runTagged(ctx, gh, cfg) if b.dashSrv != nil { b.dashSrv.RecordSweepCadence(job.name, started, job.interval(cfg), true, false) } diff --git a/src/cmd/hive/sweep_workers_test.go b/src/cmd/hive/sweep_workers_test.go index 21b3837319..a3a546dc87 100644 --- a/src/cmd/hive/sweep_workers_test.go +++ b/src/cmd/hive/sweep_workers_test.go @@ -2,6 +2,8 @@ package main import ( "context" + "net/http" + "net/http/httptest" "sync/atomic" "testing" "time" @@ -249,3 +251,34 @@ func TestSweepFirstDelaysSpreadNonEssential(t *testing.T) { prev = d } } + +// A sweep worker's GitHub calls are attributed to "hive:" in REST +// accounting (/api/gh-rate-limits), not the undifferentiated "hive" that hid +// ~4,400 calls an hour on the knuckle hive. The timeline read stands in for +// any untagged call a sweep makes. +func TestSweepJobRunTaggedAttributesRESTCalls(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != "/repos/testorg/widget/issues/1/timeline" { + t.Errorf("unexpected operation: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`[]`)) + })) + defer srv.Close() + ghClient := github.NewClientForTest(srv.URL, "testorg", []string{"widget"}, sweepTestLogger(nil)) + job := sweepJob{name: "p34_caller_probe", run: func(ctx context.Context, gh *github.Client, _ *config.Config) { + if _, err := gh.IssueTimelineSince(ctx, "widget", 1, time.Time{}); err != nil { + t.Errorf("timeline read: %v", err) + } + }} + job.runTagged(context.Background(), ghClient, &config.Config{}) + + for _, c := range github.RESTTopConsumers(1000) { + if c.Caller == "hive:p34_caller_probe" && c.Endpoint == "/repos/{owner}/{repo}/issues/{number}/timeline" && c.Requests >= 1 { + return + } + } + t.Fatalf("REST accounting has no hive:p34_caller_probe timeline row: %+v", github.RESTTopConsumers(1000)) +} diff --git a/src/docs/design/agent-turn-model.md b/src/docs/design/agent-turn-model.md index 1ab9cb1d53..be699369d7 100644 --- a/src/docs/design/agent-turn-model.md +++ b/src/docs/design/agent-turn-model.md @@ -139,9 +139,9 @@ Kick *timing* lives in `pkg/governor`; kick *text* is built in `pkg/scheduler`. `deps.newTicker(initialEvalInterval)` at `src/cmd/hive/main.go:5772`, loop at `src/cmd/hive/main.go:6112`, evaluation via `deps.runEval` at `src/cmd/hive/main.go:5956` (`runEvalCycle`, `src/cmd/hive/main_eval_cycle.go:471`), - message assembly via `sched.BuildKickMessagesWithSkips` at `src/cmd/hive/main_eval_cycle.go:964` + message assembly via `sched.BuildKickMessagesWithSkips` at `src/cmd/hive/main_eval_cycle.go:1094` (`src/pkg/scheduler/kickmessage.go:261`), and delivery via - `agentMgr.SendKick` wired into `dispatchAgentKicks` at `src/cmd/hive/main_eval_cycle.go:1127`. + `agentMgr.SendKick` wired into `dispatchAgentKicks` at `src/cmd/hive/main_eval_cycle.go:1580`. This matters for the RFC: the scheduler is already **stateless with respect to turns**. It does not hold a continuation, does not await turn *N* before diff --git a/src/docs/operator-reference/sweeps-and-operations.md b/src/docs/operator-reference/sweeps-and-operations.md index 454a779286..571c88a2d8 100644 --- a/src/docs/operator-reference/sweeps-and-operations.md +++ b/src/docs/operator-reference/sweeps-and-operations.md @@ -907,6 +907,10 @@ Mode exits use hysteresis: Hive leaves conserve/critical only after quota rises **Historical PR accounting waits for a normal budget.** The worker that reads each recently merged or closed PR's merger and reviews (for the throughput and review counts) runs only in `normal` mode. In `conserve` and `critical` it makes no GitHub calls; its queue is kept and drained once the budget is back to `normal`. The hive logs `historical PR enrichment paused for GitHub API budget` once when it pauses and `historical PR enrichment resumed` once when it starts again. This matters most after a restart: the 72-hour settle replay queues every recent merged or closed PR again, at about two calls each. +**Sweeps that read "everything since a time" round that time down to the hour.** A free `304` answer needs the request URL to match the one the hive fetched before, query string included. The un-park sweep's comment reads (comments from the last seven days), the closed-issue stage sweep and the review backlog split (issues closed since the last pass), the shared-CI rerun sweep (incidents closed in its lookback) and the same-component check when an agent files an issue all used to build that time from the current clock, so the URL changed on every pass and every pass was charged in full. Now the time sent to GitHub is rounded down to the hour, so every pass in the same hour asks for the same URL. The window only gets wider, by less than an hour, and each sweep still filters on its exact cutoff, so nothing is missed and no old command is answered again. + +**`/api/gh-rate-limits` names the sweep behind each call.** In `top_consumers`, the calls a periodic sweep makes are listed as `hive:` (for example `hive:issue_unpark` or `hive:closed_issue_stage`), the governor cycle's calls as `hive:eval_cycle` (its issue listing as `hive:enumerate_actionable`), the state cache's as `hive:forgestate`, the dashboard metrics as `hive:metrics_collector`, and the calls made while serving a dashboard request as `hive:dashboard_api`. A call that a sweep already tags more narrowly keeps that name. Plain `hive` is now only what none of these cover. + - Recovery from a provider spending limit is automatic, via a probe. Withholding kicks also withholds the inference calls that would reveal the provider is serving again, so the hive suppresses only while the last refusal is recent and then lets a single kick through to test the gateway; the probe re-arms suppression the moment it is released, so at most one probe run flies per interval. A still-clipped key refuses the probe and suppression resumes for another interval; once the provider's window resets the probe succeeds, normal kicking resumes with no operator action, and a one-time recovery notification is sent (the entering notification is likewise sent once per clip, not once per cycle). Tune with `governor.provider_budget.probe_interval_s` (default 1800 — 30 minutes): ```yaml diff --git a/src/pkg/dashboard/metrics_collector.go b/src/pkg/dashboard/metrics_collector.go index 0d7607ef83..9a834b9f3a 100644 --- a/src/pkg/dashboard/metrics_collector.go +++ b/src/pkg/dashboard/metrics_collector.go @@ -322,6 +322,7 @@ func (mc *MetricsCollector) collect(ctx context.Context) { // Dashboard metrics are housekeeping against the pacer: they stand down // for the governor's enumeration while a caution window is engaged. ctx = ghpkg.WithPacingClass(ctx, ghpkg.PacingLow) + ctx = ghpkg.WithRESTCaller(ctx, "hive:metrics_collector") metrics := make(map[string]any) outreach := mc.collectOutreach(ctx) diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index cd1f3e7d1e..f09df0f037 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -2838,7 +2838,16 @@ async function poll(interval,flowId){ ` func (s *Server) Handler() http.Handler { - return s.withRequestLog(s.noIndex(s.authenticatedMux())) + return s.withRequestLog(s.noIndex(withDashboardRESTCaller(s.authenticatedMux()))) +} + +// withDashboardRESTCaller attributes the GitHub REST calls a dashboard request +// makes to "hive:dashboard_api" in /api/gh-rate-limits instead of plain +// "hive". A handler that tags its calls more narrowly keeps its own name. +func withDashboardRESTCaller(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + next.ServeHTTP(w, r.WithContext(github.WithRESTCaller(r.Context(), "hive:dashboard_api"))) + }) } func (s *Server) roleEnforcement(next http.Handler) http.Handler { diff --git a/src/pkg/github/cacheable_since_test.go b/src/pkg/github/cacheable_since_test.go new file mode 100644 index 0000000000..226937f893 --- /dev/null +++ b/src/pkg/github/cacheable_since_test.go @@ -0,0 +1,170 @@ +package github + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + gh "github.com/google/go-github/v72/github" +) + +// These tests pin the ETag fix for moving since= cutoffs: a sweep that built +// its since from time.Now() changed the URL on every pass, so the ETag cache +// (keyed on the full URL) never matched and every pass was charged. Rounded +// down to the hour, every pass in that hour asks for the same URL and the +// repeat gets a free 304. + +func TestCacheableSince(t *testing.T) { + if got := cacheableSince(time.Time{}); !got.IsZero() { + t.Fatalf("zero since = %v, want zero (no filter)", got) + } + east := time.FixedZone("UTC+5:30", 5*3600+1800) + cases := []struct { + in, want time.Time + }{ + {time.Date(2026, 11, 7, 19, 42, 17, 500, time.UTC), time.Date(2026, 11, 7, 19, 0, 0, 0, time.UTC)}, + {time.Date(2026, 11, 7, 19, 0, 0, 0, time.UTC), time.Date(2026, 11, 7, 19, 0, 0, 0, time.UTC)}, + {time.Date(2026, 11, 7, 19, 59, 59, 999, time.UTC), time.Date(2026, 11, 7, 19, 0, 0, 0, time.UTC)}, + // A half-hour zone still rounds to the UTC hour, so two hives in + // different zones ask for the same URL. + {time.Date(2026, 11, 8, 1, 12, 0, 0, east), time.Date(2026, 11, 7, 19, 0, 0, 0, time.UTC)}, + } + for _, tc := range cases { + got := cacheableSince(tc.in) + if !got.Equal(tc.want) || got.Location() != time.UTC { + t.Errorf("cacheableSince(%v) = %v, want %v in UTC", tc.in, got, tc.want) + } + if got.After(tc.in) || tc.in.Sub(got) >= time.Hour { + t.Errorf("cacheableSince(%v) = %v: must be earlier by less than an hour", tc.in, got) + } + } +} + +// etagListServer serves fixed JSON bodies by path with an ETag, and 304 when +// the client presents it. It records the since= and status of every request +// to the paths it watches. +type etagListServer struct { + t *testing.T + bodies map[string]any + watch string + + mu sync.Mutex + seen []etagListRequest +} + +type etagListRequest struct { + since string + status int +} + +func (s *etagListServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + body, ok := s.bodies[r.URL.Path] + if r.Method != http.MethodGet || !ok { + s.t.Errorf("unexpected operation: %s %s", r.Method, r.URL.String()) + w.WriteHeader(http.StatusNotFound) + return + } + etag := `W/"` + r.URL.Path + `"` + status := http.StatusOK + if r.Header.Get("If-None-Match") == etag { + status = http.StatusNotModified + } + if r.URL.Path == s.watch { + s.mu.Lock() + s.seen = append(s.seen, etagListRequest{since: r.URL.Query().Get("since"), status: status}) + s.mu.Unlock() + } + w.Header().Set("ETag", etag) + if status == http.StatusNotModified { + w.WriteHeader(status) + return + } + w.Header().Set("Content-Type", "application/json; charset=utf-8") + _ = json.NewEncoder(w).Encode(body) +} + +func (s *etagListServer) requests() []etagListRequest { + s.mu.Lock() + defer s.mu.Unlock() + return append([]etagListRequest(nil), s.seen...) +} + +// assertCacheablePasses checks three passes: the second (same hour) asks for +// the first's exact URL and is answered 304; the third (next hour) moves +// since forward to that hour. +func assertCacheablePasses(t *testing.T, got []etagListRequest, sameHour, nextHour string) { + t.Helper() + if len(got) != 3 { + t.Fatalf("watched list requests = %+v, want one per pass (3)", got) + } + if got[0].since != sameHour || got[0].status != http.StatusOK { + t.Errorf("pass 1 = %+v, want since=%s answered 200", got[0], sameHour) + } + if got[1].since != got[0].since || got[1].status != http.StatusNotModified { + t.Errorf("pass 2 in the same hour = %+v, want since=%s (identical URL) answered 304", got[1], got[0].since) + } + if got[2].since != nextHour || got[2].status != http.StatusOK { + t.Errorf("pass 3 after the hour = %+v, want since=%s answered 200", got[2], nextHour) + } +} + +func TestSweepIssueUnparkCommands_CommentListSinceIsCacheable(t *testing.T) { + t0 := time.Date(2026, 11, 7, 19, 5, 13, 0, time.UTC) + srv := &etagListServer{t: t, watch: "/repos/org/repo/issues/7/comments", bodies: map[string]any{ + "/repos/org/repo/issues": []*gh.Issue{{ + Number: gh.Ptr(7), + Comments: gh.Ptr(1), + UpdatedAt: &gh.Timestamp{Time: t0.Add(-time.Hour)}, + PullRequestLinks: &gh.PullRequestLinks{URL: gh.Ptr("pr")}, + }}, + "/repos/org/repo/issues/7/comments": []*gh.IssueComment{{ + ID: gh.Ptr(int64(1)), + User: &gh.User{Login: gh.Ptr("owner")}, + Body: gh.Ptr("looks good"), + CreatedAt: &gh.Timestamp{Time: t0.Add(-2 * time.Hour)}, + }}, + }} + ts := httptest.NewServer(srv) + defer ts.Close() + client := newTestClient(t, ts, "org", []string{"repo"}) + client.SetAppBotLogin("hive[bot]") + + for _, now := range []time.Time{t0, t0.Add(40 * time.Minute), t0.Add(time.Hour)} { + opts := IssueUnparkSweepOptions{ + Now: func() time.Time { return now }, + PRApproveReply: func(context.Context, string, int, *gh.IssueComment) (string, error) { + t.Fatal("no /hive approve command was posted") + return "", nil + }, + } + if _, err := client.SweepIssueUnparkCommands(context.Background(), opts); err != nil { + t.Fatalf("sweep at %v: %v", now, err) + } + } + assertCacheablePasses(t, srv.requests(), "2026-10-31T19:00:00Z", "2026-10-31T20:00:00Z") +} + +func TestSweepClosedIssueStageLabels_ListSinceIsCacheable(t *testing.T) { + srv := &etagListServer{t: t, watch: "/repos/org/repo/issues", bodies: map[string]any{ + "/repos/org/repo/issues": []*gh.Issue{}, + }} + ts := httptest.NewServer(srv) + defer ts.Close() + client := newTestClient(t, ts, "org", []string{"repo"}) + + // The scheduler passes lastRun-10m, which moves on every 10-minute pass. + for _, since := range []time.Time{ + time.Date(2026, 11, 7, 19, 5, 13, 0, time.UTC), + time.Date(2026, 11, 7, 19, 15, 13, 0, time.UTC), + time.Date(2026, 11, 7, 20, 5, 13, 0, time.UTC), + } { + if _, err := client.SweepClosedIssueStageLabels(context.Background(), ClosedIssueStageSweepOptions{Since: since}); err != nil { + t.Fatalf("sweep since %v: %v", since, err) + } + } + assertCacheablePasses(t, srv.requests(), "2026-11-07T19:00:00Z", "2026-11-07T20:00:00Z") +} diff --git a/src/pkg/github/closed_issue_stage_sweep.go b/src/pkg/github/closed_issue_stage_sweep.go index 32cebd7240..e48eeddc1b 100644 --- a/src/pkg/github/closed_issue_stage_sweep.go +++ b/src/pkg/github/closed_issue_stage_sweep.go @@ -121,10 +121,13 @@ func (c *Client) SweepClosedIssueStageLabels(ctx context.Context, opts ClosedIss // listClosedIssuesSince is one page per 100 closed issues updated after // since, newest activity first — the recent closes this sweep exists for. +// The since sent is rounded down to the hour (cacheableSince): both callers +// pass a since that moves on every pass, and a wider window only re-reads +// closes they already handled idempotently. func (c *Client) listClosedIssuesSince(ctx context.Context, owner, repo string, since time.Time) ([]*gh.Issue, error) { opts := &gh.IssueListByRepoOptions{ State: "closed", - Since: since, + Since: cacheableSince(since), Sort: "updated", Direction: "desc", ListOptions: gh.ListOptions{PerPage: 100}, diff --git a/src/pkg/github/etagcache.go b/src/pkg/github/etagcache.go index 722c617e67..4ac26edbb6 100644 --- a/src/pkg/github/etagcache.go +++ b/src/pkg/github/etagcache.go @@ -173,6 +173,30 @@ func etagCacheKey(req *http.Request) string { return hex.EncodeToString(sum[:8]) + " " + req.Header.Get("Accept") + " " + req.URL.String() } +// cacheableSinceGranularity is how far down a list query's since is rounded. +const cacheableSinceGranularity = time.Hour + +// cacheableSince rounds the since of a GitHub list query down to the hour (in +// UTC), so every pass inside that hour asks for the same URL. +// +// The ETag cache above is keyed on the full URL, query included. A sweep that +// builds its since from time.Now() on every pass ("comments updated in the +// last 7 days") changes the URL on every call, so the cache never matches and +// every pass is charged in full. In 50 minutes on the knuckle hive +// (2026-11-07) the hive's own issue-comment lists were charged 1357 times in +// 2776 and its issue lists 1012 times in 1399, while its pull-request lists +// and review reads came back as free 304s 83% and 91% of the time. +// +// Rounding only ever moves since earlier, by less than an hour, so the list +// is a superset of what was asked for and nothing is missed. Callers keep +// filtering on their exact cutoff. A zero since (no filter) stays zero. +func cacheableSince(since time.Time) time.Time { + if since.IsZero() { + return since + } + return since.UTC().Truncate(cacheableSinceGranularity) +} + // etagCacheable reports whether the request may be served conditionally: a // plain GET with no conditional or range semantics of its own. func etagCacheable(req *http.Request) bool { diff --git a/src/pkg/github/issue_request_watcher.go b/src/pkg/github/issue_request_watcher.go index d38fd2b537..85795ba95c 100644 --- a/src/pkg/github/issue_request_watcher.go +++ b/src/pkg/github/issue_request_watcher.go @@ -290,7 +290,7 @@ func (c *Client) StartIssueRequestWatcher(ctx context.Context, authz IssueReques if ctx.Err() != nil { return } - c.processIssueRequestsInDir(ctx, nowFn, dir) + c.processIssueRequestsInDir(WithRESTCaller(ctx, "hive:issue_request_watcher"), nowFn, dir) } } }() @@ -2284,7 +2284,7 @@ func (c *Client) countClosedSiblingsBy(ctx context.Context, owner, repo, botLogi Creator: botLogin, Sort: "updated", Direction: "desc", - Since: cutoff, + Since: cacheableSince(cutoff), // the closed_at filter below keeps the exact cutoff ListOptions: gh.ListOptions{PerPage: 100}, } for page := 1; page <= sameComponentClosedMaxPages; page++ { diff --git a/src/pkg/github/issue_unpark_command.go b/src/pkg/github/issue_unpark_command.go index c0a9e106e9..5701cc1f49 100644 --- a/src/pkg/github/issue_unpark_command.go +++ b/src/pkg/github/issue_unpark_command.go @@ -204,6 +204,8 @@ type IssueUnparkSweepOptions struct { // ClosedAudit, when set, is called once per parked issue a maintainer's // `/fixed` closed (hivecommons/dibs#459). ClosedAudit func(IssueUnparkEvent) + // Now overrides the clock in tests. + Now func() time.Time } // PullRequestDecisionEvent records one `/hive decision` on a hive-authored @@ -315,7 +317,14 @@ func (c *Client) SweepIssueUnparkCommands(ctx context.Context, opts IssueUnparkS if opts.HumanChangeRequests != nil { c.ReconcileSelfDecisionChangeRequests(opts.HumanChangeRequests) } - cutoff := time.Now().Add(-unparkCommandScanWindow) + now := time.Now() + if opts.Now != nil { + now = opts.Now() + } + // Exact cutoff for every eligibility check; the comment lists built from + // it round their since down to the hour so repeat passes stay cacheable + // (cacheableSince). + cutoff := now.Add(-unparkCommandScanWindow) for _, repo := range c.getRepos() { if result.actions() >= maxActions { diff --git a/src/pkg/github/mention_comments.go b/src/pkg/github/mention_comments.go index 45de58cfbe..c3746f3b80 100644 --- a/src/pkg/github/mention_comments.go +++ b/src/pkg/github/mention_comments.go @@ -19,6 +19,7 @@ func (c *Client) ListMentionComments(ctx context.Context, repo string, since tim if c == nil || c.client == nil { return nil, ErrNoGitHubClient } + ctx = WithRESTCaller(ctx, "hive:mention_poll") owner, repoName := c.splitRepo(repo) var out []mention.Event issueComments, err := c.listMentionIssueComments(ctx, owner, repoName, since) diff --git a/src/pkg/github/pr_approve_command_test.go b/src/pkg/github/pr_approve_command_test.go index 396857b84d..e491d86036 100644 --- a/src/pkg/github/pr_approve_command_test.go +++ b/src/pkg/github/pr_approve_command_test.go @@ -38,8 +38,9 @@ func TestSweepPRApproveReplies(t *testing.T) { json.NewEncoder(w).Encode([]*gh.Issue{{Number: gh.Ptr(7), Comments: gh.Ptr(len(comments)), UpdatedAt: &gh.Timestamp{Time: now}, PullRequestLinks: &gh.PullRequestLinks{URL: gh.Ptr("pr")}}}) case r.Method == http.MethodGet && r.URL.Path == "/repos/org/repo/issues/7/comments": commentReads++ - // Only comments updated since the cutoff are requested. - if since, err := time.Parse(time.RFC3339, r.URL.Query().Get("since")); err != nil || since.Before(now.Add(-7*24*time.Hour-time.Minute)) || since.After(now.Add(-7*24*time.Hour+time.Minute)) { + // Only comments updated since the cutoff, rounded down to the + // hour so repeat passes stay cacheable, are requested. + if since, err := time.Parse(time.RFC3339, r.URL.Query().Get("since")); err != nil || since.Before(now.Add(-7*24*time.Hour-time.Hour)) || since.After(now.Add(-7*24*time.Hour)) || !since.Equal(since.Truncate(time.Hour)) { t.Errorf("since=%q err=%v", r.URL.Query().Get("since"), err) } if listFailed { diff --git a/src/pkg/github/pr_level_hold.go b/src/pkg/github/pr_level_hold.go index 09b601485d..c79c6f00b6 100644 --- a/src/pkg/github/pr_level_hold.go +++ b/src/pkg/github/pr_level_hold.go @@ -325,11 +325,13 @@ func (c *Client) listIssueComments(ctx context.Context, owner, repo string, numb } // listIssueCommentsSince lists comments updated at or after since; a zero -// since lists every comment. +// since lists every comment. The since sent is rounded down to the hour +// (cacheableSince), so callers may get a few older comments and must keep +// filtering on their own cutoff. func (c *Client) listIssueCommentsSince(ctx context.Context, owner, repo string, number int, since time.Time) ([]*gh.IssueComment, error) { opts := &gh.IssueListCommentsOptions{ListOptions: gh.ListOptions{PerPage: 100}} if !since.IsZero() { - opts.Since = gh.Ptr(since) + opts.Since = gh.Ptr(cacheableSince(since)) } var all []*gh.IssueComment for { diff --git a/src/pkg/github/pr_request_watcher.go b/src/pkg/github/pr_request_watcher.go index adf5e4d40e..9147ce45f8 100644 --- a/src/pkg/github/pr_request_watcher.go +++ b/src/pkg/github/pr_request_watcher.go @@ -300,7 +300,7 @@ func (c *Client) StartPRRequestWatcher(ctx context.Context, authz PRRequestAutho if ctx.Err() != nil { return } - c.processPRRequestsInDir(ctx, nowFn, dir) + c.processPRRequestsInDir(WithRESTCaller(ctx, "hive:pr_request_watcher"), nowFn, dir) // Same goroutine as the open path, so the two never rewrite // one branch concurrently (#9364). Throttled internally and // a no-op unless github.app_signed_commits is on. diff --git a/src/pkg/github/shared_ci_rerun.go b/src/pkg/github/shared_ci_rerun.go index 27b71c0e95..113e5d0c7c 100644 --- a/src/pkg/github/shared_ci_rerun.go +++ b/src/pkg/github/shared_ci_rerun.go @@ -178,7 +178,7 @@ func (c *Client) recentlyClosedSharedCIIncidents(ctx context.Context, owner, rep State: "closed", Sort: "updated", Direction: "desc", - Since: since, + Since: cacheableSince(since), // the closed_at filter below keeps the exact since ListOptions: gh.ListOptions{PerPage: 100}, } out := map[int]bool{}