From 748175cab1b9c717f83b6ccc785d339227e2a02c Mon Sep 17 00:00:00 2001 From: Danathar Date: Thu, 29 Oct 2026 06:25:40 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20fix(automerge):=20cache=20contri?= =?UTF-8?q?butor=20merged-repo=20search=20across=20sweeps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #15125 memoized contributorMergedRepos within one sweep and skipped bot authors. A human author outside the registry was still searched again on every sweep, 3-4 search/issues calls per author on a 19-repo hive, even after they had passed auto-promotion. The per-author answer (which active repositories hold a merge by them) is now kept on the Engine for an hour, keyed by the active-repository set. A failed search is kept for 15 minutes, so a secondary rate limit is not re-tripped every sweep; the PR's own repository is counted alone meanwhile, as before. The per-sweep context cache is subsumed and removed. Only the list is cached. The PR's own repository and every listed one are still walked fresh each sweep for clean merges and revert/sentinel evidence, so a stale list can only leave out a repository the author first merged into within the hour. Refs #15088 Signed-off-by: Danathar Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BQYnPGFJx81c8JXR3o1sg1 --- .../fixed-15088-promotion-search-ttl.md | 1 + src/docs/scale-envelope.md | 2 +- .../automerge/automerge_contributor_lane.go | 100 ++++++++---- .../automerge_contributor_lane_test.go | 142 ++++++++++++++++-- src/pkg/github/automerge/automerge_sweep.go | 4 + 5 files changed, 203 insertions(+), 46 deletions(-) create mode 100644 changelog.d/fixed-15088-promotion-search-ttl.md diff --git a/changelog.d/fixed-15088-promotion-search-ttl.md b/changelog.d/fixed-15088-promotion-search-ttl.md new file mode 100644 index 0000000000..a33cb04d7a --- /dev/null +++ b/changelog.d/fixed-15088-promotion-search-ttl.md @@ -0,0 +1 @@ +- Contributor auto-promotion now remembers each author's merged-repository search for an hour (a failed search for 15 minutes) instead of searching again on every sweep ([#15088](https://github.com/hivecommons/hive/issues/15088)). diff --git a/src/docs/scale-envelope.md b/src/docs/scale-envelope.md index 8d0611a278..237493847b 100644 --- a/src/docs/scale-envelope.md +++ b/src/docs/scale-envelope.md @@ -134,7 +134,7 @@ operator can change it without rebuilding. | CI-evidence excerpt per PR | `redPRFixExcerptRunes`, `src/pkg/scheduler/policy_overlays.go:176` | 400 runes | **No** | Truncate with "…" | | Skill text injected per kick | `maxSkillsInjectionBytes`, `src/pkg/scheduler/kick_priming.go:138` | 8192 B | **No** | Shed load: skills are **dropped whole**, never truncated mid-body, and the drop is logged | | Issues primed into `AGENTS.md` context | `maxIssuesToPrime`, `src/pkg/scheduler/kick_priming.go:13` | 5 | **No** | Truncate | -| Auto-merge sweep merges per pass | `DefaultAutoMergeSweepMaxMerges`, `src/pkg/github/automerge/automerge_sweep.go:375` | 3 | **Yes** (`auto_merge.max_merges`) | Shed load: remaining merges wait for the next pass | +| Auto-merge sweep merges per pass | `DefaultAutoMergeSweepMaxMerges`, `src/pkg/github/automerge/automerge_sweep.go:505` | 3 | **Yes** (`auto_merge.max_merges`) | Shed load: remaining merges wait for the next pass | | Task-list sweep closures per tick | `DefaultTaskListSweepMaxCloses`, `src/pkg/github/task_list_sweep.go:20` | 5 | Partly (`MaxCloses` option) | Shed load: remaining closures wait for the next tick | | Prompt-history on disk | `promptHistoryMaxSizeMB` × backups, `src/pkg/dashboard/prompt_history.go:67` | 96 MiB worst case | **No** | Shed oldest: rotate + gzip, oldest prompts age out first | | Issue/PR enumeration | `fetchIssues` / `fetchPRs`, `src/pkg/github/client.go:1653`, `src/pkg/github/client.go:1920` | **no cap** | n/a | Pages to completion; cost grows with backlog | diff --git a/src/pkg/github/automerge/automerge_contributor_lane.go b/src/pkg/github/automerge/automerge_contributor_lane.go index b636c7a65f..6bfd918e2d 100644 --- a/src/pkg/github/automerge/automerge_contributor_lane.go +++ b/src/pkg/github/automerge/automerge_contributor_lane.go @@ -398,7 +398,6 @@ func (c *Engine) SweepContributorLaneAutoMerges(ctx context.Context, opts AutoMe } ctx = contextWithExpectedCheckCache(ctx, newExpectedCheckCache()) ctx = contextWithLaneMergedBranches(ctx) - ctx = context.WithValue(ctx, contributorRepoSearchCacheKey{}, make(map[string][]contributorRepoRef)) for _, displayRepo := range c.activeRepos() { if len(result.Merged) >= maxMerges { break @@ -828,7 +827,35 @@ type contributorRepoRef struct { // contributorSearchQueryMaxLen is GitHub's limit on a search `q` parameter. const contributorSearchQueryMaxLen = 256 -type contributorRepoSearchCacheKey struct{} +const ( + // contributorMergedReposTTL is how long contributorMergedRepos trusts an + // author's discovered repository list across sweeps. Only the list is + // kept: every repository on it, and the PR's own, is still walked fresh + // each sweep for clean merges and revert/sentinel evidence, so a stale + // list can leave out a repository the author first merged into during + // the last hour. That repository's merges, and a revert of them, are + // then seen up to an hour late; nothing on the list goes unchecked. The + // walk already stops at the first `need` clean merges, so it never read + // every repository. + contributorMergedReposTTL = time.Hour + // contributorMergedReposFailureTTL is how long a failed search is + // remembered. The PR's own repository is counted alone meanwhile, as + // on the failing sweep, rather than re-tripping a secondary rate limit + // on every sweep (#15088). + contributorMergedReposFailureTTL = 15 * time.Minute + // contributorMergedReposCacheMax bounds the cache; past it the cache + // starts over. + contributorMergedReposCacheMax = 1024 +) + +// contributorMergedReposEntry is one author's search result: the active +// repositories (the PR's own included) that hold a merge by them. repos is +// the active-repository set the search covered; a different set is a miss. +type contributorMergedReposEntry struct { + refs []contributorRepoRef + repos string + expires time.Time +} // contributorMergedRepos returns the hive's other active repositories that // hold at least one merged PR by author, in the order the transport lists @@ -838,23 +865,28 @@ type contributorRepoSearchCacheKey struct{} // calls instead of ten closed-PR listings. A search failure degrades to the // PR's own repository with a warning rather than blocking the sweep: the // result is then exactly the pre-#14517 per-repo count. +// +// The answer covers every active repository and is kept on the Engine per +// author for contributorMergedReposTTL (a failure for +// contributorMergedReposFailureTTL), so an author with PRs in several +// repositories, or the same PR on every sweep, costs one search per TTL +// rather than one per PR per sweep (#15088). func (c *Engine) contributorMergedRepos(ctx context.Context, owner, repo, author string) []contributorRepoRef { if c == nil || c.gh == nil || c.transport == nil { return nil } own := strings.ToLower(owner + "/" + repo) - cache, _ := ctx.Value(contributorRepoSearchCacheKey{}).(map[string][]contributorRepoRef) - key := strings.ToLower(strings.TrimSpace(author)) - if refs, ok := cache[key]; ok { - var others []contributorRepoRef + others := func(refs []contributorRepoRef) []contributorRepoRef { + var out []contributorRepoRef for _, ref := range refs { if !strings.EqualFold(ref.Owner+"/"+ref.Repo, own) { - others = append(others, ref) + out = append(out, ref) } } - return others + return out } var candidates []contributorRepoRef + var names []string seen := map[string]bool{} for _, display := range c.activeRepos() { o, r := c.transport.SplitRepo(display) @@ -862,15 +894,33 @@ func (c *Engine) contributorMergedRepos(ctx context.Context, owner, repo, author continue } full := strings.ToLower(o + "/" + r) - if (cache == nil && full == own) || seen[full] { + if seen[full] { continue } seen[full] = true candidates = append(candidates, contributorRepoRef{Owner: o, Repo: r}) + names = append(names, full) } - if len(candidates) == 0 { + if len(others(candidates)) == 0 { return nil } + key := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(author), "@")) + repos := strings.Join(names, ",") + now := c.clock() + c.contributorMergedReposMu.Lock() + entry, ok := c.contributorMergedReposCache[key] + c.contributorMergedReposMu.Unlock() + if ok && entry.repos == repos && now.Before(entry.expires) { + return others(entry.refs) + } + remember := func(refs []contributorRepoRef, ttl time.Duration) { + c.contributorMergedReposMu.Lock() + defer c.contributorMergedReposMu.Unlock() + if c.contributorMergedReposCache == nil || len(c.contributorMergedReposCache) >= contributorMergedReposCacheMax { + c.contributorMergedReposCache = map[string]contributorMergedReposEntry{} + } + c.contributorMergedReposCache[key] = contributorMergedReposEntry{refs: refs, repos: repos, expires: now.Add(ttl)} + } base := "is:pr is:merged author:" + strings.TrimPrefix(strings.TrimSpace(author), "@") found := map[string]bool{} query := base @@ -893,25 +943,22 @@ func (c *Engine) contributorMergedRepos(ctx context.Context, owner, repo, author } return nil } + failed := func(err error) []contributorRepoRef { + c.warn("contributor-lane auto-promotion repo search failed; counting the PR's repository only", "repo", owner+"/"+repo, "author", author, "error", err, "retry_after", contributorMergedReposFailureTTL) + remember(nil, contributorMergedReposFailureTTL) + return nil + } for _, ref := range candidates { qualifier := " repo:" + ref.Owner + "/" + ref.Repo if len(query)+len(qualifier) > contributorSearchQueryMaxLen { if err := flush(); err != nil { - c.warn("contributor-lane auto-promotion repo search failed; counting the PR's repository only", "repo", owner+"/"+repo, "author", author, "error", err) - if cache != nil { - cache[key] = nil - } - return nil + return failed(err) } } query += qualifier } if err := flush(); err != nil { - c.warn("contributor-lane auto-promotion repo search failed; counting the PR's repository only", "repo", owner+"/"+repo, "author", author, "error", err) - if cache != nil { - cache[key] = nil - } - return nil + return failed(err) } out := make([]contributorRepoRef, 0, len(found)) for _, ref := range candidates { @@ -919,17 +966,8 @@ func (c *Engine) contributorMergedRepos(ctx context.Context, owner, repo, author out = append(out, ref) } } - if cache != nil { - cache[key] = out - var others []contributorRepoRef - for _, ref := range out { - if !strings.EqualFold(ref.Owner+"/"+ref.Repo, own) { - others = append(others, ref) - } - } - return others - } - return out + remember(out, contributorMergedReposTTL) + return others(out) } // repoFromRepositoryURL extracts lower-cased owner/repo from a search diff --git a/src/pkg/github/automerge/automerge_contributor_lane_test.go b/src/pkg/github/automerge/automerge_contributor_lane_test.go index a4c305b947..bdec07a7a3 100644 --- a/src/pkg/github/automerge/automerge_contributor_lane_test.go +++ b/src/pkg/github/automerge/automerge_contributor_lane_test.go @@ -1206,8 +1206,11 @@ func TestTrySweepContributorLanePRAutoPromotionScope(t *testing.T) { t.Fatalf("search query %q missing %q", fx.searchQuery, want) } } - if strings.Contains(fx.searchQuery, "repo:acme/widget") { - t.Fatalf("search query %q must not include the PR's own repository", fx.searchQuery) + // The answer is cached per author for every active repository, + // so the PR's own repository is searched too; it is never walked + // twice (wantLists below). + if !strings.Contains(fx.searchQuery, "repo:acme/widget") { + t.Fatalf("search query %q must cover the PR's own repository for the per-author cache", fx.searchQuery) } } if strings.Join(fx.otherLists, ",") != strings.Join(tc.wantLists, ",") { @@ -1688,32 +1691,143 @@ func TestContributorLaneAssociationTrusted(t *testing.T) { } } -func TestContributorPromotionSearchCachePerSweep(t *testing.T) { - fx := validContributorFixture() +// contributorPromotionSearchEngine builds an engine whose hive-wide +// auto-promotion walk needs other repositories: widget holds one clean merge +// by each author, hive holds several. +func contributorPromotionSearchEngine(t *testing.T, fx *contributorLaneFixture) (*Engine, *contributorLaneClock) { + t.Helper() fx.issueComments[0]["body"] = contributorClaimMarkerTier(contributorLaneTestAuthor, "contributor", contributorLaneTestHive, " tier=contributor verified=true") - api := newContributorLaneAPI(t, &fx) - defer api.Close() + api := newContributorLaneAPI(t, fx) + t.Cleanup(api.Close) policy := defaultContributorLanePolicy() policy.AutoPromoteAfterMergedPR = 5 clock := &contributorLaneClock{at: time.Date(2026, 10, 15, 13, 0, 0, 0, time.UTC)} - engine := newContributorLaneEngineRepos(api.URL, []string{"widget", "hive"}, policy, clock, Options{}) - ctx := context.WithValue(context.Background(), contributorRepoSearchCacheKey{}, make(map[string][]contributorRepoRef)) - for _, author := range []string{"alice", "alice", "alice", "bob"} { - _, err := engine.contributorCleanMergedHistory(ctx, "acme", "widget", author, 5, true) + return newContributorLaneEngineRepos(api.URL, []string{"widget", "hive"}, policy, clock, Options{}), clock +} + +func TestContributorPromotionSearchCachePerSweep(t *testing.T) { + fx := validContributorFixture() + engine, clock := contributorPromotionSearchEngine(t, &fx) + history := func(author string) contributorMergedHistory { + t.Helper() + h, err := engine.contributorCleanMergedHistory(context.Background(), "acme", "widget", author, 5, true) if err != nil { t.Fatal(err) } + return h + } + for _, author := range []string{"alice", "alice", "alice", "bob"} { + history(author) } if fx.searches != 2 { t.Fatalf("searches = %d, want 2", fx.searches) } - ctx = context.WithValue(context.Background(), contributorRepoSearchCacheKey{}, make(map[string][]contributorRepoRef)) - _, err := engine.contributorCleanMergedHistory(ctx, "acme", "widget", "alice", 5, true) + // A later sweep inside the TTL reuses alice's answer (#15088). + clock.advance(contributorMergedReposTTL - time.Minute) + history("alice") + if fx.searches != 2 { + t.Fatalf("searches inside the TTL = %d, want 2", fx.searches) + } + clock.advance(time.Minute) + history("alice") + if fx.searches != 3 { + t.Fatalf("searches after the TTL = %d, want 3", fx.searches) + } +} + +// TestContributorPromotionSearchCacheWalksListedReposFresh pins the +// staleness argument: only the list of repositories is cached, so a revert +// that lands in a listed repository after the search still blocks promotion +// on the next sweep, without a new search. +func TestContributorPromotionSearchCacheWalksListedReposFresh(t *testing.T) { + fx := validContributorFixture() + fx.mergedHistory = nil + clean := func(n int) []map[string]any { + var out []map[string]any + for i := 0; i < n; i++ { + out = append(out, mergedHistoryPR(contributorLaneTestAuthor, "fix: clean change")) + } + return out + } + fx.otherHistory = map[string][]map[string]any{"hive": clean(5)} + engine, clock := contributorPromotionSearchEngine(t, &fx) + h, err := engine.contributorCleanMergedHistory(context.Background(), "acme", "widget", contributorLaneTestAuthor, 5, true) + if err != nil || !h.Clean { + t.Fatalf("first sweep = %+v, %v; want clean", h, err) + } + fx.otherHistory["hive"] = append([]map[string]any{mergedHistoryPR(contributorLaneTestAuthor, "Revert \"fix: clean change\"")}, clean(5)...) + clock.advance(time.Minute) + h, err = engine.contributorCleanMergedHistory(context.Background(), "acme", "widget", contributorLaneTestAuthor, 5, true) if err != nil { t.Fatal(err) } - if fx.searches != 3 { - t.Fatalf("new sweep searches = %d, want 3", fx.searches) + if h.Clean || h.Reason != "revert-or-sentinel" { + t.Fatalf("second sweep = %+v; want blocked by the new revert", h) + } + if fx.searches != 1 { + t.Fatalf("searches = %d, want 1", fx.searches) + } +} + +// TestContributorPromotionSearchCacheIsPerAuthor checks that one search +// serves the author's PRs in every repository: each PR's own repository is +// left out of its answer. +func TestContributorPromotionSearchCacheIsPerAuthor(t *testing.T) { + fx := validContributorFixture() + fx.otherHistory = map[string][]map[string]any{ + "widget": {mergedHistoryPR(contributorLaneTestAuthor, "fix: a")}, + "hive": {mergedHistoryPR(contributorLaneTestAuthor, "fix: b")}, + } + engine, _ := contributorPromotionSearchEngine(t, &fx) + got := func(repo string) string { + var names []string + for _, ref := range engine.contributorMergedRepos(context.Background(), "acme", repo, contributorLaneTestAuthor) { + names = append(names, ref.Repo) + } + return strings.Join(names, ",") + } + if g := got("widget"); g != "hive" { + t.Fatalf("from widget = %q, want hive", g) + } + if g := got("hive"); g != "widget" { + t.Fatalf("from hive = %q, want widget", g) + } + if fx.searches != 1 { + t.Fatalf("searches = %d, want 1", fx.searches) + } + // A changed active-repository set is a miss: the cached answer does not + // cover the new repository. + engine.transport.(*hgithub.Client).SetRepos([]string{"acme/widget", "acme/hive", "acme/docs"}) + got("widget") + if fx.searches != 2 { + t.Fatalf("searches after the repo set changed = %d, want 2", fx.searches) + } +} + +func TestContributorPromotionSearchFailureCached(t *testing.T) { + fx := validContributorFixture() + fx.searchStatus = http.StatusForbidden + fx.otherHistory = map[string][]map[string]any{"hive": {mergedHistoryPR(contributorLaneTestAuthor, "fix: b")}} + engine, clock := contributorPromotionSearchEngine(t, &fx) + lookup := func() int { + return len(engine.contributorMergedRepos(context.Background(), "acme", "widget", contributorLaneTestAuthor)) + } + if n := lookup(); n != 0 || fx.searches != 1 { + t.Fatalf("failing search: %d repos, %d searches; want 0, 1", n, fx.searches) + } + fx.searchStatus = 0 + clock.advance(contributorMergedReposFailureTTL - time.Minute) + if n := lookup(); n != 0 || fx.searches != 1 { + t.Fatalf("inside the failure TTL: %d repos, %d searches; want 0, 1", n, fx.searches) + } + clock.advance(time.Minute) + if n := lookup(); n != 1 || fx.searches != 2 { + t.Fatalf("after the failure TTL: %d repos, %d searches; want 1, 2", n, fx.searches) + } + // The success is now kept for the full TTL, not the failure one. + clock.advance(contributorMergedReposFailureTTL) + if n := lookup(); n != 1 || fx.searches != 2 { + t.Fatalf("success past the failure TTL: %d repos, %d searches; want 1, 2", n, fx.searches) } } diff --git a/src/pkg/github/automerge/automerge_sweep.go b/src/pkg/github/automerge/automerge_sweep.go index 6612e68b6b..640a1ff00e 100644 --- a/src/pkg/github/automerge/automerge_sweep.go +++ b/src/pkg/github/automerge/automerge_sweep.go @@ -331,6 +331,10 @@ type Engine struct { closingLinkMu sync.Mutex closingNonPRTargets map[string]time.Time closingCommitRefs map[string][]int + // contributorMergedReposCache keeps contributorMergedRepos' per-author + // search result across sweeps (automerge_contributor_lane.go). + contributorMergedReposMu sync.Mutex + contributorMergedReposCache map[string]contributorMergedReposEntry stageLabelPolicy func() StageLabelPolicy stageTracker *StageTracker