diff --git a/changelog.d/fixed-15088-promotion-search-ttl.md b/changelog.d/fixed-15088-promotion-search-ttl.md new file mode 100644 index 0000000000..a33cb04d7a --- /dev/null +++ b/changelog.d/fixed-15088-promotion-search-ttl.md @@ -0,0 +1 @@ +- Contributor auto-promotion now remembers each author's merged-repository search for an hour (a failed search for 15 minutes) instead of searching again on every sweep ([#15088](https://github.com/hivecommons/hive/issues/15088)). diff --git a/src/docs/scale-envelope.md b/src/docs/scale-envelope.md index 8d0611a278..237493847b 100644 --- a/src/docs/scale-envelope.md +++ b/src/docs/scale-envelope.md @@ -134,7 +134,7 @@ operator can change it without rebuilding. | CI-evidence excerpt per PR | `redPRFixExcerptRunes`, `src/pkg/scheduler/policy_overlays.go:176` | 400 runes | **No** | Truncate with "…" | | Skill text injected per kick | `maxSkillsInjectionBytes`, `src/pkg/scheduler/kick_priming.go:138` | 8192 B | **No** | Shed load: skills are **dropped whole**, never truncated mid-body, and the drop is logged | | Issues primed into `AGENTS.md` context | `maxIssuesToPrime`, `src/pkg/scheduler/kick_priming.go:13` | 5 | **No** | Truncate | -| Auto-merge sweep merges per pass | `DefaultAutoMergeSweepMaxMerges`, `src/pkg/github/automerge/automerge_sweep.go:375` | 3 | **Yes** (`auto_merge.max_merges`) | Shed load: remaining merges wait for the next pass | +| Auto-merge sweep merges per pass | `DefaultAutoMergeSweepMaxMerges`, `src/pkg/github/automerge/automerge_sweep.go:505` | 3 | **Yes** (`auto_merge.max_merges`) | Shed load: remaining merges wait for the next pass | | Task-list sweep closures per tick | `DefaultTaskListSweepMaxCloses`, `src/pkg/github/task_list_sweep.go:20` | 5 | Partly (`MaxCloses` option) | Shed load: remaining closures wait for the next tick | | Prompt-history on disk | `promptHistoryMaxSizeMB` × backups, `src/pkg/dashboard/prompt_history.go:67` | 96 MiB worst case | **No** | Shed oldest: rotate + gzip, oldest prompts age out first | | Issue/PR enumeration | `fetchIssues` / `fetchPRs`, `src/pkg/github/client.go:1653`, `src/pkg/github/client.go:1920` | **no cap** | n/a | Pages to completion; cost grows with backlog | diff --git a/src/pkg/github/automerge/automerge_contributor_lane.go b/src/pkg/github/automerge/automerge_contributor_lane.go index b636c7a65f..6bfd918e2d 100644 --- a/src/pkg/github/automerge/automerge_contributor_lane.go +++ b/src/pkg/github/automerge/automerge_contributor_lane.go @@ -398,7 +398,6 @@ func (c *Engine) SweepContributorLaneAutoMerges(ctx context.Context, opts AutoMe } ctx = contextWithExpectedCheckCache(ctx, newExpectedCheckCache()) ctx = contextWithLaneMergedBranches(ctx) - ctx = context.WithValue(ctx, contributorRepoSearchCacheKey{}, make(map[string][]contributorRepoRef)) for _, displayRepo := range c.activeRepos() { if len(result.Merged) >= maxMerges { break @@ -828,7 +827,35 @@ type contributorRepoRef struct { // contributorSearchQueryMaxLen is GitHub's limit on a search `q` parameter. const contributorSearchQueryMaxLen = 256 -type contributorRepoSearchCacheKey struct{} +const ( + // contributorMergedReposTTL is how long contributorMergedRepos trusts an + // author's discovered repository list across sweeps. Only the list is + // kept: every repository on it, and the PR's own, is still walked fresh + // each sweep for clean merges and revert/sentinel evidence, so a stale + // list can leave out a repository the author first merged into during + // the last hour. That repository's merges, and a revert of them, are + // then seen up to an hour late; nothing on the list goes unchecked. The + // walk already stops at the first `need` clean merges, so it never read + // every repository. + contributorMergedReposTTL = time.Hour + // contributorMergedReposFailureTTL is how long a failed search is + // remembered. The PR's own repository is counted alone meanwhile, as + // on the failing sweep, rather than re-tripping a secondary rate limit + // on every sweep (#15088). + contributorMergedReposFailureTTL = 15 * time.Minute + // contributorMergedReposCacheMax bounds the cache; past it the cache + // starts over. + contributorMergedReposCacheMax = 1024 +) + +// contributorMergedReposEntry is one author's search result: the active +// repositories (the PR's own included) that hold a merge by them. repos is +// the active-repository set the search covered; a different set is a miss. +type contributorMergedReposEntry struct { + refs []contributorRepoRef + repos string + expires time.Time +} // contributorMergedRepos returns the hive's other active repositories that // hold at least one merged PR by author, in the order the transport lists @@ -838,23 +865,28 @@ type contributorRepoSearchCacheKey struct{} // calls instead of ten closed-PR listings. A search failure degrades to the // PR's own repository with a warning rather than blocking the sweep: the // result is then exactly the pre-#14517 per-repo count. +// +// The answer covers every active repository and is kept on the Engine per +// author for contributorMergedReposTTL (a failure for +// contributorMergedReposFailureTTL), so an author with PRs in several +// repositories, or the same PR on every sweep, costs one search per TTL +// rather than one per PR per sweep (#15088). func (c *Engine) contributorMergedRepos(ctx context.Context, owner, repo, author string) []contributorRepoRef { if c == nil || c.gh == nil || c.transport == nil { return nil } own := strings.ToLower(owner + "/" + repo) - cache, _ := ctx.Value(contributorRepoSearchCacheKey{}).(map[string][]contributorRepoRef) - key := strings.ToLower(strings.TrimSpace(author)) - if refs, ok := cache[key]; ok { - var others []contributorRepoRef + others := func(refs []contributorRepoRef) []contributorRepoRef { + var out []contributorRepoRef for _, ref := range refs { if !strings.EqualFold(ref.Owner+"/"+ref.Repo, own) { - others = append(others, ref) + out = append(out, ref) } } - return others + return out } var candidates []contributorRepoRef + var names []string seen := map[string]bool{} for _, display := range c.activeRepos() { o, r := c.transport.SplitRepo(display) @@ -862,15 +894,33 @@ func (c *Engine) contributorMergedRepos(ctx context.Context, owner, repo, author continue } full := strings.ToLower(o + "/" + r) - if (cache == nil && full == own) || seen[full] { + if seen[full] { continue } seen[full] = true candidates = append(candidates, contributorRepoRef{Owner: o, Repo: r}) + names = append(names, full) } - if len(candidates) == 0 { + if len(others(candidates)) == 0 { return nil } + key := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(author), "@")) + repos := strings.Join(names, ",") + now := c.clock() + c.contributorMergedReposMu.Lock() + entry, ok := c.contributorMergedReposCache[key] + c.contributorMergedReposMu.Unlock() + if ok && entry.repos == repos && now.Before(entry.expires) { + return others(entry.refs) + } + remember := func(refs []contributorRepoRef, ttl time.Duration) { + c.contributorMergedReposMu.Lock() + defer c.contributorMergedReposMu.Unlock() + if c.contributorMergedReposCache == nil || len(c.contributorMergedReposCache) >= contributorMergedReposCacheMax { + c.contributorMergedReposCache = map[string]contributorMergedReposEntry{} + } + c.contributorMergedReposCache[key] = contributorMergedReposEntry{refs: refs, repos: repos, expires: now.Add(ttl)} + } base := "is:pr is:merged author:" + strings.TrimPrefix(strings.TrimSpace(author), "@") found := map[string]bool{} query := base @@ -893,25 +943,22 @@ func (c *Engine) contributorMergedRepos(ctx context.Context, owner, repo, author } return nil } + failed := func(err error) []contributorRepoRef { + c.warn("contributor-lane auto-promotion repo search failed; counting the PR's repository only", "repo", owner+"/"+repo, "author", author, "error", err, "retry_after", contributorMergedReposFailureTTL) + remember(nil, contributorMergedReposFailureTTL) + return nil + } for _, ref := range candidates { qualifier := " repo:" + ref.Owner + "/" + ref.Repo if len(query)+len(qualifier) > contributorSearchQueryMaxLen { if err := flush(); err != nil { - c.warn("contributor-lane auto-promotion repo search failed; counting the PR's repository only", "repo", owner+"/"+repo, "author", author, "error", err) - if cache != nil { - cache[key] = nil - } - return nil + return failed(err) } } query += qualifier } if err := flush(); err != nil { - c.warn("contributor-lane auto-promotion repo search failed; counting the PR's repository only", "repo", owner+"/"+repo, "author", author, "error", err) - if cache != nil { - cache[key] = nil - } - return nil + return failed(err) } out := make([]contributorRepoRef, 0, len(found)) for _, ref := range candidates { @@ -919,17 +966,8 @@ func (c *Engine) contributorMergedRepos(ctx context.Context, owner, repo, author out = append(out, ref) } } - if cache != nil { - cache[key] = out - var others []contributorRepoRef - for _, ref := range out { - if !strings.EqualFold(ref.Owner+"/"+ref.Repo, own) { - others = append(others, ref) - } - } - return others - } - return out + remember(out, contributorMergedReposTTL) + return others(out) } // repoFromRepositoryURL extracts lower-cased owner/repo from a search diff --git a/src/pkg/github/automerge/automerge_contributor_lane_test.go b/src/pkg/github/automerge/automerge_contributor_lane_test.go index a4c305b947..bdec07a7a3 100644 --- a/src/pkg/github/automerge/automerge_contributor_lane_test.go +++ b/src/pkg/github/automerge/automerge_contributor_lane_test.go @@ -1206,8 +1206,11 @@ func TestTrySweepContributorLanePRAutoPromotionScope(t *testing.T) { t.Fatalf("search query %q missing %q", fx.searchQuery, want) } } - if strings.Contains(fx.searchQuery, "repo:acme/widget") { - t.Fatalf("search query %q must not include the PR's own repository", fx.searchQuery) + // The answer is cached per author for every active repository, + // so the PR's own repository is searched too; it is never walked + // twice (wantLists below). + if !strings.Contains(fx.searchQuery, "repo:acme/widget") { + t.Fatalf("search query %q must cover the PR's own repository for the per-author cache", fx.searchQuery) } } if strings.Join(fx.otherLists, ",") != strings.Join(tc.wantLists, ",") { @@ -1688,32 +1691,143 @@ func TestContributorLaneAssociationTrusted(t *testing.T) { } } -func TestContributorPromotionSearchCachePerSweep(t *testing.T) { - fx := validContributorFixture() +// contributorPromotionSearchEngine builds an engine whose hive-wide +// auto-promotion walk needs other repositories: widget holds one clean merge +// by each author, hive holds several. +func contributorPromotionSearchEngine(t *testing.T, fx *contributorLaneFixture) (*Engine, *contributorLaneClock) { + t.Helper() fx.issueComments[0]["body"] = contributorClaimMarkerTier(contributorLaneTestAuthor, "contributor", contributorLaneTestHive, " tier=contributor verified=true") - api := newContributorLaneAPI(t, &fx) - defer api.Close() + api := newContributorLaneAPI(t, fx) + t.Cleanup(api.Close) policy := defaultContributorLanePolicy() policy.AutoPromoteAfterMergedPR = 5 clock := &contributorLaneClock{at: time.Date(2026, 10, 15, 13, 0, 0, 0, time.UTC)} - engine := newContributorLaneEngineRepos(api.URL, []string{"widget", "hive"}, policy, clock, Options{}) - ctx := context.WithValue(context.Background(), contributorRepoSearchCacheKey{}, make(map[string][]contributorRepoRef)) - for _, author := range []string{"alice", "alice", "alice", "bob"} { - _, err := engine.contributorCleanMergedHistory(ctx, "acme", "widget", author, 5, true) + return newContributorLaneEngineRepos(api.URL, []string{"widget", "hive"}, policy, clock, Options{}), clock +} + +func TestContributorPromotionSearchCachePerSweep(t *testing.T) { + fx := validContributorFixture() + engine, clock := contributorPromotionSearchEngine(t, &fx) + history := func(author string) contributorMergedHistory { + t.Helper() + h, err := engine.contributorCleanMergedHistory(context.Background(), "acme", "widget", author, 5, true) if err != nil { t.Fatal(err) } + return h + } + for _, author := range []string{"alice", "alice", "alice", "bob"} { + history(author) } if fx.searches != 2 { t.Fatalf("searches = %d, want 2", fx.searches) } - ctx = context.WithValue(context.Background(), contributorRepoSearchCacheKey{}, make(map[string][]contributorRepoRef)) - _, err := engine.contributorCleanMergedHistory(ctx, "acme", "widget", "alice", 5, true) + // A later sweep inside the TTL reuses alice's answer (#15088). + clock.advance(contributorMergedReposTTL - time.Minute) + history("alice") + if fx.searches != 2 { + t.Fatalf("searches inside the TTL = %d, want 2", fx.searches) + } + clock.advance(time.Minute) + history("alice") + if fx.searches != 3 { + t.Fatalf("searches after the TTL = %d, want 3", fx.searches) + } +} + +// TestContributorPromotionSearchCacheWalksListedReposFresh pins the +// staleness argument: only the list of repositories is cached, so a revert +// that lands in a listed repository after the search still blocks promotion +// on the next sweep, without a new search. +func TestContributorPromotionSearchCacheWalksListedReposFresh(t *testing.T) { + fx := validContributorFixture() + fx.mergedHistory = nil + clean := func(n int) []map[string]any { + var out []map[string]any + for i := 0; i < n; i++ { + out = append(out, mergedHistoryPR(contributorLaneTestAuthor, "fix: clean change")) + } + return out + } + fx.otherHistory = map[string][]map[string]any{"hive": clean(5)} + engine, clock := contributorPromotionSearchEngine(t, &fx) + h, err := engine.contributorCleanMergedHistory(context.Background(), "acme", "widget", contributorLaneTestAuthor, 5, true) + if err != nil || !h.Clean { + t.Fatalf("first sweep = %+v, %v; want clean", h, err) + } + fx.otherHistory["hive"] = append([]map[string]any{mergedHistoryPR(contributorLaneTestAuthor, "Revert \"fix: clean change\"")}, clean(5)...) + clock.advance(time.Minute) + h, err = engine.contributorCleanMergedHistory(context.Background(), "acme", "widget", contributorLaneTestAuthor, 5, true) if err != nil { t.Fatal(err) } - if fx.searches != 3 { - t.Fatalf("new sweep searches = %d, want 3", fx.searches) + if h.Clean || h.Reason != "revert-or-sentinel" { + t.Fatalf("second sweep = %+v; want blocked by the new revert", h) + } + if fx.searches != 1 { + t.Fatalf("searches = %d, want 1", fx.searches) + } +} + +// TestContributorPromotionSearchCacheIsPerAuthor checks that one search +// serves the author's PRs in every repository: each PR's own repository is +// left out of its answer. +func TestContributorPromotionSearchCacheIsPerAuthor(t *testing.T) { + fx := validContributorFixture() + fx.otherHistory = map[string][]map[string]any{ + "widget": {mergedHistoryPR(contributorLaneTestAuthor, "fix: a")}, + "hive": {mergedHistoryPR(contributorLaneTestAuthor, "fix: b")}, + } + engine, _ := contributorPromotionSearchEngine(t, &fx) + got := func(repo string) string { + var names []string + for _, ref := range engine.contributorMergedRepos(context.Background(), "acme", repo, contributorLaneTestAuthor) { + names = append(names, ref.Repo) + } + return strings.Join(names, ",") + } + if g := got("widget"); g != "hive" { + t.Fatalf("from widget = %q, want hive", g) + } + if g := got("hive"); g != "widget" { + t.Fatalf("from hive = %q, want widget", g) + } + if fx.searches != 1 { + t.Fatalf("searches = %d, want 1", fx.searches) + } + // A changed active-repository set is a miss: the cached answer does not + // cover the new repository. + engine.transport.(*hgithub.Client).SetRepos([]string{"acme/widget", "acme/hive", "acme/docs"}) + got("widget") + if fx.searches != 2 { + t.Fatalf("searches after the repo set changed = %d, want 2", fx.searches) + } +} + +func TestContributorPromotionSearchFailureCached(t *testing.T) { + fx := validContributorFixture() + fx.searchStatus = http.StatusForbidden + fx.otherHistory = map[string][]map[string]any{"hive": {mergedHistoryPR(contributorLaneTestAuthor, "fix: b")}} + engine, clock := contributorPromotionSearchEngine(t, &fx) + lookup := func() int { + return len(engine.contributorMergedRepos(context.Background(), "acme", "widget", contributorLaneTestAuthor)) + } + if n := lookup(); n != 0 || fx.searches != 1 { + t.Fatalf("failing search: %d repos, %d searches; want 0, 1", n, fx.searches) + } + fx.searchStatus = 0 + clock.advance(contributorMergedReposFailureTTL - time.Minute) + if n := lookup(); n != 0 || fx.searches != 1 { + t.Fatalf("inside the failure TTL: %d repos, %d searches; want 0, 1", n, fx.searches) + } + clock.advance(time.Minute) + if n := lookup(); n != 1 || fx.searches != 2 { + t.Fatalf("after the failure TTL: %d repos, %d searches; want 1, 2", n, fx.searches) + } + // The success is now kept for the full TTL, not the failure one. + clock.advance(contributorMergedReposFailureTTL) + if n := lookup(); n != 1 || fx.searches != 2 { + t.Fatalf("success past the failure TTL: %d repos, %d searches; want 1, 2", n, fx.searches) } } diff --git a/src/pkg/github/automerge/automerge_sweep.go b/src/pkg/github/automerge/automerge_sweep.go index 6612e68b6b..640a1ff00e 100644 --- a/src/pkg/github/automerge/automerge_sweep.go +++ b/src/pkg/github/automerge/automerge_sweep.go @@ -331,6 +331,10 @@ type Engine struct { closingLinkMu sync.Mutex closingNonPRTargets map[string]time.Time closingCommitRefs map[string][]int + // contributorMergedReposCache keeps contributorMergedRepos' per-author + // search result across sweeps (automerge_contributor_lane.go). + contributorMergedReposMu sync.Mutex + contributorMergedReposCache map[string]contributorMergedReposEntry stageLabelPolicy func() StageLabelPolicy stageTracker *StageTracker