diff --git a/changelog.d/fixed-claim-ledger-repo-spelling.md b/changelog.d/fixed-claim-ledger-repo-spelling.md new file mode 100644 index 0000000000..95246f7f08 --- /dev/null +++ b/changelog.d/fixed-claim-ledger-repo-spelling.md @@ -0,0 +1 @@ +- The duplicate-PR claim ledger now keys every claim by the configured repo spelling. A verdict claim (stored under the hub task repo, `owner/repo`) or a cross-repo `Part of owner/repo#N` reference was invisible to the enumeration's lookups by the bare name, while the ledger backfill still found it, so the issue flipped between `stage/010-actionable` and `stage/040-covered-by-pr` every pass (projectbluefin/actions#748) and `hive/likely-done` was stripped minutes after it was added. Claims stored under another spelling are re-keyed on the next guard pass, keeping the stronger claim; another org's repo of the same bare name is never folded onto the hive's. diff --git a/src/pkg/github/open_pr_links.go b/src/pkg/github/open_pr_links.go index 726ee68f73..eb94115a35 100644 --- a/src/pkg/github/open_pr_links.go +++ b/src/pkg/github/open_pr_links.go @@ -171,8 +171,25 @@ func (c *Client) InvalidateOpenPRLinks(repo string) { // canonicalRepo maps repo onto the configured spelling, so a caller naming // the repo bare or org-qualified shares one cache entry and one claim key. +// The owner counts: a bare configured name means the hive's org, so +// "otherorg/actions" is not the configured "actions" and is returned +// unchanged — the claim ledger keys cross-repo references with this, and a +// PR naming another org's issue must not claim the hive's issue of the same +// number. Without an org (bare names cannot be resolved) it falls back to +// the bare-name match. func (c *Client) canonicalRepo(repo string) string { - return canonicalConfigRepo(repo, c.getRepos()) + repos := c.getRepos() + if strings.TrimSpace(c.org) == "" { + return canonicalConfigRepo(repo, repos) + } + owner, name := c.splitRepo(strings.TrimSpace(repo)) + for _, cfg := range repos { + cfgOwner, cfgName := c.splitRepo(strings.TrimSpace(cfg)) + if strings.EqualFold(cfgOwner, owner) && strings.EqualFold(cfgName, name) { + return cfg + } + } + return repo } // openPRIndexFor returns the (possibly cached) open-PR link index for repo. diff --git a/src/pkg/github/prchurn.go b/src/pkg/github/prchurn.go index 7c5dbd159c..0d28286072 100644 --- a/src/pkg/github/prchurn.go +++ b/src/pkg/github/prchurn.go @@ -286,6 +286,7 @@ func (l *ClaimLedger) insertHistoryLocked(r IssuePRRecord) { if r.ObservedAt.IsZero() { r.ObservedAt = l.now() } + r.Repo = l.repoLocked(r.Repo) if l.history == nil { l.history = make(map[string]map[int]IssuePRRecord) } @@ -335,7 +336,7 @@ func (l *ClaimLedger) Churn(repo string, issue int) (IssueChurn, bool) { } l.mu.RLock() defer l.mu.RUnlock() - byPR, ok := l.history[claimKey(repo, issue)] + byPR, ok := l.history[l.keyLocked(repo, issue)] if !ok || len(byPR) == 0 { return IssueChurn{}, false } diff --git a/src/pkg/github/prclaims.go b/src/pkg/github/prclaims.go index 98a724b9b7..e5d7abba3f 100644 --- a/src/pkg/github/prclaims.go +++ b/src/pkg/github/prclaims.go @@ -1224,6 +1224,9 @@ type ClaimLedger struct { weakDefer time.Duration // now is the clock, overridable for tests. now func() time.Time + // canon maps a repo onto its configured spelling (SetRepoCanonicalizer); + // nil keys repos exactly as given. + canon func(string) string } // ledgerFile is the on-disk shape of the ledger. History is omitempty so a @@ -1392,6 +1395,7 @@ func (l *ClaimLedger) weakDeferExpired(c IssueClaim) bool { // external PR racing it to the map. Callers must hold l.mu (or own the ledger // exclusively, as LoadClaimLedger does before publishing it). func (l *ClaimLedger) insertLocked(c IssueClaim) { + c.Repo = l.repoLocked(c.Repo) key := c.Key() if existing, ok := l.claims[key]; ok && claimRank(c) < claimRank(existing) { return @@ -1399,6 +1403,56 @@ func (l *ClaimLedger) insertLocked(c IssueClaim) { l.claims[key] = c } +// repoLocked is repo in the configured spelling when a canonicalizer is set. +// Callers must hold l.mu (read or write). +func (l *ClaimLedger) repoLocked(repo string) string { + if l.canon == nil || repo == "" { + return repo + } + return l.canon(repo) +} + +// keyLocked is claimKey with the repo canonicalized, so a lookup spelled +// "owner/repo" and one spelled "repo" find the same entry. Callers must hold +// l.mu (read or write). +func (l *ClaimLedger) keyLocked(repo string, issue int) string { + return claimKey(l.repoLocked(repo), issue) +} + +// SetRepoCanonicalizer keys every claim and churn record by the repo +// spelling canon returns (projectbluefin/actions#748). The writers disagree: +// the scan stores the configured spelling ("actions"), while a verdict claim +// carries the hub task repo ("projectbluefin/actions") and a cross-repo +// "Part of owner/repo#N" carries the owner. The enumeration looks claims up +// with the configured spelling, so an owner-qualified claim was invisible to +// the claimed-by-PR filter and the label sync while BackfillCoveredByLedger, +// which walks the ledger itself, still found it: the issue flipped 010↔040 +// every pass and hive/likely-done was stripped minutes after it was added. +// Entries already stored under another spelling are re-keyed here, keeping +// the higher-ranked claim when both spellings hold one. canon must be +// owner-aware (Client.canonicalRepo): a hive can watch two repos with the +// same bare name in different orgs. +func (l *ClaimLedger) SetRepoCanonicalizer(canon func(string) string) { + if l == nil || canon == nil { + return + } + l.mu.Lock() + defer l.mu.Unlock() + l.canon = canon + claims := l.claims + l.claims = make(map[string]IssueClaim, len(claims)) + for _, c := range claims { + l.insertLocked(c) + } + history := l.history + l.history = make(map[string]map[int]IssuePRRecord, len(history)) + for _, byPR := range history { + for _, r := range byPR { + l.insertHistoryLocked(r) + } + } +} + // Record inserts one claim outside the enumeration cycle and persists the // ledger (hivecommons/hive#7871). It is the write path for verdict-recovered // claims: the contribute hub has just verified, via the API, that a PR or @@ -1415,6 +1469,7 @@ func (l *ClaimLedger) Record(c IssueClaim) error { if c.ObservedAt.IsZero() { c.ObservedAt = l.now() } + c.Repo = l.repoLocked(c.Repo) l.insertLocked(l.anchorFirstObservedLocked(l.claims, c)) l.mu.Unlock() return l.Save() @@ -1431,7 +1486,7 @@ func (l *ClaimLedger) MarkCloseOnMergeSettled(repo string, issue, pr int, at tim } l.mu.Lock() defer l.mu.Unlock() - key := claimKey(repo, issue) + key := l.keyLocked(repo, issue) c, ok := l.claims[key] if !ok || c.PRNumber != pr { return false @@ -1521,7 +1576,7 @@ func (l *ClaimLedger) LookupAny(repo string, issue int) (IssueClaim, bool) { } l.mu.RLock() defer l.mu.RUnlock() - c, ok := l.claims[claimKey(repo, issue)] + c, ok := l.claims[l.keyLocked(repo, issue)] return c, ok } @@ -1553,7 +1608,7 @@ func (l *ClaimLedger) MarkIssueReopened(repo string, issue int, at time.Time) bo func (l *ClaimLedger) markIssueReopenedLocked(repo string, issue int, at time.Time) (IssueClaim, bool) { l.mu.Lock() defer l.mu.Unlock() - key := claimKey(repo, issue) + key := l.keyLocked(repo, issue) c, ok := l.claims[key] if !ok || !c.MergedPR || !c.ReopenedAt.IsZero() { return IssueClaim{}, false @@ -1572,7 +1627,7 @@ func (l *ClaimLedger) markIssueReopenedLocked(repo string, issue int, at time.Ti func (l *ClaimLedger) markReopenChecked(repo string, issue int, at time.Time) { l.mu.Lock() defer l.mu.Unlock() - key := claimKey(repo, issue) + key := l.keyLocked(repo, issue) if c, ok := l.claims[key]; ok { c.ReopenCheckedAt = at l.claims[key] = c @@ -1730,6 +1785,7 @@ func (l *ClaimLedger) reconcileOpen(live []IssueClaim, authoritative bool, open if c.Issue <= 0 || c.Repo == "" { continue } + c.Repo = l.repoLocked(c.Repo) l.insertLocked(l.anchorFirstObservedLocked(prev, c)) } if !listedAt.IsZero() { @@ -1785,6 +1841,7 @@ func (l *ClaimLedger) reconcileOpen(live []IssueClaim, authoritative bool, open if c.Issue <= 0 || c.Repo == "" { continue } + c.Repo = l.repoLocked(c.Repo) l.insertLocked(l.anchorFirstObservedLocked(l.claims, c)) } l.markOpenLocked(open, l.now()) diff --git a/src/pkg/github/prclaims_repo_spelling_test.go b/src/pkg/github/prclaims_repo_spelling_test.go new file mode 100644 index 0000000000..9f826760b8 --- /dev/null +++ b/src/pkg/github/prclaims_repo_spelling_test.go @@ -0,0 +1,214 @@ +package github + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + gh "github.com/google/go-github/v72/github" + + "github.com/hivecommons/hive/pkg/stagelabels" +) + +// spellingClient is a hive configured with bare repo names in one org, the +// default config shape the claim ledger misread (projectbluefin/actions#748). +func spellingClient(t *testing.T, repos ...string) *Client { + t.Helper() + return NewClientForTest("http://127.0.0.1:1", "projectbluefin", repos, testLogger()) +} + +func spellingLedger(t *testing.T, c *Client) *ClaimLedger { + t.Helper() + ledger := NewClaimLedger(filepath.Join(t.TempDir(), "claims.json"), testLogger()) + ledger.SetRepoCanonicalizer(c.canonicalRepo) + return ledger +} + +// The owner counts: a bare configured name means the hive's org, so another +// org's repo of the same name is never folded onto it. +func TestClientCanonicalRepoIsOwnerAware(t *testing.T) { + c := spellingClient(t, "actions", "otherorg/tools") + for in, want := range map[string]string{ + "actions": "actions", + "projectbluefin/actions": "actions", + "ProjectBluefin/Actions": "actions", + "otherorg/actions": "otherorg/actions", + "otherorg/tools": "otherorg/tools", + "OtherOrg/Tools": "otherorg/tools", + "tools": "tools", + "projectbluefin/unknown": "projectbluefin/unknown", + } { + if got := c.canonicalRepo(in); got != want { + t.Errorf("canonicalRepo(%q) = %q, want %q", in, got, want) + } + } +} + +// projectbluefin/actions#748: the verdict path records the hub task repo +// ("projectbluefin/actions") while the enumeration looks the issue up as +// "actions". The claim must be found, and the claimed-by-PR filter must +// bucket the issue, so the stage sweep stops writing 010 under it. +func TestVerdictClaimOwnerQualifiedFoundByConfiguredSpelling(t *testing.T) { + c := spellingClient(t, "actions") + ledger := spellingLedger(t, c) + if err := ledger.Record(IssueClaim{ + Repo: "projectbluefin/actions", Issue: 748, PRNumber: 731, PRRepo: "projectbluefin/actions", + PRAuthor: "kylerankin", PRState: PRStateOpen, ExternalAuthor: true, + Source: ClaimSourceVerdict, SourceReporter: "mendezr", + }); err != nil { + t.Fatal(err) + } + if ledger.Len() != 1 { + t.Fatalf("ledger holds %d claims, want 1", ledger.Len()) + } + for _, repo := range []string{"actions", "projectbluefin/actions"} { + claim, ok := ledger.Lookup(repo, 748) + if !ok || claim.PRNumber != 731 || claim.Repo != "actions" { + t.Fatalf("Lookup(%q, 748) = %+v, %v; want the #731 claim keyed by the configured spelling", repo, claim, ok) + } + } + + result := &ActionableResult{Issues: IssueResult{Items: []Issue{{Repo: "actions", Number: 748, Title: "[shared-ci] tracker"}}, Count: 1}} + if got := FilterClaimedIssues(result, ledger, nil, testLogger()); got != 1 || len(result.Issues.Items) != 0 { + t.Fatalf("filter suppressed %d, kept %+v; want the issue bucketed claimed_by_pr", got, result.Issues.Items) + } +} + +// projectbluefin/utah#1236: a PR in another watched repo saying "Part of +// owner/repo#N" claims the issue under its owner-qualified name. It is found +// by the configured spelling; the same text naming another org's repo is not +// a claim on the hive's issue. +func TestCrossRepoReferenceClaimFoundByConfiguredSpelling(t *testing.T) { + c := spellingClient(t, "utah", "utah-packages") + now := time.Now() + claimsFor := func(body string) []IssueClaim { + pr := &gh.PullRequest{ + Number: gh.Ptr(1017), Title: gh.Ptr("Translate the BuildStream elements"), Body: gh.Ptr(body), + State: gh.Ptr(PRStateOpen), User: &gh.User{Login: gh.Ptr("contributor")}, + HTMLURL: gh.Ptr("https://github.com/projectbluefin/utah-packages/pull/1017"), + } + claims := claimsFromPR(pr, "utah-packages", HiveIdentity{AppLogin: "hive[bot]"}, now) + for i := range claims { + claims[i].PRState = PRStateOpen + } + return claims + } + + ledger := spellingLedger(t, c) + ledger.Reconcile(claimsFor("Part of the BuildStream→RPM translation for projectbluefin/utah#1236"), true) + if claim, ok := ledger.Lookup("utah", 1236); !ok || claim.PRNumber != 1017 || claim.Repo != "utah" { + t.Fatalf("Lookup(utah, 1236) = %+v, %v; want the utah-packages#1017 reference claim", claim, ok) + } + + foreign := spellingLedger(t, c) + foreign.Reconcile(claimsFor("Part of otherorg/utah#1236"), true) + if claim, ok := foreign.Lookup("utah", 1236); ok { + t.Fatalf("another org's utah#1236 claimed the hive's issue: %+v", claim) + } +} + +// A ledger saved before the fix holds owner-qualified keys. Setting the +// canonicalizer re-keys them once, keeps the higher-ranked claim when both +// spellings hold one, moves the churn history too, and the next save writes +// the configured spelling. +func TestClaimLedgerRekeysOwnerQualifiedEntries(t *testing.T) { + c := spellingClient(t, "actions") + path := filepath.Join(t.TempDir(), "claims.json") + now := time.Now().UTC() + file := ledgerFile{ + SavedAt: now, + Claims: []IssueClaim{ + {Repo: "projectbluefin/actions", Issue: 748, PRNumber: 731, PRRepo: "projectbluefin/actions", PRState: PRStateOpen, ExternalAuthor: true, ObservedAt: now, FirstObservedAt: now}, + {Repo: "actions", Issue: 748, PRNumber: 900, PRRepo: "actions", PRState: PRStateOpen, ExternalAuthor: true, Reference: true, ObservedAt: now, FirstObservedAt: now}, + {Repo: "projectbluefin/actions", Issue: 12, PRNumber: 40, PRRepo: "actions", PRState: PRStateOpen, Closing: true, ObservedAt: now, FirstObservedAt: now}, + }, + History: []IssuePRRecord{ + {Repo: "projectbluefin/actions", Issue: 748, PRNumber: 731, PRRepo: "actions", State: PRStateOpen, ObservedAt: now}, + }, + } + raw, err := json.Marshal(file) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, raw, 0o600); err != nil { + t.Fatal(err) + } + ledger, err := LoadClaimLedger(path, testLogger()) + if err != nil { + t.Fatal(err) + } + if ledger.Len() != 3 { + t.Fatalf("loaded %d claims, want 3 (two spellings of #748 before re-keying)", ledger.Len()) + } + + ledger.SetRepoCanonicalizer(c.canonicalRepo) + if ledger.Len() != 2 { + t.Fatalf("after re-keying %d claims, want 2", ledger.Len()) + } + if claim, ok := ledger.Lookup("actions", 748); !ok || claim.PRNumber != 731 { + t.Fatalf("Lookup(actions, 748) = %+v, %v; want the stronger #731 claim to survive the merge", claim, ok) + } + if claim, ok := ledger.Lookup("actions", 12); !ok || claim.PRNumber != 40 { + t.Fatalf("Lookup(actions, 12) = %+v, %v; want the re-keyed #40 claim", claim, ok) + } + if churn, ok := ledger.Churn("actions", 748); !ok || len(churn.Open) != 1 { + t.Fatalf("Churn(actions, 748) = %+v, %v; want the re-keyed history record", churn, ok) + } + // Idempotent: setting it again (every guard pass does) changes nothing. + ledger.SetRepoCanonicalizer(c.canonicalRepo) + if ledger.Len() != 2 { + t.Fatalf("second re-key left %d claims, want 2", ledger.Len()) + } + + if err := ledger.Save(); err != nil { + t.Fatal(err) + } + reloaded, err := LoadClaimLedger(path, testLogger()) + if err != nil { + t.Fatal(err) + } + for _, claim := range reloaded.Claims() { + if claim.Repo != "actions" { + t.Fatalf("saved claim keyed %q, want the configured spelling", claim.Repo) + } + } +} + +// The flip itself: with an owner-qualified claim, the stage sweep (fed by the +// claimed-by-PR filter) wrote 010 while BackfillCoveredByLedger wrote 040. +// Keyed by one spelling, the filter buckets the issue covered, the sweep's +// 040 stands, and the backfill finds nothing to correct. +func TestOwnerQualifiedClaimFilterAndBackfillAgree(t *testing.T) { + srv := newStageFactsServer(t, "help wanted", stagelabels.CoveredByPR.Label()) + c := testClient(t, srv.srv.URL) + c.SetIssueStageLabels(func() IssueStagePolicy { return IssueStagePolicy{Enabled: true} }) + store := stagelabels.Shared() + issue := c.IssueStageRef("r", 7) + forget := func() { store.Forget(issue.Key()); store.Forget(c.PRStageRef("o/r", 8).Key()) } + forget() + t.Cleanup(forget) + + ledger := NewClaimLedger(filepath.Join(t.TempDir(), "claims.json"), testLogger()) + ledger.SetRepoCanonicalizer(c.canonicalRepo) + if err := ledger.Record(IssueClaim{Repo: "o/r", Issue: 7, PRNumber: 8, PRRepo: "o/r", PRState: PRStateOpen, ExternalAuthor: true, Source: ClaimSourceVerdict}); err != nil { + t.Fatal(err) + } + + result := &ActionableResult{Issues: IssueResult{Items: []Issue{{Repo: "r", Number: 7}}, Count: 1}} + if got := FilterClaimedIssues(result, ledger, nil, testLogger()); got != 1 { + t.Fatalf("filter suppressed %d, want 1: the enumeration's bare spelling must find the claim", got) + } + // What the sweep writes for a claimed_by_pr issue. + store.Observe(issue.Key(), stagelabels.CoveredByPR, "open PR covers it") + for pass := 1; pass <= 2; pass++ { + if res := c.BackfillCoveredByLedger(context.Background(), ledger, nil); res.Checked != 0 || res.Covered != 0 { + t.Fatalf("backfill pass %d = %+v, want nothing to correct", pass, res) + } + } + if srv.prReads != 0 { + t.Fatalf("backfill read the PR %d times, want none (the two already agree)", srv.prReads) + } +}