From 428f2bef45c7996969b1f708afb0b88ef5803b04 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 08:51:44 -0500 Subject: [PATCH] fix(dashboard): deflake TestClosedDeliveryEvictsWithWebhookRefreshDisabled Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-16429.md | 1 + src/pkg/github/client.go | 4 +- src/pkg/github/pr_batch_graphql.go | 6 +-- src/pkg/github/prdetailcache.go | 2 +- .../github/review_evidence_change_failure.go | 2 +- src/pkg/github/webhook_cache.go | 45 +++++++++++-------- src/pkg/github/webhook_cache_test.go | 8 ++-- src/pkg/github/webhook_mode.go | 8 ++-- 8 files changed, 43 insertions(+), 33 deletions(-) create mode 100644 changelog.d/fixed-16429.md diff --git a/changelog.d/fixed-16429.md b/changelog.d/fixed-16429.md new file mode 100644 index 0000000000..fbf03b2134 --- /dev/null +++ b/changelog.d/fixed-16429.md @@ -0,0 +1 @@ +- The process-wide webhook tracker is now swapped and read atomically, removing a data race that failed `TestClosedDeliveryEvictsWithWebhookRefreshDisabled` under `-race` when a refresher flush from an earlier test was still reading it ([#16429](https://github.com/hivecommons/hive/issues/16429)). diff --git a/src/pkg/github/client.go b/src/pkg/github/client.go index bc9dd017e7..32e5c7c002 100644 --- a/src/pkg/github/client.go +++ b/src/pkg/github/client.go @@ -2605,13 +2605,13 @@ func (c *Client) enrichPRCI(ctx context.Context, pr *PullRequest) map[string]boo full, ok := c.cachedPRDetailFromList(pr.Repo, pr.Number, pr.HeadSHA, pr.UpdatedAt) if !ok { var err error - started := sharedWebhookTracker.now() + started := sharedWebhookTracker().now() full, _, err = c.client.PullRequests.Get(WithRESTCaller(ctx, "hive:enrich_pr_ci"), owner, repoName, pr.Number) if err != nil { c.logger.Warn("failed to fetch PR mergeability", "repo", pr.Repo, "pr", pr.Number, "error", err) } else { c.storePRDetail(pr.Repo, pr.Number, full) - sharedWebhookTracker.clearDirtyBefore(pr.Repo, pr.Number, started) + sharedWebhookTracker().clearDirtyBefore(pr.Repo, pr.Number, started) } } if full != nil { diff --git a/src/pkg/github/pr_batch_graphql.go b/src/pkg/github/pr_batch_graphql.go index dbcc0f2139..a44d0c6cf3 100644 --- a/src/pkg/github/pr_batch_graphql.go +++ b/src/pkg/github/pr_batch_graphql.go @@ -170,11 +170,11 @@ func (c *Client) prefetchOpenPRDetailsGraphQL(ctx context.Context, owner, repoNa } // With healthy webhooks and nothing marked dirty since the last batch, // the cached batch still describes every open PR (#11177). - if sharedWebhookTracker.repoCleanAndHealthy(fullRepo) && c.hasGraphQLPRBatchRepo(fullRepo) { + if sharedWebhookTracker().repoCleanAndHealthy(fullRepo) && c.hasGraphQLPRBatchRepo(fullRepo) { c.recordGraphQLPRBatchWebhookSkip() return } - started := sharedWebhookTracker.now() + started := sharedWebhookTracker().now() c.clearGraphQLPRBatchRepo(fullRepo) pageSize := c.graphQLPRBatchPageSizeEffective() var cursor string @@ -208,7 +208,7 @@ func (c *Client) prefetchOpenPRDetailsGraphQL(ctx context.Context, owner, repoNa } } c.recordGraphQLPRBatchSuccess(repoPRs, pages) - sharedWebhookTracker.clearDirtyRepoBefore(fullRepo, started) + sharedWebhookTracker().clearDirtyRepoBefore(fullRepo, started) } func (c *Client) hasGraphQLPRBatchRepo(repo string) bool { diff --git a/src/pkg/github/prdetailcache.go b/src/pkg/github/prdetailcache.go index 3811bd8cf4..4e728c1466 100644 --- a/src/pkg/github/prdetailcache.go +++ b/src/pkg/github/prdetailcache.go @@ -101,7 +101,7 @@ func (c *prDetailCache) getLocked(repo string, number int, ttl time.Duration, al return nil, false } now := c.now() - expired := now.Sub(e.fetchedAt) >= ttl && !sharedWebhookTracker.cleanAndHealthy(repo, number) + expired := now.Sub(e.fetchedAt) >= ttl && !sharedWebhookTracker().cleanAndHealthy(repo, number) if expired || !match(e) || (!allowUnknownMergeable && strings.EqualFold(strings.TrimSpace(e.pr.GetMergeableState()), "unknown")) { c.misses.Add(1) return nil, false diff --git a/src/pkg/github/review_evidence_change_failure.go b/src/pkg/github/review_evidence_change_failure.go index 64e1084b91..489ef7bb77 100644 --- a/src/pkg/github/review_evidence_change_failure.go +++ b/src/pkg/github/review_evidence_change_failure.go @@ -294,7 +294,7 @@ func webhookMergedPR(event string, p webhookEventPayload) (MergedPRText, bool) { out.MergedAt = p.PullRequest.MergedAt.UTC() } if out.MergedAt.IsZero() { - out.MergedAt = sharedWebhookTracker.now().UTC() + out.MergedAt = sharedWebhookTracker().now().UTC() } return out, true } diff --git a/src/pkg/github/webhook_cache.go b/src/pkg/github/webhook_cache.go index 92cace1e19..dd9f29d45e 100644 --- a/src/pkg/github/webhook_cache.go +++ b/src/pkg/github/webhook_cache.go @@ -6,6 +6,7 @@ import ( "sort" "strings" "sync" + "sync/atomic" "time" itemblocker "github.com/hivecommons/hive/pkg/blocker" @@ -131,7 +132,15 @@ func newWebhookTracker() *webhookTracker { } } -var sharedWebhookTracker = newWebhookTracker() +// webhookTrackerPtr is atomic because tests swap the tracker while goroutines +// from earlier tests (a refresher flush) may still be reading it. +var webhookTrackerPtr = func() *atomic.Pointer[webhookTracker] { + p := &atomic.Pointer[webhookTracker]{} + p.Store(newWebhookTracker()) + return p +}() + +func sharedWebhookTracker() *webhookTracker { return webhookTrackerPtr.Load() } // SetWebhookHealthWindow sets how recent the last webhook event must be for // webhooks to count as healthy. The governor publishes 2 × eval_interval_s. @@ -139,7 +148,7 @@ func SetWebhookHealthWindow(d time.Duration) { if d <= 0 { d = DefaultWebhookHealthWindow } - t := sharedWebhookTracker + t := sharedWebhookTracker() t.mu.Lock() t.window = d t.mu.Unlock() @@ -147,7 +156,7 @@ func SetWebhookHealthWindow(d time.Duration) { // WebhookHealthSnapshot returns the current webhook health block. func WebhookHealthSnapshot() WebhookHealth { - return sharedWebhookTracker.snapshot() + return sharedWebhookTracker().snapshot() } // ObserveWebhookHealth reports whether webhooks are healthy and whether this @@ -155,7 +164,7 @@ func WebhookHealthSnapshot() WebhookHealth { // (stale → healthy, healthy → stale), so callers can log exactly once per // transition. func ObserveWebhookHealth() (healthy, becameHealthy, becameStale bool) { - return sharedWebhookTracker.observe() + return sharedWebhookTracker().observe() } // RecordWebhookEventForTest records a delivery for repo as if a verified @@ -163,7 +172,7 @@ func ObserveWebhookHealth() (healthy, becameHealthy, becameStale bool) { // without a GitHub client. func RecordWebhookEventForTest(repo string) { key := canonicalPRDetailRepo(repo) - sharedWebhookTracker.recordEvent([]string{key}) + sharedWebhookTracker().recordEvent([]string{key}) } func (t *webhookTracker) recordEvent(aliases []string) { @@ -193,14 +202,14 @@ func (t *webhookTracker) recordInvalidations(n int) { // RecordWebhookTargetedRefreshes counts n single-item GitHub reads made on // behalf of webhook deliveries (/api/status webhooks.targeted_refreshes_1h). func RecordWebhookTargetedRefreshes(n int) { - t := sharedWebhookTracker + t := sharedWebhookTracker() t.recordStamped(&t.refreshes, n) } // RecordWebhookTrackerInvalidations counts n stage-tracker cache drops caused // by webhook deliveries (/api/status webhooks.tracker_invalidations_1h). func RecordWebhookTrackerInvalidations(n int) { - t := sharedWebhookTracker + t := sharedWebhookTracker() t.recordStamped(&t.trackerDrops, n) } @@ -268,7 +277,7 @@ func (t *webhookTracker) takeDirtyIssues() int { // a non-zero answer to run early instead of waiting out its interval; polling // stays the fallback when webhooks are not delivered. func TakeDirtyIssueCount() int { - return sharedWebhookTracker.takeDirtyIssues() + return sharedWebhookTracker().takeDirtyIssues() } // clearDirtyBefore clears a dirty mark set before since. A mark set after the @@ -388,13 +397,13 @@ func pruneWebhookTimes(in []time.Time, now time.Time) []time.Time { } func resetWebhookTrackerForTest(now func() time.Time) func() { - old := sharedWebhookTracker + old := sharedWebhookTracker() t := newWebhookTracker() if now != nil { t.now = now } - sharedWebhookTracker = t - return func() { sharedWebhookTracker = old } + webhookTrackerPtr.Store(t) + return func() { webhookTrackerPtr.Store(old) } } // ResetWebhookTrackerForTest swaps in a fresh process-wide webhook tracker and @@ -508,7 +517,7 @@ func (c *Client) HandleWebhookEvent(event string, payload []byte) (WebhookEventR res.KnownRepo = true res.ConfiguredRepo = configured aliases := webhookRepoAliases(configured, p.Repository.FullName) - sharedWebhookTracker.recordEvent(aliases) + sharedWebhookTracker().recordEvent(aliases) res.Item = webhookItemHint(event, p) if n, ok := webhookTouchedIssue(event, p); ok { @@ -540,16 +549,16 @@ func (c *Client) HandleWebhookEvent(event string, payload []byte) (WebhookEventR } if n, ok := webhookIssueDirtyTarget(event, p); ok { - sharedWebhookTracker.markIssueDirty(aliases, n) + sharedWebhookTracker().markIssueDirty(aliases, n) res.DirtyIssues = []int{n} } if n, ok := webhookIssueReopenedTarget(event, p); ok { res.ReopenedIssue = n if hook := issueReopenedHook(); hook != nil { - hook(configured, n, sharedWebhookTracker.now()) + hook(configured, n, sharedWebhookTracker().now()) } } - if d, ok := c.webhookPRClaimDelivery(event, configured, p, sharedWebhookTracker.now()); ok { + if d, ok := c.webhookPRClaimDelivery(event, configured, p, sharedWebhookTracker().now()); ok { // The PR's claims take effect now, not on the next reconcile pass // (hivecommons/hive#14814); the live open-PR index for the repo is // stale the same instant, so the next guard call lists again. @@ -577,7 +586,7 @@ func (c *Client) HandleWebhookEvent(event string, payload []byte) (WebhookEventR for _, n := range numbers { c.invalidateWebhookPR(aliases, n) } - sharedWebhookTracker.recordInvalidations(len(numbers)) + sharedWebhookTracker().recordInvalidations(len(numbers)) res.Invalidated = numbers return res, nil } @@ -664,7 +673,7 @@ func webhookIssueCommentBody(event string, p webhookEventPayload) (string, time. if event != "issue_comment" || p.Comment == nil || (p.Action != "created" && p.Action != "edited") { return "", time.Time{}, false } - at := sharedWebhookTracker.now() + at := sharedWebhookTracker().now() if p.Comment.CreatedAt != nil && !p.Comment.CreatedAt.IsZero() { at = *p.Comment.CreatedAt } @@ -799,7 +808,7 @@ func (c *Client) invalidateWebhookPR(aliases []string, number int) { sharedPRDetailCache.invalidate(repo, number) c.invalidateGraphQLPRBatchPR(repo, number) } - sharedWebhookTracker.markDirty(aliases, number) + sharedWebhookTracker().markDirty(aliases, number) } // webhookCIFailureResult pairs a failed-check delivery with the PRs it diff --git a/src/pkg/github/webhook_cache_test.go b/src/pkg/github/webhook_cache_test.go index af79112e02..8f55a3347e 100644 --- a/src/pkg/github/webhook_cache_test.go +++ b/src/pkg/github/webhook_cache_test.go @@ -60,7 +60,7 @@ func seedWebhookTestPRs(c *Client, updated time.Time) { } func webhookTestDirty(repo string, number int) bool { - t := sharedWebhookTracker + t := sharedWebhookTracker() t.mu.Lock() defer t.mu.Unlock() _, ok := t.dirty[canonicalPRDetailRepo(repo)][number] @@ -399,12 +399,12 @@ func TestWebhookDirtyMarkSurvivesInFlightFetch(t *testing.T) { defer resetWebhookTrackerForTest(clock.Now)() started := clock.now clock.now = clock.now.Add(time.Second) - sharedWebhookTracker.markDirty([]string{"repo"}, 4) - sharedWebhookTracker.clearDirtyBefore("repo", 4, started) + sharedWebhookTracker().markDirty([]string{"repo"}, 4) + sharedWebhookTracker().clearDirtyBefore("repo", 4, started) if !webhookTestDirty("repo", 4) { t.Fatal("an event that lands after the fetch started must stay dirty") } - sharedWebhookTracker.clearDirtyBefore("repo", 4, clock.now.Add(time.Second)) + sharedWebhookTracker().clearDirtyBefore("repo", 4, clock.now.Add(time.Second)) if webhookTestDirty("repo", 4) { t.Fatal("a mark older than the fetch should clear") } diff --git a/src/pkg/github/webhook_mode.go b/src/pkg/github/webhook_mode.go index 5b50d67d52..3750f8281b 100644 --- a/src/pkg/github/webhook_mode.go +++ b/src/pkg/github/webhook_mode.go @@ -168,13 +168,13 @@ func (m *webhookModeMachine) status(window time.Duration, now time.Time) Webhook // mode plus whether this observation degraded to polling or recovered to // webhook mode, so the caller logs exactly once per transition. func ObserveWebhookMode() (status WebhookModeStatus, degraded, recovered bool) { - tr := sharedWebhookTracker.observeMode() + tr := sharedWebhookTracker().observeMode() return tr.Status, tr.Degraded, tr.Recovered } // WebhookModeSnapshot reports the cadence mode without advancing it. func WebhookModeSnapshot() WebhookModeStatus { - return sharedWebhookTracker.modeSnapshot() + return sharedWebhookTracker().modeSnapshot() } // RecordGitHubActivity notes the newest `updated_at` a poll or reconcile @@ -184,7 +184,7 @@ func RecordGitHubActivity(updatedAt time.Time) { if updatedAt.IsZero() { return } - t := sharedWebhookTracker + t := sharedWebhookTracker() t.mu.Lock() if updatedAt.After(t.lastActivity) { t.lastActivity = updatedAt @@ -197,7 +197,7 @@ func RecordGitHubActivity(updatedAt time.Time) { // that did not verify or a payload that did not decode. Failures newer than // the newest good delivery degrade the cadence to polling. func RecordWebhookDeliveryFailure() { - t := sharedWebhookTracker + t := sharedWebhookTracker() t.mu.Lock() t.lastFailure = t.now() t.mu.Unlock()