Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-16429.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- The process-wide webhook tracker is now swapped and read atomically, removing a data race that failed `TestClosedDeliveryEvictsWithWebhookRefreshDisabled` under `-race` when a refresher flush from an earlier test was still reading it ([#16429](https://github.com/hivecommons/hive/issues/16429)).
4 changes: 2 additions & 2 deletions src/pkg/github/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -2605,13 +2605,13 @@ func (c *Client) enrichPRCI(ctx context.Context, pr *PullRequest) map[string]boo
full, ok := c.cachedPRDetailFromList(pr.Repo, pr.Number, pr.HeadSHA, pr.UpdatedAt)
if !ok {
var err error
started := sharedWebhookTracker.now()
started := sharedWebhookTracker().now()
full, _, err = c.client.PullRequests.Get(WithRESTCaller(ctx, "hive:enrich_pr_ci"), owner, repoName, pr.Number)
if err != nil {
c.logger.Warn("failed to fetch PR mergeability", "repo", pr.Repo, "pr", pr.Number, "error", err)
} else {
c.storePRDetail(pr.Repo, pr.Number, full)
sharedWebhookTracker.clearDirtyBefore(pr.Repo, pr.Number, started)
sharedWebhookTracker().clearDirtyBefore(pr.Repo, pr.Number, started)
}
}
if full != nil {
Expand Down
6 changes: 3 additions & 3 deletions src/pkg/github/pr_batch_graphql.go
Original file line number Diff line number Diff line change
Expand Up @@ -170,11 +170,11 @@ func (c *Client) prefetchOpenPRDetailsGraphQL(ctx context.Context, owner, repoNa
}
// With healthy webhooks and nothing marked dirty since the last batch,
// the cached batch still describes every open PR (#11177).
if sharedWebhookTracker.repoCleanAndHealthy(fullRepo) && c.hasGraphQLPRBatchRepo(fullRepo) {
if sharedWebhookTracker().repoCleanAndHealthy(fullRepo) && c.hasGraphQLPRBatchRepo(fullRepo) {
c.recordGraphQLPRBatchWebhookSkip()
return
}
started := sharedWebhookTracker.now()
started := sharedWebhookTracker().now()
c.clearGraphQLPRBatchRepo(fullRepo)
pageSize := c.graphQLPRBatchPageSizeEffective()
var cursor string
Expand Down Expand Up @@ -208,7 +208,7 @@ func (c *Client) prefetchOpenPRDetailsGraphQL(ctx context.Context, owner, repoNa
}
}
c.recordGraphQLPRBatchSuccess(repoPRs, pages)
sharedWebhookTracker.clearDirtyRepoBefore(fullRepo, started)
sharedWebhookTracker().clearDirtyRepoBefore(fullRepo, started)
}

func (c *Client) hasGraphQLPRBatchRepo(repo string) bool {
Expand Down
2 changes: 1 addition & 1 deletion src/pkg/github/prdetailcache.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ func (c *prDetailCache) getLocked(repo string, number int, ttl time.Duration, al
return nil, false
}
now := c.now()
expired := now.Sub(e.fetchedAt) >= ttl && !sharedWebhookTracker.cleanAndHealthy(repo, number)
expired := now.Sub(e.fetchedAt) >= ttl && !sharedWebhookTracker().cleanAndHealthy(repo, number)
if expired || !match(e) || (!allowUnknownMergeable && strings.EqualFold(strings.TrimSpace(e.pr.GetMergeableState()), "unknown")) {
c.misses.Add(1)
return nil, false
Expand Down
2 changes: 1 addition & 1 deletion src/pkg/github/review_evidence_change_failure.go
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,7 @@ func webhookMergedPR(event string, p webhookEventPayload) (MergedPRText, bool) {
out.MergedAt = p.PullRequest.MergedAt.UTC()
}
if out.MergedAt.IsZero() {
out.MergedAt = sharedWebhookTracker.now().UTC()
out.MergedAt = sharedWebhookTracker().now().UTC()
}
return out, true
}
45 changes: 27 additions & 18 deletions src/pkg/github/webhook_cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"sort"
"strings"
"sync"
"sync/atomic"
"time"

itemblocker "github.com/hivecommons/hive/pkg/blocker"
Expand Down Expand Up @@ -131,39 +132,47 @@ func newWebhookTracker() *webhookTracker {
}
}

var sharedWebhookTracker = newWebhookTracker()
// webhookTrackerPtr is atomic because tests swap the tracker while goroutines
// from earlier tests (a refresher flush) may still be reading it.
var webhookTrackerPtr = func() *atomic.Pointer[webhookTracker] {
p := &atomic.Pointer[webhookTracker]{}
p.Store(newWebhookTracker())
return p
}()

func sharedWebhookTracker() *webhookTracker { return webhookTrackerPtr.Load() }

// SetWebhookHealthWindow sets how recent the last webhook event must be for
// webhooks to count as healthy. The governor publishes 2 × eval_interval_s.
func SetWebhookHealthWindow(d time.Duration) {
if d <= 0 {
d = DefaultWebhookHealthWindow
}
t := sharedWebhookTracker
t := sharedWebhookTracker()
t.mu.Lock()
t.window = d
t.mu.Unlock()
}

// WebhookHealthSnapshot returns the current webhook health block.
func WebhookHealthSnapshot() WebhookHealth {
return sharedWebhookTracker.snapshot()
return sharedWebhookTracker().snapshot()
}

// ObserveWebhookHealth reports whether webhooks are healthy and whether this
// observation is the first one since they changed state in either direction
// (stale → healthy, healthy → stale), so callers can log exactly once per
// transition.
func ObserveWebhookHealth() (healthy, becameHealthy, becameStale bool) {
return sharedWebhookTracker.observe()
return sharedWebhookTracker().observe()
}

// RecordWebhookEventForTest records a delivery for repo as if a verified
// webhook had arrived, so tests outside this package can drive webhook health
// without a GitHub client.
func RecordWebhookEventForTest(repo string) {
key := canonicalPRDetailRepo(repo)
sharedWebhookTracker.recordEvent([]string{key})
sharedWebhookTracker().recordEvent([]string{key})
}

func (t *webhookTracker) recordEvent(aliases []string) {
Expand Down Expand Up @@ -193,14 +202,14 @@ func (t *webhookTracker) recordInvalidations(n int) {
// RecordWebhookTargetedRefreshes counts n single-item GitHub reads made on
// behalf of webhook deliveries (/api/status webhooks.targeted_refreshes_1h).
func RecordWebhookTargetedRefreshes(n int) {
t := sharedWebhookTracker
t := sharedWebhookTracker()
t.recordStamped(&t.refreshes, n)
}

// RecordWebhookTrackerInvalidations counts n stage-tracker cache drops caused
// by webhook deliveries (/api/status webhooks.tracker_invalidations_1h).
func RecordWebhookTrackerInvalidations(n int) {
t := sharedWebhookTracker
t := sharedWebhookTracker()
t.recordStamped(&t.trackerDrops, n)
}

Expand Down Expand Up @@ -268,7 +277,7 @@ func (t *webhookTracker) takeDirtyIssues() int {
// a non-zero answer to run early instead of waiting out its interval; polling
// stays the fallback when webhooks are not delivered.
func TakeDirtyIssueCount() int {
return sharedWebhookTracker.takeDirtyIssues()
return sharedWebhookTracker().takeDirtyIssues()
}

// clearDirtyBefore clears a dirty mark set before since. A mark set after the
Expand Down Expand Up @@ -388,13 +397,13 @@ func pruneWebhookTimes(in []time.Time, now time.Time) []time.Time {
}

func resetWebhookTrackerForTest(now func() time.Time) func() {
old := sharedWebhookTracker
old := sharedWebhookTracker()
t := newWebhookTracker()
if now != nil {
t.now = now
}
sharedWebhookTracker = t
return func() { sharedWebhookTracker = old }
webhookTrackerPtr.Store(t)
return func() { webhookTrackerPtr.Store(old) }
}

// ResetWebhookTrackerForTest swaps in a fresh process-wide webhook tracker and
Expand Down Expand Up @@ -508,7 +517,7 @@ func (c *Client) HandleWebhookEvent(event string, payload []byte) (WebhookEventR
res.KnownRepo = true
res.ConfiguredRepo = configured
aliases := webhookRepoAliases(configured, p.Repository.FullName)
sharedWebhookTracker.recordEvent(aliases)
sharedWebhookTracker().recordEvent(aliases)

res.Item = webhookItemHint(event, p)
if n, ok := webhookTouchedIssue(event, p); ok {
Expand Down Expand Up @@ -540,16 +549,16 @@ func (c *Client) HandleWebhookEvent(event string, payload []byte) (WebhookEventR
}

if n, ok := webhookIssueDirtyTarget(event, p); ok {
sharedWebhookTracker.markIssueDirty(aliases, n)
sharedWebhookTracker().markIssueDirty(aliases, n)
res.DirtyIssues = []int{n}
}
if n, ok := webhookIssueReopenedTarget(event, p); ok {
res.ReopenedIssue = n
if hook := issueReopenedHook(); hook != nil {
hook(configured, n, sharedWebhookTracker.now())
hook(configured, n, sharedWebhookTracker().now())
}
}
if d, ok := c.webhookPRClaimDelivery(event, configured, p, sharedWebhookTracker.now()); ok {
if d, ok := c.webhookPRClaimDelivery(event, configured, p, sharedWebhookTracker().now()); ok {
// The PR's claims take effect now, not on the next reconcile pass
// (hivecommons/hive#14814); the live open-PR index for the repo is
// stale the same instant, so the next guard call lists again.
Expand Down Expand Up @@ -577,7 +586,7 @@ func (c *Client) HandleWebhookEvent(event string, payload []byte) (WebhookEventR
for _, n := range numbers {
c.invalidateWebhookPR(aliases, n)
}
sharedWebhookTracker.recordInvalidations(len(numbers))
sharedWebhookTracker().recordInvalidations(len(numbers))
res.Invalidated = numbers
return res, nil
}
Expand Down Expand Up @@ -664,7 +673,7 @@ func webhookIssueCommentBody(event string, p webhookEventPayload) (string, time.
if event != "issue_comment" || p.Comment == nil || (p.Action != "created" && p.Action != "edited") {
return "", time.Time{}, false
}
at := sharedWebhookTracker.now()
at := sharedWebhookTracker().now()
if p.Comment.CreatedAt != nil && !p.Comment.CreatedAt.IsZero() {
at = *p.Comment.CreatedAt
}
Expand Down Expand Up @@ -799,7 +808,7 @@ func (c *Client) invalidateWebhookPR(aliases []string, number int) {
sharedPRDetailCache.invalidate(repo, number)
c.invalidateGraphQLPRBatchPR(repo, number)
}
sharedWebhookTracker.markDirty(aliases, number)
sharedWebhookTracker().markDirty(aliases, number)
}

// webhookCIFailureResult pairs a failed-check delivery with the PRs it
Expand Down
8 changes: 4 additions & 4 deletions src/pkg/github/webhook_cache_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ func seedWebhookTestPRs(c *Client, updated time.Time) {
}

func webhookTestDirty(repo string, number int) bool {
t := sharedWebhookTracker
t := sharedWebhookTracker()
t.mu.Lock()
defer t.mu.Unlock()
_, ok := t.dirty[canonicalPRDetailRepo(repo)][number]
Expand Down Expand Up @@ -399,12 +399,12 @@ func TestWebhookDirtyMarkSurvivesInFlightFetch(t *testing.T) {
defer resetWebhookTrackerForTest(clock.Now)()
started := clock.now
clock.now = clock.now.Add(time.Second)
sharedWebhookTracker.markDirty([]string{"repo"}, 4)
sharedWebhookTracker.clearDirtyBefore("repo", 4, started)
sharedWebhookTracker().markDirty([]string{"repo"}, 4)
sharedWebhookTracker().clearDirtyBefore("repo", 4, started)
if !webhookTestDirty("repo", 4) {
t.Fatal("an event that lands after the fetch started must stay dirty")
}
sharedWebhookTracker.clearDirtyBefore("repo", 4, clock.now.Add(time.Second))
sharedWebhookTracker().clearDirtyBefore("repo", 4, clock.now.Add(time.Second))
if webhookTestDirty("repo", 4) {
t.Fatal("a mark older than the fetch should clear")
}
Expand Down
8 changes: 4 additions & 4 deletions src/pkg/github/webhook_mode.go
Original file line number Diff line number Diff line change
Expand Up @@ -168,13 +168,13 @@ func (m *webhookModeMachine) status(window time.Duration, now time.Time) Webhook
// mode plus whether this observation degraded to polling or recovered to
// webhook mode, so the caller logs exactly once per transition.
func ObserveWebhookMode() (status WebhookModeStatus, degraded, recovered bool) {
tr := sharedWebhookTracker.observeMode()
tr := sharedWebhookTracker().observeMode()
return tr.Status, tr.Degraded, tr.Recovered
}

// WebhookModeSnapshot reports the cadence mode without advancing it.
func WebhookModeSnapshot() WebhookModeStatus {
return sharedWebhookTracker.modeSnapshot()
return sharedWebhookTracker().modeSnapshot()
}

// RecordGitHubActivity notes the newest `updated_at` a poll or reconcile
Expand All @@ -184,7 +184,7 @@ func RecordGitHubActivity(updatedAt time.Time) {
if updatedAt.IsZero() {
return
}
t := sharedWebhookTracker
t := sharedWebhookTracker()
t.mu.Lock()
if updatedAt.After(t.lastActivity) {
t.lastActivity = updatedAt
Expand All @@ -197,7 +197,7 @@ func RecordGitHubActivity(updatedAt time.Time) {
// that did not verify or a payload that did not decode. Failures newer than
// the newest good delivery degrade the cadence to polling.
func RecordWebhookDeliveryFailure() {
t := sharedWebhookTracker
t := sharedWebhookTracker()
t.mu.Lock()
t.lastFailure = t.now()
t.mu.Unlock()
Expand Down
Loading