Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-automerge-once-per-head-memory.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- automerge: the lanes' "once per PR head" memory now lasts for the life of the hive process instead of one sweep. The queued, trusted-author and contributor lanes run every minute on a fresh engine, and the memory lived on that engine, so it was empty on every tick. A refused `update-branch` (a fork head behind a base that changed workflow files) was sent again every sweep: projectbluefin `fsdk-containers#508` logged "update-branch refused" about every 75 s. A merge GitHub refused for want of an approving review was sent again every minute instead of every 30 minutes. The trusted-author merge-forward was tried every sweep instead of once per head. "Logged once per head" holds, such as "merge train: PR left out of every train", were logged on every sweep (about 15 PRs every 80 s). The memory now lives on the long-lived GitHub client and is shared by every lane's engine. A read-only integration-eligibility check keeps its own memory, so a check that makes no write never marks a head as tried.
10 changes: 10 additions & 0 deletions src/pkg/github/automerge/automerge_contributor_lane_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,10 @@ type contributorLaneFixture struct {
labelsCreated int
labelsAdded []string
labelsRemoved []string
// updateStatus, when set, is the status every PUT update-branch
// answers with, carrying updateMessage.
updateStatus int
updateMessage string
}

func newContributorLaneAPI(t *testing.T, fx *contributorLaneFixture) *httptest.Server {
Expand Down Expand Up @@ -436,6 +440,12 @@ func newContributorLaneAPI(t *testing.T, fx *contributorLaneFixture) *httptest.S
json.NewEncoder(w).Encode(commit)
case r.Method == http.MethodPut && r.URL.Path == "/repos/acme/widget/pulls/7/update-branch":
fx.updates++
if fx.updateStatus != 0 {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(fx.updateStatus)
json.NewEncoder(w).Encode(map[string]any{"message": fx.updateMessage})
return
}
json.NewEncoder(w).Encode(map[string]any{"message": "Updating pull request branch."})
case r.Method == http.MethodPost && r.URL.Path == "/repos/acme/widget/labels":
fx.labelsCreated++
Expand Down
11 changes: 1 addition & 10 deletions src/pkg/github/automerge/automerge_held_runs.go
Original file line number Diff line number Diff line change
Expand Up @@ -677,16 +677,7 @@ func (c *Engine) markRebaseNudged(owner, repo string, number int, headSHA string
// and reports whether this is the first report.
func (c *Engine) markTerminalHoldLogged(owner, repo string, number int, headSHA, reason string) bool {
key := c.evaluatedHeadKey(owner, repo, number) + "@" + strings.ToLower(headSHA) + "@" + reason
c.heldRunApprovalsMu.Lock()
defer c.heldRunApprovalsMu.Unlock()
if c.terminalHoldsLogged == nil {
c.terminalHoldsLogged = make(map[string]bool)
}
if c.terminalHoldsLogged[key] {
return false
}
c.terminalHoldsLogged[key] = true
return true
return c.memory().markHoldLogged(key, c.clock())
}

// reportTerminalHold logs a terminal hold once per PR head and reason, and
Expand Down
3 changes: 3 additions & 0 deletions src/pkg/github/automerge/automerge_integration_eligibility.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ func integrationEligibilityProbe(ctx context.Context) bool {
func (c *Engine) integrationReadOnlyOptions() Options {
options := c.integrationOptions
options.MutationBoundary = integrationEligibilityReadOnlyBoundary{}
// Its one-per-head marks (a merge-forward tried, a hold logged) must
// not reach the sweeps: the probe makes no write.
options.isolatedSweepMemory = true
options.StageTracker = NewStageTracker()
options.BasePreparation = NewBasePreparationStore("")
// A private held-run ledger: the contributor lane claims a head before it
Expand Down
23 changes: 8 additions & 15 deletions src/pkg/github/automerge/automerge_merge_forward.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ func (c *Engine) mergeForwardTrustedHumanFor(repo string) bool {
return c.mergeForwardTrustedHuman(repo)
}

// mergeForwardKey namespaces the per-head attempt in heldRunApprovals so it
// never collides with the rebase-nudge or bot-takeover entries.
// mergeForwardKey namespaces the per-head attempt in the sweep memory
// (automerge_sweep_memory.go), which every lane's engine shares.
func mergeForwardKey(prKey, headSHA string) string {
return prKey + "@" + headSHA + "#merge-forward"
}
Expand All @@ -48,22 +48,15 @@ func mergeForwardWorkflowRefusedKey(prKey, headSHA string) string {

func (c *Engine) mergeForwardWorkflowRefused(owner, repo string, number int, headSHA string) bool {
key := mergeForwardWorkflowRefusedKey(c.evaluatedHeadKey(owner, repo, number), headSHA)
c.heldRunApprovalsMu.Lock()
defer c.heldRunApprovalsMu.Unlock()
return c.heldRunApprovals[key]
return c.memory().hasOnce(key, c.clock())
}

// markMergeForwardOnce records key in the sweep memory and reports whether
// this is its first use. The memory outlives the one-shot engine the
// trusted-author lane runs on each tick, so a head is tried once per
// process, not once per sweep.
func (c *Engine) markMergeForwardOnce(key string) bool {
c.heldRunApprovalsMu.Lock()
defer c.heldRunApprovalsMu.Unlock()
if c.heldRunApprovals == nil {
c.heldRunApprovals = make(map[string]bool)
}
if c.heldRunApprovals[key] {
return false
}
c.heldRunApprovals[key] = true
return true
return c.memory().markOnce(key, c.clock())
}

// mergeForwardOrConflict handles a trusted-author PR GitHub reports as
Expand Down
61 changes: 33 additions & 28 deletions src/pkg/github/automerge/automerge_merge_refusal.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,12 +147,13 @@ func (c *Engine) requiredChecksExpected(a mergeAttempt, err error) bool {
// mergeable, so a "blocked"-only memory was never consulted there and the
// merge was re-PUT every pass (hivecommons/hive#15419).
func (c *Engine) checksExpectedKnown(a mergeAttempt, now time.Time) bool {
r, ok := c.mergeRefusals[mergeRefusalChecksKey(a)]
m := c.memory()
r, ok := m.mergeRefusals[mergeRefusalChecksKey(a)]
if !ok {
return false
}
if now.Sub(r.at) >= mergeRefusalRecheck || !strings.EqualFold(r.head, strings.TrimSpace(a.head)) || comparableMergeableState(a.state) != r.state {
delete(c.mergeRefusals, mergeRefusalChecksKey(a))
delete(m.mergeRefusals, mergeRefusalChecksKey(a))
return false
}
return true
Expand Down Expand Up @@ -187,27 +188,28 @@ func (c *Engine) knownMergeRefusal(a mergeAttempt) string {
return autoMergeReasonMergeQueueRequired
}
now := c.clock()
c.mergeRefusalsMu.Lock()
defer c.mergeRefusalsMu.Unlock()
m := c.memory()
m.mergeRefusalsMu.Lock()
defer m.mergeRefusalsMu.Unlock()
if c.checksExpectedKnown(a, now) {
return autoMergeReasonChecksExpected
}
if key := mergeRefusalMethodKey(a); strings.TrimSpace(a.method) != "" {
if r, ok := c.mergeRefusals[key]; ok {
if r, ok := m.mergeRefusals[key]; ok {
if now.Sub(r.at) < mergeRefusalRecheck {
return autoMergeReasonMergeRefusedMethod
}
delete(c.mergeRefusals, key)
delete(m.mergeRefusals, key)
}
}
key := mergeRefusalReviewKey(a)
r, ok := c.mergeRefusals[key]
r, ok := m.mergeRefusals[key]
if !ok {
return ""
}
state := comparableMergeableState(a.state)
if now.Sub(r.at) >= mergeRefusalRecheck || !strings.EqualFold(r.head, strings.TrimSpace(a.head)) || (state != "" && state != r.state) {
delete(c.mergeRefusals, key)
delete(m.mergeRefusals, key)
return ""
}
return autoMergeReasonMergeRefusedReview
Expand Down Expand Up @@ -251,20 +253,21 @@ func (c *Engine) refusedMerge(a mergeAttempt, err error) string {
// entries once the map is large.
func (c *Engine) rememberMergeRefusal(key string, r mergeRefusal) {
r.at = c.clock()
c.mergeRefusalsMu.Lock()
defer c.mergeRefusalsMu.Unlock()
if c.mergeRefusals == nil {
c.mergeRefusals = make(map[string]mergeRefusal)
m := c.memory()
m.mergeRefusalsMu.Lock()
defer m.mergeRefusalsMu.Unlock()
if m.mergeRefusals == nil {
m.mergeRefusals = make(map[string]mergeRefusal)
}
if len(c.mergeRefusals) >= mergeRefusalPruneAt {
if len(m.mergeRefusals) >= mergeRefusalPruneAt {
// Entries for PRs that merged or closed are never read again.
for k, old := range c.mergeRefusals {
for k, old := range m.mergeRefusals {
if r.at.Sub(old.at) >= mergeRefusalRecheck {
delete(c.mergeRefusals, k)
delete(m.mergeRefusals, k)
}
}
}
c.mergeRefusals[key] = r
m.mergeRefusals[key] = r
}

func nativeQueueMethodRefusalKey(claim effects.Claim) string {
Expand All @@ -281,14 +284,15 @@ func (c *Engine) knownNativeQueueMethodRefusal(claim effects.Claim, head string)
}
key := nativeQueueMethodRefusalKey(claim)
now := c.clock()
c.mergeRefusalsMu.Lock()
defer c.mergeRefusalsMu.Unlock()
r, ok := c.mergeRefusals[key]
m := c.memory()
m.mergeRefusalsMu.Lock()
defer m.mergeRefusalsMu.Unlock()
r, ok := m.mergeRefusals[key]
if !ok {
return false
}
if now.Sub(r.at) >= mergeRefusalRecheck || !strings.EqualFold(r.head, strings.TrimSpace(head)) {
delete(c.mergeRefusals, key)
delete(m.mergeRefusals, key)
return false
}
return true
Expand All @@ -301,17 +305,18 @@ func (c *Engine) noteNativeQueueMethodRefusal(claim effects.Claim, head string)
return
}
r := mergeRefusal{head: strings.TrimSpace(head), at: c.clock()}
c.mergeRefusalsMu.Lock()
defer c.mergeRefusalsMu.Unlock()
if c.mergeRefusals == nil {
c.mergeRefusals = make(map[string]mergeRefusal)
m := c.memory()
m.mergeRefusalsMu.Lock()
defer m.mergeRefusalsMu.Unlock()
if m.mergeRefusals == nil {
m.mergeRefusals = make(map[string]mergeRefusal)
}
if len(c.mergeRefusals) >= mergeRefusalPruneAt {
for k, old := range c.mergeRefusals {
if len(m.mergeRefusals) >= mergeRefusalPruneAt {
for k, old := range m.mergeRefusals {
if r.at.Sub(old.at) >= mergeRefusalRecheck {
delete(c.mergeRefusals, k)
delete(m.mergeRefusals, k)
}
}
}
c.mergeRefusals[nativeQueueMethodRefusalKey(claim)] = r
m.mergeRefusals[nativeQueueMethodRefusalKey(claim)] = r
}
23 changes: 14 additions & 9 deletions src/pkg/github/automerge/automerge_sweep.go
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,10 @@ type Options struct {
// again; nil records nothing.
HotspotRecorder HotspotRecorder
IntegrationFilesObserved func(repo, head string, files []string)
// isolatedSweepMemory gives the engine a sweep memory of its own
// instead of the transport's shared one: a read-only evaluation must
// not mark a head as tried (automerge_sweep_memory.go).
isolatedSweepMemory bool
}

// Engine owns the automerge sweep policy state.
Expand Down Expand Up @@ -430,9 +434,6 @@ type Engine struct {
workflowApprovalScans map[string]time.Time
workflowApprovalUntrusted map[string]bool
workflowApprovalWorkflowChecks map[string]okToTestWorkflowCheck
// terminalHoldsLogged remembers PR head+reason pairs already reported as a
// terminal held/never-started hold so later ticks only count them.
terminalHoldsLogged map[string]bool
// unauthorized remembers, per PR, the head and step this process marked
// `hive:unauthorized` for, so the same head logs once and the mark
// clears when the step is done; unauthorizedLabelReady remembers the
Expand All @@ -445,12 +446,13 @@ type Engine struct {
// a REST merge that cannot succeed (hivecommons/hive#14068).
mergeQueueBasesMu sync.Mutex
mergeQueueBases *MergeQueueBases
// mergeRefusals remembers merges GitHub refused because an approving
// review is required (per PR head) or the merge method is disabled (per
// repository and method), so no sweep re-issues them every pass
// (automerge_merge_refusal.go).
mergeRefusalsMu sync.Mutex
mergeRefusals map[string]mergeRefusal
// sweepMemory is what the lanes remember across sweeps: merge-forward
// and refused update-branch marks per head, the merges GitHub refused
// (automerge_merge_refusal.go) and the holds already logged. New shares
// it through the transport so the one-shot engine each tick builds
// keeps it (automerge_sweep_memory.go).
sweepMemoryOnce sync.Once
sweepMemory *sweepMemory
// adoptionOriginalChecks throttles the self lane's lookup of an
// adoption PR's original (automerge_adoption_original.go).
adoptionOriginalChecksMu sync.Mutex
Expand Down Expand Up @@ -606,6 +608,9 @@ func New(transport Transport, opts Options) *Engine {
e.mutation = provider.MutationBoundary()
}
}
if !opts.isolatedSweepMemory {
e.sweepMemory = sharedSweepMemory(transport)
}
return e
}

Expand Down
125 changes: 125 additions & 0 deletions src/pkg/github/automerge/automerge_sweep_memory.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
package automerge

import (
"sync"
"time"
)

// sweepMemory is what the lanes remember across sweeps about a PR head:
// the merge-forward tried on it, an update-branch or merge-forward GitHub
// refused for lack of the workflows permission, a merge GitHub refused
// (automerge_merge_refusal.go), and the holds already logged.
//
// It is shared by every engine built over the same GitHub client, not kept
// on one engine. The hive runs the queued, trusted-author and contributor
// lanes every minute on a one-shot engine (SweepTrustedAuthorAutoMerges,
// SweepContributorLaneAutoMerges, cmd/hive runAutoMergeSweepIfDue), next to
// the long-lived self-authored loop, so a mark on the engine was empty
// again on the next tick: projectbluefin fsdk-containers#508 sent the same
// refused update-branch to GitHub every ~75 s, and the "logged once per
// head" holds were logged every sweep.
//
// An engine over a transport that does not share state (the sweep's test
// fakes, a nil transport) keeps its own memory, as before. A read-only
// evaluation (IntegrationEligibility) gets its own too, so a probe that
// cannot act never marks a head as tried.
type sweepMemory struct {
// onceMu guards once and holdsLogged: key -> when it was last used.
onceMu sync.Mutex
once map[string]time.Time
holdsLogged map[string]time.Time
// mergeRefusalsMu guards mergeRefusals (automerge_merge_refusal.go).
mergeRefusalsMu sync.Mutex
mergeRefusals map[string]mergeRefusal
}

// sweepMemoryStateName is the name the memory is kept under on the client
// (hgithub.Client.SharedState).
const sweepMemoryStateName = "automerge.sweep-memory"

// sweepMemoryIdle is how long a once-per-head mark nothing has used stays:
// past it the PR merged, closed or moved to another head long ago.
const sweepMemoryIdle = 24 * time.Hour

// sweepMemoryPruneAt is the entry count at which idle marks are dropped on
// the next insert.
const sweepMemoryPruneAt = 1024

func newSweepMemory() *sweepMemory { return &sweepMemory{} }

// sharedSweepMemory returns the memory the transport keeps for every engine
// built over it, or nil when it keeps none.
func sharedSweepMemory(transport Transport) *sweepMemory {
provider, ok := transport.(interface {
SharedState(name string, build func() any) any
})
if !ok {
return nil
}
m, _ := provider.SharedState(sweepMemoryStateName, func() any { return newSweepMemory() }).(*sweepMemory)
return m
}

// memory returns the engine's sweep memory, building a private one for an
// engine not made by New.
func (c *Engine) memory() *sweepMemory {
c.sweepMemoryOnce.Do(func() {
if c.sweepMemory == nil {
c.sweepMemory = newSweepMemory()
}
})
return c.sweepMemory
}

// markOnce records key and reports whether this is its first use.
func (m *sweepMemory) markOnce(key string, now time.Time) bool {
m.onceMu.Lock()
defer m.onceMu.Unlock()
if _, ok := m.once[key]; ok {
m.once[key] = now
return false
}
m.once = sweepMemoryInsert(m.once, key, now)
return true
}

// hasOnce reports whether key was recorded.
func (m *sweepMemory) hasOnce(key string, now time.Time) bool {
m.onceMu.Lock()
defer m.onceMu.Unlock()
if _, ok := m.once[key]; !ok {
return false
}
m.once[key] = now
return true
}

// markHoldLogged records a logged hold and reports whether this is its
// first report.
func (m *sweepMemory) markHoldLogged(key string, now time.Time) bool {
m.onceMu.Lock()
defer m.onceMu.Unlock()
if _, ok := m.holdsLogged[key]; ok {
m.holdsLogged[key] = now
return false
}
m.holdsLogged = sweepMemoryInsert(m.holdsLogged, key, now)
return true
}

// sweepMemoryInsert adds key to set, first dropping marks idle past
// sweepMemoryIdle once the set is large. The caller holds the lock.
func sweepMemoryInsert(set map[string]time.Time, key string, now time.Time) map[string]time.Time {
if set == nil {
set = make(map[string]time.Time)
}
if len(set) >= sweepMemoryPruneAt {
for k, at := range set {
if now.Sub(at) >= sweepMemoryIdle {
delete(set, k)
}
}
}
set[key] = now
return set
}
Loading
Loading