Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-16147-16152-head-skew-first-seen.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- The merge-request CI gate now clears the min_head_age hold one window after it first sees a head whose GitHub timestamp is ahead of the host clock even when the merge request omits `expect_sha`, and both merge paths log when a head time is ahead of the host clock so a skew-handled head is distinguishable from a normally aged one.
2 changes: 1 addition & 1 deletion src/docs/scale-envelope.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ ones to:
2. **Truncation must preserve the prioritization signal, not cut an arbitrary slice.**
This holds today and is easy to assume backwards:
- Issues are sorted by actionable rank tier, then **oldest first** within each tier before any cap is applied
(`RankActionableIssues`, `src/pkg/github/client.go:5012-5022`, descending `AgeMinutes` within each tier).
(`RankActionableIssues`, `src/pkg/github/client.go:5017-5027`, descending `AgeMinutes` within each tier).
- PRs are sorted by **review class then oldest-first within class**
(`SortPullRequestsForReview`, `src/pkg/github/review_priority.go:334`;
fixes → refactors/docs → tests).
Expand Down
4 changes: 4 additions & 0 deletions src/pkg/github/automerge/automerge_sweep_queued.go
Original file line number Diff line number Diff line change
Expand Up @@ -365,6 +365,10 @@ func (c *Engine) commitGreenForPRSkipping(ctx context.Context, owner, repo, bran
if err != nil {
return false, st.Reason, err
}
if prNumber > 0 && st.HeadTimeAheadBy > 0 {
c.info("automerge head time ahead of host clock; min_head_age not evaluable from it, measured from first observation", "repo", owner+"/"+repo, "pr", prNumber,
"sha", sha, "ahead_by", st.HeadTimeAheadBy.Round(time.Second), "green", st.Green, "reason", st.Reason)
}
if len(st.Reporting) > 0 {
c.info("automerge CI reporting observation", "repo", owner+"/"+repo, "pr", prNumber,
"head", sha, "observations", st.Reporting)
Expand Down
5 changes: 5 additions & 0 deletions src/pkg/github/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -463,6 +463,11 @@ type Client struct {
mergeStatePath string
mergeState map[string]mergeRequestPRState
mergeStateLoaded bool
// mergeHeadFirstSeen is the in-memory first observation of each PR head
// the CI gate judged without an expect_sha (no persisted wait state), so
// the min_head_age skew fallback has a stable start (#16152). Guarded by
// mergeStateMu.
mergeHeadFirstSeen map[string]mergeRequestPRState
// attribution holds the invocation-attribution hooks (trailer gate,
// per-agent metadata resolver, audit sink) — see attribution.go. Guarded by
// attribMu because the hooks are installed in stages during startup while
Expand Down
7 changes: 7 additions & 0 deletions src/pkg/github/commit_ci.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,12 @@ type CommitCIState struct {
// Every one of them postdates the push, so it is a safe lower bound on
// how long the head has existed.
EarliestActivity time.Time
// HeadTimeAheadBy is how far CommitCIOptions.HeadPushedAt was ahead of
// the host clock when min_head_age was evaluated (#16143). Non-zero means
// the head's age was not evaluable from GitHub's timestamp and the hold
// was measured from HeadFirstSeenAt instead; callers log it so a
// skew-handled head is distinguishable from a normally aged one.
HeadTimeAheadBy time.Duration
}

// CommitCIOptions carries policy facts that are known to callers before the
Expand Down Expand Up @@ -667,6 +673,7 @@ func EvaluateCommitCI(ctx context.Context, client *gh.Client, owner, repo, sha s
// for the whole skew.
skewed := age < 0
if skewed {
st.HeadTimeAheadBy = -age
age = 0
if !opts.HeadFirstSeenAt.IsZero() {
age = max(now().Sub(opts.HeadFirstSeenAt), 0)
Expand Down
29 changes: 26 additions & 3 deletions src/pkg/github/commit_ci_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ func TestEvaluateCommitCIFreshHeadExpectedChecks(t *testing.T) {
opts CommitCIOptions
wantGreen bool
want string
// wantAheadBy is the CommitCIState.HeadTimeAheadBy the gate must
// report, so a skew-handled head is observable (#16147).
wantAheadBy time.Duration
}{
{
name: "fresh head with zero slow checks waits for min age",
Expand All @@ -45,7 +48,8 @@ func TestEvaluateCommitCIFreshHeadExpectedChecks(t *testing.T) {
HeadPushedAt: now.Add(time.Hour),
Now: func() time.Time { return now },
},
want: "pending: head time ahead of host clock; head first seen 0s ago (< min_head_age)",
want: "pending: head time ahead of host clock; head first seen 0s ago (< min_head_age)",
wantAheadBy: time.Hour,
},
{
name: "head time ahead of host clock still holds while first seen under min age",
Expand All @@ -56,7 +60,8 @@ func TestEvaluateCommitCIFreshHeadExpectedChecks(t *testing.T) {
HeadFirstSeenAt: now.Add(-time.Minute),
Now: func() time.Time { return now },
},
want: "pending: head time ahead of host clock; head first seen 1m0s ago (< min_head_age)",
want: "pending: head time ahead of host clock; head first seen 1m0s ago (< min_head_age)",
wantAheadBy: time.Hour,
},
{
name: "head time ahead of host clock clears once first seen reaches min age",
Expand All @@ -67,7 +72,22 @@ func TestEvaluateCommitCIFreshHeadExpectedChecks(t *testing.T) {
HeadFirstSeenAt: now.Add(-3 * time.Minute),
Now: func() time.Time { return now },
},
wantGreen: true,
wantGreen: true,
wantAheadBy: time.Hour,
},
{
// #16147: a head cleared through the skew path must not look like
// a normally aged one; the state names how far ahead it was.
name: "head time ahead of host clock is reported when the hold clears",
checks: []map[string]string{{"name": "fast", "status": "completed", "conclusion": "success"}},
opts: CommitCIOptions{
MinHeadAge: 3 * time.Minute,
HeadPushedAt: now.Add(90 * time.Minute),
HeadFirstSeenAt: now.Add(-10 * time.Minute),
Now: func() time.Time { return now },
},
wantGreen: true,
wantAheadBy: 90 * time.Minute,
},
{
name: "first seen is ignored when the head time is not ahead of the host clock",
Expand Down Expand Up @@ -175,6 +195,9 @@ func TestEvaluateCommitCIFreshHeadExpectedChecks(t *testing.T) {
if st.Green != tt.wantGreen || st.Reason != tt.want {
t.Fatalf("EvaluateCommitCI = (%v,%q), want (%v,%q)", st.Green, st.Reason, tt.wantGreen, tt.want)
}
if st.HeadTimeAheadBy != tt.wantAheadBy {
t.Fatalf("HeadTimeAheadBy = %s, want %s", st.HeadTimeAheadBy, tt.wantAheadBy)
}
})
}
}
Expand Down
11 changes: 10 additions & 1 deletion src/pkg/github/merge_ci_gate.go
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,9 @@ func (c *Client) repoNoCIOK(owner, name string) bool {
// for the never-created required-check verdict. firstSeen is the watcher's
// clock when it first parked this head on a pending verdict (zero on the
// first poll); min_head_age falls back to it when GitHub's head time is ahead
// of the host clock (#16143). The returned reason is a human-readable
// of the host clock (#16143). Without expectSHA that wait state is never
// persisted, so the gate keeps its own first observation of the resolved head
// instead (#16152). The returned reason is a human-readable
// explanation suitable for the result file and the log. A non-nil error means
// the verdict could not be computed (API failure) and the caller must treat
// that as a failed attempt, never as a pass.
Expand Down Expand Up @@ -198,6 +200,9 @@ func (c *Client) verifyMergeRequestCI(ctx context.Context, repo string, number i
minHeadAgeFrom = at
}
}
if minHeadAge > 0 && firstSeen.IsZero() && strings.TrimSpace(expectSHA) == "" {
firstSeen = c.mergeRequestHeadFirstSeen(owner+"/"+name, number, sha, now)
}
st, err := EvaluateCommitCI(ctx, c.client, owner, name, sha, CommitCIOptions{
PRNumber: number,
Required: required,
Expand All @@ -215,6 +220,10 @@ func (c *Client) verifyMergeRequestCI(ctx context.Context, repo string, number i
if cfgKnown && actualKnown {
c.warnRequiredChecksMismatch(owner, name, cfgSet, actualRequired, st.Observed)
}
if st.HeadTimeAheadBy > 0 {
c.info("merge-request CI head time ahead of host clock; min_head_age not evaluable from it, measured from first observation", "repo", owner+"/"+name, "pr", number,
"sha", sha, "ahead_by", st.HeadTimeAheadBy.Round(time.Second), "first_seen", firstSeen, "green", st.Green, "reason", st.Reason)
}
if len(st.Reporting) > 0 {
c.info("merge-request CI reporting observation", "repo", owner+"/"+name, "pr", number,
"head", sha, "observations", st.Reporting)
Expand Down
52 changes: 50 additions & 2 deletions src/pkg/github/merge_ci_gate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ type ciFixture struct {
// (to exercise the GitHub-side refusal paths); zero merges successfully.
mergeStatus int
approveCalls *atomic.Int32
// updatedAt, when set, is the PR's updated_at (RFC 3339): the head time
// the min_head_age gate measures from.
updatedAt string
}

type ciStatus struct{ context, state string }
Expand Down Expand Up @@ -107,13 +110,17 @@ func (f *ciFixture) serveCI(w http.ResponseWriter, r *http.Request) bool {
if baseRepo == "" {
baseRepo = "o/r"
}
enc(map[string]any{
pr := map[string]any{
"number": n,
"state": "open",
"user": map[string]any{"login": author, "type": authorType},
"head": map[string]any{"sha": head, "repo": map[string]any{"full_name": headRepo}},
"base": map[string]any{"ref": "main", "repo": map[string]any{"full_name": baseRepo}},
})
}
if f.updatedAt != "" {
pr["updated_at"] = f.updatedAt
}
enc(pr)
case strings.Contains(p, "/branches/") && !strings.Contains(p, "/protection"): // /repos/{o}/{r}/branches/{branch}
if f.branchStatus != 0 {
w.WriteHeader(f.branchStatus)
Expand Down Expand Up @@ -1165,3 +1172,44 @@ func TestMergeCIGate_RequiredCheckNeverCreated(t *testing.T) {
})
}
}

// #16152: a merge request without expect_sha has no persisted wait state, so
// while GitHub's head time is ahead of the host clock the min_head_age hold
// must be measured from the gate's own first observation of the head, and
// clear after one window, instead of reading "first seen 0s ago" every poll
// for as long as the skew lasts.
func TestMergeCIGate_SkewedHeadWithoutExpectSHAHoldsFromFirstObservation(t *testing.T) {
f := greenFixture()
f.updatedAt = time.Now().Add(time.Hour).UTC().Format(time.RFC3339)
var merges atomic.Int32
srv := ciGateServer(t, f, &merges)
defer srv.Close()
c := testMergeClient(t, srv.URL)
c.SetAutoMergeMinHeadAge(10 * time.Minute)
ctx := context.Background()

verdict, why, err := c.verifyMergeRequestCI(ctx, "o/r", 100, "", time.Now().Add(-5*time.Minute), time.Time{})
if err != nil || verdict != mergeCIPending || !strings.Contains(why, "head time ahead of host clock; head first seen 5m") {
t.Fatalf("first poll = (%v, %q, %v), want pending held from the first observation 5m ago", verdict, why, err)
}
// A later poll without expect_sha still has no persisted PendingSince; the
// gate must keep its first observation rather than restart at 0s.
verdict, why, err = c.verifyMergeRequestCI(ctx, "o/r", 100, "", time.Now(), time.Time{})
if err != nil || verdict != mergeCIPending || !strings.Contains(why, "head first seen 5m") {
t.Fatalf("second poll = (%v, %q, %v), want the hold measured from the first observation", verdict, why, err)
}
c.SetAutoMergeMinHeadAge(4 * time.Minute)
verdict, why, err = c.verifyMergeRequestCI(ctx, "o/r", 100, "", time.Now(), time.Time{})
if err != nil || verdict != mergeCIGreen {
t.Fatalf("poll after one min_head_age window = (%v, %q, %v), want green", verdict, why, err)
}
// A new head is a new first observation: it is held again (fail closed).
f.heads[100] = "def456"
verdict, why, err = c.verifyMergeRequestCI(ctx, "o/r", 100, "", time.Now(), time.Time{})
if err != nil || verdict != mergeCIPending || !strings.Contains(why, "head first seen 0s ago") {
t.Fatalf("new head = (%v, %q, %v), want pending from a fresh first observation", verdict, why, err)
}
if got := merges.Load(); got != 0 {
t.Fatalf("verifyMergeRequestCI must never merge, got %d", got)
}
}
30 changes: 30 additions & 0 deletions src/pkg/github/merge_request_state.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,36 @@ func (c *Client) putMergeRequestState(repo string, number int, st mergeRequestPR
c.saveMergeStateLocked()
}

// mergeRequestHeadFirstSeen returns the watcher-clock time repo#number was
// first seen at head sha, recording now on the first call for that head. It
// backs the min_head_age skew fallback (#16143) for requests without
// expect_sha: their wait state is never persisted (putMergeRequestState needs
// a SHA), so PendingSince is zero on every poll and the hold would last as
// long as the skew (#16152). In memory only; a restart re-holds for at most
// one min_head_age window.
func (c *Client) mergeRequestHeadFirstSeen(repo string, number int, sha string, now time.Time) time.Time {
sha = strings.TrimSpace(sha)
if c == nil || sha == "" {
return time.Time{}
}
c.mergeStateMu.Lock()
defer c.mergeStateMu.Unlock()
key := c.mergeRequestStateKey(repo, number)
if st, ok := c.mergeHeadFirstSeen[key]; ok && strings.EqualFold(st.SHA, sha) {
return st.PendingSince
}
if c.mergeHeadFirstSeen == nil {
c.mergeHeadFirstSeen = map[string]mergeRequestPRState{}
}
c.mergeHeadFirstSeen[key] = mergeRequestPRState{SHA: sha, PendingSince: now, At: now}
for k, prev := range c.mergeHeadFirstSeen {
if now.Sub(prev.At) > mergeRequestStateRetention {
delete(c.mergeHeadFirstSeen, k)
}
}
return now
}

func (c *Client) clearMergeRequestState(repo string, number int) {
if c == nil {
return
Expand Down
Loading