From 0b626217171201c3b7fe47932dd471a233ed28ef Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 05:19:40 -0500 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=90=9B=20fix(github):=20hold=20skewed?= =?UTF-8?q?=20heads=20from=20first=20observation=20without=20expect=5Fsha?= =?UTF-8?q?=20and=20log=20head-time=20skew?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #16147 Closes #16152 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-16147-16152-head-skew-first-seen.md | 1 + .../automerge/automerge_sweep_queued.go | 4 ++ src/pkg/github/client.go | 5 ++ src/pkg/github/commit_ci.go | 7 +++ src/pkg/github/commit_ci_test.go | 29 +++++++++-- src/pkg/github/merge_ci_gate.go | 11 +++- src/pkg/github/merge_ci_gate_test.go | 52 ++++++++++++++++++- src/pkg/github/merge_request_state.go | 30 +++++++++++ 8 files changed, 133 insertions(+), 6 deletions(-) create mode 100644 changelog.d/fixed-16147-16152-head-skew-first-seen.md diff --git a/changelog.d/fixed-16147-16152-head-skew-first-seen.md b/changelog.d/fixed-16147-16152-head-skew-first-seen.md new file mode 100644 index 0000000000..8965ff3ca4 --- /dev/null +++ b/changelog.d/fixed-16147-16152-head-skew-first-seen.md @@ -0,0 +1 @@ +- The merge-request CI gate now clears the min_head_age hold one window after it first sees a head whose GitHub timestamp is ahead of the host clock even when the merge request omits `expect_sha`, and both merge paths log when a head time is ahead of the host clock so a skew-handled head is distinguishable from a normally aged one. diff --git a/src/pkg/github/automerge/automerge_sweep_queued.go b/src/pkg/github/automerge/automerge_sweep_queued.go index bbab9b48f3..0db88ec567 100644 --- a/src/pkg/github/automerge/automerge_sweep_queued.go +++ b/src/pkg/github/automerge/automerge_sweep_queued.go @@ -365,6 +365,10 @@ func (c *Engine) commitGreenForPRSkipping(ctx context.Context, owner, repo, bran if err != nil { return false, st.Reason, err } + if prNumber > 0 && st.HeadTimeAheadBy > 0 { + c.info("automerge head time ahead of host clock; min_head_age not evaluable from it, measured from first observation", "repo", owner+"/"+repo, "pr", prNumber, + "sha", sha, "ahead_by", st.HeadTimeAheadBy.Round(time.Second), "green", st.Green, "reason", st.Reason) + } if len(st.Reporting) > 0 { c.info("automerge CI reporting observation", "repo", owner+"/"+repo, "pr", prNumber, "head", sha, "observations", st.Reporting) diff --git a/src/pkg/github/client.go b/src/pkg/github/client.go index 9e3dd29ba2..afa694b2af 100644 --- a/src/pkg/github/client.go +++ b/src/pkg/github/client.go @@ -463,6 +463,11 @@ type Client struct { mergeStatePath string mergeState map[string]mergeRequestPRState mergeStateLoaded bool + // mergeHeadFirstSeen is the in-memory first observation of each PR head + // the CI gate judged without an expect_sha (no persisted wait state), so + // the min_head_age skew fallback has a stable start (#16152). Guarded by + // mergeStateMu. + mergeHeadFirstSeen map[string]mergeRequestPRState // attribution holds the invocation-attribution hooks (trailer gate, // per-agent metadata resolver, audit sink) — see attribution.go. Guarded by // attribMu because the hooks are installed in stages during startup while diff --git a/src/pkg/github/commit_ci.go b/src/pkg/github/commit_ci.go index 3c08ef8a26..732aebdbd2 100644 --- a/src/pkg/github/commit_ci.go +++ b/src/pkg/github/commit_ci.go @@ -123,6 +123,12 @@ type CommitCIState struct { // Every one of them postdates the push, so it is a safe lower bound on // how long the head has existed. EarliestActivity time.Time + // HeadTimeAheadBy is how far CommitCIOptions.HeadPushedAt was ahead of + // the host clock when min_head_age was evaluated (#16143). Non-zero means + // the head's age was not evaluable from GitHub's timestamp and the hold + // was measured from HeadFirstSeenAt instead; callers log it so a + // skew-handled head is distinguishable from a normally aged one. + HeadTimeAheadBy time.Duration } // CommitCIOptions carries policy facts that are known to callers before the @@ -667,6 +673,7 @@ func EvaluateCommitCI(ctx context.Context, client *gh.Client, owner, repo, sha s // for the whole skew. skewed := age < 0 if skewed { + st.HeadTimeAheadBy = -age age = 0 if !opts.HeadFirstSeenAt.IsZero() { age = max(now().Sub(opts.HeadFirstSeenAt), 0) diff --git a/src/pkg/github/commit_ci_test.go b/src/pkg/github/commit_ci_test.go index 9ca1446fc1..5ff482a77b 100644 --- a/src/pkg/github/commit_ci_test.go +++ b/src/pkg/github/commit_ci_test.go @@ -22,6 +22,9 @@ func TestEvaluateCommitCIFreshHeadExpectedChecks(t *testing.T) { opts CommitCIOptions wantGreen bool want string + // wantAheadBy is the CommitCIState.HeadTimeAheadBy the gate must + // report, so a skew-handled head is observable (#16147). + wantAheadBy time.Duration }{ { name: "fresh head with zero slow checks waits for min age", @@ -45,7 +48,8 @@ func TestEvaluateCommitCIFreshHeadExpectedChecks(t *testing.T) { HeadPushedAt: now.Add(time.Hour), Now: func() time.Time { return now }, }, - want: "pending: head time ahead of host clock; head first seen 0s ago (< min_head_age)", + want: "pending: head time ahead of host clock; head first seen 0s ago (< min_head_age)", + wantAheadBy: time.Hour, }, { name: "head time ahead of host clock still holds while first seen under min age", @@ -56,7 +60,8 @@ func TestEvaluateCommitCIFreshHeadExpectedChecks(t *testing.T) { HeadFirstSeenAt: now.Add(-time.Minute), Now: func() time.Time { return now }, }, - want: "pending: head time ahead of host clock; head first seen 1m0s ago (< min_head_age)", + want: "pending: head time ahead of host clock; head first seen 1m0s ago (< min_head_age)", + wantAheadBy: time.Hour, }, { name: "head time ahead of host clock clears once first seen reaches min age", @@ -67,7 +72,22 @@ func TestEvaluateCommitCIFreshHeadExpectedChecks(t *testing.T) { HeadFirstSeenAt: now.Add(-3 * time.Minute), Now: func() time.Time { return now }, }, - wantGreen: true, + wantGreen: true, + wantAheadBy: time.Hour, + }, + { + // #16147: a head cleared through the skew path must not look like + // a normally aged one; the state names how far ahead it was. + name: "head time ahead of host clock is reported when the hold clears", + checks: []map[string]string{{"name": "fast", "status": "completed", "conclusion": "success"}}, + opts: CommitCIOptions{ + MinHeadAge: 3 * time.Minute, + HeadPushedAt: now.Add(90 * time.Minute), + HeadFirstSeenAt: now.Add(-10 * time.Minute), + Now: func() time.Time { return now }, + }, + wantGreen: true, + wantAheadBy: 90 * time.Minute, }, { name: "first seen is ignored when the head time is not ahead of the host clock", @@ -175,6 +195,9 @@ func TestEvaluateCommitCIFreshHeadExpectedChecks(t *testing.T) { if st.Green != tt.wantGreen || st.Reason != tt.want { t.Fatalf("EvaluateCommitCI = (%v,%q), want (%v,%q)", st.Green, st.Reason, tt.wantGreen, tt.want) } + if st.HeadTimeAheadBy != tt.wantAheadBy { + t.Fatalf("HeadTimeAheadBy = %s, want %s", st.HeadTimeAheadBy, tt.wantAheadBy) + } }) } } diff --git a/src/pkg/github/merge_ci_gate.go b/src/pkg/github/merge_ci_gate.go index 451d688d94..6fb3055e12 100644 --- a/src/pkg/github/merge_ci_gate.go +++ b/src/pkg/github/merge_ci_gate.go @@ -134,7 +134,9 @@ func (c *Client) repoNoCIOK(owner, name string) bool { // for the never-created required-check verdict. firstSeen is the watcher's // clock when it first parked this head on a pending verdict (zero on the // first poll); min_head_age falls back to it when GitHub's head time is ahead -// of the host clock (#16143). The returned reason is a human-readable +// of the host clock (#16143). Without expectSHA that wait state is never +// persisted, so the gate keeps its own first observation of the resolved head +// instead (#16152). The returned reason is a human-readable // explanation suitable for the result file and the log. A non-nil error means // the verdict could not be computed (API failure) and the caller must treat // that as a failed attempt, never as a pass. @@ -198,6 +200,9 @@ func (c *Client) verifyMergeRequestCI(ctx context.Context, repo string, number i minHeadAgeFrom = at } } + if minHeadAge > 0 && firstSeen.IsZero() && strings.TrimSpace(expectSHA) == "" { + firstSeen = c.mergeRequestHeadFirstSeen(owner+"/"+name, number, sha, now) + } st, err := EvaluateCommitCI(ctx, c.client, owner, name, sha, CommitCIOptions{ PRNumber: number, Required: required, @@ -215,6 +220,10 @@ func (c *Client) verifyMergeRequestCI(ctx context.Context, repo string, number i if cfgKnown && actualKnown { c.warnRequiredChecksMismatch(owner, name, cfgSet, actualRequired, st.Observed) } + if st.HeadTimeAheadBy > 0 { + c.info("merge-request CI head time ahead of host clock; min_head_age not evaluable from it, measured from first observation", "repo", owner+"/"+name, "pr", number, + "sha", sha, "ahead_by", st.HeadTimeAheadBy.Round(time.Second), "first_seen", firstSeen, "green", st.Green, "reason", st.Reason) + } if len(st.Reporting) > 0 { c.info("merge-request CI reporting observation", "repo", owner+"/"+name, "pr", number, "head", sha, "observations", st.Reporting) diff --git a/src/pkg/github/merge_ci_gate_test.go b/src/pkg/github/merge_ci_gate_test.go index 2ab08196a6..dde271d51f 100644 --- a/src/pkg/github/merge_ci_gate_test.go +++ b/src/pkg/github/merge_ci_gate_test.go @@ -45,6 +45,9 @@ type ciFixture struct { // (to exercise the GitHub-side refusal paths); zero merges successfully. mergeStatus int approveCalls *atomic.Int32 + // updatedAt, when set, is the PR's updated_at (RFC 3339): the head time + // the min_head_age gate measures from. + updatedAt string } type ciStatus struct{ context, state string } @@ -107,13 +110,17 @@ func (f *ciFixture) serveCI(w http.ResponseWriter, r *http.Request) bool { if baseRepo == "" { baseRepo = "o/r" } - enc(map[string]any{ + pr := map[string]any{ "number": n, "state": "open", "user": map[string]any{"login": author, "type": authorType}, "head": map[string]any{"sha": head, "repo": map[string]any{"full_name": headRepo}}, "base": map[string]any{"ref": "main", "repo": map[string]any{"full_name": baseRepo}}, - }) + } + if f.updatedAt != "" { + pr["updated_at"] = f.updatedAt + } + enc(pr) case strings.Contains(p, "/branches/") && !strings.Contains(p, "/protection"): // /repos/{o}/{r}/branches/{branch} if f.branchStatus != 0 { w.WriteHeader(f.branchStatus) @@ -1165,3 +1172,44 @@ func TestMergeCIGate_RequiredCheckNeverCreated(t *testing.T) { }) } } + +// #16152: a merge request without expect_sha has no persisted wait state, so +// while GitHub's head time is ahead of the host clock the min_head_age hold +// must be measured from the gate's own first observation of the head, and +// clear after one window, instead of reading "first seen 0s ago" every poll +// for as long as the skew lasts. +func TestMergeCIGate_SkewedHeadWithoutExpectSHAHoldsFromFirstObservation(t *testing.T) { + f := greenFixture() + f.updatedAt = time.Now().Add(time.Hour).UTC().Format(time.RFC3339) + var merges atomic.Int32 + srv := ciGateServer(t, f, &merges) + defer srv.Close() + c := testMergeClient(t, srv.URL) + c.SetAutoMergeMinHeadAge(10 * time.Minute) + ctx := context.Background() + + verdict, why, err := c.verifyMergeRequestCI(ctx, "o/r", 100, "", time.Now().Add(-5*time.Minute), time.Time{}) + if err != nil || verdict != mergeCIPending || !strings.Contains(why, "head time ahead of host clock; head first seen 5m") { + t.Fatalf("first poll = (%v, %q, %v), want pending held from the first observation 5m ago", verdict, why, err) + } + // A later poll without expect_sha still has no persisted PendingSince; the + // gate must keep its first observation rather than restart at 0s. + verdict, why, err = c.verifyMergeRequestCI(ctx, "o/r", 100, "", time.Now(), time.Time{}) + if err != nil || verdict != mergeCIPending || !strings.Contains(why, "head first seen 5m") { + t.Fatalf("second poll = (%v, %q, %v), want the hold measured from the first observation", verdict, why, err) + } + c.SetAutoMergeMinHeadAge(4 * time.Minute) + verdict, why, err = c.verifyMergeRequestCI(ctx, "o/r", 100, "", time.Now(), time.Time{}) + if err != nil || verdict != mergeCIGreen { + t.Fatalf("poll after one min_head_age window = (%v, %q, %v), want green", verdict, why, err) + } + // A new head is a new first observation: it is held again (fail closed). + f.heads[100] = "def456" + verdict, why, err = c.verifyMergeRequestCI(ctx, "o/r", 100, "", time.Now(), time.Time{}) + if err != nil || verdict != mergeCIPending || !strings.Contains(why, "head first seen 0s ago") { + t.Fatalf("new head = (%v, %q, %v), want pending from a fresh first observation", verdict, why, err) + } + if got := merges.Load(); got != 0 { + t.Fatalf("verifyMergeRequestCI must never merge, got %d", got) + } +} diff --git a/src/pkg/github/merge_request_state.go b/src/pkg/github/merge_request_state.go index 5a476199d8..62a31820d4 100644 --- a/src/pkg/github/merge_request_state.go +++ b/src/pkg/github/merge_request_state.go @@ -127,6 +127,36 @@ func (c *Client) putMergeRequestState(repo string, number int, st mergeRequestPR c.saveMergeStateLocked() } +// mergeRequestHeadFirstSeen returns the watcher-clock time repo#number was +// first seen at head sha, recording now on the first call for that head. It +// backs the min_head_age skew fallback (#16143) for requests without +// expect_sha: their wait state is never persisted (putMergeRequestState needs +// a SHA), so PendingSince is zero on every poll and the hold would last as +// long as the skew (#16152). In memory only; a restart re-holds for at most +// one min_head_age window. +func (c *Client) mergeRequestHeadFirstSeen(repo string, number int, sha string, now time.Time) time.Time { + sha = strings.TrimSpace(sha) + if c == nil || sha == "" { + return time.Time{} + } + c.mergeStateMu.Lock() + defer c.mergeStateMu.Unlock() + key := c.mergeRequestStateKey(repo, number) + if st, ok := c.mergeHeadFirstSeen[key]; ok && strings.EqualFold(st.SHA, sha) { + return st.PendingSince + } + if c.mergeHeadFirstSeen == nil { + c.mergeHeadFirstSeen = map[string]mergeRequestPRState{} + } + c.mergeHeadFirstSeen[key] = mergeRequestPRState{SHA: sha, PendingSince: now, At: now} + for k, prev := range c.mergeHeadFirstSeen { + if now.Sub(prev.At) > mergeRequestStateRetention { + delete(c.mergeHeadFirstSeen, k) + } + } + return now +} + func (c *Client) clearMergeRequestState(repo string, number int) { if c == nil { return From 9820c7fa528361222944bee5c7c556f649d09d31 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 05:23:57 -0500 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=93=9D=20docs(scale-envelope):=20repo?= =?UTF-8?q?int=20RankActionableIssues=20citation=20after=20client.go=20lin?= =?UTF-8?q?e=20shift?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit check-docs-citations flagged src/docs/scale-envelope.md:37 because the AgeMinutes sort in RankActionableIssues moved from client.go:5012-5022 to 5017-5027 on this branch. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/docs/scale-envelope.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/docs/scale-envelope.md b/src/docs/scale-envelope.md index fe8ecc34c1..7f436f54ec 100644 --- a/src/docs/scale-envelope.md +++ b/src/docs/scale-envelope.md @@ -34,7 +34,7 @@ ones to: 2. **Truncation must preserve the prioritization signal, not cut an arbitrary slice.** This holds today and is easy to assume backwards: - Issues are sorted by actionable rank tier, then **oldest first** within each tier before any cap is applied - (`RankActionableIssues`, `src/pkg/github/client.go:5012-5022`, descending `AgeMinutes` within each tier). + (`RankActionableIssues`, `src/pkg/github/client.go:5017-5027`, descending `AgeMinutes` within each tier). - PRs are sorted by **review class then oldest-first within class** (`SortPullRequestsForReview`, `src/pkg/github/review_priority.go:334`; fixes → refactors/docs → tests).