Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-16143.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- The `min_head_age` merge gate no longer stalls green merges for an hour when GitHub's server-generated timestamps (`created_at`/`updated_at`/`pushed_at`) run ahead of the host clock. A head time in the future is treated as clock skew: instead of clamping to "head pushed 0s ago" for the whole skew, the automerge sweep and the `hive-merge` CI gate measure the hold from the host-clock time they first observed the head (`pending: head time ahead of host clock; head first seen … ago (< min_head_age)`), so slow check-runs are still held for one `min_head_age` window and the hold then clears. Heads whose age is zero or positive still wait as before ([#16143](https://github.com/hivecommons/hive/issues/16143)).
2 changes: 1 addition & 1 deletion src/docs/operator-reference/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -450,7 +450,7 @@ advanced into the merge path.
|---|---|---|
| `auto_merge.self_authored` | **on** when unset | The only off switch. `false` disables the sweep and App-authored PRs fall back to fully manual merges. |
| `auto_merge.max_merges` | `3` (`DefaultAutoMergeSweepMaxMerges`) when 0/unset | Caps merges per sweep pass. |
| `auto_merge.min_head_age` | `3m` | Minimum PR head age before automerge trusts an unknown required-check set. A fresh head pushed more recently than this waits with `pending: head pushed ... ago (< min_head_age)` unless `auto_merge.required_checks` is declared and every required check is complete and successful. Head age is measured from the head commit's committer date (never earlier than the PR's creation), not the PR's `updated_at`, so label and comment activity — including the hive's own — does not restart it. |
| `auto_merge.min_head_age` | `3m` | Minimum PR head age before automerge trusts an unknown required-check set. A fresh head pushed more recently than this waits with `pending: head pushed ... ago (< min_head_age)` unless `auto_merge.required_checks` is declared and every required check is complete and successful. Head age is measured from the head commit's committer date (never earlier than the PR's creation), not the PR's `updated_at`, so label and comment activity — including the hive's own — does not restart it. When GitHub's head time is ahead of the host clock (clock skew), age is measured from when the hive first observed the head instead, so the hold lasts one window and then clears (`pending: head time ahead of host clock; head first seen ... ago (< min_head_age)`). |
| `auto_merge.required_checks` | unset | Default operator-declared status-check contexts / check-run names (e.g. `["build-gate"]`) that the sweep's green gate requires on repos without a per-repo override. See below. |
| `auto_merge.repos.<repo>.required_checks` | inherits default | Per-repo override for heterogeneous hives. `<repo>` may be the bare name from `project.repos` or `owner/repo`; an explicit empty list declares that GitHub has no required contexts, while Hive still requires positive CI evidence from at least one real successful check-run before merge. |
| `auto_merge.allow_unprotected_base` | deprecated no-op | Accepted for compatibility only. [`hive-merge`](../hive-merge.md) no longer refuses solely because a base branch has no GitHub branch protection; it may merge into any branch the App can write after positive CI evidence. |
Expand Down
70 changes: 56 additions & 14 deletions src/pkg/github/automerge/automerge_head_age.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,18 +26,24 @@ const headFirstSeenPruneAfter = 7 * 24 * time.Hour
// and starts its own age) and caches the head commit's committer date, which
// is immutable per SHA. One tracker is shared by every engine in the process
// because the governor sweep builds a fresh engine per tick.
//
// hostFirstSeen pins the host clock at the first observation of each head.
// It is the min_head_age fallback when GitHub's server-generated head time
// runs ahead of the host (#16143): the head existed when it was first seen,
// so measuring from there can only over-hold by one min_head_age window.
type HeadFirstSeenTracker struct {
mu sync.Mutex
firstSeen map[string]time.Time
commitAt map[string]time.Time
lastSeen map[string]time.Time
mu sync.Mutex
firstSeen map[string]time.Time
hostFirstSeen map[string]time.Time
commitAt map[string]time.Time
lastSeen map[string]time.Time
}

var sharedHeadFirstSeen = NewHeadFirstSeenTracker()

// NewHeadFirstSeenTracker returns an empty tracker.
func NewHeadFirstSeenTracker() *HeadFirstSeenTracker {
return &HeadFirstSeenTracker{firstSeen: make(map[string]time.Time), commitAt: make(map[string]time.Time), lastSeen: make(map[string]time.Time)}
return &HeadFirstSeenTracker{firstSeen: make(map[string]time.Time), hostFirstSeen: make(map[string]time.Time), commitAt: make(map[string]time.Time), lastSeen: make(map[string]time.Time)}
}

// observe returns the pinned head time for key, recording candidate when the
Expand All @@ -49,9 +55,13 @@ func (t *HeadFirstSeenTracker) observe(key string, candidate, now time.Time) tim
}
t.mu.Lock()
defer t.mu.Unlock()
if _, ok := t.hostFirstSeen[key]; !ok {
t.hostFirstSeen[key] = now
}
first, ok := t.firstSeen[key]
if !ok {
if candidate.IsZero() {
t.touchLocked(key, now)
return candidate
}
first = candidate
Expand All @@ -61,6 +71,17 @@ func (t *HeadFirstSeenTracker) observe(key string, candidate, now time.Time) tim
return first
}

// hostObservedAt returns the host clock at the first observe of key, or zero
// when the key has never been observed.
func (t *HeadFirstSeenTracker) hostObservedAt(key string) time.Time {
if t == nil {
return time.Time{}
}
t.mu.Lock()
defer t.mu.Unlock()
return t.hostFirstSeen[key]
}

// cachedCommitTime returns the head commit time recorded for key, if any.
func (t *HeadFirstSeenTracker) cachedCommitTime(key string, now time.Time) (time.Time, bool) {
if t == nil {
Expand Down Expand Up @@ -92,6 +113,7 @@ func (t *HeadFirstSeenTracker) touchLocked(key string, now time.Time) {
if now.Sub(seen) > headFirstSeenPruneAfter {
delete(t.lastSeen, k)
delete(t.firstSeen, k)
delete(t.hostFirstSeen, k)
delete(t.commitAt, k)
}
}
Expand Down Expand Up @@ -152,6 +174,12 @@ func (c *Engine) headPushedAt(ctx context.Context, owner, repo string, number in
// right after a hive comment/label write is still judged by when it was
// committed. If the commit lookup fails the pin is kept: it can only
// over-hold for one min_head_age window and cannot loop.
//
// A result ahead of the host clock is GitHub's server-generated timestamps
// running ahead of this host (#16143), not a fresh push: measuring from it
// clamped every head to "pushed 0s ago" for the whole skew. The host clock
// pinned at this head's first observation is returned instead, so the gate
// still holds slow check-runs for one min_head_age window and then clears.
func (c *Engine) headPushedAtWith(ctx context.Context, tracker *HeadFirstSeenTracker, owner, repo string, number int, sha string, pr *gh.PullRequest) time.Time {
var updated time.Time
if updatedAt := pr.GetUpdatedAt(); !updatedAt.IsZero() {
Expand All @@ -165,21 +193,35 @@ func (c *Engine) headPushedAtWith(ctx context.Context, tracker *HeadFirstSeenTra
now = c.now
}
key := contributorSoakKey(owner, repo, number, sha)
pinned := tracker.observe(key, updated, now())
if c == nil || c.minHeadAge <= 0 || pinned.IsZero() || now().Sub(pinned) >= c.minHeadAge {
at := now()
pinned := tracker.observe(key, updated, at)
if c == nil || c.minHeadAge <= 0 || pinned.IsZero() || at.Sub(pinned) >= c.minHeadAge {
return pinned
}
committed, ok := tracker.cachedCommitTime(key, now())
committed, ok := tracker.cachedCommitTime(key, at)
if !ok {
at, err := hgithub.HeadCommitPushedAt(hgithub.WithRESTCaller(ctx, "hive:automerge_sweep"), c.gh, owner, repo, sha, pr)
got, err := hgithub.HeadCommitPushedAt(hgithub.WithRESTCaller(ctx, "hive:automerge_sweep"), c.gh, owner, repo, sha, pr)
if err != nil {
return pinned
return hostClockBound(pinned, tracker.hostObservedAt(key), at)
}
tracker.recordCommitTime(key, at, now())
committed = at
tracker.recordCommitTime(key, got, at)
committed = got
}
if committed.Before(pinned) {
return committed
return hostClockBound(committed, tracker.hostObservedAt(key), at)
}
return hostClockBound(pinned, tracker.hostObservedAt(key), at)
}

// hostClockBound returns headTime unless it is ahead of the host clock now,
// in which case the host-clock first observation hostFirst (or now, when the
// head has no pin) stands in for it (#16143).
func hostClockBound(headTime, hostFirst, now time.Time) time.Time {
if !headTime.After(now) {
return headTime
}
if hostFirst.IsZero() || hostFirst.After(now) {
return now
}
return pinned
return hostFirst
}
46 changes: 46 additions & 0 deletions src/pkg/github/automerge/automerge_head_age_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -189,3 +189,49 @@ func TestHeadPushedAtFallsBackToPinWhenCommitLookupFails(t *testing.T) {
t.Fatalf("after bump fallback = %v, want pinned %v", got, first)
}
}

// hivecommons/hive#16143: GitHub's server-generated updated_at/created_at ran
// +1h ahead of the host clock, so both the pinned updated_at and the commit
// time (clamped to created_at) landed in the future and the gate read every
// head as "pushed 0s ago" for an hour. The host clock at the head's first
// observation stands in: the gate holds one min_head_age window, then clears.
func TestHeadPushedAtFallsBackToHostClockWhenHeadTimeIsAhead(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"sha":"skew","committer":{"name":"c","date":"2026-11-01T08:49:14Z"}}`))
}))
defer srv.Close()
clock := time.Date(2026, 11, 1, 8, 51, 19, 0, time.UTC)
firstSeen := clock
minHeadAge := 3 * time.Minute
e := New(hgithub.NewClient("token", "acme", []string{"widget"}, nil, srv.URL), Options{
Now: func() time.Time { return clock },
MinHeadAge: minHeadAge,
HeadFirstSeen: NewHeadFirstSeenTracker(),
})
pr := &gh.PullRequest{
CreatedAt: &gh.Timestamp{Time: clock.Add(time.Hour - 2*time.Minute)},
UpdatedAt: &gh.Timestamp{Time: clock.Add(time.Hour - time.Minute)},
}

if got := e.headPushedAt(context.Background(), "acme", "widget", 560, "skew", pr); !got.Equal(firstSeen) {
t.Fatalf("first observation headPushedAt = %v, want host clock %v", got, firstSeen)
}
clock = firstSeen.Add(time.Minute)
got := e.headPushedAt(context.Background(), "acme", "widget", 560, "skew", pr)
if !got.Equal(firstSeen) {
t.Fatalf("second observation headPushedAt = %v, want pinned host clock %v", got, firstSeen)
}
if age := clock.Sub(got); age != time.Minute {
t.Fatalf("head age = %v, want 1m measured from first observation", age)
}
clock = firstSeen.Add(minHeadAge)
if got := e.headPushedAt(context.Background(), "acme", "widget", 560, "skew", pr); clock.Sub(got) < minHeadAge {
t.Fatalf("head age %v still under min_head_age after one window from first observation", clock.Sub(got))
}
// A head whose time is not ahead of the host clock is unaffected.
sane := &gh.PullRequest{UpdatedAt: &gh.Timestamp{Time: clock.Add(-10 * time.Minute)}}
if got := e.headPushedAt(context.Background(), "acme", "widget", 561, "sane", sane); !got.Equal(sane.GetUpdatedAt().Time) {
t.Fatalf("sane head = %v, want updated_at %v", got, sane.GetUpdatedAt().Time)
}
}
27 changes: 25 additions & 2 deletions src/pkg/github/commit_ci.go
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,13 @@ type CommitCIOptions struct {
SkippedByDesign map[string]bool
MinHeadAge time.Duration
HeadPushedAt time.Time
// HeadFirstSeenAt is the host-clock time the caller first observed this
// head. It is consulted only when HeadPushedAt is ahead of the host clock
// (GitHub's server-generated timestamps running ahead of this host,
// #16143): the hold is then measured from it instead of being skipped. A
// zero value is the first observation, so the head is held for one
// min_head_age window from now.
HeadFirstSeenAt time.Time
Now func() time.Time
// RequireEvidence fails closed when no real successful CI check-run evidence
// exists. It is used when GitHub reports a known empty required-check set or
Expand Down Expand Up @@ -650,12 +657,28 @@ func EvaluateCommitCI(ctx context.Context, client *gh.Client, owner, repo, sha s
now = time.Now
}
age := now().Sub(opts.HeadPushedAt)
if age < 0 {
// A head time ahead of the host clock is skew between GitHub's
// server-generated timestamps and this host (#16143), not a fresh
// push, and the host cannot measure age from it. The hold is not
// skipped: when the required set is unknown it is the only cover for
// slow check-runs that have not registered yet. Age is measured from
// the host-clock first observation instead, which can over-hold by at
// most one min_head_age window and never clamps to "pushed 0s ago"
// for the whole skew.
skewed := age < 0
if skewed {
age = 0
if !opts.HeadFirstSeenAt.IsZero() {
age = max(now().Sub(opts.HeadFirstSeenAt), 0)
}
}
configRequiredAllGreen := opts.RequiredKnownFromConfig && len(required) > 0 && len(requiredSuccess) == len(required) && len(st.MissingRequired) == 0
if age < opts.MinHeadAge && !configRequiredAllGreen {
block(fmt.Sprintf("pending: head pushed %s ago (< min_head_age)", age.Round(time.Second)))
if skewed {
block(fmt.Sprintf("pending: head time ahead of host clock; head first seen %s ago (< min_head_age)", age.Round(time.Second)))
} else {
block(fmt.Sprintf("pending: head pushed %s ago (< min_head_age)", age.Round(time.Second)))
}
}
}
st.Green = st.Reason == ""
Expand Down
47 changes: 47 additions & 0 deletions src/pkg/github/commit_ci_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,53 @@ func TestEvaluateCommitCIFreshHeadExpectedChecks(t *testing.T) {
},
want: "pending: head pushed 1m0s ago (< min_head_age)",
},
{
// #16143: GitHub's head time is +1h ahead of the host. The hold
// is not skipped (unknown required set, slow checks may not have
// registered) and does not clamp to "pushed 0s ago" for the hour:
// on first observation it holds for one min_head_age window.
name: "head time ahead of host clock holds from first observation, not forever",
checks: []map[string]string{{"name": "fast", "status": "completed", "conclusion": "success"}},
opts: CommitCIOptions{
MinHeadAge: 3 * time.Minute,
HeadPushedAt: now.Add(time.Hour),
Now: func() time.Time { return now },
},
want: "pending: head time ahead of host clock; head first seen 0s ago (< min_head_age)",
},
{
name: "head time ahead of host clock still holds while first seen under min age",
checks: []map[string]string{{"name": "fast", "status": "completed", "conclusion": "success"}},
opts: CommitCIOptions{
MinHeadAge: 3 * time.Minute,
HeadPushedAt: now.Add(time.Hour),
HeadFirstSeenAt: now.Add(-time.Minute),
Now: func() time.Time { return now },
},
want: "pending: head time ahead of host clock; head first seen 1m0s ago (< min_head_age)",
},
{
name: "head time ahead of host clock clears once first seen reaches min age",
checks: []map[string]string{{"name": "fast", "status": "completed", "conclusion": "success"}},
opts: CommitCIOptions{
MinHeadAge: 3 * time.Minute,
HeadPushedAt: now.Add(time.Hour),
HeadFirstSeenAt: now.Add(-3 * time.Minute),
Now: func() time.Time { return now },
},
wantGreen: true,
},
{
name: "first seen is ignored when the head time is not ahead of the host clock",
checks: []map[string]string{{"name": "fast", "status": "completed", "conclusion": "success"}},
opts: CommitCIOptions{
MinHeadAge: 3 * time.Minute,
HeadPushedAt: now.Add(-time.Minute),
HeadFirstSeenAt: now.Add(-time.Hour),
Now: func() time.Time { return now },
},
want: "pending: head pushed 1m0s ago (< min_head_age)",
},
{
name: "expected name missing no longer blocks unknown required set",
checks: []map[string]string{{"name": "fast", "status": "completed", "conclusion": "success"}},
Expand Down
8 changes: 6 additions & 2 deletions src/pkg/github/merge_ci_gate.go
Original file line number Diff line number Diff line change
Expand Up @@ -131,11 +131,14 @@ func (c *Client) repoNoCIOK(owner, name string) bool {
// repo may be "owner/repo" or a bare name (resolved against c.org, like
// MergePR). expectSHA, when set, is the SHA whose CI is judged; when empty the
// PR's current head is used. now is the watcher's clock, used to age the head
// for the never-created required-check verdict. The returned reason is a human-readable
// for the never-created required-check verdict. firstSeen is the watcher's
// clock when it first parked this head on a pending verdict (zero on the
// first poll); min_head_age falls back to it when GitHub's head time is ahead
// of the host clock (#16143). The returned reason is a human-readable
// explanation suitable for the result file and the log. A non-nil error means
// the verdict could not be computed (API failure) and the caller must treat
// that as a failed attempt, never as a pass.
func (c *Client) verifyMergeRequestCI(ctx context.Context, repo string, number int, expectSHA string, now time.Time) (mergeCIVerdict, string, error) {
func (c *Client) verifyMergeRequestCI(ctx context.Context, repo string, number int, expectSHA string, now, firstSeen time.Time) (mergeCIVerdict, string, error) {
if c == nil || c.client == nil {
return mergeCIUnverified, "no GitHub client", ErrNoGitHubClient
}
Expand Down Expand Up @@ -204,6 +207,7 @@ func (c *Client) verifyMergeRequestCI(ctx context.Context, repo string, number i
ExpectedChecks: expected,
MinHeadAge: minHeadAge,
HeadPushedAt: minHeadAgeFrom,
HeadFirstSeenAt: firstSeen,
})
if err != nil {
return mergeCIUnverified, "ci gate: " + st.Reason, fmt.Errorf("ci gate: %s for %s/%s@%s: %w", st.Reason, owner, name, shortSHA(sha), err)
Expand Down
2 changes: 1 addition & 1 deletion src/pkg/github/merge_request_watcher.go
Original file line number Diff line number Diff line change
Expand Up @@ -462,7 +462,7 @@ func (c *Client) handleOneMergeRequest(ctx context.Context, path string, nowFn f
// the request without consuming an attempt; red and unverified verdicts
// are failed attempts that go through the same retry / exhaust /
// re-engage path a GitHub-side refusal does.
verdict, why, err := c.verifyMergeRequestCI(ctx, req.Repo, req.Number, req.ExpectSHA, now)
verdict, why, err := c.verifyMergeRequestCI(ctx, req.Repo, req.Number, req.ExpectSHA, now, prState.PendingSince)
if err != nil {
c.logCIVerdict(req, verdict, err.Error())
c.recordMergeFailure(path, req, attempts, err.Error(), nowFn)
Expand Down
Loading