diff --git a/changelog.d/fixed-16522-covered-issue-release-while-fixing.md b/changelog.d/fixed-16522-covered-issue-release-while-fixing.md new file mode 100644 index 0000000000..84dd8ca332 --- /dev/null +++ b/changelog.d/fixed-16522-covered-issue-release-while-fixing.md @@ -0,0 +1 @@ +- The claim-release valve no longer drops a covered issue to `stage/010-actionable` while the hive is still repairing its red covering PR ([#16522](https://github.com/hivecommons/hive/issues/16522)). A red+stale PR releases its issue only once the fix loop has given up on that head: escalated to a human, re-engagement budget spent, or red for the whole re-engagement window (`escalation.ClaimReleaseAfter`) with no owner reachable. Until then the issue stays `stage/040-covered-by-pr` narrating the PR's `070-fixing`, and the scanner is not offered work a live PR already owns. diff --git a/src/cmd/hive/escalation_gates_test.go b/src/cmd/hive/escalation_gates_test.go index aa7cbe2deb..22f2a2c034 100644 --- a/src/cmd/hive/escalation_gates_test.go +++ b/src/cmd/hive/escalation_gates_test.go @@ -214,8 +214,8 @@ func TestClaimingPRRedStale(t *testing.T) { } }) - t.Run("release only for red AND stale claiming PRs", func(t *testing.T) { - _, clock := newTestEscalationStore(t) + t.Run("release only for red AND abandoned claiming PRs", func(t *testing.T) { + store, clock := newTestEscalationStore(t) cfg := escalationTestConfig() green := redPR("widgets", 9, "hive-agent", "ggg") green.FailingChecks = nil @@ -239,13 +239,45 @@ func TestClaimingPRRedStale(t *testing.T) { t.Error("a freshly-red claiming PR must keep suppressing (false)") } *clock = clock.Add(escalation.RedPRStaleAfter + time.Minute) + if pred("widgets", 7) { + // hivecommons/hive#16522: #16475 fell to 010-actionable under a + // live #16492 the reaper was re-engaging a minute later. + t.Error("a stale red claiming PR the reaper is still re-engaging must keep suppressing (false)") + } + for i := 0; i < escalation.MaxReEngagements; i++ { + store.RecordReEngagement("acme/widgets", 7) + } if !pred("widgets", 7) { - t.Error("a red claiming PR stale past the threshold must release (true)") + t.Error("a stale red claiming PR with its re-engagement budget spent must release (true)") } if !pred("acme/widgets", 7) { t.Error("the predicate must resolve owner-prefixed claim repos too") } }) + + t.Run("escalated or long-abandoned red claiming PRs release", func(t *testing.T) { + store, clock := newTestEscalationStore(t) + cfg := escalationTestConfig() + actionable := actionableWith( + redPR("widgets", 7, "hive-agent", "abc"), + redPR("widgets", 8, "hive-agent", "def"), + ) + recordRedStaleness(cfg, actionable) + pred := claimingPRRedStale(cfg, actionable) + + *clock = clock.Add(escalation.RedPRStaleAfter + time.Minute) + store.MarkEscalated("acme/widgets", 7) + if !pred("widgets", 7) { + t.Error("a stale red claiming PR handed to a human must release (true)") + } + if pred("widgets", 8) { + t.Error("a stale red claiming PR still inside the re-engagement window must keep suppressing (false)") + } + *clock = clock.Add(escalation.ClaimReleaseAfter) + if !pred("widgets", 8) { + t.Error("a red claiming PR untouched for the whole re-engagement window must release (true)") + } + }) } func TestReapStuckRedPRs(t *testing.T) { diff --git a/src/cmd/hive/main_sweeps.go b/src/cmd/hive/main_sweeps.go index b453d5f3ca..ab636a5755 100644 --- a/src/cmd/hive/main_sweeps.go +++ b/src/cmd/hive/main_sweeps.go @@ -1848,12 +1848,17 @@ func issueReopenedHookFor(ledger *github.ClaimLedger, logger *slog.Logger) githu } // claimingPRRedStale builds the Fix #3 release predicate: given a claiming PR -// (prRepo, prNumber), report whether it is red on a required check AND stale. -// It looks up the PR's live CI state + head SHA from this pass's enumeration and -// consults the shared staleness clock. A PR not found in the enumeration, or one -// that is green/pending, or one whose red head only just appeared, returns false -// — so a HEALTHY claiming PR still suppresses its issue. Returns a nil func when -// escalation is disabled, preserving the original unconditional-suppress +// (prRepo, prNumber), report whether it is red on a required check AND the +// hive has given up repairing it (escalation.Store.AbandonedRed: stale, and +// escalated / re-engagement budget spent / red for the whole re-engagement +// window). It looks up the PR's live CI state + head SHA from this pass's +// enumeration and consults the shared staleness clock. A PR not found in the +// enumeration, one that is green/pending, one whose red head only just +// appeared, or one the reaper is still re-engaging returns false — so a +// HEALTHY or still-being-fixed claiming PR keeps suppressing its issue, and +// the issue stays 040-covered-by-pr while the PR reads 070-fixing instead of +// falling to 010 under a live PR (hivecommons/hive#16522). Returns a nil func +// when escalation is disabled, preserving the original unconditional-suppress // behavior. GENERIC: keys only off check state + staleness. func claimingPRRedStale(cfg *config.Config, actionable *github.ActionableResult) github.RedStaleFunc { if cfg.Escalation.Disabled || actionable == nil { @@ -1886,7 +1891,7 @@ func claimingPRRedStale(cfg *config.Config, actionable *github.ActionableResult) if !ok || !st.red { return false // not enumerated, or healthy → keep suppressing } - return store.StaleRed(st.repo, prNumber, st.headSHA) + return store.AbandonedRed(st.repo, prNumber, st.headSHA) } } diff --git a/src/docs/contributor-relay.md b/src/docs/contributor-relay.md index 273cd8467d..1af4c04966 100644 --- a/src/docs/contributor-relay.md +++ b/src/docs/contributor-relay.md @@ -1230,7 +1230,7 @@ Worker claims instead expire on their configured per-kind TTL, renew with task o The notes below record the evolution of claim scanning, verified verdicts, prose parsing, and settled retention. Their earlier “72h deferral/expiry” descriptions are historical, **not the current dispatch rules**; use the table above for today's operator decision. -- **Issues already claimed by an open PR — or settled by a recently-merged one — are skipped.** The hub's claim ledger records every open PR that references an issue with a closing keyword (`fixes #N`, `closes owner/repo#N`, …) — including PRs from external authors, not just hive agents ([#3792](https://github.com/hivecommons/hive/pull/3792)). Since [#6867](https://github.com/hivecommons/hive/issues/6867) the same scan also covers **recently-merged** PRs: a fix that merged *without* a closing keyword leaves its issue open, and its claim used to vanish the moment the PR left the open set, so the settled issue went straight back to contributors (and agent dispatch) whose only possible verdict was "already resolved on `main`" — measured downstream at ~34% of contributor sessions. Merged claims are graded by the same evidence tiers as open ones and suppress identically; across tiers, evidence wins, so a merged `Fixes #N` outranks an open `Refs #N`. A claimed issue is silently dropped from the contribute candidate set; if nothing else is admissible the relay receives `task_unavailable` with reason `no_matching_work` (there is no per-issue "claimed by PR #N" message). External claims — like the weaker non-closing `Refs #N` references — now DEFER the hive's own agents for a bounded 72h window, then release the issue even while the PR stays open ([#4929](https://github.com/hivecommons/hive/issues/4929)). They previously never suppressed agent work at all, which let an agent that cannot check for existing PRs re-implement an issue a live PR already covered. Nothing is frozen: the window is a bound — anchored at first observation for an open claim, and at the **merge** for a merged one, so a merged `Refs #N` still releases an epic's remainder — and a red+stale claiming PR defers nothing. +- **Issues already claimed by an open PR — or settled by a recently-merged one — are skipped.** The hub's claim ledger records every open PR that references an issue with a closing keyword (`fixes #N`, `closes owner/repo#N`, …) — including PRs from external authors, not just hive agents ([#3792](https://github.com/hivecommons/hive/pull/3792)). Since [#6867](https://github.com/hivecommons/hive/issues/6867) the same scan also covers **recently-merged** PRs: a fix that merged *without* a closing keyword leaves its issue open, and its claim used to vanish the moment the PR left the open set, so the settled issue went straight back to contributors (and agent dispatch) whose only possible verdict was "already resolved on `main`" — measured downstream at ~34% of contributor sessions. Merged claims are graded by the same evidence tiers as open ones and suppress identically; across tiers, evidence wins, so a merged `Fixes #N` outranks an open `Refs #N`. A claimed issue is silently dropped from the contribute candidate set; if nothing else is admissible the relay receives `task_unavailable` with reason `no_matching_work` (there is no per-issue "claimed by PR #N" message). External claims — like the weaker non-closing `Refs #N` references — now DEFER the hive's own agents for a bounded 72h window, then release the issue even while the PR stays open ([#4929](https://github.com/hivecommons/hive/issues/4929)). They previously never suppressed agent work at all, which let an agent that cannot check for existing PRs re-implement an issue a live PR already covered. Nothing is frozen: the window is a bound — anchored at first observation for an open claim, and at the **merge** for a merged one, so a merged `Refs #N` still releases an epic's remainder — and a red+stale claiming PR the hive has given up repairing (escalated to a human, re-engagement budget spent, or red for the whole re-engagement window — [#16522](https://github.com/hivecommons/hive/issues/16522)) defers nothing; while the reaper is still re-engaging it, the issue stays covered. - **A `no_work_needed` verdict's citation becomes a verified claim.** The merged-PR scan above only sees PRs that *reference* the issue; a fix that landed without mentioning it is invisible, and on one hosted spoke 5 of 7 tasks on a contributor ended `no_work_needed` re-discovering settlements the previous agent had already found ([#7871](https://github.com/hivecommons/hive/issues/7871)). The agent does name what settled the issue in its `verdict_reason` ("already merged via #532", "covered by merged PR #867", "already in upstream/main (e6d3de3)"), so the hub now parses that reason for `#N`, `owner/repo#N`, pull URLs and 7–40 hex SHAs (the task's own issue number excluded, at most six candidates), checks each against GitHub, and records the **first that holds up** in the same claim ledger, exactly as the scan would have: a PR merged into the task repo *before the task was dispatched* — or a commit reachable from the default branch — is a strong merged claim; an open PR in the task repo by someone else is a weak external claim — *someone else* is enforced: the hub compares the PR's author to the reporting contributor's GitHub login (case-insensitively) and drops a self-authored open PR, so the least-trusted write path cannot defer an issue by citing its own unrelated PR ([#7890](https://github.com/hivecommons/hive/issues/7890)); a merged PR is a fact whoever authored it. The text alone is never trusted: a reference the API cannot confirm (wrong repo, closed-unmerged, not on `main`, merged after dispatch, or simply fabricated) records nothing and today's cooldown applies. What the API check does *not* establish is relevance — a real but unrelated ref passes, since a settling PR by definition never referenced the issue; that residual is bounded by the claim TTL and voided by newer activity on the issue. Verdict-recovered claims carry `source: "verdict"` and survive the authoritative scan that cannot see their PR, until the ordinary 72h TTL retires them or stronger live evidence for the issue replaces them. The check runs off the contributor's read loop and requires a wired claim ledger and GitHub client; a hive without either behaves as before. - **A closing keyword written as prose counts.** The claim parser used to require GitHub's own trailer shape, `keyword #N` (optionally with a colon), so a merged PR whose body read *"Resolves architect issue #1232 (first incremental step)"* produced **no claim at all** ([#7995](https://github.com/hivecommons/hive/issues/7995)). Since that issue the closing tier accepts the same bounded run of prose the non-closing reference tier already did — at most 40 characters, no `.`, newline or `#` — so `Resolves architect issue #1232` and `Fixes the bug in #12` are closing claims. GitHub will still not auto-close on those, and hive does not pretend it will: a claim here is about whether the **work landed**, and a merged PR that says it resolves an issue is the strongest evidence the hub gets, so it suppresses rather than being released for re-verification after every merge. The prose gap must begin with a space or tab, which is what keeps a Conventional Commits type prefix (`fix:`, `fix(scope):`) from claiming the first issue number in a title. The closing tier remains single-issue, matching GitHub's one-keyword-per-number rule; only the reference tier reads lists. - **A merged fix does not expire on a clock.** A strong claim from a merged PR — or a verified `no_work_needed` verdict — used to leave the ledger 72 hours after the merge, because the settle scan looks back 72h and the ledger TTL matched it. The PR was still merged; only the clock had moved. So the issue went straight back into the offer pool, an agent re-verified "already fixed on main", and the verdict claim it left sat on the same 72h clock — one wasted task cycle every three days for as long as nobody closed the issue ([#8003](https://github.com/hivecommons/hive/issues/8003): projectbluefin/chairlift#55, fixed by #133, re-offered three times with two "please close" agent comments on it). Settled claims are now carried forward across scans for as long as the issue stays open — the 30-day retention bound counts from the last enumeration pass that listed the issue as open, not from the merge, so it only prunes claims whose issue has since closed ([#11580](https://github.com/hivecommons/hive/issues/11580)); a merged *weak* claim (a `Refs #N`, or an external author's PR) is not — it never asserted it finished the issue, and is still released with context as before. And because the hold has by then outlasted every automatic bound, after **7 days** the contribute queue stops describing it as "an open pull request already claims this issue" and withholds it under `merged_claim_stale`, whose prose is the question a maintainer has to answer: *Fixed by merged PR #133 4 days ago; the issue is still open — close it or say what remains.* diff --git a/src/pkg/escalation/abandoned_red_test.go b/src/pkg/escalation/abandoned_red_test.go new file mode 100644 index 0000000000..82eb897b07 --- /dev/null +++ b/src/pkg/escalation/abandoned_red_test.go @@ -0,0 +1,91 @@ +package escalation + +import ( + "path/filepath" + "testing" + "time" +) + +// The claim-release valve must not let a covered issue go while the hive is +// still re-engaging the covering PR: hivecommons/hive#16475 fell from +// 040-covered-by-pr to 010-actionable ten minutes after #16492 went red, with +// the reaper's first re-engagement for that same head landing a minute later +// (hivecommons/hive#16522). +func TestAbandonedRed(t *testing.T) { + newStore := func(t *testing.T) (*Store, *time.Time) { + t.Helper() + s := Load(filepath.Join(t.TempDir(), "ledger.json")) + now := time.Now().UTC() + s.SetClock(func() time.Time { return now }) + s.ObserveRed([]Observation{{Repo: "org/repo", Number: 1, HeadSHA: "sha1", Red: true}}) + return s, &now + } + + t.Run("stale inside the re-engagement budget keeps the claim", func(t *testing.T) { + s, now := newStore(t) + *now = now.Add(RedPRStaleAfter + time.Minute) + if !s.StaleRed("org/repo", 1, "sha1") { + t.Fatal("precondition: head must read stale") + } + if s.AbandonedRed("org/repo", 1, "sha1") { + t.Fatal("a stale red head the reaper is still re-engaging must not release its issue") + } + s.RecordReEngagement("org/repo", 1) + *now = now.Add(ReEngageCooldown + time.Minute) + if s.AbandonedRed("org/repo", 1, "sha1") { + t.Fatal("one delivered re-engagement is the fix loop working, not giving up") + } + }) + + t.Run("fresh red is never abandoned", func(t *testing.T) { + s, _ := newStore(t) + s.MarkEscalated("org/repo", 1) + if s.AbandonedRed("org/repo", 1, "sha1") { + t.Fatal("a head younger than RedPRStaleAfter must not release, even escalated") + } + }) + + t.Run("escalated to a human releases", func(t *testing.T) { + s, now := newStore(t) + *now = now.Add(RedPRStaleAfter + time.Minute) + s.MarkEscalated("org/repo", 1) + if !s.AbandonedRed("org/repo", 1, "sha1") { + t.Fatal("a stale red head handed to a human has left the fix loop") + } + }) + + t.Run("spent re-engagement budget releases", func(t *testing.T) { + s, now := newStore(t) + *now = now.Add(RedPRStaleAfter + time.Minute) + for i := 0; i < MaxReEngagements; i++ { + s.RecordReEngagement("org/repo", 1) + } + if !s.AbandonedRed("org/repo", 1, "sha1") { + t.Fatal("a stale red head with no re-engagements left has been given up on") + } + }) + + t.Run("unreachable owner releases after the whole window", func(t *testing.T) { + s, now := newStore(t) + *now = now.Add(ClaimReleaseAfter - time.Minute) + if s.AbandonedRed("org/repo", 1, "sha1") { + t.Fatal("still inside the re-engagement window") + } + *now = now.Add(2 * time.Minute) + if !s.AbandonedRed("org/repo", 1, "sha1") { + t.Fatal("a head red for the whole re-engagement window with nobody reachable must release") + } + }) + + t.Run("a moved head resets the verdict", func(t *testing.T) { + s, now := newStore(t) + *now = now.Add(ClaimReleaseAfter + time.Minute) + s.ObserveRed([]Observation{{Repo: "org/repo", Number: 1, HeadSHA: "sha2", Red: true}}) + if s.AbandonedRed("org/repo", 1, "sha2") { + t.Fatal("a new red head is a fix attempt: the clock and the budget start over") + } + if s.AbandonedRed("org/repo", 1, "sha1") { + t.Fatal("a head the store no longer tracks is not stale") + } + }) +} diff --git a/src/pkg/escalation/escalation.go b/src/pkg/escalation/escalation.go index 94aac1e196..b02b8aab75 100644 --- a/src/pkg/escalation/escalation.go +++ b/src/pkg/escalation/escalation.go @@ -999,6 +999,38 @@ func (s *Store) StaleRed(repo string, number int, headSHA string) bool { return s.now().Sub(e.FirstRedAt) >= RedPRStaleAfter } +// ClaimReleaseAfter is how long a red head may sit unchanged before the +// claim-release valve lets the covered issue go even though no re-engagement +// was ever delivered (an owner the fix loop could not reach). It is the +// whole re-engagement window — MaxReEngagements spaced by ReEngageCooldown — +// so an issue is never offered for a second PR while the first is still +// inside the hive's own repair budget. +const ClaimReleaseAfter = time.Duration(MaxReEngagements) * ReEngageCooldown + +// AbandonedRed reports whether the hive has stopped repairing the PR's +// CURRENT red head SHA: the head is StaleRed AND the fix loop has given up +// on it — the PR was escalated to a human, the re-engagement budget for +// this head is spent, or the head has been red for ClaimReleaseAfter with +// nobody reachable to re-engage. It is the predicate the claim-release +// valve (FilterClaimedIssues, "Fix #3") must use instead of StaleRed: a +// red head that is merely stale is still being re-engaged by the reaper +// and the merge watcher, and releasing its issue then stages the issue +// 010-actionable while the covering PR reads 070-fixing — one piece of +// work with two owners (hivecommons/hive#16522). +func (s *Store) AbandonedRed(repo string, number int, headSHA string) bool { + s.mu.Lock() + defer s.mu.Unlock() + e := s.entries[Key(repo, number)] + if e == nil || e.CurRedSHA == "" || headSHA == "" || e.CurRedSHA != headSHA || e.FirstRedAt.IsZero() { + return false + } + redFor := s.now().Sub(e.FirstRedAt) + if redFor < RedPRStaleAfter { + return false + } + return e.Escalated || e.ReEngagements >= MaxReEngagements || redFor >= ClaimReleaseAfter +} + // TryReEngage atomically decides whether the re-engagement paths (#2/#4) may // dispatch another fix for the PR's current red head SHA, and if so records the // attempt. It returns true at most MaxReEngagements times per red SHA: the cap diff --git a/src/pkg/github/prclaims.go b/src/pkg/github/prclaims.go index 0b649fdd95..b3bdb9fb8d 100644 --- a/src/pkg/github/prclaims.go +++ b/src/pkg/github/prclaims.go @@ -1836,11 +1836,13 @@ func (l *ClaimLedger) Save() error { } // RedStaleFunc reports whether the claiming PR (prRepo, prNumber) is red on a -// required check AND stale (its red head SHA unchanged past the staleness -// threshold). Fix #3 uses it to DECIDE NOT to suppress a claimed issue: a -// healthy claiming PR (green/pending, or a freshly-changed head) still -// suppresses its issue, but a red+stale one releases the issue so the work can -// be re-taken. It is supplied by the caller so this package stays free of the +// required check AND abandoned (its red head SHA unchanged past the staleness +// threshold and the hive's fix loop done with it: escalated, re-engagement +// budget spent, or red for the whole re-engagement window). Fix #3 uses it to +// DECIDE NOT to suppress a claimed issue: a healthy claiming PR (green/pending, +// or a freshly-changed head) or one the reaper is still re-engaging keeps +// suppressing its issue, but an abandoned red one releases the issue so the +// work can be re-taken (hivecommons/hive#16522). It is supplied by the caller so this package stays free of the // escalation-store dependency; a nil func means "never release" (old behavior). // GENERIC: the predicate keys only off check state + commit staleness. type RedStaleFunc func(prRepo string, prNumber int) bool