diff --git a/changelog.d/fixed-18608-prow-refusal-repo-memory.md b/changelog.d/fixed-18608-prow-refusal-repo-memory.md new file mode 100644 index 0000000000..dc1355711b --- /dev/null +++ b/changelog.d/fixed-18608-prow-refusal-repo-memory.md @@ -0,0 +1 @@ +- Remember per repository that Prow refuses the App's `/ok-to-test`, so a new fork PR there gets the needs-human note instead of a first command Prow rejects; the memory is re-derived from the repository's comments after a restart and clears when the `hive-ok-to-test` workflow is installed (#18608). diff --git a/src/docs/fork-pr-presubmit-stall-runbook.md b/src/docs/fork-pr-presubmit-stall-runbook.md index 5ce9801981..85e43a9148 100644 --- a/src/docs/fork-pr-presubmit-stall-runbook.md +++ b/src/docs/fork-pr-presubmit-stall-runbook.md @@ -49,6 +49,9 @@ ask that owner to comment `/approve`. admitted non-member fork PR. Making the hive App (or contributor agents) a Prow trusted user remains a Prow configuration choice; until then Prow rejects the App's command and step 2 applies. +- Done in #18608: once Prow has refused the App anywhere in a repository, the + lane skips the first `/ok-to-test` on later fork PRs there and leaves the + needs-human note directly; step 2 applies to those PRs too. - Relay `/lgtm` and `/approve` with the approving human's token, or add the hive App to `OWNERS`; surface Prow's rejection reply on the board instead of showing "approved". diff --git a/src/docs/hive-merge.md b/src/docs/hive-merge.md index 100db6170e..72d71874cd 100644 --- a/src/docs/hive-merge.md +++ b/src/docs/hive-merge.md @@ -864,7 +864,17 @@ mutation-boundary dedupe as the trusted-lane command. If Prow replies `skipped=prow-rejects-bot-commands`, leaves one `` needs-human note and never posts the command on that PR again; making the App a Prow trusted user is the -repository's Prow configuration. The merge-request relay's read-only +repository's Prow configuration. Prow's answer is the repository's, so it is +also remembered per repository (hivecommons/hive#18608): once Prow has refused +the App anywhere in the repository, the lane skips the first command on every +later fork PR there and leaves the `` note +directly. The memory is not stored: after a restart it is re-derived from one +bounded read of the repository's comments of the last 30 days (Prow's refusal +addressed to the App, or the hive's own note quoting it), kept for an hour, and +cleared while the repository carries `.github/workflows/hive-ok-to-test.yml`. +A repository with no refusal on record behaves as before. The note in that case +does not quote Prow's wording, so only a real refusal renews the memory and it +ages out if Prow later trusts the App. The merge-request relay's read-only evaluation posts nothing. A PR merges through this tier only when all of the following hold: diff --git a/src/pkg/github/automerge/automerge_contributor_lane_test.go b/src/pkg/github/automerge/automerge_contributor_lane_test.go index e54e5d9c31..702cfc57ee 100644 --- a/src/pkg/github/automerge/automerge_contributor_lane_test.go +++ b/src/pkg/github/automerge/automerge_contributor_lane_test.go @@ -22,6 +22,7 @@ import ( "github.com/hivecommons/hive/pkg/authorkind" hgithub "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/intent" + "github.com/hivecommons/hive/pkg/oktotest" "github.com/hivecommons/hive/pkg/review" hreview "github.com/hivecommons/hive/pkg/review" "github.com/hivecommons/hive/pkg/stageengine" @@ -117,6 +118,12 @@ type contributorLaneFixture struct { // headStatuses is what GET commits/sha7/statuses (every commit status, // with its creator) serves; nil serves an empty list. headStatuses []map[string]any + // repoComments is what GET issues/comments (the repository-wide comment + // listing the Prow refusal memory reads) serves; repoCommentLists counts + // those reads. workflowInstalled serves the hive-ok-to-test workflow file. + repoComments []map[string]any + repoCommentLists int + workflowInstalled bool // baseRef, when set, gives the PR base a branch name and serves the // branch, protection and compare routes the current-base gate // (automerge_current_base.go) and release hold need; empty leaves the @@ -336,6 +343,19 @@ func newContributorLaneAPI(t *testing.T, fx *contributorLaneFixture) *httptest.S json.NewEncoder(w).Encode(fx.issueComments) case r.Method == http.MethodGet && r.URL.Path == "/repos/acme/widget/issues/7/comments": json.NewEncoder(w).Encode(fx.prComments) + case r.Method == http.MethodGet && r.URL.Path == "/repos/acme/widget/issues/comments": + fx.repoCommentLists++ + repoComments := fx.repoComments + if repoComments == nil { + repoComments = []map[string]any{} + } + json.NewEncoder(w).Encode(repoComments) + case r.Method == http.MethodGet && r.URL.Path == "/repos/acme/widget/contents/"+oktotest.WorkflowPath: + if !fx.workflowInstalled { + http.NotFound(w, r) + return + } + json.NewEncoder(w).Encode(map[string]any{"type": "file", "name": "hive-ok-to-test.yml", "path": oktotest.WorkflowPath}) case r.Method == http.MethodGet && r.URL.Path == "/repos/acme/widget/pulls/7/files": if fx.filesStatus != 0 { http.Error(w, "files error", fx.filesStatus) diff --git a/src/pkg/github/automerge/automerge_contributor_prow_oktotest.go b/src/pkg/github/automerge/automerge_contributor_prow_oktotest.go index 04e9252929..5baa507354 100644 --- a/src/pkg/github/automerge/automerge_contributor_prow_oktotest.go +++ b/src/pkg/github/automerge/automerge_contributor_prow_oktotest.go @@ -48,7 +48,9 @@ var prowTrustedAssociations = map[string]bool{ // a fork PR by a non-member on a Prow-managed repository (Prow's labels or its // approve plugin's comment), carries no `ok-to-test` label, and Prow has // reported no presubmit status on the head. A Prow rejection of the App's -// command is recorded once per PR and stops every later post. The caller +// command is recorded once per PR and stops every later post; it is also +// remembered per repository (automerge_prow_refusal_memory.go), so a later +// fork PR there gets the needs-human note without the command being posted. The caller // must already have admitted the PR to the contributor lane. func (c *Engine) requestProwPresubmitsForFork(ctx context.Context, displayRepo, owner, repo string, pr *gh.PullRequest, number int, headSHA string) { if c == nil || c.gh == nil || pr == nil || strings.TrimSpace(headSHA) == "" || !c.isForkPR(pr) { @@ -61,6 +63,7 @@ func (c *Engine) requestProwPresubmitsForFork(ctx context.Context, displayRepo, if hasLabel(labels, prowOkToTestLabel) || c.workflowApprovalUntrustedKnown(owner, repo, number) { return } + knownRefusal := c.prowRefusesApp(ctx, displayRepo, owner, repo) _, first := c.markWorkflowApprovalCommandSeen(owner, repo, number, headSHA) state, scanned := c.workflowApprovalCommentState(ctx, displayRepo, owner, repo, number, headSHA, first) if !scanned { @@ -68,6 +71,7 @@ func (c *Engine) requestProwPresubmitsForFork(ctx context.Context, displayRepo, } if state.rejected { c.markWorkflowApprovalUntrusted(owner, repo, number) + c.rememberProwRefusal(owner, repo) args := []any{"repo", displayRepo, "pr", number, "sha", headSHA, "skipped", hgithub.PRApprovalCommandsSkippedProw} const msg = "contributor-lane ok-to-test not posted again: Prow rejected the App's command on this fork PR; a trusted user must post it" if state.rejectionNoted { @@ -84,6 +88,17 @@ func (c *Engine) requestProwPresubmitsForFork(ctx context.Context, displayRepo, if c.prowPresubmitStatusPresent(ctx, displayRepo, owner, repo, number, headSHA) { return } + if knownRefusal { + // Prow already refused the App in this repository: skip the first + // command and hand the PR to a person directly (hivecommons/hive#18608). + c.markWorkflowApprovalUntrusted(owner, repo, number) + if !state.rejectionNoted { + c.postWorkflowApprovalUntrustedNote(ctx, displayRepo, owner, repo, number, headSHA, prowKnownRefusalNote(strings.TrimSpace(c.transport.AppBotLogin()))) + } + c.debug("contributor-lane ok-to-test not posted: Prow already refused the App's command on this repository; a trusted user must post it", + "repo", displayRepo, "pr", number, "sha", headSHA, "skipped", hgithub.PRApprovalCommandsSkippedProw) + return + } c.postWorkflowApprovalCommand(ctx, displayRepo, owner, repo, number, headSHA) } diff --git a/src/pkg/github/automerge/automerge_held_runs.go b/src/pkg/github/automerge/automerge_held_runs.go index 88fd8a69f1..03882aab7a 100644 --- a/src/pkg/github/automerge/automerge_held_runs.go +++ b/src/pkg/github/automerge/automerge_held_runs.go @@ -238,6 +238,9 @@ func (c *Engine) workflowApprovalAttention(ctx context.Context, displayRepo, own // users: every further post gets the same rejection, so stop and // hand the command to a person once. c.markWorkflowApprovalUntrusted(owner, repo, number) + if state.rejected { + c.rememberProwRefusal(owner, repo) + } if !state.rejectionNoted { c.postWorkflowApprovalUntrustedNote(ctx, displayRepo, owner, repo, number, headSHA, workflowApprovalUntrustedNote(strings.TrimSpace(c.transport.AppBotLogin()), state.rejected)) } diff --git a/src/pkg/github/automerge/automerge_prow_refusal_memory.go b/src/pkg/github/automerge/automerge_prow_refusal_memory.go new file mode 100644 index 0000000000..94822baf7e --- /dev/null +++ b/src/pkg/github/automerge/automerge_prow_refusal_memory.go @@ -0,0 +1,151 @@ +package automerge + +// Repo-level memory that Prow refuses the App's /ok-to-test +// (hivecommons/hive#18608). Whether Prow trusts the App is the repository's +// Prow configuration, so one refusal on any PR answers the question for every +// later fork PR in that repository. The per-PR memory +// (workflowApprovalUntrusted) cannot: it starts empty for each new PR and for +// each restarted process, so the first /ok-to-test on every new fork PR drew +// Prow's "Cannot trigger testing" reply. +// +// The memory is a cache over the repository's own comments, so it survives a +// restart without a store: one repository-wide comment listing, bounded by +// prowRefusalLookback and prowRefusalMaxPages, looks for Prow's refusal +// addressed to the App or for the hive's own needs-human note that quotes it. +// The result, positive or negative, is kept for okToTestWorkflowCheckTTL. It +// clears when the repository carries the self-contained hive-ok-to-test +// workflow, which approves held runs whatever Prow thinks of the App. + +import ( + "context" + "strings" + "time" + + gh "github.com/google/go-github/v72/github" + hgithub "github.com/hivecommons/hive/pkg/github" +) + +const ( + // prowRefusalLookback is how far back the repository's comments are read + // for evidence of a refusal. + prowRefusalLookback = 30 * 24 * time.Hour + // prowRefusalMaxPages bounds the quota one derivation may spend. + prowRefusalMaxPages = 3 + // prowRefusalPageSize is the comments per page of that listing. + prowRefusalPageSize = 100 +) + +// prowRefusalMemo is one repository's remembered answer and when it was read. +type prowRefusalMemo struct { + at time.Time + refused bool +} + +func prowRefusalKey(owner, repo string) string { + return strings.ToLower(owner + "/" + repo) +} + +// rememberProwRefusal records that Prow refused the App's command in this +// repository, so no later PR here is asked first. +func (c *Engine) rememberProwRefusal(owner, repo string) { + c.storeProwRefusal(owner, repo, true) +} + +func (c *Engine) storeProwRefusal(owner, repo string, refused bool) { + c.workflowApprovalCommandsMu.Lock() + defer c.workflowApprovalCommandsMu.Unlock() + if c.prowRefusals == nil { + c.prowRefusals = make(map[string]prowRefusalMemo) + } + c.prowRefusals[prowRefusalKey(owner, repo)] = prowRefusalMemo{at: c.clock(), refused: refused} +} + +// prowRefusesApp reports whether Prow is known to refuse the App's +// /ok-to-test in this repository. A repository that carries the +// hive-ok-to-test workflow is never reported as refusing. Unknown, or a +// lookup that failed, reports false so the existing post-once behaviour holds. +func (c *Engine) prowRefusesApp(ctx context.Context, displayRepo, owner, repo string) bool { + key := prowRefusalKey(owner, repo) + c.workflowApprovalCommandsMu.Lock() + memo, ok := c.prowRefusals[key] + c.workflowApprovalCommandsMu.Unlock() + if !ok || c.clock().Sub(memo.at) >= okToTestWorkflowCheckTTL { + memo.refused = c.deriveProwRefusal(ctx, displayRepo, owner, repo) + c.storeProwRefusal(owner, repo, memo.refused) + } + if !memo.refused { + return false + } + if c.okToTestWorkflowInstalled(ctx, displayRepo, owner, repo) { + c.storeProwRefusal(owner, repo, false) + return false + } + return true +} + +// deriveProwRefusal reads the repository's recent comments for a Prow refusal +// addressed to the App, or the hive's own note quoting one. +func (c *Engine) deriveProwRefusal(ctx context.Context, displayRepo, owner, repo string) bool { + appLogin := "" + if c.transport != nil { + appLogin = strings.TrimSpace(c.transport.AppBotLogin()) + } + since := c.clock().Add(-prowRefusalLookback) + opts := &gh.IssueListCommentsOptions{ + Sort: gh.Ptr("created"), + Direction: gh.Ptr("desc"), + Since: &since, + ListOptions: gh.ListOptions{PerPage: prowRefusalPageSize}, + } + for page := 0; page < prowRefusalMaxPages; page++ { + comments, resp, err := c.gh.Issues.ListComments(hgithub.WithRESTCaller(ctx, "hive:automerge_sweep"), owner, repo, 0, opts) + if err != nil { + c.warn("automerge could not list repository comments for Prow ok-to-test refusals", "repo", displayRepo, "error", err) + return false + } + for _, comment := range comments { + if comment == nil || !strings.Contains(comment.GetBody(), prowOkToTestRejection) { + continue + } + body := comment.GetBody() + login := hgithub.SafeGetLogin(comment.GetUser()) + if appLogin != "" && strings.EqualFold(login, appLogin) { + if strings.Contains(body, workflowApprovalRejectedMarker) { + return true + } + continue + } + if appLogin == "" || strings.Contains(strings.ToLower(body), "@"+strings.ToLower(appLogin)) { + return true + } + } + if resp == nil || resp.NextPage == 0 { + break + } + opts.Page = resp.NextPage + } + return false +} + +// prowKnownRefusalNote is the one needs-human comment posted on a fork PR +// when the repository is already known to refuse the App's command, instead +// of posting the command to be refused. It deliberately omits Prow's refusal +// wording: only a real refusal may renew the repository's memory, so the +// memory ages out if Prow later trusts the App. +func prowKnownRefusalNote(appLogin string) string { + if appLogin == "" { + appLogin = "the Hive App" + } + var b strings.Builder + b.WriteString(workflowApprovalRejectedMarker) + b.WriteString("\nneeds-human: Prow has not run this fork PR's presubmits. Its author is not one of Prow's trusted users, so the trigger plugin waits for `") + b.WriteString(workflowApprovalCommand) + b.WriteString("` from a trusted user. Prow already refused the command from `") + b.WriteString(appLogin) + b.WriteString("` on this repository, so Hive did not post it here.\n\nA maintainer can comment `") + b.WriteString(workflowApprovalCommand) + b.WriteString("` (plus `/test all` for the current head), or make `") + b.WriteString(appLogin) + b.WriteString("` a Prow trusted user so Hive's command counts.") + return b.String() +} diff --git a/src/pkg/github/automerge/automerge_prow_refusal_memory_test.go b/src/pkg/github/automerge/automerge_prow_refusal_memory_test.go new file mode 100644 index 0000000000..7e7e56a302 --- /dev/null +++ b/src/pkg/github/automerge/automerge_prow_refusal_memory_test.go @@ -0,0 +1,166 @@ +package automerge + +// Repo-level memory that Prow refuses the App's /ok-to-test +// (hivecommons/hive#18608): once Prow refused the App anywhere in a +// repository, a later fork PR there gets the needs-human note without the +// first command being posted. + +import ( + "context" + "strings" + "testing" + "time" +) + +const prowRefusalReply = "@" + testHiveAppBotLogin + ": Cannot trigger testing until a trusted user reviews the PR and leaves an `/ok-to-test` message." + +// refusalOnEarlierPR is the repository-wide listing after Prow refused the App +// on some other PR. +func refusalOnEarlierPR() []map[string]any { + return []map[string]any{prowComment(90, prowRefusalReply)} +} + +// runProwRefusalEngines sweeps PR 7 with one fresh engine per entry in +// restarts, so each is a restarted process with an empty in-process memory. +func runProwRefusalEngines(t *testing.T, fx *contributorLaneFixture, restarts int) { + t.Helper() + api := newContributorLaneAPI(t, fx) + defer api.Close() + policy := defaultContributorLanePolicy() + clock := &contributorLaneClock{at: time.Date(2026, 11, 9, 12, 0, 0, 0, time.UTC)} + for run := 0; run < restarts; run++ { + engine := newContributorLaneEngine(api.URL, policy, clock, Options{ContributorSoak: soakedTracker(clock)}) + if _, _, err := engine.trySweepContributorLanePR(context.Background(), "widget", "acme", "widget", 7, policy); err != nil { + t.Fatalf("run %d err = %v", run, err) + } + clock.advance(workflowApprovalRecheck + time.Minute) + } +} + +func countCommand(fx *contributorLaneFixture) int { + n := 0 + for _, body := range fx.allComments { + if body == workflowApprovalCommandBody("sha7") { + n++ + } + } + return n +} + +func TestKnownRefusingRepoGetsNoFirstOkToTest(t *testing.T) { + fx := prowOkToTestFixture() + fx.repoComments = refusalOnEarlierPR() + runProwRefusalEngines(t, &fx, 1) + if got := countCommand(&fx); got != 0 { + t.Fatalf("ok-to-test posts = %d (%q), want none in a repository Prow already refused the App in", got, fx.allComments) + } + if fx.comments != 1 { + t.Fatalf("comments = %d %q, want exactly the one needs-human note", fx.comments, fx.allComments) + } + note := fx.allComments[0] + if !strings.HasPrefix(note, workflowApprovalRejectedMarker) || !strings.Contains(note, "needs-human") || !strings.Contains(note, testHiveAppBotLogin) { + t.Fatalf("note = %q, want the rejected marker and a needs-human note naming the App", note) + } + if strings.Contains(note, prowOkToTestRejection) { + t.Fatalf("note = %q quotes Prow's refusal, which would renew the memory forever", note) + } + for _, line := range strings.Split(note, "\n") { + if strings.TrimSpace(line) == workflowApprovalCommand { + t.Fatalf("note = %q carries a bare %s line Prow would act on", note, workflowApprovalCommand) + } + } +} + +func TestUnknownRepoStillPostsOkToTestOnce(t *testing.T) { + fx := prowOkToTestFixture() + runProwRefusalEngines(t, &fx, 2) + if got := countCommand(&fx); got != 1 { + t.Fatalf("ok-to-test posts = %d (%q), want exactly one in a repository with no refusal on record", got, fx.allComments) + } + if fx.repoCommentLists == 0 { + t.Fatalf("repository comments were never read, so the memory was not consulted") + } +} + +// The memory is derived from the repository's comments, so a restarted hive +// (empty in-process state) still skips the command, and posts the note once. +func TestProwRefusalMemorySurvivesRestart(t *testing.T) { + fx := prowOkToTestFixture() + fx.repoComments = refusalOnEarlierPR() + runProwRefusalEngines(t, &fx, 3) + if got := countCommand(&fx); got != 0 { + t.Fatalf("ok-to-test posts = %d (%q), want none across restarts", got, fx.allComments) + } + if fx.comments != 1 { + t.Fatalf("comments = %d %q, want one note across three restarts", fx.comments, fx.allComments) + } +} + +// The hive's own needs-human note from another PR is evidence too. +func TestProwRefusalMemoryFromHiveNote(t *testing.T) { + fx := prowOkToTestFixture() + fx.repoComments = []map[string]any{appComment(91, workflowApprovalUntrustedNote(testHiveAppBotLogin, true))} + runProwRefusalEngines(t, &fx, 1) + if got := countCommand(&fx); got != 0 { + t.Fatalf("ok-to-test posts = %d, want none", got) + } +} + +// A refusal addressed to someone else is not a refusal of the App. +func TestProwRefusalForAnotherUserIsNotMemory(t *testing.T) { + fx := prowOkToTestFixture() + fx.repoComments = []map[string]any{prowComment(92, "@someone-else: Cannot trigger testing until a trusted user reviews the PR.")} + runProwRefusalEngines(t, &fx, 1) + if got := countCommand(&fx); got != 1 { + t.Fatalf("ok-to-test posts = %d, want 1", got) + } +} + +// Once the repository carries the self-contained hive-ok-to-test workflow the +// memory clears and the lane behaves as before. +func TestProwRefusalMemoryClearsWhenWorkflowInstalled(t *testing.T) { + fx := prowOkToTestFixture() + fx.repoComments = refusalOnEarlierPR() + fx.workflowInstalled = true + runProwRefusalEngines(t, &fx, 1) + if got := countCommand(&fx); got != 1 { + t.Fatalf("ok-to-test posts = %d (%q), want 1 once the workflow is installed", got, fx.allComments) + } +} + +func TestProwRefusalMemoryIsReadOncePerTTL(t *testing.T) { + fx := prowOkToTestFixture() + fx.repoComments = refusalOnEarlierPR() + api := newContributorLaneAPI(t, &fx) + defer api.Close() + policy := defaultContributorLanePolicy() + clock := &contributorLaneClock{at: time.Date(2026, 11, 9, 12, 0, 0, 0, time.UTC)} + engine := newContributorLaneEngine(api.URL, policy, clock, Options{ContributorSoak: soakedTracker(clock)}) + for i := 0; i < 3; i++ { + engine.prowRefusesApp(context.Background(), "acme/widget", "acme", "widget") + } + if fx.repoCommentLists != 1 { + t.Fatalf("repository comment reads = %d, want 1 within the TTL", fx.repoCommentLists) + } + clock.advance(okToTestWorkflowCheckTTL + time.Minute) + engine.prowRefusesApp(context.Background(), "acme/widget", "acme", "widget") + if fx.repoCommentLists != 2 { + t.Fatalf("repository comment reads = %d, want 2 after the TTL", fx.repoCommentLists) + } +} + +// No behaviour change for a repository without Prow: no Prow signal on the PR +// means nothing is posted, with or without a refusal elsewhere. +func TestProwRefusalMemoryNoChangeForNonProwRepo(t *testing.T) { + for _, refused := range []bool{false, true} { + fx := prowOkToTestFixture() + fx.labels = nil + if refused { + fx.repoComments = refusalOnEarlierPR() + } + runProwRefusalEngines(t, &fx, 2) + if fx.comments != 0 { + t.Fatalf("refused=%v comments = %d %q, want none on a PR with no Prow signal", refused, fx.comments, fx.allComments) + } + } +} diff --git a/src/pkg/github/automerge/automerge_sweep.go b/src/pkg/github/automerge/automerge_sweep.go index 0f8d9375fe..368e3f568f 100644 --- a/src/pkg/github/automerge/automerge_sweep.go +++ b/src/pkg/github/automerge/automerge_sweep.go @@ -393,6 +393,9 @@ type Engine struct { workflowApprovalScans map[string]time.Time workflowApprovalUntrusted map[string]bool workflowApprovalWorkflowChecks map[string]okToTestWorkflowCheck + // prowRefusals is the repository-level memory that Prow refuses the + // App's /ok-to-test (automerge_prow_refusal_memory.go). + prowRefusals map[string]prowRefusalMemo // terminalHoldsLogged remembers PR head+reason pairs already reported as a // terminal held/never-started hold so later ticks only count them. terminalHoldsLogged map[string]bool