diff --git a/changelog.d/fixed-16739.md b/changelog.d/fixed-16739.md new file mode 100644 index 0000000000..c10e28352e --- /dev/null +++ b/changelog.d/fixed-16739.md @@ -0,0 +1 @@ +- Workflow auto-approval now retries a PR head on the next sweep when the approve call fails transiently (5xx, network error, cancellation, or a rate-limit `403`) instead of silently spending the head's single approval pass, and no longer trusts this hive's App as triggering actor on fork heads, since a public App installed on a fork pushes with the same bot login ([#16739](https://github.com/hivecommons/hive/issues/16739), [#16743](https://github.com/hivecommons/hive/issues/16743)). diff --git a/src/docs/operator-reference/runtime-and-quota.md b/src/docs/operator-reference/runtime-and-quota.md index 4b24dde531..fb7109f510 100644 --- a/src/docs/operator-reference/runtime-and-quota.md +++ b/src/docs/operator-reference/runtime-and-quota.md @@ -283,9 +283,10 @@ v5 and v6 spokes) tracking one repo all stay on webhook mode. to bot logins unless that is acceptable for a human login you add. Trusting the App as triggering actor covers heads the App pushed itself (its own PRs, rebases, review-fix - commits); it does not extend to contributor code, because GitHub holds - only fork `pull_request` runs behind the gate and the App cannot push to - a fork — a same-repository branch the App pushes to is never gated. Runs + commits), and only when the run's head repository is the PR's base + repository: a fork head is never trusted on the triggering-actor path, + because a public App installed on a contributor's fork pushes with the + same `[bot]` login. Runs of other events (for example `pull_request_target`) are counted in the caption but never approved. Guards: `governor.ci.auto_approve_workflows` (default `true` — unset @@ -304,7 +305,12 @@ v5 and v6 spokes) tracking one repo all stay on webhook mode. lacks `actions: write`) is recorded once, never retried for that head, logged as `workflow approval refused; needs workflow approval on GitHub`, and the caption becomes `CI waiting · needs workflow approval on GitHub` - on every later cycle, including ones that skip the sweep. That refusal + on every later cycle, including ones that skip the sweep. Any other + failure of that POST — a `5xx`, a network error, a cancelled cycle, or a + primary/secondary rate-limit `403` — is logged as `failed to approve + gated workflow run; retrying next sweep` and gives the head's pass back, + so the next sweep re-reads the PR and tries again (denials are not + counted twice). That refusal is bound to the head: granting the App **Actions: Read and write** afterwards does not make the hive retry a head it already recorded a refusal for — the operator approves that run on GitHub, or a new push to diff --git a/src/pkg/github/workflow_auto_approval.go b/src/pkg/github/workflow_auto_approval.go index 7b2d05f1a1..15a5298e32 100644 --- a/src/pkg/github/workflow_auto_approval.go +++ b/src/pkg/github/workflow_auto_approval.go @@ -136,6 +136,20 @@ func (l *workflowApprovalLedger) claim(key workflowApprovalKey) bool { return true } +// release gives the head's pass back after a transient failure, so the next +// sweep retries it. deniedCounted and permissionDenied are kept: a retry +// neither re-counts denials nor forgets a recorded refusal. +func (l *workflowApprovalLedger) release(key workflowApprovalKey) { + l.mu.Lock() + defer l.mu.Unlock() + attempt, ok := l.attempts[key] + if !ok { + return + } + attempt.attempted = false + l.attempts[key] = attempt +} + func (l *workflowApprovalLedger) record(key workflowApprovalKey, runID int64, outcome string) { l.mu.Lock() defer l.mu.Unlock() @@ -184,23 +198,25 @@ func (c *Client) workflowApprovalWithLedger(repo string, number int, head string // every gated run on the head, since the code under test is theirs. // // Trusting the App as triggering actor covers heads the App pushed itself -// (its own PRs, rebases and review-fix commits). GitHub only holds fork -// `pull_request` runs behind "Approve and run workflows", and the App -// cannot push to a fork, so this never approves a contributor's head the -// App merely pushed to: a same-repository branch is not gated, and a fork -// head is always triggered by the fork owner. Trust is per head, not per +// (its own PRs, rebases and review-fix commits), and only when the run's +// head repository is the base repository (headInBase). The login alone +// does not prove the push came from this hive: if the App is public, a +// contributor can install it on their fork, and pushes from that +// installation carry the same `[bot]` login. Fork heads therefore +// never earn trust on the actor path. Trust is per head, not per // repository: a contributor PR later pushed by the App earns no trust for // the heads the contributor pushed. -func workflowApprovalTrusted(author, actor, app string, admitted bool, actors []string) bool { +func workflowApprovalTrusted(author, actor, app string, headInBase, admitted bool, actors []string) bool { if admitted { return true } - for _, login := range []string{author, actor} { + for i, login := range []string{author, actor} { login = strings.TrimSpace(login) if login == "" { continue } - if app != "" && strings.EqualFold(login, app) { + isActor := i == 1 + if app != "" && strings.EqualFold(login, app) && (!isActor || headInBase) { return true } for _, trusted := range actors { @@ -267,13 +283,15 @@ func (c *Client) workflowApprovalAdmitted(repo string, number int, head string) } // splitTrustedWorkflowRuns partitions the approvable gated runs by policy. -func (c *Client) splitTrustedWorkflowRuns(runs []*gh.WorkflowRun, number int, head, headRepo, author string, admitted bool, actors []string) (trusted, denied []*gh.WorkflowRun) { +// baseRepo is the PR's own "owner/name". +func (c *Client) splitTrustedWorkflowRuns(runs []*gh.WorkflowRun, number int, head, headRepo, baseRepo, author string, admitted bool, actors []string) (trusted, denied []*gh.WorkflowRun) { app := c.AppBotLogin() for _, run := range runs { if !workflowRunApprovable(run, number, head, headRepo) { continue } - if workflowApprovalTrusted(author, workflowTriggeringActor(run), app, admitted, actors) { + headInBase := baseRepo != "" && strings.EqualFold(run.GetHeadRepository().GetFullName(), baseRepo) + if workflowApprovalTrusted(author, workflowTriggeringActor(run), app, headInBase, admitted, actors) { trusted = append(trusted, run) } else { denied = append(denied, run) @@ -285,7 +303,9 @@ func (c *Client) splitTrustedWorkflowRuns(runs []*gh.WorkflowRun, number int, he // SweepWorkflowApprovals approves gated workflow runs that EnrichCIStatus // found on the PRs' heads, when policy trusts them, at most once per PR // head. Call it right after EnrichCIStatus on the same slice. A 403/404 is -// recorded once and surfaces as PermissionDenied instead of being retried. +// recorded once and surfaces as PermissionDenied instead of being retried; +// any other failure (5xx, network, rate limit, cancellation) gives the pass +// back so the next sweep retries the head. func (c *Client) SweepWorkflowApprovals(ctx context.Context, prs []PullRequest, policy config.GovernorCIConfig) { if c == nil || c.client == nil || !policy.AutoApproveWorkflowsEnabled() { return @@ -312,7 +332,7 @@ func (c *Client) SweepWorkflowApprovals(ctx context.Context, prs []PullRequest, admitted := c.workflowApprovalAdmitted(key.repo, pr.Number, pr.HeadSHA) // This first split only counts denials off the listing; its trusted // set is discarded and recomputed after the fresh PR read below. - trusted, denied := c.splitTrustedWorkflowRuns(wa.runs, pr.Number, pr.HeadSHA, "", pr.Author, admitted, actors) + trusted, denied := c.splitTrustedWorkflowRuns(wa.runs, pr.Number, pr.HeadSHA, "", key.repo, pr.Author, admitted, actors) if len(denied) > 0 && !previous.deniedCounted { c.workflowApprovals.markDeniedCounted(key) for _, run := range denied { @@ -334,7 +354,7 @@ func (c *Client) SweepWorkflowApprovals(ctx context.Context, prs []PullRequest, if c.workflowApprovalHeld(labels) || headRepo == "" { continue } - trusted, _ = c.splitTrustedWorkflowRuns(wa.runs, pr.Number, pr.HeadSHA, headRepo, fresh.GetUser().GetLogin(), admitted, actors) + trusted, _ = c.splitTrustedWorkflowRuns(wa.runs, pr.Number, pr.HeadSHA, headRepo, key.repo, fresh.GetUser().GetLogin(), admitted, actors) if len(trusted) == 0 { continue } @@ -344,11 +364,12 @@ func (c *Client) SweepWorkflowApprovals(ctx context.Context, prs []PullRequest, continue } // Claim before the first POST: neither a concurrent cycle nor an - // unchanged head repeats the pass, even after a partial failure. + // unchanged head repeats the pass. Only a transient failure gives + // it back (below). if !c.workflowApprovals.claim(key) { continue } - approved := false + approved, permissionRefused, transient := false, false, false for _, run := range trusted { err := c.approveWorkflowRun(ctx, owner, name, run.GetID()) if err == nil { @@ -357,8 +378,8 @@ func (c *Client) SweepWorkflowApprovals(ctx context.Context, prs []PullRequest, c.logger.Info("approved gated workflow run", "repo", key.repo, "pr", pr.Number, "head", pr.HeadSHA, "run_id", run.GetID(), "actor", workflowTriggeringActor(run)) continue } - var apiErr *gh.ErrorResponse - if errors.As(err, &apiErr) && apiErr.Response != nil && (apiErr.Response.StatusCode == http.StatusForbidden || apiErr.Response.StatusCode == http.StatusNotFound) { + if workflowApprovalPermissionRefused(err) { + permissionRefused = true c.workflowApprovals.record(key, run.GetID(), "permission_error") c.logger.Warn("workflow approval refused; needs workflow approval on GitHub (App lacks actions: write?)", "repo", key.repo, "pr", pr.Number, "head", pr.HeadSHA, "run_id", run.GetID(), "error", err) refused := *wa @@ -368,7 +389,11 @@ func (c *Client) SweepWorkflowApprovals(ctx context.Context, prs []PullRequest, pr.CIActivity = activity break } - c.logger.Warn("failed to approve gated workflow run", "repo", key.repo, "pr", pr.Number, "head", pr.HeadSHA, "run_id", run.GetID(), "error", err) + transient = true + c.logger.Warn("failed to approve gated workflow run; retrying next sweep", "repo", key.repo, "pr", pr.Number, "head", pr.HeadSHA, "run_id", run.GetID(), "error", err) + } + if transient && !permissionRefused { + c.workflowApprovals.release(key) } if approved { // Let the next cycle observe the runs starting. @@ -377,6 +402,17 @@ func (c *Client) SweepWorkflowApprovals(ctx context.Context, prs []PullRequest, } } +// workflowApprovalPermissionRefused reports whether an approve POST failed +// because the token may not approve runs here: a plain 403/404 API error. +// go-github types rate-limit 403s as *RateLimitError/*AbuseRateLimitError, +// not *ErrorResponse, so those count as transient like 5xx, network errors +// and cancellation. +func workflowApprovalPermissionRefused(err error) bool { + var apiErr *gh.ErrorResponse + return errors.As(err, &apiErr) && apiErr.Response != nil && + (apiErr.Response.StatusCode == http.StatusForbidden || apiErr.Response.StatusCode == http.StatusNotFound) +} + // approveWorkflowRun calls POST /repos/{o}/{r}/actions/runs/{id}/approve, // which go-github v72 does not bind. func (c *Client) approveWorkflowRun(ctx context.Context, owner, repo string, runID int64) error { diff --git a/src/pkg/github/workflow_auto_approval_test.go b/src/pkg/github/workflow_auto_approval_test.go index 3ec65ee761..7e1c683b14 100644 --- a/src/pkg/github/workflow_auto_approval_test.go +++ b/src/pkg/github/workflow_auto_approval_test.go @@ -3,6 +3,8 @@ package github import ( "context" "encoding/json" + "errors" + "fmt" "net/http" "net/http/httptest" "strings" @@ -19,14 +21,19 @@ func TestWorkflowApprovalTrusted(t *testing.T) { defaults := (config.GovernorCIConfig{}).WorkflowApprovalActors() for _, tc := range []struct { name, author, actor, app string - admitted bool + headInBase, admitted bool actors []string want bool }{ {name: "dependabot author", author: "dependabot[bot]", actor: "someone", actors: defaults, want: true}, {name: "renovate triggering actor", author: "outsider", actor: "renovate[bot]", actors: defaults, want: true}, {name: "app author", author: "hive[bot]", actor: "outsider", app: "hive[bot]", want: true}, - {name: "app triggering actor", author: "outsider", actor: "hive[bot]", app: "hive[bot]", want: true}, + {name: "app triggering actor", author: "outsider", actor: "hive[bot]", app: "hive[bot]", headInBase: true, want: true}, + // A public App installed on a contributor's fork pushes with the + // same login: the actor path never trusts a fork head. + {name: "app triggering actor on fork head", author: "outsider", actor: "hive[bot]", app: "hive[bot]"}, + {name: "app author on fork head", author: "hive[bot]", actor: "outsider", app: "hive[bot]", want: true}, + {name: "allowlisted actor on fork head", author: "outsider", actor: "dependabot[bot]", actors: defaults, want: true}, {name: "owner admission", author: "outsider", actor: "outsider", admitted: true, want: true}, {name: "case and whitespace", author: "outsider", actor: "CUSTOM[bot]", actors: []string{" custom[bot] "}, want: true}, {name: "untrusted human", author: "outsider", actor: "outsider", actors: defaults}, @@ -35,7 +42,7 @@ func TestWorkflowApprovalTrusted(t *testing.T) { {name: "explicit empty allowlist", author: "dependabot[bot]", actor: "dependabot[bot]", actors: []string{}}, } { t.Run(tc.name, func(t *testing.T) { - if got := workflowApprovalTrusted(tc.author, tc.actor, tc.app, tc.admitted, tc.actors); got != tc.want { + if got := workflowApprovalTrusted(tc.author, tc.actor, tc.app, tc.headInBase, tc.admitted, tc.actors); got != tc.want { t.Fatalf("trusted = %v, want %v", got, tc.want) } }) @@ -113,6 +120,8 @@ type workflowApprovalFixture struct { posts []string freshReads int approvalStatus int + // transientFailures fails that many approve POSTs with a 502 first. + transientFailures int } func (f *workflowApprovalFixture) counts() (int, int) { @@ -139,6 +148,11 @@ func newWorkflowApprovalFixture(t *testing.T) (*Client, *workflowApprovalFixture _ = json.NewEncoder(w).Encode(f.fresh) case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/approve"): f.posts = append(f.posts, r.URL.Path) + if f.transientFailures > 0 { + f.transientFailures-- + http.Error(w, `{"message":"Bad Gateway"}`, http.StatusBadGateway) + return + } if f.approvalStatus != 0 { http.Error(w, `{"message":"Resource not accessible by integration"}`, f.approvalStatus) return @@ -257,6 +271,92 @@ func TestSweepWorkflowApprovalsPermissionRefusalOnce(t *testing.T) { } } +func TestWorkflowApprovalPermissionRefused(t *testing.T) { + status := func(code int) *http.Response { return &http.Response{StatusCode: code} } + for _, tc := range []struct { + name string + err error + want bool + }{ + {name: "403", err: &gh.ErrorResponse{Response: status(http.StatusForbidden)}, want: true}, + {name: "404", err: &gh.ErrorResponse{Response: status(http.StatusNotFound)}, want: true}, + {name: "wrapped 403", err: fmt.Errorf("approve: %w", &gh.ErrorResponse{Response: status(http.StatusForbidden)}), want: true}, + {name: "500", err: &gh.ErrorResponse{Response: status(http.StatusInternalServerError)}}, + {name: "502", err: &gh.ErrorResponse{Response: status(http.StatusBadGateway)}}, + {name: "primary rate limit", err: &gh.RateLimitError{Response: status(http.StatusForbidden)}}, + {name: "secondary rate limit", err: &gh.AbuseRateLimitError{Response: status(http.StatusForbidden)}}, + {name: "network error", err: errors.New("connection reset by peer")}, + {name: "context canceled", err: context.Canceled}, + {name: "no response", err: &gh.ErrorResponse{}}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := workflowApprovalPermissionRefused(tc.err); got != tc.want { + t.Fatalf("refused = %v, want %v", got, tc.want) + } + }) + } +} + +// A transient POST failure must not spend the head's one pass: the next +// sweep retries, without counting the head's denials a second time. +func TestSweepWorkflowApprovalsRetriesTransientFailure(t *testing.T) { + c, f, _ := newWorkflowApprovalFixture(t) + f.fresh.User.Login = gh.Ptr("outsider") + f.transientFailures = 1 + sweep := func() []PullRequest { + prs := []PullRequest{gatedPR("outsider", gatedRun(11, "head1", "renovate[bot]"), gatedRun(12, "head1", "outsider"))} + c.SweepWorkflowApprovals(context.Background(), prs, config.GovernorCIConfig{}) + return prs + } + prs := sweep() + if posts, _ := f.counts(); posts != 1 { + t.Fatalf("first pass: posts=%d", posts) + } + if wa := prs[0].CIActivity.WorkflowApproval; wa == nil || wa.PermissionDenied { + t.Fatalf("transient failure surfaced as a permission refusal: %+v", wa) + } + if stats := c.WorkflowApprovalsStatus(); stats.Approved != 0 || stats.PermissionErrors != 0 || stats.Denied != 1 { + t.Fatalf("after transient failure: stats=%+v", stats) + } + sweep() + if posts, _ := f.counts(); posts != 2 { + t.Fatalf("transient failure consumed the pass: posts=%d", posts) + } + if stats := c.WorkflowApprovalsStatus(); stats.Approved != 1 || stats.Denied != 1 { + t.Fatalf("after retry: stats=%+v", stats) + } + // Approved: the pass is spent again. + sweep() + if posts, _ := f.counts(); posts != 2 { + t.Fatalf("approved head repeated: posts=%d", posts) + } +} + +// The App as triggering actor is trusted only on a head in the base +// repository; on a fork head it may be a public App installed by the fork. +func TestSweepWorkflowApprovalsAppActorNeedsBaseRepoHead(t *testing.T) { + for _, tc := range []struct { + name, headRepo string + want int + }{ + {name: "base repository head", headRepo: "org/repo", want: 1}, + {name: "fork head", headRepo: "fork/repo"}, + } { + t.Run(tc.name, func(t *testing.T) { + c, f, _ := newWorkflowApprovalFixture(t) + c.SetAppBotLogin("hive[bot]") + f.fresh.User.Login = gh.Ptr("outsider") + f.fresh.Head.Repo.FullName = gh.Ptr(tc.headRepo) + run := gatedRun(11, "head1", "hive[bot]") + run.HeadRepository.FullName = gh.Ptr(tc.headRepo) + c.SweepWorkflowApprovals(context.Background(), []PullRequest{gatedPR("outsider", run)}, config.GovernorCIConfig{}) + if posts, _ := f.counts(); posts != tc.want { + t.Fatalf("posts=%d, want %d", posts, tc.want) + } + }) + } +} + func TestSweepWorkflowApprovalsUntrustedThenAdmitted(t *testing.T) { c, f, _ := newWorkflowApprovalFixture(t) f.fresh.User.Login = gh.Ptr("outsider")