Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-16739.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Workflow auto-approval now retries a PR head on the next sweep when the approve call fails transiently (5xx, network error, cancellation, or a rate-limit `403`) instead of silently spending the head's single approval pass, and no longer trusts this hive's App as triggering actor on fork heads, since a public App installed on a fork pushes with the same bot login ([#16739](https://github.com/hivecommons/hive/issues/16739), [#16743](https://github.com/hivecommons/hive/issues/16743)).
14 changes: 10 additions & 4 deletions src/docs/operator-reference/runtime-and-quota.md
Original file line number Diff line number Diff line change
Expand Up @@ -283,9 +283,10 @@ v5 and v6 spokes) tracking one repo all stay on webhook mode.
to bot logins unless that is acceptable for a human login you add.
Trusting the App as triggering actor
covers heads the App pushed itself (its own PRs, rebases, review-fix
commits); it does not extend to contributor code, because GitHub holds
only fork `pull_request` runs behind the gate and the App cannot push to
a fork — a same-repository branch the App pushes to is never gated. Runs
commits), and only when the run's head repository is the PR's base
repository: a fork head is never trusted on the triggering-actor path,
because a public App installed on a contributor's fork pushes with the
same `<slug>[bot]` login. Runs
of other events (for example
`pull_request_target`) are counted in the caption but never approved.
Guards: `governor.ci.auto_approve_workflows` (default `true` — unset
Expand All @@ -304,7 +305,12 @@ v5 and v6 spokes) tracking one repo all stay on webhook mode.
lacks `actions: write`) is recorded once, never retried for that head,
logged as `workflow approval refused; needs workflow approval on GitHub`,
and the caption becomes `CI waiting · needs workflow approval on GitHub`
on every later cycle, including ones that skip the sweep. That refusal
on every later cycle, including ones that skip the sweep. Any other
failure of that POST — a `5xx`, a network error, a cancelled cycle, or a
primary/secondary rate-limit `403` — is logged as `failed to approve
gated workflow run; retrying next sweep` and gives the head's pass back,
so the next sweep re-reads the PR and tries again (denials are not
counted twice). That refusal
is bound to the head: granting the App **Actions: Read and write**
afterwards does not make the hive retry a head it already recorded a
refusal for — the operator approves that run on GitHub, or a new push to
Expand Down
72 changes: 54 additions & 18 deletions src/pkg/github/workflow_auto_approval.go
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,20 @@ func (l *workflowApprovalLedger) claim(key workflowApprovalKey) bool {
return true
}

// release gives the head's pass back after a transient failure, so the next
// sweep retries it. deniedCounted and permissionDenied are kept: a retry
// neither re-counts denials nor forgets a recorded refusal.
func (l *workflowApprovalLedger) release(key workflowApprovalKey) {
l.mu.Lock()
defer l.mu.Unlock()
attempt, ok := l.attempts[key]
if !ok {
return
}
attempt.attempted = false
l.attempts[key] = attempt
}

func (l *workflowApprovalLedger) record(key workflowApprovalKey, runID int64, outcome string) {
l.mu.Lock()
defer l.mu.Unlock()
Expand Down Expand Up @@ -184,23 +198,25 @@ func (c *Client) workflowApprovalWithLedger(repo string, number int, head string
// every gated run on the head, since the code under test is theirs.
//
// Trusting the App as triggering actor covers heads the App pushed itself
// (its own PRs, rebases and review-fix commits). GitHub only holds fork
// `pull_request` runs behind "Approve and run workflows", and the App
// cannot push to a fork, so this never approves a contributor's head the
// App merely pushed to: a same-repository branch is not gated, and a fork
// head is always triggered by the fork owner. Trust is per head, not per
// (its own PRs, rebases and review-fix commits), and only when the run's
// head repository is the base repository (headInBase). The login alone
// does not prove the push came from this hive: if the App is public, a
// contributor can install it on their fork, and pushes from that
// installation carry the same `<slug>[bot]` login. Fork heads therefore
// never earn trust on the actor path. Trust is per head, not per
// repository: a contributor PR later pushed by the App earns no trust for
// the heads the contributor pushed.
func workflowApprovalTrusted(author, actor, app string, admitted bool, actors []string) bool {
func workflowApprovalTrusted(author, actor, app string, headInBase, admitted bool, actors []string) bool {
if admitted {
return true
}
for _, login := range []string{author, actor} {
for i, login := range []string{author, actor} {
login = strings.TrimSpace(login)
if login == "" {
continue
}
if app != "" && strings.EqualFold(login, app) {
isActor := i == 1
if app != "" && strings.EqualFold(login, app) && (!isActor || headInBase) {
return true
}
for _, trusted := range actors {
Expand Down Expand Up @@ -267,13 +283,15 @@ func (c *Client) workflowApprovalAdmitted(repo string, number int, head string)
}

// splitTrustedWorkflowRuns partitions the approvable gated runs by policy.
func (c *Client) splitTrustedWorkflowRuns(runs []*gh.WorkflowRun, number int, head, headRepo, author string, admitted bool, actors []string) (trusted, denied []*gh.WorkflowRun) {
// baseRepo is the PR's own "owner/name".
func (c *Client) splitTrustedWorkflowRuns(runs []*gh.WorkflowRun, number int, head, headRepo, baseRepo, author string, admitted bool, actors []string) (trusted, denied []*gh.WorkflowRun) {
app := c.AppBotLogin()
for _, run := range runs {
if !workflowRunApprovable(run, number, head, headRepo) {
continue
}
if workflowApprovalTrusted(author, workflowTriggeringActor(run), app, admitted, actors) {
headInBase := baseRepo != "" && strings.EqualFold(run.GetHeadRepository().GetFullName(), baseRepo)
if workflowApprovalTrusted(author, workflowTriggeringActor(run), app, headInBase, admitted, actors) {
trusted = append(trusted, run)
} else {
denied = append(denied, run)
Expand All @@ -285,7 +303,9 @@ func (c *Client) splitTrustedWorkflowRuns(runs []*gh.WorkflowRun, number int, he
// SweepWorkflowApprovals approves gated workflow runs that EnrichCIStatus
// found on the PRs' heads, when policy trusts them, at most once per PR
// head. Call it right after EnrichCIStatus on the same slice. A 403/404 is
// recorded once and surfaces as PermissionDenied instead of being retried.
// recorded once and surfaces as PermissionDenied instead of being retried;
// any other failure (5xx, network, rate limit, cancellation) gives the pass
// back so the next sweep retries the head.
func (c *Client) SweepWorkflowApprovals(ctx context.Context, prs []PullRequest, policy config.GovernorCIConfig) {
if c == nil || c.client == nil || !policy.AutoApproveWorkflowsEnabled() {
return
Expand All @@ -312,7 +332,7 @@ func (c *Client) SweepWorkflowApprovals(ctx context.Context, prs []PullRequest,
admitted := c.workflowApprovalAdmitted(key.repo, pr.Number, pr.HeadSHA)
// This first split only counts denials off the listing; its trusted
// set is discarded and recomputed after the fresh PR read below.
trusted, denied := c.splitTrustedWorkflowRuns(wa.runs, pr.Number, pr.HeadSHA, "", pr.Author, admitted, actors)
trusted, denied := c.splitTrustedWorkflowRuns(wa.runs, pr.Number, pr.HeadSHA, "", key.repo, pr.Author, admitted, actors)
if len(denied) > 0 && !previous.deniedCounted {
c.workflowApprovals.markDeniedCounted(key)
for _, run := range denied {
Expand All @@ -334,7 +354,7 @@ func (c *Client) SweepWorkflowApprovals(ctx context.Context, prs []PullRequest,
if c.workflowApprovalHeld(labels) || headRepo == "" {
continue
}
trusted, _ = c.splitTrustedWorkflowRuns(wa.runs, pr.Number, pr.HeadSHA, headRepo, fresh.GetUser().GetLogin(), admitted, actors)
trusted, _ = c.splitTrustedWorkflowRuns(wa.runs, pr.Number, pr.HeadSHA, headRepo, key.repo, fresh.GetUser().GetLogin(), admitted, actors)
if len(trusted) == 0 {
continue
}
Expand All @@ -344,11 +364,12 @@ func (c *Client) SweepWorkflowApprovals(ctx context.Context, prs []PullRequest,
continue
}
// Claim before the first POST: neither a concurrent cycle nor an
// unchanged head repeats the pass, even after a partial failure.
// unchanged head repeats the pass. Only a transient failure gives
// it back (below).
if !c.workflowApprovals.claim(key) {
continue
}
approved := false
approved, permissionRefused, transient := false, false, false
for _, run := range trusted {
err := c.approveWorkflowRun(ctx, owner, name, run.GetID())
if err == nil {
Expand All @@ -357,8 +378,8 @@ func (c *Client) SweepWorkflowApprovals(ctx context.Context, prs []PullRequest,
c.logger.Info("approved gated workflow run", "repo", key.repo, "pr", pr.Number, "head", pr.HeadSHA, "run_id", run.GetID(), "actor", workflowTriggeringActor(run))
continue
}
var apiErr *gh.ErrorResponse
if errors.As(err, &apiErr) && apiErr.Response != nil && (apiErr.Response.StatusCode == http.StatusForbidden || apiErr.Response.StatusCode == http.StatusNotFound) {
if workflowApprovalPermissionRefused(err) {
permissionRefused = true
c.workflowApprovals.record(key, run.GetID(), "permission_error")
c.logger.Warn("workflow approval refused; needs workflow approval on GitHub (App lacks actions: write?)", "repo", key.repo, "pr", pr.Number, "head", pr.HeadSHA, "run_id", run.GetID(), "error", err)
refused := *wa
Expand All @@ -368,7 +389,11 @@ func (c *Client) SweepWorkflowApprovals(ctx context.Context, prs []PullRequest,
pr.CIActivity = activity
break
}
c.logger.Warn("failed to approve gated workflow run", "repo", key.repo, "pr", pr.Number, "head", pr.HeadSHA, "run_id", run.GetID(), "error", err)
transient = true
c.logger.Warn("failed to approve gated workflow run; retrying next sweep", "repo", key.repo, "pr", pr.Number, "head", pr.HeadSHA, "run_id", run.GetID(), "error", err)
}
if transient && !permissionRefused {
c.workflowApprovals.release(key)
}
if approved {
// Let the next cycle observe the runs starting.
Expand All @@ -377,6 +402,17 @@ func (c *Client) SweepWorkflowApprovals(ctx context.Context, prs []PullRequest,
}
}

// workflowApprovalPermissionRefused reports whether an approve POST failed
// because the token may not approve runs here: a plain 403/404 API error.
// go-github types rate-limit 403s as *RateLimitError/*AbuseRateLimitError,
// not *ErrorResponse, so those count as transient like 5xx, network errors
// and cancellation.
func workflowApprovalPermissionRefused(err error) bool {
var apiErr *gh.ErrorResponse
return errors.As(err, &apiErr) && apiErr.Response != nil &&
(apiErr.Response.StatusCode == http.StatusForbidden || apiErr.Response.StatusCode == http.StatusNotFound)
}

// approveWorkflowRun calls POST /repos/{o}/{r}/actions/runs/{id}/approve,
// which go-github v72 does not bind.
func (c *Client) approveWorkflowRun(ctx context.Context, owner, repo string, runID int64) error {
Expand Down
106 changes: 103 additions & 3 deletions src/pkg/github/workflow_auto_approval_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package github
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"strings"
Expand All @@ -19,14 +21,19 @@ func TestWorkflowApprovalTrusted(t *testing.T) {
defaults := (config.GovernorCIConfig{}).WorkflowApprovalActors()
for _, tc := range []struct {
name, author, actor, app string
admitted bool
headInBase, admitted bool
actors []string
want bool
}{
{name: "dependabot author", author: "dependabot[bot]", actor: "someone", actors: defaults, want: true},
{name: "renovate triggering actor", author: "outsider", actor: "renovate[bot]", actors: defaults, want: true},
{name: "app author", author: "hive[bot]", actor: "outsider", app: "hive[bot]", want: true},
{name: "app triggering actor", author: "outsider", actor: "hive[bot]", app: "hive[bot]", want: true},
{name: "app triggering actor", author: "outsider", actor: "hive[bot]", app: "hive[bot]", headInBase: true, want: true},
// A public App installed on a contributor's fork pushes with the
// same login: the actor path never trusts a fork head.
{name: "app triggering actor on fork head", author: "outsider", actor: "hive[bot]", app: "hive[bot]"},
{name: "app author on fork head", author: "hive[bot]", actor: "outsider", app: "hive[bot]", want: true},
{name: "allowlisted actor on fork head", author: "outsider", actor: "dependabot[bot]", actors: defaults, want: true},
{name: "owner admission", author: "outsider", actor: "outsider", admitted: true, want: true},
{name: "case and whitespace", author: "outsider", actor: "CUSTOM[bot]", actors: []string{" custom[bot] "}, want: true},
{name: "untrusted human", author: "outsider", actor: "outsider", actors: defaults},
Expand All @@ -35,7 +42,7 @@ func TestWorkflowApprovalTrusted(t *testing.T) {
{name: "explicit empty allowlist", author: "dependabot[bot]", actor: "dependabot[bot]", actors: []string{}},
} {
t.Run(tc.name, func(t *testing.T) {
if got := workflowApprovalTrusted(tc.author, tc.actor, tc.app, tc.admitted, tc.actors); got != tc.want {
if got := workflowApprovalTrusted(tc.author, tc.actor, tc.app, tc.headInBase, tc.admitted, tc.actors); got != tc.want {
t.Fatalf("trusted = %v, want %v", got, tc.want)
}
})
Expand Down Expand Up @@ -113,6 +120,8 @@ type workflowApprovalFixture struct {
posts []string
freshReads int
approvalStatus int
// transientFailures fails that many approve POSTs with a 502 first.
transientFailures int
}

func (f *workflowApprovalFixture) counts() (int, int) {
Expand All @@ -139,6 +148,11 @@ func newWorkflowApprovalFixture(t *testing.T) (*Client, *workflowApprovalFixture
_ = json.NewEncoder(w).Encode(f.fresh)
case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/approve"):
f.posts = append(f.posts, r.URL.Path)
if f.transientFailures > 0 {
f.transientFailures--
http.Error(w, `{"message":"Bad Gateway"}`, http.StatusBadGateway)
return
}
if f.approvalStatus != 0 {
http.Error(w, `{"message":"Resource not accessible by integration"}`, f.approvalStatus)
return
Expand Down Expand Up @@ -257,6 +271,92 @@ func TestSweepWorkflowApprovalsPermissionRefusalOnce(t *testing.T) {
}
}

func TestWorkflowApprovalPermissionRefused(t *testing.T) {
status := func(code int) *http.Response { return &http.Response{StatusCode: code} }
for _, tc := range []struct {
name string
err error
want bool
}{
{name: "403", err: &gh.ErrorResponse{Response: status(http.StatusForbidden)}, want: true},
{name: "404", err: &gh.ErrorResponse{Response: status(http.StatusNotFound)}, want: true},
{name: "wrapped 403", err: fmt.Errorf("approve: %w", &gh.ErrorResponse{Response: status(http.StatusForbidden)}), want: true},
{name: "500", err: &gh.ErrorResponse{Response: status(http.StatusInternalServerError)}},
{name: "502", err: &gh.ErrorResponse{Response: status(http.StatusBadGateway)}},
{name: "primary rate limit", err: &gh.RateLimitError{Response: status(http.StatusForbidden)}},
{name: "secondary rate limit", err: &gh.AbuseRateLimitError{Response: status(http.StatusForbidden)}},
{name: "network error", err: errors.New("connection reset by peer")},
{name: "context canceled", err: context.Canceled},
{name: "no response", err: &gh.ErrorResponse{}},
} {
t.Run(tc.name, func(t *testing.T) {
if got := workflowApprovalPermissionRefused(tc.err); got != tc.want {
t.Fatalf("refused = %v, want %v", got, tc.want)
}
})
}
}

// A transient POST failure must not spend the head's one pass: the next
// sweep retries, without counting the head's denials a second time.
func TestSweepWorkflowApprovalsRetriesTransientFailure(t *testing.T) {
c, f, _ := newWorkflowApprovalFixture(t)
f.fresh.User.Login = gh.Ptr("outsider")
f.transientFailures = 1
sweep := func() []PullRequest {
prs := []PullRequest{gatedPR("outsider", gatedRun(11, "head1", "renovate[bot]"), gatedRun(12, "head1", "outsider"))}
c.SweepWorkflowApprovals(context.Background(), prs, config.GovernorCIConfig{})
return prs
}
prs := sweep()
if posts, _ := f.counts(); posts != 1 {
t.Fatalf("first pass: posts=%d", posts)
}
if wa := prs[0].CIActivity.WorkflowApproval; wa == nil || wa.PermissionDenied {
t.Fatalf("transient failure surfaced as a permission refusal: %+v", wa)
}
if stats := c.WorkflowApprovalsStatus(); stats.Approved != 0 || stats.PermissionErrors != 0 || stats.Denied != 1 {
t.Fatalf("after transient failure: stats=%+v", stats)
}
sweep()
if posts, _ := f.counts(); posts != 2 {
t.Fatalf("transient failure consumed the pass: posts=%d", posts)
}
if stats := c.WorkflowApprovalsStatus(); stats.Approved != 1 || stats.Denied != 1 {
t.Fatalf("after retry: stats=%+v", stats)
}
// Approved: the pass is spent again.
sweep()
if posts, _ := f.counts(); posts != 2 {
t.Fatalf("approved head repeated: posts=%d", posts)
}
}

// The App as triggering actor is trusted only on a head in the base
// repository; on a fork head it may be a public App installed by the fork.
func TestSweepWorkflowApprovalsAppActorNeedsBaseRepoHead(t *testing.T) {
for _, tc := range []struct {
name, headRepo string
want int
}{
{name: "base repository head", headRepo: "org/repo", want: 1},
{name: "fork head", headRepo: "fork/repo"},
} {
t.Run(tc.name, func(t *testing.T) {
c, f, _ := newWorkflowApprovalFixture(t)
c.SetAppBotLogin("hive[bot]")
f.fresh.User.Login = gh.Ptr("outsider")
f.fresh.Head.Repo.FullName = gh.Ptr(tc.headRepo)
run := gatedRun(11, "head1", "hive[bot]")
run.HeadRepository.FullName = gh.Ptr(tc.headRepo)
c.SweepWorkflowApprovals(context.Background(), []PullRequest{gatedPR("outsider", run)}, config.GovernorCIConfig{})
if posts, _ := f.counts(); posts != tc.want {
t.Fatalf("posts=%d, want %d", posts, tc.want)
}
})
}
}

func TestSweepWorkflowApprovalsUntrustedThenAdmitted(t *testing.T) {
c, f, _ := newWorkflowApprovalFixture(t)
f.fresh.User.Login = gh.Ptr("outsider")
Expand Down
Loading