Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-shared-ci-permission-recovery.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Surface shared-CI recovery permission refusals as an actionable operator handoff, retry after the permission cooldown even on unchanged PRs, and align bot-PR squash attribution with the existing required check.
21 changes: 21 additions & 0 deletions src/docs/github-app-setup.md
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,27 @@ Organization permission:
| --- | --- | --- |
| Members | Read-only | Identify contributors and owners where configured. |

### Optional shared-CI recovery

The shared-CI rerun sweep (`shared_ci_rerun.enabled`, default on) can inspect
failed runs with Actions read, but `rerun-failed-jobs` requires **Actions: Read
and write** on the repository's recovery identity. This is optional, not a new
baseline permission for every Hive App. A refusal is an operator handoff, not
a source-code defect: Hive records `unauthorized:rerun-shared-ci`, posts one
head-scoped actionable note using the existing unauthorized mechanism, and
retries no more than once per repository per permission cooldown (one hour).
An unchanged PR remains eligible after the cooldown; a successful rerun or
observing no failed runs clears the recovery block.

A maintainer may re-run the named workflow with their already-authorized
identity. That manual recovery does **not** change the App's permissions. For
automatic recovery, the App owner must explicitly request Actions read and
write for the intended recovery installation, and its installation owner must
accept the update in **Settings → GitHub Apps → Configure**. Do not broaden
unrelated Apps or repositories or silently substitute a PAT. Leaving the grant
at read is supported; an operator may disable the optional sweep instead.
Neither action grants merge approval or bypasses required checks.

### The optional Visual Hive App (#4030)

Visual Hive needs two grants the Hive App does not otherwise require — Actions
Expand Down
14 changes: 12 additions & 2 deletions src/pkg/github/automerge/automerge_stage_dco.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ func dcoCommitStatsDetailedForAuthor(commits []*gh.RepositoryCommit, appLogin, p
} else {
stats.unsignedOther++
}
} else if dcoCommitByHive(rc, appLogin) && !messageHasPRAuthorSignoff(msg, prAuthor) {
} else if !botSquashAuthor(prAuthor) && dcoCommitByHive(rc, appLogin) && !messageHasPRAuthorSignoff(msg, prAuthor) {
stats.squashUnsafeHive++
}
}
Expand Down Expand Up @@ -96,7 +96,11 @@ func (c *Engine) stageDCO(ctx context.Context, store *stagelabels.Store, key str
// none; an in-place adoption decides repair-vs-rehome on it,
// so read the commits once more for that case only.
if (u > 0 || rec.DCOSquashUnsafeKnown) && (u == 0 || rec.DCOSplitKnown(obs.HeadSHA) || !rec.AdoptedInPlace()) {
return dcoStats{total: t, unsigned: u, unsignedHive: rec.DCOUnsignedHive, unsignedOther: u - rec.DCOUnsignedHive, squashUnsafeHive: rec.DCOSquashUnsafeHive}, nil
unsafe := rec.DCOSquashUnsafeHive
if botSquashAuthor(obs.Author) {
unsafe = 0
}
return dcoStats{total: t, unsigned: u, unsignedHive: rec.DCOUnsignedHive, unsignedOther: u - rec.DCOUnsignedHive, squashUnsafeHive: unsafe}, nil
}
}
}
Expand All @@ -116,6 +120,12 @@ func (c *Engine) stageDCO(ctx context.Context, store *stagelabels.Store, key str
return dcoCommitStatsDetailedForAuthor(all, c.transport.AppBotLogin(), obs.Author), nil
}

// Matches the bot-PR policy in check-squash-signoff-attribution.sh: a squash
// remains bot-authored. Unsigned commits are still counted and rejected.
func botSquashAuthor(login string) bool {
return strings.HasSuffix(strings.ToLower(strings.TrimSpace(login)), "[bot]")
}

func messageHasPRAuthorSignoff(message, prAuthor string) bool {
login := strings.ToLower(strings.TrimSpace(prAuthor))
if login == "" {
Expand Down
8 changes: 8 additions & 0 deletions src/pkg/github/automerge/automerge_stage_labels.go
Original file line number Diff line number Diff line change
Expand Up @@ -1138,6 +1138,14 @@ func (c *Engine) ApplyStageLabels(ctx context.Context) (StageLabelResult, error)
route := c.fixerExhaustedRoute(obs, fixer)
obs.Route = &route
}
if step := c.sharedCIRecoveryUnauthorized(ctx, obs); step != "" && verdict.Stage != stagelabels.None {
pass.RecoveryUnauthorized = step
verdict.Stage = stagelabels.NeedsDecision
verdict.Rank = stageRankOwned
obs.Reason = unauthorizedReason(step)
route := stagelabels.Route{Stage: stagelabels.NeedsDecision}
obs.Route = &route
}
tr = store.Observe(key, verdict.Stage, obs.Reason)
// hivecommons/hive#12502: remember the author so the dashboard can
// list "PRs returned to this contributor" from the store alone.
Expand Down
52 changes: 52 additions & 0 deletions src/pkg/github/automerge/shared_ci_attribution_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
package automerge

import (
"context"
"testing"
"time"

gh "github.com/google/go-github/v72/github"
"github.com/hivecommons/hive/pkg/stagelabels"
)

func TestBotSquashAttributionMatchesRequiredCheck(t *testing.T) {
for _, tc := range []struct {
author string
message string
unsafe int
unsigned int
}{
{"dependabot[bot]", "repair\n\nSigned-off-by: Hive <hive@users.noreply.github.com>", 0, 0},
{"renovate[bot]", "repair\n\nSigned-off-by: Hive <hive@users.noreply.github.com>", 0, 0},
{"person", "repair\n\nSigned-off-by: Hive <hive@users.noreply.github.com>", 1, 0},
{"person-bot", "repair\n\nSigned-off-by: Hive <hive@users.noreply.github.com>", 1, 0},
{"dependabot[bot]", "unsigned repair", 0, 1},
} {
t.Run(tc.author+tc.message, func(t *testing.T) {
commits := []*gh.RepositoryCommit{{
Author: &gh.User{Login: gh.Ptr("hive[bot]")},
Commit: &gh.Commit{Message: gh.Ptr(tc.message)},
}}
got := dcoCommitStatsDetailedForAuthor(commits, "hive[bot]", tc.author)
if got.total != 1 || got.squashUnsafeHive != tc.unsafe || got.unsigned != tc.unsigned {
t.Fatalf("stats = %+v, want unsafe=%d unsigned=%d", got, tc.unsafe, tc.unsigned)
}
})
}
}

func TestStageDCONormalizesCachedBotAttributionWithoutChangingSignoffs(t *testing.T) {
api := &stageAPI{labels: map[string]string{}}
srv := newStageAPI(t, api)
defer srv.Close()
e, _, store := newStageEngine(t, srv.URL, &contributorLaneClock{at: time.Now()})
key := stageKey("acme", "widget", 42)
store.Observe(key, stagelabels.Fixing, stagelabels.LaneReasonHiveSignoffAttribution)
store.NoteDCO(key, "aaa", 3, 0, 0, 1)
got, err := e.stageDCO(context.Background(), store, key, stageObservation{
Owner: "acme", Repo: "widget", Number: 42, HeadSHA: "aaa", Author: "dependabot[bot]",
})
if err != nil || got.total != 3 || got.unsigned != 0 || got.squashUnsafeHive != 0 {
t.Fatalf("cached bot attribution: %+v, %v", got, err)
}
}
43 changes: 43 additions & 0 deletions src/pkg/github/automerge/shared_ci_recovery.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package automerge

import (
"context"
"fmt"

gh "github.com/google/go-github/v72/github"
hgithub "github.com/hivecommons/hive/pkg/github"
)

const unauthorizedStepRerunSharedCI = "rerun-shared-ci"

type sharedCIRecoveryTransport interface {
SharedCIRerunBlocked(repo string, number int, head string) (hgithub.SharedCIRerunBlock, bool)
}

func (c *Engine) sharedCIRecoveryUnauthorized(ctx context.Context, obs stageObservation) string {
t, ok := c.transport.(sharedCIRecoveryTransport)
if !ok {
return ""
}
pr := &gh.PullRequest{Head: &gh.PullRequestBranch{SHA: gh.Ptr(obs.HeadSHA)}}
for _, name := range obs.Labels {
pr.Labels = append(pr.Labels, &gh.Label{Name: gh.Ptr(name)})
}
block, blocked := t.SharedCIRerunBlocked(obs.Owner+"/"+obs.Repo, obs.Number, obs.HeadSHA)
if !blocked {
c.unauthorizedMu.Lock()
marked := c.unauthorized[c.unauthorizedKey(obs.Owner, obs.Repo, obs.Number)].Step == unauthorizedStepRerunSharedCI
c.unauthorizedMu.Unlock()
if marked {
c.clearUnauthorized(ctx, obs.Display, obs.Owner, obs.Repo, pr, obs.Number, unauthorizedStepRerunSharedCI)
}
return ""
}
action := fmt.Sprintf("A repository maintainer with Actions write can re-run failed jobs at https://github.com/%s/%s/actions/runs/%d . "+
"Alternatively, the App owner can request Actions read and write for this recovery installation and its owner must accept the update. "+
"This is a CI recovery permission block, not a source-code failure or merge approval. Hive retries after its bounded permission cooldown; no alternate credential is selected.",
obs.Owner, obs.Repo, block.RunID)
c.markUnauthorized(ctx, obs.Display, obs.Owner, obs.Repo, pr, obs.Number, obs.HeadSHA,
unauthorizedStepRerunSharedCI, unauthorizedPermissionActionsWrite, action)
return unauthorizedStepRerunSharedCI
}
48 changes: 48 additions & 0 deletions src/pkg/github/automerge/shared_ci_recovery_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package automerge

import (
"context"
"strings"
"testing"
"time"

hgithub "github.com/hivecommons/hive/pkg/github"
)

type recoveryBlockTransport struct {
Transport
blocked bool
}

func (t *recoveryBlockTransport) SharedCIRerunBlocked(repo string, number int, head string) (hgithub.SharedCIRerunBlock, bool) {
return hgithub.SharedCIRerunBlock{HeadSHA: head, RunID: 11}, t.blocked
}

func TestSharedCIRecoveryUsesUnauthorizedHandoffAndClearsOnRecovery(t *testing.T) {
api := &stageAPI{statefulComments: true}
srv := newStageAPI(t, api)
defer srv.Close()
e, _, _ := newStageEngine(t, srv.URL, &contributorLaneClock{at: time.Now()})
transport := &recoveryBlockTransport{Transport: e.transport, blocked: true}
e.transport = transport
obs := stageObservation{Owner: "acme", Repo: "widget", Display: "acme/widget", Number: 42, HeadSHA: "aaa"}
ctx := context.Background()
if step := e.sharedCIRecoveryUnauthorized(ctx, obs); step != unauthorizedStepRerunSharedCI {
t.Fatalf("step = %q", step)
}
e.sharedCIRecoveryUnauthorized(ctx, obs)
if len(api.comments) != 1 || !strings.Contains(api.comments[0], "actions: write") ||
!strings.Contains(api.comments[0], "/actions/runs/11") ||
!strings.Contains(api.comments[0], "not a source-code failure") {
t.Fatalf("handoff must be actionable and head-deduplicated: %v", api.comments)
}
transport.blocked = false
obs.Labels = []string{unauthorizedLabel}
obs.HeadSHA = "new-head"
if step := e.sharedCIRecoveryUnauthorized(ctx, obs); step != "" {
t.Fatalf("recovered step = %q", step)
}
if len(api.removed) != 1 || api.removed[0] != unauthorizedLabel {
t.Fatalf("recovery/head replacement did not clear its own label: %v", api.removed)
}
}
78 changes: 72 additions & 6 deletions src/pkg/github/shared_ci_rerun.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package github
import (
"context"
"fmt"
"maps"
"regexp"
"sort"
"strconv"
Expand Down Expand Up @@ -70,6 +71,34 @@ type SharedCIRerunResult struct {
type sharedCIRerunSeen struct {
updatedAt time.Time
incidents string
block SharedCIRerunBlock
}

// SharedCIRerunBlock is an observed permission refusal, not a CI verdict.
type SharedCIRerunBlock struct {
HeadSHA string
RunID int64
}

// SharedCIRerunBlocked returns the recovery step owed on this exact head.
// The block survives the retry cooldown until a sweep observes recovery.
func (c *Client) SharedCIRerunBlocked(repo string, number int, head string) (SharedCIRerunBlock, bool) {
owner, name := c.splitRepo(repo)
c.sharedCIRerunMu.Lock()
defer c.sharedCIRerunMu.Unlock()
seen := c.sharedCIRerunSeen[fmt.Sprintf("%s/%s#%d", owner, name, number)]
return seen.block, seen.block.RunID != 0 && seen.block.HeadSHA == head
}

func (c *Client) hasSharedCIRerunBlocks(owner, repo string) bool {
c.sharedCIRerunMu.Lock()
defer c.sharedCIRerunMu.Unlock()
for key, seen := range c.sharedCIRerunSeen {
if strings.HasPrefix(key, owner+"/"+repo+"#") && seen.block.RunID != 0 {
return true
}
}
return false
}

// SweepSharedCIReruns re-runs the failed jobs on open PRs that a fix lane
Expand Down Expand Up @@ -107,7 +136,7 @@ func (c *Client) SweepSharedCIReruns(ctx context.Context, opts SharedCIRerunOpti
if err != nil {
return result, err
}
if len(closed) == 0 {
if len(closed) == 0 && !c.hasSharedCIRerunBlocks(owner, repoName) {
continue
}
result.Incidents += len(closed)
Expand Down Expand Up @@ -230,12 +259,12 @@ func (c *Client) trySharedCIRerun(ctx context.Context, repo, owner, repoName str
if labels := labelNames(pr.Labels); c.isHeld(labels) || c.isExempt(labels) {
return SharedCIRerunEvent{}, false, nil
}
seenKey := fmt.Sprintf("%s#%d", repo, pr.GetNumber())
seenKey := fmt.Sprintf("%s/%s#%d", owner, repoName, pr.GetNumber())
updatedAt := pr.GetUpdatedAt().Time
c.sharedCIRerunMu.Lock()
prev, known := c.sharedCIRerunSeen[seenKey]
c.sharedCIRerunMu.Unlock()
if known && prev.incidents == incidentKey && prev.updatedAt.Equal(updatedAt) {
if known && prev.incidents == incidentKey && prev.updatedAt.Equal(updatedAt) && prev.block.RunID == 0 {
return SharedCIRerunEvent{}, false, nil
}
markSeen := func() {
Expand All @@ -246,6 +275,17 @@ func (c *Client) trySharedCIRerun(ctx context.Context, repo, owner, repoName str
c.sharedCIRerunSeen[seenKey] = sharedCIRerunSeen{updatedAt: updatedAt, incidents: incidentKey}
c.sharedCIRerunMu.Unlock()
}
markBlocked := func(runID int64) {
c.sharedCIRerunMu.Lock()
defer c.sharedCIRerunMu.Unlock()
if c.sharedCIRerunSeen == nil {
c.sharedCIRerunSeen = map[string]sharedCIRerunSeen{}
}
c.sharedCIRerunSeen[seenKey] = sharedCIRerunSeen{
updatedAt: updatedAt, incidents: incidentKey,
block: SharedCIRerunBlock{HeadSHA: pr.GetHead().GetSHA(), RunID: runID},
}
}

comments, err := c.listIssueComments(ctx, owner, repoName, pr.GetNumber())
if err != nil {
Expand All @@ -255,6 +295,20 @@ func (c *Client) trySharedCIRerun(ctx context.Context, repo, owner, repoName str
for _, cm := range comments {
texts = append(texts, cm.GetBody())
}
if prev.block.RunID != 0 && prev.block.HeadSHA == pr.GetHead().GetSHA() {
// Discovery's lookback must not strand an already-observed refusal.
// Keep its incident eligible without broadening discovery for other PRs.
closed = maps.Clone(closed)
if closed == nil {
closed = map[int]bool{}
}
for _, raw := range strings.Split(prev.incidents, ",") {
if n, err := strconv.Atoi(raw); err == nil && n > 0 {
closed[n] = true
}
}
incidentKey = sharedCIIncidentSetKey(closed)
}
incidents := pendingSharedCIReruns(texts, closed)
if len(incidents) == 0 {
markSeen()
Expand All @@ -275,8 +329,11 @@ func (c *Client) trySharedCIRerun(ctx context.Context, repo, owner, repoName str

event := SharedCIRerunEvent{Repo: repo, Number: pr.GetNumber(), HeadSHA: sha, Incidents: incidents}
var rerun []*gh.WorkflowRun
blocked := false
for _, run := range runs {
if c.sharedCIRerunRefused(repo) {
markBlocked(run.GetID())
blocked = true
break
}
if _, err := c.client.Actions.RerunFailedJobsByID(ctx, owner, repoName, run.GetID()); err != nil {
Expand All @@ -287,6 +344,8 @@ func (c *Client) trySharedCIRerun(ctx context.Context, repo, owner, repoName str
if c.markSharedCIRerunForbidden(repo) {
c.logger.Warn("shared-ci rerun forbidden: the App installation lacks Actions write permission for rerun-failed-jobs", "repo", repo, "pr", pr.GetNumber(), "run", run.GetID(), "error", err)
}
markBlocked(run.GetID())
blocked = true
break
}
c.logger.Warn("shared-ci rerun failed", "repo", repo, "pr", pr.GetNumber(), "run", run.GetID(), "error", err)
Expand All @@ -296,16 +355,23 @@ func (c *Client) trySharedCIRerun(ctx context.Context, repo, owner, repoName str
event.RunIDs = append(event.RunIDs, run.GetID())
}
if len(rerun) == 0 {
// Every re-run was refused (permissions, run too old to re-run).
// Memoize so the sweep does not retry each pass for this PR state.
markSeen()
// A permission refusal must remain retryable after the repo cooldown,
// even when neither the PR nor the incident changed.
if !blocked {
markSeen()
}
return SharedCIRerunEvent{}, false, nil
}
if blocked {
// Do not stamp an incident complete when only some runs were accepted.
return event, true, nil
}

body := renderSharedCIRerunComment(incidents, sha, rerun)
if _, _, err := c.client.Issues.CreateComment(ctx, owner, repoName, pr.GetNumber(), &gh.IssueComment{Body: &body}); err != nil {
return event, false, fmt.Errorf("stamping shared-CI rerun marker on %s#%d: %w", repo, pr.GetNumber(), err)
}
markSeen()
for _, incident := range incidents {
c.logger.Info("shared-ci rerun", "repo", repo, "pr", pr.GetNumber(), "incident", "#"+strconv.Itoa(incident), "head", sha, "runs", len(event.RunIDs))
}
Expand Down
Loading
Loading