Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-15339.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- fix: stop retrying shared-ci reruns the App may not perform (#15339)
2 changes: 2 additions & 0 deletions src/pkg/fleetreport/runtime.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ const (
ClassLabelProvisioningFailed = "label_provisioning_failed"
ClassAgentCLIRestartLoop = "agent_cli_restart_loop"
ClassTranscriptScanSlow = "transcript_scan_slow"
ClassSharedCIRerunForbidden = "shared_ci_rerun_forbidden"
// A restart or normal stop can invalidate a pending bootstrap without
// indicating a restart storm; only repeated interruptions are reportable.
ClassStartupKickInterrupted = "startup_kick_interrupted"
Expand Down Expand Up @@ -197,6 +198,7 @@ var Classes = map[string]ClassSpec{
ClassGitHubFetchTransient: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"channel commit date: fetch failed"}},
ClassLabelProvisioningFailed: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"issue stage labels: provisioning failed"}},
ClassAgentCLIRestartLoop: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"agent cli crashed or stuck on consent screen"}},
ClassSharedCIRerunForbidden: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"shared-ci rerun forbidden"}},
ClassTranscriptScanSlow: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"transcript activity scan outlived its interval"}},
}

Expand Down
2 changes: 2 additions & 0 deletions src/pkg/fleetreport/runtime_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -351,6 +351,7 @@ func TestEveryRuntimeClassTriggerAndNonTriggerThresholds(t *testing.T) {
{class: ClassAgentDirSetupFailed, message: "failed to create per-agent XDG dir"},
{class: ClassReviewEvidenceWriteFail, message: "review evidence: could not write bundle"},
{class: ClassFeedbackIssueRefreshFailed, message: "feedback: issue refresh failed"},
{class: ClassSharedCIRerunForbidden, message: "shared-ci rerun forbidden: the App installation lacks Actions write permission for rerun-failed-jobs"},
{class: ClassTranscriptScanSlow, message: "copilot transcript activity scan outlived its interval; agent liveness reads may lag"},
{class: ClassBeadFactIngestFailed, message: "failed to ingest bead fact"},
{class: ClassSelfReportClearedFailed, message: "fleet report: cleared comment failed"},
Expand Down Expand Up @@ -990,6 +991,7 @@ func TestRecurringRuntimeMessagesAreClassified(t *testing.T) {
"channel commit date: fetch failed — the row will render without a timestamp": ClassGitHubFetchTransient,
"issue stage labels: provisioning failed": ClassLabelProvisioningFailed,
"agent CLI crashed or stuck on consent screen, restarting before kick": ClassAgentCLIRestartLoop,
"shared-ci rerun forbidden: the App installation lacks Actions write permission": ClassSharedCIRerunForbidden,
"SHA poll: commit message fetch non-200": ClassSHAPollCommitMsgUnavail,
"login required detected": ClassAgentLoginRequired,
"pr-request watcher: head branch is not on the remote — the agent's push did not authenticate": ClassPRRequestHeadUnpushed,
Expand Down
4 changes: 4 additions & 0 deletions src/pkg/github/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,10 @@ type Client struct {
// to do for, keyed "owner/repo#n". Guarded by sharedCIRerunMu.
sharedCIRerunSeen map[string]sharedCIRerunSeen
sharedCIRerunMu sync.Mutex
// sharedCIRerunForbidden records when the App was last refused a rerun in
// a repo (403 "Resource not accessible by integration"), keyed
// "owner/repo". Guarded by sharedCIRerunMu.
sharedCIRerunForbidden map[string]time.Time
// combinedPerspectivesFn / maxReviewsPerHeadFn feed the per-head review
// backstop (perHeadReviewRefusal); read live so a dashboard change applies
// without a client rebuild. maxReviewsPerHeadFn returning 0 means derive;
Expand Down
39 changes: 39 additions & 0 deletions src/pkg/github/shared_ci_rerun.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package github

import (
Expand All @@ -19,6 +19,10 @@
// closes the sweep keeps looking for PRs stamped with its marker.
DefaultSharedCIRerunLookback = 72 * time.Hour

// sharedCIRerunForbiddenTTL is how long a refused rerun suppresses further
// rerun attempts (and the warning) for that repo.
sharedCIRerunForbiddenTTL = time.Hour

sharedCIIncidentTitlePrefix = "[shared-ci]"
sharedCIRerunMaxIssuePages = 3
)
Expand Down Expand Up @@ -272,7 +276,19 @@
event := SharedCIRerunEvent{Repo: repo, Number: pr.GetNumber(), HeadSHA: sha, Incidents: incidents}
var rerun []*gh.WorkflowRun
for _, run := range runs {
if c.sharedCIRerunRefused(repo) {
break
}
if _, err := c.client.Actions.RerunFailedJobsByID(ctx, owner, repoName, run.GetID()); err != nil {
if isWriteForbiddenError(err) {
// The App installation lacks the Actions write permission for
// rerun-failed-jobs: a permanent gap, so stop this pass and
// warn once per repo per TTL instead of once per run.
if c.markSharedCIRerunForbidden(repo) {
c.logger.Warn("shared-ci rerun forbidden: the App installation lacks Actions write permission for rerun-failed-jobs", "repo", repo, "pr", pr.GetNumber(), "run", run.GetID(), "error", err)
}
break
}
c.logger.Warn("shared-ci rerun failed", "repo", repo, "pr", pr.GetNumber(), "run", run.GetID(), "error", err)
continue
}
Expand All @@ -296,6 +312,29 @@
return event, true, nil
}

// sharedCIRerunRefused reports whether a rerun was refused for repo within the TTL.
func (c *Client) sharedCIRerunRefused(repo string) bool {
c.sharedCIRerunMu.Lock()
defer c.sharedCIRerunMu.Unlock()
at, ok := c.sharedCIRerunForbidden[repo]
return ok && time.Since(at) < sharedCIRerunForbiddenTTL
}

// markSharedCIRerunForbidden records a refusal and reports whether it is the
// first one in the current TTL window (so the caller logs it once).
func (c *Client) markSharedCIRerunForbidden(repo string) bool {
c.sharedCIRerunMu.Lock()
defer c.sharedCIRerunMu.Unlock()
if at, ok := c.sharedCIRerunForbidden[repo]; ok && time.Since(at) < sharedCIRerunForbiddenTTL {
return false
}
if c.sharedCIRerunForbidden == nil {
c.sharedCIRerunForbidden = map[string]time.Time{}
}
c.sharedCIRerunForbidden[repo] = time.Now()
return true
}

// latestFailedWorkflowRuns returns, per workflow, the newest run for sha when
// that run completed as failure or timed_out.
func (c *Client) latestFailedWorkflowRuns(ctx context.Context, owner, repo, sha string) ([]*gh.WorkflowRun, error) {
Expand Down
30 changes: 30 additions & 0 deletions src/pkg/github/shared_ci_rerun_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ type sharedCIRerunFake struct {
reruns []int64
posted map[int][]string
commentLists map[int]int
forbidRerun bool
}

func newSharedCIRerunFake(t *testing.T, issues []sharedCIRerunIssue, prs []*sharedCIRerunPR) (*httptest.Server, *sharedCIRerunFake) {
Expand Down Expand Up @@ -134,6 +135,11 @@ func newSharedCIRerunFake(t *testing.T, issues []sharedCIRerunIssue, prs []*shar
case rerunRE.MatchString(r.URL.Path) && r.Method == http.MethodPost:
id, _ := strconv.ParseInt(rerunRE.FindStringSubmatch(r.URL.Path)[1], 10, 64)
f.reruns = append(f.reruns, id)
if f.forbidRerun {
w.WriteHeader(http.StatusForbidden)
_ = json.NewEncoder(w).Encode(map[string]any{"message": "Resource not accessible by integration"})
return
}
w.WriteHeader(http.StatusCreated)
default:
t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
Expand Down Expand Up @@ -289,3 +295,27 @@ func TestPendingSharedCIReruns(t *testing.T) {
}
}
}

func TestSweepSharedCIReruns_ForbiddenStopsAndIsRememberedPerRepo(t *testing.T) {
now := sharedCIRerunNow()
issues := []sharedCIRerunIssue{{Number: 800, Title: "[shared-ci] test failing across org/repo", ClosedAt: now.Add(-time.Hour)}}
marker := []string{"<!-- hive-shared-ci-800 -->"}
prs := []*sharedCIRerunPR{
{Number: 1, SHA: "aaaaaaaaaa", Comments: marker, Runs: []sharedCIRerunRun{failedRun(11, 1), failedRun(12, 2), failedRun(13, 3)}},
{Number: 2, SHA: "bbbbbbbbbb", Comments: marker, Runs: []sharedCIRerunRun{failedRun(21, 1), failedRun(22, 2)}},
}
srv, fake := newSharedCIRerunFake(t, issues, prs)
fake.forbidRerun = true
c := newTestClient(t, srv, "org", []string{"org/repo"})
res, err := c.SweepSharedCIReruns(context.Background(), SharedCIRerunOptions{Now: sharedCIRerunNow})
if err != nil {
t.Fatalf("sweep: %v", err)
}
reruns, posted, _ := fake.snap()
if len(reruns) != 1 {
t.Fatalf("reruns = %v, want exactly one POST across both PRs", reruns)
}
if len(res.Rerun) != 0 || len(posted[1]) != 0 || len(posted[2]) != 0 {
t.Fatalf("expected no rerun result or marker comment: result=%+v posted=%v", res, posted)
}
}
Loading