From 89e7176677fcb24c67b3c4fbd72e780cf9ddba0c Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 02:58:55 -0400 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=90=9B=20fix:=20stop=20retrying=20sha?= =?UTF-8?q?red-ci=20reruns=20the=20App=20may=20not=20perform?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 403 'Resource not accessible by integration' on rerun-failed-jobs is a permanent installation permission gap. One PR with N red workflows emitted N identical WARNs per pass. Stop the pass on the first refusal, remember it per repo for 1h, warn once, and classify the new message in fleetreport. Closes #15336 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-15336-shared-ci-rerun-forbidden.md | 1 + src/pkg/fleetreport/runtime.go | 2 + src/pkg/fleetreport/runtime_test.go | 2 + src/pkg/github/client.go | 4 ++ src/pkg/github/shared_ci_rerun.go | 39 +++++++++++++++++++ src/pkg/github/shared_ci_rerun_test.go | 30 ++++++++++++++ 6 files changed, 78 insertions(+) create mode 100644 changelog.d/fixed-15336-shared-ci-rerun-forbidden.md diff --git a/changelog.d/fixed-15336-shared-ci-rerun-forbidden.md b/changelog.d/fixed-15336-shared-ci-rerun-forbidden.md new file mode 100644 index 0000000000..70fa0dc6ee --- /dev/null +++ b/changelog.d/fixed-15336-shared-ci-rerun-forbidden.md @@ -0,0 +1 @@ +Shared-CI rerun no longer retries or spams warnings when the GitHub App lacks permission to re-run Actions. diff --git a/src/pkg/fleetreport/runtime.go b/src/pkg/fleetreport/runtime.go index 380e1eb39a..6770bb80dd 100644 --- a/src/pkg/fleetreport/runtime.go +++ b/src/pkg/fleetreport/runtime.go @@ -86,6 +86,7 @@ const ( ClassLabelProvisioningFailed = "label_provisioning_failed" ClassAgentCLIRestartLoop = "agent_cli_restart_loop" ClassTranscriptScanSlow = "transcript_scan_slow" + ClassSharedCIRerunForbidden = "shared_ci_rerun_forbidden" // A restart or normal stop can invalidate a pending bootstrap without // indicating a restart storm; only repeated interruptions are reportable. ClassStartupKickInterrupted = "startup_kick_interrupted" @@ -197,6 +198,7 @@ var Classes = map[string]ClassSpec{ ClassGitHubFetchTransient: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"channel commit date: fetch failed"}}, ClassLabelProvisioningFailed: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"issue stage labels: provisioning failed"}}, ClassAgentCLIRestartLoop: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"agent cli crashed or stuck on consent screen"}}, + ClassSharedCIRerunForbidden: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"shared-ci rerun forbidden"}}, ClassTranscriptScanSlow: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"transcript activity scan outlived its interval"}}, } diff --git a/src/pkg/fleetreport/runtime_test.go b/src/pkg/fleetreport/runtime_test.go index ea7a84b081..03dc57dbff 100644 --- a/src/pkg/fleetreport/runtime_test.go +++ b/src/pkg/fleetreport/runtime_test.go @@ -351,6 +351,7 @@ func TestEveryRuntimeClassTriggerAndNonTriggerThresholds(t *testing.T) { {class: ClassAgentDirSetupFailed, message: "failed to create per-agent XDG dir"}, {class: ClassReviewEvidenceWriteFail, message: "review evidence: could not write bundle"}, {class: ClassFeedbackIssueRefreshFailed, message: "feedback: issue refresh failed"}, + {class: ClassSharedCIRerunForbidden, message: "shared-ci rerun forbidden: the App installation lacks Actions write permission for rerun-failed-jobs"}, {class: ClassTranscriptScanSlow, message: "copilot transcript activity scan outlived its interval; agent liveness reads may lag"}, {class: ClassBeadFactIngestFailed, message: "failed to ingest bead fact"}, {class: ClassSelfReportClearedFailed, message: "fleet report: cleared comment failed"}, @@ -990,6 +991,7 @@ func TestRecurringRuntimeMessagesAreClassified(t *testing.T) { "channel commit date: fetch failed — the row will render without a timestamp": ClassGitHubFetchTransient, "issue stage labels: provisioning failed": ClassLabelProvisioningFailed, "agent CLI crashed or stuck on consent screen, restarting before kick": ClassAgentCLIRestartLoop, + "shared-ci rerun forbidden: the App installation lacks Actions write permission": ClassSharedCIRerunForbidden, "SHA poll: commit message fetch non-200": ClassSHAPollCommitMsgUnavail, "login required detected": ClassAgentLoginRequired, "pr-request watcher: head branch is not on the remote — the agent's push did not authenticate": ClassPRRequestHeadUnpushed, diff --git a/src/pkg/github/client.go b/src/pkg/github/client.go index 4205b55eb0..75eafc815c 100644 --- a/src/pkg/github/client.go +++ b/src/pkg/github/client.go @@ -254,6 +254,10 @@ type Client struct { // to do for, keyed "owner/repo#n". Guarded by sharedCIRerunMu. sharedCIRerunSeen map[string]sharedCIRerunSeen sharedCIRerunMu sync.Mutex + // sharedCIRerunForbidden records when the App was last refused a rerun in + // a repo (403 "Resource not accessible by integration"), keyed + // "owner/repo". Guarded by sharedCIRerunMu. + sharedCIRerunForbidden map[string]time.Time // combinedPerspectivesFn / maxReviewsPerHeadFn feed the per-head review // backstop (perHeadReviewRefusal); read live so a dashboard change applies // without a client rebuild. maxReviewsPerHeadFn returning 0 means derive; diff --git a/src/pkg/github/shared_ci_rerun.go b/src/pkg/github/shared_ci_rerun.go index 805b6f0036..7cbc852550 100644 --- a/src/pkg/github/shared_ci_rerun.go +++ b/src/pkg/github/shared_ci_rerun.go @@ -19,6 +19,10 @@ const ( // closes the sweep keeps looking for PRs stamped with its marker. DefaultSharedCIRerunLookback = 72 * time.Hour + // sharedCIRerunForbiddenTTL is how long a refused rerun suppresses further + // rerun attempts (and the warning) for that repo. + sharedCIRerunForbiddenTTL = time.Hour + sharedCIIncidentTitlePrefix = "[shared-ci]" sharedCIRerunMaxIssuePages = 3 ) @@ -272,7 +276,19 @@ func (c *Client) trySharedCIRerun(ctx context.Context, repo, owner, repoName str event := SharedCIRerunEvent{Repo: repo, Number: pr.GetNumber(), HeadSHA: sha, Incidents: incidents} var rerun []*gh.WorkflowRun for _, run := range runs { + if c.sharedCIRerunRefused(repo) { + break + } if _, err := c.client.Actions.RerunFailedJobsByID(ctx, owner, repoName, run.GetID()); err != nil { + if isWriteForbiddenError(err) { + // The App installation lacks the Actions write permission for + // rerun-failed-jobs: a permanent gap, so stop this pass and + // warn once per repo per TTL instead of once per run. + if c.markSharedCIRerunForbidden(repo) { + c.logger.Warn("shared-ci rerun forbidden: the App installation lacks Actions write permission for rerun-failed-jobs", "repo", repo, "pr", pr.GetNumber(), "run", run.GetID(), "error", err) + } + break + } c.logger.Warn("shared-ci rerun failed", "repo", repo, "pr", pr.GetNumber(), "run", run.GetID(), "error", err) continue } @@ -296,6 +312,29 @@ func (c *Client) trySharedCIRerun(ctx context.Context, repo, owner, repoName str return event, true, nil } +// sharedCIRerunRefused reports whether a rerun was refused for repo within the TTL. +func (c *Client) sharedCIRerunRefused(repo string) bool { + c.sharedCIRerunMu.Lock() + defer c.sharedCIRerunMu.Unlock() + at, ok := c.sharedCIRerunForbidden[repo] + return ok && time.Since(at) < sharedCIRerunForbiddenTTL +} + +// markSharedCIRerunForbidden records a refusal and reports whether it is the +// first one in the current TTL window (so the caller logs it once). +func (c *Client) markSharedCIRerunForbidden(repo string) bool { + c.sharedCIRerunMu.Lock() + defer c.sharedCIRerunMu.Unlock() + if at, ok := c.sharedCIRerunForbidden[repo]; ok && time.Since(at) < sharedCIRerunForbiddenTTL { + return false + } + if c.sharedCIRerunForbidden == nil { + c.sharedCIRerunForbidden = map[string]time.Time{} + } + c.sharedCIRerunForbidden[repo] = time.Now() + return true +} + // latestFailedWorkflowRuns returns, per workflow, the newest run for sha when // that run completed as failure or timed_out. func (c *Client) latestFailedWorkflowRuns(ctx context.Context, owner, repo, sha string) ([]*gh.WorkflowRun, error) { diff --git a/src/pkg/github/shared_ci_rerun_test.go b/src/pkg/github/shared_ci_rerun_test.go index 8b13f9f09b..88793f533c 100644 --- a/src/pkg/github/shared_ci_rerun_test.go +++ b/src/pkg/github/shared_ci_rerun_test.go @@ -46,6 +46,7 @@ type sharedCIRerunFake struct { reruns []int64 posted map[int][]string commentLists map[int]int + forbidRerun bool } func newSharedCIRerunFake(t *testing.T, issues []sharedCIRerunIssue, prs []*sharedCIRerunPR) (*httptest.Server, *sharedCIRerunFake) { @@ -134,6 +135,11 @@ func newSharedCIRerunFake(t *testing.T, issues []sharedCIRerunIssue, prs []*shar case rerunRE.MatchString(r.URL.Path) && r.Method == http.MethodPost: id, _ := strconv.ParseInt(rerunRE.FindStringSubmatch(r.URL.Path)[1], 10, 64) f.reruns = append(f.reruns, id) + if f.forbidRerun { + w.WriteHeader(http.StatusForbidden) + _ = json.NewEncoder(w).Encode(map[string]any{"message": "Resource not accessible by integration"}) + return + } w.WriteHeader(http.StatusCreated) default: t.Errorf("unexpected request %s %s", r.Method, r.URL.Path) @@ -289,3 +295,27 @@ func TestPendingSharedCIReruns(t *testing.T) { } } } + +func TestSweepSharedCIReruns_ForbiddenStopsAndIsRememberedPerRepo(t *testing.T) { + now := sharedCIRerunNow() + issues := []sharedCIRerunIssue{{Number: 800, Title: "[shared-ci] test failing across org/repo", ClosedAt: now.Add(-time.Hour)}} + marker := []string{""} + prs := []*sharedCIRerunPR{ + {Number: 1, SHA: "aaaaaaaaaa", Comments: marker, Runs: []sharedCIRerunRun{failedRun(11, 1), failedRun(12, 2), failedRun(13, 3)}}, + {Number: 2, SHA: "bbbbbbbbbb", Comments: marker, Runs: []sharedCIRerunRun{failedRun(21, 1), failedRun(22, 2)}}, + } + srv, fake := newSharedCIRerunFake(t, issues, prs) + fake.forbidRerun = true + c := newTestClient(t, srv, "org", []string{"org/repo"}) + res, err := c.SweepSharedCIReruns(context.Background(), SharedCIRerunOptions{Now: sharedCIRerunNow}) + if err != nil { + t.Fatalf("sweep: %v", err) + } + reruns, posted, _ := fake.snap() + if len(reruns) != 1 { + t.Fatalf("reruns = %v, want exactly one POST across both PRs", reruns) + } + if len(res.Rerun) != 0 || len(posted[1]) != 0 || len(posted[2]) != 0 { + t.Fatalf("expected no rerun result or marker comment: result=%+v posted=%v", res, posted) + } +} From 0d95e8f6322cc3180e3f11c1b09aab688763a64d Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 07:00:02 +0000 Subject: [PATCH 2/3] changelog: fragment for #15339 Generated by changelog-fragment-guard from the PR title (#5675). Edit the wording freely, or replace this file with the no-changelog label if the change is not user-visible. Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-15339.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/fixed-15339.md diff --git a/changelog.d/fixed-15339.md b/changelog.d/fixed-15339.md new file mode 100644 index 0000000000..2ce9f56540 --- /dev/null +++ b/changelog.d/fixed-15339.md @@ -0,0 +1 @@ +- fix: stop retrying shared-ci reruns the App may not perform (#15339) From 884cfcb042a6074017d6adad21f312f0e7895d06 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 03:01:24 -0400 Subject: [PATCH 3/3] changelog: drop malformed duplicate fragment changelog-fragment-guard requires a '- ' entry bullet; the auto-added fixed-15339.md already carries the entry. Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-15336-shared-ci-rerun-forbidden.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 changelog.d/fixed-15336-shared-ci-rerun-forbidden.md diff --git a/changelog.d/fixed-15336-shared-ci-rerun-forbidden.md b/changelog.d/fixed-15336-shared-ci-rerun-forbidden.md deleted file mode 100644 index 70fa0dc6ee..0000000000 --- a/changelog.d/fixed-15336-shared-ci-rerun-forbidden.md +++ /dev/null @@ -1 +0,0 @@ -Shared-CI rerun no longer retries or spams warnings when the GitHub App lacks permission to re-run Actions.