From c16bf63f4a0c907c8a0b866736421c37d63c3339 Mon Sep 17 00:00:00 2001 From: pi Date: Wed, 28 Oct 2026 00:27:51 -0700 Subject: [PATCH 1/3] ci(pr-gate): treat queued Frontend Status Gate as pending, 15m timeout, auto re-run --- .github/workflows/pr-gate-frontend-rerun.yml | 65 ++++++++++++++++++++ .github/workflows/pr-gate.yml | 24 ++++++-- scripts/workflow_inventory.txt | 1 + 3 files changed, 84 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/pr-gate-frontend-rerun.yml diff --git a/.github/workflows/pr-gate-frontend-rerun.yml b/.github/workflows/pr-gate-frontend-rerun.yml new file mode 100644 index 000000000..ba48636ce --- /dev/null +++ b/.github/workflows/pr-gate-frontend-rerun.yml @@ -0,0 +1,65 @@ +# Re-runs a "PR Gate: Frontend" job that gave up while Frontend CI/CD was +# still queued. pr-gate.yml polls for "Frontend CI Status Gate" for at most +# 12 minutes; when the runner queue is backed up the gate may not exist yet. +# Once Frontend CI/CD finishes for the same head SHA, this workflow re-runs +# the failed PR Gate: Frontend job, which then mirrors the real gate result. +# +# Runs on the default branch (workflow_run) and never checks out PR code. +name: PR Gate Frontend Rerun + +on: + workflow_run: + workflows: ["Frontend CI/CD"] + types: [completed] + +permissions: {} + +concurrency: + group: pr-gate-frontend-rerun-${{ github.event.workflow_run.head_sha }} + cancel-in-progress: false + +jobs: + rerun: + name: "Re-run PR Gate: Frontend" + if: github.event.workflow_run.event == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + actions: write + steps: + - name: "Re-run a failed PR Gate: Frontend job for this head SHA" + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 + env: + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + with: + github-token: ${{ github.token }} + script: | + const { owner, repo } = context.repo; + const sha = process.env.HEAD_SHA; + const runs = await github.rest.actions.listWorkflowRunsForRepo({ + owner, repo, head_sha: sha, event: 'pull_request', per_page: 50, + }); + const gateRuns = runs.data.workflow_runs + .filter(r => r.name === 'PR Gate') + .sort((a, b) => new Date(b.created_at) - new Date(a.created_at)); + if (gateRuns.length === 0) { + core.info(`No PR Gate run for ${sha}; nothing to re-run.`); + return; + } + const latest = gateRuns[0]; + const jobs = await github.rest.actions.listJobsForWorkflowRun({ + owner, repo, run_id: latest.id, filter: 'latest', per_page: 100, + }); + const job = jobs.data.jobs.find(j => j.name === 'PR Gate: Frontend'); + if (!job) { + core.info(`PR Gate run ${latest.id} has no PR Gate: Frontend job.`); + return; + } + if (job.status !== 'completed' || !['failure', 'timed_out', 'cancelled'].includes(job.conclusion)) { + core.info(`PR Gate: Frontend is ${job.status}/${job.conclusion}; no re-run needed.`); + return; + } + core.info(`Re-running PR Gate: Frontend job ${job.id} (run ${latest.id}) for ${sha}.`); + await github.request('POST /repos/{owner}/{repo}/actions/jobs/{job_id}/rerun', { + owner, repo, job_id: job.id, + }); diff --git a/.github/workflows/pr-gate.yml b/.github/workflows/pr-gate.yml index b3d3ab681..505b77beb 100644 --- a/.github/workflows/pr-gate.yml +++ b/.github/workflows/pr-gate.yml @@ -271,7 +271,13 @@ jobs: actions: read name: "PR Gate: Frontend" runs-on: ubuntu-latest - timeout-minutes: 45 + # Short bound: when the runner queue is backed up the Status Gate may not + # exist yet. pr-gate-frontend-rerun.yml re-runs this job when Frontend + # CI/CD completes, so a timeout here is recovered automatically. + timeout-minutes: 15 + concurrency: + group: pr-gate-frontend-${{ github.event.pull_request.number }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 with: @@ -332,14 +338,16 @@ jobs: CHECK_NAME: "Frontend CI Status Gate" with: github-token: ${{ github.token }} - # See the backend job above: "Frontend CI Status Gate" is created - # only after earlier frontend jobs finish, so we poll up to 40 minutes - # for it to appear and complete. + # "Frontend CI Status Gate" is created only after earlier frontend + # jobs finish. A gate that is missing or still queued is pending, not + # failed: we poll up to 12 minutes, then fail with an explicit + # "still pending" message. pr-gate-frontend-rerun.yml re-runs this job + # once Frontend CI/CD completes, which then mirrors the real result. script: | const { owner, repo } = context.repo; const sha = process.env.HEAD_SHA; const CHECK_NAME = process.env.CHECK_NAME; - const deadline = Date.now() + 40 * 60 * 1000; + const deadline = Date.now() + 12 * 60 * 1000; const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); const callWithRetry = async (fn) => { @@ -443,7 +451,11 @@ jobs: delay = Math.min(Math.round(delay * 1.5), 120000); } - core.setFailed(`Timed out waiting for ${CHECK_NAME}`); + core.setFailed( + `${CHECK_NAME} is still pending (not created yet or queued) after 12 minutes; ` + + `this is a CI queue delay, not a frontend failure. This job is re-run ` + + `automatically when Frontend CI/CD completes (pr-gate-frontend-rerun.yml).` + ); - name: No frontend-relevant changes if: steps.detect.outputs.changed != 'true' run: echo "No frontend-relevant paths changed; Frontend CI legitimately did not run. Gate vacuously green." diff --git a/scripts/workflow_inventory.txt b/scripts/workflow_inventory.txt index 3ca30927d..c65a96458 100644 --- a/scripts/workflow_inventory.txt +++ b/scripts/workflow_inventory.txt @@ -44,6 +44,7 @@ infra-k8s-ci.yml legacy-code-scan.yml openapi-public-spec.yml pr-gate.yml +pr-gate-frontend-rerun.yml pr-guards.yml pr-secrets-non-main.yml production-uptime.yml From 9bf9b48e351140bcca8f05a23ca43f263ddafa61 Mon Sep 17 00:00:00 2001 From: pi Date: Wed, 28 Oct 2026 00:36:39 -0700 Subject: [PATCH 2/3] ci(pr-gate): skip stale SHAs, final poll before timeout, cap re-runs at attempt 3 --- .github/workflows/pr-gate-frontend-rerun.yml | 29 ++++++++++++++++++++ .github/workflows/pr-gate.yml | 14 ++++++++++ 2 files changed, 43 insertions(+) diff --git a/.github/workflows/pr-gate-frontend-rerun.yml b/.github/workflows/pr-gate-frontend-rerun.yml index ba48636ce..3acbc13c9 100644 --- a/.github/workflows/pr-gate-frontend-rerun.yml +++ b/.github/workflows/pr-gate-frontend-rerun.yml @@ -3,6 +3,8 @@ # 12 minutes; when the runner queue is backed up the gate may not exist yet. # Once Frontend CI/CD finishes for the same head SHA, this workflow re-runs # the failed PR Gate: Frontend job, which then mirrors the real gate result. +# This also covers a gate that timed out just after Frontend CI completed. +# Skips SHAs that are no longer the PR head and stops after run attempt 3. # # Runs on the default branch (workflow_run) and never checks out PR code. name: PR Gate Frontend Rerun @@ -36,6 +38,28 @@ jobs: script: | const { owner, repo } = context.repo; const sha = process.env.HEAD_SHA; + // Skip stale SHAs: only re-run when this SHA is still the PR head, + // so an old run can never cancel the current head's gate. + let prs = context.payload.workflow_run.pull_requests || []; + if (prs.length === 0) { + const assoc = await github.rest.repos.listPullRequestsAssociatedWithCommit({ + owner, repo, commit_sha: sha, + }); + prs = assoc.data.filter(p => p.state === 'open'); + } + let isCurrentHead = false; + for (const p of prs) { + const pr = await github.rest.pulls.get({ owner, repo, pull_number: p.number }); + if (pr.data.state === 'open' && pr.data.head.sha === sha) { + isCurrentHead = true; + break; + } + } + if (!isCurrentHead) { + core.info(`${sha} is not the current head of an open PR; skipping re-run.`); + return; + } + const runs = await github.rest.actions.listWorkflowRunsForRepo({ owner, repo, head_sha: sha, event: 'pull_request', per_page: 50, }); @@ -59,6 +83,11 @@ jobs: core.info(`PR Gate: Frontend is ${job.status}/${job.conclusion}; no re-run needed.`); return; } + // Cap re-runs so a persistently failing gate can't loop. + if ((latest.run_attempt || 1) >= 3) { + core.info(`PR Gate run ${latest.id} is already on attempt ${latest.run_attempt}; not re-running again.`); + return; + } core.info(`Re-running PR Gate: Frontend job ${job.id} (run ${latest.id}) for ${sha}.`); await github.request('POST /repos/{owner}/{repo}/actions/jobs/{job_id}/rerun', { owner, repo, job_id: job.id, diff --git a/.github/workflows/pr-gate.yml b/.github/workflows/pr-gate.yml index 505b77beb..efd1f06fa 100644 --- a/.github/workflows/pr-gate.yml +++ b/.github/workflows/pr-gate.yml @@ -451,6 +451,20 @@ jobs: delay = Math.min(Math.round(delay * 1.5), 120000); } + // Final poll: Frontend CI may have finished during the last sleep. + const finalRes = await github.rest.checks.listForRef({ + owner, repo, ref: sha, check_name: CHECK_NAME, filter: 'latest', per_page: 20, + }).catch(() => null); + const finalRun = (finalRes?.data?.check_runs || []).find(r => r.status === 'completed'); + if (finalRun) { + if (finalRun.conclusion === 'success') { + core.info(`Check ${CHECK_NAME} succeeded with conclusion: success`); + return; + } + core.setFailed(finalRun.conclusion); + return; + } + core.setFailed( `${CHECK_NAME} is still pending (not created yet or queued) after 12 minutes; ` + `this is a CI queue delay, not a frontend failure. This job is re-run ` + From 88a414f4206e8c212dc90a0511fcbd826c25d89f Mon Sep 17 00:00:00 2001 From: pi Date: Wed, 28 Oct 2026 00:36:49 -0700 Subject: [PATCH 3/3] ci(pr-gate): grant read access needed for the PR head check --- .github/workflows/pr-gate-frontend-rerun.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr-gate-frontend-rerun.yml b/.github/workflows/pr-gate-frontend-rerun.yml index 3acbc13c9..944b99f94 100644 --- a/.github/workflows/pr-gate-frontend-rerun.yml +++ b/.github/workflows/pr-gate-frontend-rerun.yml @@ -28,6 +28,8 @@ jobs: timeout-minutes: 5 permissions: actions: write + contents: read + pull-requests: read steps: - name: "Re-run a failed PR Gate: Frontend job for this head SHA" uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9