diff --git a/.github/workflows/ci-minimal.yml b/.github/workflows/ci-minimal.yml index 70e2c56ae..a64c15a3b 100644 --- a/.github/workflows/ci-minimal.yml +++ b/.github/workflows/ci-minimal.yml @@ -7,6 +7,7 @@ on: pull_request: branches: [main] types: [opened, labeled, unlabeled, synchronize, reopened] + merge_group: workflow_dispatch: {} concurrency: @@ -14,6 +15,24 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + cancel-superseded: + name: Cancel superseded queue runs + if: ${{ !cancelled() && github.event_name == 'merge_group' }} + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + actions: write + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + sparse-checkout: ci/cancel_superseded.py + sparse-checkout-cone-mode: false + persist-credentials: false + - name: Cancel + env: + GH_TOKEN: ${{ github.token }} + run: python3 ci/cancel_superseded.py verify: name: Verify local gate continue-on-error: true @@ -188,3 +207,33 @@ jobs: # Path-selected boards: skipped when the PR touches none. case "$PR_BOARDS" in success|skipped) ;; *) exit 1 ;; esac fi + queue-cache-janitor: + name: End-of-queue cache janitor + needs: [cancel-superseded, verify, linux, board_plan, fbuild_bin, pr_boards, reuse_decision, test, full, selected-coverage] + if: always() && github.event_name == 'merge_group' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + concurrency: + group: cache-janitor + cancel-in-progress: false + permissions: + contents: read + actions: write + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + sparse-checkout: ci.toml + sparse-checkout-cone-mode: false + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + repository: zackees/ci.yml + ref: 953c4e13691b10c1c4f1f9c39fa259fd732c5789 + path: .ci-lint + persist-credentials: false + sparse-checkout: ci_lint + - name: Sweep caches + env: + GITHUB_TOKEN: ${{ github.token }} + PYTHONPATH: .ci-lint + run: python3 -P -m ci_lint cache janitor --repo . --default-branch main diff --git a/ci.toml b/ci.toml index c3d661dbb..a1a19ed3f 100644 --- a/ci.toml +++ b/ci.toml @@ -130,7 +130,7 @@ setup-soldr = { only-in = ".github/actions/setup" } cache-actions = { only-in = ".github/actions/setup" } [allow.workflows] -"ci-minimal.yml" = ["push:main", "pull_request", "workflow_dispatch"] +"ci-minimal.yml" = ["push:main", "pull_request", "merge_group", "workflow_dispatch"] "ci-full.yml" = ["workflow_call", "workflow_dispatch"] "ci-test.yml" = ["workflow_call", "workflow_dispatch"] "ci-workflow-drift.yml" = ["workflow_dispatch", "push:main", "pull_request"] @@ -160,7 +160,7 @@ cache-actions = { only-in = ".github/actions/setup" } "template_native_build.yml" = ["workflow_call"] [allow.permissions] -"actions: write" = ["dispatch"] +"actions: write" = ["dispatch", "cancel-superseded", "queue-cache-janitor"] "contents: write" = ["publish"] "pages: write" = ["publish", "deploy-pages"] "attestations: write" = ["build", "publish"] diff --git a/ci/cancel_superseded.py b/ci/cancel_superseded.py new file mode 100644 index 000000000..312216f3e --- /dev/null +++ b/ci/cancel_superseded.py @@ -0,0 +1,95 @@ +"""Cancel merge-queue runs a newer queue run for the same PR replaced (GEN-022, ci.yml#409). + + python3 ci/cancel_superseded.py [--dry-run] + +Runs as a job of every `merge_group` CI run (GITHUB_RUN_ID, GITHUB_REPOSITORY +and a GH_TOKEN with actions: write). A queue ref carries the base commit, +`gh-readonly-queue/main/pr--`, so the workflow's concurrency group +never pairs a PR's rebuilt queue run with its old one. When main moves, +GitHub starts a new run per queued PR and the old ones keep holding runners. +The newest queue run for a PR is the live one, so every older in-flight +queue run for the same PR is cancelled. Ported from obs-rust/obs-studio's +`ci/cancel_superseded.py` (#986 there) via talos-hq/clud (#2048). +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +QUEUE_RE = re.compile(r"(?:^|/)gh-readonly-queue/[^/]+/pr-(\d+)-[0-9a-f]+$") +LIVE = ("queued", "in_progress", "waiting", "pending", "requested") + + +def queue_pr(ref: str) -> int | None: + """The PR number of a merge-queue ref, else None.""" + match = QUEUE_RE.search(ref) + return int(match.group(1)) if match else None + + +def superseded(runs: list[dict], current: dict) -> list[int]: + """Ids of in-flight queue runs for current's PR that started before it.""" + pr = queue_pr(current.get("head_branch") or "") + if pr is None: + return [] + return sorted( + { + run["id"] + for run in runs + if run["id"] != current["id"] + and run.get("status") in LIVE + and queue_pr(run.get("head_branch") or "") == pr + and run["created_at"] < current["created_at"] + } + ) + + +def api(path: str, method: str = "GET") -> dict: + request = Request( + f"https://api.github.com/{path}", + method=method, + headers={ + "Authorization": f"Bearer {os.environ['GH_TOKEN']}", + "Accept": "application/vnd.github+json", + }, + ) + with urlopen(request, timeout=30) as response: + body = response.read() + return json.loads(body) if body else {} + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=(__doc__ or "").splitlines()[0]) + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args(argv) + repo = os.environ["GITHUB_REPOSITORY"] + current = api(f"repos/{repo}/actions/runs/{os.environ['GITHUB_RUN_ID']}") + workflow = current["workflow_id"] + runs: list[dict] = [] + for status in LIVE: + listing = api( + f"repos/{repo}/actions/workflows/{workflow}/runs" + f"?event=merge_group&status={status}&per_page=100" + ) + runs += listing["workflow_runs"] + stale = superseded(runs, current) + pr = queue_pr(current.get("head_branch") or "") + print(f"PR #{pr}: cancelling {len(stale)} superseded queue run(s): {stale}") + for run_id in stale: + if args.dry_run: + continue + try: + api(f"repos/{repo}/actions/runs/{run_id}/cancel", method="POST") + except HTTPError as error: + # A run that finished meanwhile can no longer be cancelled (409). + print(f"run {run_id}: cancel returned {error.code}; leaving it") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/render_workflows.py b/ci/render_workflows.py index 62dd2f5e2..ee9f323f3 100644 --- a/ci/render_workflows.py +++ b/ci/render_workflows.py @@ -368,6 +368,85 @@ def render_reuse_decision() -> str: ) +# GEN-022 (forge-ci/ci.yml#405): the end-of-queue janitor needs ci-lint with +# the #405 repo-wide LRU budget pass, which CI_LINT_SHA predates. +QUEUE_JANITOR_CI_LINT_SHA = "953c4e13691b10c1c4f1f9c39fa259fd732c5789" +QUEUE_JANITOR_NEEDS = ( + "cancel-superseded", "verify", "linux", "board_plan", "fbuild_bin", "pr_boards", + "reuse_decision", "test", "full", "selected-coverage", +) + + +def render_cancel_superseded() -> str: + """GEN-022 / forge-ci/ci.yml#409: cancel older queue runs for the same PR. + + A queue ref carries the base commit (gh-readonly-queue/main/pr--), + so the concurrency group never pairs a PR's rebuilt queue run with the one + it replaced. No `needs:`; only the end-of-queue janitor depends on it, and + local-gate.toml exempts it from verify. + """ + return """ cancel-superseded: + name: Cancel superseded queue runs + if: ${{ !cancelled() && github.event_name == 'merge_group' }} + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + actions: write + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + sparse-checkout: ci/cancel_superseded.py + sparse-checkout-cone-mode: false + persist-credentials: false + - name: Cancel + env: + GH_TOKEN: ${{ github.token }} + run: python3 ci/cancel_superseded.py +""" + + +def render_queue_cache_janitor() -> str: + """GEN-022 (forge-ci/ci.yml#405): every merge-queue run ends with the janitor. + + It needs every other job and runs on always(), so the repository stays + under [cache].budget by ci-lint's LRU order instead of GitHub's; the + non-cancelling `cache-janitor` group keeps one sweep at a time. + """ + return ( + " queue-cache-janitor:\n" + " name: End-of-queue cache janitor\n" + f" needs: [{', '.join(QUEUE_JANITOR_NEEDS)}]\n" + " if: always() && github.event_name == 'merge_group'\n" + " runs-on: ubuntu-24.04\n" + " timeout-minutes: 5\n" + " concurrency:\n" + " group: cache-janitor\n" + " cancel-in-progress: false\n" + " permissions:\n" + " contents: read\n" + " actions: write\n" + " steps:\n" + " - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6\n" + " with:\n" + " persist-credentials: false\n" + " sparse-checkout: ci.toml\n" + " sparse-checkout-cone-mode: false\n" + " - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6\n" + " with:\n" + " repository: zackees/ci.yml\n" + f" ref: {QUEUE_JANITOR_CI_LINT_SHA}\n" + " path: .ci-lint\n" + " persist-credentials: false\n" + " sparse-checkout: ci_lint\n" + " - name: Sweep caches\n" + " env:\n" + " GITHUB_TOKEN: ${{ github.token }}\n" + " PYTHONPATH: .ci-lint\n" + " run: python3 -P -m ci_lint cache janitor --repo . --default-branch main\n" + ) + + def render_local_gate_verify() -> str: """Report optional local proof without gating hosted validation (#1698).""" return """ verify: @@ -442,6 +521,7 @@ def render_ci(boards: list[dict], tier: str, families: dict) -> str: " pull_request:\n" " branches: [main]\n" " types: [opened, labeled, unlabeled, synchronize, reopened]\n" + " merge_group:\n" " workflow_dispatch: {}\n" ) gate = ( @@ -542,6 +622,7 @@ def render_ci(boards: list[dict], tier: str, families: dict) -> str: f" group: {name}-${{{{ github.event_name == 'pull_request' && github.ref || github.run_id }}}}\n" " cancel-in-progress: ${{ github.event_name == 'pull_request' }}\n\n" "jobs:\n" + + (render_cancel_superseded() if minimal else "") + verify + " linux:\n" + (" if: github.event_name != 'pull_request' || (!contains(github.event.pull_request.labels.*.name, 'ci-test') && !contains(github.event.pull_request.labels.*.name, 'ci-full'))\n" if minimal else gate) @@ -646,6 +727,7 @@ def render_ci(boards: list[dict], tier: str, families: dict) -> str: " # Path-selected boards: skipped when the PR touches none.\n" " case \"$PR_BOARDS\" in success|skipped) ;; *) exit 1 ;; esac\n" " fi\n" + + render_queue_cache_janitor() if minimal else "") ) diff --git a/ci/test_fractional_workflows.py b/ci/test_fractional_workflows.py index 2dd38d85b..176e67300 100644 --- a/ci/test_fractional_workflows.py +++ b/ci/test_fractional_workflows.py @@ -297,7 +297,10 @@ def test_ordinary_minimal_and_opt_in_test_are_distinct(self): self.assertIn("push", minimal[True]) self.assertIn("pull_request", minimal[True]) self.assertEqual( - {"verify", "linux", "board_plan", "fbuild_bin", "pr_boards", "reuse_decision", "test", "full", "selected-coverage"}, + { + "cancel-superseded", "verify", "linux", "board_plan", "fbuild_bin", "pr_boards", + "reuse_decision", "test", "full", "selected-coverage", "queue-cache-janitor", + }, set(minimal["jobs"]), ) # GEN-021 shadow decision (zackees/ci.yml#162): main pushes only, never @@ -310,7 +313,24 @@ def test_ordinary_minimal_and_opt_in_test_are_distinct(self): step_budget = sum(step["timeout-minutes"] for step in reuse["steps"]) self.assertLess(step_budget, reuse["timeout-minutes"]) self.assertIn("--mode shadow", reuse["steps"][-1]["run"]) - self.assertFalse(any("reuse_decision" in str(job.get("needs", "")) for job in minimal["jobs"].values())) + # GEN-022's end-of-queue janitor (forge-ci/ci.yml#405) is the one job that + # needs it: it must run after every other job of a merge_group run. + self.assertFalse(any( + "reuse_decision" in str(job.get("needs", "")) + for name, job in minimal["jobs"].items() if name != "queue-cache-janitor" + )) + # GEN-022: queue runs, cancel superseded queue runs (#409), and end with + # the cache janitor that needs every other job (#405). + self.assertIn("merge_group", minimal[True]) + cancel = minimal["jobs"]["cancel-superseded"] + self.assertNotIn("needs", cancel) + self.assertIn("github.event_name == 'merge_group'", cancel["if"]) + self.assertEqual("write", cancel["permissions"]["actions"]) + janitor = minimal["jobs"]["queue-cache-janitor"] + self.assertEqual(set(minimal["jobs"]) - {"queue-cache-janitor"}, set(janitor["needs"])) + self.assertEqual("always() && github.event_name == 'merge_group'", janitor["if"]) + self.assertEqual({"group": "cache-janitor", "cancel-in-progress": False}, janitor["concurrency"]) + self.assertIn("cache janitor", janitor["steps"][-1]["run"]) # Path-selected boards run only on the default PR tier. plan_if = minimal["jobs"]["board_plan"]["if"] self.assertIn("pull_request", plan_if) diff --git a/local-gate.toml b/local-gate.toml index cc92a8c10..25741327a 100644 --- a/local-gate.toml +++ b/local-gate.toml @@ -3,6 +3,8 @@ [gate] run = ["uv", "run", "--no-project", "--python", "3.12", "python", "-m", "ci.local_gate"] verify = "ci-minimal.yml:verify" +# cancel-superseded (GEN-022, ci.yml#409) runs only on merge_group, first, with no `needs:`. +verify-exempt = ["cancel-superseded"] mode = "shadow" [gate.lanes.linux-minimal]