ci: GEN-022 merge queue trigger, cancel-superseded, end-of-queue cache janitor

.github/workflows/ci-minimal.yml

@@ -7,13 +7,32 @@ on:
   pull_request:
     branches: [main]
     types: [opened, labeled, unlabeled, synchronize, reopened]
+  merge_group:
   workflow_dispatch: {}
 
 concurrency:
   group: ci-minimal-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
+  cancel-superseded:
+    name: Cancel superseded queue runs
+    if: ${{ !cancelled() && github.event_name == 'merge_group' }}
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    permissions:
+      contents: read
+      actions: write
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+        with:
+          sparse-checkout: ci/cancel_superseded.py
+          sparse-checkout-cone-mode: false
+          persist-credentials: false
+      - name: Cancel
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: python3 ci/cancel_superseded.py
   verify:
     name: Verify local gate
     continue-on-error: true
@@ -188,3 +207,33 @@ jobs:
             # Path-selected boards: skipped when the PR touches none.
             case "$PR_BOARDS" in success|skipped) ;; *) exit 1 ;; esac
           fi
+  queue-cache-janitor:
+    name: End-of-queue cache janitor
+    needs: [cancel-superseded, verify, linux, board_plan, fbuild_bin, pr_boards, reuse_decision, test, full, selected-coverage]
+    if: always() && github.event_name == 'merge_group'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    concurrency:
+      group: cache-janitor
+      cancel-in-progress: false
+    permissions:
+      contents: read
+      actions: write
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+        with:
+          persist-credentials: false
+          sparse-checkout: ci.toml
+          sparse-checkout-cone-mode: false
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+        with:
+          repository: zackees/ci.yml
+          ref: 953c4e13691b10c1c4f1f9c39fa259fd732c5789
+          path: .ci-lint
+          persist-credentials: false
+          sparse-checkout: ci_lint
+      - name: Sweep caches
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+          PYTHONPATH: .ci-lint
+        run: python3 -P -m ci_lint cache janitor --repo . --default-branch main

ci.toml

@@ -130,7 +130,7 @@ setup-soldr       = { only-in = ".github/actions/setup" }
 cache-actions     = { only-in = ".github/actions/setup" }
 
 [allow.workflows]
-"ci-minimal.yml"                 = ["push:main", "pull_request", "workflow_dispatch"]
+"ci-minimal.yml"                 = ["push:main", "pull_request", "merge_group", "workflow_dispatch"]
 "ci-full.yml"                    = ["workflow_call", "workflow_dispatch"]
 "ci-test.yml"                    = ["workflow_call", "workflow_dispatch"]
 "ci-workflow-drift.yml"          = ["workflow_dispatch", "push:main", "pull_request"]
@@ -160,7 +160,7 @@ cache-actions     = { only-in = ".github/actions/setup" }
 "template_native_build.yml"      = ["workflow_call"]
 
 [allow.permissions]
-"actions: write"      = ["dispatch"]
+"actions: write"      = ["dispatch", "cancel-superseded", "queue-cache-janitor"]
 "contents: write"     = ["publish"]
 "pages: write"        = ["publish", "deploy-pages"]
 "attestations: write" = ["build", "publish"]

ci/cancel_superseded.py

@@ -0,0 +1,95 @@
+"""Cancel merge-queue runs a newer queue run for the same PR replaced (GEN-022, ci.yml#409).
+
+    python3 ci/cancel_superseded.py [--dry-run]
+
+Runs as a job of every `merge_group` CI run (GITHUB_RUN_ID, GITHUB_REPOSITORY
+and a GH_TOKEN with actions: write). A queue ref carries the base commit,
+`gh-readonly-queue/main/pr-<N>-<sha>`, so the workflow's concurrency group
+never pairs a PR's rebuilt queue run with its old one. When main moves,
+GitHub starts a new run per queued PR and the old ones keep holding runners.
+The newest queue run for a PR is the live one, so every older in-flight
+queue run for the same PR is cancelled. Ported from obs-rust/obs-studio's
+`ci/cancel_superseded.py` (#986 there) via talos-hq/clud (#2048).
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import sys
+from urllib.error import HTTPError
+from urllib.request import Request, urlopen
+
+QUEUE_RE = re.compile(r"(?:^|/)gh-readonly-queue/[^/]+/pr-(\d+)-[0-9a-f]+$")
+LIVE = ("queued", "in_progress", "waiting", "pending", "requested")
+
+
+def queue_pr(ref: str) -> int | None:
+    """The PR number of a merge-queue ref, else None."""
+    match = QUEUE_RE.search(ref)
+    return int(match.group(1)) if match else None
+
+
+def superseded(runs: list[dict], current: dict) -> list[int]:
+    """Ids of in-flight queue runs for current's PR that started before it."""
+    pr = queue_pr(current.get("head_branch") or "")
+    if pr is None:
+        return []
+    return sorted(
+        {
+            run["id"]
+            for run in runs
+            if run["id"] != current["id"]
+            and run.get("status") in LIVE
+            and queue_pr(run.get("head_branch") or "") == pr
+            and run["created_at"] < current["created_at"]
+        }
+    )
+
+
+def api(path: str, method: str = "GET") -> dict:
+    request = Request(
+        f"https://api.github.com/{path}",
+        method=method,
+        headers={
+            "Authorization": f"Bearer {os.environ['GH_TOKEN']}",
+            "Accept": "application/vnd.github+json",
+        },
+    )
+    with urlopen(request, timeout=30) as response:
+        body = response.read()
+    return json.loads(body) if body else {}
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=(__doc__ or "").splitlines()[0])
+    parser.add_argument("--dry-run", action="store_true")
+    args = parser.parse_args(argv)
+    repo = os.environ["GITHUB_REPOSITORY"]
+    current = api(f"repos/{repo}/actions/runs/{os.environ['GITHUB_RUN_ID']}")
+    workflow = current["workflow_id"]
+    runs: list[dict] = []
+    for status in LIVE:
+        listing = api(
+            f"repos/{repo}/actions/workflows/{workflow}/runs"
+            f"?event=merge_group&status={status}&per_page=100"
+        )
+        runs += listing["workflow_runs"]
+    stale = superseded(runs, current)
+    pr = queue_pr(current.get("head_branch") or "")
+    print(f"PR #{pr}: cancelling {len(stale)} superseded queue run(s): {stale}")
+    for run_id in stale:
+        if args.dry_run:
+            continue
+        try:
+            api(f"repos/{repo}/actions/runs/{run_id}/cancel", method="POST")
+        except HTTPError as error:
+            # A run that finished meanwhile can no longer be cancelled (409).
+            print(f"run {run_id}: cancel returned {error.code}; leaving it")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

ci/render_workflows.py

@@ -368,6 +368,85 @@ def render_reuse_decision() -> str:
     )
 
 
+# GEN-022 (forge-ci/ci.yml#405): the end-of-queue janitor needs ci-lint with
+# the #405 repo-wide LRU budget pass, which CI_LINT_SHA predates.
+QUEUE_JANITOR_CI_LINT_SHA = "953c4e13691b10c1c4f1f9c39fa259fd732c5789"
+QUEUE_JANITOR_NEEDS = (
+    "cancel-superseded", "verify", "linux", "board_plan", "fbuild_bin", "pr_boards",
+    "reuse_decision", "test", "full", "selected-coverage",
+)
+
+
+def render_cancel_superseded() -> str:
+    """GEN-022 / forge-ci/ci.yml#409: cancel older queue runs for the same PR.
+
+    A queue ref carries the base commit (gh-readonly-queue/main/pr-<N>-<sha>),
+    so the concurrency group never pairs a PR's rebuilt queue run with the one
+    it replaced. No `needs:`; only the end-of-queue janitor depends on it, and
+    local-gate.toml exempts it from verify.
+    """
+    return """  cancel-superseded:
+    name: Cancel superseded queue runs
+    if: ${{ !cancelled() && github.event_name == 'merge_group' }}
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    permissions:
+      contents: read
+      actions: write
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+        with:
+          sparse-checkout: ci/cancel_superseded.py
+          sparse-checkout-cone-mode: false
+          persist-credentials: false
+      - name: Cancel
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: python3 ci/cancel_superseded.py
+"""
+
+
+def render_queue_cache_janitor() -> str:
+    """GEN-022 (forge-ci/ci.yml#405): every merge-queue run ends with the janitor.
+
+    It needs every other job and runs on always(), so the repository stays
+    under [cache].budget by ci-lint's LRU order instead of GitHub's; the
+    non-cancelling `cache-janitor` group keeps one sweep at a time.
+    """
+    return (
+        "  queue-cache-janitor:\n"
+        "    name: End-of-queue cache janitor\n"
+        f"    needs: [{', '.join(QUEUE_JANITOR_NEEDS)}]\n"
+        "    if: always() && github.event_name == 'merge_group'\n"
+        "    runs-on: ubuntu-24.04\n"
+        "    timeout-minutes: 5\n"
+        "    concurrency:\n"
+        "      group: cache-janitor\n"
+        "      cancel-in-progress: false\n"
+        "    permissions:\n"
+        "      contents: read\n"
+        "      actions: write\n"
+        "    steps:\n"
+        "      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6\n"
+        "        with:\n"
+        "          persist-credentials: false\n"
+        "          sparse-checkout: ci.toml\n"
+        "          sparse-checkout-cone-mode: false\n"
+        "      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6\n"
+        "        with:\n"
+        "          repository: zackees/ci.yml\n"
+        f"          ref: {QUEUE_JANITOR_CI_LINT_SHA}\n"
+        "          path: .ci-lint\n"
+        "          persist-credentials: false\n"
+        "          sparse-checkout: ci_lint\n"
+        "      - name: Sweep caches\n"
+        "        env:\n"
+        "          GITHUB_TOKEN: ${{ github.token }}\n"
+        "          PYTHONPATH: .ci-lint\n"
+        "        run: python3 -P -m ci_lint cache janitor --repo . --default-branch main\n"
+    )
+
+
 def render_local_gate_verify() -> str:
     """Report optional local proof without gating hosted validation (#1698)."""
     return """  verify:
@@ -442,6 +521,7 @@ def render_ci(boards: list[dict], tier: str, families: dict) -> str:
         "  pull_request:\n"
         "    branches: [main]\n"
         "    types: [opened, labeled, unlabeled, synchronize, reopened]\n"
+        "  merge_group:\n"
         "  workflow_dispatch: {}\n"
     )
     gate = (
@@ -542,6 +622,7 @@ def render_ci(boards: list[dict], tier: str, families: dict) -> str:
         f"  group: {name}-${{{{ github.event_name == 'pull_request' && github.ref || github.run_id }}}}\n"
         "  cancel-in-progress: ${{ github.event_name == 'pull_request' }}\n\n"
         "jobs:\n"
+        + (render_cancel_superseded() if minimal else "")
         + verify
         + "  linux:\n"
         + ("    if: github.event_name != 'pull_request' || (!contains(github.event.pull_request.labels.*.name, 'ci-test') && !contains(github.event.pull_request.labels.*.name, 'ci-full'))\n" if minimal else gate)
@@ -646,6 +727,7 @@ def render_ci(boards: list[dict], tier: str, families: dict) -> str:
            "            # Path-selected boards: skipped when the PR touches none.\n"
            "            case \"$PR_BOARDS\" in success|skipped) ;; *) exit 1 ;; esac\n"
            "          fi\n"
+           + render_queue_cache_janitor()
            if minimal else "")
     )
 

ci/test_fractional_workflows.py

@@ -297,7 +297,10 @@ def test_ordinary_minimal_and_opt_in_test_are_distinct(self):
         self.assertIn("push", minimal[True])
         self.assertIn("pull_request", minimal[True])
         self.assertEqual(
-            {"verify", "linux", "board_plan", "fbuild_bin", "pr_boards", "reuse_decision", "test", "full", "selected-coverage"},
+            {
+                "cancel-superseded", "verify", "linux", "board_plan", "fbuild_bin", "pr_boards",
+                "reuse_decision", "test", "full", "selected-coverage", "queue-cache-janitor",
+            },
             set(minimal["jobs"]),
         )
         # GEN-021 shadow decision (zackees/ci.yml#162): main pushes only, never
@@ -310,7 +313,24 @@ def test_ordinary_minimal_and_opt_in_test_are_distinct(self):
         step_budget = sum(step["timeout-minutes"] for step in reuse["steps"])
         self.assertLess(step_budget, reuse["timeout-minutes"])
         self.assertIn("--mode shadow", reuse["steps"][-1]["run"])
-        self.assertFalse(any("reuse_decision" in str(job.get("needs", "")) for job in minimal["jobs"].values()))
+        # GEN-022's end-of-queue janitor (forge-ci/ci.yml#405) is the one job that
+        # needs it: it must run after every other job of a merge_group run.
+        self.assertFalse(any(
+            "reuse_decision" in str(job.get("needs", ""))
+            for name, job in minimal["jobs"].items() if name != "queue-cache-janitor"
+        ))
+        # GEN-022: queue runs, cancel superseded queue runs (#409), and end with
+        # the cache janitor that needs every other job (#405).
+        self.assertIn("merge_group", minimal[True])
+        cancel = minimal["jobs"]["cancel-superseded"]
+        self.assertNotIn("needs", cancel)
+        self.assertIn("github.event_name == 'merge_group'", cancel["if"])
+        self.assertEqual("write", cancel["permissions"]["actions"])
+        janitor = minimal["jobs"]["queue-cache-janitor"]
+        self.assertEqual(set(minimal["jobs"]) - {"queue-cache-janitor"}, set(janitor["needs"]))
+        self.assertEqual("always() && github.event_name == 'merge_group'", janitor["if"])
+        self.assertEqual({"group": "cache-janitor", "cancel-in-progress": False}, janitor["concurrency"])
+        self.assertIn("cache janitor", janitor["steps"][-1]["run"])
         # Path-selected boards run only on the default PR tier.
         plan_if = minimal["jobs"]["board_plan"]["if"]
         self.assertIn("pull_request", plan_if)

local-gate.toml

@@ -3,6 +3,8 @@
 [gate]
 run = ["uv", "run", "--no-project", "--python", "3.12", "python", "-m", "ci.local_gate"]
 verify = "ci-minimal.yml:verify"
+# cancel-superseded (GEN-022, ci.yml#409) runs only on merge_group, first, with no `needs:`.
+verify-exempt = ["cancel-superseded"]
 mode = "shadow"
 
 [gate.lanes.linux-minimal]