ci: route all workflows homeserver-first with GitHub-hosted fallback

.github/actionlint.yaml

@@ -2,6 +2,9 @@ self-hosted-runner:
   # Labels of self-hosted runner in array of strings.
   labels:
     - honeypot-home
+    # CI-feedback runner on the same homeserver as honeypot-home -- see
+    # scripts/github-ci-runner/install-ci-runner.sh and #2565.
+    - honeypot-ci
 
 # Configuration variables in array of strings defined in your repository or
 # organization. `null` means disabling configuration variables check.

.github/workflows/ci-heartbeat.yml

@@ -1,7 +1,8 @@
 name: CI heartbeat
 
-# One-job canary used by quality.yml's ci-target router as its
-# homeserver-liveness proof. The runners-listing REST endpoint answers
+# One-job canary used by every workflow's ci-target router (quality.yml's
+# inline job, and the reusable ci-router.yml) as its homeserver-liveness
+# proof. The runners-listing REST endpoint answers
 # 403 "Resource not accessible by integration" for GITHUB_TOKEN no matter
 # what permissions a workflow requests -- the registry cannot be asked,
 # so availability is MEASURED instead: router dispatches this workflow at
@@ -14,17 +15,22 @@ name: CI heartbeat
 #
 # Deliberately trivial -- checkout would waste minutes; empty steps are
 # legal. Never triggers itself on push/pull_request/schedule.
+#
+# Deliberately WITHOUT a concurrency group: one push now fans out to
+# several routing workflows at once (quality, containers, security,
+# pages), each dispatching its own canary on the same ref. With
+# cancel-in-progress, the newest dispatch would cancel the canary an
+# earlier router is actively waiting on, un-vouching exactly the evidence
+# its decision depends on and flinging that workflow to the fallback for
+# no operational reason. Canaries are sub-second no-op jobs; letting
+# siblings queue briefly on the single runner is cheaper than that race.
 
 on:
   workflow_dispatch:
 
 permissions:
   contents: none
 
-concurrency:
-  group: ci-heartbeat-${{ github.ref }}
-  cancel-in-progress: true
-
 jobs:
   ping:
     name: homeserver reachable?

.github/workflows/ci-router.yml

@@ -0,0 +1,224 @@
+name: CI executor router
+
+# Reusable homeserver-first routing decision, shared by every workflow that
+# can execute jobs on the homeserver's honeypot-ci runner (containers.yml,
+# security.yml, pages.yml -- and, once its inline copy retires, quality.yml).
+# The decision procedure and its full rationale are documented in quality.yml's
+# ci-target job and docs/CI-CD.md; the short form:
+#
+#   1. Trust gate. Fork PRs are attacker-controlled input and must never
+#      become process execution on home-network infrastructure, so they are
+#      never routed to the homeserver (the caller passes CI_HOMESERVER_PRS
+#      through, and empty means pull_request is never trusted). push to
+#      main and workflow_dispatch are trusted by construction.
+#   2. Liveness is measured, not read. GET /actions/runners answers HTTP
+#      403 to GITHUB_TOKEN under every permission shape (live-verified
+#      2026-08-27), so availability is proven by dispatching the ci-heartbeat
+#      canary at a real ref -- the triggering branch, or a same-repo PR's
+#      head branch (the "<n>/merge" merge ref GITHUB_REF_NAME holds for
+#      pull_request runs backs no branch and dispatches refuse with 422) --
+#      and requiring a fresh run to complete inside the decision windows.
+#      Anything else -- box off, paused, unregistered, service wedged --
+#      times out and the caller's jobs fall back to GitHub-hosted.
+#
+# Fail-safe: every error path resolves to homeserver=false, so routing can
+# degrade CI's speed, never its pass/fail correctness.
+#
+# Caller obligation: every calling workflow MUST grant `actions: write` at
+# its workflow-level permissions -- a called reusable workflow can never
+# exceed the caller's envelope, and under-granting it startup-fails the
+# whole run as "Invalid workflow file" (the route job below requests
+# actions:write solely for the heartbeat dispatch).
+
+on:
+  workflow_call:
+    inputs:
+      ci_homeserver_prs:
+        description: >-
+          Value of vars.CI_HOMESERVER_PRS at the caller. "true" lets
+          same-repo pull_request runs execute on the homeserver; anything
+          else keeps every pull_request on GitHub-hosted.
+        type: string
+        required: false
+        default: ""
+    outputs:
+      homeserver:
+        description: "'true' when the honeypot-ci runner may execute this run, else 'false'"
+        value: ${{ jobs.route.outputs.homeserver }}
+
+permissions:
+  contents: read
+
+jobs:
+  route:
+    name: Pick CI executor
+    runs-on: ubuntu-latest
+    # Must comfortably outlive the heartbeat windows below
+    # (dispatch + appear-wait + terminal-wait + API slack; worst case
+    # 120s + 420s plus slack).
+    timeout-minutes: 15
+    # actions:write exists solely to launch the ci-heartbeat canary.
+    permissions:
+      contents: read
+      actions: write
+    outputs:
+      # Always written ('true'/'false', never empty) so downstream
+      # `== 'true'` comparisons cannot misread an absent string as truthy.
+      homeserver: ${{ steps.route.outputs.homeserver }}
+    env:
+      GH_TOKEN: ${{ github.token }}
+      CI_HOMESERVER_PRS: ${{ inputs.ci_homeserver_prs }}
+    steps:
+      - id: route
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          # The runner always exposes the webhook payload natively as
+          # GITHUB_EVENT_PATH; there is no github.event_path expression to
+          # map through env (the key silently vanishes and, under set -u,
+          # reading the bare name aborts the step -- seen live on a PR run).
+          EVENT_PATH="${EVENT_PATH:-${GITHUB_EVENT_PATH:-}}"
+
+          # Defaults are the fallback path; the guards below may only set
+          # them to true.
+          trusted=false
+          online=false
+
+          case "$GITHUB_EVENT_NAME" in
+            push)
+              # Callers' push triggers are branch/tag scoped to
+              # already-reviewed commits (main, v* tags).
+              trusted=true ;;
+            workflow_dispatch)
+              trusted=true ;;
+            schedule)
+              trusted=true ;;
+            pull_request)
+              base="$(jq -r '.pull_request.base.repo.full_name // ""' "$EVENT_PATH")"
+              head_repo="$(jq -r '.pull_request.head.repo.full_name // ""' "$EVENT_PATH")"
+              if [[ -n "$base" && "$base" == "$head_repo" && "${CI_HOMESERVER_PRS}" == "true" ]]; then
+                trusted=true
+              fi ;;
+          esac
+
+          # Decision-window knobs: env-overridable so local unit tests run
+          # in seconds while production keeps these defaults. The two
+          # windows together bound worst-case router cost well under this
+          # job's own timeout-minutes.
+          # GH_TOKEN arrives via the job-level env mapping above and is
+          # therefore always present -- no shell-side default guard here,
+          # which scripts/check-public-leaks.py would flag as a literal
+          # credential assignment.
+
+          # 120s appear: seen live on #2568 -- the dispatch is accepted and
+          # the canary still not visible in /runs 45s later while GitHub's
+          # run indexing lags, so the window must cover API lag, not just
+          # dispatch latency (the loop exits early when the run appears,
+          # so the cost is paid only in the worst case).
+          # 420s decide: the canary is tiny, but it rides a queue behind
+          # whatever else the single box instance is executing; #2572
+          # tracks real capacity. Fan-out deeper than ~2 canaries still
+          # overflows this window and falls back -- fail-safe direction.
+          appear_window="${HEARTBEAT_APPEAR_SECONDS:-120}"   # event->run-object lag (API indexing included)
+          decide_window="${HEARTBEAT_DECIDE_SECONDS:-420}"   # queue+exec budget on the box
+          poll_every="${HEARTBEAT_POLL_SECONDS:-10}"
+
+          if [[ "$trusted" == "true" ]]; then
+            api="repos/$GITHUB_REPOSITORY/actions/workflows/ci-heartbeat.yml"
+            # The dispatch API needs a real ref. For pull_request runs
+            # GITHUB_REF_NAME is the ephemeral "<n>/merge" merge ref that no
+            # branch backs, so dispatching there is refused with 422 "No ref
+            # found for" -- seen live on #2568, and it would have routed
+            # every same-repo PR to the fallback forever. Same-repo PRs
+            # therefore dispatch at the PR's head branch (whose
+            # ci-heartbeat.yml is the branch's own; fork PRs never get this
+            # far, trusted=false). push/workflow_dispatch/schedule carry a
+            # real branch in GITHUB_REF_NAME already.
+            dispatch_ref="$GITHUB_REF_NAME"
+            if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then
+              dispatch_ref="$(jq -r '.pull_request.head.ref // ""' "$EVENT_PATH")"
+            fi
+            echo "heartbeat: dispatching ci-heartbeat on ${dispatch_ref:-<none>}"
+            http=""
+            if [[ -n "$dispatch_ref" ]]; then
+              # No -f: a refused dispatch must land its HTTP code in $http,
+              # not abort the capture (with -f a 422 printed as "422X").
+              http="$(curl -sS -o "${RUNNER_TEMP:-/tmp}/hb-disp.out" -w '%{http_code}' -X POST \
+                -H "Authorization: Bearer $GH_TOKEN" \
+                -H 'Accept: application/vnd.github+json' \
+                "$GITHUB_API_URL/$api/dispatches" \
+                -d "{\"ref\":\"${dispatch_ref}\"}" || true)"
+            fi
+            [[ "$http" == "204" ]] \
+              && echo "heartbeat: dispatch accepted" \
+              || { echo "::warning::heartbeat dispatch refused (HTTP ${http:-none}); routing fallback"; http=""; }
+
+            # cutoff predates the dispatch by 2min so minor runner/GitHub
+            # clock skew cannot out-veto a real fresh run. Must be assigned
+            # BEFORE Phase A -- under set -u an unbound $cutoff kills the
+            # pipeline subshell every poll iteration and rid silently stays
+            # empty (live on #2568: canaries succeeded while every router
+            # still concluded offline because this line was dropped in the
+            # dispatch-ref edit).
+            cutoff="$(date -u -d '2 minutes ago' +%FT%TZ)"
+
+            # Phase A: wait for a FRESH heartbeat run to appear -- created
+            # after the cutoff (dispatch minus skew allowance) and not yet
+            # completed. The ref is the branch name (the dispatch API
+            # rejects commit SHAs with 422 "No ref found for"), so
+            # freshness -- not a SHA pin -- guards against counting old
+            # runs: only something genuinely recent may vouch, and it must
+            # still reach success inside this cycle's own window. Runs
+            # dispatched concurrently by sibling workflows' routers on the
+            # same ref may be consumed as evidence; they measure the same
+            # box (ci-heartbeat.yml therefore carries no concurrency group:
+            # cancelling a sibling's canary would un-vouch exactly the
+            # evidence this router is waiting on).
+            rid=""
+            deadline=$(( $(date +%s) + appear_window ))
+            while [[ "$(date +%s)" -lt "$deadline" ]]; do
+              # system jq (not `gh api --jq`, which cannot bind
+              # variables) so the cutoff is a proper parameter.
+              runs_json="$(gh api "$api/runs?per_page=20" 2>/dev/null || true)"
+              rid="$(printf '%s' "$runs_json" | jq -r --arg cutoff "$cutoff" '
+                [.workflow_runs[]
+                 | select(.event == "workflow_dispatch"
+                          and .status != "completed"
+                          and .run_started_at >= $cutoff)
+                 | .id][0] // ""' 2>/dev/null || true)"
+              [[ -n "$rid" ]] && break
+              sleep "$poll_every"
+            done
+
+            # Phase B: give the found run its window to finish.
+            if [[ -n "$rid" ]]; then
+              deadline=$(( $(date +%s) + decide_window ))
+              while [[ "$(date +%s)" -lt "$deadline" ]]; do
+                line="$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$rid" \
+                  --jq '"\(.status) \(.conclusion)"' 2>/dev/null || true)"
+                case "$line" in
+                  "completed success") online=true ;;
+                  completed*) : ;;                # finished badly -> stays offline
+                  # empty = transient API hiccup; anything else = still
+                  # running. Both keep polling inside the window.
+                  *) sleep "$poll_every"; continue ;;
+                esac
+                break
+              done
+            fi
+            echo "heartbeat verdict: online=$online"
+          fi
+
+          homeserver=false
+          if [[ "$trusted" == "true" && "$online" == "true" ]]; then
+            homeserver=true
+          fi
+
+          {
+            echo "trusted=$trusted"
+            echo "online=$online"
+            echo "executor=$( [[ "$homeserver" == true ]] && echo honeypot-ci || echo ubuntu-latest )"
+          } >>"$GITHUB_STEP_SUMMARY"
+
+          echo "homeserver=$homeserver" >>"$GITHUB_OUTPUT"

.github/workflows/containers.yml

@@ -10,11 +10,42 @@ on:
 permissions:
   contents: read
   packages: write
+  # The ci-target router dispatches the ci-heartbeat canary with
+  # GITHUB_TOKEN, and a called reusable workflow can never exceed the
+  # caller's envelope -- under-granting it startup-fails the whole run
+  # as "Invalid workflow file" (quality.yml's inline router instead
+  # elevates at its own job level).
+  actions: write
 
 jobs:
+  # Executor routing ("homeserver first, GitHub-hosted fallback") -- the
+  # same ci-target decision quality.yml documents in full: trusted events
+  # (push to main, tags, workflow_dispatch; same-repo pull_request only
+  # when repo variable CI_HOMESERVER_PRS=true) may run on the homeserver,
+  # and only if a fresh ci-heartbeat canary proves a honeypot-ci runner
+  # actually executable right now. Fork pull_request runs never reach the
+  # box and land on GitHub-hosted exactly as before.
+  #
+  # Unlike quality.yml there is a single matrix job rather than PAIR twins:
+  # every row is executor-agnostic (buildx builds and -- on non-PR events
+  # -- ghcr pushes work identically under either runner), so the one job
+  # just picks its runs-on off the router output. The matrix name carries
+  # the "(GitHub-hosted)" suffix on fallback days per quality.yml's pair-
+  # naming rule, so a degraded day reads honestly in the checks list.
+  # Matrix rows serialize behind the single registered runner instance;
+  # the 120-min per-row ceiling only fires on a wedged pickup, mirroring
+  # the timeout-minutes-on-homeserver-only convention.
+  ci-target:
+    name: Pick CI executor
+    uses: ./.github/workflows/ci-router.yml
+    with:
+      ci_homeserver_prs: ${{ vars.CI_HOMESERVER_PRS || '' }}
+
   build:
-    name: ${{ matrix.image }}
-    runs-on: ubuntu-latest
+    name: ${{ matrix.image }}${{ needs.ci-target.outputs.homeserver != 'true' && ' (GitHub-hosted)' || '' }}
+    needs: [ci-target]
+    runs-on: ${{ needs.ci-target.outputs.homeserver == 'true' && fromJSON('["self-hosted", "linux", "x64", "honeypot-ci"]') || fromJSON('["ubuntu-latest"]') }}
+    timeout-minutes: ${{ needs.ci-target.outputs.homeserver == 'true' && 120 || 360 }}
     strategy:
       fail-fast: false
       # #1502: contexts below repointed at arcane/home/honeypot-<name>/ for

.github/workflows/pages.yml

@@ -15,15 +15,36 @@ on:
 
 permissions:
   contents: read
+  # The ci-target router dispatches the ci-heartbeat canary with
+  # GITHUB_TOKEN, and a called reusable workflow can never exceed the
+  # caller's envelope -- under-granting it startup-fails the whole run
+  # as "Invalid workflow file". The deploy job keeps its own narrower
+  # job-level envelope (pages: write, id-token: write).
+  actions: write
 
 concurrency:
   group: pages
   cancel-in-progress: false
 
 jobs:
+  # Executor routing ("homeserver first, GitHub-hosted fallback") via the
+  # shared ci-router.yml -- same trust gate and heartbeat liveness proof
+  # quality.yml's ci-target job documents. The build's whole runtime is
+  # checkout files plus a stdlib python script and an artifact upload, so
+  # it is executor-agnostic and simply picks runs-on off the router
+  # output; on fallback days it reports under the "(GitHub-hosted)"
+  # suffixed name per quality.yml's pair-naming rule.
+  ci-target:
+    name: Pick CI executor
+    uses: ./.github/workflows/ci-router.yml
+    with:
+      ci_homeserver_prs: ${{ vars.CI_HOMESERVER_PRS || '' }}
+
   build:
-    name: Build Pages artifact
-    runs-on: ubuntu-latest
+    name: Build Pages artifact${{ needs.ci-target.outputs.homeserver != 'true' && ' (GitHub-hosted)' || '' }}
+    needs: [ci-target]
+    runs-on: ${{ needs.ci-target.outputs.homeserver == 'true' && fromJSON('["self-hosted", "linux", "x64", "honeypot-ci"]') || fromJSON('["ubuntu-latest"]') }}
+    timeout-minutes: ${{ needs.ci-target.outputs.homeserver == 'true' && 15 || 360 }}
     steps:
       - name: Checkout
         uses: actions/checkout@v7
@@ -32,13 +53,17 @@ jobs:
         uses: actions/configure-pages@v6
 
       - name: Build branding site
-        run: python branding/scripts/build_pages_site.py --output _site
+        run: python3 branding/scripts/build_pages_site.py --output _site
 
       - name: Upload Pages artifact
         uses: actions/upload-pages-artifact@v5
         with:
           path: _site
 
+  # Stays GitHub-hosted unconditionally: it is a seconds-long API call
+  # against the github-pages environment (no compute to relocate), and
+  # routing it through the heartbeat would only add a router leg between
+  # the artifact upload above and the deploy.
   deploy:
     name: Deploy Pages
     if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'