From b8637f444d91de608b1115dc99095376eb58ff5e Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Fri, 25 Sep 2026 21:30:29 +0100 Subject: [PATCH 1/2] ci: run CI, mesh and Litestream tests on GitHub-hosted runners The repository is public, so GitHub-hosted minutes are free, and checking the fleet's configuration no longer depends on the fleet being up. The three workflows drop the runner-fallback routing and run on ubuntu-latest; the mesh scenarios run in parallel, since each gets its own runner. The smoke tests keep targeting the fleet they test. --- .github/workflows/ci.yml | 42 ++++++-------------------- .github/workflows/litestream.yml | 22 +++----------- .github/workflows/mesh-integration.yml | 24 +++------------ README.md | 3 +- 4 files changed, 19 insertions(+), 72 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a9581b..b0a1e40 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,13 +2,9 @@ name: CI # Lints every shell script, workflow file, YAML file, and Dockerfile in the repo. There is no application test suite here (this repo is fleet infrastructure config, not application code) - these checks are its actual verification surface, matching the README's Build/test/lint section. # -# Dogfoods the fleet: every job after determine-runner routes through ExaDev/runner-fallback-action, preferring exadev-runners and falling back to ubuntu-latest only if the fleet's heartbeat gist reports it unhealthy (or the run is a fork PR / Dependabot - that guard lives inside the action). See the README's Heartbeat section for how the gist is kept fresh, and ExaDev/runner-fallback-action's own README for the routing design. -# -# determine-runner itself is deliberately anchored on exadev-runners directly, not ubuntu-latest as the action's README recommends as the general-purpose default: this is the fleet's own repo, and it should dogfood the fleet fully rather than hedge on it. This does forgo the fallback safety net for this one bootstrap job specifically - if the fleet is entirely down, determine-runner cannot schedule and cannot hand off to ubuntu-latest either - but that trade is the whole point here, not a workaround to revert later. +# Runs on GitHub-hosted runners, which are free for a public repository, so the checks of the fleet's own configuration don't depend on the fleet being up. The fleet's own behaviour is tested by test-arc-runner.yml and test-autoscaler.yml, which target it directly. # # required-checks is the single required status check in this repo's ruleset (never the individual job names), so adding or renaming a lint job never needs a ruleset update - matching ExaDev/eslint-config and ExaDev/wire-mesh's own ci.yml. -# -# The action is pinned to a commit SHA, not @main - bump deliberately when it changes. (b774d51 = the secret-gist redesign, the current HEAD of main as of the day this workflow was written.) on: push: @@ -23,21 +19,9 @@ permissions: contents: read jobs: - determine-runner: - name: Determine runner - runs-on: exadev-runners - timeout-minutes: 5 - outputs: - selected-runner: ${{ steps.fallback.outputs.selected-runner || '["ubuntu-latest"]' }} - preferred-available: ${{ steps.fallback.outputs.preferred-available || 'false' }} - steps: - - id: fallback - uses: ExaDev/runner-fallback-action@b774d51ff5360bf53705b2eb34ff019afd4d7f07 - shellcheck: name: Shellcheck - needs: determine-runner - runs-on: ${{ fromJson(needs.determine-runner.outputs.selected-runner) }} + runs-on: ubuntu-latest timeout-minutes: 10 steps: - uses: actions/checkout@v7 @@ -54,8 +38,7 @@ jobs: actionlint: name: actionlint - needs: determine-runner - runs-on: ${{ fromJson(needs.determine-runner.outputs.selected-runner) }} + runs-on: ubuntu-latest timeout-minutes: 10 steps: - uses: actions/checkout@v7 @@ -66,8 +49,7 @@ jobs: yamllint: name: YAML lint - needs: determine-runner - runs-on: ${{ fromJson(needs.determine-runner.outputs.selected-runner) }} + runs-on: ubuntu-latest timeout-minutes: 10 steps: - uses: actions/checkout@v7 @@ -80,13 +62,12 @@ jobs: hadolint: name: hadolint - needs: determine-runner - runs-on: ${{ fromJson(needs.determine-runner.outputs.selected-runner) }} + runs-on: ubuntu-latest timeout-minutes: 10 steps: - uses: actions/checkout@v7 - name: Install hadolint - # Detect arch rather than assume amd64: exadev-runners pods are arm64 (the M3's own architecture), confirmed by a real CI run against this very workflow - an assumption that turned out wrong the first time it actually ran on the fleet instead of just being read as a comment. Downloads into the workspace (no sudo, no system path) - actions-runner pods run the workflow user without write access to /usr/local/bin. + # Detect arch rather than assume amd64, so the step also works on an arm64 runner such as this project's own fleet. Downloads into the workspace (no sudo, no system path) - actions-runner pods run the workflow user without write access to /usr/local/bin. run: | set -euo pipefail case "$(uname -m)" in @@ -108,8 +89,7 @@ jobs: ansible-syntax-check: name: Ansible syntax check - needs: determine-runner - runs-on: ${{ fromJson(needs.determine-runner.outputs.selected-runner) }} + runs-on: ubuntu-latest timeout-minutes: 10 steps: - uses: actions/checkout@v7 @@ -137,8 +117,7 @@ jobs: ansible-lint: name: ansible-lint - needs: determine-runner - runs-on: ${{ fromJson(needs.determine-runner.outputs.selected-runner) }} + runs-on: ubuntu-latest timeout-minutes: 10 steps: - uses: actions/checkout@v7 @@ -156,11 +135,10 @@ jobs: required-checks: name: Required Checks - needs: [determine-runner, shellcheck, actionlint, yamllint, hadolint, ansible-syntax-check, ansible-lint] + needs: [shellcheck, actionlint, yamllint, hadolint, ansible-syntax-check, ansible-lint] # Always runs, even if a dependency failed or was skipped, so it can report that failure itself rather than being skipped - a required check that's silently absent blocks a PR indefinitely instead of failing it visibly. if: always() - # Not ubuntu-latest: confirmed live (ExaDev/github-runner#7) that ExaDev's GitHub-hosted runners currently can't schedule at all (runner_id: 0, never assigned) - a bare ubuntu-latest job here would fail every run regardless of every other job's own result. Routes through the same fallback as everything else instead. - runs-on: ${{ fromJson(needs.determine-runner.outputs.selected-runner) }} + runs-on: ubuntu-latest timeout-minutes: 5 steps: - name: Fail if any required job failed or was cancelled diff --git a/.github/workflows/litestream.yml b/.github/workflows/litestream.yml index 651fd79..4086155 100644 --- a/.github/workflows/litestream.yml +++ b/.github/workflows/litestream.yml @@ -2,7 +2,7 @@ name: Headscale Litestream # Runs tests/litestream/run.sh: the headscale_hosted provider with Litestream and a warm standby against an S3-compatible store, a real Tailscale client, promotion of the standby, and the headscale_in_cluster guard. Only on changes that can affect them. # -# Routes through ExaDev/runner-fallback-action like ci.yml, preferring the exadev-image-builder profile: the test needs a Docker daemon, which only that profile's Docker-in-Docker sidecar provides, and ubuntu-latest has one natively. The work directory sits under runner.temp because the DinD daemon resolves Compose bind mounts on its own filesystem, and only the runner's _work volume is shared with it. +# Runs on GitHub-hosted ubuntu-latest, which has the Docker daemon the test needs. The work directory sits under runner.temp, which the job always has write access to. on: pull_request: @@ -25,22 +25,9 @@ permissions: contents: read jobs: - determine-runner: - name: Determine runner - runs-on: exadev-runners - timeout-minutes: 5 - outputs: - selected-runner: ${{ steps.fallback.outputs.selected-runner || '["ubuntu-latest"]' }} - steps: - - id: fallback - uses: ExaDev/runner-fallback-action@b774d51ff5360bf53705b2eb34ff019afd4d7f07 - with: - preferred-labels: exadev-image-builder - litestream: name: Litestream and standby promotion - needs: determine-runner - runs-on: ${{ fromJson(needs.determine-runner.outputs.selected-runner) }} + runs-on: ubuntu-latest timeout-minutes: 30 env: COMPOSE_VERSION: v5.5.1 @@ -87,10 +74,9 @@ jobs: required-checks: name: Headscale Litestream result - needs: [determine-runner, litestream] + needs: [litestream] if: always() - # The ordinary pool, like determine-runner: this job only reads results. - runs-on: exadev-runners + runs-on: ubuntu-latest timeout-minutes: 5 steps: - name: Fail if the test failed or was cancelled diff --git a/.github/workflows/mesh-integration.yml b/.github/workflows/mesh-integration.yml index 6ad4786..8b43389 100644 --- a/.github/workflows/mesh-integration.yml +++ b/.github/workflows/mesh-integration.yml @@ -2,7 +2,7 @@ name: Mesh integration # Brings up three k3s nodes in Docker and joins them through Headscale with the cluster role (tests/mesh/run.sh), once per scenario: headscale_hosted, headscale_in_cluster, and the headscale_existing policy check. Only on changes that can affect how nodes join the mesh. # -# Routes through ExaDev/runner-fallback-action like ci.yml, but prefers the exadev-image-builder profile rather than exadev-runners: this job needs a Docker daemon that can run privileged containers (k3s, and Tailscale inside it), which only that profile's Docker-in-Docker sidecar provides. It falls back to ubuntu-latest, which has one natively. The work directory sits under runner.temp because the DinD daemon resolves Compose bind mounts on its own filesystem, and only the runner's _work volume is shared with it. +# Runs on GitHub-hosted ubuntu-latest, whose Docker daemon runs the privileged containers the test needs (k3s, and Tailscale inside it). The work directory sits under runner.temp, which the job always has write access to. on: pull_request: @@ -24,27 +24,12 @@ permissions: contents: read jobs: - determine-runner: - name: Determine runner - runs-on: exadev-runners - timeout-minutes: 5 - outputs: - selected-runner: ${{ steps.fallback.outputs.selected-runner || '["ubuntu-latest"]' }} - steps: - - id: fallback - uses: ExaDev/runner-fallback-action@b774d51ff5360bf53705b2eb34ff019afd4d7f07 - with: - preferred-labels: exadev-image-builder - mesh: name: Mesh (${{ matrix.scenario }}) - needs: determine-runner - runs-on: ${{ fromJson(needs.determine-runner.outputs.selected-runner) }} + runs-on: ubuntu-latest timeout-minutes: 45 strategy: fail-fast: false - # The builder profile runs one job at a time anyway; this keeps the scenarios from queueing behind each other's timeouts on ubuntu-latest too. - max-parallel: 1 matrix: scenario: [existing, hosted, in_cluster] env: @@ -92,10 +77,9 @@ jobs: required-checks: name: Mesh integration result - needs: [determine-runner, mesh] + needs: [mesh] if: always() - # The ordinary pool, like determine-runner: this job only reads results, so it has no reason to wait for the single builder runner. - runs-on: exadev-runners + runs-on: ubuntu-latest timeout-minutes: 5 steps: - name: Fail if any scenario failed or was cancelled diff --git a/README.md b/README.md index d2a75ac..240ef9f 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ The custom runner toolchain (GCC 11/G++ for C++20, Bun, the `gh` CLI — see `Do `.github/workflows/build-runner-image.yml` is meant to build and push the runner image on GitHub-hosted `ubuntu-latest`, deliberately not on this fleet itself, since building the fleet's own image on the fleet would be circular - but this org can't currently schedule `ubuntu-latest` at all, so every run of it has failed instantly (see [issue #10](https://github.com/ExaDev/github-runner/issues/10)). See [Build, test, and smoke-test](#build-test-and-smoke-test) for the actual current path. -`.github/workflows/ci.yml` dogfoods the fleet instead: its lint jobs prefer `exadev-runners`, routed through [`ExaDev/runner-fallback-action`](https://github.com/ExaDev/runner-fallback-action) so a run falls back to `ubuntu-latest` if the fleet's heartbeat reports it unhealthy. This is safe precisely because lint jobs (unlike the image build above) don't need the fleet to already be working to run — there's no circularity. +`.github/workflows/ci.yml`, `mesh-integration.yml` and `litestream.yml` run on GitHub-hosted `ubuntu-latest`, which is free for a public repository, so checking the fleet's configuration never depends on the fleet itself being up. `test-arc-runner.yml` and `test-autoscaler.yml` are the ones that target the fleet, since the fleet's own behaviour is what they test. ### Tailscale @@ -149,7 +149,6 @@ Commits follow Conventional Commits (`fix:`, `feat:`, `tune:`, and so on). - **Calling `tailscale up --authkey=...` unconditionally on every container restart re-registers a brand new tailnet device each time, instead of reconnecting the persisted one - even with the join key's own state volume intact.** A reusable Tailscale auth key is designed to authorize provisioning *new* devices, and a naive `tailscale up` call on every restart (needed here to learn this node's own Tailscale IP before `exec`ing k3s - see above) doesn't check whether the daemon is already authenticated first, so it keeps minting new devices under names like `node1`, `node1-1`, `node1-2`... k3s's own `--vpn-auth` integration avoids exactly this by checking the Tailscale backend state first and skipping its own `tailscale up` call entirely once already `"Running"` (confirmed against `pkg/vpn/vpn.go`'s `StartVPN()`) - the manual pre-empt call needs the identical guard: check `tailscale ip -4` succeeds *before* calling `tailscale up` at all, only calling it when not already authenticated. - **Tailscale assigns an IP and wires up its own MagicDNS resolver as two separate steps, not atomically - a node having a real IP is not proof DNS resolution against another tailnet member is working yet.** A joining server needs to resolve the bootstrap host's own MagicDNS hostname (`K3S_JOIN_SERVER_URL`) before it can bootstrap against it at all. Confirmed live: k3s failed outright with `"dial tcp: lookup k3s-server.example.ts.net: no such host"` immediately after `tailscale ip -4` had already returned a real address for the *local* node. Fix: wait for a real `nslookup` success against the join server's own hostname specifically, not just local Tailscale readiness, before invoking k3s (the `k3s` service's entrypoint only does this when `K3S_JOIN_SERVER_URL` is actually set - the bootstrap host has nothing else to resolve). - **A single remaining healthy etcd server can get stuck indefinitely if another member is registered but permanently unreachable (e.g. a partial join that never completed), even though that member is a non-voting learner and shouldn't block quorum.** Symptom: `"failed to publish local member to cluster through raft"` with `context deadline exceeded`, repeating forever, alongside `kubectl` returning `503 the server is currently unable to handle the request`. The fix is k3s's own documented recovery mechanism, run once against the affected node: stop the container, then `k3s server --cluster-reset` (with the SAME flags the real service uses, including `--node-ip` set to the node's own persisted Tailscale IP - getting this wrong re-registers the single remaining member under the *wrong* address, e.g. the invoking container's own ad-hoc bridge IP, and the real service then fails to start with `"this server is a not a member of the etcd cluster"`) - this resets *etcd cluster membership* back to a single member using the *existing local data*, it does not wipe anything. It's a one-shot operation: k3s writes a guard file after a successful reset and refuses to run `--cluster-reset` again until it's manually deleted (`rm /var/lib/rancher/k3s/server/db/reset-flag`) if a second reset is genuinely needed. After a reset, restart the service normally (without the flag) and clean up the stale member's own now-orphaned Kubernetes `Node` object and `node-password` Secret before it rejoins fresh. -- **`ci.yml`'s `determine-runner` job is deliberately anchored on `exadev-runners` directly, not `ubuntu-latest`.** This repo's CI runs on ExaDev's own fleet, which this collection deploys, so it dogfoods the fleet fully rather than hedging on it — unlike a consuming repo (for example `ExaDev/spot-of-the-day`), where `ubuntu-latest` is the recommended anchor so a fully-down fleet can still fall back. The trade-off here is accepted, not a workaround: if the fleet is entirely down, `determine-runner` cannot schedule and CI cannot fall back to `ubuntu-latest` either. ## References From 9cf36f5d1effd22c656f51ca7c2e86d142b3e93c Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Fri, 25 Sep 2026 21:38:54 +0100 Subject: [PATCH 2/2] ci: always install the pinned Docker Compose for the mesh and Litestream tests ubuntu-latest ships its own Compose, and with it the mesh test's second role run recreated every node container. The tests now install the pinned version whatever the runner has, and print the engine version beside it. --- .github/workflows/litestream.yml | 9 +++------ .github/workflows/mesh-integration.yml | 9 +++------ 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/litestream.yml b/.github/workflows/litestream.yml index 4086155..22b5523 100644 --- a/.github/workflows/litestream.yml +++ b/.github/workflows/litestream.yml @@ -34,14 +34,11 @@ jobs: steps: - uses: actions/checkout@v7 - - name: Install Docker Compose if the runner lacks it - # The fleet's runner image ships the Docker CLI and buildx but not the Compose plugin. + - name: Install the pinned Docker Compose + # Always the pinned version, not whatever the runner has: ubuntu-latest's own Compose recreated every node container when the role ran a second time, which the pinned version does not. The user plugin directory takes precedence over the system one. run: | set -euo pipefail - if docker compose version >/dev/null 2>&1; then - docker compose version - exit 0 - fi + docker version --format 'Docker Engine {{.Server.Version}}' case "$(uname -m)" in x86_64) arch=x86_64 ;; aarch64 | arm64) arch=aarch64 ;; diff --git a/.github/workflows/mesh-integration.yml b/.github/workflows/mesh-integration.yml index 8b43389..9ac0151 100644 --- a/.github/workflows/mesh-integration.yml +++ b/.github/workflows/mesh-integration.yml @@ -37,14 +37,11 @@ jobs: steps: - uses: actions/checkout@v7 - - name: Install Docker Compose if the runner lacks it - # The fleet's runner image ships the Docker CLI and buildx but not the Compose plugin. + - name: Install the pinned Docker Compose + # Always the pinned version, not whatever the runner has: ubuntu-latest's own Compose recreated every node container when the role ran a second time, which the pinned version does not. The user plugin directory takes precedence over the system one. run: | set -euo pipefail - if docker compose version >/dev/null 2>&1; then - docker compose version - exit 0 - fi + docker version --format 'Docker Engine {{.Server.Version}}' case "$(uname -m)" in x86_64) arch=x86_64 ;; aarch64 | arm64) arch=aarch64 ;;