diff --git a/.github/actions/docker-host-gc/action.yml b/.github/actions/docker-host-gc/action.yml new file mode 100644 index 000000000..80b03ca6f --- /dev/null +++ b/.github/actions/docker-host-gc/action.yml @@ -0,0 +1,82 @@ +name: Docker host GC +description: > + Reclaim space on the shared remote Docker host. Runs at the end of every + workflow that stands up testcontainers, so the host is swept many times a day + rather than once a night. + +# Why this is a per-run step and not a scheduled janitor: the host is shared by +# every concurrent CI run, and it fills during the working day. A nightly cron +# cannot help a run that dies at 16:03 — by the time the sweep fires the damage +# is twelve hours old and several runs have already gone red. Sweeping after +# each run keeps the quota from ever being approached. +# +# Everything here is safe to run WHILE other runs are in flight, which is the +# constraint that shapes every filter below: +# - volumes: only unreferenced ones are touched, and a live container's +# volumes are by definition referenced. +# - images: `until` protects anything pulled recently, including an image a +# concurrent run has pulled but not yet started a container from. +# - build cache / networks: age- and size-guarded for the same reason. + +inputs: + image-min-age: + description: Only prune unused images older than this (docker `until` filter). + required: false + default: 48h + builder-keep-storage: + description: Build cache to retain as a warm cache. + required: false + default: 20GB + +runs: + using: composite + steps: + - name: Docker disk usage before GC + shell: bash + run: docker system df || true + + # Volumes FIRST, and every step non-fatal. On a host already at its quota + # even image *deletion* fails, because containerd needs a meta.db write to + # do it — while volume removal frees the same filesystem without one. The + # 2026-07 incident hit exactly this: image prunes erroring out while + # thousands of orphaned testcontainer volumes held the bulk of the usage. + # Free the bulk first and the rest starts working again. + - name: Remove orphaned anonymous volumes + shell: bash + run: | + # Deliberately NOT `docker volume prune -af`. This host also runs + # long-lived services, and `-a` includes *named* volumes — so a prune + # racing a service redeploy, at the moment its named volume is briefly + # unreferenced, would delete that service's data. There is no undo. + # + # Anonymous volumes (the only kind testcontainers leaks) are named with + # a 64-char hex string; named volumes never match that shape. Matching + # on it removes exactly the leak and structurally cannot touch anything + # a human named. This is also version-independent: the meaning of + # `volume prune` without `-a` changed in Docker 23, the hex shape did not. + # + # `|| true` guards the pipeline because `grep` exits 1 on no matches and + # these steps run under `set -o pipefail`. + before=$(docker volume ls -qf dangling=true | grep -cE '^[0-9a-f]{64}$' || true) + docker volume ls -qf dangling=true \ + | grep -E '^[0-9a-f]{64}$' \ + | xargs -r docker volume rm -f >/dev/null 2>&1 || true + after=$(docker volume ls -qf dangling=true | grep -cE '^[0-9a-f]{64}$' || true) + echo "Orphaned anonymous volumes: ${before} before, ${after} after." + # A volume still in use by a concurrently starting container refuses + # removal; that is the intended outcome, not an error worth failing on. + + - name: Prune stale images, build cache and networks + shell: bash + run: | + # Age filters keep a warm pull cache for the heavy images the nightly + # suites depend on (Playwright, Elasticsearch, Postgres, Mongo). Without + # them every run would re-pull gigabytes and get slower, trading one + # problem for another. + docker image prune -af --filter "until=${{ inputs.image-min-age }}" || true + docker builder prune -af --keep-storage "${{ inputs.builder-keep-storage }}" || true + docker network prune -f --filter "until=6h" || true + + - name: Docker disk usage after GC + shell: bash + run: docker system df || true diff --git a/.github/workflows/ci-extended.yml b/.github/workflows/ci-extended.yml index 2115dc971..76bbc9eba 100644 --- a/.github/workflows/ci-extended.yml +++ b/.github/workflows/ci-extended.yml @@ -48,6 +48,11 @@ jobs: env: MAX_CONTAINER_AGE_MIN: 90 steps: + - name: Check out the Docker host GC action + uses: actions/checkout@v5 + with: + sparse-checkout: .github/actions + - name: Remove testcontainers from this run run: | docker ps -aq --filter "label=github.run_id=${{ github.run_id }}" | xargs -r docker rm -fv || true @@ -69,3 +74,9 @@ jobs: fi done echo "Swept ${reaped} orphaned testcontainer(s)." + + # See ci.yml — same sweep, so a nightly extended run cannot leave the + # shared host dirtier than it found it. + - name: GC the Docker host + if: always() + uses: ./.github/actions/docker-host-gc diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01868978b..c147946ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -306,6 +306,14 @@ jobs: # killed mid-test. MAX_CONTAINER_AGE_MIN: 90 steps: + # Only the composite action is needed here, not the tree — a cone-mode + # sparse checkout keeps this job as cheap as it was before it had any + # checkout at all. + - name: Check out the Docker host GC action + uses: actions/checkout@v5 + with: + sparse-checkout: .github/actions + - name: Reap this run's testcontainers run: | # testcontainers-rs removes a container only from `Drop for @@ -349,6 +357,14 @@ jobs: done echo "Swept ${reaped} orphaned testcontainer(s)." + # Last, so it runs after the reaps above have released this run's volumes. + # `ci.yml` fires on every push and PR, which makes this the sweep that + # actually keeps the shared host below its quota — the other workflows + # carry it too, but far less often. + - name: GC the Docker host + if: always() + uses: ./.github/actions/docker-host-gc + test-python: name: Test Python runs-on: [self-hosted] diff --git a/.github/workflows/hts.yml b/.github/workflows/hts.yml index 18dd77474..4adaf62b4 100644 --- a/.github/workflows/hts.yml +++ b/.github/workflows/hts.yml @@ -690,6 +690,11 @@ jobs: # it can never race a concurrent hts run (e.g. nightly + a release tag). MAX_CONTAINER_AGE_MIN: 90 steps: + - name: Check out the Docker host GC action + uses: actions/checkout@v5 + with: + sparse-checkout: .github/actions + - name: Sweep orphaned testcontainers from dead runs run: | # testcontainers-rs unconditionally applies @@ -709,3 +714,9 @@ jobs: fi done echo "Swept ${reaped} orphaned testcontainer(s)." + + # This workflow runs on its own nightly cron and cannot rely on ci.yml's + # sweep (see the job comment above), so it carries the GC itself. + - name: GC the Docker host + if: always() + uses: ./.github/actions/docker-host-gc diff --git a/.github/workflows/ui-tests-matrix.yml b/.github/workflows/ui-tests-matrix.yml index a75c2a78d..c1dc97149 100644 --- a/.github/workflows/ui-tests-matrix.yml +++ b/.github/workflows/ui-tests-matrix.yml @@ -43,6 +43,15 @@ jobs: - name: Checkout code uses: actions/checkout@v5 + # GC *before* the matrix rather than after it. This suite pulls the + # multi-GB Playwright image on top of Elasticsearch, Postgres and Mongo, + # so it is the workflow most likely to be the one that finally tips the + # host over its quota — it needs the headroom up front, not afterwards. + # `build` runs once and gates the whole matrix, which makes it the right + # place: one sweep, before anything heavy is pulled. + - name: GC the Docker host + uses: ./.github/actions/docker-host-gc + - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable with: