From 8f096022272db0dc7f47a802acae9f2ee9df9226 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Thu, 5 Nov 2026 14:51:31 +0100
Subject: [PATCH] ci(e2e): use the blank second disk of the 72 GB runners for
 images and work

Some hosted runners pair a 72 GB root disk with a blank 220 GB NVMe disk
that nothing mounts; a heavy leg on one ran out of disk with 16 GB free.
The leg now formats and mounts it at /mnt, and Docker's data root and the
work area go there.
---
 .github/workflows/e2e.yml | 52 +++++++++++++++++++++++++++++++++------
 docs/lessons-learned.md   |  5 ++++
 docs/testing.md           |  2 +-
 3 files changed, 51 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 1d1eb6f2..558e11f5 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -380,7 +380,35 @@ jobs:
           echo "$cpu"
           echo "Runner: ${cpu}" >> "$GITHUB_STEP_SUMMARY"
 
-      # Images go to /var/lib/docker on /; the work area goes to whichever of
+      # Some runners come with a 72 GB root disk and a second, blank NVMe
+      # disk: 220.7 GB with no partition and no file system on an
+      # ubuntu-26.04 runner (probe run 42037810203 on the geiresx fork); the
+      # 145 GB runners have none. On a 72 GB runner a heavy leg ran out of
+      # disk with 16 GB free before its cases (run 42018365546, shard 1),
+      # where the same cases need about 26 GB (run 41995719742, shard 1).
+      # A blank disk is formatted and mounted at /mnt, where the steps below
+      # put the work area and Docker's images.
+      - name: Mount a spare disk
+        run: |
+          if mountpoint -q /mnt; then echo "/mnt is already a mount"; df -h /mnt; exit 0; fi
+          disk=""
+          while read -r name type; do
+            [ "$type" = disk ] || continue
+            # Blank: no partition, no file system, nothing mounted.
+            [ "$(lsblk -nro NAME "$name" | wc -l)" -eq 1 ] || continue
+            [ -z "$(lsblk -ndro FSTYPE,MOUNTPOINT "$name" | tr -d ' ')" ] || continue
+            disk=$name
+          done < <(lsblk -dnpo NAME,TYPE)
+          if [ -z "$disk" ]; then echo "No blank disk on this runner"; exit 0; fi
+          echo "Formatting ${disk} ($(lsblk -ndo SIZE "$disk")) and mounting it at /mnt"
+          sudo mkfs.ext4 -q -F -E nodiscard "$disk"
+          sudo mkdir -p /mnt
+          sudo mount -o noatime "$disk" /mnt
+          df -h / /mnt
+          echo "Spare disk: ${disk}, $(df -h --output=avail /mnt | tail -1 | tr -d ' ') free at /mnt" >> "$GITHUB_STEP_SUMMARY"
+
+      # Images go to /var/lib/docker on / (on /mnt when /mnt is a spare
+      # disk, see "Docker image store"); the work area goes to whichever of
       # / and /mnt has more room, so the two do not compete for one disk. The
       # base outputs unpack only under the work area they were packed in, so
       # a shard leg takes the base leg's choice when its artifact arrives.
@@ -426,21 +454,31 @@ jobs:
       # 72 GB runner (48 GB free) ran out in the pre-pull (run 41938436657,
       # shards 2 and 5). Runs before any image is restored or pulled; the
       # images already in the containerd store are pruned first, as overlay2
-      # would not see them.
+      # would not see them. When /mnt is a spare disk with more room than /,
+      # Docker's data root moves there too.
       - name: Docker image store
         run: |
+          change=.
           if docker info -f '{{json .DriverStatus}}' | grep -q io.containerd.snapshotter; then
+            change+=' | .features["containerd-snapshotter"] = false'
+          fi
+          if mountpoint -q /mnt && [ "$(df --output=avail -k /mnt | tail -1)" -gt "$(df --output=avail -k / | tail -1)" ]; then
+            change+=' | .["data-root"] = "/mnt/docker"'
+          fi
+          if [ "$change" != . ]; then
             docker image prune -a -f > /dev/null
             f=/etc/docker/daemon.json
-            { [ -s "$f" ] && sudo cat "$f" || echo '{}'; } \
-              | jq '.features["containerd-snapshotter"] = false' | sudo tee "${f}.new" > /dev/null
+            { [ -s "$f" ] && sudo cat "$f" || echo '{}'; } | jq "$change" | sudo tee "${f}.new" > /dev/null
             sudo mv "${f}.new" "$f"
             sudo systemctl restart docker
           fi
-          driver=$(docker info -f '{{.Driver}}')
-          echo "Docker $(docker version -f '{{.Server.Version}}'), storage driver ${driver}" | tee -a "$GITHUB_STEP_SUMMARY"
+          driver=$(docker info -f '{{.Driver}}') root=$(docker info -f '{{.DockerRootDir}}')
+          echo "Docker $(docker version -f '{{.Server.Version}}'), storage driver ${driver}, data root ${root}" | tee -a "$GITHUB_STEP_SUMMARY"
           [ "$driver" = overlay2 ] || { echo "::error::Docker's storage driver is ${driver}, not overlay2."; exit 1; }
-          df -h /
+          if mountpoint -q /mnt && [[ "$change" == *data-root* ]] && [ "$root" != /mnt/docker ]; then
+            echo "::error::Docker's data root is ${root}, not /mnt/docker on the spare disk."; exit 1
+          fi
+          df -h / /mnt
 
       # Images are cached in four entries, one per group of
       # `scripts/ci/e2e-run.sh --image-groups`, each keyed on its own image
diff --git a/docs/lessons-learned.md b/docs/lessons-learned.md
index e5d81388..3594bf8a 100644
--- a/docs/lessons-learned.md
+++ b/docs/lessons-learned.md
@@ -615,6 +615,11 @@ Most bioinformatics containers run as non-root users. If writing to bind-mounted
 - **Why:** `ubuntu-latest` hands out runners of two sizes, 145 GB and 72 GB (48 GB free after the clean-up step), and on the ubuntu-26.04 image Docker 29 keeps images in containerd's store, which holds each layer as a compressed blob and as an unpacked copy: DeepVariant's, GATK's and VNtyper's images took 24.8 GB there against 17.6 GB on overlay2. In that store `docker image inspect -f '{{.Size}}'` gives the compressed size (7.2 GB for the same three), so the image table understated what the disk held. Every leg also pre-pulled all 56 case images, the five largest included, whatever its cases ran.
 - **Fix:** each e2e leg turns `containerd-snapshotter` off in `/etc/docker/daemon.json` and restarts Docker before it restores an image, so images go to overlay2; the five largest images are pulled at a case's first `docker run` of them (`tests/e2e/bin/docker`) and removed when the case ends; the image cache keys moved to `e2e-images-ovl-`, because 7 of the 46 archives saved from containerd's store did not load into overlay2; and `e2e-run.sh` fails a leg with less than `E2E_MIN_FREE_GB` free before its first case, so a short disk shows as one clear error instead of a failure in a pull or a case.
 
+### A 72 GB runner hides a blank 220 GB disk
+- **Failed:** shard 1 of #297's run 42018365546 had 16 GB free before its cases, where main's run of the same cases had 89 GB free and used 26 GB at peak. DeepVariant's lazy pull inside the Nextflow run then failed with "no space left on device", and four cases failed.
+- **Why:** `ubuntu-latest` gives some legs a runner whose root disk is 72 GB. `/mnt` is a plain directory on that disk, so the work area pick found no second disk. The runner does have one: a blank NVMe disk (220.7 GB, no partition, no file system, nothing mounted), seen with `lsblk` in probe run 42037810203.
+- **Fix:** the leg's first step formats a blank whole disk and mounts it at `/mnt`; the work-area pick then takes `/mnt`, and the "Docker image store" step sets Docker's `data-root` to `/mnt/docker`.
+
 ## Downloads (2026-11)
 
 ### One bad transfer of a large file failed the whole setup
diff --git a/docs/testing.md b/docs/testing.md
index 7c323dc2..4e7ec738 100644
--- a/docs/testing.md
+++ b/docs/testing.md
@@ -94,7 +94,7 @@ How long it took in one job, before the split: a passing e2e job took about 105
 
 Before the first case of a full run, and of every leg, `e2e-run.sh` pulls every image the cases use that is not on the machine yet: the `NAME_IMAGE` lines of `versions.env` and of the case files, less a list of images no case runs. A pull that fails with a registry 5xx is tried again, up to four tries in all, 20, 40 and 80 seconds apart. Any other error, such as a missing tag or a rate limit, fails the job at once, and the tools' own `docker run` is never retried. A Docker Hub 500 on one image's manifest is what failed main's run 37980248545 the first time. `scripts/ci/e2e-run.sh --self-test-pull` checks the retry against a fake docker, and `e2e-plan` runs it. A run given a pattern skips the pre-pull, because most images reach a case through the steps it runs and cannot be picked per case; its cases pull their own images, with one try each. The five largest images (DeepVariant, PharmCAT, GATK, MEGAnE and VNtyper, 1.9 to 6.0 GB each to download, none of them in the image cache) are left out of the pre-pull: the docker shim `tests/e2e/bin/docker` pulls one, with the same retry, when a case first runs it, and removes it when that case ends, so a leg holds at most one of them at a time. The image cache keys start with `e2e-images-ovl-`: archives saved from containerd's store did not all load into overlay2.
 
-Each leg switches Docker to the classic overlay2 image store before it restores or pulls an image. Docker 29 on the ubuntu-26.04 runner image uses containerd's store, which keeps every layer twice (the compressed blob and the unpacked copy): DeepVariant's, GATK's and VNtyper's images took 24.8 GB there and 17.6 GB on overlay2, and the legs that landed on a 72 GB runner (48 GB free) ran out of disk in the pre-pull (run 41938436657). Before its first case `e2e-run.sh` fails the leg when less than `E2E_MIN_FREE_GB` is free under the work area (10 GB in `e2e.yml`), and after the cases it prints the least free space it saw while they ran.
+Each leg switches Docker to the classic overlay2 image store before it restores or pulls an image. Docker 29 on the ubuntu-26.04 runner image uses containerd's store, which keeps every layer twice (the compressed blob and the unpacked copy): DeepVariant's, GATK's and VNtyper's images took 24.8 GB there and 17.6 GB on overlay2, and the legs that landed on a 72 GB runner (48 GB free) ran out of disk in the pre-pull (run 41938436657). Before its first case `e2e-run.sh` fails the leg when less than `E2E_MIN_FREE_GB` is free under the work area (10 GB in `e2e.yml`), and after the cases it prints the least free space it saw while they ran. Some hosted runners have a 72 GB root disk (about 50 GB free after the clean-up) and a second, blank NVMe disk (220.7 GB on the one we measured); the 145 GB runners have none. Each leg formats such a disk and mounts it at `/mnt` before anything else, and then Docker's data root (`/mnt/docker`) and, on the base leg, the work area go there. A shard's work area follows the base leg's, so on a shard only the images move. A 72 GB runner without a spare disk still has only about 16 GB free before a shard's cases, which the heaviest legs do not fit (they used 26 to 33 GB); the `E2E_MIN_FREE_GB` gate stops only the legs below 10 GB.
 
 What it covers: