From 5ff03f4bb88b773797fe27442a56e9bd80afe8e8 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Thu, 5 Nov 2026 03:37:05 +0100
Subject: [PATCH 1/5] ci(e2e): keep every e2e leg inside the runner's disk

Docker's containerd store on ubuntu-26.04 keeps each layer twice, and every
leg pre-pulled all 56 images; legs on 72 GB runners ran out of disk in the
pull. Legs now switch Docker to overlay2 first, the five largest images are
pulled at their first use, and a leg with too little free disk fails before
its first case.
---
 .github/workflows/e2e.yml |  28 ++++++++++
 docs/lessons-learned.md   |   5 ++
 docs/testing.md           |   4 +-
 scripts/ci/e2e-run.sh     | 112 ++++++++++++++++++++++++++++++++++++--
 tests/e2e/bin/docker      |  11 ++++
 5 files changed, 155 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index a7ed8d1b..c0b43792 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -417,6 +417,30 @@ jobs:
         with:
           version: "26.10.5"
 
+      # Docker 29 on the ubuntu-26.04 image keeps images in containerd's
+      # store, which holds each layer twice: the compressed or saved blob in
+      # its content store and the unpacked snapshot. There the 56 images of
+      # the cases and the layout took 91 GB of disk for 33 GB of image size
+      # (run 41938436657, shard 3), more than a 72 GB runner has free.
+      # The classic overlay2 store keeps the unpacked layers only (the 44
+      # cached images took 20 GB on an ubuntu-24.04 runner, run 41947119567).
+      # Runs before any image is restored or pulled; the images already in
+      # the containerd store are pruned first, as overlay2 would not see them.
+      - name: Docker image store
+        run: |
+          if docker info -f '{{json .DriverStatus}}' | grep -q io.containerd.snapshotter; then
+            docker image prune -a -f > /dev/null
+            f=/etc/docker/daemon.json
+            { [ -s "$f" ] && sudo cat "$f" || echo '{}'; } \
+              | jq '.features["containerd-snapshotter"] = false' | sudo tee "${f}.new" > /dev/null
+            sudo mv "${f}.new" "$f"
+            sudo systemctl restart docker
+          fi
+          driver=$(docker info -f '{{.Driver}}')
+          echo "Docker $(docker version -f '{{.Server.Version}}'), storage driver ${driver}" | tee -a "$GITHUB_STEP_SUMMARY"
+          [ "$driver" = overlay2 ] || { echo "::error::Docker's storage driver is ${driver}, not overlay2."; exit 1; }
+          df -h /
+
       # Images are cached in four entries, one per group of
       # `scripts/ci/e2e-run.sh --image-groups`, each keyed on its own image
       # list: a version bump misses one group's key, and that group restores
@@ -483,6 +507,8 @@ jobs:
                 [ -f "$f" ] || continue
                 if zstd -dc "$f" | docker load -q > /dev/null; then echo "${f##*/}" >> "$loaded"
                 else echo "::warning::${f##*/} did not load; e2e-run.sh pulls it"; fi
+                # Loaded, the archive only takes disk.
+                rm -f "$f"
               done
             ) &
           done
@@ -588,6 +614,8 @@ jobs:
           E2E_CASES: ${{ steps.cases-list.outputs.list }}
           E2E_JOB: ${{ matrix.job == 'base' && 'base' || format('shard {0}', matrix.job) }}
           E2E_TRACE: ${{ needs.e2e-plan.outputs.trace }}
+          # The free disk a leg needs before its first case (e2e-run.sh).
+          E2E_MIN_FREE_GB: 10
           E2E_BASE_OUT: ${{ matrix.job == 'base' && format('{0}/base.tar.zst', runner.temp) || '' }}
           E2E_BASE_IN: ${{ matrix.job != 'base' && format('{0}/e2e-base/base.tar.zst', runner.temp) || '' }}
           # Keep the downloads to the end of the job only when a save step uses them.
diff --git a/docs/lessons-learned.md b/docs/lessons-learned.md
index b121b943..2bd71692 100644
--- a/docs/lessons-learned.md
+++ b/docs/lessons-learned.md
@@ -610,6 +610,11 @@ Most bioinformatics containers run as non-root users. If writing to bind-mounted
 - **Fix:** `tests/e2e/mei-1.sh` removes its whole sample directory and the image by its id when it ends, and logs the free disk before and after. With the image gone every case passed. The image is pinned by digest, so the e2e image cache never saves it anyway.
 - **Image sizes step:** the job's "Image sizes" step then stopped with `numfmt: invalid number: ''`. For an image that is gone, `docker image inspect -f` prints an empty line before the `|| echo 0` fallback, so the step's own "not here" branch never ran. The fix in `.github/workflows/e2e.yml` reads the size into a variable first (`size=$(docker image inspect ...) || size=0`) and prints `${size:-0}`.
 
+### Docker's containerd image store on ubuntu-26.04 doubles the e2e images on disk
+- **Failed:** shards 2 and 5 of main's run 41938436657 stopped in the pre-pull with "no space left on device" under `/var/lib/containerd`, while pulling MEGAnE's image. The same commit passed on other legs.
+- **Why:** `ubuntu-latest` hands out runners of two sizes, 145 GB and 72 GB (48 GB free after the clean-up step), and on the ubuntu-26.04 image Docker 29 keeps images in containerd's store, which holds each layer as a blob and as an unpacked copy. The 56 case images (33 GB of image size) and the layout took 91 GB there. Every leg also pre-pulled all of them, the five largest included, whatever its cases ran.
+- **Fix:** each e2e leg turns `containerd-snapshotter` off in `/etc/docker/daemon.json` and restarts Docker before it restores an image, so images go to overlay2; the five largest images are pulled at a case's first `docker run` of them (`tests/e2e/bin/docker`); and `e2e-run.sh` fails a leg with less than `E2E_MIN_FREE_GB` free before its first case, so a short disk shows as one clear error instead of a failure in a pull or a case.
+
 ## Downloads (2026-11)
 
 ### One bad transfer of a large file failed the whole setup
diff --git a/docs/testing.md b/docs/testing.md
index 08c171cd..8b947f9d 100644
--- a/docs/testing.md
+++ b/docs/testing.md
@@ -92,7 +92,9 @@ Two downloads are cached the same way, because they come from EBI's 1000 Genomes
 
 How long it took in one job, before the split: a passing e2e job took about 105 to 205 minutes with 84 cases, and 230 minutes of cases with 130 (run 41390434697, 2026-11-02). The runner decides much of it, not the cache: the same CPU model took 106 minutes in one run and 166 in another. In October 2026, with 84 cases, the same code took 106 and 166 minutes on an AMD EPYC 9V45 (runs 39197502832 and 39075176392, the second of which also saved the cache), 112 minutes on an Intel Xeon 6973P (run 39088086725), 156 and 191 minutes on an AMD EPYC 7763 (runs 39075764278 and 39075201465) and 205 minutes on an AMD EPYC 9V74 (run 39197512594, which saved three groups). The image cache saves about 7 minutes of that: pulling every image took 9 minutes, restoring and loading the four groups took 2.5 (run 39075201465 restored 39 images in 118 seconds), and saving them all took 12. Each leg has a limit of 240 minutes (355 for a traced shard leg); the base leg waits up to 75 minutes for a fixture a pull request bumps. Each leg's first steps print the runner's CPU model and CPU count to the log and the job summary, so a later timing can name the hardware.
 
-Before the first case of a full run, and of every leg, `e2e-run.sh` pulls every image the cases use that is not on the machine yet: the `NAME_IMAGE` lines of `versions.env` and of the case files, less a list of images no case runs. A pull that fails with a registry 5xx is tried again, up to four tries in all, 20, 40 and 80 seconds apart. Any other error, such as a missing tag or a rate limit, fails the job at once, and the tools' own `docker run` is never retried. A Docker Hub 500 on one image's manifest is what failed main's run 37980248545 the first time. `scripts/ci/e2e-run.sh --self-test-pull` checks the retry against a fake docker, and `e2e-plan` runs it. A run given a pattern skips the pre-pull, because most images reach a case through the steps it runs and cannot be picked per case; its cases pull their own images, with one try each.
+Before the first case of a full run, and of every leg, `e2e-run.sh` pulls every image the cases use that is not on the machine yet: the `NAME_IMAGE` lines of `versions.env` and of the case files, less a list of images no case runs. A pull that fails with a registry 5xx is tried again, up to four tries in all, 20, 40 and 80 seconds apart. Any other error, such as a missing tag or a rate limit, fails the job at once, and the tools' own `docker run` is never retried. A Docker Hub 500 on one image's manifest is what failed main's run 37980248545 the first time. `scripts/ci/e2e-run.sh --self-test-pull` checks the retry against a fake docker, and `e2e-plan` runs it. A run given a pattern skips the pre-pull, because most images reach a case through the steps it runs and cannot be picked per case; its cases pull their own images, with one try each. The five largest images (DeepVariant, PharmCAT, GATK, MEGAnE and VNtyper, 15 GB together, none of them in the image cache) are left out of the pre-pull: the docker shim `tests/e2e/bin/docker` pulls one, with the same retry, when a case first runs it, so a leg holds only those its own cases use.
+
+Each leg switches Docker to the classic overlay2 image store before it restores or pulls an image. Docker 29 on the ubuntu-26.04 runner image uses containerd's store, which keeps every layer twice (the blob and the unpacked copy): there the case images and the layout took 91 GB for 33 GB of image size (run 41938436657, shard 3), and the legs that landed on a 72 GB runner ran out of disk in the pre-pull. Before its first case `e2e-run.sh` fails the leg when less than `E2E_MIN_FREE_GB` is free under the work area (10 GB in `e2e.yml`), and after the cases it prints the least free space it saw while they ran.
 
 What it covers:
 
diff --git a/scripts/ci/e2e-run.sh b/scripts/ci/e2e-run.sh
index d47807a5..f74bc162 100755
--- a/scripts/ci/e2e-run.sh
+++ b/scripts/ci/e2e-run.sh
@@ -3,6 +3,7 @@
 #
 # Usage: scripts/ci/e2e-run.sh [pattern]
 #        scripts/ci/e2e-run.sh --self-test-pull | --image-groups | --pull-images
+#        scripts/ci/e2e-run.sh --pull-image REF
 #   pattern  optional shell glob; runs only the cases whose file name matches
 #            (e.g. '2*' or '*cyrius*'). Later cases read earlier cases' outputs,
 #            so a partial run is for debugging only.
@@ -11,6 +12,8 @@
 #   --image-groups  prints "GROUP IMAGE" for every image the cases use (the
 #            e2e workflow's image cache entries) and exits.
 #   --pull-images  pulls those images, as step 3 below does, and exits.
+#   --pull-image REF  pulls one image with the same retry (the docker shim's
+#            pulls of the lazy images at their first use), and exits.
 #
 # Env: E2E_WORK  work area (default ${RUNNER_TEMP:-/tmp}/e2e-work), about 25 GB
 #      GH_TOKEN  token for `gh release download` (the job's GITHUB_TOKEN)
@@ -18,6 +21,8 @@
 #                full run (pre-pull, all-cases-ran marker). The e2e workflow
 #                gives each of its parallel jobs its list from e2e-shards.sh.
 #      E2E_JOB   the job name written beside each result (default local)
+#      E2E_MIN_FREE_GB  fail before the first case when less is free under
+#                E2E_WORK (default 0)
 #      E2E_BASE_OUT  after the cases, pack every file they wrote or changed
 #                (under E2E_WORK and the checkout) into this .tar.zst, and
 #                write "E2E_WORK CHECKOUT" to E2E_BASE_OUT.from
@@ -41,7 +46,9 @@
 #      for the build-fixture job to publish it;
 #   2. lays out GENOME_DIR the way setup.sh and step 13 would leave it;
 #   3. pulls every image the cases use that is not on the machine yet,
-#      trying a pull again only when the registry answered with a 5xx;
+#      trying a pull again only when the registry answered with a 5xx (the
+#      five largest wait for the first case that runs them), and checks the
+#      free disk against E2E_MIN_FREE_GB;
 #   4. runs tests/e2e/*.sh in name order (C locale: numbered cases first, then
 #      the <package-key>-*.sh cases later packages add), or those of E2E_CASES,
 #      and keeps going after a failure, so one run lists every broken step;
@@ -112,6 +119,22 @@ case_images() {
       done | awk '!seen[$0]++'
 }
 
+# Images pulled at first use, not before the cases: the five largest (1.9 to
+# 6.0 GB each, 15 GB together in run 41938436657), which no image cache entry
+# holds (pinned by digest or over the group budget) and which only some legs
+# run. Pulled up front, every leg carried all five. A `docker run` or
+# `docker create` of one that is missing pulls it first, with the retry above
+# (tests/e2e/bin/docker reads E2E_LAZY_IMAGES).
+LAZY_IMAGES=" DEEPVARIANT_IMAGE PHARMCAT_IMAGE GATK_IMAGE MEGANE_IMAGE VNTYPER_IMAGE "
+# lazy_images: the references of LAZY_IMAGES, from versions.env.
+lazy_images() {
+  grep -hoE '^[A-Z0-9_]+_IMAGE="[^"]+"' "${REPO}/versions.env" \
+    | while IFS='=' read -r name ref; do
+        [[ "$LAZY_IMAGES" == *" ${name} "* ]] && echo "${ref//\"/}"
+      done
+  return 0
+}
+
 # --- Image cache groups -------------------------------------------------------
 # The e2e workflow caches the images of case_images in IMAGE_GROUPS cache
 # entries, so no single entry, and not their sum, reaches the repository's
@@ -136,17 +159,20 @@ if [ "${1:-}" = --image-groups ]; then
   exit 0
 fi
 
-# pull_all: pull every image of case_images that is not here yet.
+# pull_all: pull every image of case_images that is not here yet, but the
+# lazy ones.
 pull_all() {
-  local image pulled=0
+  local image pulled=0 lazy
   mapfile -t IMAGES < <(case_images)
   [ "${#IMAGES[@]}" -gt 0 ] || { echo "ERROR: no NAME_IMAGE line found in versions.env" >&2; return 1; }
+  lazy=$(lazy_images)
   for image in "${IMAGES[@]}"; do
+    grep -qxF "$image" <<< "$lazy" && continue
     docker image inspect "$image" > /dev/null 2>&1 && continue
     pull_image "$image" || return 1
     pulled=$((pulled + 1))
   done
-  echo "${#IMAGES[@]} images: $(( ${#IMAGES[@]} - pulled )) already here, ${pulled} pulled"
+  echo "${#IMAGES[@]} images: $(( ${#IMAGES[@]} - pulled - $(wc -l <<< "$lazy") )) already here, ${pulled} pulled, $(wc -l <<< "$lazy") left to their first use"
 }
 
 # The e2e workflow's shard legs pull while the base leg runs.
@@ -154,6 +180,26 @@ if [ "${1:-}" = --pull-images ]; then
   pull_all
   exit
 fi
+# --pull-image REF: one pull with the retry (the docker shim's lazy pulls).
+if [ "${1:-}" = --pull-image ]; then
+  pull_image "${2:?--pull-image needs an image}"
+  exit
+fi
+
+# --- Free disk ------------------------------------------------------------------
+# The legs ran out of disk pulling images on 2026-11-05 (run 41938436657). Before
+# the cases a leg needs E2E_MIN_FREE_GB free under E2E_WORK: room for what its
+# cases write and for the lazy images they pull.
+# disk_check NEED_GB: prints the free space; fails when it is under NEED_GB.
+disk_check() {
+  local free
+  free=$(( $(df --output=avail -k "$E2E_WORK" | tail -1) / 1048576 ))
+  echo "Free disk before the cases: ${free} GB under ${E2E_WORK} (the leg needs ${1} GB)"
+  if [ "$free" -lt "$1" ]; then
+    echo "ERROR: ${free} GB free under ${E2E_WORK}, under the ${1} GB the cases need." >&2
+    return 1
+  fi
+}
 
 if [ "${1:-}" = --self-test-pull ]; then
   T=$(mktemp -d)
@@ -223,6 +269,48 @@ FAKE
   else
     echo "FAIL: the cache groups do not hold the pull list once each"; st_failed=1
   fi
+  # Lazy images: each one a case image, five in all.
+  LAZY=$(lazy_images)
+  if [ "$(wc -l <<< "$LAZY")" -eq 5 ] && ! grep -qvxF -f <(echo "$LIST") <<< "$LAZY"; then
+    echo "ok: the 5 lazy images are case images"
+  else
+    echo "FAIL: the lazy images are not 5 case images: ${LAZY//$'\n'/ }"; st_failed=1
+  fi
+  # disk_check against a fake df: 10 GB free fails a need of 11 and passes 10.
+  # shellcheck disable=SC2016  # the fake df expands FAKE_AVAIL_K when it runs
+  printf '#!/usr/bin/env bash\nprintf "Avail\\n%%s\\n" "$FAKE_AVAIL_K"\n' > "${T}/df"
+  chmod +x "${T}/df"
+  E2E_WORK=$T
+  if FAKE_AVAIL_K=$((10 * 1048576)) disk_check 11 > /dev/null 2>&1; then
+    echo "FAIL: 10 GB free passed a need of 11 GB"; st_failed=1
+  else echo "ok: 10 GB free fails a need of 11 GB"; fi
+  if FAKE_AVAIL_K=$((10 * 1048576)) disk_check 10 > /dev/null 2>&1; then echo "ok: 10 GB free passes a need of 10 GB"
+  else echo "FAIL: 10 GB free failed a need of 10 GB"; st_failed=1; fi
+  # The docker shim: a run of a missing lazy image pulls it once, then runs;
+  # a run of any other image only runs.
+  mkdir -p "${T}/real"
+  cat > "${T}/real/docker" <<'FAKE'
+#!/usr/bin/env bash
+echo "$*" >> "${FAKE_DIR}/calls"
+case "$1 ${2:-}" in
+  "image inspect") awk -v i="${*: -1}" '$1 == "pull" && $NF == i {f = 1} END {exit !f}' "${FAKE_DIR}/calls" ;;
+  *) exit 0 ;;
+esac
+FAKE
+  chmod +x "${T}/real/docker"
+  : > "${T}/calls"
+  PATH="${REPO}/tests/e2e/bin:${T}/real:${PATH}" E2E_LAZY_IMAGES="big/one:1 big/two:2" \
+    docker run --rm -v /a:/a big/two:2 tool --x > /dev/null 2>&1
+  PATH="${REPO}/tests/e2e/bin:${T}/real:${PATH}" E2E_LAZY_IMAGES="big/one:1 big/two:2" \
+    docker run --rm big/two:2 tool > /dev/null 2>&1
+  PATH="${REPO}/tests/e2e/bin:${T}/real:${PATH}" E2E_LAZY_IMAGES="big/one:1 big/two:2" \
+    docker run --rm small/x:1 tool > /dev/null 2>&1
+  if [ "$(grep -c '^pull ' "${T}/calls")" = 1 ] && grep -qx 'pull -q big/two:2' "${T}/calls" \
+     && [ "$(grep -c '^run ' "${T}/calls")" = 3 ]; then
+    echo "ok: the docker shim pulls a missing lazy image once, before its run"
+  else
+    echo "FAIL: the docker shim's lazy pulls:"; sed 's/^/  /' "${T}/calls"; st_failed=1
+  fi
   exit "$st_failed"
 fi
 
@@ -317,9 +405,22 @@ fi
 if [ "$PATTERN" = "*" ]; then
   echo "=== Pulling the images the cases use ==="
   pull_all
+  E2E_LAZY_IMAGES=$(lazy_images | paste -sd' ' -)
+  export E2E_LAZY_IMAGES
 else
   echo "=== Partial run ('${PATTERN}'): no pre-pull, each case pulls what it uses ==="
 fi
+docker system df
+disk_check "${E2E_MIN_FREE_GB:-0}"
+# The least free space while the cases run, sampled every 10 s, for the report.
+LOW_FILE="${REPORT_DIR}/disk-low"
+df --output=avail -k "$E2E_WORK" | tail -1 > "$LOW_FILE"
+( while sleep 10; do
+    a=$(df --output=avail -k "$E2E_WORK" | tail -1)
+    [ "$a" -lt "$(cat "$LOW_FILE")" ] && echo "$a" > "$LOW_FILE"
+  done ) &
+LOW_PID=$!
+trap 'kill "$LOW_PID" 2> /dev/null || true' EXIT
 
 # --- Downloads kept between runs ----------------------------------------------
 # The two largest downloads of the cases come from EBI's 1000 Genomes server,
@@ -494,6 +595,9 @@ if [ "${#names[@]}" -eq 0 ]; then
 fi
 # Tells the workflow every case ran, so the pulled images are complete enough to cache.
 [ "$PATTERN" = "*" ] && touch "${E2E_WORK}/all-cases-ran"
+kill "$LOW_PID" 2> /dev/null || true
+echo "Disk (${JOB}): $(( $(cat "$LOW_FILE") / 1048576 )) GB free at the lowest during the cases, $(( $(df --output=avail -k "$E2E_WORK" | tail -1) / 1048576 )) GB after them" | tee -a "$SUMMARY"
+docker system df
 
 # --- 5. Report ----------------------------------------------------------------
 {
diff --git a/tests/e2e/bin/docker b/tests/e2e/bin/docker
index 46fce0d9..2119cd8f 100755
--- a/tests/e2e/bin/docker
+++ b/tests/e2e/bin/docker
@@ -15,6 +15,17 @@ for d in "${path_dirs[@]}"; do
 done
 [ -n "$real" ] || { echo "docker shim: no real docker on PATH" >&2; exit 127; }
 
+# E2E_LAZY_IMAGES (set by scripts/ci/e2e-run.sh): images pulled at first use.
+# A `docker run` or `docker create` of one that is not here pulls it first,
+# with e2e-run.sh's retry on a registry 5xx.
+if [ -n "${E2E_LAZY_IMAGES:-}" ] && { [ "${1:-}" = run ] || [ "${1:-}" = create ]; }; then
+  for a in "$@"; do
+    [[ " ${E2E_LAZY_IMAGES} " == *" ${a} "* ]] || continue
+    "$real" image inspect "$a" > /dev/null 2>&1 && continue
+    "$(dirname "$self_dir")/../../scripts/ci/e2e-run.sh" --pull-image "$a" >&2 || exit $?
+  done
+fi
+
 max=$(nproc)
 clamp() { awk -v v="$1" -v m="$max" 'BEGIN {print (v + 0 > m) ? m : v}'; }
 

From 4f760528468d238c60fca7acd1ce1e1cfb618b87 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Thu, 5 Nov 2026 03:57:18 +0100
Subject: [PATCH 2/5] test(e2e): name the shim self-test's fake images without
 a tag

check-images.sh reads name:tag literals as image references that bypass
versions.env.
---
 scripts/ci/e2e-run.sh | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/scripts/ci/e2e-run.sh b/scripts/ci/e2e-run.sh
index f74bc162..bf70d981 100755
--- a/scripts/ci/e2e-run.sh
+++ b/scripts/ci/e2e-run.sh
@@ -299,13 +299,13 @@ esac
 FAKE
   chmod +x "${T}/real/docker"
   : > "${T}/calls"
-  PATH="${REPO}/tests/e2e/bin:${T}/real:${PATH}" E2E_LAZY_IMAGES="big/one:1 big/two:2" \
-    docker run --rm -v /a:/a big/two:2 tool --x > /dev/null 2>&1
-  PATH="${REPO}/tests/e2e/bin:${T}/real:${PATH}" E2E_LAZY_IMAGES="big/one:1 big/two:2" \
-    docker run --rm big/two:2 tool > /dev/null 2>&1
-  PATH="${REPO}/tests/e2e/bin:${T}/real:${PATH}" E2E_LAZY_IMAGES="big/one:1 big/two:2" \
-    docker run --rm small/x:1 tool > /dev/null 2>&1
-  if [ "$(grep -c '^pull ' "${T}/calls")" = 1 ] && grep -qx 'pull -q big/two:2' "${T}/calls" \
+  PATH="${REPO}/tests/e2e/bin:${T}/real:${PATH}" E2E_LAZY_IMAGES="lazyone lazytwo" \
+    docker run --rm -v /a:/a lazytwo tool --x > /dev/null 2>&1
+  PATH="${REPO}/tests/e2e/bin:${T}/real:${PATH}" E2E_LAZY_IMAGES="lazyone lazytwo" \
+    docker run --rm lazytwo tool > /dev/null 2>&1
+  PATH="${REPO}/tests/e2e/bin:${T}/real:${PATH}" E2E_LAZY_IMAGES="lazyone lazytwo" \
+    docker run --rm plainthree tool > /dev/null 2>&1
+  if [ "$(grep -c '^pull ' "${T}/calls")" = 1 ] && grep -qx 'pull -q lazytwo' "${T}/calls" \
      && [ "$(grep -c '^run ' "${T}/calls")" = 3 ]; then
     echo "ok: the docker shim pulls a missing lazy image once, before its run"
   else

From 555e03b1f842c87f459f626b35dc911d3fbc29a7 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Thu, 5 Nov 2026 04:19:42 +0100
Subject: [PATCH 3/5] docs(e2e): give the image store's cost as measured on
 both runner images

The 33 GB of the image table was containerd's compressed size; a probe run
measured the same three images at 24.8 GB on containerd and 17.6 GB on
overlay2.
---
 .github/workflows/e2e.yml | 17 +++++++++--------
 docs/lessons-learned.md   |  2 +-
 docs/testing.md           |  4 ++--
 scripts/ci/e2e-run.sh     | 11 ++++++-----
 4 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index c0b43792..2d43858d 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -418,14 +418,15 @@ jobs:
           version: "26.10.5"
 
       # Docker 29 on the ubuntu-26.04 image keeps images in containerd's
-      # store, which holds each layer twice: the compressed or saved blob in
-      # its content store and the unpacked snapshot. There the 56 images of
-      # the cases and the layout took 91 GB of disk for 33 GB of image size
-      # (run 41938436657, shard 3), more than a 72 GB runner has free.
-      # The classic overlay2 store keeps the unpacked layers only (the 44
-      # cached images took 20 GB on an ubuntu-24.04 runner, run 41947119567).
-      # Runs before any image is restored or pulled; the images already in
-      # the containerd store are pruned first, as overlay2 would not see them.
+      # store, which holds each layer twice: the compressed blob in its
+      # content store and the unpacked snapshot. DeepVariant's, GATK's and
+      # VNtyper's images took 24.8 GB of disk there and 17.6 GB in the
+      # classic overlay2 store, on 26.04 and on 24.04 alike (probe run
+      # 41957796291 on the geiresx fork). With every case image on it, a
+      # 72 GB runner (48 GB free) ran out in the pre-pull (run 41938436657,
+      # shards 2 and 5). Runs before any image is restored or pulled; the
+      # images already in the containerd store are pruned first, as overlay2
+      # would not see them.
       - name: Docker image store
         run: |
           if docker info -f '{{json .DriverStatus}}' | grep -q io.containerd.snapshotter; then
diff --git a/docs/lessons-learned.md b/docs/lessons-learned.md
index 2bd71692..7c46202c 100644
--- a/docs/lessons-learned.md
+++ b/docs/lessons-learned.md
@@ -612,7 +612,7 @@ Most bioinformatics containers run as non-root users. If writing to bind-mounted
 
 ### Docker's containerd image store on ubuntu-26.04 doubles the e2e images on disk
 - **Failed:** shards 2 and 5 of main's run 41938436657 stopped in the pre-pull with "no space left on device" under `/var/lib/containerd`, while pulling MEGAnE's image. The same commit passed on other legs.
-- **Why:** `ubuntu-latest` hands out runners of two sizes, 145 GB and 72 GB (48 GB free after the clean-up step), and on the ubuntu-26.04 image Docker 29 keeps images in containerd's store, which holds each layer as a blob and as an unpacked copy. The 56 case images (33 GB of image size) and the layout took 91 GB there. Every leg also pre-pulled all of them, the five largest included, whatever its cases ran.
+- **Why:** `ubuntu-latest` hands out runners of two sizes, 145 GB and 72 GB (48 GB free after the clean-up step), and on the ubuntu-26.04 image Docker 29 keeps images in containerd's store, which holds each layer as a compressed blob and as an unpacked copy: DeepVariant's, GATK's and VNtyper's images took 24.8 GB there against 17.6 GB on overlay2. In that store `docker image inspect -f '{{.Size}}'` gives the compressed size (7.2 GB for the same three), so the image table understated what the disk held. Every leg also pre-pulled all 56 case images, the five largest included, whatever its cases ran.
 - **Fix:** each e2e leg turns `containerd-snapshotter` off in `/etc/docker/daemon.json` and restarts Docker before it restores an image, so images go to overlay2; the five largest images are pulled at a case's first `docker run` of them (`tests/e2e/bin/docker`); and `e2e-run.sh` fails a leg with less than `E2E_MIN_FREE_GB` free before its first case, so a short disk shows as one clear error instead of a failure in a pull or a case.
 
 ## Downloads (2026-11)
diff --git a/docs/testing.md b/docs/testing.md
index 8b947f9d..68e8bd9b 100644
--- a/docs/testing.md
+++ b/docs/testing.md
@@ -92,9 +92,9 @@ Two downloads are cached the same way, because they come from EBI's 1000 Genomes
 
 How long it took in one job, before the split: a passing e2e job took about 105 to 205 minutes with 84 cases, and 230 minutes of cases with 130 (run 41390434697, 2026-11-02). The runner decides much of it, not the cache: the same CPU model took 106 minutes in one run and 166 in another. In October 2026, with 84 cases, the same code took 106 and 166 minutes on an AMD EPYC 9V45 (runs 39197502832 and 39075176392, the second of which also saved the cache), 112 minutes on an Intel Xeon 6973P (run 39088086725), 156 and 191 minutes on an AMD EPYC 7763 (runs 39075764278 and 39075201465) and 205 minutes on an AMD EPYC 9V74 (run 39197512594, which saved three groups). The image cache saves about 7 minutes of that: pulling every image took 9 minutes, restoring and loading the four groups took 2.5 (run 39075201465 restored 39 images in 118 seconds), and saving them all took 12. Each leg has a limit of 240 minutes (355 for a traced shard leg); the base leg waits up to 75 minutes for a fixture a pull request bumps. Each leg's first steps print the runner's CPU model and CPU count to the log and the job summary, so a later timing can name the hardware.
 
-Before the first case of a full run, and of every leg, `e2e-run.sh` pulls every image the cases use that is not on the machine yet: the `NAME_IMAGE` lines of `versions.env` and of the case files, less a list of images no case runs. A pull that fails with a registry 5xx is tried again, up to four tries in all, 20, 40 and 80 seconds apart. Any other error, such as a missing tag or a rate limit, fails the job at once, and the tools' own `docker run` is never retried. A Docker Hub 500 on one image's manifest is what failed main's run 37980248545 the first time. `scripts/ci/e2e-run.sh --self-test-pull` checks the retry against a fake docker, and `e2e-plan` runs it. A run given a pattern skips the pre-pull, because most images reach a case through the steps it runs and cannot be picked per case; its cases pull their own images, with one try each. The five largest images (DeepVariant, PharmCAT, GATK, MEGAnE and VNtyper, 15 GB together, none of them in the image cache) are left out of the pre-pull: the docker shim `tests/e2e/bin/docker` pulls one, with the same retry, when a case first runs it, so a leg holds only those its own cases use.
+Before the first case of a full run, and of every leg, `e2e-run.sh` pulls every image the cases use that is not on the machine yet: the `NAME_IMAGE` lines of `versions.env` and of the case files, less a list of images no case runs. A pull that fails with a registry 5xx is tried again, up to four tries in all, 20, 40 and 80 seconds apart. Any other error, such as a missing tag or a rate limit, fails the job at once, and the tools' own `docker run` is never retried. A Docker Hub 500 on one image's manifest is what failed main's run 37980248545 the first time. `scripts/ci/e2e-run.sh --self-test-pull` checks the retry against a fake docker, and `e2e-plan` runs it. A run given a pattern skips the pre-pull, because most images reach a case through the steps it runs and cannot be picked per case; its cases pull their own images, with one try each. The five largest images (DeepVariant, PharmCAT, GATK, MEGAnE and VNtyper, 1.9 to 6.0 GB each to download, none of them in the image cache) are left out of the pre-pull: the docker shim `tests/e2e/bin/docker` pulls one, with the same retry, when a case first runs it, so a leg holds only those its own cases use.
 
-Each leg switches Docker to the classic overlay2 image store before it restores or pulls an image. Docker 29 on the ubuntu-26.04 runner image uses containerd's store, which keeps every layer twice (the blob and the unpacked copy): there the case images and the layout took 91 GB for 33 GB of image size (run 41938436657, shard 3), and the legs that landed on a 72 GB runner ran out of disk in the pre-pull. Before its first case `e2e-run.sh` fails the leg when less than `E2E_MIN_FREE_GB` is free under the work area (10 GB in `e2e.yml`), and after the cases it prints the least free space it saw while they ran.
+Each leg switches Docker to the classic overlay2 image store before it restores or pulls an image. Docker 29 on the ubuntu-26.04 runner image uses containerd's store, which keeps every layer twice (the compressed blob and the unpacked copy): DeepVariant's, GATK's and VNtyper's images took 24.8 GB there and 17.6 GB on overlay2, and the legs that landed on a 72 GB runner (48 GB free) ran out of disk in the pre-pull (run 41938436657). Before its first case `e2e-run.sh` fails the leg when less than `E2E_MIN_FREE_GB` is free under the work area (10 GB in `e2e.yml`), and after the cases it prints the least free space it saw while they ran.
 
 What it covers:
 
diff --git a/scripts/ci/e2e-run.sh b/scripts/ci/e2e-run.sh
index bf70d981..0333c1c9 100755
--- a/scripts/ci/e2e-run.sh
+++ b/scripts/ci/e2e-run.sh
@@ -120,11 +120,12 @@ case_images() {
 }
 
 # Images pulled at first use, not before the cases: the five largest (1.9 to
-# 6.0 GB each, 15 GB together in run 41938436657), which no image cache entry
-# holds (pinned by digest or over the group budget) and which only some legs
-# run. Pulled up front, every leg carried all five. A `docker run` or
-# `docker create` of one that is missing pulls it first, with the retry above
-# (tests/e2e/bin/docker reads E2E_LAZY_IMAGES).
+# 6.0 GB each compressed; DeepVariant's, GATK's and VNtyper's alone take
+# 17.6 GB unpacked), which no image cache entry holds (pinned by digest or over
+# the group budget) and which only some legs run. Pulled up front, every leg
+# carried all five. A `docker run` or `docker create` of one that is missing
+# pulls it first, with the retry above (tests/e2e/bin/docker reads
+# E2E_LAZY_IMAGES).
 LAZY_IMAGES=" DEEPVARIANT_IMAGE PHARMCAT_IMAGE GATK_IMAGE MEGANE_IMAGE VNTYPER_IMAGE "
 # lazy_images: the references of LAZY_IMAGES, from versions.env.
 lazy_images() {

From 5c12cbef5a98b176d0a48558d5e802f51634e3a8 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Thu, 5 Nov 2026 06:50:36 +0100
Subject: [PATCH 4/5] ci(e2e): re-save the image cache from overlay2 and hold
 one lazy image at a time

7 of the 46 archives saved from containerd's store did not load into
overlay2, and an exact key hit never saves again: new keys. A 72 GB runner
reached 7 GB free with two lazy images left on it (run 41960912641,
shard 2): each now goes when its case ends.
---
 .github/workflows/e2e.yml | 14 +++++++++-----
 scripts/ci/e2e-run.sh     |  6 ++++++
 2 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 2d43858d..1d1eb6f2 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -457,7 +457,11 @@ jobs:
           for g in 0 1 2 3; do
             # Only refs that can be saved: a digest bump never changes an entry.
             h=$(awk -v g="$g" '$1 == g && $2 !~ /@/ {print $2}' "${RUNNER_TEMP}/image-groups.txt" | sort | sha256sum | cut -c1-16)
-            echo "g${g}=e2e-images-g${g}-${h}" | tee -a "$GITHUB_OUTPUT"
+            # "ovl": saved from the overlay2 store (step "Docker image store").
+            # 7 of the 46 archives saved from containerd's store did not load
+            # into overlay2 (run 41960912641), and an exact key hit never
+            # saves again, so the old entries are left behind.
+            echo "g${g}=e2e-images-ovl-g${g}-${h}" | tee -a "$GITHUB_OUTPUT"
           done
 
       - name: Restore cached images, group 0
@@ -466,7 +470,7 @@ jobs:
         with:
           path: ${{ runner.temp }}/images/g0
           key: ${{ steps.keys.outputs.g0 }}
-          restore-keys: e2e-images-g0-
+          restore-keys: e2e-images-ovl-g0-
 
       - name: Restore cached images, group 1
         id: images1
@@ -474,7 +478,7 @@ jobs:
         with:
           path: ${{ runner.temp }}/images/g1
           key: ${{ steps.keys.outputs.g1 }}
-          restore-keys: e2e-images-g1-
+          restore-keys: e2e-images-ovl-g1-
 
       - name: Restore cached images, group 2
         id: images2
@@ -482,7 +486,7 @@ jobs:
         with:
           path: ${{ runner.temp }}/images/g2
           key: ${{ steps.keys.outputs.g2 }}
-          restore-keys: e2e-images-g2-
+          restore-keys: e2e-images-ovl-g2-
 
       - name: Restore cached images, group 3
         id: images3
@@ -490,7 +494,7 @@ jobs:
         with:
           path: ${{ runner.temp }}/images/g3
           key: ${{ steps.keys.outputs.g3 }}
-          restore-keys: e2e-images-g3-
+          restore-keys: e2e-images-ovl-g3-
 
       # One image per file; the four groups load side by side. An image that
       # does not load is pulled by e2e-run.sh like any missing image.
diff --git a/scripts/ci/e2e-run.sh b/scripts/ci/e2e-run.sh
index 0333c1c9..4d07f268 100755
--- a/scripts/ci/e2e-run.sh
+++ b/scripts/ci/e2e-run.sh
@@ -578,6 +578,12 @@ for c in "${CASES[@]}"; do
   rc=${PIPESTATUS[0]}
   set -e
   data_drop "$c"
+  # A lazy image goes when its case ends, so a leg holds one at a time; a
+  # later case that runs it pulls it again (DeepVariant's and PharmCAT's images
+  # took 15 GB together in the base leg of run 41960912641).
+  for i in ${E2E_LAZY_IMAGES:-}; do
+    docker image inspect "$i" > /dev/null 2>&1 && { docker image rm -f "$i" > /dev/null || true; }
+  done
   echo "::endgroup::"
   case "$rc" in
     0)   result=pass ;;

From ce02b9d7338a989f776d0e711d4cb5a2bc4b2803 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Thu, 5 Nov 2026 06:51:14 +0100
Subject: [PATCH 5/5] docs(e2e): say lazy images go after their case and why
 the cache keys moved

---
 docs/lessons-learned.md | 2 +-
 docs/testing.md         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/lessons-learned.md b/docs/lessons-learned.md
index 7c46202c..e5d81388 100644
--- a/docs/lessons-learned.md
+++ b/docs/lessons-learned.md
@@ -613,7 +613,7 @@ Most bioinformatics containers run as non-root users. If writing to bind-mounted
 ### Docker's containerd image store on ubuntu-26.04 doubles the e2e images on disk
 - **Failed:** shards 2 and 5 of main's run 41938436657 stopped in the pre-pull with "no space left on device" under `/var/lib/containerd`, while pulling MEGAnE's image. The same commit passed on other legs.
 - **Why:** `ubuntu-latest` hands out runners of two sizes, 145 GB and 72 GB (48 GB free after the clean-up step), and on the ubuntu-26.04 image Docker 29 keeps images in containerd's store, which holds each layer as a compressed blob and as an unpacked copy: DeepVariant's, GATK's and VNtyper's images took 24.8 GB there against 17.6 GB on overlay2. In that store `docker image inspect -f '{{.Size}}'` gives the compressed size (7.2 GB for the same three), so the image table understated what the disk held. Every leg also pre-pulled all 56 case images, the five largest included, whatever its cases ran.
-- **Fix:** each e2e leg turns `containerd-snapshotter` off in `/etc/docker/daemon.json` and restarts Docker before it restores an image, so images go to overlay2; the five largest images are pulled at a case's first `docker run` of them (`tests/e2e/bin/docker`); and `e2e-run.sh` fails a leg with less than `E2E_MIN_FREE_GB` free before its first case, so a short disk shows as one clear error instead of a failure in a pull or a case.
+- **Fix:** each e2e leg turns `containerd-snapshotter` off in `/etc/docker/daemon.json` and restarts Docker before it restores an image, so images go to overlay2; the five largest images are pulled at a case's first `docker run` of them (`tests/e2e/bin/docker`) and removed when the case ends; the image cache keys moved to `e2e-images-ovl-`, because 7 of the 46 archives saved from containerd's store did not load into overlay2; and `e2e-run.sh` fails a leg with less than `E2E_MIN_FREE_GB` free before its first case, so a short disk shows as one clear error instead of a failure in a pull or a case.
 
 ## Downloads (2026-11)
 
diff --git a/docs/testing.md b/docs/testing.md
index 68e8bd9b..7c323dc2 100644
--- a/docs/testing.md
+++ b/docs/testing.md
@@ -92,7 +92,7 @@ Two downloads are cached the same way, because they come from EBI's 1000 Genomes
 
 How long it took in one job, before the split: a passing e2e job took about 105 to 205 minutes with 84 cases, and 230 minutes of cases with 130 (run 41390434697, 2026-11-02). The runner decides much of it, not the cache: the same CPU model took 106 minutes in one run and 166 in another. In October 2026, with 84 cases, the same code took 106 and 166 minutes on an AMD EPYC 9V45 (runs 39197502832 and 39075176392, the second of which also saved the cache), 112 minutes on an Intel Xeon 6973P (run 39088086725), 156 and 191 minutes on an AMD EPYC 7763 (runs 39075764278 and 39075201465) and 205 minutes on an AMD EPYC 9V74 (run 39197512594, which saved three groups). The image cache saves about 7 minutes of that: pulling every image took 9 minutes, restoring and loading the four groups took 2.5 (run 39075201465 restored 39 images in 118 seconds), and saving them all took 12. Each leg has a limit of 240 minutes (355 for a traced shard leg); the base leg waits up to 75 minutes for a fixture a pull request bumps. Each leg's first steps print the runner's CPU model and CPU count to the log and the job summary, so a later timing can name the hardware.
 
-Before the first case of a full run, and of every leg, `e2e-run.sh` pulls every image the cases use that is not on the machine yet: the `NAME_IMAGE` lines of `versions.env` and of the case files, less a list of images no case runs. A pull that fails with a registry 5xx is tried again, up to four tries in all, 20, 40 and 80 seconds apart. Any other error, such as a missing tag or a rate limit, fails the job at once, and the tools' own `docker run` is never retried. A Docker Hub 500 on one image's manifest is what failed main's run 37980248545 the first time. `scripts/ci/e2e-run.sh --self-test-pull` checks the retry against a fake docker, and `e2e-plan` runs it. A run given a pattern skips the pre-pull, because most images reach a case through the steps it runs and cannot be picked per case; its cases pull their own images, with one try each. The five largest images (DeepVariant, PharmCAT, GATK, MEGAnE and VNtyper, 1.9 to 6.0 GB each to download, none of them in the image cache) are left out of the pre-pull: the docker shim `tests/e2e/bin/docker` pulls one, with the same retry, when a case first runs it, so a leg holds only those its own cases use.
+Before the first case of a full run, and of every leg, `e2e-run.sh` pulls every image the cases use that is not on the machine yet: the `NAME_IMAGE` lines of `versions.env` and of the case files, less a list of images no case runs. A pull that fails with a registry 5xx is tried again, up to four tries in all, 20, 40 and 80 seconds apart. Any other error, such as a missing tag or a rate limit, fails the job at once, and the tools' own `docker run` is never retried. A Docker Hub 500 on one image's manifest is what failed main's run 37980248545 the first time. `scripts/ci/e2e-run.sh --self-test-pull` checks the retry against a fake docker, and `e2e-plan` runs it. A run given a pattern skips the pre-pull, because most images reach a case through the steps it runs and cannot be picked per case; its cases pull their own images, with one try each. The five largest images (DeepVariant, PharmCAT, GATK, MEGAnE and VNtyper, 1.9 to 6.0 GB each to download, none of them in the image cache) are left out of the pre-pull: the docker shim `tests/e2e/bin/docker` pulls one, with the same retry, when a case first runs it, and removes it when that case ends, so a leg holds at most one of them at a time. The image cache keys start with `e2e-images-ovl-`: archives saved from containerd's store did not all load into overlay2.
 
 Each leg switches Docker to the classic overlay2 image store before it restores or pulls an image. Docker 29 on the ubuntu-26.04 runner image uses containerd's store, which keeps every layer twice (the compressed blob and the unpacked copy): DeepVariant's, GATK's and VNtyper's images took 24.8 GB there and 17.6 GB on overlay2, and the legs that landed on a 72 GB runner (48 GB free) ran out of disk in the pre-pull (run 41938436657). Before its first case `e2e-run.sh` fails the leg when less than `E2E_MIN_FREE_GB` is free under the work area (10 GB in `e2e.yml`), and after the cases it prints the least free space it saw while they ran.