diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index a7ed8d1b..1d1eb6f2 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -417,6 +417,31 @@ jobs: with: version: "26.10.5" + # Docker 29 on the ubuntu-26.04 image keeps images in containerd's + # store, which holds each layer twice: the compressed blob in its + # content store and the unpacked snapshot. DeepVariant's, GATK's and + # VNtyper's images took 24.8 GB of disk there and 17.6 GB in the + # classic overlay2 store, on 26.04 and on 24.04 alike (probe run + # 41957796291 on the geiresx fork). With every case image on it, a + # 72 GB runner (48 GB free) ran out in the pre-pull (run 41938436657, + # shards 2 and 5). Runs before any image is restored or pulled; the + # images already in the containerd store are pruned first, as overlay2 + # would not see them. + - name: Docker image store + run: | + if docker info -f '{{json .DriverStatus}}' | grep -q io.containerd.snapshotter; then + docker image prune -a -f > /dev/null + f=/etc/docker/daemon.json + { [ -s "$f" ] && sudo cat "$f" || echo '{}'; } \ + | jq '.features["containerd-snapshotter"] = false' | sudo tee "${f}.new" > /dev/null + sudo mv "${f}.new" "$f" + sudo systemctl restart docker + fi + driver=$(docker info -f '{{.Driver}}') + echo "Docker $(docker version -f '{{.Server.Version}}'), storage driver ${driver}" | tee -a "$GITHUB_STEP_SUMMARY" + [ "$driver" = overlay2 ] || { echo "::error::Docker's storage driver is ${driver}, not overlay2."; exit 1; } + df -h / + # Images are cached in four entries, one per group of # `scripts/ci/e2e-run.sh --image-groups`, each keyed on its own image # list: a version bump misses one group's key, and that group restores @@ -432,7 +457,11 @@ jobs: for g in 0 1 2 3; do # Only refs that can be saved: a digest bump never changes an entry. h=$(awk -v g="$g" '$1 == g && $2 !~ /@/ {print $2}' "${RUNNER_TEMP}/image-groups.txt" | sort | sha256sum | cut -c1-16) - echo "g${g}=e2e-images-g${g}-${h}" | tee -a "$GITHUB_OUTPUT" + # "ovl": saved from the overlay2 store (step "Docker image store"). + # 7 of the 46 archives saved from containerd's store did not load + # into overlay2 (run 41960912641), and an exact key hit never + # saves again, so the old entries are left behind. + echo "g${g}=e2e-images-ovl-g${g}-${h}" | tee -a "$GITHUB_OUTPUT" done - name: Restore cached images, group 0 @@ -441,7 +470,7 @@ jobs: with: path: ${{ runner.temp }}/images/g0 key: ${{ steps.keys.outputs.g0 }} - restore-keys: e2e-images-g0- + restore-keys: e2e-images-ovl-g0- - name: Restore cached images, group 1 id: images1 @@ -449,7 +478,7 @@ jobs: with: path: ${{ runner.temp }}/images/g1 key: ${{ steps.keys.outputs.g1 }} - restore-keys: e2e-images-g1- + restore-keys: e2e-images-ovl-g1- - name: Restore cached images, group 2 id: images2 @@ -457,7 +486,7 @@ jobs: with: path: ${{ runner.temp }}/images/g2 key: ${{ steps.keys.outputs.g2 }} - restore-keys: e2e-images-g2- + restore-keys: e2e-images-ovl-g2- - name: Restore cached images, group 3 id: images3 @@ -465,7 +494,7 @@ jobs: with: path: ${{ runner.temp }}/images/g3 key: ${{ steps.keys.outputs.g3 }} - restore-keys: e2e-images-g3- + restore-keys: e2e-images-ovl-g3- # One image per file; the four groups load side by side. An image that # does not load is pulled by e2e-run.sh like any missing image. @@ -483,6 +512,8 @@ jobs: [ -f "$f" ] || continue if zstd -dc "$f" | docker load -q > /dev/null; then echo "${f##*/}" >> "$loaded" else echo "::warning::${f##*/} did not load; e2e-run.sh pulls it"; fi + # Loaded, the archive only takes disk. + rm -f "$f" done ) & done @@ -588,6 +619,8 @@ jobs: E2E_CASES: ${{ steps.cases-list.outputs.list }} E2E_JOB: ${{ matrix.job == 'base' && 'base' || format('shard {0}', matrix.job) }} E2E_TRACE: ${{ needs.e2e-plan.outputs.trace }} + # The free disk a leg needs before its first case (e2e-run.sh). + E2E_MIN_FREE_GB: 10 E2E_BASE_OUT: ${{ matrix.job == 'base' && format('{0}/base.tar.zst', runner.temp) || '' }} E2E_BASE_IN: ${{ matrix.job != 'base' && format('{0}/e2e-base/base.tar.zst', runner.temp) || '' }} # Keep the downloads to the end of the job only when a save step uses them. diff --git a/docs/lessons-learned.md b/docs/lessons-learned.md index b121b943..e5d81388 100644 --- a/docs/lessons-learned.md +++ b/docs/lessons-learned.md @@ -610,6 +610,11 @@ Most bioinformatics containers run as non-root users. If writing to bind-mounted - **Fix:** `tests/e2e/mei-1.sh` removes its whole sample directory and the image by its id when it ends, and logs the free disk before and after. With the image gone every case passed. The image is pinned by digest, so the e2e image cache never saves it anyway. - **Image sizes step:** the job's "Image sizes" step then stopped with `numfmt: invalid number: ''`. For an image that is gone, `docker image inspect -f` prints an empty line before the `|| echo 0` fallback, so the step's own "not here" branch never ran. The fix in `.github/workflows/e2e.yml` reads the size into a variable first (`size=$(docker image inspect ...) || size=0`) and prints `${size:-0}`. +### Docker's containerd image store on ubuntu-26.04 doubles the e2e images on disk +- **Failed:** shards 2 and 5 of main's run 41938436657 stopped in the pre-pull with "no space left on device" under `/var/lib/containerd`, while pulling MEGAnE's image. The same commit passed on other legs. +- **Why:** `ubuntu-latest` hands out runners of two sizes, 145 GB and 72 GB (48 GB free after the clean-up step), and on the ubuntu-26.04 image Docker 29 keeps images in containerd's store, which holds each layer as a compressed blob and as an unpacked copy: DeepVariant's, GATK's and VNtyper's images took 24.8 GB there against 17.6 GB on overlay2. In that store `docker image inspect -f '{{.Size}}'` gives the compressed size (7.2 GB for the same three), so the image table understated what the disk held. Every leg also pre-pulled all 56 case images, the five largest included, whatever its cases ran. +- **Fix:** each e2e leg turns `containerd-snapshotter` off in `/etc/docker/daemon.json` and restarts Docker before it restores an image, so images go to overlay2; the five largest images are pulled at a case's first `docker run` of them (`tests/e2e/bin/docker`) and removed when the case ends; the image cache keys moved to `e2e-images-ovl-`, because 7 of the 46 archives saved from containerd's store did not load into overlay2; and `e2e-run.sh` fails a leg with less than `E2E_MIN_FREE_GB` free before its first case, so a short disk shows as one clear error instead of a failure in a pull or a case. + ## Downloads (2026-11) ### One bad transfer of a large file failed the whole setup diff --git a/docs/testing.md b/docs/testing.md index 08c171cd..7c323dc2 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -92,7 +92,9 @@ Two downloads are cached the same way, because they come from EBI's 1000 Genomes How long it took in one job, before the split: a passing e2e job took about 105 to 205 minutes with 84 cases, and 230 minutes of cases with 130 (run 41390434697, 2026-11-02). The runner decides much of it, not the cache: the same CPU model took 106 minutes in one run and 166 in another. In October 2026, with 84 cases, the same code took 106 and 166 minutes on an AMD EPYC 9V45 (runs 39197502832 and 39075176392, the second of which also saved the cache), 112 minutes on an Intel Xeon 6973P (run 39088086725), 156 and 191 minutes on an AMD EPYC 7763 (runs 39075764278 and 39075201465) and 205 minutes on an AMD EPYC 9V74 (run 39197512594, which saved three groups). The image cache saves about 7 minutes of that: pulling every image took 9 minutes, restoring and loading the four groups took 2.5 (run 39075201465 restored 39 images in 118 seconds), and saving them all took 12. Each leg has a limit of 240 minutes (355 for a traced shard leg); the base leg waits up to 75 minutes for a fixture a pull request bumps. Each leg's first steps print the runner's CPU model and CPU count to the log and the job summary, so a later timing can name the hardware. -Before the first case of a full run, and of every leg, `e2e-run.sh` pulls every image the cases use that is not on the machine yet: the `NAME_IMAGE` lines of `versions.env` and of the case files, less a list of images no case runs. A pull that fails with a registry 5xx is tried again, up to four tries in all, 20, 40 and 80 seconds apart. Any other error, such as a missing tag or a rate limit, fails the job at once, and the tools' own `docker run` is never retried. A Docker Hub 500 on one image's manifest is what failed main's run 37980248545 the first time. `scripts/ci/e2e-run.sh --self-test-pull` checks the retry against a fake docker, and `e2e-plan` runs it. A run given a pattern skips the pre-pull, because most images reach a case through the steps it runs and cannot be picked per case; its cases pull their own images, with one try each. +Before the first case of a full run, and of every leg, `e2e-run.sh` pulls every image the cases use that is not on the machine yet: the `NAME_IMAGE` lines of `versions.env` and of the case files, less a list of images no case runs. A pull that fails with a registry 5xx is tried again, up to four tries in all, 20, 40 and 80 seconds apart. Any other error, such as a missing tag or a rate limit, fails the job at once, and the tools' own `docker run` is never retried. A Docker Hub 500 on one image's manifest is what failed main's run 37980248545 the first time. `scripts/ci/e2e-run.sh --self-test-pull` checks the retry against a fake docker, and `e2e-plan` runs it. A run given a pattern skips the pre-pull, because most images reach a case through the steps it runs and cannot be picked per case; its cases pull their own images, with one try each. The five largest images (DeepVariant, PharmCAT, GATK, MEGAnE and VNtyper, 1.9 to 6.0 GB each to download, none of them in the image cache) are left out of the pre-pull: the docker shim `tests/e2e/bin/docker` pulls one, with the same retry, when a case first runs it, and removes it when that case ends, so a leg holds at most one of them at a time. The image cache keys start with `e2e-images-ovl-`: archives saved from containerd's store did not all load into overlay2. + +Each leg switches Docker to the classic overlay2 image store before it restores or pulls an image. Docker 29 on the ubuntu-26.04 runner image uses containerd's store, which keeps every layer twice (the compressed blob and the unpacked copy): DeepVariant's, GATK's and VNtyper's images took 24.8 GB there and 17.6 GB on overlay2, and the legs that landed on a 72 GB runner (48 GB free) ran out of disk in the pre-pull (run 41938436657). Before its first case `e2e-run.sh` fails the leg when less than `E2E_MIN_FREE_GB` is free under the work area (10 GB in `e2e.yml`), and after the cases it prints the least free space it saw while they ran. What it covers: diff --git a/scripts/ci/e2e-run.sh b/scripts/ci/e2e-run.sh index d47807a5..4d07f268 100755 --- a/scripts/ci/e2e-run.sh +++ b/scripts/ci/e2e-run.sh @@ -3,6 +3,7 @@ # # Usage: scripts/ci/e2e-run.sh [pattern] # scripts/ci/e2e-run.sh --self-test-pull | --image-groups | --pull-images +# scripts/ci/e2e-run.sh --pull-image REF # pattern optional shell glob; runs only the cases whose file name matches # (e.g. '2*' or '*cyrius*'). Later cases read earlier cases' outputs, # so a partial run is for debugging only. @@ -11,6 +12,8 @@ # --image-groups prints "GROUP IMAGE" for every image the cases use (the # e2e workflow's image cache entries) and exits. # --pull-images pulls those images, as step 3 below does, and exits. +# --pull-image REF pulls one image with the same retry (the docker shim's +# pulls of the lazy images at their first use), and exits. # # Env: E2E_WORK work area (default ${RUNNER_TEMP:-/tmp}/e2e-work), about 25 GB # GH_TOKEN token for `gh release download` (the job's GITHUB_TOKEN) @@ -18,6 +21,8 @@ # full run (pre-pull, all-cases-ran marker). The e2e workflow # gives each of its parallel jobs its list from e2e-shards.sh. # E2E_JOB the job name written beside each result (default local) +# E2E_MIN_FREE_GB fail before the first case when less is free under +# E2E_WORK (default 0) # E2E_BASE_OUT after the cases, pack every file they wrote or changed # (under E2E_WORK and the checkout) into this .tar.zst, and # write "E2E_WORK CHECKOUT" to E2E_BASE_OUT.from @@ -41,7 +46,9 @@ # for the build-fixture job to publish it; # 2. lays out GENOME_DIR the way setup.sh and step 13 would leave it; # 3. pulls every image the cases use that is not on the machine yet, -# trying a pull again only when the registry answered with a 5xx; +# trying a pull again only when the registry answered with a 5xx (the +# five largest wait for the first case that runs them), and checks the +# free disk against E2E_MIN_FREE_GB; # 4. runs tests/e2e/*.sh in name order (C locale: numbered cases first, then # the -*.sh cases later packages add), or those of E2E_CASES, # and keeps going after a failure, so one run lists every broken step; @@ -112,6 +119,23 @@ case_images() { done | awk '!seen[$0]++' } +# Images pulled at first use, not before the cases: the five largest (1.9 to +# 6.0 GB each compressed; DeepVariant's, GATK's and VNtyper's alone take +# 17.6 GB unpacked), which no image cache entry holds (pinned by digest or over +# the group budget) and which only some legs run. Pulled up front, every leg +# carried all five. A `docker run` or `docker create` of one that is missing +# pulls it first, with the retry above (tests/e2e/bin/docker reads +# E2E_LAZY_IMAGES). +LAZY_IMAGES=" DEEPVARIANT_IMAGE PHARMCAT_IMAGE GATK_IMAGE MEGANE_IMAGE VNTYPER_IMAGE " +# lazy_images: the references of LAZY_IMAGES, from versions.env. +lazy_images() { + grep -hoE '^[A-Z0-9_]+_IMAGE="[^"]+"' "${REPO}/versions.env" \ + | while IFS='=' read -r name ref; do + [[ "$LAZY_IMAGES" == *" ${name} "* ]] && echo "${ref//\"/}" + done + return 0 +} + # --- Image cache groups ------------------------------------------------------- # The e2e workflow caches the images of case_images in IMAGE_GROUPS cache # entries, so no single entry, and not their sum, reaches the repository's @@ -136,17 +160,20 @@ if [ "${1:-}" = --image-groups ]; then exit 0 fi -# pull_all: pull every image of case_images that is not here yet. +# pull_all: pull every image of case_images that is not here yet, but the +# lazy ones. pull_all() { - local image pulled=0 + local image pulled=0 lazy mapfile -t IMAGES < <(case_images) [ "${#IMAGES[@]}" -gt 0 ] || { echo "ERROR: no NAME_IMAGE line found in versions.env" >&2; return 1; } + lazy=$(lazy_images) for image in "${IMAGES[@]}"; do + grep -qxF "$image" <<< "$lazy" && continue docker image inspect "$image" > /dev/null 2>&1 && continue pull_image "$image" || return 1 pulled=$((pulled + 1)) done - echo "${#IMAGES[@]} images: $(( ${#IMAGES[@]} - pulled )) already here, ${pulled} pulled" + echo "${#IMAGES[@]} images: $(( ${#IMAGES[@]} - pulled - $(wc -l <<< "$lazy") )) already here, ${pulled} pulled, $(wc -l <<< "$lazy") left to their first use" } # The e2e workflow's shard legs pull while the base leg runs. @@ -154,6 +181,26 @@ if [ "${1:-}" = --pull-images ]; then pull_all exit fi +# --pull-image REF: one pull with the retry (the docker shim's lazy pulls). +if [ "${1:-}" = --pull-image ]; then + pull_image "${2:?--pull-image needs an image}" + exit +fi + +# --- Free disk ------------------------------------------------------------------ +# The legs ran out of disk pulling images on 2026-11-05 (run 41938436657). Before +# the cases a leg needs E2E_MIN_FREE_GB free under E2E_WORK: room for what its +# cases write and for the lazy images they pull. +# disk_check NEED_GB: prints the free space; fails when it is under NEED_GB. +disk_check() { + local free + free=$(( $(df --output=avail -k "$E2E_WORK" | tail -1) / 1048576 )) + echo "Free disk before the cases: ${free} GB under ${E2E_WORK} (the leg needs ${1} GB)" + if [ "$free" -lt "$1" ]; then + echo "ERROR: ${free} GB free under ${E2E_WORK}, under the ${1} GB the cases need." >&2 + return 1 + fi +} if [ "${1:-}" = --self-test-pull ]; then T=$(mktemp -d) @@ -223,6 +270,48 @@ FAKE else echo "FAIL: the cache groups do not hold the pull list once each"; st_failed=1 fi + # Lazy images: each one a case image, five in all. + LAZY=$(lazy_images) + if [ "$(wc -l <<< "$LAZY")" -eq 5 ] && ! grep -qvxF -f <(echo "$LIST") <<< "$LAZY"; then + echo "ok: the 5 lazy images are case images" + else + echo "FAIL: the lazy images are not 5 case images: ${LAZY//$'\n'/ }"; st_failed=1 + fi + # disk_check against a fake df: 10 GB free fails a need of 11 and passes 10. + # shellcheck disable=SC2016 # the fake df expands FAKE_AVAIL_K when it runs + printf '#!/usr/bin/env bash\nprintf "Avail\\n%%s\\n" "$FAKE_AVAIL_K"\n' > "${T}/df" + chmod +x "${T}/df" + E2E_WORK=$T + if FAKE_AVAIL_K=$((10 * 1048576)) disk_check 11 > /dev/null 2>&1; then + echo "FAIL: 10 GB free passed a need of 11 GB"; st_failed=1 + else echo "ok: 10 GB free fails a need of 11 GB"; fi + if FAKE_AVAIL_K=$((10 * 1048576)) disk_check 10 > /dev/null 2>&1; then echo "ok: 10 GB free passes a need of 10 GB" + else echo "FAIL: 10 GB free failed a need of 10 GB"; st_failed=1; fi + # The docker shim: a run of a missing lazy image pulls it once, then runs; + # a run of any other image only runs. + mkdir -p "${T}/real" + cat > "${T}/real/docker" <<'FAKE' +#!/usr/bin/env bash +echo "$*" >> "${FAKE_DIR}/calls" +case "$1 ${2:-}" in + "image inspect") awk -v i="${*: -1}" '$1 == "pull" && $NF == i {f = 1} END {exit !f}' "${FAKE_DIR}/calls" ;; + *) exit 0 ;; +esac +FAKE + chmod +x "${T}/real/docker" + : > "${T}/calls" + PATH="${REPO}/tests/e2e/bin:${T}/real:${PATH}" E2E_LAZY_IMAGES="lazyone lazytwo" \ + docker run --rm -v /a:/a lazytwo tool --x > /dev/null 2>&1 + PATH="${REPO}/tests/e2e/bin:${T}/real:${PATH}" E2E_LAZY_IMAGES="lazyone lazytwo" \ + docker run --rm lazytwo tool > /dev/null 2>&1 + PATH="${REPO}/tests/e2e/bin:${T}/real:${PATH}" E2E_LAZY_IMAGES="lazyone lazytwo" \ + docker run --rm plainthree tool > /dev/null 2>&1 + if [ "$(grep -c '^pull ' "${T}/calls")" = 1 ] && grep -qx 'pull -q lazytwo' "${T}/calls" \ + && [ "$(grep -c '^run ' "${T}/calls")" = 3 ]; then + echo "ok: the docker shim pulls a missing lazy image once, before its run" + else + echo "FAIL: the docker shim's lazy pulls:"; sed 's/^/ /' "${T}/calls"; st_failed=1 + fi exit "$st_failed" fi @@ -317,9 +406,22 @@ fi if [ "$PATTERN" = "*" ]; then echo "=== Pulling the images the cases use ===" pull_all + E2E_LAZY_IMAGES=$(lazy_images | paste -sd' ' -) + export E2E_LAZY_IMAGES else echo "=== Partial run ('${PATTERN}'): no pre-pull, each case pulls what it uses ===" fi +docker system df +disk_check "${E2E_MIN_FREE_GB:-0}" +# The least free space while the cases run, sampled every 10 s, for the report. +LOW_FILE="${REPORT_DIR}/disk-low" +df --output=avail -k "$E2E_WORK" | tail -1 > "$LOW_FILE" +( while sleep 10; do + a=$(df --output=avail -k "$E2E_WORK" | tail -1) + [ "$a" -lt "$(cat "$LOW_FILE")" ] && echo "$a" > "$LOW_FILE" + done ) & +LOW_PID=$! +trap 'kill "$LOW_PID" 2> /dev/null || true' EXIT # --- Downloads kept between runs ---------------------------------------------- # The two largest downloads of the cases come from EBI's 1000 Genomes server, @@ -476,6 +578,12 @@ for c in "${CASES[@]}"; do rc=${PIPESTATUS[0]} set -e data_drop "$c" + # A lazy image goes when its case ends, so a leg holds one at a time; a + # later case that runs it pulls it again (DeepVariant's and PharmCAT's images + # took 15 GB together in the base leg of run 41960912641). + for i in ${E2E_LAZY_IMAGES:-}; do + docker image inspect "$i" > /dev/null 2>&1 && { docker image rm -f "$i" > /dev/null || true; } + done echo "::endgroup::" case "$rc" in 0) result=pass ;; @@ -494,6 +602,9 @@ if [ "${#names[@]}" -eq 0 ]; then fi # Tells the workflow every case ran, so the pulled images are complete enough to cache. [ "$PATTERN" = "*" ] && touch "${E2E_WORK}/all-cases-ran" +kill "$LOW_PID" 2> /dev/null || true +echo "Disk (${JOB}): $(( $(cat "$LOW_FILE") / 1048576 )) GB free at the lowest during the cases, $(( $(df --output=avail -k "$E2E_WORK" | tail -1) / 1048576 )) GB after them" | tee -a "$SUMMARY" +docker system df # --- 5. Report ---------------------------------------------------------------- { diff --git a/tests/e2e/bin/docker b/tests/e2e/bin/docker index 46fce0d9..2119cd8f 100755 --- a/tests/e2e/bin/docker +++ b/tests/e2e/bin/docker @@ -15,6 +15,17 @@ for d in "${path_dirs[@]}"; do done [ -n "$real" ] || { echo "docker shim: no real docker on PATH" >&2; exit 127; } +# E2E_LAZY_IMAGES (set by scripts/ci/e2e-run.sh): images pulled at first use. +# A `docker run` or `docker create` of one that is not here pulls it first, +# with e2e-run.sh's retry on a registry 5xx. +if [ -n "${E2E_LAZY_IMAGES:-}" ] && { [ "${1:-}" = run ] || [ "${1:-}" = create ]; }; then + for a in "$@"; do + [[ " ${E2E_LAZY_IMAGES} " == *" ${a} "* ]] || continue + "$real" image inspect "$a" > /dev/null 2>&1 && continue + "$(dirname "$self_dir")/../../scripts/ci/e2e-run.sh" --pull-image "$a" >&2 || exit $? + done +fi + max=$(nproc) clamp() { awk -v v="$1" -v m="$max" 'BEGIN {print (v + 0 > m) ? m : v}'; }