From a20c55513631411a6e0a6a69ad80e7be40b04c3a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Thu, 5 Nov 2026 10:36:34 +0100 Subject: [PATCH] ci(e2e): stop e2e legs timing out on EBI when the data cache is empty With no Actions cache for the 1000 Genomes sites file or the MoChA panel (a fork, or an evicted entry), shards 2 and 5 download them from EBI, whose 1000 Genomes hosts swing between a few KB/s and MB/s, and the legs hit the 240-minute limit. No other public copy of the pinned v5c sites file exists. data_fetch now tries E2E_DATA_MIRROR (the release e2e-data-1) once per file before the source. The pin checks both copies, and a file the release lacks falls back to EBI as before. --- .github/workflows/e2e.yml | 2 ++ docs/lessons-learned.md | 5 +++++ docs/testing.md | 2 +- scripts/ci/e2e-run.sh | 21 ++++++++++++++++++--- 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 1d1eb6f2..75a68eeb 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -621,6 +621,8 @@ jobs: E2E_TRACE: ${{ needs.e2e-plan.outputs.trace }} # The free disk a leg needs before its first case (e2e-run.sh). E2E_MIN_FREE_GB: 10 + # Copies of the pinned e2e data, tried before EBI (data_fetch in e2e-run.sh). + E2E_DATA_MIRROR: https://github.com/GeiserX/OwnGenome/releases/download/e2e-data-1 E2E_BASE_OUT: ${{ matrix.job == 'base' && format('{0}/base.tar.zst', runner.temp) || '' }} E2E_BASE_IN: ${{ matrix.job != 'base' && format('{0}/e2e-base/base.tar.zst', runner.temp) || '' }} # Keep the downloads to the end of the job only when a save step uses them. diff --git a/docs/lessons-learned.md b/docs/lessons-learned.md index e5d81388..de6c9bae 100644 --- a/docs/lessons-learned.md +++ b/docs/lessons-learned.md @@ -633,3 +633,8 @@ Most bioinformatics containers run as non-root users. If writing to bind-mounted - **Failed:** `archaic-ancestry-1` ran into the 3600-second case timeout on one e2e run (run 40933547961, attempt 1) and pushed the job past its own limit; the rerun passed. - **Why:** `setup.sh --archaic` downloads the 1000 Genomes phase 3 sites VCF (1.36 GB) from EBI. On that run EBI sent it at about 440 KB/s, and the download alone took 53 minutes, so the case's Nextflow half started with minutes left. The download never stopped: it was slow, not stalled. MoChA's chr20 panel in `mocha-1` comes from the same server. - **Fix:** the e2e job caches both files (keyed on their pins) and `scripts/ci/e2e-run.sh` hands them to the two cases, so a run with a cache hit downloads neither. Separately, `_get` now bounds a real stall, which nothing did before: `curl` gets `--connect-timeout 30 --speed-limit 1024 --speed-time ${FETCH_STALL:-120}`, so a transfer under 1 KB/s for two minutes fails and `fetch` tries again, resuming the `.part`. `wget` gets `--connect-timeout=30 --read-timeout=${FETCH_STALL:-120}`, which fails only when no data arrives for that long. A slow but moving transfer like the one above is left alone. + +### With no data cache, an e2e leg spent its whole limit on EBI +- **Failed:** in November 2026, fork runs and a cache-seeding run with no Actions cache for the two data entries had shards 2 and 5 cancelled at the 240-minute leg limit, still downloading. An Actions cache belongs to one repository, so a fork starts with none, and an evicted entry leaves main in the same place. +- **Why:** both EBI hosts for 1000 Genomes (`ftp.1000genomes.ebi.ac.uk` and `ftp.ebi.ac.uk/1000g`) swung between a few KB/s and about 3 MB/s from one request to the next in the same hour. No other copy of the pinned v5c sites file exists: AWS's `1000genomes` bucket holds only v5b, and NCBI's 1000 Genomes path answers 404. +- **Fix:** the pinned files are kept as assets of the GitHub release `e2e-data-1`, and `data_fetch` in `scripts/ci/e2e-run.sh` tries `E2E_DATA_MIRROR` once before the source. The pin checks both copies, so the mirror changes only where the bytes come from. A file missing from the release falls back to EBI. diff --git a/docs/testing.md b/docs/testing.md index 7c323dc2..78339fc7 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -88,7 +88,7 @@ Every pull request starts the workflow. Its `e2e-scope` job checks whether the p The images the cases use are cached in four GitHub Actions cache entries, one per group. `scripts/ci/e2e-run.sh --image-groups` names each image's group, taken from its repository name without the tag, so a version bump changes the key of one group only; a group whose key missed restores the newest older entry of that group and pulls only what changed. A group keeps its smallest images up to 1500 MB of zstd output, and the larger ones are pulled on every run. Four groups of 1500 MB and the two data entries below stay under the repository's 10 GB cache limit even while main replaces one group: 5.9 GB of images, 1.5 GB more for the group being replaced, and 2.0 GB of data. Images pinned by digest are not cached, because `docker load` gives them back without the digest, and they are left out of the group's key, so a digest bump misses no entry. Only the base leg saves, and only on main (or in a run by hand with `save_images`, on any branch), after it pulled every image the cases use and got through its cases, and only the groups whose key missed: a cache saved by a pull request can be read by that pull request alone and would only push main's entries out. The job summary lists every image with its size and what the cache did with it: restored on an exact key hit, restored from an older entry, saved, or pulled. Every merge in the e2e scope starts a run on main, so the cache follows main's images. GitHub drops an entry no run has read for 7 days, so after a quiet week the next run on main fills the cache again. Until this layout the cache was one entry, and by October 2026 it had grown to about 11 GB, over the limit, so every run missed it. -Two downloads are cached the same way, because they come from EBI's 1000 Genomes server, which is slow and at times stalls: MoChA's chr20 panel with its manifest (0.64 GB, case `mocha-1`), keyed on `MOCHA_PANEL_MANIFEST_SHA256` and the chromosome, and the 1000 Genomes phase 3 sites VCF (1.36 GB, case `archaic-ancestry-1`), keyed on `ARCHAIC_1KG_SITES_MD5`. `scripts/ci/e2e-run.sh` hands each case its files just before it runs, at the paths `setup.sh` reads them from, so `setup.sh` downloads nothing; on a miss it downloads them first, checked against their pins. Only the leg whose cases include the case restores and saves its entry, and it saves only on main or in a run by hand with `save_images`, as with the images. The job summary says whether each key hit, and the case's log says where its files came from. Every download, there and in `setup.sh`, gives up on a transfer that stalls (`FETCH_STALL`, see `fetch` in `scripts/lib/common.sh`), and the next try resumes it. +Two downloads are cached the same way, because they come from EBI's 1000 Genomes server, which is slow and at times stalls: MoChA's chr20 panel with its manifest (0.64 GB, case `mocha-1`), keyed on `MOCHA_PANEL_MANIFEST_SHA256` and the chromosome, and the 1000 Genomes phase 3 sites VCF (1.36 GB, case `archaic-ancestry-1`), keyed on `ARCHAIC_1KG_SITES_MD5`. `scripts/ci/e2e-run.sh` hands each case its files just before it runs, at the paths `setup.sh` reads them from, so `setup.sh` downloads nothing; on a miss it downloads them first, checked against their pins. Only the leg whose cases include the case restores and saves its entry, and it saves only on main or in a run by hand with `save_images`, as with the images. The job summary says whether each key hit, and the case's log says where its files came from. On a miss, each file is tried first at `E2E_DATA_MIRROR`, the GitHub release `e2e-data-1` that holds copies of the pinned files, and then at EBI; the pin checks both. Every download, there and in `setup.sh`, gives up on a transfer that stalls (`FETCH_STALL`, see `fetch` in `scripts/lib/common.sh`), and the next try resumes it. How long it took in one job, before the split: a passing e2e job took about 105 to 205 minutes with 84 cases, and 230 minutes of cases with 130 (run 41390434697, 2026-11-02). The runner decides much of it, not the cache: the same CPU model took 106 minutes in one run and 166 in another. In October 2026, with 84 cases, the same code took 106 and 166 minutes on an AMD EPYC 9V45 (runs 39197502832 and 39075176392, the second of which also saved the cache), 112 minutes on an Intel Xeon 6973P (run 39088086725), 156 and 191 minutes on an AMD EPYC 7763 (runs 39075764278 and 39075201465) and 205 minutes on an AMD EPYC 9V74 (run 39197512594, which saved three groups). The image cache saves about 7 minutes of that: pulling every image took 9 minutes, restoring and loading the four groups took 2.5 (run 39075201465 restored 39 images in 118 seconds), and saving them all took 12. Each leg has a limit of 240 minutes (355 for a traced shard leg); the base leg waits up to 75 minutes for a fixture a pull request bumps. Each leg's first steps print the runner's CPU model and CPU count to the log and the job summary, so a later timing can name the hardware. diff --git a/scripts/ci/e2e-run.sh b/scripts/ci/e2e-run.sh index 4d07f268..9f93150c 100755 --- a/scripts/ci/e2e-run.sh +++ b/scripts/ci/e2e-run.sh @@ -437,6 +437,21 @@ trap 'kill "$LOW_PID" 2> /dev/null || true' EXIT # a run that saves the cache) a case's cached files are deleted after it, # to give the disk back. DATA_CACHE=${E2E_DATA_CACHE:-} +# E2E_DATA_MIRROR, when set, is a folder URL (a GitHub release) holding copies +# of these pinned files under their own names. Each file is tried there once +# first, then at its source. EBI's 1000 Genomes hosts swing between a few KB/s +# and MB/s from one request to the next, and a leg with no Actions cache could +# spend its whole time limit on one download. The pin checks both copies the +# same way, so the mirror can only save time, never change what a case reads. +# mfetch URL DEST KIND SUM: fetch from the mirror, then from URL. Called +# inside data_fetch, after it sources scripts/lib/common.sh for fetch. +mfetch() { + if [ -n "${E2E_DATA_MIRROR:-}" ]; then + FETCH_TRIES=1 fetch "${E2E_DATA_MIRROR%/}/${1##*/}" "$2" "$3" "$4" 2> /dev/null && return 0 + echo "Data cache: ${1##*/} not on the mirror; downloading it from its source" + fi + fetch "$@" +} # data_fetch NAME: fills E2E_DATA_CACHE/NAME with what is missing, in a # subshell that sources scripts/lib/common.sh for fetch and the pins. data_fetch() ( @@ -447,21 +462,21 @@ data_fetch() ( mocha) m="${d}/manifest.txt" { [ -s "$m" ] && [ "$(_digest sha256 "$m")" = "$MOCHA_PANEL_MANIFEST_SHA256" ]; } \ - || fetch "${MOCHA_PANEL_URL}/20220804_manifest.txt" "$m" sha256 "$MOCHA_PANEL_MANIFEST_SHA256" || exit 1 + || mfetch "${MOCHA_PANEL_URL}/20220804_manifest.txt" "$m" sha256 "$MOCHA_PANEL_MANIFEST_SHA256" || exit 1 # mocha-1 cuts the panel to the fixture's chr20 slice. f=1kGP_high_coverage_Illumina.chr20.filtered.SNV_INDEL_SV_phased_panel.vcf.gz for e in "" .tbi; do md5=$(awk -v n="${f}${e}" '$1 == n {print $2}' "$m") [ -n "$md5" ] || { echo "ERROR: ${f}${e} is not in the panel manifest" >&2; exit 1; } [ -s "${d}/raw/${f}${e}" ] && [ "$(_digest md5 "${d}/raw/${f}${e}")" = "$md5" ] && continue - fetch "${MOCHA_PANEL_URL}/${f}${e}" "${d}/raw/${f}${e}" md5 "$md5" || exit 1 + mfetch "${MOCHA_PANEL_URL}/${f}${e}" "${d}/raw/${f}${e}" md5 "$md5" || exit 1 done ;; archaic) url=$(sed -n 's/^ARCHAIC_1KG_SITES_URL=//p' "${REPO}/scripts/setup.sh") [ -n "$url" ] || { echo "ERROR: no ARCHAIC_1KG_SITES_URL line in scripts/setup.sh" >&2; exit 1; } s="${d}/1kg_sites.vcf.gz" [ -s "$s" ] && [ "$(_digest md5 "$s")" = "$ARCHAIC_1KG_SITES_MD5" ] && exit 0 - fetch "$url" "$s" md5 "$ARCHAIC_1KG_SITES_MD5" || exit 1 ;; + mfetch "$url" "$s" md5 "$ARCHAIC_1KG_SITES_MD5" || exit 1 ;; *) echo "ERROR: no data cache entry ${1}" >&2; exit 1 ;; esac )