diff --git a/.github/actions/free-disk/action.yml b/.github/actions/free-disk/action.yml index 6700d1cf..4350ab6e 100644 --- a/.github/actions/free-disk/action.yml +++ b/.github/actions/free-disk/action.yml @@ -17,23 +17,39 @@ runs: df -h / /mnt 2> /dev/null || df -h / before=$(df --output=avail -k / | tail -1) # The deletions run side by side: the Android SDK alone is about - # 100,000 files. A path that is not there is skipped. - for d in /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup \ + # 100,000 files. A path that is not there is skipped. The cleanup is + # best effort, but a part that fails is named in the log and the + # summary instead of showing only as a smaller number freed. + # FREE_DISK_PATHS replaces the list (tests/test_e2e_data_cache.sh). + paths=${FREE_DISK_PATHS:-/usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup \ /usr/local/share/boost /usr/share/swift /usr/local/share/powershell \ /usr/local/share/chromium /opt/microsoft /opt/google /opt/az \ /opt/hostedtoolcache/CodeQL /opt/hostedtoolcache/go /opt/hostedtoolcache/PyPy \ /opt/hostedtoolcache/Python \ - /opt/hostedtoolcache/Ruby /opt/hostedtoolcache/node; do - [ -e "$d" ] && sudo rm -rf "$d" & + /opt/hostedtoolcache/Ruby /opt/hostedtoolcache/node} + pids=() what=() + for d in $paths; do + [ -e "$d" ] || continue + sudo rm -rf "$d" & pids+=($!); what+=("rm -rf ${d}") done # The images the runner image ships (none of them is ours). if command -v docker > /dev/null 2>&1; then - docker image prune -a -f > /dev/null 2>&1 & + docker image prune -a -f > /dev/null 2>&1 & pids+=($!); what+=("docker image prune -a") fi - wait + failed=() + for i in "${!pids[@]}"; do + rc=0 + wait "${pids[$i]}" || rc=$? + [ "$rc" -eq 0 ] || failed+=("${what[$i]} (exit ${rc})") + done after=$(df --output=avail -k / | tail -1) echo "Free space after:" df -h / /mnt 2> /dev/null || df -h / msg="Free disk: $(( (after - before) / 1048576 )) GB freed on /, $(( after / 1048576 )) GB free now, in ${SECONDS} s" + if [ "${#failed[@]}" -gt 0 ]; then + msg="${msg}; cleanup failed (best effort, the job goes on): $(printf '%s; ' "${failed[@]}")" + msg=${msg%; } + echo "::warning::${msg}" + fi echo "$msg" echo "$msg" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 288d4250..3b7cfb45 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -698,9 +698,10 @@ jobs: # prs-scores-and-run-options-2, 268 MB), from EBI's FTP server, keyed # on the sha256 tests/e2e/lib.sh pins. The cases take it from # E2E_DATA_CACHE/hapnest, check it and then run setup.sh with EBI - # unreachable; on a miss the first one downloads it and puts it there - # (setup_ancestry_panel in lib.sh). Saved like the entries above: on - # main, or a run by hand with save_images, on a miss. + # unreachable; on a miss the first one takes it from E2E_DATA_MIRROR + # (else EBI, through setup.sh) and puts it there (setup_ancestry_panel + # in lib.sh). Saved like the entries above: on main, or a run by hand + # with save_images, on a miss. - name: Data cache, HAPNEST panel id: hapnest env: @@ -730,8 +731,12 @@ jobs: E2E_TRACE: ${{ needs.e2e-plan.outputs.trace }} # The free disk a leg needs before its first case (e2e-run.sh). E2E_MIN_FREE_GB: 10 - # Copies of the pinned e2e data, tried before EBI (data_fetch in e2e-run.sh). + # Copies of the pinned e2e data, tried before EBI (data_fetch in + # e2e-run.sh, setup_ancestry_panel in tests/e2e/lib.sh). E2E_DATA_MIRROR: https://github.com/GeiserX/OwnGenome/releases/download/e2e-data-1 + # The most seconds one data cache fill may take; a fill still + # running then fails its case instead of the leg's time limit. + E2E_DATA_BUDGET: 2400 E2E_BASE_OUT: ${{ matrix.job == 'base' && format('{0}/base.tar.zst', runner.temp) || '' }} E2E_BASE_IN: ${{ matrix.job != 'base' && format('{0}/e2e-base/base.tar.zst', runner.temp) || '' }} # Keep the downloads to the end of the job only when a save step uses them. diff --git a/.github/workflows/nextflow.yml b/.github/workflows/nextflow.yml index 7c756324..8c9eda40 100644 --- a/.github/workflows/nextflow.yml +++ b/.github/workflows/nextflow.yml @@ -122,6 +122,30 @@ jobs: python3 -m json.tool nextflow_schema.json > /dev/null python3 -m json.tool assets/schema_input.json > /dev/null + # The stub run below pulls every image of the default tools: the pinned + # leg ran out of disk in it twice on 2026-11-05 (#300, job 127077180393; + # #290, run 41966270772). The free-disk step above runs first. On that + # leg, the free space under / and under Docker's root goes to the log + # and the summary, and + # below MIN_FREE_GB the job stops here, naming the cause, instead of + # failing in a task or in Nextflow's onComplete handler. + - name: Free disk before the stub runs + if: steps.scope.outputs.run == 'true' && matrix.line == 'pinned' + env: + MIN_FREE_GB: 20 + LINE: ${{ matrix.line }} + run: | + root=$(docker info --format '{{.DockerRootDir}}' 2> /dev/null || true) + root=${root:-/var/lib/docker} + sudo df -h / "$root" "$RUNNER_TEMP" || df -h / + free=$(sudo df --output=avail -k "$root" | tail -1) + msg="Free disk before the stub runs (nextflow ${LINE}): $((free / 1048576)) GB under the Docker root ${root}, at least ${MIN_FREE_GB} GB wanted" + echo "$msg" | tee -a "$GITHUB_STEP_SUMMARY" + if [ "$free" -lt $((MIN_FREE_GB * 1048576)) ]; then + echo "::error::${msg}: the runner has too little disk for the stub runs' images; re-run the job" + exit 1 + fi + # The default tools, each stub run inside its own image. - name: Stub run of the test profile with Docker if: steps.scope.outputs.run == 'true' && matrix.line == 'pinned' @@ -664,6 +688,7 @@ jobs: - name: Show what Nextflow was waiting for if: failure() && steps.scope.outputs.run == 'true' run: | + echo "Free disk when the job failed:"; df -h / "$RUNNER_TEMP" || true log=.nextflow.log [ -f "$log" ] || { echo "No .nextflow.log: no Nextflow run had started."; exit 0; } # A report, not a check: no command here may stop the step, and a diff --git a/assets/steps.tsv b/assets/steps.tsv index 835a84b9..17290263 100644 --- a/assets/steps.tsv +++ b/assets/steps.tsv @@ -72,7 +72,7 @@ step label tool recorded processes sections upstream hours hours_from hours_low 47 Viral DNA viral yes VIRAL_READS,VIRAL_REPORT viral 02 48 Exomiser (HPO terms) exomiser yes EXOMISER_PREPARE,EXOMISER,EXOMISER_REPORT exomiser 03 49 CHIP (clonal blood cells) chip yes CHIP_PREPARE,CHIP_MUTECT2,CHIP_ANNOTATE,CHIP_REPORT chip 02 -50 Mobile element insertions mei yes MEI_DEPTH,MEI,MEI_REPORT mei 02 -51 Mosaic alterations (MoChA) mocha yes MOCHA_PREP,MOCHA_PHASE,MOCHA_CALL,MOCHA_REPORT mosaic 03 -52 MUC1 VNTR (VNtyper) vntyper yes VNTYPER vntyper 02 -53 Hard loci (Locityper) locityper yes LOCITYPER locityper 02 +50 Mobile element insertions mei yes MEI_DEPTH,MEI,MEI_REPORT mei 02 MEGANE_IMAGE +51 Mosaic alterations (MoChA) mocha yes MOCHA_PREP,MOCHA_PHASE,MOCHA_CALL,MOCHA_REPORT mosaic 03 MOCHA_PANEL_VERSION,MOCHA_IMAGE +52 MUC1 VNTR (VNtyper) vntyper yes VNTYPER vntyper 02 VNTYPER_IMAGE +53 Hard loci (Locityper) locityper yes LOCITYPER locityper 02 LOCITYPER_PANEL_VERSION,LOCITYPER_IMAGE diff --git a/bin/owngenome_recheck.py b/bin/owngenome_recheck.py index 74900bf7..6dd4c69d 100755 --- a/bin/owngenome_recheck.py +++ b/bin/owngenome_recheck.py @@ -79,6 +79,13 @@ ) +# The opt-in steps' pins: the setup.sh option that installs their data and image (the main setup leaves +# them out). VNtyper's reference data is inside its image, which the run pulls, so it needs no option. +OPT_IN_SETUP = {"MEGANE_IMAGE": "--megane-kmers", "MOCHA_PANEL_VERSION": "--mocha-data", "MOCHA_IMAGE": "--mocha-data", + "LOCITYPER_PANEL_VERSION": "--locityper-data", "LOCITYPER_IMAGE": "--locityper-data", + "VNTYPER_IMAGE": None} + + class Failed(Exception): """A lookup with no usable answer.""" @@ -387,6 +394,11 @@ def classify(item, release_tag, release_pins, release_error, clinvar=False): def action(item, sample): c, key = item["class"], item["item"] if c == "rerun-ready": + if item["kind"] == "pin" and key in OPT_IN_SETUP: + opt = OPT_IN_SETUP[key] + how = (f"run ./scripts/setup.sh {opt} $GENOME_DIR (the main setup leaves this step's data out)" + if opt else "the run pulls the image") + return f"this checkout pins {item['installed']}: {how}, then rerun: bin/owngenome_recheck.py --sample {sample} --rerun" if item["kind"] == "pin": return (f"this checkout pins {item['installed']}: run ./scripts/setup.sh (it installs the data), " f"then rerun: bin/owngenome_recheck.py --sample {sample} --rerun") @@ -437,6 +449,18 @@ def add(item, fn, *args, clinvar=False): if key == "hla_database" and not this_hla: item["error"] = "could not read this checkout's HLA_DB_RELEASE (scripts/lib/common.sh)" add(item, answers.pinned, key, lookup, pattern) + # The other versions.env pins a step reads (steps 50 to 53): no freshness source exists for them, so + # nothing is looked up; a pin here that differs from the run's makes its step rerun-ready. + known = {k for k, *_ in PINNED} + for key in dict.fromkeys(r[:-5] if r.endswith("@self") else r for s in steps for r in reads_of(s)): + if key in known or key not in this_pins: + continue + ran, inst = pin_value(key, version.get(key)), pin_value(key, this_pins.get(key)) + item = {"item": key, "label": f"{key} (no upstream lookup)", "kind": "pin", "ran_with": ran, + "installed": inst, "class": "rerun-ready" if ran and inst and ran != inst else "no-newer-release"} + item["steps"] = affected_steps(steps, [key]) + item["action"] = action(item, sample) + items.append(item) for (sec, key), value in sorted(man.items()): if sec != "data" or not key.startswith("pgs:"): continue diff --git a/docs/53-locityper.md b/docs/53-locityper.md index ffe45625..5972909c 100644 --- a/docs/53-locityper.md +++ b/docs/53-locityper.md @@ -87,7 +87,7 @@ Both reports carry a section "Locityper Second Opinion at Hard Loci" in the medi ## How it was checked -The image test (`tests/smoke/locityper.sh`, the `LOCITYPER_IMAGE` row of `tests/smoke/commands.tsv`) runs the database build and the step on the fixture's GIAB HG002 reads at HLA-DRB1, HLA-DQB1 and CYP2D6. The panel holds HG002's own two assembled haplotypes, which are the truth. Both called haplotypes at each locus must match them at QV 20 or more (one difference per 100 bases or fewer), aligned with `locityper align`. +The image test (`tests/smoke/locityper.sh`, the `LOCITYPER_IMAGE` row of `tests/smoke/commands.tsv`) runs the database build and the step on the fixture's GIAB HG002 reads at HLA-DRB1, HLA-DQB1 and CYP2D6. The panel holds HG002's own two assembled haplotypes, which are the truth. Both called haplotypes at each locus must match them at QV 20 or more (one difference per 100 bases or fewer), aligned with `locityper align`. That QV measures how close each called haplotype is to HG002's true one; it is not the `quality` column, Locityper's own genotype quality. The red control takes out of the panel HG002's haplotypes and every other haplotype within QV 20 of them, rebuilds the database and genotypes again; the same check must then fail at every locus. Leaving HG002 out with `--leave-out` is not enough: other people in the panel carry near copies of HG002's haplotypes, and the call stays close. diff --git a/docs/lessons-learned.md b/docs/lessons-learned.md index de6c9bae..01405ec5 100644 --- a/docs/lessons-learned.md +++ b/docs/lessons-learned.md @@ -638,3 +638,11 @@ Most bioinformatics containers run as non-root users. If writing to bind-mounted - **Failed:** in November 2026, fork runs and a cache-seeding run with no Actions cache for the two data entries had shards 2 and 5 cancelled at the 240-minute leg limit, still downloading. An Actions cache belongs to one repository, so a fork starts with none, and an evicted entry leaves main in the same place. - **Why:** both EBI hosts for 1000 Genomes (`ftp.1000genomes.ebi.ac.uk` and `ftp.ebi.ac.uk/1000g`) swung between a few KB/s and about 3 MB/s from one request to the next in the same hour. No other copy of the pinned v5c sites file exists: AWS's `1000genomes` bucket holds only v5b, and NCBI's 1000 Genomes path answers 404. - **Fix:** the pinned files are kept as assets of the GitHub release `e2e-data-1`, and `data_fetch` in `scripts/ci/e2e-run.sh` tries `E2E_DATA_MIRROR` once before the source. The pin checks both copies, so the mirror changes only where the bytes come from. A file missing from the release falls back to EBI. + +### A data fill with no time cap could still eat a leg's limit +- **Failed:** with the mirror in place, a data fill that falls back to EBI had no bound on its total time. `fetch` cuts only a stall (under 1 KB/s for two minutes), so a transfer that moves slowly runs on, and two shard legs reached their 240-minute limit on the 1000 Genomes sites VCF (fork run 41957608395). The PGS Catalog score files and the HAPNEST panel had no mirror copy at all. +- **Fix:** `data_fill` in `scripts/ci/e2e-run.sh` runs each fill within `E2E_DATA_BUDGET` seconds (2400 in `e2e.yml`) and stops it after that, with a message; its case then fails without running. The four score files and the HAPNEST panel are in `e2e-data-1` too, pinned by md5 (`PGS_MD5`) and sha256 (`HAPNEST_SHA256`), and tried there before EBI. + +### free-disk hid a cleanup that failed +- **Failed:** `.github/actions/free-disk` ran its deletions and the image prune in the background and never read their exit codes, so a failed part showed only as fewer GB freed. +- **Fix:** each part is waited for; a failure is a warning and a line in the job summary, and the step still passes. `tests/test_e2e_data_cache.sh` plants a failing `rm` and a failing prune. diff --git a/docs/testing.md b/docs/testing.md index 558dbc6b..f4040791 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -77,12 +77,12 @@ You can build it yourself on Linux with Docker, `bgzip`, `tabix` and `python3` a In CI the cases run in parallel legs, each case at most once: a pull request runs only the cases it can affect ([Which cases a pull request runs](#which-cases-a-pull-request-runs)), main and the monthly run run them all. On 2026-11-02 the 130 cases took 230 minutes one after another on one runner (run 41390434697), against a 300-minute job limit. Now: - `e2e-plan` splits the cases with [`scripts/ci/e2e-shards.sh`](https://github.com/GeiserX/OwnGenome/blob/main/scripts/ci/e2e-shards.sh). The leg `e2e base` runs the cases of stage `base` in [`scripts/ci/e2e-case-times.tsv`](https://github.com/GeiserX/OwnGenome/blob/main/scripts/ci/e2e-case-times.tsv) (every numbered case: the fixture check, setup validation, alignment, DeepVariant, GATK, the single steps on HG002, the reports and the Nextflow run; and `bash-step-08-hla`), because most other cases read what those wrote. Every other picked case goes to one of the shard legs (`e2e shard 1`, `e2e shard 2`, ...). A full run uses all `E2E_SHARDS` (6) legs. For picked cases, `e2e-shards.sh count` sets how many: the cases' total seconds divided by the heaviest group's seconds (the critical path, since a group is never split) or by 900 seconds when that is larger, rounded down, plus one more leg when the rest is 900 seconds or more, and never more than `E2E_SHARDS` (6). A shard leg spends 18 to 20 minutes on top of its cases (pulling images, fetching and unpacking the base outputs, the parity checks and the uploads; run 41847167444), so a leg with less than 15 minutes of cases costs more runner time than it saves in wall time. With every case that rule gives 4 or 5 legs, and that is why a full run does not use it: the runner's disk, not time, is what a leg runs out of. With 5 legs, shards of 30 and 37 cases filled one runner's disk until the runner itself died (run 41995851301, shard 3), while 6 legs passed on the same main (run 41995719742). -- The table holds each case's median time on the passing runs of nine e2e runs of 2026-11-04 (41766267038, 41767270253, 41773233871, 41774860458, 41779067613, 41790595358, 41799741941, 41807346887 and 41821518376), and its `needs`: the non-base cases whose outputs it reads. The needs were written from what each case says it reads (its header and its checks) and from the cases that failed in a split run without them (run 41455829120). A case and every case it needs, followed through, form a group that stays in one shard, in name order. Groups go to the least loaded shard, heaviest first. A new case with no row counts as 300 seconds and needs nothing. A case that runs well over 300 seconds also needs a row with its measured seconds, or the split balances it as a 300-second case. +- The table holds each case's median time on the passing runs of nine e2e runs of 2026-11-04 (41766267038, 41767270253, 41773233871, 41774860458, 41779067613, 41790595358, 41799741941, 41807346887 and 41821518376), and its `needs`: the non-base cases whose outputs it reads. The needs were written from what each case says it reads (its header and its checks) and from the cases that failed in a split run without them (run 41455829120). A case and every case it needs, followed through, form a group that stays in one shard, in name order. Groups go to the least loaded shard, heaviest first. Every case needs a row: `e2e-shards.sh check`, which `tests/test_e2e_shards.sh` runs on the real table in the unit tests, fails and names each case without one. The split itself still runs a case with no row, as a 300-second case that needs nothing, so add the row with the case's measured seconds from its first passing run. - The base leg runs first. It packs every file its cases wrote or changed (at its absolute path, with owners, modes and times) and uploads it as the artifact `e2e-base`, kept for 3 days so that "Re-run failed jobs" the next day still finds it. The shard legs start only after it passed (`needs:`), so a shard never holds a runner while it waits: when many runs share the account's runners, shards that started early and waited took the runners the base legs needed (run 41506673662 had three shards running and its base leg still queued). Each shard fetches the artifact, tried up to five times, takes the base leg's work area, unpacks the archive there, deletes it (about 4 GB the cases need) and runs its cases. When the base leg fails, the shards are skipped and `e2e` is red. - The shard that holds `nextflow-from-fastq-2-nextflow` and the one that holds `nextflow-hardening-2-nf` run the parity checks; the plan fails when either case runs and the case before it in its pair is in a different shard. - The required check is the job `e2e`. It merges every leg's results into one table of case, result, time and job, followed by the failed checks of each failed case, and it is red unless the plan and every leg succeeded, every picked case reported exactly one pass and each parity check passed once when its cases ran (and not at all when they did not). It is red too when `e2e-scope` did not succeed. A leg that was skipped, cancelled or cut by its limit leaves cases with no result, and that is red too. The full logs are in the `e2e-logs-` artifacts. -`tests/test_e2e_shards.sh` (unit tests) checks the split against planted faults: a case listed twice or missing, a stale or malformed table row, a need on a case that runs later, a pick that leaves out a need, and results missing, failed or reported twice; and the leg count. `tests/test_e2e_scope.sh` checks the pick: a case with no paths row, a second row, a row for no case and a glob that matches no file must each fail `e2e-scope.sh check`, and a docs-only change, a change to `main.nf`, a change to one helper, to one module, to a case and to a table row must each pick the cases listed below. +`tests/test_e2e_shards.sh` (unit tests) checks the split against planted faults: a case listed twice or missing, a stale or malformed table row, a need on a case that runs later, a pick that leaves out a need, and results missing, failed or reported twice; the leg count; and a case with no row, which `check` must name. `tests/test_e2e_scope.sh` checks the pick: a case with no paths row, a second row, a row for no case and a glob that matches no file must each fail `e2e-scope.sh check`, and a docs-only change, a change to `main.nf`, a change to one helper, to one module, to a case and to a table row must each pick the cases listed below. To measure the table again, run the E2E workflow by hand with `job: e2e` and `trace: true`. That run puts every non-base case in one shard, so it runs as long as one serial run. After each case `e2e-run.sh` records its time and, with the file systems remounted `strictatime`, every file under the work area and the checkout whose atime moved during the case, attributed to the case whose run last changed that file's ctime. The job `e2e` then prints a new `e2e-case-times.tsv` in its summary and uploads it as the artifact `e2e-case-times`; it keeps each row's stage and needs and adds the traced needs. It refuses a trace that saw no case read another case's file. The traced needs are an upper bound, not the table to commit: a file counts for the case that changed it last, so a case that rewrites or touches a shared file (a report, an index) shows up as a need of every later case that reads that file, and a case that scans a folder reads everything in it. On the trace of run 41454755531 they joined almost every case into one group, so check each traced need against the case before you add it. A read the trace cannot see, such as a test for whether a file exists, is not in the table: such a case fails in its shard, and its row then needs a hand-written `needs`. @@ -102,7 +102,7 @@ The base leg runs every base case whenever any case runs, because every shard st The images the cases use are cached in four GitHub Actions cache entries, one per group. `scripts/ci/e2e-run.sh --image-groups` names each image's group, taken from its repository name without the tag, so a version bump changes the key of one group only; a group whose key missed restores the newest older entry of that group and pulls only what changed. A group keeps its smallest images up to 1500 MB of zstd output, and the larger ones are pulled on every run. Four groups of 1500 MB and the five data entries below stay under the repository's 10 GB cache limit even while main replaces one group: 5.9 GB of images, 1.5 GB more for the group being replaced, and about 2.3 GB of data. Images pinned by digest are not cached, because `docker load` gives them back without the digest, and they are left out of the group's key, so a digest bump misses no entry. Only the base leg saves, and only on main (or in a run by hand with `save_images`, on any branch), after it pulled every image the cases use and got through its cases, and only the groups whose key missed: a cache saved by a pull request can be read by that pull request alone and would only push main's entries out. The job summary lists every image with its size and what the cache did with it: restored on an exact key hit, restored from an older entry, saved, or pulled. Every merge in the e2e scope starts a run on main, so the cache follows main's images. GitHub drops an entry no run has read for 7 days, so after a quiet week the next run on main fills the cache again. Until this layout the cache was one entry, and by October 2026 it had grown to about 11 GB, over the limit, so every run missed it. -Five downloads are cached the same way. Two come from EBI's 1000 Genomes server, which is slow and at times stalls: MoChA's chr20 panel with its manifest (0.64 GB, case `mocha-1`), keyed on `MOCHA_PANEL_MANIFEST_SHA256` and the chromosome, and the 1000 Genomes phase 3 sites VCF (1.36 GB, case `archaic-ancestry-1`), keyed on `ARCHAIC_1KG_SITES_MD5`. On a miss, each of those two is tried first at `E2E_DATA_MIRROR`, the GitHub release `e2e-data-1` that holds copies of the pinned files, and then at EBI; the pin checks both. Two are small and failed whole runs on 2026-11-04, because their case is in the base leg or a shard that cannot start without it: `hla.dat` of the two IPD-IMGT/HLA releases `bash-step-08-hla` installs, from raw.githubusercontent.com (which served it under 1 KB/s), with GENCODE's gene and exon files from EBI's FTP server (curl exit 7 and 28), and the four real PGS Catalog score files of `prs-scores-and-run-options-1-strand-sex`, from the same server. `scripts/ci/e2e-run.sh --data-key` names their keys: the releases and a hash of the code in `scripts/lib/common.sh` that installs them, and a hash of the score ids and their server. `scripts/ci/e2e-run.sh` hands each case its files just before it runs, at the paths `setup.sh` and `install_data_file` read them from, so they download nothing; the score files the case copies from the cache itself. On a miss it downloads them first, checked against their pins or the checksum files the servers publish beside them (`MD5SUMS`, `md5checksum.txt`, the `.md5` of each score), trying up to 6 times 30 s apart. A restored entry is checked again before use (`--data-check`): the IPD-IMGT/HLA and GENCODE entry against the sha256 list written when it was filled, the score files against their `.md5`; one that fails is downloaded again. `bash-step-08-hla` then runs with every HTTP and HTTPS request sent to a closed port, after a control shows that a request under that block fails at the closed port, so a warm cache cannot reach either host. Only the leg whose cases include the case restores and saves its entry, and it saves only on main or in a run by hand with `save_images`, as with the images. The job summary says whether each key hit, and the case's log says where its files came from. The fifth is the PGS Catalog's synthetic HAPNEST panel (0.27 GB, cases `prs-2-ancestry` and `prs-scores-and-run-options-2-panel-chrx`) from EBI's FTP server, which failed cases with curl exit 7 on 2026-11-04. It is keyed on the sha256 `tests/e2e/lib.sh` pins (`HAPNEST_SHA256`). The cases handle it themselves (`setup_ancestry_panel` in `lib.sh`): with the panel in the cache they check the sha256 and run `setup.sh --ancestry-panel` with every HTTP and HTTPS request sent to a closed port, so an EBI outage cannot fail them; on a miss `setup.sh` downloads the panel as a user's run does, and the first case puts it in the cache for the save step. A red control in `prs-2-ancestry` runs `setup.sh` with the network blocked and no panel, which must fail. Every download, there and in `setup.sh`, gives up on a transfer that stalls (`FETCH_STALL`, see `fetch` in `scripts/lib/common.sh`), and the next try resumes it. +Five downloads are cached the same way. Two come from EBI's 1000 Genomes server, which is slow and at times stalls: MoChA's chr20 panel with its manifest (0.64 GB, case `mocha-1`), keyed on `MOCHA_PANEL_MANIFEST_SHA256` and the chromosome, and the 1000 Genomes phase 3 sites VCF (1.36 GB, case `archaic-ancestry-1`), keyed on `ARCHAIC_1KG_SITES_MD5`. On a miss, each of those two is tried first at `E2E_DATA_MIRROR`, the GitHub release `e2e-data-1` that holds copies of the pinned files, and then at EBI; the pin checks both. The release also holds the four PGS Catalog score files and the HAPNEST panel below, so an EBI outage costs none of them on a cold cache. Two are small and failed whole runs on 2026-11-04, because their case is in the base leg or a shard that cannot start without it: `hla.dat` of the two IPD-IMGT/HLA releases `bash-step-08-hla` installs, from raw.githubusercontent.com (which served it under 1 KB/s), with GENCODE's gene and exon files from EBI's FTP server (curl exit 7 and 28), and the four real PGS Catalog score files of `prs-scores-and-run-options-1-strand-sex`, from the same server. `scripts/ci/e2e-run.sh --data-key` names their keys: the releases and a hash of the code in `scripts/lib/common.sh` that installs them, and a hash of the score ids and their server. `scripts/ci/e2e-run.sh` hands each case its files just before it runs, at the paths `setup.sh` and `install_data_file` read them from, so they download nothing; the score files the case copies from the cache itself. On a miss it downloads them first, checked against their pins or the checksum files the servers publish beside them (`MD5SUMS`, `md5checksum.txt`, the `.md5` of each score), trying up to 6 times 30 s apart. The four score files are pinned by md5 in `PGS_MD5` of `e2e-run.sh` (the md5 the PGS Catalog served beside each on 2026-11-06), tried on the mirror first and then at EBI; a restored or downloaded score must match its pin as well as its `.md5`. A file gets at most `E2E_MIRROR_BUDGET` seconds on the mirror (900, or half the fill's budget when that is less), so a mirror that trickles leaves EBI its turn. One fill may take at most `E2E_DATA_BUDGET` seconds (2400 in `e2e.yml`): a source that trickles above `fetch`'s stall limit would otherwise run on to the leg's 240-minute limit, as EBI's 1000 Genomes server did twice on 2026-11-05. A fill still running then is stopped and its case fails as `FAIL (data fill over 2400s)` without running, since its own download would be as slow. A restored entry is checked again before use (`--data-check`): the IPD-IMGT/HLA and GENCODE entry against the sha256 list written when it was filled, the score files against their `.md5`; one that fails is downloaded again. `bash-step-08-hla` then runs with every HTTP and HTTPS request sent to a closed port, after a control shows that a request under that block fails at the closed port, so a warm cache cannot reach either host. Only the leg whose cases include the case restores and saves its entry, and it saves only on main or in a run by hand with `save_images`, as with the images. The job summary says whether each key hit, and the case's log says where its files came from. The fifth is the PGS Catalog's synthetic HAPNEST panel (0.27 GB, cases `prs-2-ancestry` and `prs-scores-and-run-options-2-panel-chrx`) from EBI's FTP server, which failed cases with curl exit 7 on 2026-11-04. It is keyed on the sha256 `tests/e2e/lib.sh` pins (`HAPNEST_SHA256`). The cases handle it themselves (`setup_ancestry_panel` in `lib.sh`): with the panel in the cache, or on a miss from `E2E_DATA_MIRROR`, they check the sha256 and run `setup.sh --ancestry-panel` with every HTTP and HTTPS request sent to a closed port, so an EBI outage cannot fail them; when neither has it, `setup.sh` downloads the panel from EBI as a user's run does. The first case puts a panel that did not come from the cache in the cache for the save step. A red control in `prs-2-ancestry` runs `setup.sh` with the network blocked and no panel, which must fail. Every download, there and in `setup.sh`, gives up on a transfer that stalls (`FETCH_STALL`, see `fetch` in `scripts/lib/common.sh`), and the next try resumes it. How long it took in one job, before the split: a passing e2e job took about 105 to 205 minutes with 84 cases, and 230 minutes of cases with 130 (run 41390434697, 2026-11-02). The runner decides much of it, not the cache: the same CPU model took 106 minutes in one run and 166 in another. In October 2026, with 84 cases, the same code took 106 and 166 minutes on an AMD EPYC 9V45 (runs 39197502832 and 39075176392, the second of which also saved the cache), 112 minutes on an Intel Xeon 6973P (run 39088086725), 156 and 191 minutes on an AMD EPYC 7763 (runs 39075764278 and 39075201465) and 205 minutes on an AMD EPYC 9V74 (run 39197512594, which saved three groups). The image cache saves about 7 minutes of that: pulling every image took 9 minutes, restoring and loading the four groups took 2.5 (run 39075201465 restored 39 images in 118 seconds), and saving them all took 12. Each leg has a limit of 240 minutes (355 for a traced shard leg); the base leg waits up to 75 minutes for a fixture a pull request bumps. Each leg's first steps print the runner's CPU model and CPU count to the log and the job summary, so a later timing can name the hardware. diff --git a/scripts/ci/e2e-case-times.tsv b/scripts/ci/e2e-case-times.tsv index 21198267..96e74ca1 100644 --- a/scripts/ci/e2e-case-times.tsv +++ b/scripts/ci/e2e-case-times.tsv @@ -8,7 +8,9 @@ # needs the shard-stage cases whose outputs it reads, comma-separated, or - # for none; a case runs in the same shard as every case it needs, # after them (name order) -# A case with no row runs as a shard case of 300 seconds that needs nothing. +# Every case needs a row (e2e-shards.sh check, run by tests/test_e2e_shards.sh); +# the split still runs a case with no row, as a shard case of 300 seconds that +# needs nothing. # Stage base holds the numbered cases and bash-step-08-hla: most other cases # read what they wrote. The needs come from what each case says it reads (its # header and its checks) and from the cases that failed when a need was @@ -78,11 +80,14 @@ containers-software-versions shard 0 - couple-carrier-1 shard 101 - couple-common-sv-1 shard 46 - cpic-phenoconversion-1-medications shard 36 - +cyp21a2-short-reads-1-hg002 shard 11 - exomiser-1 shard 300 - expansions-and-rfc1-1-motif shard 9 - families-file-1 shard 76 - family-joint-1 shard 515 - fixitfelix-chr21-1 shard 98 - +followups-clinvar-traits-1-rules-html shard 2 - +followups-cn-1-couple-report shard 69 - genome-lookup-1 shard 7 - genotype-rule-rows-1-str-extra shard 8 - genotype-rules-1-phase shard 37 - @@ -141,6 +146,7 @@ setup-gencode-exons-and-nextflow-clip-check-1 shard 57 clinvar-clip-check-1 short-sample-id-stub shard 48 - single-source-lib-ownership shard 1 - str-catalog-1-strchive shard 12 - +str-extra-repeats-1-readings shard 16 - sv-mito-telomere-steps-1-sv-merge shard 5 - sv-mito-telomere-steps-2-mito shard 6 bash-step-20-numt sv-mito-telomere-steps-3-y-haplogroup shard 13 - diff --git a/scripts/ci/e2e-run.sh b/scripts/ci/e2e-run.sh index bc5de979..c0496df1 100755 --- a/scripts/ci/e2e-run.sh +++ b/scripts/ci/e2e-run.sh @@ -17,6 +17,8 @@ # --data-key NAME prints the Actions cache key of data entry NAME (hla, pgs). # --data-check NAME exits 0 when entry NAME in E2E_DATA_CACHE is whole and # matches its checksums (the e2e workflow saves only such an entry). +# --data-fill NAME fills entry NAME in E2E_DATA_CACHE as a case's run does, +# within E2E_DATA_BUDGET, and exits (124 when the budget ran out). # # Env: E2E_WORK work area (default ${RUNNER_TEMP:-/tmp}/e2e-work), about 25 GB # GH_TOKEN token for `gh release download` (the job's GITHUB_TOKEN) @@ -38,6 +40,13 @@ # $E2E_WORK/report/trace.tsv for `e2e-shards.sh times` # E2E_DATA_CACHE a folder of downloads kept between runs (see "Downloads # kept between runs" below); E2E_DATA_KEEP=true keeps it whole +# E2E_DATA_MIRROR a folder URL holding copies of the pinned downloads, +# tried before their sources +# E2E_DATA_BUDGET the most seconds one data cache fill may take +# (default 2400); a fill still running then fails its case +# E2E_MIRROR_BUDGET the most seconds one file may take on the mirror +# before its source is tried (default 900, or half of +# E2E_DATA_BUDGET when that is less) # # Every run writes $E2E_WORK/report/results.tsv ("CASE RESULT SECONDS JOB", # tab-separated) and report/extra.md (failed checks and notes), which the @@ -171,6 +180,13 @@ fi DATA_CACHE=${E2E_DATA_CACHE:-} HLA_CASE="${REPO}/tests/e2e/bash-step-08-hla.sh" PGS_CASE="${REPO}/tests/e2e/prs-scores-and-run-options-1-strand-sex.sh" +# PGS_MD5: "ID MD5" of the score files the e2e-data-1 release holds, the md5 +# the PGS Catalog served beside each on 2026-11-06 (its .md5 file). +# E2E_PGS_MD5 replaces the list (tests/test_e2e_data_cache.sh). +PGS_MD5=${E2E_PGS_MD5-"PGS002257 bb1b2b11a499c5930e8c835647e48296 +PGS000065 90ac7aabbbde9bdaa681dbdb66c0ebd8 +PGS000064 43b39c4450f423f0772bbd8779dd640f +PGS000066 b34f31b8b3a7dbbbb58882a70e14649d"} # hla_releases: the IPD-IMGT/HLA releases bash-step-08-hla installs (the # release before the pin, then the pin), from the case and scripts/lib/common.sh. hla_releases() { @@ -203,23 +219,190 @@ data_key() { # downloads passed their md5 checks, matches. pgs: each score of the case is # there with the md5 file the PGS Catalog serves beside it, and matches it. data_ok() { - local d="${DATA_CACHE:?}/$1" id + local d="${DATA_CACHE:?}/$1" id pin sum case "$1" in hla) [ -s "${d}/SHA256SUMS" ] && [ "$(wc -l < "${d}/SHA256SUMS")" -eq 4 ] \ && (cd "$d" && sha256sum --quiet -c SHA256SUMS) ;; pgs) + # A score pinned in PGS_MD5 must match its pin as well as its .md5. for id in $(sed -n 's/^NEW_IDS="\(.*\)"$/\1/p' "$PGS_CASE"); do + pin=$(awk -v i="$id" '$1 == i {print $2}' <<< "$PGS_MD5") + sum=$(md5sum "${d}/${id}.txt.gz" 2> /dev/null | cut -d' ' -f1) [ -s "${d}/${id}.txt.gz" ] && [ -s "${d}/${id}.txt.gz.md5" ] \ - && [ "$(md5sum "${d}/${id}.txt.gz" | cut -d' ' -f1)" = "$(awk '{print $1; exit}' "${d}/${id}.txt.gz.md5")" ] \ + && [ "$sum" = "$(awk '{print $1; exit}' "${d}/${id}.txt.gz.md5")" ] \ + && { [ -z "$pin" ] || [ "$sum" = "$pin" ]; } \ || return 1 done ;; *) return 1 ;; esac } +# --- Downloads kept between runs ---------------------------------------------- +# The two largest downloads of the cases come from EBI's 1000 Genomes server, +# which is slow and stalls at times: MoChA's chr20 panel (mocha-1, about +# 655 MB, with its manifest) and the 1000 Genomes phase 3 sites VCF +# (archaic-ancestry-1, about 1.5 GB). With E2E_DATA_CACHE set (the e2e +# workflow restores it from the Actions cache), the runner gives each of +# these cases its files before it starts, at the paths setup.sh reads them +# from, so setup.sh finds them and downloads nothing. A file missing from +# E2E_DATA_CACHE is downloaded there first, checked against its pin. The +# files are hard links where they can be (setup.sh deletes its copy, the +# cache keeps its own). Without E2E_DATA_KEEP=true (the workflow sets it on +# a run that saves the cache) a case's cached files are deleted after it, +# to give the disk back. +# E2E_DATA_MIRROR, when set, is a folder URL (a GitHub release) holding copies +# of these pinned files under their own names. Each file is tried there once +# first, then at its source. EBI's 1000 Genomes hosts swing between a few KB/s +# and MB/s from one request to the next, and a leg with no Actions cache could +# spend its whole time limit on one download. The pin checks both copies the +# same way, so the mirror can only save time, never change what a case reads. +# kill_tree PID: PID and every process under it (data_fetch's subshells and +# their curl). +kill_tree() { + local c + for c in $(pgrep -P "$1" 2> /dev/null); do kill_tree "$c"; done + kill -TERM "$1" 2> /dev/null || true +} +# within SECS CMD...: runs CMD (a function too) and stops it with everything +# under it after SECS seconds; returns CMD's status, or 124 when it was stopped. +within() { + local secs=$1 pid wd rc=0 over + shift + over=$(mktemp) + "$@" & pid=$! + ( sleep "$secs"; echo over > "$over"; kill_tree "$pid" ) & wd=$! + wait "$pid" || rc=$? + kill_tree "$wd" + wait "$wd" 2> /dev/null || true + if [ -s "$over" ]; then rm -f "$over"; return 124; fi + rm -f "$over" + return "$rc" +} +# mfetch URL DEST KIND SUM: fetch from the mirror, then from URL. Called +# inside data_fetch, after it sources scripts/lib/common.sh for fetch. The +# mirror try gets E2E_MIRROR_BUDGET seconds (default 900, or half of +# E2E_DATA_BUDGET when that is less), so a mirror that trickles leaves the +# source its turn within the fill's budget. +mfetch() { + local secs=${E2E_MIRROR_BUDGET:-} rc=0 + if [ -n "${E2E_DATA_MIRROR:-}" ]; then + if [ -z "$secs" ]; then + secs=$(( ${E2E_DATA_BUDGET:-2400} / 2 )) + [ "$secs" -le 900 ] || secs=900 + [ "$secs" -ge 1 ] || secs=1 + fi + FETCH_TRIES=1 within "$secs" fetch "${E2E_DATA_MIRROR%/}/${1##*/}" "$2" "$3" "$4" 2> /dev/null || rc=$? + if [ "$rc" -eq 0 ]; then + echo "Data cache: ${1##*/} from the mirror" + return 0 + fi + if [ "$rc" -eq 124 ]; then + echo "Data cache: ${1##*/} took more than ${secs} s on the mirror; downloading it from its source" + else + echo "Data cache: ${1##*/} not on the mirror; downloading it from its source" + fi + fi + fetch "$@" +} +# data_fetch NAME: fills E2E_DATA_CACHE/NAME with what is missing, in a +# subshell that sources scripts/lib/common.sh for fetch and the pins. +data_fetch() ( + d="${DATA_CACHE}/$1" + # shellcheck source=../lib/common.sh + . "${REPO}/scripts/lib/common.sh" || exit 1 + # More and longer-spaced tries than a user's run: EBI and GitHub's raw host + # stayed down for minutes at a time on 2026-11-04 (curl exit 7 and 28). + FETCH_TRIES=${E2E_FETCH_TRIES:-6} FETCH_WAIT=${E2E_FETCH_WAIT:-30} + case "$1" in + mocha) + m="${d}/manifest.txt" + { [ -s "$m" ] && [ "$(_digest sha256 "$m")" = "$MOCHA_PANEL_MANIFEST_SHA256" ]; } \ + || mfetch "${MOCHA_PANEL_URL}/20220804_manifest.txt" "$m" sha256 "$MOCHA_PANEL_MANIFEST_SHA256" || exit 1 + # mocha-1 cuts the panel to the fixture's chr20 slice. + f=1kGP_high_coverage_Illumina.chr20.filtered.SNV_INDEL_SV_phased_panel.vcf.gz + for e in "" .tbi; do + md5=$(awk -v n="${f}${e}" '$1 == n {print $2}' "$m") + [ -n "$md5" ] || { echo "ERROR: ${f}${e} is not in the panel manifest" >&2; exit 1; } + [ -s "${d}/raw/${f}${e}" ] && [ "$(_digest md5 "${d}/raw/${f}${e}")" = "$md5" ] && continue + mfetch "${MOCHA_PANEL_URL}/${f}${e}" "${d}/raw/${f}${e}" md5 "$md5" || exit 1 + done ;; + archaic) + url=$(sed -n 's/^ARCHAIC_1KG_SITES_URL=//p' "${REPO}/scripts/setup.sh") + [ -n "$url" ] || { echo "ERROR: no ARCHAIC_1KG_SITES_URL line in scripts/setup.sh" >&2; exit 1; } + s="${d}/1kg_sites.vcf.gz" + [ -s "$s" ] && [ "$(_digest md5 "$s")" = "$ARCHAIC_1KG_SITES_MD5" ] && exit 0 + mfetch "$url" "$s" md5 "$ARCHAIC_1KG_SITES_MD5" || exit 1 ;; + hla) + # The files install_data_file writes, made by install_data_file itself + # with the entry as GENOME_DIR: hla.dat of both releases (md5 from + # IPD-IMGT/HLA's md5checksum.txt) and GENCODE's genes and exons (md5 + # from its MD5SUMS). Their sha256 go to SHA256SUMS, which data_ok checks. + data_ok hla && exit 0 + rm -rf "$d" && mkdir -p "$d" || exit 1 + read -r other pin <<< "$(hla_releases)" + [ -n "$pin" ] || exit 1 + # Each install is tried again as a whole: install_data_file reads + # hla.dat.zip's md5 from md5checksum.txt in one request, outside fetch's + # tries. What an earlier try installed is kept. + ( GENOME_DIR=$d + unset SAMPLE + for job in "$other hla_dat" "$pin hla_dat" "$pin gencode_genes" "$pin gencode_exons"; do + read -r rel name <<< "$job" + for ((t = 1; ; t++)); do + HLA_DB_RELEASE=$rel install_data_file "$name" && break + [ "$t" -lt 3 ] || exit 1 + echo "Data cache: installing ${name} (${rel}) failed, try ${t} of 3; again in ${FETCH_WAIT} s" >&2 + sleep "$FETCH_WAIT" + done + done ) || exit 1 + (cd "$d" && find . -type f ! -name 'SHA256SUMS*' ! -name '*.part' | LC_ALL=C sort | xargs sha256sum) > "${d}/SHA256SUMS.tmp" \ + && mv "${d}/SHA256SUMS.tmp" "${d}/SHA256SUMS" && data_ok hla || exit 1 ;; + pgs) + # The four real PGS Catalog files of prs-scores-and-run-options-1, each + # with the .md5 file the PGS Catalog serves beside it. A score pinned + # in PGS_MD5 is tried on the mirror first, checked against the pin, and + # its .md5 file is written from the pin; any other score comes from + # the PGS Catalog with its served .md5, as before. + base=$(sed -n 's/^PGS_BASE=//p' "$PGS_CASE") + [ -n "$base" ] || { echo "ERROR: no PGS_BASE= line in ${PGS_CASE}" >&2; exit 1; } + for id in $(sed -n 's/^NEW_IDS="\(.*\)"$/\1/p' "$PGS_CASE"); do + f="${d}/${id}.txt.gz" url="${base}/${id}/ScoringFiles/Harmonized/${id}_hmPOS_GRCh38.txt.gz" + pin=$(awk -v i="$id" '$1 == i {print $2}' <<< "$PGS_MD5") + [ -s "$f" ] && [ -s "${f}.md5" ] && [ "$(_digest md5 "$f")" = "$(awk '{print $1; exit}' "${f}.md5")" ] \ + && { [ -z "$pin" ] || [ "$(_digest md5 "$f")" = "$pin" ]; } && continue + rm -f "$f" "${f}.md5" + if [ -n "$pin" ]; then + mfetch "$url" "$f" md5 "$pin" || exit 1 + echo "${pin} ${url##*/}" > "${f}.md5" + else + fetch "${url}.md5" "${f}.md5" || exit 1 + fetch "$url" "$f" md5 "$(awk '{print $1; exit}' "${f}.md5")" || exit 1 + fi + done + data_ok pgs || exit 1 ;; + *) echo "ERROR: no data cache entry ${1}" >&2; exit 1 ;; + esac +) +# data_fill NAME: data_fetch NAME within E2E_DATA_BUDGET seconds (default +# 2400). A fill that is still running then is stopped, its partial files +# kept for the next try, and data_fill returns 124 with a message that says +# so. Each try of fetch has its own stall limit, but a source that trickles +# without stalling is never cut by it: on 2026-11-05 EBI served the 1000 +# Genomes sites VCF slowly enough that two shard legs reached their 240-minute +# limit on that one download (fork run 41957608395). +data_fill() { + local budget=${E2E_DATA_BUDGET:-2400} rc=0 + case "$budget" in ''|*[!0-9]*) echo "ERROR: E2E_DATA_BUDGET must be a whole number of seconds, got '${budget}'" >&2; return 1 ;; esac + within "$budget" data_fetch "$1" || rc=$? + if [ "$rc" -eq 124 ]; then + echo "ERROR: Data cache: filling ${1} took more than E2E_DATA_BUDGET=${budget} s and was stopped; a source this slow would eat the job's time limit" >&2 + fi + return "$rc" +} case "${1:-}" in --data-key) data_key "${2:?--data-key NAME}"; exit ;; --data-check) data_ok "${2:?--data-check NAME}"; exit ;; + --data-fill) data_fill "${2:?--data-fill NAME}"; exit ;; esac # pull_all: pull every image of case_images that is not here yet, but the @@ -485,104 +668,6 @@ df --output=avail -k "$E2E_WORK" | tail -1 > "$LOW_FILE" LOW_PID=$! trap 'kill "$LOW_PID" 2> /dev/null || true' EXIT -# --- Downloads kept between runs ---------------------------------------------- -# The two largest downloads of the cases come from EBI's 1000 Genomes server, -# which is slow and stalls at times: MoChA's chr20 panel (mocha-1, about -# 655 MB, with its manifest) and the 1000 Genomes phase 3 sites VCF -# (archaic-ancestry-1, about 1.5 GB). With E2E_DATA_CACHE set (the e2e -# workflow restores it from the Actions cache), the runner gives each of -# these cases its files before it starts, at the paths setup.sh reads them -# from, so setup.sh finds them and downloads nothing. A file missing from -# E2E_DATA_CACHE is downloaded there first, checked against its pin. The -# files are hard links where they can be (setup.sh deletes its copy, the -# cache keeps its own). Without E2E_DATA_KEEP=true (the workflow sets it on -# a run that saves the cache) a case's cached files are deleted after it, -# to give the disk back. -DATA_CACHE=${E2E_DATA_CACHE:-} -# E2E_DATA_MIRROR, when set, is a folder URL (a GitHub release) holding copies -# of these pinned files under their own names. Each file is tried there once -# first, then at its source. EBI's 1000 Genomes hosts swing between a few KB/s -# and MB/s from one request to the next, and a leg with no Actions cache could -# spend its whole time limit on one download. The pin checks both copies the -# same way, so the mirror can only save time, never change what a case reads. -# mfetch URL DEST KIND SUM: fetch from the mirror, then from URL. Called -# inside data_fetch, after it sources scripts/lib/common.sh for fetch. -mfetch() { - if [ -n "${E2E_DATA_MIRROR:-}" ]; then - FETCH_TRIES=1 fetch "${E2E_DATA_MIRROR%/}/${1##*/}" "$2" "$3" "$4" 2> /dev/null && return 0 - echo "Data cache: ${1##*/} not on the mirror; downloading it from its source" - fi - fetch "$@" -} -# data_fetch NAME: fills E2E_DATA_CACHE/NAME with what is missing, in a -# subshell that sources scripts/lib/common.sh for fetch and the pins. -data_fetch() ( - d="${DATA_CACHE}/$1" - # shellcheck source=../lib/common.sh - . "${REPO}/scripts/lib/common.sh" || exit 1 - # More and longer-spaced tries than a user's run: EBI and GitHub's raw host - # stayed down for minutes at a time on 2026-11-04 (curl exit 7 and 28). - FETCH_TRIES=${E2E_FETCH_TRIES:-6} FETCH_WAIT=${E2E_FETCH_WAIT:-30} - case "$1" in - mocha) - m="${d}/manifest.txt" - { [ -s "$m" ] && [ "$(_digest sha256 "$m")" = "$MOCHA_PANEL_MANIFEST_SHA256" ]; } \ - || mfetch "${MOCHA_PANEL_URL}/20220804_manifest.txt" "$m" sha256 "$MOCHA_PANEL_MANIFEST_SHA256" || exit 1 - # mocha-1 cuts the panel to the fixture's chr20 slice. - f=1kGP_high_coverage_Illumina.chr20.filtered.SNV_INDEL_SV_phased_panel.vcf.gz - for e in "" .tbi; do - md5=$(awk -v n="${f}${e}" '$1 == n {print $2}' "$m") - [ -n "$md5" ] || { echo "ERROR: ${f}${e} is not in the panel manifest" >&2; exit 1; } - [ -s "${d}/raw/${f}${e}" ] && [ "$(_digest md5 "${d}/raw/${f}${e}")" = "$md5" ] && continue - mfetch "${MOCHA_PANEL_URL}/${f}${e}" "${d}/raw/${f}${e}" md5 "$md5" || exit 1 - done ;; - archaic) - url=$(sed -n 's/^ARCHAIC_1KG_SITES_URL=//p' "${REPO}/scripts/setup.sh") - [ -n "$url" ] || { echo "ERROR: no ARCHAIC_1KG_SITES_URL line in scripts/setup.sh" >&2; exit 1; } - s="${d}/1kg_sites.vcf.gz" - [ -s "$s" ] && [ "$(_digest md5 "$s")" = "$ARCHAIC_1KG_SITES_MD5" ] && exit 0 - mfetch "$url" "$s" md5 "$ARCHAIC_1KG_SITES_MD5" || exit 1 ;; - hla) - # The files install_data_file writes, made by install_data_file itself - # with the entry as GENOME_DIR: hla.dat of both releases (md5 from - # IPD-IMGT/HLA's md5checksum.txt) and GENCODE's genes and exons (md5 - # from its MD5SUMS). Their sha256 go to SHA256SUMS, which data_ok checks. - data_ok hla && exit 0 - rm -rf "$d" && mkdir -p "$d" || exit 1 - read -r other pin <<< "$(hla_releases)" - [ -n "$pin" ] || exit 1 - # Each install is tried again as a whole: install_data_file reads - # hla.dat.zip's md5 from md5checksum.txt in one request, outside fetch's - # tries. What an earlier try installed is kept. - ( GENOME_DIR=$d - unset SAMPLE - for job in "$other hla_dat" "$pin hla_dat" "$pin gencode_genes" "$pin gencode_exons"; do - read -r rel name <<< "$job" - for ((t = 1; ; t++)); do - HLA_DB_RELEASE=$rel install_data_file "$name" && break - [ "$t" -lt 3 ] || exit 1 - echo "Data cache: installing ${name} (${rel}) failed, try ${t} of 3; again in ${FETCH_WAIT} s" >&2 - sleep "$FETCH_WAIT" - done - done ) || exit 1 - (cd "$d" && find . -type f ! -name 'SHA256SUMS*' ! -name '*.part' | LC_ALL=C sort | xargs sha256sum) > "${d}/SHA256SUMS.tmp" \ - && mv "${d}/SHA256SUMS.tmp" "${d}/SHA256SUMS" && data_ok hla || exit 1 ;; - pgs) - # The four real PGS Catalog files of prs-scores-and-run-options-1, each - # with the .md5 file served beside it, checked against it. - base=$(sed -n 's/^PGS_BASE=//p' "$PGS_CASE") - [ -n "$base" ] || { echo "ERROR: no PGS_BASE= line in ${PGS_CASE}" >&2; exit 1; } - for id in $(sed -n 's/^NEW_IDS="\(.*\)"$/\1/p' "$PGS_CASE"); do - f="${d}/${id}.txt.gz" url="${base}/${id}/ScoringFiles/Harmonized/${id}_hmPOS_GRCh38.txt.gz" - [ -s "$f" ] && [ -s "${f}.md5" ] && [ "$(_digest md5 "$f")" = "$(awk '{print $1; exit}' "${f}.md5")" ] && continue - rm -f "$f" "${f}.md5" - fetch "${url}.md5" "${f}.md5" || exit 1 - fetch "$url" "$f" md5 "$(awk '{print $1; exit}' "${f}.md5")" || exit 1 - done - data_ok pgs || exit 1 ;; - *) echo "ERROR: no data cache entry ${1}" >&2; exit 1 ;; - esac -) # data_name CASE: the E2E_DATA_CACHE entry CASE reads, and where setup.sh # wants it ("-": the case reads the entry itself), then "offline" when the # case runs with the network blocked once its files are in place. @@ -611,11 +696,13 @@ offline_blocked() { } # data_seed CASE: before CASE, its files from E2E_DATA_CACHE. A failure here # only warns and costs the download: setup.sh in the case fetches what is -# not there. Sets CASE_ENV to OFFLINE when the case's entry says "offline" +# not there. A fill that runs past E2E_DATA_BUDGET sets CASE_DATA_FAILED +# instead: the case does not run (its own download would be as slow) and +# counts as failed. Sets CASE_ENV to OFFLINE when the case's entry says "offline" # and every file is in place, after a control shows the block holds. data_seed() { local n dest flag f t placed=true - CASE_ENV=() CASE_CONTROL_FAILED=false + CASE_ENV=() CASE_CONTROL_FAILED=false CASE_DATA_FAILED=false [ -n "$DATA_CACHE" ] || return 0 read -r n dest flag <<< "$(data_name "$1")" [ -n "$n" ] || return 0 @@ -624,7 +711,14 @@ data_seed() { else echo "Data cache: ${n} not in the Actions cache; downloading it" fi - data_fetch "$n" || { echo "::warning::Data cache: could not fill ${n}; ${1%.sh} downloads it itself"; return 0; } + local rc=0 + data_fill "$n" || rc=$? + if [ "$rc" -eq 124 ]; then + echo "::error::Data cache: filling ${n} took more than E2E_DATA_BUDGET=${E2E_DATA_BUDGET:-2400} s; ${1%.sh} does not run and counts as failed" + CASE_DATA_FAILED=true + return 0 + fi + [ "$rc" -eq 0 ] || { echo "::warning::Data cache: could not fill ${n}; ${1%.sh} downloads it itself"; return 0; } [ "$dest" = - ] && { echo "Data cache: ${n} is whole; ${1%.sh} reads it from ${DATA_CACHE}/${n}"; return 0; } while IFS= read -r f; do t="${dest}/${f#./}" @@ -730,10 +824,15 @@ for c in "${CASES[@]}"; do tstart=$(date +%s.%N) echo "::group::${name}" data_seed "$c" - set +e - timeout -k 30 "$CASE_TIMEOUT" "${CASE_ENV[@]}" bash "${REPO}/tests/e2e/${c}" 2>&1 | tee "$log" - rc=${PIPESTATUS[0]} - set -e + if $CASE_DATA_FAILED; then + echo "${name}: not run: its data fill ran past E2E_DATA_BUDGET" | tee "$log" + rc=-2 + else + set +e + timeout -k 30 "$CASE_TIMEOUT" "${CASE_ENV[@]}" bash "${REPO}/tests/e2e/${c}" 2>&1 | tee "$log" + rc=${PIPESTATUS[0]} + set -e + fi data_drop "$c" # A lazy image goes when its case ends, so a leg holds one at a time; a # later case that runs it pulls it again (DeepVariant's and PharmCAT's images @@ -747,6 +846,7 @@ for c in "${CASES[@]}"; do case "$rc" in 0) result=pass ;; -1) result="FAIL (network-block control)"; failed=$((failed + 1)) ;; + -2) result="FAIL (data fill over ${E2E_DATA_BUDGET:-2400}s)"; failed=$((failed + 1)) ;; 124) result="FAIL (timeout ${CASE_TIMEOUT}s)"; failed=$((failed + 1)) ;; *) result="FAIL (exit ${rc})"; failed=$((failed + 1)) ;; esac diff --git a/scripts/ci/e2e-shards.sh b/scripts/ci/e2e-shards.sh index d384d278..99741c04 100755 --- a/scripts/ci/e2e-shards.sh +++ b/scripts/ci/e2e-shards.sh @@ -7,6 +7,7 @@ # scripts/ci/e2e-shards.sh cover FILE # scripts/ci/e2e-shards.sh merge EXPECTED RESULTS... # scripts/ci/e2e-shards.sh times TRACE... +# scripts/ci/e2e-shards.sh check # # base prints the base cases, one per line, in name order. The job # "e2e base" runs them once, and every shard starts from what @@ -35,6 +36,10 @@ # scripts/ci/e2e-run.sh with E2E_TRACE=true). Keeps each case's # stage and needs, adds the traced needs. Exits 1 when the trace # saw no case read another case's files, which means it saw nothing. +# check exits 1 unless the table is well formed and every case has a +# row, naming each case with none (tests/test_e2e_shards.sh runs it +# on the real table). The split itself still runs a case with no +# row, as a shard case of 300 seconds that needs nothing. # # Env: E2E_CASE_DIR the case directory (default tests/e2e) # E2E_TIMES the table (default scripts/ci/e2e-case-times.tsv) @@ -281,6 +286,17 @@ times() { }' <(cases) <(rows) "$@" } +# check: the table checks of every command, plus a row for every case. +check() { + local missing + [ -r "$TIMES" ] || die "no table to check: ${TIMES}" + [ -n "$(cases)" ] || die "no case in ${CASE_DIR}" + check_table + missing=$(awk -F'\t' 'FILENAME == ARGV[1] { r[$1] = 1; next } !($1 in r)' <(rows) <(cases)) + [ -z "$missing" ] || die "$(printf '%s\n' "in ${TIMES}, no row for:" "$missing" "Add each with its stage, its measured seconds and its needs.")" + echo "OK: every case of ${CASE_DIR} has a row in ${TIMES}" +} + cmd=${1:-} [ "$#" -gt 0 ] && shift case "$cmd" in @@ -290,5 +306,6 @@ case "$cmd" in cover) cover "${1:?Usage: $0 cover FILE}" ;; merge) [ "$#" -ge 1 ] || die "Usage: $0 merge EXPECTED RESULTS..."; merge "$@" ;; times) times "$@" ;; - *) sed -n '2,46p' "$0" >&2; exit 2 ;; + check) check ;; + *) sed -n '2,51p' "$0" >&2; exit 2 ;; esac diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh index 6a515856..6d5de4ec 100755 --- a/tests/e2e/lib.sh +++ b/tests/e2e/lib.sh @@ -110,7 +110,7 @@ print("\n".join(sorted(out))) PY } -# --- The ancestry panel, from the Actions cache when it is there ----------------- +# --- The ancestry panel, from the Actions cache or the mirror when it is there --- # EBI's FTP server fails now and then (curl exit 7). The e2e workflow keeps the # PGS Catalog's HAPNEST panel in the Actions cache (E2E_DATA_CACHE/hapnest, # keyed on the pin below) for prs-2-ancestry and @@ -123,40 +123,61 @@ NET_BLOCKED=(env http_proxy=http://127.0.0.1:9 https_proxy=http://127.0.0.1:9 HT HTTPS_PROXY=http://127.0.0.1:9 no_proxy= NO_PROXY= FETCH_TRIES=1 FETCH_WAIT=0) _sha256() { sha256sum "$1" 2>/dev/null | cut -d' ' -f1; } # setup_ancestry_panel NAME PANEL: setup.sh --ancestry-panel for panel NAME -# (installed at PANEL), its log in STEP_LOG, with checks. From the cache, the -# panel is checked against HAPNEST_SHA256 and setup.sh runs with the network -# blocked; on a miss setup.sh downloads it as a user's run does, and with -# E2E_HAPNEST_KEEP=true a panel that matches the pin goes into the cache. +# (installed at PANEL), its log in STEP_LOG, with checks. The HAPNEST panel +# comes from the Actions cache, an earlier install at PANEL, or else from +# E2E_DATA_MIRROR (the e2e-data-1 release, which holds a copy), each checked +# against HAPNEST_SHA256; then setup.sh runs with the network blocked. When +# none has it, setup.sh downloads it from EBI as a user's run does. With E2E_HAPNEST_KEEP=true a +# panel that matches the pin and did not come from the cache goes into it. setup_ancestry_panel() { - local name=$1 panel=$2 cached="${E2E_DATA_CACHE:-}/hapnest/${1}.tar.zst" from=false + local name=$1 panel=$2 cached="${E2E_DATA_CACHE:-}/hapnest/${1}.tar.zst" from="" if [ "$name" = GRCh38_HAPNEST_reference ] && [ -n "${E2E_DATA_CACHE:-}" ] && [ -s "$cached" ]; then if [ "$(_sha256 "$cached")" = "$HAPNEST_SHA256" ]; then mkdir -p "${panel%/*}" ln -f "$cached" "$panel" 2>/dev/null || cp -f "$cached" "$panel" - from=true + from="the Actions cache" else - echo "Data cache: ${cached} does not match its pin; setup.sh downloads the panel" + echo "Data cache: ${cached} does not match its pin; trying the mirror, then setup.sh downloads it" rm -f "$cached" fi fi - echo "+ ANCESTRY_PANEL_NAME=${name} scripts/setup.sh --ancestry-panel (panel from the Actions cache: ${from})" - if $from; then + # A panel an earlier case installed, when it matches the pin, is used as it is. + if [ -z "$from" ] && [ "$name" = GRCh38_HAPNEST_reference ] && [ -s "$panel" ] \ + && [ "$(_sha256 "$panel")" = "$HAPNEST_SHA256" ]; then + from="an earlier install" + fi + if [ -z "$from" ] && [ "$name" = GRCh38_HAPNEST_reference ] && [ -n "${E2E_DATA_MIRROR:-}" ]; then + mkdir -p "${panel%/*}" + # One try, cut when it stalls or takes over 15 minutes: EBI is next. + if curl -fsSL --connect-timeout 30 --speed-limit 1024 --speed-time 120 --max-time 900 \ + -o "${panel}.mirror" "${E2E_DATA_MIRROR%/}/${name}.tar.zst" \ + && [ "$(_sha256 "${panel}.mirror")" = "$HAPNEST_SHA256" ]; then + mv -f "${panel}.mirror" "$panel" + from="the mirror" + else + rm -f "${panel}.mirror" + echo "Data cache: ${name} not on the mirror or not matching its pin; setup.sh downloads it from EBI" + fi + fi + echo "+ ANCESTRY_PANEL_NAME=${name} scripts/setup.sh --ancestry-panel (panel from: ${from:-EBI, by setup.sh})" + if [ -n "$from" ]; then "${NET_BLOCKED[@]}" ANCESTRY_PANEL_NAME="$name" "${REPO}/scripts/setup.sh" --ancestry-panel "$GENOME_DIR" 2>&1 | tee "$STEP_LOG" - check_eq "setup.sh --ancestry-panel exits 0 from the cached panel, with EBI unreachable" "${PIPESTATUS[0]}" 0 + check_eq "setup.sh --ancestry-panel exits 0 from the panel of ${from}, with EBI unreachable" "${PIPESTATUS[0]}" 0 check "and downloads nothing" lacks 'Downloading' "$(cat "$STEP_LOG")" else ANCESTRY_PANEL_NAME="$name" "${REPO}/scripts/setup.sh" --ancestry-panel "$GENOME_DIR" 2>&1 | tee "$STEP_LOG" check_eq "setup.sh --ancestry-panel exits 0" "${PIPESTATUS[0]}" 0 if [ "$name" = GRCh38_HAPNEST_reference ]; then check_eq "the downloaded panel matches its pin" "$(_sha256 "$panel")" "$HAPNEST_SHA256" - if [ -n "${E2E_DATA_CACHE:-}" ] && [ "${E2E_HAPNEST_KEEP:-false}" = true ] && [ "$(_sha256 "$panel")" = "$HAPNEST_SHA256" ]; then - mkdir -p "${cached%/*}" - ln -f "$panel" "$cached" 2>/dev/null || cp -f "$panel" "$cached" - echo "Data cache: the panel is in ${cached} for the Actions cache" - fi fi fi - echo "${CASE_NAME}: ancestry panel from the Actions cache ${from}" >> "$E2E_NOTES" + if [ "$name" = GRCh38_HAPNEST_reference ] && [ "$from" != "the Actions cache" ] && [ -n "${E2E_DATA_CACHE:-}" ] \ + && [ "${E2E_HAPNEST_KEEP:-false}" = true ] && [ "$(_sha256 "$panel")" = "$HAPNEST_SHA256" ]; then + mkdir -p "${cached%/*}" + ln -f "$panel" "$cached" 2>/dev/null || cp -f "$panel" "$cached" + echo "Data cache: the panel is in ${cached} for the Actions cache" + fi + echo "${CASE_NAME}: ancestry panel from ${from:-EBI}" >> "$E2E_NOTES" } finish() { diff --git a/tests/test_e2e_data_cache.sh b/tests/test_e2e_data_cache.sh index 0462bb35..05c47bcd 100755 --- a/tests/test_e2e_data_cache.sh +++ b/tests/test_e2e_data_cache.sh @@ -2,7 +2,10 @@ # The data cache entries hla and pgs of scripts/ci/e2e-run.sh: --data-check # passes a whole entry and fails a changed, missing or partial one (the e2e # workflow saves and uses only an entry that passes), and --data-key names -# the releases. No network, no docker. +# the releases; a data fill that runs past E2E_DATA_BUDGET stops with its +# message (a planted slow mirror on 127.0.0.1, the sources blocked); and +# .github/actions/free-disk names a cleanup that failed in +# its summary (fake sudo, docker and df). No network, no docker. set -uo pipefail REPO="$(cd "$(dirname "$0")/.." && pwd)" RUN="${REPO}/scripts/ci/e2e-run.sh" @@ -12,7 +15,8 @@ failed=0 # expect DESC 0|1 NAME: --data-check NAME on $T exits as said. expect() { local rc=0 - E2E_DATA_CACHE=$T "$RUN" --data-check "$3" > /dev/null 2>&1 || rc=$? + # PINS, when set, replaces e2e-run.sh's PGS_MD5. + env ${PINS+"E2E_PGS_MD5=$PINS"} E2E_DATA_CACHE="$T" "$RUN" --data-check "$3" > /dev/null 2>&1 || rc=$? if { [ "$2" = 0 ] && [ "$rc" -eq 0 ]; } || { [ "$2" = 1 ] && [ "$rc" -ne 0 ]; }; then echo "PASS: $1" else echo "FAIL: $1 (exit ${rc})"; failed=1; fi } @@ -41,7 +45,12 @@ for id in $ids; do echo "$id" | gzip -c > "${T}/pgs/${id}.txt.gz" echo "$(md5sum < "${T}/pgs/${id}.txt.gz" | cut -d' ' -f1) ${id}_hmPOS_GRCh38.txt.gz" > "${T}/pgs/${id}.txt.gz.md5" done -expect "pgs: every score with a matching .md5 passes" 0 pgs +# PINS stands in for PGS_MD5 of e2e-run.sh: these files are made up. +PINS=$(for id in $ids; do echo "${id} $(awk '{print $1}' "${T}/pgs/${id}.txt.gz.md5")"; done) +expect "pgs: every score that matches its .md5 and its pin passes" 0 pgs +saved=$PINS; unset PINS +expect "pgs: a score that matches its .md5 but not its pin (the real PGS_MD5) fails" 1 pgs +PINS=$saved last=${ids##* } echo x | gzip -c > "${T}/pgs/${last}.txt.gz" expect "pgs: a score that does not match its .md5 fails" 1 pgs @@ -59,6 +68,143 @@ if [ -n "$other" ] && [ -n "$pin" ] && [ -n "$gencode" ] \ else echo "FAIL: the hla key does not match its inputs (${other:-?} ${pin:-?} ${gencode:-?}): ${key}"; failed=1; fi if "$RUN" --data-key pgs | grep -qE '^e2e-data-pgs-[0-9a-f]{16}$'; then echo "PASS: the pgs key is a hash" else echo "FAIL: the pgs key"; failed=1; fi + +# --- the data fill: the mirror, its deadline and the budget ------------------ +# A mirror on 127.0.0.1; every other host is sent to a closed port, so the +# PGS Catalog, the source after the mirror, always fails. Its modes: +# slow answers every file at about 2 KB/s, fast enough that fetch's stall +# limit (1 KB/s) never cuts it; +# 404 has no file; +# DIR serves the files of DIR. +# Checks: a slow mirror is cut at E2E_MIRROR_BUDGET and the source is tried +# within the fill's budget; a fill still running at E2E_DATA_BUDGET stops +# with its message (exit 124); a mirror with every file pinned fills the +# entry; a mirror whose bytes do not match the pins fills nothing. The 404 +# mirror is the control: it fails at once, with no budget message. +SL="${T}/slow" +mkdir -p "$SL" +cat > "${SL}/server.py" <<'EOF' +import http.server, os, sys, time +MODE = sys.argv[1] +class H(http.server.BaseHTTPRequestHandler): + def do_GET(self): + if MODE == "404": + self.send_error(404); return + if MODE != "slow": + f = os.path.join(MODE, os.path.basename(self.path)) + if not os.path.isfile(f): + self.send_error(404); return + data = open(f, "rb").read() + self.send_response(200) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data); return + self.send_response(200) + self.send_header("Content-Length", str(10 ** 8)) + self.end_headers() + try: + while True: + self.wfile.write(b"x" * 512); self.wfile.flush(); time.sleep(0.25) + except (BrokenPipeError, ConnectionResetError): + pass + def log_message(self, *a): + pass +s = http.server.ThreadingHTTPServer(("127.0.0.1", 0), H) +print(s.server_address[1], flush=True) +s.serve_forever() +EOF +# fill TAG MODE DATA_BUDGET MIRROR_BUDGET: "EXIT SECONDS" of --data-fill pgs +# against the mirror; its log in ${SL}/TAG.log, the entry in ${SL}/TAG.cache. +fill() { + local port pid rc=0 t0 + python3 "${SL}/server.py" "$2" > "${SL}/port.$1" & pid=$! + for _ in $(seq 50); do [ -s "${SL}/port.$1" ] && break; sleep 0.1; done + port=$(head -n 1 "${SL}/port.$1") + t0=$(date +%s) + env http_proxy=http://127.0.0.1:9 https_proxy=http://127.0.0.1:9 HTTP_PROXY=http://127.0.0.1:9 HTTPS_PROXY=http://127.0.0.1:9 \ + no_proxy=127.0.0.1 NO_PROXY=127.0.0.1 E2E_FETCH_TRIES=1 E2E_FETCH_WAIT=0 E2E_DATA_BUDGET="$3" E2E_MIRROR_BUDGET="$4" \ + E2E_PGS_MD5="$PINS" E2E_DATA_CACHE="${SL}/$1.cache" E2E_DATA_MIRROR="http://127.0.0.1:${port}" \ + timeout 60 "$RUN" --data-fill pgs > "${SL}/$1.log" 2>&1 || rc=$? + kill "$pid" 2> /dev/null; wait "$pid" 2> /dev/null + echo "$rc $(( $(date +%s) - t0 ))" +} +read -r rc secs <<< "$(fill slow slow 3 30)" +if [ "$rc" = 124 ] && [ "$secs" -lt 30 ] && grep -q 'filling pgs took more than E2E_DATA_BUDGET=3 s' "${SL}/slow.log"; then + echo "PASS: a slow mirror stops the fill at its 3 s budget (exit ${rc} after ${secs} s) with its message" +else echo "FAIL: the slow fill gave exit ${rc} after ${secs} s: $(tail -n 3 "${SL}/slow.log")"; failed=1; fi +if pgrep -f "${SL}/slow.cache" > /dev/null 2>&1; then echo "FAIL: a download of the stopped fill still runs"; failed=1 +else echo "PASS: nothing of the stopped fill still runs"; fi +read -r rc secs <<< "$(fill deadline slow 50 2)" +if [ "$rc" != 0 ] && [ "$rc" != 124 ] && [ "$secs" -lt 30 ] && grep -q 'took more than 2 s on the mirror; downloading it from its source' "${SL}/deadline.log" \ + && ! grep -q 'E2E_DATA_BUDGET' "${SL}/deadline.log"; then + echo "PASS: a slow mirror is cut at its 2 s deadline and the source gets its turn inside the budget (exit ${rc} after ${secs} s)" +else echo "FAIL: the mirror deadline gave exit ${rc} after ${secs} s: $(tail -n 3 "${SL}/deadline.log")"; failed=1; fi +read -r rc secs <<< "$(fill 404 404 30 10)" +if [ "$rc" != 0 ] && [ "$rc" != 124 ] && [ "$secs" -lt 30 ] && ! grep -q 'E2E_DATA_BUDGET' "${SL}/404.log"; then + echo "PASS: control: a mirror that answers 404, with the source blocked, fails at once (exit ${rc} after ${secs} s) with no budget message" +else echo "FAIL: control gave exit ${rc} after ${secs} s: $(tail -n 3 "${SL}/404.log")"; failed=1; fi +# The release names the score files as the PGS Catalog does. +mkdir -p "${SL}/good" "${SL}/bad" +for id in $ids; do + echo "$id" | gzip -c > "${SL}/good/${id}_hmPOS_GRCh38.txt.gz" + echo "not ${id}" | gzip -c > "${SL}/bad/${id}_hmPOS_GRCh38.txt.gz" +done +PINS=$(for id in $ids; do echo "${id} $(md5sum < "${SL}/good/${id}_hmPOS_GRCh38.txt.gz" | cut -d' ' -f1)"; done) +read -r rc secs <<< "$(fill good "${SL}/good" 30 10)" +if [ "$rc" = 0 ] && E2E_PGS_MD5=$PINS E2E_DATA_CACHE="${SL}/good.cache" "$RUN" --data-check pgs \ + && grep -q "_hmPOS_GRCh38.txt.gz from the mirror" "${SL}/good.log"; then + echo "PASS: a mirror with every score pinned fills the entry, with the source blocked, logs each file as from the mirror, and it checks out (${secs} s)" +else echo "FAIL: the good mirror gave exit ${rc}: $(tail -n 3 "${SL}/good.log")"; failed=1; fi +read -r rc secs <<< "$(fill bad "${SL}/bad" 30 10)" +if [ "$rc" != 0 ] && [ -z "$(find "${SL}/bad.cache" -name '*.txt.gz' 2> /dev/null)" ] && grep -q 'not on the mirror' "${SL}/bad.log"; then + echo "PASS: a mirror whose bytes do not match the pins is refused and leaves no score in the entry (exit ${rc})" +else echo "FAIL: the bad mirror gave exit ${rc}: $(find "${SL}/bad.cache" -type f 2> /dev/null | head -3) $(tail -n 3 "${SL}/bad.log")"; failed=1; fi + +# --- free-disk: a cleanup that fails is named in the summary ----------------- +# The action's script, run as GitHub runs a composite bash step, with sudo, +# docker and df faked on PATH: sudo fails for the planted path, docker's prune +# fails in the second run. The control run, with nothing failing, must not +# name a failure. +FD="${T}/free-disk" +mkdir -p "${FD}/bin" "${FD}/ok" "${FD}/planted-fail" +awk '/^ run: \|/ {on = 1; next} on && /^ / {sub(/^ /, ""); print; next} on && NF {exit}' \ + "${REPO}/.github/actions/free-disk/action.yml" > "${FD}/run.sh" +grep -q 'failed' "${FD}/run.sh" || { echo "FAIL: could not read the free-disk script"; failed=1; } +cat > "${FD}/bin/sudo" <<'EOF' +#!/usr/bin/env bash +case "$*" in + *planted-fail*) echo "rm: cannot remove: Permission denied" >&2; exit 1 ;; + *"${FD_ROOT:?}"/*) exec "$@" ;; + *) echo "fake sudo: refused, outside the test folder: $*" >&2; exit 9 ;; +esac +EOF +cat > "${FD}/bin/docker" <<'EOF' +#!/usr/bin/env bash +[ "${FAKE_PRUNE_FAILS:-false}" = true ] && exit 3 +exit 0 +EOF +cat > "${FD}/bin/df" <<'EOF' +#!/usr/bin/env bash +printf 'Avail\n10485760\n' +EOF +chmod +x "${FD}/bin/"* +freedisk() { # freedisk PATHS [PRUNE_FAILS]: the summary the script wrote + : > "${FD}/summary" + FD_ROOT=$FD PATH="${FD}/bin:${PATH}" FREE_DISK_PATHS="$1" FAKE_PRUNE_FAILS="${2:-false}" GITHUB_STEP_SUMMARY="${FD}/summary" \ + bash --noprofile --norc -eo pipefail "${FD}/run.sh" > "${FD}/log" 2>&1 + echo "exit $?: $(cat "${FD}/summary")" +} +out=$(freedisk "${FD}/ok") +if [[ "$out" == "exit 0: Free disk:"* ]] && [[ "$out" != *failed* ]] && [ ! -e "${FD}/ok" ]; then + echo "PASS: free-disk control: nothing failed, nothing named (${out})" +else echo "FAIL: free-disk control: ${out}"; failed=1; fi +mkdir -p "${FD}/ok" +out=$(freedisk "${FD}/ok ${FD}/planted-fail" true) +if [[ "$out" == "exit 0: "* ]] && [[ "$out" == *"cleanup failed"*"rm -rf ${FD}/planted-fail (exit 1)"* ]] \ + && [[ "$out" == *"docker image prune -a (exit 3)"* ]] && [[ "$out" != *"rm -rf ${FD}/ok "* ]]; then + echo "PASS: free-disk names the planted failing rm and the failed prune, and still exits 0 (${out})" +else echo "FAIL: free-disk did not name the planted failures: ${out}"; failed=1; fi + # The network-block control: curl's error must name the closed proxy, in # either wording curl uses, and an error that never reached the proxy must # not pass for it (a host that is down, a name that does not resolve). diff --git a/tests/test_e2e_shards.sh b/tests/test_e2e_shards.sh index f57b5cd5..ca5a3c0e 100755 --- a/tests/test_e2e_shards.sh +++ b/tests/test_e2e_shards.sh @@ -114,6 +114,27 @@ broken "a base case with a need" "z9${TAB}base${TAB}5${TAB}c1" broken "a row of three columns" "z9${TAB}shard${TAB}5" broken "an unknown stage" "z9${TAB}later${TAB}5${TAB}-" +# --- check: every case has a row --------------------------------------------- +# z9 has no row in the planted table: check names it, while plan still runs it. +if out=$("$S" check 2>&1); then bad "check passed a table with no row for z9" +elif grep -qx 'z9' <<< "$out"; then ok "check names the case with no row: $(tr '\n' ' ' <<< "$out")" +else bad "check did not name z9: ${out}"; fi +table | grep -v "^e1${TAB}" > "${WORK}/two-missing.tsv" +if out=$(E2E_TIMES="${WORK}/two-missing.tsv" "$S" check 2>&1); then bad "check passed a table with no row for e1 and z9" +elif grep -qx 'e1' <<< "$out" && grep -qx 'z9' <<< "$out"; then ok "check names both cases with no row" +else bad "check did not name both e1 and z9: ${out}"; fi +if E2E_TIMES="${WORK}/no-such.tsv" "$S" check > /dev/null 2>&1; then bad "check passed with no table" +else ok "check refuses a table that is not there"; fi +mkdir -p "${WORK}/empty" +if E2E_CASE_DIR="${WORK}/empty" E2E_TIMES=/dev/null "$S" check > /dev/null 2>&1; then bad "check passed with no case and an empty table" +else ok "check refuses a case directory with no case"; fi +{ table; printf '%s\n' "z9${TAB}shard${TAB}5${TAB}-"; } > "${WORK}/whole.tsv" +if out=$(E2E_TIMES="${WORK}/whole.tsv" "$S" check 2>&1); then ok "check passes once z9 has its row: ${out}" +else bad "check failed a table with a row for every case: ${out}"; fi +{ table; printf '%s\n' "z9${TAB}shard${TAB}5${TAB}-" "z9${TAB}shard${TAB}6${TAB}-"; } > "${WORK}/twice.tsv" +if E2E_TIMES="${WORK}/twice.tsv" "$S" check > /dev/null 2>&1; then bad "check passed a table with two rows for z9" +else ok "check refuses a second row too"; fi + # --- merge: one pass per case, or red --------------------------------------- res() { printf '%s\t%s\t%s\t%s\n' "$@"; } while read -r c; do res "$c" pass 1 "shard 1"; done < "${WORK}/all.txt" > "${WORK}/r-all.tsv" @@ -163,5 +184,7 @@ n=$(awk '/^ E2E_SHARDS:/ {print $2}' "${REPO}/.github/workflows/e2e.yml") if { "$S" base; "$S" plan "$n" | cut -f2; } > "${WORK}/real.txt" && out=$("$S" cover "${WORK}/real.txt"); then ok "real table, ${n} shards: ${out}" else bad "real split: ${out:-plan failed}"; fi +if out=$("$S" check 2>&1); then ok "real table: ${out}" +else bad "real table: ${out}"; fi [ "$FAILS" -eq 0 ] diff --git a/tests/test_owngenome_recheck.py b/tests/test_owngenome_recheck.py index 8c8a72cd..bb5b8a50 100644 --- a/tests/test_owngenome_recheck.py +++ b/tests/test_owngenome_recheck.py @@ -21,6 +21,12 @@ 10. the live-site plugin (bin/owngenome_live_recheck.py) lets the browser fetch recheck/state.json, and its state hook gives each step its badges ("newer ClinVar available", "redone ") and {} when there is no check. + 11. a bumped pin of the opt-in steps 50 to 53 (LOCITYPER_PANEL_VERSION and + LOCITYPER_IMAGE for 53, MOCHA_PANEL_VERSION and MOCHA_IMAGE for 51, + VNTYPER_IMAGE for 52, MEGANE_IMAGE for 50) marks its step. + 12. check() itself names step 53 rerun-ready when the run used an older + LOCITYPER_PANEL_VERSION than this checkout pins, and not when they match; + its action names setup.sh --locityper-data, which the main setup leaves out. Each check is then run against a planted mistake and must fail there. Run: python3 tests/test_owngenome_recheck.py @@ -518,7 +524,55 @@ def plugin_from_source(src, name): return mod +# Steps 50-53 read pins of versions.env that change what they find: the data +# setup.sh builds or downloads with them, or the reference data inside the +# image. A bump of each must mark its step (steps.tsv's reads column). +OPT_IN_PINS = [("MEGANE_IMAGE", "50"), ("MOCHA_PANEL_VERSION", "51"), ("MOCHA_IMAGE", "51"), + ("VNTYPER_IMAGE", "52"), ("LOCITYPER_PANEL_VERSION", "53"), ("LOCITYPER_IMAGE", "53")] + + +def pins_ok(m, steps): + """Each pin of OPT_IN_PINS, bumped alone, makes affected_steps list exactly its step and the + HTML report (no step reads the outputs of steps 50 to 53).""" + missed = [(k, n, got) for k, n in OPT_IN_PINS + for got in [m.affected_steps(steps, [k])] if sorted(got) != sorted([n, m.REPORT_STEP])] + return not missed, missed + + +def check_marks_53(m, steps): + """check() on a run that used an older LOCITYPER_PANEL_VERSION than this checkout pins puts 53 in + rerun_ready_steps; the same pin on both sides puts nothing there.""" + tmp = tempfile.mkdtemp(prefix="recheck.", dir=os.path.join(REPO, "tmp") if os.path.isdir(os.path.join(REPO, "tmp")) else None) + try: + g = genome(tmp, installed_clinvar="20260901") + with open(os.path.join(g, "S1", "run_manifest.tsv"), "a") as f: + f.write("version\tLOCITYPER_PANEL_VERSION\tpanel-1.0\n") + st = m.check("S1", g, answers(m, g), steps, dict(THIS_PINS, LOCITYPER_PANEL_VERSION="panel-2.0"), "3.66.0") + same = m.check("S1", g, answers(m, g), steps, dict(THIS_PINS, LOCITYPER_PANEL_VERSION="panel-1.0"), "3.66.0") + act = by_item(st).get("LOCITYPER_PANEL_VERSION", {}).get("action", "") + return ("53" in st["rerun_ready_steps"] and "53" not in same["rerun_ready_steps"] + and "./scripts/setup.sh --locityper-data $GENOME_DIR" in act, + (st["rerun_ready_steps"], same["rerun_ready_steps"], act)) + finally: + shutil.rmtree(tmp, ignore_errors=True) + + def main(): + m = load() + steps = m.run_state.read_steps(m.STEPS_TSV) + ok, detail = check_marks_53(m, steps) + check("a run on an older LOCITYPER_PANEL_VERSION makes check() name step 53 rerun-ready, with setup.sh --locityper-data", ok, detail) + blank = [dict(st, reads="") if st["step"] == "53" else st for st in steps] + check("fails on a planted mistake: a run on an older LOCITYPER_PANEL_VERSION makes check() name step 53 " + "(row 53 with no reads)", not check_marks_53(m, blank)[0], "passed on the mutant") + ok, detail = pins_ok(m, steps) + check("a bumped pin of steps 50-53 (LOCITYPER_PANEL_VERSION and the rest) marks its step", ok, detail) + blank = [dict(st, reads="") if st["step"] == "53" else st for st in steps] + check("fails on a planted mistake: a bumped pin of steps 50-53 marks its step (row 53 with no reads)", + not pins_ok(m, blank)[0], "passed on the mutant") + no_report = [st for st in steps if st["step"] != m.REPORT_STEP] + check("fails on a planted mistake: a bumped pin of steps 50-53 marks its step (no report step in the catalogue)", + not pins_ok(m, no_report)[0], "passed on the mutant") ok, detail = plugin_ok(load_plugin()) check("the live-site plugin serves the check and gives each step its badges", ok, detail) bad = load_plugin()