diff --git a/.github/actions/free-disk/action.yml b/.github/actions/free-disk/action.yml new file mode 100644 index 00000000..6700d1cf --- /dev/null +++ b/.github/actions/free-disk/action.yml @@ -0,0 +1,39 @@ +name: Free disk space +description: >- + Removes the preinstalled toolchains no OwnGenome job uses and the + preinstalled Docker images, and prints the free space before and after. + The free space on / varies with the runner image, and the e2e legs and the + Nextflow stub runs ran out of it on 2026-11-04. Java stays in the tool + cache for setup-java; the scripts run the system python3, not the tool + cache's. + +runs: + using: composite + steps: + - name: Free disk space + shell: bash + run: | + echo "Free space before:" + df -h / /mnt 2> /dev/null || df -h / + before=$(df --output=avail -k / | tail -1) + # The deletions run side by side: the Android SDK alone is about + # 100,000 files. A path that is not there is skipped. + for d in /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup \ + /usr/local/share/boost /usr/share/swift /usr/local/share/powershell \ + /usr/local/share/chromium /opt/microsoft /opt/google /opt/az \ + /opt/hostedtoolcache/CodeQL /opt/hostedtoolcache/go /opt/hostedtoolcache/PyPy \ + /opt/hostedtoolcache/Python \ + /opt/hostedtoolcache/Ruby /opt/hostedtoolcache/node; do + [ -e "$d" ] && sudo rm -rf "$d" & + done + # The images the runner image ships (none of them is ours). + if command -v docker > /dev/null 2>&1; then + docker image prune -a -f > /dev/null 2>&1 & + fi + wait + after=$(df --output=avail -k / | tail -1) + echo "Free space after:" + df -h / /mnt 2> /dev/null || df -h / + msg="Free disk: $(( (after - before) / 1048576 )) GB freed on /, $(( after / 1048576 )) GB free now, in ${SECONDS} s" + echo "$msg" + echo "$msg" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index cd3a5d5a..2746e4d6 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -437,20 +437,16 @@ jobs: echo "$cpu" echo "Runner: ${cpu}" >> "$GITHUB_STEP_SUMMARY" + # Before anything is pulled or restored: the legs ran out of disk on + # 2026-11-04 with only part of the preinstalled software removed. + - uses: ./.github/actions/free-disk + # Images go to /var/lib/docker on /; the work area goes to whichever of # / and /mnt has more room, so the two do not compete for one disk. The # base outputs unpack only under the work area they were packed in, so # a shard leg takes the base leg's choice when its artifact arrives. - - name: Free disk space and pick the work area - # A shard starts its cases with about 15 GB free on /, and shard 1 - # needs most of it: the toolchains below are not used by any case. + - name: Pick the work area run: | - df -h / /mnt - sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup \ - /opt/hostedtoolcache/CodeQL /usr/local/share/boost \ - /usr/share/swift /usr/local/share/powershell /usr/local/share/chromium /opt/microsoft /opt/az \ - /opt/hostedtoolcache/Python /opt/hostedtoolcache/PyPy /opt/hostedtoolcache/go /opt/hostedtoolcache/Ruby - df -h / /mnt work="${RUNNER_TEMP}/e2e-work" if [ "$(df --output=avail -k /mnt | tail -1)" -gt "$(df --output=avail -k / | tail -1)" ]; then work=/mnt/e2e-work @@ -600,6 +596,8 @@ jobs: grep -qx nextflow-hardening-2-nf "$list" && echo "parity_hardening=true" >> "$GITHUB_OUTPUT" grep -qx mocha-1 "$list" && echo "data_mocha=true" >> "$GITHUB_OUTPUT" grep -qx archaic-ancestry-1 "$list" && echo "data_archaic=true" >> "$GITHUB_OUTPUT" + grep -qx bash-step-08-hla "$list" && echo "data_hla=true" >> "$GITHUB_OUTPUT" + grep -qx prs-scores-and-run-options-1-strand-sex "$list" && echo "data_pgs=true" >> "$GITHUB_OUTPUT" echo "list=${list}" >> "$GITHUB_OUTPUT" # Before the base outputs: the base leg pulled the same images. @@ -651,12 +649,18 @@ jobs: # runs" there) and downloads what a miss leaves out. Saved like the # images: on main, or a run by hand with save_images, on a miss. Only # the leg whose cases include the case restores and saves its entry. + # Two small ones the same way, keyed by e2e-run.sh --data-key: hla.dat + # of two IPD-IMGT/HLA releases and GENCODE's genes and exons + # (bash-step-08-hla, which then runs with the network blocked), and the + # four real PGS Catalog score files of prs-scores-and-run-options-1. - name: Data cache keys id: data-keys run: | . versions.env echo "mocha=e2e-data-mocha-${MOCHA_PANEL_MANIFEST_SHA256:0:16}-chr20" | tee -a "$GITHUB_OUTPUT" echo "archaic=e2e-data-1kg-sites-${ARCHAIC_1KG_SITES_MD5}" | tee -a "$GITHUB_OUTPUT" + echo "hla=$(scripts/ci/e2e-run.sh --data-key hla)" | tee -a "$GITHUB_OUTPUT" + echo "pgs=$(scripts/ci/e2e-run.sh --data-key pgs)" | tee -a "$GITHUB_OUTPUT" - name: Restore cached data, MoChA panel id: data-mocha @@ -674,6 +678,49 @@ jobs: path: ${{ env.E2E_DATA_CACHE }}/archaic key: ${{ steps.data-keys.outputs.archaic }} + - name: Restore cached data, IPD-IMGT/HLA and GENCODE + id: data-hla + if: steps.cases-list.outputs.data_hla == 'true' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ env.E2E_DATA_CACHE }}/hla + key: ${{ steps.data-keys.outputs.hla }} + + - name: Restore cached data, PGS Catalog scores + id: data-pgs + if: steps.cases-list.outputs.data_pgs == 'true' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ env.E2E_DATA_CACHE }}/pgs + key: ${{ steps.data-keys.outputs.pgs }} + + # The PGS Catalog's synthetic HAPNEST panel (prs-2-ancestry and + # prs-scores-and-run-options-2, 268 MB), from EBI's FTP server, keyed + # on the sha256 tests/e2e/lib.sh pins. The cases take it from + # E2E_DATA_CACHE/hapnest, check it and then run setup.sh with EBI + # unreachable; on a miss the first one downloads it and puts it there + # (setup_ancestry_panel in lib.sh). Saved like the entries above: on + # main, or a run by hand with save_images, on a miss. + - name: Data cache, HAPNEST panel + id: hapnest + env: + LIST: ${{ steps.cases-list.outputs.list }} + SAVE: ${{ (github.ref == 'refs/heads/main' || inputs.save_images) && 'true' || 'false' }} + run: | + grep -qxE 'prs-2-ancestry|prs-scores-and-run-options-2-panel-chrx' "$LIST" || exit 0 + sha=$(sed -n 's/^HAPNEST_SHA256=//p' tests/e2e/lib.sh) + [ "${#sha}" -eq 64 ] || { echo "::error::No HAPNEST_SHA256= line in tests/e2e/lib.sh"; exit 1; } + { echo "run=true"; echo "key=e2e-data-hapnest-${sha:0:16}"; } | tee -a "$GITHUB_OUTPUT" + echo "E2E_HAPNEST_KEEP=${SAVE}" >> "$GITHUB_ENV" + + - name: Restore cached data, HAPNEST panel + id: data-hapnest + if: steps.hapnest.outputs.run == 'true' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ env.E2E_DATA_CACHE }}/hapnest + key: ${{ steps.hapnest.outputs.key }} + - name: Run the e2e cases id: cases env: @@ -686,14 +733,15 @@ jobs: E2E_BASE_OUT: ${{ matrix.job == 'base' && format('{0}/base.tar.zst', runner.temp) || '' }} E2E_BASE_IN: ${{ matrix.job != 'base' && format('{0}/e2e-base/base.tar.zst', runner.temp) || '' }} # Keep the downloads to the end of the job only when a save step uses them. - E2E_DATA_KEEP: ${{ (github.ref == 'refs/heads/main' || inputs.save_images) && ((steps.cases-list.outputs.data_mocha == 'true' && steps.data-mocha.outputs.cache-hit != 'true') || (steps.cases-list.outputs.data_archaic == 'true' && steps.data-archaic.outputs.cache-hit != 'true')) && 'true' || 'false' }} + E2E_DATA_KEEP: ${{ (github.ref == 'refs/heads/main' || inputs.save_images) && ((steps.cases-list.outputs.data_mocha == 'true' && steps.data-mocha.outputs.cache-hit != 'true') || (steps.cases-list.outputs.data_archaic == 'true' && steps.data-archaic.outputs.cache-hit != 'true') || (steps.cases-list.outputs.data_hla == 'true' && steps.data-hla.outputs.cache-hit != 'true') || (steps.cases-list.outputs.data_pgs == 'true' && steps.data-pgs.outputs.cache-hit != 'true')) && 'true' || 'false' }} HIT_MOCHA: ${{ steps.data-mocha.outputs.cache-hit || 'false' }} HIT_ARCHAIC: ${{ steps.data-archaic.outputs.cache-hit || 'false' }} - DATA_MOCHA: ${{ steps.cases-list.outputs.data_mocha || 'false' }} - DATA_ARCHAIC: ${{ steps.cases-list.outputs.data_archaic || 'false' }} + HIT_HLA: ${{ steps.data-hla.outputs.cache-hit || 'false' }} + HIT_PGS: ${{ steps.data-pgs.outputs.cache-hit || 'false' }} + DATA_ANY: ${{ steps.cases-list.outputs.data_mocha || steps.cases-list.outputs.data_archaic || steps.cases-list.outputs.data_hla || steps.cases-list.outputs.data_pgs || 'false' }} run: | - if [ "$DATA_MOCHA" = true ] || [ "$DATA_ARCHAIC" = true ]; then - echo "Data cache (${JOB}): exact key hit, MoChA panel ${HIT_MOCHA}, 1000 Genomes sites ${HIT_ARCHAIC}" | tee -a "$GITHUB_STEP_SUMMARY" + if [ "$DATA_ANY" = true ]; then + echo "Data cache (${JOB}): exact key hit, MoChA panel ${HIT_MOCHA}, 1000 Genomes sites ${HIT_ARCHAIC}, IPD-IMGT/HLA and GENCODE ${HIT_HLA}, PGS scores ${HIT_PGS}" | tee -a "$GITHUB_STEP_SUMMARY" fi scripts/ci/e2e-run.sh @@ -707,6 +755,8 @@ jobs: env: HIT_MOCHA: ${{ steps.data-mocha.outputs.cache-hit }} HIT_ARCHAIC: ${{ steps.data-archaic.outputs.cache-hit }} + HIT_HLA: ${{ steps.data-hla.outputs.cache-hit }} + HIT_PGS: ${{ steps.data-pgs.outputs.cache-hit }} run: | d=$E2E_DATA_CACHE if [ "$HIT_MOCHA" != true ] && [ -s "${d}/mocha/manifest.txt" ] \ @@ -716,6 +766,13 @@ jobs: if [ "$HIT_ARCHAIC" != true ] && [ -s "${d}/archaic/1kg_sites.vcf.gz" ]; then echo "archaic=true" >> "$GITHUB_OUTPUT" fi + # Whole and checked (data_ok in e2e-run.sh), or not saved. + for n in hla pgs; do + hit=HIT_${n^^} + if [ "${!hit}" != true ] && [ -d "${d}/${n}" ] && scripts/ci/e2e-run.sh --data-check "$n"; then + echo "${n}=true" >> "$GITHUB_OUTPUT" + fi + done du -sh "${d}"/* 2> /dev/null || echo "Data cache: nothing downloaded" - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -730,6 +787,36 @@ jobs: path: ${{ env.E2E_DATA_CACHE }}/archaic key: ${{ steps.data-keys.outputs.archaic }} + - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + if: always() && steps.data-save.outputs.hla == 'true' + with: + path: ${{ env.E2E_DATA_CACHE }}/hla + key: ${{ steps.data-keys.outputs.hla }} + + - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + if: always() && steps.data-save.outputs.pgs == 'true' + with: + path: ${{ env.E2E_DATA_CACHE }}/pgs + key: ${{ steps.data-keys.outputs.pgs }} + + # The case leaves the panel there only once its sha256 matched. + - name: Data cache, HAPNEST panel, what to save + id: hapnest-save + if: >- + always() && steps.hapnest.outputs.run == 'true' && env.E2E_HAPNEST_KEEP == 'true' && + steps.data-hapnest.outputs.cache-hit != 'true' + run: | + if [ -s "${E2E_DATA_CACHE}/hapnest/GRCh38_HAPNEST_reference.tar.zst" ]; then + echo "save=true" >> "$GITHUB_OUTPUT" + fi + du -sh "${E2E_DATA_CACHE}/hapnest" 2> /dev/null || echo "Data cache: no HAPNEST panel to save" + + - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + if: always() && steps.hapnest-save.outputs.save == 'true' + with: + path: ${{ env.E2E_DATA_CACHE }}/hapnest + key: ${{ steps.hapnest.outputs.key }} + - name: Upload the base outputs if: matrix.job == 'base' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 @@ -739,7 +826,8 @@ jobs: ${{ runner.temp }}/base.tar.zst ${{ runner.temp }}/base.tar.zst.from compression-level: 0 - retention-days: 1 + # Three days, so "Re-run failed jobs" the next day still finds it. + retention-days: 3 overwrite: true # The cases nextflow-from-fastq-1-bash and -2-nextflow ran the bash steps @@ -803,8 +891,9 @@ jobs: # Each group keeps its smallest images up to GROUP_BUDGET_MB of zstd # output; the rest are pulled on every run. Four groups of 1500 MB are # 5.9 GB, and while main replaces one group the old and the new entry - # both count: 7.3 GB. With the two data entries above (2.0 GB) that is - # 9.3 GB, under the 10 GB limit. An image pinned by digest + # both count: 7.3 GB. With the five data entries above (about 2.3 GB: + # MoChA 0.64, 1000 Genomes sites 1.36, HAPNEST 0.27, hla and GENCODE + # 0.03, PGS a few KB) that is 9.6 GB, under the 10 GB limit. An image pinned by digest # is never saved: docker load gives it back without its digest, so # e2e-run.sh would pull it again anyway. - name: Image sizes, and save images for the next run diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index 9e28d78d..0b628f8a 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -83,7 +83,12 @@ jobs: - name: Self-test (planted bgzip/tabix calls behind if !, time and VAR=, and an orad option orad does not list, must be caught) run: scripts/ci/check-container-helpers.sh --self-test + # tests/test_exec_bits.sh (unit-tests) plants a helper without the bit. + - name: Every bin/ helper the pipeline calls by name is executable + run: scripts/ci/check-exec-bits.sh + - name: Table of helper calls derived from scripts and modules + if: ${{ !cancelled() }} # runs even when a step above failed, so each result shows run: scripts/ci/check-container-helpers.sh --list - name: Each helper binary exists in the image that calls it diff --git a/.github/workflows/nextflow.yml b/.github/workflows/nextflow.yml index b2d95304..7c756324 100644 --- a/.github/workflows/nextflow.yml +++ b/.github/workflows/nextflow.yml @@ -60,6 +60,11 @@ jobs: echo "run=${run}" >> "$GITHUB_OUTPUT" [ "$run" = true ] || echo "Only docs changed: nothing here can affect the pipeline, skipping the Nextflow steps." + # The stub runs pull every image of the pipeline: the pinned leg ran + # out of disk on 2026-11-04. + - uses: ./.github/actions/free-disk + if: steps.scope.outputs.run == 'true' + - name: Resolve the Nextflow version if: steps.scope.outputs.run == 'true' id: nf diff --git a/.gitignore b/.gitignore index f6733320..c6733184 100644 --- a/.gitignore +++ b/.gitignore @@ -105,3 +105,6 @@ site/ # Beads: the tracker lives in a private Dolt remote, bead text never enters git .beads/*.jsonl + +# Scratch space of a checkout: logs, downloads, CI output +/tmp/ diff --git a/docs/testing.md b/docs/testing.md index e884aa21..0409c543 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -6,6 +6,8 @@ The stub runs in `nextflow.yml` check that the Nextflow wiring holds together, a Every `nextflow run` step in `nextflow.yml` has a time limit of its own, well under the job's 45 minutes. It is 10 minutes for the runs without containers, which take under a minute, and 20 for the stub run that pulls images, which took about four minutes in the run we timed. When a step of that job fails or is cut by its limit, the job prints which tasks were running and which were waiting to start, and uploads `.nextflow.log` as the artifact `nextflow-logs-` (`pinned` or `26.10`). Without that log the first two hangs of the retry step could not be explained ([Lessons learned](lessons-learned.md#the-local-executor-can-lose-track-of-free-memory-and-a-task-that-needs-all-of-it-then-never-starts)). +The stub runs never execute a `bin/` helper the way a task does, by its bare name from the PATH, so a helper committed without its executable bit passes them and fails only in a real task (exit 126). The `helper-binaries` job of `guard.yml` runs [`scripts/ci/check-exec-bits.sh`](https://github.com/GeiserX/OwnGenome/blob/main/scripts/ci/check-exec-bits.sh): every `bin/` file that `main.nf`, `workflows/` or `modules/` calls by name must be committed as mode 100755 and start with a `#!` line. `tests/test_exec_bits.sh` plants a helper without the bit, which must fail. + ## The fixture The test data is a slice of **HG002**, the Genome in a Bottle (GIAB) son of the Ashkenazi trio. HG002 is a public, consented reference sample, so nothing in the fixture is personal data. @@ -76,7 +78,7 @@ In CI the cases run in parallel legs, each case at most once: a pull request run - `e2e-plan` splits the cases with [`scripts/ci/e2e-shards.sh`](https://github.com/GeiserX/OwnGenome/blob/main/scripts/ci/e2e-shards.sh). The leg `e2e base` runs the cases of stage `base` in [`scripts/ci/e2e-case-times.tsv`](https://github.com/GeiserX/OwnGenome/blob/main/scripts/ci/e2e-case-times.tsv) (every numbered case: the fixture check, setup validation, alignment, DeepVariant, GATK, the single steps on HG002, the reports and the Nextflow run; and `bash-step-08-hla`), because most other cases read what those wrote. Every other picked case goes to one of the shard legs (`e2e shard 1`, `e2e shard 2`, ...). A full run uses all `E2E_SHARDS` (6) legs. For picked cases, `e2e-shards.sh count` sets how many: the cases' total seconds divided by the heaviest group's seconds (the critical path, since a group is never split) or by 900 seconds when that is larger, rounded down, plus one more leg when the rest is 900 seconds or more, and never more than `E2E_SHARDS` (6). A shard leg spends 18 to 20 minutes on top of its cases (pulling images, fetching and unpacking the base outputs, the parity checks and the uploads; run 41847167444), so a leg with less than 15 minutes of cases costs more runner time than it saves in wall time. With every case that rule gives 4 or 5 legs, and that is why a full run does not use it: the runner's disk, not time, is what a leg runs out of. With 5 legs, shards of 30 and 37 cases filled one runner's disk until the runner itself died (run 41995851301, shard 3), while 6 legs passed on the same main (run 41995719742). - The table holds each case's median time on the passing runs of nine e2e runs of 2026-11-04 (41766267038, 41767270253, 41773233871, 41774860458, 41779067613, 41790595358, 41799741941, 41807346887 and 41821518376), and its `needs`: the non-base cases whose outputs it reads. The needs were written from what each case says it reads (its header and its checks) and from the cases that failed in a split run without them (run 41455829120). A case and every case it needs, followed through, form a group that stays in one shard, in name order. Groups go to the least loaded shard, heaviest first. A new case with no row counts as 300 seconds and needs nothing. A case that runs well over 300 seconds also needs a row with its measured seconds, or the split balances it as a 300-second case. -- The base leg runs first. It packs every file its cases wrote or changed (at its absolute path, with owners, modes and times) and uploads it as the artifact `e2e-base`. The shard legs start only after it passed (`needs:`), so a shard never holds a runner while it waits: when many runs share the account's runners, shards that started early and waited took the runners the base legs needed (run 41506673662 had three shards running and its base leg still queued). Each shard fetches the artifact, tried up to five times, takes the base leg's work area, unpacks the archive there, deletes it (about 4 GB the cases need) and runs its cases. When the base leg fails, the shards are skipped and `e2e` is red. +- The base leg runs first. It packs every file its cases wrote or changed (at its absolute path, with owners, modes and times) and uploads it as the artifact `e2e-base`, kept for 3 days so that "Re-run failed jobs" the next day still finds it. The shard legs start only after it passed (`needs:`), so a shard never holds a runner while it waits: when many runs share the account's runners, shards that started early and waited took the runners the base legs needed (run 41506673662 had three shards running and its base leg still queued). Each shard fetches the artifact, tried up to five times, takes the base leg's work area, unpacks the archive there, deletes it (about 4 GB the cases need) and runs its cases. When the base leg fails, the shards are skipped and `e2e` is red. - The shard that holds `nextflow-from-fastq-2-nextflow` and the one that holds `nextflow-hardening-2-nf` run the parity checks; the plan fails when either case runs and the case before it in its pair is in a different shard. - The required check is the job `e2e`. It merges every leg's results into one table of case, result, time and job, followed by the failed checks of each failed case, and it is red unless the plan and every leg succeeded, every picked case reported exactly one pass and each parity check passed once when its cases ran (and not at all when they did not). It is red too when `e2e-scope` did not succeed. A leg that was skipped, cancelled or cut by its limit leaves cases with no result, and that is red too. The full logs are in the `e2e-logs-` artifacts. @@ -84,7 +86,7 @@ In CI the cases run in parallel legs, each case at most once: a pull request run To measure the table again, run the E2E workflow by hand with `job: e2e` and `trace: true`. That run puts every non-base case in one shard, so it runs as long as one serial run. After each case `e2e-run.sh` records its time and, with the file systems remounted `strictatime`, every file under the work area and the checkout whose atime moved during the case, attributed to the case whose run last changed that file's ctime. The job `e2e` then prints a new `e2e-case-times.tsv` in its summary and uploads it as the artifact `e2e-case-times`; it keeps each row's stage and needs and adds the traced needs. It refuses a trace that saw no case read another case's file. The traced needs are an upper bound, not the table to commit: a file counts for the case that changed it last, so a case that rewrites or touches a shared file (a report, an index) shows up as a need of every later case that reads that file, and a case that scans a folder reads everything in it. On the trace of run 41454755531 they joined almost every case into one group, so check each traced need against the case before you add it. A read the trace cannot see, such as a test for whether a file exists, is not in the table: such a case fails in its shard, and its row then needs a hand-written `needs`. -Every pull request starts the workflow, so main can require the e2e check. Its `e2e-scope` job picks the cases (next section); when the pull request changes no path in the e2e scope the plan and the legs are skipped and the job `e2e` passes without running anything, so a docs-only change is never blocked. The e2e job also runs once a month and by hand (`job: e2e`), with every case. It also runs, with every case, on main after every merge that touches the e2e scope, because main does not require a branch to be up to date before it merges: two pull requests that land close together each passed alone, and only the run on main tests them together. A docs-only merge starts no run. A run on main is never cancelled half way. A merge that lands while one runs waits, and GitHub keeps one waiting run, so a newer merge replaces the waiting one and the next run tests every merge since the last run. A red run on main names its commit, the newest merge it tested; the merges between the last green run on main and that commit are the suspects. The push paths in `e2e.yml` and the scope in `scripts/ci/e2e-scope.sh` (`e2e-scope.sh scope` prints it) are the same list written twice, and the first step of `e2e-plan` fails when they differ. A push to any other branch starts no run: its pull request runs e2e, and an `e2e` check from a push run would sit on the same commit as the pull request's required `e2e` check. The DeepVariant case passes the fixture slices as `INTERVALS`, so DeepVariant calls only those slices instead of walking all 1.8 Gb of the reference, which alone took 27 to 48 minutes. It uses a standard GitHub-hosted runner (4 CPUs, 16 GB of RAM) after deleting preinstalled toolchains it does not use (Android, .NET, Haskell, CodeQL, Boost) to make disk room. +Every pull request starts the workflow, so main can require the e2e check. Its `e2e-scope` job picks the cases (next section); when the pull request changes no path in the e2e scope the plan and the legs are skipped and the job `e2e` passes without running anything, so a docs-only change is never blocked. The e2e job also runs once a month and by hand (`job: e2e`), with every case. It also runs, with every case, on main after every merge that touches the e2e scope, because main does not require a branch to be up to date before it merges: two pull requests that land close together each passed alone, and only the run on main tests them together. A docs-only merge starts no run. A run on main is never cancelled half way. A merge that lands while one runs waits, and GitHub keeps one waiting run, so a newer merge replaces the waiting one and the next run tests every merge since the last run. A red run on main names its commit, the newest merge it tested; the merges between the last green run on main and that commit are the suspects. The push paths in `e2e.yml` and the scope in `scripts/ci/e2e-scope.sh` (`e2e-scope.sh scope` prints it) are the same list written twice, and the first step of `e2e-plan` fails when they differ. A push to any other branch starts no run: its pull request runs e2e, and an `e2e` check from a push run would sit on the same commit as the pull request's required `e2e` check. The DeepVariant case passes the fixture slices as `INTERVALS`, so DeepVariant calls only those slices instead of walking all 1.8 Gb of the reference, which alone took 27 to 48 minutes. It uses a standard GitHub-hosted runner (4 CPUs, 16 GB of RAM). Every e2e leg and both `nextflow.yml` legs first run the composite action [`.github/actions/free-disk`](https://github.com/GeiserX/OwnGenome/blob/main/.github/actions/free-disk/action.yml): it deletes the preinstalled toolchains no job uses (Android, .NET, Haskell, Boost, Swift, PowerShell, Chromium, the Azure CLI, and CodeQL, Go, Python, PyPy, Ruby and Node from the tool cache; Java stays, and the scripts use the system `python3`) and the preinstalled Docker images, and prints the free space before and after to the log and the job summary. Legs ran out of disk on 2026-11-04 with only part of that removed. ### Which cases a pull request runs @@ -98,9 +100,9 @@ The base leg runs every base case whenever any case runs, because every shard st [`scripts/ci/e2e-case-paths.tsv`](https://github.com/GeiserX/OwnGenome/blob/main/scripts/ci/e2e-case-paths.tsv) holds, for each case, the files it runs: the step scripts, bin helpers, modules (`modules/local//*`) and assets it calls, and what those call in turn. The globs are matched as in bash `[[ PATH == GLOB ]]`, so `*` matches slashes too. A base case's row is `-`. The table was first written from the text of each case, followed through the scripts, helpers and modules it names (comments left out). `e2e-scope.sh check` runs first in `e2e-scope` and fails when a case has no row or two, a row names no case, a base row lists paths, or a glob matches no tracked file; then `e2e-scope` is red and so is `e2e`. A glob too many costs runner time; a glob too few lets a break through to main, where the full run after the merge finds it. -The images the cases use are cached in four GitHub Actions cache entries, one per group. `scripts/ci/e2e-run.sh --image-groups` names each image's group, taken from its repository name without the tag, so a version bump changes the key of one group only; a group whose key missed restores the newest older entry of that group and pulls only what changed. A group keeps its smallest images up to 1500 MB of zstd output, and the larger ones are pulled on every run. Four groups of 1500 MB and the two data entries below stay under the repository's 10 GB cache limit even while main replaces one group: 5.9 GB of images, 1.5 GB more for the group being replaced, and 2.0 GB of data. Images pinned by digest are not cached, because `docker load` gives them back without the digest, and they are left out of the group's key, so a digest bump misses no entry. Only the base leg saves, and only on main (or in a run by hand with `save_images`, on any branch), after it pulled every image the cases use and got through its cases, and only the groups whose key missed: a cache saved by a pull request can be read by that pull request alone and would only push main's entries out. The job summary lists every image with its size and what the cache did with it: restored on an exact key hit, restored from an older entry, saved, or pulled. Every merge in the e2e scope starts a run on main, so the cache follows main's images. GitHub drops an entry no run has read for 7 days, so after a quiet week the next run on main fills the cache again. Until this layout the cache was one entry, and by October 2026 it had grown to about 11 GB, over the limit, so every run missed it. +The images the cases use are cached in four GitHub Actions cache entries, one per group. `scripts/ci/e2e-run.sh --image-groups` names each image's group, taken from its repository name without the tag, so a version bump changes the key of one group only; a group whose key missed restores the newest older entry of that group and pulls only what changed. A group keeps its smallest images up to 1500 MB of zstd output, and the larger ones are pulled on every run. Four groups of 1500 MB and the five data entries below stay under the repository's 10 GB cache limit even while main replaces one group: 5.9 GB of images, 1.5 GB more for the group being replaced, and about 2.3 GB of data. Images pinned by digest are not cached, because `docker load` gives them back without the digest, and they are left out of the group's key, so a digest bump misses no entry. Only the base leg saves, and only on main (or in a run by hand with `save_images`, on any branch), after it pulled every image the cases use and got through its cases, and only the groups whose key missed: a cache saved by a pull request can be read by that pull request alone and would only push main's entries out. The job summary lists every image with its size and what the cache did with it: restored on an exact key hit, restored from an older entry, saved, or pulled. Every merge in the e2e scope starts a run on main, so the cache follows main's images. GitHub drops an entry no run has read for 7 days, so after a quiet week the next run on main fills the cache again. Until this layout the cache was one entry, and by October 2026 it had grown to about 11 GB, over the limit, so every run missed it. -Two downloads are cached the same way, because they come from EBI's 1000 Genomes server, which is slow and at times stalls: MoChA's chr20 panel with its manifest (0.64 GB, case `mocha-1`), keyed on `MOCHA_PANEL_MANIFEST_SHA256` and the chromosome, and the 1000 Genomes phase 3 sites VCF (1.36 GB, case `archaic-ancestry-1`), keyed on `ARCHAIC_1KG_SITES_MD5`. `scripts/ci/e2e-run.sh` hands each case its files just before it runs, at the paths `setup.sh` reads them from, so `setup.sh` downloads nothing; on a miss it downloads them first, checked against their pins. Only the leg whose cases include the case restores and saves its entry, and it saves only on main or in a run by hand with `save_images`, as with the images. The job summary says whether each key hit, and the case's log says where its files came from. Every download, there and in `setup.sh`, gives up on a transfer that stalls (`FETCH_STALL`, see `fetch` in `scripts/lib/common.sh`), and the next try resumes it. +Five downloads are cached the same way. Two come from EBI's 1000 Genomes server, which is slow and at times stalls: MoChA's chr20 panel with its manifest (0.64 GB, case `mocha-1`), keyed on `MOCHA_PANEL_MANIFEST_SHA256` and the chromosome, and the 1000 Genomes phase 3 sites VCF (1.36 GB, case `archaic-ancestry-1`), keyed on `ARCHAIC_1KG_SITES_MD5`. Two are small and failed whole runs on 2026-11-04, because their case is in the base leg or a shard that cannot start without it: `hla.dat` of the two IPD-IMGT/HLA releases `bash-step-08-hla` installs, from raw.githubusercontent.com (which served it under 1 KB/s), with GENCODE's gene and exon files from EBI's FTP server (curl exit 7 and 28), and the four real PGS Catalog score files of `prs-scores-and-run-options-1-strand-sex`, from the same server. `scripts/ci/e2e-run.sh --data-key` names their keys: the releases and a hash of the code in `scripts/lib/common.sh` that installs them, and a hash of the score ids and their server. `scripts/ci/e2e-run.sh` hands each case its files just before it runs, at the paths `setup.sh` and `install_data_file` read them from, so they download nothing; the score files the case copies from the cache itself. On a miss it downloads them first, checked against their pins or the checksum files the servers publish beside them (`MD5SUMS`, `md5checksum.txt`, the `.md5` of each score), trying up to 6 times 30 s apart. A restored entry is checked again before use (`--data-check`): the IPD-IMGT/HLA and GENCODE entry against the sha256 list written when it was filled, the score files against their `.md5`; one that fails is downloaded again. `bash-step-08-hla` then runs with every HTTP and HTTPS request sent to a closed port, after a control shows that a request under that block fails at the closed port, so a warm cache cannot reach either host. Only the leg whose cases include the case restores and saves its entry, and it saves only on main or in a run by hand with `save_images`, as with the images. The job summary says whether each key hit, and the case's log says where its files came from. The fifth is the PGS Catalog's synthetic HAPNEST panel (0.27 GB, cases `prs-2-ancestry` and `prs-scores-and-run-options-2-panel-chrx`) from EBI's FTP server, which failed cases with curl exit 7 on 2026-11-04. It is keyed on the sha256 `tests/e2e/lib.sh` pins (`HAPNEST_SHA256`). The cases handle it themselves (`setup_ancestry_panel` in `lib.sh`): with the panel in the cache they check the sha256 and run `setup.sh --ancestry-panel` with every HTTP and HTTPS request sent to a closed port, so an EBI outage cannot fail them; on a miss `setup.sh` downloads the panel as a user's run does, and the first case puts it in the cache for the save step. A red control in `prs-2-ancestry` runs `setup.sh` with the network blocked and no panel, which must fail. Every download, there and in `setup.sh`, gives up on a transfer that stalls (`FETCH_STALL`, see `fetch` in `scripts/lib/common.sh`), and the next try resumes it. How long it took in one job, before the split: a passing e2e job took about 105 to 205 minutes with 84 cases, and 230 minutes of cases with 130 (run 41390434697, 2026-11-02). The runner decides much of it, not the cache: the same CPU model took 106 minutes in one run and 166 in another. In October 2026, with 84 cases, the same code took 106 and 166 minutes on an AMD EPYC 9V45 (runs 39197502832 and 39075176392, the second of which also saved the cache), 112 minutes on an Intel Xeon 6973P (run 39088086725), 156 and 191 minutes on an AMD EPYC 7763 (runs 39075764278 and 39075201465) and 205 minutes on an AMD EPYC 9V74 (run 39197512594, which saved three groups). The image cache saves about 7 minutes of that: pulling every image took 9 minutes, restoring and loading the four groups took 2.5 (run 39075201465 restored 39 images in 118 seconds), and saving them all took 12. Each leg has a limit of 240 minutes (355 for a traced shard leg); the base leg waits up to 75 minutes for a fixture a pull request bumps. Each leg's first steps print the runner's CPU model and CPU count to the log and the job summary, so a later timing can name the hardware. diff --git a/scripts/ci/check-exec-bits.sh b/scripts/ci/check-exec-bits.sh new file mode 100755 index 00000000..8fd7d85b --- /dev/null +++ b/scripts/ci/check-exec-bits.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# check-exec-bits.sh [ROOT]: every bin/ helper the pipeline calls by name is +# committed with its executable bit, and starts with a #! line. +# +# Nextflow puts bin/ on the PATH of every task, so a module calls a helper by +# its bare name (`cdna_scan.py --out ...`). A helper committed as 100644 then +# fails only inside a real task ("Permission denied", exit 126): the stub runs +# and the unit tests, which call it through python3, never see it. That is how +# bin/cdna_scan.py reached main without the bit. +# +# A helper counts as called when its file name appears in main.nf, +# workflows/*.nf or modules/**/*.nf as a word that no "/" or word character +# comes before (`bin/foo.py` in a comment is a path, not a call). A bare name +# after an interpreter (`python3 foo.py`) or in a comment counts as a call +# too: the guard asks for a bit it may not need rather than tell a mention +# from a call, since a missing bit only shows inside a real task. The mode is +# the one git records (git ls-files -s), which is what a checkout gets. +# ROOT is a git work tree, by default the checkout this script is in. +set -euo pipefail + +ROOT=${1:-"$(cd "$(dirname "$0")/../.." && pwd)"} +cd "$ROOT" + +mapfile -t nf < <(git ls-files -- main.nf 'workflows/*.nf' 'modules/*.nf' 'modules/**/*.nf' | LC_ALL=C sort -u) +[ "${#nf[@]}" -gt 0 ] || { echo "ERROR: no Nextflow file under ${ROOT}: nothing could call a helper" >&2; exit 1; } + +bad=0 called=0 +while read -r mode _ _ path; do + name=${path#bin/} + case "$name" in */*) continue ;; esac + # The name as a regex: dots are literal. + re="(^|[^/A-Za-z0-9_.-])${name//./\\.}([^A-Za-z0-9_.-]|\$)" + hits=$(grep -nE -- "$re" "${nf[@]}" 2> /dev/null | head -n 1 || true) + [ -n "$hits" ] || continue + called=$((called + 1)) + if [ "$mode" != 100755 ]; then + echo "FAIL ${path}: mode ${mode}, but ${hits%%:*} calls it by name (${hits#*:}); run: git update-index --chmod=+x ${path}" + bad=$((bad + 1)) + elif [ "$(head -c 2 "$path")" != '#!' ]; then + echo "FAIL ${path}: executable, but its first line is not a #! line, so the task cannot run it" + bad=$((bad + 1)) + fi +done < <(git ls-files -s -- bin) + +if [ "$called" -eq 0 ]; then + echo "ERROR: no bin/ helper is called by name from ${#nf[@]} Nextflow files: the parser no longer sees the calls" >&2 + exit 1 +fi +if [ "$bad" -gt 0 ]; then + echo "${bad} of ${called} bin/ helpers the pipeline calls cannot be run by a task" + exit 1 +fi +echo "OK: the ${called} bin/ helpers the pipeline calls by name are executable" diff --git a/scripts/ci/e2e-run.sh b/scripts/ci/e2e-run.sh index 4d07f268..24c29eda 100755 --- a/scripts/ci/e2e-run.sh +++ b/scripts/ci/e2e-run.sh @@ -14,6 +14,9 @@ # --pull-images pulls those images, as step 3 below does, and exits. # --pull-image REF pulls one image with the same retry (the docker shim's # pulls of the lazy images at their first use), and exits. +# --data-key NAME prints the Actions cache key of data entry NAME (hla, pgs). +# --data-check NAME exits 0 when entry NAME in E2E_DATA_CACHE is whole and +# matches its checksums (the e2e workflow saves only such an entry). # # Env: E2E_WORK work area (default ${RUNNER_TEMP:-/tmp}/e2e-work), about 25 GB # GH_TOKEN token for `gh release download` (the job's GITHUB_TOKEN) @@ -160,6 +163,65 @@ if [ "${1:-}" = --image-groups ]; then exit 0 fi +# --- Data cache keys and checks ------------------------------------------------- +# The e2e workflow asks here for the Actions cache key of the entries hla and +# pgs (`--data-key NAME`) and whether an entry is whole before it saves it +# (`--data-check NAME`), so the key and the check are written once. The other +# entries (mocha, archaic) are keyed in e2e.yml on their versions.env pins. +DATA_CACHE=${E2E_DATA_CACHE:-} +HLA_CASE="${REPO}/tests/e2e/bash-step-08-hla.sh" +PGS_CASE="${REPO}/tests/e2e/prs-scores-and-run-options-1-strand-sex.sh" +# hla_releases: the IPD-IMGT/HLA releases bash-step-08-hla installs (the +# release before the pin, then the pin), from the case and scripts/lib/common.sh. +hla_releases() { + local other pin + other=$(sed -n 's/^OTHER=//p' "$HLA_CASE") + pin=$(sed -n 's/^HLA_DB_RELEASE=\${HLA_DB_RELEASE:-\(.*\)}$/\1/p' "${REPO}/scripts/lib/common.sh") + [ -n "$other" ] && [ -n "$pin" ] || { echo "ERROR: no OTHER= line in ${HLA_CASE} or no HLA_DB_RELEASE default in scripts/lib/common.sh" >&2; return 1; } + echo "$other $pin" +} +# data_key NAME: the Actions cache key of entry NAME. hla: the two releases, +# GENCODE's release and a hash of the code that installs them (the "Pinned +# data files" part of scripts/lib/common.sh), since the entry holds what that +# code writes. pgs: a hash of the score ids and the server they come from. +data_key() { + local r g h + case "$1" in + hla) + r=$(hla_releases) || return 1 + g=$(sed -n 's/^GENCODE_RELEASE=//p' "${REPO}/scripts/lib/common.sh") + h=$(awk '/^# --- Pinned data files/,/^_keep_lines\(\)/' "${REPO}/scripts/lib/common.sh" | sha256sum | cut -c1-16) + echo "e2e-data-hla-${r// /-}-gencode${g}-${h}" ;; + pgs) + h=$(grep -E '^(NEW_IDS|PGS_BASE)=' "$PGS_CASE" | sha256sum | cut -c1-16) + echo "e2e-data-pgs-${h}" ;; + *) echo "ERROR: no data cache key for ${1}" >&2; return 1 ;; + esac +} +# data_ok NAME: 0 when entry NAME in E2E_DATA_CACHE is whole and checks out. +# hla: every file its SHA256SUMS lists, which data_fetch writes after the +# downloads passed their md5 checks, matches. pgs: each score of the case is +# there with the md5 file the PGS Catalog serves beside it, and matches it. +data_ok() { + local d="${DATA_CACHE:?}/$1" id + case "$1" in + hla) + [ -s "${d}/SHA256SUMS" ] && [ "$(wc -l < "${d}/SHA256SUMS")" -eq 4 ] \ + && (cd "$d" && sha256sum --quiet -c SHA256SUMS) ;; + pgs) + for id in $(sed -n 's/^NEW_IDS="\(.*\)"$/\1/p' "$PGS_CASE"); do + [ -s "${d}/${id}.txt.gz" ] && [ -s "${d}/${id}.txt.gz.md5" ] \ + && [ "$(md5sum "${d}/${id}.txt.gz" | cut -d' ' -f1)" = "$(awk '{print $1; exit}' "${d}/${id}.txt.gz.md5")" ] \ + || return 1 + done ;; + *) return 1 ;; + esac +} +case "${1:-}" in + --data-key) data_key "${2:?--data-key NAME}"; exit ;; + --data-check) data_ok "${2:?--data-check NAME}"; exit ;; +esac + # pull_all: pull every image of case_images that is not here yet, but the # lazy ones. pull_all() { @@ -443,6 +505,9 @@ data_fetch() ( d="${DATA_CACHE}/$1" # shellcheck source=../lib/common.sh . "${REPO}/scripts/lib/common.sh" || exit 1 + # More and longer-spaced tries than a user's run: EBI and GitHub's raw host + # stayed down for minutes at a time on 2026-11-04 (curl exit 7 and 28). + FETCH_TRIES=${E2E_FETCH_TRIES:-6} FETCH_WAIT=${E2E_FETCH_WAIT:-30} case "$1" in mocha) m="${d}/manifest.txt" @@ -462,24 +527,77 @@ data_fetch() ( s="${d}/1kg_sites.vcf.gz" [ -s "$s" ] && [ "$(_digest md5 "$s")" = "$ARCHAIC_1KG_SITES_MD5" ] && exit 0 fetch "$url" "$s" md5 "$ARCHAIC_1KG_SITES_MD5" || exit 1 ;; + hla) + # The files install_data_file writes, made by install_data_file itself + # with the entry as GENOME_DIR: hla.dat of both releases (md5 from + # IPD-IMGT/HLA's md5checksum.txt) and GENCODE's genes and exons (md5 + # from its MD5SUMS). Their sha256 go to SHA256SUMS, which data_ok checks. + data_ok hla && exit 0 + rm -rf "$d" && mkdir -p "$d" || exit 1 + read -r other pin <<< "$(hla_releases)" + [ -n "$pin" ] || exit 1 + # Each install is tried again as a whole: install_data_file reads + # hla.dat.zip's md5 from md5checksum.txt in one request, outside fetch's + # tries. What an earlier try installed is kept. + ( GENOME_DIR=$d + unset SAMPLE + for job in "$other hla_dat" "$pin hla_dat" "$pin gencode_genes" "$pin gencode_exons"; do + read -r rel name <<< "$job" + for ((t = 1; ; t++)); do + HLA_DB_RELEASE=$rel install_data_file "$name" && break + [ "$t" -lt 3 ] || exit 1 + echo "Data cache: installing ${name} (${rel}) failed, try ${t} of 3; again in ${FETCH_WAIT} s" >&2 + sleep "$FETCH_WAIT" + done + done ) || exit 1 + (cd "$d" && find . -type f ! -name 'SHA256SUMS*' ! -name '*.part' | LC_ALL=C sort | xargs sha256sum) > "${d}/SHA256SUMS.tmp" \ + && mv "${d}/SHA256SUMS.tmp" "${d}/SHA256SUMS" && data_ok hla || exit 1 ;; + pgs) + # The four real PGS Catalog files of prs-scores-and-run-options-1, each + # with the .md5 file served beside it, checked against it. + base=$(sed -n 's/^PGS_BASE=//p' "$PGS_CASE") + [ -n "$base" ] || { echo "ERROR: no PGS_BASE= line in ${PGS_CASE}" >&2; exit 1; } + for id in $(sed -n 's/^NEW_IDS="\(.*\)"$/\1/p' "$PGS_CASE"); do + f="${d}/${id}.txt.gz" url="${base}/${id}/ScoringFiles/Harmonized/${id}_hmPOS_GRCh38.txt.gz" + [ -s "$f" ] && [ -s "${f}.md5" ] && [ "$(_digest md5 "$f")" = "$(awk '{print $1; exit}' "${f}.md5")" ] && continue + rm -f "$f" "${f}.md5" + fetch "${url}.md5" "${f}.md5" || exit 1 + fetch "$url" "$f" md5 "$(awk '{print $1; exit}' "${f}.md5")" || exit 1 + done + data_ok pgs || exit 1 ;; *) echo "ERROR: no data cache entry ${1}" >&2; exit 1 ;; esac ) -# data_name CASE: the E2E_DATA_CACHE entry CASE reads, and where setup.sh wants it. +# data_name CASE: the E2E_DATA_CACHE entry CASE reads, and where setup.sh +# wants it ("-": the case reads the entry itself), then "offline" when the +# case runs with the network blocked once its files are in place. +# bash-step-08-hla: hla.dat of two IPD-IMGT/HLA releases (raw.githubusercontent.com, +# which served it under 1 KB/s at times) and GENCODE's genes and exons +# (ftp.ebi.ac.uk), at the paths data_file names, so install_data_file +# finds them and downloads nothing. +# prs-scores-and-run-options-1: the four real PGS Catalog score files +# (ftp.ebi.ac.uk); the case copies them from E2E_DATA_CACHE/pgs. data_name() { # shellcheck source=../../versions.env case "$1" in mocha-1.sh) echo "mocha ${GENOME_DIR}/reference/mocha-$(. "${REPO}/versions.env" && echo "$MOCHA_PANEL_VERSION")" ;; archaic-ancestry-1.sh) echo "archaic ${GENOME_DIR}/reference/archaic/$(. "${REPO}/versions.env" && echo "$ARCHAIC_DATA_VERSION")/download" ;; + bash-step-08-hla.sh) echo "hla ${GENOME_DIR} offline" ;; + prs-scores-and-run-options-1-strand-sex.sh) echo "pgs -" ;; esac } +# Every HTTP(S) request of a process run under it goes to a closed port. +OFFLINE=(env http_proxy=http://127.0.0.1:9 https_proxy=http://127.0.0.1:9 HTTP_PROXY=http://127.0.0.1:9 + HTTPS_PROXY=http://127.0.0.1:9 no_proxy= NO_PROXY= FETCH_TRIES=1 FETCH_WAIT=0) # data_seed CASE: before CASE, its files from E2E_DATA_CACHE. A failure here # only warns and costs the download: setup.sh in the case fetches what is -# not there. +# not there. Sets CASE_ENV to OFFLINE when the case's entry says "offline" +# and every file is in place, after a control shows the block holds. data_seed() { - local n dest f t + local n dest flag f t placed=true + CASE_ENV=() CASE_CONTROL_FAILED=false [ -n "$DATA_CACHE" ] || return 0 - read -r n dest <<< "$(data_name "$1")" + read -r n dest flag <<< "$(data_name "$1")" [ -n "$n" ] || return 0 if [ -n "$(find "${DATA_CACHE}/${n}" -type f ! -name '*.part' 2> /dev/null | head -n 1)" ]; then echo "Data cache: ${n} restored from the Actions cache" @@ -487,13 +605,29 @@ data_seed() { echo "Data cache: ${n} not in the Actions cache; downloading it" fi data_fetch "$n" || { echo "::warning::Data cache: could not fill ${n}; ${1%.sh} downloads it itself"; return 0; } + [ "$dest" = - ] && { echo "Data cache: ${n} is whole; ${1%.sh} reads it from ${DATA_CACHE}/${n}"; return 0; } while IFS= read -r f; do t="${dest}/${f#./}" mkdir -p "${t%/*}" \ && { ln -f "${DATA_CACHE}/${n}/${f#./}" "$t" 2> /dev/null || cp -f "${DATA_CACHE}/${n}/${f#./}" "$t"; } \ - || echo "::warning::Data cache: could not place ${f#./}; ${1%.sh} downloads it itself" - done < <(cd "${DATA_CACHE}/${n}" && find . -type f ! -name '*.part') - echo "Data cache: ${n}: $(cd "${DATA_CACHE}/${n}" && find . -type f ! -name '*.part' | wc -l | tr -d ' ') files in ${dest}" + || { echo "::warning::Data cache: could not place ${f#./}; ${1%.sh} downloads it itself"; placed=false; } + done < <(cd "${DATA_CACHE}/${n}" && find . -type f ! -name '*.part' ! -name SHA256SUMS) + echo "Data cache: ${n}: $(cd "${DATA_CACHE}/${n}" && find . -type f ! -name '*.part' ! -name SHA256SUMS | wc -l | tr -d ' ') files in ${dest}" + [ "$flag" = offline ] && $placed || return 0 + # Control: under OFFLINE a request to a host that answers must fail, or + # the offline run would prove nothing. + # Read from curl's own error, so a host that is down cannot pass for it. + # (A variable, not a pipe: under pipefail curl's own failure would fail it.) + local out + out=$("${OFFLINE[@]}" curl -fsS --max-time 20 -o /dev/null https://raw.githubusercontent.com/ 2>&1 || true) + if ! grep -q '127\.0\.0\.1.* 9\b' <<< "$out"; then + echo "$out" + echo "::error::Data cache: a request under the network block did not fail at the closed port; ${1%.sh} runs with the network and counts as failed" + CASE_CONTROL_FAILED=true + return 0 + fi + echo "Data cache: control passed (a request under the block failed); ${1%.sh} runs with every HTTP(S) request sent to a closed port" + CASE_ENV=("${OFFLINE[@]}") } # data_drop CASE: after CASE, its cache files go, unless this run saves them. data_drop() { @@ -574,7 +708,7 @@ for c in "${CASES[@]}"; do echo "::group::${name}" data_seed "$c" set +e - timeout -k 30 "$CASE_TIMEOUT" bash "${REPO}/tests/e2e/${c}" 2>&1 | tee "$log" + timeout -k 30 "$CASE_TIMEOUT" "${CASE_ENV[@]}" bash "${REPO}/tests/e2e/${c}" 2>&1 | tee "$log" rc=${PIPESTATUS[0]} set -e data_drop "$c" @@ -585,8 +719,11 @@ for c in "${CASES[@]}"; do docker image inspect "$i" > /dev/null 2>&1 && { docker image rm -f "$i" > /dev/null || true; } done echo "::endgroup::" + # A case whose network block could not be shown to hold proves nothing offline. + [ "$rc" -eq 0 ] && $CASE_CONTROL_FAILED && rc=-1 case "$rc" in 0) result=pass ;; + -1) result="FAIL (network-block control)"; failed=$((failed + 1)) ;; 124) result="FAIL (timeout ${CASE_TIMEOUT}s)"; failed=$((failed + 1)) ;; *) result="FAIL (exit ${rc})"; failed=$((failed + 1)) ;; esac diff --git a/tests/e2e/family-joint-1.sh b/tests/e2e/family-joint-1.sh index 5b466aa5..4c52f753 100755 --- a/tests/e2e/family-joint-1.sh +++ b/tests/e2e/family-joint-1.sh @@ -24,9 +24,11 @@ # - parity: step 38 and the pipeline write the same relationships, # inheritance table, segments, DNA Painter CSV, de novo list and joint # and phased records; -# - a single-sample run without --pedigree publishes the same files as main -# does, after dropping the lines that hold a date or a command line; a -# planted difference shows that the comparison sees one. +# - a single-sample run of HG002 without --pedigree publishes the same +# files, with the same content, as the trio run publishes for HG002 (same +# commit, same tools), after dropping the lines that hold a date or a +# command line: --pedigree adds the family folder and changes nothing +# else. A planted difference shows that the comparison sees one. . "$(dirname "$0")/lib.sh" command -v nextflow >/dev/null || { fail "nextflow is not on PATH"; finish; } @@ -229,6 +231,16 @@ check "nothing was phased or reported after the stop" \ test ! -e "${F}/trio_swapped/family.phased.vcf.gz" -a ! -e "${F}/trio_swapped/family_report.md" # --- 4. The pipeline ---------------------------------------------------------------------------- +# The tools of the trio runs and of the single-sample run of section 5. Not +# clinvar: with it and the partners HG003 and HG004 the trio run would also +# run the couple carrier screen, which case couple-carrier-1 covers. +NF_TOOLS=(--tools roh) +# samplesheet_row SAMPLE [GVCF]: the sample as a VCF row with its gVCF. +samplesheet_row() { + local g=${2:-${F}/$1/vcf/$1.g.vcf.gz} + printf '%s,%s,%s,%s,%s,%s\n' "$1" "${F}/$1/vcf/$1.vcf.gz" "${F}/$1/vcf/$1.vcf.gz.tbi" "$g" "${g}.tbi" "${SEX[$1]}" +} + # nf_family NAME PEDIGREE [HG004_GVCF]: the trio as VCF rows with their gVCFs. nf_family() { local name=$1 ped=$2 hg004=${3:-${F}/HG004/vcf/HG004.g.vcf.gz} dir="${CASE_TMP}/$1" s g @@ -238,14 +250,13 @@ nf_family() { for s in HG002 HG003 HG004; do g="${F}/${s}/vcf/${s}.g.vcf.gz" [ "$s" = HG004 ] && g=$hg004 - printf '%s,%s,%s,%s,%s,%s\n' "$s" "${F}/${s}/vcf/${s}.vcf.gz" "${F}/${s}/vcf/${s}.vcf.gz.tbi" "$g" "${g}.tbi" "${SEX[$s]}" \ - >> "${dir}/samplesheet.csv" + samplesheet_row "$s" "$g" >> "${dir}/samplesheet.csv" done OUT="${dir}/out" echo "+ nextflow run main.nf (${name}) --pedigree ${ped##*/}" ( cd "$dir" && nextflow run "${REPO}/main.nf" -profile docker -ansi-log false \ -work-dir "${F}/nfwork" --input "${dir}/samplesheet.csv" \ - --reference "${F}/reference/${REF_NAME}.fasta" --tools roh \ + --reference "${F}/reference/${REF_NAME}.fasta" "${NF_TOOLS[@]}" \ --pedigree "$ped" --somalier_sites "$SITES" \ --outdir "$OUT" --max_cpus 4 --max_memory 14.GB ) > "${dir}/run.log" 2>&1 RC=$? @@ -256,6 +267,7 @@ nf_family() { nf_family clean "${F}/pedigree.tsv" check_eq "pipeline, the trio: exit" "$RC" 0 +TRIO_OUT=$OUT NF="${OUT}/family" cmp_items=(relationships.tsv HG002_inheritance.tsv.gz HG002_segments.tsv HG002_dnapainter.csv de_novo.tsv) for f in "${cmp_items[@]}"; do @@ -272,10 +284,13 @@ check "the pipeline published somalier's tables under family/somalier" test -s " nf_family swapped "${F}/pedigree_swapped.tsv" check "pipeline, HG002 and HG003 swapped: the run stops (exit ${RC})" test "$RC" -ne 0 -check "FAMILY_CHECK is the task that failed" has "FAMILY:FAMILY_CHECK" "$(cat "$LOG")" +# Both read from files, never from the console: Nextflow's console shows only +# the tail of a failed task's stderr, and in one run the task's stdout lines +# overwrote the start of it there, so the ERROR lines were gone. The run's +# .nextflow.log names the failed process in full. +check "FAMILY_CHECK is the task that failed (.nextflow.log)" \ + grep -qE "Error executing process > '([A-Za-z_]+:)*FAMILY:FAMILY_CHECK" "${CASE_TMP}/swapped/.nextflow.log" # The pair message, read from the failed FAMILY_CHECK task's own .command.err. -# The console's copy of it is not reliable: in one run the task's stdout lines -# overwrote the start of its stderr there, and the ERROR lines were gone. FC_ERR="" for d in "${F}"/nfwork/*/*/; do grep -q 'family_report.py check' "${d}.command.sh" 2>/dev/null || continue @@ -312,7 +327,7 @@ echo "${DN:-no row}" check "the planted site is a de novo candidate of HG002" test -n "$DN" check_eq "with both parents homozygous reference" "$(cut -f7,8 <<< "$DN" | cut -d: -f1 | tr '\t' ' ')" "0/0 0/0" -# --- 5. A single-sample run is what main publishes --------------------------------------------- +# --- 5. A single-sample run is what the trio run publishes for the sample -------------------- # norm DIR: one line per published file (outside pipeline_info), its path and # the md5 of its text without the lines that hold a date or a command line. # An index (.tbi, .csi) is listed by name only: it holds byte offsets into the @@ -326,41 +341,35 @@ norm() { esac done ) } -MAIN="${CASE_TMP}/main" -rm -rf "$MAIN" -mkdir -p "$MAIN" -if git -C "$REPO" fetch -q --depth 1 https://github.com/GeiserX/OwnGenome.git main 2>"${CASE_TMP}/fetch.log"; then - git -C "$REPO" archive FETCH_HEAD | tar -x -C "$MAIN" -fi -check "main's pipeline is checked out" test -s "${MAIN}/main.nf" -single() { - local tree=$1 dir="${CASE_TMP}/single-$2" - rm -rf "$dir" - mkdir -p "$dir" - printf 'sample,vcf,vcf_index,sex\nHG002,%s,%s,male\n' "${F}/HG002/vcf/HG002.vcf.gz" "${F}/HG002/vcf/HG002.vcf.gz.tbi" > "${dir}/samplesheet.csv" - ( cd "$dir" && nextflow run "${tree}/main.nf" -profile docker -ansi-log false -work-dir "${dir}/work" \ - --input "${dir}/samplesheet.csv" --reference "${F}/reference/${REF_NAME}.fasta" --tools roh,clinvar \ - --clinvar "${G0}/clinvar/clinvar_pathogenic_chr.vcf.gz" --clinvar_index "${G0}/clinvar/clinvar_pathogenic_chr.vcf.gz.tbi" \ - --outdir "${dir}/out" --max_cpus 4 --max_memory 14.GB ) > "${dir}/run.log" 2>&1 - SRC=$? - grep -vE 'Pulling|Waiting|Verifying|Download complete|Pull complete|Already exists' "${dir}/run.log" | tail -n 5 - norm "${dir}/out" > "${dir}/files.txt" -} -single "$MAIN" main -check_eq "main, one sample, no --pedigree: exit" "$SRC" 0 -single "$REPO" head -check_eq "this branch, one sample, no --pedigree: exit" "$SRC" 0 -check_ge "files the single-sample run publishes" "$(wc -l < "${CASE_TMP}/single-head/files.txt" | tr -d ' ')" 3 -# A file main does not publish yet is a new output, not a change: compare only -# the files main publishes, so a missing or changed one still fails. -check "a single-sample run publishes every file main publishes, with the same content" \ - diff "${CASE_TMP}/single-main/files.txt" <(awk 'NR == FNR {keep[$1]; next} $1 in keep' "${CASE_TMP}/single-main/files.txt" "${CASE_TMP}/single-head/files.txt") -check "no family/ folder without --pedigree" test ! -e "${CASE_TMP}/single-head/out/family" -F1=$(awk 'NR == 1 {print $1}' "${CASE_TMP}/single-head/files.txt") -cp -r "${CASE_TMP}/single-head/out" "${CASE_TMP}/single-planted" +# The same commit both times, so a pull request that changes a single-sample +# output on purpose changes it in both runs alike. +SINGLE="${CASE_TMP}/single" +rm -rf "$SINGLE" +mkdir -p "$SINGLE" +{ echo 'sample,vcf,vcf_index,gvcf,gvcf_index,sex'; samplesheet_row HG002; } > "${SINGLE}/samplesheet.csv" +( cd "$SINGLE" && nextflow run "${REPO}/main.nf" -profile docker -ansi-log false -work-dir "${F}/nfwork" \ + --input "${SINGLE}/samplesheet.csv" --reference "${F}/reference/${REF_NAME}.fasta" "${NF_TOOLS[@]}" \ + --outdir "${SINGLE}/out" --max_cpus 4 --max_memory 14.GB ) > "${SINGLE}/run.log" 2>&1 +check_eq "one sample, no --pedigree: exit" "$?" 0 +grep -vE 'Pulling|Waiting|Verifying|Download complete|Pull complete|Already exists' "${SINGLE}/run.log" | tail -n 5 +# HG002's folder only: a file at the top of --outdir may sum up every sample. +norm "${SINGLE}/out" | grep '^\./HG002/' > "${SINGLE}/files.txt" +norm "$TRIO_OUT" > "${CASE_TMP}/trio-files.txt" +check_ge "files the single-sample run publishes for HG002" "$(wc -l < "${SINGLE}/files.txt" | tr -d ' ')" 2 +# The trio run publishes more (HG003, HG004, family/): compare the files the +# single-sample run publishes, so a missing or changed one fails. +check "the trio run publishes every file the single-sample run does for HG002, with the same content" \ + diff "${SINGLE}/files.txt" <(awk 'NR == FNR {keep[$1]; next} $1 in keep' "${SINGLE}/files.txt" "${CASE_TMP}/trio-files.txt") +check "no family/ folder without --pedigree" test ! -e "${SINGLE}/out/family" +# Not a .gz file: norm() reads through gzip -cdf, which drops trailing +# garbage after a gzip stream, so a line added to one would not show. +F1=$(awk '$2 != "index" && $1 !~ /\.gz$/ {print $1; exit}' "${SINGLE}/files.txt") +rm -rf "${CASE_TMP}/single-planted" +cp -r "${SINGLE}/out" "${CASE_TMP}/single-planted" echo "planted" >> "${CASE_TMP}/single-planted/${F1#./}" -norm "${CASE_TMP}/single-planted" > "${CASE_TMP}/single-planted.txt" +norm "${CASE_TMP}/single-planted" | grep '^\./HG002/' > "${CASE_TMP}/single-planted.txt" check "the comparison sees a planted difference (a line added to ${F1})" \ - sh -c "! diff -q '${CASE_TMP}/single-head/files.txt' '${CASE_TMP}/single-planted.txt' > /dev/null" + bash -c '! diff -q "$1" <(awk "NR == FNR {keep[\$1]; next} \$1 in keep" "$1" "$2") > /dev/null' \ + _ "${CASE_TMP}/single-planted.txt" "${CASE_TMP}/trio-files.txt" in_e2e rm -rf /w/genome-family finish diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh index df5a98e7..6a515856 100755 --- a/tests/e2e/lib.sh +++ b/tests/e2e/lib.sh @@ -110,6 +110,55 @@ print("\n".join(sorted(out))) PY } +# --- The ancestry panel, from the Actions cache when it is there ----------------- +# EBI's FTP server fails now and then (curl exit 7). The e2e workflow keeps the +# PGS Catalog's HAPNEST panel in the Actions cache (E2E_DATA_CACHE/hapnest, +# keyed on the pin below) for prs-2-ancestry and +# prs-scores-and-run-options-2-panel-chrx. The pin is the sha256 of the file +# EBI served on 2026-11-04 (281,185,251 bytes, md5 +# 3630f2ec3343599f3c55827ddbe38f60 as EBI's md5s.txt lists it). +HAPNEST_SHA256=8ba05cffc4b816f8f8aafd78add52b90ab80315e930936277e09861c5d134aa3 +# NET_BLOCKED: an env prefix that sends every HTTP(S) request to a closed port. +NET_BLOCKED=(env http_proxy=http://127.0.0.1:9 https_proxy=http://127.0.0.1:9 HTTP_PROXY=http://127.0.0.1:9 + HTTPS_PROXY=http://127.0.0.1:9 no_proxy= NO_PROXY= FETCH_TRIES=1 FETCH_WAIT=0) +_sha256() { sha256sum "$1" 2>/dev/null | cut -d' ' -f1; } +# setup_ancestry_panel NAME PANEL: setup.sh --ancestry-panel for panel NAME +# (installed at PANEL), its log in STEP_LOG, with checks. From the cache, the +# panel is checked against HAPNEST_SHA256 and setup.sh runs with the network +# blocked; on a miss setup.sh downloads it as a user's run does, and with +# E2E_HAPNEST_KEEP=true a panel that matches the pin goes into the cache. +setup_ancestry_panel() { + local name=$1 panel=$2 cached="${E2E_DATA_CACHE:-}/hapnest/${1}.tar.zst" from=false + if [ "$name" = GRCh38_HAPNEST_reference ] && [ -n "${E2E_DATA_CACHE:-}" ] && [ -s "$cached" ]; then + if [ "$(_sha256 "$cached")" = "$HAPNEST_SHA256" ]; then + mkdir -p "${panel%/*}" + ln -f "$cached" "$panel" 2>/dev/null || cp -f "$cached" "$panel" + from=true + else + echo "Data cache: ${cached} does not match its pin; setup.sh downloads the panel" + rm -f "$cached" + fi + fi + echo "+ ANCESTRY_PANEL_NAME=${name} scripts/setup.sh --ancestry-panel (panel from the Actions cache: ${from})" + if $from; then + "${NET_BLOCKED[@]}" ANCESTRY_PANEL_NAME="$name" "${REPO}/scripts/setup.sh" --ancestry-panel "$GENOME_DIR" 2>&1 | tee "$STEP_LOG" + check_eq "setup.sh --ancestry-panel exits 0 from the cached panel, with EBI unreachable" "${PIPESTATUS[0]}" 0 + check "and downloads nothing" lacks 'Downloading' "$(cat "$STEP_LOG")" + else + ANCESTRY_PANEL_NAME="$name" "${REPO}/scripts/setup.sh" --ancestry-panel "$GENOME_DIR" 2>&1 | tee "$STEP_LOG" + check_eq "setup.sh --ancestry-panel exits 0" "${PIPESTATUS[0]}" 0 + if [ "$name" = GRCh38_HAPNEST_reference ]; then + check_eq "the downloaded panel matches its pin" "$(_sha256 "$panel")" "$HAPNEST_SHA256" + if [ -n "${E2E_DATA_CACHE:-}" ] && [ "${E2E_HAPNEST_KEEP:-false}" = true ] && [ "$(_sha256 "$panel")" = "$HAPNEST_SHA256" ]; then + mkdir -p "${cached%/*}" + ln -f "$panel" "$cached" 2>/dev/null || cp -f "$panel" "$cached" + echo "Data cache: the panel is in ${cached} for the Actions cache" + fi + fi + fi + echo "${CASE_NAME}: ancestry panel from the Actions cache ${from}" >> "$E2E_NOTES" +} + finish() { if [ "$FAILS" -gt 0 ]; then echo "${CASE_NAME}: ${FAILS} check(s) failed" diff --git a/tests/e2e/prs-2-ancestry.sh b/tests/e2e/prs-2-ancestry.sh index 8c147edb..2fda6ce2 100644 --- a/tests/e2e/prs-2-ancestry.sh +++ b/tests/e2e/prs-2-ancestry.sh @@ -31,9 +31,19 @@ check "that line says it was skipped and how to install the panel" has '^Step 26 check "step 26 without a panel wrote no ancestry table" test ! -e "${D}/ancestry/${SAMPLE}_ancestry.tsv" # --- the panel ------------------------------------------------------------------ -echo "+ ANCESTRY_PANEL_NAME=${PANEL_NAME} scripts/setup.sh --ancestry-panel" -ANCESTRY_PANEL_NAME="$PANEL_NAME" "${REPO}/scripts/setup.sh" --ancestry-panel "$GENOME_DIR" 2>&1 | tee "$STEP_LOG" -check_eq "setup.sh --ancestry-panel exits 0" "${PIPESTATUS[0]}" 0 +# From the Actions cache when it is there, with EBI unreachable +# (setup_ancestry_panel in lib.sh, which holds the pin HAPNEST_SHA256). +# Red control: with the network blocked and no panel, setup.sh must fail, or +# the cached run would prove nothing about EBI. +mkdir -p "${CASE_TMP}/blocked" +"${NET_BLOCKED[@]}" ANCESTRY_PANEL_NAME="$PANEL_NAME" "${REPO}/scripts/setup.sh" --ancestry-panel "${CASE_TMP}/blocked" \ + > "${CASE_TMP}/blocked.log" 2>&1 +BRC=$? +tail -n 3 "${CASE_TMP}/blocked.log" +check "with EBI unreachable and no panel, setup.sh --ancestry-panel fails (exit ${BRC})" \ + bash -c '[ "$1" -ne 0 ] && grep -q "Could not download the ancestry panel" "$2"' _ "$BRC" "${CASE_TMP}/blocked.log" +rm -rf "${CASE_TMP}/blocked" +setup_ancestry_panel "$PANEL_NAME" "$PANEL" check "the panel is installed" test -s "$PANEL" check_ge "GRCh38 SNVs in the panel's site list" "$(wc -l < "$SITES" 2>/dev/null | tr -d ' ')" 1000 check "the site list is chrN, position, REF, ALT" \ @@ -58,6 +68,19 @@ done # --- step 26 with the panel (it runs step 25) ------------------------------------- ANCESTRY_PANEL="$PANEL" run_step 26-ancestry.sh "$SAMPLE" +# pgsc_calc's INTERSECT_VARIANTS sorts the panel in chunks under random +# temporary names, and reads back any file whose name holds "bim" as a plink +# .bim file (pgscatalog.match intersect_cli.py, read_var_general). A chunk +# named, say, tmpq7bimx2a then fails with KeyError: 'CHR:POS:A0:A1', under the +# script line 'No variants in intersection': about 1 run in 600 with this +# panel's 14 chunks. That one failure is run again once, with new names; any +# other failure stays a failure. +if [ "$STEP_RC" -ne 0 ] && grep -q "KeyError: 'CHR:POS:A0:A1'" "$STEP_LOG"; then + echo "pgsc_calc INTERSECT_VARIANTS read a temporary chunk as a .bim file (its random name held 'bim'): running step 26 once more" + echo "prs-2-ancestry: step 26 ran twice (pgsc_calc's 'bim' temporary name)" >> "$E2E_NOTES" + rm -rf "${D}/prs" "${D}/ancestry" + ANCESTRY_PANEL="$PANEL" run_step 26-ancestry.sh "$SAMPLE" +fi check_step_exit "26-ancestry.sh (with ${PANEL_NAME})" LOG26=$(cat "$STEP_LOG") ANC="${D}/ancestry/${SAMPLE}_ancestry.tsv" diff --git a/tests/e2e/prs-scores-and-run-options-1-strand-sex.sh b/tests/e2e/prs-scores-and-run-options-1-strand-sex.sh index ae536f91..35e28bc6 100644 --- a/tests/e2e/prs-scores-and-run-options-1-strand-sex.sh +++ b/tests/e2e/prs-scores-and-run-options-1-strand-sex.sh @@ -111,13 +111,23 @@ check_eq "chrX hom-ref rows" "$(wc -l < "$X_SITES" | tr -d ' ')" 3 SCORES="${GENOME_DIR}/prs_scores" rm -rf "$SCORES" "${D}/prs" mkdir -p "$SCORES" +# scripts/ci/e2e-run.sh keeps the four real files and the .md5 files served +# beside them in E2E_DATA_CACHE/pgs (the e2e workflow's Actions cache), so an +# EBI outage cannot fail the case; without that entry they are downloaded here. PGS_BASE=https://ftp.ebi.ac.uk/pub/databases/spot/pgs/scores +PGS_CACHE="${E2E_DATA_CACHE:-}/pgs" for id in $(awk -F'\t' '$1 ~ /^PGS[0-9]+$/ {print $1}' "${REPO}/assets/pgs_scores.tsv"); do case " ${NEW_IDS} " in *" ${id} "*) url="${PGS_BASE}/${id}/ScoringFiles/Harmonized/${id}_hmPOS_GRCh38.txt.gz" - curl -fsSL --retry 3 -o "${SCORES}/${id}.txt.gz" "$url" - want=$(curl -fsSL --retry 3 "${url}.md5" | awk '{print $1}') + if [ -n "${E2E_DATA_CACHE:-}" ] && [ -s "${PGS_CACHE}/${id}.txt.gz" ] && [ -s "${PGS_CACHE}/${id}.txt.gz.md5" ]; then + echo "${id}: from the Actions cache (${PGS_CACHE}), no download" + cp "${PGS_CACHE}/${id}.txt.gz" "${SCORES}/${id}.txt.gz" + want=$(awk '{print $1; exit}' "${PGS_CACHE}/${id}.txt.gz.md5") + else + curl -fsSL --retry 3 -o "${SCORES}/${id}.txt.gz" "$url" + want=$(curl -fsSL --retry 3 "${url}.md5" | awk '{print $1}') + fi check_eq "${id}: the PGS Catalog's GRCh38-harmonised file, md5 checked" \ "$( (md5sum "${SCORES}/${id}.txt.gz" 2>/dev/null || echo x) | awk '{print $1}')" "${want:-none}" continue ;; diff --git a/tests/e2e/prs-scores-and-run-options-2-panel-chrx.sh b/tests/e2e/prs-scores-and-run-options-2-panel-chrx.sh index 276edc1b..f2317217 100644 --- a/tests/e2e/prs-scores-and-run-options-2-panel-chrx.sh +++ b/tests/e2e/prs-scores-and-run-options-2-panel-chrx.sh @@ -27,9 +27,8 @@ PANEL_NAME=GRCh38_HAPNEST_reference PANEL="${GENOME_DIR}/reference/pgsc_calc/${PANEL_NAME}.tar.zst" PSITES="${PANEL%.tar.zst}_GRCh38_sites.tsv" -echo "+ ANCESTRY_PANEL_NAME=${PANEL_NAME} scripts/setup.sh --ancestry-panel" -ANCESTRY_PANEL_NAME="$PANEL_NAME" "${REPO}/scripts/setup.sh" --ancestry-panel "$GENOME_DIR" 2>&1 | tee "$STEP_LOG" -check_eq "setup.sh --ancestry-panel exits 0" "${PIPESTATUS[0]}" 0 +# From the Actions cache when it is there, with EBI unreachable (lib.sh). +setup_ancestry_panel "$PANEL_NAME" "$PANEL" check_ge "GRCh38 SNVs in the panel's site list" "$(wc -l < "$PSITES" 2>/dev/null | tr -d ' ')" 1000 # The panel's own chrX variants (its .pvar or .pvar.zst files: chromosome X or 23). PVARS=$(tar --zstd -tf "$PANEL" 2>/dev/null | grep -E '\.pvar(\.zst)?$' | tr '\n' ' ') diff --git a/tests/test_e2e_data_cache.sh b/tests/test_e2e_data_cache.sh new file mode 100755 index 00000000..18077390 --- /dev/null +++ b/tests/test_e2e_data_cache.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# The data cache entries hla and pgs of scripts/ci/e2e-run.sh: --data-check +# passes a whole entry and fails a changed, missing or partial one (the e2e +# workflow saves and uses only an entry that passes), and --data-key names +# the releases. No network, no docker. +set -uo pipefail +REPO="$(cd "$(dirname "$0")/.." && pwd)" +RUN="${REPO}/scripts/ci/e2e-run.sh" +T=$(mktemp -d) +trap 'rm -rf "$T"' EXIT +failed=0 +# expect DESC 0|1 NAME: --data-check NAME on $T exits as said. +expect() { + local rc=0 + E2E_DATA_CACHE=$T "$RUN" --data-check "$3" > /dev/null 2>&1 || rc=$? + if { [ "$2" = 0 ] && [ "$rc" -eq 0 ]; } || { [ "$2" = 1 ] && [ "$rc" -ne 0 ]; }; then echo "PASS: $1" + else echo "FAIL: $1 (exit ${rc})"; failed=1; fi +} + +# hla: four files and the SHA256SUMS data_fetch writes for them. +mkdir -p "${T}/hla/hla/IPD-IMGT-HLA_1/" "${T}/hla/hla/IPD-IMGT-HLA_2" "${T}/hla/reference" +for f in hla/IPD-IMGT-HLA_1/hla.dat hla/IPD-IMGT-HLA_2/hla.dat reference/genes.gtf reference/exons.bed; do + echo "$f" > "${T}/hla/${f}" +done +(cd "${T}/hla" && find . -type f ! -name SHA256SUMS | LC_ALL=C sort | xargs sha256sum > SHA256SUMS) +expect "hla: a whole entry passes" 0 hla +echo changed >> "${T}/hla/reference/genes.gtf" +expect "hla: a changed file fails" 1 hla +sed -i.bak '$d' "${T}/hla/reference/genes.gtf" && rm -f "${T}/hla/reference/genes.gtf.bak" +expect "hla: the file put back passes again" 0 hla +sed -i.bak '$d' "${T}/hla/SHA256SUMS" && rm -f "${T}/hla/SHA256SUMS.bak" +expect "hla: a SHA256SUMS with three of the four files fails" 1 hla +rm -f "${T}/hla/SHA256SUMS" +expect "hla: no SHA256SUMS fails" 1 hla + +# pgs: each score id of the case, with its .md5 beside it. +ids=$(sed -n 's/^NEW_IDS="\(.*\)"$/\1/p' "${REPO}/tests/e2e/prs-scores-and-run-options-1-strand-sex.sh") +[ -n "$ids" ] || { echo "FAIL: no NEW_IDS line in prs-scores-and-run-options-1-strand-sex.sh"; exit 1; } +mkdir -p "${T}/pgs" +for id in $ids; do + echo "$id" | gzip -c > "${T}/pgs/${id}.txt.gz" + echo "$(md5sum < "${T}/pgs/${id}.txt.gz" | cut -d' ' -f1) ${id}_hmPOS_GRCh38.txt.gz" > "${T}/pgs/${id}.txt.gz.md5" +done +expect "pgs: every score with a matching .md5 passes" 0 pgs +last=${ids##* } +echo x | gzip -c > "${T}/pgs/${last}.txt.gz" +expect "pgs: a score that does not match its .md5 fails" 1 pgs +rm -f "${T}/pgs/${last}.txt.gz" "${T}/pgs/${last}.txt.gz.md5" +expect "pgs: a missing score fails" 1 pgs +expect "an unknown entry fails" 1 nosuch + +key=$("$RUN" --data-key hla) +pin=$(sed -n 's/^HLA_DB_RELEASE=\${HLA_DB_RELEASE:-\(.*\)}$/\1/p' "${REPO}/scripts/lib/common.sh") +other=$(sed -n 's/^OTHER=//p' "${REPO}/tests/e2e/bash-step-08-hla.sh") +gencode=$(sed -n 's/^GENCODE_RELEASE=//p' "${REPO}/scripts/lib/common.sh") +if [ -n "$other" ] && [ -n "$pin" ] && [ -n "$gencode" ] \ + && [[ "$key" =~ ^e2e-data-hla-${other//./\\.}-${pin//./\\.}-gencode${gencode}-[0-9a-f]{16}$ ]]; then + echo "PASS: the hla key names ${other}, ${pin}, GENCODE ${gencode} and a hash: ${key}" +else echo "FAIL: the hla key does not match its inputs (${other:-?} ${pin:-?} ${gencode:-?}): ${key}"; failed=1; fi +if "$RUN" --data-key pgs | grep -qE '^e2e-data-pgs-[0-9a-f]{16}$'; then echo "PASS: the pgs key is a hash" +else echo "FAIL: the pgs key"; failed=1; fi +exit "$failed" diff --git a/tests/test_exec_bits.sh b/tests/test_exec_bits.sh new file mode 100644 index 00000000..56ca1aa3 --- /dev/null +++ b/tests/test_exec_bits.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# scripts/ci/check-exec-bits.sh on a planted tree: a helper a module calls by +# name, committed without its executable bit, fails the check; with the bit +# it passes; a helper named only as a path (bin/x.py in a comment) is not a +# call; a bare name after an interpreter or in a comment is one (on purpose, +# see the script); an executable helper without a #! line fails. Then the +# real tree. +set -uo pipefail + +REPO="$(cd "$(dirname "$0")/.." && pwd)" +CHECK="${REPO}/scripts/ci/check-exec-bits.sh" +T=$(mktemp -d) +trap 'rm -rf "$T"' EXIT +fails=0 +ok() { echo "PASS $*"; } +bad() { echo "FAIL $*"; fails=$((fails + 1)); } + +g() { git -C "$T" -c user.name=t -c user.email=t@example.invalid "$@"; } +g init -q +mkdir -p "${T}/bin" "${T}/modules/local/scan" +printf '#!/usr/bin/env python3\nprint(1)\n' > "${T}/bin/scan_it.py" +printf '#!/usr/bin/env python3\nprint(2)\n' > "${T}/bin/only_named.py" +printf '#!/usr/bin/env python3\nprint(3)\n' > "${T}/bin/via_python.py" +printf '#!/usr/bin/env python3\nprint(4)\n' > "${T}/bin/in_comment.py" +cat > "${T}/modules/local/scan/main.nf" <<'NF' +process SCAN { + // reads what bin/only_named.py wrote + // see in_comment.py + script: + """ + scan_it.py --out x.tsv + python3 via_python.py + """ +} +NF +printf 'workflow { SCAN() }\n' > "${T}/main.nf" +g add -A +g update-index --chmod=-x bin/scan_it.py bin/only_named.py bin/via_python.py bin/in_comment.py + +out=$(bash "$CHECK" "$T" 2>&1); rc=$? +if [ "$rc" -ne 0 ] && grep -q 'FAIL bin/scan_it.py: mode 100644' <<< "$out"; then ok "a called helper without the bit fails" +else bad "a called helper without the bit fails (exit ${rc}): ${out}"; fi +if grep -q 'only_named' <<< "$out"; then bad "a helper named only as a path is not a call: ${out}" +else ok "a helper named only as a path is not a call"; fi +# On purpose: a bare name after an interpreter or in a comment counts as a call. +for h in via_python in_comment; do + if grep -q "FAIL bin/${h}.py: mode 100644" <<< "$out"; then ok "a bare name (${h}) counts as a call and needs the bit" + else bad "a bare name (${h}) counts as a call and needs the bit: ${out}"; fi +done + +g update-index --chmod=+x bin/scan_it.py bin/via_python.py bin/in_comment.py +out=$(bash "$CHECK" "$T" 2>&1); rc=$? +if [ "$rc" -eq 0 ]; then ok "with the bit set it passes"; else bad "with the bit set it passes (exit ${rc}): ${out}"; fi + +printf 'print(1)\n' > "${T}/bin/scan_it.py" +out=$(bash "$CHECK" "$T" 2>&1); rc=$? +if [ "$rc" -ne 0 ] && grep -q 'not a #! line' <<< "$out"; then ok "an executable helper without #! fails" +else bad "an executable helper without #! fails (exit ${rc}): ${out}"; fi + +printf 'workflow { }\n' > "${T}/main.nf" +rm -rf "${T}/modules" +out=$(bash "$CHECK" "$T" 2>&1); rc=$? +if [ "$rc" -ne 0 ]; then ok "a tree where no helper is called fails (the parser saw nothing)" +else bad "a tree where no helper is called fails: ${out}"; fi + +out=$(bash "$CHECK" "$REPO" 2>&1); rc=$? +echo "$out" | tail -n 3 +if [ "$rc" -eq 0 ]; then ok "this checkout"; else bad "this checkout (exit ${rc})"; fi + +[ "$fails" -eq 0 ] || { echo "${fails} check(s) failed"; exit 1; } +echo "all checks passed"