GeiserX · Nov 3, 2026 · Nov 2, 2026 · Nov 2, 2026 · Nov 2, 2026 · Nov 2, 2026
@@ -79,7 +79,7 @@ on:
         type: boolean
         default: false
       save_images:
-        description: 'e2e: save the image cache from this branch too (only main saves by default)'
+        description: 'e2e: save the image and data caches from this branch too (only main saves by default)'
         type: boolean
         default: false
       trace:
@@ -395,6 +395,13 @@ jobs:
             sudo mkdir -p "$work" && sudo chown "$(id -u):$(id -g)" "$work"
           fi
           echo "E2E_WORK=${work}" >> "$GITHUB_ENV"
+          # One fixed path: actions/cache counts the path in an entry's
+          # version, so a path that followed the work area would miss on a
+          # runner that picked the other disk. e2e-run.sh hard-links the
+          # cached downloads into the work area, or copies them when the
+          # work area is on /mnt.
+          mkdir -p "${RUNNER_TEMP}/e2e-data"
+          echo "E2E_DATA_CACHE=${RUNNER_TEMP}/e2e-data" >> "$GITHUB_ENV"
 
       - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0
         with:
@@ -493,6 +500,8 @@ jobs:
           echo "$(wc -l < "$list") cases: $(paste -sd' ' "$list")"
           grep -qx nextflow-from-fastq-2-nextflow "$list" && echo "parity_fastq=true" >> "$GITHUB_OUTPUT"
           grep -qx nextflow-hardening-2-nf "$list" && echo "parity_hardening=true" >> "$GITHUB_OUTPUT"
+          grep -qx mocha-1 "$list" && echo "data_mocha=true" >> "$GITHUB_OUTPUT"
+          grep -qx archaic-ancestry-1 "$list" && echo "data_archaic=true" >> "$GITHUB_OUTPUT"
           echo "list=${list}" >> "$GITHUB_OUTPUT"
 
       # Before the base outputs: the base leg pulled the same images.
@@ -535,6 +544,38 @@ jobs:
           echo "::error::Could not fetch e2e-base in 5 tries."
           exit 1
 
+      # The two largest downloads of the cases, from EBI's 1000 Genomes
+      # server, which is slow and stalls at times: MoChA's chr20 panel and
+      # its manifest (mocha-1, 0.64 GB), keyed on the manifest's pin and the
+      # chromosome (the whole raw chr20 panel, which no fixture changes), and the 1000
+      # Genomes sites VCF (archaic-ancestry-1, 1.36 GB), keyed on its md5.
+      # e2e-run.sh hands them to the two cases ("Downloads kept between
+      # runs" there) and downloads what a miss leaves out. Saved like the
+      # images: on main, or a run by hand with save_images, on a miss. Only
+      # the leg whose cases include the case restores and saves its entry.
+      - name: Data cache keys
+        id: data-keys
+        run: |
+          . versions.env
+          echo "mocha=e2e-data-mocha-${MOCHA_PANEL_MANIFEST_SHA256:0:16}-chr20" | tee -a "$GITHUB_OUTPUT"
+          echo "archaic=e2e-data-1kg-sites-${ARCHAIC_1KG_SITES_MD5}" | tee -a "$GITHUB_OUTPUT"
+
+      - name: Restore cached data, MoChA panel
+        id: data-mocha
+        if: steps.cases-list.outputs.data_mocha == 'true'
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ env.E2E_DATA_CACHE }}/mocha
+          key: ${{ steps.data-keys.outputs.mocha }}
+
+      - name: Restore cached data, 1000 Genomes sites
+        id: data-archaic
+        if: steps.cases-list.outputs.data_archaic == 'true'
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ env.E2E_DATA_CACHE }}/archaic
+          key: ${{ steps.data-keys.outputs.archaic }}
+
       - name: Run the e2e cases
         id: cases
         env:
@@ -544,7 +585,50 @@ jobs:
           E2E_TRACE: ${{ needs.e2e-plan.outputs.trace }}
           E2E_BASE_OUT: ${{ matrix.job == 'base' && format('{0}/base.tar.zst', runner.temp) || '' }}
           E2E_BASE_IN: ${{ matrix.job != 'base' && format('{0}/e2e-base/base.tar.zst', runner.temp) || '' }}
-        run: scripts/ci/e2e-run.sh
+          # Keep the downloads to the end of the job only when a save step uses them.
+          E2E_DATA_KEEP: ${{ (github.ref == 'refs/heads/main' || inputs.save_images) && ((steps.cases-list.outputs.data_mocha == 'true' && steps.data-mocha.outputs.cache-hit != 'true') || (steps.cases-list.outputs.data_archaic == 'true' && steps.data-archaic.outputs.cache-hit != 'true')) && 'true' || 'false' }}
+          HIT_MOCHA: ${{ steps.data-mocha.outputs.cache-hit || 'false' }}
+          HIT_ARCHAIC: ${{ steps.data-archaic.outputs.cache-hit || 'false' }}
+          DATA_MOCHA: ${{ steps.cases-list.outputs.data_mocha || 'false' }}
+          DATA_ARCHAIC: ${{ steps.cases-list.outputs.data_archaic || 'false' }}
+        run: |
+          if [ "$DATA_MOCHA" = true ] || [ "$DATA_ARCHAIC" = true ]; then
+            echo "Data cache (${JOB}): exact key hit, MoChA panel ${HIT_MOCHA}, 1000 Genomes sites ${HIT_ARCHAIC}" | tee -a "$GITHUB_STEP_SUMMARY"
+          fi
+          scripts/ci/e2e-run.sh
+
+      # A miss saves what e2e-run.sh downloaded, checked against its pin
+      # (data_fetch there), when every file of the entry is in place.
+      - name: Data cache, what to save
+        id: data-save
+        if: >-
+          always() && steps.cases.outcome != 'skipped' &&
+          (github.ref == 'refs/heads/main' || inputs.save_images)
+        env:
+          HIT_MOCHA: ${{ steps.data-mocha.outputs.cache-hit }}
+          HIT_ARCHAIC: ${{ steps.data-archaic.outputs.cache-hit }}
+        run: |
+          d=$E2E_DATA_CACHE
+          if [ "$HIT_MOCHA" != true ] && [ -s "${d}/mocha/manifest.txt" ] \
+             && [ "$(find "${d}/mocha/raw" -type f ! -name '*.part' 2> /dev/null | wc -l)" -eq 2 ]; then
+            echo "mocha=true" >> "$GITHUB_OUTPUT"
+          fi
+          if [ "$HIT_ARCHAIC" != true ] && [ -s "${d}/archaic/1kg_sites.vcf.gz" ]; then
+            echo "archaic=true" >> "$GITHUB_OUTPUT"
+          fi
+          du -sh "${d}"/* 2> /dev/null || echo "Data cache: nothing downloaded"
+
+      - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        if: always() && steps.data-save.outputs.mocha == 'true'
+        with:
+          path: ${{ env.E2E_DATA_CACHE }}/mocha
+          key: ${{ steps.data-keys.outputs.mocha }}
+
+      - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        if: always() && steps.data-save.outputs.archaic == 'true'
+        with:
+          path: ${{ env.E2E_DATA_CACHE }}/archaic
+          key: ${{ steps.data-keys.outputs.archaic }}
 
       - name: Upload the base outputs
         if: matrix.job == 'base'
@@ -617,17 +701,18 @@ jobs:
       # readable by that pull request alone, and every entry counts against
       # the repository's 10 GB, so it would only push main's entries out.
       # Each group keeps its smallest images up to GROUP_BUDGET_MB of zstd
-      # output; the rest are pulled on every run. Four groups of 1900 MB are
-      # 7.4 GB, and while main replaces one group the old and the new entry
-      # both count: 9.3 GB, under the 10 GB limit. An image pinned by digest
+      # output; the rest are pulled on every run. Four groups of 1500 MB are
+      # 5.9 GB, and while main replaces one group the old and the new entry
+      # both count: 7.3 GB. With the two data entries above (2.0 GB) that is
+      # 9.3 GB, under the 10 GB limit. An image pinned by digest
       # is never saved: docker load gives it back without its digest, so
       # e2e-run.sh would pull it again anyway.
       - name: Image sizes, and save images for the next run
         id: save
         if: always() && matrix.job == 'base' && steps.keys.outcome == 'success'
         env:
           SAVE: ${{ (github.ref == 'refs/heads/main' || inputs.save_images) && 'true' || 'false' }}
-          GROUP_BUDGET_MB: 1900
+          GROUP_BUDGET_MB: 1500
           HIT0: ${{ steps.images0.outputs.cache-hit }}
           HIT1: ${{ steps.images1.outputs.cache-hit }}
           HIT2: ${{ steps.images2.outputs.cache-hit }}

@@ -0,0 +1,130 @@
+name: Repeat a test
+
+# Runs one test many times, each run in a fresh job on its own runner, to
+# tell a flaky test from a broken one: a flake fails a few runs out of many,
+# a broken test fails them all. Start it from the Actions tab with the test's
+# path and the number of runs. It also runs on a pull request that changes
+# this file, with the defaults, so a change to it is seen working.
+#
+# One extra run, "control", runs a test that always fails. The summary
+# counts the failed runs and fails when any did, or when the control did
+# not fail, so a count of zero failures is never a count that could not
+# see one.
+on:
+  workflow_dispatch:
+    inputs:
+      test:
+        description: "Test to repeat (a tests/test_*.sh or tests/test_*.py)"
+        default: tests/test_annotsv_benign_mane.sh
+      runs:
+        description: "How many runs (1 to 60)"
+        default: "30"
+  pull_request:
+    branches: [main]
+    paths: [.github/workflows/repeat-test.yml]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  TEST: ${{ inputs.test || 'tests/test_annotsv_benign_mane.sh' }}
+  RUNS: ${{ inputs.runs || '30' }}
+
+jobs:
+  plan:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    outputs:
+      runs: ${{ steps.plan.outputs.runs }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Check the inputs and list the runs
+        id: plan
+        run: |
+          case "$TEST" in
+            tests/test_*.sh|tests/test_*.py) ;;
+            *) echo "ERROR: '${TEST}' is not a tests/test_*.sh or tests/test_*.py"; exit 1 ;;
+          esac
+          [ -f "$TEST" ] || { echo "ERROR: ${TEST} does not exist"; exit 1; }
+          case "$RUNS" in ''|*[!0-9]*) echo "ERROR: runs must be a whole number, got '${RUNS}'"; exit 1 ;; esac
+          n=$((10#$RUNS))
+          [ "$n" -ge 1 ] && [ "$n" -le 60 ] || { echo "ERROR: runs must be 1 to 60, got ${n}"; exit 1; }
+          echo "runs=$( (seq 1 "$n"; echo control) | jq -R . | jq -sc .)" >> "$GITHUB_OUTPUT"
+          echo "${TEST}: ${n} runs and the control"
+
+  run:
+    needs: plan
+    name: run ${{ matrix.run }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      # GitHub runs at most 20 jobs at once on this account; leave room for
+      # the other workflows.
+      max-parallel: 6
+      matrix:
+        run: ${{ fromJSON(needs.plan.outputs.runs) }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      # The step records the test's exit code in the name of a small
+      # artifact and passes, so a failed test leaves the job green and the
+      # summary below is the one place that counts.
+      - name: Run the test
+        env:
+          RUN: ${{ matrix.run }}
+        run: |
+          t=$TEST
+          if [ "$RUN" = control ]; then
+            t="${RUNNER_TEMP}/test_planted.sh"
+            echo 'echo "planted failure"; exit 1' > "$t"
+          fi
+          rc=0
+          case "$t" in
+            *.py) python3 "$t" || rc=$? ;;
+            *) bash "$t" || rc=$? ;;
+          esac
+          echo "run ${RUN}: exit ${rc}"
+          echo "$rc" > "${RUNNER_TEMP}/rc"
+          echo "RC=${rc}" >> "$GITHUB_ENV"
+
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: result-${{ matrix.run }}-exit-${{ env.RC }}
+          path: ${{ runner.temp }}/rc
+          # A rerun of a job may write the same name again.
+          overwrite: true
+          retention-days: 1
+
+  summary:
+    needs: [plan, run]
+    if: always() && needs.plan.result == 'success'
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      actions: read
+    steps:
+      - name: Count the failed runs
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts" \
+            -q '.artifacts[] | select(.name | startswith("result-")) | "\(.created_at)\t\(.id)\t\(.name)"' \
+            | sort -t$'\t' -k1,1 -k2,2n \
+            | sed -n 's/^.*\tresult-\(.*\)-exit-\([0-9]*\)$/\1\t\2/p' \
+            | awk -F'\t' '{ last[$1] = $2 } END { for (s in last) print s "\t" last[s] }' > results.tsv
+          # A rerun leaves the old attempt's artifact beside the new one;
+          # the newest result of each run counts.
+          sort -V results.tsv
+          ctl=$(awk -F'\t' '$1 == "control" {print $2}' results.tsv)
+          [ -n "$ctl" ] && [ "$ctl" -ne 0 ] || { echo "FAIL: the control run should fail, its exit is '${ctl:-missing}'"; exit 1; }
+          n=$(awk -F'\t' '$1 != "control"' results.tsv | wc -l)
+          bad=$(awk -F'\t' '$1 != "control" && $2 != 0' results.tsv | wc -l)
+          echo "${TEST}: ${bad} of ${n} runs failed (the control failed, as it should)." | tee -a "$GITHUB_STEP_SUMMARY"
+          [ "$n" -eq "$((10#$RUNS))" ] || { echo "FAIL: ${n} results found, ${RUNS} runs planned"; exit 1; }
+          [ "$bad" -eq 0 ] || { echo "FAIL: runs that failed: $(awk -F'\t' '$1 != "control" && $2 != 0 {print $1}' results.tsv | paste -sd' ' -)"; exit 1; }
diff --git a/docs/lessons-learned.md b/docs/lessons-learned.md
@@ -616,3 +616,15 @@ Most bioinformatics containers run as non-root users. If writing to bind-mounted
 - **Failed:** `setup.sh --archaic` stopped on a wrong md5 for the 1000 Genomes phase 3 sites VCF (about 1.5 GB) in the e2e case `archaic-ancestry-1`. Other runs on the same day downloaded the same file with the right md5.
 - **Why:** `fetch` tried a download again only when the transfer itself failed. A transfer that finished with the wrong bytes failed its check, and `fetch` deleted it and returned 1 at once, so one bad transfer ended the run.
 - **Fix:** a download that fails its checksum, `gzip -t` or non-empty check is deleted and downloaded again from the start, up to `FETCH_TRIES` tries `FETCH_WAIT` apart, the same budget as a broken transfer. A broken transfer still keeps its `.part` and the next try resumes it. `tests/test_fetch_retry.sh` serves a bad file once and then the right one, and a bad file every time, and checks both against a mutant that returns at the first failed check.
+
+## CI flakes (2026-11)
+
+### AnnotSV's merge run in `setup.sh` failed in two seconds
+- **Failed:** `tests/test_annotsv_benign_mane.sh` failed now and then with `setup.sh --annotsv-data` exiting 1, and the rerun passed. Its log showed AnnotSV's merge run ending with exit 125 two seconds after it started, and the gnomAD-SV file left unmerged.
+- **Why:** the merge run was the first use of the AnnotSV image on the runner, so `docker run` pulled it. One pull broke off (`short read: expected 603618355 bytes but got 8388608: unexpected EOF`), and `docker run` does not try a pull again: it exits 125 before AnnotSV starts.
+- **Fix:** `annotsv_merge_benign` pulls the image first with `pull_image`, which tries `FETCH_TRIES` times. The **Repeat a test** workflow (`.github/workflows/repeat-test.yml`) runs the test 30 times on fresh runners to show it no longer flakes.
+
+### A slow download used up an e2e case's hour
+- **Failed:** `archaic-ancestry-1` ran into the 3600-second case timeout on one e2e run (run 40933547961, attempt 1) and pushed the job past its own limit; the rerun passed.
+- **Why:** `setup.sh --archaic` downloads the 1000 Genomes phase 3 sites VCF (1.36 GB) from EBI. On that run EBI sent it at about 440 KB/s, and the download alone took 53 minutes, so the case's Nextflow half started with minutes left. The download never stopped: it was slow, not stalled. MoChA's chr20 panel in `mocha-1` comes from the same server.
+- **Fix:** the e2e job caches both files (keyed on their pins) and `scripts/ci/e2e-run.sh` hands them to the two cases, so a run with a cache hit downloads neither. Separately, `_get` now bounds a real stall, which nothing did before: `curl` gets `--connect-timeout 30 --speed-limit 1024 --speed-time ${FETCH_STALL:-120}`, so a transfer under 1 KB/s for two minutes fails and `fetch` tries again, resuming the `.part`. `wget` gets `--connect-timeout=30 --read-timeout=${FETCH_STALL:-120}`, which fails only when no data arrives for that long. A slow but moving transfer like the one above is left alone.