From 6f406670d6ed5570532e37b8cf1ae9365c9f59b7 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 6 Nov 2026 22:23:16 +0100 Subject: [PATCH 1/3] fix(recheck): see image digest changes, skip opt-in steps the run left out, never crash on a bad header An image re-pinned to a new digest at the same tag compared equal, because the check reduced every image to its tag; it now compares the full reference. A pin with no upstream lookup that the run's manifest lacks is now "unknown" instead of "no-newer-release". A pin read only by steps run-all.sh recorded as skipped is left out. A malformed Last-Modified of a PGS file and an offline release with no tag are now errors of that lookup, not a crash. --refresh-clinvar now holds GENOME_DIR/clinvar/.refresh.lock exclusive while setup.sh refreshes; run-all.sh and step 06 hold it shared, so a run that starts during a refresh waits for it. Bead: pgp-5bja.23 Bead: pgp-ax3g --- bin/owngenome_recheck.py | 93 ++++++++++-- docs/recheck.md | 14 +- scripts/06-clinvar-screen.sh | 12 ++ scripts/run-all.sh | 11 ++ tests/test_owngenome_recheck.py | 249 ++++++++++++++++++++++++++++++++ 5 files changed, 360 insertions(+), 19 deletions(-) diff --git a/bin/owngenome_recheck.py b/bin/owngenome_recheck.py index 6dd4c69dd..0430e60e3 100755 --- a/bin/owngenome_recheck.py +++ b/bin/owngenome_recheck.py @@ -22,6 +22,8 @@ in-pipeline-release a newer pin is in a pipeline release: update the pipeline upstream-only upstream has a newer release that no pipeline release pins no-newer-release the lookup answered and nothing is newer + unknown a pin with no upstream lookup that the run's manifest + does not name: nothing was compared error the lookup failed (never read as up to date); exit 1 The steps an item feeds come from the reads column of assets/steps.tsv, then every step downstream of them through its upstream column, then the reports. @@ -37,7 +39,9 @@ check named, and lists the named script steps (04b, 14, 29, 03a-03d), which skip when their output exists, as not redone. --refresh-clinvar runs setup.sh --refresh clinvar first, and refuses while a run of any sample of -GENOME_DIR is going on, since they share the ClinVar files. Standard library only. +GENOME_DIR is going on, since they share the ClinVar files. It holds +GENOME_DIR/clinvar/.refresh.lock while it refreshes: run-all.sh and step 06 +wait on it. Standard library only. """ import argparse import fcntl @@ -66,8 +70,11 @@ REPORT_STEP = "24" # the HTML report; the text report is written after it # ClinVar's ##fileDate is the release; NCBI's Last-Modified comes a day or two later. CLINVAR_SLACK_DAYS = 2 -CLASSES = ("rerun-ready", "refresh-available", "in-pipeline-release", "upstream-only", "no-newer-release", "error") +CLASSES = ("rerun-ready", "refresh-available", "in-pipeline-release", "upstream-only", "no-newer-release", "unknown", + "error") LOG_HEADER = "utc\tevent\titem\tclass\tdetail\n" +# Held exclusive by --refresh-clinvar, shared by scripts/run-all.sh and scripts/06-clinvar-screen.sh. +CLINVAR_LOCK = "clinvar/.refresh.lock" CLINVAR_FILES = ("clinvar/clinvar_pathogenic_chr.vcf.gz", "clinvar/clinvar_chr.vcf.gz", "clinvar/clinvar.vcf.gz") # The pinned items: (manifest key, label, freshness lookup, how to read its upstream). PINNED = ( @@ -290,7 +297,10 @@ def pgs(self, pgs_id): except self.fr.LOOKUP_ERRORS as e: raise Failed(str(e)) import email.utils - return email.utils.parsedate_to_datetime(lm).strftime("%Y%m%d") + try: # a malformed date is a failed lookup of this score, never a crash of the whole check + return email.utils.parsedate_to_datetime(lm).strftime("%Y%m%d") + except (TypeError, ValueError, AttributeError, OverflowError): + raise Failed(f"cannot read the Last-Modified of {url}: {lm!r}") def release(self): try: @@ -334,7 +344,10 @@ def pgs(self, pgs_id): def release(self): e = self._get("release") - return e["tag"], dict(e.get("pins") or {}) + tag, pins = e.get("tag"), e.get("pins") or {} + if not isinstance(tag, str) or not tag or not isinstance(pins, dict): + raise Failed("the offline answers' release needs a tag and a pins object") + return tag, dict(pins) # --- the check ----------------------------------------------------------------------------- @@ -362,6 +375,10 @@ def classify(item, release_tag, release_pins, release_error, clinvar=False): ran, inst, up = item.get("ran_with"), item.get("installed"), item.get("upstream") if inst and ran and ver(inst) > ver(ran): return "rerun-ready" + # An image re-pinned at the same tag (a new digest): the tags compare equal, the references do not. + ran_ref, inst_ref = item.get("ran_with_ref"), item.get("installed_ref") + if inst and ran and ver(inst) == ver(ran) and ran_ref and inst_ref and ran_ref != inst_ref: + return "rerun-ready" # ClinVar's date is in the manifest whenever step 06 had a file, so a run without one cannot be compared # with the file here. A pin with no run value (HLA data not installed then) compares this checkout's. if clinvar and not ran: @@ -391,6 +408,14 @@ def classify(item, release_tag, release_pins, release_error, clinvar=False): return "upstream-only" +def skipped_steps(sample, sample_dir): + """The steps run-all.sh recorded as skipped in this run (logs/run_status.tsv).""" + try: + return {s["step"] for s in run_state.collect(sample, sample_dir)["steps"] if s["state"] == "skipped"} + except (OSError, ValueError, KeyError): + return set() + + def action(item, sample): c, key = item["class"], item["item"] if c == "rerun-ready": @@ -415,6 +440,9 @@ def action(item, sample): return "no pipeline release pins it yet; nothing to do until one does" if c == "error": return "the lookup failed; check again later" + if c == "unknown": + return ("the run's manifest does not name this pin (a run of an older version): nothing to compare; " + "the next run records it") return "" @@ -446,18 +474,31 @@ def add(item, fn, *args, clinvar=False): ran = pin_value(key, man.get(("data", key)) if key == "hla_database" else version.get(key)) inst = this_hla if key == "hla_database" else pin_value(key, this_pins.get(key)) item = {"item": key, "label": label, "kind": "pin", "ran_with": ran, "installed": inst} + if key.endswith("_IMAGE"): + item["ran_with_ref"], item["installed_ref"] = version.get(key) or None, this_pins.get(key) or None if key == "hla_database" and not this_hla: item["error"] = "could not read this checkout's HLA_DB_RELEASE (scripts/lib/common.sh)" add(item, answers.pinned, key, lookup, pattern) # The other versions.env pins a step reads (steps 50 to 53): no freshness source exists for them, so - # nothing is looked up; a pin here that differs from the run's makes its step rerun-ready. + # nothing is looked up; a pin here that differs from the run's (an image by its full reference, so a + # new digest at the same tag counts) makes its step rerun-ready. A pin the run's manifest lacks is + # unknown, never no-newer-release. A pin read only by steps run-all.sh recorded as skipped (an opt-in + # step the run left out) is left out: a rerun would not run that step either. known = {k for k, *_ in PINNED} + skipped = skipped_steps(sample, sample_dir) for key in dict.fromkeys(r[:-5] if r.endswith("@self") else r for s in steps for r in reads_of(s)): if key in known or key not in this_pins: continue - ran, inst = pin_value(key, version.get(key)), pin_value(key, this_pins.get(key)) + readers = [s["step"] for s in steps if any(key_matches(key, r) for r in reads_of(s))] + if readers and all(n in skipped for n in readers): + continue + ran_ref, inst_ref = version.get(key) or None, this_pins.get(key) or None + ran, inst = pin_value(key, ran_ref), pin_value(key, inst_ref) + cls = "unknown" if not (ran_ref and inst_ref) else "rerun-ready" if ran_ref != inst_ref else "no-newer-release" item = {"item": key, "label": f"{key} (no upstream lookup)", "kind": "pin", "ran_with": ran, - "installed": inst, "class": "rerun-ready" if ran and inst and ran != inst else "no-newer-release"} + "installed": inst, "class": cls} + if key.endswith("_IMAGE"): + item["ran_with_ref"], item["installed_ref"] = ran_ref, inst_ref item["steps"] = affected_steps(steps, [key]) item["action"] = action(item, sample) items.append(item) @@ -520,6 +561,9 @@ def print_check(state, out=None): w(f"Database re-check of {state['sample']} ({state['checked_utc']}):\n") for i in state["items"]: detail = i.get("error") or f"run used {i.get('ran_with') or 'unknown'}, here {i.get('installed') or 'unknown'}, upstream {i.get('upstream') or 'unknown'}" + if not i.get("error") and i.get("ran_with_ref") and i.get("installed_ref") and i["ran_with_ref"] != i["installed_ref"] \ + and i.get("ran_with") == i.get("installed"): + detail += f"; same tag, new image: run used {i['ran_with_ref']}, here {i['installed_ref']}" w(f" {i['label']}: {i['class']} ({detail})\n") if i["class"] not in ("no-newer-release",) and i.get("action"): w(f" {i['action']}\n") @@ -624,6 +668,31 @@ def running_samples(genome_dir): and run_state.collect(n, os.path.join(genome_dir, n))["run"]["state"] == "running"] +def refresh_clinvar(genome_dir): + """setup.sh --refresh clinvar under the ClinVar lock of GENOME_DIR (CLINVAR_LOCK). The ClinVar files + are shared by every sample, so a run of any of them blocks the refresh: run-all.sh and step 06 hold + the lock shared while they read the files, and wait while this holds it. A run with no lock (a bare + nextflow run, an older checkout) is caught by its logs/run_status.tsv. The lock is let go before + --rerun starts run-all.sh, which takes it shared.""" + os.makedirs(os.path.join(genome_dir, "clinvar"), exist_ok=True) + with open(os.path.join(genome_dir, CLINVAR_LOCK), "a") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + print(f"ERROR: a run or step 06 holds {os.path.join(genome_dir, CLINVAR_LOCK)}: not refreshing the " + "ClinVar files it reads.", file=sys.stderr) + return 3 + busy = running_samples(genome_dir) + if busy: + print(f"ERROR: a run of {', '.join(busy)} is going on: not refreshing the ClinVar files it reads.", + file=sys.stderr) + return 3 + r = subprocess.run([os.path.join(ROOT, "scripts", "setup.sh"), "--refresh", "clinvar", genome_dir]).returncode + if r != 0: + print("ERROR: setup.sh --refresh clinvar failed: see above.", file=sys.stderr) + return r + + def main(argv=None): ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) ap.add_argument("--sample") @@ -659,16 +728,8 @@ def main(argv=None): print(f"ERROR: no {sample_dir}/run_manifest.tsv: run the pipeline on {a.sample} first.", file=sys.stderr) return 2 if a.refresh_clinvar: - # The ClinVar files are shared by every sample of GENOME_DIR, so a run of any of them blocks the - # refresh. This narrows the window; a run that starts during the refresh is not stopped. - busy = running_samples(a.genome_dir) - if busy: - print(f"ERROR: a run of {', '.join(busy)} is going on: not refreshing the ClinVar files it reads.", - file=sys.stderr) - return 3 - r = subprocess.run([os.path.join(ROOT, "scripts", "setup.sh"), "--refresh", "clinvar", a.genome_dir]).returncode + r = refresh_clinvar(a.genome_dir) if r != 0: - print("ERROR: setup.sh --refresh clinvar failed: see above.", file=sys.stderr) return r fr = load_freshness() this_pins, _ = fr.read_pins(os.path.join(ROOT, "versions.env")) diff --git a/docs/recheck.md b/docs/recheck.md index e3ece402c..4eae712a0 100644 --- a/docs/recheck.md +++ b/docs/recheck.md @@ -23,6 +23,8 @@ Standard library only, so the plain `python3` on the host is enough. It needs th | IPD-IMGT/HLA | `hla_database` | The `release_version.txt` of the IPD-IMGT/HLA repository | 08 | | Each PGS score | `pgs:`, its `HmPOS_date` | The PGS Catalog's REST API, then the `Last-Modified` date of the harmonised GRCh38 file | 25, then 26 | +The pins of steps 50 to 53 (`MEGANE_IMAGE`, `MOCHA_PANEL_VERSION`, `MOCHA_IMAGE`, `VNTYPER_IMAGE`, `LOCITYPER_PANEL_VERSION`, `LOCITYPER_IMAGE`) have no upstream lookup: each is compared only with this checkout's `versions.env`. An image is compared by its full reference, so a new `@sha256` digest at the same tag counts as a change, for these and for `PHARMCAT_IMAGE`. A pin read only by a step that `run-all.sh` recorded as skipped in this run (an opt-in step it left out) is not listed: a rerun would not run that step either. + The HTML report (24) and the text report are written again after any of them. The ClinVar release inside the PCGR bundle (`cpsr_clinvar_release`) moves with the PCGR bundle, never with step 06's ClinVar file. The lookups are the ones the monthly CI check uses (`scripts/ci/freshness.py`). It also reads the newest release of this pipeline (`--owngenome-repo`, default `GeiserX/OwnGenome`) and that release's `versions.env`, to say whether a newer pin is already in a release. @@ -37,11 +39,12 @@ Each item gets one class: | Class | Meaning | What to do | |---|---|---| -| `rerun-ready` | What is installed here is newer than what the run used: a newer ClinVar file or PGS file on disk, or a pin in this checkout | Rerun (for a pin, run `./scripts/setup.sh` first so its data is there) | -| `refresh-available` | ClinVar only: NCBI's file is more than 2 days newer than the one installed | `./scripts/setup.sh --refresh clinvar $GENOME_DIR`, then rerun; `--rerun --refresh-clinvar` does both, and refuses while any sample of `$GENOME_DIR` is running, since they share the ClinVar files | +| `rerun-ready` | What is installed here is newer than what the run used: a newer ClinVar file or PGS file on disk, or a pin in this checkout (an image re-pinned to a new digest at the same tag too) | Rerun (for a pin, run `./scripts/setup.sh` first so its data is there) | +| `refresh-available` | ClinVar only: NCBI's file is more than 2 days newer than the one installed | `./scripts/setup.sh --refresh clinvar $GENOME_DIR`, then rerun; `--rerun --refresh-clinvar` does both, and refuses while any sample of `$GENOME_DIR` is running, since they share the ClinVar files ([the ClinVar lock](#the-clinvar-lock)) | | `in-pipeline-release` | A pipeline release pins a newer version | Update the pipeline to that release, run `setup.sh`, then rerun | | `upstream-only` | Upstream has a newer release that no pipeline release pins | Nothing yet for a pinned item; for a PGS score, download it again ([step 25](25-prs.md)) | | `no-newer-release` | The lookup answered and nothing is newer | Nothing | +| `unknown` | A pin of steps 50 to 53 that the run's manifest does not name (a run of an older version): nothing was compared | Nothing; the next run records it. It does not make the command exit 1 | | `error` | The lookup failed | Check again later. A failed lookup is never read as "up to date", and the command exits 1 | The 2-day margin for ClinVar is there because NCBI dates the file a day or two after the release in its `##fileDate`. A newer PGS file date means the catalog wrote the harmonised file again; it does not prove the weights changed. @@ -67,6 +70,10 @@ When the run ends, it compares the steps that ran a task again with the steps th A sample last run by an older `run-all.sh` has no `meta command` row: run the same command once by hand. +### The ClinVar lock + +The ClinVar files are shared by every sample of `$GENOME_DIR`. `--refresh-clinvar` holds `$GENOME_DIR/clinvar/.refresh.lock` exclusive while `setup.sh --refresh clinvar` replaces them, and lets go before the rerun starts. `run-all.sh` holds the same lock shared from its start to its end, and `scripts/06-clinvar-screen.sh` while it runs: a run that starts during a refresh waits for it ("Waiting for the ClinVar refresh ..."), and a refresh refuses (exit 3) while a run holds the lock. It also refuses while any sample's `logs/run_status.tsv` says a run is going on, which covers a run of an older checkout. The shell side needs `flock` (part of util-linux on Linux; on macOS `brew install flock`); without it nothing waits. A bare `nextflow run`, and `setup.sh --refresh clinvar` run by hand, take no lock. + ### A bare Nextflow run A bare `nextflow run main.nf` writes no run manifest, so there is nothing to check. To get the command back, `--launch-dir` prints the last command of that launch directory (its `.nextflow/history`) with `-resume` added. It never runs it. @@ -77,9 +84,10 @@ python3 bin/owngenome_recheck.py --launch-dir /path/to/launch/dir ## The files it writes -- `/recheck/state.json`, replaced whole on each check: `checked_utc`, the hosts asked, the pipeline release it read, one entry per item (`item`, `label`, `ran_with`, `installed`, `upstream`, `class`, `steps`, `action`, and `error` or `in_owngenome_release` when they apply), `affected_steps` (the steps a rerun would redo for the `rerun-ready` and `refresh-available` items), and after a rerun a `rerun` block (`started_utc`, `finished_utc`, `rc`, `command`, `predicted_vs_redone`). The last rerun stays until another one replaces it. +- `/recheck/state.json`, replaced whole on each check: `checked_utc`, the hosts asked, the pipeline release it read, one entry per item (`item`, `label`, `ran_with`, `installed`, `upstream`, `class`, `steps`, `action`, and `error` or `in_owngenome_release` when they apply; an image item also has `ran_with_ref` and `installed_ref`, the full references with any digest), `affected_steps` (the steps a rerun would redo for the `rerun-ready` and `refresh-available` items), and after a rerun a `rerun` block (`started_utc`, `finished_utc`, `rc`, `command`, `predicted_vs_redone`). The last rerun stays until another one replaces it. - `/recheck/log.tsv`, only ever appended: one row per item and check, and a row as each rerun starts and ends. - `/recheck/rerun.lock`, an empty file each `--rerun` locks while it runs. +- `$GENOME_DIR/clinvar/.refresh.lock`, an empty file: [the ClinVar lock](#the-clinvar-lock). `run-all.sh` and step 06 make it too when it is missing. ## In the reports and on the live site diff --git a/scripts/06-clinvar-screen.sh b/scripts/06-clinvar-screen.sh index 7023c8610..4d938d897 100755 --- a/scripts/06-clinvar-screen.sh +++ b/scripts/06-clinvar-screen.sh @@ -61,6 +61,18 @@ done echo "=== ClinVar Pathogenic Screen: ${SAMPLE} ===" +# bin/owngenome_recheck.py --refresh-clinvar holds this lock while it replaces the ClinVar files every sample +# shares; the screen holds it shared on fd 7 until it ends, so it waits for a refresh going on and a refresh +# refuses while it reads. Without flock (macOS: brew install flock) nothing waits here. +CVLOCK="${GENOME_DIR}/clinvar/.refresh.lock" +if [ -d "${GENOME_DIR}/clinvar" ] && command -v flock >/dev/null 2>&1; then + [ -e "$CVLOCK" ] || : 2>/dev/null >> "$CVLOCK" || true + if [ -r "$CVLOCK" ]; then + exec 7<"$CVLOCK" + flock -s -n 7 || { echo "Waiting for the ClinVar refresh that holds ${CVLOCK} to end ..."; flock -s 7; } + fi +fi + for f in "$VCF" "${VCF}.tbi" "$CLINVAR" "${CLINVAR}.tbi" "$REF" "${REF}.fai" "$SITES"; do if [ ! -f "$f" ]; then echo "ERROR: File not found: ${f}" >&2 diff --git a/scripts/run-all.sh b/scripts/run-all.sh index ab6c3cdf8..c5d15d59b 100755 --- a/scripts/run-all.sh +++ b/scripts/run-all.sh @@ -62,6 +62,17 @@ export GENOME_DIR=${GENOME_DIR:?Set GENOME_DIR to your data directory} validate_sample "$SAMPLE" G=$GENOME_DIR S="${GENOME_DIR}/${SAMPLE}" LOG_DIR="${GENOME_DIR}/${SAMPLE}/logs" T=${TOOLS:-} C=${EXTRA_CALLERS:-} on() { [[ "${!1:-}" =~ ^(true|1)$ ]]; } +# The ClinVar files are shared by every sample of GENOME_DIR. bin/owngenome_recheck.py --refresh-clinvar holds this +# lock while it replaces them; the run holds it shared on fd 7 until it ends, so a run waits for a refresh going on +# and a refresh refuses while a run goes on. Without flock (macOS: brew install flock) nothing waits here. +CVLOCK="${G}/clinvar/.refresh.lock" +if [ -d "${G}/clinvar" ] && command -v flock >/dev/null 2>&1; then + [ -e "$CVLOCK" ] || : 2>/dev/null >> "$CVLOCK" || true + if [ -r "$CVLOCK" ]; then + exec 7<"$CVLOCK" + flock -s -n 7 || { echo "Waiting for the ClinVar refresh that holds ${CVLOCK} to end ..."; flock -s 7; } + fi +fi java_major=$(java -version 2>&1 | awk -F'"' '/version "/ {split($2, v, "."); print (v[1] == 1 ? v[2] : v[1]); exit}') || true if ! command -v nextflow >/dev/null 2>&1 || [[ ! "${java_major:-}" =~ ^[0-9]+$ ]] || [ "$java_major" -lt 17 ]; then printf '%s\n' "ERROR: run-all.sh starts the Nextflow pipeline and needs Java 17 or later (found: ${java_major:-none}) and Nextflow ($(command -v nextflow || echo 'not on PATH'))." \ diff --git a/tests/test_owngenome_recheck.py b/tests/test_owngenome_recheck.py index bb5b8a501..ea901237c 100644 --- a/tests/test_owngenome_recheck.py +++ b/tests/test_owngenome_recheck.py @@ -27,6 +27,18 @@ 12. check() itself names step 53 rerun-ready when the run used an older LOCITYPER_PANEL_VERSION than this checkout pins, and not when they match; its action names setup.sh --locityper-data, which the main setup leaves out. + 13. an image re-pinned at the same tag with a new digest is rerun-ready, for a pin + with no upstream lookup (step 50's) and for a looked-up one (PharmCAT); the + same reference on both sides is no-newer-release; + 14. a pin with no upstream lookup that the run's manifest lacks is unknown, never + no-newer-release, names no step and does not make the command exit 1; + 15. a pin read only by a step run-all.sh recorded as skipped is left out; + 16. a malformed Last-Modified of a PGS file, and an offline release with no tag, + are errors of that lookup, never a crash; + 17. --refresh-clinvar refuses (exit 3) while a run holds the ClinVar lock shared, + and holds it exclusive while setup.sh refreshes; + 18. the ClinVar lock block of run-all.sh and of step 06 waits while a refresh holds + the lock and goes on once it ends (needs flock; it fails in CI without it). Each check is then run against a planted mistake and must fail there. Run: python3 tests/test_owngenome_recheck.py @@ -272,6 +284,8 @@ def all_checks(m): cmp["redone"] == ["06", "07"] and cmp["predicted_not_redone"] == ["13"] and cmp["redone_not_predicted"] == ["07"] and cmp["not_redone_script_steps"] == ["14"], cmp) + new_checks(m, tmp, steps, out) + ld = os.path.join(tmp, "launch") os.makedirs(os.path.join(ld, ".nextflow")) with open(os.path.join(ld, ".nextflow", "history"), "w") as f: @@ -290,6 +304,204 @@ def all_checks(m): return out +def manifest_add(g, rows): + with open(os.path.join(g, "S1", "run_manifest.tsv"), "a") as f: + f.writelines(f"version\t{k}\t{v}\n" for k, v in rows) + + +def skip_step(g, step): + with open(os.path.join(g, "S1", "logs", "run_status.tsv"), "a") as f: + f.write(f"step\t{step}\tskipped (opt-in: add mei to TOOLS)\n") + + +def new_checks(m, tmp, steps, out): + # Image references built from the fixture's own, so no image name is a literal here. + ref = THIS_PINS["PHARMCAT_IMAGE"] + d1, d2 = ref + "@sha256:" + "a" * 64, ref + "@sha256:" + "b" * 64 + g = genome(os.path.join(tmp, "digest")) + manifest_add(g, [("MEGANE_IMAGE", d1)]) + new = by_item(m.check("S1", g, answers(m, g), steps, dict(THIS_PINS, MEGANE_IMAGE=d2), "3.66.0")) + same = by_item(m.check("S1", g, answers(m, g), steps, dict(THIS_PINS, MEGANE_IMAGE=d1), "3.66.0")) + mg, ms = new.get("MEGANE_IMAGE", {}), same.get("MEGANE_IMAGE", {}) + out["a new digest at the same tag of a pin with no lookup is rerun-ready"] = ( + mg.get("class") == "rerun-ready" and "50" in mg.get("steps", []) and mg.get("installed_ref") == d2 + and ms.get("class") == "no-newer-release", (mg, ms)) + + gp = genome(os.path.join(tmp, "digest-pharmcat")) + rows = [x for x in open(os.path.join(gp, "S1", "run_manifest.tsv")) if not x.startswith("version\tPHARMCAT_IMAGE\t")] + open(os.path.join(gp, "S1", "run_manifest.tsv"), "w").writelines(rows + [f"version\tPHARMCAT_IMAGE\t{d1}\n"]) + ph = by_item(m.check("S1", gp, answers(m, gp), steps, dict(THIS_PINS, PHARMCAT_IMAGE=d2), "3.66.0"))["PHARMCAT_IMAGE"] + ph1 = by_item(m.check("S1", gp, answers(m, gp), steps, dict(THIS_PINS, PHARMCAT_IMAGE=d1), "3.66.0"))["PHARMCAT_IMAGE"] + text = io.StringIO() + m.print_check({"sample": "S1", "checked_utc": "x", "items": [ph], "affected_steps": []}, text) + out["a new digest at the same tag of a looked-up image is rerun-ready"] = ( + ph["class"] == "rerun-ready" and ph1["class"] == "no-newer-release" and "same tag, new image" in text.getvalue(), + (ph, ph1, text.getvalue())) + + gu = genome(os.path.join(tmp, "unknown")) + st = m.check("S1", gu, answers(m, gu), steps, dict(THIS_PINS, MEGANE_IMAGE=d1), "3.66.0") + mu = by_item(st).get("MEGANE_IMAGE", {}) + with contextlib.redirect_stdout(io.StringIO()): + ans = write(tmp, "unknown.json", json.load(open(os.path.join(FIX, "offline.json")))) + keep = m.check + m.check = lambda s_, g_, a_, st_, pins, hla: keep(s_, g_, a_, st_, dict(pins, MEGANE_IMAGE=d1), hla) + try: + rc = m.main(["--sample", "S1", "--genome-dir", gu, "--offline", ans]) + finally: + m.check = keep + out["a pin with no lookup the run's manifest lacks is unknown, exit 0"] = ( + mu.get("class") == "unknown" and "50" not in st["affected_steps"] and "50" not in st["rerun_ready_steps"] + and "older version" in mu.get("action", "") and rc == 0, (mu, st["affected_steps"], rc)) + + gs = genome(os.path.join(tmp, "skipped")) + manifest_add(gs, [("MEGANE_IMAGE", d1)]) + skip_step(gs, "50") + sk = m.check("S1", gs, answers(m, gs), steps, dict(THIS_PINS, MEGANE_IMAGE=d2), "3.66.0") + out["a pin read only by a step the run skipped is left out"] = ( + "MEGANE_IMAGE" not in by_item(sk) and "50" not in sk["rerun_ready_steps"], by_item(sk).get("MEGANE_IMAGE")) + + srv, port = serve_pgs("not a date") + try: + fr = m.load_freshness() + fr.RETRIES = 0 + on = m.Online(fr, {}, "o/r", pgs_api=f"http://127.0.0.1:{port}/rest/score", raw=f"http://127.0.0.1:{port}", + out=io.StringIO()) + try: + gm = genome(os.path.join(tmp, "lastmod")) + pg = by_item(m.check("S1", gm, Mixed(on, answers(m, gm)), steps, THIS_PINS, "3.66.0"))["pgs:PGS000001"] + got = (pg["class"] == "error" and "Last-Modified" in pg.get("error", ""), pg) + except Exception as e: # noqa: BLE001 the crash this check is about + got = (False, repr(e)) + finally: + srv.shutdown() + out["a malformed Last-Modified of a PGS file is error, not a crash"] = got + + try: + go = genome(os.path.join(tmp, "notag")) + so = m.check("S1", go, answers(m, go, release={"pins": {}}), steps, THIS_PINS, "3.66.0") + got = ("error" in so["pipeline_release"] and "tag" in so["pipeline_release"]["error"], so["pipeline_release"]) + except Exception as e: # noqa: BLE001 + got = (False, repr(e)) + out["an offline release with no tag is error, not a crash"] = got + + gl = genome(os.path.join(tmp, "cvlock")) + lock = os.path.join(gl, "clinvar", ".refresh.lock") + keep_sp, ran = m.subprocess, [] + m.subprocess = type("SP", (), {"run": staticmethod(lambda *a, **k: ran.append(a) or type("R", (), {"returncode": 0})())}) + e = io.StringIO() + try: + with open(lock, "a") as held: + fcntl.flock(held, fcntl.LOCK_SH | fcntl.LOCK_NB) + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(e): + rc = m.main(["--sample", "S1", "--genome-dir", gl, "--offline", os.path.join(FIX, "offline.json"), + "--rerun", "--refresh-clinvar"]) + finally: + m.subprocess = keep_sp + out["--refresh-clinvar refuses while a run holds the ClinVar lock"] = ( + rc == 3 and ".refresh.lock" in e.getvalue() and not ran, (rc, e.getvalue(), ran)) + + seen = [] + + def setup_run(argv, *a, **k): + if "--refresh" in argv: + with open(lock, "a") as f: + try: + fcntl.flock(f, fcntl.LOCK_SH | fcntl.LOCK_NB) + seen.append("free") + except OSError: + seen.append("held") + return type("R", (), {"returncode": 0})() + m.subprocess = type("SP", (), {"run": staticmethod(setup_run)}) + try: + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + rc = m.main(["--sample", "S1", "--genome-dir", gl, "--offline", os.path.join(FIX, "offline.json"), + "--rerun", "--refresh-clinvar"]) + with open(lock, "a") as f: # let go before the replay's run-all.sh takes it shared + fcntl.flock(f, fcntl.LOCK_SH | fcntl.LOCK_NB) + after = True + except OSError: + after = False + finally: + m.subprocess = keep_sp + out["--refresh-clinvar holds the ClinVar lock while setup.sh refreshes"] = ( + seen == ["held"] and after and rc == 0, (seen, after, rc)) + + +def lock_block(path): + """The ClinVar lock block of a script: from 'CVLOCK=' through its closing 'fi'.""" + x = re.search(r"^CVLOCK=.*?\n fi\nfi\n", open(path).read(), re.S | re.M) + return x.group(0) if x else "" + + +def lock_waits(block): + """block, run in bash while this process holds the lock exclusive: it must wait, then go on + once the lock is let go.""" + import subprocess + import time + tmp = tempfile.mkdtemp(prefix="recheck-cvlock.") + try: + g = os.path.join(tmp, "genome") + os.makedirs(os.path.join(g, "clinvar")) + lock = os.path.join(g, "clinvar", ".refresh.lock") + with open(lock, "a") as held: + fcntl.flock(held, fcntl.LOCK_EX | fcntl.LOCK_NB) + p = subprocess.Popen(["bash", "-c", "set -euo pipefail\n" + block + "echo through\n"], + env=dict(os.environ, G=g, GENOME_DIR=g), stdout=subprocess.PIPE, text=True) + time.sleep(1.5) + waited = p.poll() is None + try: + o, _ = p.communicate(timeout=20) + except subprocess.TimeoutExpired: + p.kill() + o, _ = p.communicate() + return (bool(block) and waited and p.returncode == 0 and "Waiting" in o and o.rstrip().endswith("through"), + (waited, p.returncode, o)) + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +def serve_pgs(last_modified): + """A PGS Catalog score answer, and a HEAD of its file with this Last-Modified.""" + class H(http.server.BaseHTTPRequestHandler): + def do_GET(self): + port = self.server.server_address[1] + body = json.dumps({"id": "PGS000001", "ftp_harmonized_scoring_files": {"GRCh38": { + "positions": f"http://127.0.0.1:{port}/f/PGS000001_hmPOS_GRCh38.txt.gz"}}}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_HEAD(self): + self.send_response(200) + self.send_header("Last-Modified", last_modified) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *a): + pass + srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), H) + threading.Thread(target=srv.serve_forever, daemon=True).start() + return srv, srv.server_address[1] + + +def from_source(old, new): + """bin/owngenome_recheck.py with old replaced by new (which must be in it), as a module.""" + import types + src = open(TOOL).read() + if src.count(old) != 1: + raise AssertionError(f"planted mistake not in the copy: {old[:60]!r}") + mod = types.ModuleType("owngenome_recheck_planted") + mod.__file__ = TOOL + keep, sys.dont_write_bytecode = sys.dont_write_bytecode, True + try: + exec(compile(src.replace(old, new), TOOL, "exec"), mod.__dict__) + finally: + sys.dont_write_bytecode = keep + return mod + + class Mixed: """The online release and PGS lookups, the offline answers for the rest.""" @@ -428,6 +640,30 @@ def regex_resume(d): return cmd if re.search(r"(^|\s)-resume(\s|$)", cmd) else cmd + " -resume" m.launch_dir_command = regex_resume ms["--launch-dir: an argument holding -resume is not the option"] = m + ms["a new digest at the same tag of a pin with no lookup is rerun-ready"] = from_source( + '"rerun-ready" if ran_ref != inst_ref else', '"rerun-ready" if ran != inst else') + ms["a new digest at the same tag of a looked-up image is rerun-ready"] = from_source( + "and ran_ref != inst_ref:\n return \"rerun-ready\"", "and False:\n return \"rerun-ready\"") + ms["a pin with no lookup the run's manifest lacks is unknown, exit 0"] = from_source( + 'cls = "unknown" if not', 'cls = "no-newer-release" if not') + m = load() + m.skipped_steps = lambda *a: set() + ms["a pin read only by a step the run skipped is left out"] = m + ms["a malformed Last-Modified of a PGS file is error, not a crash"] = from_source( + " try: # a malformed date is a failed lookup of this score, never a crash of the whole check\n" + " return email.utils.parsedate_to_datetime(lm).strftime(\"%Y%m%d\")\n" + " except (TypeError, ValueError, AttributeError, OverflowError):\n", + " return email.utils.parsedate_to_datetime(lm).strftime(\"%Y%m%d\")\n if False:\n") + ms["an offline release with no tag is error, not a crash"] = from_source( + 'tag, pins = e.get("tag"), e.get("pins") or {}\n if not isinstance(tag, str) or not tag or not isinstance(pins, dict):', + 'tag, pins = e["tag"], e.get("pins") or {}\n if not isinstance(pins, dict):') + m = load() + m.fcntl = type("F", (), {"LOCK_EX": 0, "LOCK_NB": 0, "flock": staticmethod(lambda *a: None)}) + ms["--refresh-clinvar refuses while a run holds the ClinVar lock"] = m + ms["--refresh-clinvar holds the ClinVar lock while setup.sh refreshes"] = from_source( + " fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)\n except OSError:\n" + " print(f\"ERROR: a run or step 06", + " pass\n except OSError:\n print(f\"ERROR: a run or step 06") return ms @@ -595,6 +831,19 @@ def main(): unfiltered = plugin_from_source(src.replace(stale, "if False:"), "owngenome_live_recheck_unfiltered") check("fails on a planted mistake: the live-site plugin leaves out a check older than the run's manifest", not plugin_stale_ok(unfiltered)[0], "passed on the mutant") + import shutil as _sh + for script in ("scripts/run-all.sh", "scripts/06-clinvar-screen.sh"): + block = lock_block(os.path.join(REPO, script)) + if not _sh.which("flock"): + check(f"the ClinVar lock block of {script} waits for a refresh (flock is not installed: " + f"{'FAIL in CI' if os.environ.get('CI') else 'not run here'})", not os.environ.get("CI")) + continue + ok, detail = lock_waits(block) + check(f"the ClinVar lock block of {script} waits for a refresh, then goes on", ok, detail) + planted = re.sub(r"(?m)^ flock -s -n 7 .*$", " :", block) + check(f"planted mistake is in the copy: the ClinVar lock block of {script}", planted != block) + check(f"fails on a planted mistake: the ClinVar lock block of {script} waits for a refresh (no flock call)", + not lock_waits(planted)[0], "passed on the mutant") results = all_checks(load()) for name, (ok, detail) in results.items(): check(name, ok, detail) From ae61a3455e31456cfa697c31e6953594ed930bf0 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 6 Nov 2026 22:23:31 +0100 Subject: [PATCH 2/3] build(renovate): track the nf-weblog plugin pin of the live site NF_WEBLOG_PLUGIN in scripts/run-all.sh had no Renovate manager, so no bot saw a new nf-weblog release. The rule also moves the copy in docs/live-site.md. Bead: pgp-5bja.23 --- renovate.json | 8 ++++++++ scripts/run-all.sh | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/renovate.json b/renovate.json index 4cf6b687e..768584993 100644 --- a/renovate.json +++ b/renovate.json @@ -63,6 +63,14 @@ "extractVersionTemplate": "^v(?.+)$", "versioningTemplate": "regex:^(?\\d+)\\.(?\\d+)\\.(?\\d+)$" }, + { + "customType": "regex", + "description": "nf-weblog, the Nextflow plugin of the live site: NF_WEBLOG_PLUGIN in scripts/run-all.sh and the plugins block docs/live-site.md shows, in one PR. Its releases are tagged without a v.", + "managerFilePatterns": ["/^scripts/run-all\\.sh$/", "/^docs/live-site\\.md$/"], + "matchStrings": ["NF_WEBLOG_PLUGIN=nf-weblog@(?\\d+\\.\\d+\\.\\d+)", "id 'nf-weblog@(?\\d+\\.\\d+\\.\\d+)'"], + "depNameTemplate": "nextflow-io/nf-weblog", + "datasourceTemplate": "github-releases" + }, { "customType": "regex", "description": "actionlint, downloaded by version in the lint workflow.", diff --git a/scripts/run-all.sh b/scripts/run-all.sh index c5d15d59b..9988346d1 100755 --- a/scripts/run-all.sh +++ b/scripts/run-all.sh @@ -306,7 +306,7 @@ else echo " benchmark-variants skipped (only one caller VCF: set EXTRA_CALLERS, export NXF_VER=${NXF_VER:-$NEXTFLOW_VERSION} # The live site (docs/live-site.md). Nextflow's weblog is the nf-weblog plugin, which Nextflow 26.10 loads only when # a config names it, so the block lives in a config of this run's own, passed only when OWNGENOME_LIVE_URL is set: a run -# without it neither downloads the plugin nor warns about an unknown weblog scope. Bump the pin with its release. +# without it neither downloads the plugin nor warns about an unknown weblog scope. Renovate bumps the pin (renovate.json). NF_WEBLOG_PLUGIN=nf-weblog@1.2.0 if [ -n "${OWNGENOME_LIVE_URL:-}" ]; then # The ingest listener speaks plain HTTP and serves /ingest/ at its root, so https or a path would lose every post. From acf47c99de0f426f770b65fd4389dc38722aeb6d Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 6 Nov 2026 23:03:56 +0100 Subject: [PATCH 3/3] fix(recheck): leave out the pins of opt-in steps a default run never planned run-all.sh writes no status row for an opt-in step 50-53 left out of TOOLS, so the check saw it as 'not planned', not 'skipped', and still called its changed pin rerun-ready. Count both states, and leave nothing out when the status file records no step at all. --- bin/owngenome_recheck.py | 19 +++++++++++++------ docs/recheck.md | 2 +- tests/test_owngenome_recheck.py | 30 ++++++++++++++++++++++++------ 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/bin/owngenome_recheck.py b/bin/owngenome_recheck.py index 0430e60e3..5e940be7f 100755 --- a/bin/owngenome_recheck.py +++ b/bin/owngenome_recheck.py @@ -408,10 +408,17 @@ def classify(item, release_tag, release_pins, release_error, clinvar=False): return "upstream-only" -def skipped_steps(sample, sample_dir): - """The steps run-all.sh recorded as skipped in this run (logs/run_status.tsv).""" +def steps_not_run(sample, sample_dir): + """The steps this run did not run: the ones run-all.sh recorded as skipped (no BAM, missing + data), and the ones it wrote no row for, which is how it leaves out an opt-in step not in TOOLS + (run_state's "not planned"). Empty when logs/run_status.tsv records no step at all (a run of an + older version), so nothing is left out on a guess.""" try: - return {s["step"] for s in run_state.collect(sample, sample_dir)["steps"] if s["state"] == "skipped"} + status = run_state.read_run_status(os.path.join(sample_dir, "logs", "run_status.tsv")) + if not (status or {}).get("steps"): + return set() + return {s["step"] for s in run_state.collect(sample, sample_dir)["steps"] + if s["state"] in ("skipped", "not planned")} except (OSError, ValueError, KeyError): return set() @@ -482,10 +489,10 @@ def add(item, fn, *args, clinvar=False): # The other versions.env pins a step reads (steps 50 to 53): no freshness source exists for them, so # nothing is looked up; a pin here that differs from the run's (an image by its full reference, so a # new digest at the same tag counts) makes its step rerun-ready. A pin the run's manifest lacks is - # unknown, never no-newer-release. A pin read only by steps run-all.sh recorded as skipped (an opt-in - # step the run left out) is left out: a rerun would not run that step either. + # unknown, never no-newer-release. A pin read only by steps this run did not run (an opt-in step not + # in TOOLS, or one skipped for missing data) is left out: a rerun would not run that step either. known = {k for k, *_ in PINNED} - skipped = skipped_steps(sample, sample_dir) + skipped = steps_not_run(sample, sample_dir) for key in dict.fromkeys(r[:-5] if r.endswith("@self") else r for s in steps for r in reads_of(s)): if key in known or key not in this_pins: continue diff --git a/docs/recheck.md b/docs/recheck.md index 4eae712a0..221c6fc05 100644 --- a/docs/recheck.md +++ b/docs/recheck.md @@ -23,7 +23,7 @@ Standard library only, so the plain `python3` on the host is enough. It needs th | IPD-IMGT/HLA | `hla_database` | The `release_version.txt` of the IPD-IMGT/HLA repository | 08 | | Each PGS score | `pgs:`, its `HmPOS_date` | The PGS Catalog's REST API, then the `Last-Modified` date of the harmonised GRCh38 file | 25, then 26 | -The pins of steps 50 to 53 (`MEGANE_IMAGE`, `MOCHA_PANEL_VERSION`, `MOCHA_IMAGE`, `VNTYPER_IMAGE`, `LOCITYPER_PANEL_VERSION`, `LOCITYPER_IMAGE`) have no upstream lookup: each is compared only with this checkout's `versions.env`. An image is compared by its full reference, so a new `@sha256` digest at the same tag counts as a change, for these and for `PHARMCAT_IMAGE`. A pin read only by a step that `run-all.sh` recorded as skipped in this run (an opt-in step it left out) is not listed: a rerun would not run that step either. +The pins of steps 50 to 53 (`MEGANE_IMAGE`, `MOCHA_PANEL_VERSION`, `MOCHA_IMAGE`, `VNTYPER_IMAGE`, `LOCITYPER_PANEL_VERSION`, `LOCITYPER_IMAGE`) have no upstream lookup: each is compared only with this checkout's `versions.env`. An image is compared by its full reference, so a new `@sha256` digest at the same tag counts as a change, for these and for `PHARMCAT_IMAGE`. A pin read only by steps this run did not run (an opt-in step not in `TOOLS`, or one skipped for missing data) is not listed: a rerun would not run that step either. The HTML report (24) and the text report are written again after any of them. The ClinVar release inside the PCGR bundle (`cpsr_clinvar_release`) moves with the PCGR bundle, never with step 06's ClinVar file. diff --git a/tests/test_owngenome_recheck.py b/tests/test_owngenome_recheck.py index ea901237c..bb98fcb79 100644 --- a/tests/test_owngenome_recheck.py +++ b/tests/test_owngenome_recheck.py @@ -32,7 +32,9 @@ same reference on both sides is no-newer-release; 14. a pin with no upstream lookup that the run's manifest lacks is unknown, never no-newer-release, names no step and does not make the command exit 1; - 15. a pin read only by a step run-all.sh recorded as skipped is left out; + 15. a pin read only by a step the run did not run is left out: an opt-in step not in + TOOLS (run-all.sh writes no row for it) and one recorded as skipped (no BAM); + a status file with no step rows at all leaves every pin in; 16. a malformed Last-Modified of a PGS file, and an offline release with no tag, are errors of that lookup, never a crash; 17. --refresh-clinvar refuses (exit 3) while a run holds the ClinVar lock shared, @@ -309,9 +311,10 @@ def manifest_add(g, rows): f.writelines(f"version\t{k}\t{v}\n" for k, v in rows) -def skip_step(g, step): +def step_rows(g, rows): + """run_status.tsv step rows as run-all.sh writes them; a step it left out gets no row.""" with open(os.path.join(g, "S1", "logs", "run_status.tsv"), "a") as f: - f.write(f"step\t{step}\tskipped (opt-in: add mei to TOOLS)\n") + f.writelines(f"step\t{k}\t{v}\n" for k, v in rows) def new_checks(m, tmp, steps, out): @@ -353,12 +356,25 @@ def new_checks(m, tmp, steps, out): mu.get("class") == "unknown" and "50" not in st["affected_steps"] and "50" not in st["rerun_ready_steps"] and "older version" in mu.get("action", "") and rc == 0, (mu, st["affected_steps"], rc)) + # A default run (no TOOLS): run-all.sh writes rows for the steps it ran and none for step 50. + gd = genome(os.path.join(tmp, "not-in-tools")) + manifest_add(gd, [("MEGANE_IMAGE", d1)]) + step_rows(gd, [("06", "planned"), ("06", "ok")]) + nd = m.check("S1", gd, answers(m, gd), steps, dict(THIS_PINS, MEGANE_IMAGE=d2), "3.66.0") + out["a pin read only by an opt-in step not in TOOLS is left out"] = ( + "MEGANE_IMAGE" not in by_item(nd) and "50" not in nd["rerun_ready_steps"], by_item(nd).get("MEGANE_IMAGE")) + gs = genome(os.path.join(tmp, "skipped")) manifest_add(gs, [("MEGANE_IMAGE", d1)]) - skip_step(gs, "50") + step_rows(gs, [("06", "ok"), ("50", "skipped (no BAM)")]) sk = m.check("S1", gs, answers(m, gs), steps, dict(THIS_PINS, MEGANE_IMAGE=d2), "3.66.0") + gr = genome(os.path.join(tmp, "ran")) + manifest_add(gr, [("MEGANE_IMAGE", d1)]) + step_rows(gr, [("06", "ok"), ("50", "planned"), ("50", "ok")]) + rn = by_item(m.check("S1", gr, answers(m, gr), steps, dict(THIS_PINS, MEGANE_IMAGE=d2), "3.66.0")) out["a pin read only by a step the run skipped is left out"] = ( - "MEGANE_IMAGE" not in by_item(sk) and "50" not in sk["rerun_ready_steps"], by_item(sk).get("MEGANE_IMAGE")) + "MEGANE_IMAGE" not in by_item(sk) and "50" not in sk["rerun_ready_steps"] + and rn.get("MEGANE_IMAGE", {}).get("class") == "rerun-ready", (by_item(sk).get("MEGANE_IMAGE"), rn.get("MEGANE_IMAGE"))) srv, port = serve_pgs("not a date") try: @@ -646,8 +662,10 @@ def regex_resume(d): "and ran_ref != inst_ref:\n return \"rerun-ready\"", "and False:\n return \"rerun-ready\"") ms["a pin with no lookup the run's manifest lacks is unknown, exit 0"] = from_source( 'cls = "unknown" if not', 'cls = "no-newer-release" if not') + ms["a pin read only by an opt-in step not in TOOLS is left out"] = from_source( + 'if s["state"] in ("skipped", "not planned")}', 'if s["state"] in ("skipped",)}') m = load() - m.skipped_steps = lambda *a: set() + m.steps_not_run = lambda *a: set() ms["a pin read only by a step the run skipped is left out"] = m ms["a malformed Last-Modified of a PGS file is error, not a crash"] = from_source( " try: # a malformed date is a failed lookup of this score, never a crash of the whole check\n"