From 5a47a4db337482d50577aaac25bdf9f248a73a90 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 4 Nov 2026 15:09:37 +0100 Subject: [PATCH 1/3] feat(report): a Viewer column on every findings table The report linked only the ClinVar hits, the secondary findings and the mitochondrial variants to the genome viewer, as one line of links under each table. Every findings table now ends with a Viewer column, one link per row to that row's position: ClinVar hits, secondary findings, mito variants, deep intronic sites, the cDNA scan, AnnotSV class 4-5 and the calls moved to class 3, slivar's pairs in trans and mosaic alterations. A column of its own keeps every other cell's text as it was. Bead: pgp-awpe --- bin/render_report.py | 107 +++++++++++++++++++--------------- docs/24-html-report.md | 3 +- tests/test_collect_summary.py | 10 +++- 3 files changed, 70 insertions(+), 50 deletions(-) diff --git a/bin/render_report.py b/bin/render_report.py index 27072abf..3e0f9a9a 100755 --- a/bin/render_report.py +++ b/bin/render_report.py @@ -733,10 +733,10 @@ def comphet_phase_html(d): out = [stat("Pairs in trans / cis / unknown", f"{ph.get('trans', 0)} / {ph.get('cis', 0)} / {ph.get('unknown', 0)}")] trans = d.get("compound_het_trans") or [] if trans: - out += [" ", " "] - out += [f" " - for x in trans] - out.append("
GenePair in transReads
{E(x['gene'])}{E(x['variants'])}{E(x['reads'])}
") + out += [" ", f" {VIEWER_TH}"] + out += [f" " + f"{viewer_td_of(x['variants'])}" for x in trans] + out += ["
GenePair in transReads
{E(x['gene'])}{E(x['variants'])}{E(x['reads'])}
", VIEWER_NOTE] return out + [small_note(COMPHET_NOTE)] @@ -1938,9 +1938,10 @@ def mosaic_mca_body(d): if d.get("status") != "assessed": return [stat("Status", badge("Not assessed", "gray")), small_note(d.get("message", ".")), small_note(MCA_NOTE)] rows = "".join(f"{E(x['type'])}{E(x['arms'])}{E(x['chrom'])}:{E(x['start'])}-{E(x['end'])}" - f"{E(x['length_mb'])}{E(x['cell_fraction'])}" for x in d.get("calls", [])) - table = (" " - + rows + "
TypeArmRegionMbCell fraction
") if rows else small_note("No mosaic alteration reported.") + f"{E(x['length_mb'])}{E(x['cell_fraction'])}{viewer_td(x['chrom'], x['start'], x['end'])}" + for x in d.get("calls", [])) + table = (f" {VIEWER_TH}" + + rows + "
TypeArmRegionMbCell fraction
\n" + VIEWER_NOTE) if rows else small_note("No mosaic alteration reported.") lines = mosaic_mca_lines(d) return [stat("Calls reported", E(d.get("calls_reported", "0"))), stat("Cell-fraction floor", E(d.get("floor", "."))), table, small_note(lines[0]), small_note(MCA_NOTE)] @@ -2139,25 +2140,37 @@ def mito_extended_lines(d): # scripts/owngenome-view.sh serves the genome viewer here (OWNGENOME_VIEW_PORT's default); -# the part after '#' is the locus it opens at. +# the part after '#' is the locus it opens at. The live site's viewer plugin +# (bin/owngenome_live_viewer.py) points this address and VIEWER_NOTE at its own viewer +# in the report it serves. VIEWER_URL = "http://127.0.0.1:8765/" - - -def viewer_links(sites): - """One line under a findings table: each (chrom, pos, label) as a link that - opens the reads at chrom:pos in the local genome viewer. Empty for none.""" - links = [f'{E(str(t))}' - for c, p, t in sites if c and p] - if not links: - return [] - return ['

See the reads in the genome viewer on this computer (start it first: ' - "scripts/owngenome-view.sh <sample>, docs/viewer.md): " + " · ".join(links) + "

"] - - -def variant_site(variant): - """(chrom, pos) of a "chrom:pos ref>alt" text, (None, None) for any other.""" - m = re.match(r"^([A-Za-z0-9._-]+):([0-9]+) ", variant or "") - return (m.group(1), m.group(2)) if m else (None, None) +# Every findings table ends with a Viewer column: one link per row, to the row's locus. A column +# of its own leaves every other cell's text as it was. +VIEWER_TH = "Viewer" +VIEWER_NOTE = ('

The Viewer column opens the reads at each row in the genome viewer ' + "on this computer (start it first: scripts/owngenome-view.sh <sample>, docs/viewer.md).

") +LOCUS_IN_TEXT = re.compile(r"([A-Za-z0-9_.]+):([0-9]+)(?:-([0-9]+))?(?![0-9])") + + +def viewer_td(chrom, start, end=None): + """The Viewer cell of a row: a link that opens chrom:start, or chrom:start-end, in the + genome viewer; an empty cell without a position.""" + if not chrom or not start: + return "" + where = f"{chrom}:{start}" + (f"-{end}" if end and str(end) != str(start) else "") + return (f'' + f"{E(where)}") + + +def viewer_td_of(text): + """The Viewer cell of a row whose locus is written in text ("chr1:100 A>G", "DEL chr1:5-90", + "chr1:50 A>G + chr1:90 C>T"): the span of every position on the first one's chromosome.""" + found = LOCUS_IN_TEXT.findall(str(text or "")) + if not found: + return "" + chrom = found[0][0] + pos = [int(x) for c, a, b in found if c == chrom for x in (a, b) if x] + return viewer_td(chrom, min(pos), max(pos)) EXPLORE_NOTE = ("Nothing is uploaded by this pipeline. The docs link lists the files each service reads, " @@ -2424,22 +2437,24 @@ def a(key, lines): av = S["annotsv"]["data"] body = [stat("Status", done_badge(S["annotsv"]))] + ([small_note(x) for x in annotsv_lines(av)] if av else []) if av.get("rows"): - body.append(" " + body.append(f"
SVClassGenesCallersChecks
{VIEWER_TH}" + "".join(f"" f"" - f"" for x in av["rows"]) + f"{viewer_td_of(x['sv'])}" + for x in av["rows"]) + "
SVClassGenesCallersChecks
{E(x['type'] + ' ' + x['sv'])}{E(x['acmg_class'])}{E(annotsv_genes_text(x))}{E(annotsv_support_text(x))}{E('; '.join(annotsv_flags(x)) or 'none')}
{E('; '.join(annotsv_flags(x)) or 'none')}
") if av.get("downgraded"): body.append("

Moved to class 3 by step 05's rules

") body.append(" " - "" + f"{VIEWER_TH}" + "".join(f"" f"" - f"" + f"" + f"{viewer_td_of(x['sv'])}" for x in av["downgraded"]) + "
SVAnnotSV classClassGenesRuleWhy
Why
{E(x['type'] + ' ' + x['sv'])}{E(x['annotsv_class'])}{E(x['acmg_class'])}{E(annotsv_genes_text(x))}{E(x['rule'] or 'not named')}{E(x['rule_detail'] or 'no detail')}
{E(x['rule'] or 'not named')}{E(x['rule_detail'] or 'no detail')}
") if av.get("rows") or av.get("downgraded"): - body.append(small_note(ANNOTSV_NOTE)) + body += [VIEWER_NOTE, small_note(ANNOTSV_NOTE)] a("annotsv", card(S["annotsv"], "Structural Variants, Class 4 and 5 (AnnotSV)", body, full=True)) cs = S["cpsr"]["data"] @@ -2583,13 +2598,13 @@ def a(key, lines): if ma.get("variants"): mito_body.append(" " + "".join(f"" for c in ( "Variant", "AF", "ClinVar", "Review", "Condition", "gnomAD AF hom", "gnomAD AF het", "Poly-C tract")) - + "" + "".join( + + VIEWER_TH + "" + "".join( "" + "".join(f"" for x in ( f"chrM:{v['pos']} {v['ref']}>{v['alt']}", v.get("af"), v.get("clinvar_significance"), v.get("clinvar_review"), v.get("clinvar_condition"), v.get("gnomad_af_hom"), - v.get("gnomad_af_het"), "yes" if v.get("polyc") else "")) + "" + v.get("gnomad_af_het"), "yes" if v.get("polyc") else "")) + viewer_td("chrM", v["pos"]) + "" for v in ma["variants"]) + "
{c}
{E(str(x))}
") - mito_body += viewer_links(("chrM", v["pos"], f"chrM:{v['pos']}") for v in ma["variants"]) + mito_body.append(VIEWER_NOTE) mito_body.append(small_note(f"ClinVar {ma.get('clinvar') or 'unknown'}; gnomAD {ma.get('gnomad') or 'unknown'}")) elif ma.get("note"): mito_body.append(small_note(f"ClinVar and gnomAD: not annotated ({ma['note']})")) @@ -2629,14 +2644,13 @@ def a(key, lines): sf = cl.get("acmg_sf") if sf and (sf["clinvar_hits"] or sf["high_impact"] or sf.get("structural") or sf.get("not_reported") or sf.get("unavailable")): - rows = "".join(f"{E(g)}{E(v)}{E(z)}{E(ev)}\n" + rows = "".join(f"{E(g)}{E(v)}{E(z)}{E(ev)}{viewer_td_of(v)}\n" for g, v, z, ev in sf_rows(sf)) a("clinical", card(S["clinical"], f"Secondary-Findings Genes ({sf['version']})", - [" ", " ", + ["
GeneVariantGenotypeEvidence
", f" {VIEWER_TH}", rows.rstrip("\n"), "
GeneVariantGenotypeEvidence
"] + [f"

{E(line)}

" for line in sf_apart_lines(sf)] - + viewer_links((*variant_site(x["variant"]), f"{x['gene']} {x['variant'].split(' ')[0]}") - for x in sf["clinvar_hits"] + sf["high_impact"]) + + ([VIEWER_NOTE] if rows else []) + [f"

{E(ACMG_NOTE)}

"], full=True)) sl = S["slivar"]["data"] @@ -2734,7 +2748,7 @@ def a(key, lines): f"{E(combined_cell(h))}" + (f"{E(af_text(h))}" if freq else "") + (f"{E(call_text(h))}" if call else "") - + (f"{E(cdna_text(h))}" if reads else "") + "" + + (f"{E(cdna_text(h))}" if reads else "") + viewer_td(h["chrom"], h["pos"]) + "" for h in c["hits"][:report_charts.HITS_SHOWN]] a("clinvar", card(S["clinvar"], f"ClinVar Hits (best-reviewed first, {min(nh, 50)} of {nh})", # The anchor the charts link a listed hit to (report_charts.hit_anchor). @@ -2743,29 +2757,30 @@ def a(key, lines): "SignificanceReview statusStarsCombined allele" + ("gnomAD frequency" if freq else "") + ("Call quality" if call else "") - + ("cDNA read check" if reads else "") + ""] + + ("cDNA read check" if reads else "") + VIEWER_TH + ""] + [" " + x for x in rows] + [" "] + [small_note("Hit at a known lab-construct site past the first 50: " + construct_past_cap_text(h)) for h in construct_past_cap(c["hits"], report_charts.HITS_SHOWN)] - + viewer_links((h["chrom"], h["pos"], f"{h['gene']} {h['chrom']}:{h['pos']}") for h in c["hits"][:50]), + + ([VIEWER_NOTE] if rows else []), full=True)) if S["clinvar"]["state"] in ("ok", "stale") and ("deep_intronic" in c or c.get("cdna_scan")): body = [] if "deep_intronic" in c: - body += [" ", " "] + body += ["
GeneVariantPositionCall
", f" {VIEWER_TH}"] body += [f" " f"" - f"" for x in c["deep_intronic"]] - body += ["
GeneVariantPositionCall
{E(x['gene'])}{E(x['hgvs'])}{E(x['chrom'])}:{x['pos']} {E(x['ref'])}>{E(x['alt'])}{E(deep_intronic_call(x))}
", small_note(DEEP_INTRONIC_NOTE)] + f"{E(deep_intronic_call(x))}{viewer_td(x['chrom'], x['pos'])}" for x in c["deep_intronic"]] + body += [" ", VIEWER_NOTE, small_note(DEEP_INTRONIC_NOTE)] scan = cdna_scan_lines(c) if scan: body.append(f"

{E(scan[0])}

") if len(scan) > 1: - body += [" ", " "] + body += ["
LocusJunctionsReads
", f" {VIEWER_TH}"] body += [f" " - f"" for x in c["cdna_scan"]["loci"][:20]] - body += ["
LocusJunctionsReads
{E(x['chrom'])}:{x['start']}-{x['end']}{x['junctions']}{x['reads']}
", small_note(CDNA_SCAN_NOTE)] + f"{x['reads']}{viewer_td(x['chrom'], x['start'], x['end'])}" + for x in c["cdna_scan"]["loci"][:20]] + body += [" ", VIEWER_NOTE, small_note(CDNA_SCAN_NOTE)] a("clinvar", card(S["clinvar"], "Known Deep Intronic Sites and the cDNA Scan", body, full=True)) add_charts(parts, chart_specs(s)) diff --git a/docs/24-html-report.md b/docs/24-html-report.md index 4eda6ff9..19d0ec2d 100644 --- a/docs/24-html-report.md +++ b/docs/24-html-report.md @@ -67,6 +67,7 @@ After them, "What this report leaves out" lists the steps not run and what no st - **PRS** (`#prs`): the percentiles on a 0 to 100 strip, only when pgsc_calc had an ancestry panel. A score with no percentile, or one of the other sex, says "no percentile" and gets no dot. Without a panel there is no strip: a raw score has no scale, and the caption already says so. - **Links to the step's own files**, relative to the report, when the summary says they exist: the PharmCAT HTML report, the CPIC recommendations per gene, the CPSR HTML report (or the `cpsr/` folder that holds it) and the Exomiser ranking table. The report sits in the sample folder beside each step's folder, in the bash layout and in a Nextflow `--outdir`. MultiQC and the couple report are not linked: the summary does not record them, and the couple report sits in the family folder. - **Nothing is fetched.** A Content-Security-Policy (`default-src 'none'; img-src data:; style-src 'unsafe-inline'`) lets the browser load nothing but the file itself and its inline pictures, and there is no JavaScript. The only addresses off this computer are the haplogroup links (FamilyTreeDNA, YFull, the docs) and the project link in the footer, each marked "opens a website"; the FamilyTreeDNA and YFull ones add that the address carries the haplogroup. The genome viewer links go to `127.0.0.1`, this computer. +- **A Viewer column on every findings table.** The ClinVar hits, the secondary-findings genes, the mitochondrial variants, the known deep intronic sites, the cDNA scan, the AnnotSV class 4 and 5 SVs and the calls moved to class 3, slivar's pairs in trans and the mosaic alterations each end with a Viewer column: one link per row, to that row's position (a range for an SV, a region or a pair). The link opens the reads there in the [genome viewer](viewer.md) once `scripts/owngenome-view.sh ` runs; a note under the table says so. The column is new, so every other cell keeps its text; it replaces the single line of links that sat under three of these tables. A row with no position has an empty Viewer cell. On the [live site](live-site.md) the served report points the same links at the site's own viewer of the run. - **Genes and rsIDs in these results.** The closing list is built by `bin/results_index.py`. It holds every gene symbol and rsID that a section in state complete or stale names in `summary.json`, by the same rule `scripts/owngenome-lookup.sh` uses for "in the reports" (`names_of` in `bin/genome_lookup.py`). It adds the genes of the clinical filter (step 23) and variant prioritization (step 31), whose cards only count them: from their summary tables on the bash path, from the VCFs on the Nextflow path. The two give the same genes when a variant's most severe consequence is on the transcript the report ranks first (MANE, else canonical); the step 31 table keeps the gene of the most severe consequence, so where an overlapping gene's transcript carries the worse one, the two forms name different genes. A ClinVar hit's rsIDs come from step 13's VEP output (`Existing_variation`), because the hits file's ID column holds ClinVar's own id; without step 13 a hit has none. The Nextflow `HTML_REPORT` process does not get step 13's VEP output, so its index has no ClinVar rsIDs; step 24, which `run-all.sh` runs over the output folder afterwards, adds them. A failed, missing or unreadable section adds nothing. The browser's find (Ctrl+F) searches the list; there is still no script. A position is not in it: ask `scripts/owngenome-lookup.sh` for a genotype ([genotype lookup](lookup.md)). - **Dark on screen, light on paper.** - **Print one section.** Open a section from the contents (or its "link to this section"), then print: only that section prints, with the header (sample, generation date, pipeline commit, ClinVar file date, ClinVar in CPSR's bundle, HLA database), the list of what is not assessed and the disclaimer. A chapter prints the same way from its title in the contents. Without an anchor in the address the whole report prints, as it does in a browser without CSS `:has()`. @@ -77,7 +78,7 @@ After them, "What this report leaves out" lists the steps not run and what no st `${SAMPLE}_search_index.json` is the list the report closes with, as JSON: `schema_version` 1, `sample`, `generated_utc` and `terms`, where each term (a gene in upper case, an rsID in lower case) maps to one item per section that names it: the section key, its report anchor, title, step, state, source file, a count of mentions (a variant counts once per section) and the first 20 mentions, each with its text and locus. The Nextflow `HTML_REPORT` process writes the same file beside its report, without the ClinVar hits' rsIDs (see above). `bin/results_index.py query --index BRCA2` answers one query at a shell. The file names genes of the results, so keep it on this machine like the report. On the live site its plugin (`bin/owngenome_live_search.py`, `assets/live/plugins/search.js`) serves the file only behind the site's password. The page fetches it once, and again only when the report is written anew, then filters it in the browser as you type. No request carries the query, so it is in no log. A prefix lists the terms that start with it (`BRCA` gives `BRCA1` and `BRCA2`), and a position gets a pointer to `scripts/owngenome-lookup.sh`. -`${SAMPLE}_charts.json` holds the five charts as data and as SVG (`schema_version`, `sample`, `generated_utc`, `charts`), each with its section, its caveat and one entry per mark (value, range, locus, detail). The Nextflow `HTML_REPORT` process writes the same file beside its report. One input differs: Nextflow also passes the samplesheet's sex, so when no run status or manifest records a sex, its PRS chart marks the sex-specific scores against that sex and step 24's does not. It holds findings, so treat it like the report: keep it on this machine. On the live site its plugin (`bin/owngenome_live_charts.py`, `assets/live/plugins/charts.js`) serves this file only behind the site's password and shows the same charts; a click on a mark opens a panel with its detail, its caveat, a `viewer/#` link that opens the reads once the live site serves the genome viewer, and a link to its section of the report. +`${SAMPLE}_charts.json` holds the five charts as data and as SVG (`schema_version`, `sample`, `generated_utc`, `charts`), each with its section, its caveat and one entry per mark (value, range, locus, detail). The Nextflow `HTML_REPORT` process writes the same file beside its report. One input differs: Nextflow also passes the samplesheet's sex, so when no run status or manifest records a sex, its PRS chart marks the sex-specific scores against that sex and step 24's does not. It holds findings, so treat it like the report: keep it on this machine. On the live site its plugin (`bin/owngenome_live_charts.py`, `assets/live/plugins/charts.js`) serves this file only behind the site's password and shows the same charts; a click on a mark opens a panel with its detail, its caveat, a `viewer/#` link that opens the reads in the site's genome viewer of the run (`bin/owngenome_live_viewer.py`), and a link to its section of the report. The cards, by chapter (cards not listed here are named in the chapter table above): diff --git a/tests/test_collect_summary.py b/tests/test_collect_summary.py index 22ead72a..8cc94786 100644 --- a/tests/test_collect_summary.py +++ b/tests/test_collect_summary.py @@ -1859,8 +1859,10 @@ def main(): check("HTML report: the same ClinVar count in its badge, yellow (never green) with a hit", 'class="badge badge-yellow">3' in html and 'badge-green">3<' not in html) rows = [l for l in html.splitlines() if l.strip().startswith("chr")] - check("HTML report: one row per hit, each with a Stars cell and an empty combined-allele cell", - len(rows) == 3 and rows[0].strip().endswith("3gnomAD AF unknown"), + check("HTML report: one row per hit, each with a Stars cell, an empty combined-allele cell and the viewer link", + len(rows) == 3 and rows[0].strip().endswith( + '3gnomAD AF unknownchr1:20'), rows) for r in sec["prs"]["data"]["scores"]: check(f"PRS row {r['pgs_id']} in both reports", r["pgs_id"] in txt and f"{r['pgs_id']}" in html) @@ -1965,7 +1967,9 @@ def main(): check("combined allele: the HTML table has the column and the notes, an SNV hit's cell empty", "StarsCombined allele" in html_cv and any(f"part of a combined allele: chr22:30000038 GCCAT>CATGGG{unph}" in r for r in cv_rows) - and any("30000210" in r and r.strip().endswith("") for r in cv_rows), cv_rows) + and any("30000210" in r and r.strip().endswith( + 'chr22:30000210') for r in cv_rows), cv_rows) # The same hits phased (0|1, one phase set) or homozygous: both # parts are on one haplotype, so the notes carry no phase caveat. for gt_ in ("0|1", "1/1"): From 41484276d67373ec9f778686af78744c5b5a2a86 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 4 Nov 2026 15:09:41 +0100 Subject: [PATCH 2/3] feat(viewer): serve the genome viewer per run on the live site The genome viewer ran only on 127.0.0.1 of the machine holding the data. bin/owngenome_live_viewer.py serves the same igv.js page for every run of the live site, behind its password, and points the served report's Viewer links at it. The igv.js pin moves to versions.env so both launchers share it. tests/test_live_viewer.py checks the login, the policy, the byte ranges, the refused files, the report links, the access log and the page in headless Chrome; tests/test_viewer.py checks every findings table and reads the pin from versions.env. Bead: pgp-5bja.8 Bead: pgp-awpe --- assets/viewer/serve.py | 65 ++++--- bin/owngenome_live_viewer.py | 178 ++++++++++++++++++++ docs/live-site.md | 12 +- docs/viewer.md | 21 ++- scripts/owngenome-live.sh | 21 +++ scripts/owngenome-view.sh | 8 +- tests/test_live_viewer.py | 317 +++++++++++++++++++++++++++++++++++ tests/test_viewer.py | 165 ++++++++++++++---- versions.env | 8 + 9 files changed, 731 insertions(+), 64 deletions(-) create mode 100644 bin/owngenome_live_viewer.py create mode 100644 tests/test_live_viewer.py diff --git a/assets/viewer/serve.py b/assets/viewer/serve.py index 5fc115e0..af47ac48 100755 --- a/assets/viewer/serve.py +++ b/assets/viewer/serve.py @@ -1,8 +1,11 @@ #!/usr/bin/env python3 """Serve the local genome viewer (igv.js) for one sample on 127.0.0.1. -scripts/owngenome-view.sh finds the sample's files and starts this server. It -serves exactly these, and nothing else from the disk: +scripts/owngenome-view.sh finds the sample's files and starts this server. +The live site serves the same page per run, behind its login, through +bin/owngenome_live_viewer.py, which reuses sample_files, build_config, +send_file and CSP from here. This server serves exactly these, and nothing +else from the disk: / index.html of this folder /viewer.js the page's script, from this folder @@ -73,6 +76,18 @@ def index_of(names): return next((n for n in names if os.path.isfile(n)), None) +def sample_files(sample_dir, sample): + """(reads, vcf, sv_vcf) of a sample folder, by the names the steps write, + None for one that is not there: the files owngenome-view.sh picks when the + environment names none, and the ones the live site's viewer + (bin/owngenome_live_viewer.py) serves.""" + def first(*names): + return next((p for p in (os.path.join(sample_dir, n) for n in names) if os.path.isfile(p)), None) + return (first(f"aligned/{sample}_sorted.bam", f"aligned/{sample}_sorted.cram"), + first(f"vcf/{sample}.vcf.gz"), + first(f"sv_merged/{sample}_sv_consensus.vcf.gz", "manta/results/variants/diploidSV.vcf.gz")) + + def build_config(sample, fasta, reads=None, vcf=None, sv_vcf=None, locus=None): """(config, files): the igv.js browser options and the list of files the server may send, /data// each.""" @@ -129,6 +144,31 @@ def parse_range(header, size): return start, min(end, size - 1) +def send_file(handler, head, ctype, src): + """Send the file src, or the one byte range the request's Range header + asks for (206; 416 past the end), 1 MB at a time. head(code, ctype, + length, extra headers) writes the status line and headers.""" + size = os.path.getsize(src) + rng = parse_range(handler.headers.get("Range"), size) + if rng == "bad": + return head(416, "text/plain; charset=utf-8", 0, [("Content-Range", f"bytes */{size}")]) + start, end = rng if rng else (0, size - 1) + length = end - start + 1 if size else 0 + extra = [("Accept-Ranges", "bytes")] + ([("Content-Range", f"bytes {start}-{end}/{size}")] if rng else []) + head(206 if rng else 200, ctype, length, extra) + if handler.command == "HEAD" or not length: + return + with open(src, "rb") as f: + f.seek(start) + left = length + while left > 0: + chunk = f.read(min(1 << 20, left)) + if not chunk: + break + handler.wfile.write(chunk) + left -= len(chunk) + + class Handler(http.server.BaseHTTPRequestHandler): server_version = "owngenome-view" sys_version = "" @@ -141,7 +181,6 @@ def _head(self, code, ctype, length, extra=()): self.send_response(code) self.send_header("Content-Type", ctype) self.send_header("Content-Length", str(length)) - self.send_header("Accept-Ranges", "bytes") self.send_header("Cache-Control", "no-store") self.send_header("Content-Security-Policy", CSP) self.send_header("X-Content-Type-Options", "nosniff") @@ -180,25 +219,7 @@ def _serve(self): if self.command != "HEAD": self.wfile.write(data) return - size = os.path.getsize(src) - rng = parse_range(self.headers.get("Range"), size) - if rng == "bad": - return self._head(416, "text/plain; charset=utf-8", 0, [("Content-Range", f"bytes */{size}")]) - start, end = rng if rng else (0, size - 1) - length = end - start + 1 if size else 0 - extra = [("Content-Range", f"bytes {start}-{end}/{size}")] if rng else [] - self._head(206 if rng else 200, ctype, length, extra) - if self.command == "HEAD" or not length: - return - with open(src, "rb") as f: - f.seek(start) - left = length - while left > 0: - chunk = f.read(min(1 << 20, left)) - if not chunk: - break - self.wfile.write(chunk) - left -= len(chunk) + send_file(self, self._head, ctype, src) def do_GET(self): try: diff --git a/bin/owngenome_live_viewer.py b/bin/owngenome_live_viewer.py new file mode 100644 index 00000000..e00d94c5 --- /dev/null +++ b/bin/owngenome_live_viewer.py @@ -0,0 +1,178 @@ +"""owngenome_live_viewer.py: the genome viewer on the live site, a plugin of bin/owngenome_live.py. + +scripts/owngenome-view.sh serves one sample's reads in igv.js on 127.0.0.1, +with no login. This plugin serves the same page for every run of the site, +behind the site's password, so a position in the report opens the reads from +any device that can reach the site: + + GET /runs//viewer/ the page (assets/viewer/index.html) + GET /runs//viewer/viewer.js its script (assets/viewer/viewer.js) + GET /runs//viewer/igv.min.js the pinned igv.js (IGV_JS_VERSION in versions.env) + GET /runs//viewer/config.json the reference and the run's tracks + GET /runs//viewer/data// the n-th file of that config, by byte range + +The files are the ones owngenome-view.sh picks by default +(assets/viewer/serve.py sample_files): the run folder's BAM or CRAM, its VCF +and its SV VCF, each with its index, and the reference. A file outside +GENOME_DIR (a symlink out of it) is left out. The page and the files carry the +viewer's Content-Security-Policy (serve.CSP): the page talks to this site +only. The report the site serves (report.html) gets its viewer links pointed +here. Opening the page writes one access-log row; the byte ranges igv.js reads +write none. + +Flags it adds to owngenome_live.py (also as KEY=VALUE lines in --config; +scripts/owngenome-live.sh passes both): + + --viewer-fasta FILE the reference FASTA, with its .fai beside it (REF_FASTA) + --viewer-igv-js FILE the igv.min.js scripts/owngenome-live.sh checked by sha256 + +Without both the viewer answers 404 with the reason, and the report keeps its +links to the local viewer. Standard library only. +""" +import importlib.util +import json +import os +import re +import sys +import urllib.parse + +HERE = os.path.dirname(os.path.abspath(__file__)) +VIEWER_DIR = os.path.join(os.path.dirname(HERE), "assets", "viewer") +sys.path.insert(0, HERE) +import render_report # noqa: E402 + +_spec = importlib.util.spec_from_file_location("owngenome_viewer_serve", os.path.join(VIEWER_DIR, "serve.py")) +serve = importlib.util.module_from_spec(_spec) +_keep, sys.dont_write_bytecode = sys.dont_write_bytecode, True # no __pycache__ in assets/viewer +try: + _spec.loader.exec_module(serve) +finally: + sys.dont_write_bytecode = _keep + +LOCAL_TEXT = "served from this computer only." +LIVE_TEXT = "served by the run site, behind its login." +LIVE_NOTE = ('

The Viewer column opens the reads at each row in this site\'s genome viewer ' + "(docs/live-site.md).

") +DATA_RE = re.compile(r"data/([0-9]+)/(.+)") +STATIC = {"viewer.js": "text/javascript; charset=utf-8"} + +APP = None # the App register() was given; its config holds the flags once they are read + + +def flag(name): + return getattr(APP.config, name, None) if APP and APP.config else None + + +def ready(): + """Why the viewer cannot run, or None when it can.""" + fasta, igv = flag("viewer_fasta"), flag("viewer_igv_js") + if not fasta or not igv: + return "the genome viewer is not set up: start the site with --viewer-fasta and --viewer-igv-js " \ + "(scripts/owngenome-live.sh passes both)" + if not os.path.isfile(fasta) or not os.path.isfile(fasta + ".fai"): + return "the genome viewer's reference or its .fai is missing" + if not os.path.isfile(igv): + return "igv.min.js is missing" + return None + + +def run_config(run, run_dir, root): + """(config, files) of one run: serve.build_config on the run folder's files that lie inside + GENOME_DIR. ValueError when the reference or an index is wrong.""" + top = os.path.realpath(run_dir) + + def inside(p): + real = os.path.realpath(p) + return all(real.startswith(t.rstrip(os.sep) + os.sep) for t in (top, root)) + reads, vcf, sv_vcf = (p if p and inside(p) else None for p in serve.sample_files(run_dir, run)) + return serve.build_config(run, flag("viewer_fasta"), reads, vcf, sv_vcf) + + +def head_writer(req): + def head(code, ctype, length, extra=()): + req.send_response(code) + req.send_header("Content-Type", ctype) + req.send_header("Content-Length", str(length)) + req.send_header("Cross-Origin-Resource-Policy", "same-origin") + for k, v in extra: + req.send_header(k, v) + req.headers_common(serve.CSP) + req.end_headers() + return head + + +def send(req, code, ctype, body): + body = body.encode() if isinstance(body, str) else body + head_writer(req)(code, ctype, len(body)) + if req.command != "HEAD": + req.wfile.write(body) + + +def viewer(req, m): + run, rest = urllib.parse.unquote(m.group(1)), m.group(2) + page = rest in ("", "index.html") + # The page asks for a login like every page of the site; the files answer 401. + if req.live.password_hash and req.sid is None: + req.log("viewer", status=303 if page else 401) + if page: + return req.send_body(303, "text/plain", "", [("Location", "/login?next=" + urllib.parse.quote( + urllib.parse.urlsplit(req.path).path))]) + return req.deny(401) + d = req.run_dir(run) + if not d: + req.log("viewer", status=404) + return req.deny(404) + why = ready() + if why: + req.log("viewer", run, status=404) + return req.deny(404, why) + try: + if page: + with open(os.path.join(VIEWER_DIR, "index.html")) as f: + html = f.read().replace(LOCAL_TEXT, LIVE_TEXT) + req.log("viewer", run, status=200) + return send(req, 200, "text/html; charset=utf-8", html) + if rest in STATIC: + with open(os.path.join(VIEWER_DIR, rest), "rb") as f: + return send(req, 200, STATIC[rest], f.read()) + if rest == "igv.min.js": + return serve.send_file(req, head_writer(req), "text/javascript; charset=utf-8", flag("viewer_igv_js")) + cfg, files = run_config(run, d, req.live.root) + if rest == "config.json": + return send(req, 200, "application/json", json.dumps(cfg, indent=1)) + dm = DATA_RE.fullmatch(rest) + if dm and int(dm.group(1)) < len(files) and os.path.basename(files[int(dm.group(1))]) == dm.group(2): + return serve.send_file(req, head_writer(req), "application/octet-stream", files[int(dm.group(1))]) + except ValueError as e: + req.log("viewer", run, status=409) + return req.deny(409, f"the genome viewer cannot open this run: {e}") + except (BrokenPipeError, ConnectionResetError): + return None + return req.deny(404) + + +def to_slash(req, m): + run = urllib.parse.unquote(m.group(1)) + if not req.run_dir(run): + return req.deny(404) + return req.send_body(301, "text/plain", "", [("Location", f"/runs/{urllib.parse.quote(run)}/viewer/")]) + + +def point_links(run, html): + """The served report's viewer links and notes, pointed at this site's viewer of the run.""" + if ready(): + return html + here = f"/runs/{urllib.parse.quote(run)}/viewer/" + return (html.replace(f'href="{render_report.VIEWER_URL}#', f'href="{here}#') + .replace(render_report.VIEWER_NOTE, LIVE_NOTE)) + + +def register(app): + global APP + APP = app + app.add_argument("--viewer-fasta", help="reference FASTA (with its .fai) for the genome viewer") + app.add_argument("--viewer-igv-js", help="the checked igv.min.js for the genome viewer") + app.route("GET", r"/runs/([^/]+)/viewer", to_slash) + # auth=False: viewer() asks for the login itself, so the page sends the browser to it. + app.route("GET", r"/runs/([^/]+)/viewer/(.*)", viewer, auth=False) + app.report_filter(point_links) diff --git a/docs/live-site.md b/docs/live-site.md index e1a01aab..d4dc4ab1 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -35,12 +35,21 @@ weblog { - **A step's detail.** Click a box: its state and reason, start and end, the usual hours, each process with its tasks waiting, running, done, from the cache or failed, and the headline of its sections with a link into the report. - **Found so far.** One line per finished section, worded by the same code that writes the report, caveat included. A line marked for attention means the report has something to read there; no line ever says a section is fine. The ClinVar line, with no hit, is information, never an all-clear. - **The report.** Once written, `report.html` opens the run's `_report.html`, and the files it links to beside it. +- **The genome viewer.** `/runs//viewer/` shows the run's reads and variants in igv.js, as [`scripts/owngenome-view.sh`](viewer.md) does on one computer. The report's Viewer links and the charts' "See the reads" links open it at their position. + +## The genome viewer + +`bin/owngenome_live_viewer.py` serves it, behind the same password as every page. `owngenome-live.sh` passes it the reference (`REF_FASTA`, as in every step) and igv.js, which the first start downloads into `GENOME_DIR/tools/` and checks against `IGV_JS_SHA256` in `versions.env`. Without the reference's `.fai` or igv.js the site starts without the viewer and says why; `OWNGENOME_LIVE_VIEWER=off` leaves it out on purpose. Running `owngenome_live.py` yourself (in Docker, say), pass `--viewer-fasta` and `--viewer-igv-js`. + +Per run it serves the folder's BAM or CRAM, its VCF and its SV VCF, each with its index, and the reference: the files `owngenome-view.sh` picks by default. A file that is a symlink out of `GENOME_DIR` is left out. The page and the files carry the viewer's Content-Security-Policy, so the page talks to this site only, and igv.js's calls to outside services are blocked. + +The reads cross the network to the browser that logged in: only the byte ranges igv.js asks for, but they are the sample's reads. Off loopback, use TLS (`--tls-cert` and `--tls-key`, or a proxy that ends TLS); without it the password and the reads cross the network in clear. Opening the viewer writes one access-log row; the byte ranges igv.js reads write none. `GET /` lists the runs: each folder under `GENOME_DIR` holding `logs/run_status.tsv`, `nextflow/` or `run_manifest.tsv`, and each run the site has messages for. ## Privacy and what is stored -- The site serves only its own pages, the run's state, the report and the files the report links to by a relative link. Any other file is 404, and so is a link that leaves the run folder or a symlink out of `GENOME_DIR`. It never lists a folder. +- The site serves only its own pages, the run's state, the report, the files the report links to by a relative link, and the genome viewer's files of the run (see above). Any other file is 404, and so is a link that leaves the run folder or a symlink out of `GENOME_DIR`. It never lists a folder. - Each message is cut to what a progress view needs before it is stored: the task, its process and status, times, CPU and memory figures. Nextflow's parameters, command line and task scripts are dropped. Three paths stay on the server for later views (the output folder, the CPU and memory caps, the work directory); none of them reaches a page. - Messages go to `/runs//events.jsonl` (state dir: `GENOME_DIR/.owngenome-live`, or `OWNGENOME_LIVE_STATE_DIR`). The site writes nowhere else, so `GENOME_DIR` can be mounted read-only. - The access log, `/access.log`, has one tab-separated row per page or file read and per login: time, client, the first 8 characters of a hash of the session, action, run, file, status. It never holds a query string, a request body, a result or the password. It moves to `access.log.1` past `--access-log-max-mb` (10). @@ -62,6 +71,7 @@ weblog { | `--trust-proxy CIDR` | none | Read `X-Forwarded-For` only from this proxy (repeatable) | | `--tls-cert`, `--tls-key` | none | Serve HTTPS; the cookie is then `Secure` | | `--cookie-secure` | off | Mark the cookie `Secure` when TLS ends at a proxy | +| `--viewer-fasta`, `--viewer-igv-js` | none | The genome viewer's reference and igv.js; `owngenome-live.sh` passes both | | `--config FILE` | none | `key=value` lines setting any flag above (`session_hours=8`), read before the flags | The session cookie is random (32 bytes), `HttpOnly` and `SameSite=Strict`. A POST from another site is refused. diff --git a/docs/viewer.md b/docs/viewer.md index 7a96a7bd..24672799 100644 --- a/docs/viewer.md +++ b/docs/viewer.md @@ -9,7 +9,9 @@ - its small variants (the VCF from step 3), - its structural variants (step 22's consensus, else Manta's VCF from step 4). -It opens at the locus you give. The HTML report (step 24) links to the viewer under the ClinVar hits, the secondary-findings genes and the mitochondrial variants. With the viewer running, a click on one of those links opens the reads at that position. +It opens at the locus you give. Every findings table of the HTML report (step 24) ends with a Viewer column: the ClinVar hits, the secondary-findings genes, the mitochondrial variants, the deep intronic sites, the cDNA scan, the AnnotSV class 4 and 5 SVs, slivar's pairs in trans and the mosaic alterations. With the viewer running, a click on a row's link opens the reads at that row. + +The [live site](live-site.md) serves the same viewer for each run, behind its password, so the report's links work from any device that can reach the site (see "From the live site" below). ## Why @@ -17,7 +19,7 @@ A variant call is a summary of the reads. Before you act on a finding, look at t ## Tool -- **igv.js** 3.8.10 (`IGV_JS_VERSION` in `scripts/owngenome-view.sh`, checked against a pinned sha256) draws the reads and variants in the browser. +- **igv.js** 3.8.10 (`IGV_JS_VERSION` and `IGV_JS_SHA256` in `versions.env`, shared with the live site) draws the reads and variants in the browser. - **`assets/viewer/serve.py`** (Python standard library) serves the page and the sample's files on `127.0.0.1`. ## Docker Image @@ -49,7 +51,7 @@ Open it in a browser. To jump somewhere else, type a locus in igv.js's search bo The script leaves out a file it does not find, and says so. A BAM or CRAM needs its index (`.bai`, `.crai`) beside it. A VCF with a `.tbi` is read in pieces. One without a `.tbi` is read whole, which is fine for an SV VCF. -The command needs `python3` on the host, but no Docker. The first run downloads igv.js (version 3.8.10, the `IGV_JS_VERSION` line in the script) into `GENOME_DIR/tools/igv.js-3.8.10/` and checks its sha256. After that it needs no network. +The command needs `python3` on the host, but no Docker. The first run downloads igv.js (version 3.8.10, the `IGV_JS_VERSION` line in `versions.env`) into `GENOME_DIR/tools/igv.js-3.8.10/` and checks its sha256. After that it needs no network. ## Privacy @@ -76,6 +78,12 @@ GENOME_DIR=/path/to/data ./scripts/owngenome-view.sh your_name chr17:43045000 Then open `http://127.0.0.1:8765/` on your computer. The report's links work the same way. The reads travel inside the ssh connection, and only the ranges igv.js asks for. Do not expose the port any other way: the server has no login, so anyone who can reach it can read the sample. +## From the live site + +`scripts/owngenome-live.sh` starts the [live site](live-site.md) with the viewer: each run has it at `/runs//viewer/`, behind the site's password, and the report the site serves points its Viewer links there. `bin/owngenome_live_viewer.py` does it with the same page, the same Content-Security-Policy and the same files as `owngenome-view.sh` picks by default (the run folder's BAM or CRAM, its VCF and its SV VCF, each with its index, and `REF_FASTA`). The `READS`, `VCF` and `SV_VCF` overrides do not apply there, and a file that is a symlink out of `GENOME_DIR` is left out. The first start downloads igv.js the same way. `OWNGENOME_LIVE_VIEWER=off` starts the site without it. + +This differs from the local viewer in one way that matters: the reads now cross the network, to whatever browser logged in. Only the byte ranges igv.js asks for travel, and only after the login, but they are the sample's reads. Off loopback, use TLS (`--tls-cert` and `--tls-key`, or a proxy that ends TLS), or keep the site on loopback and forward it over ssh. Opening the viewer writes one row in the site's access log; the byte ranges write none. + ## Output Nothing is written apart from igv.js in `GENOME_DIR/tools/` on the first run. Stop the viewer with Ctrl-C. @@ -92,11 +100,14 @@ The server starts at once. How fast a view loads depends on the disk and the rea - the server listens on 127.0.0.1 and not on the machine's other address; - it answers byte ranges of the BAM exactly, and refuses unknown paths and a foreign `Host`; - in headless Chrome, igv.js starts on the config, the policy blocks nothing it needs, and it reads the BAM, its index, the reference and the VCF; -- the report links a ClinVar hit to the viewer at its position. +- every findings table of the report ends with a Viewer cell that links each row to its position, with the note under it; a control with the links taken out must fail that check for every table; +- igv.js is the release `versions.env` pins, with its sha256, and the script sets no pin of its own. + +`tests/test_live_viewer.py` checks the live site's viewer on the same sample: the login, the policy, the byte ranges, the files it refuses, the report's links pointed at the site, the access log, a BAM symlinked out of `GENOME_DIR`, a site started without the viewer, and the page in headless Chrome. ## Notes -- **The report links assume port 8765.** With another `OWNGENOME_VIEW_PORT`, type the locus into the viewer yourself. +- **The report links assume port 8765.** With another `OWNGENOME_VIEW_PORT`, type the locus into the viewer yourself. On the live site the links go to the site's viewer, whatever its port. - **CRAM** is decoded in the browser against the same reference. It must be the FASTA the CRAM was written with. - **Gene names** are not searchable (see Privacy). Use the coordinates the report shows. - igv.js is MIT-licensed. Reference: Robinson JT, Thorvaldsdottir H, Turner D, Mesirov JP. igv.js: an embeddable JavaScript implementation of the Integrative Genomics Viewer (IGV). *Bioinformatics* 39(1), btac830 (2023). https://doi.org/10.1093/bioinformatics/btac830 diff --git a/scripts/owngenome-live.sh b/scripts/owngenome-live.sh index c0d5d205..d73b9048 100755 --- a/scripts/owngenome-live.sh +++ b/scripts/owngenome-live.sh @@ -16,6 +16,14 @@ # OWNGENOME_LIVE_STATE_DIR where messages and the access log go (default GENOME_DIR/.owngenome-live) # OWNGENOME_LIVE_PASSWORD_FILE the hash file (default OWNGENOME_LIVE_STATE_DIR/password) # OWNGENOME_LIVE_ALLOWED_HOSTS space-separated host names the browser may use besides loopback +# OWNGENOME_LIVE_VIEWER "off" leaves the genome viewer out (default on) +# +# The genome viewer (bin/owngenome_live_viewer.py) shows each run's reads at +# /runs//viewer/, behind the same password: the reference is REF_FASTA, as +# in every step, and the first start downloads the pinned igv.js +# (IGV_JS_VERSION in versions.env) into GENOME_DIR/tools/, checked by sha256, +# as owngenome-view.sh does. Without the reference or igv.js the site starts +# without the viewer and says why. # Then start a run with OWNGENOME_LIVE_URL=http://127.0.0.1:8788 ./scripts/run-all.sh . # Needs python3 (3.8 or later) on the host; nothing is installed. set -euo pipefail @@ -45,6 +53,19 @@ args=(--root "$GENOME_DIR" --state-dir "$STATE" --bind "${OWNGENOME_LIVE_BIND:-1 --ingest-port "${OWNGENOME_LIVE_INGEST_PORT:-8788}") if [ -s "$PWF" ]; then args+=(--password-file "$PWF"); fi for h in ${OWNGENOME_LIVE_ALLOWED_HOSTS:-}; do args+=(--allowed-host "$h"); done +IGV_JS="${GENOME_DIR}/tools/igv.js-${IGV_JS_VERSION}/igv.min.js" +if [ "${OWNGENOME_LIVE_VIEWER:-on}" = off ]; then + : +elif [ ! -f "${REF_FASTA}.fai" ]; then + echo "Note: no reference index ${REF_FASTA}.fai; the site starts without the genome viewer." >&2 +else + if [ ! -f "$IGV_JS" ]; then + echo "Downloading igv.js ${IGV_JS_VERSION} for the genome viewer (once)..." + fetch "https://cdn.jsdelivr.net/npm/igv@${IGV_JS_VERSION}/dist/igv.min.js" "$IGV_JS" sha256 "$IGV_JS_SHA256" \ + || echo "Note: igv.js could not be downloaded; the site starts without the genome viewer." >&2 + fi + if [ -f "$IGV_JS" ]; then args+=(--viewer-fasta "$REF_FASTA" --viewer-igv-js "$IGV_JS"); fi +fi scheme=http for a in "$@"; do [ "$a" != --tls-cert ] && [[ "$a" != --tls-cert=* ]] || scheme=https; done [ -z "$OPEN" ] || echo "The run's page: ${scheme}://${OWNGENOME_LIVE_BIND:-127.0.0.1}:${OWNGENOME_LIVE_PORT:-8787}/runs/${OPEN}/" diff --git a/scripts/owngenome-view.sh b/scripts/owngenome-view.sh index f950fb11..e124b308 100755 --- a/scripts/owngenome-view.sh +++ b/scripts/owngenome-view.sh @@ -11,7 +11,7 @@ # 127.0.0.1 only and its pages may not call any other server, so the data never # leaves this machine; docs/viewer.md says how to reach it from a remote box # over ssh. Needs python3 on the host. The first run downloads the pinned -# igv.js release (IGV_JS_VERSION below) into GENOME_DIR/tools/ and checks its +# igv.js release (IGV_JS_VERSION in versions.env) into GENOME_DIR/tools/ and checks its # sha256; later runs need no network. # # Files, each overridable from the environment: @@ -30,10 +30,8 @@ GENOME_DIR=${GENOME_DIR:?Set GENOME_DIR to your data directory} . "$(dirname "$0")/lib/common.sh" validate_sample "$SAMPLE" -# igv.js, MIT licence, https://github.com/igvteam/igv.js. Bump both lines -# together: the sha256 is of dist/igv.min.js of that npm release. -IGV_JS_VERSION=3.8.10 -IGV_JS_SHA256=87b991ae2a720978172e269dd5127c495e56df5af31144dff19c83124e2f5744 +# igv.js: IGV_JS_VERSION and IGV_JS_SHA256 come from versions.env (through +# lib/common.sh), shared with scripts/owngenome-live.sh. IGV_JS_URL="https://cdn.jsdelivr.net/npm/igv@${IGV_JS_VERSION}/dist/igv.min.js" PORT=${OWNGENOME_VIEW_PORT:-8765} diff --git a/tests/test_live_viewer.py b/tests/test_live_viewer.py new file mode 100644 index 00000000..cfad547b --- /dev/null +++ b/tests/test_live_viewer.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 +"""The genome viewer on the live site: bin/owngenome_live_viewer.py, started by +scripts/owngenome-live.sh, on the synthetic sample of tests/test_viewer.py. + +Checks: + 1. owngenome-live.sh downloads the igv.js pinned in versions.env (sha256 + checked) and starts the site with the viewer: /runs//viewer/ without + a login sends the browser to /login, and the config and the reads answer + 401; + 2. after the login the page carries the viewer's Content-Security-Policy + and says it is served by the site; the config names the run's BAM, VCF + and SV VCF; a byte range of the BAM is 206 with exactly those bytes, past + the end 416; a file the config does not name (a wrong name, a path out + of data/) is 404; a run that does not exist is 404; + 3. the report the site serves links each position to /runs//viewer/ + with the site's note, not to the local viewer on 127.0.0.1:8765; + 4. opening the page writes one access-log row, and igv.js's byte ranges + write none; + 5. a run whose BAM is a symlink out of GENOME_DIR gets no reads track; + 6. a site started without the viewer flags answers the viewer with 404 and + the reason, and leaves the report's links as they are; + 7. in a headless Chrome (skipped without Chrome, except in CI) the page, + loaded from the site, starts igv.js (status ready, 3 tracks) and the + policy blocks nothing. +Checks 2 and 3 are also run against planted mistakes (a viewer that serves +any file of the run folder, a report filter that does nothing) and must fail. + +Run: python3 tests/test_live_viewer.py +""" +import http.client +import importlib.util +import os +import re +import shutil +import socket +import subprocess +import sys +import tempfile +import threading +import time +import urllib.parse + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(REPO, "bin")) +sys.path.insert(0, os.path.join(REPO, "tests")) +import owngenome_live # noqa: E402 +import render_report # noqa: E402 +import test_viewer as tv # noqa: E402 + +PASSWORD = "viewer test password" +FAILED = [] + + +def check(desc, ok, detail=""): + print(("[PASS] " if ok else "[FAIL] ") + desc + ("" if ok else f" -- {str(detail)[:600]}")) + if not ok: + FAILED.append(desc) + + +def request(port, path, headers=None, method="GET", body=None): + c = http.client.HTTPConnection("127.0.0.1", port, timeout=20) + try: + c.request(method, path, body=body, headers=dict({"Host": f"127.0.0.1:{port}"}, **(headers or {}))) + r = c.getresponse() + return r.status, {k.lower(): v for k, v in r.getheaders()}, r.read() + finally: + c.close() + + +def login(port): + code, h, _ = request(port, "/login", {"Content-Type": "application/x-www-form-urlencoded"}, "POST", + urllib.parse.urlencode({"password": PASSWORD, "next": "/"})) + m = re.match(r"owngenome_live=([^;]+)", h.get("set-cookie", "")) + return {"Cookie": f"owngenome_live={m.group(1)}"} if code == 303 and m else None + + +def wait_port(port, proc, seconds=120): + for _ in range(seconds * 2): + if proc.poll() is not None: + return False + try: + with socket.create_connection(("127.0.0.1", port), timeout=1): + return True + except OSError: + time.sleep(0.5) + return False + + +def in_process(argv, plugin_dir=None): + """The site in this process (plugins from plugin_dir, default bin/), on free ports.""" + argv = argv + ["--port", "0", "--ingest-port", "0", "--quiet"] + (["--plugin-dir", plugin_dir] if plugin_dir else []) + live, web, ingest = owngenome_live.start(argv) + for s in (web, ingest): + threading.Thread(target=s.serve_forever, daemon=True).start() + return live, web, ingest + + +def stop(live, web, ingest): + live.stopping = True + for s in (web, ingest): + s.shutdown() + s.server_close() + + +def viewer_checks(port, auth, s, label=""): + """Checks 2: what the viewer serves to a signed-in browser. Returns the config.""" + import json + code, h, body = request(port, "/runs/S/viewer/", auth) + csp = h.get("content-security-policy", "") + check(f"{label}page: 200 with the viewer's policy, served by the site", + code == 200 and csp == tv.serve.CSP and b"served by the run site" in body + and b"served from this computer only" not in body, (code, csp, body[:300])) + code, _, body = request(port, "/runs/S/viewer/igv.min.js", auth) + c2, _, js = request(port, "/runs/S/viewer/viewer.js", auth) + check(f"{label}page: igv.min.js and viewer.js are served beside it", + code == 200 and len(body) > 100000 and c2 == 200 and b"config.json" in js, (code, c2)) + code, _, body = request(port, "/runs/S/viewer/config.json", auth) + cfg = json.loads(body) if code == 200 else {} + names = [t["url"].rsplit("/", 1)[1] for t in cfg.get("tracks", [])] + check(f"{label}config: the run's BAM, VCF and SV VCF, by relative URL", + names == ["S_sorted.bam", "S.vcf.gz", "S_sv_consensus.vcf.gz"] + and all(t["url"].startswith("data/") for t in cfg.get("tracks", [])), (code, cfg)) + raw = open(f"{s}/aligned/S_sorted.bam", "rb").read() + bam = f"/runs/S/viewer/{cfg['tracks'][0]['url']}" if cfg.get("tracks") else "/runs/S/viewer/data/x" + code, h, body = request(port, bam, dict(auth, Range="bytes=10-109")) + check(f"{label}reads: a BAM range is 206 with exactly those bytes", + code == 206 and body == raw[10:110] and h.get("content-range") == f"bytes 10-109/{len(raw)}", (code, h)) + code, h, _ = request(port, bam, dict(auth, Range=f"bytes={len(raw) + 5}-")) + check(f"{label}reads: a range past the end is 416", code == 416, code) + bad = [request(port, p, auth)[0] for p in ( + "/runs/S/viewer/data/2/S_report.html", "/runs/S/viewer/data/99/S_sorted.bam", + "/runs/S/viewer/S_report.html", "/runs/S/viewer/data/2/..%2F..%2FS_report.html", + "/runs/S/viewer/serve.py", "/runs/NOPE/viewer/config.json")] + check(f"{label}a file the config does not name, and a run that does not exist, are 404", + bad == [404] * len(bad), bad) + return cfg + + +def report_checks(port, auth, label=""): + """Checks 3: the served report's links point at the site's viewer.""" + code, _, body = request(port, "/runs/S/report.html", auth) + page = body.decode(errors="replace") + check(f"{label}report: positions link to /runs/S/viewer/ with the site's note, none to 127.0.0.1:8765", + code == 200 and 'href="/runs/S/viewer/#chr1:451"' in page and render_report.VIEWER_URL not in page + and "this site's genome viewer" in page + and render_report.VIEWER_NOTE not in page, (code, [x for x in page.splitlines() if "viewer" in x][:4])) + + +def mutant_dir(work, name, old, new): + """A plugin folder holding bin/owngenome_live_viewer.py with one planted mistake (beside a link + to assets/, which the plugin finds from its own folder).""" + top = os.path.join(work, f"mutant-{name}") + d = os.path.join(top, "bin") + os.makedirs(d) + os.symlink(os.path.join(REPO, "assets"), os.path.join(top, "assets")) + src = open(os.path.join(REPO, "bin", "owngenome_live_viewer.py")).read() + if old not in src: + check(f"mutant '{name}': its line is in owngenome_live_viewer.py", False, old) + with open(os.path.join(d, "owngenome_live_viewer.py"), "w") as f: + f.write(src.replace(old, new)) + return d + + +def main(): + work = tempfile.mkdtemp(prefix="pgp-live-viewer-") + proc = None + try: + gd = f"{work}/genome" + tv.make_sample(gd) + s = f"{gd}/S" + os.makedirs(f"{s}/nextflow") # the site lists a folder with nextflow/ as a run + html, _ = tv.findings_report(work) + with open(f"{s}/S_report.html", "w") as f: + f.write(html) + state = f"{work}/state" + os.makedirs(state) + with open(f"{state}/password", "w") as f: + f.write(owngenome_live.hash_password(PASSWORD) + "\n") + + # 1. owngenome-live.sh, with a password + port, iport = tv.free_port(), tv.free_port() + env = dict(os.environ, GENOME_DIR=gd, REF_FASTA="reference/synthetic.fasta", OWNGENOME_LIVE_PORT=str(port), + OWNGENOME_LIVE_INGEST_PORT=str(iport), OWNGENOME_LIVE_STATE_DIR=state) + log = open(f"{work}/site.log", "w") + proc = subprocess.Popen(["bash", f"{REPO}/scripts/owngenome-live.sh", "--quiet"], env=env, + stdout=log, stderr=subprocess.STDOUT) + up = wait_port(port, proc) + check("owngenome-live.sh: the site starts", up, open(f"{work}/site.log").read()) + if not up: + raise SystemExit(1) + pin = tv.igv_pin() + igv = f"{gd}/tools/igv.js-{pin['IGV_JS_VERSION']}/igv.min.js" + import hashlib + check("owngenome-live.sh: the igv.js pinned in versions.env is downloaded, with its sha256", + os.path.isfile(igv) and hashlib.sha256(open(igv, "rb").read()).hexdigest() == pin["IGV_JS_SHA256"], + open(f"{work}/site.log").read()) + code, h, _ = request(port, "/runs/S/viewer/") + c2, _, _ = request(port, "/runs/S/viewer/config.json") + c3, _, _ = request(port, "/runs/S/viewer/data/2/S_sorted.bam", {"Range": "bytes=0-9"}) + check("no login: the page sends the browser to /login, the config and the reads are 401", + code == 303 and h.get("location") == "/login?next=/runs/S/viewer/" and c2 == 401 and c3 == 401, + (code, h.get("location"), c2, c3)) + auth = login(port) + check("login: the password opens a session", auth is not None) + auth = auth or {} + + # 2 and 3. the viewer and the served report + acc = f"{state}/access.log" + before = open(acc).read().count("\tviewer\t") if os.path.isfile(acc) else 0 + cfg = viewer_checks(port, auth, s) + report_checks(port, auth) + code, h, _ = request(port, "/runs/S/viewer", auth) + check("/runs/S/viewer redirects to /runs/S/viewer/", code == 301 and h.get("location") == "/runs/S/viewer/", + (code, h.get("location"))) + + # 4. the access log + time.sleep(0.5) + rows = [r for r in open(acc).read().splitlines() if "\tviewer\t" in r] + mine = rows[before:] + check("access log: one row for the page, none for config.json, igv.js or the byte ranges", + len(mine) == 2 and all("\t200" in r or "\t404" in r for r in mine), mine) + proc.terminate() + proc.wait(timeout=10) + proc = None + + # 5. a BAM symlinked out of GENOME_DIR + t = f"{gd}/T" + os.makedirs(f"{t}/aligned") + os.makedirs(f"{t}/nextflow") + shutil.copy(f"{s}/aligned/S_sorted.bam", f"{work}/outside.bam") + shutil.copy(f"{s}/aligned/S_sorted.bam.bai", f"{work}/outside.bam.bai") + os.symlink(f"{work}/outside.bam", f"{t}/aligned/T_sorted.bam") + os.symlink(f"{work}/outside.bam.bai", f"{t}/aligned/T_sorted.bam.bai") + flags = ["--root", gd, "--state-dir", f"{work}/state2", "--viewer-fasta", f"{gd}/reference/synthetic.fasta", + "--viewer-igv-js", igv] + srv = in_process(flags) + try: + p = srv[1].server_address[1] + import json + code, _, body = request(p, "/runs/T/viewer/config.json") + tracks = json.loads(body).get("tracks") if code == 200 else None + check("a BAM symlinked out of GENOME_DIR is left out of the config", code == 200 and tracks == [], + (code, body[:300])) + + # 7. headless Chrome, on the site without a password (loopback) + chrome = tv.find_chrome() + if not chrome: + ci = os.environ.get("CI") == "true" + print(("[FAIL] " if ci else "[SKIP] ") + "no Chrome or Chromium to load the page in") + if ci: + FAILED.append("no Chrome in CI") + else: + r = subprocess.run([chrome, "--headless=new", "--disable-gpu", "--no-sandbox", "--no-first-run", + "--no-default-browser-check", f"--user-data-dir={work}/chrome", + "--virtual-time-budget=20000", "--dump-dom", + f"http://127.0.0.1:{p}/runs/S/viewer/#chr1:451"], + capture_output=True, text=True, timeout=180) + status = re.search(r'data-pgp-status="([^"]*)"', r.stdout) + tracks = re.search(r'data-pgp-tracks="([^"]*)"', r.stdout) + blocked = re.search(r'data-pgp-blocked="([^"]*)"', r.stdout) + check("browser: the page from the site starts igv.js with the run's 3 tracks", + status and status.group(1) == "ready" and tracks and tracks.group(1) == "3", + (status and status.group(1), r.stderr[-800:], r.stdout[:600])) + check("browser: the policy blocked nothing the page needed", not blocked, blocked and blocked.group(1)) + finally: + stop(*srv) + + # 6. a site without the viewer flags + srv = in_process(["--root", gd, "--state-dir", f"{work}/state3"]) + try: + p = srv[1].server_address[1] + code, _, body = request(p, "/runs/S/viewer/") + _, _, rep = request(p, "/runs/S/report.html") + check("without the flags: the viewer is 404 with the reason, the report keeps its local links", + code == 404 and b"not set up" in body and render_report.VIEWER_URL.encode() in rep, (code, body)) + finally: + stop(*srv) + + # Planted mistakes: checks 2 and 3 must fail on them. + for name, old, new in ( + ("serves any file", " if dm and int(dm.group(1)) < len(files) and " + "os.path.basename(files[int(dm.group(1))]) == dm.group(2):", + " if rest and os.path.isfile(os.path.join(d, rest)):\n" + " return serve.send_file(req, head_writer(req), 'application/octet-stream', " + "os.path.join(d, rest))\n if dm and int(dm.group(1)) < len(files) and " + "os.path.basename(files[int(dm.group(1))]) == dm.group(2):"), + ("no report filter", " app.report_filter(point_links)\n", "")): + pd = mutant_dir(work, name.replace(" ", "-"), old, new) + srv = in_process(flags[:3] + [f"{work}/state-{name.replace(' ', '-')}"] + flags[4:], pd) + try: + p = srv[1].server_address[1] + quiet = [] + real, globals()["check"] = check, (lambda desc, ok, detail="": quiet.append(ok)) + try: + viewer_checks(p, {}, s) + report_checks(p, {}) + finally: + globals()["check"] = real + check(f"control: the checks fail on the planted mistake '{name}'", not all(quiet), quiet) + finally: + stop(*srv) + finally: + if proc and proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + shutil.rmtree(work, ignore_errors=True) + + print() + print("RESULT: ALL PASS" if not FAILED else f"RESULT: {len(FAILED)} FAILED") + sys.exit(1 if FAILED else 0) + + +if __name__ == "__main__": + main() diff --git a/tests/test_viewer.py b/tests/test_viewer.py index 39b4c924..1fdad22e 100644 --- a/tests/test_viewer.py +++ b/tests/test_viewer.py @@ -10,7 +10,7 @@ Checks: 1. build_config names the sample's files and the locus, and refuses a locus that is not chr[:pos[-end]] of the reference, and a BAM with no index; - 2. owngenome-view.sh downloads the pinned igv.js and checks its sha256, finds the + 2. owngenome-view.sh downloads the igv.js pinned in versions.env and checks its sha256, finds the BAM, VCF and SV VCF in the sample folder, and starts the server, which listens on 127.0.0.1 and not on the machine's other address; 3. the server sends the page with a Content-Security-Policy that keeps it @@ -20,12 +20,21 @@ 4. in a headless Chrome the page starts igv.js on that config (status "ready", nothing blocked by the policy) and igv.js reads the BAM and its index by range; skipped without Chrome, except in CI; - 5. the HTML report links each ClinVar hit, each mitochondrial variant and - each secondary-findings variant to the viewer at its position; a file - name with a space, '#' or '?' is served from its percent-encoded URL. + 5. every findings table of the HTML report (ClinVar hits, secondary + findings, mitochondrial variants, deep intronic sites, the cDNA scan, + AnnotSV class 4-5 and the calls moved to class 3, slivar's pairs in + trans, mosaic alterations) links the locus cell of each row to the + viewer at its position, with the note under it; a control with the links + removed must fail that check for every table. A file name with a space, + '#' or '?' is served from its percent-encoded URL. + +The igv.js pin is read from versions.env (IGV_JS_VERSION, IGV_JS_SHA256): +the downloaded file must have that sha256, and owngenome-view.sh must set no +pin of its own. """ import gzip +import hashlib import json import os import random @@ -163,6 +172,15 @@ def make_sample(gd): # --- helpers ------------------------------------------------------------------ +def igv_pin(): + """IGV_JS_VERSION and IGV_JS_SHA256 as versions.env sets them.""" + with open(f"{REPO}/versions.env") as f: + pin = dict(re.findall(r'^(IGV_JS_VERSION|IGV_JS_SHA256)="([^"]*)"', f.read(), re.M)) + if len(pin) != 2: + raise SystemExit(f"versions.env does not set IGV_JS_VERSION and IGV_JS_SHA256: {pin}") + return pin + + def free_port(): with socket.socket() as so: so.bind(("127.0.0.1", 0)) @@ -197,6 +215,89 @@ def find_chrome(): return None +# --- the report's findings tables --------------------------------------------- +# Each findings table of the HTML report, by the start of its header row, and +# the viewer address its one row (made by findings_report) must link to. +FINDINGS_TABLES = { + "ClinVar hits": ("ChrPosition", "chr1:451"), + "secondary findings": ("GeneVariantGenotypeEvidence", "chr2:120"), + "mitochondrial variants": ("VariantAFClinVar", "chrM:3243"), + "deep intronic sites": ("GeneVariantPositionCall", "chr1:900"), + "cDNA scan": ("LocusJunctionsReads", "chr21:100-900"), + "AnnotSV class 4-5": ("SVClassGenes", "chr1:1000-1500"), + "AnnotSV moved to class 3": ("SVAnnotSV class", "chr20:2000-2400"), + "slivar pairs in trans": ("GenePair in trans", "chr1:50-90"), + "mosaic alterations": ("TypeArmRegion", "chr7:1-5000000"), +} + + +def findings_report(work): + """The HTML report of a summary with one row in every findings table.""" + import collect_summary + import render_report + d = f"{work}/findings/S" + os.makedirs(d) + summ = collect_summary.collect("S", d) + S = summ["sections"] + for k in ("clinvar", "clinical", "mito", "annotsv", "slivar", "mosaic"): + S.setdefault(k, {"title": k, "step": "-", "data": {}}) + S[k]["state"] = "ok" + S["clinvar"]["data"] = { + "hits": [{"chrom": "chr1", "pos": 451, "ref": "T", "alt": "G", "genotype": "het", "gene": "GENEB", + "significance": "Pathogenic", "review_status": "reviewed by expert panel", "stars": 3}], + "deep_intronic": [{"gene": "GENEE", "hgvs": "NM_000001.1:c.100-500A>G", "chrom": "chr1", "pos": 900, + "ref": "A", "alt": "G", "call": "het", "detail": "GQ 40"}], + "cdna_scan": {"note": "", "loci": [{"chrom": "chr21", "start": 100, "end": 900, "junctions": 3, "reads": 24}]}} + S["clinical"]["data"] = {"variants": 1, "acmg_sf": { + "version": "v3.3", "high_impact": [], + "clinvar_hits": [{"gene": "GENEC", "variant": "chr2:120 A>G", "genotype": "0/1", "significance": "Pathogenic"}]}} + S["mito"]["data"] = {"pass": 1, "heteroplasmic": 0, "annotation": { + "source": "synthetic", "variants": [{"pos": 3243, "ref": "A", "alt": "G"}]}} + sv = {"type": "DEL", "genes": ["GENEF"], "caller": "manta", "support": 2, "support_callers": ["manta", "delly"], + "exon_edge": "no", "exon_edge_detail": "", "population_af": None, "population_source": "", "common": False, + "rule": "", "rule_detail": ""} + S["annotsv"]["data"] = { + "count": 1, "single_caller": 0, "consensus": True, "exon_edge": 0, "exon_edge_checked": True, "common": 0, + "rows": [dict(sv, sv="chr1:1000-1500", acmg_class="5", annotsv_class="5")], + "downgraded": [dict(sv, sv="chr20:2000-2400", acmg_class="3", annotsv_class="4", rule="ig_tr_locus", + rule_detail="inside a locus")]} + S["slivar"]["data"] = {"prioritized": 1, "compound_het_pairs": 1, + "compound_het_phase": {"trans": 1, "cis": 0, "unknown": 0}, + "compound_het_trans": [{"gene": "G2", "variants": "chr1:50 A>G + chr1:90 C>T", + "reads": "19 of 20"}]} + S["mosaic"]["data"] = {"status": "assessed", "calls_reported": "1", "floor": "0.05", "depth": "30", + "calls": [{"type": "Loss", "arms": "7q", "chrom": "chr7", "start": "1", "end": "5000000", + "length_mb": "5.0", "cell_fraction": "0.2"}]} + return render_report.html_report(summ), summ + + +def tables_without_links(html): + """[name] of each findings table missing from html, without the Viewer column last, with a row + whose last cell does not link to the viewer at the expected address, or without the viewer + note in its card.""" + import render_report + bad = [] + chunks = html.split("", 1)[0]] + if not mine: + bad.append(f"{name}: no table") + continue + table = mine[0].split("", 1)[0] + after = html[html.index(table) + len(table):] + if not table.split("", 1)[0].endswith(render_report.VIEWER_TH): + bad.append(f"{name}: the header does not end with the Viewer column") + rows = [r.rsplit("", 1)[0] for r in table.split("{where}') + if not rows or any(not re.search(r']*>[^<]+$', r) for r in rows) \ + or not any(r.endswith(link) for r in rows): + bad.append(f"{name}: a row whose last cell is not a viewer link to {where}") + if render_report.VIEWER_NOTE.strip() not in after.split('
100000, igv) + pin = igv_pin() + igv = f"{gd}/tools/igv.js-{pin['IGV_JS_VERSION']}/igv.min.js" + check("owngenome-view.sh: the igv.js pinned in versions.env is downloaded into GENOME_DIR/tools, with its sha256", + os.path.isfile(igv) and os.path.getsize(igv) > 100000 + and hashlib.sha256(open(igv, "rb").read()).hexdigest() == pin["IGV_JS_SHA256"], (igv, pin)) + check("owngenome-view.sh: it sets no igv.js pin of its own (versions.env is the one place)", + not re.search(r"^\s*IGV_JS_(VERSION|SHA256)=", open(f"{REPO}/scripts/owngenome-view.sh").read(), re.M)) ip = other_address() if ip: try: @@ -336,32 +442,29 @@ def main(): check("browser: igv.js read the reference and the VCF", "synthetic.fasta" in served and "S.vcf.gz" in served, served[-1500:]) - # 5. the report's links - import collect_summary + # 5. the report's links: a viewer link in the locus cell of every row of every findings table import render_report - d = f"{work}/report/S" - os.makedirs(f"{d}/clinvar") - with open(f"{d}/clinvar/S_clinvar_hits.vcf", "w") as f: - f.write("##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" - "chr1\t451\t2\tT\tG\t50\tPASS\tGENEINFO=GENEB:22;CLNSIG=Pathogenic;" - "CLNREVSTAT=reviewed_by_expert_panel\tGT\t0/1\n") - html = render_report.html_report(collect_summary.collect("S", d)) - check("report: the ClinVar hit links to the viewer at its position", - 'href="http://127.0.0.1:8765/#chr1:451"' in html and "scripts/owngenome-view.sh" in html, - [l for l in html.splitlines() if "ClinVar" in l][:5]) - summ = collect_summary.collect("S", d) - for k in ("mito", "clinical"): - summ["sections"][k]["state"] = "ok" - summ["sections"]["mito"]["data"]["annotation"] = { - "source": "synthetic", "variants": [{"pos": 3243, "ref": "A", "alt": "G"}]} - summ["sections"]["clinical"]["data"]["acmg_sf"] = { - "version": "v3.3", - "clinvar_hits": [{"gene": "GENEC", "variant": "chr2:120 A>G", "genotype": "0/1", "significance": "Pathogenic"}], - "high_impact": [{"gene": "GENED", "variant": "chr1:2000 C>T", "genotype": "0/1", "consequence": "stop_gained"}]} - html = render_report.html_report(summ) - want = [f'href="http://127.0.0.1:8765/#{x}"' for x in ("chrM:3243", "chr2:120", "chr1:2000")] - check("report: the mito and secondary-findings variants link to the viewer at their positions", - all(w in html for w in want), [l for l in html.splitlines() if "8765" in l][:5]) + html, _ = findings_report(work) + bad = tables_without_links(html) + check("report: every row of every findings table ends with a Viewer cell that links to its locus, with the note in the card", + not bad, bad) + check("report: the note says how to start the viewer", "scripts/owngenome-view.sh" in render_report.VIEWER_NOTE) + # The control: with plain text in place of the link, the same check names every table. + real = render_report.viewer_td + render_report.viewer_td = lambda chrom, start, end=None: "" + try: + plain, _ = findings_report(f"{work}/control") + finally: + render_report.viewer_td = real + missed = tables_without_links(plain) + check("report: control: with the links removed the check names every findings table", + len(missed) == len(FINDINGS_TABLES) and all("not a viewer link" in m for m in missed), missed) + cell = render_report.viewer_td_of + check("report: a row's Viewer cell spans the positions on its first chromosome, and is empty without one", + cell("chr1:50 A>G + chr1:90 C>T").endswith('#chr1:50-90" target="_blank" rel="noopener noreferrer">' + "chr1:50-90") + and "#chrM:3243\"" in cell("chrM:3243 A>G") and cell("no locus ") == "", + [cell("chr1:50 A>G + chr1:90 C>T"), cell("chrM:3243 A>G"), cell("no locus ")]) finally: if proc and proc.poll() is None: proc.terminate() diff --git a/versions.env b/versions.env index 7ce1069e..7476ff09 100644 --- a/versions.env +++ b/versions.env @@ -163,6 +163,14 @@ EIGENSOFT_IMAGE="quay.io/biocontainers/eigensoft:9.0.0--h519d32d_0" # optional: VCFANNO_IMAGE="quay.io/biocontainers/vcfanno:0.3.9--h1079eea_0" SLIVAR_IMAGE="quay.io/biocontainers/slivar:0.3.4--hb56abc1_0" +# --- Genome viewer (scripts/owngenome-view.sh and the live site) --- +# igv.js (MIT licence, https://github.com/igvteam/igv.js) runs in the browser, +# so it has no image: both scripts download dist/igv.min.js of this npm +# release once into GENOME_DIR/tools/igv.js-/ and check it against +# this sha256. Change the two together. +IGV_JS_VERSION="3.8.10" +IGV_JS_SHA256="87b991ae2a720978172e269dd5127c495e56df5af31144dff19c83124e2f5744" + # --- Data versions coupled to an image (change them together) --- # VEP cache release for step 13: must equal the major of VEP_IMAGE. VEP_CACHE_RELEASE="116" From f3189dff5963048e8264ad976c8d6d0f3a290ba3 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 4 Nov 2026 16:10:04 +0100 Subject: [PATCH 3/3] fix(viewer): Exomiser rows get the Viewer column; the live viewer drops tracks with an index out of GENOME_DIR and serves quoted names The live viewer compared the still-quoted request name with the file name, so a reference named with a space, '#' or '?' answered 404. It now unquotes it. An index symlinked out of GENOME_DIR was served; now its track, file and index are left out. Each new check has a planted mistake that must fail it. --- assets/viewer/serve.py | 7 ++-- bin/owngenome_live_viewer.py | 24 +++++++++++--- bin/render_report.py | 7 ++-- docs/24-html-report.md | 2 +- docs/live-site.md | 4 +-- docs/viewer.md | 4 +-- tests/test_live_viewer.py | 64 ++++++++++++++++++++++++++---------- tests/test_viewer.py | 16 ++++++--- 8 files changed, 90 insertions(+), 38 deletions(-) diff --git a/assets/viewer/serve.py b/assets/viewer/serve.py index af47ac48..435c9dae 100755 --- a/assets/viewer/serve.py +++ b/assets/viewer/serve.py @@ -78,9 +78,10 @@ def index_of(names): def sample_files(sample_dir, sample): """(reads, vcf, sv_vcf) of a sample folder, by the names the steps write, - None for one that is not there: the files owngenome-view.sh picks when the - environment names none, and the ones the live site's viewer - (bin/owngenome_live_viewer.py) serves.""" + None for one that is not there: the default aligned/ and vcf/ folders, as + owngenome-view.sh picks them with none of READS, VCF, SV_VCF, ALIGN_DIR or + VCF_DIR set, and the ones the live site's viewer (bin/owngenome_live_viewer.py) + serves.""" def first(*names): return next((p for p in (os.path.join(sample_dir, n) for n in names) if os.path.isfile(p)), None) return (first(f"aligned/{sample}_sorted.bam", f"aligned/{sample}_sorted.cram"), diff --git a/bin/owngenome_live_viewer.py b/bin/owngenome_live_viewer.py index e00d94c5..7edbf1ba 100644 --- a/bin/owngenome_live_viewer.py +++ b/bin/owngenome_live_viewer.py @@ -13,8 +13,8 @@ The files are the ones owngenome-view.sh picks by default (assets/viewer/serve.py sample_files): the run folder's BAM or CRAM, its VCF -and its SV VCF, each with its index, and the reference. A file outside -GENOME_DIR (a symlink out of it) is left out. The page and the files carry the +and its SV VCF, each with its index, and the reference. A track whose file +or index lies outside GENOME_DIR (a symlink out of it) is left out. The page and the files carry the viewer's Content-Security-Policy (serve.CSP): the page talks to this site only. The report the site serves (report.html) gets its viewer links pointed here. Opening the page writes one access-log row; the byte ranges igv.js reads @@ -85,7 +85,19 @@ def inside(p): real = os.path.realpath(p) return all(real.startswith(t.rstrip(os.sep) + os.sep) for t in (top, root)) reads, vcf, sv_vcf = (p if p and inside(p) else None for p in serve.sample_files(run_dir, run)) - return serve.build_config(run, flag("viewer_fasta"), reads, vcf, sv_vcf) + cfg, files = serve.build_config(run, flag("viewer_fasta"), reads, vcf, sv_vcf) + # An index beside a track can be a symlink out of GENOME_DIR too: then the track goes, with + # its file and its index. Files 0 and 1 are the reference and its .fai (--viewer-fasta, any folder). + files = [f if n < 2 or inside(f) else None for n, f in enumerate(files)] + + def nums(t): + return [int(t[k].split("/")[1]) for k in ("url", "indexURL") if k in t] + for t in cfg["tracks"]: + if not all(files[n] for n in nums(t)): + for n in nums(t): + files[n] = None + cfg["tracks"] = [t for t in cfg["tracks"] if all(files[n] for n in nums(t))] + return cfg, files def head_writer(req): @@ -141,8 +153,10 @@ def viewer(req, m): if rest == "config.json": return send(req, 200, "application/json", json.dumps(cfg, indent=1)) dm = DATA_RE.fullmatch(rest) - if dm and int(dm.group(1)) < len(files) and os.path.basename(files[int(dm.group(1))]) == dm.group(2): - return serve.send_file(req, head_writer(req), "application/octet-stream", files[int(dm.group(1))]) + # The config's URLs quote the name (serve.build_config); the path is still quoted here. + n = int(dm.group(1)) if dm else len(files) + if n < len(files) and files[n] and os.path.basename(files[n]) == urllib.parse.unquote(dm.group(2)): + return serve.send_file(req, head_writer(req), "application/octet-stream", files[n]) except ValueError as e: req.log("viewer", run, status=409) return req.deny(409, f"the genome viewer cannot open this run: {e}") diff --git a/bin/render_report.py b/bin/render_report.py index 3e0f9a9a..da299c86 100755 --- a/bin/render_report.py +++ b/bin/render_report.py @@ -625,12 +625,13 @@ def exomiser_html(d, sample): f"{E(g['phenotype_score'])}") if i == 0 else "" * 5 rows.append(f" {lead}{E(v['variant'])} {E(v['genotype'])}
{E(v['consequence'])} {E(v['hgvs'])}{E(v['acmg'])}" - f"{E(v['disease'])}") + f"{E(v['disease'])}{viewer_td_of(v['variant'])}") body = [stat("Symptoms searched", E(exomiser_terms(d)))] if rows: body += [" ", " " - ""] - body += rows + ["
RankGeneInheritanceScoreSymptom matchVariantACMG (Exomiser)Linked disease
"] + "Symptom matchVariantACMG (Exomiser)Linked disease" + f"{VIEWER_TH}"] + body += rows + [" ", VIEWER_NOTE] else: body.append(stat("Ranked genes", "none passed Exomiser's filters")) body += [small_note(EXOMISER_NOTE), small_note(f"Exomiser's own report: exomiser/{sample}_exomiser.html")] diff --git a/docs/24-html-report.md b/docs/24-html-report.md index 19d0ec2d..5efddee2 100644 --- a/docs/24-html-report.md +++ b/docs/24-html-report.md @@ -67,7 +67,7 @@ After them, "What this report leaves out" lists the steps not run and what no st - **PRS** (`#prs`): the percentiles on a 0 to 100 strip, only when pgsc_calc had an ancestry panel. A score with no percentile, or one of the other sex, says "no percentile" and gets no dot. Without a panel there is no strip: a raw score has no scale, and the caption already says so. - **Links to the step's own files**, relative to the report, when the summary says they exist: the PharmCAT HTML report, the CPIC recommendations per gene, the CPSR HTML report (or the `cpsr/` folder that holds it) and the Exomiser ranking table. The report sits in the sample folder beside each step's folder, in the bash layout and in a Nextflow `--outdir`. MultiQC and the couple report are not linked: the summary does not record them, and the couple report sits in the family folder. - **Nothing is fetched.** A Content-Security-Policy (`default-src 'none'; img-src data:; style-src 'unsafe-inline'`) lets the browser load nothing but the file itself and its inline pictures, and there is no JavaScript. The only addresses off this computer are the haplogroup links (FamilyTreeDNA, YFull, the docs) and the project link in the footer, each marked "opens a website"; the FamilyTreeDNA and YFull ones add that the address carries the haplogroup. The genome viewer links go to `127.0.0.1`, this computer. -- **A Viewer column on every findings table.** The ClinVar hits, the secondary-findings genes, the mitochondrial variants, the known deep intronic sites, the cDNA scan, the AnnotSV class 4 and 5 SVs and the calls moved to class 3, slivar's pairs in trans and the mosaic alterations each end with a Viewer column: one link per row, to that row's position (a range for an SV, a region or a pair). The link opens the reads there in the [genome viewer](viewer.md) once `scripts/owngenome-view.sh ` runs; a note under the table says so. The column is new, so every other cell keeps its text; it replaces the single line of links that sat under three of these tables. A row with no position has an empty Viewer cell. On the [live site](live-site.md) the served report points the same links at the site's own viewer of the run. +- **A Viewer column on every findings table.** The ClinVar hits, the secondary-findings genes, the mitochondrial variants, the known deep intronic sites, the cDNA scan, the AnnotSV class 4 and 5 SVs and the calls moved to class 3, slivar's pairs in trans, the mosaic alterations and Exomiser's ranked variants each end with a Viewer column: one link per row, to that row's position (a range for an SV, a region or a pair). The link opens the reads there in the [genome viewer](viewer.md) once `scripts/owngenome-view.sh ` runs; a note under the table says so. The column is new, so every other cell keeps its text; it replaces the single line of links that sat under three of these tables. A row with no position has an empty Viewer cell. On the [live site](live-site.md) the served report points the same links at the site's own viewer of the run. - **Genes and rsIDs in these results.** The closing list is built by `bin/results_index.py`. It holds every gene symbol and rsID that a section in state complete or stale names in `summary.json`, by the same rule `scripts/owngenome-lookup.sh` uses for "in the reports" (`names_of` in `bin/genome_lookup.py`). It adds the genes of the clinical filter (step 23) and variant prioritization (step 31), whose cards only count them: from their summary tables on the bash path, from the VCFs on the Nextflow path. The two give the same genes when a variant's most severe consequence is on the transcript the report ranks first (MANE, else canonical); the step 31 table keeps the gene of the most severe consequence, so where an overlapping gene's transcript carries the worse one, the two forms name different genes. A ClinVar hit's rsIDs come from step 13's VEP output (`Existing_variation`), because the hits file's ID column holds ClinVar's own id; without step 13 a hit has none. The Nextflow `HTML_REPORT` process does not get step 13's VEP output, so its index has no ClinVar rsIDs; step 24, which `run-all.sh` runs over the output folder afterwards, adds them. A failed, missing or unreadable section adds nothing. The browser's find (Ctrl+F) searches the list; there is still no script. A position is not in it: ask `scripts/owngenome-lookup.sh` for a genotype ([genotype lookup](lookup.md)). - **Dark on screen, light on paper.** - **Print one section.** Open a section from the contents (or its "link to this section"), then print: only that section prints, with the header (sample, generation date, pipeline commit, ClinVar file date, ClinVar in CPSR's bundle, HLA database), the list of what is not assessed and the disclaimer. A chapter prints the same way from its title in the contents. Without an anchor in the address the whole report prints, as it does in a browser without CSS `:has()`. diff --git a/docs/live-site.md b/docs/live-site.md index d4dc4ab1..e70190c0 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -41,7 +41,7 @@ weblog { `bin/owngenome_live_viewer.py` serves it, behind the same password as every page. `owngenome-live.sh` passes it the reference (`REF_FASTA`, as in every step) and igv.js, which the first start downloads into `GENOME_DIR/tools/` and checks against `IGV_JS_SHA256` in `versions.env`. Without the reference's `.fai` or igv.js the site starts without the viewer and says why; `OWNGENOME_LIVE_VIEWER=off` leaves it out on purpose. Running `owngenome_live.py` yourself (in Docker, say), pass `--viewer-fasta` and `--viewer-igv-js`. -Per run it serves the folder's BAM or CRAM, its VCF and its SV VCF, each with its index, and the reference: the files `owngenome-view.sh` picks by default. A file that is a symlink out of `GENOME_DIR` is left out. The page and the files carry the viewer's Content-Security-Policy, so the page talks to this site only, and igv.js's calls to outside services are blocked. +Per run it serves the folder's BAM or CRAM, its VCF and its SV VCF, each with its index, and the reference: the files `owngenome-view.sh` picks by default. A track whose file or index is a symlink out of `GENOME_DIR` is left out. The page and the files carry the viewer's Content-Security-Policy, so the page talks to this site only, and igv.js's calls to outside services are blocked. The reads cross the network to the browser that logged in: only the byte ranges igv.js asks for, but they are the sample's reads. Off loopback, use TLS (`--tls-cert` and `--tls-key`, or a proxy that ends TLS); without it the password and the reads cross the network in clear. Opening the viewer writes one access-log row; the byte ranges igv.js reads write none. @@ -52,7 +52,7 @@ The reads cross the network to the browser that logged in: only the byte ranges - The site serves only its own pages, the run's state, the report, the files the report links to by a relative link, and the genome viewer's files of the run (see above). Any other file is 404, and so is a link that leaves the run folder or a symlink out of `GENOME_DIR`. It never lists a folder. - Each message is cut to what a progress view needs before it is stored: the task, its process and status, times, CPU and memory figures. Nextflow's parameters, command line and task scripts are dropped. Three paths stay on the server for later views (the output folder, the CPU and memory caps, the work directory); none of them reaches a page. - Messages go to `/runs//events.jsonl` (state dir: `GENOME_DIR/.owngenome-live`, or `OWNGENOME_LIVE_STATE_DIR`). The site writes nowhere else, so `GENOME_DIR` can be mounted read-only. -- The access log, `/access.log`, has one tab-separated row per page or file read and per login: time, client, the first 8 characters of a hash of the session, action, run, file, status. It never holds a query string, a request body, a result or the password. It moves to `access.log.1` past `--access-log-max-mb` (10). +- The access log, `/access.log`, has one tab-separated row per page or file read and per login (the genome viewer's files excepted: one row per page open, see above): time, client, the first 8 characters of a hash of the session, action, run, file, status. It never holds a query string, a request body, a result or the password. It moves to `access.log.1` past `--access-log-max-mb` (10). - Every page carries a Content-Security-Policy that lets it talk to this server only, and the page loads no script from anywhere else. The run's own HTML (the report, CPSR, PharmCAT, MultiQC) is sent sandboxed: no script runs and nothing outside is fetched. ## The password and the limits diff --git a/docs/viewer.md b/docs/viewer.md index 24672799..5c517055 100644 --- a/docs/viewer.md +++ b/docs/viewer.md @@ -9,7 +9,7 @@ - its small variants (the VCF from step 3), - its structural variants (step 22's consensus, else Manta's VCF from step 4). -It opens at the locus you give. Every findings table of the HTML report (step 24) ends with a Viewer column: the ClinVar hits, the secondary-findings genes, the mitochondrial variants, the deep intronic sites, the cDNA scan, the AnnotSV class 4 and 5 SVs, slivar's pairs in trans and the mosaic alterations. With the viewer running, a click on a row's link opens the reads at that row. +It opens at the locus you give. Every findings table of the HTML report (step 24) ends with a Viewer column: the ClinVar hits, the secondary-findings genes, the mitochondrial variants, the deep intronic sites, the cDNA scan, the AnnotSV class 4 and 5 SVs, slivar's pairs in trans, the mosaic alterations and Exomiser's ranked variants. With the viewer running, a click on a row's link opens the reads at that row. The [live site](live-site.md) serves the same viewer for each run, behind its password, so the report's links work from any device that can reach the site (see "From the live site" below). @@ -80,7 +80,7 @@ Then open `http://127.0.0.1:8765/` on your computer. The report's links work the ## From the live site -`scripts/owngenome-live.sh` starts the [live site](live-site.md) with the viewer: each run has it at `/runs//viewer/`, behind the site's password, and the report the site serves points its Viewer links there. `bin/owngenome_live_viewer.py` does it with the same page, the same Content-Security-Policy and the same files as `owngenome-view.sh` picks by default (the run folder's BAM or CRAM, its VCF and its SV VCF, each with its index, and `REF_FASTA`). The `READS`, `VCF` and `SV_VCF` overrides do not apply there, and a file that is a symlink out of `GENOME_DIR` is left out. The first start downloads igv.js the same way. `OWNGENOME_LIVE_VIEWER=off` starts the site without it. +`scripts/owngenome-live.sh` starts the [live site](live-site.md) with the viewer: each run has it at `/runs//viewer/`, behind the site's password, and the report the site serves points its Viewer links there. `bin/owngenome_live_viewer.py` does it with the same page, the same Content-Security-Policy and the same files as `owngenome-view.sh` picks by default (the run folder's BAM or CRAM, its VCF and its SV VCF, each with its index, and `REF_FASTA`). The `READS`, `VCF`, `SV_VCF`, `ALIGN_DIR` and `VCF_DIR` overrides do not apply there, and a track whose file or index is a symlink out of `GENOME_DIR` is left out. The first start downloads igv.js the same way. `OWNGENOME_LIVE_VIEWER=off` starts the site without it. This differs from the local viewer in one way that matters: the reads now cross the network, to whatever browser logged in. Only the byte ranges igv.js asks for travel, and only after the login, but they are the sample's reads. Off loopback, use TLS (`--tls-cert` and `--tls-key`, or a proxy that ends TLS), or keep the site on loopback and forward it over ssh. Opening the viewer writes one row in the site's access log; the byte ranges write none. diff --git a/tests/test_live_viewer.py b/tests/test_live_viewer.py index cfad547b..2521c816 100644 --- a/tests/test_live_viewer.py +++ b/tests/test_live_viewer.py @@ -16,14 +16,19 @@ with the site's note, not to the local viewer on 127.0.0.1:8765; 4. opening the page writes one access-log row, and igv.js's byte ranges write none; - 5. a run whose BAM is a symlink out of GENOME_DIR gets no reads track; + 5. a run whose BAM is a symlink out of GENOME_DIR gets no reads track, a + run whose VCF index is one gets no VCF track and neither file is + served, and a reference named with a space, '#' and '?' is served from + the encoded URL its config gives; 6. a site started without the viewer flags answers the viewer with 404 and the reason, and leaves the report's links as they are; 7. in a headless Chrome (skipped without Chrome, except in CI) the page, loaded from the site, starts igv.js (status ready, 3 tracks) and the policy blocks nothing. -Checks 2 and 3 are also run against planted mistakes (a viewer that serves -any file of the run folder, a report filter that does nothing) and must fail. +Checks 2, 3 and 5 are also run against planted mistakes (a viewer that serves +any file of the run folder, a report filter that does nothing, a viewer that +compares the still-quoted name, one that serves an index out of GENOME_DIR) +and must fail. Run: python3 tests/test_live_viewer.py """ @@ -146,6 +151,25 @@ def report_checks(port, auth, label=""): and render_report.VIEWER_NOTE not in page, (code, [x for x in page.splitlines() if "viewer" in x][:4])) +def outside_checks(port, auth, label=""): + """Checks 5: files out of GENOME_DIR are left out; an odd reference name is served.""" + import json + code, _, body = request(port, "/runs/T/viewer/config.json", auth) + tracks = json.loads(body).get("tracks") if code == 200 else None + check(f"{label}a BAM symlinked out of GENOME_DIR is left out of the config", code == 200 and tracks == [], + (code, body[:300])) + code, _, body = request(port, "/runs/U/viewer/config.json", auth) + cfg = json.loads(body) if code == 200 else {} + c2, _, _ = request(port, "/runs/U/viewer/data/2/U.vcf.gz", dict(auth, Range="bytes=0-9")) + c3, _, _ = request(port, "/runs/U/viewer/data/3/U.vcf.gz.tbi", dict(auth, Range="bytes=0-9")) + check(f"{label}a VCF whose index is symlinked out of GENOME_DIR: no track, neither file served", + code == 200 and cfg.get("tracks") == [] and c2 == 404 and c3 == 404, (code, cfg, c2, c3)) + url = cfg.get("reference", {}).get("fastaURL", "") + code, _, body = request(port, f"/runs/U/viewer/{url}", dict(auth, Range="bytes=0-4")) + check(f"{label}a reference named with a space, '#' or '?' is served from its encoded URL", + url == "data/0/my%20ref%231%3F.fasta" and code == 206 and body == b">chr1", (url, code, body)) + + def mutant_dir(work, name, old, new): """A plugin folder holding bin/owngenome_live_viewer.py with one planted mistake (beside a link to assets/, which the plugin finds from its own folder).""" @@ -223,7 +247,7 @@ def main(): proc.wait(timeout=10) proc = None - # 5. a BAM symlinked out of GENOME_DIR + # 5. a BAM, and a VCF index, symlinked out of GENOME_DIR; an odd reference name t = f"{gd}/T" os.makedirs(f"{t}/aligned") os.makedirs(f"{t}/nextflow") @@ -231,16 +255,20 @@ def main(): shutil.copy(f"{s}/aligned/S_sorted.bam.bai", f"{work}/outside.bam.bai") os.symlink(f"{work}/outside.bam", f"{t}/aligned/T_sorted.bam") os.symlink(f"{work}/outside.bam.bai", f"{t}/aligned/T_sorted.bam.bai") - flags = ["--root", gd, "--state-dir", f"{work}/state2", "--viewer-fasta", f"{gd}/reference/synthetic.fasta", - "--viewer-igv-js", igv] + u = f"{gd}/U" + os.makedirs(f"{u}/vcf") + os.makedirs(f"{u}/nextflow") + shutil.copy(f"{s}/vcf/S.vcf.gz", f"{u}/vcf/U.vcf.gz") + shutil.copy(f"{s}/vcf/S.vcf.gz.tbi", f"{work}/outside.vcf.gz.tbi") + os.symlink(f"{work}/outside.vcf.gz.tbi", f"{u}/vcf/U.vcf.gz.tbi") + odd = f"{gd}/reference/my ref#1?.fasta" + shutil.copy(f"{gd}/reference/synthetic.fasta", odd) + shutil.copy(f"{gd}/reference/synthetic.fasta.fai", odd + ".fai") + flags = ["--root", gd, "--state-dir", f"{work}/state2", "--viewer-fasta", odd, "--viewer-igv-js", igv] srv = in_process(flags) try: p = srv[1].server_address[1] - import json - code, _, body = request(p, "/runs/T/viewer/config.json") - tracks = json.loads(body).get("tracks") if code == 200 else None - check("a BAM symlinked out of GENOME_DIR is left out of the config", code == 200 and tracks == [], - (code, body[:300])) + outside_checks(p, {}) # 7. headless Chrome, on the site without a password (loopback) chrome = tv.find_chrome() @@ -276,15 +304,16 @@ def main(): finally: stop(*srv) - # Planted mistakes: checks 2 and 3 must fail on them. + # Planted mistakes: checks 2, 3 and 5 must fail on them. for name, old, new in ( - ("serves any file", " if dm and int(dm.group(1)) < len(files) and " - "os.path.basename(files[int(dm.group(1))]) == dm.group(2):", + ("serves any file", " n = int(dm.group(1)) if dm else len(files)\n", " if rest and os.path.isfile(os.path.join(d, rest)):\n" " return serve.send_file(req, head_writer(req), 'application/octet-stream', " - "os.path.join(d, rest))\n if dm and int(dm.group(1)) < len(files) and " - "os.path.basename(files[int(dm.group(1))]) == dm.group(2):"), - ("no report filter", " app.report_filter(point_links)\n", "")): + "os.path.join(d, rest))\n n = int(dm.group(1)) if dm else len(files)\n"), + ("no report filter", " app.report_filter(point_links)\n", ""), + ("quoted name", "== urllib.parse.unquote(dm.group(2)):", "== dm.group(2):"), + ("index out served", " files = [f if n < 2 or inside(f) else None for n, f in enumerate(files)]\n", + "")): pd = mutant_dir(work, name.replace(" ", "-"), old, new) srv = in_process(flags[:3] + [f"{work}/state-{name.replace(' ', '-')}"] + flags[4:], pd) try: @@ -294,6 +323,7 @@ def main(): try: viewer_checks(p, {}, s) report_checks(p, {}) + outside_checks(p, {}) finally: globals()["check"] = real check(f"control: the checks fail on the planted mistake '{name}'", not all(quiet), quiet) diff --git a/tests/test_viewer.py b/tests/test_viewer.py index 1fdad22e..c0f56853 100644 --- a/tests/test_viewer.py +++ b/tests/test_viewer.py @@ -23,9 +23,10 @@ 5. every findings table of the HTML report (ClinVar hits, secondary findings, mitochondrial variants, deep intronic sites, the cDNA scan, AnnotSV class 4-5 and the calls moved to class 3, slivar's pairs in - trans, mosaic alterations) links the locus cell of each row to the - viewer at its position, with the note under it; a control with the links - removed must fail that check for every table. A file name with a space, + trans, mosaic alterations, Exomiser's ranked variants) ends each row + with a Viewer cell linking to the row's position, with the note under + it; a control with the links removed must fail that check for every + table. A file name with a space, '#' or '?' is served from its percent-encoded URL. The igv.js pin is read from versions.env (IGV_JS_VERSION, IGV_JS_SHA256): @@ -228,6 +229,7 @@ def find_chrome(): "AnnotSV moved to class 3": ("SVAnnotSV class", "chr20:2000-2400"), "slivar pairs in trans": ("GenePair in trans", "chr1:50-90"), "mosaic alterations": ("TypeArmRegion", "chr7:1-5000000"), + "Exomiser ranking": ("RankGeneInheritance", "chr3:700"), } @@ -239,7 +241,7 @@ def findings_report(work): os.makedirs(d) summ = collect_summary.collect("S", d) S = summ["sections"] - for k in ("clinvar", "clinical", "mito", "annotsv", "slivar", "mosaic"): + for k in ("clinvar", "clinical", "mito", "annotsv", "slivar", "mosaic", "exomiser"): S.setdefault(k, {"title": k, "step": "-", "data": {}}) S[k]["state"] = "ok" S["clinvar"]["data"] = { @@ -268,6 +270,10 @@ def findings_report(work): S["mosaic"]["data"] = {"status": "assessed", "calls_reported": "1", "floor": "0.05", "depth": "30", "calls": [{"type": "Loss", "arms": "7q", "chrom": "chr7", "start": "1", "end": "5000000", "length_mb": "5.0", "cell_fraction": "0.2"}]} + S["exomiser"]["data"] = {"terms": [{"id": "HP:0001250", "label": "Seizure"}], "genes": [ + {"rank": "1", "gene": "GENEG", "moi": "AD", "exomiser_score": "0.9", "phenotype_score": "0.8", + "variants": [{"variant": "chr3:700 C>T", "genotype": "0/1", "consequence": "missense_variant", + "hgvs": "p.(Arg1Cys)", "acmg": "LIKELY_PATHOGENIC", "disease": "a disease"}]}]} return render_report.html_report(summ), summ @@ -442,7 +448,7 @@ def main(): check("browser: igv.js read the reference and the VCF", "synthetic.fasta" in served and "S.vcf.gz" in served, served[-1500:]) - # 5. the report's links: a viewer link in the locus cell of every row of every findings table + # 5. the report's links: a Viewer cell at the end of every row of every findings table import render_report html, _ = findings_report(work) bad = tables_without_links(html)