diff --git a/assets/live/family-page.js b/assets/live/family-page.js index 2856ead5..d9b07a3a 100644 --- a/assets/live/family-page.js +++ b/assets/live/family-page.js @@ -12,6 +12,7 @@ if (!U) { return; } var el = U.el, add = U.add, root = document.documentElement; var RESULT = { shared: 'Both carry', x_linked: 'X-linked', one_partner: 'One partner' }; + var STALE = ' (older than its input)'; // A chapter: its roman number in brass, its title in the serif. var CHAPTER = { 'Pedigree': 'I', 'Who is related how': 'II', 'Couple carrier screen': 'III', 'Each child': 'IV', @@ -342,11 +343,14 @@ function children(f) { var s = section('Each child', 'children'); if (!f.state.children.length) { add(s, el('p', { 'class': 'dim' }, 'The pedigree names no child.')); } - var plan = {}; - (f.state.plan || []).forEach(function (r) { plan[r.analysis] = r; }); - function absent(file, analysis) { - var r = plan[analysis]; - return file + ' is not here' + (r ? ': ' + r.analysis.replace(/_/g, ' ') + ' ' + r.decision + (r.reason ? ' (' + r.reason + ')' : '') : '') + '.'; + // Whether step 38 assessed this child (family.json trio_status): "None listed" only for a child it assessed; + // otherwise why not, and a file that links out of the family folder is named as one, never followed. + function empty(child, key, file) { + var st = ((f.trio_status || {})[child] || {})[key]; + if (st && st.assessed) { return 'Assessed (' + st.why + '): none listed in ' + file + ' for this child.'; } + if (st && st.assessed === null) { return 'Not known whether assessed for this child: ' + st.why + '.'; } + var why = st ? st.why : file + ' is not here'; + return 'Not assessed for this child: ' + why + '.'; } f.state.children.forEach(function (c, i) { var d = el('details', { 'data-child': c.child }); @@ -370,8 +374,7 @@ rows: dn.map(function (r) { return [r.chrom + ':' + r.pos + ' ' + r.ref + '>' + r.alt, r.child_gt, r.father_gt, r.mother_gt, r.child_dp + ' / ' + r.child_gq, r.child_ab, r.quality]; }) }, 8)); } else { - add(d, el('p', { 'class': 'dim' }, (f.trio_files || {})['de_novo.tsv'] ? 'None listed in de_novo.tsv for this child.' - : absent('de_novo.tsv', 'trio_phasing'))); + add(d, el('p', { 'class': 'dim', 'data-trio': 'de_novo' }, empty(c.child, 'de_novo', 'de_novo.tsv'))); } var ch = (f.compound_het || []).filter(function (r) { return r.child === c.child; }); add(d, el('h3', {}, 'Compound-het pairs')); @@ -380,8 +383,7 @@ add(d, U.table({ columns: ['Gene', 'Phase', 'Variant A', 'From', 'Variant B', 'From'], rows: ch.map(function (r) { return [r.gene, r.phase, r.variant_a, r.a_from, r.variant_b, r.b_from]; }) }, 8)); } else { - add(d, el('p', { 'class': 'dim' }, (f.trio_files || {})['compound_het.tsv'] ? 'None listed in compound_het.tsv for this child.' - : absent('compound_het.tsv', 'trio_phasing'))); + add(d, el('p', { 'class': 'dim', 'data-trio': 'compound_het' }, empty(c.child, 'compound_het', 'compound_het.tsv'))); } add(s, d); }); @@ -395,12 +397,21 @@ if (!sb || !sb.rows.length) { add(s, el('p', { 'class': 'dim' }, 'No member has a summary yet, or no summary lists these results.')); return s; } add(s, el('p', { 'class': 'dim' }, 'Each member’s own report, one column per person: CYP2D6, the CPIC genes ' + '(phenotypes in CPIC’s words), HLA, the haplogroups and the blood groups.')); - var group = null, rows = []; + // A value from a section older than its input (state "stale") says so in its own cell. + var group = null, rows = [], old = 0; sb.rows.forEach(function (r) { - rows.push([r.group === group ? '' : r.group, r.label].concat(r.values)); + rows.push([r.group === group ? '' : r.group, r.label].concat(r.values.map(function (v, i) { + if (!(r.stale || [])[i]) { return v; } + old++; + return v + STALE; + }))); group = r.group; }); add(s, U.table({ columns: ['', 'Gene or system'].concat(sb.members), rows: rows }, 14)); + if (old) { + add(s, el('p', { 'class': 'dim', 'data-stale': String(old) }, 'Marked' + STALE + ': the member’s result file is from an ' + + 'earlier run, and the latest run did not write it again; that member’s page says why.')); + } return s; } diff --git a/bin/owngenome_live_family.py b/bin/owngenome_live_family.py index 40afb3d1..fa23dabc 100644 --- a/bin/owngenome_live_family.py +++ b/bin/owngenome_live_family.py @@ -18,10 +18,14 @@ couple's gene rows, each child's inherited stretches (_segments.tsv), the de novo and compound-het candidates (de_novo.tsv, - compound_het.tsv), each member's run progress and - attention count, and the members side by side: - CYP2D6, the CPIC genes, HLA, the haplogroups and - the blood groups (from each summary.json only) + compound_het.tsv) with whether each child was + assessed and why not (trio_status; a file linking + out of the folder is "rejected" in trio_file_state), + each member's run progress and attention count, + and the members side by side: CYP2D6, the CPIC + genes, HLA, the haplogroups and every blood group + system, each value with its stale flag (from each + summary.json only) state.json gets plugins.family: {"kind": "family", ...} on a family's page, {"kind": "member", "families": [...]} on the page of a sample in a family, @@ -235,14 +239,127 @@ def child_depth(d, order, members): rows = read_tsv(os.path.join(d, "%s_segments.tsv" % s)) if rows: segs[s] = [{k: r.get(k) for k in SEGMENT_KEYS} for r in rows] + files = {f: file_state(d, f) for f in TRIO_FILES} comp = [{k: r.get(k) for k in COMP_KEYS} for r in read_tsv(os.path.join(d, "compound_het.tsv"))] comp.sort(key=lambda r: PHASE_ORDER.get(r.get("phase"), 1)) return {"segments": segs, "de_novo": [{k: r.get(k) for k in DE_NOVO_KEYS} for r in read_tsv(os.path.join(d, "de_novo.tsv"))], "compound_het": comp, # read_tsv's own test: a link out of the family folder counts as not there - "trio_files": {f: os.path.isfile(os.path.join(d, f)) and inside(os.path.join(d, f), os.path.abspath(d)) - for f in ("de_novo.tsv", "compound_het.tsv")}} + "trio_files": {f: files[f] == "here" for f in TRIO_FILES}, + "trio_file_state": files, + "trio_status": trio_status(d, order, members, files)} + + +TRIO_FILES = ("de_novo.tsv", "compound_het.tsv") + + +def file_state(d, f): + """'here', 'rejected' (a link out of the family folder: read_tsv does not read it, and the page never + shows where it points) or 'missing'.""" + p = os.path.join(d, f) + if not os.path.lexists(p): + return "missing" + if not inside(p, os.path.abspath(d)): + return "rejected" + return "here" if os.path.isfile(p) else "missing" + + +def trio_status(d, order, members, files): + """Per child, whether step 38 assessed it for de novo candidates and compound-het pairs, and why not: + {child: {"de_novo": {"assessed": bool, "why": text}, "compound_het": {...}}}. The rule is step 38's + (bin/family_report.py): de novo candidates need both parents sequenced, compound-het pairs a sequenced + parent; a file that is not here, or is a link out of the folder, means no child was assessed.""" + plan = {r.get("analysis"): r for r in read_tsv(os.path.join(d, "family_plan.tsv"))} + trio = plan.get("trio_phasing") or {} + + def absent(f): + if files[f] == "rejected": + return "%s is a link out of the family folder, so this page does not read it" % f + why = "%s is not here" % f + if trio.get("decision"): + why += ": trio phasing %s%s" % (trio["decision"], " (%s)" % trio["reason"] if trio.get("reason") else "") + return why + + joint = plan.get("joint_call") or {} + called = set((joint.get("members") or "").split(",")) if joint.get("decision") == "runs" else None + + def seq(p): + return bool(p) and p in members and bool(members[p]["sequenced"]) and (called is None or p in called) + + need = "sequenced" if called is None else "sequenced and in the joint call" + + def not_called(p): + if not members[p]["sequenced"]: + return "%s is not sequenced" % p + return "%s is not in the joint call (family_plan.tsv)" % p + + def gaps(m): + return "; ".join(("no %s is named" % w) if not m[w] else "%s (the %s) is not" % (m[w], w) + for w in ("father", "mother") if not seq(m[w])) + + out = {} + for s in order: + m = members[s] + if not (m["father"] or m["mother"]): + continue + st = {} + if files["de_novo.tsv"] != "here": + st["de_novo"] = {"assessed": False, "why": absent("de_novo.tsv")} + elif not seq(s): + st["de_novo"] = {"assessed": False, "why": not_called(s)} + elif seq(m["father"]) and seq(m["mother"]): + st["de_novo"] = {"assessed": True, "why": "both parents are sequenced"} + else: + st["de_novo"] = {"assessed": False, "why": "de novo candidates need both parents %s: %s" % (need, gaps(m))} + if files["compound_het.tsv"] != "here": + st["compound_het"] = {"assessed": False, "why": absent("compound_het.tsv")} + elif not seq(s): + st["compound_het"] = {"assessed": False, "why": not_called(s)} + elif seq(m["father"]) or seq(m["mother"]): + st["compound_het"] = {"assessed": True, "why": "a parent is sequenced; pairs come only from the child's " + "clinical-filter candidates (step 23), so with none given no pair is listed"} + else: + st["compound_het"] = {"assessed": False, "why": "compound-het pairs need a parent %s" % need} + out[s] = st + return out + + +def candidates_given(live, d, sample, since=None): + """Whether the clinical filter (step 23) wrote 's candidates, the only variants step 38 pairs: + "here", "absent" (the run is here without them; a file linking out of the run counts as absent), + "newer" (written after since, compound_het.tsv's mtime, so step 38 may not have had them) or None + (the run is not on this site: the page cannot tell).""" + if not d: + return None + p = os.path.join(d, "clinical", "%s_clinical.vcf.gz" % sample) + if not (os.path.isfile(p) and live.inside(p, os.path.realpath(d)) and live.inside(p, live.root)): + return "absent" + return "newer" if since is not None and os.path.getmtime(p) > since else "here" + + +def with_candidates(status, given): + """trio_status with the compound-het line told by the child's step 23 candidates ({child: one of + candidates_given's answers}). assessed is None where the page cannot tell.""" + for child, st in status.items(): + c = st.get("compound_het") or {} + if not c.get("assessed"): + continue + g = given.get(child) + if g == "here": + c["why"] = ("a parent is sequenced and the child's clinical-filter candidates (step 23) are here; " + "pairs come only from them") + elif g == "absent": + st["compound_het"] = {"assessed": False, "why": "the clinical filter (step 23) wrote no candidates for " + "%s (clinical/%s_clinical.vcf.gz), so step 38 had no variant to pair" % (child, child)} + elif g == "newer": + st["compound_het"] = {"assessed": None, "why": "%s's clinical-filter candidates (step 23) are newer than " + "compound_het.tsv, so step 38 may not have paired them" % child} + else: + st["compound_het"] = {"assessed": None, "why": "pairs come only from the child's clinical-filter " + "candidates (step 23), and %s's run is not on this site to tell whether step 23 " + "wrote them" % child} + return status def _sec(summary, key): @@ -250,54 +367,81 @@ def _sec(summary, key): return sec.get("data") or {} if sec.get("state") in ("ok", "stale") else None +def _stale(summary, *keys): + secs = (summary or {}).get("sections") or {} + return any(((secs.get(k) or {}).get("state")) == "stale" for k in keys) + + def side_values(summary): - """[(group, label, value)] of one member's summary.json, as its report prints them; nothing computed.""" + """[(group, label, value, stale)] of one member's summary.json, as its report prints them; nothing + computed. stale is true when the value comes from a section older than its input (state "stale").""" out = [] c = (summary or {}).get("cyp2d6") or {} calls = ", ".join("%s %s" % (k, v) for k, v in (c.get("calls") or {}).items() if v) if calls: agree = c.get("agree") - out.append(("Medicines", "CYP2D6 (callers)", calls + (" (callers agree)" if agree else " (callers differ)" if agree is False - else ""))) + # summary.json's cyp2d6 block reads pypgx and Cyrius only when complete; its PharmCAT call is cpic's + out.append(("Medicines", "CYP2D6 (callers)", calls + (" (callers agree)" if agree else " (callers differ)" + if agree is False else ""), + bool((c.get("calls") or {}).get("PharmCAT")) and _stale(summary, "cpic"))) cpic = _sec(summary, "cpic") for g in (cpic or {}).get("genes") or []: ph = g.get("phenotype") - out.append(("Medicines", g.get("gene"), "%s%s" % (g.get("diplotype") or ".", ' "%s"' % ph if ph else ""))) + out.append(("Medicines", g.get("gene"), "%s%s" % (g.get("diplotype") or ".", ' "%s"' % ph if ph else ""), + _stale(summary, "cpic"))) for l in (_sec(summary, "hla") or {}).get("loci") or []: - out.append(("HLA", l.get("gene"), " ".join(l.get("alleles") or []) or "not typed")) + out.append(("HLA", l.get("gene"), " ".join(l.get("alleles") or []) or "not typed", _stale(summary, "hla"))) mt = _sec(summary, "haplogroup") if mt is not None: - out.append(("Haplogroups", "Mitochondrial", mt.get("haplogroup") or ".")) + out.append(("Haplogroups", "Mitochondrial", mt.get("haplogroup") or ".", _stale(summary, "haplogroup"))) + ykey = "y_tree" if _sec(summary, "y_tree") else "y_haplogroup" y = _sec(summary, "y_tree") or _sec(summary, "y_haplogroup") if y is not None: - out.append(("Haplogroups", "Y chromosome", y.get("haplogroup") or ".")) + out.append(("Haplogroups", "Y chromosome", y.get("haplogroup") or ".", _stale(summary, ykey))) bt = _sec(summary, "bloodtype") if bt is not None: - out.append(("Blood groups", "ABO", bt.get("abo") or next((x.get("phenotype") for x in bt.get("systems") or [] - if x.get("system") == "ABO"), "."))) - out.append(("Blood groups", "RhD", bt.get("rhd") or next((x.get("phenotype") for x in bt.get("systems") or [] - if x.get("system") == "RhD"), "."))) + old = _stale(summary, "bloodtype") + systems = [x for x in bt.get("systems") or [] if x.get("system")] + by = {x["system"]: x.get("phenotype") or "." for x in systems} + out.append(("Blood groups", "ABO", bt.get("abo") or by.get("ABO", "."), old)) + out.append(("Blood groups", "RhD", bt.get("rhd") or by.get("RhD", "."), old)) + # every other system step 41 typed, in its order (Kell, Duffy, Kidd, MNS...) + for x in systems: + if x["system"] not in ("ABO", "RhD"): + out.append(("Blood groups", x["system"], by[x["system"]], old)) return out def side_by_side(names, summaries): - """The members' results in one table: {members, rows: [{group, label, values}]}. A member with no - summary.json yet says so in its column; a row a member's summary does not have is '.'.""" + """The members' results in one table: {members, rows: [{group, label, values, stale}]}. A member with + no summary.json yet says so in its column; a row a member's summary does not have is '.'. stale holds, + per member, whether that value comes from a section older than its input.""" per = {n: side_values(summaries.get(n)) for n in names} - rows, seen = [], {} + # Each member's rows keep their own order (step 41's for the blood groups): a row first seen in a later + # member goes right after the row before it in that member's list, not at the end. + keys = [] for n in names: - for g, label, _ in per[n]: - if (g, label) not in seen: - seen[(g, label)] = len(rows) - rows.append({"group": g, "label": label, "values": []}) + prev = None + for g, label, _, _ in per[n]: + if (g, label) not in keys: + keys.insert(keys.index(prev) + 1 if prev else len(keys), (g, label)) + prev = (g, label) groups = ["Medicines", "HLA", "Haplogroups", "Blood groups"] - rows.sort(key=lambda r: (groups.index(r["group"]) if r["group"] in groups else 9, seen[(r["group"], r["label"])])) + import bloodtype_report # step 41's own order of the systems, for systems two members each lack one of + step41 = {s: i for i, (s, _) in enumerate(bloodtype_report.SYSTEMS)} + pos = {k: i for i, k in enumerate(keys)} + keys.sort(key=lambda k: (groups.index(k[0]) if k[0] in groups else 9, + step41.get(k[1], len(step41)) if k[0] == "Blood groups" else 0, pos[k])) + rows = [{"group": g, "label": label, "values": [], "stale": []} for g, label in keys] for r in rows: for n in names: if summaries.get(n) is None: r["values"].append("no summary yet") + r["stale"].append(False) else: - r["values"].append(next((v for g, lab, v in per[n] if (g, lab) == (r["group"], r["label"])), ".")) + hit = next((x for x in per[n] if (x[0], x[1]) == (r["group"], r["label"])), None) + r["values"].append(hit[2] if hit else ".") + r["stale"].append(bool(hit and hit[3])) return {"members": list(names), "rows": rows} @@ -368,6 +512,14 @@ def page_data(req, m): "planned": len(planned), "done": sum(1 for s in planned if s["state"] == "done"), "attention": n, "results": mtime is not None} out["runs"] = runs + given = {} + comp_p = os.path.join(d, "compound_het.tsv") + since = os.path.getmtime(comp_p) if os.path.isfile(comp_p) else None + for c in out.get("trio_status") or {}: + r = live.run(c) if live else None + given[c] = (candidates_given(live, r["dir"] or os.path.join(live.root, r["name"]), c, since) + if r and r.get("kind") != "family" else None) + with_candidates(out.get("trio_status") or {}, given) # The members side by side, from each sequenced member's summary.json alone (read as the home # page's attention count reads it: never a file linking out of its run). seq = [p["sample"] for p in out["state"].get("members") or [] if p.get("sequenced")] diff --git a/docs/live-site.md b/docs/live-site.md index 95f63f81..ec23f65b 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -70,7 +70,7 @@ The header search finds a gene or rsID across the results and opens the card tha **A finding** (`/runs//finding//`). One finding on a page of its own: its level and how many others share it, the title, the gene and variant, a plain explanation, and **What to do** as numbered steps. Then the evidence (its numbers, how common the change is on a log scale from one in a million to everyone, the caveat in the margin), the reads with a link to the genome viewer, the rest of the family, the related findings of the same gene or step, and the section of the report it comes from. The contents on the left lead back to the report and on to the next finding. The page is 404 unless the id is a finding of that run's `results.json` now. -**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first. V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and blood groups, one column each. On a phone the grid becomes a list of pairs. +**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. On a phone the grid becomes a list of pairs. On a phone each page is one column and each table row becomes a short list of label and value. On a person's report the contents give way to a tab bar at the bottom (To act, Medicines, Family, Trust, More); under 700 px the report is short: the findings of sections 1 to 4 (to act on, medicines, family, worth a look) stay open, while the cards behind them, every information card and the bodies of sections 5 to 10 start closed, each behind one line that says what it holds (section 5 stays open when it holds a finding about data quality). The counts read one line per level, and the "How to read this" note and the links between findings are left out. A link to a card, a section or a finding, a search hit and a tab open the folds around it; a finding page drops its contents (the header links back to the report), and a family page keeps its chapter list above the chapters. The pages are dark. `tests/demo/render_pictures.py --out DIR --only live-phone` draws the picture below from the demo. diff --git a/docs/multi-sample.md b/docs/multi-sample.md index ab8c44a7..8f12ccf2 100644 --- a/docs/multi-sample.md +++ b/docs/multi-sample.md @@ -119,8 +119,8 @@ The live site (`bin/owngenome_live.py`) lists each family folder under `GENOME_D - **I, the pedigree and members.** `bin/pedigree_svg.py` draws it from `members.tsv`, or from `pedigree.tsv` when step 38 has not run, each time the page asks. A square is a male, a circle a female, a diamond a sex not known; a filled symbol is a sequenced person (with only `pedigree.tsv`, a person with a row of their own). A line joins partners, and the children hang from it, one generation per row. It draws what the file declares; the relationship check below is what tests it. Beside it, the members and the analyses step 38 planned (`family_plan.tsv`). - **II, who is related how**: the relationship check (`relationships.tsv`) and the shared segments (`ibd_pairs.tsv`) as one grid of every pair, with ⚑ where the measured relationship differs from the declared one (a list of pairs on a phone). - **III, the couple carrier screen**, per couple: how many genes `couple__.tsv` lists as `shared`, as `x_linked` and as carried by one partner, each gene both carry with its chance per pregnancy (drawn as four children when it is the carrier pattern), the whole table, and a link to `couple__.md`. A count of 0 is not an all-clear; the report says what the screen does not see. -- **IV, each child**: the stretches of `_segments.tsv` painted per chromosome, the copy from the father above the copy from the mother, coloured by grandparent where step 38 could tell, else by parent; then the child's de novo candidates (`de_novo.tsv`) and compound-het pairs (`compound_het.tsv`, in trans first). Without `_segments.tsv` the page shows `_segments.svg`, or a line saying step 38 has not drawn one; a missing de novo or compound-het file is named with the reason, never shown as an empty list. -- **V, the members side by side**: each sequenced member's CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and blood groups from their `summary.json`, one column each; `.` where a member's summary has no such row, "no summary yet" for a member without one. +- **IV, each child**: the stretches of `_segments.tsv` painted per chromosome, the copy from the father above the copy from the mother, coloured by grandparent where step 38 could tell, else by parent; then the child's de novo candidates (`de_novo.tsv`) and compound-het pairs (`compound_het.tsv`, in trans first). Without `_segments.tsv` the page shows `_segments.svg`, or a line saying step 38 has not drawn one. For each child the page says whether step 38 assessed it: "none listed" only for a child it assessed (de novo candidates need both parents sequenced and in the joint call, compound-het pairs one such parent), with the reason. Compound-het pairs come only from the child's step 23 candidates: a child whose run is on the site without `clinical/_clinical.vcf.gz` was not assessed. When that file is newer than `compound_het.tsv`, or the child's run is not on the site, the page says it cannot tell whether the child was assessed, and why. Otherwise the page says why not. A missing de novo or compound-het file is named with the reason from `family_plan.tsv`, and a file that is a link out of the family folder is named as one, without showing where it points. +- **V, the members side by side**: each sequenced member's CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system step 41 typed (ABO and RhD first, then Kell, Duffy, Kidd and the rest in step 41's order) from their `summary.json`, one column each; `.` where a member's summary has no such row, "no summary yet" for a member without one. A value from a section older than its input (state stale in `summary.json`) is marked "(older than its input)" in its cell. A person's page gets a Family card (`assets/live/plugins/family.js`) with a link to each family the person is in. The plugin reads the files the steps wrote and nothing else: it calls no tool and computes no genetics. It serves only the pedigree it draws and the result files named above, at `/family//`, behind the site's login; any other file of the folder, such as `family.vcf.gz`, is 404. `tests/test_pedigree_svg.py` and `tests/test_owngenome_live_family.py` check the drawing and the family page on an invented family (`tests/fixtures/families_live/`), with a swapped parent as the control that must fail; `tests/test_live_family_depth.py` checks the painting, the de novo and compound-het tables and the side by side on the demo's invented trio (`tests/fixtures/live/demo/DEMO-FAMILY/`). diff --git a/tests/test_live_family_depth.py b/tests/test_live_family_depth.py index 4ab1576d..afd08e46 100644 --- a/tests/test_live_family_depth.py +++ b/tests/test_live_family_depth.py @@ -12,10 +12,16 @@ column per member, '.' where a member's summary lacks the row, "no summary yet" for a member without one; CPIC's phenotype quoted, never bare; 5. a member's summary.json that links out of its run folder is never read; - 6. family.json passes the wording gate (live_results.FORBIDDEN). + 6. family.json passes the wording gate (live_results.FORBIDDEN); + 7. a value from a stale section is flagged stale in its own cell, and only that one; + 8. every blood group system step 41 typed gets a row (ABO and RhD first, then the rest in order); + 9. each child's trio status: assessed with both parents sequenced and the file here; not assessed + with the reason when a parent is not sequenced or not in the joint call; a missing file and a + file linking out of the folder told apart, the link's target never written. Each check runs again on a planted mistake and must fail there: the segments read from the wrong child (1), the pairs left in file order (2), a trio file planted (3), a member's summary dropped (4), -the inside check bypassed (5), "ok" planted in a row (6). A check that cannot fail is no check. +the inside check bypassed (5), "ok" planted in a row (6), the stale section made complete (7), the +systems list emptied (8), the link replaced by a plain file (9). A check that cannot fail is no check. Run: python3 tests/test_live_family_depth.py """ @@ -95,6 +101,55 @@ def side_problems(sb): return out +def stale_summary(state): + S = summary("*1/*1", [("DPYD", "Reference/Reference", "Normal Metabolizer")], "H1") + S["sections"]["bloodtype"] = {"state": state, "data": { + "abo": "A1", "rhd": "D+", "systems": [{"system": "ABO", "phenotype": "A1"}, {"system": "RhD", "phenotype": "D+"}, + {"system": "Kell", "phenotype": "K-k+"}, + {"system": "Duffy", "phenotype": "Fy(a+b-)"}]}} + return S + + +def source_summary(key, state): + """Every side-by-side source present and complete, with PharmCAT's CYP2D6 call, the Y haplogroup from + y_haplogroup alone (the fallback), and one section, key, in the given state.""" + S = summary("*1/*1", [("DPYD", "Reference/Reference", "Normal Metabolizer")], "H1") + S["cyp2d6"]["calls"]["PharmCAT"] = "*1/*1" + S["sections"]["hla"] = {"state": "ok", "data": {"loci": [{"gene": "HLA-A", "alleles": ["A*01:01", "A*02:01"]}]}} + S["sections"]["y_haplogroup"] = {"state": "ok", "data": {"haplogroup": "R-M269"}} + S["sections"][key]["state"] = state + return S + + +def stale_problems(sb): + """Member A's blood groups come from a stale section; B's from a complete one; C has no summary.""" + out = [] + for r in sb["rows"]: + want = [r["group"] == "Blood groups", False, False] + if r.get("stale") != want: + out.append(f"{r['label']}: stale {r.get('stale')} != {want}") + return out or ([] if sb["rows"] else ["no rows"]) + + +def blood_problems(sb): + labels = [r["label"] for r in sb["rows"] if r["group"] == "Blood groups"] + want = ["ABO", "RhD", "Kell", "Duffy"] + vals = {r["label"]: r["values"][0] for r in sb["rows"] if r["group"] == "Blood groups"} + out = [] if labels == want else [f"blood rows {labels} != {want}"] + if vals.get("Kell") != "K-k+" or vals.get("Duffy") != "Fy(a+b-)": + out.append(f"values {vals}") + return out + + +def trio_problems(st, want): + out = [] + for child, key, assessed, words in want: + got = (st.get(child) or {}).get(key) or {} + if got.get("assessed") is not assessed or words not in (got.get("why") or ""): + out.append(f"{child} {key}: {got} (want assessed={assessed}, '{words}')") + return out + + def main(): work = tempfile.mkdtemp(prefix="test_live_family_depth.") try: @@ -154,6 +209,131 @@ class Open(Live): planted = copy.deepcopy(page) planted["side_by_side"]["rows"][0]["values"][0] = "ok" control("an ok in a side-by-side row", not live_results.wording_problems(planted)) + + sums7 = {"A": stale_summary("stale"), "B": summary("*1/*1", [], "U5"), "C": None} + sb7 = fam.side_by_side(["A", "B", "C"], sums7) + bad = stale_problems(sb7) + check("7. a stale section's values are flagged stale in their own cells, and no other value is", not bad, bad) + control("the stale section made complete", + not stale_problems(fam.side_by_side(["A", "B", "C"], dict(sums7, A=stale_summary("ok"))))) + + bad = blood_problems(sb7) + check("8. every blood group system typed gets a row: ABO, RhD, then Kell and Duffy in order", not bad, bad) + emptied = stale_summary("stale") + emptied["sections"]["bloodtype"]["data"]["systems"] = [] + control("the systems list emptied", not blood_problems(fam.side_by_side(["A", "B", "C"], dict(sums7, A=emptied)))) + + flags = {"cpic": {"CYP2D6 (callers)", "DPYD"}, "hla": {"HLA-A"}, "haplogroup": {"Mitochondrial"}, + "y_haplogroup": {"Y chromosome"}} + bad = [] + for key in flags: + sb = fam.side_by_side(["A"], {"A": source_summary(key, "stale")}) + got = {r["label"] for r in sb["rows"] if r["stale"][0]} + if got != flags[key]: + bad.append(f"{key} stale flags {sorted(got)} != {sorted(flags[key])}") + check("7. each stale source flags its own cells only: CPIC (and PharmCAT's CYP2D6 call), HLA, the " + "haplogroups (the Y fallback too)", not bad, bad) + sb = fam.side_by_side(["A"], {"A": source_summary("cpic", "ok")}) + control("the CPIC section made complete", {r["label"] for r in sb["rows"] if r["stale"][0]} == flags["cpic"]) + + short = stale_summary("ok") + short["sections"]["bloodtype"]["data"]["systems"] = [x for x in short["sections"]["bloodtype"]["data"]["systems"] + if x["system"] != "Kell"] + merged = lambda a, b_: [r["label"] for r in fam.side_by_side(["A", "B"], {"A": a, "B": b_})["rows"] # noqa: E731 + if r["group"] == "Blood groups"] + got = merged(short, stale_summary("ok")) + kidd = lambda s, drop: [x for x in s + [{"system": "Kidd", "phenotype": "Jk(a+b+)"}] # noqa: E731 + if x["system"] != drop] + a_, b2 = stale_summary("ok"), stale_summary("ok") + a_["sections"]["bloodtype"]["data"]["systems"] = kidd(a_["sections"]["bloodtype"]["data"]["systems"], "Duffy") + b2["sections"]["bloodtype"]["data"]["systems"] = kidd(b2["sections"]["bloodtype"]["data"]["systems"], "Kell") + got2 = merged(a_, b2) + check("8. a system only a later member has keeps step 41's order, also when each lacks one", + got == ["ABO", "RhD", "Kell", "Duffy"] and got2 == ["ABO", "RhD", "Kell", "Duffy", "Kidd"], (got, got2)) + import bloodtype_report + saved, bloodtype_report.SYSTEMS = bloodtype_report.SYSTEMS, [] + try: + control("step 41's order taken away", merged(a_, b2) == ["ABO", "RhD", "Kell", "Duffy", "Kidd"]) + finally: + bloodtype_report.SYSTEMS = saved + + tri = os.path.join(work, "trio") + shutil.copytree(DEMO, tri) + with open(os.path.join(tri, "members.tsv"), "a") as f: + f.write("DEMO-004\tfemale\tDEMO-001\tDEMO-005\t\tyes\tyes\nDEMO-005\tfemale\t\t\t\tyes\tno\n") + plan_p = os.path.join(tri, "family_plan.tsv") + with open(plan_p) as f: + plan_text = f.read() + with open(plan_p, "w") as f: + f.write(plan_text.replace("DEMO-001,DEMO-002,DEMO-003\t", "DEMO-001,DEMO-002,DEMO-003,DEMO-004\t")) + secret = os.path.join(work, "elsewhere-private-name.tsv") + shutil.copy(os.path.join(DEMO, "compound_het.tsv"), secret) + os.remove(os.path.join(tri, "compound_het.tsv")) + os.symlink(secret, os.path.join(tri, "compound_het.tsv")) + want = [("DEMO-003", "de_novo", True, "both parents"), + ("DEMO-004", "de_novo", False, "DEMO-005 (the mother) is not"), + ("DEMO-003", "compound_het", False, "link out of the family folder"), + ("DEMO-004", "compound_het", False, "link out of the family folder")] + tp = fam.family_page("trio", tri) + bad = trio_problems(tp["trio_status"], want) + if tp["trio_file_state"] != {"de_novo.tsv": "here", "compound_het.tsv": "rejected"}: + bad.append(f"file state {tp['trio_file_state']}") + if "elsewhere-private-name" in json.dumps(tp): + bad.append("the link's target is in family.json") + check("9. trio status per child: assessed, a parent not sequenced, a file linking out refused", not bad, bad) + with open(plan_p, "w") as f: + f.write(plan_text) + out_of_call = trio_problems(fam.family_page("trio", tri)["trio_status"], + [("DEMO-004", "de_novo", False, "DEMO-004 is not in the joint call")]) + check("9. a child left out of the joint call is not assessed, and the page says so", not out_of_call, out_of_call) + os.remove(os.path.join(tri, "de_novo.tsv")) + miss = trio_problems(fam.family_page("trio", tri)["trio_status"], + [("DEMO-003", "de_novo", False, "de_novo.tsv is not here")]) + check("9. a missing de_novo.tsv is not assessed, said apart from a refused one", not miss, miss) + os.remove(os.path.join(tri, "compound_het.tsv")) + shutil.copy(secret, os.path.join(tri, "compound_het.tsv")) + control("the link replaced by a plain file", not trio_problems(fam.family_page("trio", tri)["trio_status"], want[2:])) + shutil.copy(os.path.join(DEMO, "de_novo.tsv"), tri) + with open(plan_p, "w") as f: + f.write(plan_text.replace("DEMO-001,DEMO-002,DEMO-003\t", "DEMO-002,DEMO-003,DEMO-004\t")) + st9 = fam.family_page("trio", tri)["trio_status"] + bad = trio_problems(st9, [("DEMO-003", "de_novo", False, "DEMO-001 (the father) is not"), + ("DEMO-003", "compound_het", True, "a parent is sequenced"), + ("DEMO-004", "de_novo", False, "DEMO-001 (the father) is not"), + ("DEMO-004", "compound_het", False, "need a parent sequenced and in the joint call")]) + check("9. a sequenced parent left out of the joint call: no de novo, and no pairs when it was the only one", + not bad, bad) + with open(plan_p, "w") as f: + f.write(plan_text) + control("the parent back in the joint call", not trio_problems( + fam.family_page("trio", tri)["trio_status"], [("DEMO-003", "de_novo", False, "DEMO-001 (the father) is not")])) + + st10 = fam.with_candidates(copy.deepcopy(st9), {"DEMO-003": "absent", "DEMO-004": "here"}) + bad = trio_problems(st10, [("DEMO-003", "compound_het", False, "wrote no candidates for DEMO-003"), + ("DEMO-004", "compound_het", False, "need a parent")]) + st10 = fam.with_candidates(copy.deepcopy(st9), {"DEMO-003": "here"}) + bad += trio_problems(st10, [("DEMO-003", "compound_het", True, "candidates (step 23) are here")]) + for g, words in (("newer", "newer than compound_het.tsv"), (None, "run is not on this site")): + st10 = fam.with_candidates(copy.deepcopy(st9), {"DEMO-003": g}) + bad += trio_problems(st10, [("DEMO-003", "compound_het", None, words)]) + check("10. compound-het pairs by the child's step 23 candidates: here assessed, absent not, newer or no run " + "not known", not bad, bad) + control("the candidates not known read as absent", + not trio_problems(fam.with_candidates(copy.deepcopy(st9), {"DEMO-003": None}), + [("DEMO-003", "compound_het", False, "wrote no candidates")])) + run3 = os.path.join(work, "root", "DEMO-003") + os.makedirs(os.path.join(run3, "clinical")) + vcf = os.path.join(run3, "clinical", "DEMO-003_clinical.vcf.gz") + none_yet = fam.candidates_given(Live, run3, "DEMO-003") + open(vcf, "w").close() + os.utime(vcf, (1000, 1000)) + here, newer = fam.candidates_given(Live, run3, "DEMO-003", 2000), fam.candidates_given(Live, run3, "DEMO-003", 500) + os.remove(vcf) + os.symlink(outside, vcf) + got = (none_yet, here, newer, fam.candidates_given(Live, run3, "DEMO-003"), fam.candidates_given(Live, None, "DEMO-003")) + check("10. candidates: none in the run, here, newer than compound_het.tsv, a link out of the run, no run", + got == ("absent", "here", "newer", "absent", None), got) + control("the inside check bypassed for the candidates", fam.candidates_given(Open, run3, "DEMO-003") == "absent") finally: shutil.rmtree(work, ignore_errors=True) if FAILS: diff --git a/tests/test_owngenome_live_family.py b/tests/test_owngenome_live_family.py index d38d8b10..d0421505 100755 --- a/tests/test_owngenome_live_family.py +++ b/tests/test_owngenome_live_family.py @@ -20,7 +20,11 @@ the real server: the run list, state.json's plugins.family, the route behind the login, and a family page that loads the script; in a headless Chrome (skipped without Chrome, except in CI) the card, the pedigree and a - child's map load and the page's policy blocks nothing. + child's map load and the page's policy blocks nothing; then a 1-in-4 shared + row draws the four children and a 1-in-2 row gives its chance in words with no + drawing, a missing trio file and one linking out of the folder are worded + apart (the link's target never shown), and the side by side lists every + blood group system with a stale value marked in its cell. Checks 2 and 4 are run again on a copy of the fixture with two parents swapped and must fail there: a check that cannot fail is no check. @@ -343,6 +347,7 @@ def browser_checks(port, work): check("browser: child_a's chromosome map loads", seg == "/family/family_a/child_a_segments.svg", seg) blocked = b.eval("document.documentElement.dataset.pgpBlocked || ''") check("browser: the policy blocked nothing on the family page", not blocked, blocked) + depth_checks(b, base, os.path.join(work, "genome")) b.go(base + "/runs/child_a/") href = b.wait("var a = document.querySelector('section[data-plugin=family] a'); a && a.getAttribute('href')") check("browser: child_a's page links to its family", href == "/runs/family_a/", href) @@ -350,5 +355,85 @@ def browser_checks(port, work): b.close() +# A stale blood-group section with two systems past ABO and RhD, for child_a's summary.json. +STALE_SUMMARY = {"sections": {"bloodtype": {"state": "stale", "data": { + "abo": "A1", "rhd": "D+", "systems": [{"system": "ABO", "phenotype": "A1"}, {"system": "RhD", "phenotype": "D+"}, + {"system": "Kell", "phenotype": "K-k+"}, {"system": "Duffy", "phenotype": "Fy(a+b-)"}]}}, + "haplogroup": {"state": "ok", "data": {"haplogroup": "H1"}}}} + + +def depth_checks(b, base, root): + """The couple screen's risk drawing, a child's trio files (one missing, one linking out of the folder) and the + side by side (a stale value, every blood group system), on the family page in the browser.""" + fam_dir = os.path.join(root, "family_a") + tsv = os.path.join(fam_dir, "couple_father_a_mother_a.tsv") + with open(tsv) as f: + text = f.read() + text = text.replace("\tshared\t25%\t", "\tshared\t1 in 4 per pregnancy\t") + text += ("GENE4\tAR\tACMG\tinvented condition four\tshared\t1 in 2 per pregnancy\tfather_a\tinvented finding" + "\tmother_a\tinvented finding\tinvented\n") + with open(tsv, "w") as f: + f.write(text) + secret = os.path.join(os.path.dirname(root), "outside-folder-name.tsv") + with open(secret, "w") as f: + f.write("child\tgene\tvariant_a\tvariant_b\ta_from\tb_from\tphase\n") + os.symlink(secret, os.path.join(fam_dir, "compound_het.tsv")) + with open(os.path.join(root, "child_a", "summary.json"), "w") as f: + json.dump(STALE_SUMMARY, f) + clin = os.path.join(root, "child_a", "clinical") + try: + b.go(base + "/runs/family_a/") + ready = b.wait("document.documentElement.dataset.pgpStatus === 'ready' && " + "!!document.querySelector('div[data-shared=GENE4]')") + four = b.eval("var k = document.querySelector('div[data-shared=GENE1] .kids4 .grid4'); k ? k.children.length : -1") + check("browser: a 1-in-4 shared row draws the four children", ready and four == 4, (ready, four)) + two = b.eval("var d = document.querySelector('div[data-shared=GENE4]'); d ? " + "[!!d.querySelector('.kids4'), d.textContent].join('|') : ''") + check("browser: a 1-in-2 shared row gives the chance in words, without the drawing", + two.startswith("false|") and "1 in 2 per pregnancy" in two, two) + dn = b.eval("var p = document.querySelector('details[data-child=child_a] p[data-trio=de_novo]'); p && p.textContent") + ch = b.eval("var p = document.querySelector('details[data-child=child_a] p[data-trio=compound_het]'); " + "p && p.textContent") + page = b.eval("document.body.textContent") or "" + check("browser: a missing trio file and one linking out of the folder are told apart, the link's target unshown", + dn and "Not assessed" in dn and "de_novo.tsv is not here" in dn and ch and "link out of the family folder" + in ch and "outside-folder-name" not in page, (dn, ch)) + rows = b.eval("Array.from(document.querySelectorAll('#ch-side-by-side tbody tr')).map(function (r) { " + "return Array.from(r.cells).map(function (c) { return c.textContent; }).join('|'); }).join('\\n')") or "" + kell = next((r for r in rows.split("\n") if "|Kell|" in r), "") + hap = next((r for r in rows.split("\n") if "|Mitochondrial|" in r), "") + legend = b.eval("var p = document.querySelector('#ch-side-by-side p[data-stale]'); p && p.dataset.stale") + check("browser: side by side has a row per blood group system, a stale value marked in its cell and no other", + "|Duffy|" in rows and kell.split("|")[2] == "K-k+ (older than its input)" and hap.split("|")[2] == "H1" + and legend == "4", (kell, hap, legend)) + os.remove(os.path.join(fam_dir, "compound_het.tsv")) + shutil.copy(secret, os.path.join(fam_dir, "compound_het.tsv")) + + def comp_line(): + b.go(base + "/runs/family_a/") + b.wait("document.documentElement.dataset.pgpStatus === 'ready' && " + "!!document.querySelector('details[data-child=child_a] p[data-trio=compound_het]')") + return b.eval("var p = document.querySelector('details[data-child=child_a] p[data-trio=compound_het]'); " + "p && p.textContent") or "" + ch = comp_line() + check("browser: a child whose run has no step 23 candidates is not assessed for compound-het pairs", + ch.startswith("Not assessed") and "wrote no candidates for child_a" in ch, ch) + os.makedirs(clin) + vcf = os.path.join(clin, "child_a_clinical.vcf.gz") + open(vcf, "w").close() + ch = comp_line() + check("browser: candidates newer than compound_het.tsv leave the compound-het line not known", + ch.startswith("Not known whether assessed") and "newer than compound_het.tsv" in ch, ch) + old = os.path.getmtime(os.path.join(fam_dir, "compound_het.tsv")) - 60 + os.utime(vcf, (old, old)) + ch = comp_line() + check("browser: an assessed child's empty compound-het line says pairs come only from step 23's candidates", + ch.startswith("Assessed (") and "step 23" in ch, ch) + finally: + os.remove(os.path.join(fam_dir, "compound_het.tsv")) + os.remove(os.path.join(root, "child_a", "summary.json")) + shutil.rmtree(clin, ignore_errors=True) + + if __name__ == "__main__": sys.exit(main())