diff --git a/assets/live/finding.js b/assets/live/finding.js index 390f48d7..fe74ab01 100644 --- a/assets/live/finding.js +++ b/assets/live/finding.js @@ -72,11 +72,12 @@ root.setAttribute('data-pgp-status', 'ready'); return; } + // known, not family: the click that opens this finding draws again with family as it came, UNKNOWN kept. var unknown = family === UNKNOWN; - if (unknown) { family = null; } - var me = family && (family.state.members || []).filter(function (m) { return m.sample === r.sample; })[0]; + var known = unknown ? null : family; + var me = known && (known.state.members || []).filter(function (m) { return m.sample === r.sample; })[0]; // A family that does not list this person cannot say they are not a child: closed as well. - var child = unknown || (!!family && !me) || !!(me && (me.father || me.mother)); + var child = unknown || (!!known && !me) || !!(me && (me.father || me.mother)); // Closed: no title, gene or variant anywhere on the page (its title, crumb, cover, rail) until opened. var closed = child && f.level === 'act' && !draw.open; var title = closed ? 'A result to act on' : f.title; @@ -97,7 +98,7 @@ add(rail, el('div', { 'class': 'group' }, 'This finding')); var parts = [['what', 'What it is'], ['plan-sec', 'What to do'], ['evidence', 'Evidence']]; if (f.viewer) { parts.push(['reads', 'The reads']); } - if (family) { parts.push(['family', 'In the family']); } + if (known) { parts.push(['family', 'In the family']); } parts.push(['related', 'Related results'], ['sources', 'Caveat and sources']); var tol = el('ol', {}); parts.forEach(function (p, i) { @@ -158,7 +159,7 @@ add(box, ol); add(m1, box); var a1 = el('aside', {}); - if (f.level === 'carrier' && family) { add(a1, note('Family', add(el('span', {}), 'A carrier is not affected. ', el('a', { href: abs(familyUrl) }, 'The couple screen on the family page →')))); } + if (f.level === 'carrier' && known) { add(a1, note('Family', add(el('span', {}), 'A carrier is not affected. ', el('a', { href: abs(familyUrl) }, 'The couple screen on the family page →')))); } add(row1, m1, a1); add(col, row1); // The evidence. @@ -189,14 +190,14 @@ // Related findings and the family. var rel = (f.related || []).map(function (id) { return (r.findings || []).filter(function (x) { return x.id === id; })[0]; }) .filter(function (x) { return x && !(child && x.level === 'act'); }); - if (family) { + if (known) { var sf = el('section', { 'class': 'sec', id: 'family' }); add(sf, add(el('div', { 'class': 'sechead' }), el('span', { 'class': 'secnum' }, ROMAN[parts.length - 3]), el('h2', { 'class': 'h2' }, 'In the family'))); var fam = el('p', { 'class': 'soft' }); add(fam, 'Who else in the family has a genome here: '); - (family.state.members || []).filter(function (m) { return m.sample !== r.sample; }).forEach(function (m, i) { + (known.state.members || []).filter(function (m) { return m.sample !== r.sample; }).forEach(function (m, i) { if (i) { add(fam, ', '); } - var u = family.runs && family.runs[m.sample] && family.runs[m.sample].url; + var u = known.runs && known.runs[m.sample] && known.runs[m.sample].url; add(fam, u ? el('a', { href: abs(u) }, m.sample) : el('span', {}, m.sample + ' (no genome)')); }); add(fam, '. ', el('a', { href: abs(familyUrl) }, 'The family report →')); @@ -230,7 +231,11 @@ var familyUrl = null; var UNKNOWN = { unknown: true }; function familyOf(st) { - var fam = st && st.plugins && st.plugins.family; + // No state (state.json or frames.json refused, missing or not JSON), a state with no plugins, or a family + // plugin that found a family report it could not read: the roles are unknown, never a standalone adult. + if (!st || !st.plugins) { return Promise.resolve(UNKNOWN); } + var fam = st.plugins.family; + if (fam && fam.unknown) { return Promise.resolve(UNKNOWN); } var f = fam && fam.kind === 'member' && (fam.families || [])[0]; if (!f) { return Promise.resolve(null); } familyUrl = f.url; diff --git a/assets/live/person.js b/assets/live/person.js index 641f8b04..0ef6a8e5 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -73,12 +73,18 @@ var f = fam && fam.kind === 'member' && (fam.families || [])[0]; return f ? f : null; } + // The family plugin cannot say: a state with no plugins (unavailable or cut short), or a family plugin + // that found a family report it could not read (plugins.family.unknown). Not a standalone adult then. + function familyUnknown() { + var fam = state && state.plugins ? state.plugins.family : undefined; + return !state || !state.plugins || !!(fam && fam.unknown); + } function loadFamily() { var f = familyUrl(); - var key = f ? f.url : null; + var key = f ? f.url : (familyUnknown() ? '?' : null); if (key === familyFor) { return; } familyFor = key; - if (!f) { family = null; familyStatus = 'none'; return; } + if (!f) { family = null; familyStatus = key ? 'failed' : 'none'; return; } familyStatus = 'pending'; U.getJSON(f.url + 'family.json').then(function (d) { if (familyFor !== key) { return; } @@ -316,6 +322,17 @@ // The findings of a card that a child's page keeps closed: while one is closed the card's own rows, // numbers and drawing stay closed too, or its table would name the gene and variant under the veil. function veiledOf(c) { return (c.findings || []).map(findingById).filter(function (f) { return f && veiled(f); }); } + // The search box (plugins/search.js) asks the same question for each hit, by the summary key and the anchor + // of the section it comes from: a hit from a card this page keeps closed is left out. A closed page with + // no results.json yet cannot tell which cards those are, so it leaves every hit out until it can. + // Asked with no key, it answers whether this page keeps anything closed at all. + window.PGP.searchHidden = function (key, anchor) { + if (!closedPage()) { return false; } + if (!results || key === null) { return true; } + return cards().some(function (c) { + return (c.id === anchor || (c.keys || []).indexOf(key) >= 0) && veiledOf(c).length > 0; + }); + }; function card(c) { var g = gate(c); @@ -1051,7 +1068,7 @@ function keepFolds() { foldKeys().forEach(function (x) { foldState[x[0]] = x[1].open; }); } function restoreFolds() { foldKeys().forEach(function (x) { if (foldState[x[0]]) { x[1].open = true; } }); } function redraw(force) { - var k = JSON.stringify([doneSet(), state && state.run && state.run.state, !!results, !!family, Object.keys(shown), + var k = JSON.stringify([doneSet(), state && state.run && state.run.state, !!results, !!family, familyStatus, Object.keys(shown), state && state.plugins && state.plugins.recheck && state.plugins.recheck.mtime, // Every step's state: one can move from waiting to not finished with the done set unchanged. ((state && state.steps) || []).map(function (s) { return s.step + ':' + s.state; })]); diff --git a/assets/live/plugins/family.js b/assets/live/plugins/family.js index 8875f183..b94728f8 100644 --- a/assets/live/plugins/family.js +++ b/assets/live/plugins/family.js @@ -94,6 +94,10 @@ function member(f) { box.appendChild(el('h2', {}, 'Family')); + if (f.unknown) { + box.appendChild(para('A family report could not be read, so this page cannot tell whether this person is a child ' + + 'of it: results to act on stay closed until opened.', 'muted')); + } f.families.forEach(function (x) { var p = el('p', {}, 'In the family '); p.appendChild(link(x.url, x.name)); diff --git a/assets/live/plugins/search.js b/assets/live/plugins/search.js index 6a96bb33..3523890d 100644 --- a/assets/live/plugins/search.js +++ b/assets/live/plugins/search.js @@ -85,12 +85,24 @@ return li; } + // A hit from a section the page keeps closed (a result to act on about a child, person.js's + // PGP.searchHidden) is left out, and so is a suggested term that only such sections name: the search + // must not name what the page veils. The page says so whether or not anything was left out. + function hidden(it) { return !!(root.PGP.searchHidden && root.PGP.searchHidden(it.section, it.anchor)); } + function closedNote() { + return root.PGP.searchHidden && root.PGP.searchHidden(null, null) + ? el('p', { 'class': 'muted search-closed' }, 'Results this page keeps closed are left out of the search until they are opened.') + : null; + } + function show() { if (!out) { return; } out.textContent = ''; var q = input.value; if (!index || !q.trim()) { document.documentElement.removeAttribute('data-pgp-search'); return; } var r = match(index, q); + r.items = r.items.filter(function (it) { return !hidden(it); }); + r.suggestions = r.suggestions.filter(function (t) { return (index.terms[t] || []).some(function (it) { return !hidden(it); }); }); document.documentElement.setAttribute('data-pgp-search', String(r.items.length)); if (r.kind === 'site') { out.appendChild(el('p', {}, 'The search finds gene symbols and rsIDs. A position is a question about the genotype: ' @@ -119,6 +131,8 @@ }); out.appendChild(p); } + var note = closedNote(); + if (note) { out.appendChild(note); } } function onState(state) { diff --git a/bin/live_results.py b/bin/live_results.py index bb05e5f4..f764d3c3 100644 --- a/bin/live_results.py +++ b/bin/live_results.py @@ -772,7 +772,41 @@ def _f(level, card_id, key, title, text, gene=None, detail=None, evidence=(), vi def _data(S, k): sec = S.get(k) or {} - return sec.get("data") or {} if sec.get("state") in ("ok", "stale") else {} + if not isinstance(sec, dict) or sec.get("state") not in ("ok", "stale"): + return {} + d = sec.get("data") or {} + # Data that is not a mapping gives no rows here; the section's card says it could not be drawn (card()). + return d if isinstance(d, dict) else {} + + +def _items(v): + """A summary's list of rows; a lone row that is not in a list is read as one row (and fails as one).""" + return v if isinstance(v, list) else [v] if v else [] + + +class _Row: + """with _Row(out, card[, level]): one rule turning one row of the summary into a finding. A row the rule + cannot read (a field of the wrong type, a stars value that is not a number) costs that row only: it + becomes a finding on its card that says the row could not be read, and every other row and card still + renders. Its words name no gene or variant: the row is unreadable. Its level is the highest the rule + can give (level "act" for a rule that can give one): a child's page veils a card that holds a finding + to act on, and the card's table may still draw the row the rule could not read.""" + + def __init__(self, out, card_id, level="review"): + self.out, self.card_id, self.level = out, card_id, level + + def __enter__(self): + return self + + def __exit__(self, kind, err, tb): + if kind is None or not issubclass(kind, (KeyError, TypeError, ValueError, AttributeError, IndexError, + ZeroDivisionError)): + return False + self.out.append(_f(self.level, self.card_id, ("unreadable", self.card_id, len(self.out)), + "A row of this section could not be read", + f"One row of this section holds a value of a kind the page cannot read ({kind.__name__}). " + f"The other rows show as usual; the report and its files have the row as the step wrote it.")) + return True def _copies(n): @@ -783,166 +817,193 @@ def finding_rows(summary): """[finding] from the rows of the summary, each with its level, in the order of the rules.""" S = summary.get("sections") or {} out = [] - sf = (_data(S, "clinical").get("acmg_sf") or {}) - sf_genes = {x.get("gene") for x in sf.get("clinvar_hits") or []} + # The genes the later rules leave to an earlier one. A row that is not a mapping names no gene here; + # its own rule below turns it into the one unreadable finding. + sf = _data(S, "clinical").get("acmg_sf") or {} + sf = sf if isinstance(sf, dict) else {} + def gene(x): + g = x if isinstance(x, str) else x.get("gene") if isinstance(x, dict) else None + return g if isinstance(g, str) else None + sf_genes = {gene(x) for x in _items(sf.get("clinvar_hits")) if isinstance(x, dict)} car = _data(S, "carrier") - carrier_genes = {c.get("gene") for c in car.get("carriers") or []} - two_hit = {g if isinstance(g, str) else g.get("gene") for g in (car.get("two_hits") or {}).get("genes") or []} - # 1. ACMG secondary findings: the gene's own reporting rule already said "report". - for r in rr.sf_rows(sf) if sf and not sf.get("unavailable") else []: - out.append(_f("act", "clinical-filter", ("sf", r[0], r[1]), f"Secondary finding in {r[0]}", - f"A change in {r[0]} that the ACMG secondary-findings list reports: {r[3]}.", - gene=r[0], detail=f"{r[0]} {r[1]} {r[2] if r[2] != '.' else ''}".strip(), - evidence=[["Variant", r[1]], ["Genotype", r[2]], ["Why it is reported", r[3]]])) + carrier_genes = {gene(c) for c in _items(car.get("carriers")) if isinstance(c, dict)} + th = car.get("two_hits") if isinstance(car.get("two_hits"), dict) else {} + two_hit = {gene(g) for g in _items(th.get("genes"))} - {None} + # 1. ACMG secondary findings: the gene's own reporting rule already said "report". The report's own + # rows (rr.sf_rows): one it cannot read costs this rule, never the rest. + sf_list = [] + with _Row(out, "clinical-filter", "act"): + sf_list = rr.sf_rows(sf) if sf and not sf.get("unavailable") else [] + for r in sf_list: + with _Row(out, "clinical-filter", "act"): + out.append(_f("act", "clinical-filter", ("sf", r[0], r[1]), f"Secondary finding in {r[0]}", + f"A change in {r[0]} that the ACMG secondary-findings list reports: {r[3]}.", + gene=r[0], detail=f"{r[0]} {r[1]} {r[2] if r[2] != '.' else ''}".strip(), + evidence=[["Variant", r[1]], ["Genotype", r[2]], ["Why it is reported", r[3]]])) # 2. Cancer predisposition, CPSR's pathogenic and likely pathogenic rows. - for x in _data(S, "cpsr").get("plp") or []: - out.append(_f("act", "cpsr", ("cpsr", x.get("gene"), x.get("variant")), - f"Cancer predisposition ({x.get('gene')})", - f"CPSR grades this change in {x.get('gene')} as {x.get('final')}: it can raise the risk of some " - f"cancers. It is not a diagnosis of cancer.", gene=x.get("gene"), - detail=f"{x.get('gene')} {x.get('variant')}", - evidence=[["Class", x.get("final")], ["ClinVar", x.get("clinvar")], - ["Call", "low confidence" if x.get("low_confidence") else None]])) + for x in _items(_data(S, "cpsr").get("plp")): + with _Row(out, "cpsr", "act"): + out.append(_f("act", "cpsr", ("cpsr", x.get("gene"), x.get("variant")), + f"Cancer predisposition ({x.get('gene')})", + f"CPSR grades this change in {x.get('gene')} as {x.get('final')}: it can raise the risk of some " + f"cancers. It is not a diagnosis of cancer.", gene=x.get("gene"), + detail=f"{x.get('gene')} {x.get('variant')}", + evidence=[["Class", x.get("final")], ["ClinVar", x.get("clinvar")], + ["Call", "low confidence" if x.get("low_confidence") else None]])) # 3. Two hits in one carrier-list gene: possibly both copies changed. for g in sorted(x for x in two_hit if x): - out.append(_f("act", "carrier-list", ("two-hits", g), f"Two changes in {g}", - f"Two changes were found in {g}, a gene where it takes two to cause the condition. Whether they " - f"sit on different copies decides whether it matters.", gene=g)) + with _Row(out, "carrier-list", "act"): + out.append(_f("act", "carrier-list", ("two-hits", g), f"Two changes in {g}", + f"Two changes were found in {g}, a gene where it takes two to cause the condition. Whether they " + f"sit on different copies decides whether it matters.", gene=g)) # 4. ClinVar P/LP hits not already a secondary finding or a carrier row. - for h in (_data(S, "clinvar").get("hits") or [])[:50]: - g = h.get("gene") - if g in sf_genes or g in carrier_genes or g in two_hit: - continue - var = f"{h.get('chrom')}:{h.get('pos')} {h.get('ref')}>{h.get('alt')}" - out.append(_f("review", "clinvar", ("clinvar", g, h.get("chrom"), h.get("pos"), h.get("ref"), h.get("alt")), - f"{h.get('significance') or 'Pathogenic'} change in {g}", - f"ClinVar lists this change in {g} as {str(h.get('significance') or '').lower()}; this genome " - f"has {_copies(h.get('genotype'))}. Whether it matters depends on how the gene is inherited: " - f"a genetic counsellor reads it with the rest of this report.", - gene=g, detail=f"{g} {var} {h.get('genotype') or ''}".strip(), - evidence=[["Significance", h.get("significance")], ["ClinVar review", "★" * int(h.get("stars") or 0) or "0 stars"], - ["Genotype", h.get("genotype")], ["Frequency", rr.af_text(h) or "frequency unknown"], - ["Call", clinvar_call(h)]], - viewer=f"{h.get('chrom')}:{h.get('pos')}", af=h.get("af"))) + for h in _items(_data(S, "clinvar").get("hits"))[:50]: + with _Row(out, "clinvar"): + g = h.get("gene") + if g in sf_genes or g in carrier_genes or g in two_hit: + continue + var = f"{h.get('chrom')}:{h.get('pos')} {h.get('ref')}>{h.get('alt')}" + out.append(_f("review", "clinvar", ("clinvar", g, h.get("chrom"), h.get("pos"), h.get("ref"), h.get("alt")), + f"{h.get('significance') or 'Pathogenic'} change in {g}", + f"ClinVar lists this change in {g} as {str(h.get('significance') or '').lower()}; this genome " + f"has {_copies(h.get('genotype'))}. Whether it matters depends on how the gene is inherited: " + f"a genetic counsellor reads it with the rest of this report.", + gene=g, detail=f"{g} {var} {h.get('genotype') or ''}".strip(), + evidence=[["Significance", h.get("significance")], ["ClinVar review", "★" * int(h.get("stars") or 0) or "0 stars"], + ["Genotype", h.get("genotype")], ["Frequency", rr.af_text(h) or "frequency unknown"], + ["Call", clinvar_call(h)]], + viewer=f"{h.get('chrom')}:{h.get('pos')}", af=h.get("af"))) # 5. Large deletions and duplications AnnotSV ranks class 4 or 5. - for x in _data(S, "annotsv").get("rows") or []: - genes = ", ".join(x.get("genes") or []) or "no gene" - out.append(_f("act" if str(x.get("acmg_class")) == "5" else "review", "structural-variants", - ("sv", x.get("sv")), f"Large {x.get('type') or 'structural change'} over {genes}", - f"A {x.get('type') or 'structural change'} of {abs(int(num(x.get('length')) or 0)):,} bases that " - f"AnnotSV ranks class {x.get('acmg_class')}. Short reads can misplace large changes: confirm it.", - gene=(x.get("genes") or [None])[0], detail=x.get("sv"), - evidence=[["Class", x.get("acmg_class")], ["Callers", rr.annotsv_support_text(x)], - ["Genotype", x.get("gt")]], viewer=str(x.get("sv") or "").replace(" ", "") or None)) + for x in _items(_data(S, "annotsv").get("rows")): + with _Row(out, "structural-variants", "act"): + genes = ", ".join(x.get("genes") or []) or "no gene" + out.append(_f("act" if str(x.get("acmg_class")) == "5" else "review", "structural-variants", + ("sv", x.get("sv")), f"Large {x.get('type') or 'structural change'} over {genes}", + f"A {x.get('type') or 'structural change'} of {abs(int(num(x.get('length')) or 0)):,} bases that " + f"AnnotSV ranks class {x.get('acmg_class')}. Short reads can misplace large changes: confirm it.", + gene=(x.get("genes") or [None])[0], detail=x.get("sv"), + evidence=[["Class", x.get("acmg_class")], ["Callers", rr.annotsv_support_text(x)], + ["Genotype", x.get("gt")]], viewer=str(x.get("sv") or "").replace(" ", "") or None)) # 6. Repeat expansions outside Stranger's "normal", or in an intermediate zone. - for l in _data(S, "expansions").get("loci") or []: - st = [x for x in (l.get("status") or "").split(",") if x and x != "normal"] - if not st and not l.get("zone"): - continue - full = any("full" in x for x in st) - lead_st = next((x for x in st if "full" in x), st[0]) if st else None - what = (STR_WORDS.get(lead_st, lead_st.replace("_", " ")) if lead_st - else f"{l.get('zone_label') or 'intermediate'} range") - count = rr.str_count(l) if l.get("repeat_count") else "not given" - out.append(_f("act" if full else "review", "expansions", ("str", l.get("locus")), - f"{l.get('locus')} repeat in the {what}", - f"{l.get('locus')} has {count} repeats: Stranger reads it as \"{what}\". " - f"{'See a clinical geneticist.' if full else 'It may grow when passed on; worth a word with a genetic counsellor.'}", - gene=l.get("locus"), detail=f"{l.get('locus')} {count} repeats", - evidence=[["Repeats", count], ["95% interval", l.get("ci")], - ["Pathologic from", l.get("pathologic_min")], - ["Sizing", l.get("allele_sizing") or l.get("sizing")]])) - v = _data(S, "vntyper") - if v.get("status") in ("positive", "possible"): - out.append(_f("act" if v["status"] == "positive" else "review", "vntyper", ("muc1", v["status"]), - f"MUC1 kidney gene: {v['status']}", f"VNtyper reads the MUC1 repeat as {v['status']}; it needs a " - f"clinical confirmation.", gene="MUC1", evidence=[["Confidence", v.get("confidence")]])) - lp = _data(S, "lpa") - if lp.get("flag") == "high": - out.append(_f("review", "lpa", ("lpa",), "Lp(a) likely high", - "The KIV-2 count suggests a high Lp(a); one blood measurement settles it.", gene="LPA", - evidence=[["KIV-2 repeats", lp.get("kiv2_copies")], ["Class", lp.get("class")]])) - for c in _data(S, "chip").get("calls") or []: - out.append(_f("review", "chip", ("chip", c.get("gene"), c.get("protein_change")), - f"A blood clone in {c.get('gene')} (CHIP)", - f"{c.get('gene')} {c.get('protein_change') or ''} is carried by a share of the blood cells (VAF " - f"{c.get('vaf')}). It becomes common with age; most people with it never get a blood cancer.", - gene=c.get("gene"), evidence=[["VAF", c.get("vaf")], ["Reads", f"{c.get('alt_reads')} of {c.get('depth')}"]])) - for c in _data(S, "mosaic").get("calls") or []: - out.append(_f("review", "mosaic-mca", ("mca", c.get("chrom"), c.get("start")), - f"A {c.get('type') or 'change'} on {c.get('chrom')} in some blood cells", - f"MoChA sees a {c.get('type') or 'change'} of {c.get('length_mb')} Mb on {c.get('chrom')} in a share " - f"of the blood cells ({c.get('cell_fraction')}).", - evidence=[["Where", f"{c.get('chrom')}:{c.get('start')}-{c.get('end')}"], - ["Cell fraction", c.get("cell_fraction")]])) - for x in (_data(S, "exomiser").get("genes") or [])[:3]: - out.append(_f("review", "exomiser", ("exomiser", x.get("gene")), f"Symptom ranking: {x.get('gene')} ranks {x.get('rank')}", - "A ranking to bring to a clinical geneticist, not a diagnosis.", gene=x.get("gene"), - evidence=[["Score", x.get("exomiser_score")], ["Inheritance", x.get("moi")]])) + for l in _items(_data(S, "expansions").get("loci")): + with _Row(out, "expansions", "act"): + st = [x for x in (l.get("status") or "").split(",") if x and x != "normal"] + if not st and not l.get("zone"): + continue + full = any("full" in x for x in st) + lead_st = next((x for x in st if "full" in x), st[0]) if st else None + what = (STR_WORDS.get(lead_st, lead_st.replace("_", " ")) if lead_st + else f"{l.get('zone_label') or 'intermediate'} range") + count = rr.str_count(l) if l.get("repeat_count") else "not given" + out.append(_f("act" if full else "review", "expansions", ("str", l.get("locus")), + f"{l.get('locus')} repeat in the {what}", + f"{l.get('locus')} has {count} repeats: Stranger reads it as \"{what}\". " + f"{'See a clinical geneticist.' if full else 'It may grow when passed on; worth a word with a genetic counsellor.'}", + gene=l.get("locus"), detail=f"{l.get('locus')} {count} repeats", + evidence=[["Repeats", count], ["95% interval", l.get("ci")], + ["Pathologic from", l.get("pathologic_min")], + ["Sizing", l.get("allele_sizing") or l.get("sizing")]])) + with _Row(out, "vntyper", "act"): + v = _data(S, "vntyper") + if v.get("status") in ("positive", "possible"): + out.append(_f("act" if v["status"] == "positive" else "review", "vntyper", ("muc1", v["status"]), + f"MUC1 kidney gene: {v['status']}", f"VNtyper reads the MUC1 repeat as {v['status']}; it needs a " + f"clinical confirmation.", gene="MUC1", evidence=[["Confidence", v.get("confidence")]])) + with _Row(out, "lpa"): + lp = _data(S, "lpa") + if lp.get("flag") == "high": + out.append(_f("review", "lpa", ("lpa",), "Lp(a) likely high", + "The KIV-2 count suggests a high Lp(a); one blood measurement settles it.", gene="LPA", + evidence=[["KIV-2 repeats", lp.get("kiv2_copies")], ["Class", lp.get("class")]])) + for c in _items(_data(S, "chip").get("calls")): + with _Row(out, "chip"): + out.append(_f("review", "chip", ("chip", c.get("gene"), c.get("protein_change")), + f"A blood clone in {c.get('gene')} (CHIP)", + f"{c.get('gene')} {c.get('protein_change') or ''} is carried by a share of the blood cells (VAF " + f"{c.get('vaf')}). It becomes common with age; most people with it never get a blood cancer.", + gene=c.get("gene"), evidence=[["VAF", c.get("vaf")], ["Reads", f"{c.get('alt_reads')} of {c.get('depth')}"]])) + for c in _items(_data(S, "mosaic").get("calls")): + with _Row(out, "mosaic-mca"): + out.append(_f("review", "mosaic-mca", ("mca", c.get("chrom"), c.get("start")), + f"A {c.get('type') or 'change'} on {c.get('chrom')} in some blood cells", + f"MoChA sees a {c.get('type') or 'change'} of {c.get('length_mb')} Mb on {c.get('chrom')} in a share " + f"of the blood cells ({c.get('cell_fraction')}).", + evidence=[["Where", f"{c.get('chrom')}:{c.get('start')}-{c.get('end')}"], + ["Cell fraction", c.get("cell_fraction")]])) + for x in _items(_data(S, "exomiser").get("genes"))[:3]: + with _Row(out, "exomiser"): + out.append(_f("review", "exomiser", ("exomiser", x.get("gene")), f"Symptom ranking: {x.get('gene')} ranks {x.get('rank')}", + "A ranking to bring to a clinical geneticist, not a diagnosis.", gene=x.get("gene"), + evidence=[["Score", x.get("exomiser_score")], ["Inheritance", x.get("moi")]])) # 7. Carriers: one changed copy in a gene of the carrier list, and the couple screen. - for c in car.get("carriers") or []: - if c.get("gene") in two_hit: - continue - gt = str(c.get("genotype") or "") - if gt in ("hom", "hemizygous"): - # Both copies changed, or the only copy (X in a male, or inside a deletion): not a carrier. - out.append(_f("act", "carrier-list", ("carrier", c.get("gene"), c.get("variant")), - f"{c.get('condition') or c.get('gene')}: {'two changed copies' if gt == 'hom' else 'the only copy changed'}", - f"This genome has {'two changed copies' if gt == 'hom' else 'no unchanged copy'} of " - f"{c.get('gene')}, a gene of {c.get('condition') or 'a condition on the carrier list'} " - f"({c.get('inheritance') or 'inheritance not given'}). That is not the pattern of a carrier: " - f"a doctor or genetic counsellor reads it.", - gene=c.get("gene"), detail=f"{c.get('gene')} {c.get('variant') or ''} {gt}".strip(), - evidence=[["Genotype", gt], ["Significance", c.get("significance")], ["Stars", c.get("stars")], + for c in _items(car.get("carriers")): + with _Row(out, "carrier-list", "act"): + if c.get("gene") in two_hit: + continue + gt = str(c.get("genotype") or "") + if gt in ("hom", "hemizygous"): + # Both copies changed, or the only copy (X in a male, or inside a deletion): not a carrier. + out.append(_f("act", "carrier-list", ("carrier", c.get("gene"), c.get("variant")), + f"{c.get('condition') or c.get('gene')}: {'two changed copies' if gt == 'hom' else 'the only copy changed'}", + f"This genome has {'two changed copies' if gt == 'hom' else 'no unchanged copy'} of " + f"{c.get('gene')}, a gene of {c.get('condition') or 'a condition on the carrier list'} " + f"({c.get('inheritance') or 'inheritance not given'}). That is not the pattern of a carrier: " + f"a doctor or genetic counsellor reads it.", + gene=c.get("gene"), detail=f"{c.get('gene')} {c.get('variant') or ''} {gt}".strip(), + evidence=[["Genotype", gt], ["Significance", c.get("significance")], ["Stars", c.get("stars")], + ["Inheritance", c.get("inheritance")]])) + continue + out.append(_f("carrier", "carrier-list", ("carrier", c.get("gene"), c.get("variant")), + f"{c.get('condition') or c.get('gene')} carrier" if c.get("condition") else f"Carrier in {c.get('gene')}", + f"{CARRIER_NOTE} This genome has one changed copy of {c.get('gene')}: a carrier, not affected.", + gene=c.get("gene"), detail=f"{c.get('gene')} {c.get('variant') or ''}".strip(), + evidence=[["Significance", c.get("significance")], ["Stars", c.get("stars")], ["Inheritance", c.get("inheritance")]])) - continue - out.append(_f("carrier", "carrier-list", ("carrier", c.get("gene"), c.get("variant")), - f"{c.get('condition') or c.get('gene')} carrier" if c.get("condition") else f"Carrier in {c.get('gene')}", - f"{CARRIER_NOTE} This genome has one changed copy of {c.get('gene')}: a carrier, not affected.", - gene=c.get("gene"), detail=f"{c.get('gene')} {c.get('variant') or ''}".strip(), - evidence=[["Significance", c.get("significance")], ["Stars", c.get("stars")], - ["Inheritance", c.get("inheritance")]])) cp = _data(S, "couple") for lab, key in (("Both partners carry", "shared"), ("X-linked", "x_linked")): - for x in cp.get(key) or []: - out.append(_f("carrier", "couple", ("couple", key, x.get("gene")), f"{lab} {x.get('gene')}" - if key == "shared" else f"X-linked finding in {x.get('gene')}", - f"{x.get('condition') or x.get('gene')}: {x.get('risk') or 'see the couple screen'} for each " - f"pregnancy, with {cp.get('partner')}.", gene=x.get("gene"), - evidence=[["Chance for a child", x.get("risk")], ["Partner", cp.get("partner")]])) - smn1 = [g for g in _data(S, "paralogs").get("genes") or [] if g.get("gene") == "SMN1" and g.get("status") == "called"] - one = smn1 and str(smn1[0].get("copy_number")) == "1" - by = "Paraphase" - if not smn1: # no Paraphase call: Parascopy's resolved count from short reads - cn = ((_data(S, "parascopy").get("smn1") or {}).get("cn")) - one, by = cn == 1, "Parascopy" - if one: - out.append(_f("carrier", "paralogs", ("smn1",), "Spinal muscular atrophy carrier (SMN1)", - "One copy of SMN1 where most people have two: a carrier of spinal muscular atrophy, not " - "affected.", gene="SMN1", evidence=[["SMN1 copies", "1"], ["Called by", by]])) + for x in _items(cp.get(key)): + with _Row(out, "couple"): + out.append(_f("carrier", "couple", ("couple", key, x.get("gene")), f"{lab} {x.get('gene')}" + if key == "shared" else f"X-linked finding in {x.get('gene')}", + f"{x.get('condition') or x.get('gene')}: {x.get('risk') or 'see the couple screen'} for each " + f"pregnancy, with {cp.get('partner')}.", gene=x.get("gene"), + evidence=[["Chance for a child", x.get("risk")], ["Partner", cp.get("partner")]])) + with _Row(out, "paralogs"): + smn1 = [g for g in _data(S, "paralogs").get("genes") or [] if g.get("gene") == "SMN1" and g.get("status") == "called"] + one = smn1 and str(smn1[0].get("copy_number")) == "1" + by = "Paraphase" + if not smn1: # no Paraphase call: Parascopy's resolved count from short reads + cn = ((_data(S, "parascopy").get("smn1") or {}).get("cn")) + one, by = cn == 1, "Parascopy" + if one: + out.append(_f("carrier", "paralogs", ("smn1",), "Spinal muscular atrophy carrier (SMN1)", + "One copy of SMN1 where most people have two: a carrier of spinal muscular atrophy, not " + "affected.", gene="SMN1", evidence=[["SMN1 copies", "1"], ["Called by", by]])) # 8. Medicines: each gene whose called phenotype changes function. - for g in _data(S, "cpic").get("genes") or []: - if g.get("status") not in ("non-normal", "ambiguous"): - continue - amb = g["status"] == "ambiguous" - out.append(_f("medicine", "cpic", ("pgx", g.get("gene")), - f"{g.get('gene')}: {g.get('phenotype')}" if not amb else f"{g.get('gene')}: more than one possible result", - f"{g.get('gene')} {g.get('diplotype')} reads as \"{g.get('phenotype')}\" in CPIC's words. The drugs " - f"it bears on are in the CPIC report; tell whoever prescribes.", gene=g.get("gene"), - detail=f"{g.get('gene')} {g.get('diplotype')}", - evidence=[["Diplotype", g.get("diplotype")], ["Phenotype", g.get("phenotype")], ["Status", g.get("status")]])) + for g in _items(_data(S, "cpic").get("genes")): + with _Row(out, "cpic"): + if g.get("status") not in ("non-normal", "ambiguous"): + continue + amb = g["status"] == "ambiguous" + out.append(_f("medicine", "cpic", ("pgx", g.get("gene")), + f"{g.get('gene')}: {g.get('phenotype')}" if not amb else f"{g.get('gene')}: more than one possible result", + f"{g.get('gene')} {g.get('diplotype')} reads as \"{g.get('phenotype')}\" in CPIC's words. The drugs " + f"it bears on are in the CPIC report; tell whoever prescribes.", gene=g.get("gene"), + detail=f"{g.get('gene')} {g.get('diplotype')}", + evidence=[["Diplotype", g.get("diplotype")], ["Phenotype", g.get("phenotype")], ["Status", g.get("status")]])) # 9. Trust: a problem with the data itself. - q = _data(S, "sample_qc") - if q.get("sex_check") == "mismatch": - out.append(_f("trust", "qc", ("sex",), "Declared and measured sex differ", - "The sex the reads show differs from the one declared: a swapped or mislabelled sample, or a real " - "sex-chromosome difference.", evidence=[["Declared", q.get("declared_sex")], ["From the reads", q.get("inferred_sex")]])) - if q.get("contamination") == "warn": - out.append(_f("trust", "qc", ("contamination",), "Another person's DNA may be mixed in", - rr.freemix_text(q), evidence=[["FREEMIX", q.get("freemix")]])) - if q.get("same_person_as"): - out.append(_f("trust", "qc", ("same-person",), f"The same person as {q['same_person_as']}", - "Two runs of this server hold the same genome.", evidence=[["Matches", q["same_person_as"]]])) + with _Row(out, "qc"): + q = _data(S, "sample_qc") + if q.get("sex_check") == "mismatch": + out.append(_f("trust", "qc", ("sex",), "Declared and measured sex differ", + "The sex the reads show differs from the one declared: a swapped or mislabelled sample, or a real " + "sex-chromosome difference.", evidence=[["Declared", q.get("declared_sex")], ["From the reads", q.get("inferred_sex")]])) + if q.get("contamination") == "warn": + out.append(_f("trust", "qc", ("contamination",), "Another person's DNA may be mixed in", + rr.freemix_text(q), evidence=[["FREEMIX", q.get("freemix")]])) + if q.get("same_person_as"): + out.append(_f("trust", "qc", ("same-person",), f"The same person as {q['same_person_as']}", + "Two runs of this server hold the same genome.", evidence=[["Matches", q["same_person_as"]]])) seen, uniq = set(), [] for x in out: if x["id"] not in seen: diff --git a/bin/owngenome_live_family.py b/bin/owngenome_live_family.py index fa23dabc..b65bd33a 100644 --- a/bin/owngenome_live_family.py +++ b/bin/owngenome_live_family.py @@ -29,7 +29,8 @@ state.json gets plugins.family: {"kind": "family", ...} on a family's page, {"kind": "member", "families": [...]} on the page of a sample in a family, -null otherwise. +{"kind": "member", "families": [], "unknown": true} when no readable family +lists the sample but a family's pedigree could not be read, null otherwise. The plugin computes nothing about genetics: it reads the files the steps wrote, counts their rows and draws the pedigree the file declares. @@ -415,8 +416,18 @@ def side_values(summary): def side_by_side(names, summaries): """The members' results in one table: {members, rows: [{group, label, values, stale}]}. A member with no summary.json yet says so in its column; a row a member's summary does not have is '.'. stale holds, - per member, whether that value comes from a section older than its input.""" - per = {n: side_values(summaries.get(n)) for n in names} + per member, whether that value comes from a section older than its input. A summary whose fields are + not of the shape the report writes costs its own column only: it says so there, and the family page + still draws.""" + per, bad = {}, set() + for n in names: + try: + got = side_values(summaries.get(n)) + for g, label, _, _ in got: + hash((g, label)) # a label that is a list or a dict cannot key the table's rows + per[n] = got + except (AttributeError, KeyError, TypeError, ValueError, IndexError): + per[n], bad = [], bad | {n} # Each member's rows keep their own order (step 41's for the blood groups): a row first seen in a later # member goes right after the row before it in that member's list, not at the end. keys = [] @@ -435,7 +446,10 @@ def side_by_side(names, summaries): rows = [{"group": g, "label": label, "values": [], "stale": []} for g, label in keys] for r in rows: for n in names: - if summaries.get(n) is None: + if n in bad: + r["values"].append("summary not readable") + r["stale"].append(False) + elif summaries.get(n) is None: r["values"].append("no summary yet") r["stale"].append(False) else: @@ -464,15 +478,22 @@ def member_summary(live, d): def member_state(root, sample): - fams = [] + """The families that list the sample. A family whose pedigree cannot be read may be the one that + lists it as a child: with no readable family listing it, the answer is unknown, never "in no + family" ({"kind": "member", "families": [], "unknown": true}), and its pages keep results to act + on closed, as the home page does for a run no readable family lists.""" + fams, unread = [], False for name, d in family_dirs(root): try: order, _ = members_of(d) except (OSError, ValueError): + unread = True continue if sample in order: fams.append({"name": name, "url": "/runs/%s/" % name}) - return {"kind": "member", "families": fams} if fams else None + if fams: + return {"kind": "member", "families": fams} + return {"kind": "member", "families": [], "unknown": True} if unread else None def register(app): diff --git a/bin/run_state.py b/bin/run_state.py index ac9d6d88..8e24d642 100755 --- a/bin/run_state.py +++ b/bin/run_state.py @@ -173,17 +173,21 @@ def parse_trace_time(v): def parse_duration(v): """A trace duration ('1h 2m 3s', '450ms', '2.5s', '-') in seconds, or None.""" - if not v or v == "-": + if not v or not isinstance(v, str) or v == "-": return None total, found = 0.0, False - for num, unit in re.findall(r"([\d.]+)\s*(ms|d|h|m|s)", v): + # A number is digits with at most one decimal point: '1.2.3s' or '.s' (a cut or garbled cell) is no + # duration, never a crash that loses the whole trace file. + for num, unit in re.findall(r"(?= started] + return [p for p in found if (mtime(p) or 0) >= started] def collect(sample, sample_dir, trace=None, events_path=None, steps_path=STEPS_TSV, stale=False): diff --git a/docs/live-site.md b/docs/live-site.md index 87a9bc78..7fef70b6 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -56,7 +56,7 @@ Every result carries one of six levels, with a word, a shape and a colour, so th 9. **What it cannot see**: what the run does not assess and the steps that did not run. 10. **Sources and files**: the database versions, the database re-check, and the report and its files. -A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the chromosome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. +A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the chromosome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. So is a page whose run state is unavailable or comes without its plugins, and a person no readable family lists while some family's pedigree cannot be read (the family card says so): an unknown role is never read as a standalone adult. The header search follows the same veil: a hit from a card the page keeps closed is left out, and so is a suggested gene that only such a card names. A stopped run's page says so under the name and in the summary line, with the day it was last heard from, and warns that the report may be incomplete; a section still waiting for its step says the run stopped before that step ended. @@ -68,7 +68,7 @@ While the run goes, the page still opens on the report being written. A short ba - **The report.** Once written, `report.html` opens the run's `_report.html`, and the files it links to beside it. - **The genome viewer.** `/runs//viewer/` shows the run's reads and variants in igv.js, as [`scripts/owngenome-view.sh`](viewer.md) does on one computer. The report's Viewer links, the charts' "See the reads" links and a finding page's reads open it at their position. -The header search finds a gene or rsID across the results and opens the card that names it. `#s` opens a section, `#s/
` a card (the report's own anchors), `#f/` a finding, `#q/` the section a report chapter falls in, `#progress` the timeline. +The header search finds a gene or rsID across the results and opens the card that names it; on a page that keeps results closed it leaves their hits out (above). One row of the results that the page cannot read (a field of the wrong type) costs that row only: it shows as a finding on its card, "A row of this section could not be read", at the highest level its rule can give (to act on for a rule that can give one, so a page that keeps results closed keeps that card closed too), the card whose table cannot be drawn says so, and every other card and finding is on the page. `#s` opens a section, `#s/
` a card (the report's own anchors), `#f/` a finding, `#q/` the section a report chapter falls in, `#progress` the timeline. **A finding** (`/runs//finding//`). One finding on a page of its own: its level and how many others share it, the title, the gene and variant, a plain explanation, and **What to do** as numbered steps. Then the evidence (its numbers, how common the change is on a log scale from one in a million to everyone, the caveat in the margin), the reads with a link to the genome viewer, the rest of the family, the related findings of the same gene or step, and the section of the report it comes from. The contents on the left lead back to the report and on to the next finding. The page is 404 unless the id is a finding of that run's `results.json` now. diff --git a/docs/multi-sample.md b/docs/multi-sample.md index 8f12ccf2..f9e3cef1 100644 --- a/docs/multi-sample.md +++ b/docs/multi-sample.md @@ -120,7 +120,7 @@ The live site (`bin/owngenome_live.py`) lists each family folder under `GENOME_D - **II, who is related how**: the relationship check (`relationships.tsv`) and the shared segments (`ibd_pairs.tsv`) as one grid of every pair, with ⚑ where the measured relationship differs from the declared one (a list of pairs on a phone). - **III, the couple carrier screen**, per couple: how many genes `couple__.tsv` lists as `shared`, as `x_linked` and as carried by one partner, each gene both carry with its chance per pregnancy (drawn as four children when it is the carrier pattern), the whole table, and a link to `couple__.md`. A count of 0 is not an all-clear; the report says what the screen does not see. - **IV, each child**: the stretches of `_segments.tsv` painted per chromosome, the copy from the father above the copy from the mother, coloured by grandparent where step 38 could tell, else by parent; then the child's de novo candidates (`de_novo.tsv`) and compound-het pairs (`compound_het.tsv`, in trans first). Without `_segments.tsv` the page shows `_segments.svg`, or a line saying step 38 has not drawn one. For each child the page says whether step 38 assessed it: "none listed" only for a child it assessed (de novo candidates need both parents sequenced and in the joint call, compound-het pairs one such parent), with the reason. Compound-het pairs come only from the child's step 23 candidates: a child whose run is on the site without `clinical/_clinical.vcf.gz` was not assessed. When that file is newer than `compound_het.tsv`, or the child's run is not on the site, the page says it cannot tell whether the child was assessed, and why. Otherwise the page says why not. A missing de novo or compound-het file is named with the reason from `family_plan.tsv`, and a file that is a link out of the family folder is named as one, without showing where it points. -- **V, the members side by side**: each sequenced member's CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system step 41 typed (ABO and RhD first, then Kell, Duffy, Kidd and the rest in step 41's order) from their `summary.json`, one column each; `.` where a member's summary has no such row, "no summary yet" for a member without one. A value from a section older than its input (state stale in `summary.json`) is marked "(older than its input)" in its cell. +- **V, the members side by side**: each sequenced member's CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system step 41 typed (ABO and RhD first, then Kell, Duffy, Kidd and the rest in step 41's order) from their `summary.json`, one column each; `.` where a member's summary has no such row, "no summary yet" for a member without one, and "summary not readable" for a member whose `summary.json` is not of the shape the report writes (the rest of the family page still draws). A value from a section older than its input (state stale in `summary.json`) is marked "(older than its input)" in its cell. A person's page gets a Family card (`assets/live/plugins/family.js`) with a link to each family the person is in. The plugin reads the files the steps wrote and nothing else: it calls no tool and computes no genetics. It serves only the pedigree it draws and the result files named above, at `/family//`, behind the site's login; any other file of the folder, such as `family.vcf.gz`, is 404. `tests/test_pedigree_svg.py` and `tests/test_owngenome_live_family.py` check the drawing and the family page on an invented family (`tests/fixtures/families_live/`), with a swapped parent as the control that must fail; `tests/test_live_family_depth.py` checks the painting, the de novo and compound-het tables and the side by side on the demo's invented trio (`tests/fixtures/live/demo/DEMO-FAMILY/`). diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index e3e55340..68211b93 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -42,7 +42,11 @@ a Ctrl+click on a link to the current place opens nothing; on the son's page a two planted CPSR results to act on name their genes and variants nowhere (their card included) until "Show the result" is clicked, and each click - opens one result, there and on the finding's own page; DEMO-001's page + opens one result, there and on the finding's own page; they stay closed on + both pages with family.json unavailable, with a state cut short (no + plugins) and with no state at all (frames.json unavailable), and the son's + search lists no hit and suggests no term from their closed card while a + term of an open card is still found; DEMO-001's page mid-replay and at the end leads with the report (a short run band above the lead while it runs, none once finished), ends with the step timeline folded, keeps the wire to at most eight one-sentence lines with the plain @@ -50,7 +54,8 @@ with its absent and undetermined lines folded. Checks 1, 2, 3, 5, the widgets (person.js without its widget line), the family depth (family.json with the son's stretches emptied), the phone width and the son's closed result (person.js whose cards ignore -the veil), the phone page (person.js that never folds, and a route that opens only the nearest fold), the +the veil, person.js and finding.js that read a missing state as a standalone adult, and search.js +without its filter), the phone page (person.js that never folds, and a route that opens only the nearest fold), the folds (a flagged section 5 card folded, a redraw that forgets the folds, a Ctrl+click that opens one) and the person page's layout (a wire showing every line or every plain finish, a card showing every line, a step timeline left open) also run against a planted mistake and must @@ -510,6 +515,10 @@ def veil_checks(b, base, out): with open(js, "w") as f: f.write(keep_js.replace("} else if (closedFs.length) {", "} else if (false) {")) control("person.js whose cards ignore the veil", not veil_problems(*veil_text(b, base))) + with open(js, "w") as f: + f.write(keep_js) + fid = [fd["id"] for fd in res["findings"] if fd["card"] == "cpsr"][0] + unknown_checks(b, base, out, fid) finally: with open(rj, "w") as f: f.write(keep) @@ -519,6 +528,136 @@ def veil_checks(b, base, out): shutil.rmtree(d, ignore_errors=True) +# finding.js as it was: a missing state or one with no plugins read as a standalone adult. +OLD_FINDING = ("if (!st || !st.plugins) { return Promise.resolve(UNKNOWN); }\n var fam = st.plugins.family;", + "var fam = st && st.plugins && st.plugins.family;") + + +def closed_pages(b, base, fid): + """The son's person page and one planted finding's page, before any click: their text once each has drawn + (or after 10 s, for a page whose state never comes).""" + b.go("about:blank") + b.wait("location.href === 'about:blank'") + b.go(f"{base}/runs/{OTHERS[1]}/") + b.wait("document.documentElement.dataset.pgpResults && document.documentElement.dataset.pgpStatus === 'ready' " + "&& 'drawn'", 10) + time.sleep(0.5) + person = b.eval("document.body.textContent") or "" + b.go(f"{base}/runs/{OTHERS[1]}/finding/{fid}/") + b.wait("document.documentElement.dataset.pgpStatus === 'ready' && 'drawn'", 10) + page = b.eval("document.body.textContent") or "" + return person, page + + +def closed_problems(person, page): + out = [f"{w} on the person page" for pair in VEIL for w in pair if w in person] + out += [f"{w} on the finding page" for pair in VEIL for w in pair if w in page] + if "A result to act on" not in page: + out.append("the finding page does not show its closed title: " + page[:200]) + return out + + +def search_text(b, base, q): + """[hits, the search panel's text] for q typed on the son's page.""" + b.go("about:blank") + b.wait("location.href === 'about:blank'") + b.go(f"{base}/runs/{OTHERS[1]}/") + b.wait("document.documentElement.dataset.pgpResults && document.querySelector('.veil') && " + "!document.querySelector('input[type=search]').disabled && 'ready'", 15) + b.eval("var i = document.querySelector('input[type=search]'); i.value = %s; i.dispatchEvent(new Event('input')); true" + % json.dumps(q)) + hits = b.wait("document.documentElement.dataset.pgpSearch", 5) + return hits, b.eval("(document.querySelector('.search-results') || {}).textContent || ''") or "" + + +def search_problems(b, base): + out = [] + hits, text = search_text(b, base, "GENEQ") + if hits != "0" or "c.77del" in text: + out.append(f"GENEQ, a gene of a closed card: {hits} hits, {text[:200]}") + hits, text = search_text(b, base, "GENE") + if "GENEQ" in text or "GENER" in text or "GENEOPEN" not in text: + out.append(f"the suggestions for GENE: {text[:300]}") + hits, text = search_text(b, base, "GENEOPEN") + if hits != "1": + out.append(f"GENEOPEN, a gene of an open card, not found: {hits} {text[:200]}") + return out + + +def unknown_checks(b, base, out, fid): + """The son's results to act on stay closed when the page cannot learn his family role, on the person page + and on the finding's own page, and his search names nothing his closed card holds.""" + sd = os.path.join(out, "runs", OTHERS[1]) + fj = os.path.join(out, "runs", FAM, "family.json") + frames_path = os.path.join(sd, "frames.json") + idx = os.path.join(sd, f"{OTHERS[1]}_search_index.json") + pj, fjs, sj = (os.path.join(out, "assets", x) for x in ("person.js", "finding.js", os.path.join("plugins", "search.js"))) + keep = {} + for p in (frames_path, pj, fjs, sj) + ((idx,) if os.path.isfile(idx) else ()): + with open(p) as f: + keep[p] = f.read() + frames = json.loads(keep[frames_path]) + + def write(p, text): + with open(p, "w") as f: + f.write(text) + + def item(sec, text): + return {"section": sec, "anchor": sec, "title": sec, "step": "24", "state": "ok", "source": "summary.json", + "count": 1, "mentions": [{"text": text, "locus": None}]} + try: + # The search: an index naming the two planted genes in the closed CPSR card, and one gene of an open card. + write(idx, json.dumps({"schema_version": 1, "sample": OTHERS[1], "terms": { + "GENEQ": [item("cpsr", "GENEQ c.77del")], "GENER": [item("cpsr", "GENER c.88dup")], + "GENEOPEN": [item("roh", "GENEOPEN in a run of homozygosity")]}})) + with_search = copy.deepcopy(frames) + for fr in with_search["frames"]: + fr["state"].setdefault("plugins", {})["search"] = {"file": os.path.basename(idx), "mtime": 1} + write(frames_path, json.dumps(with_search)) + bad = search_problems(b, base) + check("browser: the son's search lists no hit and suggests no term of his closed card, and still finds an open one", + not bad, bad) + write(sj, keep[sj].replace("function hidden(it) { return !!(", "function hidden(it) { return false && !!(")) + control("search.js without its filter", not search_problems(b, base)) + write(sj, keep[sj]) + write(frames_path, keep[frames_path]) + + # family.json unavailable + os.replace(fj, fj + ".away") + try: + bad = closed_problems(*closed_pages(b, base, fid)) + finally: + os.replace(fj + ".away", fj) + check("browser: with family.json unavailable the son's results to act on stay closed on both pages", not bad, bad) + + # a state cut short: no plugins at all + cut = copy.deepcopy(frames) + for fr in cut["frames"]: + fr["state"].pop("plugins", None) + write(frames_path, json.dumps(cut)) + bad = closed_problems(*closed_pages(b, base, fid)) + check("browser: with a state that has no plugins the son's results to act on stay closed on both pages", not bad, bad) + write(pj, keep[pj].replace("return !state || !state.plugins || !!(fam && fam.unknown);", "return false;")) + write(fjs, keep[fjs].replace(OLD_FINDING[0], OLD_FINDING[1])) + control("person.js and finding.js that read a state with no plugins as a standalone adult", + not closed_problems(*closed_pages(b, base, fid))) + write(pj, keep[pj]) + write(fjs, keep[fjs]) + + # no state at all + os.remove(frames_path) + bad = closed_problems(*closed_pages(b, base, fid)) + check("browser: with no state at all (frames.json unavailable) the son's results to act on stay closed on both pages", + not bad, bad) + write(fjs, keep[fjs].replace(OLD_FINDING[0], OLD_FINDING[1])) + control("finding.js that reads a missing state as a standalone adult", not closed_problems(*closed_pages(b, base, fid))) + finally: + for p, text in keep.items(): + write(p, text) + if idx not in keep and os.path.isfile(idx): + os.remove(idx) + + # The person page reads as a report: what (frame) -> the page's layout facts, read in the browser. LAYOUT_JS = """JSON.stringify((function () { var main = document.getElementById('content'), fold = document.getElementById('progress-fold'); diff --git a/tests/test_live_family_depth.py b/tests/test_live_family_depth.py index afd08e46..90323718 100644 --- a/tests/test_live_family_depth.py +++ b/tests/test_live_family_depth.py @@ -17,11 +17,16 @@ 8. every blood group system step 41 typed gets a row (ABO and RhD first, then the rest in order); 9. each child's trio status: assessed with both parents sequenced and the file here; not assessed with the reason when a parent is not sequenced or not in the joint call; a missing file and a - file linking out of the folder told apart, the link's target never written. + file linking out of the folder told apart, the link's target never written; + 10. compound-het pairs come only from the child's step 23 candidates: assessed, not, or not known; + 11. a member's summary.json of a shape the report never writes (sections a list, CYP2D6 calls a + string, a CPIC gene named by a list) costs that member's column only: it says "summary not + readable" there, and every other member's column is as before. Each check runs again on a planted mistake and must fail there: the segments read from the wrong child (1), the pairs left in file order (2), a trio file planted (3), a member's summary dropped (4), the inside check bypassed (5), "ok" planted in a row (6), the stale section made complete (7), the -systems list emptied (8), the link replaced by a plain file (9). A check that cannot fail is no check. +systems list emptied (8), the link replaced by a plain file (9), a bad summary read +silently as empty (11). A check that cannot fail is no check. Run: python3 tests/test_live_family_depth.py """ @@ -334,6 +339,36 @@ class Open(Live): check("10. candidates: none in the run, here, newer than compound_het.tsv, a link out of the run, no run", got == ("absent", "here", "newer", "absent", None), got) control("the inside check bypassed for the candidates", fam.candidates_given(Open, run3, "DEMO-003") == "absent") + + def malformed_problems(): + out = [] + for bad in ({"sections": ["not", "a", "mapping"]}, {"cyp2d6": "CYP2D6 *1/*4"}, + {"sections": {"cpic": {"state": "ok", "data": {"genes": [{"gene": ["CYP2D6"]}]}}}}): + try: + got = fam.side_by_side(["A", "B", "C"], dict(sums, B=bad)) + except Exception as e: # noqa: BLE001 -- the very thing under test + out.append(f"{bad}: raised {type(e).__name__}") + continue + cols = {r["label"]: r["values"] for r in got["rows"]} + want = {r["label"]: r["values"] for r in fam.side_by_side(["A", "B", "C"], dict(sums, B=None))["rows"]} + for label, v in want.items(): + if cols.get(label, [None] * 3)[0] != v[0] or cols.get(label, [None] * 3)[1] != "summary not readable": + out.append(f"{bad}: {label} {cols.get(label)}") + return out + bad = malformed_problems() + check("11. a member's summary of a shape the report never writes costs that member's column only", not bad, bad) + real = fam.side_values + + def quiet(summary): # the planted mistake: a bad summary read as one with no rows, never said + try: + return real(summary) + except (AttributeError, TypeError): + return [] + fam.side_values = quiet + try: + control("a bad summary shown as '.' in every row, as if it had none of them", not malformed_problems()) + finally: + fam.side_values = real finally: shutil.rmtree(work, ignore_errors=True) if FAILS: diff --git a/tests/test_live_journal.py b/tests/test_live_journal.py index 136fa560..776746f6 100644 --- a/tests/test_live_journal.py +++ b/tests/test_live_journal.py @@ -18,10 +18,17 @@ with the run and the id filled in and the server's Content-Security-Policy for an id of the run's findings, and a 404 for an id that is not one, for a path-like id and for a run with no such page; 4. /runs.json of a running run counts from the summary its page last read only while that read is - current (same steps done, same summary.json), and says whether a clinical screen has a result. + current (same steps done, same summary.json), and says whether a clinical screen has a result; + 5. one row the rules cannot read (a ClinVar hit whose stars is no number, a CPSR row that is a string, + a carrier row of the wrong type) costs that row only: results.json keeps every card and every other + finding, the row becomes a finding on its own card that names no gene, at the highest level its rule + can give (to act on for CPSR, the carrier list and AnnotSV, so a child's page veils the card; worth a + look for ClinVar), and the card whose table could not be drawn says so. Each check runs again on a planted mistake and must fail there: 1 with the CPIC rule's level swapped to review, 2 with the act findings left out of the lead and with their names unmarked and with the -old lead on a run with no clinical screen, 3 with the route answering any id, 4 with a current read. A check +old lead on a run with no clinical screen, 3 with the route answering any id, 4 with a current read, 5 with the rules run without the row guard +and with the guard's level hard-coded to review. +A check that cannot fail is no check. Run: python3 tests/test_live_journal.py @@ -179,11 +186,15 @@ def route_problems(port, fid): code, h, _ = request(port, "GET", path) if code != 303 or h.get("location") != "/login?next=" + urllib.parse.quote(path): out.append(f"finding page without a session: {code} {h.get('location')}") - code, h, _ = request(port, "POST", "/login", urllib.parse.urlencode({"password": PASSWORD, "next": "/"}), + # The login form posts the next the redirect gave it: the browser lands back on the finding. + nxt = urllib.parse.parse_qs(urllib.parse.urlsplit(h.get("location") or "").query).get("next", ["/"])[0] + code, h, _ = request(port, "POST", "/login", urllib.parse.urlencode({"password": PASSWORD, "next": nxt}), {"Content-Type": "application/x-www-form-urlencoded"}) m = re.match(r"owngenome_live=([^;]+)", h.get("set-cookie", "")) if not m: return out + [f"login {code}"] + if code != 303 or h.get("location") != path: + out.append(f"the login with the finding as next goes to {code} {h.get('location')}, not back to {path}") ck = {"Cookie": f"owngenome_live={m.group(1)}"} code, h, body = request(port, "GET", path, headers=ck) page = body.decode("utf-8", "replace") @@ -200,6 +211,53 @@ def route_problems(port, fid): return out +def bad_row_problems(): + """What one unreadable row of each kind takes from results.json beyond the row itself.""" + good = live_results.build("MADE-UP", copy.deepcopy(SUMMARY)) + s = copy.deepcopy(SUMMARY) + hit = s["sections"]["clinvar"]["data"]["hits"][0] + s["sections"]["clinvar"]["data"]["hits"].append(dict(hit, gene="GENEZ", pos=2000, stars="two")) + s["sections"]["cpsr"]["data"]["plp"].append("GENEY c.1del") + s["sections"]["carrier"]["data"]["carriers"].append(["GENEX"]) + try: + res = live_results.build("MADE-UP", s) + except Exception as e: # noqa: BLE001 -- the very thing under test + return [f"build raised {type(e).__name__}: {e}"] + out = [] + + def cards(r): + return [c["id"] for ch in r["chapters"] for c in ch["sections"]] + if cards(res) != cards(good): + out.append(f"cards {cards(res)} != {cards(good)}") + ids = {f["id"] for f in res["findings"]} + out += [f"lost {f['id']}" for f in good["findings"] if f["id"] not in ids] + new = [f for f in res["findings"] if f["id"] not in {g["id"] for g in good["findings"]}] + # A rule that can give a finding to act on gives its unreadable row that level, so a child's page veils + # the card (its table may still draw the row); the ClinVar rule never gives one. + want = {"carrier-list": "act", "clinvar": "review", "cpsr": "act"} + if sorted(f["card"] for f in new) != sorted(want) or any( + f["level"] != want[f["card"]] or f["gene"] or re.search(r"GENE[XYZ]", f["title"] + f["text"]) for f in new): + out.append(f"the unreadable rows: {[(f['card'], f['level'], f['gene'], f['title']) for f in new]}") + # An AnnotSV class-5 row whose gene list the rule cannot join: the card's table draws its first genes, + # so the card must hold a finding to act on, or a child's page shows the row unveiled. + s = copy.deepcopy(SUMMARY) + s["sections"]["annotsv"] = ok({"rows": [{"sv": "chr1:1000-9000", "type": "deletion", "length": -8000, + "acmg_class": "5", "genes": ["GENEX", "GENEY", "GENEZ", None], "gt": "0/1"}]}) + try: + sv_res = live_results.build("MADE-UP", s) + except Exception as e: # noqa: BLE001 -- the very thing under test + return out + [f"build with an unreadable AnnotSV row raised {type(e).__name__}: {e}"] + byid = {f["id"]: f for f in sv_res["findings"]} + sv = [c for ch in sv_res["chapters"] for c in ch["sections"] if c["id"] == "structural-variants"] + lv = [byid[i]["level"] for i in sv[0]["findings"]] if sv else None + if not lv or "act" not in lv: + out.append(f"the AnnotSV card with an unreadable class-5 row holds no finding to act on: {lv}") + cv = [c for ch in res["chapters"] for c in ch["sections"] if c["id"] == "clinvar"][0] + if not any("could not be drawn" in x for x in cv["lines"]): + out.append(f"the ClinVar card does not say it could not be drawn: {cv['lines'][:2]}") + return out + + def main(): work = tempfile.mkdtemp(prefix="test_live_journal.") srvs = [] @@ -316,6 +374,28 @@ def act_count(key): finally: live.base_state = real_state live.headline_cache.pop(SAMPLE, None) + # 5. one bad row costs one row + bad = bad_row_problems() + check("5. one unreadable row costs that row only: every card and every other finding stays", not bad, bad) + real_row = live_results._Row + + class Unguarded(real_row): # the planted mistake: the rules run without the row guard + def __exit__(self, *a): + return False + live_results._Row = Unguarded + try: + control("the rules without the row guard", not bad_row_problems()) + finally: + live_results._Row = real_row + + class AlwaysReview(real_row): # the planted mistake: every unreadable row worth a look only + def __init__(self, out, card_id, level="review"): + super().__init__(out, card_id, "review") + live_results._Row = AlwaysReview + try: + control("the row guard with its level hard-coded to review", not bad_row_problems()) + finally: + live_results._Row = real_row finally: for s in srvs: stop(s) diff --git a/tests/test_owngenome_live_family.py b/tests/test_owngenome_live_family.py index d0421505..8cb7c867 100755 --- a/tests/test_owngenome_live_family.py +++ b/tests/test_owngenome_live_family.py @@ -9,7 +9,9 @@ chromosome map (child_a has one, child_b none), the relationship check, the shared segments, the couple screen counted from step 39's table, the plan, and exactly the result files that exist; - 3. child_a's state links to family_a; a sample in no family gets null; + 3. child_a's state links to family_a; a sample in no family gets null; while a + family's pedigree cannot be read, a sample no readable family lists gets + "unknown" (it may be that family's child), and child_a still its family; 4. GET /family//pedigree.svg is the drawing of the declared parents; a listed file is served as written; notes.txt (in the folder, written by no step), a name with '..', a symlink out of the folder, a sample's @@ -238,7 +240,12 @@ def main(): code = app.get("/family/family_empty/pedigree.svg")[0] check("a members file with a header and no rows: an error on the page, 404 for the pedigree", "error" in (es or {}) and code == 404, (es, code)) + got = (hook(os.path.join(root, "loner"), {}, []), hook(os.path.join(root, "child_a"), {}, [])) + check("while a family cannot be read, a sample no readable family lists is unknown, never in no family", + got == ({"kind": "member", "families": [], "unknown": True}, + {"kind": "member", "families": [{"name": "family_a", "url": "/runs/family_a/"}]}), got) shutil.rmtree(empty) + check("...and once it reads again, in no family", hook(os.path.join(root, "loner"), {}, []) is None) # Planted mistake: child_a's father swapped for its grandfather in the copy. bad = os.path.join(work, "swapped") diff --git a/tests/test_run_state_trace_cache.py b/tests/test_run_state_trace_cache.py index bc997004..639427a1 100644 --- a/tests/test_run_state_trace_cache.py +++ b/tests/test_run_state_trace_cache.py @@ -3,14 +3,19 @@ /runs.json builds the state of every run on the server, and each build read every trace file of every run again. read_trace now keeps the parsed rows of each file, -keyed on its path, mtime and size. Checks: +keyed on its path and the file's identity (device, inode, mtime, ctime, size). Checks: 1. the second read of an unchanged trace file opens nothing and gives the same rows; 2. a file that grew (a task row appended) is read again and gives the new row; 3. the rows handed out are copies: a caller that changes one does not change the cache; 4. two samples of one trace file each get only their own rows from the one parse; - 5. the cache keeps at most TRACE_CACHE_MAX files. + 5. the cache keeps at most TRACE_CACHE_MAX files; + 6. a trace replaced by another file of the same size and mtime (another run's + file moved in, its times kept) is read again and gives the new rows; + 7. a duration cell that is no number ('1.2.3s', '.s') costs that row its end + time only: the file still reads, every other row as before. Check 1 is also run against a planted mistake (a cache that keeps nothing) and must -fail there: a check that cannot fail is no check. +fail there, and check 6 against the old key (mtime and size alone): a check that +cannot fail is no check. Run: python3 tests/test_run_state_trace_cache.py """ @@ -112,6 +117,66 @@ def main(): run_state.read_trace([p], "demo1") check("5. the cache keeps at most TRACE_CACHE_MAX files", len(run_state.TRACE_CACHE) == run_state.TRACE_CACHE_MAX, len(run_state.TRACE_CACHE)) + + # 6. a file replaced by one of the same size and mtime + def replaced(path): + with open(FIX) as f: + text = f.read() + with open(path, "w") as f: + f.write(text) + run_state.TRACE_CACHE.clear() + old = run_state.read_trace([path], "demo1") + st = os.stat(path) + new = path + ".new" + with open(new, "w") as f: # the same length: one task's status COMPLETED becomes FAILED... + f.write(text.replace("\tCOMPLETED\t0\t", "\tFAILED...\t1\t", 1)) + os.utime(new, ns=(st.st_atime_ns, st.st_mtime_ns)) + os.replace(new, path) + st2 = os.stat(path) + got = run_state.read_trace([path], "demo1") + return old, got, (st.st_size, st.st_mtime_ns) == (st2.st_size, st2.st_mtime_ns) + p6 = os.path.join(work, "replaced.txt") + old, got, same = replaced(p6) + check("6. a trace replaced by a file of the same size and mtime is read again", + same and [r["status"] for r in got] != [r["status"] for r in old] and "FAILED..." in [r["status"] for r in got], + (same, [r["status"] for r in old], [r["status"] for r in got])) + real_stat = os.stat + + class OldKey: # the key before: mtime and size alone (os.stat is run_state's os.stat too) + def __init__(self, st): + self.st = st + self.st_dev = self.st_ino = self.st_ctime_ns = 0 + + def __getattr__(self, k): + return getattr(self.st, k) + run_state.os.stat = lambda p, *a, **k: OldKey(real_stat(p, *a, **k)) + try: + old, got, same = replaced(p6) + finally: + run_state.os.stat = real_stat + check("control: a key of mtime and size alone fails check 6", + same and [r["status"] for r in got] == [r["status"] for r in old], (same, [r["status"] for r in got])) + + # 7. a malformed duration + with open(FIX) as f: + text = f.read() + lines = text.splitlines(True) + bad = lines[2].split("\t") + bad[7] = "1.2.3s" + lines[2] = "\t".join(bad) + lines[1] = lines[1].replace("\t2s\t", "\t.s\t", 1) + p7 = os.path.join(work, "durations.txt") + with open(p7, "w") as f: + f.write("".join(lines)) + run_state.TRACE_CACHE.clear() + want = run_state.read_trace([FIX], "demo1") + try: + got = run_state.read_trace([p7], "demo1") + except ValueError as e: + got = e + ok = isinstance(got, list) and len(got) == len(want) and got[0]["end"] is None and got[1]["end"] is None \ + and got[2:] == want[2:] and got[1]["start"] == want[1]["start"] + check("7. a duration that is no number costs its row the end time only", ok, got) finally: shutil.rmtree(work, ignore_errors=True) if FAILS: