feat(live): show which family values are stale, every blood group system, and why a trio file is missing

assets/live/family-page.js

@@ -12,6 +12,7 @@
   if (!U) { return; }
   var el = U.el, add = U.add, root = document.documentElement;
   var RESULT = { shared: 'Both carry', x_linked: 'X-linked', one_partner: 'One partner' };
+  var STALE = ' (older than its input)';
 
   // A chapter: its roman number in brass, its title in the serif.
   var CHAPTER = { 'Pedigree': 'I', 'Who is related how': 'II', 'Couple carrier screen': 'III', 'Each child': 'IV',
@@ -342,11 +343,14 @@
   function children(f) {
     var s = section('Each child', 'children');
     if (!f.state.children.length) { add(s, el('p', { 'class': 'dim' }, 'The pedigree names no child.')); }
-    var plan = {};
-    (f.state.plan || []).forEach(function (r) { plan[r.analysis] = r; });
-    function absent(file, analysis) {
-      var r = plan[analysis];
-      return file + ' is not here' + (r ? ': ' + r.analysis.replace(/_/g, ' ') + ' ' + r.decision + (r.reason ? ' (' + r.reason + ')' : '') : '') + '.';
+    // Whether step 38 assessed this child (family.json trio_status): "None listed" only for a child it assessed;
+    // otherwise why not, and a file that links out of the family folder is named as one, never followed.
+    function empty(child, key, file) {
+      var st = ((f.trio_status || {})[child] || {})[key];
+      if (st && st.assessed) { return 'Assessed (' + st.why + '): none listed in ' + file + ' for this child.'; }
+      if (st && st.assessed === null) { return 'Not known whether assessed for this child: ' + st.why + '.'; }
+      var why = st ? st.why : file + ' is not here';
+      return 'Not assessed for this child: ' + why + '.';
     }
     f.state.children.forEach(function (c, i) {
       var d = el('details', { 'data-child': c.child });
@@ -370,8 +374,7 @@
           rows: dn.map(function (r) { return [r.chrom + ':' + r.pos + ' ' + r.ref + '>' + r.alt, r.child_gt, r.father_gt,
             r.mother_gt, r.child_dp + ' / ' + r.child_gq, r.child_ab, r.quality]; }) }, 8));
       } else {
-        add(d, el('p', { 'class': 'dim' }, (f.trio_files || {})['de_novo.tsv'] ? 'None listed in de_novo.tsv for this child.'
-          : absent('de_novo.tsv', 'trio_phasing')));
+        add(d, el('p', { 'class': 'dim', 'data-trio': 'de_novo' }, empty(c.child, 'de_novo', 'de_novo.tsv')));
       }
       var ch = (f.compound_het || []).filter(function (r) { return r.child === c.child; });
       add(d, el('h3', {}, 'Compound-het pairs'));
@@ -380,8 +383,7 @@
         add(d, U.table({ columns: ['Gene', 'Phase', 'Variant A', 'From', 'Variant B', 'From'],
           rows: ch.map(function (r) { return [r.gene, r.phase, r.variant_a, r.a_from, r.variant_b, r.b_from]; }) }, 8));
       } else {
-        add(d, el('p', { 'class': 'dim' }, (f.trio_files || {})['compound_het.tsv'] ? 'None listed in compound_het.tsv for this child.'
-          : absent('compound_het.tsv', 'trio_phasing')));
+        add(d, el('p', { 'class': 'dim', 'data-trio': 'compound_het' }, empty(c.child, 'compound_het', 'compound_het.tsv')));
       }
       add(s, d);
     });
@@ -395,12 +397,21 @@
     if (!sb || !sb.rows.length) { add(s, el('p', { 'class': 'dim' }, 'No member has a summary yet, or no summary lists these results.')); return s; }
     add(s, el('p', { 'class': 'dim' }, 'Each member’s own report, one column per person: CYP2D6, the CPIC genes ' +
       '(phenotypes in CPIC’s words), HLA, the haplogroups and the blood groups.'));
-    var group = null, rows = [];
+    // A value from a section older than its input (state "stale") says so in its own cell.
+    var group = null, rows = [], old = 0;
     sb.rows.forEach(function (r) {
-      rows.push([r.group === group ? '' : r.group, r.label].concat(r.values));
+      rows.push([r.group === group ? '' : r.group, r.label].concat(r.values.map(function (v, i) {
+        if (!(r.stale || [])[i]) { return v; }
+        old++;
+        return v + STALE;
+      })));
       group = r.group;
     });
     add(s, U.table({ columns: ['', 'Gene or system'].concat(sb.members), rows: rows }, 14));
+    if (old) {
+      add(s, el('p', { 'class': 'dim', 'data-stale': String(old) }, 'Marked' + STALE + ': the member’s result file is from an ' +
+        'earlier run, and the latest run did not write it again; that member’s page says why.'));
+    }
     return s;
   }
 

bin/owngenome_live_family.py

@@ -18,10 +18,14 @@
                                 couple's gene rows, each child's inherited
                                 stretches (<child>_segments.tsv), the de novo and
                                 compound-het candidates (de_novo.tsv,
-                                compound_het.tsv), each member's run progress and
-                                attention count, and the members side by side:
-                                CYP2D6, the CPIC genes, HLA, the haplogroups and
-                                the blood groups (from each summary.json only)
+                                compound_het.tsv) with whether each child was
+                                assessed and why not (trio_status; a file linking
+                                out of the folder is "rejected" in trio_file_state),
+                                each member's run progress and attention count,
+                                and the members side by side: CYP2D6, the CPIC
+                                genes, HLA, the haplogroups and every blood group
+                                system, each value with its stale flag (from each
+                                summary.json only)
 
 state.json gets plugins.family: {"kind": "family", ...} on a family's page,
 {"kind": "member", "families": [...]} on the page of a sample in a family,
@@ -235,69 +239,209 @@ def child_depth(d, order, members):
         rows = read_tsv(os.path.join(d, "%s_segments.tsv" % s))
         if rows:
             segs[s] = [{k: r.get(k) for k in SEGMENT_KEYS} for r in rows]
+    files = {f: file_state(d, f) for f in TRIO_FILES}
     comp = [{k: r.get(k) for k in COMP_KEYS} for r in read_tsv(os.path.join(d, "compound_het.tsv"))]
     comp.sort(key=lambda r: PHASE_ORDER.get(r.get("phase"), 1))
     return {"segments": segs,
             "de_novo": [{k: r.get(k) for k in DE_NOVO_KEYS} for r in read_tsv(os.path.join(d, "de_novo.tsv"))],
             "compound_het": comp,
             # read_tsv's own test: a link out of the family folder counts as not there
-            "trio_files": {f: os.path.isfile(os.path.join(d, f)) and inside(os.path.join(d, f), os.path.abspath(d))
-                           for f in ("de_novo.tsv", "compound_het.tsv")}}
+            "trio_files": {f: files[f] == "here" for f in TRIO_FILES},
+            "trio_file_state": files,
+            "trio_status": trio_status(d, order, members, files)}
+
+
+TRIO_FILES = ("de_novo.tsv", "compound_het.tsv")
+
+
+def file_state(d, f):
+    """'here', 'rejected' (a link out of the family folder: read_tsv does not read it, and the page never
+    shows where it points) or 'missing'."""
+    p = os.path.join(d, f)
+    if not os.path.lexists(p):
+        return "missing"
+    if not inside(p, os.path.abspath(d)):
+        return "rejected"
+    return "here" if os.path.isfile(p) else "missing"
+
+
+def trio_status(d, order, members, files):
+    """Per child, whether step 38 assessed it for de novo candidates and compound-het pairs, and why not:
+    {child: {"de_novo": {"assessed": bool, "why": text}, "compound_het": {...}}}. The rule is step 38's
+    (bin/family_report.py): de novo candidates need both parents sequenced, compound-het pairs a sequenced
+    parent; a file that is not here, or is a link out of the folder, means no child was assessed."""
+    plan = {r.get("analysis"): r for r in read_tsv(os.path.join(d, "family_plan.tsv"))}
+    trio = plan.get("trio_phasing") or {}
+
+    def absent(f):
+        if files[f] == "rejected":
+            return "%s is a link out of the family folder, so this page does not read it" % f
+        why = "%s is not here" % f
+        if trio.get("decision"):
+            why += ": trio phasing %s%s" % (trio["decision"], " (%s)" % trio["reason"] if trio.get("reason") else "")
+        return why
+
+    joint = plan.get("joint_call") or {}
+    called = set((joint.get("members") or "").split(",")) if joint.get("decision") == "runs" else None
+
+    def seq(p):
+        return bool(p) and p in members and bool(members[p]["sequenced"]) and (called is None or p in called)
+
+    need = "sequenced" if called is None else "sequenced and in the joint call"
+
+    def not_called(p):
+        if not members[p]["sequenced"]:
+            return "%s is not sequenced" % p
+        return "%s is not in the joint call (family_plan.tsv)" % p
+
+    def gaps(m):
+        return "; ".join(("no %s is named" % w) if not m[w] else "%s (the %s) is not" % (m[w], w)
+                         for w in ("father", "mother") if not seq(m[w]))
+
+    out = {}
+    for s in order:
+        m = members[s]
+        if not (m["father"] or m["mother"]):
+            continue
+        st = {}
+        if files["de_novo.tsv"] != "here":
+            st["de_novo"] = {"assessed": False, "why": absent("de_novo.tsv")}
+        elif not seq(s):
+            st["de_novo"] = {"assessed": False, "why": not_called(s)}
+        elif seq(m["father"]) and seq(m["mother"]):
+            st["de_novo"] = {"assessed": True, "why": "both parents are sequenced"}
+        else:
+            st["de_novo"] = {"assessed": False, "why": "de novo candidates need both parents %s: %s" % (need, gaps(m))}
+        if files["compound_het.tsv"] != "here":
+            st["compound_het"] = {"assessed": False, "why": absent("compound_het.tsv")}
+        elif not seq(s):
+            st["compound_het"] = {"assessed": False, "why": not_called(s)}
+        elif seq(m["father"]) or seq(m["mother"]):
+            st["compound_het"] = {"assessed": True, "why": "a parent is sequenced; pairs come only from the child's "
+                                  "clinical-filter candidates (step 23), so with none given no pair is listed"}
+        else:
+            st["compound_het"] = {"assessed": False, "why": "compound-het pairs need a parent %s" % need}
+        out[s] = st
+    return out
+
+
+def candidates_given(live, d, sample, since=None):
+    """Whether the clinical filter (step 23) wrote <sample>'s candidates, the only variants step 38 pairs:
+    "here", "absent" (the run is here without them; a file linking out of the run counts as absent),
+    "newer" (written after since, compound_het.tsv's mtime, so step 38 may not have had them) or None
+    (the run is not on this site: the page cannot tell)."""
+    if not d:
+        return None
+    p = os.path.join(d, "clinical", "%s_clinical.vcf.gz" % sample)
+    if not (os.path.isfile(p) and live.inside(p, os.path.realpath(d)) and live.inside(p, live.root)):
+        return "absent"
+    return "newer" if since is not None and os.path.getmtime(p) > since else "here"
+
+
+def with_candidates(status, given):
+    """trio_status with the compound-het line told by the child's step 23 candidates ({child: one of
+    candidates_given's answers}). assessed is None where the page cannot tell."""
+    for child, st in status.items():
+        c = st.get("compound_het") or {}
+        if not c.get("assessed"):
+            continue
+        g = given.get(child)
+        if g == "here":
+            c["why"] = ("a parent is sequenced and the child's clinical-filter candidates (step 23) are here; "
+                        "pairs come only from them")
+        elif g == "absent":
+            st["compound_het"] = {"assessed": False, "why": "the clinical filter (step 23) wrote no candidates for "
+                                  "%s (clinical/%s_clinical.vcf.gz), so step 38 had no variant to pair" % (child, child)}
+        elif g == "newer":
+            st["compound_het"] = {"assessed": None, "why": "%s's clinical-filter candidates (step 23) are newer than "
+                                  "compound_het.tsv, so step 38 may not have paired them" % child}
+        else:
+            st["compound_het"] = {"assessed": None, "why": "pairs come only from the child's clinical-filter "
+                                  "candidates (step 23), and %s's run is not on this site to tell whether step 23 "
+                                  "wrote them" % child}
+    return status
 
 
 def _sec(summary, key):
     sec = ((summary or {}).get("sections") or {}).get(key) or {}
     return sec.get("data") or {} if sec.get("state") in ("ok", "stale") else None
 
 
+def _stale(summary, *keys):
+    secs = (summary or {}).get("sections") or {}
+    return any(((secs.get(k) or {}).get("state")) == "stale" for k in keys)
+
+
 def side_values(summary):
-    """[(group, label, value)] of one member's summary.json, as its report prints them; nothing computed."""
+    """[(group, label, value, stale)] of one member's summary.json, as its report prints them; nothing
+    computed. stale is true when the value comes from a section older than its input (state "stale")."""
     out = []
     c = (summary or {}).get("cyp2d6") or {}
     calls = ", ".join("%s %s" % (k, v) for k, v in (c.get("calls") or {}).items() if v)
     if calls:
         agree = c.get("agree")
-        out.append(("Medicines", "CYP2D6 (callers)", calls + (" (callers agree)" if agree else " (callers differ)" if agree is False
-                                                     else "")))
+        # summary.json's cyp2d6 block reads pypgx and Cyrius only when complete; its PharmCAT call is cpic's
+        out.append(("Medicines", "CYP2D6 (callers)", calls + (" (callers agree)" if agree else " (callers differ)"
+                                                             if agree is False else ""),
+                    bool((c.get("calls") or {}).get("PharmCAT")) and _stale(summary, "cpic")))
     cpic = _sec(summary, "cpic")
     for g in (cpic or {}).get("genes") or []:
         ph = g.get("phenotype")
-        out.append(("Medicines", g.get("gene"), "%s%s" % (g.get("diplotype") or ".", ' "%s"' % ph if ph else "")))
+        out.append(("Medicines", g.get("gene"), "%s%s" % (g.get("diplotype") or ".", ' "%s"' % ph if ph else ""),
+                    _stale(summary, "cpic")))
     for l in (_sec(summary, "hla") or {}).get("loci") or []:
-        out.append(("HLA", l.get("gene"), " ".join(l.get("alleles") or []) or "not typed"))
+        out.append(("HLA", l.get("gene"), " ".join(l.get("alleles") or []) or "not typed", _stale(summary, "hla")))
     mt = _sec(summary, "haplogroup")
     if mt is not None:
-        out.append(("Haplogroups", "Mitochondrial", mt.get("haplogroup") or "."))
+        out.append(("Haplogroups", "Mitochondrial", mt.get("haplogroup") or ".", _stale(summary, "haplogroup")))
+    ykey = "y_tree" if _sec(summary, "y_tree") else "y_haplogroup"
     y = _sec(summary, "y_tree") or _sec(summary, "y_haplogroup")
     if y is not None:
-        out.append(("Haplogroups", "Y chromosome", y.get("haplogroup") or "."))
+        out.append(("Haplogroups", "Y chromosome", y.get("haplogroup") or ".", _stale(summary, ykey)))
     bt = _sec(summary, "bloodtype")
     if bt is not None:
-        out.append(("Blood groups", "ABO", bt.get("abo") or next((x.get("phenotype") for x in bt.get("systems") or []
-                                                                  if x.get("system") == "ABO"), ".")))
-        out.append(("Blood groups", "RhD", bt.get("rhd") or next((x.get("phenotype") for x in bt.get("systems") or []
-                                                                  if x.get("system") == "RhD"), ".")))
+        old = _stale(summary, "bloodtype")
+        systems = [x for x in bt.get("systems") or [] if x.get("system")]
+        by = {x["system"]: x.get("phenotype") or "." for x in systems}
+        out.append(("Blood groups", "ABO", bt.get("abo") or by.get("ABO", "."), old))
+        out.append(("Blood groups", "RhD", bt.get("rhd") or by.get("RhD", "."), old))
+        # every other system step 41 typed, in its order (Kell, Duffy, Kidd, MNS...)
+        for x in systems:
+            if x["system"] not in ("ABO", "RhD"):
+                out.append(("Blood groups", x["system"], by[x["system"]], old))
     return out
 
 
 def side_by_side(names, summaries):
-    """The members' results in one table: {members, rows: [{group, label, values}]}. A member with no
-    summary.json yet says so in its column; a row a member's summary does not have is '.'."""
+    """The members' results in one table: {members, rows: [{group, label, values, stale}]}. A member with
+    no summary.json yet says so in its column; a row a member's summary does not have is '.'. stale holds,
+    per member, whether that value comes from a section older than its input."""
     per = {n: side_values(summaries.get(n)) for n in names}
-    rows, seen = [], {}
+    # Each member's rows keep their own order (step 41's for the blood groups): a row first seen in a later
+    # member goes right after the row before it in that member's list, not at the end.
+    keys = []
     for n in names:
-        for g, label, _ in per[n]:
-            if (g, label) not in seen:
-                seen[(g, label)] = len(rows)
-                rows.append({"group": g, "label": label, "values": []})
+        prev = None
+        for g, label, _, _ in per[n]:
+            if (g, label) not in keys:
+                keys.insert(keys.index(prev) + 1 if prev else len(keys), (g, label))
+            prev = (g, label)
     groups = ["Medicines", "HLA", "Haplogroups", "Blood groups"]
-    rows.sort(key=lambda r: (groups.index(r["group"]) if r["group"] in groups else 9, seen[(r["group"], r["label"])]))
+    import bloodtype_report  # step 41's own order of the systems, for systems two members each lack one of
+    step41 = {s: i for i, (s, _) in enumerate(bloodtype_report.SYSTEMS)}
+    pos = {k: i for i, k in enumerate(keys)}
+    keys.sort(key=lambda k: (groups.index(k[0]) if k[0] in groups else 9,
+                             step41.get(k[1], len(step41)) if k[0] == "Blood groups" else 0, pos[k]))
+    rows = [{"group": g, "label": label, "values": [], "stale": []} for g, label in keys]
     for r in rows:
         for n in names:
             if summaries.get(n) is None:
                 r["values"].append("no summary yet")
+                r["stale"].append(False)
             else:
-                r["values"].append(next((v for g, lab, v in per[n] if (g, lab) == (r["group"], r["label"])), "."))
+                hit = next((x for x in per[n] if (x[0], x[1]) == (r["group"], r["label"])), None)
+                r["values"].append(hit[2] if hit else ".")
+                r["stale"].append(bool(hit and hit[3]))
     return {"members": list(names), "rows": rows}
 
 
@@ -368,6 +512,14 @@ def page_data(req, m):
                                  "planned": len(planned), "done": sum(1 for s in planned if s["state"] == "done"),
                                  "attention": n, "results": mtime is not None}
         out["runs"] = runs
+        given = {}
+        comp_p = os.path.join(d, "compound_het.tsv")
+        since = os.path.getmtime(comp_p) if os.path.isfile(comp_p) else None
+        for c in out.get("trio_status") or {}:
+            r = live.run(c) if live else None
+            given[c] = (candidates_given(live, r["dir"] or os.path.join(live.root, r["name"]), c, since)
+                        if r and r.get("kind") != "family" else None)
+        with_candidates(out.get("trio_status") or {}, given)
         # The members side by side, from each sequenced member's summary.json alone (read as the home
         # page's attention count reads it: never a file linking out of its run).
         seq = [p["sample"] for p in out["state"].get("members") or [] if p.get("sequenced")]