From 1058e27299ccfd8f2702832ca00067fc2fe7f734 Mon Sep 17 00:00:00 2001 From: Benjamin Nemoz Date: Tue, 17 Nov 2026 16:18:24 +0000 Subject: [PATCH 1/2] feat: directional UMI collapsing per (well, barcode) before UMI counting Adds pairplex/umi.py:collapse_umis_directional (UMI-tools directional adjacency, neighbour generation, N-containing UMIs absorbed but never absorbing). process_droplet collapses UMIs per droplet partition: umi_raw keeps the raw UMI, metadata umis is the corrected count and umis_raw the raw count; min_cluster_umis filters on corrected counts. run(umi_collapse='directional'|'none'), CLI --umi-collapse; run(legacy=True) / --legacy forces 'none' (v2-base behaviour). Co-Authored-By: Claude Opus 5.5 --- pairplex/cli/pairplex.py | 16 +++ pairplex/pairplex.py | 25 ++++ pairplex/tests/test_umi.py | 278 +++++++++++++++++++++++++++++++++++++ pairplex/umi.py | 131 +++++++++++++++++ pairplex/utils.py | 26 ++++ 5 files changed, 476 insertions(+) create mode 100644 pairplex/tests/test_umi.py create mode 100644 pairplex/umi.py diff --git a/pairplex/cli/pairplex.py b/pairplex/cli/pairplex.py index 8a39506..ce40bb1 100644 --- a/pairplex/cli/pairplex.py +++ b/pairplex/cli/pairplex.py @@ -98,6 +98,18 @@ def version(): default="human", help="Germline database for V(D)J assignment", ) +@click.option( + "--umi-collapse", + "umi_collapse", + type=click.Choice(["directional", "none"]), + default="directional", + help="UMI error correction within each (well, barcode): UMI-tools directional (default) or none", +) +@click.option( + "--legacy", + is_flag=True, + help="Reproduce v2-base behaviour exactly (forces --umi-collapse none)", +) @click.option("--quiet", is_flag=True, help="Whether to suppress output") @click.option( "--debug", @@ -118,6 +130,8 @@ def run( merge_paired_reads: bool, receptor: str, germline_database: str, + umi_collapse: str, + legacy: bool, quiet: bool, debug: bool, ): @@ -135,6 +149,8 @@ def run( merge_paired_reads=merge_paired_reads, receptor=receptor, germline_database=germline_database, + umi_collapse=umi_collapse, + legacy=legacy, quiet=quiet, debug=debug, ) diff --git a/pairplex/pairplex.py b/pairplex/pairplex.py index 85c4ed5..e311fc6 100644 --- a/pairplex/pairplex.py +++ b/pairplex/pairplex.py @@ -49,6 +49,8 @@ def run( merge_paired_reads: bool = False, receptor: str = "bcr", germline_database: str = "human", + umi_collapse: str = "directional", + legacy: bool = False, quiet: bool = False, debug: bool = False, ) -> list: @@ -105,6 +107,19 @@ def run( The germline database to use. Options are anything accepted by `abstar.run`, including custom databases if available. + umi_collapse : str + UMI error correction, applied within each (well, droplet barcode) before any UMI counting. + ``"directional"`` (default) uses the UMI-tools directional adjacency method: UMI ``b`` is + absorbed by UMI ``a`` when they differ at one position and ``reads(a) >= 2 * reads(b) - 1``, + transitively, processing UMIs by descending read count (see :mod:`pairplex.umi`, which also + documents how UMIs containing ``N`` are handled). The metadata ``umis`` column is then the + corrected count, a ``umis_raw`` column holds the count before collapsing, and + ``min_cluster_umis`` filters on corrected counts. ``"none"`` counts raw UMIs (v2-base + behaviour, no ``umis_raw`` column). + + legacy : bool + If ``True``, reproduce the v2-base behaviour exactly: forces ``umi_collapse="none"``. + quiet : bool Whether to suppress the progress bar and other update messages. @@ -120,6 +135,15 @@ def run( """ parameters = dict(locals()) # captured first: exactly the arguments of this call + # legacy mode forces every v2 behaviour option to its v2-base value + if legacy: + umi_collapse = "none" + if umi_collapse not in ("directional", "none"): + raise ValueError( + f"umi_collapse must be 'directional' or 'none', got {umi_collapse!r}" + ) + parameters["umi_collapse_effective"] = umi_collapse + # print splash screen if not quiet: print_splash(include_version=True) @@ -346,6 +370,7 @@ def run( "min_cluster_fraction": min_cluster_fraction, "consensus_downsample": consensus_downsample, "clustering_threshold": clustering_threshold, + "umi_collapse": umi_collapse, "quiet": quiet, "debug": debug, } diff --git a/pairplex/tests/test_umi.py b/pairplex/tests/test_umi.py new file mode 100644 index 0000000..be295db --- /dev/null +++ b/pairplex/tests/test_umi.py @@ -0,0 +1,278 @@ +import itertools +import random + +import polars as pl +import pytest + +from pairplex.umi import collapse_umi_column, collapse_umis_directional +from pairplex.utils import process_droplet + + +def _hamming(a, b): + return sum(x != y for x, y in zip(a, b)) + + +def _reference_directional(cnt): + """O(n^2) reference implementation, written the UMI-tools way: build the full directed + adjacency list, BFS components from nodes by descending count (ties by UMI), first come first + served, N-containing UMIs never absorb.""" + umis = sorted(cnt, key=lambda u: (-cnt[u], u)) + adj = {u: [] for u in umis} + for a, b in itertools.permutations(umis, 2): + if "N" in a or len(a) != len(b): + continue + if _hamming(a, b) == 1 and cnt[a] >= 2 * cnt[b] - 1: + adj[a].append(b) + rep = {} + for root in umis: + if root in rep: + continue + seen, queue = {root}, [root] + while queue: + n = queue.pop(0) + for m in adj[n]: + if m not in seen: + seen.add(m) + queue.append(m) + for u in seen: + rep.setdefault(u, root) + return rep + + +# ---------------------------------------------------------------- classic examples + + +def test_singleton(): + assert collapse_umis_directional({"AAAAAAAAAA": 1}) == {"AAAAAAAAAA": "AAAAAAAAAA"} + + +def test_empty(): + assert collapse_umis_directional({}) == {} + + +def test_simple_absorption(): + m = collapse_umis_directional({"AAAAAAAAAA": 10, "AAAAAAAAAC": 2}) + assert m == {"AAAAAAAAAA": "AAAAAAAAAA", "AAAAAAAAAC": "AAAAAAAAAA"} + + +def test_count_rule_boundary(): + # 2*count(b) - 1: a=5, b=3 -> 5 >= 5 absorbs; a=5, b=4 -> 5 >= 7 is false + assert collapse_umis_directional({"AAAAAAAAAA": 5, "AAAAAAAAAC": 3})["AAAAAAAAAC"] == "AAAAAAAAAA" + assert collapse_umis_directional({"AAAAAAAAAA": 5, "AAAAAAAAAC": 4})["AAAAAAAAAC"] == "AAAAAAAAAC" + + +def test_hamming_two_not_absorbed(): + m = collapse_umis_directional({"AAAAAAAAAA": 100, "AAAAAAAACC": 1}) + assert m["AAAAAAAACC"] == "AAAAAAAACC" + + +def test_chain_is_transitive(): + # a(9) -> b(4) -> c(1): c is 2 substitutions from a, still absorbed through b + a, b, c = "AAAAAAAAAA", "AAAAAAAAAC", "AAAAAAAACC" + m = collapse_umis_directional({a: 9, b: 4, c: 1}) + assert m == {a: a, b: a, c: a} + + +def test_chain_broken_by_count_rule(): + # a(9) -> b(4), but b(4) vs c(3): 4 >= 5 false, so c stays separate + a, b, c = "AAAAAAAAAA", "AAAAAAAAAC", "AAAAAAAACC" + m = collapse_umis_directional({a: 9, b: 4, c: 3}) + assert m == {a: a, b: a, c: c} + + +def test_ties_of_one_merge_deterministically(): + # 1 >= 2*1 - 1: two singleton UMIs at Hamming 1 merge; representative is the smaller string + m = collapse_umis_directional({"AAAAAAAAAC": 1, "AAAAAAAAAA": 1}) + assert m == {"AAAAAAAAAA": "AAAAAAAAAA", "AAAAAAAAAC": "AAAAAAAAAA"} + # order of input does not matter + assert m == collapse_umis_directional({"AAAAAAAAAA": 1, "AAAAAAAAAC": 1}) + + +def test_ties_above_one_do_not_merge(): + m = collapse_umis_directional({"AAAAAAAAAA": 3, "AAAAAAAAAC": 3}) + assert m == {"AAAAAAAAAA": "AAAAAAAAAA", "AAAAAAAAAC": "AAAAAAAAAC"} + + +def test_umi_tools_paper_example(): + # Smith et al. 2017 Fig. 1 style: one true UMI with errors, one independent UMI nearby + cnt = { + "ACGTACGTAC": 456, # true + "ACGTACGTAA": 72, # error of the true one + "ACGTACGTTA": 2, # error of the error (chain) + "ACGTACGTCC": 90, # error of the true one + "TTTTACGTAC": 30, # independent + "TTTTACGTAA": 6, # error of the independent one + } + m = collapse_umis_directional(cnt) + assert {u for u, r in m.items() if r == "ACGTACGTAC"} == { + "ACGTACGTAC", "ACGTACGTAA", "ACGTACGTTA", "ACGTACGTCC" + } + assert m["TTTTACGTAA"] == "TTTTACGTAC" + assert len(set(m.values())) == 2 + + +def test_node_claimed_by_first_root(): + # c is reachable from both a(10) and b(6, not reachable from a); a is processed first + a, b, c = "AAAAAAAAAA", "AAAAAAAACC", "AAAAAAAAAC" + m = collapse_umis_directional({a: 10, b: 6, c: 1}) + assert m == {a: a, b: b, c: a} + + +# ---------------------------------------------------------------- N handling + + +def test_n_umi_absorbed_but_never_absorbs(): + real, nn = "ACGTACGTAC", "ACGTNCGTAC" + assert collapse_umis_directional({real: 5, nn: 1})[nn] == real + # an abundant N UMI does not absorb an N-free neighbour + m = collapse_umis_directional({nn: 50, real: 1}) + assert m == {nn: nn, real: real} + + +def test_n_umi_respects_count_rule_and_double_n(): + real = "ACGTACGTAC" + assert collapse_umis_directional({real: 2, "ACGTNCGTAC": 2})["ACGTNCGTAC"] == "ACGTNCGTAC" + assert collapse_umis_directional({real: 50, "ACGTNNGTAC": 1})["ACGTNNGTAC"] == "ACGTNNGTAC" + + +def test_short_umis_only_compared_at_equal_length(): + m = collapse_umis_directional({"ACGTACGTAC": 10, "ACGTACGTA": 1}) + assert m["ACGTACGTA"] == "ACGTACGTA" + + +# ---------------------------------------------------------------- API forms + + +def test_api_forms_agree(): + reads = ["AAAAAAAAAA"] * 4 + ["AAAAAAAAAC"] + ["GGGGGGGGGG"] * 2 + by_reads = collapse_umis_directional(reads) + by_map = collapse_umis_directional({"AAAAAAAAAA": 4, "AAAAAAAAAC": 1, "GGGGGGGGGG": 2}) + by_lists = collapse_umis_directional(["AAAAAAAAAA", "AAAAAAAAAC", "GGGGGGGGGG"], [4, 1, 2]) + assert by_reads == by_map == by_lists + assert collapse_umi_column(reads) == ["AAAAAAAAAA"] * 5 + ["GGGGGGGGGG"] * 2 + with pytest.raises(ValueError): + collapse_umis_directional(["A", "A"], [1, 2]) + with pytest.raises(ValueError): + collapse_umis_directional(["A"], [1, 2]) + + +# ---------------------------------------------------------------- randomized vs reference + + +@pytest.mark.parametrize("seed", range(20)) +def test_matches_reference_on_random_sets(seed): + rng = random.Random(seed) + L = 6 # short UMIs so Hamming-1 collisions are frequent + true = ["".join(rng.choice("ACGT") for _ in range(L)) for _ in range(rng.randint(1, 15))] + cnt = {} + for t in true: + n = rng.randint(1, 60) + for _ in range(n): + u = list(t) + if rng.random() < 0.25: + u[rng.randrange(L)] = rng.choice("ACGTN" if rng.random() < 0.9 else "N") + u = "".join(u) + cnt[u] = cnt.get(u, 0) + 1 + assert collapse_umis_directional(cnt) == _reference_directional(cnt) + + +def test_large_barcode_is_fast(): + import time + + rng = random.Random(0) + reads = [] + for _ in range(3000): + t = "".join(rng.choice("ACGT") for _ in range(10)) + for _ in range(rng.randint(1, 20)): + u = list(t) + if rng.random() < 0.2: + u[rng.randrange(10)] = rng.choice("ACGT") + reads.append("".join(u)) + t0 = time.perf_counter() + m = collapse_umis_directional(reads) + assert time.perf_counter() - t0 < 5 + assert len(set(m.values())) < len(set(reads)) + + +# ---------------------------------------------------------------- integration via process_droplet + +HEAVY = "CAGGTGCAGCTGGTGCAGTCTGGGGCTGAGGTGAAGAAGCCTGGGGCCTCAGTGAAGGTCTCCTGCAAGGCTTCTGGATACACC" +LIGHT = "GACATCCAGATGACCCAGTCTCCATCCTCCCTGTCTGCATCTGTAGGAGACAGAGTCACCATCACTTGCCGGGCAAGTCAG" + + +def _droplet_df(): + rows = [] + # heavy: true UMI A (6 reads) + 1-error copy (1 read) + independent UMI G (3 reads) + heavy_umis = ["AAAAAAAAAA"] * 6 + ["AAAAAAAAAT"] + ["GGGGGGGGGG"] * 3 + # light: one true UMI (4 reads) + two 1-error copies (1 read each) + light_umis = ["CCCCCCCCCC"] * 4 + ["CCCCCCCCCA", "CCCCCCCCCT"] + for i, u in enumerate(heavy_umis): + rows.append({"umi": u, "barcode": "BC", "seq_id": f"h{i}", "sequence": HEAVY}) + for i, u in enumerate(light_umis): + rows.append({"umi": u, "barcode": "BC", "seq_id": f"l{i}", "sequence": LIGHT}) + return pl.DataFrame(rows) + + +def _by_reads(meta): + return {m["reads"]: m for m in meta} + + +def test_process_droplet_directional(tmp_path): + meta = process_droplet( + "well_BC", _droplet_df(), temp_directory=tmp_path, clustering_threshold=0.9, + min_cluster_reads=3, min_cluster_umis=2, + ) + h, l = _by_reads(meta)[10], _by_reads(meta)[6] + assert (h["umis"], h["umis_raw"]) == (2, 3) + assert (l["umis"], l["umis_raw"]) == (1, 3) + # min_cluster_umis filters on corrected counts: light had 3 raw UMIs but only 1 molecule + assert h["pass_filters"] and not l["pass_filters"] + assert list(h) == ["name", "reads", "umis", "umis_raw", "cluster_fraction", "consensus", "pass_filters"] + + +def test_process_droplet_none_is_v2_base_layout(tmp_path): + meta = process_droplet( + "well_BC", _droplet_df(), temp_directory=tmp_path, clustering_threshold=0.9, + min_cluster_reads=3, min_cluster_umis=2, umi_collapse="none", + ) + h, l = _by_reads(meta)[10], _by_reads(meta)[6] + assert h["umis"] == 3 and l["umis"] == 3 + assert h["pass_filters"] and l["pass_filters"] + assert list(h) == ["name", "reads", "umis", "cluster_fraction", "consensus", "pass_filters"] + + +def test_process_droplet_rejects_unknown_mode(tmp_path): + with pytest.raises(ValueError): + process_droplet("well_BC", _droplet_df(), temp_directory=tmp_path, umi_collapse="cluster") + + +def test_run_legacy_forces_none(tmp_path, monkeypatch): + """legacy=True must pass umi_collapse='none' to process_droplet, whatever umi_collapse says.""" + import pairplex.pairplex as pp + + seen = {} + + class _Stop(Exception): + pass + + def fake_pool(*a, **k): + raise _Stop + + monkeypatch.setattr(pp, "ProcessPoolExecutor", fake_pool) + real_runlog = pp.RunLog + + def capture(path, tool, parameters, inputs): + seen.update(parameters) + return real_runlog(path, tool=tool, parameters=parameters, inputs=inputs) + + monkeypatch.setattr(pp, "RunLog", capture) + fq = tmp_path / "w.fastq" + fq.write_text("@r\nACGT\n+\nIIII\n") + with pytest.raises(_Stop): + pp.run(str(fq), tmp_path / "out", quiet=True, legacy=True, umi_collapse="directional") + assert seen["umi_collapse_effective"] == "none" + with pytest.raises(_Stop): + pp.run(str(fq), tmp_path / "out2", quiet=True) + assert seen["umi_collapse_effective"] == "directional" + with pytest.raises(ValueError): + pp.run(str(fq), tmp_path / "out3", quiet=True, umi_collapse="bogus") diff --git a/pairplex/umi.py b/pairplex/umi.py new file mode 100644 index 0000000..8d0f863 --- /dev/null +++ b/pairplex/umi.py @@ -0,0 +1,131 @@ +# Copyright (c) 2025 Benjamin Nemoz +# Distributed under the terms of the MIT License. +# SPDX-License-Identifier: MIT + +# This file is part of PairPlex. +# PairPlex is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# PairPlex is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# You should have received a copy of the GNU General Public License +# along with PairPlex. If not, see . + +"""UMI error correction. + +PairPlex collapses UMIs within one (well, droplet barcode) group with the *directional* +adjacency method of UMI-tools (Smith, Heger & Sudbery, Genome Res. 2017): + +- nodes are the distinct raw UMIs, weighted by their read count; +- there is a directed edge ``a -> b`` when ``Hamming(a, b) == 1`` and + ``count(a) >= 2 * count(b) - 1``; +- UMIs are visited in descending count; each UMI not yet absorbed becomes a representative and + absorbs every UMI reachable from it along directed edges, transitively: ``a -> b -> c`` puts + ``c`` under ``a`` although ``c`` is two substitutions away from ``a``. A UMI reached from an + earlier representative is never re-assigned. + +Ties in count are broken by the UMI string (lexicographic), so the result is deterministic. +UMI-tools leaves tie order to dict order; the groups are the same, only which tied UMI is named +representative can differ. + +UMIs containing ``N``: ``N`` counts as a mismatch at its position. An N-containing UMI never +absorbs anything (it would be Hamming-1 from up to four different real UMIs, so it is no evidence +for any of them), but it can be absorbed by an N-free UMI at Hamming distance 1 that satisfies the +count rule (e.g. ``ACGTNCGTAC`` under ``ACGTACGTAC``). An N-containing UMI with no such parent +stays its own UMI. Characters other than ``ACGTN`` are compared literally and never generate +neighbours. + +Neighbour generation (``4 * L`` single substitutions over ``ACGTN`` per UMI) keeps the cost +linear in the number of distinct UMIs, so barcodes with thousands of reads are cheap. +""" + +from __future__ import annotations + +from collections import Counter +from collections.abc import Iterable, Mapping + +__all__ = ["collapse_umis_directional", "collapse_umi_column"] + +_ALPHABET = "ACGTN" + + +def _neighbours(umi: str): + """All strings at Hamming distance 1 from ``umi`` over ``ACGTN`` (positions with other + characters are skipped).""" + for i, c in enumerate(umi): + if c not in _ALPHABET: + continue + head, tail = umi[:i], umi[i + 1 :] + for b in _ALPHABET: + if b != c: + yield head + b + tail + + +def collapse_umis_directional( + umis: Iterable[str] | Mapping[str, int], + counts: Iterable[int] | None = None, +) -> dict[str, str]: + """ + Directional UMI collapsing (UMI-tools) within one group of reads. + + Parameters + ---------- + umis : iterable of str, or mapping UMI -> count + Distinct raw UMIs. If a mapping is given, its values are the counts and ``counts`` must be + ``None``. If ``counts`` is ``None`` and ``umis`` is a plain iterable, it is treated as one + entry per READ and counted (duplicates allowed). + + counts : iterable of int, optional + Reads per raw UMI, aligned with ``umis`` (which must then be distinct). + + Returns + ------- + dict[str, str] + Mapping from every input UMI to its representative (a representative maps to itself). + """ + if isinstance(umis, Mapping): + if counts is not None: + raise ValueError("pass counts either in the mapping or in `counts`, not both") + cnt = dict(umis) + elif counts is None: + cnt = dict(Counter(umis)) + else: + umis = list(umis) + counts = list(counts) + if len(umis) != len(counts): + raise ValueError("umis and counts must have the same length") + cnt = {} + for u, c in zip(umis, counts): + if u in cnt: + raise ValueError(f"duplicate UMI with explicit counts: {u}") + cnt[u] = int(c) + + order = sorted(cnt, key=lambda u: (-cnt[u], u)) + rep: dict[str, str] = {} + for root in order: + if root in rep: + continue + rep[root] = root + stack = [root] + while stack: + parent = stack.pop() + if "N" in parent: # N-containing UMIs never absorb + continue + threshold = cnt[parent] + for nb in _neighbours(parent): + c = cnt.get(nb) + if c is None or nb in rep: + continue + if threshold >= 2 * c - 1: + rep[nb] = root + stack.append(nb) + return rep + + +def collapse_umi_column(umis: list[str]) -> list[str]: + """Per-read convenience wrapper: corrected UMI for each read UMI in ``umis`` (one group).""" + mapping = collapse_umis_directional(umis) + return [mapping[u] for u in umis] diff --git a/pairplex/utils.py b/pairplex/utils.py index 0c599a3..eb89083 100644 --- a/pairplex/utils.py +++ b/pairplex/utils.py @@ -29,6 +29,7 @@ from natsort import natsorted from spoa import poa +from .umi import collapse_umis_directional from .version import __version__ BARCODE_DIR = Path(__file__).resolve().parent / "barcodes" @@ -333,6 +334,7 @@ def process_droplet( min_cluster_reads: int = 3, min_cluster_umis: int = 2, min_cluster_fraction: float = 0.0, + umi_collapse: str = "directional", quiet: bool = False, debug: bool = False, ) -> list[dict]: @@ -365,6 +367,14 @@ def process_droplet( min_cluster_fraction : float Minimum fraction of reads in a cluster. + umi_collapse : str + UMI error correction applied to the whole droplet (all reads of this barcode in this well) + before any counting. ``"directional"`` (default) collapses UMIs with the UMI-tools + directional method (see :mod:`pairplex.umi`): the raw UMI of each read is kept in + ``umi_raw``, ``umi`` becomes the corrected representative, the metadata ``umis`` is the + corrected count and ``umis_raw`` the count before collapsing. ``"none"`` keeps raw UMIs and + the v2-base metadata layout (no ``umis_raw`` column). + quiet : bool Whether to suppress output. @@ -381,6 +391,16 @@ def process_droplet( barcode = name.split("_")[-1] metadata = [] + # UMI error correction, per (well, barcode): this partition is exactly one barcode of one well + if umi_collapse == "directional": + mapping = collapse_umis_directional(partition_df["umi"].to_list()) + partition_df = partition_df.with_columns( + pl.col("umi").alias("umi_raw"), + pl.col("umi").replace_strict(mapping, return_dtype=pl.String).alias("umi"), + ) + elif umi_collapse != "none": + raise ValueError(f"unknown umi_collapse: {umi_collapse!r}") + # sequences = abutils.io.from_polars( # partition_df, id_key="seq_id", sequence_key="sequence" # ) @@ -398,6 +418,8 @@ def process_droplet( contig_name = f"{barcode}_contig-{i}" cluster_df = partition_df.filter(pl.col("seq_id").is_in(clust)) n_umis = len(cluster_df["umi"].unique()) + if umi_collapse != "none": + n_umis_raw = len(cluster_df["umi_raw"].unique()) cluster_fraction = len(clust) / partition_df.shape[0] # consensus @@ -417,6 +439,10 @@ def process_droplet( "name": contig_name, "reads": len(clust), "umis": n_umis, + } + if umi_collapse != "none": + meta["umis_raw"] = n_umis_raw + meta |= { "cluster_fraction": cluster_fraction, "consensus": consensus, } From 6f5f888f9fca454430804434384afc2f072bd668 Mon Sep 17 00:00:00 2001 From: Benjamin Nemoz Date: Tue, 17 Nov 2026 16:25:34 +0000 Subject: [PATCH 2/2] feat(qc): support columns, per-stage drop counts, abstar failure table, QC report - annotated outputs carry cluster support: umis, reads, cluster_fraction, barcode, contig (unpaired) and umis:0/1, reads:0/1, ... (paired); the always-null abstar `umi` column is dropped (a consensus contig has no single UMI; the count lives in `umis`) - qc/stage_counts.csv: drop counts at every stage, per well and TOTAL (reads, barcode groups, empty sequences skipped by greedy_clustering, clusters failing each filter, annotation outcomes, chain composition, emitted pairs) - qc/annotation_failures.csv: abstar record errors (from its failures.tsv warning), unassigned reasons, productivity issues, missing junctions - qc/report.{json,md}: fragment fraction by locus, UMI Hamming-1 excess vs shuffled null, heavy-junction multi-barcode share, leakage f by chain (>= 8 wells), pair confidence distribution, clonal light-chain purity with chance and a memory-only caveat - new run() kwargs support_columns=True, qc=True, legacy=False (legacy forces both off and reproduces v2-base); matching CLI options - pairing is unchanged; logic lives in pairplex/qc.py behind small hooks Co-Authored-By: Claude Opus 5.5 --- pairplex/cli/pairplex.py | 21 ++ pairplex/pairplex.py | 75 +++- pairplex/qc.py | 729 ++++++++++++++++++++++++++++++++++++++ pairplex/tests/test_qc.py | 325 +++++++++++++++++ 4 files changed, 1144 insertions(+), 6 deletions(-) create mode 100644 pairplex/qc.py create mode 100644 pairplex/tests/test_qc.py diff --git a/pairplex/cli/pairplex.py b/pairplex/cli/pairplex.py index 8a39506..e03afbc 100644 --- a/pairplex/cli/pairplex.py +++ b/pairplex/cli/pairplex.py @@ -104,6 +104,21 @@ def version(): is_flag=True, help="Whether to enable debug mode, which saves all temporary files to ease troubleshooting", ) +@click.option( + "--support_columns/--no_support_columns", + default=True, + help="Carry cluster support (umis, reads, cluster_fraction) into the annotated outputs", +) +@click.option( + "--qc/--no_qc", + default=True, + help="Write qc/stage_counts.csv, qc/annotation_failures.csv and qc/report.{json,md}", +) +@click.option( + "--legacy", + is_flag=True, + help="Reproduce PairPlex v2-base exactly (disables support columns and QC outputs)", +) def run( sequences: str | Path, output_directory: str | Path, @@ -120,6 +135,9 @@ def run( germline_database: str, quiet: bool, debug: bool, + support_columns: bool, + qc: bool, + legacy: bool, ): run_pairplex( sequences=sequences, @@ -137,6 +155,9 @@ def run( germline_database=germline_database, quiet=quiet, debug=debug, + support_columns=support_columns, + qc=qc, + legacy=legacy, ) diff --git a/pairplex/pairplex.py b/pairplex/pairplex.py index 85c4ed5..df38125 100644 --- a/pairplex/pairplex.py +++ b/pairplex/pairplex.py @@ -18,6 +18,7 @@ import multiprocessing as mp import os import time +from contextlib import nullcontext from concurrent.futures import ProcessPoolExecutor, as_completed from pathlib import Path @@ -27,6 +28,7 @@ from natsort import natsorted from tqdm.auto import tqdm +from . import qc as qc_mod from .provenance import RunLog, run_log_path from .utils import available_cpus, parse_barcodes, print_splash, process_droplet @@ -51,6 +53,9 @@ def run( germline_database: str = "human", quiet: bool = False, debug: bool = False, + support_columns: bool = True, + qc: bool = True, + legacy: bool = False, ) -> list: """ Run PairPlex. @@ -111,6 +116,24 @@ def run( debug : bool If ``True``, all temporary files will be retained to ease debugging. + support_columns : bool + If ``True`` (default), the annotated outputs carry the cluster support of each contig: + ``umis``, ``reads``, ``cluster_fraction``, ``barcode`` and ``contig`` in the unpaired + outputs, ``umis:0/1``, ``reads:0/1``, ``cluster_fraction:0/1`` (etc.) in the paired outputs. + abstar's ``umi`` column, which is always null for PairPlex consensus contigs, is dropped. + Pairing is unchanged. + + qc : bool + If ``True`` (default), write ``qc/stage_counts.csv`` (drop counts at every stage, per + well and in total), ``qc/annotation_failures.csv`` (abstar failure and exclusion modes) + and ``qc/report.json`` / ``qc/report.md`` (fragment fraction, UMI error signature, + ambient proxy, leakage by chain, pair confidence, clonal light-chain coherence). + Pairing is unchanged. + + legacy : bool + If ``True``, reproduce PairPlex v2-base exactly: forces ``support_columns=False`` and + ``qc=False``. + Returns ------- @@ -119,6 +142,9 @@ def run( """ parameters = dict(locals()) # captured first: exactly the arguments of this call + if legacy: + support_columns = False + qc = False # print splash screen if not quiet: @@ -218,6 +244,9 @@ def run( f"valid barcodes: {all_valid_barcodes} | consensus sequences: {all_consensus_sequences} | pairs: {all_pairs}" ) + # QC accumulators (qc=True): per-well stage counts, abstar failure modes, fragments, UMI stats + qc_rows, qc_failures, qc_fragments, qc_umis, qc_wells = [], [], {}, [], [] + # initialize the Process Pool with ProcessPoolExecutor( max_workers=available_cpus(), mp_context=mp.get_context("spawn") @@ -239,6 +268,10 @@ def run( input_file = Path(input_file) name = input_file.stem name_printer.set_description_str(f"---- {name} ----") + if qc: + wc = qc_mod.new_counts(name) + qc_rows.append(wc) + qc_wells.append(name) # check to make sure the input file is FASTA/Q (and not empty) if abutils.io.determine_fastx_format(str(input_file)) is None: @@ -257,6 +290,8 @@ def run( for s in abutils.io.parse_fastx(str(input_file)): input_count += 1 seqs_printer.set_description_str(f"{input_count} input sequences") + if qc: + wc["input_reads"] = wc["dropped_orientation_or_barcode"] = input_count # split input file into chunks main_pbar.set_postfix_str("splitting input file", refresh=True) @@ -297,6 +332,10 @@ def run( ) df = pl.read_parquet(concat_parquet) seqs_with_barcodes = df.shape[0] + if qc: + wc.update(qc_mod.read_stage_counts(input_count, df)) + wc.update(qc_mod.barcode_stage_counts(df, min_cluster_reads)) + qc_umis.append(qc_mod.umi_hamming1_stats(df)) all_valid_barcodes += seqs_with_barcodes valids_printer.set_description_str( f"{seqs_with_barcodes} sequences with valid barcodes" @@ -377,6 +416,12 @@ def run( ) metadata_df = pl.DataFrame(metadata) consensus_pbar.close() + if qc: + wc.update( + qc_mod.cluster_stage_counts( + metadata_df, min_cluster_reads, min_cluster_umis, min_cluster_fraction + ) + ) # write metadata to file metadata_file = metadata_directory / f"{name}.csv" @@ -415,12 +460,21 @@ def run( if consensus_count < 1000: mmseqs_threads = 1 # run abstar - sequences = abstar.run( - sequences=str(consensus_file), - germline_database=germline_database, - receptor=receptor, - mmseqs_threads=mmseqs_threads, - ) + with qc_mod.capture_abstar_failures() if qc else nullcontext([]) as record_failures: + sequences = abstar.run( + sequences=str(consensus_file), + germline_database=germline_database, + receptor=receptor, + mmseqs_threads=mmseqs_threads, + ) + if qc: + ann_df = qc_mod.annotation_frame(sequences) + n_rec = len(record_failures) + wc.update(qc_mod.annotation_stage_counts(ann_df, consensus_count, n_rec)) + qc_failures.extend(qc_mod.annotation_failure_rows(name, ann_df, record_failures)) + qc_mod.merge_fragment_counts(qc_fragments, qc_mod.fragment_counts(ann_df, n_rec)) + if support_columns: + qc_mod.add_support_columns(sequences, metadata_df) if len(sequences) > 1: # A single sequence cannot be paired and will cause issues @@ -444,7 +498,11 @@ def run( paired_airr_file = annotated_directory / f"{name}_paired.tsv" paired_parquet_file = annotated_directory / f"{name}_paired.parquet" pairs = abutils.tl.assign_pairs(sequences, delim="_", delim_occurance=-1) + if qc: + wc.update(qc_mod.chain_composition_counts(pairs)) pairs = [p for p in pairs if len(p.heavies) == 1 and len(p.lights) == 1] + if qc: + wc["emitted_pairs"] = len(pairs) pairs_printer.set_description_str(f"{len(pairs)} paired sequences") abutils.io.to_airr(pairs, str(paired_airr_file), drop_na_columns=False) abutils.io.to_parquet( @@ -481,6 +539,11 @@ def run( # update the main progress bar main_pbar.update(1) + if qc: + qc_mod.write_qc( + output_directory, qc_rows, qc_failures, qc_fragments, qc_umis, qc_wells + ) + run_log.finish( stats={ "input_files": len(input_files), diff --git a/pairplex/qc.py b/pairplex/qc.py new file mode 100644 index 0000000..37367a8 --- /dev/null +++ b/pairplex/qc.py @@ -0,0 +1,729 @@ +# Copyright (c) 2025 Benjamin Nemoz +# Distributed under the terms of the MIT License. +# SPDX-License-Identifier: MIT + +""" +Run QC for ``pairplex run``: support columns, per-stage drop counts, abstar failure table and a +per-run QC report. + +Everything here counts from data ``run()`` already holds (the parsed reads of a well, the cluster +metadata, the abstar results, the assigned pairs) or reads back a few columns of the per-well +outputs at the end of the run. Nothing here changes which pairs are emitted. + +Outputs (under ``/qc/``): + +- ``stage_counts.csv``: one row per well plus a ``TOTAL`` row, columns ``STAGE_FIELDS``; +- ``annotation_failures.csv``: abstar failure / exclusion modes with counts per well and in total; +- ``report.json`` and ``report.md``: run-level QC metrics (see ``build_report``). +""" + +from __future__ import annotations + +import csv +import json +import math +import re +import warnings +from collections import Counter +from contextlib import contextmanager +from pathlib import Path + +import numpy as np +import polars as pl + +# -------------------------------------------------------------------------------------------- +# per-stage counts +# -------------------------------------------------------------------------------------------- + +# 0H0L = a barcode whose contigs all failed annotation (no locus); "other" = any other combination +COMPOSITIONS = ["1H1L", "1H0L", "0H1L", "1H2L", "2H1L", "2H2L", "0H0L", "other"] + +STAGE_FIELDS = [ + # reads + "input_reads", + "valid_barcode_reads", + "dropped_orientation_or_barcode", + # barcode groups + "barcode_groups", + "groups_dropped_min_cluster_reads", + "reads_in_dropped_groups", + "empty_sequences_skipped", + # clusters + "clusters", + "clusters_fail_reads", + "clusters_fail_umis", + "clusters_fail_fraction", + "clusters_pass", + # annotation + "consensus_to_annotation", + "annotation_returned", + "annotated", + "failed_annotation", + "failed_unassigned", + "failed_record_error", + "productive", + "unproductive", + "no_junction", + # pairing + "barcodes_with_chains", + *[f"barcodes_{c}" for c in COMPOSITIONS], + "emitted_pairs", +] + + +def new_counts(well: str) -> dict: + """An all-zero stage-count row for ``well``.""" + return {"well": well, **{f: 0 for f in STAGE_FIELDS}} + + +def read_stage_counts(input_reads: int, parsed: pl.DataFrame) -> dict: + """Reads in, reads with a valid barcode, reads dropped (no orientation gave a valid barcode).""" + valid = int(parsed.height) + return { + "input_reads": int(input_reads), + "valid_barcode_reads": valid, + "dropped_orientation_or_barcode": int(input_reads) - valid, + } + + +def barcode_stage_counts(parsed: pl.DataFrame, min_cluster_reads: int) -> dict: + """ + Barcode groups, groups (and their reads) dropped by ``min_cluster_reads``, and the empty + sequences in kept groups, which ``greedy_clustering`` skips silently. + """ + if parsed.height == 0: + return {"barcode_groups": 0, "groups_dropped_min_cluster_reads": 0, + "reads_in_dropped_groups": 0, "empty_sequences_skipped": 0} + g = parsed.group_by("barcode").agg( + pl.len().alias("n"), + (pl.col("sequence").fill_null("").str.len_chars() == 0).sum().alias("empty"), + ) + dropped = g.filter(pl.col("n") < min_cluster_reads) + kept = g.filter(pl.col("n") >= min_cluster_reads) + return { + "barcode_groups": int(g.height), + "groups_dropped_min_cluster_reads": int(dropped.height), + "reads_in_dropped_groups": int(dropped["n"].sum()), + "empty_sequences_skipped": int(kept["empty"].sum()), + } + + +def cluster_stage_counts( + metadata: pl.DataFrame, + min_cluster_reads: int, + min_cluster_umis: int, + min_cluster_fraction: float, +) -> dict: + """Clusters and clusters failing each filter (a cluster can fail several).""" + if metadata.height == 0: + return {"clusters": 0, "clusters_fail_reads": 0, "clusters_fail_umis": 0, + "clusters_fail_fraction": 0, "clusters_pass": 0} + return { + "clusters": int(metadata.height), + "clusters_fail_reads": int((metadata["reads"] < min_cluster_reads).sum()), + "clusters_fail_umis": int((metadata["umis"] < min_cluster_umis).sum()), + "clusters_fail_fraction": int((metadata["cluster_fraction"] < min_cluster_fraction).sum()), + "clusters_pass": int(metadata["pass_filters"].sum()), + } + + +# columns pulled from abstar results (kept small: this is per consensus sequence) +ANN_SCHEMA = { + "sequence_id": pl.String, + "annotation_status": pl.String, + "failure_reason": pl.String, + "locus": pl.String, + "productive": pl.Boolean, + "productivity_issues": pl.String, + "junction": pl.String, + "complete_vdj": pl.Boolean, + "v_germline_start": pl.Int64, +} + + +def annotation_frame(sequences) -> pl.DataFrame: + """A small polars frame (``ANN_SCHEMA`` columns) from abstar's returned ``Sequence`` objects.""" + if sequences is None: + sequences = [] + elif not isinstance(sequences, list | tuple): + sequences = [sequences] + rows = [] + for s in sequences: + ann = s.annotations or {} + rows.append({c: ann.get(c) for c in ANN_SCHEMA}) + return pl.DataFrame(rows, schema=ANN_SCHEMA, strict=False) + + +def annotation_stage_counts(ann: pl.DataFrame, n_sent: int, n_record_failures: int = 0) -> dict: + """ + Consensus sequences sent to abstar and what came back. + + ``failed_annotation`` = ``failed_unassigned`` (returned with ``annotation_status`` other than + ``annotated``) + ``failed_record_error`` (no row returned: abstar record failures). + ``unproductive`` and ``no_junction`` are counted among annotated sequences. + """ + annotated = ann.filter(pl.col("annotation_status") == "annotated") + unassigned = ann.height - annotated.height + record = max(int(n_record_failures), int(n_sent) - ann.height) + return { + "consensus_to_annotation": int(n_sent), + "annotation_returned": int(ann.height), + "annotated": int(annotated.height), + "failed_unassigned": int(unassigned), + "failed_record_error": int(record), + "failed_annotation": int(unassigned + record), + "productive": int(annotated["productive"].fill_null(False).sum()), + "unproductive": int((~annotated["productive"].fill_null(False)).sum()), + "no_junction": int(annotated["junction"].is_null().sum()), + } + + +def composition_label(n_heavy: int, n_light: int) -> str: + label = f"{n_heavy}H{n_light}L" + return label if label in COMPOSITIONS else "other" + + +def chain_composition_counts(pairs) -> dict: + """Barcodes by chain composition, from ``abutils.tl.assign_pairs`` output (before the 1H1L rule).""" + c = Counter(composition_label(len(p.heavies), len(p.lights)) for p in pairs) + out = {f"barcodes_{k}": int(c.get(k, 0)) for k in COMPOSITIONS} + out["barcodes_with_chains"] = int(sum(c.values())) + return out + + +def stage_counts_frame(rows: list[dict]) -> pl.DataFrame: + """Per-well rows plus a ``TOTAL`` row.""" + schema = {"well": pl.String, **{f: pl.Int64 for f in STAGE_FIELDS}} + df = pl.DataFrame(rows, schema=schema, strict=False).fill_null(0) + total = df.select(pl.lit("TOTAL").alias("well"), *[pl.col(f).sum() for f in STAGE_FIELDS]) + return pl.concat([df, total], how="vertical_relaxed") + + +# -------------------------------------------------------------------------------------------- +# abstar failure modes +# -------------------------------------------------------------------------------------------- + +_FAILURES_TSV = re.compile(r"see (\S*failures\.tsv)") + + +@contextmanager +def capture_abstar_failures(): + """ + Collect abstar record failures (records abstar drops, with no output row). + + abstar reports them only through a ``RuntimeWarning`` pointing at a ``failures.tsv`` in its + log directory. The warning is re-emitted unchanged; the parsed rows (``stage``, ``category``, + ``exception_type``, ``message``) are appended to the yielded list. + """ + failures: list[dict] = [] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + yield failures + for w in caught: + m = _FAILURES_TSV.search(str(w.message)) + if m and issubclass(w.category, RuntimeWarning): + failures.extend(read_failures_tsv(m.group(1))) + warnings.warn_explicit(w.message, w.category, w.filename, w.lineno) + + +def read_failures_tsv(path: str | Path) -> list[dict]: + keep = ("sequence_id", "stage", "category", "exception_type", "message") + try: + with open(path, newline="") as f: + return [{k: r.get(k) for k in keep} for r in csv.DictReader(f, delimiter="\t")] + except OSError: + return [] + + +FAILURE_COLUMNS = {"well": pl.String, "kind": pl.String, "mode": pl.String, "count": pl.Int64} + + +def annotation_failure_rows(well: str, ann: pl.DataFrame, record_failures: list[dict]) -> list[dict]: + """ + Failure / exclusion modes for one well. ``kind`` is one of: + + - ``record_error``: abstar dropped the record (mode = ``stage/category exception: message``); + - ``unassigned``: returned but not annotated (mode = abstar ``failure_reason``); + - ``unproductive``: annotated but unproductive (mode = each ``productivity_issues`` item, so a + sequence with several issues counts once per issue); + - ``no_junction``: annotated with no junction. + """ + c: Counter = Counter() + for r in record_failures: + exc = f" {r['exception_type']}" if r.get("exception_type") else "" + c[("record_error", f"{r.get('stage')}/{r.get('category')}{exc}: {r.get('message')}"[:200])] += 1 + annotated = ann.filter(pl.col("annotation_status") == "annotated") + for reason in ann.filter(pl.col("annotation_status") != "annotated")["failure_reason"].to_list(): + c[("unassigned", reason or "unknown")] += 1 + unprod = annotated.filter(~pl.col("productive").fill_null(False)) + for issues in unprod["productivity_issues"].to_list(): + items = [i.strip() for i in (issues or "").split("|") if i.strip()] or ["unspecified"] + for i in items: + c[("unproductive", i)] += 1 + n_nojunc = int(annotated["junction"].is_null().sum()) + if n_nojunc: + c[("no_junction", "junction is null")] += n_nojunc + return [{"well": well, "kind": k, "mode": m, "count": n} for (k, m), n in sorted(c.items())] + + +def failure_table(rows: list[dict]) -> pl.DataFrame: + """Per-well failure rows plus ``TOTAL`` rows (one per kind/mode).""" + df = pl.DataFrame(rows, schema=FAILURE_COLUMNS) + if df.height == 0: + return df + total = df.group_by("kind", "mode").agg(pl.col("count").sum()).with_columns(pl.lit("TOTAL").alias("well")) + total = total.select(list(FAILURE_COLUMNS)).sort(["kind", "count"], descending=[False, True]) + return pl.concat([df, total]) + + +# -------------------------------------------------------------------------------------------- +# fragment fraction +# -------------------------------------------------------------------------------------------- + +# a contig is "truncated" when its V alignment starts this many nt or more into the germline V +# (i.e. most of FR1 is missing) or when it has no junction +TRUNCATION_NT = 30 + + +def fragment_counts(ann: pl.DataFrame, n_record_failures: int = 0) -> dict: + """ + Per-locus counts of passing consensus sequences: ``{locus: {"n", "truncated", "incomplete_vdj"}}``, + with failed annotations (unassigned + record errors) under ``"unknown"``. + """ + out: dict = {} + failed = ann.filter(pl.col("annotation_status") != "annotated").height + int(n_record_failures) + if failed: + out["unknown"] = {"n": failed, "truncated": failed, "incomplete_vdj": failed} + annotated = ann.filter(pl.col("annotation_status") == "annotated").with_columns( + ((pl.col("v_germline_start").fill_null(10**6) > TRUNCATION_NT) | pl.col("junction").is_null()).alias("_trunc"), + (~pl.col("complete_vdj").fill_null(False)).alias("_inc"), + ) + for r in annotated.group_by(pl.col("locus").fill_null("unknown")).agg( + pl.len().alias("n"), pl.col("_trunc").sum().alias("truncated"), pl.col("_inc").sum().alias("incomplete_vdj") + ).iter_rows(named=True): + d = out.setdefault(r["locus"], {"n": 0, "truncated": 0, "incomplete_vdj": 0}) + for k in ("n", "truncated", "incomplete_vdj"): + d[k] += int(r[k]) + return out + + +def merge_fragment_counts(acc: dict, new: dict) -> dict: + for locus, d in new.items(): + a = acc.setdefault(locus, {"n": 0, "truncated": 0, "incomplete_vdj": 0}) + for k, v in d.items(): + a[k] += v + return acc + + +def fragment_summary(frag: dict) -> dict: + n = sum(d["n"] for d in frag.values()) + bad = sum(d["truncated"] for d in frag.values()) + by_locus = { + locus: {**d, "fragment_fraction": _ratio(d["truncated"], d["n"])} for locus, d in sorted(frag.items()) + } + return { + "definition": ( + "share of passing consensus sequences that fail annotation or are truncated " + f"(V alignment starts > {TRUNCATION_NT} nt into germline V, or no junction); " + "failed annotations have no locus ('unknown')" + ), + "passing_consensus": n, + "fragment_fraction": _ratio(bad, n), + "failed_annotation_fraction": _ratio(frag.get("unknown", {}).get("n", 0), n), + "by_locus": by_locus, + } + + +# -------------------------------------------------------------------------------------------- +# UMI error signature +# -------------------------------------------------------------------------------------------- + + +def _hamming1_pairs(u: pl.DataFrame) -> tuple[int, int]: + """(Hamming-1 UMI pairs within barcode, distinct UMIs with >=1 Hamming-1 neighbour) for unique (barcode, umi).""" + if u.height < 2: + return 0, 0 + L = int(u["umi"].str.len_chars().max()) + keyed = pl.concat( + [ + u.select( + "barcode", "umi", pl.lit(i, pl.Int8).alias("pos"), + (pl.col("umi").str.slice(0, i) + pl.col("umi").str.slice(i + 1)).alias("key"), + ) + for i in range(L) + ] + ) + g = keyed.group_by("barcode", "pos", "key").agg(pl.len().alias("k"), pl.col("umi")) + g = g.filter(pl.col("k") >= 2) + if g.height == 0: + return 0, 0 + pairs = int((g["k"] * (g["k"] - 1) // 2).sum()) + with_nb = g.select("barcode", "umi").explode("umi").unique().height + return pairs, with_nb + + +def umi_hamming1_stats(parsed: pl.DataFrame, max_umis: int = 200_000, seed: int = 0) -> dict: + """ + Hamming-1 UMI pairs within barcode, observed vs a null that shuffles barcodes across the + distinct (barcode, UMI) rows (keeps each barcode's UMI count and the UMI composition). + + Exact (no pairwise loop): each Hamming-1 pair shares exactly one "UMI with position i + masked" key. Barcodes are subsampled when a well has more than ``max_umis`` distinct + (barcode, UMI) rows. + """ + empty = {"umis": 0, "barcodes": 0, "sampled": False, "h1_pairs": 0, "h1_pairs_null": 0, + "umis_with_h1_neighbour": 0, "umis_with_h1_neighbour_null": 0} + if parsed.height == 0: + return empty + u = parsed.select("barcode", "umi").drop_nulls().unique() + if u.height == 0: + return empty + L = u["umi"].str.len_chars().mode().sort()[-1] + u = u.filter(pl.col("umi").str.len_chars() == L) + sampled = False + if u.height > max_umis: + bcs = u["barcode"].unique().sort() + keep = bcs.sample(fraction=max_umis / u.height, seed=seed) + u = u.filter(pl.col("barcode").is_in(keep.implode())) + sampled = True + obs, obs_nb = _hamming1_pairs(u) + shuffled = u.with_columns(pl.col("barcode").shuffle(seed=seed)).unique() + null, null_nb = _hamming1_pairs(shuffled) + return {"umis": int(u.height), "barcodes": int(u["barcode"].n_unique()), "sampled": sampled, + "h1_pairs": obs, "h1_pairs_null": null, + "umis_with_h1_neighbour": obs_nb, "umis_with_h1_neighbour_null": null_nb} + + +def umi_summary(per_well: list[dict]) -> dict: + keys = ["umis", "h1_pairs", "h1_pairs_null", "umis_with_h1_neighbour", "umis_with_h1_neighbour_null"] + t = {k: int(sum(d.get(k, 0) for d in per_well)) for k in keys} + t["sampled_wells"] = int(sum(bool(d.get("sampled")) for d in per_well)) + t["excess_h1_pairs"] = _ratio(t["h1_pairs"], t["h1_pairs_null"]) if t["h1_pairs_null"] else None + obs_f = _ratio(t["umis_with_h1_neighbour"], t["umis"]) + null_f = _ratio(t["umis_with_h1_neighbour_null"], t["umis"]) + t["error_like_umi_fraction"] = obs_f + t["error_like_umi_fraction_null"] = null_f + t["excess_error_like_umi_fraction"] = ( + None if obs_f is None else round(max(0.0, obs_f - (null_f or 0.0)), 6) + ) + t["definition"] = ( + "Hamming-1 UMI pairs among distinct UMIs of the same barcode vs a null with barcodes " + "shuffled across (barcode, UMI) rows; excess_h1_pairs = observed / null" + ) + return t + + +# -------------------------------------------------------------------------------------------- +# run-level metrics from the per-well outputs +# -------------------------------------------------------------------------------------------- + +_LIGHT = ["IGK", "IGL"] + + +def load_unpaired(files: list[tuple[str, Path]]) -> pl.DataFrame: + """Productive annotated contigs (barcode, well, locus, v_gene, j_gene, junction, cdr3_aa) of the run.""" + cols = ["sequence_id", "locus", "productive", "v_gene", "j_gene", "junction", "cdr3_aa"] + out = [] + for well, f in files: + schema = pl.read_parquet_schema(f) + if not all(c in schema for c in cols): + continue + out.append( + pl.read_parquet(f, columns=cols) + .filter(pl.col("productive").fill_null(False) & pl.col("junction").is_not_null()) + .with_columns(pl.col("sequence_id").str.split("_").list.first().alias("barcode"), pl.lit(well).alias("well")) + .drop("productive") + ) + if not out: + return pl.DataFrame(schema={c: pl.String for c in ["sequence_id", "locus", "v_gene", "j_gene", "junction", "cdr3_aa", "barcode", "well"]}) + return pl.concat(out, how="diagonal_relaxed") + + +def ambient_proxy(unpaired: pl.DataFrame) -> dict: + """Share of heavy-chain junctions (nt) seen under >=2 and >=10 distinct droplet barcodes, run-wide.""" + h = unpaired.filter(pl.col("locus") == "IGH") + if h.height == 0: + return {"heavy_junctions": 0, "share_ge2_barcodes": None, "share_ge10_barcodes": None} + nb = h.group_by("junction").agg(pl.col("barcode").n_unique().alias("nb")) + n = nb.height + occ = h.join(nb, on="junction") + return { + "heavy_junctions": int(n), + "share_ge2_barcodes": _ratio(int((nb["nb"] >= 2).sum()), n), + "share_ge10_barcodes": _ratio(int((nb["nb"] >= 10).sum()), n), + "contigs_with_junction_ge2_barcodes": _ratio(int((occ["nb"] >= 2).sum()), occ.height), + "contigs_with_junction_ge10_barcodes": _ratio(int((occ["nb"] >= 10).sum()), occ.height), + "definition": "distinct productive IGH junctions (nt) by number of distinct barcodes carrying them, across wells", + } + + +def leakage_by_chain(output_directory: str | Path, min_wells: int = 8) -> dict: + """ + Leakage f per chain with ``pooled_pairing.estimate_leakage`` on this run's per-well outputs. + Skipped (``None`` values) when fewer than ``min_wells`` wells have outputs. + """ + from .pooled_pairing import SPECIES, estimate_leakage, load_run + + try: + occ = load_run(output_directory) + except FileNotFoundError: + return {"wells": 0, "skipped": "no per-well outputs"} + W = int(occ["well"].n_unique()) + if W < min_wells: + return {"wells": W, "skipped": f"fewer than {min_wells} wells"} + sw = occ.group_by(*SPECIES, "well").agg(pl.col("umis").sum().alias("n")) + out = {"wells": W} + for name, loci in (("heavy", ["IGH"]), ("light", _LIGHT), ("IGK", ["IGK"]), ("IGL", ["IGL"])): + f = estimate_leakage(sw.filter(pl.col("locus").is_in(loci)), W) + out[name] = None if f is None else round(f, 4) + return out + + +CONF_BINS = [0.0, 0.1, 0.3, 0.6, 0.9, 1.0001] +CONF_LABELS = ["<0.1", "0.1-0.3", "0.3-0.6", "0.6-0.9", ">=0.9"] + + +def confidence_distribution(conf: pl.Series | None) -> dict | None: + if conf is None: + return None + c = conf.drop_nulls().cast(pl.Float64) + if c.len() == 0: + return {"n": 0} + bins = {} + for lo, hi, lab in zip(CONF_BINS[:-1], CONF_BINS[1:], CONF_LABELS): + bins[lab] = int(((c >= lo) & (c < hi)).sum()) + q = {f"q{int(p * 100):02d}": round(float(c.quantile(p, "linear")), 4) for p in (0.1, 0.25, 0.5, 0.75, 0.9)} + return {"n": int(c.len()), "mean": round(float(c.mean()), 4), **q, "bins": bins} + + +def _norm_light_v(expr: pl.Expr) -> pl.Expr: + # IGKV1D-39 and IGKV1-39 are the same gene in practice: count them as one light V + return expr.str.replace(r"^(IG[KL]V\d+)D-", "${1}-") + + +def clonal_coherence(paired: pl.DataFrame) -> dict: + """ + Clonal light-chain coherence. Clone = heavy V gene + CDRH3 aa; purity = share of a clone's + cells carrying its modal light V gene, pooled over clones with >= 2 cells (cell-weighted); + ``purity_mean_clone`` is the unweighted mean over those clones. ``chance`` = sum p^2 over the + light V frequencies of all pairs (the purity expected if light chains were random). + """ + need = ["v_gene:0", "cdr3_aa:0", "v_gene:1"] + if paired.height == 0 or not all(c in paired.columns for c in need): + return {"pairs": int(paired.height), "clones_ge2": 0, "purity": None, "chance": None} + d = paired.select( + pl.col("v_gene:0").alias("hv"), pl.col("cdr3_aa:0").alias("hcdr3"), _norm_light_v(pl.col("v_gene:1")).alias("lv") + ).drop_nulls() + p = d["lv"].value_counts()["count"].to_numpy().astype(float) + chance = round(float(((p / p.sum()) ** 2).sum()), 4) if p.size else None + c = d.group_by("hv", "hcdr3", "lv").len() + cl = c.group_by("hv", "hcdr3").agg(pl.col("len").sum().alias("size"), pl.col("len").max().alias("top")) + cl = cl.filter(pl.col("size") >= 2) + if cl.height == 0: + return {"pairs": int(paired.height), "clones_ge2": 0, "purity": None, "chance": chance} + return { + "pairs": int(paired.height), + "clones_ge2": int(cl.height), + "cells_in_clones": int(cl["size"].sum()), + "purity": round(float(cl["top"].sum() / cl["size"].sum()), 4), + "purity_mean_clone": round(float((cl["top"] / cl["size"]).mean()), 4), + "chance": chance, + } + + +# -------------------------------------------------------------------------------------------- +# report +# -------------------------------------------------------------------------------------------- + +PURITY_CAVEAT = ( + "Clonal light-chain purity is only meaningful for memory / clonally expanded repertoires; " + "in naive repertoires clones with >= 2 cells are rare and mostly chance coincidences, " + "so purity there says little about pairing accuracy. Use it to rank runs, not as an accuracy." +) + + +def build_report( + output_directory: str | Path, + stage_counts: pl.DataFrame, + failures: pl.DataFrame, + fragments: dict, + umi_stats: list[dict], + wells: list[str], +) -> dict: + """Assemble the run-level QC report from the counters and the per-well outputs on disk.""" + output_directory = Path(output_directory) + ann_dir = output_directory / "annotated" + unpaired_files = [(w, ann_dir / f"{w}_unpaired.parquet") for w in wells] + unpaired_files = [(w, f) for w, f in unpaired_files if f.exists() and f.stat().st_size > 0] + paired_files = [ann_dir / f"{w}_paired.parquet" for w in wells] + paired_files = [f for f in paired_files if f.exists() and f.stat().st_size > 0] + + paired = ( + pl.concat([pl.read_parquet(f) for f in paired_files], how="diagonal_relaxed") + if paired_files else pl.DataFrame() + ) + conf = None + for src in (output_directory / "pairs" / "pairs.parquet",): + if src.exists(): + p = pl.read_parquet(src) + if "confidence" in p.columns: + conf = p["confidence"] + if conf is None and "confidence" in paired.columns: + conf = paired["confidence"] + + total = stage_counts.filter(pl.col("well") == "TOTAL").to_dicts() + total = total[0] if total else {} + total.pop("well", None) + + try: + leakage = leakage_by_chain(output_directory) + except Exception as e: # QC must never fail a run + leakage = {"error": repr(e)} + + top_fail = ( + failures.filter(pl.col("well") == "TOTAL").sort("count", descending=True).head(10).drop("well").to_dicts() + if failures.height else [] + ) + return { + "wells": len(wells), + "stage_totals": total, + "heavy_fragment_fraction": fragment_summary(fragments), + "umi_error_signature": umi_summary(umi_stats), + "ambient_proxy": ambient_proxy(load_unpaired(unpaired_files)), + "leakage": leakage, + "pair_confidence": confidence_distribution(conf), + "clonal_coherence": {**clonal_coherence(paired), "caveat": PURITY_CAVEAT}, + "top_annotation_failures": top_fail, + } + + +def _fmt(v) -> str: + if v is None: + return "n/a" + if isinstance(v, float): + return f"{v:.4g}" + return str(v) + + +def report_markdown(rep: dict) -> str: + L = ["# PairPlex QC report", "", f"Wells: {rep['wells']}", ""] + L += ["## Stage totals", "", "| stage | count |", "|---|---|"] + L += [f"| {k} | {v} |" for k, v in rep["stage_totals"].items()] + fr = rep["heavy_fragment_fraction"] + L += ["", "## Fragment fraction", "", fr["definition"], "", + f"Passing consensus sequences: {fr['passing_consensus']}; fragment fraction: {_fmt(fr['fragment_fraction'])}; " + f"failed annotation: {_fmt(fr['failed_annotation_fraction'])}", "", + "| locus | n | truncated | incomplete VDJ | fragment fraction |", "|---|---|---|---|---|"] + L += [f"| {k} | {d['n']} | {d['truncated']} | {d['incomplete_vdj']} | {_fmt(d['fragment_fraction'])} |" + for k, d in fr["by_locus"].items()] + u = rep["umi_error_signature"] + L += ["", "## UMI error signature", "", u["definition"], "", + f"- distinct (barcode, UMI): {u['umis']} (wells subsampled: {u['sampled_wells']})", + f"- Hamming-1 pairs: {u['h1_pairs']} observed vs {u['h1_pairs_null']} shuffled null " + f"(excess x{_fmt(u['excess_h1_pairs'])})", + f"- UMIs with a Hamming-1 neighbour: {_fmt(u['error_like_umi_fraction'])} " + f"(null {_fmt(u['error_like_umi_fraction_null'])}; excess {_fmt(u['excess_error_like_umi_fraction'])})"] + a = rep["ambient_proxy"] + L += ["", "## Ambient proxy", "", a.get("definition", ""), "", + f"- heavy junctions: {a['heavy_junctions']}", + f"- share under >=2 barcodes: {_fmt(a['share_ge2_barcodes'])}", + f"- share under >=10 barcodes: {_fmt(a['share_ge10_barcodes'])}"] + lk = rep["leakage"] + L += ["", "## Leakage f (pooled_pairing.estimate_leakage)", ""] + if "skipped" in lk or "error" in lk: + L.append(f"Not computed: {lk.get('skipped') or lk.get('error')} (wells: {lk.get('wells', 'n/a')})") + else: + L += [f"- {k}: {_fmt(lk.get(k))}" for k in ("heavy", "light", "IGK", "IGL")] + [f"- wells: {lk['wells']}"] + c = rep["pair_confidence"] + L += ["", "## Pair confidence", ""] + if not c: + L.append("No confidence column in this run's pair outputs.") + else: + L.append(f"n = {c['n']}" + (f", mean {_fmt(c['mean'])}, median {_fmt(c['q50'])}" if c.get("n") else "")) + if c.get("bins"): + L += ["", "| bin | pairs |", "|---|---|"] + [f"| {k} | {v} |" for k, v in c["bins"].items()] + cc = rep["clonal_coherence"] + L += ["", "## Clonal light-chain coherence", "", f"> **Caveat:** {cc['caveat']}", "", + "Clone = heavy V gene + CDRH3 aa; purity = share of a clone's cells with its modal light V " + "(clones with >= 2 cells, cell-weighted); chance = sum p^2 over light V genes.", "", + f"- pairs: {cc['pairs']}; clones with >= 2 cells: {cc['clones_ge2']}", + f"- purity: {_fmt(cc.get('purity'))} (mean per clone {_fmt(cc.get('purity_mean_clone'))}); chance: {_fmt(cc.get('chance'))}"] + if rep["top_annotation_failures"]: + L += ["", "## Top annotation failure modes", "", "| kind | mode | count |", "|---|---|---|"] + L += [f"| {r['kind']} | {r['mode']} | {r['count']} |" for r in rep["top_annotation_failures"]] + return "\n".join(L) + "\n" + + +def write_qc( + output_directory: str | Path, + stage_rows: list[dict], + failure_rows: list[dict], + fragments: dict, + umi_stats: list[dict], + wells: list[str], +) -> dict: + """Write ``qc/stage_counts.csv``, ``qc/annotation_failures.csv``, ``qc/report.json`` and ``qc/report.md``.""" + qc_dir = Path(output_directory) / "qc" + qc_dir.mkdir(parents=True, exist_ok=True) + sc = stage_counts_frame(stage_rows) + sc.write_csv(qc_dir / "stage_counts.csv") + ft = failure_table(failure_rows) + ft.write_csv(qc_dir / "annotation_failures.csv") + rep = build_report(output_directory, sc, ft, fragments, umi_stats, wells) + (qc_dir / "report.json").write_text(json.dumps(rep, indent=2, default=_json_default) + "\n") + (qc_dir / "report.md").write_text(report_markdown(rep)) + return rep + + +# -------------------------------------------------------------------------------------------- +# support columns +# -------------------------------------------------------------------------------------------- + +SUPPORT_FIELDS = ("umis", "reads", "cluster_fraction") + + +def add_support_columns(sequences, metadata: pl.DataFrame) -> None: + """ + Add cluster support to abstar's ``Sequence`` objects in place, joined on the contig name + (= ``sequence_id``): ``umis``, ``reads``, ``cluster_fraction``, ``barcode`` and ``contig`` + (``contig-N``, N = rank by reads within the barcode). Paired outputs built from these objects + get them as ``umis:0/1`` etc. + + abstar's ``umi`` field is removed: it holds one read's UMI parsed from the read header with + ``umi_pattern``, which PairPlex never sets; a consensus contig stands for many UMIs, so the + field is always null here and the count lives in ``umis``. + """ + if sequences is None: + return + if not isinstance(sequences, list | tuple): + sequences = [sequences] + cols = [c for c in ("name", *SUPPORT_FIELDS) if c in metadata.columns] + support = {r[0]: r[1:] for r in metadata.select(cols).iter_rows()} + names = cols[1:] + for s in sequences: + sid = s.id + vals = support.get(sid) + for k, v in zip(names, vals if vals is not None else [None] * len(names)): + s[k] = v + bc, _, contig = (sid or "").rpartition("_") + s["barcode"] = bc or None + s["contig"] = contig or None + if s.annotations is not None: + s.annotations.pop("umi", None) + + +# -------------------------------------------------------------------------------------------- +# helpers +# -------------------------------------------------------------------------------------------- + + +def _ratio(a, b): + return None if not b else round(float(a) / float(b), 6) + + +def _json_default(o): + if isinstance(o, np.integer): + return int(o) + if isinstance(o, np.floating): + return None if math.isnan(o) else float(o) + return str(o) diff --git a/pairplex/tests/test_qc.py b/pairplex/tests/test_qc.py new file mode 100644 index 0000000..c094849 --- /dev/null +++ b/pairplex/tests/test_qc.py @@ -0,0 +1,325 @@ +"""Tests for pairplex.qc: stage counters, failure table, support columns and QC report metrics.""" + +import json +import warnings + +import polars as pl +import pytest +from abutils import Sequence + +from pairplex import qc + + +def _seq(sid, **ann): + rec = {"sequence_id": sid, "sequence": "ACGT", "umi": None, **ann} + return Sequence(rec, id_key="sequence_id", seq_key="sequence") + + +class _Pair: + def __init__(self, nh, nl): + self.heavies = [object()] * nh + self.lights = [object()] * nl + + +# ---------------------------------------------------------------- stage counters + + +def test_read_stage_counts(): + parsed = pl.DataFrame({"barcode": ["A", "B"], "umi": ["x", "y"], "sequence": ["AC", "GT"]}) + assert qc.read_stage_counts(5, parsed) == { + "input_reads": 5, "valid_barcode_reads": 2, "dropped_orientation_or_barcode": 3, + } + + +def test_barcode_stage_counts_groups_and_empty(): + parsed = pl.DataFrame({ + "barcode": ["A"] * 4 + ["B"] * 2 + ["C"], + "umi": ["u"] * 7, + "sequence": ["ACG", "", "ACG", "", "", "ACG", "A"], + }) + out = qc.barcode_stage_counts(parsed, min_cluster_reads=3) + # B (2 reads) and C (1 read) dropped; empties counted only in kept group A + assert out == {"barcode_groups": 3, "groups_dropped_min_cluster_reads": 2, + "reads_in_dropped_groups": 3, "empty_sequences_skipped": 2} + + +def test_barcode_stage_counts_empty_frame(): + parsed = pl.DataFrame(schema={"barcode": pl.String, "umi": pl.String, "sequence": pl.String}) + assert qc.barcode_stage_counts(parsed, 3)["barcode_groups"] == 0 + + +def test_cluster_stage_counts_each_filter(): + meta = pl.DataFrame({ + "reads": [10, 2, 5, 1], + "umis": [3, 1, 1, 1], + "cluster_fraction": [0.6, 0.1, 0.25, 0.05], + "pass_filters": [True, False, False, False], + }) + out = qc.cluster_stage_counts(meta, min_cluster_reads=3, min_cluster_umis=2, min_cluster_fraction=0.2) + assert out == {"clusters": 4, "clusters_fail_reads": 2, "clusters_fail_umis": 3, + "clusters_fail_fraction": 2, "clusters_pass": 1} + + +def _ann_seqs(): + return [ + _seq("A_contig-0", annotation_status="annotated", locus="IGH", productive=True, + junction="TGT", complete_vdj=True, v_germline_start=1), + _seq("A_contig-1", annotation_status="annotated", locus="IGK", productive=False, + productivity_issues="junction does not end with conserved F|V/J junction is out of frame", + junction="TGC", complete_vdj=False, v_germline_start=2), + _seq("B_contig-0", annotation_status="annotated", locus="IGH", productive=False, + productivity_issues="", junction=None, complete_vdj=False, v_germline_start=250), + _seq("C_contig-0", annotation_status="unassigned", failure_reason="no compatible V gene assignment", + locus=None, productive=None, junction=None), + ] + + +def test_annotation_frame_and_stage_counts(): + ann = qc.annotation_frame(_ann_seqs()) + assert ann.height == 4 and set(qc.ANN_SCHEMA) == set(ann.columns) + out = qc.annotation_stage_counts(ann, n_sent=6, n_record_failures=2) + assert out == { + "consensus_to_annotation": 6, "annotation_returned": 4, "annotated": 3, + "failed_unassigned": 1, "failed_record_error": 2, "failed_annotation": 3, + "productive": 1, "unproductive": 2, "no_junction": 1, + } + + +def test_annotation_frame_single_sequence_and_none(): + assert qc.annotation_frame(_seq("A_contig-0", annotation_status="annotated")).height == 1 + assert qc.annotation_frame(None).height == 0 + + +def test_record_errors_inferred_from_missing_rows(): + ann = qc.annotation_frame(_ann_seqs()) + assert qc.annotation_stage_counts(ann, n_sent=7)["failed_record_error"] == 3 + + +def test_chain_composition_counts(): + pairs = [_Pair(1, 1), _Pair(1, 1), _Pair(1, 0), _Pair(0, 1), _Pair(1, 2), _Pair(2, 1), + _Pair(2, 2), _Pair(0, 0), _Pair(3, 1), _Pair(0, 2)] + out = qc.chain_composition_counts(pairs) + assert out["barcodes_1H1L"] == 2 + assert out["barcodes_0H0L"] == 1 + assert out["barcodes_other"] == 2 + assert out["barcodes_with_chains"] == 10 + assert sum(out[f"barcodes_{c}"] for c in qc.COMPOSITIONS) == 10 + + +def test_stage_counts_frame_total(): + a, b = qc.new_counts("w1"), qc.new_counts("w2") + a["input_reads"], b["input_reads"], b["emitted_pairs"] = 3, 4, 2 + df = qc.stage_counts_frame([a, b]) + assert df["well"].to_list() == ["w1", "w2", "TOTAL"] + tot = df.filter(pl.col("well") == "TOTAL").row(0, named=True) + assert tot["input_reads"] == 7 and tot["emitted_pairs"] == 2 + assert df.columns == ["well", *qc.STAGE_FIELDS] + + +# ---------------------------------------------------------------- abstar failures + + +def test_annotation_failure_rows_and_table(): + ann = qc.annotation_frame(_ann_seqs()) + rec = [{"sequence_id": "D_contig-0", "stage": "annotation", "category": "internal_error", + "exception_type": "ValueError", "message": "CDR3 coordinates fall outside"}] + rows = qc.annotation_failure_rows("w1", ann, rec) + got = {(r["kind"], r["mode"]): r["count"] for r in rows} + assert got[("record_error", "annotation/internal_error ValueError: CDR3 coordinates fall outside")] == 1 + assert got[("unassigned", "no compatible V gene assignment")] == 1 + assert got[("unproductive", "junction does not end with conserved F")] == 1 + assert got[("unproductive", "V/J junction is out of frame")] == 1 + assert got[("unproductive", "unspecified")] == 1 + assert got[("no_junction", "junction is null")] == 1 + table = qc.failure_table(rows + qc.annotation_failure_rows("w2", ann, [])) + tot = table.filter((pl.col("well") == "TOTAL") & (pl.col("kind") == "unassigned")) + assert tot["count"].to_list() == [2] + assert qc.failure_table([]).height == 0 + + +def test_capture_abstar_failures_reads_tsv(tmp_path): + tsv = tmp_path / "failures.tsv" + tsv.write_text( + "input_file\tsample\trow_id\tsequence_id\tstage\tcategory\texception_type\tmessage\ttraceback_location\tdiagnostic_path\n" + "f\ts\t1\tX_contig-0\tannotation\tinternal_error\tValueError\tboom\tloc\tp\n" + ) + with warnings.catch_warnings(record=True) as outer: + warnings.simplefilter("always") + with qc.capture_abstar_failures() as fails: + warnings.warn(f"1 sequences failed annotation; see {tsv}", RuntimeWarning) + assert fails == [{"sequence_id": "X_contig-0", "stage": "annotation", "category": "internal_error", + "exception_type": "ValueError", "message": "boom"}] + assert any("failed annotation" in str(w.message) for w in outer) # re-emitted + + +# ---------------------------------------------------------------- fragments + + +def test_fragment_counts_by_locus(): + ann = qc.annotation_frame(_ann_seqs()) + frag = qc.fragment_counts(ann, n_record_failures=1) + assert frag["unknown"] == {"n": 2, "truncated": 2, "incomplete_vdj": 2} + assert frag["IGH"] == {"n": 2, "truncated": 1, "incomplete_vdj": 1} + assert frag["IGK"] == {"n": 1, "truncated": 0, "incomplete_vdj": 1} + acc = qc.merge_fragment_counts({}, frag) + acc = qc.merge_fragment_counts(acc, frag) + summ = qc.fragment_summary(acc) + assert summ["passing_consensus"] == 10 + assert summ["fragment_fraction"] == pytest.approx(6 / 10) + assert summ["by_locus"]["IGH"]["fragment_fraction"] == pytest.approx(0.5) + + +# ---------------------------------------------------------------- UMI signature + + +def test_hamming1_pairs_exact(): + u = pl.DataFrame({ + "barcode": ["A", "A", "A", "A", "B", "B"], + "umi": ["AAAA", "AAAT", "AATT", "CCCC", "AAAA", "GGGG"], + }) + pairs, with_nb = qc._hamming1_pairs(u) + # A: AAAA-AAAT, AAAT-AATT; cross-barcode AAAA (B) must not count + assert pairs == 2 and with_nb == 3 + + +def test_umi_hamming1_stats_excess_over_null(): + import random + + rng = random.Random(1) + rows = [] + for b in range(200): + bc = f"BC{b:04d}" + for _ in range(5): + umi = "".join(rng.choice("ACGT") for _ in range(10)) + rows.append((bc, umi)) + err = umi[:3] + ("A" if umi[3] != "A" else "C") + umi[4:] + rows.append((bc, err)) # one error copy per molecule + parsed = pl.DataFrame(rows, schema=["barcode", "umi"], orient="row") + st = qc.umi_hamming1_stats(parsed) + assert st["h1_pairs"] >= 1000 and st["h1_pairs_null"] < 50 + summ = qc.umi_summary([st]) + assert summ["excess_error_like_umi_fraction"] > 0.9 + sub = qc.umi_hamming1_stats(parsed, max_umis=500) + assert sub["sampled"] and sub["umis"] < parsed.height + + +# ---------------------------------------------------------------- run-level metrics + + +def test_ambient_proxy(): + rows = [("IGH", "J1", f"B{i}") for i in range(10)] + [("IGH", "J2", "B0"), ("IGH", "J2", "B1"), + ("IGH", "J3", "B5"), ("IGK", "J1", "B9")] + df = pl.DataFrame(rows, schema=["locus", "junction", "barcode"], orient="row") + out = qc.ambient_proxy(df) + assert out["heavy_junctions"] == 3 + assert out["share_ge2_barcodes"] == pytest.approx(2 / 3, abs=1e-6) + assert out["share_ge10_barcodes"] == pytest.approx(1 / 3, abs=1e-6) + + +def test_clonal_coherence_and_chance(): + df = pl.DataFrame({ + "v_gene:0": ["H1", "H1", "H1", "H2", "H2", "H3"], + "cdr3_aa:0": ["C1", "C1", "C1", "C2", "C2", "C3"], + "v_gene:1": ["IGKV1-39", "IGKV1D-39", "IGKV3-20", "IGLV2-14", "IGLV2-14", "IGKV3-20"], + }) + out = qc.clonal_coherence(df) + # clone H1/C1: KV1-39 x2 (D- normalised) + KV3-20 -> top 2/3; clone H2/C2: 2/2 + assert out["clones_ge2"] == 2 and out["cells_in_clones"] == 5 + assert out["purity"] == pytest.approx(4 / 5) + assert out["purity_mean_clone"] == pytest.approx((2 / 3 + 1) / 2, abs=1e-4) + # light V frequencies 2/6, 2/6, 2/6 + assert out["chance"] == pytest.approx(1 / 3, abs=1e-4) + assert qc.clonal_coherence(pl.DataFrame())["purity"] is None + + +def test_confidence_distribution(): + assert qc.confidence_distribution(None) is None + out = qc.confidence_distribution(pl.Series([0.05, 0.5, 0.95, 1.0, None])) + assert out["n"] == 4 + assert out["bins"] == {"<0.1": 1, "0.1-0.3": 0, "0.3-0.6": 1, "0.6-0.9": 0, ">=0.9": 2} + + +def _write_well(ann_dir, meta_dir, well, rows): + """rows: (barcode, locus, junction, umis)""" + name = f"well{well:03d}.fastq" + df = pl.DataFrame({ + "sequence_id": [f"{b}_contig-{i}" for i, (b, *_r) in enumerate(rows)], + "locus": [r[1] for r in rows], + "productive": [True] * len(rows), + "v_gene": ["IGHV1-2" if r[1] == "IGH" else "IGKV1-39" for r in rows], + "j_gene": ["IGHJ4" if r[1] == "IGH" else "IGKJ1" for r in rows], + "junction": [r[2] for r in rows], + "cdr3_aa": ["CARW" for _ in rows], + "sequence": ["ACGT" for _ in rows], + }) + df.write_parquet(ann_dir / f"{name}_unpaired.parquet") + # pooled_pairing skips files < 100 bytes, so carry a realistic consensus column + pl.DataFrame({"name": df["sequence_id"], "umis": [r[3] for r in rows], + "consensus": ["ACGT" * 30 for _ in rows]}).write_csv(meta_dir / f"{name}.csv") + return name + + +def test_leakage_by_chain_needs_8_wells(tmp_path): + ann, meta = tmp_path / "annotated", tmp_path / "metadata" + ann.mkdir() + meta.mkdir() + # heavy species: 9 UMIs at home, 1 elsewhere; light: 5 at home, 5 spread + for w in range(8): + rows = [(f"B{w}", "IGH", f"TGTH{w}", 9), (f"B{w}", "IGK", f"TGTK{w}", 5)] + other = (w + 1) % 8 + rows += [(f"B{other}", "IGH", f"TGTH{other}", 1), (f"B{other}", "IGK", f"TGTK{other}", 5)] + _write_well(ann, meta, w, rows) + out = qc.leakage_by_chain(tmp_path) + assert out["wells"] == 8 + assert out["heavy"] == pytest.approx(0.1 / (1 - 1 / 8), abs=1e-3) + assert out["light"] == pytest.approx(0.5 / (1 - 1 / 8), abs=1e-3) + few = qc.leakage_by_chain(tmp_path, min_wells=9) + assert "skipped" in few + + +def test_write_qc_end_to_end(tmp_path): + ann, meta = tmp_path / "annotated", tmp_path / "metadata" + ann.mkdir() + meta.mkdir() + name = _write_well(ann, meta, 0, [("B0", "IGH", "TGTA", 3), ("B1", "IGH", "TGTA", 2)]) + pl.DataFrame({"name": ["B0"], "v_gene:0": ["IGHV1-2"], "cdr3_aa:0": ["CARW"], "v_gene:1": ["IGKV1-39"], + "confidence": [0.95]}).write_parquet(ann / f"{name}_paired.parquet") + row = qc.new_counts(name) + row["input_reads"] = 10 + rep = qc.write_qc(tmp_path, [row], [], {}, [], [name]) + for f in ("stage_counts.csv", "annotation_failures.csv", "report.json", "report.md"): + assert (tmp_path / "qc" / f).exists() + assert json.loads((tmp_path / "qc" / "report.json").read_text())["stage_totals"]["input_reads"] == 10 + assert rep["ambient_proxy"]["share_ge2_barcodes"] == 1.0 + assert rep["pair_confidence"]["n"] == 1 + assert "memory" in (tmp_path / "qc" / "report.md").read_text() + + +# ---------------------------------------------------------------- support columns + + +def test_add_support_columns(): + seqs = [_seq("AAAC_contig-0"), _seq("AAAC_contig-1"), _seq("GGGT_contig-0")] + meta = pl.DataFrame({"name": ["AAAC_contig-0", "AAAC_contig-1"], "reads": [10, 3], "umis": [4, 1], + "cluster_fraction": [0.77, 0.23], "consensus": ["A", "C"], "pass_filters": [True, True]}) + qc.add_support_columns(seqs, meta) + a = seqs[0].annotations + assert (a["umis"], a["reads"], a["cluster_fraction"]) == (4, 10, 0.77) + assert (a["barcode"], a["contig"]) == ("AAAC", "contig-0") + assert "umi" not in a + assert seqs[2]["umis"] is None and seqs[2]["barcode"] == "GGGT" + + +def test_support_columns_reach_paired_output(): + import abutils + + seqs = [_seq("AAAC_contig-0", locus="IGH"), _seq("AAAC_contig-1", locus="IGK")] + meta = pl.DataFrame({"name": ["AAAC_contig-0", "AAAC_contig-1"], "reads": [10, 3], "umis": [4, 1], + "cluster_fraction": [0.77, 0.23]}) + qc.add_support_columns(seqs, meta) + pairs = abutils.tl.assign_pairs(seqs, delim="_", delim_occurance=-1) + df = abutils.io.to_polars(pairs) if hasattr(abutils.io, "to_polars") else None + if df is None: + pytest.skip("abutils.io.to_polars unavailable") + assert df["umis:0"].to_list() == [4] and df["umis:1"].to_list() == [1] + assert "umi:0" not in df.columns