From 1058e27299ccfd8f2702832ca00067fc2fe7f734 Mon Sep 17 00:00:00 2001 From: Benjamin Nemoz Date: Tue, 17 Nov 2026 16:18:24 +0000 Subject: [PATCH 1/4] feat: directional UMI collapsing per (well, barcode) before UMI counting Adds pairplex/umi.py:collapse_umis_directional (UMI-tools directional adjacency, neighbour generation, N-containing UMIs absorbed but never absorbing). process_droplet collapses UMIs per droplet partition: umi_raw keeps the raw UMI, metadata umis is the corrected count and umis_raw the raw count; min_cluster_umis filters on corrected counts. run(umi_collapse='directional'|'none'), CLI --umi-collapse; run(legacy=True) / --legacy forces 'none' (v2-base behaviour). Co-Authored-By: Claude Opus 5.5 --- pairplex/cli/pairplex.py | 16 +++ pairplex/pairplex.py | 25 ++++ pairplex/tests/test_umi.py | 278 +++++++++++++++++++++++++++++++++++++ pairplex/umi.py | 131 +++++++++++++++++ pairplex/utils.py | 26 ++++ 5 files changed, 476 insertions(+) create mode 100644 pairplex/tests/test_umi.py create mode 100644 pairplex/umi.py diff --git a/pairplex/cli/pairplex.py b/pairplex/cli/pairplex.py index 8a39506..ce40bb1 100644 --- a/pairplex/cli/pairplex.py +++ b/pairplex/cli/pairplex.py @@ -98,6 +98,18 @@ def version(): default="human", help="Germline database for V(D)J assignment", ) +@click.option( + "--umi-collapse", + "umi_collapse", + type=click.Choice(["directional", "none"]), + default="directional", + help="UMI error correction within each (well, barcode): UMI-tools directional (default) or none", +) +@click.option( + "--legacy", + is_flag=True, + help="Reproduce v2-base behaviour exactly (forces --umi-collapse none)", +) @click.option("--quiet", is_flag=True, help="Whether to suppress output") @click.option( "--debug", @@ -118,6 +130,8 @@ def run( merge_paired_reads: bool, receptor: str, germline_database: str, + umi_collapse: str, + legacy: bool, quiet: bool, debug: bool, ): @@ -135,6 +149,8 @@ def run( merge_paired_reads=merge_paired_reads, receptor=receptor, germline_database=germline_database, + umi_collapse=umi_collapse, + legacy=legacy, quiet=quiet, debug=debug, ) diff --git a/pairplex/pairplex.py b/pairplex/pairplex.py index 85c4ed5..e311fc6 100644 --- a/pairplex/pairplex.py +++ b/pairplex/pairplex.py @@ -49,6 +49,8 @@ def run( merge_paired_reads: bool = False, receptor: str = "bcr", germline_database: str = "human", + umi_collapse: str = "directional", + legacy: bool = False, quiet: bool = False, debug: bool = False, ) -> list: @@ -105,6 +107,19 @@ def run( The germline database to use. Options are anything accepted by `abstar.run`, including custom databases if available. + umi_collapse : str + UMI error correction, applied within each (well, droplet barcode) before any UMI counting. + ``"directional"`` (default) uses the UMI-tools directional adjacency method: UMI ``b`` is + absorbed by UMI ``a`` when they differ at one position and ``reads(a) >= 2 * reads(b) - 1``, + transitively, processing UMIs by descending read count (see :mod:`pairplex.umi`, which also + documents how UMIs containing ``N`` are handled). The metadata ``umis`` column is then the + corrected count, a ``umis_raw`` column holds the count before collapsing, and + ``min_cluster_umis`` filters on corrected counts. ``"none"`` counts raw UMIs (v2-base + behaviour, no ``umis_raw`` column). + + legacy : bool + If ``True``, reproduce the v2-base behaviour exactly: forces ``umi_collapse="none"``. + quiet : bool Whether to suppress the progress bar and other update messages. @@ -120,6 +135,15 @@ def run( """ parameters = dict(locals()) # captured first: exactly the arguments of this call + # legacy mode forces every v2 behaviour option to its v2-base value + if legacy: + umi_collapse = "none" + if umi_collapse not in ("directional", "none"): + raise ValueError( + f"umi_collapse must be 'directional' or 'none', got {umi_collapse!r}" + ) + parameters["umi_collapse_effective"] = umi_collapse + # print splash screen if not quiet: print_splash(include_version=True) @@ -346,6 +370,7 @@ def run( "min_cluster_fraction": min_cluster_fraction, "consensus_downsample": consensus_downsample, "clustering_threshold": clustering_threshold, + "umi_collapse": umi_collapse, "quiet": quiet, "debug": debug, } diff --git a/pairplex/tests/test_umi.py b/pairplex/tests/test_umi.py new file mode 100644 index 0000000..be295db --- /dev/null +++ b/pairplex/tests/test_umi.py @@ -0,0 +1,278 @@ +import itertools +import random + +import polars as pl +import pytest + +from pairplex.umi import collapse_umi_column, collapse_umis_directional +from pairplex.utils import process_droplet + + +def _hamming(a, b): + return sum(x != y for x, y in zip(a, b)) + + +def _reference_directional(cnt): + """O(n^2) reference implementation, written the UMI-tools way: build the full directed + adjacency list, BFS components from nodes by descending count (ties by UMI), first come first + served, N-containing UMIs never absorb.""" + umis = sorted(cnt, key=lambda u: (-cnt[u], u)) + adj = {u: [] for u in umis} + for a, b in itertools.permutations(umis, 2): + if "N" in a or len(a) != len(b): + continue + if _hamming(a, b) == 1 and cnt[a] >= 2 * cnt[b] - 1: + adj[a].append(b) + rep = {} + for root in umis: + if root in rep: + continue + seen, queue = {root}, [root] + while queue: + n = queue.pop(0) + for m in adj[n]: + if m not in seen: + seen.add(m) + queue.append(m) + for u in seen: + rep.setdefault(u, root) + return rep + + +# ---------------------------------------------------------------- classic examples + + +def test_singleton(): + assert collapse_umis_directional({"AAAAAAAAAA": 1}) == {"AAAAAAAAAA": "AAAAAAAAAA"} + + +def test_empty(): + assert collapse_umis_directional({}) == {} + + +def test_simple_absorption(): + m = collapse_umis_directional({"AAAAAAAAAA": 10, "AAAAAAAAAC": 2}) + assert m == {"AAAAAAAAAA": "AAAAAAAAAA", "AAAAAAAAAC": "AAAAAAAAAA"} + + +def test_count_rule_boundary(): + # 2*count(b) - 1: a=5, b=3 -> 5 >= 5 absorbs; a=5, b=4 -> 5 >= 7 is false + assert collapse_umis_directional({"AAAAAAAAAA": 5, "AAAAAAAAAC": 3})["AAAAAAAAAC"] == "AAAAAAAAAA" + assert collapse_umis_directional({"AAAAAAAAAA": 5, "AAAAAAAAAC": 4})["AAAAAAAAAC"] == "AAAAAAAAAC" + + +def test_hamming_two_not_absorbed(): + m = collapse_umis_directional({"AAAAAAAAAA": 100, "AAAAAAAACC": 1}) + assert m["AAAAAAAACC"] == "AAAAAAAACC" + + +def test_chain_is_transitive(): + # a(9) -> b(4) -> c(1): c is 2 substitutions from a, still absorbed through b + a, b, c = "AAAAAAAAAA", "AAAAAAAAAC", "AAAAAAAACC" + m = collapse_umis_directional({a: 9, b: 4, c: 1}) + assert m == {a: a, b: a, c: a} + + +def test_chain_broken_by_count_rule(): + # a(9) -> b(4), but b(4) vs c(3): 4 >= 5 false, so c stays separate + a, b, c = "AAAAAAAAAA", "AAAAAAAAAC", "AAAAAAAACC" + m = collapse_umis_directional({a: 9, b: 4, c: 3}) + assert m == {a: a, b: a, c: c} + + +def test_ties_of_one_merge_deterministically(): + # 1 >= 2*1 - 1: two singleton UMIs at Hamming 1 merge; representative is the smaller string + m = collapse_umis_directional({"AAAAAAAAAC": 1, "AAAAAAAAAA": 1}) + assert m == {"AAAAAAAAAA": "AAAAAAAAAA", "AAAAAAAAAC": "AAAAAAAAAA"} + # order of input does not matter + assert m == collapse_umis_directional({"AAAAAAAAAA": 1, "AAAAAAAAAC": 1}) + + +def test_ties_above_one_do_not_merge(): + m = collapse_umis_directional({"AAAAAAAAAA": 3, "AAAAAAAAAC": 3}) + assert m == {"AAAAAAAAAA": "AAAAAAAAAA", "AAAAAAAAAC": "AAAAAAAAAC"} + + +def test_umi_tools_paper_example(): + # Smith et al. 2017 Fig. 1 style: one true UMI with errors, one independent UMI nearby + cnt = { + "ACGTACGTAC": 456, # true + "ACGTACGTAA": 72, # error of the true one + "ACGTACGTTA": 2, # error of the error (chain) + "ACGTACGTCC": 90, # error of the true one + "TTTTACGTAC": 30, # independent + "TTTTACGTAA": 6, # error of the independent one + } + m = collapse_umis_directional(cnt) + assert {u for u, r in m.items() if r == "ACGTACGTAC"} == { + "ACGTACGTAC", "ACGTACGTAA", "ACGTACGTTA", "ACGTACGTCC" + } + assert m["TTTTACGTAA"] == "TTTTACGTAC" + assert len(set(m.values())) == 2 + + +def test_node_claimed_by_first_root(): + # c is reachable from both a(10) and b(6, not reachable from a); a is processed first + a, b, c = "AAAAAAAAAA", "AAAAAAAACC", "AAAAAAAAAC" + m = collapse_umis_directional({a: 10, b: 6, c: 1}) + assert m == {a: a, b: b, c: a} + + +# ---------------------------------------------------------------- N handling + + +def test_n_umi_absorbed_but_never_absorbs(): + real, nn = "ACGTACGTAC", "ACGTNCGTAC" + assert collapse_umis_directional({real: 5, nn: 1})[nn] == real + # an abundant N UMI does not absorb an N-free neighbour + m = collapse_umis_directional({nn: 50, real: 1}) + assert m == {nn: nn, real: real} + + +def test_n_umi_respects_count_rule_and_double_n(): + real = "ACGTACGTAC" + assert collapse_umis_directional({real: 2, "ACGTNCGTAC": 2})["ACGTNCGTAC"] == "ACGTNCGTAC" + assert collapse_umis_directional({real: 50, "ACGTNNGTAC": 1})["ACGTNNGTAC"] == "ACGTNNGTAC" + + +def test_short_umis_only_compared_at_equal_length(): + m = collapse_umis_directional({"ACGTACGTAC": 10, "ACGTACGTA": 1}) + assert m["ACGTACGTA"] == "ACGTACGTA" + + +# ---------------------------------------------------------------- API forms + + +def test_api_forms_agree(): + reads = ["AAAAAAAAAA"] * 4 + ["AAAAAAAAAC"] + ["GGGGGGGGGG"] * 2 + by_reads = collapse_umis_directional(reads) + by_map = collapse_umis_directional({"AAAAAAAAAA": 4, "AAAAAAAAAC": 1, "GGGGGGGGGG": 2}) + by_lists = collapse_umis_directional(["AAAAAAAAAA", "AAAAAAAAAC", "GGGGGGGGGG"], [4, 1, 2]) + assert by_reads == by_map == by_lists + assert collapse_umi_column(reads) == ["AAAAAAAAAA"] * 5 + ["GGGGGGGGGG"] * 2 + with pytest.raises(ValueError): + collapse_umis_directional(["A", "A"], [1, 2]) + with pytest.raises(ValueError): + collapse_umis_directional(["A"], [1, 2]) + + +# ---------------------------------------------------------------- randomized vs reference + + +@pytest.mark.parametrize("seed", range(20)) +def test_matches_reference_on_random_sets(seed): + rng = random.Random(seed) + L = 6 # short UMIs so Hamming-1 collisions are frequent + true = ["".join(rng.choice("ACGT") for _ in range(L)) for _ in range(rng.randint(1, 15))] + cnt = {} + for t in true: + n = rng.randint(1, 60) + for _ in range(n): + u = list(t) + if rng.random() < 0.25: + u[rng.randrange(L)] = rng.choice("ACGTN" if rng.random() < 0.9 else "N") + u = "".join(u) + cnt[u] = cnt.get(u, 0) + 1 + assert collapse_umis_directional(cnt) == _reference_directional(cnt) + + +def test_large_barcode_is_fast(): + import time + + rng = random.Random(0) + reads = [] + for _ in range(3000): + t = "".join(rng.choice("ACGT") for _ in range(10)) + for _ in range(rng.randint(1, 20)): + u = list(t) + if rng.random() < 0.2: + u[rng.randrange(10)] = rng.choice("ACGT") + reads.append("".join(u)) + t0 = time.perf_counter() + m = collapse_umis_directional(reads) + assert time.perf_counter() - t0 < 5 + assert len(set(m.values())) < len(set(reads)) + + +# ---------------------------------------------------------------- integration via process_droplet + +HEAVY = "CAGGTGCAGCTGGTGCAGTCTGGGGCTGAGGTGAAGAAGCCTGGGGCCTCAGTGAAGGTCTCCTGCAAGGCTTCTGGATACACC" +LIGHT = "GACATCCAGATGACCCAGTCTCCATCCTCCCTGTCTGCATCTGTAGGAGACAGAGTCACCATCACTTGCCGGGCAAGTCAG" + + +def _droplet_df(): + rows = [] + # heavy: true UMI A (6 reads) + 1-error copy (1 read) + independent UMI G (3 reads) + heavy_umis = ["AAAAAAAAAA"] * 6 + ["AAAAAAAAAT"] + ["GGGGGGGGGG"] * 3 + # light: one true UMI (4 reads) + two 1-error copies (1 read each) + light_umis = ["CCCCCCCCCC"] * 4 + ["CCCCCCCCCA", "CCCCCCCCCT"] + for i, u in enumerate(heavy_umis): + rows.append({"umi": u, "barcode": "BC", "seq_id": f"h{i}", "sequence": HEAVY}) + for i, u in enumerate(light_umis): + rows.append({"umi": u, "barcode": "BC", "seq_id": f"l{i}", "sequence": LIGHT}) + return pl.DataFrame(rows) + + +def _by_reads(meta): + return {m["reads"]: m for m in meta} + + +def test_process_droplet_directional(tmp_path): + meta = process_droplet( + "well_BC", _droplet_df(), temp_directory=tmp_path, clustering_threshold=0.9, + min_cluster_reads=3, min_cluster_umis=2, + ) + h, l = _by_reads(meta)[10], _by_reads(meta)[6] + assert (h["umis"], h["umis_raw"]) == (2, 3) + assert (l["umis"], l["umis_raw"]) == (1, 3) + # min_cluster_umis filters on corrected counts: light had 3 raw UMIs but only 1 molecule + assert h["pass_filters"] and not l["pass_filters"] + assert list(h) == ["name", "reads", "umis", "umis_raw", "cluster_fraction", "consensus", "pass_filters"] + + +def test_process_droplet_none_is_v2_base_layout(tmp_path): + meta = process_droplet( + "well_BC", _droplet_df(), temp_directory=tmp_path, clustering_threshold=0.9, + min_cluster_reads=3, min_cluster_umis=2, umi_collapse="none", + ) + h, l = _by_reads(meta)[10], _by_reads(meta)[6] + assert h["umis"] == 3 and l["umis"] == 3 + assert h["pass_filters"] and l["pass_filters"] + assert list(h) == ["name", "reads", "umis", "cluster_fraction", "consensus", "pass_filters"] + + +def test_process_droplet_rejects_unknown_mode(tmp_path): + with pytest.raises(ValueError): + process_droplet("well_BC", _droplet_df(), temp_directory=tmp_path, umi_collapse="cluster") + + +def test_run_legacy_forces_none(tmp_path, monkeypatch): + """legacy=True must pass umi_collapse='none' to process_droplet, whatever umi_collapse says.""" + import pairplex.pairplex as pp + + seen = {} + + class _Stop(Exception): + pass + + def fake_pool(*a, **k): + raise _Stop + + monkeypatch.setattr(pp, "ProcessPoolExecutor", fake_pool) + real_runlog = pp.RunLog + + def capture(path, tool, parameters, inputs): + seen.update(parameters) + return real_runlog(path, tool=tool, parameters=parameters, inputs=inputs) + + monkeypatch.setattr(pp, "RunLog", capture) + fq = tmp_path / "w.fastq" + fq.write_text("@r\nACGT\n+\nIIII\n") + with pytest.raises(_Stop): + pp.run(str(fq), tmp_path / "out", quiet=True, legacy=True, umi_collapse="directional") + assert seen["umi_collapse_effective"] == "none" + with pytest.raises(_Stop): + pp.run(str(fq), tmp_path / "out2", quiet=True) + assert seen["umi_collapse_effective"] == "directional" + with pytest.raises(ValueError): + pp.run(str(fq), tmp_path / "out3", quiet=True, umi_collapse="bogus") diff --git a/pairplex/umi.py b/pairplex/umi.py new file mode 100644 index 0000000..8d0f863 --- /dev/null +++ b/pairplex/umi.py @@ -0,0 +1,131 @@ +# Copyright (c) 2025 Benjamin Nemoz +# Distributed under the terms of the MIT License. +# SPDX-License-Identifier: MIT + +# This file is part of PairPlex. +# PairPlex is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# PairPlex is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# You should have received a copy of the GNU General Public License +# along with PairPlex. If not, see . + +"""UMI error correction. + +PairPlex collapses UMIs within one (well, droplet barcode) group with the *directional* +adjacency method of UMI-tools (Smith, Heger & Sudbery, Genome Res. 2017): + +- nodes are the distinct raw UMIs, weighted by their read count; +- there is a directed edge ``a -> b`` when ``Hamming(a, b) == 1`` and + ``count(a) >= 2 * count(b) - 1``; +- UMIs are visited in descending count; each UMI not yet absorbed becomes a representative and + absorbs every UMI reachable from it along directed edges, transitively: ``a -> b -> c`` puts + ``c`` under ``a`` although ``c`` is two substitutions away from ``a``. A UMI reached from an + earlier representative is never re-assigned. + +Ties in count are broken by the UMI string (lexicographic), so the result is deterministic. +UMI-tools leaves tie order to dict order; the groups are the same, only which tied UMI is named +representative can differ. + +UMIs containing ``N``: ``N`` counts as a mismatch at its position. An N-containing UMI never +absorbs anything (it would be Hamming-1 from up to four different real UMIs, so it is no evidence +for any of them), but it can be absorbed by an N-free UMI at Hamming distance 1 that satisfies the +count rule (e.g. ``ACGTNCGTAC`` under ``ACGTACGTAC``). An N-containing UMI with no such parent +stays its own UMI. Characters other than ``ACGTN`` are compared literally and never generate +neighbours. + +Neighbour generation (``4 * L`` single substitutions over ``ACGTN`` per UMI) keeps the cost +linear in the number of distinct UMIs, so barcodes with thousands of reads are cheap. +""" + +from __future__ import annotations + +from collections import Counter +from collections.abc import Iterable, Mapping + +__all__ = ["collapse_umis_directional", "collapse_umi_column"] + +_ALPHABET = "ACGTN" + + +def _neighbours(umi: str): + """All strings at Hamming distance 1 from ``umi`` over ``ACGTN`` (positions with other + characters are skipped).""" + for i, c in enumerate(umi): + if c not in _ALPHABET: + continue + head, tail = umi[:i], umi[i + 1 :] + for b in _ALPHABET: + if b != c: + yield head + b + tail + + +def collapse_umis_directional( + umis: Iterable[str] | Mapping[str, int], + counts: Iterable[int] | None = None, +) -> dict[str, str]: + """ + Directional UMI collapsing (UMI-tools) within one group of reads. + + Parameters + ---------- + umis : iterable of str, or mapping UMI -> count + Distinct raw UMIs. If a mapping is given, its values are the counts and ``counts`` must be + ``None``. If ``counts`` is ``None`` and ``umis`` is a plain iterable, it is treated as one + entry per READ and counted (duplicates allowed). + + counts : iterable of int, optional + Reads per raw UMI, aligned with ``umis`` (which must then be distinct). + + Returns + ------- + dict[str, str] + Mapping from every input UMI to its representative (a representative maps to itself). + """ + if isinstance(umis, Mapping): + if counts is not None: + raise ValueError("pass counts either in the mapping or in `counts`, not both") + cnt = dict(umis) + elif counts is None: + cnt = dict(Counter(umis)) + else: + umis = list(umis) + counts = list(counts) + if len(umis) != len(counts): + raise ValueError("umis and counts must have the same length") + cnt = {} + for u, c in zip(umis, counts): + if u in cnt: + raise ValueError(f"duplicate UMI with explicit counts: {u}") + cnt[u] = int(c) + + order = sorted(cnt, key=lambda u: (-cnt[u], u)) + rep: dict[str, str] = {} + for root in order: + if root in rep: + continue + rep[root] = root + stack = [root] + while stack: + parent = stack.pop() + if "N" in parent: # N-containing UMIs never absorb + continue + threshold = cnt[parent] + for nb in _neighbours(parent): + c = cnt.get(nb) + if c is None or nb in rep: + continue + if threshold >= 2 * c - 1: + rep[nb] = root + stack.append(nb) + return rep + + +def collapse_umi_column(umis: list[str]) -> list[str]: + """Per-read convenience wrapper: corrected UMI for each read UMI in ``umis`` (one group).""" + mapping = collapse_umis_directional(umis) + return [mapping[u] for u in umis] diff --git a/pairplex/utils.py b/pairplex/utils.py index 0c599a3..eb89083 100644 --- a/pairplex/utils.py +++ b/pairplex/utils.py @@ -29,6 +29,7 @@ from natsort import natsorted from spoa import poa +from .umi import collapse_umis_directional from .version import __version__ BARCODE_DIR = Path(__file__).resolve().parent / "barcodes" @@ -333,6 +334,7 @@ def process_droplet( min_cluster_reads: int = 3, min_cluster_umis: int = 2, min_cluster_fraction: float = 0.0, + umi_collapse: str = "directional", quiet: bool = False, debug: bool = False, ) -> list[dict]: @@ -365,6 +367,14 @@ def process_droplet( min_cluster_fraction : float Minimum fraction of reads in a cluster. + umi_collapse : str + UMI error correction applied to the whole droplet (all reads of this barcode in this well) + before any counting. ``"directional"`` (default) collapses UMIs with the UMI-tools + directional method (see :mod:`pairplex.umi`): the raw UMI of each read is kept in + ``umi_raw``, ``umi`` becomes the corrected representative, the metadata ``umis`` is the + corrected count and ``umis_raw`` the count before collapsing. ``"none"`` keeps raw UMIs and + the v2-base metadata layout (no ``umis_raw`` column). + quiet : bool Whether to suppress output. @@ -381,6 +391,16 @@ def process_droplet( barcode = name.split("_")[-1] metadata = [] + # UMI error correction, per (well, barcode): this partition is exactly one barcode of one well + if umi_collapse == "directional": + mapping = collapse_umis_directional(partition_df["umi"].to_list()) + partition_df = partition_df.with_columns( + pl.col("umi").alias("umi_raw"), + pl.col("umi").replace_strict(mapping, return_dtype=pl.String).alias("umi"), + ) + elif umi_collapse != "none": + raise ValueError(f"unknown umi_collapse: {umi_collapse!r}") + # sequences = abutils.io.from_polars( # partition_df, id_key="seq_id", sequence_key="sequence" # ) @@ -398,6 +418,8 @@ def process_droplet( contig_name = f"{barcode}_contig-{i}" cluster_df = partition_df.filter(pl.col("seq_id").is_in(clust)) n_umis = len(cluster_df["umi"].unique()) + if umi_collapse != "none": + n_umis_raw = len(cluster_df["umi_raw"].unique()) cluster_fraction = len(clust) / partition_df.shape[0] # consensus @@ -417,6 +439,10 @@ def process_droplet( "name": contig_name, "reads": len(clust), "umis": n_umis, + } + if umi_collapse != "none": + meta["umis_raw"] = n_umis_raw + meta |= { "cluster_fraction": cluster_fraction, "consensus": consensus, } From 564bb4cc093c6c998f90647b1192a005abff21ad Mon Sep 17 00:00:00 2001 From: Benjamin Nemoz Date: Tue, 17 Nov 2026 16:19:52 +0000 Subject: [PATCH 2/4] Scored, dominance-aware per-well pairing with confidence and margin (fix 7) Barcodes with several productive chains in a well were dropped by the exact 1 heavy + 1 light rule. The new default ranks productive heavy and light (IGK+IGL) chains by UMIs then reads, emits the top heavy + top light when each group's top chain has >= min_dominance x the second's UMIs (default 2), and scores each pair: confidence = s_H * s_L with s = u1/(U + 1), margin = min over groups of (u1 - u2)/(U + 1). Per-well paired files gain confidence, margin, umis:0, umis:1; all wells are written to pairs/pairs.parquet in the contract schema. pairing='exact' / legacy=True keep the v2-base rule. Co-Authored-By: Claude Opus 5.5 --- pairplex/cli/pairplex.py | 24 +++ pairplex/pairing.py | 291 +++++++++++++++++++++++++++++++++ pairplex/pairplex.py | 59 ++++++- pairplex/tests/test_pairing.py | 175 ++++++++++++++++++++ 4 files changed, 543 insertions(+), 6 deletions(-) create mode 100644 pairplex/pairing.py create mode 100644 pairplex/tests/test_pairing.py diff --git a/pairplex/cli/pairplex.py b/pairplex/cli/pairplex.py index 8a39506..abd349e 100644 --- a/pairplex/cli/pairplex.py +++ b/pairplex/cli/pairplex.py @@ -98,6 +98,24 @@ def version(): default="human", help="Germline database for V(D)J assignment", ) +@click.option( + "--pairing", + type=click.Choice(["scored", "exact"]), + default="scored", + help="Per-well pairing rule: 'scored' resolves multi-chain barcodes by UMI dominance and scores each pair; 'exact' keeps only 1 heavy + 1 light barcodes (v2-base)", +) +@click.option( + "--min-dominance", + "min_dominance", + type=float, + default=2.0, + help="Scored pairing: top chain of each locus group needs >= this many times the UMIs of the second-best chain", +) +@click.option( + "--legacy", + is_flag=True, + help="Reproduce v2-base behaviour (forces --pairing exact)", +) @click.option("--quiet", is_flag=True, help="Whether to suppress output") @click.option( "--debug", @@ -118,6 +136,9 @@ def run( merge_paired_reads: bool, receptor: str, germline_database: str, + pairing: str, + min_dominance: float, + legacy: bool, quiet: bool, debug: bool, ): @@ -135,6 +156,9 @@ def run( merge_paired_reads=merge_paired_reads, receptor=receptor, germline_database=germline_database, + pairing=pairing, + min_dominance=min_dominance, + legacy=legacy, quiet=quiet, debug=debug, ) diff --git a/pairplex/pairing.py b/pairplex/pairing.py new file mode 100644 index 0000000..9a0988e --- /dev/null +++ b/pairplex/pairing.py @@ -0,0 +1,291 @@ +# Copyright (c) 2025 Benjamin Nemoz +# Distributed under the terms of the MIT License. +# SPDX-License-Identifier: MIT + +""" +Per-well heavy/light pairing of droplet barcodes. + +Two rules are available: + +``"exact"`` (legacy, ``v2-base``) + Keep a barcode only if ``abutils.tl.assign_pairs`` finds exactly one heavy and one light + chain (any productivity). Barcodes with several chains are dropped. + +``"scored"`` (default) + Dominance-aware resolution. For each (well, barcode), the *productive* heavy chains (IGH) + and the *productive* light chains (IGK and IGL together: a cell expresses one light chain, + so kappa and lambda compete in the same group) are ranked by support: corrected UMIs from + the metadata CSV, then reads as a tie-break, then contig name for determinism. + The top heavy and the top light are emitted when, in each group, + + u1 >= min_dominance * u2 + + where ``u1``/``u2`` are the UMIs of the best and second-best chain (``u2 = 0`` for a + single chain). Each emitted pair gets + + * ``confidence = s_H * s_L`` with ``s_g = u1_g / (U_g + pseudocount)``, ``U_g`` the total + UMIs of the group. ``s_g`` is the top chain's share of the group's support after adding + ``pseudocount`` UMIs for an unobserved competing chain (Laplace-style). It is in [0, 1), + increases with the top chain's UMIs and decreases with every competitor's UMIs, so a + deep 1H+1L barcode scores close to 1, a 1-UMI 1H+1L barcode scores 0.25 and an even + split scores low (and is not emitted unless ``min_dominance <= 1``). + * ``margin = min_g (u1_g - u2_g) / (U_g + pseudocount)``: the smaller, over the two + groups, of the gap between the top and the second-best chain's share. In [0, 1); equal + to ``min(s_H, s_L)`` for a 1H+1L barcode. + * ``umis`` on each chain (``umis:0`` heavy, ``umis:1`` light in the paired outputs). + +Default ``min_dominance = 2.0``: the top chain must hold at least two thirds of a 2-chain +group's UMIs. In real data about half of the 1H+2L / 2H+1L barcodes meet it; near-even splits +(where picking the top chain is close to a coin flip) are left unpaired. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + +import abutils +import polars as pl + +HEAVY_LOCI = ("IGH",) +LIGHT_LOCI = ("IGK", "IGL") +PAIRING_MODES = ("scored", "exact") +DEFAULT_MIN_DOMINANCE = 2.0 +DEFAULT_PSEUDOCOUNT = 1.0 + +# contract columns of pairs/pairs.parquet (see diagnosis/bench/README.md), per chain +_CHAIN_COLS = ["locus", "v_gene", "j_gene", "junction", "cdr3_aa", "sequence_id", "sequence", "umis"] +_WELL_RE = re.compile(r"(\d+)") + + +@dataclass(frozen=True) +class Chain: + """Support of one contig: name, UMIs, reads.""" + + name: str + umis: int + reads: int = 0 + + +@dataclass(frozen=True) +class Resolution: + """Outcome of resolving one barcode. ``heavy``/``light`` are None when not emitted.""" + + heavy: Chain | None + light: Chain | None + confidence: float | None + margin: float | None + reason: str # "paired", "no_heavy", "no_light", "heavy_ambiguous", "light_ambiguous", "both_ambiguous" + + +def _rank(chains: Iterable[Chain]) -> list[Chain]: + return sorted(chains, key=lambda c: (-c.umis, -c.reads, c.name)) + + +def _group_stats(ranked: list[Chain], pseudocount: float) -> tuple[float, float, int, int]: + u1 = ranked[0].umis + u2 = ranked[1].umis if len(ranked) > 1 else 0 + denom = sum(c.umis for c in ranked) + pseudocount + if denom <= 0: + return 0.0, 0.0, u1, u2 + return u1 / denom, (u1 - u2) / denom, u1, u2 + + +def resolve_barcode( + heavies: Iterable[Chain], + lights: Iterable[Chain], + min_dominance: float = DEFAULT_MIN_DOMINANCE, + pseudocount: float = DEFAULT_PSEUDOCOUNT, +) -> Resolution: + """ + Pick the top heavy and top light chain of one (well, barcode) and score the pair. + + Parameters + ---------- + heavies, lights : iterable of Chain + Productive heavy chains and productive light chains (IGK and IGL together). + min_dominance : float + Emit only if, in each group, the top chain has >= ``min_dominance`` x the UMIs of the + second-best chain. Values <= 1 accept any split (ties broken by reads, then name). + pseudocount : float + UMIs added to each group's total for an unobserved competing chain (see module doc). + + Returns + ------- + Resolution + """ + hs, ls = _rank(heavies), _rank(lights) + if not hs or not ls: + return Resolution(None, None, None, None, "no_heavy" if not hs else "no_light") + sh, mh, h1, h2 = _group_stats(hs, pseudocount) + sl, ml, l1, l2 = _group_stats(ls, pseudocount) + h_ok = h1 >= min_dominance * h2 + l_ok = l1 >= min_dominance * l2 + if not (h_ok and l_ok): + reason = "both_ambiguous" if not (h_ok or l_ok) else ("heavy_ambiguous" if not h_ok else "light_ambiguous") + return Resolution(None, None, None, None, reason) + return Resolution(hs[0], ls[0], sh * sl, min(mh, ml), "paired") + + +def _barcode(sequence_id: str) -> str: + # same grouping as abutils.tl.assign_pairs(delim="_", delim_occurance=-1) + return "_".join(sequence_id.split("_")[:-1]) + + +def _productive(seq) -> bool: + p = seq["productive"] + if isinstance(p, str): + return p.strip().lower() in ("true", "t", "1", "yes") + return bool(p) + + +def exact_pairs(sequences: list) -> list: + """Legacy rule (``v2-base``): barcodes with exactly one heavy and one light chain.""" + pairs = abutils.tl.assign_pairs(sequences, delim="_", delim_occurance=-1) + return [p for p in pairs if len(p.heavies) == 1 and len(p.lights) == 1] + + +def scored_pairs( + sequences: list, + support: dict[str, tuple[int, int]], + min_dominance: float = DEFAULT_MIN_DOMINANCE, + pseudocount: float = DEFAULT_PSEUDOCOUNT, +) -> tuple[list, dict[str, int]]: + """ + Resolve every barcode of one well with ``resolve_barcode``. + + Parameters + ---------- + sequences : list of abutils Sequence + Annotated contigs of one well (``sequence_id`` = ``_``). + support : dict + ``name -> (umis, reads)`` from the metadata CSV. Missing contigs count as 1 UMI, 0 reads. + + Returns + ------- + (pairs, reasons) : list of abutils ``Pair`` (heavy and light chains carry a ``umis`` + annotation; the pair has ``confidence`` and ``margin`` attributes) and a count per + resolution reason. + """ + groups: dict[str, tuple[list, list]] = {} + by_id = {} + for s in sequences: + if not _productive(s): + continue + locus = s["locus"] + if locus in HEAVY_LOCI: + side = 0 + elif locus in LIGHT_LOCI: + side = 1 + else: + continue + sid = s["sequence_id"] + by_id[sid] = s + groups.setdefault(_barcode(sid), ([], []))[side].append(sid) + + pairs, reasons = [], {} + for bc, (h_ids, l_ids) in groups.items(): + def chains(ids): + out = [] + for i in ids: + u, r = support.get(i, (1, 0)) + out.append(Chain(i, int(u), int(r))) + return out + + res = resolve_barcode(chains(h_ids), chains(l_ids), min_dominance, pseudocount) + reasons[res.reason] = reasons.get(res.reason, 0) + 1 + if res.heavy is None: + continue + heavy, light = by_id[res.heavy.name], by_id[res.light.name] + heavy["umis"] = res.heavy.umis + light["umis"] = res.light.umis + pairs.append( + abutils.Pair( + [heavy, light], + name=bc, + properties={"confidence": res.confidence, "margin": res.margin}, + ) + ) + return pairs, reasons + + +def support_from_metadata(metadata_df: pl.DataFrame) -> dict[str, tuple[int, int]]: + """``name -> (umis, reads)`` from a PairPlex metadata table.""" + if metadata_df.is_empty() or "name" not in metadata_df.columns: + return {} + umis = metadata_df["umis"] if "umis" in metadata_df.columns else pl.Series([1] * metadata_df.height) + reads = metadata_df["reads"] if "reads" in metadata_df.columns else pl.Series([0] * metadata_df.height) + return {n: (int(u if u is not None else 1), int(r if r is not None else 0)) + for n, u, r in zip(metadata_df["name"], umis, reads)} + + +def write_paired(pairs: list, airr_file: str | Path, parquet_file: str | Path, scored: bool) -> None: + """Write per-well paired files; scored pairs get ``confidence`` and ``margin`` columns.""" + props = ["confidence", "margin"] if scored else None + abutils.io.to_airr(pairs, str(airr_file), properties=props, drop_na_columns=False) + abutils.io.to_parquet(pairs, str(parquet_file), properties=props, drop_na_columns=False) + + +def pair_well( + sequences: list, + paired_airr_file: str | Path, + paired_parquet_file: str | Path, + mode: str = "scored", + support: dict[str, tuple[int, int]] | None = None, + min_dominance: float = DEFAULT_MIN_DOMINANCE, + pseudocount: float = DEFAULT_PSEUDOCOUNT, +) -> list: + """Pair one well's annotated contigs with the chosen rule and write the paired files.""" + if mode == "exact": + pairs = exact_pairs(sequences) + # identical to v2-base output + abutils.io.to_airr(pairs, str(paired_airr_file), drop_na_columns=False) + abutils.io.to_parquet(pairs, str(paired_parquet_file), drop_na_columns=False) + return pairs + if mode != "scored": + raise ValueError(f"unknown pairing mode {mode!r}; expected one of {PAIRING_MODES}") + pairs, _ = scored_pairs(sequences, support or {}, min_dominance, pseudocount) + write_paired(pairs, paired_airr_file, paired_parquet_file, scored=True) + return pairs + + +def well_number(name: str) -> int | None: + """Well index from a file stem (first number: ``well007.fastq`` -> 7, ``VDJ_74`` -> 74).""" + m = _WELL_RE.search(name) + return int(m.group(1)) if m else None + + +def contract_frame(paired_parquet: str | Path, well: int | None) -> pl.DataFrame | None: + """One well's paired file in the ``pairs/pairs.parquet`` contract schema.""" + p = Path(paired_parquet) + if not p.exists() or p.stat().st_size == 0: + return None + df = pl.read_parquet(p) + if df.height == 0: + return None + cols = [pl.col("name").alias("barcode"), pl.lit(well, pl.Int64).alias("well"), pl.col("name")] + for c in ("confidence", "margin"): + cols.append(pl.col(c).cast(pl.Float64) if c in df.columns else pl.lit(None, pl.Float64).alias(c)) + for s in (0, 1): + for c in _CHAIN_COLS: + k = f"{c}:{s}" + cols.append(pl.col(k) if k in df.columns else pl.lit(None).alias(k)) + return df.select(cols).with_columns(pl.col("umis:0").cast(pl.Int64), pl.col("umis:1").cast(pl.Int64)) + + +def write_contract_pairs(frames: list[pl.DataFrame], output_file: str | Path) -> int: + """Concatenate per-well contract frames into ``pairs/pairs.parquet``; returns the row count.""" + out = Path(output_file) + out.parent.mkdir(parents=True, exist_ok=True) + frames = [f for f in frames if f is not None and f.height] + if frames: + df = pl.concat(frames, how="diagonal_relaxed") + else: + schema = {"barcode": pl.Utf8, "well": pl.Int64, "name": pl.Utf8, "confidence": pl.Float64, "margin": pl.Float64} + for s in (0, 1): + schema.update({f"{c}:{s}": (pl.Int64 if c == "umis" else pl.Utf8) for c in _CHAIN_COLS}) + df = pl.DataFrame(schema=schema) + df.write_parquet(out) + return df.height diff --git a/pairplex/pairplex.py b/pairplex/pairplex.py index 85c4ed5..3b0fd64 100644 --- a/pairplex/pairplex.py +++ b/pairplex/pairplex.py @@ -27,6 +27,15 @@ from natsort import natsorted from tqdm.auto import tqdm +from .pairing import ( + DEFAULT_MIN_DOMINANCE, + PAIRING_MODES, + contract_frame, + pair_well, + support_from_metadata, + well_number, + write_contract_pairs, +) from .provenance import RunLog, run_log_path from .utils import available_cpus, parse_barcodes, print_splash, process_droplet @@ -49,6 +58,9 @@ def run( merge_paired_reads: bool = False, receptor: str = "bcr", germline_database: str = "human", + pairing: str = "scored", + min_dominance: float = DEFAULT_MIN_DOMINANCE, + legacy: bool = False, quiet: bool = False, debug: bool = False, ) -> list: @@ -105,6 +117,23 @@ def run( The germline database to use. Options are anything accepted by `abstar.run`, including custom databases if available. + pairing : str + How heavy and light chains of a (well, barcode) are paired. ``"scored"`` (default): rank + the productive heavy chains and the productive light chains (IGK and IGL together) by + UMIs (then reads) and emit the top heavy + top light when each group passes + ``min_dominance``; every pair gets a ``confidence`` and a ``margin`` (see + ``pairplex.pairing``), and all wells are also written to ``pairs/pairs.parquet``. + ``"exact"``: the ``v2-base`` rule, keep only barcodes with exactly one heavy and one + light chain; no ``pairs/pairs.parquet``. + + min_dominance : float + Scored pairing only. The top chain of each locus group must have at least this many + times the UMIs of the second-best chain. Defaults to ``2.0`` (top chain holds >= 2/3 of + a 2-chain group). Values <= 1 accept any split (ties broken by reads). + + legacy : bool + If ``True``, reproduce ``v2-base`` behaviour: forces ``pairing="exact"``. + quiet : bool Whether to suppress the progress bar and other update messages. @@ -119,6 +148,11 @@ def run( """ parameters = dict(locals()) # captured first: exactly the arguments of this call + if legacy: + pairing = "exact" + if pairing not in PAIRING_MODES: + raise ValueError(f"pairing must be one of {PAIRING_MODES}, got {pairing!r}") + contract_frames = [] # print splash screen if not quiet: @@ -443,13 +477,19 @@ def run( pairs_printer = tqdm(total=0, bar_format="{desc}", position=9, leave=False) paired_airr_file = annotated_directory / f"{name}_paired.tsv" paired_parquet_file = annotated_directory / f"{name}_paired.parquet" - pairs = abutils.tl.assign_pairs(sequences, delim="_", delim_occurance=-1) - pairs = [p for p in pairs if len(p.heavies) == 1 and len(p.lights) == 1] - pairs_printer.set_description_str(f"{len(pairs)} paired sequences") - abutils.io.to_airr(pairs, str(paired_airr_file), drop_na_columns=False) - abutils.io.to_parquet( - pairs, str(paired_parquet_file), drop_na_columns=False + pairs = pair_well( + sequences, + paired_airr_file, + paired_parquet_file, + mode=pairing, + support=support_from_metadata(metadata_df), + min_dominance=min_dominance, ) + pairs_printer.set_description_str(f"{len(pairs)} paired sequences") + if pairing == "scored": + contract_frames.append( + contract_frame(paired_parquet_file, well_number(name)) + ) all_pairs += len(pairs) running_total_printer.set_description_str( @@ -481,12 +521,19 @@ def run( # update the main progress bar main_pbar.update(1) + n_contract_pairs = None + if pairing == "scored": + n_contract_pairs = write_contract_pairs( + contract_frames, output_directory / "pairs" / "pairs.parquet" + ) + run_log.finish( stats={ "input_files": len(input_files), "valid_barcodes": all_valid_barcodes, "consensus_sequences": all_consensus_sequences, "pairs": all_pairs, + **({"pairs_parquet_rows": n_contract_pairs} if n_contract_pairs is not None else {}), } ) diff --git a/pairplex/tests/test_pairing.py b/pairplex/tests/test_pairing.py new file mode 100644 index 0000000..1ef328e --- /dev/null +++ b/pairplex/tests/test_pairing.py @@ -0,0 +1,175 @@ +import os +from pathlib import Path + +import abutils +import polars as pl +import pytest + +from pairplex.pairing import ( + Chain, + contract_frame, + exact_pairs, + pair_well, + resolve_barcode, + scored_pairs, + well_number, + write_contract_pairs, +) + + +# ---------------------------------------------------------------- resolve_barcode + + +def test_1h1l_strong_support_scores_high(): + r = resolve_barcode([Chain("bc_h", 30, 300)], [Chain("bc_l", 40, 400)]) + assert r.reason == "paired" and (r.heavy.name, r.light.name) == ("bc_h", "bc_l") + assert r.confidence == pytest.approx(30 / 31 * 40 / 41) + assert r.confidence > 0.9 + assert r.margin == pytest.approx(min(30 / 31, 40 / 41)) + + +def test_1h1l_single_umi_scores_low(): + r = resolve_barcode([Chain("h", 1)], [Chain("l", 1)]) + assert r.reason == "paired" and r.confidence == pytest.approx(0.25) + + +def test_1h2l_dominant_light_is_resolved(): + r = resolve_barcode([Chain("h", 10)], [Chain("l1", 2), Chain("l2", 8)]) + assert r.reason == "paired" and r.light.name == "l2" + assert r.confidence == pytest.approx(10 / 11 * 8 / 11) + assert r.margin == pytest.approx(min(10 / 11, 6 / 11)) + # lower than the same barcode without the competing light + alone = resolve_barcode([Chain("h", 10)], [Chain("l2", 8)]) + assert r.confidence < alone.confidence + + +def test_ambiguous_2h2l_not_emitted(): + r = resolve_barcode([Chain("h1", 5), Chain("h2", 4)], [Chain("l1", 6), Chain("l2", 5)]) + assert r.heavy is None and r.confidence is None and r.reason == "both_ambiguous" + r = resolve_barcode([Chain("h1", 10), Chain("h2", 1)], [Chain("l1", 6), Chain("l2", 5)]) + assert r.heavy is None and r.reason == "light_ambiguous" + + +def test_kappa_and_lambda_compete_in_one_group(): + lights = [Chain("k", 9), Chain("lam", 3)] + r = resolve_barcode([Chain("h", 5)], lights) + assert r.light.name == "k" + assert resolve_barcode([Chain("h", 5)], [Chain("k", 4), Chain("lam", 3)]).heavy is None + + +def test_threshold_boundary_and_tie_break_by_reads(): + assert resolve_barcode([Chain("h", 4)], [Chain("l1", 4), Chain("l2", 2)]).reason == "paired" + assert resolve_barcode([Chain("h", 4)], [Chain("l1", 4), Chain("l2", 3)]).reason == "light_ambiguous" + r = resolve_barcode([Chain("h", 4)], [Chain("l1", 3, 10), Chain("l2", 3, 50)], min_dominance=1.0) + assert r.light.name == "l2" and r.margin == pytest.approx(0.0) + + +def test_confidence_is_monotonic(): + def conf(h1, h2, l1): + return resolve_barcode([Chain("a", h1)] + ([Chain("b", h2)] if h2 else []), [Chain("l", l1)], + min_dominance=1.0).confidence + assert conf(5, 0, 5) < conf(6, 0, 5) < conf(6, 0, 6) + assert conf(6, 1, 5) < conf(6, 0, 5) + assert conf(6, 3, 5) < conf(6, 1, 5) + for h1, h2, l1 in [(1, 0, 1), (100, 0, 100), (3, 3, 2)]: + assert 0.0 <= conf(h1, h2, l1) <= 1.0 + + +def test_missing_locus_group(): + assert resolve_barcode([], [Chain("l", 3)]).reason == "no_heavy" + assert resolve_barcode([Chain("h", 3)], []).reason == "no_light" + + +# ---------------------------------------------------------------- well-level + + +def _seq(bc, n, locus, productive=True): + return abutils.Sequence({"sequence_id": f"{bc}_contig-{n}", "sequence": "ACGT" * 10, "locus": locus, + "productive": productive, "v_gene": f"{locus}V1", "j_gene": f"{locus}J1", + "v_call": f"{locus}V1*01", "junction": "TGT" + "A" * n, "cdr3_aa": "CAR"}) + + +def _well(): + seqs = [ + _seq("AAA", 0, "IGH"), _seq("AAA", 1, "IGK"), # 1H+1L + _seq("BBB", 0, "IGH"), _seq("BBB", 1, "IGK"), _seq("BBB", 2, "IGL"), # 1H+2L, lambda dominant + _seq("CCC", 0, "IGH"), _seq("CCC", 1, "IGH"), _seq("CCC", 2, "IGK"), _seq("CCC", 3, "IGK"), # ambiguous + _seq("DDD", 0, "IGH"), _seq("DDD", 1, "IGH", productive=False), _seq("DDD", 2, "IGL"), # non-productive extra H + ] + support = {"AAA_contig-0": (5, 50), "AAA_contig-1": (6, 60), + "BBB_contig-0": (4, 40), "BBB_contig-1": (1, 10), "BBB_contig-2": (7, 70), + "CCC_contig-0": (3, 30), "CCC_contig-1": (3, 31), "CCC_contig-2": (2, 20), "CCC_contig-3": (2, 21), + "DDD_contig-0": (2, 20), "DDD_contig-1": (9, 90), "DDD_contig-2": (2, 20)} + return seqs, support + + +def test_scored_pairs_on_a_well(): + seqs, support = _well() + pairs, reasons = scored_pairs(seqs, support) + got = {p.name: (p.heavy["sequence_id"], p.light["sequence_id"]) for p in pairs} + assert got == {"AAA": ("AAA_contig-0", "AAA_contig-1"), "BBB": ("BBB_contig-0", "BBB_contig-2"), + "DDD": ("DDD_contig-0", "DDD_contig-2")} + assert reasons == {"paired": 3, "both_ambiguous": 1} + bbb = next(p for p in pairs if p.name == "BBB") + assert bbb.light["umis"] == 7 and bbb.confidence == pytest.approx(4 / 5 * 7 / 9) + # the legacy rule keeps only the strict 1H+1L barcode + assert {p.name for p in exact_pairs(seqs)} == {"AAA"} + + +def test_pair_well_writes_outputs_and_contract(tmp_path): + seqs, support = _well() + tsv, pq = tmp_path / "well007_paired.tsv", tmp_path / "well007_paired.parquet" + pairs = pair_well(seqs, tsv, pq, mode="scored", support=support) + df = pl.read_parquet(pq) + assert df.height == len(pairs) == 3 + for c in ("confidence", "margin", "umis:0", "umis:1", "sequence_id:0", "sequence_id:1"): + assert c in df.columns + assert "confidence" in pl.read_csv(tsv, separator="\t").columns + cf = contract_frame(pq, well_number("well007.fastq")) + out = tmp_path / "pairs" / "pairs.parquet" + assert write_contract_pairs([cf, None], out) == 3 + p = pl.read_parquet(out) + for c in ("barcode", "well", "name", "confidence") + tuple( + f"{c}:{s}" for s in (0, 1) for c in ("locus", "v_gene", "j_gene", "junction", "cdr3_aa", + "sequence_id", "sequence", "umis")): + assert c in p.columns + assert set(p["well"]) == {7} and p["confidence"].is_between(0, 1).all() + assert set(p["locus:0"]) == {"IGH"} + + +def test_empty_contract_has_schema(tmp_path): + out = tmp_path / "pairs.parquet" + assert write_contract_pairs([], out) == 0 + assert {"well", "name", "confidence", "sequence_id:0", "sequence:1"} <= set(pl.read_parquet(out).columns) + + +def test_exact_mode_matches_legacy_columns(tmp_path): + seqs, support = _well() + pq = tmp_path / "w_paired.parquet" + pair_well(seqs, tmp_path / "w_paired.tsv", pq, mode="exact", support=support) + df = pl.read_parquet(pq) + assert df["name"].to_list() == ["AAA"] and "confidence" not in df.columns + + +# ---------------------------------------------------------------- end-to-end smoke (opt-in) + +SMOKE = os.environ.get("PAIRPLEX_SMOKE_READS") + + +@pytest.mark.skipif(not SMOKE, reason="set PAIRPLEX_SMOKE_READS to a directory of per-well FASTQs") +def test_run_smoke_scored_vs_legacy(tmp_path): + from pairplex.pairplex import run + + run(SMOKE, tmp_path / "legacy", legacy=True, quiet=True) + run(SMOKE, tmp_path / "scored", quiet=True) + assert not (tmp_path / "legacy" / "pairs").exists() + pairs = pl.read_parquet(tmp_path / "scored" / "pairs" / "pairs.parquet") + n_scored = n_legacy = 0 + for f in sorted((tmp_path / "scored" / "annotated").glob("*_paired.parquet")): + df = pl.read_parquet(f) + n_scored += df.height + assert {"confidence", "margin", "umis:0", "umis:1"} <= set(df.columns) + leg = Path(str(f).replace("/scored/", "/legacy/")) + n_legacy += pl.read_parquet(leg).height + assert "confidence" not in pl.read_parquet(leg).columns + assert pairs.height == n_scored > 0 and n_legacy > 0 From 6f5f888f9fca454430804434384afc2f072bd668 Mon Sep 17 00:00:00 2001 From: Benjamin Nemoz Date: Tue, 17 Nov 2026 16:25:34 +0000 Subject: [PATCH 3/4] feat(qc): support columns, per-stage drop counts, abstar failure table, QC report - annotated outputs carry cluster support: umis, reads, cluster_fraction, barcode, contig (unpaired) and umis:0/1, reads:0/1, ... (paired); the always-null abstar `umi` column is dropped (a consensus contig has no single UMI; the count lives in `umis`) - qc/stage_counts.csv: drop counts at every stage, per well and TOTAL (reads, barcode groups, empty sequences skipped by greedy_clustering, clusters failing each filter, annotation outcomes, chain composition, emitted pairs) - qc/annotation_failures.csv: abstar record errors (from its failures.tsv warning), unassigned reasons, productivity issues, missing junctions - qc/report.{json,md}: fragment fraction by locus, UMI Hamming-1 excess vs shuffled null, heavy-junction multi-barcode share, leakage f by chain (>= 8 wells), pair confidence distribution, clonal light-chain purity with chance and a memory-only caveat - new run() kwargs support_columns=True, qc=True, legacy=False (legacy forces both off and reproduces v2-base); matching CLI options - pairing is unchanged; logic lives in pairplex/qc.py behind small hooks Co-Authored-By: Claude Opus 5.5 --- pairplex/cli/pairplex.py | 21 ++ pairplex/pairplex.py | 75 +++- pairplex/qc.py | 729 ++++++++++++++++++++++++++++++++++++++ pairplex/tests/test_qc.py | 325 +++++++++++++++++ 4 files changed, 1144 insertions(+), 6 deletions(-) create mode 100644 pairplex/qc.py create mode 100644 pairplex/tests/test_qc.py diff --git a/pairplex/cli/pairplex.py b/pairplex/cli/pairplex.py index 8a39506..e03afbc 100644 --- a/pairplex/cli/pairplex.py +++ b/pairplex/cli/pairplex.py @@ -104,6 +104,21 @@ def version(): is_flag=True, help="Whether to enable debug mode, which saves all temporary files to ease troubleshooting", ) +@click.option( + "--support_columns/--no_support_columns", + default=True, + help="Carry cluster support (umis, reads, cluster_fraction) into the annotated outputs", +) +@click.option( + "--qc/--no_qc", + default=True, + help="Write qc/stage_counts.csv, qc/annotation_failures.csv and qc/report.{json,md}", +) +@click.option( + "--legacy", + is_flag=True, + help="Reproduce PairPlex v2-base exactly (disables support columns and QC outputs)", +) def run( sequences: str | Path, output_directory: str | Path, @@ -120,6 +135,9 @@ def run( germline_database: str, quiet: bool, debug: bool, + support_columns: bool, + qc: bool, + legacy: bool, ): run_pairplex( sequences=sequences, @@ -137,6 +155,9 @@ def run( germline_database=germline_database, quiet=quiet, debug=debug, + support_columns=support_columns, + qc=qc, + legacy=legacy, ) diff --git a/pairplex/pairplex.py b/pairplex/pairplex.py index 85c4ed5..df38125 100644 --- a/pairplex/pairplex.py +++ b/pairplex/pairplex.py @@ -18,6 +18,7 @@ import multiprocessing as mp import os import time +from contextlib import nullcontext from concurrent.futures import ProcessPoolExecutor, as_completed from pathlib import Path @@ -27,6 +28,7 @@ from natsort import natsorted from tqdm.auto import tqdm +from . import qc as qc_mod from .provenance import RunLog, run_log_path from .utils import available_cpus, parse_barcodes, print_splash, process_droplet @@ -51,6 +53,9 @@ def run( germline_database: str = "human", quiet: bool = False, debug: bool = False, + support_columns: bool = True, + qc: bool = True, + legacy: bool = False, ) -> list: """ Run PairPlex. @@ -111,6 +116,24 @@ def run( debug : bool If ``True``, all temporary files will be retained to ease debugging. + support_columns : bool + If ``True`` (default), the annotated outputs carry the cluster support of each contig: + ``umis``, ``reads``, ``cluster_fraction``, ``barcode`` and ``contig`` in the unpaired + outputs, ``umis:0/1``, ``reads:0/1``, ``cluster_fraction:0/1`` (etc.) in the paired outputs. + abstar's ``umi`` column, which is always null for PairPlex consensus contigs, is dropped. + Pairing is unchanged. + + qc : bool + If ``True`` (default), write ``qc/stage_counts.csv`` (drop counts at every stage, per + well and in total), ``qc/annotation_failures.csv`` (abstar failure and exclusion modes) + and ``qc/report.json`` / ``qc/report.md`` (fragment fraction, UMI error signature, + ambient proxy, leakage by chain, pair confidence, clonal light-chain coherence). + Pairing is unchanged. + + legacy : bool + If ``True``, reproduce PairPlex v2-base exactly: forces ``support_columns=False`` and + ``qc=False``. + Returns ------- @@ -119,6 +142,9 @@ def run( """ parameters = dict(locals()) # captured first: exactly the arguments of this call + if legacy: + support_columns = False + qc = False # print splash screen if not quiet: @@ -218,6 +244,9 @@ def run( f"valid barcodes: {all_valid_barcodes} | consensus sequences: {all_consensus_sequences} | pairs: {all_pairs}" ) + # QC accumulators (qc=True): per-well stage counts, abstar failure modes, fragments, UMI stats + qc_rows, qc_failures, qc_fragments, qc_umis, qc_wells = [], [], {}, [], [] + # initialize the Process Pool with ProcessPoolExecutor( max_workers=available_cpus(), mp_context=mp.get_context("spawn") @@ -239,6 +268,10 @@ def run( input_file = Path(input_file) name = input_file.stem name_printer.set_description_str(f"---- {name} ----") + if qc: + wc = qc_mod.new_counts(name) + qc_rows.append(wc) + qc_wells.append(name) # check to make sure the input file is FASTA/Q (and not empty) if abutils.io.determine_fastx_format(str(input_file)) is None: @@ -257,6 +290,8 @@ def run( for s in abutils.io.parse_fastx(str(input_file)): input_count += 1 seqs_printer.set_description_str(f"{input_count} input sequences") + if qc: + wc["input_reads"] = wc["dropped_orientation_or_barcode"] = input_count # split input file into chunks main_pbar.set_postfix_str("splitting input file", refresh=True) @@ -297,6 +332,10 @@ def run( ) df = pl.read_parquet(concat_parquet) seqs_with_barcodes = df.shape[0] + if qc: + wc.update(qc_mod.read_stage_counts(input_count, df)) + wc.update(qc_mod.barcode_stage_counts(df, min_cluster_reads)) + qc_umis.append(qc_mod.umi_hamming1_stats(df)) all_valid_barcodes += seqs_with_barcodes valids_printer.set_description_str( f"{seqs_with_barcodes} sequences with valid barcodes" @@ -377,6 +416,12 @@ def run( ) metadata_df = pl.DataFrame(metadata) consensus_pbar.close() + if qc: + wc.update( + qc_mod.cluster_stage_counts( + metadata_df, min_cluster_reads, min_cluster_umis, min_cluster_fraction + ) + ) # write metadata to file metadata_file = metadata_directory / f"{name}.csv" @@ -415,12 +460,21 @@ def run( if consensus_count < 1000: mmseqs_threads = 1 # run abstar - sequences = abstar.run( - sequences=str(consensus_file), - germline_database=germline_database, - receptor=receptor, - mmseqs_threads=mmseqs_threads, - ) + with qc_mod.capture_abstar_failures() if qc else nullcontext([]) as record_failures: + sequences = abstar.run( + sequences=str(consensus_file), + germline_database=germline_database, + receptor=receptor, + mmseqs_threads=mmseqs_threads, + ) + if qc: + ann_df = qc_mod.annotation_frame(sequences) + n_rec = len(record_failures) + wc.update(qc_mod.annotation_stage_counts(ann_df, consensus_count, n_rec)) + qc_failures.extend(qc_mod.annotation_failure_rows(name, ann_df, record_failures)) + qc_mod.merge_fragment_counts(qc_fragments, qc_mod.fragment_counts(ann_df, n_rec)) + if support_columns: + qc_mod.add_support_columns(sequences, metadata_df) if len(sequences) > 1: # A single sequence cannot be paired and will cause issues @@ -444,7 +498,11 @@ def run( paired_airr_file = annotated_directory / f"{name}_paired.tsv" paired_parquet_file = annotated_directory / f"{name}_paired.parquet" pairs = abutils.tl.assign_pairs(sequences, delim="_", delim_occurance=-1) + if qc: + wc.update(qc_mod.chain_composition_counts(pairs)) pairs = [p for p in pairs if len(p.heavies) == 1 and len(p.lights) == 1] + if qc: + wc["emitted_pairs"] = len(pairs) pairs_printer.set_description_str(f"{len(pairs)} paired sequences") abutils.io.to_airr(pairs, str(paired_airr_file), drop_na_columns=False) abutils.io.to_parquet( @@ -481,6 +539,11 @@ def run( # update the main progress bar main_pbar.update(1) + if qc: + qc_mod.write_qc( + output_directory, qc_rows, qc_failures, qc_fragments, qc_umis, qc_wells + ) + run_log.finish( stats={ "input_files": len(input_files), diff --git a/pairplex/qc.py b/pairplex/qc.py new file mode 100644 index 0000000..37367a8 --- /dev/null +++ b/pairplex/qc.py @@ -0,0 +1,729 @@ +# Copyright (c) 2025 Benjamin Nemoz +# Distributed under the terms of the MIT License. +# SPDX-License-Identifier: MIT + +""" +Run QC for ``pairplex run``: support columns, per-stage drop counts, abstar failure table and a +per-run QC report. + +Everything here counts from data ``run()`` already holds (the parsed reads of a well, the cluster +metadata, the abstar results, the assigned pairs) or reads back a few columns of the per-well +outputs at the end of the run. Nothing here changes which pairs are emitted. + +Outputs (under ``/qc/``): + +- ``stage_counts.csv``: one row per well plus a ``TOTAL`` row, columns ``STAGE_FIELDS``; +- ``annotation_failures.csv``: abstar failure / exclusion modes with counts per well and in total; +- ``report.json`` and ``report.md``: run-level QC metrics (see ``build_report``). +""" + +from __future__ import annotations + +import csv +import json +import math +import re +import warnings +from collections import Counter +from contextlib import contextmanager +from pathlib import Path + +import numpy as np +import polars as pl + +# -------------------------------------------------------------------------------------------- +# per-stage counts +# -------------------------------------------------------------------------------------------- + +# 0H0L = a barcode whose contigs all failed annotation (no locus); "other" = any other combination +COMPOSITIONS = ["1H1L", "1H0L", "0H1L", "1H2L", "2H1L", "2H2L", "0H0L", "other"] + +STAGE_FIELDS = [ + # reads + "input_reads", + "valid_barcode_reads", + "dropped_orientation_or_barcode", + # barcode groups + "barcode_groups", + "groups_dropped_min_cluster_reads", + "reads_in_dropped_groups", + "empty_sequences_skipped", + # clusters + "clusters", + "clusters_fail_reads", + "clusters_fail_umis", + "clusters_fail_fraction", + "clusters_pass", + # annotation + "consensus_to_annotation", + "annotation_returned", + "annotated", + "failed_annotation", + "failed_unassigned", + "failed_record_error", + "productive", + "unproductive", + "no_junction", + # pairing + "barcodes_with_chains", + *[f"barcodes_{c}" for c in COMPOSITIONS], + "emitted_pairs", +] + + +def new_counts(well: str) -> dict: + """An all-zero stage-count row for ``well``.""" + return {"well": well, **{f: 0 for f in STAGE_FIELDS}} + + +def read_stage_counts(input_reads: int, parsed: pl.DataFrame) -> dict: + """Reads in, reads with a valid barcode, reads dropped (no orientation gave a valid barcode).""" + valid = int(parsed.height) + return { + "input_reads": int(input_reads), + "valid_barcode_reads": valid, + "dropped_orientation_or_barcode": int(input_reads) - valid, + } + + +def barcode_stage_counts(parsed: pl.DataFrame, min_cluster_reads: int) -> dict: + """ + Barcode groups, groups (and their reads) dropped by ``min_cluster_reads``, and the empty + sequences in kept groups, which ``greedy_clustering`` skips silently. + """ + if parsed.height == 0: + return {"barcode_groups": 0, "groups_dropped_min_cluster_reads": 0, + "reads_in_dropped_groups": 0, "empty_sequences_skipped": 0} + g = parsed.group_by("barcode").agg( + pl.len().alias("n"), + (pl.col("sequence").fill_null("").str.len_chars() == 0).sum().alias("empty"), + ) + dropped = g.filter(pl.col("n") < min_cluster_reads) + kept = g.filter(pl.col("n") >= min_cluster_reads) + return { + "barcode_groups": int(g.height), + "groups_dropped_min_cluster_reads": int(dropped.height), + "reads_in_dropped_groups": int(dropped["n"].sum()), + "empty_sequences_skipped": int(kept["empty"].sum()), + } + + +def cluster_stage_counts( + metadata: pl.DataFrame, + min_cluster_reads: int, + min_cluster_umis: int, + min_cluster_fraction: float, +) -> dict: + """Clusters and clusters failing each filter (a cluster can fail several).""" + if metadata.height == 0: + return {"clusters": 0, "clusters_fail_reads": 0, "clusters_fail_umis": 0, + "clusters_fail_fraction": 0, "clusters_pass": 0} + return { + "clusters": int(metadata.height), + "clusters_fail_reads": int((metadata["reads"] < min_cluster_reads).sum()), + "clusters_fail_umis": int((metadata["umis"] < min_cluster_umis).sum()), + "clusters_fail_fraction": int((metadata["cluster_fraction"] < min_cluster_fraction).sum()), + "clusters_pass": int(metadata["pass_filters"].sum()), + } + + +# columns pulled from abstar results (kept small: this is per consensus sequence) +ANN_SCHEMA = { + "sequence_id": pl.String, + "annotation_status": pl.String, + "failure_reason": pl.String, + "locus": pl.String, + "productive": pl.Boolean, + "productivity_issues": pl.String, + "junction": pl.String, + "complete_vdj": pl.Boolean, + "v_germline_start": pl.Int64, +} + + +def annotation_frame(sequences) -> pl.DataFrame: + """A small polars frame (``ANN_SCHEMA`` columns) from abstar's returned ``Sequence`` objects.""" + if sequences is None: + sequences = [] + elif not isinstance(sequences, list | tuple): + sequences = [sequences] + rows = [] + for s in sequences: + ann = s.annotations or {} + rows.append({c: ann.get(c) for c in ANN_SCHEMA}) + return pl.DataFrame(rows, schema=ANN_SCHEMA, strict=False) + + +def annotation_stage_counts(ann: pl.DataFrame, n_sent: int, n_record_failures: int = 0) -> dict: + """ + Consensus sequences sent to abstar and what came back. + + ``failed_annotation`` = ``failed_unassigned`` (returned with ``annotation_status`` other than + ``annotated``) + ``failed_record_error`` (no row returned: abstar record failures). + ``unproductive`` and ``no_junction`` are counted among annotated sequences. + """ + annotated = ann.filter(pl.col("annotation_status") == "annotated") + unassigned = ann.height - annotated.height + record = max(int(n_record_failures), int(n_sent) - ann.height) + return { + "consensus_to_annotation": int(n_sent), + "annotation_returned": int(ann.height), + "annotated": int(annotated.height), + "failed_unassigned": int(unassigned), + "failed_record_error": int(record), + "failed_annotation": int(unassigned + record), + "productive": int(annotated["productive"].fill_null(False).sum()), + "unproductive": int((~annotated["productive"].fill_null(False)).sum()), + "no_junction": int(annotated["junction"].is_null().sum()), + } + + +def composition_label(n_heavy: int, n_light: int) -> str: + label = f"{n_heavy}H{n_light}L" + return label if label in COMPOSITIONS else "other" + + +def chain_composition_counts(pairs) -> dict: + """Barcodes by chain composition, from ``abutils.tl.assign_pairs`` output (before the 1H1L rule).""" + c = Counter(composition_label(len(p.heavies), len(p.lights)) for p in pairs) + out = {f"barcodes_{k}": int(c.get(k, 0)) for k in COMPOSITIONS} + out["barcodes_with_chains"] = int(sum(c.values())) + return out + + +def stage_counts_frame(rows: list[dict]) -> pl.DataFrame: + """Per-well rows plus a ``TOTAL`` row.""" + schema = {"well": pl.String, **{f: pl.Int64 for f in STAGE_FIELDS}} + df = pl.DataFrame(rows, schema=schema, strict=False).fill_null(0) + total = df.select(pl.lit("TOTAL").alias("well"), *[pl.col(f).sum() for f in STAGE_FIELDS]) + return pl.concat([df, total], how="vertical_relaxed") + + +# -------------------------------------------------------------------------------------------- +# abstar failure modes +# -------------------------------------------------------------------------------------------- + +_FAILURES_TSV = re.compile(r"see (\S*failures\.tsv)") + + +@contextmanager +def capture_abstar_failures(): + """ + Collect abstar record failures (records abstar drops, with no output row). + + abstar reports them only through a ``RuntimeWarning`` pointing at a ``failures.tsv`` in its + log directory. The warning is re-emitted unchanged; the parsed rows (``stage``, ``category``, + ``exception_type``, ``message``) are appended to the yielded list. + """ + failures: list[dict] = [] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + yield failures + for w in caught: + m = _FAILURES_TSV.search(str(w.message)) + if m and issubclass(w.category, RuntimeWarning): + failures.extend(read_failures_tsv(m.group(1))) + warnings.warn_explicit(w.message, w.category, w.filename, w.lineno) + + +def read_failures_tsv(path: str | Path) -> list[dict]: + keep = ("sequence_id", "stage", "category", "exception_type", "message") + try: + with open(path, newline="") as f: + return [{k: r.get(k) for k in keep} for r in csv.DictReader(f, delimiter="\t")] + except OSError: + return [] + + +FAILURE_COLUMNS = {"well": pl.String, "kind": pl.String, "mode": pl.String, "count": pl.Int64} + + +def annotation_failure_rows(well: str, ann: pl.DataFrame, record_failures: list[dict]) -> list[dict]: + """ + Failure / exclusion modes for one well. ``kind`` is one of: + + - ``record_error``: abstar dropped the record (mode = ``stage/category exception: message``); + - ``unassigned``: returned but not annotated (mode = abstar ``failure_reason``); + - ``unproductive``: annotated but unproductive (mode = each ``productivity_issues`` item, so a + sequence with several issues counts once per issue); + - ``no_junction``: annotated with no junction. + """ + c: Counter = Counter() + for r in record_failures: + exc = f" {r['exception_type']}" if r.get("exception_type") else "" + c[("record_error", f"{r.get('stage')}/{r.get('category')}{exc}: {r.get('message')}"[:200])] += 1 + annotated = ann.filter(pl.col("annotation_status") == "annotated") + for reason in ann.filter(pl.col("annotation_status") != "annotated")["failure_reason"].to_list(): + c[("unassigned", reason or "unknown")] += 1 + unprod = annotated.filter(~pl.col("productive").fill_null(False)) + for issues in unprod["productivity_issues"].to_list(): + items = [i.strip() for i in (issues or "").split("|") if i.strip()] or ["unspecified"] + for i in items: + c[("unproductive", i)] += 1 + n_nojunc = int(annotated["junction"].is_null().sum()) + if n_nojunc: + c[("no_junction", "junction is null")] += n_nojunc + return [{"well": well, "kind": k, "mode": m, "count": n} for (k, m), n in sorted(c.items())] + + +def failure_table(rows: list[dict]) -> pl.DataFrame: + """Per-well failure rows plus ``TOTAL`` rows (one per kind/mode).""" + df = pl.DataFrame(rows, schema=FAILURE_COLUMNS) + if df.height == 0: + return df + total = df.group_by("kind", "mode").agg(pl.col("count").sum()).with_columns(pl.lit("TOTAL").alias("well")) + total = total.select(list(FAILURE_COLUMNS)).sort(["kind", "count"], descending=[False, True]) + return pl.concat([df, total]) + + +# -------------------------------------------------------------------------------------------- +# fragment fraction +# -------------------------------------------------------------------------------------------- + +# a contig is "truncated" when its V alignment starts this many nt or more into the germline V +# (i.e. most of FR1 is missing) or when it has no junction +TRUNCATION_NT = 30 + + +def fragment_counts(ann: pl.DataFrame, n_record_failures: int = 0) -> dict: + """ + Per-locus counts of passing consensus sequences: ``{locus: {"n", "truncated", "incomplete_vdj"}}``, + with failed annotations (unassigned + record errors) under ``"unknown"``. + """ + out: dict = {} + failed = ann.filter(pl.col("annotation_status") != "annotated").height + int(n_record_failures) + if failed: + out["unknown"] = {"n": failed, "truncated": failed, "incomplete_vdj": failed} + annotated = ann.filter(pl.col("annotation_status") == "annotated").with_columns( + ((pl.col("v_germline_start").fill_null(10**6) > TRUNCATION_NT) | pl.col("junction").is_null()).alias("_trunc"), + (~pl.col("complete_vdj").fill_null(False)).alias("_inc"), + ) + for r in annotated.group_by(pl.col("locus").fill_null("unknown")).agg( + pl.len().alias("n"), pl.col("_trunc").sum().alias("truncated"), pl.col("_inc").sum().alias("incomplete_vdj") + ).iter_rows(named=True): + d = out.setdefault(r["locus"], {"n": 0, "truncated": 0, "incomplete_vdj": 0}) + for k in ("n", "truncated", "incomplete_vdj"): + d[k] += int(r[k]) + return out + + +def merge_fragment_counts(acc: dict, new: dict) -> dict: + for locus, d in new.items(): + a = acc.setdefault(locus, {"n": 0, "truncated": 0, "incomplete_vdj": 0}) + for k, v in d.items(): + a[k] += v + return acc + + +def fragment_summary(frag: dict) -> dict: + n = sum(d["n"] for d in frag.values()) + bad = sum(d["truncated"] for d in frag.values()) + by_locus = { + locus: {**d, "fragment_fraction": _ratio(d["truncated"], d["n"])} for locus, d in sorted(frag.items()) + } + return { + "definition": ( + "share of passing consensus sequences that fail annotation or are truncated " + f"(V alignment starts > {TRUNCATION_NT} nt into germline V, or no junction); " + "failed annotations have no locus ('unknown')" + ), + "passing_consensus": n, + "fragment_fraction": _ratio(bad, n), + "failed_annotation_fraction": _ratio(frag.get("unknown", {}).get("n", 0), n), + "by_locus": by_locus, + } + + +# -------------------------------------------------------------------------------------------- +# UMI error signature +# -------------------------------------------------------------------------------------------- + + +def _hamming1_pairs(u: pl.DataFrame) -> tuple[int, int]: + """(Hamming-1 UMI pairs within barcode, distinct UMIs with >=1 Hamming-1 neighbour) for unique (barcode, umi).""" + if u.height < 2: + return 0, 0 + L = int(u["umi"].str.len_chars().max()) + keyed = pl.concat( + [ + u.select( + "barcode", "umi", pl.lit(i, pl.Int8).alias("pos"), + (pl.col("umi").str.slice(0, i) + pl.col("umi").str.slice(i + 1)).alias("key"), + ) + for i in range(L) + ] + ) + g = keyed.group_by("barcode", "pos", "key").agg(pl.len().alias("k"), pl.col("umi")) + g = g.filter(pl.col("k") >= 2) + if g.height == 0: + return 0, 0 + pairs = int((g["k"] * (g["k"] - 1) // 2).sum()) + with_nb = g.select("barcode", "umi").explode("umi").unique().height + return pairs, with_nb + + +def umi_hamming1_stats(parsed: pl.DataFrame, max_umis: int = 200_000, seed: int = 0) -> dict: + """ + Hamming-1 UMI pairs within barcode, observed vs a null that shuffles barcodes across the + distinct (barcode, UMI) rows (keeps each barcode's UMI count and the UMI composition). + + Exact (no pairwise loop): each Hamming-1 pair shares exactly one "UMI with position i + masked" key. Barcodes are subsampled when a well has more than ``max_umis`` distinct + (barcode, UMI) rows. + """ + empty = {"umis": 0, "barcodes": 0, "sampled": False, "h1_pairs": 0, "h1_pairs_null": 0, + "umis_with_h1_neighbour": 0, "umis_with_h1_neighbour_null": 0} + if parsed.height == 0: + return empty + u = parsed.select("barcode", "umi").drop_nulls().unique() + if u.height == 0: + return empty + L = u["umi"].str.len_chars().mode().sort()[-1] + u = u.filter(pl.col("umi").str.len_chars() == L) + sampled = False + if u.height > max_umis: + bcs = u["barcode"].unique().sort() + keep = bcs.sample(fraction=max_umis / u.height, seed=seed) + u = u.filter(pl.col("barcode").is_in(keep.implode())) + sampled = True + obs, obs_nb = _hamming1_pairs(u) + shuffled = u.with_columns(pl.col("barcode").shuffle(seed=seed)).unique() + null, null_nb = _hamming1_pairs(shuffled) + return {"umis": int(u.height), "barcodes": int(u["barcode"].n_unique()), "sampled": sampled, + "h1_pairs": obs, "h1_pairs_null": null, + "umis_with_h1_neighbour": obs_nb, "umis_with_h1_neighbour_null": null_nb} + + +def umi_summary(per_well: list[dict]) -> dict: + keys = ["umis", "h1_pairs", "h1_pairs_null", "umis_with_h1_neighbour", "umis_with_h1_neighbour_null"] + t = {k: int(sum(d.get(k, 0) for d in per_well)) for k in keys} + t["sampled_wells"] = int(sum(bool(d.get("sampled")) for d in per_well)) + t["excess_h1_pairs"] = _ratio(t["h1_pairs"], t["h1_pairs_null"]) if t["h1_pairs_null"] else None + obs_f = _ratio(t["umis_with_h1_neighbour"], t["umis"]) + null_f = _ratio(t["umis_with_h1_neighbour_null"], t["umis"]) + t["error_like_umi_fraction"] = obs_f + t["error_like_umi_fraction_null"] = null_f + t["excess_error_like_umi_fraction"] = ( + None if obs_f is None else round(max(0.0, obs_f - (null_f or 0.0)), 6) + ) + t["definition"] = ( + "Hamming-1 UMI pairs among distinct UMIs of the same barcode vs a null with barcodes " + "shuffled across (barcode, UMI) rows; excess_h1_pairs = observed / null" + ) + return t + + +# -------------------------------------------------------------------------------------------- +# run-level metrics from the per-well outputs +# -------------------------------------------------------------------------------------------- + +_LIGHT = ["IGK", "IGL"] + + +def load_unpaired(files: list[tuple[str, Path]]) -> pl.DataFrame: + """Productive annotated contigs (barcode, well, locus, v_gene, j_gene, junction, cdr3_aa) of the run.""" + cols = ["sequence_id", "locus", "productive", "v_gene", "j_gene", "junction", "cdr3_aa"] + out = [] + for well, f in files: + schema = pl.read_parquet_schema(f) + if not all(c in schema for c in cols): + continue + out.append( + pl.read_parquet(f, columns=cols) + .filter(pl.col("productive").fill_null(False) & pl.col("junction").is_not_null()) + .with_columns(pl.col("sequence_id").str.split("_").list.first().alias("barcode"), pl.lit(well).alias("well")) + .drop("productive") + ) + if not out: + return pl.DataFrame(schema={c: pl.String for c in ["sequence_id", "locus", "v_gene", "j_gene", "junction", "cdr3_aa", "barcode", "well"]}) + return pl.concat(out, how="diagonal_relaxed") + + +def ambient_proxy(unpaired: pl.DataFrame) -> dict: + """Share of heavy-chain junctions (nt) seen under >=2 and >=10 distinct droplet barcodes, run-wide.""" + h = unpaired.filter(pl.col("locus") == "IGH") + if h.height == 0: + return {"heavy_junctions": 0, "share_ge2_barcodes": None, "share_ge10_barcodes": None} + nb = h.group_by("junction").agg(pl.col("barcode").n_unique().alias("nb")) + n = nb.height + occ = h.join(nb, on="junction") + return { + "heavy_junctions": int(n), + "share_ge2_barcodes": _ratio(int((nb["nb"] >= 2).sum()), n), + "share_ge10_barcodes": _ratio(int((nb["nb"] >= 10).sum()), n), + "contigs_with_junction_ge2_barcodes": _ratio(int((occ["nb"] >= 2).sum()), occ.height), + "contigs_with_junction_ge10_barcodes": _ratio(int((occ["nb"] >= 10).sum()), occ.height), + "definition": "distinct productive IGH junctions (nt) by number of distinct barcodes carrying them, across wells", + } + + +def leakage_by_chain(output_directory: str | Path, min_wells: int = 8) -> dict: + """ + Leakage f per chain with ``pooled_pairing.estimate_leakage`` on this run's per-well outputs. + Skipped (``None`` values) when fewer than ``min_wells`` wells have outputs. + """ + from .pooled_pairing import SPECIES, estimate_leakage, load_run + + try: + occ = load_run(output_directory) + except FileNotFoundError: + return {"wells": 0, "skipped": "no per-well outputs"} + W = int(occ["well"].n_unique()) + if W < min_wells: + return {"wells": W, "skipped": f"fewer than {min_wells} wells"} + sw = occ.group_by(*SPECIES, "well").agg(pl.col("umis").sum().alias("n")) + out = {"wells": W} + for name, loci in (("heavy", ["IGH"]), ("light", _LIGHT), ("IGK", ["IGK"]), ("IGL", ["IGL"])): + f = estimate_leakage(sw.filter(pl.col("locus").is_in(loci)), W) + out[name] = None if f is None else round(f, 4) + return out + + +CONF_BINS = [0.0, 0.1, 0.3, 0.6, 0.9, 1.0001] +CONF_LABELS = ["<0.1", "0.1-0.3", "0.3-0.6", "0.6-0.9", ">=0.9"] + + +def confidence_distribution(conf: pl.Series | None) -> dict | None: + if conf is None: + return None + c = conf.drop_nulls().cast(pl.Float64) + if c.len() == 0: + return {"n": 0} + bins = {} + for lo, hi, lab in zip(CONF_BINS[:-1], CONF_BINS[1:], CONF_LABELS): + bins[lab] = int(((c >= lo) & (c < hi)).sum()) + q = {f"q{int(p * 100):02d}": round(float(c.quantile(p, "linear")), 4) for p in (0.1, 0.25, 0.5, 0.75, 0.9)} + return {"n": int(c.len()), "mean": round(float(c.mean()), 4), **q, "bins": bins} + + +def _norm_light_v(expr: pl.Expr) -> pl.Expr: + # IGKV1D-39 and IGKV1-39 are the same gene in practice: count them as one light V + return expr.str.replace(r"^(IG[KL]V\d+)D-", "${1}-") + + +def clonal_coherence(paired: pl.DataFrame) -> dict: + """ + Clonal light-chain coherence. Clone = heavy V gene + CDRH3 aa; purity = share of a clone's + cells carrying its modal light V gene, pooled over clones with >= 2 cells (cell-weighted); + ``purity_mean_clone`` is the unweighted mean over those clones. ``chance`` = sum p^2 over the + light V frequencies of all pairs (the purity expected if light chains were random). + """ + need = ["v_gene:0", "cdr3_aa:0", "v_gene:1"] + if paired.height == 0 or not all(c in paired.columns for c in need): + return {"pairs": int(paired.height), "clones_ge2": 0, "purity": None, "chance": None} + d = paired.select( + pl.col("v_gene:0").alias("hv"), pl.col("cdr3_aa:0").alias("hcdr3"), _norm_light_v(pl.col("v_gene:1")).alias("lv") + ).drop_nulls() + p = d["lv"].value_counts()["count"].to_numpy().astype(float) + chance = round(float(((p / p.sum()) ** 2).sum()), 4) if p.size else None + c = d.group_by("hv", "hcdr3", "lv").len() + cl = c.group_by("hv", "hcdr3").agg(pl.col("len").sum().alias("size"), pl.col("len").max().alias("top")) + cl = cl.filter(pl.col("size") >= 2) + if cl.height == 0: + return {"pairs": int(paired.height), "clones_ge2": 0, "purity": None, "chance": chance} + return { + "pairs": int(paired.height), + "clones_ge2": int(cl.height), + "cells_in_clones": int(cl["size"].sum()), + "purity": round(float(cl["top"].sum() / cl["size"].sum()), 4), + "purity_mean_clone": round(float((cl["top"] / cl["size"]).mean()), 4), + "chance": chance, + } + + +# -------------------------------------------------------------------------------------------- +# report +# -------------------------------------------------------------------------------------------- + +PURITY_CAVEAT = ( + "Clonal light-chain purity is only meaningful for memory / clonally expanded repertoires; " + "in naive repertoires clones with >= 2 cells are rare and mostly chance coincidences, " + "so purity there says little about pairing accuracy. Use it to rank runs, not as an accuracy." +) + + +def build_report( + output_directory: str | Path, + stage_counts: pl.DataFrame, + failures: pl.DataFrame, + fragments: dict, + umi_stats: list[dict], + wells: list[str], +) -> dict: + """Assemble the run-level QC report from the counters and the per-well outputs on disk.""" + output_directory = Path(output_directory) + ann_dir = output_directory / "annotated" + unpaired_files = [(w, ann_dir / f"{w}_unpaired.parquet") for w in wells] + unpaired_files = [(w, f) for w, f in unpaired_files if f.exists() and f.stat().st_size > 0] + paired_files = [ann_dir / f"{w}_paired.parquet" for w in wells] + paired_files = [f for f in paired_files if f.exists() and f.stat().st_size > 0] + + paired = ( + pl.concat([pl.read_parquet(f) for f in paired_files], how="diagonal_relaxed") + if paired_files else pl.DataFrame() + ) + conf = None + for src in (output_directory / "pairs" / "pairs.parquet",): + if src.exists(): + p = pl.read_parquet(src) + if "confidence" in p.columns: + conf = p["confidence"] + if conf is None and "confidence" in paired.columns: + conf = paired["confidence"] + + total = stage_counts.filter(pl.col("well") == "TOTAL").to_dicts() + total = total[0] if total else {} + total.pop("well", None) + + try: + leakage = leakage_by_chain(output_directory) + except Exception as e: # QC must never fail a run + leakage = {"error": repr(e)} + + top_fail = ( + failures.filter(pl.col("well") == "TOTAL").sort("count", descending=True).head(10).drop("well").to_dicts() + if failures.height else [] + ) + return { + "wells": len(wells), + "stage_totals": total, + "heavy_fragment_fraction": fragment_summary(fragments), + "umi_error_signature": umi_summary(umi_stats), + "ambient_proxy": ambient_proxy(load_unpaired(unpaired_files)), + "leakage": leakage, + "pair_confidence": confidence_distribution(conf), + "clonal_coherence": {**clonal_coherence(paired), "caveat": PURITY_CAVEAT}, + "top_annotation_failures": top_fail, + } + + +def _fmt(v) -> str: + if v is None: + return "n/a" + if isinstance(v, float): + return f"{v:.4g}" + return str(v) + + +def report_markdown(rep: dict) -> str: + L = ["# PairPlex QC report", "", f"Wells: {rep['wells']}", ""] + L += ["## Stage totals", "", "| stage | count |", "|---|---|"] + L += [f"| {k} | {v} |" for k, v in rep["stage_totals"].items()] + fr = rep["heavy_fragment_fraction"] + L += ["", "## Fragment fraction", "", fr["definition"], "", + f"Passing consensus sequences: {fr['passing_consensus']}; fragment fraction: {_fmt(fr['fragment_fraction'])}; " + f"failed annotation: {_fmt(fr['failed_annotation_fraction'])}", "", + "| locus | n | truncated | incomplete VDJ | fragment fraction |", "|---|---|---|---|---|"] + L += [f"| {k} | {d['n']} | {d['truncated']} | {d['incomplete_vdj']} | {_fmt(d['fragment_fraction'])} |" + for k, d in fr["by_locus"].items()] + u = rep["umi_error_signature"] + L += ["", "## UMI error signature", "", u["definition"], "", + f"- distinct (barcode, UMI): {u['umis']} (wells subsampled: {u['sampled_wells']})", + f"- Hamming-1 pairs: {u['h1_pairs']} observed vs {u['h1_pairs_null']} shuffled null " + f"(excess x{_fmt(u['excess_h1_pairs'])})", + f"- UMIs with a Hamming-1 neighbour: {_fmt(u['error_like_umi_fraction'])} " + f"(null {_fmt(u['error_like_umi_fraction_null'])}; excess {_fmt(u['excess_error_like_umi_fraction'])})"] + a = rep["ambient_proxy"] + L += ["", "## Ambient proxy", "", a.get("definition", ""), "", + f"- heavy junctions: {a['heavy_junctions']}", + f"- share under >=2 barcodes: {_fmt(a['share_ge2_barcodes'])}", + f"- share under >=10 barcodes: {_fmt(a['share_ge10_barcodes'])}"] + lk = rep["leakage"] + L += ["", "## Leakage f (pooled_pairing.estimate_leakage)", ""] + if "skipped" in lk or "error" in lk: + L.append(f"Not computed: {lk.get('skipped') or lk.get('error')} (wells: {lk.get('wells', 'n/a')})") + else: + L += [f"- {k}: {_fmt(lk.get(k))}" for k in ("heavy", "light", "IGK", "IGL")] + [f"- wells: {lk['wells']}"] + c = rep["pair_confidence"] + L += ["", "## Pair confidence", ""] + if not c: + L.append("No confidence column in this run's pair outputs.") + else: + L.append(f"n = {c['n']}" + (f", mean {_fmt(c['mean'])}, median {_fmt(c['q50'])}" if c.get("n") else "")) + if c.get("bins"): + L += ["", "| bin | pairs |", "|---|---|"] + [f"| {k} | {v} |" for k, v in c["bins"].items()] + cc = rep["clonal_coherence"] + L += ["", "## Clonal light-chain coherence", "", f"> **Caveat:** {cc['caveat']}", "", + "Clone = heavy V gene + CDRH3 aa; purity = share of a clone's cells with its modal light V " + "(clones with >= 2 cells, cell-weighted); chance = sum p^2 over light V genes.", "", + f"- pairs: {cc['pairs']}; clones with >= 2 cells: {cc['clones_ge2']}", + f"- purity: {_fmt(cc.get('purity'))} (mean per clone {_fmt(cc.get('purity_mean_clone'))}); chance: {_fmt(cc.get('chance'))}"] + if rep["top_annotation_failures"]: + L += ["", "## Top annotation failure modes", "", "| kind | mode | count |", "|---|---|---|"] + L += [f"| {r['kind']} | {r['mode']} | {r['count']} |" for r in rep["top_annotation_failures"]] + return "\n".join(L) + "\n" + + +def write_qc( + output_directory: str | Path, + stage_rows: list[dict], + failure_rows: list[dict], + fragments: dict, + umi_stats: list[dict], + wells: list[str], +) -> dict: + """Write ``qc/stage_counts.csv``, ``qc/annotation_failures.csv``, ``qc/report.json`` and ``qc/report.md``.""" + qc_dir = Path(output_directory) / "qc" + qc_dir.mkdir(parents=True, exist_ok=True) + sc = stage_counts_frame(stage_rows) + sc.write_csv(qc_dir / "stage_counts.csv") + ft = failure_table(failure_rows) + ft.write_csv(qc_dir / "annotation_failures.csv") + rep = build_report(output_directory, sc, ft, fragments, umi_stats, wells) + (qc_dir / "report.json").write_text(json.dumps(rep, indent=2, default=_json_default) + "\n") + (qc_dir / "report.md").write_text(report_markdown(rep)) + return rep + + +# -------------------------------------------------------------------------------------------- +# support columns +# -------------------------------------------------------------------------------------------- + +SUPPORT_FIELDS = ("umis", "reads", "cluster_fraction") + + +def add_support_columns(sequences, metadata: pl.DataFrame) -> None: + """ + Add cluster support to abstar's ``Sequence`` objects in place, joined on the contig name + (= ``sequence_id``): ``umis``, ``reads``, ``cluster_fraction``, ``barcode`` and ``contig`` + (``contig-N``, N = rank by reads within the barcode). Paired outputs built from these objects + get them as ``umis:0/1`` etc. + + abstar's ``umi`` field is removed: it holds one read's UMI parsed from the read header with + ``umi_pattern``, which PairPlex never sets; a consensus contig stands for many UMIs, so the + field is always null here and the count lives in ``umis``. + """ + if sequences is None: + return + if not isinstance(sequences, list | tuple): + sequences = [sequences] + cols = [c for c in ("name", *SUPPORT_FIELDS) if c in metadata.columns] + support = {r[0]: r[1:] for r in metadata.select(cols).iter_rows()} + names = cols[1:] + for s in sequences: + sid = s.id + vals = support.get(sid) + for k, v in zip(names, vals if vals is not None else [None] * len(names)): + s[k] = v + bc, _, contig = (sid or "").rpartition("_") + s["barcode"] = bc or None + s["contig"] = contig or None + if s.annotations is not None: + s.annotations.pop("umi", None) + + +# -------------------------------------------------------------------------------------------- +# helpers +# -------------------------------------------------------------------------------------------- + + +def _ratio(a, b): + return None if not b else round(float(a) / float(b), 6) + + +def _json_default(o): + if isinstance(o, np.integer): + return int(o) + if isinstance(o, np.floating): + return None if math.isnan(o) else float(o) + return str(o) diff --git a/pairplex/tests/test_qc.py b/pairplex/tests/test_qc.py new file mode 100644 index 0000000..c094849 --- /dev/null +++ b/pairplex/tests/test_qc.py @@ -0,0 +1,325 @@ +"""Tests for pairplex.qc: stage counters, failure table, support columns and QC report metrics.""" + +import json +import warnings + +import polars as pl +import pytest +from abutils import Sequence + +from pairplex import qc + + +def _seq(sid, **ann): + rec = {"sequence_id": sid, "sequence": "ACGT", "umi": None, **ann} + return Sequence(rec, id_key="sequence_id", seq_key="sequence") + + +class _Pair: + def __init__(self, nh, nl): + self.heavies = [object()] * nh + self.lights = [object()] * nl + + +# ---------------------------------------------------------------- stage counters + + +def test_read_stage_counts(): + parsed = pl.DataFrame({"barcode": ["A", "B"], "umi": ["x", "y"], "sequence": ["AC", "GT"]}) + assert qc.read_stage_counts(5, parsed) == { + "input_reads": 5, "valid_barcode_reads": 2, "dropped_orientation_or_barcode": 3, + } + + +def test_barcode_stage_counts_groups_and_empty(): + parsed = pl.DataFrame({ + "barcode": ["A"] * 4 + ["B"] * 2 + ["C"], + "umi": ["u"] * 7, + "sequence": ["ACG", "", "ACG", "", "", "ACG", "A"], + }) + out = qc.barcode_stage_counts(parsed, min_cluster_reads=3) + # B (2 reads) and C (1 read) dropped; empties counted only in kept group A + assert out == {"barcode_groups": 3, "groups_dropped_min_cluster_reads": 2, + "reads_in_dropped_groups": 3, "empty_sequences_skipped": 2} + + +def test_barcode_stage_counts_empty_frame(): + parsed = pl.DataFrame(schema={"barcode": pl.String, "umi": pl.String, "sequence": pl.String}) + assert qc.barcode_stage_counts(parsed, 3)["barcode_groups"] == 0 + + +def test_cluster_stage_counts_each_filter(): + meta = pl.DataFrame({ + "reads": [10, 2, 5, 1], + "umis": [3, 1, 1, 1], + "cluster_fraction": [0.6, 0.1, 0.25, 0.05], + "pass_filters": [True, False, False, False], + }) + out = qc.cluster_stage_counts(meta, min_cluster_reads=3, min_cluster_umis=2, min_cluster_fraction=0.2) + assert out == {"clusters": 4, "clusters_fail_reads": 2, "clusters_fail_umis": 3, + "clusters_fail_fraction": 2, "clusters_pass": 1} + + +def _ann_seqs(): + return [ + _seq("A_contig-0", annotation_status="annotated", locus="IGH", productive=True, + junction="TGT", complete_vdj=True, v_germline_start=1), + _seq("A_contig-1", annotation_status="annotated", locus="IGK", productive=False, + productivity_issues="junction does not end with conserved F|V/J junction is out of frame", + junction="TGC", complete_vdj=False, v_germline_start=2), + _seq("B_contig-0", annotation_status="annotated", locus="IGH", productive=False, + productivity_issues="", junction=None, complete_vdj=False, v_germline_start=250), + _seq("C_contig-0", annotation_status="unassigned", failure_reason="no compatible V gene assignment", + locus=None, productive=None, junction=None), + ] + + +def test_annotation_frame_and_stage_counts(): + ann = qc.annotation_frame(_ann_seqs()) + assert ann.height == 4 and set(qc.ANN_SCHEMA) == set(ann.columns) + out = qc.annotation_stage_counts(ann, n_sent=6, n_record_failures=2) + assert out == { + "consensus_to_annotation": 6, "annotation_returned": 4, "annotated": 3, + "failed_unassigned": 1, "failed_record_error": 2, "failed_annotation": 3, + "productive": 1, "unproductive": 2, "no_junction": 1, + } + + +def test_annotation_frame_single_sequence_and_none(): + assert qc.annotation_frame(_seq("A_contig-0", annotation_status="annotated")).height == 1 + assert qc.annotation_frame(None).height == 0 + + +def test_record_errors_inferred_from_missing_rows(): + ann = qc.annotation_frame(_ann_seqs()) + assert qc.annotation_stage_counts(ann, n_sent=7)["failed_record_error"] == 3 + + +def test_chain_composition_counts(): + pairs = [_Pair(1, 1), _Pair(1, 1), _Pair(1, 0), _Pair(0, 1), _Pair(1, 2), _Pair(2, 1), + _Pair(2, 2), _Pair(0, 0), _Pair(3, 1), _Pair(0, 2)] + out = qc.chain_composition_counts(pairs) + assert out["barcodes_1H1L"] == 2 + assert out["barcodes_0H0L"] == 1 + assert out["barcodes_other"] == 2 + assert out["barcodes_with_chains"] == 10 + assert sum(out[f"barcodes_{c}"] for c in qc.COMPOSITIONS) == 10 + + +def test_stage_counts_frame_total(): + a, b = qc.new_counts("w1"), qc.new_counts("w2") + a["input_reads"], b["input_reads"], b["emitted_pairs"] = 3, 4, 2 + df = qc.stage_counts_frame([a, b]) + assert df["well"].to_list() == ["w1", "w2", "TOTAL"] + tot = df.filter(pl.col("well") == "TOTAL").row(0, named=True) + assert tot["input_reads"] == 7 and tot["emitted_pairs"] == 2 + assert df.columns == ["well", *qc.STAGE_FIELDS] + + +# ---------------------------------------------------------------- abstar failures + + +def test_annotation_failure_rows_and_table(): + ann = qc.annotation_frame(_ann_seqs()) + rec = [{"sequence_id": "D_contig-0", "stage": "annotation", "category": "internal_error", + "exception_type": "ValueError", "message": "CDR3 coordinates fall outside"}] + rows = qc.annotation_failure_rows("w1", ann, rec) + got = {(r["kind"], r["mode"]): r["count"] for r in rows} + assert got[("record_error", "annotation/internal_error ValueError: CDR3 coordinates fall outside")] == 1 + assert got[("unassigned", "no compatible V gene assignment")] == 1 + assert got[("unproductive", "junction does not end with conserved F")] == 1 + assert got[("unproductive", "V/J junction is out of frame")] == 1 + assert got[("unproductive", "unspecified")] == 1 + assert got[("no_junction", "junction is null")] == 1 + table = qc.failure_table(rows + qc.annotation_failure_rows("w2", ann, [])) + tot = table.filter((pl.col("well") == "TOTAL") & (pl.col("kind") == "unassigned")) + assert tot["count"].to_list() == [2] + assert qc.failure_table([]).height == 0 + + +def test_capture_abstar_failures_reads_tsv(tmp_path): + tsv = tmp_path / "failures.tsv" + tsv.write_text( + "input_file\tsample\trow_id\tsequence_id\tstage\tcategory\texception_type\tmessage\ttraceback_location\tdiagnostic_path\n" + "f\ts\t1\tX_contig-0\tannotation\tinternal_error\tValueError\tboom\tloc\tp\n" + ) + with warnings.catch_warnings(record=True) as outer: + warnings.simplefilter("always") + with qc.capture_abstar_failures() as fails: + warnings.warn(f"1 sequences failed annotation; see {tsv}", RuntimeWarning) + assert fails == [{"sequence_id": "X_contig-0", "stage": "annotation", "category": "internal_error", + "exception_type": "ValueError", "message": "boom"}] + assert any("failed annotation" in str(w.message) for w in outer) # re-emitted + + +# ---------------------------------------------------------------- fragments + + +def test_fragment_counts_by_locus(): + ann = qc.annotation_frame(_ann_seqs()) + frag = qc.fragment_counts(ann, n_record_failures=1) + assert frag["unknown"] == {"n": 2, "truncated": 2, "incomplete_vdj": 2} + assert frag["IGH"] == {"n": 2, "truncated": 1, "incomplete_vdj": 1} + assert frag["IGK"] == {"n": 1, "truncated": 0, "incomplete_vdj": 1} + acc = qc.merge_fragment_counts({}, frag) + acc = qc.merge_fragment_counts(acc, frag) + summ = qc.fragment_summary(acc) + assert summ["passing_consensus"] == 10 + assert summ["fragment_fraction"] == pytest.approx(6 / 10) + assert summ["by_locus"]["IGH"]["fragment_fraction"] == pytest.approx(0.5) + + +# ---------------------------------------------------------------- UMI signature + + +def test_hamming1_pairs_exact(): + u = pl.DataFrame({ + "barcode": ["A", "A", "A", "A", "B", "B"], + "umi": ["AAAA", "AAAT", "AATT", "CCCC", "AAAA", "GGGG"], + }) + pairs, with_nb = qc._hamming1_pairs(u) + # A: AAAA-AAAT, AAAT-AATT; cross-barcode AAAA (B) must not count + assert pairs == 2 and with_nb == 3 + + +def test_umi_hamming1_stats_excess_over_null(): + import random + + rng = random.Random(1) + rows = [] + for b in range(200): + bc = f"BC{b:04d}" + for _ in range(5): + umi = "".join(rng.choice("ACGT") for _ in range(10)) + rows.append((bc, umi)) + err = umi[:3] + ("A" if umi[3] != "A" else "C") + umi[4:] + rows.append((bc, err)) # one error copy per molecule + parsed = pl.DataFrame(rows, schema=["barcode", "umi"], orient="row") + st = qc.umi_hamming1_stats(parsed) + assert st["h1_pairs"] >= 1000 and st["h1_pairs_null"] < 50 + summ = qc.umi_summary([st]) + assert summ["excess_error_like_umi_fraction"] > 0.9 + sub = qc.umi_hamming1_stats(parsed, max_umis=500) + assert sub["sampled"] and sub["umis"] < parsed.height + + +# ---------------------------------------------------------------- run-level metrics + + +def test_ambient_proxy(): + rows = [("IGH", "J1", f"B{i}") for i in range(10)] + [("IGH", "J2", "B0"), ("IGH", "J2", "B1"), + ("IGH", "J3", "B5"), ("IGK", "J1", "B9")] + df = pl.DataFrame(rows, schema=["locus", "junction", "barcode"], orient="row") + out = qc.ambient_proxy(df) + assert out["heavy_junctions"] == 3 + assert out["share_ge2_barcodes"] == pytest.approx(2 / 3, abs=1e-6) + assert out["share_ge10_barcodes"] == pytest.approx(1 / 3, abs=1e-6) + + +def test_clonal_coherence_and_chance(): + df = pl.DataFrame({ + "v_gene:0": ["H1", "H1", "H1", "H2", "H2", "H3"], + "cdr3_aa:0": ["C1", "C1", "C1", "C2", "C2", "C3"], + "v_gene:1": ["IGKV1-39", "IGKV1D-39", "IGKV3-20", "IGLV2-14", "IGLV2-14", "IGKV3-20"], + }) + out = qc.clonal_coherence(df) + # clone H1/C1: KV1-39 x2 (D- normalised) + KV3-20 -> top 2/3; clone H2/C2: 2/2 + assert out["clones_ge2"] == 2 and out["cells_in_clones"] == 5 + assert out["purity"] == pytest.approx(4 / 5) + assert out["purity_mean_clone"] == pytest.approx((2 / 3 + 1) / 2, abs=1e-4) + # light V frequencies 2/6, 2/6, 2/6 + assert out["chance"] == pytest.approx(1 / 3, abs=1e-4) + assert qc.clonal_coherence(pl.DataFrame())["purity"] is None + + +def test_confidence_distribution(): + assert qc.confidence_distribution(None) is None + out = qc.confidence_distribution(pl.Series([0.05, 0.5, 0.95, 1.0, None])) + assert out["n"] == 4 + assert out["bins"] == {"<0.1": 1, "0.1-0.3": 0, "0.3-0.6": 1, "0.6-0.9": 0, ">=0.9": 2} + + +def _write_well(ann_dir, meta_dir, well, rows): + """rows: (barcode, locus, junction, umis)""" + name = f"well{well:03d}.fastq" + df = pl.DataFrame({ + "sequence_id": [f"{b}_contig-{i}" for i, (b, *_r) in enumerate(rows)], + "locus": [r[1] for r in rows], + "productive": [True] * len(rows), + "v_gene": ["IGHV1-2" if r[1] == "IGH" else "IGKV1-39" for r in rows], + "j_gene": ["IGHJ4" if r[1] == "IGH" else "IGKJ1" for r in rows], + "junction": [r[2] for r in rows], + "cdr3_aa": ["CARW" for _ in rows], + "sequence": ["ACGT" for _ in rows], + }) + df.write_parquet(ann_dir / f"{name}_unpaired.parquet") + # pooled_pairing skips files < 100 bytes, so carry a realistic consensus column + pl.DataFrame({"name": df["sequence_id"], "umis": [r[3] for r in rows], + "consensus": ["ACGT" * 30 for _ in rows]}).write_csv(meta_dir / f"{name}.csv") + return name + + +def test_leakage_by_chain_needs_8_wells(tmp_path): + ann, meta = tmp_path / "annotated", tmp_path / "metadata" + ann.mkdir() + meta.mkdir() + # heavy species: 9 UMIs at home, 1 elsewhere; light: 5 at home, 5 spread + for w in range(8): + rows = [(f"B{w}", "IGH", f"TGTH{w}", 9), (f"B{w}", "IGK", f"TGTK{w}", 5)] + other = (w + 1) % 8 + rows += [(f"B{other}", "IGH", f"TGTH{other}", 1), (f"B{other}", "IGK", f"TGTK{other}", 5)] + _write_well(ann, meta, w, rows) + out = qc.leakage_by_chain(tmp_path) + assert out["wells"] == 8 + assert out["heavy"] == pytest.approx(0.1 / (1 - 1 / 8), abs=1e-3) + assert out["light"] == pytest.approx(0.5 / (1 - 1 / 8), abs=1e-3) + few = qc.leakage_by_chain(tmp_path, min_wells=9) + assert "skipped" in few + + +def test_write_qc_end_to_end(tmp_path): + ann, meta = tmp_path / "annotated", tmp_path / "metadata" + ann.mkdir() + meta.mkdir() + name = _write_well(ann, meta, 0, [("B0", "IGH", "TGTA", 3), ("B1", "IGH", "TGTA", 2)]) + pl.DataFrame({"name": ["B0"], "v_gene:0": ["IGHV1-2"], "cdr3_aa:0": ["CARW"], "v_gene:1": ["IGKV1-39"], + "confidence": [0.95]}).write_parquet(ann / f"{name}_paired.parquet") + row = qc.new_counts(name) + row["input_reads"] = 10 + rep = qc.write_qc(tmp_path, [row], [], {}, [], [name]) + for f in ("stage_counts.csv", "annotation_failures.csv", "report.json", "report.md"): + assert (tmp_path / "qc" / f).exists() + assert json.loads((tmp_path / "qc" / "report.json").read_text())["stage_totals"]["input_reads"] == 10 + assert rep["ambient_proxy"]["share_ge2_barcodes"] == 1.0 + assert rep["pair_confidence"]["n"] == 1 + assert "memory" in (tmp_path / "qc" / "report.md").read_text() + + +# ---------------------------------------------------------------- support columns + + +def test_add_support_columns(): + seqs = [_seq("AAAC_contig-0"), _seq("AAAC_contig-1"), _seq("GGGT_contig-0")] + meta = pl.DataFrame({"name": ["AAAC_contig-0", "AAAC_contig-1"], "reads": [10, 3], "umis": [4, 1], + "cluster_fraction": [0.77, 0.23], "consensus": ["A", "C"], "pass_filters": [True, True]}) + qc.add_support_columns(seqs, meta) + a = seqs[0].annotations + assert (a["umis"], a["reads"], a["cluster_fraction"]) == (4, 10, 0.77) + assert (a["barcode"], a["contig"]) == ("AAAC", "contig-0") + assert "umi" not in a + assert seqs[2]["umis"] is None and seqs[2]["barcode"] == "GGGT" + + +def test_support_columns_reach_paired_output(): + import abutils + + seqs = [_seq("AAAC_contig-0", locus="IGH"), _seq("AAAC_contig-1", locus="IGK")] + meta = pl.DataFrame({"name": ["AAAC_contig-0", "AAAC_contig-1"], "reads": [10, 3], "umis": [4, 1], + "cluster_fraction": [0.77, 0.23]}) + qc.add_support_columns(seqs, meta) + pairs = abutils.tl.assign_pairs(seqs, delim="_", delim_occurance=-1) + df = abutils.io.to_polars(pairs) if hasattr(abutils.io, "to_polars") else None + if df is None: + pytest.skip("abutils.io.to_polars unavailable") + assert df["umis:0"].to_list() == [4] and df["umis:1"].to_list() == [1] + assert "umi:0" not in df.columns From a32d67da3268c2b772ebb8a07b2bca9f21e44d5e Mon Sep 17 00:00:00 2001 From: Benjamin Nemoz Date: Tue, 17 Nov 2026 18:14:39 +0000 Subject: [PATCH 4/4] Scored pairing: merge same-clonotype contigs before dominance ranking Two same-locus contigs of one (well, barcode) sharing V gene, J gene and CDR3 aa are one chain split into two consensus clusters, or clonal relatives. Ranked separately they split the UMIs and fail the dominance test, dropping the barcode (and the lineage). Merge them first (summed UMIs/reads, best contig kept). Option collapse_clonal (default True), CLI --collapse-clonal/--no-collapse-clonal. Co-Authored-By: Claude Opus 5.5 --- pairplex/cli/pairplex.py | 8 ++++++ pairplex/pairing.py | 47 +++++++++++++++++++++++++++++----- pairplex/pairplex.py | 7 +++++ pairplex/tests/test_pairing.py | 25 ++++++++++++++++-- 4 files changed, 78 insertions(+), 9 deletions(-) diff --git a/pairplex/cli/pairplex.py b/pairplex/cli/pairplex.py index abd349e..14c0435 100644 --- a/pairplex/cli/pairplex.py +++ b/pairplex/cli/pairplex.py @@ -111,6 +111,12 @@ def version(): default=2.0, help="Scored pairing: top chain of each locus group needs >= this many times the UMIs of the second-best chain", ) +@click.option( + "--collapse-clonal/--no-collapse-clonal", + "collapse_clonal", + default=True, + help="Scored pairing: merge same-locus contigs sharing V, J and CDR3 aa before ranking", +) @click.option( "--legacy", is_flag=True, @@ -138,6 +144,7 @@ def run( germline_database: str, pairing: str, min_dominance: float, + collapse_clonal: bool, legacy: bool, quiet: bool, debug: bool, @@ -158,6 +165,7 @@ def run( germline_database=germline_database, pairing=pairing, min_dominance=min_dominance, + collapse_clonal=collapse_clonal, legacy=legacy, quiet=quiet, debug=debug, diff --git a/pairplex/pairing.py b/pairplex/pairing.py index 9a0988e..e20e4c2 100644 --- a/pairplex/pairing.py +++ b/pairplex/pairing.py @@ -16,6 +16,9 @@ and the *productive* light chains (IGK and IGL together: a cell expresses one light chain, so kappa and lambda compete in the same group) are ranked by support: corrected UMIs from the metadata CSV, then reads as a tie-break, then contig name for determinism. + Same-locus contigs sharing V gene, J gene and CDR3 aa are first merged into one chain + (summed support; ``collapse_clonal``), so a chain split into two consensus clusters, or two + clonal relatives, never competes with itself. The top heavy and the top light are emitted when, in each group, u1 >= min_dominance * u2 @@ -32,7 +35,8 @@ * ``margin = min_g (u1_g - u2_g) / (U_g + pseudocount)``: the smaller, over the two groups, of the gap between the top and the second-best chain's share. In [0, 1); equal to ``min(s_H, s_L)`` for a 1H+1L barcode. - * ``umis`` on each chain (``umis:0`` heavy, ``umis:1`` light in the paired outputs). + * ``umis`` on each chain (``umis:0`` heavy, ``umis:1`` light in the paired outputs): the + chain's support used for ranking (summed over merged same-clonotype contigs). Default ``min_dominance = 2.0``: the top chain must hold at least two thirds of a 2-chain group's UMIs. In real data about half of the 1H+2L / 2H+1L barcodes meet it; near-even splits @@ -129,6 +133,33 @@ def resolve_barcode( return Resolution(hs[0], ls[0], sh * sl, min(mh, ml), "paired") +def _gene(name) -> str | None: + return name.split("*")[0] if isinstance(name, str) else None + + +def collapse_clonotypes(chains: list[Chain], annotations: dict) -> list[Chain]: + """ + Merge chains of one locus group that share a clonotype (locus, V gene, J gene, CDR3 aa). + + Within one (well, barcode), two contigs with the same clonotype are one chain split into two + consensus clusters (sequencing or PCR errors, fragments) or clonal relatives; either way they + support the same pairing and must not compete. The merged chain keeps the name of its best + contig (UMIs, reads, name) and the summed UMIs and reads. Contigs without a CDR3 aa are kept + as they are. + """ + groups: dict[tuple, list[Chain]] = {} + for c in chains: + a = annotations.get(c.name) + cdr3 = a["cdr3_aa"] if a is not None else None + key = (a["locus"], _gene(a["v_gene"]), _gene(a["j_gene"]), cdr3) if cdr3 else ("_id", c.name) + groups.setdefault(key, []).append(c) + out = [] + for members in groups.values(): + best = _rank(members)[0] + out.append(Chain(best.name, sum(m.umis for m in members), sum(m.reads for m in members))) + return out + + def _barcode(sequence_id: str) -> str: # same grouping as abutils.tl.assign_pairs(delim="_", delim_occurance=-1) return "_".join(sequence_id.split("_")[:-1]) @@ -152,6 +183,7 @@ def scored_pairs( support: dict[str, tuple[int, int]], min_dominance: float = DEFAULT_MIN_DOMINANCE, pseudocount: float = DEFAULT_PSEUDOCOUNT, + collapse_clonal: bool = True, ) -> tuple[list, dict[str, int]]: """ Resolve every barcode of one well with ``resolve_barcode``. @@ -162,6 +194,9 @@ def scored_pairs( Annotated contigs of one well (``sequence_id`` = ``_``). support : dict ``name -> (umis, reads)`` from the metadata CSV. Missing contigs count as 1 UMI, 0 reads. + collapse_clonal : bool + Merge same-locus contigs of a barcode that share V gene, J gene and CDR3 aa before + ranking (see ``collapse_clonotypes``). Returns ------- @@ -188,11 +223,8 @@ def scored_pairs( pairs, reasons = [], {} for bc, (h_ids, l_ids) in groups.items(): def chains(ids): - out = [] - for i in ids: - u, r = support.get(i, (1, 0)) - out.append(Chain(i, int(u), int(r))) - return out + out = [Chain(i, *(int(x) for x in support.get(i, (1, 0)))) for i in ids] + return collapse_clonotypes(out, by_id) if collapse_clonal else out res = resolve_barcode(chains(h_ids), chains(l_ids), min_dominance, pseudocount) reasons[res.reason] = reasons.get(res.reason, 0) + 1 @@ -236,6 +268,7 @@ def pair_well( support: dict[str, tuple[int, int]] | None = None, min_dominance: float = DEFAULT_MIN_DOMINANCE, pseudocount: float = DEFAULT_PSEUDOCOUNT, + collapse_clonal: bool = True, ) -> list: """Pair one well's annotated contigs with the chosen rule and write the paired files.""" if mode == "exact": @@ -246,7 +279,7 @@ def pair_well( return pairs if mode != "scored": raise ValueError(f"unknown pairing mode {mode!r}; expected one of {PAIRING_MODES}") - pairs, _ = scored_pairs(sequences, support or {}, min_dominance, pseudocount) + pairs, _ = scored_pairs(sequences, support or {}, min_dominance, pseudocount, collapse_clonal) write_paired(pairs, paired_airr_file, paired_parquet_file, scored=True) return pairs diff --git a/pairplex/pairplex.py b/pairplex/pairplex.py index 3b0fd64..9405517 100644 --- a/pairplex/pairplex.py +++ b/pairplex/pairplex.py @@ -60,6 +60,7 @@ def run( germline_database: str = "human", pairing: str = "scored", min_dominance: float = DEFAULT_MIN_DOMINANCE, + collapse_clonal: bool = True, legacy: bool = False, quiet: bool = False, debug: bool = False, @@ -131,6 +132,11 @@ def run( times the UMIs of the second-best chain. Defaults to ``2.0`` (top chain holds >= 2/3 of a 2-chain group). Values <= 1 accept any split (ties broken by reads). + collapse_clonal : bool + Scored pairing only. Before ranking, merge same-locus contigs of a (well, barcode) that + share V gene, J gene and CDR3 aa (one chain split into several consensus clusters, or + clonal relatives), summing their UMIs, so they do not compete. Defaults to ``True``. + legacy : bool If ``True``, reproduce ``v2-base`` behaviour: forces ``pairing="exact"``. @@ -484,6 +490,7 @@ def run( mode=pairing, support=support_from_metadata(metadata_df), min_dominance=min_dominance, + collapse_clonal=collapse_clonal, ) pairs_printer.set_description_str(f"{len(pairs)} paired sequences") if pairing == "scored": diff --git a/pairplex/tests/test_pairing.py b/pairplex/tests/test_pairing.py index 1ef328e..462cbb2 100644 --- a/pairplex/tests/test_pairing.py +++ b/pairplex/tests/test_pairing.py @@ -83,10 +83,11 @@ def test_missing_locus_group(): # ---------------------------------------------------------------- well-level -def _seq(bc, n, locus, productive=True): +def _seq(bc, n, locus, productive=True, cdr3=None): return abutils.Sequence({"sequence_id": f"{bc}_contig-{n}", "sequence": "ACGT" * 10, "locus": locus, "productive": productive, "v_gene": f"{locus}V1", "j_gene": f"{locus}J1", - "v_call": f"{locus}V1*01", "junction": "TGT" + "A" * n, "cdr3_aa": "CAR"}) + "v_call": f"{locus}V1*01", "junction": "TGT" + "A" * n, + "cdr3_aa": cdr3 or "CAR" + "W" * n}) def _well(): @@ -116,6 +117,26 @@ def test_scored_pairs_on_a_well(): assert {p.name for p in exact_pairs(seqs)} == {"AAA"} +def test_clonal_relatives_do_not_compete(): + # 2H+1L where both heavies are the same clonotype (split consensus / clonal relatives) + seqs = [_seq("EEE", 0, "IGH", cdr3="CARDY"), _seq("EEE", 1, "IGH", cdr3="CARDY"), _seq("EEE", 2, "IGK")] + support = {"EEE_contig-0": (3, 30), "EEE_contig-1": (3, 40), "EEE_contig-2": (5, 50)} + pairs, reasons = scored_pairs(seqs, support) + assert reasons == {"paired": 1} + p = pairs[0] + assert p.heavy["sequence_id"] == "EEE_contig-1" and p.heavy["umis"] == 6 + assert p.confidence == pytest.approx(6 / 7 * 5 / 6) + # without collapsing, the even split is ambiguous and the lineage is lost + _, reasons = scored_pairs(seqs, support, collapse_clonal=False) + assert reasons == {"heavy_ambiguous": 1} + + +def test_different_clonotypes_still_compete(): + seqs = [_seq("FFF", 0, "IGH", cdr3="CARDY"), _seq("FFF", 1, "IGH", cdr3="CTTGG"), _seq("FFF", 2, "IGK")] + support = {"FFF_contig-0": (3, 30), "FFF_contig-1": (3, 40), "FFF_contig-2": (5, 50)} + assert scored_pairs(seqs, support)[1] == {"heavy_ambiguous": 1} + + def test_pair_well_writes_outputs_and_contract(tmp_path): seqs, support = _well() tsv, pq = tmp_path / "well007_paired.tsv", tmp_path / "well007_paired.parquet"