From 1058e27299ccfd8f2702832ca00067fc2fe7f734 Mon Sep 17 00:00:00 2001 From: Benjamin Nemoz Date: Tue, 17 Nov 2026 16:18:24 +0000 Subject: [PATCH] feat: directional UMI collapsing per (well, barcode) before UMI counting Adds pairplex/umi.py:collapse_umis_directional (UMI-tools directional adjacency, neighbour generation, N-containing UMIs absorbed but never absorbing). process_droplet collapses UMIs per droplet partition: umi_raw keeps the raw UMI, metadata umis is the corrected count and umis_raw the raw count; min_cluster_umis filters on corrected counts. run(umi_collapse='directional'|'none'), CLI --umi-collapse; run(legacy=True) / --legacy forces 'none' (v2-base behaviour). Co-Authored-By: Claude Opus 5.5 --- pairplex/cli/pairplex.py | 16 +++ pairplex/pairplex.py | 25 ++++ pairplex/tests/test_umi.py | 278 +++++++++++++++++++++++++++++++++++++ pairplex/umi.py | 131 +++++++++++++++++ pairplex/utils.py | 26 ++++ 5 files changed, 476 insertions(+) create mode 100644 pairplex/tests/test_umi.py create mode 100644 pairplex/umi.py diff --git a/pairplex/cli/pairplex.py b/pairplex/cli/pairplex.py index 8a39506..ce40bb1 100644 --- a/pairplex/cli/pairplex.py +++ b/pairplex/cli/pairplex.py @@ -98,6 +98,18 @@ def version(): default="human", help="Germline database for V(D)J assignment", ) +@click.option( + "--umi-collapse", + "umi_collapse", + type=click.Choice(["directional", "none"]), + default="directional", + help="UMI error correction within each (well, barcode): UMI-tools directional (default) or none", +) +@click.option( + "--legacy", + is_flag=True, + help="Reproduce v2-base behaviour exactly (forces --umi-collapse none)", +) @click.option("--quiet", is_flag=True, help="Whether to suppress output") @click.option( "--debug", @@ -118,6 +130,8 @@ def run( merge_paired_reads: bool, receptor: str, germline_database: str, + umi_collapse: str, + legacy: bool, quiet: bool, debug: bool, ): @@ -135,6 +149,8 @@ def run( merge_paired_reads=merge_paired_reads, receptor=receptor, germline_database=germline_database, + umi_collapse=umi_collapse, + legacy=legacy, quiet=quiet, debug=debug, ) diff --git a/pairplex/pairplex.py b/pairplex/pairplex.py index 85c4ed5..e311fc6 100644 --- a/pairplex/pairplex.py +++ b/pairplex/pairplex.py @@ -49,6 +49,8 @@ def run( merge_paired_reads: bool = False, receptor: str = "bcr", germline_database: str = "human", + umi_collapse: str = "directional", + legacy: bool = False, quiet: bool = False, debug: bool = False, ) -> list: @@ -105,6 +107,19 @@ def run( The germline database to use. Options are anything accepted by `abstar.run`, including custom databases if available. + umi_collapse : str + UMI error correction, applied within each (well, droplet barcode) before any UMI counting. + ``"directional"`` (default) uses the UMI-tools directional adjacency method: UMI ``b`` is + absorbed by UMI ``a`` when they differ at one position and ``reads(a) >= 2 * reads(b) - 1``, + transitively, processing UMIs by descending read count (see :mod:`pairplex.umi`, which also + documents how UMIs containing ``N`` are handled). The metadata ``umis`` column is then the + corrected count, a ``umis_raw`` column holds the count before collapsing, and + ``min_cluster_umis`` filters on corrected counts. ``"none"`` counts raw UMIs (v2-base + behaviour, no ``umis_raw`` column). + + legacy : bool + If ``True``, reproduce the v2-base behaviour exactly: forces ``umi_collapse="none"``. + quiet : bool Whether to suppress the progress bar and other update messages. @@ -120,6 +135,15 @@ def run( """ parameters = dict(locals()) # captured first: exactly the arguments of this call + # legacy mode forces every v2 behaviour option to its v2-base value + if legacy: + umi_collapse = "none" + if umi_collapse not in ("directional", "none"): + raise ValueError( + f"umi_collapse must be 'directional' or 'none', got {umi_collapse!r}" + ) + parameters["umi_collapse_effective"] = umi_collapse + # print splash screen if not quiet: print_splash(include_version=True) @@ -346,6 +370,7 @@ def run( "min_cluster_fraction": min_cluster_fraction, "consensus_downsample": consensus_downsample, "clustering_threshold": clustering_threshold, + "umi_collapse": umi_collapse, "quiet": quiet, "debug": debug, } diff --git a/pairplex/tests/test_umi.py b/pairplex/tests/test_umi.py new file mode 100644 index 0000000..be295db --- /dev/null +++ b/pairplex/tests/test_umi.py @@ -0,0 +1,278 @@ +import itertools +import random + +import polars as pl +import pytest + +from pairplex.umi import collapse_umi_column, collapse_umis_directional +from pairplex.utils import process_droplet + + +def _hamming(a, b): + return sum(x != y for x, y in zip(a, b)) + + +def _reference_directional(cnt): + """O(n^2) reference implementation, written the UMI-tools way: build the full directed + adjacency list, BFS components from nodes by descending count (ties by UMI), first come first + served, N-containing UMIs never absorb.""" + umis = sorted(cnt, key=lambda u: (-cnt[u], u)) + adj = {u: [] for u in umis} + for a, b in itertools.permutations(umis, 2): + if "N" in a or len(a) != len(b): + continue + if _hamming(a, b) == 1 and cnt[a] >= 2 * cnt[b] - 1: + adj[a].append(b) + rep = {} + for root in umis: + if root in rep: + continue + seen, queue = {root}, [root] + while queue: + n = queue.pop(0) + for m in adj[n]: + if m not in seen: + seen.add(m) + queue.append(m) + for u in seen: + rep.setdefault(u, root) + return rep + + +# ---------------------------------------------------------------- classic examples + + +def test_singleton(): + assert collapse_umis_directional({"AAAAAAAAAA": 1}) == {"AAAAAAAAAA": "AAAAAAAAAA"} + + +def test_empty(): + assert collapse_umis_directional({}) == {} + + +def test_simple_absorption(): + m = collapse_umis_directional({"AAAAAAAAAA": 10, "AAAAAAAAAC": 2}) + assert m == {"AAAAAAAAAA": "AAAAAAAAAA", "AAAAAAAAAC": "AAAAAAAAAA"} + + +def test_count_rule_boundary(): + # 2*count(b) - 1: a=5, b=3 -> 5 >= 5 absorbs; a=5, b=4 -> 5 >= 7 is false + assert collapse_umis_directional({"AAAAAAAAAA": 5, "AAAAAAAAAC": 3})["AAAAAAAAAC"] == "AAAAAAAAAA" + assert collapse_umis_directional({"AAAAAAAAAA": 5, "AAAAAAAAAC": 4})["AAAAAAAAAC"] == "AAAAAAAAAC" + + +def test_hamming_two_not_absorbed(): + m = collapse_umis_directional({"AAAAAAAAAA": 100, "AAAAAAAACC": 1}) + assert m["AAAAAAAACC"] == "AAAAAAAACC" + + +def test_chain_is_transitive(): + # a(9) -> b(4) -> c(1): c is 2 substitutions from a, still absorbed through b + a, b, c = "AAAAAAAAAA", "AAAAAAAAAC", "AAAAAAAACC" + m = collapse_umis_directional({a: 9, b: 4, c: 1}) + assert m == {a: a, b: a, c: a} + + +def test_chain_broken_by_count_rule(): + # a(9) -> b(4), but b(4) vs c(3): 4 >= 5 false, so c stays separate + a, b, c = "AAAAAAAAAA", "AAAAAAAAAC", "AAAAAAAACC" + m = collapse_umis_directional({a: 9, b: 4, c: 3}) + assert m == {a: a, b: a, c: c} + + +def test_ties_of_one_merge_deterministically(): + # 1 >= 2*1 - 1: two singleton UMIs at Hamming 1 merge; representative is the smaller string + m = collapse_umis_directional({"AAAAAAAAAC": 1, "AAAAAAAAAA": 1}) + assert m == {"AAAAAAAAAA": "AAAAAAAAAA", "AAAAAAAAAC": "AAAAAAAAAA"} + # order of input does not matter + assert m == collapse_umis_directional({"AAAAAAAAAA": 1, "AAAAAAAAAC": 1}) + + +def test_ties_above_one_do_not_merge(): + m = collapse_umis_directional({"AAAAAAAAAA": 3, "AAAAAAAAAC": 3}) + assert m == {"AAAAAAAAAA": "AAAAAAAAAA", "AAAAAAAAAC": "AAAAAAAAAC"} + + +def test_umi_tools_paper_example(): + # Smith et al. 2017 Fig. 1 style: one true UMI with errors, one independent UMI nearby + cnt = { + "ACGTACGTAC": 456, # true + "ACGTACGTAA": 72, # error of the true one + "ACGTACGTTA": 2, # error of the error (chain) + "ACGTACGTCC": 90, # error of the true one + "TTTTACGTAC": 30, # independent + "TTTTACGTAA": 6, # error of the independent one + } + m = collapse_umis_directional(cnt) + assert {u for u, r in m.items() if r == "ACGTACGTAC"} == { + "ACGTACGTAC", "ACGTACGTAA", "ACGTACGTTA", "ACGTACGTCC" + } + assert m["TTTTACGTAA"] == "TTTTACGTAC" + assert len(set(m.values())) == 2 + + +def test_node_claimed_by_first_root(): + # c is reachable from both a(10) and b(6, not reachable from a); a is processed first + a, b, c = "AAAAAAAAAA", "AAAAAAAACC", "AAAAAAAAAC" + m = collapse_umis_directional({a: 10, b: 6, c: 1}) + assert m == {a: a, b: b, c: a} + + +# ---------------------------------------------------------------- N handling + + +def test_n_umi_absorbed_but_never_absorbs(): + real, nn = "ACGTACGTAC", "ACGTNCGTAC" + assert collapse_umis_directional({real: 5, nn: 1})[nn] == real + # an abundant N UMI does not absorb an N-free neighbour + m = collapse_umis_directional({nn: 50, real: 1}) + assert m == {nn: nn, real: real} + + +def test_n_umi_respects_count_rule_and_double_n(): + real = "ACGTACGTAC" + assert collapse_umis_directional({real: 2, "ACGTNCGTAC": 2})["ACGTNCGTAC"] == "ACGTNCGTAC" + assert collapse_umis_directional({real: 50, "ACGTNNGTAC": 1})["ACGTNNGTAC"] == "ACGTNNGTAC" + + +def test_short_umis_only_compared_at_equal_length(): + m = collapse_umis_directional({"ACGTACGTAC": 10, "ACGTACGTA": 1}) + assert m["ACGTACGTA"] == "ACGTACGTA" + + +# ---------------------------------------------------------------- API forms + + +def test_api_forms_agree(): + reads = ["AAAAAAAAAA"] * 4 + ["AAAAAAAAAC"] + ["GGGGGGGGGG"] * 2 + by_reads = collapse_umis_directional(reads) + by_map = collapse_umis_directional({"AAAAAAAAAA": 4, "AAAAAAAAAC": 1, "GGGGGGGGGG": 2}) + by_lists = collapse_umis_directional(["AAAAAAAAAA", "AAAAAAAAAC", "GGGGGGGGGG"], [4, 1, 2]) + assert by_reads == by_map == by_lists + assert collapse_umi_column(reads) == ["AAAAAAAAAA"] * 5 + ["GGGGGGGGGG"] * 2 + with pytest.raises(ValueError): + collapse_umis_directional(["A", "A"], [1, 2]) + with pytest.raises(ValueError): + collapse_umis_directional(["A"], [1, 2]) + + +# ---------------------------------------------------------------- randomized vs reference + + +@pytest.mark.parametrize("seed", range(20)) +def test_matches_reference_on_random_sets(seed): + rng = random.Random(seed) + L = 6 # short UMIs so Hamming-1 collisions are frequent + true = ["".join(rng.choice("ACGT") for _ in range(L)) for _ in range(rng.randint(1, 15))] + cnt = {} + for t in true: + n = rng.randint(1, 60) + for _ in range(n): + u = list(t) + if rng.random() < 0.25: + u[rng.randrange(L)] = rng.choice("ACGTN" if rng.random() < 0.9 else "N") + u = "".join(u) + cnt[u] = cnt.get(u, 0) + 1 + assert collapse_umis_directional(cnt) == _reference_directional(cnt) + + +def test_large_barcode_is_fast(): + import time + + rng = random.Random(0) + reads = [] + for _ in range(3000): + t = "".join(rng.choice("ACGT") for _ in range(10)) + for _ in range(rng.randint(1, 20)): + u = list(t) + if rng.random() < 0.2: + u[rng.randrange(10)] = rng.choice("ACGT") + reads.append("".join(u)) + t0 = time.perf_counter() + m = collapse_umis_directional(reads) + assert time.perf_counter() - t0 < 5 + assert len(set(m.values())) < len(set(reads)) + + +# ---------------------------------------------------------------- integration via process_droplet + +HEAVY = "CAGGTGCAGCTGGTGCAGTCTGGGGCTGAGGTGAAGAAGCCTGGGGCCTCAGTGAAGGTCTCCTGCAAGGCTTCTGGATACACC" +LIGHT = "GACATCCAGATGACCCAGTCTCCATCCTCCCTGTCTGCATCTGTAGGAGACAGAGTCACCATCACTTGCCGGGCAAGTCAG" + + +def _droplet_df(): + rows = [] + # heavy: true UMI A (6 reads) + 1-error copy (1 read) + independent UMI G (3 reads) + heavy_umis = ["AAAAAAAAAA"] * 6 + ["AAAAAAAAAT"] + ["GGGGGGGGGG"] * 3 + # light: one true UMI (4 reads) + two 1-error copies (1 read each) + light_umis = ["CCCCCCCCCC"] * 4 + ["CCCCCCCCCA", "CCCCCCCCCT"] + for i, u in enumerate(heavy_umis): + rows.append({"umi": u, "barcode": "BC", "seq_id": f"h{i}", "sequence": HEAVY}) + for i, u in enumerate(light_umis): + rows.append({"umi": u, "barcode": "BC", "seq_id": f"l{i}", "sequence": LIGHT}) + return pl.DataFrame(rows) + + +def _by_reads(meta): + return {m["reads"]: m for m in meta} + + +def test_process_droplet_directional(tmp_path): + meta = process_droplet( + "well_BC", _droplet_df(), temp_directory=tmp_path, clustering_threshold=0.9, + min_cluster_reads=3, min_cluster_umis=2, + ) + h, l = _by_reads(meta)[10], _by_reads(meta)[6] + assert (h["umis"], h["umis_raw"]) == (2, 3) + assert (l["umis"], l["umis_raw"]) == (1, 3) + # min_cluster_umis filters on corrected counts: light had 3 raw UMIs but only 1 molecule + assert h["pass_filters"] and not l["pass_filters"] + assert list(h) == ["name", "reads", "umis", "umis_raw", "cluster_fraction", "consensus", "pass_filters"] + + +def test_process_droplet_none_is_v2_base_layout(tmp_path): + meta = process_droplet( + "well_BC", _droplet_df(), temp_directory=tmp_path, clustering_threshold=0.9, + min_cluster_reads=3, min_cluster_umis=2, umi_collapse="none", + ) + h, l = _by_reads(meta)[10], _by_reads(meta)[6] + assert h["umis"] == 3 and l["umis"] == 3 + assert h["pass_filters"] and l["pass_filters"] + assert list(h) == ["name", "reads", "umis", "cluster_fraction", "consensus", "pass_filters"] + + +def test_process_droplet_rejects_unknown_mode(tmp_path): + with pytest.raises(ValueError): + process_droplet("well_BC", _droplet_df(), temp_directory=tmp_path, umi_collapse="cluster") + + +def test_run_legacy_forces_none(tmp_path, monkeypatch): + """legacy=True must pass umi_collapse='none' to process_droplet, whatever umi_collapse says.""" + import pairplex.pairplex as pp + + seen = {} + + class _Stop(Exception): + pass + + def fake_pool(*a, **k): + raise _Stop + + monkeypatch.setattr(pp, "ProcessPoolExecutor", fake_pool) + real_runlog = pp.RunLog + + def capture(path, tool, parameters, inputs): + seen.update(parameters) + return real_runlog(path, tool=tool, parameters=parameters, inputs=inputs) + + monkeypatch.setattr(pp, "RunLog", capture) + fq = tmp_path / "w.fastq" + fq.write_text("@r\nACGT\n+\nIIII\n") + with pytest.raises(_Stop): + pp.run(str(fq), tmp_path / "out", quiet=True, legacy=True, umi_collapse="directional") + assert seen["umi_collapse_effective"] == "none" + with pytest.raises(_Stop): + pp.run(str(fq), tmp_path / "out2", quiet=True) + assert seen["umi_collapse_effective"] == "directional" + with pytest.raises(ValueError): + pp.run(str(fq), tmp_path / "out3", quiet=True, umi_collapse="bogus") diff --git a/pairplex/umi.py b/pairplex/umi.py new file mode 100644 index 0000000..8d0f863 --- /dev/null +++ b/pairplex/umi.py @@ -0,0 +1,131 @@ +# Copyright (c) 2025 Benjamin Nemoz +# Distributed under the terms of the MIT License. +# SPDX-License-Identifier: MIT + +# This file is part of PairPlex. +# PairPlex is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# PairPlex is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# You should have received a copy of the GNU General Public License +# along with PairPlex. If not, see . + +"""UMI error correction. + +PairPlex collapses UMIs within one (well, droplet barcode) group with the *directional* +adjacency method of UMI-tools (Smith, Heger & Sudbery, Genome Res. 2017): + +- nodes are the distinct raw UMIs, weighted by their read count; +- there is a directed edge ``a -> b`` when ``Hamming(a, b) == 1`` and + ``count(a) >= 2 * count(b) - 1``; +- UMIs are visited in descending count; each UMI not yet absorbed becomes a representative and + absorbs every UMI reachable from it along directed edges, transitively: ``a -> b -> c`` puts + ``c`` under ``a`` although ``c`` is two substitutions away from ``a``. A UMI reached from an + earlier representative is never re-assigned. + +Ties in count are broken by the UMI string (lexicographic), so the result is deterministic. +UMI-tools leaves tie order to dict order; the groups are the same, only which tied UMI is named +representative can differ. + +UMIs containing ``N``: ``N`` counts as a mismatch at its position. An N-containing UMI never +absorbs anything (it would be Hamming-1 from up to four different real UMIs, so it is no evidence +for any of them), but it can be absorbed by an N-free UMI at Hamming distance 1 that satisfies the +count rule (e.g. ``ACGTNCGTAC`` under ``ACGTACGTAC``). An N-containing UMI with no such parent +stays its own UMI. Characters other than ``ACGTN`` are compared literally and never generate +neighbours. + +Neighbour generation (``4 * L`` single substitutions over ``ACGTN`` per UMI) keeps the cost +linear in the number of distinct UMIs, so barcodes with thousands of reads are cheap. +""" + +from __future__ import annotations + +from collections import Counter +from collections.abc import Iterable, Mapping + +__all__ = ["collapse_umis_directional", "collapse_umi_column"] + +_ALPHABET = "ACGTN" + + +def _neighbours(umi: str): + """All strings at Hamming distance 1 from ``umi`` over ``ACGTN`` (positions with other + characters are skipped).""" + for i, c in enumerate(umi): + if c not in _ALPHABET: + continue + head, tail = umi[:i], umi[i + 1 :] + for b in _ALPHABET: + if b != c: + yield head + b + tail + + +def collapse_umis_directional( + umis: Iterable[str] | Mapping[str, int], + counts: Iterable[int] | None = None, +) -> dict[str, str]: + """ + Directional UMI collapsing (UMI-tools) within one group of reads. + + Parameters + ---------- + umis : iterable of str, or mapping UMI -> count + Distinct raw UMIs. If a mapping is given, its values are the counts and ``counts`` must be + ``None``. If ``counts`` is ``None`` and ``umis`` is a plain iterable, it is treated as one + entry per READ and counted (duplicates allowed). + + counts : iterable of int, optional + Reads per raw UMI, aligned with ``umis`` (which must then be distinct). + + Returns + ------- + dict[str, str] + Mapping from every input UMI to its representative (a representative maps to itself). + """ + if isinstance(umis, Mapping): + if counts is not None: + raise ValueError("pass counts either in the mapping or in `counts`, not both") + cnt = dict(umis) + elif counts is None: + cnt = dict(Counter(umis)) + else: + umis = list(umis) + counts = list(counts) + if len(umis) != len(counts): + raise ValueError("umis and counts must have the same length") + cnt = {} + for u, c in zip(umis, counts): + if u in cnt: + raise ValueError(f"duplicate UMI with explicit counts: {u}") + cnt[u] = int(c) + + order = sorted(cnt, key=lambda u: (-cnt[u], u)) + rep: dict[str, str] = {} + for root in order: + if root in rep: + continue + rep[root] = root + stack = [root] + while stack: + parent = stack.pop() + if "N" in parent: # N-containing UMIs never absorb + continue + threshold = cnt[parent] + for nb in _neighbours(parent): + c = cnt.get(nb) + if c is None or nb in rep: + continue + if threshold >= 2 * c - 1: + rep[nb] = root + stack.append(nb) + return rep + + +def collapse_umi_column(umis: list[str]) -> list[str]: + """Per-read convenience wrapper: corrected UMI for each read UMI in ``umis`` (one group).""" + mapping = collapse_umis_directional(umis) + return [mapping[u] for u in umis] diff --git a/pairplex/utils.py b/pairplex/utils.py index 0c599a3..eb89083 100644 --- a/pairplex/utils.py +++ b/pairplex/utils.py @@ -29,6 +29,7 @@ from natsort import natsorted from spoa import poa +from .umi import collapse_umis_directional from .version import __version__ BARCODE_DIR = Path(__file__).resolve().parent / "barcodes" @@ -333,6 +334,7 @@ def process_droplet( min_cluster_reads: int = 3, min_cluster_umis: int = 2, min_cluster_fraction: float = 0.0, + umi_collapse: str = "directional", quiet: bool = False, debug: bool = False, ) -> list[dict]: @@ -365,6 +367,14 @@ def process_droplet( min_cluster_fraction : float Minimum fraction of reads in a cluster. + umi_collapse : str + UMI error correction applied to the whole droplet (all reads of this barcode in this well) + before any counting. ``"directional"`` (default) collapses UMIs with the UMI-tools + directional method (see :mod:`pairplex.umi`): the raw UMI of each read is kept in + ``umi_raw``, ``umi`` becomes the corrected representative, the metadata ``umis`` is the + corrected count and ``umis_raw`` the count before collapsing. ``"none"`` keeps raw UMIs and + the v2-base metadata layout (no ``umis_raw`` column). + quiet : bool Whether to suppress output. @@ -381,6 +391,16 @@ def process_droplet( barcode = name.split("_")[-1] metadata = [] + # UMI error correction, per (well, barcode): this partition is exactly one barcode of one well + if umi_collapse == "directional": + mapping = collapse_umis_directional(partition_df["umi"].to_list()) + partition_df = partition_df.with_columns( + pl.col("umi").alias("umi_raw"), + pl.col("umi").replace_strict(mapping, return_dtype=pl.String).alias("umi"), + ) + elif umi_collapse != "none": + raise ValueError(f"unknown umi_collapse: {umi_collapse!r}") + # sequences = abutils.io.from_polars( # partition_df, id_key="seq_id", sequence_key="sequence" # ) @@ -398,6 +418,8 @@ def process_droplet( contig_name = f"{barcode}_contig-{i}" cluster_df = partition_df.filter(pl.col("seq_id").is_in(clust)) n_umis = len(cluster_df["umi"].unique()) + if umi_collapse != "none": + n_umis_raw = len(cluster_df["umi_raw"].unique()) cluster_fraction = len(clust) / partition_df.shape[0] # consensus @@ -417,6 +439,10 @@ def process_droplet( "name": contig_name, "reads": len(clust), "umis": n_umis, + } + if umi_collapse != "none": + meta["umis_raw"] = n_umis_raw + meta |= { "cluster_fraction": cluster_fraction, "consensus": consensus, }