Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions pairplex/cli/pairplex.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,12 @@ def version():
default="human",
help="Germline database for V(D)J assignment",
)
@click.option(
"--mmseqs_threads",
type=click.IntRange(min=1),
default=1,
help="Threads for MMseqs2 during annotation (always explicit; None made MMseqs2 use every node CPU and crash)",
)
@click.option("--quiet", is_flag=True, help="Whether to suppress output")
@click.option(
"--debug",
Expand All @@ -118,6 +124,7 @@ def run(
merge_paired_reads: bool,
receptor: str,
germline_database: str,
mmseqs_threads: int,
quiet: bool,
debug: bool,
):
Expand All @@ -135,6 +142,7 @@ def run(
merge_paired_reads=merge_paired_reads,
receptor=receptor,
germline_database=germline_database,
mmseqs_threads=mmseqs_threads,
quiet=quiet,
debug=debug,
)
Expand Down
6 changes: 3 additions & 3 deletions pairplex/correct_barcodes.py
Original file line number Diff line number Diff line change
Expand Up @@ -177,9 +177,9 @@ def run(
annotation_printer = tqdm(
total=0, bar_format="{desc}", position=5, leave=False
)
mmseqs_threads = None
if input_count < 1000:
mmseqs_threads = 1
# always an explicit thread count: with None, MMseqs2 uses every online
# node CPU and crashes when that exceeds an abstar sub-query batch
mmseqs_threads = 1
# run abstar
annotation_printer.set_description_str("annotating with abstar...")
sequences = abstar.run(
Expand Down
15 changes: 10 additions & 5 deletions pairplex/pairplex.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ def run(
merge_paired_reads: bool = False,
receptor: str = "bcr",
germline_database: str = "human",
mmseqs_threads: int = 1,
quiet: bool = False,
debug: bool = False,
) -> list:
Expand Down Expand Up @@ -105,6 +106,12 @@ def run(
The germline database to use. Options are anything accepted by `abstar.run`, including
custom databases if available.

mmseqs_threads : int
Number of threads MMseqs2 uses inside ``abstar.run``. Defaults to ``1``. Always passed explicitly: with
``None`` MMseqs2 uses every online CPU of the node (ignoring ``OMP_NUM_THREADS`` and CPU affinity) and
crashes when its thread count exceeds the size of an abstar sub-query batch. Annotation is not the
bottleneck of a run, so ``1`` costs little. Does not change results.

quiet : bool
Whether to suppress the progress bar and other update messages.

Expand All @@ -119,6 +126,8 @@ def run(

"""
parameters = dict(locals()) # captured first: exactly the arguments of this call
if not isinstance(mmseqs_threads, int) or mmseqs_threads < 1:
raise ValueError(f"mmseqs_threads must be a positive integer, got {mmseqs_threads!r}")

# print splash screen
if not quiet:
Expand Down Expand Up @@ -409,12 +418,8 @@ def run(
# failsafe for empty consensus files
if consensus_count <= 1 or os.path.getsize(consensus_file) == 0: # Changed consensus_count to include 1 (a single sequence can't be paired)
continue
# guard against MMSeqs threading issues with small datasets
consensus_count = filtered_df.shape[0]
mmseqs_threads = None
if consensus_count < 1000:
mmseqs_threads = 1
# run abstar
# run abstar (always with an explicit MMseqs2 thread count, see docstring)
sequences = abstar.run(
sequences=str(consensus_file),
germline_database=germline_database,
Expand Down
80 changes: 80 additions & 0 deletions pairplex/tests/test_run_annotation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
"""run() must always hand abstar.run an explicit MMseqs2 thread count.

With ``mmseqs_threads=None`` MMseqs2 uses every online CPU of the node and crashes when that
exceeds the size of an abstar sub-query batch (seen: 256 threads vs 67 sequences).
"""

import random
from unittest import mock

import pytest

from pairplex import pairplex as pp
from pairplex.utils import DEFAULT_WHITELIST

TSO = "TTTCTTATATGGG"


def _write_reads(path, n_barcodes=3, reads_per_chain=4):
rng = random.Random(0)
with open(DEFAULT_WHITELIST) as f:
barcodes = [next(f).strip() for _ in range(n_barcodes)]
chains = ["".join(rng.choice("ACT") + rng.choice("ACGT") for _ in range(150)) for _ in range(2)]
with open(path, "w") as f:
i = 0
for bc in barcodes:
for chain in chains:
for _ in range(reads_per_chain):
umi = "".join(rng.choice("ACGT") for _ in range(10))
seq = bc + umi + TSO + chain
f.write(f"@r{i}\n{seq}\n+\n{'I' * len(seq)}\n")
i += 1


@pytest.mark.parametrize("threads", [None, 4])
def test_abstar_receives_explicit_mmseqs_threads(tmp_path, monkeypatch, threads):
monkeypatch.setenv("PAIRPLEX_MAX_WORKERS", "2")
reads = tmp_path / "well001.fastq"
_write_reads(reads)
kwargs = {} if threads is None else {"mmseqs_threads": threads}
with mock.patch.object(pp.abstar, "run", return_value=[]) as abstar_run:
pp.run(str(reads), tmp_path / "out", quiet=True, **kwargs)
assert abstar_run.call_count == 1
got = abstar_run.call_args.kwargs["mmseqs_threads"]
assert got is not None
assert got == (1 if threads is None else threads)


def test_large_well_still_gets_explicit_threads(tmp_path, monkeypatch):
"""v2-base passed None once a well had >= 1000 consensus sequences: the crashing case."""
monkeypatch.setenv("PAIRPLEX_MAX_WORKERS", "2")
reads = tmp_path / "well001.fastq"
_write_reads(reads, n_barcodes=510, reads_per_chain=3)
with mock.patch.object(pp.abstar, "run", return_value=[]) as abstar_run:
pp.run(str(reads), tmp_path / "out", quiet=True)
consensus = (tmp_path / "out" / "consensus" / "well001.fasta").read_text().count(">")
assert consensus >= 1000
assert abstar_run.call_args.kwargs["mmseqs_threads"] == 1


@pytest.mark.parametrize("bad", [0, -2, None, 1.5])
def test_invalid_mmseqs_threads_rejected(tmp_path, bad):
with pytest.raises(ValueError):
pp.run(str(tmp_path), tmp_path / "out", quiet=True, mmseqs_threads=bad)


def test_correct_barcodes_large_input_gets_explicit_threads(tmp_path, monkeypatch):
"""correct-barcodes annotates the raw input; v2-base passed None once it had >= 1000 reads."""
from pairplex import correct_barcodes as cb

monkeypatch.setenv("PAIRPLEX_MAX_WORKERS", "2")
reads = tmp_path / "well001.fastq"
_write_reads(reads, n_barcodes=101, reads_per_chain=6) # 1212 reads

class _Stop(Exception):
pass

with mock.patch.object(cb.abstar, "run", side_effect=_Stop) as abstar_run:
with pytest.raises(_Stop):
cb.run(str(reads), tmp_path / "out", temp_directory=tmp_path / "tmp", quiet=True)
assert abstar_run.call_args.kwargs["mmseqs_threads"] == 1
Loading