diff --git a/pairplex/cli/pairplex.py b/pairplex/cli/pairplex.py index 8a39506..0f40715 100644 --- a/pairplex/cli/pairplex.py +++ b/pairplex/cli/pairplex.py @@ -98,6 +98,12 @@ def version(): default="human", help="Germline database for V(D)J assignment", ) +@click.option( + "--mmseqs_threads", + type=click.IntRange(min=1), + default=1, + help="Threads for MMseqs2 during annotation (always explicit; None made MMseqs2 use every node CPU and crash)", +) @click.option("--quiet", is_flag=True, help="Whether to suppress output") @click.option( "--debug", @@ -118,6 +124,7 @@ def run( merge_paired_reads: bool, receptor: str, germline_database: str, + mmseqs_threads: int, quiet: bool, debug: bool, ): @@ -135,6 +142,7 @@ def run( merge_paired_reads=merge_paired_reads, receptor=receptor, germline_database=germline_database, + mmseqs_threads=mmseqs_threads, quiet=quiet, debug=debug, ) diff --git a/pairplex/correct_barcodes.py b/pairplex/correct_barcodes.py index 9e1b577..801775e 100644 --- a/pairplex/correct_barcodes.py +++ b/pairplex/correct_barcodes.py @@ -177,9 +177,9 @@ def run( annotation_printer = tqdm( total=0, bar_format="{desc}", position=5, leave=False ) - mmseqs_threads = None - if input_count < 1000: - mmseqs_threads = 1 + # always an explicit thread count: with None, MMseqs2 uses every online + # node CPU and crashes when that exceeds an abstar sub-query batch + mmseqs_threads = 1 # run abstar annotation_printer.set_description_str("annotating with abstar...") sequences = abstar.run( diff --git a/pairplex/pairplex.py b/pairplex/pairplex.py index 85c4ed5..e13da8d 100644 --- a/pairplex/pairplex.py +++ b/pairplex/pairplex.py @@ -49,6 +49,7 @@ def run( merge_paired_reads: bool = False, receptor: str = "bcr", germline_database: str = "human", + mmseqs_threads: int = 1, quiet: bool = False, debug: bool = False, ) -> list: @@ -105,6 +106,12 @@ def run( The germline database to use. Options are anything accepted by `abstar.run`, including custom databases if available. + mmseqs_threads : int + Number of threads MMseqs2 uses inside ``abstar.run``. Defaults to ``1``. Always passed explicitly: with + ``None`` MMseqs2 uses every online CPU of the node (ignoring ``OMP_NUM_THREADS`` and CPU affinity) and + crashes when its thread count exceeds the size of an abstar sub-query batch. Annotation is not the + bottleneck of a run, so ``1`` costs little. Does not change results. + quiet : bool Whether to suppress the progress bar and other update messages. @@ -119,6 +126,8 @@ def run( """ parameters = dict(locals()) # captured first: exactly the arguments of this call + if not isinstance(mmseqs_threads, int) or mmseqs_threads < 1: + raise ValueError(f"mmseqs_threads must be a positive integer, got {mmseqs_threads!r}") # print splash screen if not quiet: @@ -409,12 +418,8 @@ def run( # failsafe for empty consensus files if consensus_count <= 1 or os.path.getsize(consensus_file) == 0: # Changed consensus_count to include 1 (a single sequence can't be paired) continue - # guard against MMSeqs threading issues with small datasets consensus_count = filtered_df.shape[0] - mmseqs_threads = None - if consensus_count < 1000: - mmseqs_threads = 1 - # run abstar + # run abstar (always with an explicit MMseqs2 thread count, see docstring) sequences = abstar.run( sequences=str(consensus_file), germline_database=germline_database, diff --git a/pairplex/tests/test_run_annotation.py b/pairplex/tests/test_run_annotation.py new file mode 100644 index 0000000..8d44664 --- /dev/null +++ b/pairplex/tests/test_run_annotation.py @@ -0,0 +1,80 @@ +"""run() must always hand abstar.run an explicit MMseqs2 thread count. + +With ``mmseqs_threads=None`` MMseqs2 uses every online CPU of the node and crashes when that +exceeds the size of an abstar sub-query batch (seen: 256 threads vs 67 sequences). +""" + +import random +from unittest import mock + +import pytest + +from pairplex import pairplex as pp +from pairplex.utils import DEFAULT_WHITELIST + +TSO = "TTTCTTATATGGG" + + +def _write_reads(path, n_barcodes=3, reads_per_chain=4): + rng = random.Random(0) + with open(DEFAULT_WHITELIST) as f: + barcodes = [next(f).strip() for _ in range(n_barcodes)] + chains = ["".join(rng.choice("ACT") + rng.choice("ACGT") for _ in range(150)) for _ in range(2)] + with open(path, "w") as f: + i = 0 + for bc in barcodes: + for chain in chains: + for _ in range(reads_per_chain): + umi = "".join(rng.choice("ACGT") for _ in range(10)) + seq = bc + umi + TSO + chain + f.write(f"@r{i}\n{seq}\n+\n{'I' * len(seq)}\n") + i += 1 + + +@pytest.mark.parametrize("threads", [None, 4]) +def test_abstar_receives_explicit_mmseqs_threads(tmp_path, monkeypatch, threads): + monkeypatch.setenv("PAIRPLEX_MAX_WORKERS", "2") + reads = tmp_path / "well001.fastq" + _write_reads(reads) + kwargs = {} if threads is None else {"mmseqs_threads": threads} + with mock.patch.object(pp.abstar, "run", return_value=[]) as abstar_run: + pp.run(str(reads), tmp_path / "out", quiet=True, **kwargs) + assert abstar_run.call_count == 1 + got = abstar_run.call_args.kwargs["mmseqs_threads"] + assert got is not None + assert got == (1 if threads is None else threads) + + +def test_large_well_still_gets_explicit_threads(tmp_path, monkeypatch): + """v2-base passed None once a well had >= 1000 consensus sequences: the crashing case.""" + monkeypatch.setenv("PAIRPLEX_MAX_WORKERS", "2") + reads = tmp_path / "well001.fastq" + _write_reads(reads, n_barcodes=510, reads_per_chain=3) + with mock.patch.object(pp.abstar, "run", return_value=[]) as abstar_run: + pp.run(str(reads), tmp_path / "out", quiet=True) + consensus = (tmp_path / "out" / "consensus" / "well001.fasta").read_text().count(">") + assert consensus >= 1000 + assert abstar_run.call_args.kwargs["mmseqs_threads"] == 1 + + +@pytest.mark.parametrize("bad", [0, -2, None, 1.5]) +def test_invalid_mmseqs_threads_rejected(tmp_path, bad): + with pytest.raises(ValueError): + pp.run(str(tmp_path), tmp_path / "out", quiet=True, mmseqs_threads=bad) + + +def test_correct_barcodes_large_input_gets_explicit_threads(tmp_path, monkeypatch): + """correct-barcodes annotates the raw input; v2-base passed None once it had >= 1000 reads.""" + from pairplex import correct_barcodes as cb + + monkeypatch.setenv("PAIRPLEX_MAX_WORKERS", "2") + reads = tmp_path / "well001.fastq" + _write_reads(reads, n_barcodes=101, reads_per_chain=6) # 1212 reads + + class _Stop(Exception): + pass + + with mock.patch.object(cb.abstar, "run", side_effect=_Stop) as abstar_run: + with pytest.raises(_Stop): + cb.run(str(reads), tmp_path / "out", temp_directory=tmp_path / "tmp", quiet=True) + assert abstar_run.call_args.kwargs["mmseqs_threads"] == 1