diff --git a/pairplex/pairing.py b/pairplex/pairing.py index 2eba149..0ec6bd5 100644 --- a/pairplex/pairing.py +++ b/pairplex/pairing.py @@ -55,6 +55,7 @@ import abutils import polars as pl +from natsort import natsorted HEAVY_LOCI = ("IGH",) LIGHT_LOCI = ("IGK", "IGL") @@ -275,9 +276,14 @@ def pair_well( pseudocount: float = DEFAULT_PSEUDOCOUNT, collapse_clonal: bool = True, ) -> list: - """Pair one well's annotated contigs with the chosen rule and write the paired files.""" + """ + Pair one well's annotated contigs with the chosen rule and write the paired files. + + Pairs are written (and returned) in natural barcode order, so the output row order does not + depend on the order of ``sequences``. + """ if mode == "exact": - pairs = exact_pairs(sequences) + pairs = natsorted(exact_pairs(sequences), key=lambda p: p.name) # identical to v2-base output abutils.io.to_airr(pairs, str(paired_airr_file), drop_na_columns=False) abutils.io.to_parquet(pairs, str(paired_parquet_file), drop_na_columns=False) @@ -285,6 +291,7 @@ def pair_well( if mode != "scored": raise ValueError(f"unknown pairing mode {mode!r}; expected one of {PAIRING_MODES}") pairs, _ = scored_pairs(sequences, support or {}, min_dominance, pseudocount, collapse_clonal) + pairs = natsorted(pairs, key=lambda p: p.name) write_paired(pairs, paired_airr_file, paired_parquet_file, scored=True) return pairs diff --git a/pairplex/pairplex.py b/pairplex/pairplex.py index b0d5439..daf7dbb 100644 --- a/pairplex/pairplex.py +++ b/pairplex/pairplex.py @@ -177,6 +177,11 @@ def run( If ``True``, reproduce v2-base exactly: forces ``tso_g_max`` to strip every leading G (``lstrip("G")``), ``barcode_prefer_exact=False``, ``umi_collapse="none"``, ``support_columns=False``, ``qc=False`` and ``pairing="exact"``. + Row order is not a legacy behaviour: in every mode (``legacy=True`` included) reads enter + clustering in input-file order and every output is written in a fixed order (see + Notes), whatever the worker count. v2-base collected worker results with ``as_completed``, + so its read order, and hence greedy-clustering and POA tie-breaks, varied from run to + run; the fixed order is one of the outcomes v2-base could produce. quiet : bool Whether to suppress the progress bar and other update messages. @@ -204,6 +209,15 @@ def run( list A list of paths to the output files. + Notes + ----- + Determinism: the same input and options give the same outputs, independent of + ``PAIRPLEX_MAX_WORKERS``. Barcode-parsing chunks are concatenated in chunk (= input) order, so + the reads of a droplet reach clustering and POA in input-file order. Droplets are processed + in sorted barcode order, so ``metadata/`` and ``consensus/`` rows are ordered by barcode, then + contig index. Annotated contigs are naturally sorted by ``sequence_id`` before they are + written, and paired outputs are naturally sorted by barcode. + """ parameters = dict(locals()) # captured first: exactly the arguments of this call if not isinstance(mmseqs_threads, int) or mmseqs_threads < 1: @@ -331,6 +345,25 @@ def run( # QC accumulators (qc=True): per-well stage counts, abstar failure modes, fragments, UMI stats qc_rows, qc_failures, qc_fragments, qc_umis, qc_wells = [], [], {}, [], [] + def finish_well(to_delete, printers): + """End of one input file (also on early exits): delete temp files, close its printers, advance.""" + if not debug: + main_pbar.set_postfix_str("cleaning up", refresh=True) + for f in to_delete: + if f is not None: + if os.path.exists(f): + os.remove(f) + if os.path.isdir(temp_directory) and not os.listdir(temp_directory): + os.rmdir(temp_directory) + + # close out sub-progress bars + time.sleep(2) + for printer in printers: + printer.close() + + # update the main progress bar + main_pbar.update(1) + # initialize the Process Pool with ProcessPoolExecutor( max_workers=available_cpus(), mp_context=mp.get_context("spawn") @@ -339,6 +372,7 @@ def run( # Checking for empty file, as this will break later steps if os.path.getsize(input_file) == 0: main_pbar.set_postfix_str("partitioning barcodes", refresh=True) + main_pbar.update(1) continue to_delete = [] @@ -347,6 +381,7 @@ def run( name_printer = tqdm(total=0, bar_format="{desc}", position=4, leave=False) seqs_printer = tqdm(total=0, bar_format="{desc}", position=5, leave=False) valids_printer = tqdm(total=0, bar_format="{desc}", position=6, leave=False) + printers = [name_printer, seqs_printer, valids_printer] # process the input file input_file = Path(input_file) @@ -401,7 +436,8 @@ def run( executor.submit(parse_barcodes, chunk, temp_directory, **parse_kwargs) for chunk in fastq_chunks ] - for future in as_completed(futures): + # results in chunk order (not completion order): reads keep their input order + for future in futures: res = future.result() if barcode_prefer_exact: res, counts = res @@ -417,6 +453,7 @@ def run( running_total_printer.set_description_str( f"valid barcodes: {all_valid_barcodes} | consensus sequences: {all_consensus_sequences} | pairs: {all_pairs}" ) + finish_well(to_delete, printers) continue concat_parquet = abutils.io.concatenate_parquet( parquet_chunks, parsed_directory / f"{name}.parquet" @@ -466,6 +503,7 @@ def run( consensus_printer = tqdm( total=0, bar_format="{desc}", position=8, leave=False ) + printers.append(consensus_printer) # make consensus sequences for each droplet futures = [] @@ -480,7 +518,7 @@ def run( "quiet": quiet, "debug": debug, } - for bc, bc_df in partitions.items(): + for bc, bc_df in sorted(partitions.items()): # fixed droplet order if isinstance(bc, tuple): bc = bc[0] futures.append( @@ -494,18 +532,23 @@ def run( # collect metadata metadata = [] + droplet_results = [None] * len(futures) + future_index = {f: i for i, f in enumerate(futures)} droplet_count = 0 consensus_count = 0 for future in as_completed(futures): res = future.result() + droplet_results[future_index[future]] = res if res is not None: - metadata.extend(res) droplet_count += 1 consensus_count += len([r for r in res if r["pass_filters"]]) consensus_pbar.update(1) consensus_printer.set_description_str( f"{droplet_count} droplet barcodes | {consensus_count} consensus sequences" ) + for res in droplet_results: # submission (= sorted barcode) order + if res is not None: + metadata.extend(res) metadata_df = pl.DataFrame(metadata) consensus_pbar.close() if qc: @@ -526,6 +569,7 @@ def run( f"{droplet_count} droplet barcodes | 0 consensus sequences" ) consensus_file.touch() + finish_well(to_delete, printers) continue filtered_df = metadata_df.filter(pl.col("pass_filters")) with consensus_file.open("w") as f: @@ -545,6 +589,7 @@ def run( main_pbar.set_postfix_str("annotating sequences", refresh=True) # failsafe for empty consensus files if consensus_count <= 1 or os.path.getsize(consensus_file) == 0: # Changed consensus_count to include 1 (a single sequence can't be paired) + finish_well(to_delete, printers) continue consensus_count = filtered_df.shape[0] # run abstar (always with an explicit MMseqs2 thread count, see docstring) @@ -555,6 +600,8 @@ def run( receptor=receptor, mmseqs_threads=mmseqs_threads, ) + if isinstance(sequences, list): # fixed row order for every annotated output + sequences = natsorted(sequences, key=lambda s: s.id) if qc: ann_df = qc_mod.annotation_frame(sequences) n_rec = len(record_failures) @@ -614,26 +661,9 @@ def run( # cleanup # -------------------- - if not debug: - main_pbar.set_postfix_str("cleaning up", refresh=True) - for f in to_delete: - if f is not None: - if os.path.exists(f): - os.remove(f) - if os.path.isdir(temp_directory) and not os.listdir(temp_directory): - os.rmdir(temp_directory) - - # close out sub-progress bars - time.sleep(2) - name_printer.close() - seqs_printer.close() - valids_printer.close() - consensus_printer.close() if pairs_printer is not None: - pairs_printer.close() - - # update the main progress bar - main_pbar.update(1) + printers.append(pairs_printer) + finish_well(to_delete, printers) n_contract_pairs = None if pairing == "scored": diff --git a/pairplex/pooled_pairing.py b/pairplex/pooled_pairing.py index c03533f..d54bfb1 100644 --- a/pairplex/pooled_pairing.py +++ b/pairplex/pooled_pairing.py @@ -31,6 +31,11 @@ Output rows carry ``well`` (= home well), ``sequence_id:0/1`` and ``sequence:0/1`` so they can be scored directly with ``simplex.score``. + +Determinism: every tie is broken explicitly (representative contig: most UMIs, then smallest +``sequence_id``; reciprocal-best candidates: highest confidence, then smallest well, heavy and +light ``sequence_id``), and the posterior sums run in a fixed order, so the same per-well run +always gives the same pairs in the same row order. """ from __future__ import annotations @@ -94,7 +99,7 @@ def load_run(run_dir: str | Path, wells: tuple[int, int] | None = None) -> pl.Da occ = occ.join(pl.concat(meta), left_on=["sequence_id", "well"], right_on=["name", "well"], how="left") else: occ = occ.with_columns(pl.lit(None, pl.Int64).alias("umis")) - return occ.with_columns(pl.col("umis").fill_null(1).cast(pl.Int64)) + return occ.with_columns(pl.col("umis").fill_null(1).cast(pl.Int64)).sort("well", "sequence_id") def estimate_leakage(species_wells: pl.DataFrame, n_wells: int, min_umis: int = 6) -> float | None: @@ -177,10 +182,13 @@ def pair_pooled( """ W = int(n_wells or occ["well"].n_unique()) # per (species, well): UMIs, and the representative contig there (most UMIs) - sw = (occ.sort("umis", descending=True) + # representative tie-break: most UMIs, then smallest sequence_id; rows in a fixed order so the + # posterior sums below always add up in the same order + sw = (occ.sort(["umis", "sequence_id"], descending=[True, False], maintain_order=True) .group_by(*SPECIES, "well") .agg(pl.col("umis").sum().alias("n"), pl.col("sequence_id").first(), pl.col("sequence").first(), - pl.col("cdr3_aa").first())) + pl.col("cdr3_aa").first()) + .sort(*SPECIES, "well", nulls_last=True)) est = leakage is None f = estimate_leakage(sw, W) if est else float(leakage) if f is None: @@ -205,7 +213,11 @@ def side(locus_filter, s): H = side(pl.col("locus") == "IGH", 0) L = side(pl.col("locus") != "IGH", 1) - cand = H.join(L, on=["barcode", "well"]).with_columns((pl.col("p_home:0") * pl.col("p_home:1")).alias("confidence")) + # fixed candidate order: "ordinal" ranks below break equal confidences by this order + tie = ["well", "sequence_id:0", "sequence_id:1"] + cand = (H.join(L, on=["barcode", "well"]) + .with_columns((pl.col("p_home:0") * pl.col("p_home:1")).alias("confidence")) + .sort(["confidence", *tie], descending=[True, False, False, False])) hkey = ["barcode", "locus:0", "v_gene:0", "j_gene:0", "junction:0"] lkey = ["barcode", "locus:1", "v_gene:1", "j_gene:1", "junction:1"] cand = cand.with_columns( @@ -219,7 +231,7 @@ def side(locus_filter, s): .drop("_rh", "_rl", "_h2", "_l2") .filter(pl.col("confidence") >= min_confidence) .with_columns(pl.col("barcode").alias("name")) - .sort("confidence", descending=True)) + .sort(["confidence", "barcode", *tie], descending=[True, False, False, False, False])) info = {"wells": W, "leakage": f, "leakage_estimated": est, "species": int(pooled.height), "pairs": int(pairs.height), "ambient_flagged": n_ambient} return pairs, info diff --git a/pairplex/qc.py b/pairplex/qc.py index 37367a8..0e880d9 100644 --- a/pairplex/qc.py +++ b/pairplex/qc.py @@ -272,7 +272,7 @@ def failure_table(rows: list[dict]) -> pl.DataFrame: if df.height == 0: return df total = df.group_by("kind", "mode").agg(pl.col("count").sum()).with_columns(pl.lit("TOTAL").alias("well")) - total = total.select(list(FAILURE_COLUMNS)).sort(["kind", "count"], descending=[False, True]) + total = total.select(list(FAILURE_COLUMNS)).sort(["kind", "count", "mode"], descending=[False, True, False]) return pl.concat([df, total]) @@ -375,7 +375,7 @@ def umi_hamming1_stats(parsed: pl.DataFrame, max_umis: int = 200_000, seed: int "umis_with_h1_neighbour": 0, "umis_with_h1_neighbour_null": 0} if parsed.height == 0: return empty - u = parsed.select("barcode", "umi").drop_nulls().unique() + u = parsed.select("barcode", "umi").drop_nulls().unique().sort("barcode", "umi") # fixed order: seeded shuffle if u.height == 0: return empty L = u["umi"].str.len_chars().mode().sort()[-1] diff --git a/pairplex/tests/test_determinism.py b/pairplex/tests/test_determinism.py new file mode 100644 index 0000000..8fb38f2 --- /dev/null +++ b/pairplex/tests/test_determinism.py @@ -0,0 +1,91 @@ +"""Same input + same options -> identical outputs, whatever the worker count. + +Runs PairPlex (default options) on the two golden wells plus a third, multi-chunk well +(> 1000 reads, so barcode parsing is split over several chunks), once with 1 worker and once +with 2, then compares every output file byte for byte, and the ``pair-pooled`` result of both. +""" + +import gzip +import os +import shutil +from pathlib import Path + +import pytest +from polars.testing import assert_frame_equal + +import pairplex.pairplex as ppx +from pairplex.pooled_pairing import run_pooled_pairing +from pairplex.tests.golden import INPUT + +# outputs that must not change between runs (logs carry timestamps and the worker count) +COMPARED = ["metadata", "consensus", "annotated", "pairs", "qc"] + + +def _multi_chunk_well(path: Path) -> None: + """well002 = well000 + well001 + well000 again (read ids suffixed): 1499 reads, 2 chunks.""" + with gzip.open(path, "wt") as out: + for src, suffix in (("well000", ""), ("well001", ""), ("well000", "-b")): + with gzip.open(INPUT / f"{src}.fastq.gz", "rt") as f: + for i, line in enumerate(f): + if i % 4 == 0 and suffix: + line = line.split()[0] + suffix + "\n" + out.write(line) + + +def _run(work: Path, inp: Path, workers: int) -> Path: + old = os.environ.get("PAIRPLEX_MAX_WORKERS") + os.environ["PAIRPLEX_MAX_WORKERS"] = str(workers) + try: + ppx.run(str(inp), str(work), quiet=True) + finally: + if old is None: + os.environ.pop("PAIRPLEX_MAX_WORKERS", None) + else: + os.environ["PAIRPLEX_MAX_WORKERS"] = old + return work + + +@pytest.fixture(scope="module") +def two_runs(tmp_path_factory): + base = tmp_path_factory.mktemp("determinism") + inp = base / "input" + shutil.copytree(INPUT, inp) + _multi_chunk_well(inp / "well002.fastq.gz") + return _run(base / "w1", inp, 1), _run(base / "w2", inp, 2) + + +def _files(out: Path) -> dict[str, Path]: + return { + str(p.relative_to(out)): p + for d in COMPARED + if (out / d).is_dir() + for p in sorted((out / d).rglob("*")) + if p.is_file() + } + + +def test_outputs_byte_identical_across_worker_counts(two_runs): + a, b = (_files(o) for o in two_runs) + assert a.keys() == b.keys() + assert any(k.startswith("metadata/well002") for k in a), "multi-chunk well missing" + differ = [k for k in a if a[k].read_bytes() != b[k].read_bytes()] + assert not differ, f"outputs differ between 1 and 2 workers: {differ}" + + +def test_metadata_in_fixed_row_order(two_runs): + import polars as pl + from natsort import natsorted + + for f in sorted((two_runs[0] / "metadata").glob("*.csv")): + names = pl.read_csv(f)["name"].to_list() + assert names == natsorted(names), f.name + + +def test_pooled_pairing_identical(two_runs): + pa, ia = run_pooled_pairing(two_runs[0], output_file=two_runs[0] / "pooled" / "p.parquet") + pb, ib = run_pooled_pairing(two_runs[1], output_file=two_runs[1] / "pooled" / "p.parquet") + assert ia["leakage"] == ib["leakage"] + assert_frame_equal(pa, pb) + # the same input twice also gives the same pooled result + pc, _ = run_pooled_pairing(two_runs[0], output_file=two_runs[0] / "pooled" / "p2.parquet") + assert_frame_equal(pa, pc)