Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions pairplex/pairing.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@

import abutils
import polars as pl
from natsort import natsorted

HEAVY_LOCI = ("IGH",)
LIGHT_LOCI = ("IGK", "IGL")
Expand Down Expand Up @@ -275,16 +276,22 @@ def pair_well(
pseudocount: float = DEFAULT_PSEUDOCOUNT,
collapse_clonal: bool = True,
) -> list:
"""Pair one well's annotated contigs with the chosen rule and write the paired files."""
"""
Pair one well's annotated contigs with the chosen rule and write the paired files.

Pairs are written (and returned) in natural barcode order, so the output row order does not
depend on the order of ``sequences``.
"""
if mode == "exact":
pairs = exact_pairs(sequences)
pairs = natsorted(exact_pairs(sequences), key=lambda p: p.name)
# identical to v2-base output
abutils.io.to_airr(pairs, str(paired_airr_file), drop_na_columns=False)
abutils.io.to_parquet(pairs, str(paired_parquet_file), drop_na_columns=False)
return pairs
if mode != "scored":
raise ValueError(f"unknown pairing mode {mode!r}; expected one of {PAIRING_MODES}")
pairs, _ = scored_pairs(sequences, support or {}, min_dominance, pseudocount, collapse_clonal)
pairs = natsorted(pairs, key=lambda p: p.name)
write_paired(pairs, paired_airr_file, paired_parquet_file, scored=True)
return pairs

Expand Down
74 changes: 52 additions & 22 deletions pairplex/pairplex.py
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,11 @@ def run(
If ``True``, reproduce v2-base exactly: forces ``tso_g_max`` to strip every leading G
(``lstrip("G")``), ``barcode_prefer_exact=False``, ``umi_collapse="none"``, ``support_columns=False``,
``qc=False`` and ``pairing="exact"``.
Row order is not a legacy behaviour: in every mode (``legacy=True`` included) reads enter
clustering in input-file order and every output is written in a fixed order (see
Notes), whatever the worker count. v2-base collected worker results with ``as_completed``,
so its read order, and hence greedy-clustering and POA tie-breaks, varied from run to
run; the fixed order is one of the outcomes v2-base could produce.

quiet : bool
Whether to suppress the progress bar and other update messages.
Expand Down Expand Up @@ -204,6 +209,15 @@ def run(
list
A list of paths to the output files.

Notes
-----
Determinism: the same input and options give the same outputs, independent of
``PAIRPLEX_MAX_WORKERS``. Barcode-parsing chunks are concatenated in chunk (= input) order, so
the reads of a droplet reach clustering and POA in input-file order. Droplets are processed
in sorted barcode order, so ``metadata/`` and ``consensus/`` rows are ordered by barcode, then
contig index. Annotated contigs are naturally sorted by ``sequence_id`` before they are
written, and paired outputs are naturally sorted by barcode.

"""
parameters = dict(locals()) # captured first: exactly the arguments of this call
if not isinstance(mmseqs_threads, int) or mmseqs_threads < 1:
Expand Down Expand Up @@ -331,6 +345,25 @@ def run(
# QC accumulators (qc=True): per-well stage counts, abstar failure modes, fragments, UMI stats
qc_rows, qc_failures, qc_fragments, qc_umis, qc_wells = [], [], {}, [], []

def finish_well(to_delete, printers):
"""End of one input file (also on early exits): delete temp files, close its printers, advance."""
if not debug:
main_pbar.set_postfix_str("cleaning up", refresh=True)
for f in to_delete:
if f is not None:
if os.path.exists(f):
os.remove(f)
if os.path.isdir(temp_directory) and not os.listdir(temp_directory):
os.rmdir(temp_directory)

# close out sub-progress bars
time.sleep(2)
for printer in printers:
printer.close()

# update the main progress bar
main_pbar.update(1)

# initialize the Process Pool
with ProcessPoolExecutor(
max_workers=available_cpus(), mp_context=mp.get_context("spawn")
Expand All @@ -339,6 +372,7 @@ def run(
# Checking for empty file, as this will break later steps
if os.path.getsize(input_file) == 0:
main_pbar.set_postfix_str("partitioning barcodes", refresh=True)
main_pbar.update(1)
continue

to_delete = []
Expand All @@ -347,6 +381,7 @@ def run(
name_printer = tqdm(total=0, bar_format="{desc}", position=4, leave=False)
seqs_printer = tqdm(total=0, bar_format="{desc}", position=5, leave=False)
valids_printer = tqdm(total=0, bar_format="{desc}", position=6, leave=False)
printers = [name_printer, seqs_printer, valids_printer]

# process the input file
input_file = Path(input_file)
Expand Down Expand Up @@ -401,7 +436,8 @@ def run(
executor.submit(parse_barcodes, chunk, temp_directory, **parse_kwargs)
for chunk in fastq_chunks
]
for future in as_completed(futures):
# results in chunk order (not completion order): reads keep their input order
for future in futures:
res = future.result()
if barcode_prefer_exact:
res, counts = res
Expand All @@ -417,6 +453,7 @@ def run(
running_total_printer.set_description_str(
f"valid barcodes: {all_valid_barcodes} | consensus sequences: {all_consensus_sequences} | pairs: {all_pairs}"
)
finish_well(to_delete, printers)
continue
concat_parquet = abutils.io.concatenate_parquet(
parquet_chunks, parsed_directory / f"{name}.parquet"
Expand Down Expand Up @@ -466,6 +503,7 @@ def run(
consensus_printer = tqdm(
total=0, bar_format="{desc}", position=8, leave=False
)
printers.append(consensus_printer)

# make consensus sequences for each droplet
futures = []
Expand All @@ -480,7 +518,7 @@ def run(
"quiet": quiet,
"debug": debug,
}
for bc, bc_df in partitions.items():
for bc, bc_df in sorted(partitions.items()): # fixed droplet order
if isinstance(bc, tuple):
bc = bc[0]
futures.append(
Expand All @@ -494,18 +532,23 @@ def run(

# collect metadata
metadata = []
droplet_results = [None] * len(futures)
future_index = {f: i for i, f in enumerate(futures)}
droplet_count = 0
consensus_count = 0
for future in as_completed(futures):
res = future.result()
droplet_results[future_index[future]] = res
if res is not None:
metadata.extend(res)
droplet_count += 1
consensus_count += len([r for r in res if r["pass_filters"]])
consensus_pbar.update(1)
consensus_printer.set_description_str(
f"{droplet_count} droplet barcodes | {consensus_count} consensus sequences"
)
for res in droplet_results: # submission (= sorted barcode) order
if res is not None:
metadata.extend(res)
metadata_df = pl.DataFrame(metadata)
consensus_pbar.close()
if qc:
Expand All @@ -526,6 +569,7 @@ def run(
f"{droplet_count} droplet barcodes | 0 consensus sequences"
)
consensus_file.touch()
finish_well(to_delete, printers)
continue
filtered_df = metadata_df.filter(pl.col("pass_filters"))
with consensus_file.open("w") as f:
Expand All @@ -545,6 +589,7 @@ def run(
main_pbar.set_postfix_str("annotating sequences", refresh=True)
# failsafe for empty consensus files
if consensus_count <= 1 or os.path.getsize(consensus_file) == 0: # Changed consensus_count to include 1 (a single sequence can't be paired)
finish_well(to_delete, printers)
continue
consensus_count = filtered_df.shape[0]
# run abstar (always with an explicit MMseqs2 thread count, see docstring)
Expand All @@ -555,6 +600,8 @@ def run(
receptor=receptor,
mmseqs_threads=mmseqs_threads,
)
if isinstance(sequences, list): # fixed row order for every annotated output
sequences = natsorted(sequences, key=lambda s: s.id)
if qc:
ann_df = qc_mod.annotation_frame(sequences)
n_rec = len(record_failures)
Expand Down Expand Up @@ -614,26 +661,9 @@ def run(
# cleanup
# --------------------

if not debug:
main_pbar.set_postfix_str("cleaning up", refresh=True)
for f in to_delete:
if f is not None:
if os.path.exists(f):
os.remove(f)
if os.path.isdir(temp_directory) and not os.listdir(temp_directory):
os.rmdir(temp_directory)

# close out sub-progress bars
time.sleep(2)
name_printer.close()
seqs_printer.close()
valids_printer.close()
consensus_printer.close()
if pairs_printer is not None:
pairs_printer.close()

# update the main progress bar
main_pbar.update(1)
printers.append(pairs_printer)
finish_well(to_delete, printers)

n_contract_pairs = None
if pairing == "scored":
Expand Down
22 changes: 17 additions & 5 deletions pairplex/pooled_pairing.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,11 @@

Output rows carry ``well`` (= home well), ``sequence_id:0/1`` and ``sequence:0/1`` so they can be
scored directly with ``simplex.score``.

Determinism: every tie is broken explicitly (representative contig: most UMIs, then smallest
``sequence_id``; reciprocal-best candidates: highest confidence, then smallest well, heavy and
light ``sequence_id``), and the posterior sums run in a fixed order, so the same per-well run
always gives the same pairs in the same row order.
"""

from __future__ import annotations
Expand Down Expand Up @@ -94,7 +99,7 @@ def load_run(run_dir: str | Path, wells: tuple[int, int] | None = None) -> pl.Da
occ = occ.join(pl.concat(meta), left_on=["sequence_id", "well"], right_on=["name", "well"], how="left")
else:
occ = occ.with_columns(pl.lit(None, pl.Int64).alias("umis"))
return occ.with_columns(pl.col("umis").fill_null(1).cast(pl.Int64))
return occ.with_columns(pl.col("umis").fill_null(1).cast(pl.Int64)).sort("well", "sequence_id")


def estimate_leakage(species_wells: pl.DataFrame, n_wells: int, min_umis: int = 6) -> float | None:
Expand Down Expand Up @@ -177,10 +182,13 @@ def pair_pooled(
"""
W = int(n_wells or occ["well"].n_unique())
# per (species, well): UMIs, and the representative contig there (most UMIs)
sw = (occ.sort("umis", descending=True)
# representative tie-break: most UMIs, then smallest sequence_id; rows in a fixed order so the
# posterior sums below always add up in the same order
sw = (occ.sort(["umis", "sequence_id"], descending=[True, False], maintain_order=True)
.group_by(*SPECIES, "well")
.agg(pl.col("umis").sum().alias("n"), pl.col("sequence_id").first(), pl.col("sequence").first(),
pl.col("cdr3_aa").first()))
pl.col("cdr3_aa").first())
.sort(*SPECIES, "well", nulls_last=True))
est = leakage is None
f = estimate_leakage(sw, W) if est else float(leakage)
if f is None:
Expand All @@ -205,7 +213,11 @@ def side(locus_filter, s):

H = side(pl.col("locus") == "IGH", 0)
L = side(pl.col("locus") != "IGH", 1)
cand = H.join(L, on=["barcode", "well"]).with_columns((pl.col("p_home:0") * pl.col("p_home:1")).alias("confidence"))
# fixed candidate order: "ordinal" ranks below break equal confidences by this order
tie = ["well", "sequence_id:0", "sequence_id:1"]
cand = (H.join(L, on=["barcode", "well"])
.with_columns((pl.col("p_home:0") * pl.col("p_home:1")).alias("confidence"))
.sort(["confidence", *tie], descending=[True, False, False, False]))
hkey = ["barcode", "locus:0", "v_gene:0", "j_gene:0", "junction:0"]
lkey = ["barcode", "locus:1", "v_gene:1", "j_gene:1", "junction:1"]
cand = cand.with_columns(
Expand All @@ -219,7 +231,7 @@ def side(locus_filter, s):
.drop("_rh", "_rl", "_h2", "_l2")
.filter(pl.col("confidence") >= min_confidence)
.with_columns(pl.col("barcode").alias("name"))
.sort("confidence", descending=True))
.sort(["confidence", "barcode", *tie], descending=[True, False, False, False, False]))
info = {"wells": W, "leakage": f, "leakage_estimated": est, "species": int(pooled.height), "pairs": int(pairs.height),
"ambient_flagged": n_ambient}
return pairs, info
Expand Down
4 changes: 2 additions & 2 deletions pairplex/qc.py
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,7 @@ def failure_table(rows: list[dict]) -> pl.DataFrame:
if df.height == 0:
return df
total = df.group_by("kind", "mode").agg(pl.col("count").sum()).with_columns(pl.lit("TOTAL").alias("well"))
total = total.select(list(FAILURE_COLUMNS)).sort(["kind", "count"], descending=[False, True])
total = total.select(list(FAILURE_COLUMNS)).sort(["kind", "count", "mode"], descending=[False, True, False])
return pl.concat([df, total])


Expand Down Expand Up @@ -375,7 +375,7 @@ def umi_hamming1_stats(parsed: pl.DataFrame, max_umis: int = 200_000, seed: int
"umis_with_h1_neighbour": 0, "umis_with_h1_neighbour_null": 0}
if parsed.height == 0:
return empty
u = parsed.select("barcode", "umi").drop_nulls().unique()
u = parsed.select("barcode", "umi").drop_nulls().unique().sort("barcode", "umi") # fixed order: seeded shuffle
if u.height == 0:
return empty
L = u["umi"].str.len_chars().mode().sort()[-1]
Expand Down
91 changes: 91 additions & 0 deletions pairplex/tests/test_determinism.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
"""Same input + same options -> identical outputs, whatever the worker count.

Runs PairPlex (default options) on the two golden wells plus a third, multi-chunk well
(> 1000 reads, so barcode parsing is split over several chunks), once with 1 worker and once
with 2, then compares every output file byte for byte, and the ``pair-pooled`` result of both.
"""

import gzip
import os
import shutil
from pathlib import Path

import pytest
from polars.testing import assert_frame_equal

import pairplex.pairplex as ppx
from pairplex.pooled_pairing import run_pooled_pairing
from pairplex.tests.golden import INPUT

# outputs that must not change between runs (logs carry timestamps and the worker count)
COMPARED = ["metadata", "consensus", "annotated", "pairs", "qc"]


def _multi_chunk_well(path: Path) -> None:
"""well002 = well000 + well001 + well000 again (read ids suffixed): 1499 reads, 2 chunks."""
with gzip.open(path, "wt") as out:
for src, suffix in (("well000", ""), ("well001", ""), ("well000", "-b")):
with gzip.open(INPUT / f"{src}.fastq.gz", "rt") as f:
for i, line in enumerate(f):
if i % 4 == 0 and suffix:
line = line.split()[0] + suffix + "\n"
out.write(line)


def _run(work: Path, inp: Path, workers: int) -> Path:
old = os.environ.get("PAIRPLEX_MAX_WORKERS")
os.environ["PAIRPLEX_MAX_WORKERS"] = str(workers)
try:
ppx.run(str(inp), str(work), quiet=True)
finally:
if old is None:
os.environ.pop("PAIRPLEX_MAX_WORKERS", None)
else:
os.environ["PAIRPLEX_MAX_WORKERS"] = old
return work


@pytest.fixture(scope="module")
def two_runs(tmp_path_factory):
base = tmp_path_factory.mktemp("determinism")
inp = base / "input"
shutil.copytree(INPUT, inp)
_multi_chunk_well(inp / "well002.fastq.gz")
return _run(base / "w1", inp, 1), _run(base / "w2", inp, 2)


def _files(out: Path) -> dict[str, Path]:
return {
str(p.relative_to(out)): p
for d in COMPARED
if (out / d).is_dir()
for p in sorted((out / d).rglob("*"))
if p.is_file()
}


def test_outputs_byte_identical_across_worker_counts(two_runs):
a, b = (_files(o) for o in two_runs)
assert a.keys() == b.keys()
assert any(k.startswith("metadata/well002") for k in a), "multi-chunk well missing"
differ = [k for k in a if a[k].read_bytes() != b[k].read_bytes()]
assert not differ, f"outputs differ between 1 and 2 workers: {differ}"


def test_metadata_in_fixed_row_order(two_runs):
import polars as pl
from natsort import natsorted

for f in sorted((two_runs[0] / "metadata").glob("*.csv")):
names = pl.read_csv(f)["name"].to_list()
assert names == natsorted(names), f.name


def test_pooled_pairing_identical(two_runs):
pa, ia = run_pooled_pairing(two_runs[0], output_file=two_runs[0] / "pooled" / "p.parquet")
pb, ib = run_pooled_pairing(two_runs[1], output_file=two_runs[1] / "pooled" / "p.parquet")
assert ia["leakage"] == ib["leakage"]
assert_frame_equal(pa, pb)
# the same input twice also gives the same pooled result
pc, _ = run_pooled_pairing(two_runs[0], output_file=two_runs[0] / "pooled" / "p2.parquet")
assert_frame_equal(pa, pc)
Loading