Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions pairplex/cli/pairplex.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,13 @@ def version():
default=True,
help="Prefer the read orientation with an exact barcode whitelist hit over a Hamming-1 correction in the other",
)
@click.option(
"--umi-collapse",
"umi_collapse",
type=click.Choice(["directional", "none"]),
default="directional",
help="UMI error correction within each (well, barcode): UMI-tools directional (default) or none",
)
@click.option(
"--legacy",
is_flag=True,
Expand All @@ -126,6 +133,16 @@ def version():
is_flag=True,
help="Whether to enable debug mode, which saves all temporary files to ease troubleshooting",
)
@click.option(
"--support_columns/--no_support_columns",
default=True,
help="Carry cluster support (umis, reads, cluster_fraction) into the annotated outputs",
)
@click.option(
"--qc/--no_qc",
default=True,
help="Write qc/stage_counts.csv, qc/annotation_failures.csv and qc/report.{json,md}",
)
def run(
sequences: str | Path,
output_directory: str | Path,
Expand All @@ -143,9 +160,12 @@ def run(
mmseqs_threads: int,
tso_g_max: int,
barcode_prefer_exact: bool,
umi_collapse: str,
legacy: bool,
quiet: bool,
debug: bool,
support_columns: bool,
qc: bool,
):
run_pairplex(
sequences=sequences,
Expand All @@ -164,9 +184,12 @@ def run(
mmseqs_threads=mmseqs_threads,
tso_g_max=tso_g_max,
barcode_prefer_exact=barcode_prefer_exact,
umi_collapse=umi_collapse,
legacy=legacy,
quiet=quiet,
debug=debug,
support_columns=support_columns,
qc=qc,
)


Expand Down
93 changes: 86 additions & 7 deletions pairplex/pairplex.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import multiprocessing as mp
import os
import time
from contextlib import nullcontext
from concurrent.futures import ProcessPoolExecutor, as_completed
from pathlib import Path

Expand All @@ -27,6 +28,7 @@
from natsort import natsorted
from tqdm.auto import tqdm

from . import qc as qc_mod
from .provenance import RunLog, run_log_path
from .utils import available_cpus, parse_barcodes, print_splash, process_droplet

Expand All @@ -52,9 +54,12 @@ def run(
mmseqs_threads: int = 1,
tso_g_max: int = 3,
barcode_prefer_exact: bool = True,
umi_collapse: str = "directional",
legacy: bool = False,
quiet: bool = False,
debug: bool = False,
support_columns: bool = True,
qc: bool = True,
) -> list:
"""
Run PairPlex.
Expand Down Expand Up @@ -127,16 +132,41 @@ def run(
when neither orientation is exact. Ties keep the forward orientation. Orientation counts are written to
the run log (``stats.barcode_orientation``). v2-base took the first orientation whose barcode corrected.

umi_collapse : str
UMI error correction, applied within each (well, droplet barcode) before any UMI counting.
``"directional"`` (default) uses the UMI-tools directional adjacency method: UMI ``b`` is
absorbed by UMI ``a`` when they differ at one position and ``reads(a) >= 2 * reads(b) - 1``,
transitively, processing UMIs by descending read count (see :mod:`pairplex.umi`, which also
documents how UMIs containing ``N`` are handled). The metadata ``umis`` column is then the
corrected count, a ``umis_raw`` column holds the count before collapsing, and
``min_cluster_umis`` filters on corrected counts. ``"none"`` counts raw UMIs (v2-base
behaviour, no ``umis_raw`` column).

legacy : bool
If ``True``, reproduce v2-base exactly: forces ``tso_g_max`` to strip every leading G
(``lstrip("G")``) and ``barcode_prefer_exact=False``.
(``lstrip("G")``), ``barcode_prefer_exact=False``, ``umi_collapse="none"``, ``support_columns=False`` and
``qc=False``.

quiet : bool
Whether to suppress the progress bar and other update messages.

debug : bool
If ``True``, all temporary files will be retained to ease debugging.

support_columns : bool
If ``True`` (default), the annotated outputs carry the cluster support of each contig:
``umis``, ``reads``, ``cluster_fraction``, ``barcode`` and ``contig`` in the unpaired
outputs, ``umis:0/1``, ``reads:0/1``, ``cluster_fraction:0/1`` (etc.) in the paired outputs.
abstar's ``umi`` column, which is always null for PairPlex consensus contigs, is dropped.
Pairing is unchanged.

qc : bool
If ``True`` (default), write ``qc/stage_counts.csv`` (drop counts at every stage, per
well and in total), ``qc/annotation_failures.csv`` (abstar failure and exclusion modes)
and ``qc/report.json`` / ``qc/report.md`` (fragment fraction, UMI error signature,
ambient proxy, leakage by chain, pair confidence, clonal light-chain coherence).
Pairing is unchanged.


Returns
-------
Expand All @@ -153,6 +183,17 @@ def run(
elif not isinstance(tso_g_max, int) or tso_g_max < 0:
raise ValueError(f"tso_g_max must be a non-negative integer, got {tso_g_max!r}")

# legacy mode forces every v2 behaviour option to its v2-base value
if legacy:
umi_collapse = "none"
support_columns = False
qc = False
if umi_collapse not in ("directional", "none"):
raise ValueError(
f"umi_collapse must be 'directional' or 'none', got {umi_collapse!r}"
)
parameters["umi_collapse_effective"] = umi_collapse

# print splash screen
if not quiet:
print_splash(include_version=True)
Expand Down Expand Up @@ -252,6 +293,9 @@ def run(
f"valid barcodes: {all_valid_barcodes} | consensus sequences: {all_consensus_sequences} | pairs: {all_pairs}"
)

# QC accumulators (qc=True): per-well stage counts, abstar failure modes, fragments, UMI stats
qc_rows, qc_failures, qc_fragments, qc_umis, qc_wells = [], [], {}, [], []

# initialize the Process Pool
with ProcessPoolExecutor(
max_workers=available_cpus(), mp_context=mp.get_context("spawn")
Expand All @@ -273,6 +317,10 @@ def run(
input_file = Path(input_file)
name = input_file.stem
name_printer.set_description_str(f"---- {name} ----")
if qc:
wc = qc_mod.new_counts(name)
qc_rows.append(wc)
qc_wells.append(name)

# check to make sure the input file is FASTA/Q (and not empty)
if abutils.io.determine_fastx_format(str(input_file)) is None:
Expand All @@ -291,6 +339,8 @@ def run(
for s in abutils.io.parse_fastx(str(input_file)):
input_count += 1
seqs_printer.set_description_str(f"{input_count} input sequences")
if qc:
wc["input_reads"] = wc["dropped_orientation_or_barcode"] = input_count

# split input file into chunks
main_pbar.set_postfix_str("splitting input file", refresh=True)
Expand Down Expand Up @@ -338,6 +388,10 @@ def run(
)
df = pl.read_parquet(concat_parquet)
seqs_with_barcodes = df.shape[0]
if qc:
wc.update(qc_mod.read_stage_counts(input_count, df))
wc.update(qc_mod.barcode_stage_counts(df, min_cluster_reads))
qc_umis.append(qc_mod.umi_hamming1_stats(df))
all_valid_barcodes += seqs_with_barcodes
valids_printer.set_description_str(
f"{seqs_with_barcodes} sequences with valid barcodes"
Expand Down Expand Up @@ -387,6 +441,7 @@ def run(
"min_cluster_fraction": min_cluster_fraction,
"consensus_downsample": consensus_downsample,
"clustering_threshold": clustering_threshold,
"umi_collapse": umi_collapse,
"quiet": quiet,
"debug": debug,
}
Expand Down Expand Up @@ -418,6 +473,12 @@ def run(
)
metadata_df = pl.DataFrame(metadata)
consensus_pbar.close()
if qc:
wc.update(
qc_mod.cluster_stage_counts(
metadata_df, min_cluster_reads, min_cluster_umis, min_cluster_fraction
)
)

# write metadata to file
metadata_file = metadata_directory / f"{name}.csv"
Expand Down Expand Up @@ -452,12 +513,21 @@ def run(
continue
consensus_count = filtered_df.shape[0]
# run abstar (always with an explicit MMseqs2 thread count, see docstring)
sequences = abstar.run(
sequences=str(consensus_file),
germline_database=germline_database,
receptor=receptor,
mmseqs_threads=mmseqs_threads,
)
with qc_mod.capture_abstar_failures() if qc else nullcontext([]) as record_failures:
sequences = abstar.run(
sequences=str(consensus_file),
germline_database=germline_database,
receptor=receptor,
mmseqs_threads=mmseqs_threads,
)
if qc:
ann_df = qc_mod.annotation_frame(sequences)
n_rec = len(record_failures)
wc.update(qc_mod.annotation_stage_counts(ann_df, consensus_count, n_rec))
qc_failures.extend(qc_mod.annotation_failure_rows(name, ann_df, record_failures))
qc_mod.merge_fragment_counts(qc_fragments, qc_mod.fragment_counts(ann_df, n_rec))
if support_columns:
qc_mod.add_support_columns(sequences, metadata_df)

if len(sequences) > 1: # A single sequence cannot be paired and will cause issues

Expand All @@ -481,7 +551,11 @@ def run(
paired_airr_file = annotated_directory / f"{name}_paired.tsv"
paired_parquet_file = annotated_directory / f"{name}_paired.parquet"
pairs = abutils.tl.assign_pairs(sequences, delim="_", delim_occurance=-1)
if qc:
wc.update(qc_mod.chain_composition_counts(pairs))
pairs = [p for p in pairs if len(p.heavies) == 1 and len(p.lights) == 1]
if qc:
wc["emitted_pairs"] = len(pairs)
pairs_printer.set_description_str(f"{len(pairs)} paired sequences")
abutils.io.to_airr(pairs, str(paired_airr_file), drop_na_columns=False)
abutils.io.to_parquet(
Expand Down Expand Up @@ -518,6 +592,11 @@ def run(
# update the main progress bar
main_pbar.update(1)

if qc:
qc_mod.write_qc(
output_directory, qc_rows, qc_failures, qc_fragments, qc_umis, qc_wells
)

run_log.finish(
stats={
"input_files": len(input_files),
Expand Down
Loading
Loading