Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions pairplex/cli/pairplex.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,11 @@ def version():
default=3,
help="Maximum number of Gs stripped after the TSO prefix (the TSO's rGrGrG run); genuine 5' Gs are kept",
)
@click.option(
"--barcode_prefer_exact/--no-barcode_prefer_exact",
default=True,
help="Prefer the read orientation with an exact barcode whitelist hit over a Hamming-1 correction in the other",
)
@click.option(
"--legacy",
is_flag=True,
Expand Down Expand Up @@ -137,6 +142,7 @@ def run(
germline_database: str,
mmseqs_threads: int,
tso_g_max: int,
barcode_prefer_exact: bool,
legacy: bool,
quiet: bool,
debug: bool,
Expand All @@ -157,6 +163,7 @@ def run(
germline_database=germline_database,
mmseqs_threads=mmseqs_threads,
tso_g_max=tso_g_max,
barcode_prefer_exact=barcode_prefer_exact,
legacy=legacy,
quiet=quiet,
debug=debug,
Expand Down
30 changes: 22 additions & 8 deletions pairplex/pairplex.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ def run(
germline_database: str = "human",
mmseqs_threads: int = 1,
tso_g_max: int = 3,
barcode_prefer_exact: bool = True,
legacy: bool = False,
quiet: bool = False,
debug: bool = False,
Expand Down Expand Up @@ -120,8 +121,15 @@ def run(
genuine 5' Gs of the transcript beyond the TSO's run are kept. v2-base removed every leading
G (``lstrip("G")``).

barcode_prefer_exact : bool
Read orientation choice. If ``True`` (default), an exact barcode whitelist hit in either orientation
(forward or reverse complement) wins over a Hamming-1 correction in the other; corrections are used only
when neither orientation is exact. Ties keep the forward orientation. Orientation counts are written to
the run log (``stats.barcode_orientation``). v2-base took the first orientation whose barcode corrected.

legacy : bool
If ``True``, reproduce v2-base exactly: forces ``tso_g_max`` to strip every leading G.
If ``True``, reproduce v2-base exactly: forces ``tso_g_max`` to strip every leading G
(``lstrip("G")``) and ``barcode_prefer_exact=False``.

quiet : bool
Whether to suppress the progress bar and other update messages.
Expand All @@ -141,6 +149,7 @@ def run(
raise ValueError(f"mmseqs_threads must be a positive integer, got {mmseqs_threads!r}")
if legacy:
tso_g_max = None # v2-base: lstrip("G")
barcode_prefer_exact = False # v2-base: first orientation that corrects
elif not isinstance(tso_g_max, int) or tso_g_max < 0:
raise ValueError(f"tso_g_max must be a non-negative integer, got {tso_g_max!r}")

Expand Down Expand Up @@ -236,6 +245,7 @@ def run(
running_total_printer = tqdm(total=0, bar_format="{desc}", position=2, leave=True)
blank2_printer = tqdm(total=0, bar_format=" ", position=3, leave=True)
all_valid_barcodes = 0
orientation_counts = {}
all_consensus_sequences = 0
all_pairs = 0
running_total_printer.set_description_str(
Expand Down Expand Up @@ -297,18 +307,21 @@ def run(

main_pbar.set_postfix_str("parsing barcodes", refresh=True)
parquet_chunks = []
parse_kwargs = {"whitelist_path": whitelist_path, "tso_g_max": tso_g_max}
if barcode_prefer_exact:
parse_kwargs.update(prefer_exact=True, return_counts=True)
else:
parse_kwargs.update(prefer_exact=False)
futures = [
executor.submit(
parse_barcodes,
chunk,
temp_directory,
whitelist_path=whitelist_path,
tso_g_max=tso_g_max,
)
executor.submit(parse_barcodes, chunk, temp_directory, **parse_kwargs)
for chunk in fastq_chunks
]
for future in as_completed(futures):
res = future.result()
if barcode_prefer_exact:
res, counts = res
for k, v in counts.items():
orientation_counts[k] = orientation_counts.get(k, 0) + v
if res is not None:
parquet_chunks.append(res)
to_delete.extend(parquet_chunks)
Expand Down Expand Up @@ -511,6 +524,7 @@ def run(
"valid_barcodes": all_valid_barcodes,
"consensus_sequences": all_consensus_sequences,
"pairs": all_pairs,
**({"barcode_orientation": orientation_counts} if barcode_prefer_exact else {}),
}
)

Expand Down
159 changes: 159 additions & 0 deletions pairplex/tests/test_barcode_orientation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
"""Orientation choice in parse_barcodes: an exact barcode hit beats a Hamming-1 correction."""

from unittest import mock

import abutils
import polars as pl
import pytest

from pairplex import pairplex as pp
from pairplex.utils import parse_barcodes

A = "AAAACCCCGGGGTTTT"
B = "ACGTTGCAACGTTGCA"
C = "CATGCATGGTACGTAC"
TSO = "TTTCTTATATGGG"
UMI_F = "AAAAAAAAAA"
UMI_R = "CCCCCCCCCC"
INSERT = "ACTGACTGACTGATTACA"


def mut(bc, i=5):
"""Hamming-1 variant of ``bc`` at position ``i``."""
return bc[:i] + ("T" if bc[i] != "T" else "A") + bc[i + 1 :]


def rc(s):
return abutils.tl.reverse_complement(s)


def read(fwd_bc, rc_bc):
"""A read whose forward start is ``fwd_bc`` and whose reverse-complement start is ``rc_bc``."""
return fwd_bc + UMI_F + TSO + INSERT + rc(rc_bc + UMI_R + TSO)


@pytest.fixture
def whitelist(tmp_path):
path = tmp_path / "wl.txt"
path.write_text("\n".join([A, B, C]) + "\n")
return str(path)


def _parse(tmp_path, whitelist, reads, **kwargs):
tmp_path.mkdir(parents=True, exist_ok=True)
fq = tmp_path / "chunk.fastq"
fq.write_text("".join(f"@r{i}\n{s}\n+\n{'I' * len(s)}\n" for i, s in enumerate(reads)))
res = parse_barcodes(fq, tmp_path / "out", whitelist_path=whitelist, return_counts=True, **kwargs)
path, counts = res
df = pl.read_parquet(path) if path else pl.DataFrame({"seq_id": [], "barcode": [], "umi": []})
return df, counts


def test_exact_rc_beats_corrected_fwd(tmp_path, whitelist):
df, counts = _parse(tmp_path, whitelist, [read(mut(A), B)])
assert df["barcode"].to_list() == [B]
assert df["umi"].to_list() == [UMI_R]
assert counts["exact_rc"] == 1 and counts["switched"] == 1


def test_legacy_takes_corrected_fwd(tmp_path, whitelist):
df, counts = _parse(tmp_path, whitelist, [read(mut(A), B)], prefer_exact=False)
assert df["barcode"].to_list() == [A]
assert df["umi"].to_list() == [UMI_F]
assert all(v == 0 for v in counts.values())


def test_exact_fwd_beats_corrected_rc(tmp_path, whitelist):
df, counts = _parse(tmp_path, whitelist, [read(A, mut(B))])
assert df["barcode"].to_list() == [A]
assert counts["exact_fwd"] == 1 and counts["both_exact"] == 0


def test_both_exact_keeps_forward_and_counts(tmp_path, whitelist):
df, counts = _parse(tmp_path, whitelist, [read(A, B)])
assert df["barcode"].to_list() == [A]
assert df["umi"].to_list() == [UMI_F]
assert counts["both_exact"] == 1


def test_both_corrected_keeps_forward_and_counts(tmp_path, whitelist):
df, counts = _parse(tmp_path, whitelist, [read(mut(A), mut(B))])
assert df["barcode"].to_list() == [A]
assert counts["corrected_fwd"] == 1 and counts["both_corrected"] == 1


def test_only_rc_correctable(tmp_path, whitelist):
df, counts = _parse(tmp_path, whitelist, [read("GGGGGGGGGGGGGGGG", mut(C))])
assert df["barcode"].to_list() == [C]
assert df["umi"].to_list() == [UMI_R]
assert counts["corrected_rc"] == 1


def test_no_hit_dropped(tmp_path, whitelist):
df, counts = _parse(tmp_path, whitelist, [read("GGGGGGGGGGGGGGGG", "GGGGGGGGGGGGGGGG")])
assert df.height == 0
assert sum(counts.values()) == 0


def test_check_rc_false_ignores_rc(tmp_path, whitelist):
df, counts = _parse(tmp_path, whitelist, [read(mut(A), B)], check_rc=False)
assert df["barcode"].to_list() == [A]
assert counts["corrected_fwd"] == 1 and counts["switched"] == 0


@pytest.mark.parametrize("prefer_exact", [True, False])
def test_same_records_when_no_conflict(tmp_path, whitelist, prefer_exact):
"""Reads without an orientation conflict parse identically in both modes (incl. sequence)."""
reads = [read(A, "G" * 16), read(mut(B), "G" * 16), read("G" * 16, C), read("G" * 16, mut(C))]
df, _ = _parse(tmp_path, whitelist, reads, prefer_exact=prefer_exact)
legacy, _ = _parse(tmp_path / "legacy", whitelist, reads, prefer_exact=False)
assert df.sort("seq_id").equals(legacy.sort("seq_id"))


def test_default_return_is_path_only(tmp_path, whitelist):
fq = tmp_path / "c.fastq"
s = read(A, B)
fq.write_text(f"@r\n{s}\n+\n{'I' * len(s)}\n")
out = parse_barcodes(fq, tmp_path / "o", whitelist_path=whitelist)
assert isinstance(out, str)


def _run(tmp_path, monkeypatch, whitelist, **kwargs):
monkeypatch.setenv("PAIRPLEX_MAX_WORKERS", "2")
tmp_path.mkdir(parents=True, exist_ok=True)
fq = tmp_path / "well001.fastq"
reads = [read(mut(A), B)] * 3 + [read(A, B)] * 3
fq.write_text("".join(f"@r{i}\n{s}\n+\n{'I' * len(s)}\n" for i, s in enumerate(reads)))
out = tmp_path / "out"
with mock.patch.object(pp.abstar, "run", return_value=[]):
pp.run(str(fq), out, whitelist_path=whitelist, quiet=True, debug=True, **kwargs)
return pl.read_parquet(out / "parsed" / "well001.parquet")


def test_run_default_and_legacy(tmp_path, monkeypatch, whitelist):
new = _run(tmp_path / "new", monkeypatch, whitelist)
assert sorted(new["barcode"].to_list()) == sorted([B] * 3 + [A] * 3)
old = _run(tmp_path / "old", monkeypatch, whitelist, legacy=True)
assert old["barcode"].to_list() == [A] * 6
old2 = _run(tmp_path / "old2", monkeypatch, whitelist, barcode_prefer_exact=False)
assert old2["barcode"].to_list() == [A] * 6


# --- interaction with the TSO G-run fix (#15): every orientation path must use strip_tso_g ---


@pytest.mark.parametrize("prefer_exact", [True, False])
def test_tso_g_strip_applies_in_every_orientation_path(tmp_path, whitelist, prefer_exact):
"""The read carries the TSO's GGG plus a genuine 5' G ("G" + INSERT). With tso_g_max=3 that G must
survive whichever orientation code path parses the read (exact-hit path or legacy first-hit path)."""
insert = "G" + INSERT
r = A + UMI_F + TSO + insert + rc(mut(B) + UMI_R + TSO)
df, _ = _parse(tmp_path / str(prefer_exact), whitelist, [r], prefer_exact=prefer_exact, tso_g_max=3)
assert df["barcode"].to_list() == [A]
assert df["sequence"][0].startswith(insert)


def test_tso_g_legacy_lstrip_in_exact_path(tmp_path, whitelist):
r = A + UMI_F + TSO + "G" + INSERT + rc(mut(B) + UMI_R + TSO)
df, _ = _parse(tmp_path, whitelist, [r], prefer_exact=True, tso_g_max=None)
assert df["sequence"][0].startswith(INSERT)
81 changes: 79 additions & 2 deletions pairplex/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,9 @@ def parse_barcodes(
whitelist_path: str | Path | None = None,
check_rc: bool = True,
tso_g_max: int | None = TSO_G_MAX,
) -> str | None:
prefer_exact: bool = True,
return_counts: bool = False,
) -> str | None | tuple[str | None, dict]:
"""
Process a chunk of fastq file to extract barcodes, UMIs and TSO sequences.

Expand Down Expand Up @@ -240,6 +242,20 @@ def parse_barcodes(
the TSO's own rGrGrG run. Genuine 5' Gs beyond it are kept. ``None`` removes every leading G
(``lstrip("G")``, the v2-base behaviour).

prefer_exact : bool = True
Orientation choice when ``check_rc`` is set. If ``True``, an exact whitelist hit in either orientation
wins over a Hamming-1 correction in the other; only if neither orientation is exact is the first
correctable orientation (forward, then reverse complement) used. If both orientations are exact, or
both need correction, the forward orientation is kept (and counted). If ``False``, the first
orientation whose barcode corrects is used even when the other one is exact (v2-base behaviour).

return_counts : bool = False
If ``True``, return ``(path, counts)``, where ``counts`` holds orientation-choice counters:
``exact_fwd``, ``exact_rc``, ``corrected_fwd``, ``corrected_rc``, ``both_exact`` (both orientations
exact, forward kept), ``both_corrected`` (both need correction, forward kept) and ``switched`` (reads
where an exact reverse-complement hit replaced the corrected forward barcode v2-base used).
Only filled when ``prefer_exact`` is ``True``.

enforce_bc_whitelist : bool = True
Whether to enforce the barcode whitelist.

Expand All @@ -258,7 +274,24 @@ def parse_barcodes(
whitelist = load_barcode_whitelist(whitelist_path)

records = []
counts = dict.fromkeys(
["exact_fwd", "exact_rc", "corrected_fwd", "corrected_rc", "both_exact", "both_corrected", "switched"], 0
)
for seq in abutils.io.parse_fastx(str(input_file)):
if prefer_exact:
# same fixed offsets as below; see the NOTE there
chosen = _choose_orientation(seq.sequence, whitelist, check_rc, counts)
if chosen is not None:
s, corrected = chosen
records.append(
{
"umi": s[16:26],
"barcode": corrected,
"seq_id": seq.id,
"sequence": strip_tso_g(s[36:], tso_g_max), # remove the TSO's G-run
}
)
continue
seqs = [seq.sequence]
if check_rc:
seqs.append(abutils.tl.reverse_complement(seq.sequence))
Expand Down Expand Up @@ -288,13 +321,57 @@ def parse_barcodes(
# all done!
break

output_file = None
if records:
output_directory = Path(output_directory)
output_directory.mkdir(parents=True, exist_ok=True)
output_file = output_directory / f"bcdf_{output_name}.parquet"
df = pl.DataFrame(records)
df.write_parquet(output_file)
return str(output_file)
output_file = str(output_file)
if return_counts:
return output_file, counts
return output_file


def _choose_orientation(
sequence: str, whitelist: set[str], check_rc: bool, counts: dict
) -> tuple[str, str] | None:
"""
Pick the read orientation for ``parse_barcodes(prefer_exact=True)``.

Returns ``(oriented_sequence, barcode)`` or ``None``. An exact whitelist hit in either orientation beats a
Hamming-1 correction; ties (both exact, or both corrected) keep the forward orientation. Updates ``counts``.
"""
fwd_bc = sequence[:16]
fwd_exact = fwd_bc in whitelist
rc = rc_bc = None
rc_exact = False
if check_rc:
rc = abutils.tl.reverse_complement(sequence)
rc_bc = rc[:16]
rc_exact = rc_bc in whitelist
if fwd_exact:
counts["exact_fwd"] += 1
if rc_exact:
counts["both_exact"] += 1
return sequence, fwd_bc
if rc_exact:
counts["exact_rc"] += 1
if correct_barcode(fwd_bc, whitelist) is not None:
counts["switched"] += 1
return rc, rc_bc
fwd_corrected = correct_barcode(fwd_bc, whitelist)
rc_corrected = correct_barcode(rc_bc, whitelist) if check_rc else None
if fwd_corrected is not None:
counts["corrected_fwd"] += 1
if rc_corrected is not None:
counts["both_corrected"] += 1
return sequence, fwd_corrected
if rc_corrected is not None:
counts["corrected_rc"] += 1
return rc, rc_corrected
return None


def greedy_clustering(
Expand Down
Loading