diff --git a/pairplex/cli/pairplex.py b/pairplex/cli/pairplex.py index 04a113a..fa9f6ff 100644 --- a/pairplex/cli/pairplex.py +++ b/pairplex/cli/pairplex.py @@ -110,6 +110,11 @@ def version(): default=3, help="Maximum number of Gs stripped after the TSO prefix (the TSO's rGrGrG run); genuine 5' Gs are kept", ) +@click.option( + "--barcode_prefer_exact/--no-barcode_prefer_exact", + default=True, + help="Prefer the read orientation with an exact barcode whitelist hit over a Hamming-1 correction in the other", +) @click.option( "--legacy", is_flag=True, @@ -137,6 +142,7 @@ def run( germline_database: str, mmseqs_threads: int, tso_g_max: int, + barcode_prefer_exact: bool, legacy: bool, quiet: bool, debug: bool, @@ -157,6 +163,7 @@ def run( germline_database=germline_database, mmseqs_threads=mmseqs_threads, tso_g_max=tso_g_max, + barcode_prefer_exact=barcode_prefer_exact, legacy=legacy, quiet=quiet, debug=debug, diff --git a/pairplex/pairplex.py b/pairplex/pairplex.py index 13a1893..262ff73 100644 --- a/pairplex/pairplex.py +++ b/pairplex/pairplex.py @@ -51,6 +51,7 @@ def run( germline_database: str = "human", mmseqs_threads: int = 1, tso_g_max: int = 3, + barcode_prefer_exact: bool = True, legacy: bool = False, quiet: bool = False, debug: bool = False, @@ -120,8 +121,15 @@ def run( genuine 5' Gs of the transcript beyond the TSO's run are kept. v2-base removed every leading G (``lstrip("G")``). + barcode_prefer_exact : bool + Read orientation choice. If ``True`` (default), an exact barcode whitelist hit in either orientation + (forward or reverse complement) wins over a Hamming-1 correction in the other; corrections are used only + when neither orientation is exact. Ties keep the forward orientation. Orientation counts are written to + the run log (``stats.barcode_orientation``). v2-base took the first orientation whose barcode corrected. + legacy : bool - If ``True``, reproduce v2-base exactly: forces ``tso_g_max`` to strip every leading G. + If ``True``, reproduce v2-base exactly: forces ``tso_g_max`` to strip every leading G + (``lstrip("G")``) and ``barcode_prefer_exact=False``. quiet : bool Whether to suppress the progress bar and other update messages. @@ -141,6 +149,7 @@ def run( raise ValueError(f"mmseqs_threads must be a positive integer, got {mmseqs_threads!r}") if legacy: tso_g_max = None # v2-base: lstrip("G") + barcode_prefer_exact = False # v2-base: first orientation that corrects elif not isinstance(tso_g_max, int) or tso_g_max < 0: raise ValueError(f"tso_g_max must be a non-negative integer, got {tso_g_max!r}") @@ -236,6 +245,7 @@ def run( running_total_printer = tqdm(total=0, bar_format="{desc}", position=2, leave=True) blank2_printer = tqdm(total=0, bar_format=" ", position=3, leave=True) all_valid_barcodes = 0 + orientation_counts = {} all_consensus_sequences = 0 all_pairs = 0 running_total_printer.set_description_str( @@ -297,18 +307,21 @@ def run( main_pbar.set_postfix_str("parsing barcodes", refresh=True) parquet_chunks = [] + parse_kwargs = {"whitelist_path": whitelist_path, "tso_g_max": tso_g_max} + if barcode_prefer_exact: + parse_kwargs.update(prefer_exact=True, return_counts=True) + else: + parse_kwargs.update(prefer_exact=False) futures = [ - executor.submit( - parse_barcodes, - chunk, - temp_directory, - whitelist_path=whitelist_path, - tso_g_max=tso_g_max, - ) + executor.submit(parse_barcodes, chunk, temp_directory, **parse_kwargs) for chunk in fastq_chunks ] for future in as_completed(futures): res = future.result() + if barcode_prefer_exact: + res, counts = res + for k, v in counts.items(): + orientation_counts[k] = orientation_counts.get(k, 0) + v if res is not None: parquet_chunks.append(res) to_delete.extend(parquet_chunks) @@ -511,6 +524,7 @@ def run( "valid_barcodes": all_valid_barcodes, "consensus_sequences": all_consensus_sequences, "pairs": all_pairs, + **({"barcode_orientation": orientation_counts} if barcode_prefer_exact else {}), } ) diff --git a/pairplex/tests/test_barcode_orientation.py b/pairplex/tests/test_barcode_orientation.py new file mode 100644 index 0000000..949a521 --- /dev/null +++ b/pairplex/tests/test_barcode_orientation.py @@ -0,0 +1,159 @@ +"""Orientation choice in parse_barcodes: an exact barcode hit beats a Hamming-1 correction.""" + +from unittest import mock + +import abutils +import polars as pl +import pytest + +from pairplex import pairplex as pp +from pairplex.utils import parse_barcodes + +A = "AAAACCCCGGGGTTTT" +B = "ACGTTGCAACGTTGCA" +C = "CATGCATGGTACGTAC" +TSO = "TTTCTTATATGGG" +UMI_F = "AAAAAAAAAA" +UMI_R = "CCCCCCCCCC" +INSERT = "ACTGACTGACTGATTACA" + + +def mut(bc, i=5): + """Hamming-1 variant of ``bc`` at position ``i``.""" + return bc[:i] + ("T" if bc[i] != "T" else "A") + bc[i + 1 :] + + +def rc(s): + return abutils.tl.reverse_complement(s) + + +def read(fwd_bc, rc_bc): + """A read whose forward start is ``fwd_bc`` and whose reverse-complement start is ``rc_bc``.""" + return fwd_bc + UMI_F + TSO + INSERT + rc(rc_bc + UMI_R + TSO) + + +@pytest.fixture +def whitelist(tmp_path): + path = tmp_path / "wl.txt" + path.write_text("\n".join([A, B, C]) + "\n") + return str(path) + + +def _parse(tmp_path, whitelist, reads, **kwargs): + tmp_path.mkdir(parents=True, exist_ok=True) + fq = tmp_path / "chunk.fastq" + fq.write_text("".join(f"@r{i}\n{s}\n+\n{'I' * len(s)}\n" for i, s in enumerate(reads))) + res = parse_barcodes(fq, tmp_path / "out", whitelist_path=whitelist, return_counts=True, **kwargs) + path, counts = res + df = pl.read_parquet(path) if path else pl.DataFrame({"seq_id": [], "barcode": [], "umi": []}) + return df, counts + + +def test_exact_rc_beats_corrected_fwd(tmp_path, whitelist): + df, counts = _parse(tmp_path, whitelist, [read(mut(A), B)]) + assert df["barcode"].to_list() == [B] + assert df["umi"].to_list() == [UMI_R] + assert counts["exact_rc"] == 1 and counts["switched"] == 1 + + +def test_legacy_takes_corrected_fwd(tmp_path, whitelist): + df, counts = _parse(tmp_path, whitelist, [read(mut(A), B)], prefer_exact=False) + assert df["barcode"].to_list() == [A] + assert df["umi"].to_list() == [UMI_F] + assert all(v == 0 for v in counts.values()) + + +def test_exact_fwd_beats_corrected_rc(tmp_path, whitelist): + df, counts = _parse(tmp_path, whitelist, [read(A, mut(B))]) + assert df["barcode"].to_list() == [A] + assert counts["exact_fwd"] == 1 and counts["both_exact"] == 0 + + +def test_both_exact_keeps_forward_and_counts(tmp_path, whitelist): + df, counts = _parse(tmp_path, whitelist, [read(A, B)]) + assert df["barcode"].to_list() == [A] + assert df["umi"].to_list() == [UMI_F] + assert counts["both_exact"] == 1 + + +def test_both_corrected_keeps_forward_and_counts(tmp_path, whitelist): + df, counts = _parse(tmp_path, whitelist, [read(mut(A), mut(B))]) + assert df["barcode"].to_list() == [A] + assert counts["corrected_fwd"] == 1 and counts["both_corrected"] == 1 + + +def test_only_rc_correctable(tmp_path, whitelist): + df, counts = _parse(tmp_path, whitelist, [read("GGGGGGGGGGGGGGGG", mut(C))]) + assert df["barcode"].to_list() == [C] + assert df["umi"].to_list() == [UMI_R] + assert counts["corrected_rc"] == 1 + + +def test_no_hit_dropped(tmp_path, whitelist): + df, counts = _parse(tmp_path, whitelist, [read("GGGGGGGGGGGGGGGG", "GGGGGGGGGGGGGGGG")]) + assert df.height == 0 + assert sum(counts.values()) == 0 + + +def test_check_rc_false_ignores_rc(tmp_path, whitelist): + df, counts = _parse(tmp_path, whitelist, [read(mut(A), B)], check_rc=False) + assert df["barcode"].to_list() == [A] + assert counts["corrected_fwd"] == 1 and counts["switched"] == 0 + + +@pytest.mark.parametrize("prefer_exact", [True, False]) +def test_same_records_when_no_conflict(tmp_path, whitelist, prefer_exact): + """Reads without an orientation conflict parse identically in both modes (incl. sequence).""" + reads = [read(A, "G" * 16), read(mut(B), "G" * 16), read("G" * 16, C), read("G" * 16, mut(C))] + df, _ = _parse(tmp_path, whitelist, reads, prefer_exact=prefer_exact) + legacy, _ = _parse(tmp_path / "legacy", whitelist, reads, prefer_exact=False) + assert df.sort("seq_id").equals(legacy.sort("seq_id")) + + +def test_default_return_is_path_only(tmp_path, whitelist): + fq = tmp_path / "c.fastq" + s = read(A, B) + fq.write_text(f"@r\n{s}\n+\n{'I' * len(s)}\n") + out = parse_barcodes(fq, tmp_path / "o", whitelist_path=whitelist) + assert isinstance(out, str) + + +def _run(tmp_path, monkeypatch, whitelist, **kwargs): + monkeypatch.setenv("PAIRPLEX_MAX_WORKERS", "2") + tmp_path.mkdir(parents=True, exist_ok=True) + fq = tmp_path / "well001.fastq" + reads = [read(mut(A), B)] * 3 + [read(A, B)] * 3 + fq.write_text("".join(f"@r{i}\n{s}\n+\n{'I' * len(s)}\n" for i, s in enumerate(reads))) + out = tmp_path / "out" + with mock.patch.object(pp.abstar, "run", return_value=[]): + pp.run(str(fq), out, whitelist_path=whitelist, quiet=True, debug=True, **kwargs) + return pl.read_parquet(out / "parsed" / "well001.parquet") + + +def test_run_default_and_legacy(tmp_path, monkeypatch, whitelist): + new = _run(tmp_path / "new", monkeypatch, whitelist) + assert sorted(new["barcode"].to_list()) == sorted([B] * 3 + [A] * 3) + old = _run(tmp_path / "old", monkeypatch, whitelist, legacy=True) + assert old["barcode"].to_list() == [A] * 6 + old2 = _run(tmp_path / "old2", monkeypatch, whitelist, barcode_prefer_exact=False) + assert old2["barcode"].to_list() == [A] * 6 + + +# --- interaction with the TSO G-run fix (#15): every orientation path must use strip_tso_g --- + + +@pytest.mark.parametrize("prefer_exact", [True, False]) +def test_tso_g_strip_applies_in_every_orientation_path(tmp_path, whitelist, prefer_exact): + """The read carries the TSO's GGG plus a genuine 5' G ("G" + INSERT). With tso_g_max=3 that G must + survive whichever orientation code path parses the read (exact-hit path or legacy first-hit path).""" + insert = "G" + INSERT + r = A + UMI_F + TSO + insert + rc(mut(B) + UMI_R + TSO) + df, _ = _parse(tmp_path / str(prefer_exact), whitelist, [r], prefer_exact=prefer_exact, tso_g_max=3) + assert df["barcode"].to_list() == [A] + assert df["sequence"][0].startswith(insert) + + +def test_tso_g_legacy_lstrip_in_exact_path(tmp_path, whitelist): + r = A + UMI_F + TSO + "G" + INSERT + rc(mut(B) + UMI_R + TSO) + df, _ = _parse(tmp_path, whitelist, [r], prefer_exact=True, tso_g_max=None) + assert df["sequence"][0].startswith(INSERT) diff --git a/pairplex/utils.py b/pairplex/utils.py index 04eb729..a058a24 100644 --- a/pairplex/utils.py +++ b/pairplex/utils.py @@ -211,7 +211,9 @@ def parse_barcodes( whitelist_path: str | Path | None = None, check_rc: bool = True, tso_g_max: int | None = TSO_G_MAX, -) -> str | None: + prefer_exact: bool = True, + return_counts: bool = False, +) -> str | None | tuple[str | None, dict]: """ Process a chunk of fastq file to extract barcodes, UMIs and TSO sequences. @@ -240,6 +242,20 @@ def parse_barcodes( the TSO's own rGrGrG run. Genuine 5' Gs beyond it are kept. ``None`` removes every leading G (``lstrip("G")``, the v2-base behaviour). + prefer_exact : bool = True + Orientation choice when ``check_rc`` is set. If ``True``, an exact whitelist hit in either orientation + wins over a Hamming-1 correction in the other; only if neither orientation is exact is the first + correctable orientation (forward, then reverse complement) used. If both orientations are exact, or + both need correction, the forward orientation is kept (and counted). If ``False``, the first + orientation whose barcode corrects is used even when the other one is exact (v2-base behaviour). + + return_counts : bool = False + If ``True``, return ``(path, counts)``, where ``counts`` holds orientation-choice counters: + ``exact_fwd``, ``exact_rc``, ``corrected_fwd``, ``corrected_rc``, ``both_exact`` (both orientations + exact, forward kept), ``both_corrected`` (both need correction, forward kept) and ``switched`` (reads + where an exact reverse-complement hit replaced the corrected forward barcode v2-base used). + Only filled when ``prefer_exact`` is ``True``. + enforce_bc_whitelist : bool = True Whether to enforce the barcode whitelist. @@ -258,7 +274,24 @@ def parse_barcodes( whitelist = load_barcode_whitelist(whitelist_path) records = [] + counts = dict.fromkeys( + ["exact_fwd", "exact_rc", "corrected_fwd", "corrected_rc", "both_exact", "both_corrected", "switched"], 0 + ) for seq in abutils.io.parse_fastx(str(input_file)): + if prefer_exact: + # same fixed offsets as below; see the NOTE there + chosen = _choose_orientation(seq.sequence, whitelist, check_rc, counts) + if chosen is not None: + s, corrected = chosen + records.append( + { + "umi": s[16:26], + "barcode": corrected, + "seq_id": seq.id, + "sequence": strip_tso_g(s[36:], tso_g_max), # remove the TSO's G-run + } + ) + continue seqs = [seq.sequence] if check_rc: seqs.append(abutils.tl.reverse_complement(seq.sequence)) @@ -288,13 +321,57 @@ def parse_barcodes( # all done! break + output_file = None if records: output_directory = Path(output_directory) output_directory.mkdir(parents=True, exist_ok=True) output_file = output_directory / f"bcdf_{output_name}.parquet" df = pl.DataFrame(records) df.write_parquet(output_file) - return str(output_file) + output_file = str(output_file) + if return_counts: + return output_file, counts + return output_file + + +def _choose_orientation( + sequence: str, whitelist: set[str], check_rc: bool, counts: dict +) -> tuple[str, str] | None: + """ + Pick the read orientation for ``parse_barcodes(prefer_exact=True)``. + + Returns ``(oriented_sequence, barcode)`` or ``None``. An exact whitelist hit in either orientation beats a + Hamming-1 correction; ties (both exact, or both corrected) keep the forward orientation. Updates ``counts``. + """ + fwd_bc = sequence[:16] + fwd_exact = fwd_bc in whitelist + rc = rc_bc = None + rc_exact = False + if check_rc: + rc = abutils.tl.reverse_complement(sequence) + rc_bc = rc[:16] + rc_exact = rc_bc in whitelist + if fwd_exact: + counts["exact_fwd"] += 1 + if rc_exact: + counts["both_exact"] += 1 + return sequence, fwd_bc + if rc_exact: + counts["exact_rc"] += 1 + if correct_barcode(fwd_bc, whitelist) is not None: + counts["switched"] += 1 + return rc, rc_bc + fwd_corrected = correct_barcode(fwd_bc, whitelist) + rc_corrected = correct_barcode(rc_bc, whitelist) if check_rc else None + if fwd_corrected is not None: + counts["corrected_fwd"] += 1 + if rc_corrected is not None: + counts["both_corrected"] += 1 + return sequence, fwd_corrected + if rc_corrected is not None: + counts["corrected_rc"] += 1 + return rc, rc_corrected + return None def greedy_clustering(