From e918d25e26065c6a1c3bf2d9173b086a4d76f608 Mon Sep 17 00:00:00 2001 From: Benjamin Nemoz Date: Tue, 17 Nov 2026 16:29:29 +0000 Subject: [PATCH] fix(parser): prefer the read orientation with an exact barcode hit parse_barcodes took the first orientation (forward, then reverse complement) whose barcode corrected to the whitelist, even when that was a Hamming-1 correction and the other orientation was an exact hit. Now an exact hit in either orientation wins; corrections are used only when neither is exact. Ties (both exact / both corrected) keep the forward orientation and are counted. Orientation counts go to the run log (stats.barcode_orientation). New run() options barcode_prefer_exact (CLI --barcode_prefer_exact/--no-barcode_prefer_exact) and legacy (CLI --legacy); legacy=True restores the v2-base choice exactly. Co-Authored-By: Claude Opus 5.5 --- pairplex/cli/pairplex.py | 14 +++ pairplex/pairplex.py | 28 ++++- pairplex/tests/test_barcode_orientation.py | 139 +++++++++++++++++++++ pairplex/utils.py | 81 +++++++++++- 4 files changed, 257 insertions(+), 5 deletions(-) create mode 100644 pairplex/tests/test_barcode_orientation.py diff --git a/pairplex/cli/pairplex.py b/pairplex/cli/pairplex.py index 8a39506..a3def7e 100644 --- a/pairplex/cli/pairplex.py +++ b/pairplex/cli/pairplex.py @@ -98,6 +98,16 @@ def version(): default="human", help="Germline database for V(D)J assignment", ) +@click.option( + "--barcode_prefer_exact/--no-barcode_prefer_exact", + default=True, + help="Prefer the read orientation with an exact barcode whitelist hit over a Hamming-1 correction in the other", +) +@click.option( + "--legacy", + is_flag=True, + help="Reproduce v2-base behaviour exactly (overrides the options of newer fixes)", +) @click.option("--quiet", is_flag=True, help="Whether to suppress output") @click.option( "--debug", @@ -118,6 +128,8 @@ def run( merge_paired_reads: bool, receptor: str, germline_database: str, + barcode_prefer_exact: bool, + legacy: bool, quiet: bool, debug: bool, ): @@ -135,6 +147,8 @@ def run( merge_paired_reads=merge_paired_reads, receptor=receptor, germline_database=germline_database, + barcode_prefer_exact=barcode_prefer_exact, + legacy=legacy, quiet=quiet, debug=debug, ) diff --git a/pairplex/pairplex.py b/pairplex/pairplex.py index 85c4ed5..4237e70 100644 --- a/pairplex/pairplex.py +++ b/pairplex/pairplex.py @@ -49,6 +49,8 @@ def run( merge_paired_reads: bool = False, receptor: str = "bcr", germline_database: str = "human", + barcode_prefer_exact: bool = True, + legacy: bool = False, quiet: bool = False, debug: bool = False, ) -> list: @@ -105,6 +107,15 @@ def run( The germline database to use. Options are anything accepted by `abstar.run`, including custom databases if available. + barcode_prefer_exact : bool + Read orientation choice. If ``True`` (default), an exact barcode whitelist hit in either orientation + (forward or reverse complement) wins over a Hamming-1 correction in the other; corrections are used only + when neither orientation is exact. Ties keep the forward orientation. Orientation counts are written to + the run log (``stats.barcode_orientation``). v2-base took the first orientation whose barcode corrected. + + legacy : bool + If ``True``, reproduce v2-base exactly: forces ``barcode_prefer_exact=False``. + quiet : bool Whether to suppress the progress bar and other update messages. @@ -119,6 +130,8 @@ def run( """ parameters = dict(locals()) # captured first: exactly the arguments of this call + if legacy: + barcode_prefer_exact = False # v2-base: first orientation that corrects # print splash screen if not quiet: @@ -212,6 +225,7 @@ def run( running_total_printer = tqdm(total=0, bar_format="{desc}", position=2, leave=True) blank2_printer = tqdm(total=0, bar_format=" ", position=3, leave=True) all_valid_barcodes = 0 + orientation_counts = {} all_consensus_sequences = 0 all_pairs = 0 running_total_printer.set_description_str( @@ -273,14 +287,21 @@ def run( main_pbar.set_postfix_str("parsing barcodes", refresh=True) parquet_chunks = [] + parse_kwargs = {"whitelist_path": whitelist_path} + if barcode_prefer_exact: + parse_kwargs.update(prefer_exact=True, return_counts=True) + else: + parse_kwargs.update(prefer_exact=False) futures = [ - executor.submit( - parse_barcodes, chunk, temp_directory, whitelist_path=whitelist_path - ) + executor.submit(parse_barcodes, chunk, temp_directory, **parse_kwargs) for chunk in fastq_chunks ] for future in as_completed(futures): res = future.result() + if barcode_prefer_exact: + res, counts = res + for k, v in counts.items(): + orientation_counts[k] = orientation_counts.get(k, 0) + v if res is not None: parquet_chunks.append(res) to_delete.extend(parquet_chunks) @@ -487,6 +508,7 @@ def run( "valid_barcodes": all_valid_barcodes, "consensus_sequences": all_consensus_sequences, "pairs": all_pairs, + **({"barcode_orientation": orientation_counts} if barcode_prefer_exact else {}), } ) diff --git a/pairplex/tests/test_barcode_orientation.py b/pairplex/tests/test_barcode_orientation.py new file mode 100644 index 0000000..b1eb1a7 --- /dev/null +++ b/pairplex/tests/test_barcode_orientation.py @@ -0,0 +1,139 @@ +"""Orientation choice in parse_barcodes: an exact barcode hit beats a Hamming-1 correction.""" + +from unittest import mock + +import abutils +import polars as pl +import pytest + +from pairplex import pairplex as pp +from pairplex.utils import parse_barcodes + +A = "AAAACCCCGGGGTTTT" +B = "ACGTTGCAACGTTGCA" +C = "CATGCATGGTACGTAC" +TSO = "TTTCTTATATGGG" +UMI_F = "AAAAAAAAAA" +UMI_R = "CCCCCCCCCC" +INSERT = "ACTGACTGACTGATTACA" + + +def mut(bc, i=5): + """Hamming-1 variant of ``bc`` at position ``i``.""" + return bc[:i] + ("T" if bc[i] != "T" else "A") + bc[i + 1 :] + + +def rc(s): + return abutils.tl.reverse_complement(s) + + +def read(fwd_bc, rc_bc): + """A read whose forward start is ``fwd_bc`` and whose reverse-complement start is ``rc_bc``.""" + return fwd_bc + UMI_F + TSO + INSERT + rc(rc_bc + UMI_R + TSO) + + +@pytest.fixture +def whitelist(tmp_path): + path = tmp_path / "wl.txt" + path.write_text("\n".join([A, B, C]) + "\n") + return str(path) + + +def _parse(tmp_path, whitelist, reads, **kwargs): + tmp_path.mkdir(parents=True, exist_ok=True) + fq = tmp_path / "chunk.fastq" + fq.write_text("".join(f"@r{i}\n{s}\n+\n{'I' * len(s)}\n" for i, s in enumerate(reads))) + res = parse_barcodes(fq, tmp_path / "out", whitelist_path=whitelist, return_counts=True, **kwargs) + path, counts = res + df = pl.read_parquet(path) if path else pl.DataFrame({"seq_id": [], "barcode": [], "umi": []}) + return df, counts + + +def test_exact_rc_beats_corrected_fwd(tmp_path, whitelist): + df, counts = _parse(tmp_path, whitelist, [read(mut(A), B)]) + assert df["barcode"].to_list() == [B] + assert df["umi"].to_list() == [UMI_R] + assert counts["exact_rc"] == 1 and counts["switched"] == 1 + + +def test_legacy_takes_corrected_fwd(tmp_path, whitelist): + df, counts = _parse(tmp_path, whitelist, [read(mut(A), B)], prefer_exact=False) + assert df["barcode"].to_list() == [A] + assert df["umi"].to_list() == [UMI_F] + assert all(v == 0 for v in counts.values()) + + +def test_exact_fwd_beats_corrected_rc(tmp_path, whitelist): + df, counts = _parse(tmp_path, whitelist, [read(A, mut(B))]) + assert df["barcode"].to_list() == [A] + assert counts["exact_fwd"] == 1 and counts["both_exact"] == 0 + + +def test_both_exact_keeps_forward_and_counts(tmp_path, whitelist): + df, counts = _parse(tmp_path, whitelist, [read(A, B)]) + assert df["barcode"].to_list() == [A] + assert df["umi"].to_list() == [UMI_F] + assert counts["both_exact"] == 1 + + +def test_both_corrected_keeps_forward_and_counts(tmp_path, whitelist): + df, counts = _parse(tmp_path, whitelist, [read(mut(A), mut(B))]) + assert df["barcode"].to_list() == [A] + assert counts["corrected_fwd"] == 1 and counts["both_corrected"] == 1 + + +def test_only_rc_correctable(tmp_path, whitelist): + df, counts = _parse(tmp_path, whitelist, [read("GGGGGGGGGGGGGGGG", mut(C))]) + assert df["barcode"].to_list() == [C] + assert df["umi"].to_list() == [UMI_R] + assert counts["corrected_rc"] == 1 + + +def test_no_hit_dropped(tmp_path, whitelist): + df, counts = _parse(tmp_path, whitelist, [read("GGGGGGGGGGGGGGGG", "GGGGGGGGGGGGGGGG")]) + assert df.height == 0 + assert sum(counts.values()) == 0 + + +def test_check_rc_false_ignores_rc(tmp_path, whitelist): + df, counts = _parse(tmp_path, whitelist, [read(mut(A), B)], check_rc=False) + assert df["barcode"].to_list() == [A] + assert counts["corrected_fwd"] == 1 and counts["switched"] == 0 + + +@pytest.mark.parametrize("prefer_exact", [True, False]) +def test_same_records_when_no_conflict(tmp_path, whitelist, prefer_exact): + """Reads without an orientation conflict parse identically in both modes (incl. sequence).""" + reads = [read(A, "G" * 16), read(mut(B), "G" * 16), read("G" * 16, C), read("G" * 16, mut(C))] + df, _ = _parse(tmp_path, whitelist, reads, prefer_exact=prefer_exact) + legacy, _ = _parse(tmp_path / "legacy", whitelist, reads, prefer_exact=False) + assert df.sort("seq_id").equals(legacy.sort("seq_id")) + + +def test_default_return_is_path_only(tmp_path, whitelist): + fq = tmp_path / "c.fastq" + s = read(A, B) + fq.write_text(f"@r\n{s}\n+\n{'I' * len(s)}\n") + out = parse_barcodes(fq, tmp_path / "o", whitelist_path=whitelist) + assert isinstance(out, str) + + +def _run(tmp_path, monkeypatch, whitelist, **kwargs): + monkeypatch.setenv("PAIRPLEX_MAX_WORKERS", "2") + tmp_path.mkdir(parents=True, exist_ok=True) + fq = tmp_path / "well001.fastq" + reads = [read(mut(A), B)] * 3 + [read(A, B)] * 3 + fq.write_text("".join(f"@r{i}\n{s}\n+\n{'I' * len(s)}\n" for i, s in enumerate(reads))) + out = tmp_path / "out" + with mock.patch.object(pp.abstar, "run", return_value=[]): + pp.run(str(fq), out, whitelist_path=whitelist, quiet=True, debug=True, **kwargs) + return pl.read_parquet(out / "parsed" / "well001.parquet") + + +def test_run_default_and_legacy(tmp_path, monkeypatch, whitelist): + new = _run(tmp_path / "new", monkeypatch, whitelist) + assert sorted(new["barcode"].to_list()) == sorted([B] * 3 + [A] * 3) + old = _run(tmp_path / "old", monkeypatch, whitelist, legacy=True) + assert old["barcode"].to_list() == [A] * 6 + old2 = _run(tmp_path / "old2", monkeypatch, whitelist, barcode_prefer_exact=False) + assert old2["barcode"].to_list() == [A] * 6 diff --git a/pairplex/utils.py b/pairplex/utils.py index 0c599a3..74baa5b 100644 --- a/pairplex/utils.py +++ b/pairplex/utils.py @@ -192,7 +192,9 @@ def parse_barcodes( output_directory: str | Path, whitelist_path: str | Path | None = None, check_rc: bool = True, -) -> str | None: + prefer_exact: bool = True, + return_counts: bool = False, +) -> str | None | tuple[str | None, dict]: """ Process a chunk of fastq file to extract barcodes, UMIs and TSO sequences. @@ -216,6 +218,20 @@ def parse_barcodes( check_rc : bool = True Whether to check the reverse complement of the sequences. + prefer_exact : bool = True + Orientation choice when ``check_rc`` is set. If ``True``, an exact whitelist hit in either orientation + wins over a Hamming-1 correction in the other; only if neither orientation is exact is the first + correctable orientation (forward, then reverse complement) used. If both orientations are exact, or + both need correction, the forward orientation is kept (and counted). If ``False``, the first + orientation whose barcode corrects is used even when the other one is exact (v2-base behaviour). + + return_counts : bool = False + If ``True``, return ``(path, counts)``, where ``counts`` holds orientation-choice counters: + ``exact_fwd``, ``exact_rc``, ``corrected_fwd``, ``corrected_rc``, ``both_exact`` (both orientations + exact, forward kept), ``both_corrected`` (both need correction, forward kept) and ``switched`` (reads + where an exact reverse-complement hit replaced the corrected forward barcode v2-base used). + Only filled when ``prefer_exact`` is ``True``. + enforce_bc_whitelist : bool = True Whether to enforce the barcode whitelist. @@ -234,7 +250,24 @@ def parse_barcodes( whitelist = load_barcode_whitelist(whitelist_path) records = [] + counts = dict.fromkeys( + ["exact_fwd", "exact_rc", "corrected_fwd", "corrected_rc", "both_exact", "both_corrected", "switched"], 0 + ) for seq in abutils.io.parse_fastx(str(input_file)): + if prefer_exact: + # same fixed offsets as below; see the NOTE there + chosen = _choose_orientation(seq.sequence, whitelist, check_rc, counts) + if chosen is not None: + s, corrected = chosen + records.append( + { + "umi": s[16:26], + "barcode": corrected, + "seq_id": seq.id, + "sequence": s[36:].lstrip("G"), # remove any remaining Gs from the TSO + } + ) + continue seqs = [seq.sequence] if check_rc: seqs.append(abutils.tl.reverse_complement(seq.sequence)) @@ -264,13 +297,57 @@ def parse_barcodes( # all done! break + output_file = None if records: output_directory = Path(output_directory) output_directory.mkdir(parents=True, exist_ok=True) output_file = output_directory / f"bcdf_{output_name}.parquet" df = pl.DataFrame(records) df.write_parquet(output_file) - return str(output_file) + output_file = str(output_file) + if return_counts: + return output_file, counts + return output_file + + +def _choose_orientation( + sequence: str, whitelist: set[str], check_rc: bool, counts: dict +) -> tuple[str, str] | None: + """ + Pick the read orientation for ``parse_barcodes(prefer_exact=True)``. + + Returns ``(oriented_sequence, barcode)`` or ``None``. An exact whitelist hit in either orientation beats a + Hamming-1 correction; ties (both exact, or both corrected) keep the forward orientation. Updates ``counts``. + """ + fwd_bc = sequence[:16] + fwd_exact = fwd_bc in whitelist + rc = rc_bc = None + rc_exact = False + if check_rc: + rc = abutils.tl.reverse_complement(sequence) + rc_bc = rc[:16] + rc_exact = rc_bc in whitelist + if fwd_exact: + counts["exact_fwd"] += 1 + if rc_exact: + counts["both_exact"] += 1 + return sequence, fwd_bc + if rc_exact: + counts["exact_rc"] += 1 + if correct_barcode(fwd_bc, whitelist) is not None: + counts["switched"] += 1 + return rc, rc_bc + fwd_corrected = correct_barcode(fwd_bc, whitelist) + rc_corrected = correct_barcode(rc_bc, whitelist) if check_rc else None + if fwd_corrected is not None: + counts["corrected_fwd"] += 1 + if rc_corrected is not None: + counts["both_corrected"] += 1 + return sequence, fwd_corrected + if rc_corrected is not None: + counts["corrected_rc"] += 1 + return rc, rc_corrected + return None def greedy_clustering(