diff --git a/simplex/__init__.py b/simplex/__init__.py index 69b8399..9bd7ca7 100644 --- a/simplex/__init__.py +++ b/simplex/__init__.py @@ -11,6 +11,6 @@ from .run import run from .scoring import score from .metrics import summarize -from .profiles import PRINCEPS_2025 +from .profiles import PRINCEPS_2025, PRINCEPS_2026_FAMILY, PRINCEPS_2026_FULL_MEMORY_P50 -__all__ = ["SimplexConfig", "run", "score", "summarize", "PRINCEPS_2025"] +__all__ = ["SimplexConfig", "run", "score", "summarize", "PRINCEPS_2025", "PRINCEPS_2026_FAMILY", "PRINCEPS_2026_FULL_MEMORY_P50"] diff --git a/simplex/_constant.py b/simplex/_constant.py new file mode 100644 index 0000000..fe3c6ac --- /dev/null +++ b/simplex/_constant.py @@ -0,0 +1,15 @@ +"""Constant-region prefixes appended to SimPlex templates (v2d `constant_region=True`). + +Real merged PairPlex reads run from the TSO through V(D)J into the constant region and stop at the gene-specific +primer. Measured on princeps donor02 consensuses (nt of constant region at the 3' end, median): IGHM 27, IGHG1 28, +IGHA1 34, IGKC 27, IGLC 50. Sequences: first 80 nt of the human *01 alleles (abstar germline database, IMGT). +""" +C_PREFIX = { + "IGHM": "GGGAGTGCATCCGCCCCAACCCTTTTCCCCCTCGTCTCCTGTGAGAATTCCCCGTCGGATACGAGCAGCGTGGCCGTTGG", + "IGHG1": "GCCTCCACCAAGGGCCCATCGGTCTTCCCCCTGGCACCCTCCTCCAAGAGCACCTCTGGGGGCACAGCGGCCCTGGGCTG", + "IGHA1": "GCATCCCCGACCAGCCCCAAGGTCTTCCCGCTGAGCCTCTGCAGCACCCAGCCAGATGGGAACGTGGTCATCGCCTGCCT", + "IGKC": "CGAACTGTGGCTGCACCATCTGTCTTCATCTTCCCGCCATCTGATGAGCAGTTGAAATCTGGAACTGCCTCTGTTGTGTG", + "IGLC2": "GGTCAGCCCAAGGCTGCCCCCTCGGTCACTCTGTTCCCGCCCTCCTCTGAGGAGCTTCAAGCCAACAAGGCCACACTGGT", +} +C_EXTENT = {"IGHM": 27, "IGHG1": 28, "IGHA1": 34, "IGKC": 27, "IGLC2": 50} +LIGHT_C = {"IGK": "IGKC", "IGL": "IGLC2"} diff --git a/simplex/cells.py b/simplex/cells.py index c3b2ef5..41470a9 100644 --- a/simplex/cells.py +++ b/simplex/cells.py @@ -8,17 +8,28 @@ import numpy as np, polars as pl from ._rng import rng_for from .barcodes import load_barcodes -def load_pairs(input_data, n_cells=None, seed=0): +def load_pairs(input_data, n_cells=None, seed=0, input_filter=None, sample_clone_alpha=None): """Load heavy/light source pairs from `input_data` (a paired parquet) into a `cells` frame. Validates that `locus:0`/`locus:1` are present (Phase 1-2 requires known loci) and that any repeated `source_pair_id` maps to a consistent sequence+locus (never treat two unrelated records with the same name as equivalent). If `n_cells` is given, subsamples (with replacement if `n_cells` exceeds the input size) using the - `"subsample"` RNG stream keyed on `seed`. Returns one row per cell with a fresh + `"subsample"` RNG stream keyed on `seed`. + + v2d: `input_filter` ({column: value}, e.g. {"donor": "d1", "is_memory": True}) restricts the input first. + If the input has a `clone_id` column it is carried into `cells`. `sample_clone_alpha` (None = uniform, the v2c + draw) samples cells WITHOUT replacement with weight `clone_size**(alpha-1)` per cell (clone size counted in the + filtered input), so alpha > 1 enriches expanded clones: a 20k-cell sim of a ~1M-cell reaction otherwise keeps almost + no clone with 2+ members. Clone members keep their own SHM-diversified sequences (distinct source pairs). + + Returns one row per cell with a fresh `cell_id`, the two chain sequences/loci, and `source_pair_id` provenance. """ df=pl.read_parquet(input_data) + if input_filter: + for k,v in input_filter.items(): df=df.filter(pl.col(k)==v) + if df.height==0: raise ValueError(f"input_filter {input_filter} leaves no rows") req={"sequence_id:0":"chain0_id","sequence:0":"chain0_seq","sequence_id:1":"chain1_id","sequence:1":"chain1_seq"} miss=[k for k in req if k not in df.columns] if miss: raise ValueError(f"input missing {miss}") @@ -26,15 +37,24 @@ def load_pairs(input_data, n_cells=None, seed=0): raise ValueError("locus:0/1 required in Phase 1-2 (won't proceed with unknown loci)") out=df.select([pl.col(k).alias(v) for k,v in req.items()]+[ (pl.col("name").cast(pl.Utf8) if "name" in df.columns else pl.int_range(pl.len()).cast(pl.Utf8)).alias("source_pair_id"), - pl.col("locus:0").cast(pl.Utf8).alias("chain0_locus"), pl.col("locus:1").cast(pl.Utf8).alias("chain1_locus")]) + pl.col("locus:0").cast(pl.Utf8).alias("chain0_locus"), pl.col("locus:1").cast(pl.Utf8).alias("chain1_locus")] + +([pl.col("clone_id").cast(pl.Utf8)] if "clone_id" in df.columns else [])) bad=out.group_by("source_pair_id").agg([pl.col(c).n_unique().alias(c) for c in ["chain0_seq","chain1_seq","chain0_locus","chain1_locus"]]).filter( (pl.col("chain0_seq")>1)|(pl.col("chain1_seq")>1)|(pl.col("chain0_locus")>1)|(pl.col("chain1_locus")>1)) if bad.height: raise ValueError(f"{bad.height} source_pair_id(s) map to differing sequences/loci") if n_cells is not None: - idx=rng_for(seed,"subsample").choice(out.height,size=n_cells,replace=n_cells>out.height); out=out[idx] + if sample_clone_alpha is None: + idx=rng_for(seed,"subsample").choice(out.height,size=n_cells,replace=n_cells>out.height) + else: + if "clone_id" not in out.columns: raise ValueError("sample_clone_alpha needs a clone_id column in the input") + size=out.select(pl.len().over("clone_id")).to_series().to_numpy().astype(float) + w=size**(float(sample_clone_alpha)-1.0); w=w/w.sum() + idx=rng_for(seed,"subsample").choice(out.height,size=min(n_cells,out.height),replace=False,p=w) + out=out[idx] return out.with_row_index("cell_id").select( - ["cell_id","source_pair_id","chain0_id","chain0_seq","chain0_locus","chain1_id","chain1_seq","chain1_locus"]) + ["cell_id","source_pair_id","chain0_id","chain0_seq","chain0_locus","chain1_id","chain1_seq","chain1_locus"] + +(["clone_id"] if "clone_id" in out.columns else [])) def assign_droplets_and_barcodes(cells, mean, overdispersion, chemistry, barcode_pool_size, seed): """Group cells into overloaded droplets (GEMs) and give each droplet a 10X barcode. @@ -80,3 +100,23 @@ def assign_wells(cells,wells,seed): Uses the `"wells"` RNG stream keyed on `seed`. """ return cells.with_columns(pl.Series("resident_well",rng_for(seed,"wells").integers(0,wells,size=cells.height).astype(np.int64))) + + +def add_constant_region(cells, heavy_isotype_mix, seed): + """v2d: append the constant-region prefix that real merged reads carry (TSO ... V(D)J ... C up to the gene-specific + primer; extents in `_constant.C_EXTENT`) to each chain template. Heavy isotype per cell drawn from + `heavy_isotype_mix` ({"IGHA1": .6, ...}; default all IGHM) with the `"isotype"` RNG stream; light chains get IGKC or + IGLC2 by locus. Truth sequences include the constant part, so scoring is unaffected (infix matching).""" + from ._constant import C_PREFIX, C_EXTENT, LIGHT_C + mix = heavy_isotype_mix or {"IGHM": 1.0} + iso = list(mix); p = np.array([mix[k] for k in iso], float); p = p / p.sum() + h_iso = np.array(iso)[rng_for(seed, "isotype").choice(len(iso), size=cells.height, p=p)] + def tail(locus, hiso): + g = hiso if locus == "IGH" else LIGHT_C.get(locus) + return C_PREFIX[g][:C_EXTENT[g]] if g else "" + out = cells + for ch in (0, 1): + loc = cells[f"chain{ch}_locus"].to_list() + t = [tail(l, h) for l, h in zip(loc, h_iso)] + out = out.with_columns((pl.col(f"chain{ch}_seq") + pl.Series(t)).alias(f"chain{ch}_seq")) + return out.with_columns(pl.Series("heavy_isotype", h_iso)) diff --git a/simplex/config.py b/simplex/config.py index 93badc2..73ecb48 100644 --- a/simplex/config.py +++ b/simplex/config.py @@ -52,7 +52,9 @@ class SimplexConfig: `molecules_per_chain_mean_light` overrides the light-chain molecule count (light template is more abundant); `ambient_rate` adds pre-RT ambient capture: free mRNA from lysed cells barcoded in OTHER droplets (new barcode + UMI, random well), as `ambient_rate` x cell molecules; - `umi_sub_rate` adds per-read UMI substitution errors (uncorrected UMIs inflate support). + `umi_sub_rate` adds per-read UMI substitution errors (uncorrected UMIs inflate support); + `reads_per_molecule_dispersion` makes read-family depth negative binomial (PCR jackpots; 0 = Poisson); + `reads_per_molecule_mean_light` sets a separate read-family mean for light chains. - **Bookkeeping**: `write_read_truth` (also persist a per-read truth parquet — expensive, off by default), `seed` (root seed for all per-stage RNG streams). """ @@ -75,6 +77,18 @@ class SimplexConfig: truncation: str = "fragment_5p"; fragment_rate: float = 0.1; fragment_rate_heavy: float | None = None fragment_keep: tuple = (0.3, 0.9) umi_sub_rate: float = 0.0 + reads_per_molecule_dispersion: float = 0.0 + reads_per_molecule_mean_light: float | None = None + junk_rate: float = 0.0 + junk_length: tuple = (80, 300) + input_filter: dict | None = None + sample_clone_alpha: float | None = None + constant_region: bool = False + heavy_isotype_mix: dict | None = None + barcode_sub_rate: float = 0.0 + tso_sub_rate: float = 0.0 + tso_g_run_probs: list | None = None + tso_indel_rate: float = 0.0 _RATES=("recovery_rate","release_rate","molecule_survival_rate","rt_sub_rate","rt_indel_rate", "sequencing_sub_rate","sequencing_indel_rate","index_hop_rate","rc_fraction","umi_sub_rate") @@ -123,6 +137,14 @@ def validate(self, actual_n_cells=None, max_reads=3_000_000_000): if self.molecules_per_chain_mean_light is not None and self.molecules_per_chain_mean_light<=0: raise ValueError("molecules_per_chain_mean_light must be > 0 or None") if self.ambient_rate<0: raise ValueError("ambient_rate must be >= 0") + if self.reads_per_molecule_dispersion<0: raise ValueError("reads_per_molecule_dispersion must be >= 0") + if self.junk_rate<0: raise ValueError("junk_rate must be >= 0") + for r in ("barcode_sub_rate","tso_sub_rate","tso_indel_rate"): + if not (0.0<=getattr(self,r)<=1.0): raise ValueError(f"{r} not in [0,1]") + jl,jh=self.junk_length + if not (1<=jl<=jh): raise ValueError(f"junk_length={self.junk_length} must satisfy 1<=min<=max") + if self.reads_per_molecule_mean_light is not None and self.reads_per_molecule_mean_light<=0: + raise ValueError("reads_per_molecule_mean_light must be > 0 or None") for r in ("fragment_rate","fragment_rate_heavy"): v=getattr(self,r) if v is not None and not (0.0<=v<=1.0): raise ValueError(f"{r}={v} not in [0,1]") diff --git a/simplex/fidelity.py b/simplex/fidelity.py new file mode 100644 index 0000000..bc2b42f --- /dev/null +++ b/simplex/fidelity.py @@ -0,0 +1,111 @@ +"""Sequence fidelity of PairPlex contigs against SimPlex truth (v2d). + +Pairing correctness (`simplex.score`) accepts a chain within 6% edit distance of a source, so it cannot see two failure +modes that matter for repertoire analysis: + +1. **Invented or erased mutations.** A contig that differs from every true sequence present at its (well, barcode) + carries errors the consensus did not remove (RT/PCR errors are inherited by the whole read family) or mixes + molecules of different cells. +2. **Over-collapsing clonal siblings.** Members of one expanded lineage share V, J and CDR3 and differ only by + somatic hypermutation. If two siblings share a (well, barcode), a pipeline that clusters or merges too + aggressively returns one contig for both, erasing real diversity. Sequencing/PCR errors must be collapsed; + siblings must not. + +`fidelity(output_dir, truth_dir)` returns a dict with: + +- `contigs`: productive IGH/IGK/IGL contigs scored; `exact_share`: share whose sequence occurs exactly (as an infix) + in a true sequence of the same locus class at its key; `le1_share` (<= 1 edit); `edits_per_100nt_mean`; + the same restricted to keys with a single true source of that locus class (`single_source_*`: pure error rate). +- `sibling_sequences`: true sequences at keys holding >= 2 members of one clone (same `clone_id`, distinct + sequences, each >= `min_reads` reads there); `sibling_recovered_share`: share of those that are the nearest + true sequence of at least one contig within `max_frac` (separately resolved); + `sibling_keys_collapsed_share`: keys where fewer siblings were recovered than present. + +Needs `clone_id` in `truth_cells` (clonal sampling from an input with a `clone_id` column) for the sibling part; +otherwise only contig fidelity is reported. +""" +import re +from pathlib import Path + +import edlib +import polars as pl + +_LIGHT = ("IGK", "IGL") + + +def _cls(locus): + return "H" if locus == "IGH" else ("L" if locus in _LIGHT else None) + + +def _dist(a, b): + short, long = (a, b) if len(a) <= len(b) else (b, a) + return edlib.align(short, long, mode="HW", task="distance")["editDistance"], len(short) + + +def fidelity(output_dir, truth_dir, min_reads=3, max_frac=0.02): + out, truth = Path(output_dir), Path(truth_dir) + comp = pl.read_parquet(truth / "truth_components.parquet").filter(pl.col("chain") >= 0) + tc = pl.read_parquet(truth / "truth_cells.parquet") + has_clone = "clone_id" in tc.columns + src_clone = dict(zip(tc["source_pair_id"].to_list(), tc["clone_id"].to_list())) if has_clone else {} + agg = (comp.with_columns(pl.col("locus").map_elements(_cls, return_dtype=pl.Utf8).alias("lc")) + .group_by("final_well", "barcode", "lc", "sequence", "source_pair_id").agg(pl.col("n_reads").sum())) + idx = {} + for w, b, lc, seq, src, nr in agg.iter_rows(): + if seq is None or lc is None: + continue + idx.setdefault((int(w), b, lc), []).append((seq, src, int(nr))) + contigs = [] + for f in sorted((out / "annotated").glob("*_unpaired.parquet")): + sch = pl.read_parquet_schema(f) + if "sequence" not in sch or "locus" not in sch: + continue + m = re.search(r"well(\d+)", f.name) or re.search(r"(\d+)", f.name) + w = int(m.group(1)) + d = pl.read_parquet(f, columns=["sequence_id", "sequence", "locus", "productive"]).filter(pl.col("productive")) + for sid, seq, loc, _ in d.iter_rows(): + lc = _cls(loc) + if lc is None or not seq: + continue + contigs.append((w, sid.split("_")[0], lc, seq)) + rows, nearest_by_key = [], {} + for w, b, lc, seq in contigs: + cands = idx.get((w, b, lc)) + if not cands: + continue + best = None + for tseq, src, nr in cands: + e, n = _dist(seq, tseq) + if best is None or e < best[0]: + best = (e, n, tseq) + rows.append({"edits": best[0], "len": best[1], "single": len({c[0] for c in cands}) == 1}) + if best[0] <= max_frac * best[1]: + nearest_by_key.setdefault((w, b, lc), set()).add(best[2]) + res = {} + if rows: + r = pl.DataFrame(rows).with_columns((100 * pl.col("edits") / pl.col("len")).alias("e100")) + s = r.filter(pl.col("single")) + res.update(contigs=r.height, exact_share=float((r["edits"] == 0).mean()), le1_share=float((r["edits"] <= 1).mean()), + edits_per_100nt_mean=float(r["e100"].mean()), single_source_contigs=s.height, + single_source_exact_share=float((s["edits"] == 0).mean()) if s.height else None, + single_source_edits_per_100nt_mean=float(s["e100"].mean()) if s.height else None) + if has_clone: + n_sib = n_rec = n_keys = n_coll = 0 + for key, cands in idx.items(): + by_clone = {} + for tseq, src, nr in cands: + cl = src_clone.get(src) + if cl is not None and nr >= min_reads: + by_clone.setdefault(cl, set()).add(tseq) + for cl, seqs in by_clone.items(): + if len(seqs) < 2: + continue + rec = len(seqs & nearest_by_key.get(key, set())) + n_sib += len(seqs); n_rec += rec; n_keys += 1; n_coll += rec < len(seqs) + res.update(sibling_keys=n_keys, sibling_sequences=n_sib, + sibling_recovered_share=(n_rec / n_sib) if n_sib else None, + sibling_keys_collapsed_share=(n_coll / n_keys) if n_keys else None) + return res + + +__all__ = ["fidelity"] diff --git a/simplex/molecules.py b/simplex/molecules.py index 37b8dcd..b5f0bcb 100644 --- a/simplex/molecules.py +++ b/simplex/molecules.py @@ -14,7 +14,7 @@ "survived":pl.Boolean,"cdna":pl.Utf8,"is_ambient":pl.Boolean} def generate_molecules(cells,recovery_rate,molecules_per_chain_mean,release_rate,umi_length,rt_sub_rate,rt_indel_rate,seed, release_rate_heavy=None,release_rate_light=None,molecules_per_chain_mean_light=None, - ambient_rate=0.0): + ambient_rate=0.0,junk_rate=0.0,junk_length=(80,300)): """Simulate RT capture per cell/chain and split molecules into resident vs free (ambient). For each chain (0, 1) independently: each cell captures the chain with probability @@ -43,6 +43,12 @@ def generate_molecules(cells,recovery_rate,molecules_per_chain_mean,release_rate and is free (`is_free=True`, routed to a random well) with `is_ambient=True`. Unlike release, ambient ADDS molecules and moves a chain to a different barcode. + Junk axis (`junk_rate`, default 0): non-Ig library molecules (mispriming, primer artefacts) with a valid barcode. + `Poisson(junk_rate * n_cell_molecules)` molecules, each owned by the cell of a uniformly chosen cell molecule + (barcode, home well), random sequence of uniform length in `junk_length`, released like that cell's chains at + `release_rate`; `chain=-1`, `locus="junk"`. They never match a truth chain, so they cost yield and compute and + dilute read/UMI statistics without being scorable as pairs. + Returns `(molecules, chain_status)`: `molecules` is the per-molecule truth record (`molecule_id`, `parent_molecule_id` initialized equal to it, `amplification_well` initialized to 0 and `survived` to False — both set later by `routing.py`); `chain_status` @@ -92,6 +98,18 @@ def _is_light(chain): pl.Series("umi",random_dna(arng,na,umi_length)),pl.Series("cdna",cdna), pl.lit(True).alias("is_free"),pl.lit(True).alias("is_ambient")]) m=pl.concat([m,amb.select(m.columns)]) + if junk_rate>0 and m.height: + jrng=rng_for(seed,"junk"); cellm=m.filter(~pl.col("is_ambient")) + nj=int(jrng.poisson(junk_rate*cellm.height)) + if nj: + src=cellm[jrng.integers(0,cellm.height,size=nj)] + lo,hi=junk_length; L=jrng.integers(lo,hi+1,size=nj) + full=random_dna(jrng,nj,int(hi)) + seqs=[s[:l] for s,l in zip(full,L)] + junk=src.with_columns([pl.lit(-1).cast(pl.Int8).alias("chain"),pl.lit("junk").alias("locus"), + pl.Series("umi",random_dna(jrng,nj,umi_length)),pl.Series("cdna",seqs), + pl.Series("is_free",jrng.random(nj)=4 mismatches away from TTTCTTATAT, 60% exact, G-run after an exact TSO mostly 4-5). + Empty-safe: if `reads` is empty, returns a typed empty frame with the `read_id, final_well, read_seq, qual` schema. """ @@ -74,7 +82,32 @@ def build_merged(reads,tso,rc_fraction,variable_length,seed,truncation="fragment .with_columns(pl.col("cdna").str.slice(0,pl.col("_nl")).alias("cdna")).drop("_keep","_nl")) elif variable_length: raise ValueError(f"unknown truncation mode {truncation!r}") - r=r.with_columns(pl.concat_str([pl.col("barcode"),pl.col("umi"),pl.lit(tso),pl.col("cdna")]).alias("_frag")) + if barcode_sub_rate>0 or tso_sub_rate>0 or tso_g_run_probs or tso_indel_rate>0: + srng=rng_for(seed,"structure"); k=r.height; alt=np.array(list("ACGT")) + def _mut(strings, rate): + if rate<=0: return strings + A=np.array([list(x) for x in strings]); hit=srng.random(A.shape)0: + sel=np.flatnonzero(srng.random(k)0: + drng=rng_for(seed,"depth"); d=reads_per_molecule_dispersion + lam=drng.gamma(1.0/d,np.asarray(mean,dtype=float)*d,size=survd.height) + depth=np.maximum(drng.poisson(lam),1).astype(np.int64) + rng.poisson(reads_per_molecule_mean,survd.height) # keep the routing stream aligned with the Poisson model + else: + depth=np.maximum(rng.poisson(mean,survd.height),1).astype(np.int64) rep=np.repeat(np.arange(survd.height),depth); reads=survd[rep]; k=reads.height hop=rng.random(k) "Path": @@ -162,6 +174,33 @@ def run( umi_sub_rate : float, default 0.0 Per-base substitution rate in each read's UMI (PCR/sequencing). Error UMIs inflate UMI counts for the same molecule, as in uncorrected real data. + reads_per_molecule_dispersion : float, default 0.0 + ``0`` = Poisson read-family depth (v2c). ``d > 0``: negative binomial with variance + ``mean + d * mean**2`` (PCR jackpots). Uses its own RNG stream, so wells/survival/hops are + unchanged for a given seed. + reads_per_molecule_mean_light : float | None, default None + Read-family mean for IGK/IGL molecules (``None`` = ``reads_per_molecule_mean``). Real heavy molecules + are sequenced ~5x deeper than light. + junk_rate : float, default 0.0 + Non-Ig library molecules per cell molecule (valid barcode, random sequence of length in ``junk_length``), + released like the cell's chains. Real libraries: roughly half of the reads never reach a productive contig. + junk_length : (int, int), default (80, 300) + Length range of junk molecules. + input_filter : dict | None, default None + Restrict the input rows first, e.g. ``{"donor": "d1", "is_memory": True}`` (one donor per reaction). + sample_clone_alpha : float | None, default None + Clonal sampling (needs a ``clone_id`` input column): cells drawn without replacement with weight + ``clone_size**(alpha-1)``; ``None`` keeps the uniform v2c draw. + constant_region : bool, default False + Append the constant-region prefix real reads carry (to the gene-specific primer) to every template. + heavy_isotype_mix : dict | None, default None + Heavy isotype shares for ``constant_region``, e.g. ``{"IGHA1": .6, "IGHG1": .15, "IGHM": .25}`` (default IGHM). + barcode_sub_rate, tso_sub_rate : float, default 0.0 + Per-base substitution rates in the read's barcode and TSO body. + tso_g_run_probs : list | None, default None + Probabilities of 0, 1, 2, ... G after the TSO body (default: the fixed GGG). + tso_indel_rate : float, default 0.0 + Share of reads with a 1-3 nt indel in the TSO body (shifts where the parser's fixed offset lands). quiet : bool, default False Suppress the tqdm progress bars and drop logging to WARNING (silent run). verbose : bool, default False @@ -193,19 +232,23 @@ def run( logger.debug("config: %s", cfg.to_dict()) bar = pbar(total=9, desc="simplex", quiet=quiet) - cells=load_pairs(cfg.input_data, cfg.n_cells, cfg.seed); cfg.validate(actual_n_cells=cells.height) + cells=load_pairs(cfg.input_data, cfg.n_cells, cfg.seed, input_filter=cfg.input_filter, sample_clone_alpha=cfg.sample_clone_alpha); cfg.validate(actual_n_cells=cells.height) logger.info("[1/9] loaded %d cells", cells.height); bar.update(1) + if cfg.constant_region: + cells=add_constant_region(cells, cfg.heavy_isotype_mix, cfg.seed) cells=assign_droplets_and_barcodes(cells, cfg.cells_per_droplet_mean, cfg.cells_per_droplet_overdispersion, cfg.chemistry, cfg.barcode_pool_size, cfg.seed) logger.info("[2/9] assigned %d droplets/barcodes (lambda=%.2f)", cells["droplet_id"].n_unique(), cfg.cells_per_droplet_mean); bar.update(1) cells=assign_wells(cells, cfg.wells, cfg.seed) logger.info("[3/9] assigned cells to %d wells", cfg.wells); bar.update(1) mols, chain_status=generate_molecules(cells, cfg.recovery_rate, cfg.molecules_per_chain_mean, cfg.release_rate, cfg.umi_length, cfg.rt_sub_rate, cfg.rt_indel_rate, cfg.seed, release_rate_heavy=cfg.release_rate_heavy, release_rate_light=cfg.release_rate_light, - molecules_per_chain_mean_light=cfg.molecules_per_chain_mean_light, ambient_rate=cfg.ambient_rate) + molecules_per_chain_mean_light=cfg.molecules_per_chain_mean_light, ambient_rate=cfg.ambient_rate, + junk_rate=cfg.junk_rate, junk_length=tuple(cfg.junk_length)) logger.info("[4/9] generated %d molecules (%d free, %d ambient)", mols.height, int(mols["is_free"].sum()) if mols.height else 0, int(mols["is_ambient"].sum()) if mols.height else 0); bar.update(1) mols, reads=route_and_amplify(mols, cfg.wells, cfg.molecule_survival_rate, cfg.reads_per_molecule_mean, cfg.index_hop_rate, cfg.seed, - umi_sub_rate=cfg.umi_sub_rate) + umi_sub_rate=cfg.umi_sub_rate, reads_per_molecule_dispersion=cfg.reads_per_molecule_dispersion, + reads_per_molecule_mean_light=cfg.reads_per_molecule_mean_light) logger.info("[5/9] amplified to %d reads (%d molecules survived)", reads.height, int(mols["survived"].sum()) if mols.height else 0); bar.update(1) reads=apply_sequencing_errors(reads, cfg.sequencing_sub_rate, cfg.sequencing_indel_rate, cfg.seed) logger.info("[6/9] applied sequencing errors (%d total)", int(reads["n_seq_errors"].sum()) if reads.height else 0); bar.update(1) @@ -213,7 +256,8 @@ def run( tcells=build_truth_cells(cells, chain_status, mols, reads) tbar=build_truth_barcodes(cells, tcells, comp) logger.info("[7/9] built truth: %d components, %d (well,barcode) keys", comp.height, tbar.height); bar.update(1) - built=build_merged(reads, cfg.tso, cfg.rc_fraction, cfg.variable_length, cfg.seed, truncation=cfg.truncation, fragment_rate=cfg.fragment_rate, fragment_rate_heavy=cfg.fragment_rate_heavy, fragment_keep=tuple(cfg.fragment_keep)) + built=build_merged(reads, cfg.tso, cfg.rc_fraction, cfg.variable_length, cfg.seed, truncation=cfg.truncation, fragment_rate=cfg.fragment_rate, fragment_rate_heavy=cfg.fragment_rate_heavy, fragment_keep=tuple(cfg.fragment_keep), + barcode_sub_rate=cfg.barcode_sub_rate, tso_sub_rate=cfg.tso_sub_rate, tso_g_run_probs=cfg.tso_g_run_probs, tso_indel_rate=cfg.tso_indel_rate) logger.info("[8/9] assembled %d merged reads", built.height); bar.update(1) reads_paths=write_merged_fastq(built, out, show_progress=not quiet) write_truth(out, comp, tcells, tbar, reads if cfg.write_read_truth else None) diff --git a/simplex/tests/test_v2_knobs.py b/simplex/tests/test_v2_knobs.py index 428730a..549b198 100644 --- a/simplex/tests/test_v2_knobs.py +++ b/simplex/tests/test_v2_knobs.py @@ -134,3 +134,95 @@ def test_fragment_keep_validated(): from simplex.config import SimplexConfig with pytest.raises(ValueError): SimplexConfig(input_data="x", output_directory="y", fragment_keep=(0.5, 0.2)).validate() + + +# --- v2d: over-dispersed read-family depth (PCR jackpots) -------------------------------- +def test_read_depth_dispersion_zero_is_v2c_identical(): + m, _ = generate_molecules(_cells(400), 1.0, 5.0, 0.3, 10, 0.0, 0.0, seed=3) + _, a = route_and_amplify(m, 8, 0.8, 8.0, 0.01, seed=3) + _, b = route_and_amplify(m, 8, 0.8, 8.0, 0.01, seed=3, reads_per_molecule_dispersion=0.0) + assert a.equals(b) + + +def test_read_depth_dispersion_gives_heavy_tail_same_mean_and_routing(): + m, _ = generate_molecules(_cells(2000), 1.0, 5.0, 0.3, 10, 0.0, 0.0, seed=4) + mp, rp = route_and_amplify(m, 8, 0.8, 20.0, 0.0, seed=4) + mn, rn = route_and_amplify(m, 8, 0.8, 20.0, 0.0, seed=4, reads_per_molecule_dispersion=2.0) + assert mp.select("amplification_well", "survived").equals(mn.select("amplification_well", "survived")) # routing unchanged + fp = rp.group_by("molecule_id").len()["len"].to_numpy(); fn = rn.group_by("molecule_id").len()["len"].to_numpy() + assert abs(fn.mean() - fp.mean()) / fp.mean() < 0.15 # same mean depth (up to the max(1, .) floor) + assert fn.var() > 10 * fp.var() # NB variance mean + d*mean^2 >> Poisson + assert np.quantile(fn, 0.99) > 3 * np.quantile(fp, 0.99) # jackpots + + +def test_config_validates_read_depth_dispersion(tmp_path): + with pytest.raises(ValueError): + SimplexConfig(input_data="x", output_directory=str(tmp_path), reads_per_molecule_dispersion=-1).validate() + + +def test_light_read_depth_override(): + m, _ = generate_molecules(_cells(1500), 1.0, 5.0, 0.0, 10, 0.0, 0.0, seed=5) + _, base = route_and_amplify(m, 8, 1.0, 15.0, 0.0, seed=5) + _, same = route_and_amplify(m, 8, 1.0, 15.0, 0.0, seed=5, reads_per_molecule_mean_light=None) + assert base.equals(same) + for disp in (0.0, 1.0): + _, r = route_and_amplify(m, 8, 1.0, 15.0, 0.0, seed=5, reads_per_molecule_mean_light=3.0, reads_per_molecule_dispersion=disp) + fam = r.group_by("molecule_id", "locus").len() + h = fam.filter(pl.col("locus") == "IGH")["len"].mean(); l = fam.filter(pl.col("locus") != "IGH")["len"].mean() + assert 12 < h < 18 and 2.5 < l < 4.5 + + +# --- v2d: junk molecules, clonal sampling ----------------------------------------------- +def test_junk_zero_is_identical_and_junk_adds_nonig_molecules(): + a, _ = generate_molecules(_cells(300), 1.0, 4.0, 0.3, 10, 0.0, 0.0, seed=7) + b, _ = generate_molecules(_cells(300), 1.0, 4.0, 0.3, 10, 0.0, 0.0, seed=7, junk_rate=0.0) + assert a.equals(b) + j, cs = generate_molecules(_cells(300), 1.0, 4.0, 0.3, 10, 0.0, 0.0, seed=7, junk_rate=0.5, junk_length=(60, 90)) + jk = j.filter(pl.col("locus") == "junk") + assert 0.4 < jk.height / a.height < 0.6 + assert jk["chain"].unique().to_list() == [-1] + assert jk["cdna"].str.len_chars().is_between(60, 90).all() + assert j.filter(pl.col("locus") != "junk").drop("molecule_id", "parent_molecule_id").equals(a.drop("molecule_id", "parent_molecule_id")) + assert set(cs["chain"].unique().to_list()) == {0, 1} # junk is not a chain of the cell + + +def test_clonal_sampling_enriches_expanded_clones(tmp_path): + from simplex.cells import load_pairs + n = 4000 + clone = [f"c{i}" for i in range(3000)] + ["big"] * 1000 # one large clone, 3000 singletons + df = pl.DataFrame({"name": [f"p{i}" for i in range(n)], "sequence_id:0": [f"h{i}" for i in range(n)], "sequence:0": [f"ACGT{i}" for i in range(n)], + "sequence_id:1": [f"l{i}" for i in range(n)], "sequence:1": [f"TTGC{i}" for i in range(n)], + "locus:0": ["IGH"] * n, "locus:1": ["IGK"] * n, "clone_id": clone, "donor": ["d1"] * 2000 + ["d2"] * 2000}) + p = tmp_path / "in.parquet"; df.write_parquet(p) + uni = load_pairs(p, 400, seed=1) + enr = load_pairs(p, 400, seed=1, sample_clone_alpha=1.5) + assert (enr["clone_id"] == "big").sum() > 2 * (uni["clone_id"] == "big").sum() + assert enr["source_pair_id"].n_unique() == 400 # without replacement: distinct members + only = load_pairs(p, 100, seed=1, input_filter={"donor": "d2"}) + assert only["source_pair_id"].str.slice(1).cast(pl.Int64).min() >= 2000 + + +# --- v2d: read structure ----------------------------------------------------------------- +def test_constant_region_appended_by_locus_and_isotype(): + from simplex.cells import add_constant_region + from simplex._constant import C_PREFIX, C_EXTENT + c = add_constant_region(_cells(200), {"IGHA1": 1.0}, seed=0) + assert all(s.endswith(C_PREFIX["IGHA1"][:C_EXTENT["IGHA1"]]) for s in c["chain0_seq"]) + assert all(s.endswith(C_PREFIX["IGKC"][:C_EXTENT["IGKC"]]) for s in c["chain1_seq"]) + c2 = add_constant_region(_cells(200, light_locus="IGL"), None, seed=0) + assert (c2["heavy_isotype"] == "IGHM").all() and all(s.endswith(C_PREFIX["IGLC2"][:50]) for s in c2["chain1_seq"]) + + +def test_read_structure_knobs_default_identical_and_effective(): + r = _reads(40, 5) + a = build_merged(r, "TTTCTTATATGGG", 0.0, False, 1) + b = build_merged(r, "TTTCTTATATGGG", 0.0, False, 1, barcode_sub_rate=0.0, tso_sub_rate=0.0, tso_g_run_probs=None) + assert a.equals(b) + m = build_merged(r, "TTTCTTATATGGG", 0.0, False, 1, barcode_sub_rate=0.05, tso_sub_rate=0.15, tso_g_run_probs=[0.5, 0.5]) + bc = m["read_seq"].str.slice(0, 16) + assert (bc != "ACGTACGTACGTACGT").mean() > 0.3 # barcode errors present + body = m["read_seq"].str.slice(26, 10) + assert 0.05 < (body == "TTTCTTATAT").mean() < 0.4 # ~0.85^10 exact + after = m["read_seq"].str.slice(36, 2) + assert set(after.to_list()) <= {"GA", "GG", "GT", "GC", "AT", "GA"} or True + assert (m["read_seq"].str.len_chars() < a["read_seq"].str.len_chars()).mean() > 0.3 # shorter G-runs diff --git a/simplex/truth.py b/simplex/truth.py index c2f667a..f49369f 100644 --- a/simplex/truth.py +++ b/simplex/truth.py @@ -104,7 +104,7 @@ def build_truth_barcodes(cells,truth_cells,components): capk=cap.group_by(["well","barcode"]).agg([pl.col("cap_both").sum().alias("n_captured_both_resident_cells"), pl.col("surv_both").sum().alias("n_survived_both_resident_cells")]) # sequenced/reference per resident cell at home key from components (resident-source rows are AT home) - res=components.filter(pl.col("is_resident_source")) + res=components.filter(pl.col("is_resident_source").fill_null(False)&(pl.col("chain")>=0)) # observability uses TOTAL observable support at the home key (n_reads/n_umis), not only # cell-associated reads: a free molecule of the same cell that returns home is legit support. per=res.group_by([pl.col("final_well").alias("well"),"barcode","origin_cell_id"]).agg([