Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions simplex/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,6 @@
from .run import run
from .scoring import score
from .metrics import summarize
from .profiles import PRINCEPS_2025
from .profiles import PRINCEPS_2025, PRINCEPS_2026_FAMILY, PRINCEPS_2026_FULL_MEMORY_P50

__all__ = ["SimplexConfig", "run", "score", "summarize", "PRINCEPS_2025"]
__all__ = ["SimplexConfig", "run", "score", "summarize", "PRINCEPS_2025", "PRINCEPS_2026_FAMILY", "PRINCEPS_2026_FULL_MEMORY_P50"]
15 changes: 15 additions & 0 deletions simplex/_constant.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
"""Constant-region prefixes appended to SimPlex templates (v2d `constant_region=True`).

Real merged PairPlex reads run from the TSO through V(D)J into the constant region and stop at the gene-specific
primer. Measured on princeps donor02 consensuses (nt of constant region at the 3' end, median): IGHM 27, IGHG1 28,
IGHA1 34, IGKC 27, IGLC 50. Sequences: first 80 nt of the human *01 alleles (abstar germline database, IMGT).
"""
C_PREFIX = {
"IGHM": "GGGAGTGCATCCGCCCCAACCCTTTTCCCCCTCGTCTCCTGTGAGAATTCCCCGTCGGATACGAGCAGCGTGGCCGTTGG",
"IGHG1": "GCCTCCACCAAGGGCCCATCGGTCTTCCCCCTGGCACCCTCCTCCAAGAGCACCTCTGGGGGCACAGCGGCCCTGGGCTG",
"IGHA1": "GCATCCCCGACCAGCCCCAAGGTCTTCCCGCTGAGCCTCTGCAGCACCCAGCCAGATGGGAACGTGGTCATCGCCTGCCT",
"IGKC": "CGAACTGTGGCTGCACCATCTGTCTTCATCTTCCCGCCATCTGATGAGCAGTTGAAATCTGGAACTGCCTCTGTTGTGTG",
"IGLC2": "GGTCAGCCCAAGGCTGCCCCCTCGGTCACTCTGTTCCCGCCCTCCTCTGAGGAGCTTCAAGCCAACAAGGCCACACTGGT",
}
C_EXTENT = {"IGHM": 27, "IGHG1": 28, "IGHA1": 34, "IGKC": 27, "IGLC2": 50}
LIGHT_C = {"IGK": "IGKC", "IGL": "IGLC2"}
50 changes: 45 additions & 5 deletions simplex/cells.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,33 +8,53 @@
import numpy as np, polars as pl
from ._rng import rng_for
from .barcodes import load_barcodes
def load_pairs(input_data, n_cells=None, seed=0):
def load_pairs(input_data, n_cells=None, seed=0, input_filter=None, sample_clone_alpha=None):
"""Load heavy/light source pairs from `input_data` (a paired parquet) into a `cells` frame.

Validates that `locus:0`/`locus:1` are present (Phase 1-2 requires known loci) and
that any repeated `source_pair_id` maps to a consistent sequence+locus (never treat
two unrelated records with the same name as equivalent). If `n_cells` is given,
subsamples (with replacement if `n_cells` exceeds the input size) using the
`"subsample"` RNG stream keyed on `seed`. Returns one row per cell with a fresh
`"subsample"` RNG stream keyed on `seed`.

v2d: `input_filter` ({column: value}, e.g. {"donor": "d1", "is_memory": True}) restricts the input first.
If the input has a `clone_id` column it is carried into `cells`. `sample_clone_alpha` (None = uniform, the v2c
draw) samples cells WITHOUT replacement with weight `clone_size**(alpha-1)` per cell (clone size counted in the
filtered input), so alpha > 1 enriches expanded clones: a 20k-cell sim of a ~1M-cell reaction otherwise keeps almost
no clone with 2+ members. Clone members keep their own SHM-diversified sequences (distinct source pairs).

Returns one row per cell with a fresh
`cell_id`, the two chain sequences/loci, and `source_pair_id` provenance.
"""
df=pl.read_parquet(input_data)
if input_filter:
for k,v in input_filter.items(): df=df.filter(pl.col(k)==v)
if df.height==0: raise ValueError(f"input_filter {input_filter} leaves no rows")
req={"sequence_id:0":"chain0_id","sequence:0":"chain0_seq","sequence_id:1":"chain1_id","sequence:1":"chain1_seq"}
miss=[k for k in req if k not in df.columns]
if miss: raise ValueError(f"input missing {miss}")
if "locus:0" not in df.columns or "locus:1" not in df.columns:
raise ValueError("locus:0/1 required in Phase 1-2 (won't proceed with unknown loci)")
out=df.select([pl.col(k).alias(v) for k,v in req.items()]+[
(pl.col("name").cast(pl.Utf8) if "name" in df.columns else pl.int_range(pl.len()).cast(pl.Utf8)).alias("source_pair_id"),
pl.col("locus:0").cast(pl.Utf8).alias("chain0_locus"), pl.col("locus:1").cast(pl.Utf8).alias("chain1_locus")])
pl.col("locus:0").cast(pl.Utf8).alias("chain0_locus"), pl.col("locus:1").cast(pl.Utf8).alias("chain1_locus")]
+([pl.col("clone_id").cast(pl.Utf8)] if "clone_id" in df.columns else []))
bad=out.group_by("source_pair_id").agg([pl.col(c).n_unique().alias(c) for c in
["chain0_seq","chain1_seq","chain0_locus","chain1_locus"]]).filter(
(pl.col("chain0_seq")>1)|(pl.col("chain1_seq")>1)|(pl.col("chain0_locus")>1)|(pl.col("chain1_locus")>1))
if bad.height: raise ValueError(f"{bad.height} source_pair_id(s) map to differing sequences/loci")
if n_cells is not None:
idx=rng_for(seed,"subsample").choice(out.height,size=n_cells,replace=n_cells>out.height); out=out[idx]
if sample_clone_alpha is None:
idx=rng_for(seed,"subsample").choice(out.height,size=n_cells,replace=n_cells>out.height)
else:
if "clone_id" not in out.columns: raise ValueError("sample_clone_alpha needs a clone_id column in the input")
size=out.select(pl.len().over("clone_id")).to_series().to_numpy().astype(float)
w=size**(float(sample_clone_alpha)-1.0); w=w/w.sum()
idx=rng_for(seed,"subsample").choice(out.height,size=min(n_cells,out.height),replace=False,p=w)
out=out[idx]
return out.with_row_index("cell_id").select(
["cell_id","source_pair_id","chain0_id","chain0_seq","chain0_locus","chain1_id","chain1_seq","chain1_locus"])
["cell_id","source_pair_id","chain0_id","chain0_seq","chain0_locus","chain1_id","chain1_seq","chain1_locus"]
+(["clone_id"] if "clone_id" in out.columns else []))
def assign_droplets_and_barcodes(cells, mean, overdispersion, chemistry, barcode_pool_size, seed):
"""Group cells into overloaded droplets (GEMs) and give each droplet a 10X barcode.

Expand Down Expand Up @@ -80,3 +100,23 @@ def assign_wells(cells,wells,seed):
Uses the `"wells"` RNG stream keyed on `seed`.
"""
return cells.with_columns(pl.Series("resident_well",rng_for(seed,"wells").integers(0,wells,size=cells.height).astype(np.int64)))


def add_constant_region(cells, heavy_isotype_mix, seed):
"""v2d: append the constant-region prefix that real merged reads carry (TSO ... V(D)J ... C up to the gene-specific
primer; extents in `_constant.C_EXTENT`) to each chain template. Heavy isotype per cell drawn from
`heavy_isotype_mix` ({"IGHA1": .6, ...}; default all IGHM) with the `"isotype"` RNG stream; light chains get IGKC or
IGLC2 by locus. Truth sequences include the constant part, so scoring is unaffected (infix matching)."""
from ._constant import C_PREFIX, C_EXTENT, LIGHT_C
mix = heavy_isotype_mix or {"IGHM": 1.0}
iso = list(mix); p = np.array([mix[k] for k in iso], float); p = p / p.sum()
h_iso = np.array(iso)[rng_for(seed, "isotype").choice(len(iso), size=cells.height, p=p)]
def tail(locus, hiso):
g = hiso if locus == "IGH" else LIGHT_C.get(locus)
return C_PREFIX[g][:C_EXTENT[g]] if g else ""
out = cells
for ch in (0, 1):
loc = cells[f"chain{ch}_locus"].to_list()
t = [tail(l, h) for l, h in zip(loc, h_iso)]
out = out.with_columns((pl.col(f"chain{ch}_seq") + pl.Series(t)).alias(f"chain{ch}_seq"))
return out.with_columns(pl.Series("heavy_isotype", h_iso))
24 changes: 23 additions & 1 deletion simplex/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,9 @@ class SimplexConfig:
`molecules_per_chain_mean_light` overrides the light-chain molecule count (light template is
more abundant); `ambient_rate` adds pre-RT ambient capture: free mRNA from lysed cells barcoded
in OTHER droplets (new barcode + UMI, random well), as `ambient_rate` x cell molecules;
`umi_sub_rate` adds per-read UMI substitution errors (uncorrected UMIs inflate support).
`umi_sub_rate` adds per-read UMI substitution errors (uncorrected UMIs inflate support);
`reads_per_molecule_dispersion` makes read-family depth negative binomial (PCR jackpots; 0 = Poisson);
`reads_per_molecule_mean_light` sets a separate read-family mean for light chains.
- **Bookkeeping**: `write_read_truth` (also persist a per-read truth parquet —
expensive, off by default), `seed` (root seed for all per-stage RNG streams).
"""
Expand All @@ -75,6 +77,18 @@ class SimplexConfig:
truncation: str = "fragment_5p"; fragment_rate: float = 0.1; fragment_rate_heavy: float | None = None
fragment_keep: tuple = (0.3, 0.9)
umi_sub_rate: float = 0.0
reads_per_molecule_dispersion: float = 0.0
reads_per_molecule_mean_light: float | None = None
junk_rate: float = 0.0
junk_length: tuple = (80, 300)
input_filter: dict | None = None
sample_clone_alpha: float | None = None
constant_region: bool = False
heavy_isotype_mix: dict | None = None
barcode_sub_rate: float = 0.0
tso_sub_rate: float = 0.0
tso_g_run_probs: list | None = None
tso_indel_rate: float = 0.0

_RATES=("recovery_rate","release_rate","molecule_survival_rate","rt_sub_rate","rt_indel_rate",
"sequencing_sub_rate","sequencing_indel_rate","index_hop_rate","rc_fraction","umi_sub_rate")
Expand Down Expand Up @@ -123,6 +137,14 @@ def validate(self, actual_n_cells=None, max_reads=3_000_000_000):
if self.molecules_per_chain_mean_light is not None and self.molecules_per_chain_mean_light<=0:
raise ValueError("molecules_per_chain_mean_light must be > 0 or None")
if self.ambient_rate<0: raise ValueError("ambient_rate must be >= 0")
if self.reads_per_molecule_dispersion<0: raise ValueError("reads_per_molecule_dispersion must be >= 0")
if self.junk_rate<0: raise ValueError("junk_rate must be >= 0")
for r in ("barcode_sub_rate","tso_sub_rate","tso_indel_rate"):
if not (0.0<=getattr(self,r)<=1.0): raise ValueError(f"{r} not in [0,1]")
jl,jh=self.junk_length
if not (1<=jl<=jh): raise ValueError(f"junk_length={self.junk_length} must satisfy 1<=min<=max")
if self.reads_per_molecule_mean_light is not None and self.reads_per_molecule_mean_light<=0:
raise ValueError("reads_per_molecule_mean_light must be > 0 or None")
for r in ("fragment_rate","fragment_rate_heavy"):
v=getattr(self,r)
if v is not None and not (0.0<=v<=1.0): raise ValueError(f"{r}={v} not in [0,1]")
Expand Down
111 changes: 111 additions & 0 deletions simplex/fidelity.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
"""Sequence fidelity of PairPlex contigs against SimPlex truth (v2d).

Pairing correctness (`simplex.score`) accepts a chain within 6% edit distance of a source, so it cannot see two failure
modes that matter for repertoire analysis:

1. **Invented or erased mutations.** A contig that differs from every true sequence present at its (well, barcode)
carries errors the consensus did not remove (RT/PCR errors are inherited by the whole read family) or mixes
molecules of different cells.
2. **Over-collapsing clonal siblings.** Members of one expanded lineage share V, J and CDR3 and differ only by
somatic hypermutation. If two siblings share a (well, barcode), a pipeline that clusters or merges too
aggressively returns one contig for both, erasing real diversity. Sequencing/PCR errors must be collapsed;
siblings must not.

`fidelity(output_dir, truth_dir)` returns a dict with:

- `contigs`: productive IGH/IGK/IGL contigs scored; `exact_share`: share whose sequence occurs exactly (as an infix)
in a true sequence of the same locus class at its key; `le1_share` (<= 1 edit); `edits_per_100nt_mean`;
the same restricted to keys with a single true source of that locus class (`single_source_*`: pure error rate).
- `sibling_sequences`: true sequences at keys holding >= 2 members of one clone (same `clone_id`, distinct
sequences, each >= `min_reads` reads there); `sibling_recovered_share`: share of those that are the nearest
true sequence of at least one contig within `max_frac` (separately resolved);
`sibling_keys_collapsed_share`: keys where fewer siblings were recovered than present.

Needs `clone_id` in `truth_cells` (clonal sampling from an input with a `clone_id` column) for the sibling part;
otherwise only contig fidelity is reported.
"""
import re
from pathlib import Path

import edlib
import polars as pl

_LIGHT = ("IGK", "IGL")


def _cls(locus):
return "H" if locus == "IGH" else ("L" if locus in _LIGHT else None)


def _dist(a, b):
short, long = (a, b) if len(a) <= len(b) else (b, a)
return edlib.align(short, long, mode="HW", task="distance")["editDistance"], len(short)


def fidelity(output_dir, truth_dir, min_reads=3, max_frac=0.02):
out, truth = Path(output_dir), Path(truth_dir)
comp = pl.read_parquet(truth / "truth_components.parquet").filter(pl.col("chain") >= 0)
tc = pl.read_parquet(truth / "truth_cells.parquet")
has_clone = "clone_id" in tc.columns
src_clone = dict(zip(tc["source_pair_id"].to_list(), tc["clone_id"].to_list())) if has_clone else {}
agg = (comp.with_columns(pl.col("locus").map_elements(_cls, return_dtype=pl.Utf8).alias("lc"))
.group_by("final_well", "barcode", "lc", "sequence", "source_pair_id").agg(pl.col("n_reads").sum()))
idx = {}
for w, b, lc, seq, src, nr in agg.iter_rows():
if seq is None or lc is None:
continue
idx.setdefault((int(w), b, lc), []).append((seq, src, int(nr)))
contigs = []
for f in sorted((out / "annotated").glob("*_unpaired.parquet")):
sch = pl.read_parquet_schema(f)
if "sequence" not in sch or "locus" not in sch:
continue
m = re.search(r"well(\d+)", f.name) or re.search(r"(\d+)", f.name)
w = int(m.group(1))
d = pl.read_parquet(f, columns=["sequence_id", "sequence", "locus", "productive"]).filter(pl.col("productive"))
for sid, seq, loc, _ in d.iter_rows():
lc = _cls(loc)
if lc is None or not seq:
continue
contigs.append((w, sid.split("_")[0], lc, seq))
rows, nearest_by_key = [], {}
for w, b, lc, seq in contigs:
cands = idx.get((w, b, lc))
if not cands:
continue
best = None
for tseq, src, nr in cands:
e, n = _dist(seq, tseq)
if best is None or e < best[0]:
best = (e, n, tseq)
rows.append({"edits": best[0], "len": best[1], "single": len({c[0] for c in cands}) == 1})
if best[0] <= max_frac * best[1]:
nearest_by_key.setdefault((w, b, lc), set()).add(best[2])
res = {}
if rows:
r = pl.DataFrame(rows).with_columns((100 * pl.col("edits") / pl.col("len")).alias("e100"))
s = r.filter(pl.col("single"))
res.update(contigs=r.height, exact_share=float((r["edits"] == 0).mean()), le1_share=float((r["edits"] <= 1).mean()),
edits_per_100nt_mean=float(r["e100"].mean()), single_source_contigs=s.height,
single_source_exact_share=float((s["edits"] == 0).mean()) if s.height else None,
single_source_edits_per_100nt_mean=float(s["e100"].mean()) if s.height else None)
if has_clone:
n_sib = n_rec = n_keys = n_coll = 0
for key, cands in idx.items():
by_clone = {}
for tseq, src, nr in cands:
cl = src_clone.get(src)
if cl is not None and nr >= min_reads:
by_clone.setdefault(cl, set()).add(tseq)
for cl, seqs in by_clone.items():
if len(seqs) < 2:
continue
rec = len(seqs & nearest_by_key.get(key, set()))
n_sib += len(seqs); n_rec += rec; n_keys += 1; n_coll += rec < len(seqs)
res.update(sibling_keys=n_keys, sibling_sequences=n_sib,
sibling_recovered_share=(n_rec / n_sib) if n_sib else None,
sibling_keys_collapsed_share=(n_coll / n_keys) if n_keys else None)
return res


__all__ = ["fidelity"]
20 changes: 19 additions & 1 deletion simplex/molecules.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"survived":pl.Boolean,"cdna":pl.Utf8,"is_ambient":pl.Boolean}
def generate_molecules(cells,recovery_rate,molecules_per_chain_mean,release_rate,umi_length,rt_sub_rate,rt_indel_rate,seed,
release_rate_heavy=None,release_rate_light=None,molecules_per_chain_mean_light=None,
ambient_rate=0.0):
ambient_rate=0.0,junk_rate=0.0,junk_length=(80,300)):
"""Simulate RT capture per cell/chain and split molecules into resident vs free (ambient).

For each chain (0, 1) independently: each cell captures the chain with probability
Expand Down Expand Up @@ -43,6 +43,12 @@ def generate_molecules(cells,recovery_rate,molecules_per_chain_mean,release_rate
and is free (`is_free=True`, routed to a random well) with `is_ambient=True`. Unlike release,
ambient ADDS molecules and moves a chain to a different barcode.

Junk axis (`junk_rate`, default 0): non-Ig library molecules (mispriming, primer artefacts) with a valid barcode.
`Poisson(junk_rate * n_cell_molecules)` molecules, each owned by the cell of a uniformly chosen cell molecule
(barcode, home well), random sequence of uniform length in `junk_length`, released like that cell's chains at
`release_rate`; `chain=-1`, `locus="junk"`. They never match a truth chain, so they cost yield and compute and
dilute read/UMI statistics without being scorable as pairs.

Returns `(molecules, chain_status)`: `molecules` is the per-molecule truth record
(`molecule_id`, `parent_molecule_id` initialized equal to it, `amplification_well`
initialized to 0 and `survived` to False — both set later by `routing.py`); `chain_status`
Expand Down Expand Up @@ -92,6 +98,18 @@ def _is_light(chain):
pl.Series("umi",random_dna(arng,na,umi_length)),pl.Series("cdna",cdna),
pl.lit(True).alias("is_free"),pl.lit(True).alias("is_ambient")])
m=pl.concat([m,amb.select(m.columns)])
if junk_rate>0 and m.height:
jrng=rng_for(seed,"junk"); cellm=m.filter(~pl.col("is_ambient"))
nj=int(jrng.poisson(junk_rate*cellm.height))
if nj:
src=cellm[jrng.integers(0,cellm.height,size=nj)]
lo,hi=junk_length; L=jrng.integers(lo,hi+1,size=nj)
full=random_dna(jrng,nj,int(hi))
seqs=[s[:l] for s,l in zip(full,L)]
junk=src.with_columns([pl.lit(-1).cast(pl.Int8).alias("chain"),pl.lit("junk").alias("locus"),
pl.Series("umi",random_dna(jrng,nj,umi_length)),pl.Series("cdna",seqs),
pl.Series("is_free",jrng.random(nj)<release_rate),pl.lit(False).alias("is_ambient")])
m=pl.concat([m,junk.select(m.columns)])
m=m.with_row_index("molecule_id").with_columns([
pl.col("molecule_id").cast(pl.Int64),pl.col("molecule_id").cast(pl.Int64).alias("parent_molecule_id"),
pl.lit(0).cast(pl.Int64).alias("amplification_well"), pl.lit(False).alias("survived")])
Expand Down
Loading
Loading