From 68db0dac2494b5ffb2b5fc02892744457e2947ca Mon Sep 17 00:00:00 2001 From: Josh Garton Date: Thu, 3 Sep 2026 18:38:56 +0000 Subject: [PATCH] Fuse QC/HVG scans, implicit PCA, and parallel markers. Speed up the existing H5/H5AD/CSV cell-matrix path without changing QC thresholds, HVG flavors, or packet artifacts. Two CSR nnz passes replace separate QC, normalize, and HVG scans. PCA applies scale+clip during sparse matmuls and clamps n_pcs so tiny post-QC matrices do not panic. Wilcoxon markers run per-cluster on rayon. Co-authored-by: Josh Garton --- PACKET.md | 2 +- src/fused.rs | 304 +++++++++++++++++++++++++++++++++++++++++++++++ src/hvg_sc.rs | 110 ++++++++++++----- src/main.rs | 198 ++++++++++++++++++++++++------ src/markers.rs | 222 +++++++++++++++++++++++----------- src/normalize.rs | 133 +++++++++++++++++++++ src/pca.rs | 181 ++++++++++++++++++++++++++-- src/qc.rs | 103 +++++++++++++++- src/scale.rs | 15 ++- src/sparse.rs | 154 ++++++++++++++++++++++++ 10 files changed, 1271 insertions(+), 151 deletions(-) create mode 100644 src/fused.rs diff --git a/PACKET.md b/PACKET.md index d5b3115..1b18d94 100644 --- a/PACKET.md +++ b/PACKET.md @@ -42,7 +42,7 @@ What this crate actually writes today (v0.3.0). Column names are as emitted; do | `knn.csv` | `neighbor_1`,`distance_1`,… for k neighbors | | `clusters.csv` | `barcode`, `cluster` | | `markers.csv` | `cluster`, `gene`, `score`, `pval`, `pval_adj`, `log2fc` | -| `pipeline_timings.json` | `pipeline`=`rustpipe-sc`, `version`, `seed`, `input`, cell/gene counts, `hvg_flavor`, `n_pcs`, `knn_k`, `steps[{step,seconds}]`, `total_seconds` | +| `pipeline_timings.json` | `pipeline`=`rustpipe-sc`, `version`, `seed`, `input`, cell/gene counts, `hvg_flavor`, `n_pcs`, `knn_k`, `steps[{step,seconds}]`, `total_seconds`. Step names are unchanged (`load`, `qc_filter`, `normalize`, `hvg`, `scale`, `pca`, `knn`, `leiden`, `markers`). Internally QC+normalize+HVG share two CSR nnz passes; `scale` is mean/std only (z-score+clip is implicit in PCA). | | `libqc/` | only if `--qc-bam`; rustqc tree. Not the cell matrix. | The `qc` subcommand also writes `filtered_barcodes.csv`. The full `pipeline` subcommand does **not** write `filtered_cells.csv` / `normalized.csv` / `pca_loadings.csv` (those names in older README lists are not produced). diff --git a/src/fused.rs b/src/fused.rs new file mode 100644 index 0000000..d186f9e --- /dev/null +++ b/src/fused.rs @@ -0,0 +1,304 @@ +//! Two-pass fused QC → (optional) normalize → HVG stats over CSR nonzeros. +//! +//! Replaces three full matrix scans (`compute_qc_metrics`, `normalize_log1p_sparse`, +//! `sparse_gene_stats`) plus a separate gene-filter scan. Biology is unchanged: +//! same QC thresholds, same per-cell totals on gene-filtered counts, same +//! seurat / seurat_v3 ranking. + +use crate::hvg_sc::{self, HvgResult}; +use crate::normalize; +use crate::qc::{self, QcMetrics}; +use crate::sparse::SpMat; +use anyhow::Result; + +/// Filtered, optionally log-normalized matrix plus HVG-ready gene stats. +pub struct FusedPrep { + pub metrics: QcMetrics, + pub keep_cells: Vec, + pub filtered_names: Vec, + pub kept_var: Vec, + pub mat: SpMat, + pub gene_means: Vec, + pub gene_variances: Vec, +} + +/// QC + gene filter + normalize + per-gene mean/variance in two CSR passes. +pub fn qc_normalize_hvg_stats( + mat: &SpMat, + var_names: &[String], + obs_names: &[String], + min_genes: u32, + max_pct_mt: f32, + min_cells: usize, + target_sum: f32, + skip_normalize: bool, +) -> FusedPrep { + let (metrics, keep_cells, gene_counts) = + qc::qc_metrics_and_kept_gene_counts(mat, var_names, obs_names, min_genes, max_pct_mt); + let keep_genes = qc::gene_keep_from_counts(&gene_counts, min_cells); + + let filtered_names: Vec = obs_names + .iter() + .zip(keep_cells.iter()) + .filter(|(_, &k)| k) + .map(|(n, _)| n.clone()) + .collect(); + let kept_var: Vec = var_names + .iter() + .zip(keep_genes.iter()) + .filter(|(_, &k)| k) + .map(|(n, _)| n.clone()) + .collect(); + + let (normed, gene_means, gene_variances) = normalize::normalize_log1p_subset_with_stats( + mat, + &keep_cells, + &keep_genes, + target_sum, + skip_normalize, + ); + + FusedPrep { + metrics, + keep_cells, + filtered_names, + kept_var, + mat: normed, + gene_means, + gene_variances, + } +} + +/// Rank HVGs from fused stats (no extra matrix scan). +pub fn select_hvg_from_stats( + prep: &FusedPrep, + n_top_genes: usize, + hvg_flavor: &str, +) -> Result { + let n_top = n_top_genes.min(prep.kept_var.len()); + match hvg_flavor { + "seurat_v3" => hvg_sc::select_hvg_vst_from_stats( + &prep.gene_means, + &prep.gene_variances, + &prep.kept_var, + n_top, + prep.mat.rows(), + ), + _ => hvg_sc::select_hvg_seurat_from_stats( + &prep.gene_means, + &prep.gene_variances, + &prep.kept_var, + n_top, + 20, + ), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::normalize::normalize_log1p_sparse; + use crate::qc::{apply_cell_filter, compute_qc_metrics, filter_cells_fixed, filter_genes}; + use crate::sparse::from_triplets; + use rand::prelude::*; + + fn toy_sc_matrix() -> (SpMat, Vec, Vec) { + let mat = from_triplets( + 4, + 5, + &[0, 0, 0, 1, 1, 2, 2, 2, 3, 3], + &[0, 1, 4, 0, 2, 1, 3, 4, 0, 1], + &[10.0, 20.0, 5.0, 15.0, 25.0, 30.0, 10.0, 8.0, 3.0, 2.0], + ); + let genes = vec![ + "BRCA1".into(), + "TP53".into(), + "EGFR".into(), + "MYC".into(), + "MT-CO1".into(), + ]; + let cells = vec![ + "CELL_A".into(), + "CELL_B".into(), + "CELL_C".into(), + "CELL_D".into(), + ]; + (mat, genes, cells) + } + + fn make_sc_test_matrix( + n_cells: usize, + n_high: usize, + n_low: usize, + seed: u64, + ) -> (SpMat, Vec, Vec) { + let n_genes = n_high + n_low; + let mut rng = StdRng::seed_from_u64(seed); + let mut rows = Vec::new(); + let mut cols = Vec::new(); + let mut vals = Vec::new(); + + for c in 0..n_cells { + for g in 0..n_genes { + let val = if g < n_high { + if rng.random::() < 0.9 { + rng.random::() * 500.0 + 10.0 + } else { + 0.0 + } + } else if rng.random::() < 0.9 { + 50.0 + rng.random::() * 2.0 + } else { + 0.0 + }; + if val > 0.0 { + rows.push(c); + cols.push(g); + vals.push(val); + } + } + } + + let mat = from_triplets(n_cells, n_genes, &rows, &cols, &vals); + let gene_names: Vec = (0..n_genes).map(|g| format!("GENE{:05}", g)).collect(); + let cell_names: Vec = (0..n_cells).map(|c| format!("CELL{:04}", c)).collect(); + (mat, gene_names, cell_names) + } + + fn sequential_prep( + mat: &SpMat, + var_names: &[String], + obs_names: &[String], + min_genes: u32, + max_pct_mt: f32, + min_cells: usize, + target_sum: f32, + skip_normalize: bool, + ) -> (QcMetrics, Vec, Vec, Vec, SpMat) { + let metrics = compute_qc_metrics(mat, var_names, obs_names); + let keep = filter_cells_fixed(&metrics, min_genes, max_pct_mt); + let (filtered_mat, filtered_names) = apply_cell_filter(mat, obs_names, &keep); + let (gene_filtered, kept_genes) = filter_genes(&filtered_mat, min_cells); + let kept_var: Vec = kept_genes.iter().map(|&i| var_names[i].clone()).collect(); + let normed = if skip_normalize { + gene_filtered + } else { + normalize_log1p_sparse(&gene_filtered, target_sum) + }; + (metrics, keep, filtered_names, kept_var, normed) + } + + fn assert_qc_close(a: &QcMetrics, b: &QcMetrics) { + assert_eq!(a.barcodes, b.barcodes); + assert_eq!(a.n_genes_by_counts, b.n_genes_by_counts); + assert_eq!(a.total_counts.len(), b.total_counts.len()); + for i in 0..a.total_counts.len() { + assert!( + (a.total_counts[i] - b.total_counts[i]).abs() < 1e-5, + "total_counts[{}] fused={} sequential={}", + i, + a.total_counts[i], + b.total_counts[i] + ); + assert!( + (a.pct_counts_mt[i] - b.pct_counts_mt[i]).abs() < 1e-5, + "pct_mt[{}] fused={} sequential={}", + i, + a.pct_counts_mt[i], + b.pct_counts_mt[i] + ); + } + } + + fn assert_sparse_close(a: &SpMat, b: &SpMat) { + assert_eq!(a.rows(), b.rows()); + assert_eq!(a.cols(), b.cols()); + assert_eq!(a.nnz(), b.nnz()); + for i in 0..a.rows() { + let ra = a.outer_view(i).unwrap(); + let rb = b.outer_view(i).unwrap(); + let va: Vec<(usize, f32)> = ra.iter().map(|(c, &v)| (c, v)).collect(); + let vb: Vec<(usize, f32)> = rb.iter().map(|(c, &v)| (c, v)).collect(); + assert_eq!(va.len(), vb.len(), "row {} nnz", i); + for ((ca, xa), (cb, xb)) in va.iter().zip(vb.iter()) { + assert_eq!(ca, cb, "row {} col mismatch", i); + assert!( + (xa - xb).abs() < 1e-5, + "row {} col {}: fused={} sequential={}", + i, + ca, + xa, + xb + ); + } + } + } + + #[test] + fn fused_matches_sequential_on_qc_fixture() { + let (mat, genes, cells) = toy_sc_matrix(); + let fused = qc_normalize_hvg_stats(&mat, &genes, &cells, 3, 100.0, 2, 1e4, false); + let (metrics, keep, names, kept_var, normed) = + sequential_prep(&mat, &genes, &cells, 3, 100.0, 2, 1e4, false); + + assert_qc_close(&fused.metrics, &metrics); + assert_eq!(fused.keep_cells, keep); + assert_eq!(fused.filtered_names, names); + assert_eq!(fused.kept_var, kept_var); + assert_sparse_close(&fused.mat, &normed); + } + + #[test] + fn fused_hvg_matches_sequential_seurat_fixture() { + let (mat, genes, cells) = make_sc_test_matrix(200, 10, 90, 42); + let fused = qc_normalize_hvg_stats(&mat, &genes, &cells, 1, 100.0, 3, 1e4, false); + let (_m, _k, _n, kept_var, normed) = + sequential_prep(&mat, &genes, &cells, 1, 100.0, 3, 1e4, false); + + assert_eq!(fused.kept_var, kept_var); + let n_top = 15.min(kept_var.len()); + let hvg_fused = select_hvg_from_stats(&fused, n_top, "seurat").unwrap(); + let hvg_seq = hvg_sc::select_hvg_seurat(&normed, &kept_var, n_top, 20).unwrap(); + assert_eq!(hvg_fused.gene_names, hvg_seq.gene_names); + assert_eq!(hvg_fused.gene_indices, hvg_seq.gene_indices); + assert_eq!(hvg_fused.variances_norm.len(), hvg_seq.variances_norm.len()); + for (i, (a, b)) in hvg_fused + .variances_norm + .iter() + .zip(hvg_seq.variances_norm.iter()) + .enumerate() + { + assert!( + (a - b).abs() < 1e-5, + "disp_norm[{}] fused={} sequential={}", + i, + a, + b + ); + } + } + + #[test] + fn fused_hvg_matches_sequential_seurat_v3_fixture() { + let (mat, genes, cells) = make_sc_test_matrix(200, 10, 90, 42); + let fused = qc_normalize_hvg_stats(&mat, &genes, &cells, 1, 100.0, 3, 1e4, false); + let (_m, _k, _n, kept_var, normed) = + sequential_prep(&mat, &genes, &cells, 1, 100.0, 3, 1e4, false); + + let n_top = 15.min(kept_var.len()); + let hvg_fused = select_hvg_from_stats(&fused, n_top, "seurat_v3").unwrap(); + let hvg_seq = hvg_sc::select_hvg_sparse(&normed, &kept_var, n_top).unwrap(); + assert_eq!(hvg_fused.gene_names, hvg_seq.gene_names); + assert_eq!(hvg_fused.gene_indices, hvg_seq.gene_indices); + } + + #[test] + fn fused_skip_normalize_matches_sequential() { + let (mat, genes, cells) = make_sc_test_matrix(80, 8, 32, 7); + let fused = qc_normalize_hvg_stats(&mat, &genes, &cells, 1, 100.0, 3, 1e4, true); + let (_m, _k, _n, kept_var, raw) = + sequential_prep(&mat, &genes, &cells, 1, 100.0, 3, 1e4, true); + assert_eq!(fused.kept_var, kept_var); + assert_sparse_close(&fused.mat, &raw); + } +} diff --git a/src/hvg_sc.rs b/src/hvg_sc.rs index a547556..c831178 100644 --- a/src/hvg_sc.rs +++ b/src/hvg_sc.rs @@ -87,18 +87,53 @@ pub fn select_hvg_sparse( } let start = std::time::Instant::now(); - - // Step 1: per-gene mean and variance from sparse data let (means, variances) = sparse::sparse_gene_stats(mat, n_cells); + let result = select_hvg_vst_from_stats(&means, &variances, var_names, n_top_genes, n_cells)?; + info!( + "Selected {} HVGs from {} genes in {:.3}s", + n_top_genes, + n_genes, + start.elapsed().as_secs_f64() + ); + Ok(result) +} - // Step 2: fit mean-variance trend - let (a, b) = fit_mean_variance_trend(&means, &variances); +/// Seurat v3 VST ranking from precomputed per-gene mean/variance. +/// +/// `n_cells` is the cell count used for the sqrt(n) clip (same as a matrix scan). +pub fn select_hvg_vst_from_stats( + means: &[f32], + variances: &[f32], + var_names: &[String], + n_top_genes: usize, + n_cells: usize, +) -> Result { + let n_genes = means.len(); + if variances.len() != n_genes || var_names.len() != n_genes { + bail!( + "means ({}), variances ({}), and var_names ({}) must have the same length", + n_genes, + variances.len(), + var_names.len() + ); + } + if n_top_genes > n_genes { + bail!( + "n_top_genes ({}) exceeds total genes ({})", + n_top_genes, + n_genes + ); + } + if n_cells < 2 { + bail!("Need at least 2 cells to compute variance"); + } + + let (a, b) = fit_mean_variance_trend(means, variances); info!( "HVG mean-variance trend: ln(var) = {:.4} + {:.4} * ln(mean)", a, b ); - // Step 3: standardized variance with Seurat v3 clipping let clip_val = (n_cells as f32).sqrt(); let variances_norm: Vec = means .par_iter() @@ -113,14 +148,13 @@ pub fn select_hvg_sparse( } let sv = v / expected; if sv.is_finite() { - sv.min(clip_val) // Seurat v3 clipping! + sv.min(clip_val) } else { 0.0 } }) .collect(); - // Step 4: rank by standardized variance (descending), take top N let mut ranked_indices: Vec = (0..n_genes).collect(); ranked_indices.sort_by(|&a_idx, &b_idx| { variances_norm[b_idx] @@ -128,8 +162,6 @@ pub fn select_hvg_sparse( .unwrap_or(std::cmp::Ordering::Equal) }); ranked_indices.truncate(n_top_genes); - - // Sort for stable ordering ranked_indices.sort(); let selected_names: Vec = ranked_indices @@ -137,18 +169,11 @@ pub fn select_hvg_sparse( .map(|&i| var_names[i].clone()) .collect(); - info!( - "Selected {} HVGs from {} genes in {:.3}s", - n_top_genes, - n_genes, - start.elapsed().as_secs_f64() - ); - Ok(HvgResult { gene_names: selected_names, gene_indices: ranked_indices, - means, - variances, + means: means.to_vec(), + variances: variances.to_vec(), variances_norm, }) } @@ -184,9 +209,42 @@ pub fn select_hvg_seurat( } let start = std::time::Instant::now(); - - // Step 1: per-gene mean and variance from sparse data let (means, variances) = sparse::sparse_gene_stats(mat, n_cells); + let result = select_hvg_seurat_from_stats(&means, &variances, var_names, n_top_genes, n_bins)?; + info!( + "Selected {} HVGs (seurat flavor, {} bins) from {} genes in {:.3}s", + n_top_genes, + n_bins, + n_genes, + start.elapsed().as_secs_f64() + ); + Ok(result) +} + +/// Seurat dispersion-bin ranking from precomputed per-gene mean/variance. +pub fn select_hvg_seurat_from_stats( + means: &[f32], + variances: &[f32], + var_names: &[String], + n_top_genes: usize, + n_bins: usize, +) -> Result { + let n_genes = means.len(); + if variances.len() != n_genes || var_names.len() != n_genes { + bail!( + "means ({}), variances ({}), and var_names ({}) must have the same length", + n_genes, + variances.len(), + var_names.len() + ); + } + if n_top_genes > n_genes { + bail!( + "n_top_genes ({}) exceeds total genes ({})", + n_top_genes, + n_genes + ); + } // Step 2: dispersion = var / mean (Scanpy: mean_zero → 1e-12, disp_zero → NaN) let dispersions: Vec = means @@ -337,19 +395,11 @@ pub fn select_hvg_seurat( let selected_names: Vec = ranked.iter().map(|&i| var_names[i].clone()).collect(); - info!( - "Selected {} HVGs (seurat flavor, {} bins) from {} genes in {:.3}s", - n_top_genes, - n_bins, - n_genes, - start.elapsed().as_secs_f64() - ); - Ok(HvgResult { gene_names: selected_names, gene_indices: ranked, - means, - variances, + means: means.to_vec(), + variances: variances.to_vec(), variances_norm: disp_norm, }) } diff --git a/src/main.rs b/src/main.rs index 6c83a92..e4217ee 100644 --- a/src/main.rs +++ b/src/main.rs @@ -14,6 +14,7 @@ clippy::doc_overindented_list_items )] +mod fused; mod graph; mod gsea; mod h5ad; @@ -119,7 +120,6 @@ enum Commands { /// HVG selection flavor: "seurat" (dispersion-bin) or "seurat_v3" (VST) #[arg(long, default_value = "seurat")] hvg_flavor: String, - }, /// PCA on expression data (dense or sparse) @@ -582,9 +582,8 @@ fn run_pipeline( let mut timings: Vec<(&str, f64)> = Vec::new(); if let Some(bam) = qc_bam { - let gtf = qc_gtf.ok_or_else(|| { - anyhow::anyhow!("--qc-gtf is required when --qc-bam is set") - })?; + let gtf = + qc_gtf.ok_or_else(|| anyhow::anyhow!("--qc-gtf is required when --qc-bam is set"))?; let t0 = Instant::now(); libqc::run_libqc( std::path::Path::new(bam), @@ -604,16 +603,26 @@ fn run_pipeline( mat.cols() ); - // Step 2: QC + // Steps 2–4: two CSR nnz passes (QC+gene counts, then normalize+HVG stats). + // pipeline_timings.json keeps qc_filter / normalize / hvg. let t0 = Instant::now(); - let metrics = qc::compute_qc_metrics(&mat, &var_names, &obs_names); + let (metrics, keep_cells, gene_counts) = + qc::qc_metrics_and_kept_gene_counts(&mat, &var_names, &obs_names, min_genes, max_pct_mt); qc::write_qc_csv(&metrics, &out_path.join("qc_metrics.csv"))?; - - let keep = qc::filter_cells_fixed(&metrics, min_genes, max_pct_mt); - let n_kept = keep.iter().filter(|&&b| b).count(); - let (filtered_mat, filtered_names) = qc::apply_cell_filter(&mat, &obs_names, &keep); - let (gene_filtered, kept_genes) = qc::filter_genes(&filtered_mat, 3); - let kept_var: Vec = kept_genes.iter().map(|&i| var_names[i].clone()).collect(); + let keep_genes = qc::gene_keep_from_counts(&gene_counts, 3); + let filtered_names: Vec = obs_names + .iter() + .zip(keep_cells.iter()) + .filter(|(_, &k)| k) + .map(|(n, _)| n.clone()) + .collect(); + let kept_var: Vec = var_names + .iter() + .zip(keep_genes.iter()) + .filter(|(_, &k)| k) + .map(|(n, _)| n.clone()) + .collect(); + let n_kept = filtered_names.len(); timings.push(("qc_filter", t0.elapsed().as_secs_f64())); eprintln!("[pipeline] QC: {} cells, {} genes", n_kept, kept_var.len()); if n_kept < 2 { @@ -627,23 +636,37 @@ fn run_pipeline( ); } - // Step 3: Normalize (skip if input is already normalized) - let normed = if skip_normalize { + let t0 = Instant::now(); + let (normed, gene_means, gene_variances) = normalize::normalize_log1p_subset_with_stats( + &mat, + &keep_cells, + &keep_genes, + 1e4, + skip_normalize, + ); + if skip_normalize { eprintln!("[pipeline] skipping normalization (--skip-normalize)"); - gene_filtered } else { - let t0 = Instant::now(); - let n = normalize::normalize_log1p_sparse(&gene_filtered, 1e4); - timings.push(("normalize", t0.elapsed().as_secs_f64())); - eprintln!("[pipeline] normalized, nnz={}", n.nnz()); - n - }; + eprintln!("[pipeline] normalized, nnz={}", normed.nnz()); + } + timings.push(("normalize", t0.elapsed().as_secs_f64())); - // Step 4: HVG selection let t0 = Instant::now(); let hvg_result = match hvg_flavor { - "seurat_v3" => hvg_sc::select_hvg_sparse(&normed, &kept_var, n_hvg)?, - _ => hvg_sc::select_hvg_seurat(&normed, &kept_var, n_hvg, 20)?, + "seurat_v3" => hvg_sc::select_hvg_vst_from_stats( + &gene_means, + &gene_variances, + &kept_var, + n_hvg, + normed.rows(), + )?, + _ => hvg_sc::select_hvg_seurat_from_stats( + &gene_means, + &gene_variances, + &kept_var, + n_hvg, + 20, + )?, }; let hvg_mat = sparse::subset_genes(&normed, &hvg_result.gene_indices); timings.push(("hvg", t0.elapsed().as_secs_f64())); @@ -665,25 +688,25 @@ fn run_pipeline( eprintln!("[pipeline] wrote {}", hvg_path.display()); } - // Step 5: Scale (z-score normalization with clipping) + // Step 5: Scale stats only (no densify). Clip/z-score applied in PCA matmuls. let t0 = Instant::now(); - let (scaled, _means, _stds) = scale::scale_sparse(&hvg_mat, max_value); + let (scale_means, scale_stds) = scale::scale_stats(&hvg_mat); timings.push(("scale", t0.elapsed().as_secs_f64())); eprintln!( - "[pipeline] scaled: {} x {}, max_value={}", - scaled.nrows(), - scaled.ncols(), + "[pipeline] scale stats: {} cells x {} HVG, max_value={} (implicit, not densified)", + hvg_mat.rows(), + hvg_mat.cols(), max_value ); - // Step 6: PCA (on scaled dense matrix) - let n_pcs = n_pcs - .min(n_kept.saturating_sub(1)) - .min(scaled.ncols().saturating_sub(1)) - .max(1); + // Step 6: PCA — power-iteration RSVD with implicit scale+clip+center + let n_pcs = sparse::clamp_n_pcs(n_pcs, n_kept, hvg_mat.cols()); let t0 = Instant::now(); - let pca_result = pca::run_pca_scaled( - &scaled, + let pca_result = pca::run_pca_sparse_scaled( + &hvg_mat, + &scale_means, + &scale_stds, + max_value, &hvg_result.gene_names, n_pcs, 3, // power iterations @@ -868,3 +891,106 @@ fn load_input(path: &str) -> Result<(sparse::SpMat, Vec, Vec)> { } } } + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + fn write_toy_cells_csv(path: &std::path::Path, n_cells: usize, n_genes: usize) { + let mut f = std::fs::File::create(path).unwrap(); + write!(f, "barcode").unwrap(); + for g in 0..n_genes { + write!(f, ",G{g:02}").unwrap(); + } + writeln!(f).unwrap(); + for c in 0..n_cells { + write!(f, "CELL{c:03}").unwrap(); + for g in 0..n_genes { + // Deterministic sparse-ish counts; every cell expresses enough genes. + let v = if (c + g) % 4 == 0 { + 0.0 + } else { + ((c * 7 + g * 13) % 40) as f32 + 2.0 + }; + write!(f, ",{v}").unwrap(); + } + writeln!(f).unwrap(); + } + } + + #[test] + fn pipeline_writes_packet_artifacts_and_timing_steps() { + let dir = tempfile::tempdir().unwrap(); + let csv = dir.path().join("counts.csv"); + write_toy_cells_csv(&csv, 40, 30); + let out = dir.path().join("out"); + + run_pipeline( + csv.to_str().unwrap(), + out.to_str().unwrap(), + 8, + 50, + 5, + 1, + 100.0, + 10.0, + false, + "seurat", + 42, + None, + None, + ) + .unwrap(); + + for name in [ + "qc_metrics.csv", + "hvg_genes.csv", + "pca_scores.csv", + "knn.csv", + "clusters.csv", + "markers.csv", + "pipeline_timings.json", + ] { + assert!(out.join(name).is_file(), "missing packet artifact {name}"); + } + + let timings: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(out.join("pipeline_timings.json")).unwrap(), + ) + .unwrap(); + assert_eq!(timings["pipeline"], "rustpipe-sc"); + let steps: Vec = timings["steps"] + .as_array() + .unwrap() + .iter() + .map(|s| s["step"].as_str().unwrap().to_string()) + .collect(); + for expected in [ + "load", + "qc_filter", + "normalize", + "hvg", + "scale", + "pca", + "knn", + "leiden", + "markers", + ] { + assert!( + steps.iter().any(|s| s == expected), + "missing timing step {expected} in {steps:?}" + ); + } + + let pca_header = std::fs::read_to_string(out.join("pca_scores.csv")) + .unwrap() + .lines() + .next() + .unwrap() + .to_string(); + // 40 cells, 8 HVG, request 50 → clamp to min(50, 39, 8) = 8 + assert!(pca_header.contains("PC8")); + assert!(!pca_header.contains("PC9")); + } +} diff --git a/src/markers.rs b/src/markers.rs index d22f055..9c3b3a9 100644 --- a/src/markers.rs +++ b/src/markers.rs @@ -141,11 +141,71 @@ fn extract_gene_columns(mat: &SpMat) -> Vec> { columns } +fn score_cluster( + cluster_id: usize, + gene_columns: &[Vec], + clusters: &[usize], + var_names: &[String], +) -> MarkerResult { + let n_cells = clusters.len(); + let n_genes = gene_columns.len(); + let is_cluster: Vec = clusters.iter().map(|&c| c == cluster_id).collect(); + let n_in = is_cluster.iter().filter(|&&b| b).count(); + let n_out = n_cells - n_in; + + let mut scores = Vec::with_capacity(n_genes); + let mut pvals = Vec::with_capacity(n_genes); + let mut log2fcs = Vec::with_capacity(n_genes); + + for g in 0..n_genes { + let vals = &gene_columns[g]; + let (score, pval) = wilcoxon_rank_sum(vals, &is_cluster); + + let mean_in: f64 = vals + .iter() + .enumerate() + .filter(|(i, _)| is_cluster[*i]) + .map(|(_, &v)| v as f64) + .sum::() + / n_in.max(1) as f64; + let mean_out: f64 = vals + .iter() + .enumerate() + .filter(|(i, _)| !is_cluster[*i]) + .map(|(_, &v)| v as f64) + .sum::() + / n_out.max(1) as f64; + + let log2fc = if mean_out > 1e-10 { + (mean_in / mean_out).log2() + } else if mean_in > 1e-10 { + 10.0 + } else { + 0.0 + }; + + scores.push(score); + pvals.push(pval); + log2fcs.push(log2fc); + } + + let mut pvals_adj = pvals.clone(); + stats_sc::bh_adjust(&mut pvals_adj); + + MarkerResult { + cluster_id, + gene_names: var_names.to_vec(), + scores, + pvals, + pvals_adj, + log2fc: log2fcs, + } +} + /// Compute marker genes for all clusters via Wilcoxon rank-sum test. /// -/// Operates on normalized (log1p) expression in sparse format. For each -/// cluster, tests every gene for differential expression between cells -/// in the cluster vs all other cells. +/// One cluster per rayon task. Ranking is deterministic (no RNG): same +/// top-N genes and order as the serial path. /// /// # Arguments /// * `mat` - Sparse CSR matrix (cells x genes), normalized log1p expression @@ -158,75 +218,38 @@ pub fn find_markers( var_names: &[String], _n_top: usize, ) -> Result> { - let n_cells = mat.rows(); - let n_genes = mat.cols(); - let n_clusters = *clusters.iter().max().unwrap_or(&0) + 1; + find_markers_impl(mat, clusters, var_names, true) +} - // Extract all gene columns in a single pass (CSR -> columnar) +/// Serial Wilcoxon markers (same ranking as `find_markers`; for tests). +pub fn find_markers_serial( + mat: &SpMat, + clusters: &[usize], + var_names: &[String], + _n_top: usize, +) -> Result> { + find_markers_impl(mat, clusters, var_names, false) +} + +fn find_markers_impl( + mat: &SpMat, + clusters: &[usize], + var_names: &[String], + parallel: bool, +) -> Result> { + let n_clusters = *clusters.iter().max().unwrap_or(&0) + 1; let gene_columns = extract_gene_columns(mat); - // For each cluster, compute markers with parallel gene iteration - let results: Vec = (0..n_clusters) - .map(|cluster_id| { - let is_cluster: Vec = clusters.iter().map(|&c| c == cluster_id).collect(); - let n_in = is_cluster.iter().filter(|&&b| b).count(); - let n_out = n_cells - n_in; - - // Parallel across genes: Wilcoxon test + fold change - let gene_results: Vec<(f64, f64, f64)> = (0..n_genes) - .into_par_iter() - .map(|g| { - let vals = &gene_columns[g]; - - // Wilcoxon rank-sum test - let (score, pval) = wilcoxon_rank_sum(vals, &is_cluster); - - // Log2 fold change: mean(cluster) / mean(rest) - let mean_in: f64 = vals - .iter() - .enumerate() - .filter(|(i, _)| is_cluster[*i]) - .map(|(_, &v)| v as f64) - .sum::() - / n_in.max(1) as f64; - let mean_out: f64 = vals - .iter() - .enumerate() - .filter(|(i, _)| !is_cluster[*i]) - .map(|(_, &v)| v as f64) - .sum::() - / n_out.max(1) as f64; - - let log2fc = if mean_out > 1e-10 { - (mean_in / mean_out).log2() - } else if mean_in > 1e-10 { - 10.0 // cap when denominator is ~zero - } else { - 0.0 - }; - - (score, pval, log2fc) - }) - .collect(); - - let scores: Vec = gene_results.iter().map(|r| r.0).collect(); - let pvals: Vec = gene_results.iter().map(|r| r.1).collect(); - let log2fcs: Vec = gene_results.iter().map(|r| r.2).collect(); - - // Benjamini-Hochberg FDR correction - let mut pvals_adj = pvals.clone(); - stats_sc::bh_adjust(&mut pvals_adj); - - MarkerResult { - cluster_id, - gene_names: var_names.to_vec(), - scores, - pvals, - pvals_adj, - log2fc: log2fcs, - } - }) - .collect(); + let results = if parallel { + (0..n_clusters) + .into_par_iter() + .map(|cluster_id| score_cluster(cluster_id, &gene_columns, clusters, var_names)) + .collect() + } else { + (0..n_clusters) + .map(|cluster_id| score_cluster(cluster_id, &gene_columns, clusters, var_names)) + .collect() + }; Ok(results) } @@ -496,4 +519,65 @@ mod tests { // Gene 1: [0, 2, 0] assert_eq!(cols[1], vec![0.0, 2.0, 0.0]); } + + fn ranked_top_genes(result: &MarkerResult, n_top: usize) -> Vec { + let mut indices: Vec = (0..result.gene_names.len()).collect(); + indices.sort_by(|&a, &b| { + result.pvals_adj[a] + .partial_cmp(&result.pvals_adj[b]) + .unwrap_or(std::cmp::Ordering::Equal) + .then( + result.log2fc[b] + .abs() + .partial_cmp(&result.log2fc[a].abs()) + .unwrap_or(std::cmp::Ordering::Equal), + ) + }); + indices + .iter() + .take(n_top) + .map(|&i| result.gene_names[i].clone()) + .collect() + } + + #[test] + fn test_find_markers_parallel_matches_serial() { + // 12 cells x 5 genes, 3 clusters — small enough to compare bit-stable ranks. + let mat = sparse::from_triplets( + 12, + 5, + &[ + 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, + ], + &[ + 0, 1, 0, 1, 0, 2, 1, 2, 1, 3, 2, 3, 2, 4, 3, 4, 3, 0, 4, 0, 4, 1, 0, 4, + ], + &[ + 9.0, 1.0, 8.0, 2.0, 10.0, 1.0, 2.0, 8.0, 1.0, 9.0, 2.0, 7.0, 8.0, 1.0, 9.0, 2.0, + 7.0, 1.0, 10.0, 1.0, 8.0, 2.0, 1.0, 9.0, + ], + ); + let clusters = vec![0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2]; + let names: Vec = (0..5).map(|g| format!("G{g}")).collect(); + + let parallel = find_markers(&mat, &clusters, &names, 3).unwrap(); + let serial = find_markers_serial(&mat, &clusters, &names, 3).unwrap(); + assert_eq!(parallel.len(), serial.len()); + for (p, s) in parallel.iter().zip(serial.iter()) { + assert_eq!(p.cluster_id, s.cluster_id); + assert_eq!(ranked_top_genes(p, 3), ranked_top_genes(s, 3)); + for i in 0..p.gene_names.len() { + assert!((p.scores[i] - s.scores[i]).abs() < 1e-10); + assert!((p.pvals[i] - s.pvals[i]).abs() < 1e-12); + assert!((p.pvals_adj[i] - s.pvals_adj[i]).abs() < 1e-12); + let same_fc = + p.log2fc[i] == s.log2fc[i] || ((p.log2fc[i] - s.log2fc[i]).abs() < 1e-10); + assert!( + same_fc, + "log2fc[{}] cluster {} parallel={} serial={}", + i, p.cluster_id, p.log2fc[i], s.log2fc[i] + ); + } + } + } } diff --git a/src/normalize.rs b/src/normalize.rs index 113e840..877009a 100644 --- a/src/normalize.rs +++ b/src/normalize.rs @@ -50,6 +50,108 @@ pub fn normalize_log1p_sparse(mat: &SpMat, target_sum: f32) -> SpMat { CsMat::new((n_cells, n_genes), indptr, indices, data) } +/// Pass 2 of the fused prep: subset kept cells/genes, optionally +/// normalize+log1p, and accumulate per-gene mean/variance. +/// +/// One scan over kept-cell nonzeros. Cell totals use only kept genes +/// (same biology as `filter_genes` then `normalize_log1p_sparse`). +/// Variance uses Bessel correction, matching `sparse_gene_stats`. +pub fn normalize_log1p_subset_with_stats( + mat: &SpMat, + keep_cells: &[bool], + keep_genes: &[bool], + target_sum: f32, + skip_normalize: bool, +) -> (SpMat, Vec, Vec) { + let n_new_cells = keep_cells.iter().filter(|&&b| b).count(); + let n_new_genes = keep_genes.iter().filter(|&&b| b).count(); + + let mut col_map = vec![None; mat.cols()]; + let mut new_idx = 0usize; + for (orig, &kept) in keep_genes.iter().enumerate() { + if kept { + col_map[orig] = Some(new_idx); + new_idx += 1; + } + } + + let mut indptr: Vec = Vec::with_capacity(n_new_cells + 1); + let mut indices: Vec = Vec::with_capacity(mat.nnz()); + let mut data: Vec = Vec::with_capacity(mat.nnz()); + indptr.push(0); + + let mut sums = vec![0.0f64; n_new_genes]; + let mut sum_sq = vec![0.0f64; n_new_genes]; + + for i in 0..mat.rows() { + if !keep_cells[i] { + continue; + } + let row = mat.outer_view(i).unwrap(); + + if skip_normalize { + for (col, &val) in row.iter() { + if val > 0.0 { + if let Some(new_col) = col_map[col] { + indices.push(new_col); + data.push(val); + let v = val as f64; + sums[new_col] += v; + sum_sq[new_col] += v * v; + } + } + } + } else { + let mut cell_total = 0.0f32; + for (col, &val) in row.iter() { + if val > 0.0 && keep_genes[col] { + cell_total += val; + } + } + if cell_total > 0.0 { + let scale = target_sum / cell_total; + for (col, &val) in row.iter() { + if val > 0.0 { + if let Some(new_col) = col_map[col] { + let normalized = (val * scale + 1.0).ln(); + if normalized > 0.0 { + indices.push(new_col); + data.push(normalized); + let v = normalized as f64; + sums[new_col] += v; + sum_sq[new_col] += v * v; + } + } + } + } + } + } + indptr.push(indices.len()); + } + + let n = n_new_cells as f64; + let means: Vec = if n > 0.0 { + sums.iter().map(|&s| (s / n) as f32).collect() + } else { + vec![0.0; n_new_genes] + }; + let variances: Vec = if n > 1.0 { + sums.iter() + .zip(sum_sq.iter()) + .map(|(&s, &sq)| { + let mean = s / n; + let var = (sq - n * mean * mean) / (n - 1.0); + var.max(0.0) as f32 + }) + .collect() + } else { + vec![0.0; n_new_genes] + }; + + let out = CsMat::new((n_new_cells, n_new_genes), indptr, indices, data); + (out, means, variances) +} + /// Normalize without log transform (just scale to target_sum per cell). /// Builds output CSR directly (no TriMat overhead). #[allow(dead_code)] @@ -151,4 +253,35 @@ mod tests { assert!((val0 - 2000.0).abs() < 0.1); assert!((val1 - 8000.0).abs() < 0.1); } + + #[test] + fn test_subset_with_stats_matches_normalize() { + let mat = from_triplets( + 2, + 3, + &[0, 0, 1, 1], + &[0, 2, 0, 1], + &[100.0, 200.0, 300.0, 100.0], + ); + let keep_cells = vec![true, true]; + let keep_genes = vec![true, true, true]; + let (fused, means, vars) = + normalize_log1p_subset_with_stats(&mat, &keep_cells, &keep_genes, 1e4, false); + let seq = normalize_log1p_sparse(&mat, 1e4); + let (seq_means, seq_vars) = crate::sparse::sparse_gene_stats(&seq, seq.rows()); + + assert_eq!(fused.nnz(), seq.nnz()); + for i in 0..2 { + let ra = fused.outer_view(i).unwrap(); + let rb = seq.outer_view(i).unwrap(); + for ((c1, &v1), (c2, &v2)) in ra.iter().zip(rb.iter()) { + assert_eq!(c1, c2); + assert!((v1 - v2).abs() < 1e-6); + } + } + for i in 0..3 { + assert!((means[i] - seq_means[i]).abs() < 1e-6); + assert!((vars[i] - seq_vars[i]).abs() < 1e-5); + } + } } diff --git a/src/pca.rs b/src/pca.rs index 91fe62d..91f89ae 100644 --- a/src/pca.rs +++ b/src/pca.rs @@ -582,10 +582,7 @@ pub fn run_pca_sparse( let total_t0 = Instant::now(); let n_cells = mat.rows(); let n_genes = mat.cols(); - let n_components = n_components - .min(n_cells.saturating_sub(1)) - .min(n_genes.saturating_sub(1)) - .max(1); + let n_components = crate::sparse::clamp_n_pcs(n_components, n_cells, n_genes); let k = (n_components + oversampling).min(n_genes).min(n_cells); eprintln!( @@ -740,10 +737,7 @@ pub fn run_pca_scaled( let total_t0 = Instant::now(); let n_cells = scaled.nrows(); let n_genes = scaled.ncols(); - let n_components = n_components - .min(n_cells.saturating_sub(1)) - .min(n_genes.saturating_sub(1)) - .max(1); + let n_components = crate::sparse::clamp_n_pcs(n_components, n_cells, n_genes); let k = (n_components + oversampling).min(n_genes).min(n_cells); eprintln!( @@ -849,6 +843,119 @@ pub fn run_pca_scaled( } } +/// Randomized SVD on a sparse HVG matrix with implicit scale+clip+center. +/// +/// Same power-iteration Halko–Martinsson–Tropp scheme as `run_pca_scaled`, +/// but never densifies the HVG matrix. Scaling matches `scale::scale_sparse`. +pub fn run_pca_sparse_scaled( + mat: &crate::sparse::SpMat, + means: &[f32], + stds: &[f32], + max_value: f32, + gene_names: &[String], + n_components: usize, + n_power_iter: usize, + oversampling: usize, + seed: u64, +) -> PcaResult { + use crate::sparse::{ + clamp_n_pcs, scaled_clipped_frobenius_sq, spmm_at_scaled_clipped, spmm_scaled_clipped, + }; + use ndarray::Array2; + use rand::SeedableRng; + use rand_distr::StandardNormal; + + let total_t0 = Instant::now(); + let n_cells = mat.rows(); + let n_genes = mat.cols(); + let n_components = clamp_n_pcs(n_components, n_cells, n_genes); + let k = (n_components + oversampling).min(n_genes).min(n_cells); + + eprintln!( + "[pca-sparse] {}x{} HVG, {} components, {} power iterations (implicit scale+clip)", + n_cells, n_genes, n_components, n_power_iter + ); + + let svd_t0 = Instant::now(); + let mut rng = rand::rngs::StdRng::seed_from_u64(seed); + let omega = Array2::from_shape_fn((n_genes, k), |_| { + use rand::Rng; + rng.sample::(StandardNormal) + }); + + let mut y = spmm_scaled_clipped(mat, means, stds, max_value, &omega); + + for _iter in 0..n_power_iter { + let (q_flat, _) = qr_mgs_f32(&y); + let q = Array2::from_shape_vec((n_cells, k), q_flat).unwrap(); + let z = spmm_at_scaled_clipped(mat, means, stds, max_value, &q); + let (z_q_flat, _) = qr_mgs_f32(&z); + let z_q = Array2::from_shape_vec((n_genes, k), z_q_flat).unwrap(); + y = spmm_scaled_clipped(mat, means, stds, max_value, &z_q); + } + + let (q_flat, _) = qr_mgs_f32(&y); + let q = Array2::from_shape_vec((n_cells, k), q_flat).unwrap(); + let b_t = spmm_at_scaled_clipped(mat, means, stds, max_value, &q); + + let k_actual = k.min(n_genes).min(n_cells); + let n_components = n_components.min(k_actual).max(1); + let mut b_flat = vec![0.0f64; k_actual * n_genes]; + for i in 0..k_actual { + for j in 0..n_genes { + b_flat[i * n_genes + j] = b_t[[j, i]] as f64; + } + } + + let (u_b, sigma, _vt) = small_svd(&b_flat, k_actual, n_genes); + + let mut pc_scores: Vec> = Vec::with_capacity(n_cells); + for c in 0..n_cells { + let mut row = Vec::with_capacity(n_components); + for pc in 0..n_components { + let mut dot = 0.0f64; + for j in 0..k_actual { + dot += q[[c, j]] as f64 * u_b[j * k_actual + pc]; + } + row.push(dot * sigma[pc]); + } + pc_scores.push(row); + } + + let total_var = scaled_clipped_frobenius_sq(mat, means, stds, max_value); + let var_explained: Vec = if total_var > 0.0 { + sigma[..n_components] + .iter() + .map(|s| (s * s) / total_var) + .collect() + } else { + vec![0.0; n_components] + }; + + let svd_ms = svd_t0.elapsed().as_secs_f64() * 1000.0; + let total_ms = total_t0.elapsed().as_secs_f64() * 1000.0; + + eprintln!("[pca-sparse] SVD: {:.1}ms", svd_ms); + eprintln!("[pca-sparse] total: {:.1}ms", total_ms); + eprintln!( + "[pca-sparse] variance captured: {:.2}%", + var_explained.iter().sum::() * 100.0 + ); + + PcaResult { + pc_scores, + variance_explained: var_explained, + selected_genes: gene_names.to_vec(), + timings: PcaTimings { + variance_calc_ms: 0.0, + gene_selection_ms: 0.0, + centering_ms: 0.0, + svd_ms, + total_ms, + }, + } +} + /// QR decomposition via modified Gram-Schmidt for f32 ndarray. /// Returns (q_flat_f32, r_flat_f32) as Vec. fn qr_mgs_f32(a: &ndarray::Array2) -> (Vec, Vec) { @@ -982,4 +1089,62 @@ mod tests { assert!((sigma[0] - 3.0).abs() < 1e-8, "sigma[0] = {}", sigma[0]); assert!(sigma[1].abs() < 1e-8, "sigma[1] = {}", sigma[1]); } + + #[test] + fn n_pcs_clamp_tiny_post_qc_does_not_panic() { + // Locally seen: 21 cells remaining after tight QC, default n_pcs=50. + let n_cells = 21; + let n_hvg = 15; + let mut rows = Vec::new(); + let mut cols = Vec::new(); + let mut vals = Vec::new(); + for c in 0..n_cells { + for g in 0..n_hvg { + let v = ((c * 3 + g * 5) % 11) as f32 + 1.0; + rows.push(c); + cols.push(g); + vals.push(v); + } + } + let mat = crate::sparse::from_triplets(n_cells, n_hvg, &rows, &cols, &vals); + let names: Vec = (0..n_hvg).map(|g| format!("G{g}")).collect(); + let (means, stds) = crate::scale::scale_stats(&mat); + let res = run_pca_sparse_scaled(&mat, &means, &stds, 10.0, &names, 50, 2, 4, 42); + assert_eq!(res.pc_scores.len(), n_cells); + assert_eq!( + res.pc_scores[0].len(), + crate::sparse::clamp_n_pcs(50, n_cells, n_hvg) + ); + assert_eq!(res.variance_explained.len(), 15); + } + + #[test] + fn sparse_scaled_pca_matches_dense_scaled() { + let mat = crate::sparse::from_triplets( + 8, + 6, + &[0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7], + &[0, 2, 1, 3, 0, 4, 1, 5, 2, 3, 0, 5, 1, 4, 2, 5], + &[ + 4.0, 1.0, 3.0, 2.0, 5.0, 1.0, 2.0, 4.0, 3.0, 1.0, 6.0, 2.0, 1.0, 3.0, 2.0, 5.0, + ], + ); + let names: Vec = (0..6).map(|g| format!("G{g}")).collect(); + let (means, stds) = crate::scale::scale_stats(&mat); + let (dense, _, _) = crate::scale::scale_sparse(&mat, 10.0); + let sparse_res = run_pca_sparse_scaled(&mat, &means, &stds, 10.0, &names, 3, 2, 2, 7); + let dense_res = run_pca_scaled(&dense, &names, 3, 2, 2, 7); + assert_eq!(sparse_res.pc_scores.len(), dense_res.pc_scores.len()); + for (i, (a, b)) in sparse_res + .pc_scores + .iter() + .zip(dense_res.pc_scores.iter()) + .enumerate() + { + assert_eq!(a.len(), b.len()); + for (j, (x, y)) in a.iter().zip(b.iter()).enumerate() { + assert!((x - y).abs() < 1e-3, "PC[{i},{j}] sparse={x} dense={y}"); + } + } + } } diff --git a/src/qc.rs b/src/qc.rs index 3a12bba..66e85b1 100644 --- a/src/qc.rs +++ b/src/qc.rs @@ -65,6 +65,82 @@ pub fn compute_qc_metrics(mat: &SpMat, var_names: &[String], obs_names: &[String } } +/// Pass 1 of the fused prep: QC metrics, cell keep mask, and per-gene +/// detection counts among kept cells. One scan over CSR nonzeros. +/// +/// Equivalent to `compute_qc_metrics` + `filter_cells_fixed` + the gene +/// detection half of `filter_genes` on the cell-filtered matrix. +pub fn qc_metrics_and_kept_gene_counts( + mat: &SpMat, + var_names: &[String], + obs_names: &[String], + min_genes: u32, + max_pct_mt: f32, +) -> (QcMetrics, Vec, Vec) { + let n_cells = mat.rows(); + let n_genes = mat.cols(); + + let is_mito: Vec = var_names + .iter() + .map(|g| g.starts_with("MT-") || g.starts_with("mt-")) + .collect(); + + let mut n_genes_by_counts = Vec::with_capacity(n_cells); + let mut total_counts = Vec::with_capacity(n_cells); + let mut pct_counts_mt = Vec::with_capacity(n_cells); + let mut keep = Vec::with_capacity(n_cells); + let mut gene_cell_counts = vec![0usize; n_genes]; + + for i in 0..n_cells { + let row = mat.outer_view(i).unwrap(); + let mut n_genes_i = 0u32; + let mut total = 0.0f32; + let mut mito_total = 0.0f32; + + for (col, &val) in row.iter() { + if val > 0.0 { + n_genes_i += 1; + total += val; + if is_mito[col] { + mito_total += val; + } + } + } + + let pct_mt = if total > 0.0 { + (mito_total / total) * 100.0 + } else { + 0.0 + }; + n_genes_by_counts.push(n_genes_i); + total_counts.push(total); + pct_counts_mt.push(pct_mt); + + let keep_cell = n_genes_i >= min_genes && pct_mt <= max_pct_mt; + keep.push(keep_cell); + if keep_cell { + for (col, &val) in row.iter() { + if val > 0.0 { + gene_cell_counts[col] += 1; + } + } + } + } + + let metrics = QcMetrics { + barcodes: obs_names.to_vec(), + n_genes_by_counts, + total_counts, + pct_counts_mt, + }; + (metrics, keep, gene_cell_counts) +} + +/// Gene keep mask from per-gene detection counts (`count >= min_cells`). +pub fn gene_keep_from_counts(gene_cell_counts: &[usize], min_cells: usize) -> Vec { + gene_cell_counts.iter().map(|&c| c >= min_cells).collect() +} + /// Filter cells based on fixed thresholds. /// /// Returns a boolean mask: `true` = keep cell. @@ -208,7 +284,7 @@ mod tests { #[test] fn test_apply_cell_filter() { - let (mat, genes, cells) = toy_sc_matrix(); + let (mat, _genes, cells) = toy_sc_matrix(); let keep = vec![true, false, true, false]; let (filtered, filtered_names) = apply_cell_filter(&mat, &cells, &keep); @@ -227,4 +303,29 @@ mod tests { // MYC (index 3) should not be in kept assert!(!kept.contains(&3)); } + + #[test] + fn test_qc_pass1_matches_sequential() { + let (mat, genes, cells) = toy_sc_matrix(); + let sequential = compute_qc_metrics(&mat, &genes, &cells); + let keep_seq = filter_cells_fixed(&sequential, 3, 100.0); + let (filtered, _) = apply_cell_filter(&mat, &cells, &keep_seq); + let (_fg, kept_seq) = filter_genes(&filtered, 2); + + let (fused_qc, keep, counts) = + qc_metrics_and_kept_gene_counts(&mat, &genes, &cells, 3, 100.0); + let keep_genes = gene_keep_from_counts(&counts, 2); + let kept_fused: Vec = keep_genes + .iter() + .enumerate() + .filter(|(_, &k)| k) + .map(|(i, _)| i) + .collect(); + + assert_eq!(fused_qc.n_genes_by_counts, sequential.n_genes_by_counts); + assert_eq!(fused_qc.total_counts, sequential.total_counts); + assert_eq!(fused_qc.pct_counts_mt, sequential.pct_counts_mt); + assert_eq!(keep, keep_seq); + assert_eq!(kept_fused, kept_seq); + } } diff --git a/src/scale.rs b/src/scale.rs index af182c4..0059cc7 100644 --- a/src/scale.rs +++ b/src/scale.rs @@ -25,12 +25,10 @@ use crate::sparse::SpMat; /// * `scaled` - dense cells x genes Array2 /// * `means` - per-gene population means (length n_genes) /// * `stds` - per-gene population stds (length n_genes) -pub fn scale_sparse(mat: &SpMat, max_value: f32) -> (Array2, Vec, Vec) { +/// Per-gene population mean and std (ddof=0), without densifying. +pub fn scale_stats(mat: &SpMat) -> (Vec, Vec) { let n_cells = mat.rows(); let n_genes = mat.cols(); - - // Step 1: Compute per-gene sum and sum-of-squares from sparse data. - // Use population statistics (ddof=0) to match Scanpy. let mut sums = vec![0.0f64; n_genes]; let mut sum_sq = vec![0.0f64; n_genes]; @@ -50,13 +48,18 @@ pub fn scale_sparse(mat: &SpMat, max_value: f32) -> (Array2, Vec, Vec< .zip(sum_sq.iter()) .map(|(&s, &sq)| { let mean = s / n; - // Population variance: E[x^2] - E[x]^2 let var = (sq / n) - mean * mean; let std = var.max(0.0).sqrt(); - // Floor at 1e-12 to avoid division by zero std.max(1e-12) as f32 }) .collect(); + (means, stds) +} + +pub fn scale_sparse(mat: &SpMat, max_value: f32) -> (Array2, Vec, Vec) { + let n_cells = mat.rows(); + let n_genes = mat.cols(); + let (means, stds) = scale_stats(mat); // Step 2: Build dense scaled matrix. // Most entries are zero in the sparse matrix, so we first fill the entire diff --git a/src/sparse.rs b/src/sparse.rs index 654ea7f..ae28a86 100644 --- a/src/sparse.rs +++ b/src/sparse.rs @@ -93,6 +93,122 @@ pub fn cell_n_genes(mat: &SpMat) -> Vec { .collect() } +/// Clamp PCA rank: `min(requested, n_cells-1, n_hvg)`, at least 1. +pub fn clamp_n_pcs(requested: usize, n_cells: usize, n_hvg: usize) -> usize { + requested.min(n_cells.saturating_sub(1)).min(n_hvg).max(1) +} + +/// Clipped z-score of implicit zeros for each gene (population scale). +pub fn scaled_zero_z(means: &[f32], stds: &[f32], max_value: f32) -> Vec { + means + .iter() + .zip(stds.iter()) + .map(|(&m, &s)| (-m / s).clamp(-max_value, max_value)) + .collect() +} + +/// Y = scale_clip(A) * Omega without materializing the dense scaled matrix. +/// +/// `scale_clip` is Scanpy `sc.pp.scale`: (x - mean) / std, clipped to +/// `[-max_value, max_value]`. Implicit zeros contribute `clip(-mean/std)`. +pub fn spmm_scaled_clipped( + mat: &SpMat, + means: &[f32], + stds: &[f32], + max_value: f32, + omega: &Array2, +) -> Array2 { + let n_cells = mat.rows(); + let n_genes = mat.cols(); + let k = omega.ncols(); + let z0 = scaled_zero_z(means, stds, max_value); + + let correction: Vec = (0..k) + .map(|j| (0..n_genes).map(|g| z0[g] * omega[[g, j]]).sum()) + .collect(); + + let mut y = Array2::::zeros((n_cells, k)); + for i in 0..n_cells { + for j in 0..k { + y[[i, j]] = correction[j]; + } + let row = mat.outer_view(i).unwrap(); + for (g, &val) in row.iter() { + let z = ((val - means[g]) / stds[g]).clamp(-max_value, max_value); + let delta = z - z0[g]; + if delta == 0.0 { + continue; + } + for j in 0..k { + y[[i, j]] += delta * omega[[g, j]]; + } + } + } + y +} + +/// Z = scale_clip(A)^T * Q without densifying. +pub fn spmm_at_scaled_clipped( + mat: &SpMat, + means: &[f32], + stds: &[f32], + max_value: f32, + q: &Array2, +) -> Array2 { + let n_genes = mat.cols(); + let n_cells = mat.rows(); + let k = q.ncols(); + let z0 = scaled_zero_z(means, stds, max_value); + let col_sums: Array1 = q.sum_axis(Axis(0)); + + let mut z = Array2::::zeros((n_genes, k)); + for g in 0..n_genes { + for j in 0..k { + z[[g, j]] = z0[g] * col_sums[j]; + } + } + + for i in 0..n_cells { + let row = mat.outer_view(i).unwrap(); + for (g, &val) in row.iter() { + let zg = ((val - means[g]) / stds[g]).clamp(-max_value, max_value); + let delta = zg - z0[g]; + if delta == 0.0 { + continue; + } + for j in 0..k { + z[[g, j]] += delta * q[[i, j]]; + } + } + } + z +} + +/// Frobenius norm squared of the scaled+clipped matrix, without densifying. +pub fn scaled_clipped_frobenius_sq( + mat: &SpMat, + means: &[f32], + stds: &[f32], + max_value: f32, +) -> f64 { + let n_cells = mat.rows(); + let n_genes = mat.cols(); + let z0 = scaled_zero_z(means, stds, max_value); + let mut total = 0.0f64; + for g in 0..n_genes { + total += n_cells as f64 * (z0[g] as f64) * (z0[g] as f64); + } + for i in 0..n_cells { + let row = mat.outer_view(i).unwrap(); + for (g, &val) in row.iter() { + let z = ((val - means[g]) / stds[g]).clamp(-max_value, max_value); + let z0g = z0[g]; + total += (z as f64) * (z as f64) - (z0g as f64) * (z0g as f64); + } + } + total.max(0.0) +} + /// Y = (A - mu) * Omega — sparse-dense matmul with implicit centering. /// /// Computes `A * Omega` (sparse x dense) then subtracts the rank-1 correction @@ -358,6 +474,44 @@ mod tests { } } + #[test] + fn test_clamp_n_pcs() { + // Tiny post-QC matrix: 21 cells, 2000 HVG, request 50 → 20 + assert_eq!(clamp_n_pcs(50, 21, 2000), 20); + // Rank limited by HVG count + assert_eq!(clamp_n_pcs(50, 21, 15), 15); + assert_eq!(clamp_n_pcs(50, 100, 40), 40); + assert_eq!(clamp_n_pcs(10, 100, 2000), 10); + assert_eq!(clamp_n_pcs(0, 21, 15), 1); + assert_eq!(clamp_n_pcs(50, 1, 100), 1); + } + + #[test] + fn test_spmm_scaled_clipped_vs_dense() { + let mat = toy_matrix(); + let (means, stds) = crate::scale::scale_stats(&mat); + let max_value = 10.0f32; + let omega = + Array2::from_shape_vec((4, 2), vec![1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0]).unwrap(); + + let y_sparse = spmm_scaled_clipped(&mat, &means, &stds, max_value, &omega); + let (dense_scaled, _, _) = crate::scale::scale_sparse(&mat, max_value); + let y_dense = dense_scaled.dot(&omega); + + for i in 0..3 { + for j in 0..2 { + assert!( + (y_sparse[[i, j]] - y_dense[[i, j]]).abs() < 1e-4, + "mismatch at [{}, {}]: sparse={}, dense={}", + i, + j, + y_sparse[[i, j]], + y_dense[[i, j]] + ); + } + } + } + #[test] fn test_subset_genes() { let mat = toy_matrix();