diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c1a47b..1a596c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,3 +16,28 @@ jobs: - uses: dtolnay/rust-toolchain@stable - name: Test (CSV + fail-closed; no HDF5) run: cargo test + + - name: Install HDF5 + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libhdf5-dev pkg-config + + - name: Test (HDF5 + 10x fixture pipeline) + run: cargo test --features hdf5 + + - name: Pipeline H5 fixture writes PACKET.md artifacts + run: | + cargo build --features hdf5 + ./target/debug/rustpipe-sc pipeline \ + --input testdata/filtered_feature_bc_matrix.h5 \ + --output packet-out + test -s packet-out/qc_metrics.csv + test -s packet-out/pca_scores.csv + test -s packet-out/clusters.csv + test -s packet-out/markers.csv + test -s packet-out/pipeline_timings.json + grep -q '"pipeline": "rustpipe-sc"' packet-out/pipeline_timings.json + + - name: Upload PACKET artifacts + uses: actions/upload-artifact@v4 + with: + name: packet + path: packet-out/ diff --git a/README.md b/README.md index 19ab705..0a926b0 100644 --- a/README.md +++ b/README.md @@ -225,10 +225,14 @@ nf-core/scrnaseq (cellranger/alevin/starsolo) ## Testing ```bash -cargo test # 104 unit tests, ~0.06s -cargo test --features hdf5 # Includes H5AD I/O tests (requires HDF5) +cargo test # unit tests, CSV + fail-closed (no HDF5) +cargo test --features hdf5 # + 10x H5 fixture through `pipeline` (needs libhdf5) ``` +CI installs `libhdf5-dev` and runs default `rustpipe-sc pipeline` on +`testdata/filtered_feature_bc_matrix.h5` (synthetic Cell Ranger `/matrix`, +not a public PBMC file). That job must write the PACKET.md artifacts. + ## Known Limitations - kNN is brute-force O(n^2): fast up to ~50K cells, needs approximate NN (HNSW) for larger diff --git a/scripts/gen_10x_ci_fixture.py b/scripts/gen_10x_ci_fixture.py new file mode 100755 index 0000000..b020fad --- /dev/null +++ b/scripts/gen_10x_ci_fixture.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Write testdata/filtered_feature_bc_matrix.h5 — tiny synthetic 10x /matrix H5. + +Layout matches Cell Ranger filtered_feature_bc_matrix.h5 so rustpipe-sc +load_input → read_10x_h5 runs. Sized to pass default pipeline flags: + + --min-genes 200 --max-pct-mt 5 --n-hvg 2000 --n-pcs 50 --knn-k 15 + +Not a public PBMC file. Regenerates the committed binary; do not vendor RustQC. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import h5py +import numpy as np + +# Human MT- prefixes so qc.rs pct_counts_mt is real and stays well under 5%. +MT_NAMES = ( + "MT-ND1", + "MT-ND2", + "MT-CO1", + "MT-CO2", + "MT-ATP6", + "MT-ATP8", + "MT-CYB", + "MT-ND4", +) + +N_CELLS = 80 +N_GENES = 2120 # 8 MT + 2112 others; default --n-hvg 2000 needs ≥2000 after min_cells=3 +N_GENES_PER_CELL = 240 +SEED = 42 + + +def gene_names() -> list[bytes]: + names = [n.encode("ascii") for n in MT_NAMES] + for i in range(len(MT_NAMES), N_GENES): + names.append(f"GENE{i:04d}".encode("ascii")) + return names + + +def barcodes() -> list[bytes]: + # 16-base + "-1", Cell Ranger style, unique and deterministic. + out = [] + alphabet = b"ACGT" + for i in range(N_CELLS): + seq = bytearray(16) + n = i + for pos in range(16): + seq[pos] = alphabet[n & 3] + n >>= 2 + out.append(bytes(seq) + b"-1") + return out + + +def build_csc(rng: np.random.Generator) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """CSC genes × cells. Every gene in ≥3 cells; every cell has 240 genes.""" + cells_for_gene: list[set[int]] = [set() for _ in range(N_GENES)] + genes_for_cell: list[set[int]] = [set() for _ in range(N_CELLS)] + + # Round-robin so min_cells=3 holds for all genes (HVG default 2000). + for g in range(N_GENES): + for k in range(3): + c = (g + k * 17) % N_CELLS + cells_for_gene[g].add(c) + genes_for_cell[c].add(g) + + for c in range(N_CELLS): + while len(genes_for_cell[c]) < N_GENES_PER_CELL: + g = int(rng.integers(0, N_GENES)) + genes_for_cell[c].add(g) + cells_for_gene[g].add(c) + + data: list[int] = [] + indices: list[int] = [] + indptr = np.zeros(N_CELLS + 1, dtype=np.int64) + for c in range(N_CELLS): + genes = sorted(genes_for_cell[c]) + for g in genes: + if g < len(MT_NAMES): + count = 1 # ~0.2% MT if the rest average ~4 UMIs + else: + count = 1 + int(rng.integers(0, 8)) + data.append(count) + indices.append(g) + indptr[c + 1] = len(data) + + return ( + np.asarray(data, dtype=np.int32), + np.asarray(indices, dtype=np.int32), + indptr, + ) + + +def write_h5(path: Path) -> None: + rng = np.random.default_rng(SEED) + data, indices, indptr = build_csc(rng) + names = gene_names() + bc = barcodes() + ids = [f"ENSG{i:011d}".encode("ascii") for i in range(N_GENES)] + feature_type = [b"Gene Expression"] * N_GENES + genome = [b"GRCh38"] * N_GENES + + path.parent.mkdir(parents=True, exist_ok=True) + with h5py.File(path, "w") as f: + matrix = f.create_group("matrix") + matrix.create_dataset("shape", data=np.array([N_GENES, N_CELLS], dtype=np.int64)) + matrix.create_dataset("data", data=data) + matrix.create_dataset("indices", data=indices) + matrix.create_dataset("indptr", data=indptr) + matrix.create_dataset("barcodes", data=np.array(bc, dtype="S18")) + features = matrix.create_group("features") + features.create_dataset("name", data=np.array(names, dtype="S16")) + features.create_dataset("id", data=np.array(ids, dtype="S15")) + features.create_dataset("feature_type", data=np.array(feature_type, dtype="S16")) + features.create_dataset("genome", data=np.array(genome, dtype="S8")) + + +def main() -> None: + root = Path(__file__).resolve().parents[1] + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "-o", + "--output", + type=Path, + default=root / "testdata" / "filtered_feature_bc_matrix.h5", + ) + args = parser.parse_args() + write_h5(args.output) + size = args.output.stat().st_size + print(f"wrote {args.output} ({size} bytes) {N_CELLS} cells x {N_GENES} genes") + + +if __name__ == "__main__": + main() diff --git a/testdata/README.md b/testdata/README.md new file mode 100644 index 0000000..e61c992 --- /dev/null +++ b/testdata/README.md @@ -0,0 +1,17 @@ +# CI 10x H5 fixture + +`filtered_feature_bc_matrix.h5` is a **synthetic** Cell Ranger `/matrix` file +(80 cells × 2120 genes). It is not a public PBMC download. + +Regenerate: + +```bash +python3 scripts/gen_10x_ci_fixture.py +``` + +Sized so default `rustpipe-sc pipeline` flags succeed: + +- each cell has 240 detected genes (`--min-genes 200`) +- `MT-*` genes are named and low-count (`--max-pct-mt 5`) +- ≥2000 genes pass `min_cells=3` (`--n-hvg 2000`) +- 80 cells (`--knn-k 15`; `n_pcs` already clamps to rank) diff --git a/testdata/filtered_feature_bc_matrix.h5 b/testdata/filtered_feature_bc_matrix.h5 new file mode 100644 index 0000000..6c39156 Binary files /dev/null and b/testdata/filtered_feature_bc_matrix.h5 differ diff --git a/tests/h5_ci_fixture.rs b/tests/h5_ci_fixture.rs new file mode 100644 index 0000000..d455292 --- /dev/null +++ b/tests/h5_ci_fixture.rs @@ -0,0 +1,58 @@ +//! Exercise `rustpipe-sc pipeline` on the committed 10x H5 fixture. +//! +//! Gated on `--features hdf5`. Without HDF5 the binary cannot read `.h5`. + +#[cfg(feature = "hdf5")] +mod with_hdf5 { + use std::path::PathBuf; + use std::process::Command; + + fn fixture() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("testdata/filtered_feature_bc_matrix.h5") + } + + #[test] + fn pipeline_h5_writes_packet_artifacts() { + let fixture = fixture(); + assert!( + fixture.is_file(), + "missing committed 10x fixture at {}", + fixture.display() + ); + + let out = tempfile::tempdir().expect("tempdir"); + let status = Command::new(env!("CARGO_BIN_EXE_rustpipe-sc")) + .args([ + "pipeline", + "--input", + fixture.to_str().unwrap(), + "--output", + out.path().to_str().unwrap(), + ]) + .status() + .expect("spawn rustpipe-sc"); + assert!(status.success(), "pipeline exited {status}"); + + for name in [ + "qc_metrics.csv", + "pca_scores.csv", + "clusters.csv", + "markers.csv", + "pipeline_timings.json", + ] { + let path = out.path().join(name); + assert!(path.is_file(), "PACKET.md artifact missing: {name}"); + assert!( + path.metadata().unwrap().len() > 0, + "PACKET.md artifact empty: {name}" + ); + } + + let timings = std::fs::read_to_string(out.path().join("pipeline_timings.json")).unwrap(); + assert!( + timings.contains("\"pipeline\": \"rustpipe-sc\""), + "pipeline_timings.json must name rustpipe-sc" + ); + } +}