Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,3 +16,28 @@ jobs:
- uses: dtolnay/rust-toolchain@stable
- name: Test (CSV + fail-closed; no HDF5)
run: cargo test

- name: Install HDF5
run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libhdf5-dev pkg-config

- name: Test (HDF5 + 10x fixture pipeline)
run: cargo test --features hdf5

- name: Pipeline H5 fixture writes PACKET.md artifacts
run: |
cargo build --features hdf5
./target/debug/rustpipe-sc pipeline \
--input testdata/filtered_feature_bc_matrix.h5 \
--output packet-out
test -s packet-out/qc_metrics.csv
test -s packet-out/pca_scores.csv
test -s packet-out/clusters.csv
test -s packet-out/markers.csv
test -s packet-out/pipeline_timings.json
grep -q '"pipeline": "rustpipe-sc"' packet-out/pipeline_timings.json

- name: Upload PACKET artifacts
uses: actions/upload-artifact@v4
with:
name: packet
path: packet-out/
8 changes: 6 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -225,10 +225,14 @@ nf-core/scrnaseq (cellranger/alevin/starsolo)
## Testing

```bash
cargo test # 104 unit tests, ~0.06s
cargo test --features hdf5 # Includes H5AD I/O tests (requires HDF5)
cargo test # unit tests, CSV + fail-closed (no HDF5)
cargo test --features hdf5 # + 10x H5 fixture through `pipeline` (needs libhdf5)
```

CI installs `libhdf5-dev` and runs default `rustpipe-sc pipeline` on
`testdata/filtered_feature_bc_matrix.h5` (synthetic Cell Ranger `/matrix`,
not a public PBMC file). That job must write the PACKET.md artifacts.

## Known Limitations

- kNN is brute-force O(n^2): fast up to ~50K cells, needs approximate NN (HNSW) for larger
Expand Down
138 changes: 138 additions & 0 deletions scripts/gen_10x_ci_fixture.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
#!/usr/bin/env python3
"""Write testdata/filtered_feature_bc_matrix.h5 — tiny synthetic 10x /matrix H5.

Layout matches Cell Ranger filtered_feature_bc_matrix.h5 so rustpipe-sc
load_input → read_10x_h5 runs. Sized to pass default pipeline flags:

--min-genes 200 --max-pct-mt 5 --n-hvg 2000 --n-pcs 50 --knn-k 15

Not a public PBMC file. Regenerates the committed binary; do not vendor RustQC.
"""

from __future__ import annotations

import argparse
from pathlib import Path

import h5py
import numpy as np

# Human MT- prefixes so qc.rs pct_counts_mt is real and stays well under 5%.
MT_NAMES = (
"MT-ND1",
"MT-ND2",
"MT-CO1",
"MT-CO2",
"MT-ATP6",
"MT-ATP8",
"MT-CYB",
"MT-ND4",
)

N_CELLS = 80
N_GENES = 2120 # 8 MT + 2112 others; default --n-hvg 2000 needs ≥2000 after min_cells=3
N_GENES_PER_CELL = 240
SEED = 42


def gene_names() -> list[bytes]:
names = [n.encode("ascii") for n in MT_NAMES]
for i in range(len(MT_NAMES), N_GENES):
names.append(f"GENE{i:04d}".encode("ascii"))
return names


def barcodes() -> list[bytes]:
# 16-base + "-1", Cell Ranger style, unique and deterministic.
out = []
alphabet = b"ACGT"
for i in range(N_CELLS):
seq = bytearray(16)
n = i
for pos in range(16):
seq[pos] = alphabet[n & 3]
n >>= 2
out.append(bytes(seq) + b"-1")
return out


def build_csc(rng: np.random.Generator) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
"""CSC genes × cells. Every gene in ≥3 cells; every cell has 240 genes."""
cells_for_gene: list[set[int]] = [set() for _ in range(N_GENES)]
genes_for_cell: list[set[int]] = [set() for _ in range(N_CELLS)]

# Round-robin so min_cells=3 holds for all genes (HVG default 2000).
for g in range(N_GENES):
for k in range(3):
c = (g + k * 17) % N_CELLS
cells_for_gene[g].add(c)
genes_for_cell[c].add(g)

for c in range(N_CELLS):
while len(genes_for_cell[c]) < N_GENES_PER_CELL:
g = int(rng.integers(0, N_GENES))
genes_for_cell[c].add(g)
cells_for_gene[g].add(c)

data: list[int] = []
indices: list[int] = []
indptr = np.zeros(N_CELLS + 1, dtype=np.int64)
for c in range(N_CELLS):
genes = sorted(genes_for_cell[c])
for g in genes:
if g < len(MT_NAMES):
count = 1 # ~0.2% MT if the rest average ~4 UMIs
else:
count = 1 + int(rng.integers(0, 8))
data.append(count)
indices.append(g)
indptr[c + 1] = len(data)

return (
np.asarray(data, dtype=np.int32),
np.asarray(indices, dtype=np.int32),
indptr,
)


def write_h5(path: Path) -> None:
rng = np.random.default_rng(SEED)
data, indices, indptr = build_csc(rng)
names = gene_names()
bc = barcodes()
ids = [f"ENSG{i:011d}".encode("ascii") for i in range(N_GENES)]
feature_type = [b"Gene Expression"] * N_GENES
genome = [b"GRCh38"] * N_GENES

path.parent.mkdir(parents=True, exist_ok=True)
with h5py.File(path, "w") as f:
matrix = f.create_group("matrix")
matrix.create_dataset("shape", data=np.array([N_GENES, N_CELLS], dtype=np.int64))
matrix.create_dataset("data", data=data)
matrix.create_dataset("indices", data=indices)
matrix.create_dataset("indptr", data=indptr)
matrix.create_dataset("barcodes", data=np.array(bc, dtype="S18"))
features = matrix.create_group("features")
features.create_dataset("name", data=np.array(names, dtype="S16"))
features.create_dataset("id", data=np.array(ids, dtype="S15"))
features.create_dataset("feature_type", data=np.array(feature_type, dtype="S16"))
features.create_dataset("genome", data=np.array(genome, dtype="S8"))


def main() -> None:
root = Path(__file__).resolve().parents[1]
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"-o",
"--output",
type=Path,
default=root / "testdata" / "filtered_feature_bc_matrix.h5",
)
args = parser.parse_args()
write_h5(args.output)
size = args.output.stat().st_size
print(f"wrote {args.output} ({size} bytes) {N_CELLS} cells x {N_GENES} genes")


if __name__ == "__main__":
main()
17 changes: 17 additions & 0 deletions testdata/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# CI 10x H5 fixture

`filtered_feature_bc_matrix.h5` is a **synthetic** Cell Ranger `/matrix` file
(80 cells × 2120 genes). It is not a public PBMC download.

Regenerate:

```bash
python3 scripts/gen_10x_ci_fixture.py
```

Sized so default `rustpipe-sc pipeline` flags succeed:

- each cell has 240 detected genes (`--min-genes 200`)
- `MT-*` genes are named and low-count (`--max-pct-mt 5`)
- ≥2000 genes pass `min_cells=3` (`--n-hvg 2000`)
- 80 cells (`--knn-k 15`; `n_pcs` already clamps to rank)
Binary file added testdata/filtered_feature_bc_matrix.h5
Binary file not shown.
58 changes: 58 additions & 0 deletions tests/h5_ci_fixture.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
//! Exercise `rustpipe-sc pipeline` on the committed 10x H5 fixture.
//!
//! Gated on `--features hdf5`. Without HDF5 the binary cannot read `.h5`.

#[cfg(feature = "hdf5")]
mod with_hdf5 {
use std::path::PathBuf;
use std::process::Command;

fn fixture() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("testdata/filtered_feature_bc_matrix.h5")
}

#[test]
fn pipeline_h5_writes_packet_artifacts() {
let fixture = fixture();
assert!(
fixture.is_file(),
"missing committed 10x fixture at {}",
fixture.display()
);

let out = tempfile::tempdir().expect("tempdir");
let status = Command::new(env!("CARGO_BIN_EXE_rustpipe-sc"))
.args([
"pipeline",
"--input",
fixture.to_str().unwrap(),
"--output",
out.path().to_str().unwrap(),
])
.status()
.expect("spawn rustpipe-sc");
assert!(status.success(), "pipeline exited {status}");

for name in [
"qc_metrics.csv",
"pca_scores.csv",
"clusters.csv",
"markers.csv",
"pipeline_timings.json",
] {
let path = out.path().join(name);
assert!(path.is_file(), "PACKET.md artifact missing: {name}");
assert!(
path.metadata().unwrap().len() > 0,
"PACKET.md artifact empty: {name}"
);
}

let timings = std::fs::read_to_string(out.path().join("pipeline_timings.json")).unwrap();
assert!(
timings.contains("\"pipeline\": \"rustpipe-sc\""),
"pipeline_timings.json must name rustpipe-sc"
);
}
}
Loading