From 0bf3761713f4fb117a7da5f520c8b3e9682d6dd0 Mon Sep 17 00:00:00 2001 From: Josh Garton Date: Wed, 2 Sep 2026 10:20:54 -0500 Subject: [PATCH] Add GHCR image, packet contract, and fail-closed bulk inputs. linux/amd64 image builds with Debian HDF5 so Cell Ranger H5 / H5AD work. Reject STAR BAM and rustqc featureCounts as --input. Pull the published digest; never rebuild this image inside prairie-rna-stream. --- .dockerignore | 5 + .github/workflows/ci.yml | 18 +++ .github/workflows/publish-ghcr.yml | 78 ++++++++++++ Dockerfile | 19 ++- PACKET.md | 60 +++++++++ README.md | 52 +++++++- src/input_guard.rs | 198 +++++++++++++++++++++++++++++ src/main.rs | 3 + 8 files changed, 423 insertions(+), 10 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/publish-ghcr.yml create mode 100644 PACKET.md create mode 100644 src/input_guard.rs diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..119dc41 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +target +overnight-smoke +.git +.github +**/.DS_Store diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..3c1a47b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,18 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - name: Test (CSV + fail-closed; no HDF5) + run: cargo test diff --git a/.github/workflows/publish-ghcr.yml b/.github/workflows/publish-ghcr.yml new file mode 100644 index 0000000..6b85ab6 --- /dev/null +++ b/.github/workflows/publish-ghcr.yml @@ -0,0 +1,78 @@ +name: Publish GHCR + +on: + push: + branches: [main] + tags: ['v*'] + workflow_dispatch: + +permissions: + contents: read + packages: write + +env: + IMAGE: ghcr.io/jwg054000/rustpipe-sc + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.IMAGE }} + labels: | + org.opencontainers.image.source=https://github.com/jwg054000/RustPipe-SC + org.opencontainers.image.description=Fast single-cell RNA-seq downstream of Cell Ranger / nf-core/scrnaseq + org.opencontainers.image.licenses=GPL-3.0 + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=raw,value=0.3.0,enable={{is_default_branch}} + type=sha,format=long + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Build and push + id: build + uses: docker/build-push-action@v6 + with: + context: . + file: Dockerfile + push: true + platforms: linux/amd64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: Record digest + run: | + { + echo "## rustpipe-sc GHCR digest" + echo "" + echo "linux/amd64. Pin this string. Do not float on \`latest\`. Do not \`docker build\` this image inside prairie-rna-stream." + echo "" + echo '```' + echo "${IMAGE}:0.3.0@${{ steps.build.outputs.digest }}" + echo '```' + echo "" + echo "First package from GITHUB_TOKEN is private. Make it public:" + echo "" + echo '```' + echo "gh api --method POST -H 'Accept: application/vnd.github+json' \\" + echo " /user/packages/container/rustpipe-sc/visibility \\" + echo " -f visibility=public" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/Dockerfile b/Dockerfile index 9c1dd84..a427c6c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,27 @@ +# linux/amd64 image for ghcr.io/jwg054000/rustpipe-sc. +# Runtime copies the stripped binary only (no rustc). HDF5 is Debian bookworm +# 1.10 via apt — not Homebrew HDF5 2.x. +# +# Downstream pipelines pull the published digest. Do not rebuild this image +# inside prairie-rna-stream and do not vendor this repo there. + FROM rust:1.88-bookworm AS builder WORKDIR /build +RUN apt-get update \ + && apt-get install -y --no-install-recommends libhdf5-dev pkg-config \ + && rm -rf /var/lib/apt/lists/* COPY Cargo.toml Cargo.lock ./ COPY src/ src/ -RUN cargo build --release && strip target/release/rustpipe-sc +RUN cargo build --release --features hdf5 && strip target/release/rustpipe-sc -FROM debian:bookworm-slim +FROM debian:bookworm-slim@sha256:5ae3c39ebd15e229dcedd5cee596b2497182493d41ff162e824ba13fc1b2b867 LABEL org.opencontainers.image.source="https://github.com/jwg054000/RustPipe-SC" -LABEL org.opencontainers.image.description="Fast single-cell and spatial transcriptomics in Rust" +LABEL org.opencontainers.image.description="Fast single-cell RNA-seq downstream of Cell Ranger / nf-core/scrnaseq" LABEL org.opencontainers.image.licenses="GPL-3.0" LABEL org.opencontainers.image.version="0.3.0" +RUN apt-get update \ + && apt-get install -y --no-install-recommends libhdf5-103-1 ca-certificates \ + && rm -rf /var/lib/apt/lists/* COPY --from=builder /build/target/release/rustpipe-sc /usr/local/bin/rustpipe-sc RUN rustpipe-sc --version ENTRYPOINT ["rustpipe-sc"] diff --git a/PACKET.md b/PACKET.md new file mode 100644 index 0000000..be4fe74 --- /dev/null +++ b/PACKET.md @@ -0,0 +1,60 @@ +# Packet contract (10x Chromium / scRNA) + +RustPipe-SC runs **downstream** of [Cell Ranger](https://www.10xgenomics.com/support/software/cell-ranger) or [nf-core/scrnaseq](https://nf-co.re/scrnaseq). It accelerates QC → normalize → HVG → PCA → kNN → Leiden → markers. + +It is **not** a Cell Ranger replacement: no BCL → FASTQ, no barcode/UMI counting, no STARSolo. It is **not** prairie-rna-stream (bulk STAR → rustqc → rustpipe). Never vendor this repo into prairie-rna-stream. + +## Inputs (`--input`) + +Exactly one cell-count matrix: + +| path | source | +| --- | --- | +| `filtered_feature_bc_matrix.h5` (or other 10x `/matrix` H5) | Cell Ranger, cellranger-arc | +| `*.h5ad` | Scanpy / nf-core/scrnaseq AnnData | +| cells × genes CSV (`barcode,gene1,gene2,…`) | custom / Scanpy export | +| genes × cells CSV (`gene,cell1,cell2,…`) | legacy; auto-transposed | + +`--qc-bam` / `--qc-gtf` are optional library QC (PATH `rustqc rna` into `output/libqc/`). They are **not** the cell matrix. `libqc/featurecounts/` must be ignored as cells. + +## Fail closed (wrong assay) + +Reject `--input` and exit non-zero. Do not transpose, do not “fix”, do not count a BAM. + +| handed in | why | +| --- | --- | +| `*.bam` / `*.sam` / `*.cram` (including STAR `*.Aligned.sortedByCoord.out.bam`) | alignments; bulk STAR belongs in prairie-rna-stream | +| Cell Ranger `possorted_genome_bam.bam` as `--input` | that path is `--qc-bam` only | +| rustqc `*.featureCounts.tsv` (header `Geneid,Chr,Start,End,Strand,Length` or `# Program:featureCounts`) | library QC table, not a cell matrix | +| `matrix.mtx` | MTX directory is not an input; use the 10x H5 or H5AD | + +A prairie-rna-stream packet (`counts.parquet` + `rustpipe_out/` + `run.json`) is the wrong product. Do not ingest it here. + +## Outputs (`rustpipe-sc pipeline --output DIR`) + +What this crate actually writes today (v0.3.0). Column names are as emitted; do not invent aliases. + +| file | columns / role | +| --- | --- | +| `qc_metrics.csv` | `barcode`, `n_genes_by_counts`, `total_counts`, `pct_counts_mt` | +| `hvg_genes.csv` | `gene` | +| `pca_scores.csv` | `sample` (cell barcode values), `PC1`…`PCk` | +| `knn.csv` | `neighbor_1`,`distance_1`,… for k neighbors | +| `clusters.csv` | `barcode`, `cluster` | +| `markers.csv` | `cluster`, `gene`, `score`, `pval`, `pval_adj`, `log2fc` | +| `pipeline_timings.json` | `pipeline`=`rustpipe-sc`, `version`, `seed`, `input`, cell/gene counts, `hvg_flavor`, `n_pcs`, `knn_k`, `steps[{step,seconds}]`, `total_seconds` | +| `libqc/` | only if `--qc-bam`; rustqc tree. Not the cell matrix. | + +The `qc` subcommand also writes `filtered_barcodes.csv`. The full `pipeline` subcommand does **not** write `filtered_cells.csv` / `normalized.csv` / `pca_loadings.csv` (those names in older README lists are not produced). + +No hashed `run.json` yet. `pipeline_timings.json` is the provenance file. A Prairie SDK ingest should fail-closed if `pipeline` ≠ `"rustpipe-sc"`, if a bulk STAR/featureCounts file is present as the matrix, or if `clusters.csv` / `pca_scores.csv` / `pipeline_timings.json` are missing. + +## Image pin + +Pull the published linux/amd64 image. Never `docker build` this tree from prairie-rna-stream. + +``` +ghcr.io/jwg054000/rustpipe-sc:0.3.0@sha256: +``` + +Until the first successful publish + public visibility click, that digest does not exist. See README (Container). diff --git a/README.md b/README.md index d50d2bd..f96c3e7 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,9 @@ Fast single-cell and spatial transcriptomics in Rust. Accelerates the compute-bound steps of scRNA-seq analysis (QC, normalization, HVG, PCA, kNN, Leiden clustering, marker detection) while maintaining concordance with Scanpy. -Designed to run downstream of [Seqera](https://seqera.io/) / [nf-core/scrnaseq](https://nf-co.re/scrnaseq) pipelines that output H5AD or 10x CellRanger H5 files. +Designed to run **downstream** of [Cell Ranger](https://www.10xgenomics.com/support/software/cell-ranger) or [nf-core/scrnaseq](https://nf-co.re/scrnaseq) (H5AD or 10x Cell Ranger H5). This crate is **not** a Cell Ranger replacement (no BCL → FASTQ, no barcode/UMI counting). Bulk RNA (STAR → rustqc → rustpipe) lives in [prairie-rna-stream](https://github.com/jwg054000/prairie-rna-stream); never vendor this repo there. + +Ingest contract: [`PACKET.md`](PACKET.md). ## Performance @@ -51,6 +53,8 @@ At 10K cells, RustPipe-SC achieves **22x speedup** over Scanpy on compute-bound Auto-detection: `.h5` and `.h5ad` files are automatically detected as 10x or AnnData format. +**Fail-closed.** `--input` must be a cell-count matrix. A bulk STAR BAM, Cell Ranger BAM, rustqc featureCounts TSV, or MTX directory is rejected (see [`PACKET.md`](PACKET.md)). A Cell Ranger BAM is optional `--qc-bam` only. + ### HDF5 Support H5AD and 10x H5 reading requires the `hdf5` feature and a system HDF5 library: @@ -65,6 +69,40 @@ HDF5_DIR=$(brew --prefix hdf5) cargo build --release --features hdf5 Without the `hdf5` feature, CSV input is still supported. +## Container (GHCR) + +Published image: `ghcr.io/jwg054000/rustpipe-sc` (linux/amd64, HDF5 enabled). + +**Digest-pin.** Downstream pipelines must `docker pull` a digest-pinned reference. They must **never** `docker build` this tree, never vendor this repo, and never compile rustpipe-sc inside prairie-rna-stream. + +``` +ghcr.io/jwg054000/rustpipe-sc:0.3.0@sha256: +``` + +Do not float on `latest`. The publish workflow writes the digest to the Actions job summary. After the first successful push the package is **private** until made public: + +```bash +# run publish (after this workflow exists on main) +gh workflow run publish-ghcr.yml --repo jwg054000/RustPipe-SC + +# make the package pullable without a GHCR token +gh api --method POST -H "Accept: application/vnd.github+json" \ + /user/packages/container/rustpipe-sc/visibility \ + -f visibility=public +``` + +UI equivalent: [GHCR package settings](https://github.com/users/jwg054000/packages/container/package/rustpipe-sc) → Change visibility → Public. + +```bash +docker pull ghcr.io/jwg054000/rustpipe-sc:0.3.0 +docker run --rm \ + -v "$PWD/in:/in:ro" -v "$PWD/out:/out" \ + ghcr.io/jwg054000/rustpipe-sc:0.3.0 \ + pipeline --input /in/filtered_feature_bc_matrix.h5 --output /out +``` + +The image `ENTRYPOINT` is `rustpipe-sc`. Nextflow must override the entrypoint/shell the same way it does for rustpipe. + ## Installation ```bash @@ -146,18 +184,17 @@ rustpipe-sc morans-i \ ## Pipeline Outputs +Canonical list and column names: [`PACKET.md`](PACKET.md). `pipeline` writes: + ``` output/ - qc_metrics.csv # Per-cell QC: n_genes, total_counts, pct_mt - filtered_cells.csv # Cells passing QC - normalized.csv # Library-size normalized, log1p transformed + qc_metrics.csv # Per-cell QC: barcode, n_genes_by_counts, total_counts, pct_counts_mt hvg_genes.csv # Selected highly variable genes - pca_scores.csv # PCA embeddings (cells x PCs) - pca_loadings.csv # Gene loadings per PC - pca_variance.csv # Variance explained per PC + pca_scores.csv # PCA embeddings (sample=barcode, PC1…PCk) knn.csv # k-nearest neighbor graph clusters.csv # Leiden cluster assignments markers.csv # Cluster marker genes (Wilcoxon rank-sum) + pipeline_timings.json # Provenance + step timings libqc/ # Optional: rustqc rna outputs (--qc-bam). Ignore featurecounts/ as cells. ``` @@ -166,6 +203,7 @@ output/ ``` src/ main.rs CLI + pipeline orchestration + input_guard.rs Fail-closed: reject STAR BAM / featureCounts as --input h5ad.rs H5AD and 10x H5 I/O (feature-gated) io.rs CSV I/O sparse.rs CSR sparse matrix ops, implicit-centering matmul diff --git a/src/input_guard.rs b/src/input_guard.rs new file mode 100644 index 0000000..19f1382 --- /dev/null +++ b/src/input_guard.rs @@ -0,0 +1,198 @@ +//! Fail-closed input checks for the 10x / scRNA lane. +//! +//! `--input` is a cell-count matrix (Cell Ranger H5, H5AD, or cell CSV). +//! A bulk STAR BAM or rustqc featureCounts TSV is the wrong assay. + +use anyhow::{bail, Result}; +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::path::Path; + +const FEATURECOUNTS_COLS: [&str; 6] = ["Geneid", "Chr", "Start", "End", "Strand", "Length"]; + +/// Reject bulk STAR / rustqc files handed in as the cell matrix. +pub fn reject_wrong_assay(path: &Path) -> Result<()> { + let name = path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("") + .to_ascii_lowercase(); + + if name.ends_with(".bam") || name.ends_with(".sam") || name.ends_with(".cram") { + bail!( + "refusing alignment file as --input ({}). \ + RustPipe-SC consumes Cell Ranger H5 / H5AD / cell-count CSV, not BAM/SAM/CRAM. \ + A bulk STAR BAM belongs in prairie-rna-stream. \ + A Cell Ranger possorted_genome_bam.bam is optional --qc-bam (library QC), never the cell matrix.", + path.display() + ); + } + + if name.ends_with(".mtx") { + bail!( + "refusing MTX as --input ({}). \ + Use Cell Ranger filtered_feature_bc_matrix.h5 or an H5AD from nf-core/scrnaseq. \ + This crate does not read the MTX directory.", + path.display() + ); + } + + if name.contains("featurecounts") { + bail!("{}", featurecounts_msg(path)); + } + + if should_peek_text(&name) { + if let Some(reason) = peek_featurecounts(path)? { + bail!("{} ({})", featurecounts_msg(path), reason); + } + } + + Ok(()) +} + +fn should_peek_text(lower_name: &str) -> bool { + !lower_name.ends_with(".h5") + && !lower_name.ends_with(".h5ad") + && !lower_name.ends_with(".bam") + && !lower_name.ends_with(".sam") + && !lower_name.ends_with(".cram") + && !lower_name.ends_with(".mtx") +} + +fn featurecounts_msg(path: &Path) -> String { + format!( + "refusing rustqc/featureCounts table as --input ({}). \ + That file is bulk library QC (Geneid/Chr/Start/End/Strand/Length), not a cell-count matrix. \ + Use filtered_feature_bc_matrix.h5 or an H5AD from Cell Ranger / nf-core/scrnaseq.", + path.display() + ) +} + +fn peek_featurecounts(path: &Path) -> Result> { + let file = match File::open(path) { + Ok(f) => f, + Err(_) => return Ok(None), + }; + let mut reader = BufReader::new(file); + let mut line = String::new(); + + for _ in 0..8 { + line.clear(); + let n = reader.read_line(&mut line)?; + if n == 0 { + break; + } + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + if trimmed.starts_with('#') { + if trimmed.to_ascii_lowercase().contains("featurecounts") { + return Ok(Some("comment line names featureCounts")); + } + continue; + } + if looks_like_featurecounts_header(trimmed) { + return Ok(Some("header has Geneid/Chr/Start/End/Strand/Length")); + } + break; + } + Ok(None) +} + +fn looks_like_featurecounts_header(header: &str) -> bool { + let cols: Vec<&str> = header + .split(['\t', ',']) + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .collect(); + FEATURECOUNTS_COLS + .iter() + .all(|need| cols.iter().any(|got| got.eq_ignore_ascii_case(need))) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + fn tmp_with(name: &str, body: &str) -> tempfile::NamedTempFile { + let mut f = tempfile::Builder::new() + .prefix("rp_sc_") + .suffix(name) + .tempfile() + .unwrap(); + f.write_all(body.as_bytes()).unwrap(); + f.flush().unwrap(); + f + } + + #[test] + fn rejects_star_bam_by_extension() { + let err = reject_wrong_assay(Path::new("WT_REP1.Aligned.sortedByCoord.out.bam")) + .unwrap_err() + .to_string(); + assert!(err.contains("alignment file"), "{err}"); + assert!(err.contains("prairie-rna-stream"), "{err}"); + } + + #[test] + fn rejects_cellranger_bam_as_input() { + let err = reject_wrong_assay(Path::new("possorted_genome_bam.bam")) + .unwrap_err() + .to_string(); + assert!(err.contains("--qc-bam"), "{err}"); + } + + #[test] + fn rejects_featurecounts_filename() { + let err = reject_wrong_assay(Path::new("RAP1_IAA.featureCounts.tsv")) + .unwrap_err() + .to_string(); + assert!(err.contains("featureCounts"), "{err}"); + } + + #[test] + fn rejects_featurecounts_header() { + let f = tmp_with( + ".tsv", + "Geneid\tChr\tStart\tEnd\tStrand\tLength\tWT.bam\nYAL069W\tI\t335\t649\t+\t315\t19\n", + ); + let err = reject_wrong_assay(f.path()).unwrap_err().to_string(); + assert!(err.contains("featureCounts"), "{err}"); + } + + #[test] + fn rejects_featurecounts_comment() { + let f = tmp_with( + ".tsv", + "# Program:featureCounts v2.0.6, generated by RustQC v0.2.1\n\ + Geneid\tChr\tStart\tEnd\tStrand\tLength\tWT.bam\n", + ); + let err = reject_wrong_assay(f.path()).unwrap_err().to_string(); + assert!(err.contains("featureCounts"), "{err}"); + } + + #[test] + fn rejects_mtx() { + let err = reject_wrong_assay(Path::new("matrix.mtx")) + .unwrap_err() + .to_string(); + assert!(err.contains("MTX"), "{err}"); + } + + #[test] + fn accepts_cell_csv() { + let f = tmp_with( + ".csv", + "barcode,GENE_A,GENE_B\nAAACCTGAGAAACCAT-1,1,0\n", + ); + reject_wrong_assay(f.path()).unwrap(); + } + + #[test] + fn does_not_peek_h5_as_text() { + reject_wrong_assay(Path::new("filtered_feature_bc_matrix.h5")).unwrap(); + reject_wrong_assay(Path::new("adata.h5ad")).unwrap(); + } +} diff --git a/src/main.rs b/src/main.rs index 760b9ca..5a66e51 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,6 +18,7 @@ mod graph; mod gsea; mod h5ad; mod hvg_sc; +mod input_guard; mod io; mod knn; mod leiden; @@ -790,6 +791,7 @@ fn run_leiden(input: &str, resolution: f64, n_iterations: usize, output: &str) - fn run_load(input: &str, gene: Option<&str>) -> Result<()> { let t0 = Instant::now(); + input_guard::reject_wrong_assay(std::path::Path::new(input))?; let (gene_names, sample_names, data, n_genes, n_samples) = io::read_expression_matrix(input).map_err(|e| anyhow::anyhow!("{}", e))?; @@ -823,6 +825,7 @@ fn run_load(input: &str, gene: Option<&str>) -> Result<()> { /// - .csv: check if first column header looks like gene names (genes x cells) /// or cell barcodes (cells x genes) fn load_input(path: &str) -> Result<(sparse::SpMat, Vec, Vec)> { + input_guard::reject_wrong_assay(std::path::Path::new(path))?; if path.ends_with(".h5ad") || path.ends_with(".h5") { // Auto-detect H5AD vs 10x CellRanger H5 format let (mat, obs, var) = h5ad::read_h5_auto(std::path::Path::new(path))?;