Dev 2

.github/workflows/c-cpp.yml

@@ -17,14 +17,30 @@ jobs:
       volumes:
         - /node20217:/node20217:rw,rshared
         - /node20217:/__e/node20:ro,rshared
+        - /node20217:/__e/node24:ro,rshared
     steps:
     - name: install git
-      run: apt-get update && apt-get install -y software-properties-common && add-apt-repository ppa:git-core/ppa && apt-get update -y && apt-get install git -y
+      run: |
+        set -e
+        apt-get update
+        apt-get install -y software-properties-common wget ca-certificates
+        . /etc/os-release
+        # add-apt-repository fetches PPA keys on port 11371 which times out on the
+        # runners, so pull the git-core PPA key over https and add the repo by hand
+        for i in 1 2 3; do
+          wget -qO /tmp/git-core.key 'https://keyserver.ubuntu.com/pks/lookup?op=get&options=mr&search=0xA1715D88E1DF1F24' && break
+          sleep 5
+        done
+        apt-key add /tmp/git-core.key
+        echo "deb http://ppa.launchpad.net/git-core/ppa/ubuntu $UBUNTU_CODENAME main" > /etc/apt/sources.list.d/git-core.list
+        apt-get update -y
+        apt-get install git -y
     - name: nodejs
       run: |
         apt-get install -y wget xz-utils
-        wget https://unofficial-builds.nodejs.org/download/release/v20.9.0/node-v20.9.0-linux-x64-glibc-217.tar.xz
-        tar -xf node-v20.9.0-linux-x64-glibc-217.tar.xz --strip-components 1 -C /node20217
+        wget https://unofficial-builds.nodejs.org/download/release/v22.20.0/node-v22.20.0-linux-x64-glibc-217.tar.xz
+        tar -xf node-v22.20.0-linux-x64-glibc-217.tar.xz --strip-components 1 -C /node20217
+        /__e/node24/bin/node --version
     - uses: actions/checkout@v2
       with:
         submodules: recursive
@@ -44,9 +60,24 @@ jobs:
       volumes:
         - /node20217:/node20217:rw,rshared
         - /node20217:/__e/node20:ro,rshared
+        - /node20217:/__e/node24:ro,rshared
     steps:
     - name: install git
-      run: apt-get update && apt-get install -y software-properties-common && add-apt-repository ppa:git-core/ppa && apt-get update -y && apt-get install git -y
+      run: |
+        set -e
+        apt-get update
+        apt-get install -y software-properties-common wget ca-certificates
+        . /etc/os-release
+        # add-apt-repository fetches PPA keys on port 11371 which times out on the
+        # runners, so pull the git-core PPA key over https and add the repo by hand
+        for i in 1 2 3; do
+          wget -qO /tmp/git-core.key 'https://keyserver.ubuntu.com/pks/lookup?op=get&options=mr&search=0xA1715D88E1DF1F24' && break
+          sleep 5
+        done
+        apt-key add /tmp/git-core.key
+        echo "deb http://ppa.launchpad.net/git-core/ppa/ubuntu $UBUNTU_CODENAME main" > /etc/apt/sources.list.d/git-core.list
+        apt-get update -y
+        apt-get install git -y
     - name: nodejs
       run: |
         apt-get install -y wget xz-utils
@@ -78,9 +109,24 @@ jobs:
       volumes:
         - /node20217:/node20217:rw,rshared
         - /node20217:/__e/node20:ro,rshared
+        - /node20217:/__e/node24:ro,rshared
     steps:
     - name: install git
-      run: apt-get update && apt-get install -y software-properties-common && add-apt-repository ppa:git-core/ppa && apt-get update -y && apt-get install git -y
+      run: |
+        set -e
+        apt-get update
+        apt-get install -y software-properties-common wget ca-certificates
+        . /etc/os-release
+        # add-apt-repository fetches PPA keys on port 11371 which times out on the
+        # runners, so pull the git-core PPA key over https and add the repo by hand
+        for i in 1 2 3; do
+          wget -qO /tmp/git-core.key 'https://keyserver.ubuntu.com/pks/lookup?op=get&options=mr&search=0xA1715D88E1DF1F24' && break
+          sleep 5
+        done
+        apt-key add /tmp/git-core.key
+        echo "deb http://ppa.launchpad.net/git-core/ppa/ubuntu $UBUNTU_CODENAME main" > /etc/apt/sources.list.d/git-core.list
+        apt-get update -y
+        apt-get install git -y
     - name: nodejs
       run: |
         apt-get install -y wget xz-utils
@@ -95,7 +141,14 @@ jobs:
     - name: install gcc
       run: |
         set -xe
-        add-apt-repository ppa:ubuntu-toolchain-r/test -y
+        . /etc/os-release
+        # same keyserver timeout as above: fetch the toolchain PPA key over https
+        for i in 1 2 3; do
+          wget -qO /tmp/toolchain-r.key 'https://keyserver.ubuntu.com/pks/lookup?op=get&options=mr&search=0x1E9377A2BA9EF27F' && break
+          sleep 5
+        done
+        apt-key add /tmp/toolchain-r.key
+        echo "deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu $UBUNTU_CODENAME main" > /etc/apt/sources.list.d/ubuntu-toolchain-r.list
         apt-get update
         apt-get install gcc-11 g++-11 -y
     - name: cuda
@@ -127,9 +180,24 @@ jobs:
       volumes:
         - /node20217:/node20217:rw,rshared
         - /node20217:/__e/node20:ro,rshared
+        - /node20217:/__e/node24:ro,rshared
     steps:
     - name: install git
-      run: apt-get update && apt-get install -y software-properties-common && add-apt-repository ppa:git-core/ppa && apt-get update -y && apt-get install git -y
+      run: |
+        set -e
+        apt-get update
+        apt-get install -y software-properties-common wget ca-certificates
+        . /etc/os-release
+        # add-apt-repository fetches PPA keys on port 11371 which times out on the
+        # runners, so pull the git-core PPA key over https and add the repo by hand
+        for i in 1 2 3; do
+          wget -qO /tmp/git-core.key 'https://keyserver.ubuntu.com/pks/lookup?op=get&options=mr&search=0xA1715D88E1DF1F24' && break
+          sleep 5
+        done
+        apt-key add /tmp/git-core.key
+        echo "deb http://ppa.launchpad.net/git-core/ppa/ubuntu $UBUNTU_CODENAME main" > /etc/apt/sources.list.d/git-core.list
+        apt-get update -y
+        apt-get install git -y
     - name: nodejs
       run: |
         apt-get install -y wget xz-utils
@@ -143,7 +211,7 @@ jobs:
     - name: torch
       run: scripts/install-torch2.sh cpu
     - name: build
-      run: make -j2 
+      run: make -j2
     - name: test
       run: export DEVICE=cpu && make test
   ubuntu_20:

.gitignore

@@ -42,6 +42,9 @@ Thumbs.db
 /.vscode
 .idea/
 
+# Local Claude Code instructions (kept locally, not in repo)
+CLAUDE.md
+
 # binaries
 /slorado
 /thirdparty/torch
@@ -67,4 +70,14 @@ test/slorado*
 *.paf
 *.bedmethyl
 *.pdf
-*.temp*
+*.temp*
+
+tmp/*
+pyvenv
+*.svg
+*.png
+*.json
+
+beam_exp
+fluke
+minimap2

Makefile

@@ -148,7 +148,7 @@ $(BUILD_DIR)/toml.o: thirdparty/tomlc99/toml.c
 	$(CC) $(CFLAGS) $(CPPFLAGS) $(DEPFLAGS) $< -c -o $@
 
 openfish/lib/libopenfish.a:
-	$(MAKE) -C openfish cuda=$(cuda) rocm=$(rocm) ROCM_ROOT=$(ROCM_ROOT) ROCM_ARCH=$(ROCM_ARCH) CUDA_ROOT=$(CUDA_ROOT) CUDA_ARCH=$(CUDA_ARCH) lib/libopenfish.a
+	$(MAKE) -C openfish cuda=$(cuda) rocm=$(rocm) ROCM_ROOT=$(ROCM_ROOT) ROCM_ARCH="$(ROCM_ARCH)" CUDA_ROOT=$(CUDA_ROOT) CUDA_ARCH="$(CUDA_ARCH)" lib/libopenfish.a
 
 slow5lib/lib/libslow5.a:
 	$(MAKE) -C slow5lib zstd=$(zstd) no_simd=$(no_simd) zstd_local=$(zstd_local) lib/libslow5.a

README.md

@@ -31,9 +31,8 @@ Binaries for the CPU-only version are not provided as basecalling on the CPU is
 
 Refer to [troubleshoot](docs/troubleshoot.md) for help resolving common problems.
 
-## Compilation and running
 
-### Compilation
+## Compilation
 
 Compilation instructions differ based on the system. Please pick one of the following that matches your system:
 
@@ -44,9 +43,9 @@ Compilation instructions differ based on the system. Please pick one of the foll
 
 Note: building from source will first require downloading and extracting Libtorch, which may take up to an hour depending on your network speed. Compilation should only take up to several minutes.
 
-### Running
+## Running
 
-We have tested slorado on a limited number of basecalling models listed [below](#tested-model). You can download them using the provided script (the binary releases already include these):
+We have tested slorado on a limited number of basecalling models listed [below](#tested-models). You can download them using the provided script (the binary releases already include these):
 
 ```
 scripts/download-models.sh
@@ -62,6 +61,33 @@ Now run on a test dataset:
 
 Refer to [troubleshoot](docs/troubleshoot.md) for help resolving common problems.
 
+### Basecalling POD5 files
+
+Slorado reads signal data in S/BLOW5 format. POD5 files from ONT sequencers can be converted to BLOW5 using [blue-crab](https://github.com/Psy-Fer/blue-crab):
+
+```
+# install blue-crab
+pip install blue-crab
+
+# convert a pod5 file (or a directory of pod5 files) to blow5
+blue-crab p2s reads.pod5 -o reads.blow5
+blue-crab p2s pod5_dir/ -o reads.blow5
+
+# then basecall as usual
+./slorado basecaller models/dna_r10.4.1_e8.2_400bps_hac@v5.0.0 reads.blow5 -o reads.fastq -x cuda:all
+```
+
+For convenience, we also provide a wrapper script [pod5-slorado](scripts/pod5-slorado) that converts a POD5 file to a temporary BLOW5 file and then invokes slorado on it. It takes the same arguments as `slorado basecaller`, but with a POD5 file as input:
+
+```
+# set environment variable SLORADO, if slorado is not in PATH (export SLORADO=/path/to/slorado).
+# set environment variable BLUECRAB, if blue-crab is not in PATH (export BLUECRAB=/path/to/blue-crab).
+scripts/pod5-slorado basecaller models/dna_r10.4.1_e8.2_400bps_fast@v5.0.0 reads.pod5 -o reads.fastq
+```
+
+### Docker
+
+Pre-built image are available on [Docker Hub](https://hub.docker.com/r/hasindu2008/slorado). Refer to [here](docs/docker.md) for detailed instructions.
 
 ## Testing
 
@@ -74,14 +100,25 @@ wget -O hg38noAlt.fa.gz seq.bioinf.science/hg38noAlt && gunzip hg38noAlt.fa.gz
 
 # set environment variable MINIMAP2, if minimap2 is not in PATH (export MINIMAP2=/path/to/minimap2).
 # set environment variable DATAMASH, if datamash is not in PATH (export DATAMASH=/path/to/datamash).
-scripts/calculate_basecalling_accuarcy.sh hg38noAlt.fa reads.fastq
+scripts/calculate_basecalling_accuracy.sh hg38noAlt.fa reads.fastq
 
 # expected median identity scores for test/PGXXXX230339/reads_1k.blow5:
 # FAST v5.0.0: 0.940696
 # HAC v5.0.0:  0.976852
 # SUP v5.0.0:  0.988194
 ```
 
+The repo also includes a minimal test that runs FAST v4.2.0 in [test.sh](test/test.sh), which will automatically install minimap2 and run on a single small dataset and reference included in the repo.
+```
+# run on a single read:
+./test/test.sh
+# accuracy: 0.944928
+
+# run on 1k reads mapped to chr22:
+./test/test.sh chr22
+# accuracy: 0.939607
+```
+
 For a more exhaustive test of slorado's features (on GPU setups), we have provided an [extensive test script](test/extensive.sh). This will automatically download the requisite test data and tools to test DNA/RNA basecalling, methylation detection, and flash attention support on your device. We highly recommend running this to ensure basecalling works on your machine. Excluding the automated binary release test mode, this script is meant to work on both ARM and x86 architectures.
 
 Here is an example of how to run it:
@@ -111,6 +148,7 @@ cd slorado
 ./test/extensive rocm build
 
 ```
+
 ## Known issues
 
 As of May 1st 2026, LSTM models (HAC and FAST or SUP < v5.0.0) on the 9700 AI Pro (and possibly other newer AMD GPUs) produce incorrect outputs (Transformer models unaffected). This issue is known and can be tracked [here](https://github.com/pytorch/pytorch/issues/177834).

docs/docker.md

@@ -0,0 +1,53 @@
+
+# Docker
+
+Pre-built images are available on [Docker Hub](https://hub.docker.com/r/hasindu2008/slorado). The image bundles the `slorado` binary (on `PATH`, at `/slorado/bin`) along with the well tested models under `/slorado/models`.
+The image also bundles `blue-crab` and `slow5tools`.
+
+Builds for both AMD (rocm) and NVIDIA (cuda) are available.
+
+## For AMD (rocm)
+
+```
+docker pull hasindu2008/slorado:0.3.0-beta-rocm
+```
+
+Print the help message to check the image works:
+
+```
+docker run --rm hasindu2008/slorado:0.3.0-beta-rocm slorado basecaller --help
+```
+
+To basecall on an AMD GPU, expose the GPU devices to the container and bind mount the directory holding your BLOW5 file:
+
+```
+# gid of the render group on the host, needed to access /dev/dri/renderD*
+export RENDER_GID=$(stat -c '%g' /dev/dri/renderD128)
+
+docker run -v "$PWD":/data  \
+  --device=/dev/kfd --device=/dev/dri \
+  --security-opt seccomp=unconfined --group-add video
+  hasindu2008/slorado:0.6.0-beta-rocm \
+  slorado basecaller /slorado/models/dna_r10.4.1_e8.2_400bps_hac@v5.0.0 /data/reads.blow5 -o /data/reads.fastq -x cuda:all
+```
+
+Note that `-x cuda:all` selects all GPUs for the rocm build too. If `--group-add video` fails with `Unable to find group video`, pass the numeric gid instead (`$(getent group video | cut -d: -f3)`).
+
+
+## For NVIDIA (cuda)
+
+```
+docker pull hasindu2008/slorado:0.6.0-cuda
+```
+
+Print the help message to check the image works:
+```
+docker run --rm hasindu2008/slorado:0.6.0-beta-cuda slorado basecaller --help
+```
+
+To basecall on an NVIDIA GPU, expose the GPU devices to the container and bind mount the directory holding your BLOW5 file:
+
+docker run -v "$PWD":/data --gpus all \
+  hasindu2008/slorado:0.6.0-cuda \
+  slorado basecaller /slorado/models/dna_r10.4.1_e8.2_400bps_hac@v5.0.0 /data/reads.blow5 -o /data/reads.fastq -x cuda:all
+```

scripts/calculate_basecalling_accuracy.sh

@@ -17,9 +17,9 @@ fi
 ${MINIMAP2} --version &> /dev/null || { echo -e $RED"minimap2 not found! Either put minimap2 under path or set MINIMAP2 variable, e.g.,export MINIMAP2=/path/to/minimap2"$NORMAL; exit 1;}
 ${DATAMASH} --version &> /dev/null || { echo -e $RED"datamash not found! Either put datamash under path or set DATAMASH variable, e.g.,export DATAMASH=/path/to/datamash, or if not installed: \`apt-get install datamash\`"$NORMAL; exit 1;}
 
-REFERENC_GENOME=$1 #path to reference genome
-FASTQ_FILE=$2 #path to basecalled fastq file
+REF=$1 #path to reference genome
+FASTQ=$2 #path to basecalled fastq file
 
-echo "identity scores:"
-echo -e "mean\tstdev\tq1\tmedian\tq3\tn"
-${MINIMAP2} -cx map-ont ${REFERENC_GENOME} -t${NTHREADS} --secondary=no ${FASTQ_FILE} | awk '{print $10/$11}' | ${DATAMASH} mean 1 sstdev 1 q1 1 median 1 q3 1 count 1 || die "Error in identity calculation"
+OUTPUT=$(${MINIMAP2} -cx map-ont ${REF} -t${NTHREADS} --secondary=no ${FASTQ} | awk '{print $10/$11}' | ${DATAMASH} mean 1 sstdev 1 q1 1 median 1 q3 1 count 1 || die "Error in identity calculation")
+echo -en "sample\tmean\tsstdev\tq1\tmedian\tq3\tn\n"
+echo -en "${FASTQ}\t$OUTPUT\n"

scripts/calculate_basecalling_accuracy_rna.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+#!/bin/bash
+
+NTHREADS="${NTHREADS:-32}"
+
+die() {
+    echo "$@" >&2
+    exit 1
+}
+
+if [ $# -ne 2 ]; then
+    die "Usage: $0 <reference genome> <fastq file> "
+fi
+
+[ -z ${MINIMAP2} ] && MINIMAP2=minimap2
+[ -z ${DATAMASH} ] && DATAMASH=datamash
+
+${MINIMAP2} --version &> /dev/null || { echo -e $RED"minimap2 not found! Either put minimap2 under path or set MINIMAP2 variable, e.g.,export MINIMAP2=/path/to/minimap2"$NORMAL; exit 1;}
+${DATAMASH} --version &> /dev/null || { echo -e $RED"datamash not found! Either put datamash under path or set DATAMASH variable, e.g.,export DATAMASH=/path/to/datamash, or if not installed: \`apt-get install datamash\`"$NORMAL; exit 1;}
+
+REF=$1 #path to reference genome
+FASTQ=$2 #path to basecalled fastq file
+
+OUTPUT=$(${MINIMAP2} -cx splice -uf -k14  ${REF} -t${NTHREADS} --secondary=no ${FASTQ} | awk '{print $10/$11}' | datamash mean 1 sstdev 1 q1 1 median 1 q3 1 count 1)
+echo -en "sample\tmean\tsstdev\tq1\tmedian\tq3\tn\n"
+echo -en "${FASTQ}\t$OUTPUT\n"