diff --git a/pawsey_minion/README.md b/pawsey_minion/README.md index 69920ef..c00d6c7 100644 --- a/pawsey_minion/README.md +++ b/pawsey_minion/README.md @@ -23,6 +23,13 @@ mamba env create --yes --prefix /scratch/$PAWSEY_PROJECT/$USER/software/minicond mamba env create --yes --prefix /scratch/$PAWSEY_PROJECT/$USER/software/miniconda3/atavide_lite_vamb --file ../atavide_lite_vamb.yaml ``` +Make sure you build atavide_lite/bin: + +``` +pushd $HOME/GitHubs/atavide_lite/bin +make all +popd +``` ## 0. Get your data. @@ -122,7 +129,7 @@ We use these assemblies, e.g. for binning with VAMB. The assemblies are not used to complete before submitting the subsequent jobs. ``` -MEGAHITJOB=$(sbatch --parsable --dependency=afterok:$HOSTJOB --export=ATAVIDE_CONDA=$ATAVIDE_CONDA --array=1-$NUM_READS:1 $SRC/megahit.slurm) +MEGAHITJOB=$(sbatch --parsable --dependency=afterok:$HOSTJOB --array=1-$NUM_READS:1 $SRC/megahit.slurm) ``` ## 9. Convert to fasta. @@ -180,10 +187,10 @@ SANKEYJOB=$(sbatch --parsable --dependency=afterok:$COUNTSSJOB $SRC/sankey_plot. If you have the assembly from above, you can use VAMB for binning the contigs ``` -VCJOB=$(sbatch --parsable --dependency=afterok:$MEGAHITJOB --export=ATAVIDE_CONDA=$ATAVIDE_CONDA $SRC/vamb_concat.slurm) -VMJOB=$(sbatch --parsable --dependency=afterok:$VCJOB --array=1-$NUM_READS:1 --export=ATAVIDE_CONDA=$ATAVIDE_CONDA $SRC/vamb_minimap.slurm) -VAMBJOB=$(sbatch --parsable --dependency=afterany:$VMJOB --export=ATAVIDE_CONDA=$ATAVIDE_CONDA $SRC/vamb.slurm) -CHECKMJOB=$(sbatch --parsable --dependency=afterany:$VAMBJOB --export=ATAVIDE_CONDA=$ATAVIDE_CONDA $SRC/checkm.slurm vamb/bins/ vamb/checkm) +VCJOB=$(sbatch --parsable --dependency=afterok:$MEGAHITJOB $SRC/vamb_concat.slurm) +VMJOB=$(sbatch --parsable --dependency=afterok:$VCJOB --array=1-$NUM_READS:1 $SRC/vamb_minimap.slurm) +VAMBJOB=$(sbatch --parsable --dependency=afterany:$VMJOB $SRC/vamb.slurm) +CHECKMJOB=$(sbatch --parsable --dependency=afterany:$VAMBJOB $SRC/checkm.slurm vamb/bins/ vamb/checkm) ``` @@ -221,11 +228,11 @@ SSJOB=$(sbatch --parsable --dependency=afterok:$MMSEQSJOB --array=1-$NUM_READS:1 COUNTSSJOB=$(sbatch --parsable --dependency=afterok:$SSJOB $SRC/count_subsystems.slurm) SANKEYJOB=$(sbatch --parsable --dependency=afterok:$COUNTSSJOB $SRC/sankey_plot.slurm) -MEGAHITJOB=$(sbatch --parsable --dependency=afterok:$HOSTJOB --export=ATAVIDE_CONDA=$ATAVIDE_CONDA --array=1-$NUM_READS:1 $SRC/megahit.slurm) -VCJOB=$(sbatch --parsable --dependency=afterok:$MEGAHITJOB --export=ATAVIDE_CONDA=$ATAVIDE_CONDA $SRC/vamb_concat.slurm) -VMJOB=$(sbatch --parsable --dependency=afterok:$VCJOB --array=1-$NUM_READS:1 --export=ATAVIDE_CONDA=$ATAVIDE_CONDA $SRC/vamb_minimap.slurm) -VAMBJOB=$(sbatch --parsable --dependency=afterany:$VMJOB --export=ATAVIDE_CONDA=$ATAVIDE_CONDA $SRC/vamb.slurm) -CHECKMJOB=$(sbatch --parsable --dependency=afterany:$VAMBJOB --export=ATAVIDE_CONDA=$ATAVIDE_CONDA $SRC/checkm.slurm vamb/bins/ vamb/checkm) +MEGAHITJOB=$(sbatch --parsable --dependency=afterok:$HOSTJOB --array=1-$NUM_READS:1 $SRC/megahit.slurm) +VCJOB=$(sbatch --parsable --dependency=afterok:$MEGAHITJOB $SRC/vamb_concat.slurm) +VMJOB=$(sbatch --parsable --dependency=afterok:$VCJOB --array=1-$NUM_READS:1 $SRC/vamb_minimap.slurm) +VAMBJOB=$(sbatch --parsable --dependency=afterany:$VMJOB $SRC/vamb.slurm) +CHECKMJOB=$(sbatch --parsable --dependency=afterany:$VAMBJOB $SRC/checkm.slurm vamb/bins/ vamb/checkm) ``` diff --git a/pawsey_minion/download_human.slurm b/pawsey_minion/download_human.slurm new file mode 100644 index 0000000..af67df8 --- /dev/null +++ b/pawsey_minion/download_human.slurm @@ -0,0 +1,22 @@ +#!/bin/bash +#SBATCH --job-name=human_download +#SBATCH --time=1-0 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=1 +#SBATCH --mem=32G +#SBATCH -o %x-%j.out +#SBATCH -e %x-%j.err + +DEST=/scratch/$PAWSEY_PROJECT/$USER/Databases/human + +mkdir -p $DEST +cd $DEST + +if [[ -e GCA_000001405.15_GRCh38_no_alt_plus_hs38d1_analysis_set.fna.gz ]]; then + echo "GCA_000001405.15_GRCh38_no_alt_plus_hs38d1_analysis_set.fna.gz already exists. Nothing to do"; + touch GCA_000001405.15_GRCh38_no_alt_plus_hs38d1_analysis_set.fna.gz; + exit 0 +fi + +curl -LO https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/405/GCF_000001405.40_GRCh38.p14/GRCh38_major_release_seqs_for_alignment_pipelines/GCA_000001405.15_GRCh38_no_alt_plus_hs38d1_analysis_set.fna.gz +echo "Downloaded human db to $DEST" diff --git a/pawsey_minion/download_taxon_db.slurm b/pawsey_minion/download_taxon_db.slurm new file mode 100644 index 0000000..db3d3d5 --- /dev/null +++ b/pawsey_minion/download_taxon_db.slurm @@ -0,0 +1,38 @@ +#!/bin/bash +#SBATCH --job-name=taxon_dld +#SBATCH --time=1-0 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=1 +#SBATCH --mem=32G +#SBATCH -o %x-%j.out +#SBATCH -e %x-%j.err + +TAXDIR="/scratch/$PAWSEY_PROJECT/$USER/Databases/NCBI/taxonomy/current" + +mkdir -p $TAXDIR +cd $TAXDIR; + +required_files=( + citations.dmp delnodes.dmp division.dmp gc.prt + gencode.dmp images.dmp merged.dmp names.dmp + nodes.dmp readme.txt +) + +missing=0 + +for file in "${required_files[@]}"; do + if [[ ! -f "$file" ]]; then + echo "Missing: $file" + missing=1 + fi +done + +if [[ $missing -eq 1 ]]; then + echo "One or more files are missing. Downloading taxdump.tar.gz..." + curl -LO https://ftp.ncbi.nih.gov/pub/taxonomy/taxdump.tar.gz + tar zxf taxdump.tar.gz && rm -f taxdump.tar.gz +else + echo "All required files are present." +fi + +echo "Done" diff --git a/pawsey_minion/download_uniref100.slurm b/pawsey_minion/download_uniref100.slurm new file mode 100644 index 0000000..53f75b4 --- /dev/null +++ b/pawsey_minion/download_uniref100.slurm @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=MMSeqsDld100 +#SBATCH --time=1-0 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=1 +#SBATCH --mem=128G +#SBATCH -o %x-%j.out +#SBATCH -e %x-%j.err + +# Use `mmseqs databases` to find information about the preformatted databases +# NOTE: Do not download the NT database unless you _really_ need it. The database is huge! + + +set -euo pipefail +trap 's=$?; echo "$0: Error on line "$LINENO": $BASH_COMMAND"; exit $s' ERR + +# test we have atavide_lite conda installed, and then activate it +eval "$(conda shell.bash hook)" +"$HOME/GitHubs/atavide_lite/pawsey_lib/check_atavide_lite_env.sh" +ATAVIDE_CONDA="/scratch/$PAWSEY_PROJECT/$USER/software/miniconda3/atavide_lite" +conda activate $ATAVIDE_CONDA + +CWD=`pwd` + +#DB can also be UniRef100 UniRef100 GTDB etc +DB=UniRef100 + + +required_files=( + UniRef100 UniRef100.dbtype UniRef100_h UniRef100_h.dbtype + UniRef100_h.index UniRef100.index UniRef100.lookup UniRef100_mapping + UniRef100.source UniRef100_taxonomy UniRef100.version +) + +missing=0 + +DESTINATION=/scratch/$PAWSEY_PROJECT/$USER/Databases/$DB +TMP=/scratch/$PAWSEY_PROJECT/$USER/tmp +mkdir -p $DESTINATION $TMP + +cd $DESTINATION + +for file in "${required_files[@]}"; do + if [[ ! -f "$DB/$file" ]]; then + echo "Missing: $DB/$file" + missing=1 + fi +done + +if [[ $missing -eq 0 ]]; then + echo "All required files are present." + exit 0 +fi + +echo "Downloading required files" + +cd $CWD +# delete the old Destination and make the dir to remove partial files. + +rm -rf $DESTINATION && mkdir -p $DESTINATION +TPD=$(mktemp -d -p $TMP) +echo -e "Downloading to $DESTINATION/$DB and using temp location $TPD.\nNote we download the tarball and then uncompress, so there is usually nothing in the output for a while!" >&2 +mmseqs databases --threads 8 $DB $DESTINATION/$DB $TPD + + + diff --git a/pawsey_minion/download_uniref50.slurm b/pawsey_minion/download_uniref50.slurm new file mode 100644 index 0000000..e67d271 --- /dev/null +++ b/pawsey_minion/download_uniref50.slurm @@ -0,0 +1,67 @@ +#!/bin/bash +#SBATCH --job-name=MMSeqsDld +#SBATCH --time=1-0 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=1 +#SBATCH --mem=128G +#SBATCH -o %x-%j.out +#SBATCH -e %x-%j.err + +# Use `mmseqs databases` to find information about the preformatted databases +# NOTE: Do not download the NT database unless you _really_ need it. The database is huge! + + + +set -euo pipefail +trap 's=$?; echo "$0: Error on line "$LINENO": $BASH_COMMAND"; exit $s' ERR + +# test we have atavide_lite conda installed, and then activate it +eval "$(conda shell.bash hook)" +"$HOME/GitHubs/atavide_lite/pawsey_lib/check_atavide_lite_env.sh" +ATAVIDE_CONDA="/scratch/$PAWSEY_PROJECT/$USER/software/miniconda3/atavide_lite" +conda activate $ATAVIDE_CONDA + +CWD=`pwd` + +#DB can also be UniRef50 UniRef100 GTDB etc +DB=UniRef50 + + +required_files=( + UniRef50 UniRef50.dbtype UniRef50_h UniRef50_h.dbtype + UniRef50_h.index UniRef50.index UniRef50.lookup UniRef50_mapping + UniRef50.source UniRef50_taxonomy UniRef50.version +) + +missing=0 + +DESTINATION=/scratch/$PAWSEY_PROJECT/$USER/Databases/$DB +TMP=/scratch/$PAWSEY_PROJECT/$USER/tmp +mkdir -p $DESTINATION $TMP + +cd $DESTINATION + +for file in "${required_files[@]}"; do + if [[ ! -f "$DB/$file" ]]; then + echo "Missing: $DB/$file" + missing=1 + fi +done + +if [[ $missing -eq 0 ]]; then + echo "All required files are present." + exit 0 +fi + +echo "Downloading required files" + +cd $CWD +# delete the old Destination and make the dir to remove partial files. + +rm -rf $DESTINATION && mkdir -p $DESTINATION +TPD=$(mktemp -d -p $TMP) +echo -e "Downloading to $DESTINATION/$DB and using temp location $TPD.\nNote we download the tarball and then uncompress, so there is usually nothing in the output for a while!" >&2 +mmseqs databases --threads 8 $DB $DESTINATION/$DB $TPD + + + diff --git a/pawsey_minion/vamb.slurm b/pawsey_minion/vamb.slurm index 370246f..9238864 100644 --- a/pawsey_minion/vamb.slurm +++ b/pawsey_minion/vamb.slurm @@ -14,10 +14,11 @@ trap 's=$?; echo "$0: Error on line "$LINENO": $BASH_COMMAND"; exit $s' ERR # test we have atavide_lite conda installed, and then activate it eval "$(conda shell.bash hook)" "$HOME/GitHubs/atavide_lite/pawsey_lib/check_atavide_lite_env.sh" -ATAVIDE_CONDA="/scratch/$PAWSEY_PROJECT/$USER/software/miniconda3/atavide_lite" +ATAVIDE_CONDA="/scratch/$PAWSEY_PROJECT/$USER/software/miniconda3/atavide_lite_vamb" conda activate $ATAVIDE_CONDA -module load pytorch/2.2.0-rocm5.7.3 +# module load pytorch/2.2.0-rocm5.7.3 +module load pytorch/2.7.1-rocm6.3.3 if [[ ! -e DEFINITIONS.sh ]]; then echo "Please create a DEFINITIONS.sh file with SOURCE, FILEEND, HOSTREMOVED" >&2 @@ -35,11 +36,14 @@ if [[ ! -e $OUTDIR/contigs.fna.gz ]]; then exit 1; fi +mkdir -p $OUTDIR/mapped_reads + for READS in $(sort -R reads.txt); do BAM=${READS/$FILEEND/.bam} echo "READS: $READS BAM: $BAM" >&2; - if [[ -e $OUTDIR/mapped_reads/$BAM ]]; then + if [[ ! -e $OUTDIR/mapped_reads/$BAM ]]; then echo "Bam file: '$OUTDIR/mapped_reads/$BAM' not found. Data generated" >&2; + mkdir -p $OUTDIR/mapped_reads minimap2 -t 16 -N 5 -ax map-ont $OUTDIR/contigs.mmi --split-prefix mmsplit$$ fastq/$READS | samtools view -F 3584 -b --threads 16 | samtools sort -@ 16 -o $OUTDIR/mapped_reads/$BAM - fi done diff --git a/pawsey_minion/vamb_concat.slurm b/pawsey_minion/vamb_concat.slurm index 111d237..fe610b7 100644 --- a/pawsey_minion/vamb_concat.slurm +++ b/pawsey_minion/vamb_concat.slurm @@ -13,7 +13,7 @@ trap 's=$?; echo "$0: Error on line "$LINENO": $BASH_COMMAND"; exit $s' ERR # test we have atavide_lite conda installed, and then activate it eval "$(conda shell.bash hook)" "$HOME/GitHubs/atavide_lite/pawsey_lib/check_atavide_lite_env.sh" -ATAVIDE_CONDA="/scratch/$PAWSEY_PROJECT/$USER/software/miniconda3/atavide_lite" +ATAVIDE_CONDA="/scratch/$PAWSEY_PROJECT/$USER/software/miniconda3/atavide_lite_vamb" conda activate $ATAVIDE_CONDA mkdir -p vamb diff --git a/pawsey_shortread/README.md b/pawsey_shortread/README.md index 5ed8b5f..1c1b080 100644 --- a/pawsey_shortread/README.md +++ b/pawsey_shortread/README.md @@ -17,6 +17,13 @@ mamba env create --yes --prefix /scratch/$PAWSEY_PROJECT/$USER/software/minicond This creates an `atavide_lite` and `atavide_lite_vamb` conda environments for us to use. +Make sure you build atavide_lite/bin: + +``` +pushd $HOME/GitHubs/atavide_lite/bin +make all +popd +``` ## 0. Get your data. diff --git a/pawsey_shortread/download_human.slurm b/pawsey_shortread/download_human.slurm index d31129e..af67df8 100644 --- a/pawsey_shortread/download_human.slurm +++ b/pawsey_shortread/download_human.slurm @@ -4,8 +4,8 @@ #SBATCH --ntasks=1 #SBATCH --cpus-per-task=1 #SBATCH --mem=32G -#SBATCH -o slurm/human_download-%j.out -#SBATCH -e slurm/human_download-%j.err +#SBATCH -o %x-%j.out +#SBATCH -e %x-%j.err DEST=/scratch/$PAWSEY_PROJECT/$USER/Databases/human diff --git a/pawsey_shortread/download_taxon_db.slurm b/pawsey_shortread/download_taxon_db.slurm index fa811d4..db3d3d5 100644 --- a/pawsey_shortread/download_taxon_db.slurm +++ b/pawsey_shortread/download_taxon_db.slurm @@ -4,8 +4,8 @@ #SBATCH --ntasks=1 #SBATCH --cpus-per-task=1 #SBATCH --mem=32G -#SBATCH -o slurm_output/taxon_download-%j.out -#SBATCH -e slurm_output/taxon_download-%j.err +#SBATCH -o %x-%j.out +#SBATCH -e %x-%j.err TAXDIR="/scratch/$PAWSEY_PROJECT/$USER/Databases/NCBI/taxonomy/current" diff --git a/pawsey_shortread/download_uniref50.slurm b/pawsey_shortread/download_uniref50.slurm index 5b9de17..e67d271 100644 --- a/pawsey_shortread/download_uniref50.slurm +++ b/pawsey_shortread/download_uniref50.slurm @@ -4,8 +4,8 @@ #SBATCH --ntasks=1 #SBATCH --cpus-per-task=1 #SBATCH --mem=128G -#SBATCH -o mmseqsdld-%j.out -#SBATCH -e mmseqsdld-%j.err +#SBATCH -o %x-%j.out +#SBATCH -e %x-%j.err # Use `mmseqs databases` to find information about the preformatted databases # NOTE: Do not download the NT database unless you _really_ need it. The database is huge! diff --git a/pawsey_shortread/mmseqs_easy_taxonomy100.slurm b/pawsey_shortread/mmseqs_easy_taxonomy100.slurm index 23e4999..ce8fd18 100644 --- a/pawsey_shortread/mmseqs_easy_taxonomy100.slurm +++ b/pawsey_shortread/mmseqs_easy_taxonomy100.slurm @@ -13,7 +13,7 @@ #SBATCH --job-name=mmseqsET100 -#SBATCH --time=1-0 +#SBATCH --time=06:00:00 #SBATCH --ntasks=1 #SBATCH --cpus-per-task=1 #SBATCH --mem=480G