diff --git a/CHANGELOG.md b/CHANGELOG.md index 59ae85b..afc95d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## ASPEN development version +- Add attempt-aware resource scaling for the main Snakemake rules, document how baseline `cluster.json` resources interact with retry-based `resources.gres` scaling, and fix dry-run failure handling so failed dry-runs now exit non-zero instead of reporting success. (#152, @kopardev) - Detect SLURM-killed Snakemake child jobs in classic cluster mode by adding a `--cluster-status` hook and `scancel` integration, so timed out, OOM-killed, cancelled, or otherwise failed child jobs now retry/fail cleanly instead of leaving the master workflow hanging indefinitely. (#148, @kopardev) - Clarify motif enrichment outputs and interpretation in the documentation: fix the reported location of HOMER/AME result folders, explain how ASPEN reuses HOMER-generated `target.fa`/`background.fa` in AME, document the AME parallelization strategy, and add simpler guidance for interpreting `knownResults.txt` and `ame_results.txt`. (#146, @kopardev) diff --git a/aspen b/aspen index 4b4ee2d..e64d38a 100755 --- a/aspen +++ b/aspen @@ -326,7 +326,7 @@ function set_cluster_arg(){ CLUSTER_SBATCH_CMD="sbatch --cpus-per-task {cluster.threads} -p {cluster.partition} -t {cluster.time} --mem {cluster.mem} --job-name {cluster.name} --output {cluster.output} --error {cluster.error}" clustername=$(scontrol show config|grep ClusterName|awk '{print $NF}') if [[ "$clustername" == "biowulf" ]];then - CLUSTER_SBATCH_CMD="${CLUSTER_SBATCH_CMD} --gres {cluster.gres}" + CLUSTER_SBATCH_CMD="${CLUSTER_SBATCH_CMD} --gres {resources.gres}" fi } @@ -397,7 +397,14 @@ function dryrun() { mv ${WORKDIR}/dryrun_git_commit.txt ${WORKDIR}/logs/dryrun_git_commit.${modtime}.txt fi fi - run "--dry-run" | tee ${WORKDIR}/dryrun.log + set -o pipefail + run "--dry-run" 2>&1 | tee ${WORKDIR}/dryrun.log + dryrun_rc=${PIPESTATUS[0]} + set +o pipefail + if [ "${dryrun_rc}" -ne 0 ];then + log_error "Dry-run failed with exit code ${dryrun_rc}. See ${WORKDIR}/dryrun.log for details." + exit "${dryrun_rc}" + fi echo "Git Commit/Tag: $GIT_COMMIT_TAG" > ${WORKDIR}/dryrun_git_commit.txt log_ok "Dry-run completed." log_info "Full dry-run output captured in ${WORKDIR}/dryrun.log" @@ -694,7 +701,7 @@ _START_TS=\$(date -u +"%Y-%m-%dT%H:%M:%SZ") _progress_monitor & _monitor_pid=\$! -snakemake -s $SNAKEFILE --directory $WORKDIR $SINGULARITY_STR --printshellcmds --latency-wait 120 --configfile $CONFIGFILE --cluster-config $CLUSTERFILE --cluster "$CLUSTER_SBATCH_CMD" --cluster-status "$CLUSTERSTATUSCMD" --cluster-cancel scancel -j 500 --rerun-incomplete ${RERUNTRIGGERS} --restart-times ${retries} --keep-going --stats ${WORKDIR}/snakemake.stats 2>&1 | tee ${WORKDIR}/snakemake.log +snakemake -s $SNAKEFILE --directory $WORKDIR $SINGULARITY_STR --printshellcmds --latency-wait 120 --configfile $CONFIGFILE --cluster-config $CLUSTERFILE --default-resources "gres='lscratch:256'" --cluster "$CLUSTER_SBATCH_CMD" --cluster-status "$CLUSTERSTATUSCMD" --cluster-cancel scancel -j 500 --rerun-incomplete ${RERUNTRIGGERS} --restart-times ${retries} --keep-going --stats ${WORKDIR}/snakemake.stats 2>&1 | tee ${WORKDIR}/snakemake.log snakemake_rc="\${PIPESTATUS[0]}" kill "\$_monitor_pid" 2>/dev/null || true wait "\$_monitor_pid" 2>/dev/null || true @@ -770,6 +777,7 @@ snakemake $1 -s $SNAKEFILE \ --latency-wait 120 \ --configfile $CONFIGFILE \ --cluster-config $CLUSTERFILE \ +--default-resources "gres='lscratch:256'" \ --cluster "$CLUSTER_SBATCH_CMD" \ -j 500 \ --rerun-incomplete \ diff --git a/docs/deployment.md b/docs/deployment.md index 818b7ef..3ed5f6a 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -262,6 +262,8 @@ aspen -m=run -w= This command submits a master job to the Slurm workload manager, which orchestrates the entire analysis workflow, managing job submissions and monitoring progress. +ASPEN submits the generated Snakemake job with rule-level resources exposed to Slurm. The heavy rules that may need extra scratch space now define an attempt-aware `resources.gres` value, so the first submission uses the baseline `cluster.json` request and later retries can scale the requested `lscratch` allocation automatically if a rule is retried by Snakemake. This keeps the default configuration in `cluster.json` while still allowing individual rules to become more conservative on repeated failures. + - 🛠️ **Optional Argument**: `--singcache` or `-c`: Specify a Singularity cache directory. The default is `/data/${USER}/.singularity` if available; otherwise, it defaults to `${WORKDIR}/snakemake/.singularity`. diff --git a/docs/outputs.md b/docs/outputs.md index 3778d30..579b884 100644 --- a/docs/outputs.md +++ b/docs/outputs.md @@ -33,29 +33,29 @@ WORKDIR Here are more details about these files: -| **File** | **File Type** | **Mode (`-m`) When This File is Created/Overwritten** | **Description** | -| ----------------------------------------------------------------------------------- | ------------- | ----------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `cluster.json` | JSON | init | Defines cluster resources per snakemake rule; this file can be edited to override default computate resource allocations per snakemake rule | -| `config.yaml` | YAML | init; can be edited later | Configurable parameters for this specific run | -| `contrasts.tsv` | TSV | Needs to be added in after init | List of contrasts to run, one per line; has no header | -| `dryrun_git_commit.txt` | TXT | dryrun | The git commit hash of the version of ASPEN used at dryrun | -| `dryrun.log` | TXT | dryrun | Log from `-m=dryrun` | -| `fastqs` | FOLDER | dryrun | Folder containing symlinks to raw data | -| `logs` | FOLDER | dryrun | Folder containing all logs including Slurm `.out` and `.err` files. Also contains older timestamped `runinfo.yaml` and `snakemake.stats` files. | -| `results` | FOLDER | Created at dryrun but populated during run | Main outputs folder | -| `pipeline.status.json` | JSON | run | Machine-readable state sidecar: `state`, `reason`, `slurm_job_id`, start/end timestamps, `duration_seconds`, `tasks_done`/`tasks_total`, `exit_code` | -| `pipeline.running` / `pipeline.completed` / `pipeline.failed` / `pipeline.canceled` | TXT | run | Human-readable state marker file; exactly one of these exists at a time. `pipeline.running` is periodically refreshed with a step-completion progress summary while the pipeline is executing | -| `runinfo.yaml` | YAML | After completion of run | Metadata about the run executor, etc. | -| `runslurm_snakemake_report.html` | HTML | After completion of run | HTML report including DAG and resource utilization | -| `sampleinfo.txt` | TXT | dryrun, run | Tab-delimited mappings between `replicateNames` and `sampleNames` | -| `samples.tsv` | TSV | init; can be edited later | Tab-delimited manifest with `replicateName`, `sampleName`, `path_to_R1_fastq`, `path_to_R2_fastq`. This file has a header. | -| `scripts` | FOLDER | init | Folder keeps local copy of scripts called by various rules | -| `run_git_commit.txt` | TXT | run | The git commit hash of the version of ASPEN used at run | -| `slurm-XXXXXXX.out` | TXT | run | Slurm `.out` file for the master job | -| `snakemake.log` | TXT | run | Snakemake `.log` file for the master job; older copies timestamped and moved into `logs` folder | -| `snakemake.stats` | JSON | run | per rule runtime stats | -| `submit_script.sbatch` | TXT | run | Slurm script to kickstart the main Snakemake job | -| `tools.yaml` | YAML | run | YAML containing the version of tools used in the pipeline (obsolete; was used to load specific module versions prior to moving over to Docker/Singularity containers) | +| **File** | **File Type** | **Mode (`-m`) When This File is Created/Overwritten** | **Description** | +| ----------------------------------------------------------------------------------- | ------------- | ----------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `cluster.json` | JSON | init | Defines the baseline cluster resources per Snakemake rule. ASPEN now forwards rule-level `resources.gres` to Slurm, so if a rule declares an attempt-aware resource expression, retries can request more `lscratch` automatically while still starting from the defaults recorded here. | +| `config.yaml` | YAML | init; can be edited later | Configurable parameters for this specific run | +| `contrasts.tsv` | TSV | Needs to be added in after init | List of contrasts to run, one per line; has no header | +| `dryrun_git_commit.txt` | TXT | dryrun | The git commit hash of the version of ASPEN used at dryrun | +| `dryrun.log` | TXT | dryrun | Log from `-m=dryrun` | +| `fastqs` | FOLDER | dryrun | Folder containing symlinks to raw data | +| `logs` | FOLDER | dryrun | Folder containing all logs including Slurm `.out` and `.err` files. Also contains older timestamped `runinfo.yaml` and `snakemake.stats` files. | +| `results` | FOLDER | Created at dryrun but populated during run | Main outputs folder | +| `pipeline.status.json` | JSON | run | Machine-readable state sidecar: `state`, `reason`, `slurm_job_id`, start/end timestamps, `duration_seconds`, `tasks_done`/`tasks_total`, `exit_code` | +| `pipeline.running` / `pipeline.completed` / `pipeline.failed` / `pipeline.canceled` | TXT | run | Human-readable state marker file; exactly one of these exists at a time. `pipeline.running` is periodically refreshed with a step-completion progress summary while the pipeline is executing | +| `runinfo.yaml` | YAML | After completion of run | Metadata about the run executor, etc. | +| `runslurm_snakemake_report.html` | HTML | After completion of run | HTML report including DAG and resource utilization | +| `sampleinfo.txt` | TXT | dryrun, run | Tab-delimited mappings between `replicateNames` and `sampleNames` | +| `samples.tsv` | TSV | init; can be edited later | Tab-delimited manifest with `replicateName`, `sampleName`, `path_to_R1_fastq`, `path_to_R2_fastq`. This file has a header. | +| `scripts` | FOLDER | init | Folder keeps local copy of scripts called by various rules | +| `run_git_commit.txt` | TXT | run | The git commit hash of the version of ASPEN used at run | +| `slurm-XXXXXXX.out` | TXT | run | Slurm `.out` file for the master job | +| `snakemake.log` | TXT | run | Snakemake `.log` file for the master job; older copies timestamped and moved into `logs` folder | +| `snakemake.stats` | JSON | run | per rule runtime stats | +| `submit_script.sbatch` | TXT | run | Slurm script to kickstart the main Snakemake job | +| `tools.yaml` | YAML | run | YAML containing the version of tools used in the pipeline (obsolete; was used to load specific module versions prior to moving over to Docker/Singularity containers) | ## 📊 `results` folder diff --git a/resources/README.md b/resources/README.md index 7b2d4dc..66ad035 100644 --- a/resources/README.md +++ b/resources/README.md @@ -7,6 +7,6 @@ This folder, `resources/`, is meant to contain all resources (except larger geno - **motif**: HOCOMOCO v11 motifs in HOMER and MEME formats for human and mouse. - **tssBed**: BED files pin-pointing the transcription start sites of annotated genes for hg19/38 and mm10. -`cluster.json` file can make specific resource requests to biowulf via slurm. Each Snakemake rule can make a unique hardware request for slurm execution. +`cluster.json` file can make specific resource requests to biowulf via slurm. ASPEN uses these values as the baseline per-rule resource settings, and the workflow now passes `resources.gres` through to Slurm so retry attempts can request more `lscratch` when a rule defines an attempt-aware `resources:` expression. In practice, this means `cluster.json` still controls the default resource profile, while the rule definitions can increase the `gres` request automatically on later Snakemake attempts. `tools.yaml` file would typically contain the modules required to be loaded for rule execution on the Biowulf cluster. Since we are using dockers for all rules in this pipeline, `tools.yaml` will be empty. diff --git a/workflow/rules/align.smk b/workflow/rules/align.smk index 8d6900a..c8a7b2c 100644 --- a/workflow/rules/align.smk +++ b/workflow/rules/align.smk @@ -36,6 +36,8 @@ rule trim: script="ccbr_cutadapt_pe.bash" container: config["cutadaptdocker"] threads: getthreads("trim") + resources: + gres=lambda wildcards, attempt: scale_gres("trim", attempt) shell:""" if [ -w "/lscratch/${{SLURM_JOB_ID}}" ];then cd /lscratch/${{SLURM_JOB_ID}};else cd /dev/shm;fi unset PYTHONPATH @@ -64,6 +66,8 @@ rule create_BL_index: genome=GENOME, container: config["masterdocker"] threads: getthreads("create_BL_index") + resources: + gres=lambda wildcards, attempt: scale_gres("create_BL_index", attempt) shell:""" blindexdir=$(dirname {output.sa}) sabasename=$(basename {output.sa}) @@ -95,6 +99,8 @@ rule remove_BL: script="ccbr_remove_blacklisted_reads_pe.bash" container: config["masterdocker"] threads: getthreads("remove_BL") + resources: + gres=lambda wildcards, attempt: scale_gres("remove_BL", attempt) shell:""" if [ -w "/lscratch/${{SLURM_JOB_ID}}" ];then cd /lscratch/${{SLURM_JOB_ID}};else cd /dev/shm;fi sa={input.sa} @@ -154,6 +160,8 @@ rule align: mem=getmemG("align") container: config["masterdocker"] threads: getthreads("align") + resources: + gres=lambda wildcards, attempt: scale_gres("align", attempt) shell:""" set -exo pipefail if [ -w "/lscratch/${{SLURM_JOB_ID}}" ];then cd /lscratch/${{SLURM_JOB_ID}};else cd /dev/shm;fi @@ -255,6 +263,8 @@ rule align2spikein: mem=getmemG("align2spikein") container: config["bwadocker"] threads: getthreads("align2spikein") + resources: + gres=lambda wildcards, attempt: scale_gres("align2spikein", attempt) shell:""" set -exo pipefail if [ -w "/lscratch/${{SLURM_JOB_ID}}" ];then cd /lscratch/${{SLURM_JOB_ID}};else cd /dev/shm;fi @@ -361,6 +371,8 @@ rule create_tn5bams: script="ccbr_atac_bam2tn5bed.py" container: config["masterdocker"] threads: getthreads("create_tn5bams") + resources: + gres=lambda wildcards, attempt: scale_gres("create_tn5bams", attempt) shell:""" set -exo pipefail unset PYTHONPATH @@ -433,6 +445,8 @@ rule create_bigwigs: script="_print_replicate_scaling_factor.py" container: config["deeptoolsdocker"] threads: getthreads("create_bigwigs") + resources: + gres=lambda wildcards, attempt: scale_gres("create_bigwigs", attempt) shell:""" set -exo pipefail unset PYTHONPATH diff --git a/workflow/rules/init.smk b/workflow/rules/init.smk index e03ab45..f0269df 100644 --- a/workflow/rules/init.smk +++ b/workflow/rules/init.smk @@ -231,6 +231,15 @@ with open(CLUSTERJSON) as json_file: getthreads=lambda rname:int(CLUSTER[rname]["threads"]) if rname in CLUSTER and "threads" in CLUSTER[rname] else int(CLUSTER["__default__"]["threads"]) getmemg=lambda rname:CLUSTER[rname]["mem"] if rname in CLUSTER else CLUSTER["__default__"]["mem"] getmemG=lambda rname:getmemg(rname).replace("g","G") +getgres=lambda rname:CLUSTER[rname]["gres"] if rname in CLUSTER and "gres" in CLUSTER[rname] else CLUSTER["__default__"]["gres"] + +def scale_gres(rname, attempt): + base_gres = getgres(rname) + if ":" in str(base_gres): + prefix, size = str(base_gres).rsplit(":", 1) + if size.isdigit(): + return f"{prefix}:{int(size) * int(attempt)}" + return base_gres ######################################################### #########################################################