diff --git a/CHANGELOG.md b/CHANGELOG.md index b711aa0..59ae85b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## ASPEN development version +- Detect SLURM-killed Snakemake child jobs in classic cluster mode by adding a `--cluster-status` hook and `scancel` integration, so timed out, OOM-killed, cancelled, or otherwise failed child jobs now retry/fail cleanly instead of leaving the master workflow hanging indefinitely. (#148, @kopardev) - Clarify motif enrichment outputs and interpretation in the documentation: fix the reported location of HOMER/AME result folders, explain how ASPEN reuses HOMER-generated `target.fa`/`background.fa` in AME, document the AME parallelization strategy, and add simpler guidance for interpreting `knownResults.txt` and `ame_results.txt`. (#146, @kopardev) ## ASPEN 1.4.0 diff --git a/aspen b/aspen index 9e52b7a..4b4ee2d 100755 --- a/aspen +++ b/aspen @@ -224,7 +224,7 @@ fi # copy essential folders for f in $ESSENTIAL_FOLDERS;do # rsync -az --progress ${PIPELINE_HOME}/$f $WORKDIR/ - cp -rv ${PIPELINE_HOME}/$f ${WORKDIR}/ + cp -r ${PIPELINE_HOME}/$f ${WORKDIR}/ done cd ${WORKDIR} @@ -400,6 +400,7 @@ function dryrun() { run "--dry-run" | tee ${WORKDIR}/dryrun.log echo "Git Commit/Tag: $GIT_COMMIT_TAG" > ${WORKDIR}/dryrun_git_commit.txt log_ok "Dry-run completed." + log_info "Full dry-run output captured in ${WORKDIR}/dryrun.log" log_next "Submit run with: aspen --workdir=$WORKDIR --runmode=run" } @@ -451,6 +452,9 @@ function runslurm() { # Submit the execution of the pipeline to the biowulf job scheduler (slurm) runcheck + if [[ ! -r "$CLUSTERSTATUSCMD" || ! -x "$CLUSTERSTATUSCMD" ]]; then + err "Cluster status script '$CLUSTERSTATUSCMD' is missing or not executable." + fi set_singularity_binds set_cluster_arg log_divider @@ -690,7 +694,7 @@ _START_TS=\$(date -u +"%Y-%m-%dT%H:%M:%SZ") _progress_monitor & _monitor_pid=\$! -snakemake -s $SNAKEFILE --directory $WORKDIR $SINGULARITY_STR --printshellcmds --latency-wait 120 --configfile $CONFIGFILE --cluster-config $CLUSTERFILE --cluster "$CLUSTER_SBATCH_CMD" -j 500 --rerun-incomplete ${RERUNTRIGGERS} --restart-times ${retries} --keep-going --stats ${WORKDIR}/snakemake.stats 2>&1 | tee ${WORKDIR}/snakemake.log +snakemake -s $SNAKEFILE --directory $WORKDIR $SINGULARITY_STR --printshellcmds --latency-wait 120 --configfile $CONFIGFILE --cluster-config $CLUSTERFILE --cluster "$CLUSTER_SBATCH_CMD" --cluster-status "$CLUSTERSTATUSCMD" --cluster-cancel scancel -j 500 --rerun-incomplete ${RERUNTRIGGERS} --restart-times ${retries} --keep-going --stats ${WORKDIR}/snakemake.stats 2>&1 | tee ${WORKDIR}/snakemake.log snakemake_rc="\${PIPESTATUS[0]}" kill "\$_monitor_pid" 2>/dev/null || true wait "\$_monitor_pid" 2>/dev/null || true @@ -878,7 +882,7 @@ function main(){ CONFIGFILE="${WORKDIR}/config.yaml" CLUSTERFILE="${WORKDIR}/cluster.json" - # CLUSTERSTATUSCMD="${PIPELINE_HOME}/resources/cluster_status.sh" + CLUSTERSTATUSCMD="${PIPELINE_HOME}/resources/cluster_status.sh" if [[ -z "$SING_CACHE_DIR" ]]; then if [[ -n "$SIFCACHE" && -d "$SIFCACHE" ]]; then diff --git a/resources/cluster_status.sh b/resources/cluster_status.sh new file mode 100755 index 0000000..028283b --- /dev/null +++ b/resources/cluster_status.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -euo pipefail + +job_id="${1:-}" + +if [[ -z "${job_id}" ]]; then + echo "failed" + exit 0 +fi + +state=$(sacct -X -j "${job_id}" -n -P --format=State 2>/dev/null | awk 'NF {print; exit}') +state=${state%% *} + +if [[ -z "${state}" ]]; then + # sacct can briefly lag behind submission; treat unknown-new jobs as running + # to avoid false failures immediately after launch. + echo "running" + exit 0 +fi + +case "${state}" in + COMPLETED) + echo "success" + ;; + PENDING|CONFIGURING|RUNNING|COMPLETING|SUSPENDED|RESIZING|REQUEUED|REQUEUE_HOLD) + echo "running" + ;; + FAILED|CANCELLED|TIMEOUT|OUT_OF_MEMORY|NODE_FAIL|PREEMPTED|BOOT_FAIL|DEADLINE|*) + echo "failed" + ;; +esac