fix(snakemake): detect cancelled child jobs

CHANGELOG.md

@@ -1,5 +1,6 @@
 ## ASPEN development version
 
+- Detect SLURM-killed Snakemake child jobs in classic cluster mode by adding a `--cluster-status` hook and `scancel` integration, so timed out, OOM-killed, cancelled, or otherwise failed child jobs now retry/fail cleanly instead of leaving the master workflow hanging indefinitely. (#148, @kopardev)
 - Clarify motif enrichment outputs and interpretation in the documentation: fix the reported location of HOMER/AME result folders, explain how ASPEN reuses HOMER-generated `target.fa`/`background.fa` in AME, document the AME parallelization strategy, and add simpler guidance for interpreting `knownResults.txt` and `ame_results.txt`. (#146, @kopardev)
 
 ## ASPEN 1.4.0

aspen

@@ -224,7 +224,7 @@ fi
 # copy essential folders
 for f in $ESSENTIAL_FOLDERS;do
   # rsync -az --progress ${PIPELINE_HOME}/$f $WORKDIR/
-  cp -rv ${PIPELINE_HOME}/$f ${WORKDIR}/
+  cp -r ${PIPELINE_HOME}/$f ${WORKDIR}/
 done
 
 cd ${WORKDIR}
@@ -400,6 +400,7 @@ function dryrun() {
   run "--dry-run" | tee ${WORKDIR}/dryrun.log
   echo "Git Commit/Tag: $GIT_COMMIT_TAG" > ${WORKDIR}/dryrun_git_commit.txt
   log_ok "Dry-run completed."
+  log_info "Full dry-run output captured in ${WORKDIR}/dryrun.log"
   log_next "Submit run with: aspen --workdir=$WORKDIR --runmode=run"
 }
 
@@ -451,6 +452,9 @@ function runslurm() {
 # Submit the execution of the pipeline to the biowulf job scheduler (slurm)
 
   runcheck
+  if [[ ! -r "$CLUSTERSTATUSCMD" || ! -x "$CLUSTERSTATUSCMD" ]]; then
+    err "Cluster status script '$CLUSTERSTATUSCMD' is missing or not executable."
+  fi
   set_singularity_binds
   set_cluster_arg
   log_divider
@@ -690,7 +694,7 @@ _START_TS=\$(date -u +"%Y-%m-%dT%H:%M:%SZ")
 _progress_monitor &
 _monitor_pid=\$!
 
-snakemake -s $SNAKEFILE --directory $WORKDIR $SINGULARITY_STR --printshellcmds --latency-wait 120 --configfile $CONFIGFILE --cluster-config $CLUSTERFILE --cluster "$CLUSTER_SBATCH_CMD" -j 500 --rerun-incomplete ${RERUNTRIGGERS} --restart-times ${retries} --keep-going --stats ${WORKDIR}/snakemake.stats 2>&1 | tee ${WORKDIR}/snakemake.log
+snakemake -s $SNAKEFILE --directory $WORKDIR $SINGULARITY_STR --printshellcmds --latency-wait 120 --configfile $CONFIGFILE --cluster-config $CLUSTERFILE --cluster "$CLUSTER_SBATCH_CMD" --cluster-status "$CLUSTERSTATUSCMD" --cluster-cancel scancel -j 500 --rerun-incomplete ${RERUNTRIGGERS} --restart-times ${retries} --keep-going --stats ${WORKDIR}/snakemake.stats 2>&1 | tee ${WORKDIR}/snakemake.log
 snakemake_rc="\${PIPESTATUS[0]}"
 kill "\$_monitor_pid" 2>/dev/null || true
 wait "\$_monitor_pid" 2>/dev/null || true
@@ -878,7 +882,7 @@ function main(){
   CONFIGFILE="${WORKDIR}/config.yaml"
   CLUSTERFILE="${WORKDIR}/cluster.json"
 
-  # CLUSTERSTATUSCMD="${PIPELINE_HOME}/resources/cluster_status.sh"
+  CLUSTERSTATUSCMD="${PIPELINE_HOME}/resources/cluster_status.sh"
 
   if [[ -z "$SING_CACHE_DIR" ]]; then
     if [[ -n "$SIFCACHE" && -d "$SIFCACHE" ]]; then

resources/cluster_status.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+job_id="${1:-}"
+
+if [[ -z "${job_id}" ]]; then
+  echo "failed"
+  exit 0
+fi
+
+state=$(sacct -X -j "${job_id}" -n -P --format=State 2>/dev/null | awk 'NF {print; exit}')
+state=${state%% *}
+
+if [[ -z "${state}" ]]; then
+  # sacct can briefly lag behind submission; treat unknown-new jobs as running
+  # to avoid false failures immediately after launch.
+  echo "running"
+  exit 0
+fi
+
+case "${state}" in
+  COMPLETED)
+    echo "success"
+    ;;
+  PENDING|CONFIGURING|RUNNING|COMPLETING|SUSPENDED|RESIZING|REQUEUED|REQUEUE_HOLD)
+    echo "running"
+    ;;
+  FAILED|CANCELLED|TIMEOUT|OUT_OF_MEMORY|NODE_FAIL|PREEMPTED|BOOT_FAIL|DEADLINE|*)
+    echo "failed"
+    ;;
+esac