From 98a21223ae55f570a57d07b39a32c6b352eab83a Mon Sep 17 00:00:00 2001
From: kopardev <vishal.koparde@nih.gov>
Date: Thu, 24 Sep 2026 15:30:36 -0400
Subject: [PATCH 1/3] fix(snakemake): detect cancelled child jobs

---
 CHANGELOG.md                |  2 ++
 aspen                       | 10 +++++++---
 resources/cluster_status.sh | 32 ++++++++++++++++++++++++++++++++
 3 files changed, 41 insertions(+), 3 deletions(-)
 create mode 100755 resources/cluster_status.sh

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 553eb91..940ae0f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,7 @@
 ## ASPEN development version
 
+- Detect SLURM-killed Snakemake child jobs in classic cluster mode by adding a `--cluster-status` hook and `scancel` integration, so timed out, OOM-killed, cancelled, or otherwise failed child jobs now retry/fail cleanly instead of leaving the master workflow hanging indefinitely. (#141, @kopardev)
+
 ## ASPEN 1.4.0
 
 - Add structured pipeline state tracking: a `pipeline.status.json` sidecar and `pipeline.{running,completed,failed,canceled}` marker files are now written to `WORKDIR` during `run`, with a live progress monitor that periodically updates `pipeline.running` with a step-completion percentage; a best-effort `jobby` TSV summary (`snakemake.log.jobby`) is now also generated when Slurm submission fails before Snakemake starts. Consolidated shared logging/banner/version-tag helper functions into a new staged library (`workflow/scripts/ccbr_pipeline_logging.sh`, slated for future migration to `ccbr_tools`); redesigned the `--version`/banner output. (#119, @kopardev)
diff --git a/aspen b/aspen
index 9e52b7a..4b4ee2d 100755
--- a/aspen
+++ b/aspen
@@ -224,7 +224,7 @@ fi
 # copy essential folders
 for f in $ESSENTIAL_FOLDERS;do
   # rsync -az --progress ${PIPELINE_HOME}/$f $WORKDIR/
-  cp -rv ${PIPELINE_HOME}/$f ${WORKDIR}/
+  cp -r ${PIPELINE_HOME}/$f ${WORKDIR}/
 done
 
 cd ${WORKDIR}
@@ -400,6 +400,7 @@ function dryrun() {
   run "--dry-run" | tee ${WORKDIR}/dryrun.log
   echo "Git Commit/Tag: $GIT_COMMIT_TAG" > ${WORKDIR}/dryrun_git_commit.txt
   log_ok "Dry-run completed."
+  log_info "Full dry-run output captured in ${WORKDIR}/dryrun.log"
   log_next "Submit run with: aspen --workdir=$WORKDIR --runmode=run"
 }
 
@@ -451,6 +452,9 @@ function runslurm() {
 # Submit the execution of the pipeline to the biowulf job scheduler (slurm)
 
   runcheck
+  if [[ ! -r "$CLUSTERSTATUSCMD" || ! -x "$CLUSTERSTATUSCMD" ]]; then
+    err "Cluster status script '$CLUSTERSTATUSCMD' is missing or not executable."
+  fi
   set_singularity_binds
   set_cluster_arg
   log_divider
@@ -690,7 +694,7 @@ _START_TS=\$(date -u +"%Y-%m-%dT%H:%M:%SZ")
 _progress_monitor &
 _monitor_pid=\$!
 
-snakemake -s $SNAKEFILE --directory $WORKDIR $SINGULARITY_STR --printshellcmds --latency-wait 120 --configfile $CONFIGFILE --cluster-config $CLUSTERFILE --cluster "$CLUSTER_SBATCH_CMD" -j 500 --rerun-incomplete ${RERUNTRIGGERS} --restart-times ${retries} --keep-going --stats ${WORKDIR}/snakemake.stats 2>&1 | tee ${WORKDIR}/snakemake.log
+snakemake -s $SNAKEFILE --directory $WORKDIR $SINGULARITY_STR --printshellcmds --latency-wait 120 --configfile $CONFIGFILE --cluster-config $CLUSTERFILE --cluster "$CLUSTER_SBATCH_CMD" --cluster-status "$CLUSTERSTATUSCMD" --cluster-cancel scancel -j 500 --rerun-incomplete ${RERUNTRIGGERS} --restart-times ${retries} --keep-going --stats ${WORKDIR}/snakemake.stats 2>&1 | tee ${WORKDIR}/snakemake.log
 snakemake_rc="\${PIPESTATUS[0]}"
 kill "\$_monitor_pid" 2>/dev/null || true
 wait "\$_monitor_pid" 2>/dev/null || true
@@ -878,7 +882,7 @@ function main(){
   CONFIGFILE="${WORKDIR}/config.yaml"
   CLUSTERFILE="${WORKDIR}/cluster.json"
 
-  # CLUSTERSTATUSCMD="${PIPELINE_HOME}/resources/cluster_status.sh"
+  CLUSTERSTATUSCMD="${PIPELINE_HOME}/resources/cluster_status.sh"
 
   if [[ -z "$SING_CACHE_DIR" ]]; then
     if [[ -n "$SIFCACHE" && -d "$SIFCACHE" ]]; then
diff --git a/resources/cluster_status.sh b/resources/cluster_status.sh
new file mode 100755
index 0000000..721ea86
--- /dev/null
+++ b/resources/cluster_status.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+job_id="${1:-}"
+
+if [[ -z "${job_id}" ]]; then
+  echo "failed"
+  exit 0
+fi
+
+state=$(sacct -X -j "${job_id}" -n -P --format=State 2>/dev/null | awk 'NF {print; exit}')
+state=${state%% *}
+
+if [[ -z "${state}" ]]; then
+  # sacct can briefly lag behind submission; treat unknown-new jobs as running
+  # to avoid false failures immediately after launch.
+  echo "running"
+  exit 0
+fi
+
+case "${state}" in
+  COMPLETED)
+    echo "success"
+    ;;
+  PENDING|CONFIGURING|RUNNING|COMPLETING|SUSPENDED|RESIZING|REQUEUED|REQUEUE_HOLD)
+    echo "running"
+    ;;
+  FAILED|CANCELLED|TIMEOUT|OUT_OF_MEMORY|NODE_FAIL|PREEMPTED|BOOT_FAIL|DEADLINE|*)
+    echo "failed"
+    ;;
+esac
\ No newline at end of file

From 24d94916263f4785438e5f9cc9310c3907d08c12 Mon Sep 17 00:00:00 2001
From: kopardev <vishal.koparde@nih.gov>
Date: Thu, 24 Sep 2026 15:31:20 -0400
Subject: [PATCH 2/3] docs(changelog): reference PR 148

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 940ae0f..414d8e6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,6 @@
 ## ASPEN development version
 
-- Detect SLURM-killed Snakemake child jobs in classic cluster mode by adding a `--cluster-status` hook and `scancel` integration, so timed out, OOM-killed, cancelled, or otherwise failed child jobs now retry/fail cleanly instead of leaving the master workflow hanging indefinitely. (#141, @kopardev)
+- Detect SLURM-killed Snakemake child jobs in classic cluster mode by adding a `--cluster-status` hook and `scancel` integration, so timed out, OOM-killed, cancelled, or otherwise failed child jobs now retry/fail cleanly instead of leaving the master workflow hanging indefinitely. (#148, @kopardev)
 
 ## ASPEN 1.4.0
 

From 9e74e222380fe901cb797650407e29cfff8a964f Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 24 Sep 2026 19:31:36 +0000
Subject: [PATCH 3/3] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 resources/cluster_status.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/resources/cluster_status.sh b/resources/cluster_status.sh
index 721ea86..028283b 100755
--- a/resources/cluster_status.sh
+++ b/resources/cluster_status.sh
@@ -29,4 +29,4 @@ case "${state}" in
   FAILED|CANCELLED|TIMEOUT|OUT_OF_MEMORY|NODE_FAIL|PREEMPTED|BOOT_FAIL|DEADLINE|*)
     echo "failed"
     ;;
-esac
\ No newline at end of file
+esac