From 321a3ccf8a2e59f848d5548e38445345e3a7748e Mon Sep 17 00:00:00 2001 From: Alexander Froch Date: Tue, 4 Aug 2026 14:28:00 +0200 Subject: [PATCH 01/13] Add container image docs and generic Slurm submission scripts - Document the CI-built container image (apptainer/docker) in the setup docs - Add a 'Running on HPC' docs page covering interactive and batch usage - Ship generic Slurm scripts (submit.sh/batch.sh/run_stage.sh) that derive the job list from the preprocessing config - Add a list_components console script to enumerate config components --- changelog.md | 2 + docs/hpc.md | 133 ++++++++ docs/setup.md | 51 +++ pyproject.toml | 1 + scripts/slurm/README.md | 6 + scripts/slurm/batch.sh | 15 + scripts/slurm/run_stage.sh | 100 ++++++ scripts/slurm/submit.sh | 399 +++++++++++++++++++++++ tests/unit/utils/test_list_components.py | 21 ++ upp/utils/list_components.py | 84 +++++ zensical.toml | 1 + 11 files changed, 813 insertions(+) create mode 100644 docs/hpc.md create mode 100644 scripts/slurm/README.md create mode 100755 scripts/slurm/batch.sh create mode 100755 scripts/slurm/run_stage.sh create mode 100755 scripts/slurm/submit.sh create mode 100644 tests/unit/utils/test_list_components.py create mode 100644 upp/utils/list_components.py diff --git a/changelog.md b/changelog.md index f084858..b58f9cf 100644 --- a/changelog.md +++ b/changelog.md @@ -2,6 +2,8 @@ ### [Latest] +- Add container image documentation and generic Slurm submission scripts with config-driven component enumeration [#XXX](https://github.com/umami-hep/umami-preprocessing/pull/XXX) + ### [v0.3.1](https://github.com/umami-hep/umami-preprocessing/releases/tag/v0.3.1) (19.06.2026) - Make skip-resampling work end-to-end; support `num_jets: -1` to write all jets passing cuts, and record the resampling method in the output metadata [#153](https://github.com/umami-hep/umami-preprocessing/pull/153) diff --git a/docs/hpc.md b/docs/hpc.md new file mode 100644 index 0000000..80e7784 --- /dev/null +++ b/docs/hpc.md @@ -0,0 +1,133 @@ +# Running on HPC clusters + +The preprocessing stages can be parallelized over components, regions and splits using the +`--component`, `--region` and `--split` flags described in [Run](run.md). On a Slurm cluster, each +of these units of work can run as its own batch job inside the [container image](setup.md#container-image). +UPP ships a small set of submission scripts in `scripts/slurm/` that automate this. + +## Prerequisites + +- A cluster with Slurm and apptainer. +- The UPP container image (see [Container image](setup.md#container-image)). By default the scripts + use `docker://gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest`. To avoid the + repeated `docker://` to SIF conversion in every job, pull the image once and point `UPP_IMAGE` at + the local file: + + ```bash + apptainer pull upp_latest.sif docker://gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest + export UPP_IMAGE=/path/to/upp_latest.sif + ``` + +## Interactive use + +For quick tests, run UPP inside the container on an interactive allocation: + +```bash +salloc --ntasks 1 --cpus-per-task 4 --time 2:00:00 +srun apptainer exec --contain --pwd "$PWD" -B /home -B /tmp \ + "$UPP_IMAGE" preprocess --config --prep +``` + +## Batch submission scripts + +The `scripts/slurm/` directory contains three scripts: + +- `submit.sh` runs on the login node. It reads the components from your preprocessing config and + submits one Slurm job per unit of work via `sbatch`. +- `batch.sh` is the sbatch payload. It carries the `#SBATCH` resource header and starts the + container on the compute node. +- `run_stage.sh` runs inside the container and maps the submitted mode onto the `preprocess` + command line flags. + +To use them, create a run directory, copy the scripts, and adapt the `#SBATCH` header in `batch.sh` +to your cluster (partition, account, time and memory limits): + +```bash +mkdir my_preprocessing && cd my_preprocessing +cp -r /scripts/slurm . +$EDITOR slurm/batch.sh +``` + +Then submit the stages in order, waiting for all jobs of one stage to finish before submitting the +next: + +```bash +./slurm/submit.sh --config --dry-run prepare # preview only +./slurm/submit.sh --config prepare +./slurm/submit.sh --config resampling +./slurm/submit.sh --config merge +./slurm/submit.sh --config normalise +./slurm/submit.sh --config plotting +``` + +Job logs are written to `logs/` in the current directory. Running `submit.sh` without a mode enters +an interactive prompt for the mode and filters. + +The available modes and the jobs they submit: + +| Mode | Jobs | `preprocess` flags per job | +|------|------|----------------------------| +| `sequential` | 1 | full chain (`--prep`, `--resample`, `--merge`, `--norm`, `--plot`) | +| `prepare` | one per component and split | `--prep --component --split ` | +| `resampling` | one per region and split | `--resample --region --split ` | +| `fine_resampling` | one per component and split | `--resample --region --component --split ` | +| `merge` | one per split | `--merge --split ` | +| `normalise` | 1 | `--norm` | +| `plotting` | one per split | `--plot --split ` | + +!!!warning "Stage ordering and parallel h5py access" + + All jobs of a stage must finish before the next stage is submitted, e.g. all `prepare` jobs + before `resampling`. Also run the [initial sample check](run.md#additional-scripts-initial-sample-check) + once before submitting `prepare` jobs in parallel — it creates the virtual datasets which can + get corrupted when created by multiple jobs at once. + +## Config-driven job lists + +`submit.sh` never hardcodes which components exist. It calls the `list_components` script (part of +UPP) to enumerate the components defined in the `components:` block of your config: + +```bash +list_components --config +``` + +```text +lowpt ttbar bjets lowpt_ttbar_bjets +highpt zprime bjets highpt_zprime_bjets +... +``` + +Only combinations actually defined in the config are submitted. The selection can be narrowed with +filter flags, each taking a comma- or space-separated list: + +```bash +./slurm/submit.sh --config --regions lowpt --splits train prepare +./slurm/submit.sh --config --samples ttbar --flavs "bjets,cjets" fine_resampling +``` + +Note that enumerating the components fully validates the config, so a broken config fails directly +on the login node instead of inside the batch jobs. + +## Environment variables + +| Variable | Default | Purpose | +|----------|---------|---------| +| `UPP_IMAGE` | `docker://...upp-images/upp:latest` | Container image (`docker://` URI or local `.sif`) | +| `UPP_BINDS` | `/home,/tmp` | Comma-separated paths bound into the container | +| `THROTTLE` | `30` | Seconds between `sbatch` calls (`0` disables) | +| `DRY_RUN` | `0` | Set to `1` to print the `sbatch` commands instead of submitting | + +Make sure `UPP_BINDS` covers your input ntuples and output directory if they live outside `/home` +(e.g. on a scratch filesystem), and export `UPP_IMAGE`/`UPP_BINDS` in your shell so they are also +picked up by the batch jobs. + +!!!warning "Keep the throttle enabled" + + The delay between `sbatch` calls avoids hammering the scheduler and gives jobs time to start + up without all of them hitting the shared filesystem at once. Only disable it for small + submissions. + +!!!info "Configs outside the repository" + + When you copy a config out of the repository, `!include` directives with relative paths no + longer resolve. Use absolute paths in `!include` lines of copied configs. diff --git a/docs/setup.md b/docs/setup.md index c9e04e8..91bc98c 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -128,6 +128,57 @@ python -m pip install . ``` +### Container image + +If you don't want to set up a Python environment at all, you can use the UPP container image. +The CI builds `gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest` on every merge to +`main` and a tagged image `upp:` (e.g. `upp:v0.3.1`) for every release. The image comes with UPP +and its command line scripts (`preprocess`, `check_input_samples`, `list_components`) pre-installed. + +=== "apptainer" + + On clusters (lxplus, HPC sites), apptainer can run the image directly from the registry: + + ```bash + apptainer exec docker://gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest \ + preprocess --config + ``` + + The first `docker://` invocation converts the image to apptainer's SIF format, which takes a + while. If you run UPP repeatedly, pull the image once and use the local file instead: + + ```bash + apptainer pull upp_latest.sif docker://gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest + apptainer exec upp_latest.sif preprocess --config + ``` + + By default apptainer shares your home directory and working directory with the container. For a + cleaner environment use `--contain` and bind only the paths you need (your input ntuples and + output directory) with `-B`, keeping the working directory with `--pwd`: + + ```bash + apptainer exec --contain --pwd "$PWD" -B /home -B /tmp -B \ + upp_latest.sif preprocess --config + ``` + +=== "docker" + + With docker, mount your working directory and data paths into the container: + + ```bash + docker run --rm -it -v $PWD:$PWD -w $PWD \ + gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest \ + preprocess --config + ``` + +!!!info "Pin a release tag for production" + + `upp:latest` follows the `main` branch and changes over time. For reproducible production + preprocessing, use a tagged release image like `upp:v0.3.1` instead. + +For running UPP as batch jobs on Slurm clusters with the container image, see +[Running on HPC](hpc.md). + ### Run the tests (Optional) To ensure that the package is working correctly, you can run the tests using the pytest framework. diff --git a/pyproject.toml b/pyproject.toml index 7c65c07..fa06dd1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ dev = [ [project.scripts] preprocess = "upp.main:main" check_input_samples = "upp.utils.check_input_samples:main" +list_components = "upp.utils.list_components:main" [tool.setuptools] packages = ["upp", "upp.classes", "upp.stages", "upp.utils"] diff --git a/scripts/slurm/README.md b/scripts/slurm/README.md new file mode 100644 index 0000000..7162b17 --- /dev/null +++ b/scripts/slurm/README.md @@ -0,0 +1,6 @@ +# Slurm submission scripts + +Submit UPP preprocessing stages as Slurm batch jobs running inside the UPP container image. + +See the [Running on HPC](https://umami-hep.github.io/umami-preprocessing/hpc/) documentation page +for usage instructions. diff --git a/scripts/slurm/batch.sh b/scripts/slurm/batch.sh new file mode 100755 index 0000000..5433150 --- /dev/null +++ b/scripts/slurm/batch.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +#SBATCH --ntasks 1 +#SBATCH --cpus-per-task 4 +#SBATCH --mem-per-cpu 8000 +#SBATCH --time 1-00:00:00 +# Edit the header above for your cluster, e.g. +# #SBATCH --partition +# #SBATCH --account + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +IMAGE="${UPP_IMAGE:-docker://gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest}" +BINDS="${UPP_BINDS:-/home,/tmp}" + +srun apptainer exec --contain --pwd "${PWD}" -B "${BINDS}" \ + "${IMAGE}" "${SCRIPT_DIR}/run_stage.sh" "$@" diff --git a/scripts/slurm/run_stage.sh b/scripts/slurm/run_stage.sh new file mode 100755 index 0000000..ae0f800 --- /dev/null +++ b/scripts/slurm/run_stage.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Runs one preprocessing stage inside the container. Called by batch.sh. +set -e + +usage() { + cat < + # full chain (prep+resample+merge+norm+plot) with split=all + + run_stage.sh normalise|normalize + run_stage.sh merge + run_stage.sh plotting + run_stage.sh prepare + run_stage.sh resampling + run_stage.sh fine_resampling +EOF +} + +if [ "$#" -eq 0 ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then + usage + exit 2 +fi + +CONFIG="$1" +shift + +run_preprocess() { + preprocess --config "${CONFIG}" "$@" +} + +need_args() { + # Usage: need_args "$@" + local n_required="$1" + local mode="$2" + shift 2 + if [ "$#" -lt "$n_required" ]; then + echo "ERROR: '$mode' needs ${n_required} argument(s), got $#." >&2 + usage + exit 2 + fi +} + +if [ "$#" -eq 0 ]; then + echo "No mode given. Processing full chain." + run_preprocess --prep --split=all + run_preprocess --resample --split=all + run_preprocess --merge --split=all + run_preprocess --norm + run_preprocess --plot --split=all + exit 0 +fi + +MODE="$1" +shift + +case "${MODE}" in + normalise|normalize) + echo "Normalisation selected. Processing..." + run_preprocess --norm + ;; + + merge) + need_args 1 "merge" "$@" + echo "Start merging for $1. Processing..." + run_preprocess --merge --split "$1" + ;; + + plotting) + need_args 1 "plotting" "$@" + echo "Plotting selected ($1). Processing..." + run_preprocess --plot --split "$1" + ;; + + prepare) + need_args 2 "prepare" "$@" + echo "Start preparation for $1 ($2). Processing..." + run_preprocess --prep --component "$1" --split "$2" + ;; + + resampling) + need_args 2 "resampling" "$@" + echo "Start resampling for $1 ($2). Processing..." + run_preprocess --resample --region "$1" --split "$2" + ;; + + fine_resampling) + need_args 3 "fine_resampling" "$@" + echo "Start resampling for $2 ($3). Processing..." + run_preprocess --resample --region "$1" --component "$2" --split "$3" + ;; + + *) + echo "Step '${MODE}' not supported!" >&2 + usage + exit 2 + ;; +esac + +echo "Done!" diff --git a/scripts/slurm/submit.sh b/scripts/slurm/submit.sh new file mode 100755 index 0000000..3b191de --- /dev/null +++ b/scripts/slurm/submit.sh @@ -0,0 +1,399 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +IMAGE="${UPP_IMAGE:-docker://gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest}" + +# Throttle between sbatch calls (seconds). Set to 0 to disable. +THROTTLE="${THROTTLE:-30}" + +# Dry-run: if 1, print commands but do not execute sbatch. +DRY_RUN="${DRY_RUN:-0}" + +CONFIG="" + +# Filters on the components enumerated from the config; empty means "everything". +declare -a REGION_FILTER=() +declare -a SAMPLE_FILTER=() +declare -a FLAV_FILTER=() +declare -a SPLIT_FILTER=() + +# ---- Helpers -------------------------------------------------------------- +slugify() { + # keep alnum, dash, underscore; replace others with dash; squish repeats; trim + local s="${*:-}" + s="${s//[^[:alnum:]_-]/-}" + s="$(printf '%s' "$s" | sed -E 's/-+/-/g; s/^-+//; s/-+$//')" + # Slurm JobName limit is 128 chars; leave margin + printf '%.*s' 120 "$s" +} + +build_job_name() { + local mode="${1:-sequential}" + shift || true + local prefix="upp-$(basename "${CONFIG%.*}")" + case "$mode" in + prepare) slugify "${prefix}-prepare-${1:-component}-${2:-split}" ;; + fine_resampling) slugify "${prefix}-fres-${1:-region}-${2:-component}-${3:-split}" ;; + resampling) slugify "${prefix}-resampling-${1:-region}-${2:-split}" ;; + merge) slugify "${prefix}-merge-${1:-split}" ;; + normalise|normalize) slugify "${prefix}-normalise" ;; + plotting) slugify "${prefix}-plotting-${1:-split}" ;; + sequential|*) slugify "${prefix}-seq" ;; + esac +} + +enumerate() { + # Use a local UPP installation when available, otherwise the container image + if command -v list_components >/dev/null 2>&1; then + list_components --config "${CONFIG}" "$@" + else + apptainer exec "${IMAGE}" list_components --config "${CONFIG}" "$@" + fi +} + +append_list() { + # Append a comma/space separated list to the named array + local input="$1" + local out_name="$2" + local -a items=() + local x + IFS=', ' read -r -a items <<< "$input" + for x in "${items[@]}"; do + if [[ -n "$x" ]]; then + eval "$out_name+=(\"\$x\")" + fi + done +} + +add_unique() { + # Append the value to the named array if not already present + local x="$1" + local out_name="$2" + local -a arr=() + eval "arr=(\"\${${out_name}[@]}\")" + local i + for i in "${arr[@]}"; do + if [[ "$i" == "$x" ]]; then + return 0 + fi + done + eval "$out_name+=(\"\$x\")" +} + +in_list() { + # in_list [items...]; an empty list matches everything + local x="$1" + shift + if [[ $# -eq 0 ]]; then + return 0 + fi + local i + for i in "$@"; do + if [[ "$i" == "$x" ]]; then + return 0 + fi + done + return 1 +} + +print_resolved_config() { + printf '\nResolved configuration:\n' + printf ' CONFIG : %s\n' "$CONFIG" + printf ' IMAGE : %s\n' "$IMAGE" + printf ' THROTTLE : %s\n' "$THROTTLE" + printf ' COMPONENTS : %s\n' "${#COMPONENTS[@]}" + printf ' SPLITS : %s\n' "${SPLITS[*]}" + printf '\n' +} + +prompt_with_default() { + # Usage: prompt_with_default "Question" "default value" + local prompt="$1" + local default="$2" + local reply + + read -r -p "${prompt} [${default}]: " reply + if [[ -z "${reply}" ]]; then + printf '%s\n' "${default}" + else + printf '%s\n' "${reply}" + fi +} + +interactive_mode() { + echo + echo "No mode provided. Entering interactive mode." + echo + echo "Available modes:" + echo " sequential prepare fine_resampling resampling merge normalise plotting" + echo + + MODE="$(prompt_with_default "Select mode" "prepare")" + + local regions samples flavs splits + regions="$(prompt_with_default "Regions" "${ALL_REGIONS[*]}")" + samples="$(prompt_with_default "Samples" "${ALL_SAMPLES[*]}")" + flavs="$(prompt_with_default "Flavours" "${ALL_FLAVS[*]}")" + splits="$(prompt_with_default "Splits" "${SPLITS[*]}")" + + append_list "${regions}" REGION_FILTER + append_list "${samples}" SAMPLE_FILTER + append_list "${flavs}" FLAV_FILTER + SPLIT_FILTER=() + append_list "${splits}" SPLIT_FILTER +} + +submit() { + # Usage: submit [mode args...] + local jobname + jobname="$(build_job_name "$@")" + + local -a cmd=( + sbatch + --job-name="$jobname" + --output="${PWD}/logs/%j_%x.out" + --error="${PWD}/logs/%j_%x.err" + "${SCRIPT_DIR}/batch.sh" + "$CONFIG" + "$@" + ) + + echo "Submitting: $jobname" + + if [[ "$DRY_RUN" == "1" ]]; then + printf 'DRY-RUN: ' + printf '%q ' "${cmd[@]}" + printf '\n' + else + "${cmd[@]}" + if [[ "$THROTTLE" != "0" ]]; then + sleep "$THROTTLE" + fi + fi +} + +usage() { + cat <<'EOF' +Usage: + submit.sh --config [options] [mode] + +Modes: + (no mode) # interactive mode: prompts for mode and filters + sequential # one job running the full chain + prepare # one job per component and split + fine_resampling # one job per component and split + resampling # one job per region and split + merge # one job per split + normalise|normalize # one job + plotting # one job per split + +Options: + --config # preprocessing config (required) + --dry-run # do not call sbatch; print commands instead + --regions "" # only submit components in these regions + --samples "" # only submit components from these samples + --flavs "" # only submit components with these flavours + --splits "" # only submit these splits (default: train val test) + --throttle N # seconds between sbatch calls (default 30; 0 disables) + +Env (still supported): + UPP_IMAGE= # container image (docker:// URI or local .sif) + THROTTLE= # same as --throttle + DRY_RUN=1 # same as --dry-run + +List format: + Comma and/or space separated, e.g. "lowpt,highpt" or "lowpt highpt" + +Examples: + ./submit.sh --config configs/my-config.yaml --dry-run prepare + ./submit.sh --config configs/my-config.yaml --regions lowpt --splits train prepare + ./submit.sh --config configs/my-config.yaml resampling +EOF +} + +parse_args() { + local -a rest=() + while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + --config) + [[ $# -ge 2 ]] || { echo "ERROR: --config requires a value" >&2; exit 2; } + CONFIG="$2" + shift 2 + ;; + --dry-run) + DRY_RUN=1 + shift + ;; + --throttle) + [[ $# -ge 2 ]] || { echo "ERROR: --throttle requires a value" >&2; exit 2; } + THROTTLE="$2" + shift 2 + ;; + --regions) + [[ $# -ge 2 ]] || { echo "ERROR: --regions requires a value" >&2; exit 2; } + append_list "$2" REGION_FILTER + shift 2 + ;; + --samples) + [[ $# -ge 2 ]] || { echo "ERROR: --samples requires a value" >&2; exit 2; } + append_list "$2" SAMPLE_FILTER + shift 2 + ;; + --flavs|--flavors|--flavours) + [[ $# -ge 2 ]] || { echo "ERROR: --flavs requires a value" >&2; exit 2; } + append_list "$2" FLAV_FILTER + shift 2 + ;; + --splits) + [[ $# -ge 2 ]] || { echo "ERROR: --splits requires a value" >&2; exit 2; } + append_list "$2" SPLIT_FILTER + shift 2 + ;; + --) # end of options + shift + rest+=("$@") + break + ;; + -*) + echo "ERROR: unknown option: $1" >&2 + usage + exit 2 + ;; + *) + rest+=("$1") + shift + ;; + esac + done + + ARGS_REST=("${rest[@]}") +} + +# ---- Main ----------------------------------------------------------------- +main() { + declare -a ARGS_REST=() + parse_args "$@" + set -- "${ARGS_REST[@]}" + + if [[ -z "$CONFIG" ]]; then + echo "ERROR: --config is required" >&2 + usage + exit 2 + fi + + # Enumerate all components defined in the config (TSV: region sample flavour name) + local -a ALL_ROWS=() + mapfile -t ALL_ROWS < <(enumerate) + if [[ ${#ALL_ROWS[@]} -eq 0 ]]; then + echo "ERROR: no components found in $CONFIG" >&2 + exit 1 + fi + + local -a ALL_REGIONS=() ALL_SAMPLES=() ALL_FLAVS=() + local row region sample flavour name + for row in "${ALL_ROWS[@]}"; do + IFS=$'\t' read -r region sample flavour name <<< "$row" + add_unique "$region" ALL_REGIONS + add_unique "$sample" ALL_SAMPLES + add_unique "$flavour" ALL_FLAVS + done + + MODE="${1:-}" + declare -a SPLITS=(train val test) + if [[ -z "$MODE" ]]; then + interactive_mode + fi + if [[ ${#SPLIT_FILTER[@]} -gt 0 ]]; then + SPLITS=("${SPLIT_FILTER[@]}") + fi + + # Apply the filters + declare -a COMPONENTS=() REGIONS=() + for row in "${ALL_ROWS[@]}"; do + IFS=$'\t' read -r region sample flavour name <<< "$row" + if in_list "$region" "${REGION_FILTER[@]}" \ + && in_list "$sample" "${SAMPLE_FILTER[@]}" \ + && in_list "$flavour" "${FLAV_FILTER[@]}"; then + COMPONENTS+=("$row") + add_unique "$region" REGIONS + fi + done + + if [[ "$DRY_RUN" == "1" ]]; then + print_resolved_config + fi + + if [[ "$DRY_RUN" != "1" ]]; then + mkdir -p "${PWD}/logs" + fi + + local split + case "$MODE" in + sequential) + echo "Start submission for sequential processing..." + submit + ;; + + prepare) + echo "Start submission for preparation..." + for row in "${COMPONENTS[@]}"; do + IFS=$'\t' read -r region sample flavour name <<< "$row" + for split in "${SPLITS[@]}"; do + submit "prepare" "$name" "$split" + done + done + ;; + + fine_resampling) + echo "Start submission for fine resampling..." + for row in "${COMPONENTS[@]}"; do + IFS=$'\t' read -r region sample flavour name <<< "$row" + for split in "${SPLITS[@]}"; do + submit "fine_resampling" "$region" "$name" "$split" + done + done + ;; + + resampling) + echo "Start submission for resampling..." + for region in "${REGIONS[@]}"; do + for split in "${SPLITS[@]}"; do + submit "resampling" "$region" "$split" + done + done + ;; + + merge) + echo "Start submission for merging..." + for split in "${SPLITS[@]}"; do + submit "merge" "$split" + done + ;; + + normalise|normalize) + echo "Start submission for normalise..." + submit "normalise" + ;; + + plotting) + echo "Start submission for plotting..." + for split in "${SPLITS[@]}"; do + submit "plotting" "$split" + done + ;; + + *) + echo "Unsupported mode: '$MODE'" >&2 + usage + exit 2 + ;; + esac + echo "Done!" +} + +main "$@" diff --git a/tests/unit/utils/test_list_components.py b/tests/unit/utils/test_list_components.py new file mode 100644 index 0000000..ab5f446 --- /dev/null +++ b/tests/unit/utils/test_list_components.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from pathlib import Path + +from upp.utils.list_components import main + +CONFIG = Path(__file__).parents[3] / "upp/configs/test.yaml" + + +def test_list_components(capsys): + main(["--config", str(CONFIG)]) + rows = [line.split("\t") for line in capsys.readouterr().out.splitlines()] + assert len(rows) == 6 + assert all(len(row) == 4 for row in rows) + assert ["lowpt", "ttbar", "bjets", "lowpt_ttbar_bjets"] in rows + assert ["highpt", "zprime", "cjets", "highpt_zprime_cjets"] in rows + + +def test_list_components_regions(capsys): + main(["--config", str(CONFIG), "--regions"]) + assert capsys.readouterr().out.splitlines() == ["lowpt", "highpt"] diff --git a/upp/utils/list_components.py b/upp/utils/list_components.py new file mode 100644 index 0000000..85b9cf0 --- /dev/null +++ b/upp/utils/list_components.py @@ -0,0 +1,84 @@ +"""List the components defined in a preprocessing config.""" + +from __future__ import annotations + +import argparse +from typing import Any + +from ftag.cli_utils import HelpFormatter, valid_path + +from upp.classes.preprocessing_config import PreprocessingConfig + + +def parse_args(args: Any) -> argparse.Namespace: + """Parse the command line arguments. + + Parameters + ---------- + args : Any + Command line arguments. + + Returns + ------- + argparse.Namespace + Namespace with the parsed command line arguments + """ + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=HelpFormatter, + ) + parser.add_argument( + "--config", + required=True, + type=valid_path, + help="Path to config file", + ) + parser.add_argument( + "--split", + default="train", + choices=["train", "val", "test"], + help="Split to load the config for (component names are split-independent)", + ) + parser.add_argument( + "--regions", + action="store_true", + help="Only print the unique region names", + ) + + return parser.parse_args(args) + + +def main(args: Any | None = None) -> None: + """List components as tab-separated `region sample flavour name` rows. + + Parameters + ---------- + args : Any | None, optional + Command line arguments, by default None + """ + args = parse_args(args) + + config = PreprocessingConfig.from_file( + config_path=args.config, + split=args.split, + skip_checks=True, + skip_config_copy=True, + ) + + if args.regions: + for region in config.components.regions: + print(region.name) + return + + for component in config.components: + print( + component.region.name, + component.sample.name, + component.flavour.name, + component.name, + sep="\t", + ) + + +if __name__ == "__main__": + main() diff --git a/zensical.toml b/zensical.toml index 97af816..46ac56f 100644 --- a/zensical.toml +++ b/zensical.toml @@ -18,6 +18,7 @@ nav = [ { "Reweighting" = "reweighting.md" }, { "Configuration" = "configuration.md" }, { "Run" = "run.md" }, + { "Running on HPC" = "hpc.md" }, { "Umami integration" = "umami_int.md" }, { "Contributing" = "contributing.md" }, { "Docs development" = "docs_development.md" }, From 0c040f3149f3ee24039b85b7ec8be80b1b278b3f Mon Sep 17 00:00:00 2001 From: Alexander Froch Date: Tue, 4 Aug 2026 15:02:16 +0200 Subject: [PATCH 02/13] Set PR number in changelog entry --- changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index b58f9cf..df9a17f 100644 --- a/changelog.md +++ b/changelog.md @@ -2,7 +2,7 @@ ### [Latest] -- Add container image documentation and generic Slurm submission scripts with config-driven component enumeration [#XXX](https://github.com/umami-hep/umami-preprocessing/pull/XXX) +- Add container image documentation and generic Slurm submission scripts with config-driven component enumeration [#160](https://github.com/umami-hep/umami-preprocessing/pull/160) ### [v0.3.1](https://github.com/umami-hep/umami-preprocessing/releases/tag/v0.3.1) (19.06.2026) From 8d5ac068268a74e285e388c56276b00fcbf294f6 Mon Sep 17 00:00:00 2001 From: Alexander Froch Date: Fri, 7 Aug 2026 12:59:53 +0200 Subject: [PATCH 03/13] Add HTCondor support and CVMFS-unpacked image default to batch scripts --- changelog.md | 2 +- docs/hpc.md | 63 ++++++----- docs/setup.md | 13 ++- scripts/batch/README.md | 7 ++ scripts/batch/condor_batch.sh | 15 +++ scripts/batch/condor_job.sub | 20 ++++ scripts/{slurm => batch}/run_stage.sh | 2 +- .../{slurm/batch.sh => batch/slurm_batch.sh} | 8 +- scripts/{slurm => batch}/submit.sh | 105 ++++++++++++++++-- scripts/slurm/README.md | 6 - 10 files changed, 198 insertions(+), 43 deletions(-) create mode 100644 scripts/batch/README.md create mode 100755 scripts/batch/condor_batch.sh create mode 100644 scripts/batch/condor_job.sub rename scripts/{slurm => batch}/run_stage.sh (95%) rename scripts/{slurm/batch.sh => batch/slurm_batch.sh} (55%) rename scripts/{slurm => batch}/submit.sh (77%) delete mode 100644 scripts/slurm/README.md diff --git a/changelog.md b/changelog.md index 8321fed..9e75f40 100644 --- a/changelog.md +++ b/changelog.md @@ -2,7 +2,7 @@ ### [Latest] -- Add container image documentation and generic Slurm submission scripts with config-driven component enumeration [#160](https://github.com/umami-hep/umami-preprocessing/pull/160) +- Add container image documentation and generic Slurm/HTCondor submission scripts with config-driven component enumeration [#160](https://github.com/umami-hep/umami-preprocessing/pull/160) ### [v0.3.2](https://github.com/umami-hep/umami-preprocessing/releases/tag/v0.3.2) (04.08.2026) - Add new configs for central dataset [#159](https://github.com/umami-hep/umami-preprocessing/pull/159) diff --git a/docs/hpc.md b/docs/hpc.md index 80e7784..75bd639 100644 --- a/docs/hpc.md +++ b/docs/hpc.md @@ -1,17 +1,20 @@ # Running on HPC clusters The preprocessing stages can be parallelized over components, regions and splits using the -`--component`, `--region` and `--split` flags described in [Run](run.md). On a Slurm cluster, each -of these units of work can run as its own batch job inside the [container image](setup.md#container-image). -UPP ships a small set of submission scripts in `scripts/slurm/` that automate this. +`--component`, `--region` and `--split` flags described in [Run](run.md). On a Slurm or HTCondor +cluster, each of these units of work can run as its own batch job inside the +[container image](setup.md#container-image). UPP ships a small set of submission scripts in +`scripts/batch/` that automate this. ## Prerequisites -- A cluster with Slurm and apptainer. -- The UPP container image (see [Container image](setup.md#container-image)). By default the scripts - use `docker://gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest`. To avoid the - repeated `docker://` to SIF conversion in every job, pull the image once and point `UPP_IMAGE` at - the local file: +- A cluster with Slurm or HTCondor and apptainer. +- The UPP container image (see [Container image](setup.md#container-image)). The scripts default to + the CVMFS-unpacked image + `/cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest` when + it exists and fall back to `docker://gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest` + otherwise. On clusters without CVMFS, avoid the repeated `docker://` to SIF conversion in every + job by pulling the image once and pointing `UPP_IMAGE` at the local file: ```bash apptainer pull upp_latest.sif docker://gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest @@ -30,39 +33,47 @@ srun apptainer exec --contain --pwd "$PWD" -B /home -B /tmp \ ## Batch submission scripts -The `scripts/slurm/` directory contains three scripts: +The `scripts/batch/` directory contains: - `submit.sh` runs on the login node. It reads the components from your preprocessing config and - submits one Slurm job per unit of work via `sbatch`. -- `batch.sh` is the sbatch payload. It carries the `#SBATCH` resource header and starts the + submits one batch job per unit of work. The scheduler is auto-detected (`sbatch` found → Slurm, + `condor_submit` found → HTCondor) and can be forced with `--scheduler slurm|condor`. +- `slurm_batch.sh` is the sbatch payload. It carries the `#SBATCH` resource header and starts the container on the compute node. +- `condor_job.sub` and `condor_batch.sh` are the HTCondor equivalents: the submit description with + the resource requests, and the job executable starting the container. - `run_stage.sh` runs inside the container and maps the submitted mode onto the `preprocess` command line flags. -To use them, create a run directory, copy the scripts, and adapt the `#SBATCH` header in `batch.sh` -to your cluster (partition, account, time and memory limits): +To use them, create a run directory, copy the scripts, and adapt the resources to your cluster +(partition/pool, account, time and memory limits) — the `#SBATCH` header in `slurm_batch.sh` for +Slurm, or the requests at the top of `condor_job.sub` for HTCondor: ```bash mkdir my_preprocessing && cd my_preprocessing -cp -r /scripts/slurm . -$EDITOR slurm/batch.sh +cp -r /scripts/batch . +$EDITOR batch/slurm_batch.sh # or batch/condor_job.sub ``` Then submit the stages in order, waiting for all jobs of one stage to finish before submitting the next: ```bash -./slurm/submit.sh --config --dry-run prepare # preview only -./slurm/submit.sh --config prepare -./slurm/submit.sh --config resampling -./slurm/submit.sh --config merge -./slurm/submit.sh --config normalise -./slurm/submit.sh --config plotting +./batch/submit.sh --config --dry-run prepare # preview only +./batch/submit.sh --config prepare +./batch/submit.sh --config resampling +./batch/submit.sh --config merge +./batch/submit.sh --config normalise +./batch/submit.sh --config plotting ``` Job logs are written to `logs/` in the current directory. Running `submit.sh` without a mode enters an interactive prompt for the mode and filters. +On Slurm each job is submitted with its own `sbatch` call and job name. On HTCondor all jobs of one +`submit.sh` invocation are submitted as a single cluster (one `condor_submit` with one process per +job), with the job arguments written to `logs/condor_.args`. + The available modes and the jobs they submit: | Mode | Jobs | `preprocess` flags per job | @@ -101,8 +112,8 @@ Only combinations actually defined in the config are submitted. The selection ca filter flags, each taking a comma- or space-separated list: ```bash -./slurm/submit.sh --config --regions lowpt --splits train prepare -./slurm/submit.sh --config --samples ttbar --flavs "bjets,cjets" fine_resampling +./batch/submit.sh --config --regions lowpt --splits train prepare +./batch/submit.sh --config --samples ttbar --flavs "bjets,cjets" fine_resampling ``` Note that enumerating the components fully validates the config, so a broken config fails directly @@ -112,10 +123,10 @@ on the login node instead of inside the batch jobs. | Variable | Default | Purpose | |----------|---------|---------| -| `UPP_IMAGE` | `docker://...upp-images/upp:latest` | Container image (`docker://` URI or local `.sif`) | +| `UPP_IMAGE` | CVMFS-unpacked image if present, else `docker://...upp-images/upp:latest` | Container image (unpacked directory, local `.sif` or `docker://` URI) | | `UPP_BINDS` | `/home,/tmp` | Comma-separated paths bound into the container | -| `THROTTLE` | `30` | Seconds between `sbatch` calls (`0` disables) | -| `DRY_RUN` | `0` | Set to `1` to print the `sbatch` commands instead of submitting | +| `THROTTLE` | `30` | Seconds between `sbatch` calls (`0` disables; Slurm only) | +| `DRY_RUN` | `0` | Set to `1` to print the submission commands instead of submitting | Make sure `UPP_BINDS` covers your input ntuples and output directory if they live outside `/home` (e.g. on a scratch filesystem), and export `UPP_IMAGE`/`UPP_BINDS` in your shell so they are also diff --git a/docs/setup.md b/docs/setup.md index 91bc98c..6bea982 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -152,6 +152,17 @@ and its command line scripts (`preprocess`, `check_input_samples`, `list_compone apptainer exec upp_latest.sif preprocess --config ``` + On machines with CVMFS (lxplus, most HPC sites) the image is also distributed pre-unpacked + via [unpacked.cern.ch](https://gitlab.cern.ch/unpacked/sync), which skips the pull and + conversion entirely: + + ```bash + apptainer exec /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest \ + preprocess --config + ``` + + The release tags are available there as well (e.g. `upp:v0.3.2`). + By default apptainer shares your home directory and working directory with the container. For a cleaner environment use `--contain` and bind only the paths you need (your input ntuples and output directory) with `-B`, keeping the working directory with `--pwd`: @@ -176,7 +187,7 @@ and its command line scripts (`preprocess`, `check_input_samples`, `list_compone `upp:latest` follows the `main` branch and changes over time. For reproducible production preprocessing, use a tagged release image like `upp:v0.3.1` instead. -For running UPP as batch jobs on Slurm clusters with the container image, see +For running UPP as batch jobs on Slurm or HTCondor clusters with the container image, see [Running on HPC](hpc.md). ### Run the tests (Optional) diff --git a/scripts/batch/README.md b/scripts/batch/README.md new file mode 100644 index 0000000..857cbd1 --- /dev/null +++ b/scripts/batch/README.md @@ -0,0 +1,7 @@ +# Batch submission scripts + +Submit UPP preprocessing stages as Slurm or HTCondor batch jobs running inside the UPP container +image. + +See the [Running on HPC](https://umami-hep.github.io/umami-preprocessing/hpc/) documentation page +for usage instructions. diff --git a/scripts/batch/condor_batch.sh b/scripts/batch/condor_batch.sh new file mode 100755 index 0000000..6c7eea5 --- /dev/null +++ b/scripts/batch/condor_batch.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# HTCondor job executable: starts the container on the worker node. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Prefer the CVMFS-unpacked image when available, fall back to the registry +DEFAULT_IMAGE="/cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest" +if [[ ! -e "${DEFAULT_IMAGE}" ]]; then + DEFAULT_IMAGE="docker://gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest" +fi +IMAGE="${UPP_IMAGE:-${DEFAULT_IMAGE}}" +BINDS="${UPP_BINDS:-/home,/tmp}" + +apptainer exec --contain --pwd "${PWD}" -B "${BINDS}" \ + "${IMAGE}" "${SCRIPT_DIR}/run_stage.sh" "$@" diff --git a/scripts/batch/condor_job.sub b/scripts/batch/condor_job.sub new file mode 100644 index 0000000..2333a26 --- /dev/null +++ b/scripts/batch/condor_job.sub @@ -0,0 +1,20 @@ +# HTCondor submit description used by submit.sh: +# condor_submit batch_dir=... batch_name=... upp_image=... upp_binds=... \ +# condor_job.sub -queue "args from " +# Edit the resources below for your pool. +request_cpus = 4 +request_memory = 32 GB +# CERN lxplus walltime flavour; ignored on other pools ++JobFlavour = "tomorrow" + +universe = vanilla +executable = $(batch_dir)/condor_batch.sh +arguments = $(args) +batch_name = $(batch_name) +environment = "UPP_IMAGE=$(upp_image) UPP_BINDS=$(upp_binds)" +output = logs/$(ClusterId).$(ProcId).out +error = logs/$(ClusterId).$(ProcId).err +log = logs/$(ClusterId).log + +# Assumes a shared filesystem between submit and worker nodes +should_transfer_files = NO diff --git a/scripts/slurm/run_stage.sh b/scripts/batch/run_stage.sh similarity index 95% rename from scripts/slurm/run_stage.sh rename to scripts/batch/run_stage.sh index ae0f800..dcba587 100755 --- a/scripts/slurm/run_stage.sh +++ b/scripts/batch/run_stage.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Runs one preprocessing stage inside the container. Called by batch.sh. +# Runs one preprocessing stage inside the container. Called by slurm_batch.sh / condor_batch.sh. set -e usage() { diff --git a/scripts/slurm/batch.sh b/scripts/batch/slurm_batch.sh similarity index 55% rename from scripts/slurm/batch.sh rename to scripts/batch/slurm_batch.sh index 5433150..e3fef6a 100755 --- a/scripts/slurm/batch.sh +++ b/scripts/batch/slurm_batch.sh @@ -8,7 +8,13 @@ # #SBATCH --account SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -IMAGE="${UPP_IMAGE:-docker://gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest}" + +# Prefer the CVMFS-unpacked image when available, fall back to the registry +DEFAULT_IMAGE="/cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest" +if [[ ! -e "${DEFAULT_IMAGE}" ]]; then + DEFAULT_IMAGE="docker://gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest" +fi +IMAGE="${UPP_IMAGE:-${DEFAULT_IMAGE}}" BINDS="${UPP_BINDS:-/home,/tmp}" srun apptainer exec --contain --pwd "${PWD}" -B "${BINDS}" \ diff --git a/scripts/slurm/submit.sh b/scripts/batch/submit.sh similarity index 77% rename from scripts/slurm/submit.sh rename to scripts/batch/submit.sh index 3b191de..c209a0c 100755 --- a/scripts/slurm/submit.sh +++ b/scripts/batch/submit.sh @@ -2,9 +2,19 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -IMAGE="${UPP_IMAGE:-docker://gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest}" -# Throttle between sbatch calls (seconds). Set to 0 to disable. +# Prefer the CVMFS-unpacked image when available, fall back to the registry +DEFAULT_IMAGE="/cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest" +if [[ ! -e "${DEFAULT_IMAGE}" ]]; then + DEFAULT_IMAGE="docker://gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest" +fi +IMAGE="${UPP_IMAGE:-${DEFAULT_IMAGE}}" +BINDS="${UPP_BINDS:-/home,/tmp}" + +# Batch scheduler: slurm or condor; empty means auto-detect +SCHEDULER="" + +# Throttle between sbatch calls (seconds, Slurm only). Set to 0 to disable. THROTTLE="${THROTTLE:-30}" # Dry-run: if 1, print commands but do not execute sbatch. @@ -18,6 +28,9 @@ declare -a SAMPLE_FILTER=() declare -a FLAV_FILTER=() declare -a SPLIT_FILTER=() +# Argument lines collected for a single condor_submit call +declare -a CONDOR_JOBS=() + # ---- Helpers -------------------------------------------------------------- slugify() { # keep alnum, dash, underscore; replace others with dash; squish repeats; trim @@ -43,6 +56,26 @@ build_job_name() { esac } +detect_scheduler() { + if [[ -z "$SCHEDULER" ]]; then + if command -v sbatch >/dev/null 2>&1; then + SCHEDULER="slurm" + elif command -v condor_submit >/dev/null 2>&1; then + SCHEDULER="condor" + else + echo "ERROR: neither sbatch nor condor_submit found; use --scheduler slurm|condor" >&2 + exit 2 + fi + fi + case "$SCHEDULER" in + slurm|condor) ;; + *) + echo "ERROR: unknown scheduler: '$SCHEDULER' (expected slurm or condor)" >&2 + exit 2 + ;; + esac +} + enumerate() { # Use a local UPP installation when available, otherwise the container image if command -v list_components >/dev/null 2>&1; then @@ -100,6 +133,7 @@ in_list() { print_resolved_config() { printf '\nResolved configuration:\n' printf ' CONFIG : %s\n' "$CONFIG" + printf ' SCHEDULER : %s\n' "$SCHEDULER" printf ' IMAGE : %s\n' "$IMAGE" printf ' THROTTLE : %s\n' "$THROTTLE" printf ' COMPONENTS : %s\n' "${#COMPONENTS[@]}" @@ -145,7 +179,16 @@ interactive_mode() { } submit() { - # Usage: submit [mode args...] + # Usage: submit [mode args...]; Slurm submits directly, condor collects for one condor_submit + if [[ "$SCHEDULER" == "condor" ]]; then + local line="$CONFIG" + if [[ $# -gt 0 ]]; then + line+=" $*" + fi + CONDOR_JOBS+=("$line") + return 0 + fi + local jobname jobname="$(build_job_name "$@")" @@ -154,7 +197,7 @@ submit() { --job-name="$jobname" --output="${PWD}/logs/%j_%x.out" --error="${PWD}/logs/%j_%x.err" - "${SCRIPT_DIR}/batch.sh" + "${SCRIPT_DIR}/slurm_batch.sh" "$CONFIG" "$@" ) @@ -173,6 +216,41 @@ submit() { fi } +condor_submit_all() { + # Submit all collected jobs as a single condor cluster + if [[ ${#CONDOR_JOBS[@]} -eq 0 ]]; then + echo "Nothing to submit." + return 0 + fi + + local bname + bname="$(slugify "upp-$(basename "${CONFIG%.*}")-${MODE:-sequential}")" + local args_file="${PWD}/logs/condor_${MODE:-sequential}.args" + + local -a cmd=( + condor_submit + "batch_dir=${SCRIPT_DIR}" + "batch_name=${bname}" + "upp_image=${IMAGE}" + "upp_binds=${BINDS}" + "${SCRIPT_DIR}/condor_job.sub" + -queue "args from ${args_file}" + ) + + echo "Submitting ${#CONDOR_JOBS[@]} job(s) as batch $bname" + + if [[ "$DRY_RUN" == "1" ]]; then + printf 'DRY-RUN: job arguments:\n' + printf ' %s\n' "${CONDOR_JOBS[@]}" + printf 'DRY-RUN: ' + printf '%q ' "${cmd[@]}" + printf '\n' + else + printf '%s\n' "${CONDOR_JOBS[@]}" > "$args_file" + "${cmd[@]}" + fi +} + usage() { cat <<'EOF' Usage: @@ -190,15 +268,17 @@ Modes: Options: --config # preprocessing config (required) - --dry-run # do not call sbatch; print commands instead + --scheduler # slurm or condor (default: auto-detect) + --dry-run # do not submit; print commands instead --regions "" # only submit components in these regions --samples "" # only submit components from these samples --flavs "" # only submit components with these flavours --splits "" # only submit these splits (default: train val test) - --throttle N # seconds between sbatch calls (default 30; 0 disables) + --throttle N # seconds between sbatch calls (default 30; 0 disables; Slurm only) Env (still supported): - UPP_IMAGE= # container image (docker:// URI or local .sif) + UPP_IMAGE= # container image (unpacked dir, local .sif or docker:// URI) + UPP_BINDS= # comma-separated bind paths (default: /home,/tmp) THROTTLE= # same as --throttle DRY_RUN=1 # same as --dry-run @@ -225,6 +305,11 @@ parse_args() { CONFIG="$2" shift 2 ;; + --scheduler) + [[ $# -ge 2 ]] || { echo "ERROR: --scheduler requires a value" >&2; exit 2; } + SCHEDULER="$2" + shift 2 + ;; --dry-run) DRY_RUN=1 shift @@ -286,6 +371,8 @@ main() { exit 2 fi + detect_scheduler + # Enumerate all components defined in the config (TSV: region sample flavour name) local -a ALL_ROWS=() mapfile -t ALL_ROWS < <(enumerate) @@ -393,6 +480,10 @@ main() { exit 2 ;; esac + + if [[ "$SCHEDULER" == "condor" ]]; then + condor_submit_all + fi echo "Done!" } diff --git a/scripts/slurm/README.md b/scripts/slurm/README.md deleted file mode 100644 index 7162b17..0000000 --- a/scripts/slurm/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Slurm submission scripts - -Submit UPP preprocessing stages as Slurm batch jobs running inside the UPP container image. - -See the [Running on HPC](https://umami-hep.github.io/umami-preprocessing/hpc/) documentation page -for usage instructions. From 5b923a8c50785d260048e6fd6284d02866ac2c32 Mon Sep 17 00:00:00 2001 From: Alexander Froch Date: Fri, 7 Aug 2026 13:08:49 +0200 Subject: [PATCH 04/13] Document obtaining the batch scripts without a repository clone --- docs/hpc.md | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/docs/hpc.md b/docs/hpc.md index 75bd639..7389b95 100644 --- a/docs/hpc.md +++ b/docs/hpc.md @@ -45,13 +45,37 @@ The `scripts/batch/` directory contains: - `run_stage.sh` runs inside the container and maps the submitted mode onto the `preprocess` command line flags. -To use them, create a run directory, copy the scripts, and adapt the resources to your cluster -(partition/pool, account, time and memory limits) — the `#SBATCH` header in `slurm_batch.sh` for -Slurm, or the requests at the top of `condor_job.sub` for HTCondor: +To use them, create a run directory and copy the scripts. A clone of the repository is not +required — the image contains the repository at `/workspace`, so the scripts can be taken straight +from there: + +=== "CVMFS" + + ```bash + mkdir my_preprocessing && cd my_preprocessing + cp -r /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest/workspace/scripts/batch . + ``` + +=== "apptainer" + + ```bash + mkdir my_preprocessing && cd my_preprocessing + apptainer exec docker://gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest \ + cp -r /workspace/scripts/batch . + ``` + +=== "local clone" + + ```bash + mkdir my_preprocessing && cd my_preprocessing + cp -r /scripts/batch . + ``` + +Then adapt the resources to your cluster (partition/pool, account, time and memory limits) — the +`#SBATCH` header in `slurm_batch.sh` for Slurm, or the requests at the top of `condor_job.sub` for +HTCondor: ```bash -mkdir my_preprocessing && cd my_preprocessing -cp -r /scripts/batch . $EDITOR batch/slurm_batch.sh # or batch/condor_job.sub ``` From ab33415281bdbf3d9d293ff376811d3005e6b3b7 Mon Sep 17 00:00:00 2001 From: Alexander Froch Date: Fri, 7 Aug 2026 13:53:02 +0200 Subject: [PATCH 05/13] Clarify apptainer image caching and cache location in the docs --- docs/hpc.md | 24 ++++++++++++++++-------- docs/setup.md | 12 ++++++++---- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/docs/hpc.md b/docs/hpc.md index 7389b95..074da29 100644 --- a/docs/hpc.md +++ b/docs/hpc.md @@ -12,14 +12,22 @@ cluster, each of these units of work can run as its own batch job inside the - The UPP container image (see [Container image](setup.md#container-image)). The scripts default to the CVMFS-unpacked image `/cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest` when - it exists and fall back to `docker://gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest` - otherwise. On clusters without CVMFS, avoid the repeated `docker://` to SIF conversion in every - job by pulling the image once and pointing `UPP_IMAGE` at the local file: - - ```bash - apptainer pull upp_latest.sif docker://gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest - export UPP_IMAGE=/path/to/upp_latest.sif - ``` + it exists (no download or conversion needed) and fall back to + `docker://gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest` otherwise. Running + from `docker://` directly is fine: apptainer caches the converted image and only downloads again + when a new version is published. Alternatively, pin a specific local file with + `export UPP_IMAGE=/path/to/upp.sif` after an `apptainer pull`. + +!!!info "Apptainer cache location" + + The apptainer cache defaults to `~/.apptainer/cache` and the conversion uses `/tmp` as + scratch space. On clusters with a small home quota (e.g. lxplus) point them to a larger + filesystem: + + ```bash + export APPTAINER_CACHEDIR=/path/to/big/storage/apptainer_cache + export APPTAINER_TMPDIR=/path/to/big/storage/apptainer_tmp + ``` ## Interactive use diff --git a/docs/setup.md b/docs/setup.md index 6bea982..6c2f3a3 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -145,11 +145,13 @@ and its command line scripts (`preprocess`, `check_input_samples`, `list_compone ``` The first `docker://` invocation converts the image to apptainer's SIF format, which takes a - while. If you run UPP repeatedly, pull the image once and use the local file instead: + while. The result is cached (default `~/.apptainer/cache`), so later invocations start quickly + and only download again when a new image version is published. If your home quota is small + (e.g. on lxplus), point the cache and the conversion scratch space to a larger filesystem: ```bash - apptainer pull upp_latest.sif docker://gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest - apptainer exec upp_latest.sif preprocess --config + export APPTAINER_CACHEDIR=/path/to/big/storage/apptainer_cache + export APPTAINER_TMPDIR=/path/to/big/storage/apptainer_tmp ``` On machines with CVMFS (lxplus, most HPC sites) the image is also distributed pre-unpacked @@ -169,9 +171,11 @@ and its command line scripts (`preprocess`, `check_input_samples`, `list_compone ```bash apptainer exec --contain --pwd "$PWD" -B /home -B /tmp -B \ - upp_latest.sif preprocess --config + preprocess --config ``` + where `` is any of the above (`docker://` URL, CVMFS path or local `.sif` file). + === "docker" With docker, mount your working directory and data paths into the container: From bd15d57aaa4aa024e0908504d43102bd5bd8a46b Mon Sep 17 00:00:00 2001 From: Alexander Froch Date: Fri, 7 Aug 2026 13:53:36 +0200 Subject: [PATCH 06/13] Mention submit.sh --help in the HPC docs --- docs/hpc.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/hpc.md b/docs/hpc.md index 074da29..e7e71bc 100644 --- a/docs/hpc.md +++ b/docs/hpc.md @@ -100,7 +100,8 @@ next: ``` Job logs are written to `logs/` in the current directory. Running `submit.sh` without a mode enters -an interactive prompt for the mode and filters. +an interactive prompt for the mode and filters, and `./batch/submit.sh --help` prints all modes and +options. On Slurm each job is submitted with its own `sbatch` call and job name. On HTCondor all jobs of one `submit.sh` invocation are submitted as a single cluster (one `condor_submit` with one process per From 511f0202c13312ac410ede4198b9715159368683 Mon Sep 17 00:00:00 2001 From: Alexander Froch Date: Fri, 7 Aug 2026 14:02:42 +0200 Subject: [PATCH 07/13] Reword resource adaptation instructions and drop EDITOR snippet --- docs/hpc.md | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/docs/hpc.md b/docs/hpc.md index e7e71bc..dfda61d 100644 --- a/docs/hpc.md +++ b/docs/hpc.md @@ -79,13 +79,9 @@ from there: cp -r /scripts/batch . ``` -Then adapt the resources to your cluster (partition/pool, account, time and memory limits) — the -`#SBATCH` header in `slurm_batch.sh` for Slurm, or the requests at the top of `condor_job.sub` for -HTCondor: - -```bash -$EDITOR batch/slurm_batch.sh # or batch/condor_job.sub -``` +Adapt the resource specifications (number of CPUs, memory allocation, time limit, partition or +account etc.) in the `slurm_batch.sh` (Slurm) or `condor_job.sub` (HTCondor) files to fit your +needs. Then submit the stages in order, waiting for all jobs of one stage to finish before submitting the next: From 3536e86cfe68e306029e0873b11893c8a4942c72 Mon Sep 17 00:00:00 2001 From: Alexander Froch Date: Fri, 7 Aug 2026 14:06:03 +0200 Subject: [PATCH 08/13] Only enumerate components for the modes that need them --- docs/hpc.md | 6 +++++- scripts/batch/submit.sh | 47 +++++++++++++++++++++++++---------------- 2 files changed, 34 insertions(+), 19 deletions(-) diff --git a/docs/hpc.md b/docs/hpc.md index dfda61d..9ae1917 100644 --- a/docs/hpc.md +++ b/docs/hpc.md @@ -125,7 +125,11 @@ The available modes and the jobs they submit: ## Config-driven job lists `submit.sh` never hardcodes which components exist. It calls the `list_components` script (part of -UPP) to enumerate the components defined in the `components:` block of your config: +UPP) to enumerate the components defined in the `components:` block of your config. No local UPP +installation is needed for this: when `list_components` is not on the `PATH`, it is run inside the +container image automatically. Only the `prepare`, `fine_resampling` and `resampling` modes (and +the interactive mode) enumerate at all — the other modes submit without running UPP on the login +node. ```bash list_components --config diff --git a/scripts/batch/submit.sh b/scripts/batch/submit.sh index c209a0c..997a0a6 100755 --- a/scripts/batch/submit.sh +++ b/scripts/batch/submit.sh @@ -85,6 +85,25 @@ enumerate() { fi } +ensure_enumerated() { + # Fill ALL_ROWS/ALL_REGIONS/ALL_SAMPLES/ALL_FLAVS from the config (TSV rows), once + if [[ ${#ALL_ROWS[@]} -gt 0 ]]; then + return 0 + fi + mapfile -t ALL_ROWS < <(enumerate) + if [[ ${#ALL_ROWS[@]} -eq 0 ]]; then + echo "ERROR: no components found in $CONFIG" >&2 + exit 1 + fi + local row region sample flavour name + for row in "${ALL_ROWS[@]}"; do + IFS=$'\t' read -r region sample flavour name <<< "$row" + add_unique "$region" ALL_REGIONS + add_unique "$sample" ALL_SAMPLES + add_unique "$flavour" ALL_FLAVS + done +} + append_list() { # Append a comma/space separated list to the named array local input="$1" @@ -136,7 +155,9 @@ print_resolved_config() { printf ' SCHEDULER : %s\n' "$SCHEDULER" printf ' IMAGE : %s\n' "$IMAGE" printf ' THROTTLE : %s\n' "$THROTTLE" - printf ' COMPONENTS : %s\n' "${#COMPONENTS[@]}" + if [[ ${#ALL_ROWS[@]} -gt 0 ]]; then + printf ' COMPONENTS : %s\n' "${#COMPONENTS[@]}" + fi printf ' SPLITS : %s\n' "${SPLITS[*]}" printf '\n' } @@ -373,34 +394,24 @@ main() { detect_scheduler - # Enumerate all components defined in the config (TSV: region sample flavour name) - local -a ALL_ROWS=() - mapfile -t ALL_ROWS < <(enumerate) - if [[ ${#ALL_ROWS[@]} -eq 0 ]]; then - echo "ERROR: no components found in $CONFIG" >&2 - exit 1 - fi - - local -a ALL_REGIONS=() ALL_SAMPLES=() ALL_FLAVS=() - local row region sample flavour name - for row in "${ALL_ROWS[@]}"; do - IFS=$'\t' read -r region sample flavour name <<< "$row" - add_unique "$region" ALL_REGIONS - add_unique "$sample" ALL_SAMPLES - add_unique "$flavour" ALL_FLAVS - done + declare -a ALL_ROWS=() ALL_REGIONS=() ALL_SAMPLES=() ALL_FLAVS=() MODE="${1:-}" declare -a SPLITS=(train val test) if [[ -z "$MODE" ]]; then + ensure_enumerated interactive_mode fi if [[ ${#SPLIT_FILTER[@]} -gt 0 ]]; then SPLITS=("${SPLIT_FILTER[@]}") fi - # Apply the filters + # Only the component/region level modes need the component list from the config declare -a COMPONENTS=() REGIONS=() + local row region sample flavour name + case "$MODE" in prepare|fine_resampling|resampling) + ensure_enumerated + esac for row in "${ALL_ROWS[@]}"; do IFS=$'\t' read -r region sample flavour name <<< "$row" if in_list "$region" "${REGION_FILTER[@]}" \ From 260563e390f95564c61501df76c578c1c7060158 Mon Sep 17 00:00:00 2001 From: Alexander Froch Date: Fri, 7 Aug 2026 14:13:19 +0200 Subject: [PATCH 09/13] Note the shared filesystem assumption and link the CERN batch docs --- docs/hpc.md | 10 ++++++++-- scripts/batch/condor_job.sub | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/hpc.md b/docs/hpc.md index 9ae1917..10ab91a 100644 --- a/docs/hpc.md +++ b/docs/hpc.md @@ -8,7 +8,12 @@ cluster, each of these units of work can run as its own batch job inside the ## Prerequisites -- A cluster with Slurm or HTCondor and apptainer. +- A cluster with Slurm or HTCondor and apptainer. On lxplus, HTCondor is provided by the CERN + batch service — see the [quickstart](https://batchdocs.web.cern.ch/local/quick.html) and + [job submission](https://batchdocs.web.cern.ch/local/submit.html) documentation. +- A shared filesystem between the submitting node and the workers — the input, intermediate and + output files must be visible to all jobs. This is the case on lxplus (AFS/EOS) and typical + institute clusters. - The UPP container image (see [Container image](setup.md#container-image)). The scripts default to the CVMFS-unpacked image `/cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest` when @@ -81,7 +86,8 @@ from there: Adapt the resource specifications (number of CPUs, memory allocation, time limit, partition or account etc.) in the `slurm_batch.sh` (Slurm) or `condor_job.sub` (HTCondor) files to fit your -needs. +needs. The `+JobFlavour` walltime flavours available on lxplus are listed in the +[CERN batch documentation](https://batchdocs.web.cern.ch/local/submit.html). Then submit the stages in order, waiting for all jobs of one stage to finish before submitting the next: diff --git a/scripts/batch/condor_job.sub b/scripts/batch/condor_job.sub index 2333a26..d1713a5 100644 --- a/scripts/batch/condor_job.sub +++ b/scripts/batch/condor_job.sub @@ -5,6 +5,7 @@ request_cpus = 4 request_memory = 32 GB # CERN lxplus walltime flavour; ignored on other pools +# Available flavours: https://batchdocs.web.cern.ch/local/submit.html +JobFlavour = "tomorrow" universe = vanilla From b88f4e2c319ca69cc9143fb8a9c9ca11f39f0f05 Mon Sep 17 00:00:00 2001 From: Alexander Froch Date: Fri, 7 Aug 2026 14:16:09 +0200 Subject: [PATCH 10/13] Document lxplus AFS/EOS input and output handling for HTCondor jobs --- docs/hpc.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/hpc.md b/docs/hpc.md index 10ab91a..59bd2a7 100644 --- a/docs/hpc.md +++ b/docs/hpc.md @@ -14,6 +14,23 @@ cluster, each of these units of work can run as its own batch job inside the - A shared filesystem between the submitting node and the workers — the input, intermediate and output files must be visible to all jobs. This is the case on lxplus (AFS/EOS) and typical institute clusters. + +!!!info "Input/output data on lxplus (AFS/EOS)" + + HTCondor on lxplus rejects submit files that reference EOS paths (executable, `output`, + `error`, `log`), so create the run directory — with the copied scripts and the `logs/` + directory — in your AFS work area and submit from there. The jobs themselves run with your + Kerberos credentials and can read and write `/afs` and `/eos` directly, so keep the large + input ntuples and outputs on EOS and bind both filesystems into the container: + + ```bash + export UPP_BINDS=/afs,/eos,/tmp + ``` + + For very I/O-heavy workflows the batch service recommends staging data through the local + pool space of the job instead of writing to EOS directly — see + [Data flows](https://batchdocs.web.cern.ch/concepts/dataflow.html) and + [EOS](https://batchdocs.web.cern.ch/troubleshooting/eos.html) in the CERN batch docs. - The UPP container image (see [Container image](setup.md#container-image)). The scripts default to the CVMFS-unpacked image `/cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest` when From dc714b7f4b539e8df2c73e1cc53cbd5b98b9babe Mon Sep 17 00:00:00 2001 From: Alexander Froch Date: Fri, 7 Aug 2026 14:29:29 +0200 Subject: [PATCH 11/13] Mention the experimental EosSubmit schedds for EOS-based run directories --- docs/hpc.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/hpc.md b/docs/hpc.md index 59bd2a7..a56b2da 100644 --- a/docs/hpc.md +++ b/docs/hpc.md @@ -31,6 +31,12 @@ cluster, each of these units of work can run as its own batch job inside the pool space of the job instead of writing to EOS directly — see [Data flows](https://batchdocs.web.cern.ch/concepts/dataflow.html) and [EOS](https://batchdocs.web.cern.ch/troubleshooting/eos.html) in the CERN batch docs. + + If the run directory has to live on EOS, the experimental + [EosSubmit schedds](https://batchdocs.web.cern.ch/local/eossubmit.html) + (`module load lxbatch/eossubmit`) accept submit files with EOS paths, transferring all job + files via xrootd instead of using a shared filesystem. All submit file paths must then be on + EOS, and this mode has not been tested with these scripts. - The UPP container image (see [Container image](setup.md#container-image)). The scripts default to the CVMFS-unpacked image `/cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/aft/training-images/upp-images/upp:latest` when From b88fd7856d5ff3c966ef746b241af225f1df36c0 Mon Sep 17 00:00:00 2001 From: Alexander Froch Date: Fri, 7 Aug 2026 14:34:11 +0200 Subject: [PATCH 12/13] Always bind the run and script directories into the container --- docs/hpc.md | 4 ++-- docs/setup.md | 2 +- scripts/batch/condor_batch.sh | 2 +- scripts/batch/slurm_batch.sh | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/hpc.md b/docs/hpc.md index a56b2da..41a3b4a 100644 --- a/docs/hpc.md +++ b/docs/hpc.md @@ -63,7 +63,7 @@ For quick tests, run UPP inside the container on an interactive allocation: ```bash salloc --ntasks 1 --cpus-per-task 4 --time 2:00:00 -srun apptainer exec --contain --pwd "$PWD" -B /home -B /tmp \ +srun apptainer exec --contain --pwd "$PWD" -B "$PWD" -B /home -B /tmp \ "$UPP_IMAGE" preprocess --config --prep ``` @@ -186,7 +186,7 @@ on the login node instead of inside the batch jobs. | Variable | Default | Purpose | |----------|---------|---------| | `UPP_IMAGE` | CVMFS-unpacked image if present, else `docker://...upp-images/upp:latest` | Container image (unpacked directory, local `.sif` or `docker://` URI) | -| `UPP_BINDS` | `/home,/tmp` | Comma-separated paths bound into the container | +| `UPP_BINDS` | `/home,/tmp` | Comma-separated paths bound into the container (the run and script directories are always bound in addition) | | `THROTTLE` | `30` | Seconds between `sbatch` calls (`0` disables; Slurm only) | | `DRY_RUN` | `0` | Set to `1` to print the submission commands instead of submitting | diff --git a/docs/setup.md b/docs/setup.md index 6c2f3a3..bd8f2ae 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -170,7 +170,7 @@ and its command line scripts (`preprocess`, `check_input_samples`, `list_compone output directory) with `-B`, keeping the working directory with `--pwd`: ```bash - apptainer exec --contain --pwd "$PWD" -B /home -B /tmp -B \ + apptainer exec --contain --pwd "$PWD" -B "$PWD" -B /home -B /tmp -B \ preprocess --config ``` diff --git a/scripts/batch/condor_batch.sh b/scripts/batch/condor_batch.sh index 6c7eea5..02234a4 100755 --- a/scripts/batch/condor_batch.sh +++ b/scripts/batch/condor_batch.sh @@ -11,5 +11,5 @@ fi IMAGE="${UPP_IMAGE:-${DEFAULT_IMAGE}}" BINDS="${UPP_BINDS:-/home,/tmp}" -apptainer exec --contain --pwd "${PWD}" -B "${BINDS}" \ +apptainer exec --contain --pwd "${PWD}" -B "${BINDS}" -B "${PWD}" -B "${SCRIPT_DIR}" \ "${IMAGE}" "${SCRIPT_DIR}/run_stage.sh" "$@" diff --git a/scripts/batch/slurm_batch.sh b/scripts/batch/slurm_batch.sh index e3fef6a..3fbee28 100755 --- a/scripts/batch/slurm_batch.sh +++ b/scripts/batch/slurm_batch.sh @@ -17,5 +17,5 @@ fi IMAGE="${UPP_IMAGE:-${DEFAULT_IMAGE}}" BINDS="${UPP_BINDS:-/home,/tmp}" -srun apptainer exec --contain --pwd "${PWD}" -B "${BINDS}" \ +srun apptainer exec --contain --pwd "${PWD}" -B "${BINDS}" -B "${PWD}" -B "${SCRIPT_DIR}" \ "${IMAGE}" "${SCRIPT_DIR}/run_stage.sh" "$@" From e905f2f3de573c400e3c199d3208d7288c2aa16b Mon Sep 17 00:00:00 2001 From: Alexander Froch Date: Fri, 7 Aug 2026 15:14:14 +0200 Subject: [PATCH 13/13] Split job stdout and stderr into logs subdirectories --- docs/hpc.md | 3 ++- scripts/batch/condor_job.sub | 4 ++-- scripts/batch/submit.sh | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/hpc.md b/docs/hpc.md index 41a3b4a..f96f1fe 100644 --- a/docs/hpc.md +++ b/docs/hpc.md @@ -124,7 +124,8 @@ next: ./batch/submit.sh --config plotting ``` -Job logs are written to `logs/` in the current directory. Running `submit.sh` without a mode enters +Job logs are written to the `logs/output/` and `logs/error/` subdirectories of the current +directory. Running `submit.sh` without a mode enters an interactive prompt for the mode and filters, and `./batch/submit.sh --help` prints all modes and options. diff --git a/scripts/batch/condor_job.sub b/scripts/batch/condor_job.sub index d1713a5..a0c735e 100644 --- a/scripts/batch/condor_job.sub +++ b/scripts/batch/condor_job.sub @@ -13,8 +13,8 @@ executable = $(batch_dir)/condor_batch.sh arguments = $(args) batch_name = $(batch_name) environment = "UPP_IMAGE=$(upp_image) UPP_BINDS=$(upp_binds)" -output = logs/$(ClusterId).$(ProcId).out -error = logs/$(ClusterId).$(ProcId).err +output = logs/output/$(ClusterId).$(ProcId).out +error = logs/error/$(ClusterId).$(ProcId).err log = logs/$(ClusterId).log # Assumes a shared filesystem between submit and worker nodes diff --git a/scripts/batch/submit.sh b/scripts/batch/submit.sh index 997a0a6..32799a6 100755 --- a/scripts/batch/submit.sh +++ b/scripts/batch/submit.sh @@ -216,8 +216,8 @@ submit() { local -a cmd=( sbatch --job-name="$jobname" - --output="${PWD}/logs/%j_%x.out" - --error="${PWD}/logs/%j_%x.err" + --output="${PWD}/logs/output/%j_%x.out" + --error="${PWD}/logs/error/%j_%x.err" "${SCRIPT_DIR}/slurm_batch.sh" "$CONFIG" "$@" @@ -427,7 +427,7 @@ main() { fi if [[ "$DRY_RUN" != "1" ]]; then - mkdir -p "${PWD}/logs" + mkdir -p "${PWD}/logs/output" "${PWD}/logs/error" fi local split