diff --git a/noxfile.py b/noxfile.py index a749ebbf..033ad7be 100644 --- a/noxfile.py +++ b/noxfile.py @@ -250,6 +250,8 @@ def benchmarks(session: nox.Session) -> None: _check_revision_count(session.posargs, expected_count=1) revision = session.posargs[0] + _setup_array_backend(session) + # overwrite current package with specified revision session.install(f"git+{GLASS_REPO_URL}@{revision}") session.run( @@ -272,8 +274,9 @@ def regression_tests(session: nox.Session) -> None: Note it is not possible to pass extra options to pytest. """ - _check_revision_count(session.posargs, expected_count=2) - before_revision, after_revision = session.posargs + minimum_posargs = 2 + _check_revision_count(session.posargs, expected_count=minimum_posargs) + before_revision, after_revision = session.posargs[:minimum_posargs] _setup_array_backend(session) diff --git a/tests/benchmarks/archer2/README.md b/tests/benchmarks/archer2/README.md new file mode 100644 index 00000000..ac874873 --- /dev/null +++ b/tests/benchmarks/archer2/README.md @@ -0,0 +1,68 @@ +# GLASS benchmarks on Archer2 + +For a more consistent benchmarking and regression testing environment we have +been trialing using the UCL machine [Archer2](https://www.archer2.ac.uk/). + +## Setting up + +1. **Install uv:** Firstly, install uv via curl onto the `/work` partition + + ```sh + cd "${HOME/home/work}" + curl -LsSf https://astral.sh/uv/install.sh | sh + ``` + + Then we must make sure uv is available on the login node and the worker node. + To do this we can update our start up scripts (`.profile`) on both + partitions. Therefore, execute the following + + ```sh + cat <<'EOF' >> "$HOME/.profile" + WORK_DIR="${HOME/home/work}" + cd "$WORK_DIR" + source "$WORK_DIR/.profile" + EOF + ``` + + and similarly + + ```sh + cat <<'EOF' >> "${HOME/home/work}/.profile" + export HOME="${HOME/home/work}" + source "$HOME/.local/bin/env" + EOF + ``` + + Now when you next login to archer2, uv will be in your path and you will be + on the `/work` partition as your `HOME` dir. + +2. **Clone GLASS:** Clone the glass repo into the `/work` partition of Archer2: + + ```sh + cd "${HOME/home/work}" + git clone https://github.com/glass-dev/glass.git + ``` + +## Run the regression tests + +Now we have cloned glass, we can run the script +[run_regression_test.sh](./run_regression_test.sh) which will setup the required +environments and submit regression test job to slurm. A help message is +provided. Just run `./tests/benchmarks/run_regression_test.sh -h` from the root +of the GLASS repo. + +### Example execution + +If I wished to to run a test to check for regressions from `main` to my feature +branch `feature` using the budget from account code `myaccount`, I could run the +following command from the root of the glass repo. + +```sh +./tests/benchmarks/archer2/run_regression_test.sh \ + -d "$(pwd)" \ + -s main \ + -e feature \ + -a myaccount +``` + +> Note that this script does not have to be ran from the root of the glass repo. diff --git a/tests/benchmarks/archer2/run_regression_test.sh b/tests/benchmarks/archer2/run_regression_test.sh new file mode 100755 index 00000000..fb45d4a6 --- /dev/null +++ b/tests/benchmarks/archer2/run_regression_test.sh @@ -0,0 +1,126 @@ +#!/bin/bash -l +# shellcheck disable=SC1090 + +GLASS_DIR="" +GLASS_REPO_URL="https://github.com/glass-dev/glass" +START_REF="" +END_REF="" +START_VENV=".venv-start" +END_VENV=".venv-end" +ACCOUNT="" +SETUP_ENVS="true" + +help() { + echo "Usage:" + echo " $0 " + echo "" + echo "ARGS:" + echo " -h | --help Display this help message." + echo " -d | --glass-dir Path to the cloned glass directory." + echo " -s | --start-ref The git ref to be used as the initial state." + echo " -e | --end-ref The git ref to be used as the final state." + echo " -a | --account The archer2 account code to run jobs against." + echo " --skip-setup Flag to state if the setup (installation of " + echo " dependencies) can be skipped. Good for simply" + echo " re-submitting" +} + +# check for no input arguments and show help +if [ $# -eq 0 ]; +then + help + exit 1 +fi + +# parse input arguments +while [ $# -gt 0 ] ; do + case $1 in + -h | --help) + help + exit 0 + ;; + -d | --glass-dir) + GLASS_DIR=$2 + shift 2 + continue + ;; + -s | --start-ref) + START_REF=$2 + shift 2 + continue + ;; + -e | --end-ref) + END_REF=$2 + shift 2 + continue + ;; + -a | --account) + ACCOUNT=$2 + shift 2 + continue + ;; + --skip-setup) + SETUP_ENVS="false" + shift 1 + continue + ;; + *) + echo "Invalid option: $1" >&2; + help + exit 1 + ;; + esac + shift 1 +done + +if [[ "$START_REF" == "" || "$END_REF" == "" ]] +then + echo "START_REF and END_REF must be provided" + help + exit 1 +fi + +if [[ "$GLASS_DIR" == "" ]] +then + echo "GLASS_DIR must be provided" + help + exit 1 +fi + +if [[ "$ACCOUNT" == "" ]] +then + echo "ACCOUNT must be provided" + help + exit 1 +fi + +BENCHMARKS_DIR="$GLASS_DIR/tests/benchmarks" + + +if [[ "$SETUP_ENVS" == "true" ]] +then + # Move to the glass directory + pushd "$GLASS_DIR" || exit + + # Setup base environment + rm -rf "${GLASS_DIR:?}/$START_VENV" # Cleanup old venv + uv venv "$GLASS_DIR/$START_VENV" + source "$GLASS_DIR/$START_VENV/bin/activate" + uv sync --active --group test --no-install-project + uv pip install --no-deps "git+$GLASS_REPO_URL@$START_REF" + deactivate + + # Setup head environment + rm -rf "${GLASS_DIR:?}/$END_VENV" # Cleanup old venv + uv venv "$GLASS_DIR/$END_VENV" + source "$GLASS_DIR/$END_VENV/bin/activate" + uv sync --active --group test --no-install-project + uv pip install --no-deps "git+$GLASS_REPO_URL@$END_REF" + deactivate + + # Return to the starting directory + popd || exit +fi + +# Submit job +sbatch --account="$ACCOUNT" "$BENCHMARKS_DIR/archer2/submission_script_cpu.sh" "$GLASS_DIR" diff --git a/tests/benchmarks/archer2/submission_script_cpu.sh b/tests/benchmarks/archer2/submission_script_cpu.sh new file mode 100755 index 00000000..b5ee8700 --- /dev/null +++ b/tests/benchmarks/archer2/submission_script_cpu.sh @@ -0,0 +1,56 @@ +#!/bin/bash --login +# shellcheck disable=SC1091 + +#SBATCH --job-name=glass_reg_test_cpu +#SBATCH --output=%x-%j.out +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=1 +#SBATCH --time=0:30:0 +#SBATCH --partition=standard +#SBATCH --qos=standard + +# Ensure uv is available +source "${HOME/home/work}/.profile" # HOME starts as /home/... but uv needs to be on /work/... + +# Recommended environment settings +# Stop unintentional multi-threading within software libraries +export OMP_NUM_THREADS=1 +# Ensure the cpus-per-task option is propagated to srun commands +export SRUN_CPUS_PER_TASK=$SLURM_CPUS_PER_TASK + +# Set path to class and select base or head +GLASS_DIR="$1" +BENCHMARKS_DIR="$GLASS_DIR/tests/benchmarks" +BENCHMARK_OUTPUT_PATH="$BENCHMARKS_DIR/archer2/.benchmarks" +BENCHMARKS_SHARED_FLAGS=( + "--benchmark-storage=file://$BENCHMARK_OUTPUT_PATH" + "--benchmark-calibration-precision=1000" + "--benchmark-columns=mean,stddev,rounds" + "--benchmark-max-time=5.0" + "--benchmark-sort=name" + "--benchmark-timer=time.process_time" +) +START_VENV_BIN="$GLASS_DIR/.venv-start/bin" +END_VENV_BIN="$GLASS_DIR/.venv-end/bin" + +# Remove old benchmark results +rm -rf "$BENCHMARK_OUTPUT_PATH" + +# Change into archer2 dir to ensure we don't pickup the glass directory as an import +cd "$BENCHMARKS_DIR/archer2" || exit + +# Generate the base report for comparison later +source "$START_VENV_BIN/activate" +srun "$START_VENV_BIN/python" -m pytest "$BENCHMARKS_DIR" \ + --benchmark-autosave "${BENCHMARKS_SHARED_FLAGS[@]}" +deactivate + +# Run the stable and unstable benchmarks and compare to the base ref +source "$END_VENV_BIN/activate" +srun "$END_VENV_BIN/python" -m pytest "$BENCHMARKS_DIR" -m stable \ + --benchmark-compare=0001 \ + --benchmark-compare-fail=mean:5% "${BENCHMARKS_SHARED_FLAGS[@]}" +srun "$END_VENV_BIN/python" -m pytest "$BENCHMARKS_DIR" -m unstable \ + --benchmark-compare=0001 \ + --benchmark-compare-fail=mean:0.0005 "${BENCHMARKS_SHARED_FLAGS[@]}" diff --git a/tests/benchmarks/myriad/run_regression_test.sh b/tests/benchmarks/myriad/run_regression_test.sh index dab2544f..38458a7c 100755 --- a/tests/benchmarks/myriad/run_regression_test.sh +++ b/tests/benchmarks/myriad/run_regression_test.sh @@ -1,9 +1,9 @@ #!/bin/bash -l -# Request ten minutes of wallclock time (format hours:minutes:seconds). +# Request three hours of wallclock time (format hours:minutes:seconds). #$ -l h_rt=3:0:0 -# Request 1 gigabyte of RAM (must be an integer followed by M, G, or T) +# Request 4 gigabyte of RAM (must be an integer followed by M, G, or T) #$ -l mem=4G # Request exclusive access to a node