gh-984: Add config to run on Archer2

noxfile.py

@@ -251,6 +251,8 @@ def benchmarks(session: nox.Session) -> None:
     _check_revision_count(session.posargs, expected_count=1)
     revision = session.posargs[0]
 
+    _setup_array_backend(session)
+
     # overwrite current package with specified revision
     session.install(f"git+{GLASS_REPO_URL}@{revision}")
     session.run(
@@ -273,8 +275,9 @@ def regression_tests(session: nox.Session) -> None:
     Note it is not possible to pass extra options to pytest.
 
     """
-    _check_revision_count(session.posargs, expected_count=2)
-    before_revision, after_revision = session.posargs
+    minimum_posargs = 2
+    _check_revision_count(session.posargs, expected_count=minimum_posargs)
+    before_revision, after_revision = session.posargs[:minimum_posargs]
 
     _setup_array_backend(session)
 

tests/benchmarks/archer2/README.md

@@ -0,0 +1,68 @@
+# GLASS benchmarks on Archer2
+
+For a more consistent benchmarking and regression testing environment we have
+been trialing using the UCL machine [Archer2](https://www.archer2.ac.uk/).
+
+## Setting up
+
+1. **Install uv:** Firstly, install uv via curl onto the `/work` partition
+
+   ```sh
+   cd "${HOME/home/work}"
+   curl -LsSf https://astral.sh/uv/install.sh | sh
+   ```
+
+   Then we must make sure uv is available on the login node and the worker node.
+   To do this we can update our start up scripts (`.profile`) on both
+   partitions. Therefore, execute the following
+
+   ```sh
+   cat <<'EOF' >> "$HOME/.profile"
+   WORK_DIR="${HOME/home/work}"
+   cd "$WORK_DIR"
+   source "$WORK_DIR/.profile"
+   EOF
+   ```
+
+   and similarly
+
+   ```sh
+   cat <<'EOF' >> "${HOME/home/work}/.profile"
+   export HOME="${HOME/home/work}"
+   source "$HOME/.local/bin/env"
+   EOF
+   ```
+
+   Now when you next login to archer2, uv will be in your path and you will be
+   on the `/work` partition as your `HOME` dir.
+
+2. **Clone GLASS:** Clone the glass repo into the `/work` partition of Archer2:
+
+   ```sh
+   cd "${HOME/home/work}"
+   git clone https://github.com/glass-dev/glass.git
+   ```
+
+## Run the regression tests
+
+Now we have cloned glass, we can run the script
+[run_regression_test.sh](./run_regression_test.sh) which will setup the required
+environments and submit regression test job to slurm. A help message is
+provided. Just run `./tests/benchmarks/run_regression_test.sh -h` from the root
+of the GLASS repo.
+
+### Example execution
+
+If I wished to to run a test to check for regressions from `main` to my feature
+branch `feature` using the budget from account code `myaccount`, I could run the
+following command from the root of the glass repo.
+
+```sh
+./tests/benchmarks/archer2/run_regression_test.sh \
+   -d "$(pwd)"                                    \
+   -s main                                        \
+   -e feature                                     \
+   -a myaccount
+```
+
+> Note that this script does not have to be ran from the root of the glass repo.

tests/benchmarks/archer2/run_regression_test.sh

@@ -0,0 +1,126 @@
+#!/bin/bash -l
+# shellcheck disable=SC1090
+
+GLASS_DIR=""
+GLASS_REPO_URL="https://github.com/glass-dev/glass"
+START_REF=""
+END_REF=""
+START_VENV=".venv-start"
+END_VENV=".venv-end"
+ACCOUNT=""
+SETUP_ENVS="true"
+
+help() {
+  echo "Usage:"
+  echo "    $0 "
+  echo ""
+  echo "ARGS:"
+  echo "    -h | --help                      Display this help message."
+  echo "    -d | --glass-dir <glass/dir>     Path to the cloned glass directory."
+  echo "    -s | --start-ref <start_ref>     The git ref to be used as the initial state."
+  echo "    -e | --end-ref <end_ref>         The git ref to be used as the final state."
+  echo "    -a | --account <archer2_account> The archer2 account code to run jobs against."
+  echo "    --skip-setup                     Flag to state if the setup (installation of "
+  echo "                                     dependencies) can be skipped. Good for simply"
+  echo "                                     re-submitting"
+}
+
+# check for no input arguments and show help
+if [ $# -eq 0 ];
+then
+    help
+    exit 1
+fi
+
+# parse input arguments
+while [ $# -gt 0 ] ; do
+    case $1 in
+        -h | --help)
+            help
+            exit 0
+            ;;
+        -d | --glass-dir)
+            GLASS_DIR=$2
+            shift 2
+            continue
+            ;;
+        -s | --start-ref)
+            START_REF=$2
+            shift 2
+            continue
+            ;;
+        -e | --end-ref)
+            END_REF=$2
+            shift 2
+            continue
+            ;;
+        -a | --account)
+            ACCOUNT=$2
+            shift 2
+            continue
+            ;;
+        --skip-setup)
+            SETUP_ENVS="false"
+            shift 1
+            continue
+            ;;
+        *)
+            echo "Invalid option: $1" >&2;
+            help
+            exit 1
+            ;;
+    esac
+    shift 1
+done
+
+if [[ "$START_REF" == "" || "$END_REF" == "" ]]
+then
+  echo "START_REF and END_REF must be provided"
+  help
+  exit 1
+fi
+
+if [[ "$GLASS_DIR" == "" ]]
+then
+  echo "GLASS_DIR must be provided"
+  help
+  exit 1
+fi
+
+if [[ "$ACCOUNT" == "" ]]
+then
+  echo "ACCOUNT must be provided"
+  help
+  exit 1
+fi
+
+BENCHMARKS_DIR="$GLASS_DIR/tests/benchmarks"
+
+
+if [[ "$SETUP_ENVS" == "true" ]]
+then
+  # Move to the glass directory
+  pushd "$GLASS_DIR" || exit
+
+  # Setup base environment
+  rm -rf "${GLASS_DIR:?}/$START_VENV" # Cleanup old venv
+  uv venv  "$GLASS_DIR/$START_VENV"
+  source "$GLASS_DIR/$START_VENV/bin/activate"
+  uv sync --active --group test --no-install-project
+  uv pip install --no-deps "git+$GLASS_REPO_URL@$START_REF"
+  deactivate
+
+  # Setup head environment
+  rm -rf "${GLASS_DIR:?}/$END_VENV" # Cleanup old venv
+  uv venv "$GLASS_DIR/$END_VENV"
+  source "$GLASS_DIR/$END_VENV/bin/activate"
+  uv sync --active --group test --no-install-project
+  uv pip install --no-deps "git+$GLASS_REPO_URL@$END_REF"
+  deactivate
+
+  # Return to the starting directory
+  popd || exit
+fi
+
+# Submit job
+sbatch --account="$ACCOUNT" "$BENCHMARKS_DIR/archer2/submission_script_cpu.sh" "$GLASS_DIR"

tests/benchmarks/archer2/submission_script_cpu.sh

@@ -0,0 +1,56 @@
+#!/bin/bash --login
+# shellcheck disable=SC1091
+
+#SBATCH --job-name=glass_reg_test_cpu
+#SBATCH --output=%x-%j.out
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=1
+#SBATCH --time=0:30:0
+#SBATCH --partition=standard
+#SBATCH --qos=standard
+
+# Ensure uv is available
+source "${HOME/home/work}/.profile" # HOME starts as /home/... but uv needs to be on /work/...
+
+# Recommended environment settings
+# Stop unintentional multi-threading within software libraries
+export OMP_NUM_THREADS=1
+# Ensure the cpus-per-task option is propagated to srun commands
+export SRUN_CPUS_PER_TASK=$SLURM_CPUS_PER_TASK
+
+# Set path to class and select base or head
+GLASS_DIR="$1"
+BENCHMARKS_DIR="$GLASS_DIR/tests/benchmarks"
+BENCHMARK_OUTPUT_PATH="$BENCHMARKS_DIR/archer2/.benchmarks"
+BENCHMARKS_SHARED_FLAGS=(
+  "--benchmark-storage=file://$BENCHMARK_OUTPUT_PATH"
+  "--benchmark-calibration-precision=1000"
+  "--benchmark-columns=mean,stddev,rounds"
+  "--benchmark-max-time=5.0"
+  "--benchmark-sort=name"
+  "--benchmark-timer=time.process_time"
+)
+START_VENV_BIN="$GLASS_DIR/.venv-start/bin"
+END_VENV_BIN="$GLASS_DIR/.venv-end/bin"
+
+# Remove old benchmark results
+rm -rf "$BENCHMARK_OUTPUT_PATH"
+
+# Change into archer2 dir to ensure we don't pickup the glass directory as an import
+cd "$BENCHMARKS_DIR/archer2" || exit
+
+# Generate the base report for comparison later
+source "$START_VENV_BIN/activate"
+srun "$START_VENV_BIN/python" -m pytest "$BENCHMARKS_DIR" \
+    --benchmark-autosave "${BENCHMARKS_SHARED_FLAGS[@]}"
+deactivate
+
+# Run the stable and unstable benchmarks and compare to the base ref
+source "$END_VENV_BIN/activate"
+srun "$END_VENV_BIN/python" -m pytest "$BENCHMARKS_DIR" -m stable \
+    --benchmark-compare=0001 \
+    --benchmark-compare-fail=mean:5% "${BENCHMARKS_SHARED_FLAGS[@]}"
+srun "$END_VENV_BIN/python" -m pytest "$BENCHMARKS_DIR" -m unstable \
+    --benchmark-compare=0001 \
+    --benchmark-compare-fail=mean:0.0005 "${BENCHMARKS_SHARED_FLAGS[@]}"

tests/benchmarks/myriad/run_regression_test.sh

@@ -1,9 +1,9 @@
 #!/bin/bash -l
 
-# Request ten minutes of wallclock time (format hours:minutes:seconds).
+# Request three hours of wallclock time (format hours:minutes:seconds).
 #$ -l h_rt=3:0:0
 
-# Request 1 gigabyte of RAM (must be an integer followed by M, G, or T)
+# Request 4 gigabyte of RAM (must be an integer followed by M, G, or T)
 #$ -l mem=4G
 
 # Request exclusive access to a node