diff --git a/conda/dev.yml b/conda/dev.yml index 85f64d32..492918c1 100644 --- a/conda/dev.yml +++ b/conda/dev.yml @@ -12,7 +12,7 @@ dependencies: - globus-sdk >=3.15.0,<4.0 # Developer Tools # ================= - # If versions are updated, also update 'rev' in `.pre-commit.config.yaml` + # If versions are updated, also update 'rev' in `.pre-commit-config.yaml` - black ==25.1.0 - flake8 ==7.3.0 - isort ==6.0.1 @@ -25,7 +25,7 @@ dependencies: - pytest-cov # Documentation # ================= - # If versions are updated, also update in `.github/workflows/workflow.yml` + # If versions are updated, also update in `.github/workflows/build_workflow.yml` and `.github/workflows/release_workflow.yml` - jinja2 <3.1 - sphinx >=5.2.0 - sphinx-multiversion >=0.2.4 diff --git a/conda/perf.yml b/conda/perf.yml new file mode 100644 index 00000000..f254e53e --- /dev/null +++ b/conda/perf.yml @@ -0,0 +1,38 @@ +name: zstash_perf +channels: + - conda-forge +dependencies: + # Base + # ================= + - pip + - python >=3.11,<3.15 + - setuptools + - sqlite + - six >=1.16.0 + - globus-sdk >=3.15.0,<4.0 + # Developer Tools + # ================= + # If versions are updated, also update 'rev' in `.pre-commit-config.yaml` + - black ==25.1.0 + - flake8 ==7.3.0 + - isort ==6.0.1 + - mypy ==1.18.2 + - pre-commit ==4.3.0 + - tbump >=6.9.0 + # Testing + # ======================= + - pytest + - pytest-cov + # Documentation + # ================= + # If versions are updated, also update in `.github/workflows/build_workflow.yml` and `.github/workflows/release_workflow.yml` + - jinja2 <3.1 + - sphinx >=5.2.0 + - sphinx-multiversion >=0.2.4 + - sphinx_rtd_theme >=1.0.0 + - docutils >=0.17.1 + # Performance profiling + # ================= + - matplotlib-base + - pandas + - numpy diff --git a/tests/README.md b/tests/README.md index c4552471..3c825fac 100644 --- a/tests/README.md +++ b/tests/README.md @@ -109,3 +109,7 @@ GitHub Actions runs the tests according to `.github/workflows/build_workflow.yml python -m unittest tests/integration/python_tests/group_by_command/test_*.py python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py ``` + +## Performance + +For performance profiling, see `tests/performance/README.md`. diff --git a/tests/performance/README.md b/tests/performance/README.md new file mode 100644 index 00000000..9658cab4 --- /dev/null +++ b/tests/performance/README.md @@ -0,0 +1,180 @@ +# How to profile zstash's performance + +Performance profiling should be done on Perlmutter. We're keeping the performance records in a long-term directory specified by `performance_archive_dir` in your config file (see below). (NOTE: this is currently user-specific. If we start having many other developers running performance profiling, we may try to find a more centralized location.) + +## Setup + +To run the visualizer (`visualize_performance.py`), you need `matplotlib`, `numpy`, and `pandas`. The repo provides a minimal conda environment for this in `conda/perf.yml`: + +```bash +conda env create -f conda/perf.yml -n zstash_perf +conda activate zstash_perf +python -m pip install . +``` + +Each script has its own config file (`key=value`) so you never need to edit the scripts themselves. Start by copying the templates and filling in your values: + +```bash +# Generator config +cp tests/performance/generate/perf.cfg tests/performance/generate/my_run.cfg + +# Visualizer config +cp tests/performance/visualize/perf.cfg tests/performance/visualize/my_run.cfg +``` + +The config files use a simple `key=value` format (lines starting with `#` are comments). + +> **Perlmutter path convention:** home and scratch directories follow the pattern +> `/global/homes/u/username/...` and `/pscratch/sd/u/username/...` +> where `u` is the first letter of your username. +> The placeholder `u/username` in the template should be replaced accordingly. + +## Generate performance data + +Edit the run metadata section of `generate/my_run.cfg`: + +```ini +# Use /pscratch since a lot of data will be transferred. +# The results csv alone will be copied to a long-term directory at the end. +work_dir=/pscratch/sd/u/username/zstash_performance/ +gen_run_id=performance_20260603 + +# The environment that zstash will be run in. +# Using Unified environment: +environment_commands=source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh +# Example dev environment: +# environment_commands=source /global/homes/u/username/miniforge3/etc/profile.d/conda.sh ; conda activate zstash-pr427-20260603 + +# Long-term directory where the results CSV is archived after the run. +performance_archive_dir=/global/homes/u/username/zstash_performance_records +``` + +These parameters you probably won't have a need to change: + +```ini +# Directories to run zstash create/update/extract on. +dir_to_copy_from=/global/cfs/cdirs/e3sm/forsyth/E3SMv2/v2.LR.historical_0201/ +subdir0=build/ +subdir1=run/ +subdir2=init/ + +# Which --hpss settings to run (space-separated; comment out any to skip): +HPSS_OPTIONS=none hpss globus + +# Used for the "hpss" option: +dst_hpss_path=/home/u/username/zstash_performance + +# Used for the "globus" option: +fresh_globus=true # prompts a fresh Globus authentication +dst_endpoint_uuid=15288284-7006-4041-ba1a-6b52501e49f1 # LCRC's endpoint +dst_endpoint_archive_dir=/lcrc/group/e3sm/username/zstash_performance_dst_dir/ +``` + +Once you have the parameters set up, run: + +```bash +cd tests/performance/generate/ +./generate_performance_data.bash my_run.cfg +``` + +If no cfg file argument is given, the script looks for `perf.cfg` in the same directory. + +Results will be saved to `${work_dir}${gen_run_id}/results.csv`. To keep all records together in a non-scratch space, the results csv is also copied to `${performance_archive_dir}/${gen_run_id}_results.csv`. + +## Visualize performance + +The visualizer lives in `tests/performance/visualize/`. Edit `visualize/my_run.cfg`: + +```ini +# Path to the results CSV to show in Figure 1. +# This should be the results.csv you just generated in the step above. +results_csv=/pscratch/sd/u/username/zstash_performance/performance_20260603/results.csv + +# Path to a baseline results CSV to compare against in Figure 2. +# Leave blank to skip Figure 2. +# This will typically be the second-to-latest results.csv in the records space. +baseline_results_csv=/pscratch/sd/u/username/zstash_performance/performance_20260414/results.csv + +# Output path for the saved figures. +# Leave blank to display interactively instead of saving. +# Make sure to use the web server path, i.e., /global/cfs/cdirs/e3sm/www/... +output_path=/global/cfs/cdirs/e3sm/www/username/zstash_performance/performance_20260603.png +``` + +The following options are available for finer control over the visualizer output: + +```ini +# Subset of HPSS modes to include in every figure (comma-separated). +# Valid values: none, hpss, globus. Leave blank to include all three. +# Example: hpss_filter=none,hpss +hpss_filter= + +# Top-level subdirectory for all output files. +# When set (together with most_recent_gen_run_id), figures are placed under +# //. +# When blank, the existing behaviour (stem of output_path as filename stem) is used. +viz_run_id=pr427_20260603 + +# Identifier for the most recent generate run; used as the filename stem for +# Figures 1 & 2 and as a subdirectory under viz_run_id/. +# Requires viz_run_id to also be set. +most_recent_gen_run_id=performance_20260603 + +# Which figures to produce (comma-separated). Valid values: 1, 2, 3, 4. +# Leave blank to produce all applicable figures. +# Note: Figure 2 still requires baseline_results_csv; Figures 3/4 still require +# performance_archive_dir, regardless of this setting. +figures=1,2,3,4 +``` + +When both `viz_run_id` and `most_recent_gen_run_id` are set, output files are laid out as: + +``` +//.png +//_vs_baseline.png +//record_create_and_update.png +//record_extract.png +``` + +Once you have the parameters set up, run: + +```bash +cd tests/performance/visualize/ +python visualize_performance.py --cfg my_run.cfg +``` + +If `--cfg` is omitted, the script looks for `perf.cfg` in the same directory. + +The script will print both the file path and the URL to access the plots. + +### Figures produced + +**Figure 1 – Performance overview** (`results_csv` required): A 2×2 grid of subplots (one per operation: create, update, extract_seq, extract_par) plus a 5th subplot comparing sequential vs parallel extract side-by-side. Bars represent HPSS mode (none / hpss / globus); individual data points are overlaid as dots when multiple test configs share the same directory. + +**Figure 2 – Baseline comparison** (`baseline_results_csv` required): Same layout as Figure 1, but each cell shows two bars (current = solid, baseline = hatched) with a current/baseline ratio annotation. Ratio > 1 indicates a regression (slower); ratio < 1 indicates an improvement (faster). + +**Figure 3 – Historical archive for create & update** (`performance_archive_dir` required): A 2×2 grid of time-series and box plots for create and update operations across all historical CSVs in the archive directory. + +**Figure 4 – Historical archive for extract** (`performance_archive_dir` required): Same layout as Figure 3, for extract_seq and extract_par operations. + +## For reference + +Records made before the long-term record space was made have been copied to it via: +```bash +SCRATCH_SPACE=/pscratch/sd/u/username/zstash_performance +RECORDS_SPACE=/global/homes/u/username/zstash_performance_records + +# Example: +for gen_run_id in \ + performance_20260225 \ + performance_20260226_pr402 \ + performance_20260226_pr424 \ + performance_20260226_pr428 \ + performance_20260402 \ + performance_20260414 \ + performance_pr416_20260403 \ + performance_pr416_20260406 +do + cp "${SCRATCH_SPACE}/${gen_run_id}/results.csv" "${RECORDS_SPACE}/${gen_run_id}_results.csv" +done +``` diff --git a/tests/performance/generate/developer_run.cfg b/tests/performance/generate/developer_run.cfg new file mode 100644 index 00000000..18395d7e --- /dev/null +++ b/tests/performance/generate/developer_run.cfg @@ -0,0 +1,19 @@ +# This version of perf.cfg has filled-in username paths. + +work_dir=/pscratch/sd/f/forsyth/zstash_performance/ +gen_run_id=performance_20260618 +environment_commands=source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh + +dir_to_copy_from=/global/cfs/cdirs/e3sm/forsyth/E3SMv2/v2.LR.historical_0201/ +subdir0=build/ +subdir1=run/ +subdir2=init/ + +HPSS_OPTIONS=none hpss globus +dst_hpss_path=/home/f/forsyth/zstash_performance + +fresh_globus=true +dst_endpoint_uuid=15288284-7006-4041-ba1a-6b52501e49f1 +dst_endpoint_archive_dir=/lcrc/group/e3sm/ac.forsyth2/zstash_performance_dst_dir/ + +performance_archive_dir=/global/homes/f/forsyth/zstash_performance_records diff --git a/tests/performance/generate/generate_performance_data.bash b/tests/performance/generate/generate_performance_data.bash new file mode 100755 index 00000000..349d9ce1 --- /dev/null +++ b/tests/performance/generate/generate_performance_data.bash @@ -0,0 +1,483 @@ +#!/bin/bash +set -e +set -o pipefail + +# Analogous to CI/CD matrix testing of Python versions, +# here we will do a matrix performance profiling +# by comparing runtimes for create/update/extract: +# - On multiple directories +# - With `--hpss=none`, with HPSS path, with Globus + +# We will also compare `zstash extract` in sequential-mode and parallel-mode + +############################################################################### +# Configuration file loader +# +# Reads a key=value file (default: perf.cfg in the same directory as this +# script, or the path given as the first argument). +# Lines starting with '#' and blank lines are ignored. +# Multi-word values (e.g. HPSS_OPTIONS) are stored as bash arrays. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CFG_FILE="${1:-${SCRIPT_DIR}/perf.cfg}" + +if [ ! -f "${CFG_FILE}" ]; then + echo "ERROR: config file not found: ${CFG_FILE}" + echo "Usage: $0 [path/to/config.cfg]" + echo "Copy ${SCRIPT_DIR}/perf.cfg and edit it for your run." + exit 1 +fi + +# Parse key=value pairs; skip comments and blank lines. +declare -A _cfg +while IFS='=' read -r key value; do + # Strip leading/trailing whitespace from key + key="${key//[[:space:]]/}" + # Strip leading whitespace from value + value="${value#"${value%%[![:space:]]*}"}" + # Strip trailing whitespace from value + value="${value%"${value##*[![:space:]]}"}" + [[ -z "$key" || "$key" == \#* ]] && continue + _cfg["$key"]="$value" +done < <(grep -v '^[[:space:]]*#' "${CFG_FILE}" | grep -v '^[[:space:]]*$') + +# Helper: get a required value or exit +cfg_require() { + local k="$1" + if [[ -z "${_cfg[$k]+_}" ]]; then + echo "ERROR: required key '${k}' is missing from ${CFG_FILE}" + exit 1 + fi + printf '%s' "${_cfg[$k]}" +} + +# Helper: get an optional value with a default +cfg_get() { + local k="$1" default="$2" + printf '%s' "${_cfg[$k]:-$default}" +} + +############################################################################### +# Load parameters from cfg file + +# Run from Perlmutter, so that we can do both +# a direct transfer to HPSS & a Globus transfer to Chrysalis +work_dir="$(cfg_require work_dir)" +work_dir="${work_dir%/}/" +gen_run_id="$(cfg_require gen_run_id)" +environment_commands="$(cfg_require environment_commands)" + +############################################################################### +# These parameters don't usually need to be changed, +# but can be changed for further customization. + +dir_to_copy_from="$(cfg_require dir_to_copy_from)" +dir_to_copy_from="${dir_to_copy_from%/}/" +subdir0="$(cfg_get subdir0 none)" +subdir1="$(cfg_get subdir1 none)" +subdir2="$(cfg_get subdir2 none)" +### +# For reference, these files have these sizes and number of files +# (Paths are from Chrysalis, but the data is identical on Perlmutter) + +# Analyzing: /lcrc/group/e3sm/ac.forsyth2/E3SMv2/v2.LR.historical_0201/build/ +# Total size: 1.2GiB +# Number of files: 7046 +# => Lots of small files + +# Analyzing: /lcrc/group/e3sm/ac.forsyth2/E3SMv2/v2.LR.historical_0201/run/ +# Total size: 11GiB +# Number of files: 111 + +# Analyzing: /lcrc/group/e3sm/ac.forsyth2/E3SMv2/v2.LR.historical_0201/init/ +# Total size: 6.9GiB +# Number of files: 14 +# => A few large files +### + +# For `--hpss=...` +# Which HPSS options to run. Space-separated in the cfg file -> bash array here. +# Options: "none" "hpss" "globus" +IFS=' ' read -r -a HPSS_OPTIONS <<< "$(cfg_require HPSS_OPTIONS)" + +dst_hpss_path="$(cfg_get dst_hpss_path "")" + +# For `--hpss=globus...` +fresh_globus="$(cfg_get fresh_globus false)" +# ENDPOINT UUIDS: +# LCRC_IMPROV_DTN_ENDPOINT=15288284-7006-4041-ba1a-6b52501e49f1 +# NERSC_PERLMUTTER_ENDPOINT=6bdc7956-fc0f-4ad2-989c-7aa5ee643a79 +# NERSC_HPSS_ENDPOINT=9cd89cfd-6d04-11e5-ba46-22000b92c6ec +# PIC_COMPY_DTN_ENDPOINT=68fbd2fa-83d7-11e9-8e63-029d279f7e24 +# GLOBUS_TUTORIAL_COLLECTION_1_ENDPOINT=6c54cade-bde5-45c1-bdea-f4bd71dba2cc +dst_endpoint_uuid="$(cfg_get dst_endpoint_uuid "")" +dst_endpoint_archive_dir="$(cfg_get dst_endpoint_archive_dir "")" + +performance_archive_dir="$(cfg_require performance_archive_dir)" + +echo "[INFO] Loaded configuration from: ${CFG_FILE}" +echo "[INFO] work_dir=${work_dir} gen_run_id=${gen_run_id}" + +############################################################################### +# Utility functions + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +NC='\033[0m' # No Color + +# Functions to print colored messages +print_step() { + echo -e "${CYAN}[STEP]${NC} $1" +} + +print_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +print_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +print_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +confirm() +{ + read -p "$1 (y/n): " -n 1 -r + echo + [[ $REPLY =~ ^[Yy]$ ]] +} + +validate_configuration() +{ + local dir_to_copy_from="${1}" + local subdir0="${2}" + local subdir1="${3}" + local subdir2="${4}" + + print_step "Validating configuration..." + + if [ ! -d "$dir_to_copy_from" ]; then + print_error "Source directory does not exist: $dir_to_copy_from" + exit 1 + fi + + if [ "$subdir0" != "none" ] && [ -n "$subdir0" ]; then + if [ ! -d "${dir_to_copy_from}${subdir0}" ]; then + print_error "subdir0 does not exist: ${dir_to_copy_from}${subdir0}" + exit 1 + fi + fi + + if [ "$subdir1" != "none" ] && [ -n "$subdir1" ]; then + if [ ! -d "${dir_to_copy_from}${subdir1}" ]; then + print_error "subdir1 does not exist: ${dir_to_copy_from}${subdir1}" + exit 1 + fi + fi + + if [ "$subdir2" != "none" ] && [ -n "$subdir2" ]; then + if [ ! -d "${dir_to_copy_from}${subdir2}" ]; then + print_error "subdir2 does not exist: ${dir_to_copy_from}${subdir2}" + exit 1 + fi + fi + + print_success "Configuration validated" +} + +refresh_globus() +{ + print_step "Setting up fresh Globus authentication..." + + INI_PATH="${HOME}/.zstash.ini" + TOKEN_FILE="${HOME}/.zstash_globus_tokens.json" + + if ! confirm "This will delete ${INI_PATH} and ${TOKEN_FILE} to start fresh. Is that ok?"; then + exit 1 + fi + + # 1. Activate endpoints + echo "Go to https://app.globus.org/file-manager?two_pane=true > For 'Collection', choose the endpoints you're using, and authenticate if needed:" + echo "LCRC Improv DTN, NERSC Perlmutter, NERSC HPSS, pic#compy-dtn" + if ! confirm "Have you authenticated into the correct endpoints?"; then + exit 1 + fi + + # 2. Reset authentication token files + if [ -f "${INI_PATH}" ]; then + rm -f "${INI_PATH}" + print_info "Removed ${INI_PATH}" + fi + + if [ -f "${TOKEN_FILE}" ]; then + rm -f "${TOKEN_FILE}" + print_info "Removed ${TOKEN_FILE}" + fi + + # 3. Reset Globus consents + echo "https://auth.globus.org/v2/web/consents > Globus Endpoint Performance Monitoring > rescind all" + if ! confirm "Have you revoked Globus consents?"; then + exit 1 + fi + + print_success "Globus authentication reset complete" +} + +# Parse the real-time (wall clock) seconds from the output of `time`. +# `time` writes to stderr a block like: +# real 1m23.456s +# user 0m12.345s +# sys 0m 1.234s +# We capture both stdout+stderr into the log, then grep for the real line. +parse_elapsed_seconds() +{ + local log_file="${1}" + # Extract "Xm Y.ZZZs" and convert to total seconds + awk '/^real/ { + split($2, a, "m"); + mins = a[1]; + secs = substr(a[2], 1, length(a[2])-1); + printf "%.3f\n", mins*60 + secs + }' "${log_file}" +} + +############################################################################### +# Core functions + +run_create() +{ + local dir_to_copy_from="${1}" + local subdir="${2}" + local archive_dir="${3}" + local hpss_path="${4}" + local cache_dir="${5}" + local create_log="${6}" + + print_step "Starting CREATE operation..." + + print_info "Copying data from ${dir_to_copy_from}${subdir}" + cp -r "${dir_to_copy_from}${subdir}" "${archive_dir}${subdir}" + + print_info "Running zstash create..." + print_info "Command: zstash create --hpss=${hpss_path} --cache=${cache_dir} -v ${archive_dir}" + + # We must be outside archive_dir when running create + if { time zstash create --hpss="${hpss_path}" --cache="${cache_dir}" -v "${archive_dir}" ; } 2>&1 | tee "${create_log}"; then + print_success "zstash create completed successfully" + else + print_error "zstash create failed with exit code $?" + exit 1 + fi +} + +run_update() +{ + local dir_to_copy_from="${1}" + local subdir="${2}" + local archive_dir="${3}" + local hpss_path="${4}" + local cache_dir="${5}" + local update_log="${6}" + + print_step "Starting UPDATE operation..." + + print_info "Copying additional data from ${dir_to_copy_from}${subdir}" + cp -r "${dir_to_copy_from}${subdir}" "${archive_dir}${subdir}" + + print_info "Running zstash update..." + print_info "Command: zstash update --hpss=${hpss_path} --cache=${cache_dir} -v" + + # zstash update must be run from within the archive directory + pushd "${archive_dir}" > /dev/null + if { time zstash update --hpss="${hpss_path}" --cache="${cache_dir}" -v ; } 2>&1 | tee "${update_log}"; then + print_success "zstash update completed successfully" + else + print_error "zstash update failed with exit code $?" + popd > /dev/null + exit 1 + fi + popd > /dev/null +} + +run_extract() +{ + local extract_dir="${1}" + local hpss_path="${2}" + local num_workers="${3}" + local cache_dir="${4}" + local extract_log="${5}" + + print_step "Starting EXTRACT operation (workers=${num_workers})..." + + print_info "Running zstash extract..." + print_info "Command: zstash extract --hpss=${hpss_path} --workers=${num_workers} --cache=${cache_dir} -v" + + # zstash extract must be run from within the extraction directory + pushd "${extract_dir}" > /dev/null + if { time zstash extract --hpss="${hpss_path}" --workers="${num_workers}" --cache="${cache_dir}" -v ; } 2>&1 | tee "${extract_log}"; then + print_success "zstash extract completed successfully" + else + print_error "zstash extract failed with exit code $?" + popd > /dev/null + exit 1 + fi + popd > /dev/null +} + +############################################################################### +# Results tracking + +# CSV file to collect all runtimes for later visualization +results_csv="${work_dir}${gen_run_id}/results.csv" + +record_result() +{ + local test_label="${1}" # e.g. "01" + local create_subdir="${2}" + local update_subdir="${3}" + local hpss_label="${4}" # "none", "hpss", "globus" + local operation="${5}" # "create", "update", "extract_seq", "extract_par" + local log_file="${6}" + + local elapsed + elapsed=$(parse_elapsed_seconds "${log_file}") + echo "${test_label},${create_subdir},${update_subdir},${hpss_label},${operation},${elapsed}" >> "${results_csv}" + print_info "Recorded: test=${test_label} op=${operation} hpss=${hpss_label} elapsed=${elapsed}s" +} + +############################################################################### +# Main script: + +# Make sure we're running from the correct environment. +# It might not necessarily be a dev environment built off this branch! +${environment_commands} + +validate_configuration "$dir_to_copy_from" "$subdir0" "$subdir1" "$subdir2" + +if [ "${fresh_globus}" == "true" ] && [[ " ${HPSS_OPTIONS[*]} " == *" globus "* ]]; then + refresh_globus +fi + +# Create the top-level results directory and CSV header +mkdir -p "${work_dir}${gen_run_id}" +echo "test_label,create_subdir,update_subdir,hpss_label,operation,elapsed_seconds" > "${results_csv}" +print_info "Results CSV: ${results_csv}" + +# Array of subdirectories (the test matrix below assumes all 3 are provided) +subdirs=("$subdir0" "$subdir1" "$subdir2") +for s in "${subdirs[@]}"; do + if [ -z "$s" ] || [ "$s" = "none" ]; then + print_error "subdir0/subdir1/subdir2 must be set (not 'none') in ${CFG_FILE}" + exit 1 + fi +done + +# Define the 6 possible permutations as test configurations. +# Each string contains two space-separated indices into the subdirs array: +# first index = subdir used for create +# second index = subdir used for update +declare -a test_configs=( + "0 1" + "0 2" + "1 0" + "1 2" + "2 0" + "2 1" +) +declare -a test_labels=("01" "02" "10" "12" "20" "21") + +# Loop through the 6 test configurations +for test_idx in 0 1 2 3 4 5; do + # Parse the configuration + read -r -a config <<< "${test_configs[$test_idx]}" + i=${config[0]} # index for create subdir + j=${config[1]} # index for update subdir + + # Get the subdirectories for this test + create_subdir="${subdirs[$i]}" + update_subdir="${subdirs[$j]}" + + test_label="${test_labels[$test_idx]}" + + print_step "==========================================" + print_step "Running Test ${test_label}" + print_step " Create subdir: $create_subdir" + print_step " Update subdir: $update_subdir" + print_step "==========================================" + + # Create unique work directories for this test + work_subdir="${work_dir}${gen_run_id}/test${test_label}/" + mkdir -p "${work_subdir}" + + log_dir="${work_subdir}logs/" + mkdir -p "${log_dir}" + + dst_globus_path="globus://${dst_endpoint_uuid}/${dst_endpoint_archive_dir}${gen_run_id}/test${test_label}/" + + # Iterate over the three HPSS modes + declare -A hpss_path_map=( + ["none"]="none" + ["hpss"]="${dst_hpss_path}" + ["globus"]="${dst_globus_path}" + ) + + for hpss_label in "${HPSS_OPTIONS[@]}"; do + hpss_path="${hpss_path_map[$hpss_label]}" + print_step "--- HPSS mode: ${hpss_label} (${hpss_path}) ---" + + # Each hpss mode gets its own subdirectories to avoid cross-contamination + mode_dir="${work_subdir}${hpss_label}/" + archive_dir="${mode_dir}archive_dir/" + cache_dir="${mode_dir}cache/" + mkdir -p "${archive_dir}" "${cache_dir}" + + create_log="${log_dir}create_${hpss_label}.log" + update_log="${log_dir}update_${hpss_label}.log" + + # --- CREATE --- + run_create "$dir_to_copy_from" "$create_subdir" "$archive_dir" "$hpss_path" "$cache_dir" "$create_log" + record_result "$test_label" "$create_subdir" "$update_subdir" "$hpss_label" "create" "$create_log" + + # --- UPDATE --- + run_update "$dir_to_copy_from" "$update_subdir" "$archive_dir" "$hpss_path" "$cache_dir" "$update_log" + record_result "$test_label" "$create_subdir" "$update_subdir" "$hpss_label" "update" "$update_log" + + # --- EXTRACT (sequential=1 worker, parallel=2 workers) --- + for num_workers in 1 2; do + extract_log="${log_dir}extract_${hpss_label}_${num_workers}workers.log" + extract_dir="${mode_dir}extract_${num_workers}workers/" + mkdir -p "${extract_dir}" + + run_extract "$extract_dir" "$hpss_path" "$num_workers" "$cache_dir" "$extract_log" + + if [ "$num_workers" -eq 1 ]; then + op_label="extract_seq" + else + op_label="extract_par" + fi + record_result "$test_label" "$create_subdir" "$update_subdir" "$hpss_label" "$op_label" "$extract_log" + done + done + + print_success "Test ${test_label} completed" + echo "" +done + +print_success "All tests completed. Results saved to: ${results_csv}" + +mkdir -p "${performance_archive_dir}" +performance_archive_path="${performance_archive_dir}/${gen_run_id}_results.csv" +cp "${results_csv}" "${performance_archive_path}" +print_success "Results copied to: ${performance_archive_path}" + +print_info "Now run: python ${SCRIPT_DIR}/../visualize/visualize_performance.py --cfg ${SCRIPT_DIR}/../visualize/perf.cfg (copy/edit this cfg to point at ${results_csv})" diff --git a/tests/performance/generate/perf.cfg b/tests/performance/generate/perf.cfg new file mode 100644 index 00000000..06a58485 --- /dev/null +++ b/tests/performance/generate/perf.cfg @@ -0,0 +1,77 @@ +# Performance profiling configuration – shared by both scripts: +# generate_performance_data.bash (pass as first argument, or name it perf.cfg) +# Usage: +# ./generate_performance_data.bash [path/to/this/file] +# Default cfg file name (when no argument given): perf.cfg +# +# NOTE: Perlmutter home/scratch paths follow the pattern: +# /global/homes/u/username/... +# /pscratch/sd/u/username/... +# where u is the first letter of your username, e.g. user "forsyth" goes under "f". + +# --------------------------------------------------------------------------- +# Run metadata <- edit these for each new run +# --------------------------------------------------------------------------- + +# Scratch directory where intermediate data and logs are written. +# Use /pscratch since a lot of data will be transferred. +work_dir=/pscratch/sd/u/username/zstash_performance/ + +# Unique identifier for this run (used as a sub-directory name and CSV prefix). +gen_run_id=performance_20260603 + +# Shell command(s) that activate the zstash environment. +# Separate multiple commands with ' ; ' (space-semicolon-space). +# Example - Unified environment: +# environment_commands=source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh +# Example - dev environment: +# environment_commands=source /global/homes/u/username/miniforge3/etc/profile.d/conda.sh ; conda activate zstash-pr427-20260603 +environment_commands=source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh + +# --------------------------------------------------------------------------- +# Data source <- usually no need to change these +# --------------------------------------------------------------------------- + +dir_to_copy_from=/global/cfs/cdirs/e3sm/forsyth/E3SMv2/v2.LR.historical_0201/ +subdir0=build/ +subdir1=run/ +subdir2=init/ + +# --------------------------------------------------------------------------- +# HPSS options +# --------------------------------------------------------------------------- + +# Space-separated list of HPSS modes to exercise. +# Valid values: none hpss globus +HPSS_OPTIONS=none hpss globus + +# Destination path on HPSS (used when hpss is in HPSS_OPTIONS). +dst_hpss_path=/home/u/username/zstash_performance + +# --------------------------------------------------------------------------- +# Globus options <- used when globus is in HPSS_OPTIONS +# --------------------------------------------------------------------------- + +# Set to true to force a fresh Globus authentication at the start of the run. +# NOTE: This will delete your ~/.zstash.ini & ~/.zstash_globus_tokens.json files. +fresh_globus=true + +# UUID of the destination Globus endpoint. +# Common endpoints: +# LCRC Improv DTN 15288284-7006-4041-ba1a-6b52501e49f1 +# NERSC Perlmutter 6bdc7956-fc0f-4ad2-989c-7aa5ee643a79 +# NERSC HPSS 9cd89cfd-6d04-11e5-ba46-22000b92c6ec +# PIC Compy DTN 68fbd2fa-83d7-11e9-8e63-029d279f7e24 +dst_endpoint_uuid=15288284-7006-4041-ba1a-6b52501e49f1 + +# Destination directory on the Globus endpoint. +dst_endpoint_archive_dir=/lcrc/group/e3sm/username/zstash_performance_dst_dir/ + +# --------------------------------------------------------------------------- +# Results archiving +# --------------------------------------------------------------------------- + +# Long-term (non-scratch) directory where the results CSV is copied at the end +# of a run. The file will be saved as: +# ${performance_archive_dir}/${gen_run_id}_results.csv +performance_archive_dir=/global/homes/u/username/zstash_performance_records diff --git a/tests/performance/visualize/developer_run.cfg b/tests/performance/visualize/developer_run.cfg new file mode 100644 index 00000000..9d5d2b17 --- /dev/null +++ b/tests/performance/visualize/developer_run.cfg @@ -0,0 +1,14 @@ +# This version of perf.cfg has filled-in username paths. + +performance_archive_dir=/global/homes/f/forsyth/zstash_performance_records + +results_csv=/pscratch/sd/f/forsyth/zstash_performance/performance_20260618/results.csv +baseline_results_csv=/pscratch/sd/f/forsyth/zstash_performance/performance_20260603/results.csv +output_path=/global/cfs/cdirs/e3sm/www/forsyth/zstash_performance/performance_20260618.png + +viz_run_id=viz_test_20260618_try3 +most_recent_gen_run_id=performance_20260618 + +hpss_filter=none,hpss,globus + +figures= diff --git a/tests/performance/visualize/perf.cfg b/tests/performance/visualize/perf.cfg new file mode 100644 index 00000000..17278fe4 --- /dev/null +++ b/tests/performance/visualize/perf.cfg @@ -0,0 +1,94 @@ +# Performance profiling configuration – shared by both scripts: +# visualize_performance.py (pass via --cfg, or name it perf.cfg) +# Usage: +# python visualize_performance.py [--cfg path/to/this/file] +# Default cfg file name (when no argument given): perf.cfg +# +# NOTE: Perlmutter home/scratch paths follow the pattern: +# /global/homes/u/username/... +# /pscratch/sd/u/username/... +# where u is the first letter of your username, e.g. user "forsyth" goes under "f". + +# --------------------------------------------------------------------------- +# Results archiving +# --------------------------------------------------------------------------- + +# Long-term (non-scratch) directory where results CSVs are copied at the end +# of a run. +performance_archive_dir=/global/homes/u/username/zstash_performance_records + +# --------------------------------------------------------------------------- +# visualize_performance.py options <- edit these for each new run +# --------------------------------------------------------------------------- + +# Path to the results CSV to show in Figure 1. +# This is the CSV produced by generate_performance_data.bash for this run. +results_csv=/pscratch/sd/u/username/zstash_performance/performance_20260603/results.csv + +# Path to a baseline results CSV to compare against in Figure 2. +# Leave blank (or comment out) to skip Figure 2. +baseline_results_csv=/pscratch/sd/u/username/zstash_performance/performance_20260414/results.csv + +# Output path for the saved figures. +# Leave blank (or comment out) to display interactively instead of saving. +# Make sure to use the web-server path so the URL is printed correctly, e.g.: +# /global/cfs/cdirs/e3sm/www/... +# When viz_run_id and most_recent_gen_run_id are both set, figures are saved +# under // using the filenames described below. +output_path=/global/cfs/cdirs/e3sm/www/username/zstash_performance/performance_pr427_20260603.png + +# --------------------------------------------------------------------------- +# Output filename control +# --------------------------------------------------------------------------- + +# Short identifier used as a top-level subdirectory for all output figures. +# When set together with most_recent_gen_run_id, figures are written to +# // +# and named as follows: +# .png +# _vs_baseline.png +# record_create_and_update.png +# record_extract.png +# Leave blank (or comment out) to use the stem of output_path (original behaviour). +# Example: viz_run_id=pr427_20260603 +viz_run_id= + +# Identifier for the most recent generate_performance_data.bash run. +# Used as the filename stem for Figures 1 & 2 and as a subdirectory under +# viz_run_id/. Requires viz_run_id to also be set; ignored when viz_run_id +# is blank. +# Example: most_recent_gen_run_id=performance_20260603 +most_recent_gen_run_id= + +# --------------------------------------------------------------------------- +# HPSS mode filter +# --------------------------------------------------------------------------- + +# Comma-separated list of HPSS modes to include in every figure. +# Valid values: none, hpss, globus (case-insensitive, any order). +# Leave blank (or comment out) to include all three modes. +# Examples: +# hpss_filter=none # local-only runs, no HPSS bars +# hpss_filter=none,hpss # skip globus +# hpss_filter=hpss,globus # skip the no-HPSS baseline +hpss_filter= + +# --------------------------------------------------------------------------- +# Figure selection +# --------------------------------------------------------------------------- + +# Comma-separated list of figure numbers to produce. +# Valid values: 1, 2, 3, 4 +# 1 = Performance overview (always available) +# 2 = Baseline comparison (requires baseline_results_csv) +# 3 = Archive: create & update (requires performance_archive_dir) +# 4 = Archive: extract_seq & extract_par (requires performance_archive_dir) +# Leave blank (or comment out) to produce all applicable figures. +# Note: listing a figure number does not bypass its data requirements — +# the data must still be present. Figures whose data is missing are skipped +# with a warning regardless of this setting. +# Examples: +# figures=1 # overview only, fast +# figures=1,2 # overview + baseline comparison +# figures=3,4 # archive history only +figures= diff --git a/tests/performance/visualize/visualize_performance.py b/tests/performance/visualize/visualize_performance.py new file mode 100644 index 00000000..eb7d8c3e --- /dev/null +++ b/tests/performance/visualize/visualize_performance.py @@ -0,0 +1,1855 @@ +#!/usr/bin/env python3 +""" +visualize_performance.py – Plot zstash performance profiling results. + +Usage: + python visualize_performance.py [--cfg path/to/perf.cfg] [--dpi 150] + +Pass --cfg (default: perf.cfg next to this script) instead of editing +hard-coded constants. The cfg file uses the same key=value *format* as +generate_performance_data.bash (see tests/performance/README.md for templates). + +The CSV is produced by generate_performance_data.bash and has columns: + test_label, create_subdir, update_subdir, hpss_label, operation, elapsed_seconds + +Visualization strategy +---------------------- +Four dimensions: + 1. Operation : create | update | extract_seq | extract_par + 2. Directory : build/ (many small) | run/ (medium) | init/ (few large) + 3. HPSS mode : none | hpss | globus + 4. Parallelism: already encoded in operation (extract_seq vs extract_par) + +Figure 1 – Performance overview: + Layout: 2×2 grid of subplots, one per operation. + Within each subplot: + - X-axis groups = directory processed (create_subdir or update_subdir) + for create/update; or (create_subdir, update_subdir) + archive config for extract_seq/extract_par. + - Bars = HPSS mode (none / hpss / globus), colour-coded + - Each test config contributes one bar per (directory, hpss_mode) cell; + if multiple configs share the same directory for an operation, their + runtimes are shown as individual dots and the bar shows the mean. + An additional 5th subplot compares extract_seq vs extract_par side-by-side + to make the parallelism speed-up immediately visible. + +Figure 2 – Baseline comparison (current branch vs main): + Produced only when baseline_results_csv is set to a valid path in the cfg + AND figure 2 is included in the figures list. + Same 2×2 + comparison layout, but each cell shows two bars + (current = solid, baseline = hatched) with a ratio annotation + (current/baseline) above each pair. Ratio > 1 = regression (slower), + ratio < 1 = improvement (faster). + +Figure 3 – Full record archive for create & update + (all historical CSVs in performance_archive_dir): + Produced only when performance_archive_dir is set in the cfg and contains + *results*.csv files with YYYYMMDD in their names, AND figure 3 is included + in the figures list. + Layout: 2×2 grid (create | update) × (time-series | box plot). + Time-series: x = record date, y = runtime, color = hpss mode, + line style = subdir (solid=build, dashed=run, dotted=init). + Box plots: vertical box-and-whisker for each (subdir, hpss) combination, + with individual data-point dots overlaid. + +Figure 4 – Full record archive for extract_seq & extract_par: + Produced only when performance_archive_dir is set and contains extract data, + AND figure 4 is included in the figures list. + Layout: 2×2 grid (extract_seq | extract_par) × (time-series | box plot). + X-axis groups for box plots: (create_subdir, update_subdir) archive config pairs. + Same color/line-style encoding as Figure 3. + +Outlier removal +--------------- +All plotting functions apply IQR-based outlier filtering before computing +means or drawing boxes/lines. Values outside + [Q1 - 1.5 * IQR, Q3 + 1.5 * IQR] +are dropped silently. This prevents a single aberrant run from dominating +axis scales while preserving legitimate spread. + +New cfg options +--------------- +hpss_filter + Comma-separated list of HPSS modes to include in every figure. + Valid values: none, hpss, globus (case-insensitive). + Leave blank or omit to include all three. + Example: hpss_filter=none,hpss + +viz_run_id + A short identifier string used as a top-level subdirectory for all output + files. When set (together with most_recent_gen_run_id), figures are placed + under //. + When blank, the existing behaviour (stem of output_path as filename stem, + parent directory of output_path as output directory) is used. + Example: viz_run_id=pr427_20260603 + +most_recent_gen_run_id + Identifier for the most recent generate_performance_data.bash run. + Used as the filename stem for Figures 1 & 2 and as a subdirectory under + viz_run_id/. Requires viz_run_id to also be set. + Example: most_recent_gen_run_id=performance_20260603 + +Output file layout when both viz_run_id and most_recent_gen_run_id are set: + //.png + //_vs_baseline.png + //record_create_and_update.png + //record_extract.png + +figures + Comma-separated list of figure numbers to produce. + Valid values: 1, 2, 3, 4. + Leave blank or omit to produce all applicable figures. + Specifying a figure number does not override data requirements: + Figure 2 still needs baseline_results_csv; Figures 3/4 still need + performance_archive_dir. But figures NOT in this list are skipped + entirely (data is not even loaded for them). + Example: figures=1,2 +""" + +import argparse +import configparser +import datetime +import os +import re +import sys +from collections import Counter +from pathlib import Path +from typing import Optional + +import matplotlib.dates +import matplotlib.lines +import matplotlib.patches as mpatches +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd + +# --------------------------------------------------------------------------- +# Cfg-file helpers +# --------------------------------------------------------------------------- + +_SCRIPT_DIR = Path(__file__).parent + + +def _load_cfg(cfg_path: Path) -> dict: + """ + Parse a key=value cfg file (same format used by generate_performance_data.bash). + Returns a plain dict. Section headers are not required; if present they are + ignored so the same file can be shared between the bash script and this one. + """ + # configparser needs at least one section header; inject a fake one. + text = "[run]\n" + cfg_path.read_text() + cp = configparser.ConfigParser( + inline_comment_prefixes=("#",), + strict=False, + ) + cp.read_string(text) + return dict(cp["run"]) + + +def _cfg_optional(cfg: dict, key: str) -> Optional[str]: + """Return the value for *key*, or None if missing / blank.""" + v = cfg.get(key, "").strip() + return v if v else None + + +def _cfg_require(cfg: dict, key: str, cfg_path: Path) -> str: + v = _cfg_optional(cfg, key) + if v is None: + print( + f"ERROR: required key '{key}' is missing from {cfg_path}", + file=sys.stderr, + ) + sys.exit(1) + return v + + +# --------------------------------------------------------------------------- +# New cfg option parsers +# --------------------------------------------------------------------------- + +_ALL_HPSS = ["none", "hpss", "globus"] +_ALL_FIGURES = {1, 2, 3, 4} + + +def _parse_hpss_filter(raw: Optional[str]) -> list: + """ + Parse the hpss_filter cfg value into an ordered list of HPSS mode strings. + + Validates each token against the known set. Preserves the canonical order + (none → hpss → globus) regardless of the order given in the cfg. Returns + the full list when *raw* is None or blank. + """ + if not raw: + return list(_ALL_HPSS) + tokens = [t.strip().lower() for t in raw.split(",") if t.strip()] + invalid = [t for t in tokens if t not in _ALL_HPSS] + if invalid: + print( + f"ERROR: hpss_filter contains unknown mode(s): {invalid}\n" + f" Valid values: {_ALL_HPSS}", + file=sys.stderr, + ) + sys.exit(1) + # Return in canonical order so plots are always consistent. + return [h for h in _ALL_HPSS if h in tokens] + + +def _parse_figures(raw: Optional[str]) -> set: + """ + Parse the figures cfg value into a set of integer figure numbers. + + Validates each token. Returns the full set {1,2,3,4} when *raw* is None + or blank. + """ + if not raw: + return set(_ALL_FIGURES) + tokens = [t.strip() for t in raw.split(",") if t.strip()] + result = set() + for t in tokens: + if not t.isdigit() or int(t) not in _ALL_FIGURES: + print( + f"ERROR: figures contains invalid value: {t!r}\n" + f" Valid values: 1, 2, 3, 4", + file=sys.stderr, + ) + sys.exit(1) + result.add(int(t)) + return result + + +# --------------------------------------------------------------------------- +# Config (styling – not user-configurable) +# --------------------------------------------------------------------------- + +# HPSS_ORDER and HPSS_COLORS / HPSS_LABELS remain the full canonical sets. +# The active subset selected by hpss_filter is stored in the module-level +# variable ACTIVE_HPSS, set once in main() before any plotting begins. +HPSS_ORDER = ["none", "hpss", "globus"] +HPSS_COLORS = {"none": "#4C72B0", "hpss": "#DD8452", "globus": "#55A868"} +HPSS_LABELS = {"none": "No HPSS", "hpss": "Direct HPSS", "globus": "Globus"} + +# Module-level active HPSS list; overwritten in main() from cfg. +# All plotting functions reference ACTIVE_HPSS instead of HPSS_ORDER directly +# so a single assignment here propagates everywhere. +ACTIVE_HPSS: list = list(HPSS_ORDER) + +OP_ORDER = ["create", "update", "extract_seq", "extract_par"] +OP_TITLES = { + "create": "zstash create", + "update": "zstash update", + "extract_seq": "zstash extract (sequential, 1 worker)", + "extract_par": "zstash extract (parallel, 2 workers)", +} + +# Map an operation to the column that holds the "relevant directory". +# Extract is intentionally absent: it operates on the combined create+update +# archive, so both subdirs are needed and it is handled separately. +OP_DIR_COL = { + "create": "create_subdir", + "update": "update_subdir", +} + +BAR_WIDTH = 0.22 +DOT_ALPHA = 0.55 +DOT_SIZE = 40 + +# --------------------------------------------------------------------------- +# Figure 3/4 – per-subdir line styles (encode which directory is plotted) +# --------------------------------------------------------------------------- +# build/ = many small files → solid +# run/ = mixed → dashed +# init/ = few large files → dotted +SUBDIR_LINESTYLES: dict[str, str] = { + "build": "solid", + "run": "dashed", + "init": "dotted", +} +SUBDIR_ORDER = ["build", "run", "init"] + + +# --------------------------------------------------------------------------- +# Outlier removal +# --------------------------------------------------------------------------- + + +def remove_outliers_iqr(vals: np.ndarray, k: float = 1.5) -> np.ndarray: + """ + Return a copy of *vals* with IQR-based outliers removed. + + Values outside [Q1 - k*IQR, Q3 + k*IQR] are dropped. + Returns the original array unchanged when it has fewer than 4 elements + (too few to estimate quartiles reliably). + """ + if len(vals) < 4: + return vals + q1, q3 = np.percentile(vals, [25, 75]) + iqr = q3 - q1 + lo = q1 - k * iqr + hi = q3 + k * iqr + return vals[(vals >= lo) & (vals <= hi)] + + +def _filter_df_outliers(df: pd.DataFrame, group_cols: list) -> pd.DataFrame: + """ + Apply IQR outlier removal to elapsed_seconds within each group defined + by *group_cols*. Returns a new DataFrame with outlier rows dropped. + Duplicate values that survive the IQR filter are all retained; only values + that fall outside the fence are removed. + """ + keep = [] + for _, grp in df.groupby(group_cols, dropna=False): + vals = grp["elapsed_seconds"].dropna().values + clean = remove_outliers_iqr(vals) + clean_counts = Counter(clean.tolist()) + used: Counter = Counter() + row_mask = [] + for v in grp["elapsed_seconds"]: + if pd.isna(v): + row_mask.append(False) + continue + if used[v] < clean_counts[v]: + row_mask.append(True) + used[v] += 1 + else: + row_mask.append(False) + keep.append(grp[row_mask]) + if not keep: + return df.iloc[0:0] + return pd.concat(keep, ignore_index=True) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def load_data(csv_path: str) -> pd.DataFrame: + df = pd.read_csv(csv_path) + df.columns = df.columns.str.strip() + df["elapsed_seconds"] = pd.to_numeric(df["elapsed_seconds"], errors="coerce") + # Normalise subdir names: strip trailing slashes for display + for col in ("create_subdir", "update_subdir"): + df[col] = df[col].str.strip().str.rstrip("/") + df["hpss_label"] = df["hpss_label"].str.strip() + df["operation"] = df["operation"].str.strip() + return df + + +def dir_sort_key(name: str) -> int: + """Sort dirs in a consistent order: build, run, init.""" + order = {"build": 0, "run": 1, "init": 2} + return order.get(name.lower(), 99) + + +def _add_dir_annotation(ax, dirs, x_positions): + """ + Add a small file-count hint below each directory group label. + + Parameters + ---------- + ax : the Axes to annotate + dirs : list of directory names in display order + x_positions : list of x-axis data coordinates for each dir group centre. + These are passed in explicitly so the function works for both + Fig. 1 (groups at 0, 1, 2, …) and Fig. 2 (wider group_span). + """ + hints = { + "build": "many small files\n(~7k files, 1.2 GiB)", + "run": "mixed\n(~111 files, 11 GiB)", + "init": "few large files\n(14 files, 6.9 GiB)", + } + for x_centre, d in zip(x_positions, dirs): + if d in hints: + ax.annotate( + hints[d], + xy=(x_centre, 0), + xycoords=("data", "axes fraction"), + xytext=(0, -46), + textcoords="offset points", + ha="center", + va="top", + fontsize=6.5, + color="#555555", + annotation_clip=False, + ) + + +def plot_operation(ax, df_op: pd.DataFrame, operation: str, dirs: list): + """Draw grouped bars for one operation subplot (outliers removed).""" + dir_col = OP_DIR_COL[operation] + df_op = _filter_df_outliers(df_op.copy(), [dir_col, "hpss_label"]) + + n_dirs = len(dirs) + n_hpss = len(ACTIVE_HPSS) + + x_base = np.arange(n_dirs) + offsets = np.linspace(-(n_hpss - 1) / 2, (n_hpss - 1) / 2, n_hpss) * BAR_WIDTH + + for h_idx, hpss in enumerate(ACTIVE_HPSS): + df_h = df_op[df_op["hpss_label"] == hpss] + means, all_vals, xs = [], [], [] + + for d_idx, d in enumerate(dirs): + vals = df_h[df_h[dir_col] == d]["elapsed_seconds"].dropna().values + mean = vals.mean() if len(vals) > 0 else 0.0 + means.append(mean) + all_vals.append(vals) + xs.append(x_base[d_idx] + offsets[h_idx]) + + color = HPSS_COLORS[hpss] + ax.bar( + xs, + means, + width=BAR_WIDTH, + color=color, + alpha=0.85, + label=HPSS_LABELS[hpss], + zorder=2, + ) + # Overlay individual data points so scatter is visible + for x_pos, vals in zip(xs, all_vals): + if len(vals) > 1: + jitter = np.random.uniform( + -BAR_WIDTH * 0.25, BAR_WIDTH * 0.25, size=len(vals) + ) + ax.scatter( + x_pos + jitter, + vals, + color="white", + edgecolors=color, + s=DOT_SIZE, + zorder=3, + alpha=DOT_ALPHA, + linewidths=1.2, + ) + + ax.set_title(OP_TITLES[operation], fontsize=10, fontweight="bold", pad=6) + ax.set_xticks(x_base) + ax.set_xticklabels([d + "/" for d in dirs], fontsize=9) + ax.set_ylabel("Wall-clock time (s)", fontsize=8) + ax.set_xlabel("Directory processed", fontsize=8, labelpad=14) + ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) + ax.set_axisbelow(True) + _add_dir_annotation(ax, dirs, list(x_base)) + + # Value labels on bars + for rect in ax.patches: + h = rect.get_height() + if h > 0: + ax.text( + rect.get_x() + rect.get_width() / 2, + h * 1.01, + f"{h:.0f}s", + ha="center", + va="bottom", + fontsize=6, + color="#333333", + ) + + +def _extract_configs(df: pd.DataFrame) -> list: + """ + Return the sorted list of (create_subdir, update_subdir) pairs that + actually appear in the extract rows of *df*. These represent the + combined archives that were extracted from. + """ + mask = df["operation"].isin(["extract_seq", "extract_par"]) + pairs = ( + df[mask][["create_subdir", "update_subdir"]] + .drop_duplicates() + .apply(tuple, axis=1) + .tolist() + ) + return sorted(pairs, key=lambda p: (dir_sort_key(p[0]), dir_sort_key(p[1]))) + + +def _extract_tick_label(create_sub: str, update_sub: str) -> str: + """Short two-line tick label for a (create, update) archive config.""" + return f"create: {create_sub}/\nupdate: {update_sub}/" + + +def _plot_extract_single_op(ax, df: pd.DataFrame, operation: str): + """ + Draw grouped bars for one extract operation (extract_seq or extract_par). + Outliers removed per (create_subdir, update_subdir, hpss_label) group. + """ + df_op = df[df["operation"] == operation].copy() + df_op = _filter_df_outliers(df_op, ["create_subdir", "update_subdir", "hpss_label"]) + + configs = _extract_configs(df) + n_configs = len(configs) + n_hpss = len(ACTIVE_HPSS) + + x_base = np.arange(n_configs, dtype=float) + offsets = np.linspace(-(n_hpss - 1) / 2, (n_hpss - 1) / 2, n_hpss) * BAR_WIDTH + + for h_idx, hpss in enumerate(ACTIVE_HPSS): + means, all_vals, xs = [], [], [] + for c_idx, (create_sub, update_sub) in enumerate(configs): + vals = ( + df_op[ + (df_op["hpss_label"] == hpss) + & (df_op["create_subdir"] == create_sub) + & (df_op["update_subdir"] == update_sub) + ]["elapsed_seconds"] + .dropna() + .values + ) + mean = vals.mean() if len(vals) > 0 else 0.0 + means.append(mean) + all_vals.append(vals) + xs.append(x_base[c_idx] + offsets[h_idx]) + + color = HPSS_COLORS[hpss] + ax.bar( + xs, + means, + width=BAR_WIDTH, + color=color, + alpha=0.85, + label=HPSS_LABELS[hpss], + zorder=2, + ) + for x_pos, vals in zip(xs, all_vals): + if len(vals) > 1: + jitter = np.random.uniform( + -BAR_WIDTH * 0.25, BAR_WIDTH * 0.25, size=len(vals) + ) + ax.scatter( + x_pos + jitter, + vals, + color="white", + edgecolors=color, + s=DOT_SIZE, + zorder=3, + alpha=DOT_ALPHA, + linewidths=1.2, + ) + + ax.set_title(OP_TITLES[operation], fontsize=10, fontweight="bold", pad=6) + ax.set_xticks(x_base) + ax.set_xticklabels([_extract_tick_label(c, u) for c, u in configs], fontsize=7) + ax.set_ylabel("Wall-clock time (s)", fontsize=8) + ax.set_xlabel("Archive contents (create → update)", fontsize=8, labelpad=6) + ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) + ax.set_axisbelow(True) + + for rect in ax.patches: + h = rect.get_height() + if h > 0: + ax.text( + rect.get_x() + rect.get_width() / 2, + h * 1.01, + f"{h:.0f}s", + ha="center", + va="bottom", + fontsize=6, + color="#333333", + ) + + +def plot_extract_comparison(ax, df: pd.DataFrame): + """ + Extra subplot: sequential vs parallel extract, grouped by (archive config, hpss). + Outliers removed per (operation, create_subdir, update_subdir, hpss_label). + """ + df_ext = df[df["operation"].isin(["extract_seq", "extract_par"])].copy() + df_ext = _filter_df_outliers( + df_ext, ["operation", "create_subdir", "update_subdir", "hpss_label"] + ) + + configs = _extract_configs(df) + n_configs = len(configs) + ops = ["extract_seq", "extract_par"] + hatches = {"extract_seq": "", "extract_par": "////"} + n_bars = len(ACTIVE_HPSS) * len(ops) + + group_width = n_bars * BAR_WIDTH + 0.15 + x_base = np.arange(n_configs) * group_width + + for c_idx, (create_sub, update_sub) in enumerate(configs): + for h_idx, hpss in enumerate(ACTIVE_HPSS): + for op_idx, op in enumerate(ops): + df_cell = df_ext[ + (df_ext["operation"] == op) + & (df_ext["hpss_label"] == hpss) + & (df_ext["create_subdir"] == create_sub) + & (df_ext["update_subdir"] == update_sub) + ] + vals = df_cell["elapsed_seconds"].dropna().values + mean = vals.mean() if len(vals) > 0 else 0.0 + bar_x = x_base[c_idx] + (h_idx * len(ops) + op_idx) * BAR_WIDTH + ax.bar( + bar_x, + mean, + width=BAR_WIDTH, + color=HPSS_COLORS[hpss], + hatch=hatches[op], + alpha=0.85, + zorder=2, + ) + + tick_positions = x_base + (n_bars / 2 - 0.5) * BAR_WIDTH + ax.set_xticks(tick_positions) + ax.set_xticklabels([_extract_tick_label(c, u) for c, u in configs], fontsize=7.5) + ax.set_ylabel("Wall-clock time (s)", fontsize=8) + ax.set_xlabel( + "Archive contents (create subdir → update subdir)", fontsize=8, labelpad=14 + ) + ax.set_title( + "Extract: Sequential vs Parallel (speed-up comparison)\n" + "Each group = archive built from create subdir + update subdir", + fontsize=10, + fontweight="bold", + pad=6, + ) + ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) + ax.set_axisbelow(True) + + hpss_patches = [ + mpatches.Patch(color=HPSS_COLORS[h], label=HPSS_LABELS[h]) for h in ACTIVE_HPSS + ] + seq_patch = mpatches.Patch( + facecolor="grey", hatch="", label="Sequential (1 worker)" + ) + par_patch = mpatches.Patch( + facecolor="grey", hatch="////", label="Parallel (2 workers)" + ) + ax.legend( + handles=hpss_patches + [seq_patch, par_patch], + fontsize=7, + loc="upper right", + ncol=2, + ) + + +# --------------------------------------------------------------------------- +# Baseline comparison figure +# --------------------------------------------------------------------------- + +RATIO_REGRESSION = 1.10 +RATIO_IMPROVEMENT = 0.90 +RATIO_NEUTRAL_COLOR = "#333333" +RATIO_REGRESSION_COLOR = "#CC3311" +RATIO_IMPROVEMENT_COLOR = "#228833" + + +def _ratio_color(ratio: float) -> str: + if ratio >= RATIO_REGRESSION: + return RATIO_REGRESSION_COLOR + if ratio <= RATIO_IMPROVEMENT: + return RATIO_IMPROVEMENT_COLOR + return RATIO_NEUTRAL_COLOR + + +def plot_comparison_operation( + ax, + df_cur: pd.DataFrame, + df_bas: pd.DataFrame, + operation: str, + dirs: list, +): + """Paired bars (current vs baseline) per (directory, hpss) cell. Outliers removed.""" + dir_col = OP_DIR_COL[operation] + df_cur = _filter_df_outliers( + df_cur[df_cur["operation"] == operation].copy(), [dir_col, "hpss_label"] + ) + df_bas = _filter_df_outliers( + df_bas[df_bas["operation"] == operation].copy(), [dir_col, "hpss_label"] + ) + + n_dirs = len(dirs) + n_hpss = len(ACTIVE_HPSS) + + pair_width = BAR_WIDTH + gap = BAR_WIDTH * 0.3 + group_span = n_hpss * (2 * pair_width + gap) + 0.2 + x_base = np.arange(n_dirs) * group_span + + for h_idx, hpss in enumerate(ACTIVE_HPSS): + color = HPSS_COLORS[hpss] + pair_offset = h_idx * (2 * pair_width + gap) + + for d_idx, d in enumerate(dirs): + x_left = x_base[d_idx] + pair_offset + x_right = x_base[d_idx] + pair_offset + pair_width + + def mean_for(df, _h=hpss, _d=d): + v = ( + df[(df["hpss_label"] == _h) & (df[dir_col] == _d)][ + "elapsed_seconds" + ] + .dropna() + .values + ) + return v.mean() if len(v) > 0 else 0.0 + + cur_mean = mean_for(df_cur) + bas_mean = mean_for(df_bas) + + ax.bar( + x_left, + bas_mean, + width=pair_width, + color=color, + alpha=0.40, + hatch="////", + zorder=2, + edgecolor=color, + ) + ax.bar( + x_right, + cur_mean, + width=pair_width, + color=color, + alpha=0.85, + zorder=2, + label=HPSS_LABELS[hpss] if d_idx == 0 else "", + ) + + if bas_mean > 0 and cur_mean > 0: + ratio = cur_mean / bas_mean + top = max(cur_mean, bas_mean) + rat_color = _ratio_color(ratio) + arrow = ( + "▲" + if ratio >= RATIO_REGRESSION + else ("▼" if ratio <= RATIO_IMPROVEMENT else "") + ) + ax.text( + (x_left + x_right) / 2, + top * 1.03, + f"{arrow}{ratio:.2f}×", + ha="center", + va="bottom", + fontsize=6.5, + fontweight="bold", + color=rat_color, + zorder=4, + ) + + ax.set_title(OP_TITLES[operation], fontsize=10, fontweight="bold", pad=6) + group_centre_offset = (n_hpss * (2 * pair_width + gap) - gap) / 2 + x_ticks = x_base + group_centre_offset + ax.set_xticks(x_ticks) + ax.set_xticklabels([d + "/" for d in dirs], fontsize=9) + ax.set_ylabel("Wall-clock time (s)", fontsize=8) + ax.set_xlabel("Directory processed", fontsize=8, labelpad=14) + ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) + ax.set_axisbelow(True) + _add_dir_annotation(ax, dirs, list(x_ticks)) + + +def _plot_comparison_extract_single_op( + ax, + df_cur: pd.DataFrame, + df_bas: pd.DataFrame, + operation: str, +): + """Fig. 2 extract subplot: current vs baseline, outliers removed.""" + df_cur = _filter_df_outliers( + df_cur[df_cur["operation"] == operation].copy(), + ["create_subdir", "update_subdir", "hpss_label"], + ) + df_bas = _filter_df_outliers( + df_bas[df_bas["operation"] == operation].copy(), + ["create_subdir", "update_subdir", "hpss_label"], + ) + + configs = _extract_configs(df_cur) + n_configs = len(configs) + n_hpss = len(ACTIVE_HPSS) + + pair_width = BAR_WIDTH + gap = BAR_WIDTH * 0.3 + group_span = n_hpss * (2 * pair_width + gap) + 0.2 + x_base = np.arange(n_configs) * group_span + + for h_idx, hpss in enumerate(ACTIVE_HPSS): + color = HPSS_COLORS[hpss] + pair_offset = h_idx * (2 * pair_width + gap) + for c_idx, (create_sub, update_sub) in enumerate(configs): + x_left = x_base[c_idx] + pair_offset + x_right = x_left + pair_width + + def mean_for(df, _h=hpss, _cs=create_sub, _us=update_sub): + v = ( + df[ + (df["hpss_label"] == _h) + & (df["create_subdir"] == _cs) + & (df["update_subdir"] == _us) + ]["elapsed_seconds"] + .dropna() + .values + ) + return v.mean() if len(v) > 0 else 0.0 + + cur_mean = mean_for(df_cur) + bas_mean = mean_for(df_bas) + + ax.bar( + x_left, + bas_mean, + width=pair_width, + color=color, + alpha=0.40, + hatch="////", + zorder=2, + edgecolor=color, + ) + ax.bar( + x_right, + cur_mean, + width=pair_width, + color=color, + alpha=0.85, + zorder=2, + label=HPSS_LABELS[hpss] if c_idx == 0 else "", + ) + + if bas_mean > 0 and cur_mean > 0: + ratio = cur_mean / bas_mean + top = max(cur_mean, bas_mean) + arrow = ( + "▲" + if ratio >= RATIO_REGRESSION + else ("▼" if ratio <= RATIO_IMPROVEMENT else "") + ) + ax.text( + (x_left + x_right) / 2, + top * 1.03, + f"{arrow}{ratio:.2f}×", + ha="center", + va="bottom", + fontsize=6.5, + fontweight="bold", + color=_ratio_color(ratio), + zorder=4, + ) + + group_centre_offset = (n_hpss * (2 * pair_width + gap) - gap) / 2 + x_ticks = x_base + group_centre_offset + ax.set_xticks(x_ticks) + ax.set_xticklabels([_extract_tick_label(c, u) for c, u in configs], fontsize=7) + ax.set_ylabel("Wall-clock time (s)", fontsize=8) + ax.set_xlabel("Archive contents (create → update)", fontsize=8, labelpad=6) + ax.set_title(OP_TITLES[operation], fontsize=10, fontweight="bold", pad=6) + ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) + ax.set_axisbelow(True) + + +def plot_comparison_extract(ax, df_cur: pd.DataFrame, df_bas: pd.DataFrame): + """Seq vs par extract, current vs baseline. Outliers removed per group.""" + df_cur = _filter_df_outliers( + df_cur[df_cur["operation"].isin(["extract_seq", "extract_par"])].copy(), + ["operation", "create_subdir", "update_subdir", "hpss_label"], + ) + df_bas = _filter_df_outliers( + df_bas[df_bas["operation"].isin(["extract_seq", "extract_par"])].copy(), + ["operation", "create_subdir", "update_subdir", "hpss_label"], + ) + + configs = _extract_configs(df_cur) + n_configs = len(configs) + ops = ["extract_seq", "extract_par"] + op_hatches = {"extract_seq": "", "extract_par": "xxxx"} + + pair_width = BAR_WIDTH + inner_gap = BAR_WIDTH * 0.15 + op_gap = BAR_WIDTH * 0.55 + hpss_gap = BAR_WIDTH * 0.30 + + pair_span = 2 * pair_width + inner_gap + hpss_group_span = 2 * pair_span + op_gap + + group_span = len(ACTIVE_HPSS) * (hpss_group_span + hpss_gap) + 0.3 + x_base = np.arange(n_configs) * group_span + + for c_idx, (create_sub, update_sub) in enumerate(configs): + for h_idx, hpss in enumerate(ACTIVE_HPSS): + color = HPSS_COLORS[hpss] + hpss_origin = x_base[c_idx] + h_idx * (hpss_group_span + hpss_gap) + for op_idx, op in enumerate(ops): + hatch = op_hatches[op] + op_origin = hpss_origin + op_idx * (pair_span + op_gap) + x_bas = op_origin + x_cur = op_origin + pair_width + inner_gap + + def mean_for(df, _op=op, _h=hpss, _cs=create_sub, _us=update_sub): + v = ( + df[ + (df["operation"] == _op) + & (df["hpss_label"] == _h) + & (df["create_subdir"] == _cs) + & (df["update_subdir"] == _us) + ]["elapsed_seconds"] + .dropna() + .values + ) + return v.mean() if len(v) > 0 else 0.0 + + cur_mean = mean_for(df_cur) + bas_mean = mean_for(df_bas) + + bas_hatch = hatch + "////" + ax.bar( + x_bas, + bas_mean, + width=pair_width, + color=color, + hatch=bas_hatch, + alpha=0.35, + zorder=2, + edgecolor=color, + ) + ax.bar( + x_cur, + cur_mean, + width=pair_width, + color=color, + hatch=hatch, + alpha=0.85, + zorder=2, + ) + + if bas_mean > 0 and cur_mean > 0: + ratio = cur_mean / bas_mean + top = max(cur_mean, bas_mean) + arrow = ( + "▲" + if ratio >= RATIO_REGRESSION + else ("▼" if ratio <= RATIO_IMPROVEMENT else "") + ) + ax.text( + (x_cur + x_bas) / 2, + top * 1.03, + f"{arrow}{ratio:.2f}×", + ha="center", + va="bottom", + fontsize=5.5, + fontweight="bold", + color=_ratio_color(ratio), + zorder=4, + ) + + group_total_bar_span = len(ACTIVE_HPSS) * (hpss_group_span + hpss_gap) - hpss_gap + x_ticks = x_base + group_total_bar_span / 2 + ax.set_xticks(x_ticks) + ax.set_xticklabels([_extract_tick_label(c, u) for c, u in configs], fontsize=7.5) + ax.set_ylabel("Wall-clock time (s)", fontsize=8) + ax.set_xlabel( + "Archive contents (create subdir → update subdir)", fontsize=8, labelpad=14 + ) + ax.set_title( + "Extract: Sequential vs Parallel — current vs baseline\n" + "Each group = archive built from create subdir + update subdir", + fontsize=10, + fontweight="bold", + pad=6, + ) + ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) + ax.set_axisbelow(True) + + hpss_patches = [ + mpatches.Patch(color=HPSS_COLORS[h], label=HPSS_LABELS[h]) for h in ACTIVE_HPSS + ] + seq_patch = mpatches.Patch( + facecolor="grey", hatch="", alpha=0.85, label="Sequential, current" + ) + seq_bas_patch = mpatches.Patch( + facecolor="grey", hatch="////", alpha=0.35, label="Sequential, baseline" + ) + par_patch = mpatches.Patch( + facecolor="grey", hatch="xxxx", alpha=0.85, label="Parallel, current" + ) + par_bas_patch = mpatches.Patch( + facecolor="grey", hatch="xxxx////", alpha=0.35, label="Parallel, baseline" + ) + ax.legend( + handles=hpss_patches + [seq_patch, seq_bas_patch, par_patch, par_bas_patch], + fontsize=6.5, + loc="upper right", + ncol=3, + ) + + +def build_comparison_figure( + df_cur: pd.DataFrame, + df_bas: pd.DataFrame, + all_dirs: list, + cur_label: str, + bas_label: str, +) -> plt.Figure: + """Build and return the full baseline-comparison figure (Figure 2).""" + fig = plt.figure(figsize=(16, 17)) + fig.suptitle( + f"zstash Performance: Current vs Baseline\n" + f"current = {cur_label} | baseline (main) = {bas_label}\n" + f"Ratio = current / baseline — " + f"▲ {RATIO_REGRESSION_COLOR_LABEL} ≥{RATIO_REGRESSION:.0%} slower " + f"▼ {RATIO_IMPROVEMENT_COLOR_LABEL} ≤{RATIO_IMPROVEMENT:.0%} faster " + f"= within ±10%", + fontsize=11, + fontweight="bold", + y=0.98, + ) + + gs = fig.add_gridspec( + 3, 2, hspace=0.58, wspace=0.35, top=0.92, bottom=0.07, left=0.07, right=0.97 + ) + axes = { + "create": fig.add_subplot(gs[0, 0]), + "update": fig.add_subplot(gs[0, 1]), + "extract_seq": fig.add_subplot(gs[1, 0]), + "extract_par": fig.add_subplot(gs[1, 1]), + } + ax_cmp = fig.add_subplot(gs[2, :]) + + for op in OP_ORDER: + if op in OP_DIR_COL: + plot_comparison_operation(axes[op], df_cur, df_bas, op, all_dirs) + else: + _plot_comparison_extract_single_op(axes[op], df_cur, df_bas, op) + + cur_patch = mpatches.Patch(facecolor="grey", alpha=0.85, label="Current branch") + bas_patch = mpatches.Patch( + facecolor="grey", alpha=0.40, hatch="////", label="Baseline (main)" + ) + hpss_patches = [ + mpatches.Patch(color=HPSS_COLORS[h], label=HPSS_LABELS[h]) for h in ACTIVE_HPSS + ] + axes["create"].legend( + handles=[cur_patch, bas_patch] + hpss_patches, fontsize=7, loc="upper right" + ) + + plot_comparison_extract(ax_cmp, df_cur, df_bas) + return fig + + +RATIO_REGRESSION_COLOR_LABEL = "red" +RATIO_IMPROVEMENT_COLOR_LABEL = "green" + + +# --------------------------------------------------------------------------- +# Archive data loading +# --------------------------------------------------------------------------- + + +def _archive_date_from_path(csv_path: Path) -> Optional[datetime.date]: + """Parse YYYYMMDD from a CSV filename; return None if not found.""" + m = re.search(r"(\d{8})", csv_path.stem) + if not m: + return None + try: + s = m.group(1) + return datetime.date(int(s[:4]), int(s[4:6]), int(s[6:])) + except ValueError: + return None + + +def load_archive_data(archive_dir: str) -> pd.DataFrame: + """ + Load and concatenate every ``*results*.csv`` in *archive_dir*. + Adds a ``record_date`` column (pandas Timestamp) from the filename. + """ + _empty = pd.DataFrame( + columns=[ + "test_label", + "create_subdir", + "update_subdir", + "hpss_label", + "operation", + "elapsed_seconds", + "record_date", + ] + ) + archive_path = Path(archive_dir) + if not archive_path.is_dir(): + print( + f"WARNING: performance_archive_dir not found: {archive_path}", + file=sys.stderr, + ) + return _empty + + csv_files = sorted(archive_path.glob("*results*.csv")) + if not csv_files: + print( + f"WARNING: no *results*.csv files found in {archive_path}", file=sys.stderr + ) + return _empty + + frames = [] + for p in csv_files: + record_date = _archive_date_from_path(p) + if record_date is None: + print( + f"WARNING: cannot parse date from {p.name!r}, skipping.", + file=sys.stderr, + ) + continue + try: + df_i = load_data(str(p)) + except Exception as exc: + print(f"WARNING: failed to load {p}: {exc}", file=sys.stderr) + continue + df_i["record_date"] = record_date + frames.append(df_i) + + if not frames: + return _empty + + df_all = pd.concat(frames, ignore_index=True) + df_all["record_date"] = pd.to_datetime(df_all["record_date"]) + return df_all + + +# --------------------------------------------------------------------------- +# Figure 3 – archive: create & update +# --------------------------------------------------------------------------- + + +def plot_archive_timeseries(ax, df_arch: pd.DataFrame, operation: str) -> None: + """ + Time-series for create/update over the full archive. + Outliers removed within each (date, subdir, hpss) group before aggregating. + Color = hpss mode; line style = subdir. + """ + dir_col = OP_DIR_COL[operation] + df_op = df_arch[df_arch["operation"] == operation].copy() + df_op = _filter_df_outliers(df_op, ["record_date", dir_col, "hpss_label"]) + + for hpss in ACTIVE_HPSS: + color = HPSS_COLORS[hpss] + for subdir in SUBDIR_ORDER: + ls = SUBDIR_LINESTYLES.get(subdir, "solid") + mask = (df_op["hpss_label"] == hpss) & (df_op[dir_col] == subdir) + df_line = ( + df_op[mask] + .groupby("record_date")["elapsed_seconds"] + .mean() + .reset_index() + .sort_values("record_date") + ) + if df_line.empty: + continue + ax.plot( + df_line["record_date"], + df_line["elapsed_seconds"], + color=color, + linestyle=ls, + linewidth=1.6, + marker="o", + markersize=4, + label=f"{HPSS_LABELS[hpss]} – {subdir}/", + zorder=3, + ) + + ax.set_title( + f"zstash {operation} – runtime over time", + fontsize=10, + fontweight="bold", + pad=6, + ) + ax.set_xlabel("Record date", fontsize=8) + ax.set_ylabel("Wall-clock time (s)", fontsize=8) + ax.xaxis.set_major_formatter(matplotlib.dates.DateFormatter("%Y-%m-%d")) + plt.setp(ax.get_xticklabels(), rotation=30, ha="right", fontsize=7) + ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) + ax.set_axisbelow(True) + + color_handles = [ + matplotlib.lines.Line2D( + [], [], color=HPSS_COLORS[h], linewidth=2, label=HPSS_LABELS[h] + ) + for h in ACTIVE_HPSS + ] + style_handles = [ + matplotlib.lines.Line2D( + [], + [], + color="grey", + linewidth=2, + linestyle=SUBDIR_LINESTYLES[s], + label=f"{s}/", + ) + for s in SUBDIR_ORDER + ] + ax.legend( + handles=color_handles + style_handles, + fontsize=6.5, + loc="upper left", + ncol=2, + framealpha=0.8, + ) + + +def plot_archive_boxplot(ax, df_arch: pd.DataFrame, operation: str) -> None: + """ + Box-and-whisker for create/update across the full archive. + Outliers removed per (subdir, hpss) group before drawing. + """ + dir_col = OP_DIR_COL[operation] + df_op = df_arch[df_arch["operation"] == operation].copy() + df_op = _filter_df_outliers(df_op, [dir_col, "hpss_label"]) + + n_hpss = len(ACTIVE_HPSS) + group_width = n_hpss * BAR_WIDTH + 0.10 + x_base = np.arange(len(SUBDIR_ORDER)) * group_width + offsets = np.linspace(0, (n_hpss - 1) * BAR_WIDTH, n_hpss) + + tick_positions, tick_labels = [], [] + + for s_idx, subdir in enumerate(SUBDIR_ORDER): + tick_positions.append(x_base[s_idx] + offsets.mean()) + tick_labels.append(f"{subdir}/") + + for h_idx, hpss in enumerate(ACTIVE_HPSS): + mask = (df_op["hpss_label"] == hpss) & (df_op[dir_col] == subdir) + vals = df_op[mask]["elapsed_seconds"].dropna().values + x_pos = x_base[s_idx] + offsets[h_idx] + if len(vals) == 0: + continue + color = HPSS_COLORS[hpss] + ax.boxplot( + vals, + positions=[x_pos], + widths=BAR_WIDTH * 0.85, + patch_artist=True, + orientation="vertical", + manage_ticks=False, + zorder=2, + boxprops=dict(facecolor=color, alpha=0.55, linewidth=0.8), + medianprops=dict(color="black", linewidth=1.5), + whiskerprops=dict(linewidth=0.8), + capprops=dict(linewidth=0.8), + flierprops=dict(marker="", linestyle="none"), + ) + jitter = np.random.uniform( + -BAR_WIDTH * 0.2, BAR_WIDTH * 0.2, size=len(vals) + ) + ax.scatter( + x_pos + jitter, + vals, + color="white", + edgecolors=color, + s=DOT_SIZE, + zorder=3, + alpha=DOT_ALPHA, + linewidths=1.2, + ) + + ax.set_xticks(tick_positions) + ax.set_xticklabels(tick_labels, fontsize=9) + ax.set_title( + f"zstash {operation} – runtime distribution (all records)", + fontsize=10, + fontweight="bold", + pad=6, + ) + ax.set_xlabel("Directory processed", fontsize=8) + ax.set_ylabel("Wall-clock time (s)", fontsize=8) + ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) + ax.set_axisbelow(True) + hpss_patches = [ + mpatches.Patch(color=HPSS_COLORS[h], alpha=0.75, label=HPSS_LABELS[h]) + for h in ACTIVE_HPSS + ] + ax.legend(handles=hpss_patches, fontsize=7, loc="upper right") + + +def build_archive_figure(df_arch: pd.DataFrame) -> plt.Figure: + """ + Figure 3 – full archive overview for create & update. + + Layout (2 rows × 2 cols): + [0,0] create time-series | [0,1] update time-series + [1,0] create box plot | [1,1] update box plot + """ + fig = plt.figure(figsize=(15, 12)) + fig.suptitle( + "zstash Performance – Full Record Archive (create & update)\n" + "Time series: color = HPSS mode · line style = directory " + "(solid = build/, dashed = run/, dotted = init/)\n" + "Box plots: every recorded runtime per (directory, HPSS) combination\n" + "Outliers removed via IQR method before plotting", + fontsize=11, + fontweight="bold", + y=0.995, + ) + gs = fig.add_gridspec( + 2, 2, hspace=0.48, wspace=0.30, top=0.90, bottom=0.08, left=0.07, right=0.97 + ) + for col_idx, op in enumerate(["create", "update"]): + plot_archive_timeseries(fig.add_subplot(gs[0, col_idx]), df_arch, op) + plot_archive_boxplot(fig.add_subplot(gs[1, col_idx]), df_arch, op) + return fig + + +# --------------------------------------------------------------------------- +# Figure 4 – archive: extract_seq & extract_par +# --------------------------------------------------------------------------- + + +def plot_extract_archive_timeseries(ax, df_arch: pd.DataFrame, operation: str) -> None: + """ + Time-series for an extract operation over the full archive. + + Since extract has no single directory column, lines are keyed by the + combined (create_subdir, update_subdir) archive config pair. + Color = hpss mode; line style = create_subdir. + Outliers removed within each (date, create_subdir, update_subdir, hpss) group. + """ + df_op = df_arch[df_arch["operation"] == operation].copy() + df_op = _filter_df_outliers( + df_op, ["record_date", "create_subdir", "update_subdir", "hpss_label"] + ) + + all_pairs = sorted( + df_op[["create_subdir", "update_subdir"]] + .drop_duplicates() + .apply(tuple, axis=1) + .tolist(), + key=lambda p: (dir_sort_key(p[0]), dir_sort_key(p[1])), + ) + + markers = ["o", "s", "^", "D", "v", "P", "X", "*", "h"] + + for hpss in ACTIVE_HPSS: + color = HPSS_COLORS[hpss] + for p_idx, (create_sub, update_sub) in enumerate(all_pairs): + ls = SUBDIR_LINESTYLES.get(create_sub, "solid") + marker = markers[p_idx % len(markers)] + mask = ( + (df_op["hpss_label"] == hpss) + & (df_op["create_subdir"] == create_sub) + & (df_op["update_subdir"] == update_sub) + ) + df_line = ( + df_op[mask] + .groupby("record_date")["elapsed_seconds"] + .mean() + .reset_index() + .sort_values("record_date") + ) + if df_line.empty: + continue + ax.plot( + df_line["record_date"], + df_line["elapsed_seconds"], + color=color, + linestyle=ls, + linewidth=1.6, + marker=marker, + markersize=4, + label=f"{HPSS_LABELS[hpss]} – create:{create_sub}/ update:{update_sub}/", + zorder=3, + ) + + ax.set_title( + f"{OP_TITLES[operation]} – runtime over time", + fontsize=10, + fontweight="bold", + pad=6, + ) + ax.set_xlabel("Record date", fontsize=8) + ax.set_ylabel("Wall-clock time (s)", fontsize=8) + ax.xaxis.set_major_formatter(matplotlib.dates.DateFormatter("%Y-%m-%d")) + plt.setp(ax.get_xticklabels(), rotation=30, ha="right", fontsize=7) + ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) + ax.set_axisbelow(True) + + color_handles = [ + matplotlib.lines.Line2D( + [], [], color=HPSS_COLORS[h], linewidth=2, label=HPSS_LABELS[h] + ) + for h in ACTIVE_HPSS + ] + style_handles = [ + matplotlib.lines.Line2D( + [], + [], + color="grey", + linewidth=2, + linestyle=SUBDIR_LINESTYLES.get(s, "solid"), + label=f"create: {s}/", + ) + for s in SUBDIR_ORDER + if any(p[0] == s for p in all_pairs) + ] + ax.legend( + handles=color_handles + style_handles, + fontsize=6.5, + loc="upper left", + ncol=2, + framealpha=0.8, + ) + + +def plot_extract_archive_boxplot(ax, df_arch: pd.DataFrame, operation: str) -> None: + """ + Box-and-whisker for an extract operation across the full archive. + + X-axis groups = (create_subdir, update_subdir) archive config pairs + (matching the x-axis used in Figures 1 and 2). + Within each group the three HPSS modes sit side by side. + Outliers removed per (archive config pair, hpss_label) group. + """ + df_op = df_arch[df_arch["operation"] == operation].copy() + df_op = _filter_df_outliers(df_op, ["create_subdir", "update_subdir", "hpss_label"]) + + all_pairs = sorted( + df_op[["create_subdir", "update_subdir"]] + .drop_duplicates() + .apply(tuple, axis=1) + .tolist(), + key=lambda p: (dir_sort_key(p[0]), dir_sort_key(p[1])), + ) + + n_hpss = len(ACTIVE_HPSS) + group_width = n_hpss * BAR_WIDTH + 0.10 + x_base = np.arange(len(all_pairs)) * group_width + offsets = np.linspace(0, (n_hpss - 1) * BAR_WIDTH, n_hpss) + + tick_positions, tick_labels = [], [] + + for p_idx, (create_sub, update_sub) in enumerate(all_pairs): + tick_positions.append(x_base[p_idx] + offsets.mean()) + tick_labels.append(_extract_tick_label(create_sub, update_sub)) + + for h_idx, hpss in enumerate(ACTIVE_HPSS): + mask = ( + (df_op["hpss_label"] == hpss) + & (df_op["create_subdir"] == create_sub) + & (df_op["update_subdir"] == update_sub) + ) + vals = df_op[mask]["elapsed_seconds"].dropna().values + x_pos = x_base[p_idx] + offsets[h_idx] + if len(vals) == 0: + continue + color = HPSS_COLORS[hpss] + ax.boxplot( + vals, + positions=[x_pos], + widths=BAR_WIDTH * 0.85, + patch_artist=True, + orientation="vertical", + manage_ticks=False, + zorder=2, + boxprops=dict(facecolor=color, alpha=0.55, linewidth=0.8), + medianprops=dict(color="black", linewidth=1.5), + whiskerprops=dict(linewidth=0.8), + capprops=dict(linewidth=0.8), + flierprops=dict(marker="", linestyle="none"), + ) + jitter = np.random.uniform( + -BAR_WIDTH * 0.2, BAR_WIDTH * 0.2, size=len(vals) + ) + ax.scatter( + x_pos + jitter, + vals, + color="white", + edgecolors=color, + s=DOT_SIZE, + zorder=3, + alpha=DOT_ALPHA, + linewidths=1.2, + ) + + ax.set_xticks(tick_positions) + ax.set_xticklabels(tick_labels, fontsize=7) + ax.set_title( + f"{OP_TITLES[operation]} – runtime distribution (all records)", + fontsize=10, + fontweight="bold", + pad=6, + ) + ax.set_xlabel("Archive contents (create → update)", fontsize=8) + ax.set_ylabel("Wall-clock time (s)", fontsize=8) + ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) + ax.set_axisbelow(True) + hpss_patches = [ + mpatches.Patch(color=HPSS_COLORS[h], alpha=0.75, label=HPSS_LABELS[h]) + for h in ACTIVE_HPSS + ] + ax.legend(handles=hpss_patches, fontsize=7, loc="upper right") + + +def build_extract_archive_figure(df_arch: pd.DataFrame) -> plt.Figure: + """ + Figure 4 – full archive overview for extract_seq & extract_par. + + Layout (2 rows × 2 cols): + [0,0] extract_seq time-series | [0,1] extract_par time-series + [1,0] extract_seq box plot | [1,1] extract_par box plot + + Only produced when the archive contains extract operation rows. + """ + fig = plt.figure(figsize=(15, 12)) + fig.suptitle( + "zstash Performance – Full Record Archive (extract_seq & extract_par)\n" + "Time series: color = HPSS mode · line style = create_subdir " + "(solid = build/, dashed = run/, dotted = init/)\n" + "Box plots: every recorded runtime per (archive config, HPSS) combination\n" + "Outliers removed via IQR method before plotting", + fontsize=11, + fontweight="bold", + y=0.995, + ) + gs = fig.add_gridspec( + 2, 2, hspace=0.55, wspace=0.32, top=0.90, bottom=0.10, left=0.07, right=0.97 + ) + for col_idx, op in enumerate(["extract_seq", "extract_par"]): + plot_extract_archive_timeseries(fig.add_subplot(gs[0, col_idx]), df_arch, op) + plot_extract_archive_boxplot(fig.add_subplot(gs[1, col_idx]), df_arch, op) + return fig + + +# --------------------------------------------------------------------------- +# Main – helpers +# --------------------------------------------------------------------------- + + +def _parse_args(): + parser = argparse.ArgumentParser( + description="Visualise zstash performance results." + ) + parser.add_argument( + "--cfg", + default=str(_SCRIPT_DIR / "perf.cfg"), + help="Path to the key=value config file (default: perf.cfg next to this script).", + ) + parser.add_argument( + "--dpi", type=int, default=150, help="Output DPI (default: 150)" + ) + return parser.parse_args() + + +def _load_results(results_csv: str) -> pd.DataFrame: + """Load and validate the primary results CSV; exit on error.""" + results_path = Path(results_csv) + if not results_path.is_file(): + print(f"ERROR: results_csv not found: {results_csv!r}", file=sys.stderr) + sys.exit(1) + df = load_data(str(results_path)) + if df.empty: + print( + f"ERROR: results_csv empty or unparseable: {results_csv!r}", file=sys.stderr + ) + sys.exit(1) + return df + + +def build_overview_figure(df: pd.DataFrame, all_dirs: list) -> plt.Figure: + """ + Figure 1 – performance overview. + + Layout: 3 rows × 2 cols + Row 0: create | update + Row 1: extract_seq | extract_par + Row 2: extract seq-vs-par comparison (spans both columns) + """ + fig = plt.figure(figsize=(15, 16)) + fig.suptitle( + "zstash Performance Profiling\n" + "(bars = mean over test configs; dots = individual runs; " + "outliers removed via IQR)", + fontsize=13, + fontweight="bold", + y=0.98, + ) + gs = fig.add_gridspec( + 3, 2, hspace=0.55, wspace=0.35, top=0.93, bottom=0.07, left=0.07, right=0.97 + ) + axes = { + "create": fig.add_subplot(gs[0, 0]), + "update": fig.add_subplot(gs[0, 1]), + "extract_seq": fig.add_subplot(gs[1, 0]), + "extract_par": fig.add_subplot(gs[1, 1]), + } + ax_cmp = fig.add_subplot(gs[2, :]) + + legend_handles = None + for op in OP_ORDER: + ax = axes[op] + if op in OP_DIR_COL: + plot_operation(ax, df[df["operation"] == op], op, all_dirs) + else: + _plot_extract_single_op(ax, df, op) + if legend_handles is None: + legend_handles = [ + mpatches.Patch(color=HPSS_COLORS[h], label=HPSS_LABELS[h]) + for h in ACTIVE_HPSS + ] + ax.legend(handles=legend_handles, fontsize=7, loc="upper right") + + plot_extract_comparison(ax_cmp, df) + return fig + + +def _try_build_comparison_figure( + df: pd.DataFrame, + all_dirs: list, + results_csv: str, + baseline_results_csv: Optional[str], +) -> Optional[plt.Figure]: + """Figure 2 – baseline comparison. Returns None when not applicable.""" + if not baseline_results_csv: + print("INFO: baseline_results_csv not set; skipping Figure 2.", file=sys.stderr) + return None + bas_path = Path(baseline_results_csv) + if not bas_path.exists(): + print(f"WARNING: baseline_results_csv not found: {bas_path}", file=sys.stderr) + print("Skipping baseline comparison figure.", file=sys.stderr) + return None + df_bas = load_data(str(bas_path)) + bas_label = bas_path.parent.name + cur_label = Path(results_csv).parent.name + return build_comparison_figure(df, df_bas, all_dirs, cur_label, bas_label) + + +def _try_build_archive_figures( + archive_dir: Optional[str], + want_fig3: bool, + want_fig4: bool, +) -> tuple: + """ + Figures 3 & 4 – full record archive. + + *want_fig3* and *want_fig4* gate whether each figure is actually built. + Returns a (fig_arch, fig_arch_extract) tuple; either element may be None + when the corresponding data is unavailable or the figure was not requested. + """ + if not archive_dir: + print( + "INFO: performance_archive_dir not set; skipping Figures 3 & 4.", + file=sys.stderr, + ) + return None, None + df_arch = load_archive_data(archive_dir) + if df_arch.empty: + print( + "WARNING: no archive data found; skipping Figures 3 & 4.", file=sys.stderr + ) + return None, None + + fig_arch = build_archive_figure(df_arch) if want_fig3 else None + if not want_fig3: + print("INFO: Figure 3 not in figures list; skipping.", file=sys.stderr) + + fig_arch_extract = None + if want_fig4: + has_extract = df_arch["operation"].isin(["extract_seq", "extract_par"]).any() + if has_extract: + fig_arch_extract = build_extract_archive_figure(df_arch) + else: + print( + "INFO: no extract data in archive; skipping Figure 4.", file=sys.stderr + ) + else: + print("INFO: Figure 4 not in figures list; skipping.", file=sys.stderr) + + return fig_arch, fig_arch_extract + + +def _output_paths( + output_path: str, + viz_run_id: Optional[str], + most_recent_gen_run_id: Optional[str], +) -> tuple: + """ + Return the four output file paths as a tuple: (fig1, fig2, fig3, fig4). + + When both *viz_run_id* and *most_recent_gen_run_id* are set, the new + directory-based layout is used: + //.png + //_vs_baseline.png + //record_create_and_update.png + //record_extract.png + + Otherwise the legacy flat layout is used (all files in the parent + directory of output_path, stem from viz_run_id or output_path): + /.png + /_vs_baseline.png + /_archive.png + /_archive_extract.png + """ + p = Path(output_path) + suffix = p.suffix or ".png" + + if viz_run_id and most_recent_gen_run_id: + out_dir = p.parent / viz_run_id + stem = most_recent_gen_run_id + return ( + str(out_dir / (stem + suffix)), + str(out_dir / (stem + "_vs_baseline" + suffix)), + str(out_dir / ("record_create_and_update" + suffix)), + str(out_dir / ("record_extract" + suffix)), + ) + + # Legacy behaviour: flat files in the parent directory of output_path. + out_dir = p.parent + stem = viz_run_id if viz_run_id else p.stem + return ( + str(out_dir / (stem + suffix)), + str(out_dir / (stem + "_vs_baseline" + suffix)), + str(out_dir / (stem + "_archive" + suffix)), + str(out_dir / (stem + "_archive_extract" + suffix)), + ) + + +def _save_figure(figure: plt.Figure, out_path_str: str, label: str, dpi: int) -> None: + """Save *figure* to *out_path_str* and print the destination.""" + out_path = Path(out_path_str) + out_path.parent.mkdir(parents=True, exist_ok=True) + try: + os.chmod(out_path.parent, 0o755) + except OSError: + pass + figure.savefig(out_path, dpi=dpi, bbox_inches="tight") + print(f"{label} saved to: {out_path}") + try: + os.chmod(out_path, 0o644) + except OSError: + pass + web_path = str(out_path).replace( + "/global/cfs/cdirs/e3sm/www/", + "https://portal.nersc.gov/cfs/e3sm/", + ) + print(f" Accessible at: {web_path}") + + +def _save_all_figures( + fig: Optional[plt.Figure], + fig_cmp: Optional[plt.Figure], + fig_arch: Optional[plt.Figure], + fig_arch_extract: Optional[plt.Figure], + output_path: str, + viz_run_id: Optional[str], + most_recent_gen_run_id: Optional[str], + dpi: int, +) -> None: + """ + Save every non-None figure using paths from _output_paths(). + """ + path1, path2, path3, path4 = _output_paths( + output_path, viz_run_id, most_recent_gen_run_id + ) + + if fig is not None: + _save_figure(fig, path1, "Figure 1 (overview)", dpi) + if fig_cmp is not None: + _save_figure(fig_cmp, path2, "Figure 2 (baseline comparison)", dpi) + if fig_arch is not None: + _save_figure(fig_arch, path3, "Figure 3 (full archive: create & update)", dpi) + if fig_arch_extract is not None: + _save_figure( + fig_arch_extract, + path4, + "Figure 4 (full archive: extract_seq & extract_par)", + dpi, + ) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def main(): + args = _parse_args() + + cfg_path = Path(args.cfg) + if not cfg_path.is_file(): + print(f"ERROR: config file not found: {cfg_path}", file=sys.stderr) + print( + f"Copy {_SCRIPT_DIR / 'perf.cfg'} and edit it for your run.", + file=sys.stderr, + ) + sys.exit(1) + + cfg = _load_cfg(cfg_path) + results_csv: str = _cfg_require(cfg, "results_csv", cfg_path) + baseline_results_csv: Optional[str] = _cfg_optional(cfg, "baseline_results_csv") + output_path: Optional[str] = _cfg_optional(cfg, "output_path") + archive_dir: Optional[str] = _cfg_optional(cfg, "performance_archive_dir") + + # --- New options -------------------------------------------------------- + # hpss_filter: subset of HPSS modes to plot (default: all three) + global ACTIVE_HPSS + ACTIVE_HPSS = _parse_hpss_filter(_cfg_optional(cfg, "hpss_filter")) + if ACTIVE_HPSS != HPSS_ORDER: + print( + f"INFO: hpss_filter active — plotting only: {ACTIVE_HPSS}", file=sys.stderr + ) + + # viz_run_id: top-level subdirectory for output files + viz_run_id: Optional[str] = _cfg_optional(cfg, "viz_run_id") + if viz_run_id: + print(f"INFO: viz_run_id = {viz_run_id!r}", file=sys.stderr) + + # most_recent_gen_run_id: stem for Figures 1 & 2, subdir under viz_run_id/ + most_recent_gen_run_id: Optional[str] = _cfg_optional(cfg, "most_recent_gen_run_id") + if most_recent_gen_run_id: + print( + f"INFO: most_recent_gen_run_id = {most_recent_gen_run_id!r}", + file=sys.stderr, + ) + if viz_run_id and not most_recent_gen_run_id: + print( + "WARNING: viz_run_id is set but most_recent_gen_run_id is not; " + "falling back to legacy flat filename layout.", + file=sys.stderr, + ) + + # figures: which figures to produce (default: all applicable) + figures_set: set = _parse_figures(_cfg_optional(cfg, "figures")) + if figures_set != _ALL_FIGURES: + print( + f"INFO: figures filter active — producing only: {sorted(figures_set)}", + file=sys.stderr, + ) + # ------------------------------------------------------------------------ + + df = _load_results(results_csv) + all_dirs = sorted( + set(df["create_subdir"].dropna()) | set(df["update_subdir"].dropna()), + key=dir_sort_key, + ) + + # Build only the requested figures. + fig = build_overview_figure(df, all_dirs) if 1 in figures_set else None + if 1 not in figures_set: + print("INFO: Figure 1 not in figures list; skipping.", file=sys.stderr) + + fig_cmp = ( + _try_build_comparison_figure(df, all_dirs, results_csv, baseline_results_csv) + if 2 in figures_set + else None + ) + if 2 not in figures_set: + print("INFO: Figure 2 not in figures list; skipping.", file=sys.stderr) + + fig_arch, fig_arch_extract = _try_build_archive_figures( + archive_dir, + want_fig3=3 in figures_set, + want_fig4=4 in figures_set, + ) + + if output_path: + _save_all_figures( + fig, + fig_cmp, + fig_arch, + fig_arch_extract, + output_path, + viz_run_id, + most_recent_gen_run_id, + args.dpi, + ) + else: + plt.show() + + +if __name__ == "__main__": + np.random.seed(42) # reproducible jitter + main()