From 53e5dc32ea38d26a0db32f0d941dede3d7e0f6de Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 25 Feb 2026 18:23:13 -0800 Subject: [PATCH 01/27] Add performance profiling as a standard zstash test --- .../generate_performance_data.bash | 330 ++++++++++++++++++ tests/performance/visualize_performance.py | 0 2 files changed, 330 insertions(+) create mode 100644 tests/performance/generate_performance_data.bash create mode 100644 tests/performance/visualize_performance.py diff --git a/tests/performance/generate_performance_data.bash b/tests/performance/generate_performance_data.bash new file mode 100644 index 00000000..54d7eeff --- /dev/null +++ b/tests/performance/generate_performance_data.bash @@ -0,0 +1,330 @@ +#!/bin/bash +set -e + +# Analagous to CI/CD matrix testing of Python versions, +# here we will do a matrix performance profiling +# by comparing runtimes for create/update/extract: +# - On multiple directories +# - With `--hpss=none` with HPSS path, with Globus + +# We will also compare `zstash extract` in sequential-mode and parallel-mode + +############################################################################### +# Manually edit parameters here: + +# Run from Perlmutter, so that we can do both +# a direct transfer to HPSS & a Globus transfer to Chrysalis +work_dir=/global/cfs/cdirs/e3sm/forsyth/zstash_performance/ +unique_id=performance_20260225 + +dir_to_copy_from=/global/cfs/cdirs/e3sm/forsyth/E3SMv2/v2.LR.historical_0201/ +subdir0=build/ +subdir1=run/ +subdir2=init/ +### +# For reference, these files have these sizes and number of files +# (Paths are from Chrysalis, but the data is identical on Perlmutter) + +# Analyzing: /lcrc/group/e3sm/ac.forsyth2/E3SMv2/v2.LR.historical_0201/build/ +# Total size: 1.2GiB +# Number of files: 7046 +# => Lots of small files + +# Analyzing: /lcrc/group/e3sm/ac.forsyth2/E3SMv2/v2.LR.historical_0201/run/ +# Total size: 11GiB +# Number of files: 111 + +# Analyzing: /lcrc/group/e3sm/ac.forsyth2/E3SMv2/v2.LR.historical_0201/init/ +# Total size: 6.9GiB +# Number of files: 14 +# => A few large files +### + + +# For `--hpss=...` +dst_hpss=/home/f/forsyth/zstash_performance + +# For `--hpss=globus...` +fresh_globus=true +# ENDPOINT UUIDS: +# LCRC_IMPROV_DTN_ENDPOINT=15288284-7006-4041-ba1a-6b52501e49f1 +# NERSC_PERLMUTTER_ENDPOINT=6bdc7956-fc0f-4ad2-989c-7aa5ee643a79 +# NERSC_HPSS_ENDPOINT=9cd89cfd-6d04-11e5-ba46-22000b92c6ec +# PIC_COMPY_DTN_ENDPOINT=68fbd2fa-83d7-11e9-8e63-029d279f7e24 +# GLOBUS_TUTORIAL_COLLECTION_1_ENDPOINT=6c54cade-bde5-45c1-bdea-f4bd71dba2cc +dst_endpoint_uuid=15288284-7006-4041-ba1a-6b52501e49f1 +dst_endpoint_archive_dir=/lcrc/group/e3sm/ac.forsyth2/zstash_performance_dst_dir/ + +############################################################################### +# Utility functions + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +NC='\033[0m' # No Color + +# Functions to print colored messages +print_step() { + echo -e "${CYAN}[STEP]${NC} $1" +} + +print_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +print_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +print_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +confirm() +{ + read -p "$1 (y/n): " -n 1 -r + echo + [[ $REPLY =~ ^[Yy]$ ]] +} + +valiate_configuration() +{ + local dir_to_copy_from="${1}" + local subdir0="${2}" + local subdir1="${3}" + local subdir2="${4}" + + print_step "Validating configuration..." + + if [ ! -d "$dir_to_copy_from" ]; then + print_error "Source directory does not exist: $dir_to_copy_from" + exit 1 + fi + + if [ "$subdir0" != "none" ] && [ -n "$subdir0" ]; then + if [ ! -d "${dir_to_copy_from}${subdir0}" ]; then + print_error "subdir0 does not exist: ${dir_to_copy_from}${subdir0}" + exit 1 + fi + fi + + if [ "$subdir1" != "none" ] && [ -n "$subdir1" ]; then + if [ ! -d "${dir_to_copy_from}${subdir1}" ]; then + print_error "subdir1 does not exist: ${dir_to_copy_from}${subdir1}" + exit 1 + fi + fi + + if [ "$subdir2" != "none" ] && [ -n "$subdir2" ]; then + if [ ! -d "${dir_to_copy_from}${subdir2}" ]; then + print_error "subdir2 does not exist: ${dir_to_copy_from}${subdir2}" + exit 1 + fi + fi + + print_success "Configuration validated" +} + +refresh_globus() +{ + print_step "Setting up fresh Globus authentication..." + + # 1. Activate endpoints + echo "Go to https://app.globus.org/file-manager?two_pane=true > For 'Collection', choose the endpoints you're using, and authenticate if needed:" + echo "LCRC Improv DTN, NERSC Perlmutter, NERSC HPSS, pic#compy-dtn" + if ! confirm "Have you authenticated into the correct endpoints?"; then + exit 1 + fi + + # 2. Reset authentication token files + INI_PATH=${HOME}/.zstash.ini + TOKEN_FILE=${HOME}/.zstash_globus_tokens.json + + if [ -f "${INI_PATH}" ]; then + rm -f "${INI_PATH}" + print_info "Removed ${INI_PATH}" + fi + + if [ -f "${TOKEN_FILE}" ]; then + rm -f "${TOKEN_FILE}" + print_info "Removed ${TOKEN_FILE}" + fi + + # 3. Reset Globus consents + echo "https://auth.globus.org/v2/web/consents > Globus Endpoint Performance Monitoring > rescind all" + if ! confirm "Have you revoked Globus consents?"; then + exit 1 + fi + + print_success "Globus authentication reset complete" +} + +############################################################################### +# Core functions + +run_create() +{ + local dir_to_copy_from="${1}" + local subdir="${2}" + local archive_dir="${3}" + local dst_hpss="${4}" + local cache_dir="${5}" + local create_log="${6}" + print_step "Starting CREATE operation..." + + print_info "Copying data from ${dir_to_copy_from}${subdir}" + cp -r "${dir_to_copy_from}${subdir}" "${archive_dir}${subdir}" + + print_info "Running zstash create..." + print_info "Command: zstash create --hpss=${dst_hpss} --cache=${cache_dir} -v ${archive_dir}" + + if { time zstash create --hpss="${dst_hpss}" --cache="${cache_dir}" -v "${archive_dir}" ; } 2>&1 | tee "${create_log}"; then + print_success "zstash create completed successfully" + else + print_error "zstash create failed with exit code $?" + exit 1 + fi +} + +run_update() +{ + local dir_to_copy_from="${1}" + local subdir="${2}" + local archive_dir="${3}" + local dst_hpss="${4}" + local cache_dir="${5}" + local update_log="${6}" + + print_step "Starting UPDATE operation..." + + print_info "Copying additional data from ${dir_to_copy_from}${subdir}" + cp -r "${dir_to_copy_from}${subdir}" "${archive_dir}${subdir}" + + print_info "Running zstash update..." + print_info "Command: zstash update --hpss=${dst_hpss} --cache=${cache_dir} -v" + + if { time zstash update --hpss="${dst_hpss}" --cache="${cache_dir}" -v ; } 2>&1 | tee "${update_log}"; then + print_success "zstash update completed successfully" + else + print_error "zstash update failed with exit code $?" + exit 1 + fi +} + +run_extract() +{ + local archive_dir="${1}" + local src_hpss="${2}" + local num_workers="${3}" + local cache_dir="${4}" + local extract_log="${5}" + + print_step "Starting EXTRACT operation..." + + print_info "Running zstash extract..." + print_info "Command: zstash extract --hpss=${src_hpss} --workers=${num_workers} --cache=${cache_dir} -v" + + if { time zstash extract --hpss="${src_hpss}" --workers="${num_workers}" --cache="${cache_dir}" -v ; } 2>&1 | tee "${extract_log}"; then + print_success "zstash extract completed successfully" + else + print_error "zstash extract failed with exit code $?" + exit 1 + fi +} + +############################################################################### +# Main script: + +valiate_configuration $dir_to_copy_from $subdir0 $subdir1 $subdir2 + +if [ "${fresh_globus}" == "true" ]; then + refresh_globus +fi + +# Array of subdirectories +subdirs=("$subdir0" "$subdir1" "$subdir2") +# Define the 6 possible permutations as test configurations +# Each array contains indices into the subdirs array +declare -a test_configs=( + "0 1" + "0 2" + "1 0" + "1 2" + "2 0" + "2 1" +) +declare -a test_labels=("01" "02" "10", "12", "20", "21") +declare -a test_names + +# Loop through the 6 test configurations +for test_idx in 0 1 2 3 4 5; do + # Parse the configuration + config=(${test_configs[$test_idx]}) + i=${config[0]} # 1st element (create-subdir) + j=${config[1]} # 2nd element (update-subdir) + + # Get the subdirectories for this test + create_subdir="${subdirs[$i]}" + update_subdir="${subdirs[$j]}" + + # Create a label for this test + test_label="${test_labels[$test_idx]}" + test_names+=("Test_${test_label}") + + print_step "==========================================" + print_step "Running Test ${test_label}" + print_step " Create: $create_subdir" + print_step " Update: $update_subdir" + print_step "==========================================" + + # Create unique work directories for this test + dst_endpoint_archive_subdir="${dst_endpoint_archive_dir}${unique_id}/test${test_label}/" + work_subdir="${work_dir}${unique_id}/test${test_label}/" + mkdir -p "${work_subdir}" + archive_dir="${work_subdir}archive_dir/" + cache_dir="${work_subdir}cache/" + log_dir="${work_subdir}logs/" + mkdir -p "${archive_dir}" + mkdir -p "${cache_dir}" + mkdir -p "${log_dir}" + + # Define log file paths + create_log="${log_dir}create.log" + update_log="${log_dir}update.log" + + print_success "Work directories created at ${work_subdir}" + + dst_globus="globus://${dst_endpoint_uuid}/${dst_endpoint_archive_subdir}" + dst_hpss="" + + + for hpss_path in "none" "${dst_hpss}" "${dst_globus}"; do + cd "${work_subdir}" + run_create "$dir_to_copy_from" "$create_subdir" "$archive_dir" "$hpss_path" "$cache_dir" "$create_log" + # For update, we need to be in archive_dir: + cd "${archive_dir}" + run_update "$dir_to_copy_from" "$update_subdir" "$archive_dir" "$hpss_path" "$cache_dir" "$update1_log" + + cd "${work_subdir}" + + # For extraction, dst should really be thought of as the src + for num_workers in 1 2; do + # For extraction, we must NOT be in archive_dir + extraction_subdir=${work_subdir}extract_subdir_${num_workers}_workers + mkdir ${extract_subdir} + cd ${extract_subdir} + run_extract "$archive_dir" "$hpss_path" "$cache_dir" "$extract_log" + cd "${work_subdir}" + done + done + + print_success "Test ${test_label} completed" + echo "" +done diff --git a/tests/performance/visualize_performance.py b/tests/performance/visualize_performance.py new file mode 100644 index 00000000..e69de29b From 35a69d2da1cfcd2c183b74b640e260bccdf97bce Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 25 Feb 2026 18:38:02 -0800 Subject: [PATCH 02/27] Apply changes from Claude --- .../generate_performance_data.bash | 172 ++++++--- tests/performance/visualize_performance.py | 353 ++++++++++++++++++ 2 files changed, 474 insertions(+), 51 deletions(-) diff --git a/tests/performance/generate_performance_data.bash b/tests/performance/generate_performance_data.bash index 54d7eeff..1c4c9ab2 100644 --- a/tests/performance/generate_performance_data.bash +++ b/tests/performance/generate_performance_data.bash @@ -1,11 +1,11 @@ #!/bin/bash set -e -# Analagous to CI/CD matrix testing of Python versions, +# Analogous to CI/CD matrix testing of Python versions, # here we will do a matrix performance profiling # by comparing runtimes for create/update/extract: # - On multiple directories -# - With `--hpss=none` with HPSS path, with Globus +# - With `--hpss=none`, with HPSS path, with Globus # We will also compare `zstash extract` in sequential-mode and parallel-mode @@ -42,7 +42,7 @@ subdir2=init/ # For `--hpss=...` -dst_hpss=/home/f/forsyth/zstash_performance +dst_hpss_path=/home/f/forsyth/zstash_performance # For `--hpss=globus...` fresh_globus=true @@ -94,7 +94,7 @@ confirm() [[ $REPLY =~ ^[Yy]$ ]] } -valiate_configuration() +validate_configuration() { local dir_to_copy_from="${1}" local subdir0="${2}" @@ -166,6 +166,24 @@ refresh_globus() print_success "Globus authentication reset complete" } +# Parse the real-time (wall clock) seconds from the output of `time`. +# `time` writes to stderr a block like: +# real 1m23.456s +# user 0m12.345s +# sys 0m 1.234s +# We capture both stdout+stderr into the log, then grep for the real line. +parse_elapsed_seconds() +{ + local log_file="${1}" + # Extract "Xm Y.ZZZs" and convert to total seconds + awk '/^real/ { + split($2, a, "m"); + mins = a[1]; + secs = substr(a[2], 1, length(a[2])-1); + printf "%.3f\n", mins*60 + secs + }' "${log_file}" +} + ############################################################################### # Core functions @@ -174,18 +192,20 @@ run_create() local dir_to_copy_from="${1}" local subdir="${2}" local archive_dir="${3}" - local dst_hpss="${4}" + local hpss_path="${4}" local cache_dir="${5}" local create_log="${6}" + print_step "Starting CREATE operation..." print_info "Copying data from ${dir_to_copy_from}${subdir}" cp -r "${dir_to_copy_from}${subdir}" "${archive_dir}${subdir}" print_info "Running zstash create..." - print_info "Command: zstash create --hpss=${dst_hpss} --cache=${cache_dir} -v ${archive_dir}" + print_info "Command: zstash create --hpss=${hpss_path} --cache=${cache_dir} -v ${archive_dir}" - if { time zstash create --hpss="${dst_hpss}" --cache="${cache_dir}" -v "${archive_dir}" ; } 2>&1 | tee "${create_log}"; then + # We must be outside archive_dir when running create + if { time zstash create --hpss="${hpss_path}" --cache="${cache_dir}" -v "${archive_dir}" ; } 2>&1 | tee "${create_log}"; then print_success "zstash create completed successfully" else print_error "zstash create failed with exit code $?" @@ -198,7 +218,7 @@ run_update() local dir_to_copy_from="${1}" local subdir="${2}" local archive_dir="${3}" - local dst_hpss="${4}" + local hpss_path="${4}" local cache_dir="${5}" local update_log="${6}" @@ -208,50 +228,87 @@ run_update() cp -r "${dir_to_copy_from}${subdir}" "${archive_dir}${subdir}" print_info "Running zstash update..." - print_info "Command: zstash update --hpss=${dst_hpss} --cache=${cache_dir} -v" + print_info "Command: zstash update --hpss=${hpss_path} --cache=${cache_dir} -v" - if { time zstash update --hpss="${dst_hpss}" --cache="${cache_dir}" -v ; } 2>&1 | tee "${update_log}"; then + # zstash update must be run from within the archive directory + pushd "${archive_dir}" > /dev/null + if { time zstash update --hpss="${hpss_path}" --cache="${cache_dir}" -v ; } 2>&1 | tee "${update_log}"; then print_success "zstash update completed successfully" else print_error "zstash update failed with exit code $?" + popd > /dev/null exit 1 fi + popd > /dev/null } run_extract() { - local archive_dir="${1}" - local src_hpss="${2}" + local extract_dir="${1}" + local hpss_path="${2}" local num_workers="${3}" local cache_dir="${4}" local extract_log="${5}" - print_step "Starting EXTRACT operation..." + print_step "Starting EXTRACT operation (workers=${num_workers})..." print_info "Running zstash extract..." - print_info "Command: zstash extract --hpss=${src_hpss} --workers=${num_workers} --cache=${cache_dir} -v" + print_info "Command: zstash extract --hpss=${hpss_path} --workers=${num_workers} --cache=${cache_dir} -v" - if { time zstash extract --hpss="${src_hpss}" --workers="${num_workers}" --cache="${cache_dir}" -v ; } 2>&1 | tee "${extract_log}"; then + # zstash extract must be run from within the extraction directory + pushd "${extract_dir}" > /dev/null + if { time zstash extract --hpss="${hpss_path}" --workers="${num_workers}" --cache="${cache_dir}" -v ; } 2>&1 | tee "${extract_log}"; then print_success "zstash extract completed successfully" else print_error "zstash extract failed with exit code $?" + popd > /dev/null exit 1 fi + popd > /dev/null +} + +############################################################################### +# Results tracking + +# CSV file to collect all runtimes for later visualization +results_csv="${work_dir}${unique_id}/results.csv" + +record_result() +{ + local test_label="${1}" # e.g. "01" + local create_subdir="${2}" + local update_subdir="${3}" + local hpss_label="${4}" # "none", "hpss", "globus" + local operation="${5}" # "create", "update", "extract_seq", "extract_par" + local log_file="${6}" + + local elapsed + elapsed=$(parse_elapsed_seconds "${log_file}") + echo "${test_label},${create_subdir},${update_subdir},${hpss_label},${operation},${elapsed}" >> "${results_csv}" + print_info "Recorded: test=${test_label} op=${operation} hpss=${hpss_label} elapsed=${elapsed}s" } ############################################################################### # Main script: -valiate_configuration $dir_to_copy_from $subdir0 $subdir1 $subdir2 +validate_configuration "$dir_to_copy_from" "$subdir0" "$subdir1" "$subdir2" if [ "${fresh_globus}" == "true" ]; then refresh_globus fi +# Create the top-level results directory and CSV header +mkdir -p "${work_dir}${unique_id}" +echo "test_label,create_subdir,update_subdir,hpss_label,operation,elapsed_seconds" > "${results_csv}" +print_info "Results CSV: ${results_csv}" + # Array of subdirectories subdirs=("$subdir0" "$subdir1" "$subdir2") -# Define the 6 possible permutations as test configurations -# Each array contains indices into the subdirs array + +# Define the 6 possible permutations as test configurations. +# Each string contains two space-separated indices into the subdirs array: +# first index = subdir used for create +# second index = subdir used for update declare -a test_configs=( "0 1" "0 2" @@ -260,71 +317,84 @@ declare -a test_configs=( "2 0" "2 1" ) -declare -a test_labels=("01" "02" "10", "12", "20", "21") -declare -a test_names +# FIX: removed erroneous commas that were present in the original array +declare -a test_labels=("01" "02" "10" "12" "20" "21") # Loop through the 6 test configurations for test_idx in 0 1 2 3 4 5; do # Parse the configuration - config=(${test_configs[$test_idx]}) - i=${config[0]} # 1st element (create-subdir) - j=${config[1]} # 2nd element (update-subdir) + read -r -a config <<< "${test_configs[$test_idx]}" + i=${config[0]} # index for create subdir + j=${config[1]} # index for update subdir # Get the subdirectories for this test create_subdir="${subdirs[$i]}" update_subdir="${subdirs[$j]}" - # Create a label for this test test_label="${test_labels[$test_idx]}" - test_names+=("Test_${test_label}") print_step "==========================================" print_step "Running Test ${test_label}" - print_step " Create: $create_subdir" - print_step " Update: $update_subdir" + print_step " Create subdir: $create_subdir" + print_step " Update subdir: $update_subdir" print_step "==========================================" # Create unique work directories for this test - dst_endpoint_archive_subdir="${dst_endpoint_archive_dir}${unique_id}/test${test_label}/" work_subdir="${work_dir}${unique_id}/test${test_label}/" mkdir -p "${work_subdir}" - archive_dir="${work_subdir}archive_dir/" - cache_dir="${work_subdir}cache/" + log_dir="${work_subdir}logs/" - mkdir -p "${archive_dir}" - mkdir -p "${cache_dir}" mkdir -p "${log_dir}" - # Define log file paths - create_log="${log_dir}create.log" - update_log="${log_dir}update.log" + # FIX: renamed `dst_hpss` local var to `dst_globus_path` so it does not + # shadow/overwrite the top-level `dst_hpss_path` parameter. + dst_globus_path="globus://${dst_endpoint_uuid}/${dst_endpoint_archive_dir}${unique_id}/test${test_label}/" + + # Iterate over the three HPSS modes + for hpss_entry in "none:none" "hpss:${dst_hpss_path}" "globus:${dst_globus_path}"; do + hpss_label="${hpss_entry%%:*}" + hpss_path="${hpss_entry#*:}" - print_success "Work directories created at ${work_subdir}" + print_step "--- HPSS mode: ${hpss_label} (${hpss_path}) ---" - dst_globus="globus://${dst_endpoint_uuid}/${dst_endpoint_archive_subdir}" - dst_hpss="" + # Each hpss mode gets its own subdirectories to avoid cross-contamination + mode_dir="${work_subdir}${hpss_label}/" + archive_dir="${mode_dir}archive_dir/" + cache_dir="${mode_dir}cache/" + mkdir -p "${archive_dir}" "${cache_dir}" + create_log="${log_dir}create_${hpss_label}.log" + update_log="${log_dir}update_${hpss_label}.log" - for hpss_path in "none" "${dst_hpss}" "${dst_globus}"; do - cd "${work_subdir}" + # --- CREATE --- run_create "$dir_to_copy_from" "$create_subdir" "$archive_dir" "$hpss_path" "$cache_dir" "$create_log" - # For update, we need to be in archive_dir: - cd "${archive_dir}" - run_update "$dir_to_copy_from" "$update_subdir" "$archive_dir" "$hpss_path" "$cache_dir" "$update1_log" + record_result "$test_label" "$create_subdir" "$update_subdir" "$hpss_label" "create" "$create_log" - cd "${work_subdir}" + # --- UPDATE --- + run_update "$dir_to_copy_from" "$update_subdir" "$archive_dir" "$hpss_path" "$cache_dir" "$update_log" + record_result "$test_label" "$create_subdir" "$update_subdir" "$hpss_label" "update" "$update_log" - # For extraction, dst should really be thought of as the src + # --- EXTRACT (sequential=1 worker, parallel=2 workers) --- for num_workers in 1 2; do - # For extraction, we must NOT be in archive_dir - extraction_subdir=${work_subdir}extract_subdir_${num_workers}_workers - mkdir ${extract_subdir} - cd ${extract_subdir} - run_extract "$archive_dir" "$hpss_path" "$cache_dir" "$extract_log" - cd "${work_subdir}" + extract_log="${log_dir}extract_${hpss_label}_${num_workers}workers.log" + extract_dir="${mode_dir}extract_${num_workers}workers/" + mkdir -p "${extract_dir}" + + # FIX: pass num_workers argument (was missing in original) + run_extract "$extract_dir" "$hpss_path" "$num_workers" "$cache_dir" "$extract_log" + + if [ "$num_workers" -eq 1 ]; then + op_label="extract_seq" + else + op_label="extract_par" + fi + record_result "$test_label" "$create_subdir" "$update_subdir" "$hpss_label" "$op_label" "$extract_log" done done print_success "Test ${test_label} completed" echo "" done + +print_success "All tests completed. Results saved to: ${results_csv}" +print_info "Run: python visualize_performance.py ${results_csv}" diff --git a/tests/performance/visualize_performance.py b/tests/performance/visualize_performance.py index e69de29b..d9824253 100644 --- a/tests/performance/visualize_performance.py +++ b/tests/performance/visualize_performance.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python3 +""" +visualize_performance.py – Plot zstash performance profiling results. + +Usage: + python visualize_performance.py results.csv + python visualize_performance.py results.csv --output perf_report.png + +The CSV is produced by performance_profile.sh and has columns: + test_label, create_subdir, update_subdir, hpss_label, operation, elapsed_seconds + +Visualization strategy +---------------------- +Four dimensions: + 1. Operation : create | update | extract_seq | extract_par + 2. Directory : build/ (many small) | run/ (medium) | init/ (few large) + 3. HPSS mode : none | hpss | globus + 4. Parallelism: already encoded in operation (extract_seq vs extract_par) + +Layout: 2×2 grid of subplots, one per operation. + Within each subplot: + - X-axis groups = directory processed (create_subdir or update_subdir) + - Bars = HPSS mode (none / hpss / globus), colour-coded + - Each test config contributes one bar per (directory, hpss_mode) cell; + if multiple configs share the same directory for an operation, their + runtimes are shown as individual dots and the bar shows the mean. + +An additional 5th subplot compares extract_seq vs extract_par side-by-side +to make the parallelism speed-up immediately visible. +""" + +import argparse +import sys +from pathlib import Path + +import matplotlib.patches as mpatches +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- + +HPSS_ORDER = ["none", "hpss", "globus"] +HPSS_COLORS = {"none": "#4C72B0", "hpss": "#DD8452", "globus": "#55A868"} +HPSS_LABELS = {"none": "No HPSS", "hpss": "Direct HPSS", "globus": "Globus"} + +OP_ORDER = ["create", "update", "extract_seq", "extract_par"] +OP_TITLES = { + "create": "zstash create", + "update": "zstash update", + "extract_seq": "zstash extract (sequential, 1 worker)", + "extract_par": "zstash extract (parallel, 2 workers)", +} + +# Map an operation to the column that holds the "relevant directory" +OP_DIR_COL = { + "create": "create_subdir", + "update": "update_subdir", + "extract_seq": "create_subdir", # extraction exercises the create archive + "extract_par": "create_subdir", +} + +BAR_WIDTH = 0.22 +DOT_ALPHA = 0.55 +DOT_SIZE = 40 + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def load_data(csv_path: str) -> pd.DataFrame: + df = pd.read_csv(csv_path) + df.columns = df.columns.str.strip() + df["elapsed_seconds"] = pd.to_numeric(df["elapsed_seconds"], errors="coerce") + # Normalise subdir names: strip trailing slashes for display + for col in ("create_subdir", "update_subdir"): + df[col] = df[col].str.strip().str.rstrip("/") + df["hpss_label"] = df["hpss_label"].str.strip() + df["operation"] = df["operation"].str.strip() + return df + + +def dir_sort_key(name: str) -> int: + """Sort dirs in a consistent order: build, run, init.""" + order = {"build": 0, "run": 1, "init": 2} + return order.get(name.lower(), 99) + + +def _add_dir_annotation(ax, dirs): + """Add a small file-count hint below each directory group label.""" + hints = { + "build": "many small files\n(~7k files, 1.2 GiB)", + "run": "mixed\n(~111 files, 11 GiB)", + "init": "few large files\n(14 files, 6.9 GiB)", + } + for idx, d in enumerate(dirs): + if d in hints: + # position in data coords: centre of the dir group + x_centre = idx + ax.annotate( + hints[d], + xy=(x_centre, 0), + xycoords=("data", "axes fraction"), + xytext=(0, -46), + textcoords="offset points", + ha="center", + va="top", + fontsize=6.5, + color="#555555", + annotation_clip=False, + ) + + +def plot_operation(ax, df_op: pd.DataFrame, operation: str, dirs: list[str]): + """Draw grouped bars for one operation subplot.""" + dir_col = OP_DIR_COL[operation] + n_dirs = len(dirs) + n_hpss = len(HPSS_ORDER) + + x_base = np.arange(n_dirs) + offsets = np.linspace(-(n_hpss - 1) / 2, (n_hpss - 1) / 2, n_hpss) * BAR_WIDTH + + for h_idx, hpss in enumerate(HPSS_ORDER): + df_h = df_op[df_op["hpss_label"] == hpss] + means, all_vals, xs = [], [], [] + + for d_idx, d in enumerate(dirs): + vals = df_h[df_h[dir_col] == d]["elapsed_seconds"].dropna().values + mean = vals.mean() if len(vals) > 0 else 0.0 + means.append(mean) + all_vals.append(vals) + xs.append(x_base[d_idx] + offsets[h_idx]) + + color = HPSS_COLORS[hpss] + ax.bar( + xs, + means, + width=BAR_WIDTH, + color=color, + alpha=0.85, + label=HPSS_LABELS[hpss], + zorder=2, + ) + # Overlay individual data points so scatter is visible + for x_pos, vals in zip(xs, all_vals): + if len(vals) > 1: + jitter = np.random.uniform( + -BAR_WIDTH * 0.25, BAR_WIDTH * 0.25, size=len(vals) + ) + ax.scatter( + x_pos + jitter, + vals, + color="white", + edgecolors=color, + s=DOT_SIZE, + zorder=3, + alpha=DOT_ALPHA, + linewidths=1.2, + ) + + ax.set_title(OP_TITLES[operation], fontsize=10, fontweight="bold", pad=6) + ax.set_xticks(x_base) + ax.set_xticklabels([d + "/" for d in dirs], fontsize=9) + ax.set_ylabel("Wall-clock time (s)", fontsize=8) + ax.set_xlabel("Directory processed", fontsize=8, labelpad=14) + ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) + ax.set_axisbelow(True) + _add_dir_annotation(ax, dirs) + + # Value labels on bars + for rect in ax.patches: + h = rect.get_height() + if h > 0: + ax.text( + rect.get_x() + rect.get_width() / 2, + h * 1.01, + f"{h:.0f}s", + ha="center", + va="bottom", + fontsize=6, + color="#333333", + ) + + +def plot_extract_comparison(ax, df: pd.DataFrame, dirs: list[str]): + """ + Extra subplot: sequential vs parallel extract, grouped by (directory, hpss). + Uses a hatch pattern to distinguish seq/par within each hpss colour. + """ + dir_col = "create_subdir" + n_dirs = len(dirs) + n_hpss = len(HPSS_ORDER) + ops = ["extract_seq", "extract_par"] + hatches = {"extract_seq": "", "extract_par": "////"} + n_bars = n_hpss * len(ops) + + group_width = n_bars * BAR_WIDTH + 0.15 # total width per dir group + x_base = np.arange(n_dirs) * group_width + bar_positions = [] # (x, height, color, hatch, label_shown) + + for d_idx, d in enumerate(dirs): + for h_idx, hpss in enumerate(HPSS_ORDER): + for op_idx, op in enumerate(ops): + df_cell = df[ + (df["operation"] == op) + & (df["hpss_label"] == hpss) + & (df[dir_col] == d) + ] + vals = df_cell["elapsed_seconds"].dropna().values + mean = vals.mean() if len(vals) > 0 else 0.0 + bar_x = x_base[d_idx] + (h_idx * len(ops) + op_idx) * BAR_WIDTH + bar_positions.append( + (bar_x, mean, HPSS_COLORS[hpss], hatches[op], hpss, op) + ) + + for bar_x, mean, color, hatch, hpss, op in bar_positions: + ax.bar( + bar_x, mean, width=BAR_WIDTH, color=color, hatch=hatch, alpha=0.85, zorder=2 + ) + + ax.set_xticks(x_base + (n_bars / 2 - 0.5) * BAR_WIDTH) + ax.set_xticklabels([d + "/" for d in dirs], fontsize=9) + ax.set_ylabel("Wall-clock time (s)", fontsize=8) + ax.set_xlabel("Directory (archive source)", fontsize=8, labelpad=14) + ax.set_title( + "Extract: Sequential vs Parallel (speed-up comparison)", + fontsize=10, + fontweight="bold", + pad=6, + ) + ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) + ax.set_axisbelow(True) + _add_dir_annotation(ax, dirs) + + # Custom legend: colour = hpss, hatch = seq/par + hpss_patches = [ + mpatches.Patch(color=HPSS_COLORS[h], label=HPSS_LABELS[h]) for h in HPSS_ORDER + ] + seq_patch = mpatches.Patch( + facecolor="grey", hatch="", label="Sequential (1 worker)" + ) + par_patch = mpatches.Patch( + facecolor="grey", hatch="////", label="Parallel (2 workers)" + ) + ax.legend( + handles=hpss_patches + [seq_patch, par_patch], + fontsize=7, + loc="upper right", + ncol=2, + ) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def main(): + parser = argparse.ArgumentParser( + description="Visualise zstash performance results." + ) + parser.add_argument( + "csv", help="Path to results.csv produced by performance_profile.sh" + ) + parser.add_argument( + "--output", + default=None, + help="Save figure to this path (e.g. report.png). " + "If omitted, the figure is displayed interactively.", + ) + parser.add_argument( + "--dpi", type=int, default=150, help="Output DPI (default: 150)" + ) + args = parser.parse_args() + + df = load_data(args.csv) + + if df.empty: + print("ERROR: CSV is empty or could not be parsed.", file=sys.stderr) + sys.exit(1) + + # Determine the sorted list of directories that actually appear in the data + all_dirs = sorted( + set(df["create_subdir"].dropna()) | set(df["update_subdir"].dropna()), + key=dir_sort_key, + ) + + # ----------------------------------------------------------------------- + # Figure layout: 3 rows × 2 cols + # Row 0: create | update + # Row 1: extract_seq | extract_par + # Row 2: extract seq-vs-par comparison (spans both columns) + # ----------------------------------------------------------------------- + fig = plt.figure(figsize=(15, 16)) + fig.suptitle( + "zstash Performance Profiling\n" + "(bars = mean over test configs; dots = individual runs)", + fontsize=13, + fontweight="bold", + y=0.98, + ) + + gs = fig.add_gridspec( + 3, 2, hspace=0.55, wspace=0.35, top=0.93, bottom=0.07, left=0.07, right=0.97 + ) + + axes = { + "create": fig.add_subplot(gs[0, 0]), + "update": fig.add_subplot(gs[0, 1]), + "extract_seq": fig.add_subplot(gs[1, 0]), + "extract_par": fig.add_subplot(gs[1, 1]), + } + ax_cmp = fig.add_subplot(gs[2, :]) + + # ----------------------------------------------------------------------- + # Draw the four single-operation subplots + # ----------------------------------------------------------------------- + legend_handles = None + for op in OP_ORDER: + ax = axes[op] + df_op = df[df["operation"] == op] + plot_operation(ax, df_op, op, all_dirs) + + if legend_handles is None: + legend_handles = [ + mpatches.Patch(color=HPSS_COLORS[h], label=HPSS_LABELS[h]) + for h in HPSS_ORDER + ] + ax.legend(handles=legend_handles, fontsize=7, loc="upper right") + + # ----------------------------------------------------------------------- + # Draw the sequential vs parallel comparison subplot + # ----------------------------------------------------------------------- + plot_extract_comparison(ax_cmp, df, all_dirs) + + # ----------------------------------------------------------------------- + # Save or show + # ----------------------------------------------------------------------- + if args.output: + out_path = Path(args.output) + fig.savefig(out_path, dpi=args.dpi, bbox_inches="tight") + print(f"Figure saved to: {out_path}") + else: + plt.show() + + +if __name__ == "__main__": + np.random.seed(42) # reproducible jitter + main() From 7302b54a7b1de7fc2078a44d30283aaa48523b96 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 25 Feb 2026 18:43:47 -0800 Subject: [PATCH 03/27] Send output to web server --- tests/performance/generate_performance_data.bash | 2 ++ tests/performance/visualize_performance.py | 13 ++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/performance/generate_performance_data.bash b/tests/performance/generate_performance_data.bash index 1c4c9ab2..6f9408e3 100644 --- a/tests/performance/generate_performance_data.bash +++ b/tests/performance/generate_performance_data.bash @@ -398,3 +398,5 @@ done print_success "All tests completed. Results saved to: ${results_csv}" print_info "Run: python visualize_performance.py ${results_csv}" +print_info "Plot will be saved to the web server and accessible at:" +print_info " https://portal.nersc.gov/cfs/e3sm/forsyth/zstash_performance.png" diff --git a/tests/performance/visualize_performance.py b/tests/performance/visualize_performance.py index d9824253..ba3dbee1 100644 --- a/tests/performance/visualize_performance.py +++ b/tests/performance/visualize_performance.py @@ -268,9 +268,10 @@ def main(): ) parser.add_argument( "--output", - default=None, - help="Save figure to this path (e.g. report.png). " - "If omitted, the figure is displayed interactively.", + default="/global/cfs/cdirs/e3sm/www/forsyth/zstash_performance.png", + help="Save figure to this path. " + "Defaults to the NERSC web server output directory. " + "Pass --output '' to display interactively instead.", ) parser.add_argument( "--dpi", type=int, default=150, help="Output DPI (default: 150)" @@ -342,8 +343,14 @@ def main(): # ----------------------------------------------------------------------- if args.output: out_path = Path(args.output) + out_path.parent.mkdir(parents=True, exist_ok=True) fig.savefig(out_path, dpi=args.dpi, bbox_inches="tight") print(f"Figure saved to: {out_path}") + web_path = str(out_path).replace( + "/global/cfs/cdirs/e3sm/www/", + "https://portal.nersc.gov/cfs/e3sm/", + ) + print(f"Accessible at: {web_path}") else: plt.show() From 648f7e6fedca0377a6557734a535cea2939440e0 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 26 Feb 2026 08:21:11 -0800 Subject: [PATCH 04/27] Add regression testing --- tests/performance/visualize_performance.py | 371 ++++++++++++++++++++- 1 file changed, 358 insertions(+), 13 deletions(-) diff --git a/tests/performance/visualize_performance.py b/tests/performance/visualize_performance.py index ba3dbee1..99a5c811 100644 --- a/tests/performance/visualize_performance.py +++ b/tests/performance/visualize_performance.py @@ -17,16 +17,23 @@ 3. HPSS mode : none | hpss | globus 4. Parallelism: already encoded in operation (extract_seq vs extract_par) -Layout: 2×2 grid of subplots, one per operation. +Figure 1 – Performance overview: + Layout: 2×2 grid of subplots, one per operation. Within each subplot: - X-axis groups = directory processed (create_subdir or update_subdir) - Bars = HPSS mode (none / hpss / globus), colour-coded - Each test config contributes one bar per (directory, hpss_mode) cell; if multiple configs share the same directory for an operation, their runtimes are shown as individual dots and the bar shows the mean. - -An additional 5th subplot compares extract_seq vs extract_par side-by-side -to make the parallelism speed-up immediately visible. + An additional 5th subplot compares extract_seq vs extract_par side-by-side + to make the parallelism speed-up immediately visible. + +Figure 2 – Baseline comparison (current branch vs main): + Produced only when BASELINE_RESULTS_CSV is set to a valid path. + Same 2×2 + comparison layout, but each cell shows two bars + (current = solid, baseline = hatched) with a ratio annotation + (current/baseline) above each pair. Ratio > 1 = regression (slower), + ratio < 1 = improvement (faster). """ import argparse @@ -38,6 +45,15 @@ import numpy as np import pandas as pd +# --------------------------------------------------------------------------- +# Baseline config ← EDIT THIS to point at the main-branch profiling run +# --------------------------------------------------------------------------- + +# Set to the results.csv of the baseline (main branch) run, e.g.: +# /global/cfs/cdirs/e3sm/forsyth/zstash_performance/performance_20260101/results.csv +# Set to None to skip the baseline comparison figure. +BASELINE_RESULTS_CSV = None + # --------------------------------------------------------------------------- # Config # --------------------------------------------------------------------------- @@ -254,6 +270,302 @@ def plot_extract_comparison(ax, df: pd.DataFrame, dirs: list[str]): ) +# --------------------------------------------------------------------------- +# Baseline comparison figure +# --------------------------------------------------------------------------- + +# Ratio colouring thresholds +RATIO_REGRESSION = 1.10 # >= 10% slower → red +RATIO_IMPROVEMENT = 0.90 # <= 10% faster → green +RATIO_NEUTRAL_COLOR = "#333333" +RATIO_REGRESSION_COLOR = "#CC3311" +RATIO_IMPROVEMENT_COLOR = "#228833" + + +def _ratio_color(ratio: float) -> str: + if ratio >= RATIO_REGRESSION: + return RATIO_REGRESSION_COLOR + if ratio <= RATIO_IMPROVEMENT: + return RATIO_IMPROVEMENT_COLOR + return RATIO_NEUTRAL_COLOR + + +def plot_comparison_operation( + ax, + df_cur: pd.DataFrame, + df_bas: pd.DataFrame, + operation: str, + dirs: list[str], +): + """ + For one operation, draw paired bars (current vs baseline) per + (directory, hpss_mode) cell, with a ratio annotation above each pair. + """ + dir_col = OP_DIR_COL[operation] + n_dirs = len(dirs) + n_hpss = len(HPSS_ORDER) + + # Each hpss group occupies 2 bars (current + baseline) + a small gap + pair_width = BAR_WIDTH + gap = BAR_WIDTH * 0.3 + group_span = n_hpss * (2 * pair_width + gap) + 0.2 + x_base = np.arange(n_dirs) * group_span + + for h_idx, hpss in enumerate(HPSS_ORDER): + color = HPSS_COLORS[hpss] + pair_offset = h_idx * (2 * pair_width + gap) + + for d_idx, d in enumerate(dirs): + x_left = x_base[d_idx] + pair_offset # current bar + x_right = x_base[d_idx] + pair_offset + pair_width # baseline bar + + def mean_for(df): + v = ( + df[ + (df["operation"] == operation) + & (df["hpss_label"] == hpss) + & (df[dir_col] == d) + ]["elapsed_seconds"] + .dropna() + .values + ) + return v.mean() if len(v) > 0 else 0.0 + + cur_mean = mean_for(df_cur) + bas_mean = mean_for(df_bas) + + # Current bar (solid) + ax.bar( + x_left, + cur_mean, + width=pair_width, + color=color, + alpha=0.85, + zorder=2, + label=HPSS_LABELS[hpss] if d_idx == 0 else "", + ) + # Baseline bar (hatched, lighter) + ax.bar( + x_right, + bas_mean, + width=pair_width, + color=color, + alpha=0.40, + hatch="////", + zorder=2, + edgecolor=color, + ) + + # Ratio annotation + if bas_mean > 0 and cur_mean > 0: + ratio = cur_mean / bas_mean + top = max(cur_mean, bas_mean) + rat_color = _ratio_color(ratio) + arrow = ( + "▲" + if ratio >= RATIO_REGRESSION + else ("▼" if ratio <= RATIO_IMPROVEMENT else "") + ) + ax.text( + (x_left + x_right) / 2, + top * 1.03, + f"{arrow}{ratio:.2f}×", + ha="center", + va="bottom", + fontsize=6.5, + fontweight="bold", + color=rat_color, + zorder=4, + ) + + ax.set_title(OP_TITLES[operation], fontsize=10, fontweight="bold", pad=6) + ax.set_xticks(x_base + (n_hpss * (2 * pair_width + gap) - gap) / 2) + ax.set_xticklabels([d + "/" for d in dirs], fontsize=9) + ax.set_ylabel("Wall-clock time (s)", fontsize=8) + ax.set_xlabel("Directory processed", fontsize=8, labelpad=14) + ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) + ax.set_axisbelow(True) + _add_dir_annotation(ax, dirs) + + +def plot_comparison_extract( + ax, + df_cur: pd.DataFrame, + df_bas: pd.DataFrame, + dirs: list[str], +): + """Seq vs par extract comparison with current/baseline pairing.""" + dir_col = "create_subdir" + n_dirs = len(dirs) + ops = ["extract_seq", "extract_par"] + op_hatches = {"extract_seq": "", "extract_par": "xxxx"} + + pair_width = BAR_WIDTH + gap = BAR_WIDTH * 0.3 + group_span = len(HPSS_ORDER) * len(ops) * (2 * pair_width + gap) + 0.3 + x_base = np.arange(n_dirs) * group_span + + for d_idx, d in enumerate(dirs): + slot = 0 + for h_idx, hpss in enumerate(HPSS_ORDER): + color = HPSS_COLORS[hpss] + for op in ops: + x_left = x_base[d_idx] + slot * (2 * pair_width + gap) + x_right = x_left + pair_width + hatch = op_hatches[op] + + def mean_for(df): + v = ( + df[ + (df["operation"] == op) + & (df["hpss_label"] == hpss) + & (df[dir_col] == d) + ]["elapsed_seconds"] + .dropna() + .values + ) + return v.mean() if len(v) > 0 else 0.0 + + cur_mean = mean_for(df_cur) + bas_mean = mean_for(df_bas) + + ax.bar( + x_left, + cur_mean, + width=pair_width, + color=color, + hatch=hatch, + alpha=0.85, + zorder=2, + ) + ax.bar( + x_right, + bas_mean, + width=pair_width, + color=color, + hatch=hatch, + alpha=0.35, + zorder=2, + edgecolor=color, + ) + + if bas_mean > 0 and cur_mean > 0: + ratio = cur_mean / bas_mean + top = max(cur_mean, bas_mean) + rat_color = _ratio_color(ratio) + arrow = ( + "▲" + if ratio >= RATIO_REGRESSION + else ("▼" if ratio <= RATIO_IMPROVEMENT else "") + ) + ax.text( + (x_left + x_right) / 2, + top * 1.03, + f"{arrow}{ratio:.2f}×", + ha="center", + va="bottom", + fontsize=5.5, + fontweight="bold", + color=rat_color, + zorder=4, + ) + slot += 1 + + ax.set_xticks(x_base + group_span / 2 - (2 * pair_width + gap) / 2) + ax.set_xticklabels([d + "/" for d in dirs], fontsize=9) + ax.set_ylabel("Wall-clock time (s)", fontsize=8) + ax.set_xlabel("Directory (archive source)", fontsize=8, labelpad=14) + ax.set_title( + "Extract: Sequential vs Parallel — current vs baseline", + fontsize=10, + fontweight="bold", + pad=6, + ) + ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) + ax.set_axisbelow(True) + _add_dir_annotation(ax, dirs) + + # Legend + hpss_patches = [ + mpatches.Patch(color=HPSS_COLORS[h], label=HPSS_LABELS[h]) for h in HPSS_ORDER + ] + seq_patch = mpatches.Patch( + facecolor="grey", hatch="", label="Sequential (1 worker)" + ) + par_patch = mpatches.Patch( + facecolor="grey", hatch="xxxx", label="Parallel (2 workers)" + ) + cur_patch = mpatches.Patch(facecolor="grey", alpha=0.85, label="Current branch") + bas_patch = mpatches.Patch( + facecolor="grey", alpha=0.35, label="Baseline (main)", hatch="////" + ) + ax.legend( + handles=hpss_patches + [seq_patch, par_patch, cur_patch, bas_patch], + fontsize=6.5, + loc="upper right", + ncol=3, + ) + + +def build_comparison_figure( + df_cur: pd.DataFrame, + df_bas: pd.DataFrame, + all_dirs: list[str], + cur_label: str, + bas_label: str, +) -> plt.Figure: + """Build and return the full baseline-comparison figure.""" + fig = plt.figure(figsize=(16, 17)) + fig.suptitle( + f"zstash Performance: Current vs Baseline\n" + f"current = {cur_label} | baseline (main) = {bas_label}\n" + f"Ratio = current / baseline — " + f"▲ {RATIO_REGRESSION_COLOR_LABEL} ≥{RATIO_REGRESSION:.0%} slower " + f"▼ {RATIO_IMPROVEMENT_COLOR_LABEL} ≤{RATIO_IMPROVEMENT:.0%} faster " + f"= within ±10%", + fontsize=11, + fontweight="bold", + y=0.98, + ) + + gs = fig.add_gridspec( + 3, 2, hspace=0.58, wspace=0.35, top=0.92, bottom=0.07, left=0.07, right=0.97 + ) + + axes = { + "create": fig.add_subplot(gs[0, 0]), + "update": fig.add_subplot(gs[0, 1]), + "extract_seq": fig.add_subplot(gs[1, 0]), + "extract_par": fig.add_subplot(gs[1, 1]), + } + ax_cmp = fig.add_subplot(gs[2, :]) + + for op in OP_ORDER: + plot_comparison_operation(axes[op], df_cur, df_bas, op, all_dirs) + + # Shared legend for solid=current, hatched=baseline + cur_patch = mpatches.Patch(facecolor="grey", alpha=0.85, label="Current branch") + bas_patch = mpatches.Patch( + facecolor="grey", alpha=0.40, hatch="////", label="Baseline (main)" + ) + hpss_patches = [ + mpatches.Patch(color=HPSS_COLORS[h], label=HPSS_LABELS[h]) for h in HPSS_ORDER + ] + axes["create"].legend( + handles=[cur_patch, bas_patch] + hpss_patches, + fontsize=7, + loc="upper right", + ) + + plot_comparison_extract(ax_cmp, df_cur, df_bas, all_dirs) + + return fig + + +# String labels used in the suptitle (avoids referencing undefined vars earlier) +RATIO_REGRESSION_COLOR_LABEL = "red" +RATIO_IMPROVEMENT_COLOR_LABEL = "green" + # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- @@ -338,19 +650,52 @@ def main(): # ----------------------------------------------------------------------- plot_extract_comparison(ax_cmp, df, all_dirs) + # ----------------------------------------------------------------------- + # Baseline comparison figure (Figure 2) + # ----------------------------------------------------------------------- + fig_cmp = None + if BASELINE_RESULTS_CSV is not None: + bas_path = Path(BASELINE_RESULTS_CSV) + if not bas_path.exists(): + print( + f"WARNING: BASELINE_RESULTS_CSV not found: {bas_path}", file=sys.stderr + ) + print("Skipping baseline comparison figure.", file=sys.stderr) + else: + df_bas = load_data(str(bas_path)) + # Derive a short label from the baseline CSV path for titles + # e.g. ".../performance_20260101/results.csv" → "performance_20260101" + bas_label = bas_path.parent.name + cur_label = Path(args.csv).parent.name + fig_cmp = build_comparison_figure( + df, df_bas, all_dirs, cur_label, bas_label + ) + # ----------------------------------------------------------------------- # Save or show # ----------------------------------------------------------------------- + def save_or_show(figure, out_path_str, label): + if out_path_str: + out_path = Path(out_path_str) + out_path.parent.mkdir(parents=True, exist_ok=True) + figure.savefig(out_path, dpi=args.dpi, bbox_inches="tight") + print(f"{label} saved to: {out_path}") + web_path = str(out_path).replace( + "/global/cfs/cdirs/e3sm/www/", + "https://portal.nersc.gov/cfs/e3sm/", + ) + print(f" Accessible at: {web_path}") + else: + plt.show() + if args.output: - out_path = Path(args.output) - out_path.parent.mkdir(parents=True, exist_ok=True) - fig.savefig(out_path, dpi=args.dpi, bbox_inches="tight") - print(f"Figure saved to: {out_path}") - web_path = str(out_path).replace( - "/global/cfs/cdirs/e3sm/www/", - "https://portal.nersc.gov/cfs/e3sm/", - ) - print(f"Accessible at: {web_path}") + save_or_show(fig, args.output, "Figure 1 (overview)") + if fig_cmp is not None: + # Derive comparison output path by inserting "_vs_baseline" before + # the extension, e.g. zstash_performance.png → zstash_performance_vs_baseline.png + p = Path(args.output) + cmp_output = str(p.with_stem(p.stem + "_vs_baseline")) + save_or_show(fig_cmp, cmp_output, "Figure 2 (baseline comparison)") else: plt.show() From d9cf130f6fea01fd518799eaae7694e405efed67 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 26 Feb 2026 08:26:49 -0800 Subject: [PATCH 05/27] Add ability to configure hpss options --- .../performance/generate_performance_data.bash | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/performance/generate_performance_data.bash b/tests/performance/generate_performance_data.bash index 6f9408e3..384c3362 100644 --- a/tests/performance/generate_performance_data.bash +++ b/tests/performance/generate_performance_data.bash @@ -42,6 +42,10 @@ subdir2=init/ # For `--hpss=...` +# Which HPSS options to run. Comment out any you want to skip. +# Options: "none" "hpss" "globus" +HPSS_OPTIONS=("none" "hpss" "globus") + dst_hpss_path=/home/f/forsyth/zstash_performance # For `--hpss=globus...` @@ -293,7 +297,7 @@ record_result() validate_configuration "$dir_to_copy_from" "$subdir0" "$subdir1" "$subdir2" -if [ "${fresh_globus}" == "true" ]; then +if [ "${fresh_globus}" == "true" ] && [[ " ${HPSS_OPTIONS[*]} " == *" globus "* ]]; then refresh_globus fi @@ -351,10 +355,14 @@ for test_idx in 0 1 2 3 4 5; do dst_globus_path="globus://${dst_endpoint_uuid}/${dst_endpoint_archive_dir}${unique_id}/test${test_label}/" # Iterate over the three HPSS modes - for hpss_entry in "none:none" "hpss:${dst_hpss_path}" "globus:${dst_globus_path}"; do - hpss_label="${hpss_entry%%:*}" - hpss_path="${hpss_entry#*:}" - + declare -A hpss_path_map=( + ["none"]="none" + ["hpss"]="${dst_hpss_path}" + ["globus"]="${dst_globus_path}" + ) + + for hpss_label in "${HPSS_OPTIONS[@]}"; do + hpss_path="${hpss_path_map[$hpss_label]}" print_step "--- HPSS mode: ${hpss_label} (${hpss_path}) ---" # Each hpss mode gets its own subdirectories to avoid cross-contamination From e6364ccb781085bf6f99b3d41a07db5704358374 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 26 Feb 2026 10:14:07 -0800 Subject: [PATCH 06/27] Fixes to generate plots --- conda/dev.yml | 3 +++ tests/performance/generate_performance_data.bash | 8 +++----- tests/performance/visualize_performance.py | 5 ++++- 3 files changed, 10 insertions(+), 6 deletions(-) mode change 100644 => 100755 tests/performance/generate_performance_data.bash diff --git a/conda/dev.yml b/conda/dev.yml index 85f64d32..e5a9b2a0 100644 --- a/conda/dev.yml +++ b/conda/dev.yml @@ -23,6 +23,9 @@ dependencies: # ======================= - pytest - pytest-cov + - matplotlib-base + - pandas + - numpy # Documentation # ================= # If versions are updated, also update in `.github/workflows/workflow.yml` diff --git a/tests/performance/generate_performance_data.bash b/tests/performance/generate_performance_data.bash old mode 100644 new mode 100755 index 384c3362..f2a8d290 --- a/tests/performance/generate_performance_data.bash +++ b/tests/performance/generate_performance_data.bash @@ -14,7 +14,7 @@ set -e # Run from Perlmutter, so that we can do both # a direct transfer to HPSS & a Globus transfer to Chrysalis -work_dir=/global/cfs/cdirs/e3sm/forsyth/zstash_performance/ +work_dir=/pscratch/sd/f/forsyth/zstash_performance/ unique_id=performance_20260225 dir_to_copy_from=/global/cfs/cdirs/e3sm/forsyth/E3SMv2/v2.LR.historical_0201/ @@ -44,7 +44,7 @@ subdir2=init/ # For `--hpss=...` # Which HPSS options to run. Comment out any you want to skip. # Options: "none" "hpss" "globus" -HPSS_OPTIONS=("none" "hpss" "globus") +HPSS_OPTIONS=("none" "hpss") # globus endpoint currently down dst_hpss_path=/home/f/forsyth/zstash_performance @@ -405,6 +405,4 @@ for test_idx in 0 1 2 3 4 5; do done print_success "All tests completed. Results saved to: ${results_csv}" -print_info "Run: python visualize_performance.py ${results_csv}" -print_info "Plot will be saved to the web server and accessible at:" -print_info " https://portal.nersc.gov/cfs/e3sm/forsyth/zstash_performance.png" +print_info "Now run: python visualize_performance.py ${results_csv}" diff --git a/tests/performance/visualize_performance.py b/tests/performance/visualize_performance.py index 99a5c811..5b46353d 100644 --- a/tests/performance/visualize_performance.py +++ b/tests/performance/visualize_performance.py @@ -37,6 +37,7 @@ """ import argparse +import os import sys from pathlib import Path @@ -580,7 +581,7 @@ def main(): ) parser.add_argument( "--output", - default="/global/cfs/cdirs/e3sm/www/forsyth/zstash_performance.png", + default="/global/cfs/cdirs/e3sm/www/forsyth/zstash_performance/performance.png", help="Save figure to this path. " "Defaults to the NERSC web server output directory. " "Pass --output '' to display interactively instead.", @@ -680,6 +681,8 @@ def save_or_show(figure, out_path_str, label): out_path.parent.mkdir(parents=True, exist_ok=True) figure.savefig(out_path, dpi=args.dpi, bbox_inches="tight") print(f"{label} saved to: {out_path}") + web_permissions = 0o755 + os.chmod(out_path_str, web_permissions) web_path = str(out_path).replace( "/global/cfs/cdirs/e3sm/www/", "https://portal.nersc.gov/cfs/e3sm/", From e415ec90a8ac7e9f3cf7045ddb9e92df9dbe02e0 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 26 Feb 2026 13:06:30 -0800 Subject: [PATCH 07/27] Improve plots --- tests/performance/visualize_performance.py | 361 +++++++++++++++++---- 1 file changed, 303 insertions(+), 58 deletions(-) diff --git a/tests/performance/visualize_performance.py b/tests/performance/visualize_performance.py index 5b46353d..85f5bcf4 100644 --- a/tests/performance/visualize_performance.py +++ b/tests/performance/visualize_performance.py @@ -71,12 +71,12 @@ "extract_par": "zstash extract (parallel, 2 workers)", } -# Map an operation to the column that holds the "relevant directory" +# Map an operation to the column that holds the "relevant directory". +# Extract is intentionally absent: it operates on the combined create+update +# archive, so both subdirs are needed and it is handled separately. OP_DIR_COL = { "create": "create_subdir", "update": "update_subdir", - "extract_seq": "create_subdir", # extraction exercises the create archive - "extract_par": "create_subdir", } BAR_WIDTH = 0.22 @@ -107,17 +107,25 @@ def dir_sort_key(name: str) -> int: return order.get(name.lower(), 99) -def _add_dir_annotation(ax, dirs): - """Add a small file-count hint below each directory group label.""" +def _add_dir_annotation(ax, dirs, x_positions): + """ + Add a small file-count hint below each directory group label. + + Parameters + ---------- + ax : the Axes to annotate + dirs : list of directory names in display order + x_positions : list of x-axis data coordinates for each dir group centre. + These are passed in explicitly so the function works for both + Fig. 1 (groups at 0, 1, 2, …) and Fig. 2 (wider group_span). + """ hints = { "build": "many small files\n(~7k files, 1.2 GiB)", "run": "mixed\n(~111 files, 11 GiB)", "init": "few large files\n(14 files, 6.9 GiB)", } - for idx, d in enumerate(dirs): + for x_centre, d in zip(x_positions, dirs): if d in hints: - # position in data coords: centre of the dir group - x_centre = idx ax.annotate( hints[d], xy=(x_centre, 0), @@ -186,7 +194,8 @@ def plot_operation(ax, df_op: pd.DataFrame, operation: str, dirs: list[str]): ax.set_xlabel("Directory processed", fontsize=8, labelpad=14) ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) ax.set_axisbelow(True) - _add_dir_annotation(ax, dirs) + # Fig. 1: group centres are at integer positions 0, 1, 2, … + _add_dir_annotation(ax, dirs, list(x_base)) # Value labels on bars for rect in ax.patches: @@ -203,55 +212,163 @@ def plot_operation(ax, df_op: pd.DataFrame, operation: str, dirs: list[str]): ) -def plot_extract_comparison(ax, df: pd.DataFrame, dirs: list[str]): +def _extract_configs(df: pd.DataFrame) -> list[tuple[str, str]]: + """ + Return the sorted list of (create_subdir, update_subdir) pairs that + actually appear in the extract rows of *df*. These represent the + combined archives that were extracted from. + """ + mask = df["operation"].isin(["extract_seq", "extract_par"]) + pairs = ( + df[mask][["create_subdir", "update_subdir"]] + .drop_duplicates() + .apply(tuple, axis=1) + .tolist() + ) + # Sort by create_subdir first, then update_subdir + return sorted(pairs, key=lambda p: (dir_sort_key(p[0]), dir_sort_key(p[1]))) + + +def _extract_tick_label(create_sub: str, update_sub: str) -> str: + """Short two-line tick label for a (create, update) archive config.""" + return f"create: {create_sub}/\nupdate: {update_sub}/" + + +def _plot_extract_single_op(ax, df: pd.DataFrame, operation: str): + """ + Draw grouped bars for one extract operation (extract_seq or extract_par). + X-axis groups are the combined (create_subdir, update_subdir) archive + configs, since extract operates on the archive built by both operations. + """ + configs = _extract_configs(df) + n_configs = len(configs) + n_hpss = len(HPSS_ORDER) + + x_base = np.arange(n_configs, dtype=float) + offsets = np.linspace(-(n_hpss - 1) / 2, (n_hpss - 1) / 2, n_hpss) * BAR_WIDTH + + for h_idx, hpss in enumerate(HPSS_ORDER): + means, all_vals, xs = [], [], [] + for c_idx, (create_sub, update_sub) in enumerate(configs): + vals = ( + df[ + (df["operation"] == operation) + & (df["hpss_label"] == hpss) + & (df["create_subdir"] == create_sub) + & (df["update_subdir"] == update_sub) + ]["elapsed_seconds"] + .dropna() + .values + ) + mean = vals.mean() if len(vals) > 0 else 0.0 + means.append(mean) + all_vals.append(vals) + xs.append(x_base[c_idx] + offsets[h_idx]) + + color = HPSS_COLORS[hpss] + ax.bar( + xs, + means, + width=BAR_WIDTH, + color=color, + alpha=0.85, + label=HPSS_LABELS[hpss], + zorder=2, + ) + for x_pos, vals in zip(xs, all_vals): + if len(vals) > 1: + jitter = np.random.uniform( + -BAR_WIDTH * 0.25, BAR_WIDTH * 0.25, size=len(vals) + ) + ax.scatter( + x_pos + jitter, + vals, + color="white", + edgecolors=color, + s=DOT_SIZE, + zorder=3, + alpha=DOT_ALPHA, + linewidths=1.2, + ) + + ax.set_title(OP_TITLES[operation], fontsize=10, fontweight="bold", pad=6) + ax.set_xticks(x_base) + ax.set_xticklabels([_extract_tick_label(c, u) for c, u in configs], fontsize=7) + ax.set_ylabel("Wall-clock time (s)", fontsize=8) + ax.set_xlabel("Archive contents (create → update)", fontsize=8, labelpad=6) + ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) + ax.set_axisbelow(True) + + for rect in ax.patches: + h = rect.get_height() + if h > 0: + ax.text( + rect.get_x() + rect.get_width() / 2, + h * 1.01, + f"{h:.0f}s", + ha="center", + va="bottom", + fontsize=6, + color="#333333", + ) + + +def plot_extract_comparison(ax, df: pd.DataFrame): """ - Extra subplot: sequential vs parallel extract, grouped by (directory, hpss). + Extra subplot: sequential vs parallel extract, grouped by + (archive config, hpss). Each archive config is the *combined* + create+update directory pair, since extract operates on the full + archive assembled by both operations. Uses a hatch pattern to distinguish seq/par within each hpss colour. """ - dir_col = "create_subdir" - n_dirs = len(dirs) + configs = _extract_configs(df) + n_configs = len(configs) n_hpss = len(HPSS_ORDER) ops = ["extract_seq", "extract_par"] hatches = {"extract_seq": "", "extract_par": "////"} n_bars = n_hpss * len(ops) - group_width = n_bars * BAR_WIDTH + 0.15 # total width per dir group - x_base = np.arange(n_dirs) * group_width - bar_positions = [] # (x, height, color, hatch, label_shown) + group_width = n_bars * BAR_WIDTH + 0.15 # total width per config group + x_base = np.arange(n_configs) * group_width - for d_idx, d in enumerate(dirs): + for c_idx, (create_sub, update_sub) in enumerate(configs): for h_idx, hpss in enumerate(HPSS_ORDER): for op_idx, op in enumerate(ops): df_cell = df[ (df["operation"] == op) & (df["hpss_label"] == hpss) - & (df[dir_col] == d) + & (df["create_subdir"] == create_sub) + & (df["update_subdir"] == update_sub) ] vals = df_cell["elapsed_seconds"].dropna().values mean = vals.mean() if len(vals) > 0 else 0.0 - bar_x = x_base[d_idx] + (h_idx * len(ops) + op_idx) * BAR_WIDTH - bar_positions.append( - (bar_x, mean, HPSS_COLORS[hpss], hatches[op], hpss, op) + bar_x = x_base[c_idx] + (h_idx * len(ops) + op_idx) * BAR_WIDTH + ax.bar( + bar_x, + mean, + width=BAR_WIDTH, + color=HPSS_COLORS[hpss], + hatch=hatches[op], + alpha=0.85, + zorder=2, ) - for bar_x, mean, color, hatch, hpss, op in bar_positions: - ax.bar( - bar_x, mean, width=BAR_WIDTH, color=color, hatch=hatch, alpha=0.85, zorder=2 - ) - - ax.set_xticks(x_base + (n_bars / 2 - 0.5) * BAR_WIDTH) - ax.set_xticklabels([d + "/" for d in dirs], fontsize=9) + tick_positions = x_base + (n_bars / 2 - 0.5) * BAR_WIDTH + ax.set_xticks(tick_positions) + ax.set_xticklabels([_extract_tick_label(c, u) for c, u in configs], fontsize=7.5) ax.set_ylabel("Wall-clock time (s)", fontsize=8) - ax.set_xlabel("Directory (archive source)", fontsize=8, labelpad=14) + ax.set_xlabel( + "Archive contents (create subdir → update subdir)", fontsize=8, labelpad=14 + ) ax.set_title( - "Extract: Sequential vs Parallel (speed-up comparison)", + "Extract: Sequential vs Parallel (speed-up comparison)\n" + "Each group = archive built from create subdir + update subdir", fontsize=10, fontweight="bold", pad=6, ) ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) ax.set_axisbelow(True) - _add_dir_annotation(ax, dirs) # Custom legend: colour = hpss, hatch = seq/par hpss_patches = [ @@ -380,13 +497,111 @@ def mean_for(df): ) ax.set_title(OP_TITLES[operation], fontsize=10, fontweight="bold", pad=6) - ax.set_xticks(x_base + (n_hpss * (2 * pair_width + gap) - gap) / 2) + # Tick at the centre of each directory's group of bars + group_centre_offset = (n_hpss * (2 * pair_width + gap) - gap) / 2 + x_ticks = x_base + group_centre_offset + ax.set_xticks(x_ticks) ax.set_xticklabels([d + "/" for d in dirs], fontsize=9) ax.set_ylabel("Wall-clock time (s)", fontsize=8) ax.set_xlabel("Directory processed", fontsize=8, labelpad=14) ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) ax.set_axisbelow(True) - _add_dir_annotation(ax, dirs) + # Pass actual tick x-positions so annotations align with tick labels + _add_dir_annotation(ax, dirs, list(x_ticks)) + + +def _plot_comparison_extract_single_op( + ax, + df_cur: pd.DataFrame, + df_bas: pd.DataFrame, + operation: str, +): + """ + Fig. 2 version of a single extract-op subplot (extract_seq or extract_par). + X-axis = combined (create, update) archive config; bars = current vs baseline + paired within each HPSS group. + """ + configs = _extract_configs(df_cur) + n_configs = len(configs) + n_hpss = len(HPSS_ORDER) + + pair_width = BAR_WIDTH + gap = BAR_WIDTH * 0.3 + group_span = n_hpss * (2 * pair_width + gap) + 0.2 + x_base = np.arange(n_configs) * group_span + + for h_idx, hpss in enumerate(HPSS_ORDER): + color = HPSS_COLORS[hpss] + pair_offset = h_idx * (2 * pair_width + gap) + for c_idx, (create_sub, update_sub) in enumerate(configs): + x_left = x_base[c_idx] + pair_offset + x_right = x_left + pair_width + + def mean_for(df, _op=operation, _h=hpss, _cs=create_sub, _us=update_sub): + v = ( + df[ + (df["operation"] == _op) + & (df["hpss_label"] == _h) + & (df["create_subdir"] == _cs) + & (df["update_subdir"] == _us) + ]["elapsed_seconds"] + .dropna() + .values + ) + return v.mean() if len(v) > 0 else 0.0 + + cur_mean = mean_for(df_cur) + bas_mean = mean_for(df_bas) + + ax.bar( + x_left, + cur_mean, + width=pair_width, + color=color, + alpha=0.85, + zorder=2, + label=HPSS_LABELS[hpss] if c_idx == 0 else "", + ) + ax.bar( + x_right, + bas_mean, + width=pair_width, + color=color, + alpha=0.40, + hatch="////", + zorder=2, + edgecolor=color, + ) + + if bas_mean > 0 and cur_mean > 0: + ratio = cur_mean / bas_mean + top = max(cur_mean, bas_mean) + arrow = ( + "▲" + if ratio >= RATIO_REGRESSION + else ("▼" if ratio <= RATIO_IMPROVEMENT else "") + ) + ax.text( + (x_left + x_right) / 2, + top * 1.03, + f"{arrow}{ratio:.2f}×", + ha="center", + va="bottom", + fontsize=6.5, + fontweight="bold", + color=_ratio_color(ratio), + zorder=4, + ) + + group_centre_offset = (n_hpss * (2 * pair_width + gap) - gap) / 2 + x_ticks = x_base + group_centre_offset + ax.set_xticks(x_ticks) + ax.set_xticklabels([_extract_tick_label(c, u) for c, u in configs], fontsize=7) + ax.set_ylabel("Wall-clock time (s)", fontsize=8) + ax.set_xlabel("Archive contents (create → update)", fontsize=8, labelpad=6) + ax.set_title(OP_TITLES[operation], fontsize=10, fontweight="bold", pad=6) + ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) + ax.set_axisbelow(True) def plot_comparison_extract( @@ -395,32 +610,51 @@ def plot_comparison_extract( df_bas: pd.DataFrame, dirs: list[str], ): - """Seq vs par extract comparison with current/baseline pairing.""" + """ + Seq vs par extract comparison with current/baseline pairing. + + Bar order within each HPSS × op cell (innermost grouping): + [current/seq] [baseline/seq] [current/par] [baseline/par] + + The seq pair and par pair are visually separated by a slightly larger + intra-group gap, making it clear which two bars belong together. + """ dir_col = "create_subdir" n_dirs = len(dirs) ops = ["extract_seq", "extract_par"] + # Solid = current, hatched = baseline (matches the other Fig. 2 subplots) op_hatches = {"extract_seq": "", "extract_par": "xxxx"} - pair_width = BAR_WIDTH - gap = BAR_WIDTH * 0.3 - group_span = len(HPSS_ORDER) * len(ops) * (2 * pair_width + gap) + 0.3 + pair_width = BAR_WIDTH # width of each individual bar + inner_gap = BAR_WIDTH * 0.15 # gap between current/baseline within a pair + op_gap = BAR_WIDTH * 0.55 # larger gap between seq-pair and par-pair + hpss_gap = BAR_WIDTH * 0.30 # gap between HPSS groups + + # Width of one HPSS group: seq-pair + op_gap + par-pair + # Each pair = 2 * pair_width + inner_gap + pair_span = 2 * pair_width + inner_gap + hpss_group_span = 2 * pair_span + op_gap + + group_span = len(HPSS_ORDER) * (hpss_group_span + hpss_gap) + 0.3 x_base = np.arange(n_dirs) * group_span for d_idx, d in enumerate(dirs): - slot = 0 for h_idx, hpss in enumerate(HPSS_ORDER): color = HPSS_COLORS[hpss] - for op in ops: - x_left = x_base[d_idx] + slot * (2 * pair_width + gap) - x_right = x_left + pair_width + hpss_origin = x_base[d_idx] + h_idx * (hpss_group_span + hpss_gap) + for op_idx, op in enumerate(ops): hatch = op_hatches[op] + op_origin = hpss_origin + op_idx * (pair_span + op_gap) - def mean_for(df): + x_cur = op_origin # current branch bar + x_bas = op_origin + pair_width + inner_gap # baseline bar + + def mean_for(df, _op=op, _hpss=hpss, _d=d): v = ( df[ - (df["operation"] == op) - & (df["hpss_label"] == hpss) - & (df[dir_col] == d) + (df["operation"] == _op) + & (df["hpss_label"] == _hpss) + & (df[dir_col] == _d) ]["elapsed_seconds"] .dropna() .values @@ -430,8 +664,9 @@ def mean_for(df): cur_mean = mean_for(df_cur) bas_mean = mean_for(df_bas) + # Current bar (solid, full alpha) ax.bar( - x_left, + x_cur, cur_mean, width=pair_width, color=color, @@ -439,8 +674,9 @@ def mean_for(df): alpha=0.85, zorder=2, ) + # Baseline bar (hatched overlay, lighter alpha) ax.bar( - x_right, + x_bas, bas_mean, width=pair_width, color=color, @@ -460,7 +696,7 @@ def mean_for(df): else ("▼" if ratio <= RATIO_IMPROVEMENT else "") ) ax.text( - (x_left + x_right) / 2, + (x_cur + x_bas) / 2 + pair_width / 2, top * 1.03, f"{arrow}{ratio:.2f}×", ha="center", @@ -470,9 +706,11 @@ def mean_for(df): color=rat_color, zorder=4, ) - slot += 1 - ax.set_xticks(x_base + group_span / 2 - (2 * pair_width + gap) / 2) + # Tick at centre of each directory's full group + group_total_bar_span = len(HPSS_ORDER) * (hpss_group_span + hpss_gap) - hpss_gap + x_ticks = x_base + group_total_bar_span / 2 + ax.set_xticks(x_ticks) ax.set_xticklabels([d + "/" for d in dirs], fontsize=9) ax.set_ylabel("Wall-clock time (s)", fontsize=8) ax.set_xlabel("Directory (archive source)", fontsize=8, labelpad=14) @@ -484,9 +722,9 @@ def mean_for(df): ) ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) ax.set_axisbelow(True) - _add_dir_annotation(ax, dirs) + _add_dir_annotation(ax, dirs, list(x_ticks)) - # Legend + # Legend: colour=hpss, hatch=seq/par, alpha=current/baseline hpss_patches = [ mpatches.Patch(color=HPSS_COLORS[h], label=HPSS_LABELS[h]) for h in HPSS_ORDER ] @@ -542,7 +780,10 @@ def build_comparison_figure( ax_cmp = fig.add_subplot(gs[2, :]) for op in OP_ORDER: - plot_comparison_operation(axes[op], df_cur, df_bas, op, all_dirs) + if op in OP_DIR_COL: + plot_comparison_operation(axes[op], df_cur, df_bas, op, all_dirs) + else: + _plot_comparison_extract_single_op(axes[op], df_cur, df_bas, op) # Shared legend for solid=current, hatched=baseline cur_patch = mpatches.Patch(facecolor="grey", alpha=0.85, label="Current branch") @@ -559,7 +800,6 @@ def build_comparison_figure( ) plot_comparison_extract(ax_cmp, df_cur, df_bas, all_dirs) - return fig @@ -634,10 +874,15 @@ def main(): # Draw the four single-operation subplots # ----------------------------------------------------------------------- legend_handles = None - for op in OP_ORDER: + for op in ["create", "update", "extract_seq", "extract_par"]: ax = axes[op] - df_op = df[df["operation"] == op] - plot_operation(ax, df_op, op, all_dirs) + if op in OP_DIR_COL: + df_op = df[df["operation"] == op] + plot_operation(ax, df_op, op, all_dirs) + else: + # extract_seq / extract_par each get a dedicated single-op view + # that still uses the combined archive config as the x-axis. + _plot_extract_single_op(ax, df, op) if legend_handles is None: legend_handles = [ @@ -649,7 +894,7 @@ def main(): # ----------------------------------------------------------------------- # Draw the sequential vs parallel comparison subplot # ----------------------------------------------------------------------- - plot_extract_comparison(ax_cmp, df, all_dirs) + plot_extract_comparison(ax_cmp, df) # ----------------------------------------------------------------------- # Baseline comparison figure (Figure 2) From 10539e080a38550ec9539465cd099eefa0ea6364 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 26 Feb 2026 13:21:15 -0800 Subject: [PATCH 08/27] Update parameters --- .../generate_performance_data.bash | 2 +- tests/performance/visualize_performance.py | 36 +++++++++---------- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/tests/performance/generate_performance_data.bash b/tests/performance/generate_performance_data.bash index f2a8d290..bc285fe2 100755 --- a/tests/performance/generate_performance_data.bash +++ b/tests/performance/generate_performance_data.bash @@ -405,4 +405,4 @@ for test_idx in 0 1 2 3 4 5; do done print_success "All tests completed. Results saved to: ${results_csv}" -print_info "Now run: python visualize_performance.py ${results_csv}" +print_info "Now edit IO paths and run: python visualize_performance.py" diff --git a/tests/performance/visualize_performance.py b/tests/performance/visualize_performance.py index 85f5bcf4..d598fffd 100644 --- a/tests/performance/visualize_performance.py +++ b/tests/performance/visualize_performance.py @@ -50,11 +50,19 @@ # Baseline config ← EDIT THIS to point at the main-branch profiling run # --------------------------------------------------------------------------- -# Set to the results.csv of the baseline (main branch) run, e.g.: -# /global/cfs/cdirs/e3sm/forsyth/zstash_performance/performance_20260101/results.csv -# Set to None to skip the baseline comparison figure. +# The results to show in Fig. 1 +RESULTS_CSV = ( + "/pscratch/sd/f/forsyth/zstash_performance/performance_20260225/results.csv" +) + +# The results to compare against in Fig. 2 +# Set to None to skip Fig. 2. BASELINE_RESULTS_CSV = None +# Set to None to display interactively instead of saving. +OUTPUT_PATH = "/global/cfs/cdirs/e3sm/www/forsyth/zstash_performance/performance.png" + + # --------------------------------------------------------------------------- # Config # --------------------------------------------------------------------------- @@ -816,22 +824,12 @@ def main(): parser = argparse.ArgumentParser( description="Visualise zstash performance results." ) - parser.add_argument( - "csv", help="Path to results.csv produced by performance_profile.sh" - ) - parser.add_argument( - "--output", - default="/global/cfs/cdirs/e3sm/www/forsyth/zstash_performance/performance.png", - help="Save figure to this path. " - "Defaults to the NERSC web server output directory. " - "Pass --output '' to display interactively instead.", - ) parser.add_argument( "--dpi", type=int, default=150, help="Output DPI (default: 150)" ) args = parser.parse_args() - df = load_data(args.csv) + df = load_data(RESULTS_CSV) if df.empty: print("ERROR: CSV is empty or could not be parsed.", file=sys.stderr) @@ -912,7 +910,7 @@ def main(): # Derive a short label from the baseline CSV path for titles # e.g. ".../performance_20260101/results.csv" → "performance_20260101" bas_label = bas_path.parent.name - cur_label = Path(args.csv).parent.name + cur_label = Path(RESULTS_CSV).parent.name fig_cmp = build_comparison_figure( df, df_bas, all_dirs, cur_label, bas_label ) @@ -936,12 +934,10 @@ def save_or_show(figure, out_path_str, label): else: plt.show() - if args.output: - save_or_show(fig, args.output, "Figure 1 (overview)") + if OUTPUT_PATH: + save_or_show(fig, OUTPUT_PATH, "Figure 1 (overview)") if fig_cmp is not None: - # Derive comparison output path by inserting "_vs_baseline" before - # the extension, e.g. zstash_performance.png → zstash_performance_vs_baseline.png - p = Path(args.output) + p = Path(OUTPUT_PATH) cmp_output = str(p.with_stem(p.stem + "_vs_baseline")) save_or_show(fig_cmp, cmp_output, "Figure 2 (baseline comparison)") else: From a8206eeef262b9d49c03ac855c36b5af46116867 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 26 Feb 2026 15:37:38 -0800 Subject: [PATCH 09/27] Fixes made comparing pr402 and pr424 --- tests/performance/visualize_performance.py | 93 ++++++++++------------ 1 file changed, 44 insertions(+), 49 deletions(-) diff --git a/tests/performance/visualize_performance.py b/tests/performance/visualize_performance.py index d598fffd..180357ee 100644 --- a/tests/performance/visualize_performance.py +++ b/tests/performance/visualize_performance.py @@ -3,8 +3,9 @@ visualize_performance.py – Plot zstash performance profiling results. Usage: - python visualize_performance.py results.csv - python visualize_performance.py results.csv --output perf_report.png + python visualize_performance.py + +Edit the constants at the top of this file to point at the CSV(s) to plot. The CSV is produced by performance_profile.sh and has columns: test_label, create_subdir, update_subdir, hpss_label, operation, elapsed_seconds @@ -21,6 +22,8 @@ Layout: 2×2 grid of subplots, one per operation. Within each subplot: - X-axis groups = directory processed (create_subdir or update_subdir) + for create/update; or (create_subdir, update_subdir) + archive config for extract_seq/extract_par. - Bars = HPSS mode (none / hpss / globus), colour-coded - Each test config contributes one bar per (directory, hpss_mode) cell; if multiple configs share the same directory for an operation, their @@ -47,21 +50,23 @@ import pandas as pd # --------------------------------------------------------------------------- -# Baseline config ← EDIT THIS to point at the main-branch profiling run +# ← EDIT THESE for each new run # --------------------------------------------------------------------------- # The results to show in Fig. 1 RESULTS_CSV = ( - "/pscratch/sd/f/forsyth/zstash_performance/performance_20260225/results.csv" + "/pscratch/sd/f/forsyth/zstash_performance/performance_20260226_pr424/results.csv" ) -# The results to compare against in Fig. 2 +# The results to compare against in Fig. 2. # Set to None to skip Fig. 2. -BASELINE_RESULTS_CSV = None +BASELINE_RESULTS_CSV = ( + "/pscratch/sd/f/forsyth/zstash_performance/performance_20260225/results.csv" +) +# Output path for the saved figures. # Set to None to display interactively instead of saving. -OUTPUT_PATH = "/global/cfs/cdirs/e3sm/www/forsyth/zstash_performance/performance.png" - +OUTPUT_PATH = "/global/cfs/cdirs/e3sm/www/forsyth/zstash_performance/performance_pr424_vs_main.png" # --------------------------------------------------------------------------- # Config @@ -331,10 +336,9 @@ def plot_extract_comparison(ax, df: pd.DataFrame): """ configs = _extract_configs(df) n_configs = len(configs) - n_hpss = len(HPSS_ORDER) ops = ["extract_seq", "extract_par"] hatches = {"extract_seq": "", "extract_par": "////"} - n_bars = n_hpss * len(ops) + n_bars = len(HPSS_ORDER) * len(ops) group_width = n_bars * BAR_WIDTH + 0.15 # total width per config group x_base = np.arange(n_configs) * group_width @@ -445,12 +449,12 @@ def plot_comparison_operation( x_left = x_base[d_idx] + pair_offset # current bar x_right = x_base[d_idx] + pair_offset + pair_width # baseline bar - def mean_for(df): + def mean_for(df, _op=operation, _h=hpss, _d=d): v = ( df[ - (df["operation"] == operation) - & (df["hpss_label"] == hpss) - & (df[dir_col] == d) + (df["operation"] == _op) + & (df["hpss_label"] == _h) + & (df[dir_col] == _d) ]["elapsed_seconds"] .dropna() .values @@ -616,53 +620,47 @@ def plot_comparison_extract( ax, df_cur: pd.DataFrame, df_bas: pd.DataFrame, - dirs: list[str], ): """ - Seq vs par extract comparison with current/baseline pairing. + Seq vs par extract comparison with current/baseline pairing, grouped by + the combined (create_subdir, update_subdir) archive config. Bar order within each HPSS × op cell (innermost grouping): - [current/seq] [baseline/seq] [current/par] [baseline/par] - - The seq pair and par pair are visually separated by a slightly larger - intra-group gap, making it clear which two bars belong together. + [current/seq] [baseline/seq] ‹op_gap› [current/par] [baseline/par] """ - dir_col = "create_subdir" - n_dirs = len(dirs) + configs = _extract_configs(df_cur) + n_configs = len(configs) ops = ["extract_seq", "extract_par"] - # Solid = current, hatched = baseline (matches the other Fig. 2 subplots) op_hatches = {"extract_seq": "", "extract_par": "xxxx"} - pair_width = BAR_WIDTH # width of each individual bar + pair_width = BAR_WIDTH inner_gap = BAR_WIDTH * 0.15 # gap between current/baseline within a pair op_gap = BAR_WIDTH * 0.55 # larger gap between seq-pair and par-pair hpss_gap = BAR_WIDTH * 0.30 # gap between HPSS groups - # Width of one HPSS group: seq-pair + op_gap + par-pair - # Each pair = 2 * pair_width + inner_gap pair_span = 2 * pair_width + inner_gap hpss_group_span = 2 * pair_span + op_gap group_span = len(HPSS_ORDER) * (hpss_group_span + hpss_gap) + 0.3 - x_base = np.arange(n_dirs) * group_span + x_base = np.arange(n_configs) * group_span - for d_idx, d in enumerate(dirs): + for c_idx, (create_sub, update_sub) in enumerate(configs): for h_idx, hpss in enumerate(HPSS_ORDER): color = HPSS_COLORS[hpss] - hpss_origin = x_base[d_idx] + h_idx * (hpss_group_span + hpss_gap) + hpss_origin = x_base[c_idx] + h_idx * (hpss_group_span + hpss_gap) for op_idx, op in enumerate(ops): hatch = op_hatches[op] op_origin = hpss_origin + op_idx * (pair_span + op_gap) + x_cur = op_origin + x_bas = op_origin + pair_width + inner_gap - x_cur = op_origin # current branch bar - x_bas = op_origin + pair_width + inner_gap # baseline bar - - def mean_for(df, _op=op, _hpss=hpss, _d=d): + def mean_for(df, _op=op, _h=hpss, _cs=create_sub, _us=update_sub): v = ( df[ (df["operation"] == _op) - & (df["hpss_label"] == _hpss) - & (df[dir_col] == _d) + & (df["hpss_label"] == _h) + & (df["create_subdir"] == _cs) + & (df["update_subdir"] == _us) ]["elapsed_seconds"] .dropna() .values @@ -672,7 +670,6 @@ def mean_for(df, _op=op, _hpss=hpss, _d=d): cur_mean = mean_for(df_cur) bas_mean = mean_for(df_bas) - # Current bar (solid, full alpha) ax.bar( x_cur, cur_mean, @@ -682,7 +679,6 @@ def mean_for(df, _op=op, _hpss=hpss, _d=d): alpha=0.85, zorder=2, ) - # Baseline bar (hatched overlay, lighter alpha) ax.bar( x_bas, bas_mean, @@ -697,7 +693,6 @@ def mean_for(df, _op=op, _hpss=hpss, _d=d): if bas_mean > 0 and cur_mean > 0: ratio = cur_mean / bas_mean top = max(cur_mean, bas_mean) - rat_color = _ratio_color(ratio) arrow = ( "▲" if ratio >= RATIO_REGRESSION @@ -711,26 +706,27 @@ def mean_for(df, _op=op, _hpss=hpss, _d=d): va="bottom", fontsize=5.5, fontweight="bold", - color=rat_color, + color=_ratio_color(ratio), zorder=4, ) - # Tick at centre of each directory's full group group_total_bar_span = len(HPSS_ORDER) * (hpss_group_span + hpss_gap) - hpss_gap x_ticks = x_base + group_total_bar_span / 2 ax.set_xticks(x_ticks) - ax.set_xticklabels([d + "/" for d in dirs], fontsize=9) + ax.set_xticklabels([_extract_tick_label(c, u) for c, u in configs], fontsize=7.5) ax.set_ylabel("Wall-clock time (s)", fontsize=8) - ax.set_xlabel("Directory (archive source)", fontsize=8, labelpad=14) + ax.set_xlabel( + "Archive contents (create subdir → update subdir)", fontsize=8, labelpad=14 + ) ax.set_title( - "Extract: Sequential vs Parallel — current vs baseline", + "Extract: Sequential vs Parallel — current vs baseline\n" + "Each group = archive built from create subdir + update subdir", fontsize=10, fontweight="bold", pad=6, ) ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) ax.set_axisbelow(True) - _add_dir_annotation(ax, dirs, list(x_ticks)) # Legend: colour=hpss, hatch=seq/par, alpha=current/baseline hpss_patches = [ @@ -807,7 +803,7 @@ def build_comparison_figure( loc="upper right", ) - plot_comparison_extract(ax_cmp, df_cur, df_bas, all_dirs) + plot_comparison_extract(ax_cmp, df_cur, df_bas) return fig @@ -872,7 +868,7 @@ def main(): # Draw the four single-operation subplots # ----------------------------------------------------------------------- legend_handles = None - for op in ["create", "update", "extract_seq", "extract_par"]: + for op in OP_ORDER: ax = axes[op] if op in OP_DIR_COL: df_op = df[df["operation"] == op] @@ -907,7 +903,7 @@ def main(): print("Skipping baseline comparison figure.", file=sys.stderr) else: df_bas = load_data(str(bas_path)) - # Derive a short label from the baseline CSV path for titles + # Derive a short label from the CSV path for titles, # e.g. ".../performance_20260101/results.csv" → "performance_20260101" bas_label = bas_path.parent.name cur_label = Path(RESULTS_CSV).parent.name @@ -924,8 +920,7 @@ def save_or_show(figure, out_path_str, label): out_path.parent.mkdir(parents=True, exist_ok=True) figure.savefig(out_path, dpi=args.dpi, bbox_inches="tight") print(f"{label} saved to: {out_path}") - web_permissions = 0o755 - os.chmod(out_path_str, web_permissions) + os.chmod(out_path_str, 0o755) web_path = str(out_path).replace( "/global/cfs/cdirs/e3sm/www/", "https://portal.nersc.gov/cfs/e3sm/", From 85f093d795320f81d42b17e6943dcfd4120727ea Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Fri, 3 Apr 2026 13:24:59 -0700 Subject: [PATCH 10/27] Updates for 2060402 profiling --- tests/performance/generate_performance_data.bash | 4 ++-- tests/performance/visualize_performance.py | 13 +++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/performance/generate_performance_data.bash b/tests/performance/generate_performance_data.bash index bc285fe2..83bc9131 100755 --- a/tests/performance/generate_performance_data.bash +++ b/tests/performance/generate_performance_data.bash @@ -15,7 +15,7 @@ set -e # Run from Perlmutter, so that we can do both # a direct transfer to HPSS & a Globus transfer to Chrysalis work_dir=/pscratch/sd/f/forsyth/zstash_performance/ -unique_id=performance_20260225 +unique_id=performance_20260402 dir_to_copy_from=/global/cfs/cdirs/e3sm/forsyth/E3SMv2/v2.LR.historical_0201/ subdir0=build/ @@ -44,7 +44,7 @@ subdir2=init/ # For `--hpss=...` # Which HPSS options to run. Comment out any you want to skip. # Options: "none" "hpss" "globus" -HPSS_OPTIONS=("none" "hpss") # globus endpoint currently down +HPSS_OPTIONS=("none" "hpss" "globus") dst_hpss_path=/home/f/forsyth/zstash_performance diff --git a/tests/performance/visualize_performance.py b/tests/performance/visualize_performance.py index 180357ee..79e5f7f5 100644 --- a/tests/performance/visualize_performance.py +++ b/tests/performance/visualize_performance.py @@ -43,6 +43,7 @@ import os import sys from pathlib import Path +from typing import Optional import matplotlib.patches as mpatches import matplotlib.pyplot as plt @@ -54,19 +55,19 @@ # --------------------------------------------------------------------------- # The results to show in Fig. 1 -RESULTS_CSV = ( - "/pscratch/sd/f/forsyth/zstash_performance/performance_20260226_pr424/results.csv" +RESULTS_CSV: str = ( + "/pscratch/sd/f/forsyth/zstash_performance/performance_20260402/results.csv" ) # The results to compare against in Fig. 2. # Set to None to skip Fig. 2. -BASELINE_RESULTS_CSV = ( - "/pscratch/sd/f/forsyth/zstash_performance/performance_20260225/results.csv" -) +BASELINE_RESULTS_CSV: Optional[str] = None # Output path for the saved figures. # Set to None to display interactively instead of saving. -OUTPUT_PATH = "/global/cfs/cdirs/e3sm/www/forsyth/zstash_performance/performance_pr424_vs_main.png" +OUTPUT_PATH: Optional[str] = ( + "/global/cfs/cdirs/e3sm/www/forsyth/zstash_performance/performance__20260402_pr427.png" +) # --------------------------------------------------------------------------- # Config From 1768a8f66376eef00427c1b25b7ad38c1357c0a8 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Tue, 14 Apr 2026 15:22:26 -0700 Subject: [PATCH 11/27] Enable changing environment --- tests/performance/generate_performance_data.bash | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/performance/generate_performance_data.bash b/tests/performance/generate_performance_data.bash index 83bc9131..3a72df4f 100755 --- a/tests/performance/generate_performance_data.bash +++ b/tests/performance/generate_performance_data.bash @@ -16,6 +16,7 @@ set -e # a direct transfer to HPSS & a Globus transfer to Chrysalis work_dir=/pscratch/sd/f/forsyth/zstash_performance/ unique_id=performance_20260402 +environment_commands="source /global/common/software/e3sm/anaconda_envs/test_e3sm_unified_1.13.0rc5_pm-cpu.sh" dir_to_copy_from=/global/cfs/cdirs/e3sm/forsyth/E3SMv2/v2.LR.historical_0201/ subdir0=build/ @@ -295,6 +296,10 @@ record_result() ############################################################################### # Main script: +# Make sure we're running from the correct environment. +# It might not necessarily be a dev environment built off this branch! +${environment_commands} + validate_configuration "$dir_to_copy_from" "$subdir0" "$subdir1" "$subdir2" if [ "${fresh_globus}" == "true" ] && [[ " ${HPSS_OPTIONS[*]} " == *" globus "* ]]; then From 1cb3eb8720be4a76b81cfc44007a71f5df41361f Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Tue, 2 Jun 2026 18:02:20 -0700 Subject: [PATCH 12/27] Fix issues in comparison plots --- tests/performance/visualize_performance.py | 75 ++++++++++++---------- 1 file changed, 41 insertions(+), 34 deletions(-) diff --git a/tests/performance/visualize_performance.py b/tests/performance/visualize_performance.py index 79e5f7f5..45692301 100644 --- a/tests/performance/visualize_performance.py +++ b/tests/performance/visualize_performance.py @@ -447,8 +447,8 @@ def plot_comparison_operation( pair_offset = h_idx * (2 * pair_width + gap) for d_idx, d in enumerate(dirs): - x_left = x_base[d_idx] + pair_offset # current bar - x_right = x_base[d_idx] + pair_offset + pair_width # baseline bar + x_left = x_base[d_idx] + pair_offset # baseline bar + x_right = x_base[d_idx] + pair_offset + pair_width # current bar def mean_for(df, _op=operation, _h=hpss, _d=d): v = ( @@ -465,26 +465,26 @@ def mean_for(df, _op=operation, _h=hpss, _d=d): cur_mean = mean_for(df_cur) bas_mean = mean_for(df_bas) - # Current bar (solid) + # Baseline bar (hatched, lighter) — left ax.bar( x_left, - cur_mean, + bas_mean, width=pair_width, color=color, - alpha=0.85, + alpha=0.40, + hatch="////", zorder=2, - label=HPSS_LABELS[hpss] if d_idx == 0 else "", + edgecolor=color, ) - # Baseline bar (hatched, lighter) + # Current bar (solid) — right ax.bar( x_right, - bas_mean, + cur_mean, width=pair_width, color=color, - alpha=0.40, - hatch="////", + alpha=0.85, zorder=2, - edgecolor=color, + label=HPSS_LABELS[hpss] if d_idx == 0 else "", ) # Ratio annotation @@ -566,24 +566,26 @@ def mean_for(df, _op=operation, _h=hpss, _cs=create_sub, _us=update_sub): cur_mean = mean_for(df_cur) bas_mean = mean_for(df_bas) + # Baseline bar (hatched, lighter) — left ax.bar( x_left, - cur_mean, + bas_mean, width=pair_width, color=color, - alpha=0.85, + alpha=0.40, + hatch="////", zorder=2, - label=HPSS_LABELS[hpss] if c_idx == 0 else "", + edgecolor=color, ) + # Current bar (solid) — right ax.bar( x_right, - bas_mean, + cur_mean, width=pair_width, color=color, - alpha=0.40, - hatch="////", + alpha=0.85, zorder=2, - edgecolor=color, + label=HPSS_LABELS[hpss] if c_idx == 0 else "", ) if bas_mean > 0 and cur_mean > 0: @@ -652,8 +654,8 @@ def plot_comparison_extract( for op_idx, op in enumerate(ops): hatch = op_hatches[op] op_origin = hpss_origin + op_idx * (pair_span + op_gap) - x_cur = op_origin - x_bas = op_origin + pair_width + inner_gap + x_bas = op_origin # baseline — left + x_cur = op_origin + pair_width + inner_gap # current — right def mean_for(df, _op=op, _h=hpss, _cs=create_sub, _us=update_sub): v = ( @@ -671,24 +673,27 @@ def mean_for(df, _op=op, _h=hpss, _cs=create_sub, _us=update_sub): cur_mean = mean_for(df_cur) bas_mean = mean_for(df_bas) + # Baseline bar (left): op-hatch + //// to mark it as baseline + bas_hatch = hatch + "////" ax.bar( - x_cur, - cur_mean, + x_bas, + bas_mean, width=pair_width, color=color, - hatch=hatch, - alpha=0.85, + hatch=bas_hatch, + alpha=0.35, zorder=2, + edgecolor=color, ) + # Current bar (right): op-hatch only ax.bar( - x_bas, - bas_mean, + x_cur, + cur_mean, width=pair_width, color=color, hatch=hatch, - alpha=0.35, + alpha=0.85, zorder=2, - edgecolor=color, ) if bas_mean > 0 and cur_mean > 0: @@ -734,17 +739,19 @@ def mean_for(df, _op=op, _h=hpss, _cs=create_sub, _us=update_sub): mpatches.Patch(color=HPSS_COLORS[h], label=HPSS_LABELS[h]) for h in HPSS_ORDER ] seq_patch = mpatches.Patch( - facecolor="grey", hatch="", label="Sequential (1 worker)" + facecolor="grey", hatch="", alpha=0.85, label="Sequential, current" + ) + seq_bas_patch = mpatches.Patch( + facecolor="grey", hatch="////", alpha=0.35, label="Sequential, baseline" ) par_patch = mpatches.Patch( - facecolor="grey", hatch="xxxx", label="Parallel (2 workers)" + facecolor="grey", hatch="xxxx", alpha=0.85, label="Parallel, current" ) - cur_patch = mpatches.Patch(facecolor="grey", alpha=0.85, label="Current branch") - bas_patch = mpatches.Patch( - facecolor="grey", alpha=0.35, label="Baseline (main)", hatch="////" + par_bas_patch = mpatches.Patch( + facecolor="grey", hatch="xxxx////", alpha=0.35, label="Parallel, baseline" ) ax.legend( - handles=hpss_patches + [seq_patch, par_patch, cur_patch, bas_patch], + handles=hpss_patches + [seq_patch, seq_bas_patch, par_patch, par_bas_patch], fontsize=6.5, loc="upper right", ncol=3, From 511233135dd83d4c9dbf0d038a1ee513a1937ba3 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 3 Jun 2026 12:25:03 -0700 Subject: [PATCH 13/27] Update paths --- tests/performance/visualize_performance.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/performance/visualize_performance.py b/tests/performance/visualize_performance.py index 45692301..920eec06 100644 --- a/tests/performance/visualize_performance.py +++ b/tests/performance/visualize_performance.py @@ -56,17 +56,19 @@ # The results to show in Fig. 1 RESULTS_CSV: str = ( - "/pscratch/sd/f/forsyth/zstash_performance/performance_20260402/results.csv" + "/pscratch/sd/f/forsyth/zstash_performance/performance_20260414/results.csv" ) # The results to compare against in Fig. 2. # Set to None to skip Fig. 2. -BASELINE_RESULTS_CSV: Optional[str] = None +BASELINE_RESULTS_CSV: Optional[str] = ( + "/pscratch/sd/f/forsyth/zstash_performance/performance_20260402/results.csv" +) # Output path for the saved figures. # Set to None to display interactively instead of saving. OUTPUT_PATH: Optional[str] = ( - "/global/cfs/cdirs/e3sm/www/forsyth/zstash_performance/performance__20260402_pr427.png" + "/global/cfs/cdirs/e3sm/www/forsyth/zstash_performance/performance_pr427_20260603.png" ) # --------------------------------------------------------------------------- From 4ff7f4399afc1a59220e0efc95355769f7704296 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 3 Jun 2026 13:12:47 -0700 Subject: [PATCH 14/27] Document process in README --- tests/README.md | 4 + tests/performance/README.md | 85 +++++++++++++++++++ .../generate_performance_data.bash | 9 ++ 3 files changed, 98 insertions(+) create mode 100644 tests/performance/README.md diff --git a/tests/README.md b/tests/README.md index c4552471..3c825fac 100644 --- a/tests/README.md +++ b/tests/README.md @@ -109,3 +109,7 @@ GitHub Actions runs the tests according to `.github/workflows/build_workflow.yml python -m unittest tests/integration/python_tests/group_by_command/test_*.py python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py ``` + +## Performance + +For performance profiling, see `tests/performance/README.md`. diff --git a/tests/performance/README.md b/tests/performance/README.md new file mode 100644 index 00000000..b1abd30f --- /dev/null +++ b/tests/performance/README.md @@ -0,0 +1,85 @@ +# How to profile zstash's performance + +Performance profiling should be done on Perlmutter. We're keeping the performance records in `/global/homes/f/forsyth/zstash_performance_records`. + +## Generate performance data.bash + +In `zstash/tests/performance/generate_performance_data.bash`, edit the run metadata: + +```bash +# The performance data will end up in `results_csv="${work_dir}${unique_id}/results.csv"` +# Use `/pscratch` since a lot of data will be transferred. +# The results csv alone will be copied to a long-term (i.e., non-scratch) directory at the end. +work_dir=/pscratch/sd/f/forsyth/zstash_performance/ +unique_id=performance_20260603 + +# This is the environment that zstash will be run in. +# Using Unified environment: +environment_commands="source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh" +# Example dev environment: +environment_commands="source /global/homes/f/forsyth/miniforge3/etc/profile.d/conda.sh; conda activate zstash-pr427-20260603" +``` + +These parameters you probably won't have a need to change: +```bash +# These are the directories to run `zstash create`, `zstash update`, and `zstash extract` on. +dir_to_copy_from=/global/cfs/cdirs/e3sm/forsyth/E3SMv2/v2.LR.historical_0201/ +subdir0=build/ +subdir1=run/ +subdir2=init/ + +# This specifies which `--hpss` settings will be run: +HPSS_OPTIONS=("none" "hpss" "globus") + +# This is what will be used for the "hpss" option: +dst_hpss_path=/home/f/forsyth/zstash_performance + +# This is what will be used the "globus" option: +fresh_globus=true # This will prompt a fresh Globus authentication +dst_endpoint_uuid=15288284-7006-4041-ba1a-6b52501e49f1 # This is LCRC's endpoint +dst_endpoint_archive_dir=/lcrc/group/e3sm/ac.forsyth2/zstash_performance_dst_dir/ +``` + +Results will be saved to `${results_csv}` (recall `results_csv="${work_dir}${unique_id}/results.csv`). To keep all records together in a non-scratch space, the results csv is also copied to: `/global/homes/f/forsyth/zstash_performance_records/${unique_id}_results.csv`. + +## Visualize performance + +In `zstash/tests/performance/visualize_performance.py`, edit the run metadata: + +```python +# The results to show in Fig. 1 +# This should be the results.csv you just generated in the step above. +RESULTS_CSV: str = "" + +# The results to compare against in Fig. 2. +# Set to None to skip Fig. 2. +# This will typically be the second-to-oldest results.csv in the records space +BASELINE_RESULTS_CSV: Optional[str] = "" + +# Output path for the saved figures. +# Set to None to display interactively instead of saving. +# Make sure to put this on the web server path, +# i.e., /global/cfs/cdirs/e3sm/www/... +OUTPUT_PATH: Optional[str] = "" +``` + +## For reference + +Records made before the long-term record space was made have been copied to it via: +```bash +SCRATCH_SPACE=/pscratch/sd/f/forsyth/zstash_performance +RECORDS_SPACE=/global/homes/f/forsyth/zstash_performance_records + +for unique_id in \ + performance_20260225 \ + performance_20260226_pr402 \ + performance_20260226_pr424 \ + performance_20260226_pr428 \ + performance_20260402 \ + performance_20260414 \ + performance_pr416_20260403 \ + performance_pr416_20260406 +do + cp "${SCRATCH_SPACE}/${unique_id}/results.csv" "${RECORDS_SPACE}/${unique_id}_results.csv" +done +``` diff --git a/tests/performance/generate_performance_data.bash b/tests/performance/generate_performance_data.bash index 3a72df4f..d401cf7a 100755 --- a/tests/performance/generate_performance_data.bash +++ b/tests/performance/generate_performance_data.bash @@ -18,6 +18,10 @@ work_dir=/pscratch/sd/f/forsyth/zstash_performance/ unique_id=performance_20260402 environment_commands="source /global/common/software/e3sm/anaconda_envs/test_e3sm_unified_1.13.0rc5_pm-cpu.sh" +############################################################################### +# These parameters don't usually need to be changed, +# but can be changed for further customization. + dir_to_copy_from=/global/cfs/cdirs/e3sm/forsyth/E3SMv2/v2.LR.historical_0201/ subdir0=build/ subdir1=run/ @@ -410,4 +414,9 @@ for test_idx in 0 1 2 3 4 5; do done print_success "All tests completed. Results saved to: ${results_csv}" + +performance_archive_path=/global/homes/f/forsyth/zstash_performance_records/${unique_id}_results.csv +cp ${results_csv} ${performance_archive_path} +print_success "Results copied to: ${performance_archive_path}" + print_info "Now edit IO paths and run: python visualize_performance.py" From 258c90cd03622061ebde12d7ce2054184693f64d Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 3 Jun 2026 14:39:08 -0700 Subject: [PATCH 15/27] Address comments --- tests/performance/README.md | 8 ++++---- .../performance/generate_performance_data.bash | 3 ++- tests/performance/visualize_performance.py | 18 ++++++++++++------ 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/tests/performance/README.md b/tests/performance/README.md index b1abd30f..6df40f11 100644 --- a/tests/performance/README.md +++ b/tests/performance/README.md @@ -1,6 +1,6 @@ # How to profile zstash's performance -Performance profiling should be done on Perlmutter. We're keeping the performance records in `/global/homes/f/forsyth/zstash_performance_records`. +Performance profiling should be done on Perlmutter. We're keeping the performance records in `/global/homes/f/forsyth/zstash_performance_records`. (NOTE: this is currently user-specific. If we start having many other developers running performance profiling, we may try to find a more centralized location.) ## Generate performance data.bash @@ -49,18 +49,18 @@ In `zstash/tests/performance/visualize_performance.py`, edit the run metadata: ```python # The results to show in Fig. 1 # This should be the results.csv you just generated in the step above. -RESULTS_CSV: str = "" +RESULTS_CSV: str = "/pscratch/sd/f/forsyth/zstash_performance/performance_20260414/results.csv" # The results to compare against in Fig. 2. # Set to None to skip Fig. 2. # This will typically be the second-to-oldest results.csv in the records space -BASELINE_RESULTS_CSV: Optional[str] = "" +BASELINE_RESULTS_CSV: Optional[str] = "/pscratch/sd/f/forsyth/zstash_performance/performance_20260402/results.csv" # Output path for the saved figures. # Set to None to display interactively instead of saving. # Make sure to put this on the web server path, # i.e., /global/cfs/cdirs/e3sm/www/... -OUTPUT_PATH: Optional[str] = "" +OUTPUT_PATH: Optional[str] = "/global/cfs/cdirs/e3sm/www/forsyth/zstash_performance/performance_pr427_20260603.png" ``` ## For reference diff --git a/tests/performance/generate_performance_data.bash b/tests/performance/generate_performance_data.bash index d401cf7a..a663d11d 100755 --- a/tests/performance/generate_performance_data.bash +++ b/tests/performance/generate_performance_data.bash @@ -1,5 +1,6 @@ #!/bin/bash set -e +set -o pipefail # Analogous to CI/CD matrix testing of Python versions, # here we will do a matrix performance profiling @@ -416,7 +417,7 @@ done print_success "All tests completed. Results saved to: ${results_csv}" performance_archive_path=/global/homes/f/forsyth/zstash_performance_records/${unique_id}_results.csv -cp ${results_csv} ${performance_archive_path} +cp "${results_csv}" "${performance_archive_path}" print_success "Results copied to: ${performance_archive_path}" print_info "Now edit IO paths and run: python visualize_performance.py" diff --git a/tests/performance/visualize_performance.py b/tests/performance/visualize_performance.py index 920eec06..fa69eede 100644 --- a/tests/performance/visualize_performance.py +++ b/tests/performance/visualize_performance.py @@ -7,7 +7,7 @@ Edit the constants at the top of this file to point at the CSV(s) to plot. -The CSV is produced by performance_profile.sh and has columns: + The CSV is produced by generate_performance_data.bash and has columns: test_label, create_subdir, update_subdir, hpss_label, operation, elapsed_seconds Visualization strategy @@ -835,10 +835,16 @@ def main(): ) args = parser.parse_args() - df = load_data(RESULTS_CSV) - + results_path = Path(RESULTS_CSV) + if not RESULTS_CSV or not results_path.is_file(): + print(f"ERROR: RESULTS_CSV not found: {RESULTS_CSV!r}", file=sys.stderr) + sys.exit(1) + df = load_data(str(results_path)) if df.empty: - print("ERROR: CSV is empty or could not be parsed.", file=sys.stderr) + print( + f"ERROR: RESULTS_CSV is empty or could not be parsed: {RESULTS_CSV!r}", + file=sys.stderr, + ) sys.exit(1) # Determine the sorted list of directories that actually appear in the data @@ -904,7 +910,7 @@ def main(): # Baseline comparison figure (Figure 2) # ----------------------------------------------------------------------- fig_cmp = None - if BASELINE_RESULTS_CSV is not None: + if BASELINE_RESULTS_CSV: bas_path = Path(BASELINE_RESULTS_CSV) if not bas_path.exists(): print( @@ -930,7 +936,7 @@ def save_or_show(figure, out_path_str, label): out_path.parent.mkdir(parents=True, exist_ok=True) figure.savefig(out_path, dpi=args.dpi, bbox_inches="tight") print(f"{label} saved to: {out_path}") - os.chmod(out_path_str, 0o755) + os.chmod(out_path_str, 0o644) web_path = str(out_path).replace( "/global/cfs/cdirs/e3sm/www/", "https://portal.nersc.gov/cfs/e3sm/", From 2fab8b2caf3704547c5e45d94721925e03d2ea48 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 3 Jun 2026 16:57:33 -0700 Subject: [PATCH 16/27] Update IO paths --- tests/performance/generate_performance_data.bash | 4 ++-- tests/performance/visualize_performance.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/performance/generate_performance_data.bash b/tests/performance/generate_performance_data.bash index a663d11d..ae067702 100755 --- a/tests/performance/generate_performance_data.bash +++ b/tests/performance/generate_performance_data.bash @@ -16,8 +16,8 @@ set -o pipefail # Run from Perlmutter, so that we can do both # a direct transfer to HPSS & a Globus transfer to Chrysalis work_dir=/pscratch/sd/f/forsyth/zstash_performance/ -unique_id=performance_20260402 -environment_commands="source /global/common/software/e3sm/anaconda_envs/test_e3sm_unified_1.13.0rc5_pm-cpu.sh" +unique_id=performance_20260603 +environment_commands="source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh" ############################################################################### # These parameters don't usually need to be changed, diff --git a/tests/performance/visualize_performance.py b/tests/performance/visualize_performance.py index fa69eede..f6df996e 100644 --- a/tests/performance/visualize_performance.py +++ b/tests/performance/visualize_performance.py @@ -56,19 +56,19 @@ # The results to show in Fig. 1 RESULTS_CSV: str = ( - "/pscratch/sd/f/forsyth/zstash_performance/performance_20260414/results.csv" + "/pscratch/sd/f/forsyth/zstash_performance/performance_20260603/results.csv" ) # The results to compare against in Fig. 2. # Set to None to skip Fig. 2. BASELINE_RESULTS_CSV: Optional[str] = ( - "/pscratch/sd/f/forsyth/zstash_performance/performance_20260402/results.csv" + "/pscratch/sd/f/forsyth/zstash_performance/performance_20260414/results.csv" ) # Output path for the saved figures. # Set to None to display interactively instead of saving. OUTPUT_PATH: Optional[str] = ( - "/global/cfs/cdirs/e3sm/www/forsyth/zstash_performance/performance_pr427_20260603.png" + "/global/cfs/cdirs/e3sm/www/forsyth/zstash_performance/performance_pr427_20260603_run2.png" ) # --------------------------------------------------------------------------- From 01beff5872da2789befe7d98bb09226ee47dc961 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 4 Jun 2026 09:04:06 -0700 Subject: [PATCH 17/27] Address comments --- tests/performance/README.md | 14 ++++++++++++++ tests/performance/generate_performance_data.bash | 4 ---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/performance/README.md b/tests/performance/README.md index 6df40f11..dbaa3045 100644 --- a/tests/performance/README.md +++ b/tests/performance/README.md @@ -40,6 +40,12 @@ dst_endpoint_uuid=15288284-7006-4041-ba1a-6b52501e49f1 # This is LCRC's endpoint dst_endpoint_archive_dir=/lcrc/group/e3sm/ac.forsyth2/zstash_performance_dst_dir/ ``` +Once you have the parameters set up, run: +```bash +cd tests/performance/ +./generate_performance_data.bash +``` + Results will be saved to `${results_csv}` (recall `results_csv="${work_dir}${unique_id}/results.csv`). To keep all records together in a non-scratch space, the results csv is also copied to: `/global/homes/f/forsyth/zstash_performance_records/${unique_id}_results.csv`. ## Visualize performance @@ -63,6 +69,14 @@ BASELINE_RESULTS_CSV: Optional[str] = "/pscratch/sd/f/forsyth/zstash_performance OUTPUT_PATH: Optional[str] = "/global/cfs/cdirs/e3sm/www/forsyth/zstash_performance/performance_pr427_20260603.png" ``` +Once you have the parameters set up, run: +```bash +cd tests/performance/ +python visualize_performance.py +``` + +The script will print both the file path and the URL to access the plots. + ## For reference Records made before the long-term record space was made have been copied to it via: diff --git a/tests/performance/generate_performance_data.bash b/tests/performance/generate_performance_data.bash index ae067702..160d9540 100755 --- a/tests/performance/generate_performance_data.bash +++ b/tests/performance/generate_performance_data.bash @@ -331,7 +331,6 @@ declare -a test_configs=( "2 0" "2 1" ) -# FIX: removed erroneous commas that were present in the original array declare -a test_labels=("01" "02" "10" "12" "20" "21") # Loop through the 6 test configurations @@ -360,8 +359,6 @@ for test_idx in 0 1 2 3 4 5; do log_dir="${work_subdir}logs/" mkdir -p "${log_dir}" - # FIX: renamed `dst_hpss` local var to `dst_globus_path` so it does not - # shadow/overwrite the top-level `dst_hpss_path` parameter. dst_globus_path="globus://${dst_endpoint_uuid}/${dst_endpoint_archive_dir}${unique_id}/test${test_label}/" # Iterate over the three HPSS modes @@ -398,7 +395,6 @@ for test_idx in 0 1 2 3 4 5; do extract_dir="${mode_dir}extract_${num_workers}workers/" mkdir -p "${extract_dir}" - # FIX: pass num_workers argument (was missing in original) run_extract "$extract_dir" "$hpss_path" "$num_workers" "$cache_dir" "$extract_log" if [ "$num_workers" -eq 1 ]; then From da4823978a91192b2c9bc18a5bd6a0430e06d6bf Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 18 Jun 2026 11:08:35 -0700 Subject: [PATCH 18/27] Address Copilot review comments --- conda/dev.yml | 3 - conda/perf.yml | 17 ++++ tests/performance/README.md | 94 +++++++++++------- .../generate_performance_data.bash | 89 +++++++++++++---- tests/performance/perf.cfg | 97 +++++++++++++++++++ tests/performance/visualize_performance.py | 91 ++++++++++++----- 6 files changed, 311 insertions(+), 80 deletions(-) create mode 100644 conda/perf.yml create mode 100644 tests/performance/perf.cfg diff --git a/conda/dev.yml b/conda/dev.yml index e5a9b2a0..85f64d32 100644 --- a/conda/dev.yml +++ b/conda/dev.yml @@ -23,9 +23,6 @@ dependencies: # ======================= - pytest - pytest-cov - - matplotlib-base - - pandas - - numpy # Documentation # ================= # If versions are updated, also update in `.github/workflows/workflow.yml` diff --git a/conda/perf.yml b/conda/perf.yml new file mode 100644 index 00000000..64f4c8d6 --- /dev/null +++ b/conda/perf.yml @@ -0,0 +1,17 @@ +name: zstash_perf +channels: + - conda-forge +dependencies: + # Base (minimal subset needed to run zstash) + # ================= + - pip + - python >=3.11,<3.15 + - setuptools + - sqlite + - six >=1.16.0 + - globus-sdk >=3.15.0,<4.0 + # Performance profiling + # ================= + - matplotlib-base + - pandas + - numpy diff --git a/tests/performance/README.md b/tests/performance/README.md index dbaa3045..009b6f34 100644 --- a/tests/performance/README.md +++ b/tests/performance/README.md @@ -1,80 +1,104 @@ # How to profile zstash's performance -Performance profiling should be done on Perlmutter. We're keeping the performance records in `/global/homes/f/forsyth/zstash_performance_records`. (NOTE: this is currently user-specific. If we start having many other developers running performance profiling, we may try to find a more centralized location.) +Performance profiling should be done on Perlmutter. We're keeping the performance records in a long-term directory specified by `performance_archive_dir` in your config file (see below). (NOTE: this is currently user-specific. If we start having many other developers running performance profiling, we may try to find a more centralized location.) -## Generate performance data.bash +## Setup -In `zstash/tests/performance/generate_performance_data.bash`, edit the run metadata: +All parameters for both scripts live in a single config file (`perf.cfg`) so you never need to edit the scripts themselves. Copy the provided template and fill in your values: ```bash -# The performance data will end up in `results_csv="${work_dir}${unique_id}/results.csv"` -# Use `/pscratch` since a lot of data will be transferred. -# The results csv alone will be copied to a long-term (i.e., non-scratch) directory at the end. -work_dir=/pscratch/sd/f/forsyth/zstash_performance/ +cd tests/performance/ +cp perf.cfg my_run.cfg # or just edit perf.cfg in place +``` + +The config file uses a simple `key=value` format (lines starting with `#` are comments). It is shared between the bash script and the Python visualizer. + +> **Perlmutter path convention:** home and scratch directories follow the pattern +> `/global/homes/u/username/...` and `/pscratch/sd/u/username/...` +> where `u` is the first letter of your username. +> The placeholder `u/username` in the template should be replaced accordingly. + +## Generate performance data + +Edit the run metadata section of your cfg file: + +```ini +# Use /pscratch since a lot of data will be transferred. +# The results csv alone will be copied to a long-term directory at the end. +work_dir=/pscratch/sd/u/username/zstash_performance/ unique_id=performance_20260603 -# This is the environment that zstash will be run in. +# The environment that zstash will be run in. # Using Unified environment: -environment_commands="source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh" +environment_commands=source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh # Example dev environment: -environment_commands="source /global/homes/f/forsyth/miniforge3/etc/profile.d/conda.sh; conda activate zstash-pr427-20260603" +# environment_commands=source /global/homes/u/username/miniforge3/etc/profile.d/conda.sh ; conda activate zstash-pr427-20260603 + +# Long-term directory where the results CSV is archived after the run. +performance_archive_dir=/global/homes/u/username/zstash_performance_records ``` These parameters you probably won't have a need to change: -```bash -# These are the directories to run `zstash create`, `zstash update`, and `zstash extract` on. + +```ini +# Directories to run zstash create/update/extract on. dir_to_copy_from=/global/cfs/cdirs/e3sm/forsyth/E3SMv2/v2.LR.historical_0201/ subdir0=build/ subdir1=run/ subdir2=init/ -# This specifies which `--hpss` settings will be run: -HPSS_OPTIONS=("none" "hpss" "globus") +# Which --hpss settings to run (space-separated; comment out any to skip): +HPSS_OPTIONS=none hpss globus -# This is what will be used for the "hpss" option: -dst_hpss_path=/home/f/forsyth/zstash_performance +# Used for the "hpss" option: +dst_hpss_path=/home/u/username/zstash_performance -# This is what will be used the "globus" option: -fresh_globus=true # This will prompt a fresh Globus authentication -dst_endpoint_uuid=15288284-7006-4041-ba1a-6b52501e49f1 # This is LCRC's endpoint -dst_endpoint_archive_dir=/lcrc/group/e3sm/ac.forsyth2/zstash_performance_dst_dir/ +# Used for the "globus" option: +fresh_globus=true # prompts a fresh Globus authentication +dst_endpoint_uuid=15288284-7006-4041-ba1a-6b52501e49f1 # LCRC's endpoint +dst_endpoint_archive_dir=/lcrc/group/e3sm/username/zstash_performance_dst_dir/ ``` Once you have the parameters set up, run: + ```bash cd tests/performance/ -./generate_performance_data.bash +./generate_performance_data.bash my_run.cfg ``` -Results will be saved to `${results_csv}` (recall `results_csv="${work_dir}${unique_id}/results.csv`). To keep all records together in a non-scratch space, the results csv is also copied to: `/global/homes/f/forsyth/zstash_performance_records/${unique_id}_results.csv`. +If no cfg file argument is given, the script looks for `perf.cfg` in the same directory. + +Results will be saved to `${work_dir}${unique_id}/results.csv`. To keep all records together in a non-scratch space, the results csv is also copied to `${performance_archive_dir}/${unique_id}_results.csv`. ## Visualize performance -In `zstash/tests/performance/visualize_performance.py`, edit the run metadata: +Edit the visualizer section of your cfg file: -```python -# The results to show in Fig. 1 +```ini +# Path to the results CSV to show in Figure 1. # This should be the results.csv you just generated in the step above. -RESULTS_CSV: str = "/pscratch/sd/f/forsyth/zstash_performance/performance_20260414/results.csv" +results_csv=/pscratch/sd/u/username/zstash_performance/performance_20260603/results.csv -# The results to compare against in Fig. 2. -# Set to None to skip Fig. 2. -# This will typically be the second-to-oldest results.csv in the records space -BASELINE_RESULTS_CSV: Optional[str] = "/pscratch/sd/f/forsyth/zstash_performance/performance_20260402/results.csv" +# Path to a baseline results CSV to compare against in Figure 2. +# Leave blank to skip Figure 2. +# This will typically be the second-to-latest results.csv in the records space. +baseline_results_csv=/pscratch/sd/u/username/zstash_performance/performance_20260414/results.csv # Output path for the saved figures. -# Set to None to display interactively instead of saving. -# Make sure to put this on the web server path, -# i.e., /global/cfs/cdirs/e3sm/www/... -OUTPUT_PATH: Optional[str] = "/global/cfs/cdirs/e3sm/www/forsyth/zstash_performance/performance_pr427_20260603.png" +# Leave blank to display interactively instead of saving. +# Make sure to use the web server path, i.e., /global/cfs/cdirs/e3sm/www/... +output_path=/global/cfs/cdirs/e3sm/www/username/zstash_performance/performance_pr427_20260603.png ``` Once you have the parameters set up, run: + ```bash cd tests/performance/ -python visualize_performance.py +python visualize_performance.py --cfg my_run.cfg ``` +If `--cfg` is omitted, the script looks for `perf.cfg` in the same directory. + The script will print both the file path and the URL to access the plots. ## For reference diff --git a/tests/performance/generate_performance_data.bash b/tests/performance/generate_performance_data.bash index 160d9540..77275cb6 100755 --- a/tests/performance/generate_performance_data.bash +++ b/tests/performance/generate_performance_data.bash @@ -11,22 +11,69 @@ set -o pipefail # We will also compare `zstash extract` in sequential-mode and parallel-mode ############################################################################### -# Manually edit parameters here: +# Configuration file loader +# +# Reads a key=value file (default: perf.cfg in the same directory as this +# script, or the path given as the first argument). +# Lines starting with '#' and blank lines are ignored. +# Multi-word values (e.g. HPSS_OPTIONS) are stored as bash arrays. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CFG_FILE="${1:-${SCRIPT_DIR}/perf.cfg}" + +if [ ! -f "${CFG_FILE}" ]; then + echo "ERROR: config file not found: ${CFG_FILE}" + echo "Usage: $0 [path/to/config.cfg]" + echo "Copy ${SCRIPT_DIR}/perf.cfg and edit it for your run." + exit 1 +fi + +# Parse key=value pairs; skip comments and blank lines. +declare -A _cfg +while IFS='=' read -r key value; do + # Strip leading/trailing whitespace from key + key="${key//[[:space:]]/}" + # Strip leading whitespace from value + value="${value#"${value%%[![:space:]]*}"}" + # Strip trailing whitespace from value + value="${value%"${value##*[![:space:]]}"}" + [[ -z "$key" || "$key" == \#* ]] && continue + _cfg["$key"]="$value" +done < <(grep -v '^[[:space:]]*#' "${CFG_FILE}" | grep -v '^[[:space:]]*$') + +# Helper: get a required value or exit +cfg_require() { + local k="$1" + if [[ -z "${_cfg[$k]+_}" ]]; then + echo "ERROR: required key '${k}' is missing from ${CFG_FILE}" + exit 1 + fi + printf '%s' "${_cfg[$k]}" +} + +# Helper: get an optional value with a default +cfg_get() { + local k="$1" default="$2" + printf '%s' "${_cfg[$k]:-$default}" +} + +############################################################################### +# Load parameters from cfg file # Run from Perlmutter, so that we can do both # a direct transfer to HPSS & a Globus transfer to Chrysalis -work_dir=/pscratch/sd/f/forsyth/zstash_performance/ -unique_id=performance_20260603 -environment_commands="source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh" +work_dir="$(cfg_require work_dir)" +unique_id="$(cfg_require unique_id)" +environment_commands="$(cfg_require environment_commands)" ############################################################################### # These parameters don't usually need to be changed, # but can be changed for further customization. -dir_to_copy_from=/global/cfs/cdirs/e3sm/forsyth/E3SMv2/v2.LR.historical_0201/ -subdir0=build/ -subdir1=run/ -subdir2=init/ +dir_to_copy_from="$(cfg_require dir_to_copy_from)" +subdir0="$(cfg_get subdir0 none)" +subdir1="$(cfg_get subdir1 none)" +subdir2="$(cfg_get subdir2 none)" ### # For reference, these files have these sizes and number of files # (Paths are from Chrysalis, but the data is identical on Perlmutter) @@ -46,24 +93,28 @@ subdir2=init/ # => A few large files ### - # For `--hpss=...` -# Which HPSS options to run. Comment out any you want to skip. +# Which HPSS options to run. Space-separated in the cfg file -> bash array here. # Options: "none" "hpss" "globus" -HPSS_OPTIONS=("none" "hpss" "globus") +IFS=' ' read -r -a HPSS_OPTIONS <<< "$(cfg_require HPSS_OPTIONS)" -dst_hpss_path=/home/f/forsyth/zstash_performance +dst_hpss_path="$(cfg_get dst_hpss_path "")" # For `--hpss=globus...` -fresh_globus=true +fresh_globus="$(cfg_get fresh_globus false)" # ENDPOINT UUIDS: # LCRC_IMPROV_DTN_ENDPOINT=15288284-7006-4041-ba1a-6b52501e49f1 # NERSC_PERLMUTTER_ENDPOINT=6bdc7956-fc0f-4ad2-989c-7aa5ee643a79 # NERSC_HPSS_ENDPOINT=9cd89cfd-6d04-11e5-ba46-22000b92c6ec # PIC_COMPY_DTN_ENDPOINT=68fbd2fa-83d7-11e9-8e63-029d279f7e24 # GLOBUS_TUTORIAL_COLLECTION_1_ENDPOINT=6c54cade-bde5-45c1-bdea-f4bd71dba2cc -dst_endpoint_uuid=15288284-7006-4041-ba1a-6b52501e49f1 -dst_endpoint_archive_dir=/lcrc/group/e3sm/ac.forsyth2/zstash_performance_dst_dir/ +dst_endpoint_uuid="$(cfg_get dst_endpoint_uuid "")" +dst_endpoint_archive_dir="$(cfg_get dst_endpoint_archive_dir "")" + +performance_archive_dir="$(cfg_require performance_archive_dir)" + +echo "[INFO] Loaded configuration from: ${CFG_FILE}" +echo "[INFO] work_dir=${work_dir} unique_id=${unique_id}" ############################################################################### # Utility functions @@ -145,6 +196,9 @@ validate_configuration() refresh_globus() { print_step "Setting up fresh Globus authentication..." + if ! confirm "This will delete ${INI_PATH} and ${TOKEN_FILE} to start fresh. Is that ok?"; then + exit 1 + fi # 1. Activate endpoints echo "Go to https://app.globus.org/file-manager?two_pane=true > For 'Collection', choose the endpoints you're using, and authenticate if needed:" @@ -412,8 +466,9 @@ done print_success "All tests completed. Results saved to: ${results_csv}" -performance_archive_path=/global/homes/f/forsyth/zstash_performance_records/${unique_id}_results.csv +mkdir -p "${performance_archive_dir}" +performance_archive_path="${performance_archive_dir}/${unique_id}_results.csv" cp "${results_csv}" "${performance_archive_path}" print_success "Results copied to: ${performance_archive_path}" -print_info "Now edit IO paths and run: python visualize_performance.py" +print_info "Now run: python visualize_performance.py --cfg ${CFG_FILE}" diff --git a/tests/performance/perf.cfg b/tests/performance/perf.cfg new file mode 100644 index 00000000..01027b9f --- /dev/null +++ b/tests/performance/perf.cfg @@ -0,0 +1,97 @@ +# Performance profiling configuration – shared by both scripts: +# generate_performance_data.bash (pass as first argument, or name it perf.cfg) +# visualize_performance.py (pass via --cfg, or name it perf.cfg) +# Usage: +# ./generate_performance_data.bash [path/to/this/file] +# python visualize_performance.py [--cfg path/to/this/file] +# Default cfg file name (when no argument given): perf.cfg +# +# NOTE: Perlmutter home/scratch paths follow the pattern: +# /global/homes/u/username/... +# /pscratch/sd/u/username/... +# where u is the first letter of your username, e.g. user "forsyth" goes under "f". + +# --------------------------------------------------------------------------- +# Run metadata <- edit these for each new run +# --------------------------------------------------------------------------- + +# Scratch directory where intermediate data and logs are written. +# Use /pscratch since a lot of data will be transferred. +work_dir=/pscratch/sd/u/username/zstash_performance/ + +# Unique identifier for this run (used as a sub-directory name and CSV prefix). +unique_id=performance_20260603 + +# Shell command(s) that activate the zstash environment. +# Separate multiple commands with ' ; ' (space-semicolon-space). +# Example - Unified environment: +# environment_commands=source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh +# Example - dev environment: +# environment_commands=source /global/homes/u/username/miniforge3/etc/profile.d/conda.sh ; conda activate zstash-pr427-20260603 +environment_commands=source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh + +# --------------------------------------------------------------------------- +# Data source <- usually no need to change these +# --------------------------------------------------------------------------- + +dir_to_copy_from=/global/cfs/cdirs/e3sm/forsyth/E3SMv2/v2.LR.historical_0201/ +subdir0=build/ +subdir1=run/ +subdir2=init/ + +# --------------------------------------------------------------------------- +# HPSS options +# --------------------------------------------------------------------------- + +# Space-separated list of HPSS modes to exercise. +# Valid values: none hpss globus +HPSS_OPTIONS=none hpss globus + +# Destination path on HPSS (used when hpss is in HPSS_OPTIONS). +dst_hpss_path=/home/u/username/zstash_performance + +# --------------------------------------------------------------------------- +# Globus options <- used when globus is in HPSS_OPTIONS +# --------------------------------------------------------------------------- + +# Set to true to force a fresh Globus authentication at the start of the run. +# NOTE: This will delete your ~/.zstash.ini & ~/.zstash_globus_tokens.json files. +fresh_globus=true + +# UUID of the destination Globus endpoint. +# Common endpoints: +# LCRC Improv DTN 15288284-7006-4041-ba1a-6b52501e49f1 +# NERSC Perlmutter 6bdc7956-fc0f-4ad2-989c-7aa5ee643a79 +# NERSC HPSS 9cd89cfd-6d04-11e5-ba46-22000b92c6ec +# PIC Compy DTN 68fbd2fa-83d7-11e9-8e63-029d279f7e24 +dst_endpoint_uuid=15288284-7006-4041-ba1a-6b52501e49f1 + +# Destination directory on the Globus endpoint. +dst_endpoint_archive_dir=/lcrc/group/e3sm/username/zstash_performance_dst_dir/ + +# --------------------------------------------------------------------------- +# Results archiving +# --------------------------------------------------------------------------- + +# Long-term (non-scratch) directory where the results CSV is copied at the end +# of a run. The file will be saved as: +# ${performance_archive_dir}/${unique_id}_results.csv +performance_archive_dir=/global/homes/u/username/zstash_performance_records + +# --------------------------------------------------------------------------- +# visualize_performance.py options <- edit these for each new run +# --------------------------------------------------------------------------- + +# Path to the results CSV to show in Figure 1. +# This is the CSV produced by generate_performance_data.bash for this run. +results_csv=/pscratch/sd/u/username/zstash_performance/performance_20260603/results.csv + +# Path to a baseline results CSV to compare against in Figure 2. +# Leave blank (or comment out) to skip Figure 2. +baseline_results_csv=/pscratch/sd/u/username/zstash_performance/performance_20260414/results.csv + +# Output path for the saved figures. +# Leave blank (or comment out) to display interactively instead of saving. +# Make sure to use the web-server path so the URL is printed correctly, e.g.: +# /global/cfs/cdirs/e3sm/www/... +output_path=/global/cfs/cdirs/e3sm/www/username/zstash_performance/performance_pr427_20260603.png diff --git a/tests/performance/visualize_performance.py b/tests/performance/visualize_performance.py index f6df996e..de4d825e 100644 --- a/tests/performance/visualize_performance.py +++ b/tests/performance/visualize_performance.py @@ -3,12 +3,14 @@ visualize_performance.py – Plot zstash performance profiling results. Usage: - python visualize_performance.py + python visualize_performance.py [--cfg path/to/perf.cfg] [--dpi 150] -Edit the constants at the top of this file to point at the CSV(s) to plot. +Pass --cfg (default: perf.cfg next to this script) instead of editing +hard-coded constants. The cfg file uses the same key=value format as +generate_performance_data.bash and can be shared between both scripts. - The CSV is produced by generate_performance_data.bash and has columns: - test_label, create_subdir, update_subdir, hpss_label, operation, elapsed_seconds +The CSV is produced by generate_performance_data.bash and has columns: + test_label, create_subdir, update_subdir, hpss_label, operation, elapsed_seconds Visualization strategy ---------------------- @@ -32,7 +34,7 @@ to make the parallelism speed-up immediately visible. Figure 2 – Baseline comparison (current branch vs main): - Produced only when BASELINE_RESULTS_CSV is set to a valid path. + Produced only when baseline_results_csv is set to a valid path in the cfg. Same 2×2 + comparison layout, but each cell shows two bars (current = solid, baseline = hatched) with a ratio annotation (current/baseline) above each pair. Ratio > 1 = regression (slower), @@ -40,6 +42,7 @@ """ import argparse +import configparser import os import sys from pathlib import Path @@ -51,28 +54,47 @@ import pandas as pd # --------------------------------------------------------------------------- -# ← EDIT THESE for each new run +# Cfg-file helpers # --------------------------------------------------------------------------- -# The results to show in Fig. 1 -RESULTS_CSV: str = ( - "/pscratch/sd/f/forsyth/zstash_performance/performance_20260603/results.csv" -) +_SCRIPT_DIR = Path(__file__).parent -# The results to compare against in Fig. 2. -# Set to None to skip Fig. 2. -BASELINE_RESULTS_CSV: Optional[str] = ( - "/pscratch/sd/f/forsyth/zstash_performance/performance_20260414/results.csv" -) -# Output path for the saved figures. -# Set to None to display interactively instead of saving. -OUTPUT_PATH: Optional[str] = ( - "/global/cfs/cdirs/e3sm/www/forsyth/zstash_performance/performance_pr427_20260603_run2.png" -) +def _load_cfg(cfg_path: Path) -> dict: + """ + Parse a key=value cfg file (same format used by generate_performance_data.bash). + Returns a plain dict. Section headers are not required; if present they are + ignored so the same file can be shared between the bash script and this one. + """ + # configparser needs at least one section header; inject a fake one. + text = "[run]\n" + cfg_path.read_text() + cp = configparser.ConfigParser( + inline_comment_prefixes=("#",), + strict=False, + ) + cp.read_string(text) + return dict(cp["run"]) + + +def _cfg_optional(cfg: dict, key: str) -> Optional[str]: + """Return the value for *key*, or None if missing / blank.""" + v = cfg.get(key, "").strip() + return v if v else None + + +def _cfg_require(cfg: dict, key: str, cfg_path: Path) -> str: + v = _cfg_optional(cfg, key) + if v is None: + print( + f"ERROR: required key '{key}' is missing from {cfg_path}", + file=sys.stderr, + ) + sys.exit(1) + return v + # --------------------------------------------------------------------------- -# Config +# Config (styling – not user-configurable) # --------------------------------------------------------------------------- HPSS_ORDER = ["none", "hpss", "globus"] @@ -707,7 +729,7 @@ def mean_for(df, _op=op, _h=hpss, _cs=create_sub, _us=update_sub): else ("▼" if ratio <= RATIO_IMPROVEMENT else "") ) ax.text( - (x_cur + x_bas) / 2 + pair_width / 2, + (x_cur + x_bas) / 2, top * 1.03, f"{arrow}{ratio:.2f}×", ha="center", @@ -830,19 +852,38 @@ def main(): parser = argparse.ArgumentParser( description="Visualise zstash performance results." ) + parser.add_argument( + "--cfg", + default=str(_SCRIPT_DIR / "perf.cfg"), + help="Path to the key=value config file (default: perf.cfg next to this script).", + ) parser.add_argument( "--dpi", type=int, default=150, help="Output DPI (default: 150)" ) args = parser.parse_args() + cfg_path = Path(args.cfg) + if not cfg_path.is_file(): + print(f"ERROR: config file not found: {cfg_path}", file=sys.stderr) + print( + f"Copy {_SCRIPT_DIR / 'perf.cfg'} and edit it for your run.", + file=sys.stderr, + ) + sys.exit(1) + + cfg = _load_cfg(cfg_path) + RESULTS_CSV: str = _cfg_require(cfg, "results_csv", cfg_path) + BASELINE_RESULTS_CSV: Optional[str] = _cfg_optional(cfg, "baseline_results_csv") + OUTPUT_PATH: Optional[str] = _cfg_optional(cfg, "output_path") + results_path = Path(RESULTS_CSV) if not RESULTS_CSV or not results_path.is_file(): - print(f"ERROR: RESULTS_CSV not found: {RESULTS_CSV!r}", file=sys.stderr) + print(f"ERROR: results_csv not found: {RESULTS_CSV!r}", file=sys.stderr) sys.exit(1) df = load_data(str(results_path)) if df.empty: print( - f"ERROR: RESULTS_CSV is empty or could not be parsed: {RESULTS_CSV!r}", + f"ERROR: results_csv is empty or could not be parsed: {RESULTS_CSV!r}", file=sys.stderr, ) sys.exit(1) @@ -914,7 +955,7 @@ def main(): bas_path = Path(BASELINE_RESULTS_CSV) if not bas_path.exists(): print( - f"WARNING: BASELINE_RESULTS_CSV not found: {bas_path}", file=sys.stderr + f"WARNING: baseline_results_csv not found: {bas_path}", file=sys.stderr ) print("Skipping baseline comparison figure.", file=sys.stderr) else: From 47821a4687dd4832a7f0f1c67ee3e6b96afd075a Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 18 Jun 2026 11:38:34 -0700 Subject: [PATCH 19/27] Address Copilot review comments 2 --- tests/performance/README.md | 8 ++++++++ .../performance/generate_performance_data.bash | 17 +++++++++++++---- tests/performance/visualize_performance.py | 5 ++++- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/tests/performance/README.md b/tests/performance/README.md index 009b6f34..6f8ea724 100644 --- a/tests/performance/README.md +++ b/tests/performance/README.md @@ -4,6 +4,14 @@ Performance profiling should be done on Perlmutter. We're keeping the performanc ## Setup +To run the visualizer (`visualize_performance.py`), you need `matplotlib`, `numpy`, and `pandas`. The repo provides a minimal conda environment for this in `conda/perf.yml`: + +```bash +conda env create -f conda/perf.yml -n zstash_perf +conda activate zstash_perf +python -m pip install . +``` + All parameters for both scripts live in a single config file (`perf.cfg`) so you never need to edit the scripts themselves. Copy the provided template and fill in your values: ```bash diff --git a/tests/performance/generate_performance_data.bash b/tests/performance/generate_performance_data.bash index 77275cb6..79957ba7 100755 --- a/tests/performance/generate_performance_data.bash +++ b/tests/performance/generate_performance_data.bash @@ -63,6 +63,7 @@ cfg_get() { # Run from Perlmutter, so that we can do both # a direct transfer to HPSS & a Globus transfer to Chrysalis work_dir="$(cfg_require work_dir)" +work_dir="${work_dir%/}/" unique_id="$(cfg_require unique_id)" environment_commands="$(cfg_require environment_commands)" @@ -71,6 +72,7 @@ environment_commands="$(cfg_require environment_commands)" # but can be changed for further customization. dir_to_copy_from="$(cfg_require dir_to_copy_from)" +dir_to_copy_from="${dir_to_copy_from%/}/" subdir0="$(cfg_get subdir0 none)" subdir1="$(cfg_get subdir1 none)" subdir2="$(cfg_get subdir2 none)" @@ -196,6 +198,10 @@ validate_configuration() refresh_globus() { print_step "Setting up fresh Globus authentication..." + + INI_PATH="${HOME}/.zstash.ini" + TOKEN_FILE="${HOME}/.zstash_globus_tokens.json" + if ! confirm "This will delete ${INI_PATH} and ${TOKEN_FILE} to start fresh. Is that ok?"; then exit 1 fi @@ -208,9 +214,6 @@ refresh_globus() fi # 2. Reset authentication token files - INI_PATH=${HOME}/.zstash.ini - TOKEN_FILE=${HOME}/.zstash_globus_tokens.json - if [ -f "${INI_PATH}" ]; then rm -f "${INI_PATH}" print_info "Removed ${INI_PATH}" @@ -370,8 +373,14 @@ mkdir -p "${work_dir}${unique_id}" echo "test_label,create_subdir,update_subdir,hpss_label,operation,elapsed_seconds" > "${results_csv}" print_info "Results CSV: ${results_csv}" -# Array of subdirectories +# Array of subdirectories (the test matrix below assumes all 3 are provided) subdirs=("$subdir0" "$subdir1" "$subdir2") +for s in "${subdirs[@]}"; do + if [ -z "$s" ] || [ "$s" = "none" ]; then + print_error "subdir0/subdir1/subdir2 must be set (not 'none') in ${CFG_FILE}" + exit 1 + fi +done # Define the 6 possible permutations as test configurations. # Each string contains two space-separated indices into the subdirs array: diff --git a/tests/performance/visualize_performance.py b/tests/performance/visualize_performance.py index de4d825e..074bd669 100644 --- a/tests/performance/visualize_performance.py +++ b/tests/performance/visualize_performance.py @@ -977,7 +977,10 @@ def save_or_show(figure, out_path_str, label): out_path.parent.mkdir(parents=True, exist_ok=True) figure.savefig(out_path, dpi=args.dpi, bbox_inches="tight") print(f"{label} saved to: {out_path}") - os.chmod(out_path_str, 0o644) + try: + os.chmod(out_path, 0o644) + except OSError: + pass web_path = str(out_path).replace( "/global/cfs/cdirs/e3sm/www/", "https://portal.nersc.gov/cfs/e3sm/", From ea14728ed6d5ff6029744088f7491c3587adcd35 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 18 Jun 2026 12:04:02 -0700 Subject: [PATCH 20/27] Add archive plots --- tests/performance/visualize_performance.py | 335 +++++++++++++++++++++ 1 file changed, 335 insertions(+) diff --git a/tests/performance/visualize_performance.py b/tests/performance/visualize_performance.py index 074bd669..546f4556 100644 --- a/tests/performance/visualize_performance.py +++ b/tests/performance/visualize_performance.py @@ -39,15 +39,30 @@ (current = solid, baseline = hatched) with a ratio annotation (current/baseline) above each pair. Ratio > 1 = regression (slower), ratio < 1 = improvement (faster). + +Figure 3 – Full record archive (all historical CSVs in performance_archive_dir): + Produced only when performance_archive_dir is set in the cfg and contains + *results*.csv files with YYYYMMDD in their names. + Layout: 2×2 grid (create | update) × (time-series | box plot). + Time-series: x = record date, y = runtime, color = hpss mode, + line style = subdir (solid=build, dashed=run, dotted=init). + 9 lines per subplot (3 subdirs × 3 hpss modes). + Box plots: vertical box-and-whisker for each (subdir, hpss) combination, + with individual data-point dots overlaid. + 9 boxes per subplot. """ import argparse import configparser +import datetime import os +import re import sys from pathlib import Path from typing import Optional +import matplotlib.dates +import matplotlib.lines import matplotlib.patches as mpatches import matplotlib.pyplot as plt import numpy as np @@ -121,6 +136,19 @@ def _cfg_require(cfg: dict, key: str, cfg_path: Path) -> str: DOT_ALPHA = 0.55 DOT_SIZE = 40 +# --------------------------------------------------------------------------- +# Figure 3 – per-subdir line styles (encode which directory is plotted) +# --------------------------------------------------------------------------- +# build/ = many small files → solid +# run/ = mixed → dashed +# init/ = few large files → dotted +SUBDIR_LINESTYLES: dict[str, str] = { + "build": "solid", + "run": "dashed", + "init": "dotted", +} +SUBDIR_ORDER = ["build", "run", "init"] + # --------------------------------------------------------------------------- # Helpers @@ -843,6 +871,294 @@ def build_comparison_figure( RATIO_REGRESSION_COLOR_LABEL = "red" RATIO_IMPROVEMENT_COLOR_LABEL = "green" + +# --------------------------------------------------------------------------- +# Figure 3 – full record archive +# --------------------------------------------------------------------------- + + +def _archive_date_from_path(csv_path: Path) -> Optional[datetime.date]: + """ + Parse the record date from a CSV filename that contains YYYYMMDD. + + E.g. ``performance_20260603_results.csv`` → ``date(2026, 6, 3)`` + Returns *None* when no eight-digit date string is found. + """ + m = re.search(r"(\d{8})", csv_path.stem) + if not m: + return None + try: + s = m.group(1) + return datetime.date(int(s[:4]), int(s[4:6]), int(s[6:])) + except ValueError: + return None + + +def load_archive_data(archive_dir: str) -> pd.DataFrame: + """ + Load and concatenate every ``*results*.csv`` in *archive_dir*. + + Adds a ``record_date`` column (pandas Timestamp) derived from the + filename. Files with no parseable date are skipped with a warning. + + Returns an empty DataFrame (with expected columns) on failure. + """ + _empty = pd.DataFrame( + columns=[ + "test_label", + "create_subdir", + "update_subdir", + "hpss_label", + "operation", + "elapsed_seconds", + "record_date", + ] + ) + archive_path = Path(archive_dir) + if not archive_path.is_dir(): + print( + f"WARNING: performance_archive_dir not found: {archive_path}", + file=sys.stderr, + ) + return _empty + + csv_files = sorted(archive_path.glob("*results*.csv")) + if not csv_files: + print( + f"WARNING: no *results*.csv files found in {archive_path}", + file=sys.stderr, + ) + return _empty + + frames = [] + for p in csv_files: + record_date = _archive_date_from_path(p) + if record_date is None: + print( + f"WARNING: cannot parse date from filename {p.name!r}, skipping.", + file=sys.stderr, + ) + continue + try: + df_i = load_data(str(p)) + except Exception as exc: + print(f"WARNING: failed to load {p}: {exc}", file=sys.stderr) + continue + df_i["record_date"] = record_date + frames.append(df_i) + + if not frames: + return _empty + + df_all = pd.concat(frames, ignore_index=True) + df_all["record_date"] = pd.to_datetime(df_all["record_date"]) + return df_all + + +def plot_archive_timeseries(ax, df_arch: pd.DataFrame, operation: str) -> None: + """ + Time-series line graph for *operation* over the full record archive. + + Axes + ---- + X : record date + Y : elapsed_seconds (mean over all test configs sharing the same + subdir × hpss × date) + + Visual encoding + --------------- + Color : hpss_label – blue (none) / orange (hpss) / green (globus) + Line style: subdir – solid (build) / dashed (run) / dotted (init) + → 9 lines total (3 subdirs × 3 hpss modes) + """ + dir_col = OP_DIR_COL[operation] + df_op = df_arch[df_arch["operation"] == operation].copy() + + for hpss in HPSS_ORDER: + color = HPSS_COLORS[hpss] + for subdir in SUBDIR_ORDER: + ls = SUBDIR_LINESTYLES.get(subdir, "solid") + mask = (df_op["hpss_label"] == hpss) & (df_op[dir_col] == subdir) + df_line = ( + df_op[mask] + .groupby("record_date")["elapsed_seconds"] + .mean() + .reset_index() + .sort_values("record_date") + ) + if df_line.empty: + continue + ax.plot( + df_line["record_date"], + df_line["elapsed_seconds"], + color=color, + linestyle=ls, + linewidth=1.6, + marker="o", + markersize=4, + label=f"{HPSS_LABELS[hpss]} – {subdir}/", + zorder=3, + ) + + ax.set_title( + f"zstash {operation} – runtime over time", + fontsize=10, + fontweight="bold", + pad=6, + ) + ax.set_xlabel("Record date", fontsize=8) + ax.set_ylabel("Wall-clock time (s)", fontsize=8) + ax.xaxis.set_major_formatter(matplotlib.dates.DateFormatter("%Y-%m-%d")) + plt.setp(ax.get_xticklabels(), rotation=30, ha="right", fontsize=7) + ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) + ax.set_axisbelow(True) + + # Two-section legend: top = hpss color, bottom = subdir line style + color_handles = [ + matplotlib.lines.Line2D( + [], [], color=HPSS_COLORS[h], linewidth=2, label=HPSS_LABELS[h] + ) + for h in HPSS_ORDER + ] + style_handles = [ + matplotlib.lines.Line2D( + [], + [], + color="grey", + linewidth=2, + linestyle=SUBDIR_LINESTYLES[s], + label=f"{s}/", + ) + for s in SUBDIR_ORDER + ] + ax.legend( + handles=color_handles + style_handles, + fontsize=6.5, + loc="upper left", + ncol=2, + framealpha=0.8, + ) + + +def plot_archive_boxplot(ax, df_arch: pd.DataFrame, operation: str) -> None: + """ + Vertical box-and-whisker plots for *operation* across the full archive. + + Layout + ------ + X axis : one tick-group per subdir (build, run, init); within each group + the three hpss modes are placed side by side. + Color : hpss_label (same palette as all other figures) + Overlay : individual data-point dots (jittered, matching other figures) + → 9 boxes total (3 subdirs × 3 hpss modes) + """ + dir_col = OP_DIR_COL[operation] + df_op = df_arch[df_arch["operation"] == operation].copy() + + n_hpss = len(HPSS_ORDER) + group_width = n_hpss * BAR_WIDTH + 0.10 + x_base = np.arange(len(SUBDIR_ORDER)) * group_width + offsets = np.linspace(0, (n_hpss - 1) * BAR_WIDTH, n_hpss) + + tick_positions = [] + tick_labels = [] + + for s_idx, subdir in enumerate(SUBDIR_ORDER): + tick_positions.append(x_base[s_idx] + offsets.mean()) + tick_labels.append(f"{subdir}/") + + for h_idx, hpss in enumerate(HPSS_ORDER): + mask = (df_op["hpss_label"] == hpss) & (df_op[dir_col] == subdir) + vals = df_op[mask]["elapsed_seconds"].dropna().values + x_pos = x_base[s_idx] + offsets[h_idx] + + if len(vals) == 0: + continue + + color = HPSS_COLORS[hpss] + + ax.boxplot( + vals, + positions=[x_pos], + widths=BAR_WIDTH * 0.85, + patch_artist=True, + vert=True, + manage_ticks=False, + zorder=2, + boxprops=dict(facecolor=color, alpha=0.55, linewidth=0.8), + medianprops=dict(color="black", linewidth=1.5), + whiskerprops=dict(linewidth=0.8), + capprops=dict(linewidth=0.8), + flierprops=dict(marker="", linestyle="none"), + ) + + # Overlay individual data points + jitter = np.random.uniform( + -BAR_WIDTH * 0.2, BAR_WIDTH * 0.2, size=len(vals) + ) + ax.scatter( + x_pos + jitter, + vals, + color="white", + edgecolors=color, + s=DOT_SIZE, + zorder=3, + alpha=DOT_ALPHA, + linewidths=1.2, + ) + + ax.set_xticks(tick_positions) + ax.set_xticklabels(tick_labels, fontsize=9) + ax.set_title( + f"zstash {operation} – runtime distribution (all records)", + fontsize=10, + fontweight="bold", + pad=6, + ) + ax.set_xlabel("Directory processed", fontsize=8) + ax.set_ylabel("Wall-clock time (s)", fontsize=8) + ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) + ax.set_axisbelow(True) + + hpss_patches = [ + mpatches.Patch(color=HPSS_COLORS[h], alpha=0.75, label=HPSS_LABELS[h]) + for h in HPSS_ORDER + ] + ax.legend(handles=hpss_patches, fontsize=7, loc="upper right") + + +def build_archive_figure(df_arch: pd.DataFrame) -> plt.Figure: + """ + Figure 3 – full archive overview. + + Layout (2 rows × 2 cols): + [0,0] create time-series | [0,1] update time-series + [1,0] create box plot | [1,1] update box plot + """ + fig = plt.figure(figsize=(15, 12)) + fig.suptitle( + "zstash Performance – Full Record Archive\n" + "Time series: color = HPSS mode · line style = directory " + "(solid = build/, dashed = run/, dotted = init/)\n" + "Box plots: every recorded runtime for each (directory, HPSS) combination", + fontsize=11, + fontweight="bold", + y=0.995, + ) + + gs = fig.add_gridspec( + 2, 2, hspace=0.48, wspace=0.30, top=0.90, bottom=0.08, left=0.07, right=0.97 + ) + + for col_idx, op in enumerate(["create", "update"]): + ax_ts = fig.add_subplot(gs[0, col_idx]) + ax_box = fig.add_subplot(gs[1, col_idx]) + plot_archive_timeseries(ax_ts, df_arch, op) + plot_archive_boxplot(ax_box, df_arch, op) + + return fig + + # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- @@ -875,6 +1191,7 @@ def main(): RESULTS_CSV: str = _cfg_require(cfg, "results_csv", cfg_path) BASELINE_RESULTS_CSV: Optional[str] = _cfg_optional(cfg, "baseline_results_csv") OUTPUT_PATH: Optional[str] = _cfg_optional(cfg, "output_path") + ARCHIVE_DIR: Optional[str] = _cfg_optional(cfg, "performance_archive_dir") results_path = Path(RESULTS_CSV) if not RESULTS_CSV or not results_path.is_file(): @@ -968,6 +1285,20 @@ def main(): df, df_bas, all_dirs, cur_label, bas_label ) + # ----------------------------------------------------------------------- + # Full archive figure (Figure 3) + # ----------------------------------------------------------------------- + fig_arch = None + if ARCHIVE_DIR: + df_arch = load_archive_data(ARCHIVE_DIR) + if not df_arch.empty: + fig_arch = build_archive_figure(df_arch) + else: + print( + "WARNING: no archive data found; skipping Figure 3.", + file=sys.stderr, + ) + # ----------------------------------------------------------------------- # Save or show # ----------------------------------------------------------------------- @@ -995,6 +1326,10 @@ def save_or_show(figure, out_path_str, label): p = Path(OUTPUT_PATH) cmp_output = str(p.with_stem(p.stem + "_vs_baseline")) save_or_show(fig_cmp, cmp_output, "Figure 2 (baseline comparison)") + if fig_arch is not None: + p = Path(OUTPUT_PATH) + arch_output = str(p.with_stem(p.stem + "_archive")) + save_or_show(fig_arch, arch_output, "Figure 3 (full archive)") else: plt.show() From 6e6c925a6eda4608e619202869920bd589595cd7 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 18 Jun 2026 12:21:38 -0700 Subject: [PATCH 21/27] Updates to run --- conda/perf.yml | 23 ++++++- tests/performance/develop_run.cfg | 99 +++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 tests/performance/develop_run.cfg diff --git a/conda/perf.yml b/conda/perf.yml index 64f4c8d6..7b447c74 100644 --- a/conda/perf.yml +++ b/conda/perf.yml @@ -2,7 +2,7 @@ name: zstash_perf channels: - conda-forge dependencies: - # Base (minimal subset needed to run zstash) + # Base # ================= - pip - python >=3.11,<3.15 @@ -10,6 +10,27 @@ dependencies: - sqlite - six >=1.16.0 - globus-sdk >=3.15.0,<4.0 + # Developer Tools + # ================= + # If versions are updated, also update 'rev' in `.pre-commit.config.yaml` + - black ==25.1.0 + - flake8 ==7.3.0 + - isort ==6.0.1 + - mypy ==1.18.2 + - pre-commit ==4.3.0 + - tbump >=6.9.0 + # Testing + # ======================= + - pytest + - pytest-cov + # Documentation + # ================= + # If versions are updated, also update in `.github/workflows/workflow.yml` + - jinja2 <3.1 + - sphinx >=5.2.0 + - sphinx-multiversion >=0.2.4 + - sphinx_rtd_theme >=1.0.0 + - docutils >=0.17.1 # Performance profiling # ================= - matplotlib-base diff --git a/tests/performance/develop_run.cfg b/tests/performance/develop_run.cfg new file mode 100644 index 00000000..f48b2d20 --- /dev/null +++ b/tests/performance/develop_run.cfg @@ -0,0 +1,99 @@ +# This version of perf.cfg has filled-in username paths. + +# Performance profiling configuration – shared by both scripts: +# generate_performance_data.bash (pass as first argument, or name it perf.cfg) +# visualize_performance.py (pass via --cfg, or name it perf.cfg) +# Usage: +# ./generate_performance_data.bash [path/to/this/file] +# python visualize_performance.py [--cfg path/to/this/file] +# Default cfg file name (when no argument given): perf.cfg +# +# NOTE: Perlmutter home/scratch paths follow the pattern: +# /global/homes/u/username/... +# /pscratch/sd/u/username/... +# where u is the first letter of your username, e.g. user "forsyth" goes under "f". + +# --------------------------------------------------------------------------- +# Run metadata <- edit these for each new run +# --------------------------------------------------------------------------- + +# Scratch directory where intermediate data and logs are written. +# Use /pscratch since a lot of data will be transferred. +work_dir=/pscratch/sd/f/forsyth/zstash_performance/ + +# Unique identifier for this run (used as a sub-directory name and CSV prefix). +unique_id=performance_20260603 + +# Shell command(s) that activate the zstash environment. +# Separate multiple commands with ' ; ' (space-semicolon-space). +# Example - Unified environment: +# environment_commands=source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh +# Example - dev environment: +# environment_commands=source /global/homes/u/username/miniforge3/etc/profile.d/conda.sh ; conda activate zstash-pr427-20260603 +environment_commands=source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh + +# --------------------------------------------------------------------------- +# Data source <- usually no need to change these +# --------------------------------------------------------------------------- + +dir_to_copy_from=/global/cfs/cdirs/e3sm/forsyth/E3SMv2/v2.LR.historical_0201/ +subdir0=build/ +subdir1=run/ +subdir2=init/ + +# --------------------------------------------------------------------------- +# HPSS options +# --------------------------------------------------------------------------- + +# Space-separated list of HPSS modes to exercise. +# Valid values: none hpss globus +HPSS_OPTIONS=none hpss globus + +# Destination path on HPSS (used when hpss is in HPSS_OPTIONS). +dst_hpss_path=/home/f/forsyth/zstash_performance + +# --------------------------------------------------------------------------- +# Globus options <- used when globus is in HPSS_OPTIONS +# --------------------------------------------------------------------------- + +# Set to true to force a fresh Globus authentication at the start of the run. +# NOTE: This will delete your ~/.zstash.ini & ~/.zstash_globus_tokens.json files. +fresh_globus=true + +# UUID of the destination Globus endpoint. +# Common endpoints: +# LCRC Improv DTN 15288284-7006-4041-ba1a-6b52501e49f1 +# NERSC Perlmutter 6bdc7956-fc0f-4ad2-989c-7aa5ee643a79 +# NERSC HPSS 9cd89cfd-6d04-11e5-ba46-22000b92c6ec +# PIC Compy DTN 68fbd2fa-83d7-11e9-8e63-029d279f7e24 +dst_endpoint_uuid=15288284-7006-4041-ba1a-6b52501e49f1 + +# Destination directory on the Globus endpoint. +dst_endpoint_archive_dir=/lcrc/group/e3sm/ac.forsyth2/zstash_performance_dst_dir/ + +# --------------------------------------------------------------------------- +# Results archiving +# --------------------------------------------------------------------------- + +# Long-term (non-scratch) directory where the results CSV is copied at the end +# of a run. The file will be saved as: +# ${performance_archive_dir}/${unique_id}_results.csv +performance_archive_dir=/global/homes/f/forsyth/zstash_performance_records + +# --------------------------------------------------------------------------- +# visualize_performance.py options <- edit these for each new run +# --------------------------------------------------------------------------- + +# Path to the results CSV to show in Figure 1. +# This is the CSV produced by generate_performance_data.bash for this run. +results_csv=/pscratch/sd/f/forsyth/zstash_performance/performance_20260603/results.csv + +# Path to a baseline results CSV to compare against in Figure 2. +# Leave blank (or comment out) to skip Figure 2. +baseline_results_csv=/pscratch/sd/f/forsyth/zstash_performance/performance_20260414/results.csv + +# Output path for the saved figures. +# Leave blank (or comment out) to display interactively instead of saving. +# Make sure to use the web-server path so the URL is printed correctly, e.g.: +# /global/cfs/cdirs/e3sm/www/... +output_path=/global/cfs/cdirs/e3sm/www/forsyth/zstash_performance/performance_pr427_20260603.png From 7a9af786c1b0f11d98d58cc0563a8103611dc106 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 18 Jun 2026 14:28:30 -0700 Subject: [PATCH 22/27] Remove outliers and add extract archive plots --- tests/performance/visualize_performance.py | 796 ++++++++++++++------- 1 file changed, 543 insertions(+), 253 deletions(-) diff --git a/tests/performance/visualize_performance.py b/tests/performance/visualize_performance.py index 546f4556..385ed9cd 100644 --- a/tests/performance/visualize_performance.py +++ b/tests/performance/visualize_performance.py @@ -40,16 +40,29 @@ (current/baseline) above each pair. Ratio > 1 = regression (slower), ratio < 1 = improvement (faster). -Figure 3 – Full record archive (all historical CSVs in performance_archive_dir): +Figure 3 – Full record archive for create & update + (all historical CSVs in performance_archive_dir): Produced only when performance_archive_dir is set in the cfg and contains *results*.csv files with YYYYMMDD in their names. Layout: 2×2 grid (create | update) × (time-series | box plot). Time-series: x = record date, y = runtime, color = hpss mode, line style = subdir (solid=build, dashed=run, dotted=init). - 9 lines per subplot (3 subdirs × 3 hpss modes). Box plots: vertical box-and-whisker for each (subdir, hpss) combination, with individual data-point dots overlaid. - 9 boxes per subplot. + +Figure 4 – Full record archive for extract_seq & extract_par: + Produced only when performance_archive_dir is set and contains extract data. + Layout: 2×2 grid (extract_seq | extract_par) × (time-series | box plot). + X-axis groups for box plots: (create_subdir, update_subdir) archive config pairs. + Same color/line-style encoding as Figure 3. + +Outlier removal +--------------- +All plotting functions apply IQR-based outlier filtering before computing +means or drawing boxes/lines. Values outside + [Q1 - 1.5 * IQR, Q3 + 1.5 * IQR] +are dropped silently. This prevents a single aberrant run from dominating +axis scales while preserving legitimate spread. """ import argparse @@ -58,6 +71,7 @@ import os import re import sys +from collections import Counter from pathlib import Path from typing import Optional @@ -137,7 +151,7 @@ def _cfg_require(cfg: dict, key: str, cfg_path: Path) -> str: DOT_SIZE = 40 # --------------------------------------------------------------------------- -# Figure 3 – per-subdir line styles (encode which directory is plotted) +# Figure 3/4 – per-subdir line styles (encode which directory is plotted) # --------------------------------------------------------------------------- # build/ = many small files → solid # run/ = mixed → dashed @@ -150,6 +164,57 @@ def _cfg_require(cfg: dict, key: str, cfg_path: Path) -> str: SUBDIR_ORDER = ["build", "run", "init"] +# --------------------------------------------------------------------------- +# Outlier removal +# --------------------------------------------------------------------------- + + +def remove_outliers_iqr(vals: np.ndarray, k: float = 1.5) -> np.ndarray: + """ + Return a copy of *vals* with IQR-based outliers removed. + + Values outside [Q1 - k*IQR, Q3 + k*IQR] are dropped. + Returns the original array unchanged when it has fewer than 4 elements + (too few to estimate quartiles reliably). + """ + if len(vals) < 4: + return vals + q1, q3 = np.percentile(vals, [25, 75]) + iqr = q3 - q1 + lo = q1 - k * iqr + hi = q3 + k * iqr + return vals[(vals >= lo) & (vals <= hi)] + + +def _filter_df_outliers(df: pd.DataFrame, group_cols: list) -> pd.DataFrame: + """ + Apply IQR outlier removal to elapsed_seconds within each group defined + by *group_cols*. Returns a new DataFrame with outlier rows dropped. + Duplicate values that survive the IQR filter are all retained; only values + that fall outside the fence are removed. + """ + keep = [] + for _, grp in df.groupby(group_cols, dropna=False): + vals = grp["elapsed_seconds"].dropna().values + clean = remove_outliers_iqr(vals) + clean_counts = Counter(clean.tolist()) + used: Counter = Counter() + row_mask = [] + for v in grp["elapsed_seconds"]: + if pd.isna(v): + row_mask.append(False) + continue + if used[v] < clean_counts[v]: + row_mask.append(True) + used[v] += 1 + else: + row_mask.append(False) + keep.append(grp[row_mask]) + if not keep: + return df.iloc[0:0] + return pd.concat(keep, ignore_index=True) + + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- @@ -206,9 +271,11 @@ def _add_dir_annotation(ax, dirs, x_positions): ) -def plot_operation(ax, df_op: pd.DataFrame, operation: str, dirs: list[str]): - """Draw grouped bars for one operation subplot.""" +def plot_operation(ax, df_op: pd.DataFrame, operation: str, dirs: list): + """Draw grouped bars for one operation subplot (outliers removed).""" dir_col = OP_DIR_COL[operation] + df_op = _filter_df_outliers(df_op.copy(), [dir_col, "hpss_label"]) + n_dirs = len(dirs) n_hpss = len(HPSS_ORDER) @@ -260,7 +327,6 @@ def plot_operation(ax, df_op: pd.DataFrame, operation: str, dirs: list[str]): ax.set_xlabel("Directory processed", fontsize=8, labelpad=14) ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) ax.set_axisbelow(True) - # Fig. 1: group centres are at integer positions 0, 1, 2, … _add_dir_annotation(ax, dirs, list(x_base)) # Value labels on bars @@ -278,7 +344,7 @@ def plot_operation(ax, df_op: pd.DataFrame, operation: str, dirs: list[str]): ) -def _extract_configs(df: pd.DataFrame) -> list[tuple[str, str]]: +def _extract_configs(df: pd.DataFrame) -> list: """ Return the sorted list of (create_subdir, update_subdir) pairs that actually appear in the extract rows of *df*. These represent the @@ -291,7 +357,6 @@ def _extract_configs(df: pd.DataFrame) -> list[tuple[str, str]]: .apply(tuple, axis=1) .tolist() ) - # Sort by create_subdir first, then update_subdir return sorted(pairs, key=lambda p: (dir_sort_key(p[0]), dir_sort_key(p[1]))) @@ -303,9 +368,11 @@ def _extract_tick_label(create_sub: str, update_sub: str) -> str: def _plot_extract_single_op(ax, df: pd.DataFrame, operation: str): """ Draw grouped bars for one extract operation (extract_seq or extract_par). - X-axis groups are the combined (create_subdir, update_subdir) archive - configs, since extract operates on the archive built by both operations. + Outliers removed per (create_subdir, update_subdir, hpss_label) group. """ + df_op = df[df["operation"] == operation].copy() + df_op = _filter_df_outliers(df_op, ["create_subdir", "update_subdir", "hpss_label"]) + configs = _extract_configs(df) n_configs = len(configs) n_hpss = len(HPSS_ORDER) @@ -317,11 +384,10 @@ def _plot_extract_single_op(ax, df: pd.DataFrame, operation: str): means, all_vals, xs = [], [], [] for c_idx, (create_sub, update_sub) in enumerate(configs): vals = ( - df[ - (df["operation"] == operation) - & (df["hpss_label"] == hpss) - & (df["create_subdir"] == create_sub) - & (df["update_subdir"] == update_sub) + df_op[ + (df_op["hpss_label"] == hpss) + & (df_op["create_subdir"] == create_sub) + & (df_op["update_subdir"] == update_sub) ]["elapsed_seconds"] .dropna() .values @@ -381,29 +447,31 @@ def _plot_extract_single_op(ax, df: pd.DataFrame, operation: str): def plot_extract_comparison(ax, df: pd.DataFrame): """ - Extra subplot: sequential vs parallel extract, grouped by - (archive config, hpss). Each archive config is the *combined* - create+update directory pair, since extract operates on the full - archive assembled by both operations. - Uses a hatch pattern to distinguish seq/par within each hpss colour. + Extra subplot: sequential vs parallel extract, grouped by (archive config, hpss). + Outliers removed per (operation, create_subdir, update_subdir, hpss_label). """ + df_ext = df[df["operation"].isin(["extract_seq", "extract_par"])].copy() + df_ext = _filter_df_outliers( + df_ext, ["operation", "create_subdir", "update_subdir", "hpss_label"] + ) + configs = _extract_configs(df) n_configs = len(configs) ops = ["extract_seq", "extract_par"] hatches = {"extract_seq": "", "extract_par": "////"} n_bars = len(HPSS_ORDER) * len(ops) - group_width = n_bars * BAR_WIDTH + 0.15 # total width per config group + group_width = n_bars * BAR_WIDTH + 0.15 x_base = np.arange(n_configs) * group_width for c_idx, (create_sub, update_sub) in enumerate(configs): for h_idx, hpss in enumerate(HPSS_ORDER): for op_idx, op in enumerate(ops): - df_cell = df[ - (df["operation"] == op) - & (df["hpss_label"] == hpss) - & (df["create_subdir"] == create_sub) - & (df["update_subdir"] == update_sub) + df_cell = df_ext[ + (df_ext["operation"] == op) + & (df_ext["hpss_label"] == hpss) + & (df_ext["create_subdir"] == create_sub) + & (df_ext["update_subdir"] == update_sub) ] vals = df_cell["elapsed_seconds"].dropna().values mean = vals.mean() if len(vals) > 0 else 0.0 @@ -435,7 +503,6 @@ def plot_extract_comparison(ax, df: pd.DataFrame): ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) ax.set_axisbelow(True) - # Custom legend: colour = hpss, hatch = seq/par hpss_patches = [ mpatches.Patch(color=HPSS_COLORS[h], label=HPSS_LABELS[h]) for h in HPSS_ORDER ] @@ -457,9 +524,8 @@ def plot_extract_comparison(ax, df: pd.DataFrame): # Baseline comparison figure # --------------------------------------------------------------------------- -# Ratio colouring thresholds -RATIO_REGRESSION = 1.10 # >= 10% slower → red -RATIO_IMPROVEMENT = 0.90 # <= 10% faster → green +RATIO_REGRESSION = 1.10 +RATIO_IMPROVEMENT = 0.90 RATIO_NEUTRAL_COLOR = "#333333" RATIO_REGRESSION_COLOR = "#CC3311" RATIO_IMPROVEMENT_COLOR = "#228833" @@ -478,17 +544,20 @@ def plot_comparison_operation( df_cur: pd.DataFrame, df_bas: pd.DataFrame, operation: str, - dirs: list[str], + dirs: list, ): - """ - For one operation, draw paired bars (current vs baseline) per - (directory, hpss_mode) cell, with a ratio annotation above each pair. - """ + """Paired bars (current vs baseline) per (directory, hpss) cell. Outliers removed.""" dir_col = OP_DIR_COL[operation] + df_cur = _filter_df_outliers( + df_cur[df_cur["operation"] == operation].copy(), [dir_col, "hpss_label"] + ) + df_bas = _filter_df_outliers( + df_bas[df_bas["operation"] == operation].copy(), [dir_col, "hpss_label"] + ) + n_dirs = len(dirs) n_hpss = len(HPSS_ORDER) - # Each hpss group occupies 2 bars (current + baseline) + a small gap pair_width = BAR_WIDTH gap = BAR_WIDTH * 0.3 group_span = n_hpss * (2 * pair_width + gap) + 0.2 @@ -499,16 +568,14 @@ def plot_comparison_operation( pair_offset = h_idx * (2 * pair_width + gap) for d_idx, d in enumerate(dirs): - x_left = x_base[d_idx] + pair_offset # baseline bar - x_right = x_base[d_idx] + pair_offset + pair_width # current bar + x_left = x_base[d_idx] + pair_offset + x_right = x_base[d_idx] + pair_offset + pair_width - def mean_for(df, _op=operation, _h=hpss, _d=d): + def mean_for(df, _h=hpss, _d=d): v = ( - df[ - (df["operation"] == _op) - & (df["hpss_label"] == _h) - & (df[dir_col] == _d) - ]["elapsed_seconds"] + df[(df["hpss_label"] == _h) & (df[dir_col] == _d)][ + "elapsed_seconds" + ] .dropna() .values ) @@ -517,7 +584,6 @@ def mean_for(df, _op=operation, _h=hpss, _d=d): cur_mean = mean_for(df_cur) bas_mean = mean_for(df_bas) - # Baseline bar (hatched, lighter) — left ax.bar( x_left, bas_mean, @@ -528,7 +594,6 @@ def mean_for(df, _op=operation, _h=hpss, _d=d): zorder=2, edgecolor=color, ) - # Current bar (solid) — right ax.bar( x_right, cur_mean, @@ -539,7 +604,6 @@ def mean_for(df, _op=operation, _h=hpss, _d=d): label=HPSS_LABELS[hpss] if d_idx == 0 else "", ) - # Ratio annotation if bas_mean > 0 and cur_mean > 0: ratio = cur_mean / bas_mean top = max(cur_mean, bas_mean) @@ -562,7 +626,6 @@ def mean_for(df, _op=operation, _h=hpss, _d=d): ) ax.set_title(OP_TITLES[operation], fontsize=10, fontweight="bold", pad=6) - # Tick at the centre of each directory's group of bars group_centre_offset = (n_hpss * (2 * pair_width + gap) - gap) / 2 x_ticks = x_base + group_centre_offset ax.set_xticks(x_ticks) @@ -571,7 +634,6 @@ def mean_for(df, _op=operation, _h=hpss, _d=d): ax.set_xlabel("Directory processed", fontsize=8, labelpad=14) ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) ax.set_axisbelow(True) - # Pass actual tick x-positions so annotations align with tick labels _add_dir_annotation(ax, dirs, list(x_ticks)) @@ -581,11 +643,16 @@ def _plot_comparison_extract_single_op( df_bas: pd.DataFrame, operation: str, ): - """ - Fig. 2 version of a single extract-op subplot (extract_seq or extract_par). - X-axis = combined (create, update) archive config; bars = current vs baseline - paired within each HPSS group. - """ + """Fig. 2 extract subplot: current vs baseline, outliers removed.""" + df_cur = _filter_df_outliers( + df_cur[df_cur["operation"] == operation].copy(), + ["create_subdir", "update_subdir", "hpss_label"], + ) + df_bas = _filter_df_outliers( + df_bas[df_bas["operation"] == operation].copy(), + ["create_subdir", "update_subdir", "hpss_label"], + ) + configs = _extract_configs(df_cur) n_configs = len(configs) n_hpss = len(HPSS_ORDER) @@ -602,11 +669,10 @@ def _plot_comparison_extract_single_op( x_left = x_base[c_idx] + pair_offset x_right = x_left + pair_width - def mean_for(df, _op=operation, _h=hpss, _cs=create_sub, _us=update_sub): + def mean_for(df, _h=hpss, _cs=create_sub, _us=update_sub): v = ( df[ - (df["operation"] == _op) - & (df["hpss_label"] == _h) + (df["hpss_label"] == _h) & (df["create_subdir"] == _cs) & (df["update_subdir"] == _us) ]["elapsed_seconds"] @@ -618,7 +684,6 @@ def mean_for(df, _op=operation, _h=hpss, _cs=create_sub, _us=update_sub): cur_mean = mean_for(df_cur) bas_mean = mean_for(df_bas) - # Baseline bar (hatched, lighter) — left ax.bar( x_left, bas_mean, @@ -629,7 +694,6 @@ def mean_for(df, _op=operation, _h=hpss, _cs=create_sub, _us=update_sub): zorder=2, edgecolor=color, ) - # Current bar (solid) — right ax.bar( x_right, cur_mean, @@ -671,27 +735,26 @@ def mean_for(df, _op=operation, _h=hpss, _cs=create_sub, _us=update_sub): ax.set_axisbelow(True) -def plot_comparison_extract( - ax, - df_cur: pd.DataFrame, - df_bas: pd.DataFrame, -): - """ - Seq vs par extract comparison with current/baseline pairing, grouped by - the combined (create_subdir, update_subdir) archive config. +def plot_comparison_extract(ax, df_cur: pd.DataFrame, df_bas: pd.DataFrame): + """Seq vs par extract, current vs baseline. Outliers removed per group.""" + df_cur = _filter_df_outliers( + df_cur[df_cur["operation"].isin(["extract_seq", "extract_par"])].copy(), + ["operation", "create_subdir", "update_subdir", "hpss_label"], + ) + df_bas = _filter_df_outliers( + df_bas[df_bas["operation"].isin(["extract_seq", "extract_par"])].copy(), + ["operation", "create_subdir", "update_subdir", "hpss_label"], + ) - Bar order within each HPSS × op cell (innermost grouping): - [current/seq] [baseline/seq] ‹op_gap› [current/par] [baseline/par] - """ configs = _extract_configs(df_cur) n_configs = len(configs) ops = ["extract_seq", "extract_par"] op_hatches = {"extract_seq": "", "extract_par": "xxxx"} pair_width = BAR_WIDTH - inner_gap = BAR_WIDTH * 0.15 # gap between current/baseline within a pair - op_gap = BAR_WIDTH * 0.55 # larger gap between seq-pair and par-pair - hpss_gap = BAR_WIDTH * 0.30 # gap between HPSS groups + inner_gap = BAR_WIDTH * 0.15 + op_gap = BAR_WIDTH * 0.55 + hpss_gap = BAR_WIDTH * 0.30 pair_span = 2 * pair_width + inner_gap hpss_group_span = 2 * pair_span + op_gap @@ -706,8 +769,8 @@ def plot_comparison_extract( for op_idx, op in enumerate(ops): hatch = op_hatches[op] op_origin = hpss_origin + op_idx * (pair_span + op_gap) - x_bas = op_origin # baseline — left - x_cur = op_origin + pair_width + inner_gap # current — right + x_bas = op_origin + x_cur = op_origin + pair_width + inner_gap def mean_for(df, _op=op, _h=hpss, _cs=create_sub, _us=update_sub): v = ( @@ -725,7 +788,6 @@ def mean_for(df, _op=op, _h=hpss, _cs=create_sub, _us=update_sub): cur_mean = mean_for(df_cur) bas_mean = mean_for(df_bas) - # Baseline bar (left): op-hatch + //// to mark it as baseline bas_hatch = hatch + "////" ax.bar( x_bas, @@ -737,7 +799,6 @@ def mean_for(df, _op=op, _h=hpss, _cs=create_sub, _us=update_sub): zorder=2, edgecolor=color, ) - # Current bar (right): op-hatch only ax.bar( x_cur, cur_mean, @@ -786,7 +847,6 @@ def mean_for(df, _op=op, _h=hpss, _cs=create_sub, _us=update_sub): ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) ax.set_axisbelow(True) - # Legend: colour=hpss, hatch=seq/par, alpha=current/baseline hpss_patches = [ mpatches.Patch(color=HPSS_COLORS[h], label=HPSS_LABELS[h]) for h in HPSS_ORDER ] @@ -813,11 +873,11 @@ def mean_for(df, _op=op, _h=hpss, _cs=create_sub, _us=update_sub): def build_comparison_figure( df_cur: pd.DataFrame, df_bas: pd.DataFrame, - all_dirs: list[str], + all_dirs: list, cur_label: str, bas_label: str, ) -> plt.Figure: - """Build and return the full baseline-comparison figure.""" + """Build and return the full baseline-comparison figure (Figure 2).""" fig = plt.figure(figsize=(16, 17)) fig.suptitle( f"zstash Performance: Current vs Baseline\n" @@ -834,7 +894,6 @@ def build_comparison_figure( gs = fig.add_gridspec( 3, 2, hspace=0.58, wspace=0.35, top=0.92, bottom=0.07, left=0.07, right=0.97 ) - axes = { "create": fig.add_subplot(gs[0, 0]), "update": fig.add_subplot(gs[0, 1]), @@ -849,7 +908,6 @@ def build_comparison_figure( else: _plot_comparison_extract_single_op(axes[op], df_cur, df_bas, op) - # Shared legend for solid=current, hatched=baseline cur_patch = mpatches.Patch(facecolor="grey", alpha=0.85, label="Current branch") bas_patch = mpatches.Patch( facecolor="grey", alpha=0.40, hatch="////", label="Baseline (main)" @@ -858,32 +916,24 @@ def build_comparison_figure( mpatches.Patch(color=HPSS_COLORS[h], label=HPSS_LABELS[h]) for h in HPSS_ORDER ] axes["create"].legend( - handles=[cur_patch, bas_patch] + hpss_patches, - fontsize=7, - loc="upper right", + handles=[cur_patch, bas_patch] + hpss_patches, fontsize=7, loc="upper right" ) plot_comparison_extract(ax_cmp, df_cur, df_bas) return fig -# String labels used in the suptitle (avoids referencing undefined vars earlier) RATIO_REGRESSION_COLOR_LABEL = "red" RATIO_IMPROVEMENT_COLOR_LABEL = "green" # --------------------------------------------------------------------------- -# Figure 3 – full record archive +# Archive data loading # --------------------------------------------------------------------------- def _archive_date_from_path(csv_path: Path) -> Optional[datetime.date]: - """ - Parse the record date from a CSV filename that contains YYYYMMDD. - - E.g. ``performance_20260603_results.csv`` → ``date(2026, 6, 3)`` - Returns *None* when no eight-digit date string is found. - """ + """Parse YYYYMMDD from a CSV filename; return None if not found.""" m = re.search(r"(\d{8})", csv_path.stem) if not m: return None @@ -897,11 +947,7 @@ def _archive_date_from_path(csv_path: Path) -> Optional[datetime.date]: def load_archive_data(archive_dir: str) -> pd.DataFrame: """ Load and concatenate every ``*results*.csv`` in *archive_dir*. - - Adds a ``record_date`` column (pandas Timestamp) derived from the - filename. Files with no parseable date are skipped with a warning. - - Returns an empty DataFrame (with expected columns) on failure. + Adds a ``record_date`` column (pandas Timestamp) from the filename. """ _empty = pd.DataFrame( columns=[ @@ -925,8 +971,7 @@ def load_archive_data(archive_dir: str) -> pd.DataFrame: csv_files = sorted(archive_path.glob("*results*.csv")) if not csv_files: print( - f"WARNING: no *results*.csv files found in {archive_path}", - file=sys.stderr, + f"WARNING: no *results*.csv files found in {archive_path}", file=sys.stderr ) return _empty @@ -935,7 +980,7 @@ def load_archive_data(archive_dir: str) -> pd.DataFrame: record_date = _archive_date_from_path(p) if record_date is None: print( - f"WARNING: cannot parse date from filename {p.name!r}, skipping.", + f"WARNING: cannot parse date from {p.name!r}, skipping.", file=sys.stderr, ) continue @@ -955,24 +1000,20 @@ def load_archive_data(archive_dir: str) -> pd.DataFrame: return df_all +# --------------------------------------------------------------------------- +# Figure 3 – archive: create & update +# --------------------------------------------------------------------------- + + def plot_archive_timeseries(ax, df_arch: pd.DataFrame, operation: str) -> None: """ - Time-series line graph for *operation* over the full record archive. - - Axes - ---- - X : record date - Y : elapsed_seconds (mean over all test configs sharing the same - subdir × hpss × date) - - Visual encoding - --------------- - Color : hpss_label – blue (none) / orange (hpss) / green (globus) - Line style: subdir – solid (build) / dashed (run) / dotted (init) - → 9 lines total (3 subdirs × 3 hpss modes) + Time-series for create/update over the full archive. + Outliers removed within each (date, subdir, hpss) group before aggregating. + Color = hpss mode; line style = subdir. """ dir_col = OP_DIR_COL[operation] df_op = df_arch[df_arch["operation"] == operation].copy() + df_op = _filter_df_outliers(df_op, ["record_date", dir_col, "hpss_label"]) for hpss in HPSS_ORDER: color = HPSS_COLORS[hpss] @@ -1013,7 +1054,6 @@ def plot_archive_timeseries(ax, df_arch: pd.DataFrame, operation: str) -> None: ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) ax.set_axisbelow(True) - # Two-section legend: top = hpss color, bottom = subdir line style color_handles = [ matplotlib.lines.Line2D( [], [], color=HPSS_COLORS[h], linewidth=2, label=HPSS_LABELS[h] @@ -1042,26 +1082,19 @@ def plot_archive_timeseries(ax, df_arch: pd.DataFrame, operation: str) -> None: def plot_archive_boxplot(ax, df_arch: pd.DataFrame, operation: str) -> None: """ - Vertical box-and-whisker plots for *operation* across the full archive. - - Layout - ------ - X axis : one tick-group per subdir (build, run, init); within each group - the three hpss modes are placed side by side. - Color : hpss_label (same palette as all other figures) - Overlay : individual data-point dots (jittered, matching other figures) - → 9 boxes total (3 subdirs × 3 hpss modes) + Box-and-whisker for create/update across the full archive. + Outliers removed per (subdir, hpss) group before drawing. """ dir_col = OP_DIR_COL[operation] df_op = df_arch[df_arch["operation"] == operation].copy() + df_op = _filter_df_outliers(df_op, [dir_col, "hpss_label"]) n_hpss = len(HPSS_ORDER) group_width = n_hpss * BAR_WIDTH + 0.10 x_base = np.arange(len(SUBDIR_ORDER)) * group_width offsets = np.linspace(0, (n_hpss - 1) * BAR_WIDTH, n_hpss) - tick_positions = [] - tick_labels = [] + tick_positions, tick_labels = [], [] for s_idx, subdir in enumerate(SUBDIR_ORDER): tick_positions.append(x_base[s_idx] + offsets.mean()) @@ -1071,12 +1104,9 @@ def plot_archive_boxplot(ax, df_arch: pd.DataFrame, operation: str) -> None: mask = (df_op["hpss_label"] == hpss) & (df_op[dir_col] == subdir) vals = df_op[mask]["elapsed_seconds"].dropna().values x_pos = x_base[s_idx] + offsets[h_idx] - if len(vals) == 0: continue - color = HPSS_COLORS[hpss] - ax.boxplot( vals, positions=[x_pos], @@ -1091,8 +1121,6 @@ def plot_archive_boxplot(ax, df_arch: pd.DataFrame, operation: str) -> None: capprops=dict(linewidth=0.8), flierprops=dict(marker="", linestyle="none"), ) - - # Overlay individual data points jitter = np.random.uniform( -BAR_WIDTH * 0.2, BAR_WIDTH * 0.2, size=len(vals) ) @@ -1119,7 +1147,6 @@ def plot_archive_boxplot(ax, df_arch: pd.DataFrame, operation: str) -> None: ax.set_ylabel("Wall-clock time (s)", fontsize=8) ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) ax.set_axisbelow(True) - hpss_patches = [ mpatches.Patch(color=HPSS_COLORS[h], alpha=0.75, label=HPSS_LABELS[h]) for h in HPSS_ORDER @@ -1129,7 +1156,7 @@ def plot_archive_boxplot(ax, df_arch: pd.DataFrame, operation: str) -> None: def build_archive_figure(df_arch: pd.DataFrame) -> plt.Figure: """ - Figure 3 – full archive overview. + Figure 3 – full archive overview for create & update. Layout (2 rows × 2 cols): [0,0] create time-series | [0,1] update time-series @@ -1137,34 +1164,249 @@ def build_archive_figure(df_arch: pd.DataFrame) -> plt.Figure: """ fig = plt.figure(figsize=(15, 12)) fig.suptitle( - "zstash Performance – Full Record Archive\n" + "zstash Performance – Full Record Archive (create & update)\n" "Time series: color = HPSS mode · line style = directory " "(solid = build/, dashed = run/, dotted = init/)\n" - "Box plots: every recorded runtime for each (directory, HPSS) combination", + "Box plots: every recorded runtime per (directory, HPSS) combination\n" + "Outliers removed via IQR method before plotting", fontsize=11, fontweight="bold", y=0.995, ) - gs = fig.add_gridspec( 2, 2, hspace=0.48, wspace=0.30, top=0.90, bottom=0.08, left=0.07, right=0.97 ) - for col_idx, op in enumerate(["create", "update"]): - ax_ts = fig.add_subplot(gs[0, col_idx]) - ax_box = fig.add_subplot(gs[1, col_idx]) - plot_archive_timeseries(ax_ts, df_arch, op) - plot_archive_boxplot(ax_box, df_arch, op) + plot_archive_timeseries(fig.add_subplot(gs[0, col_idx]), df_arch, op) + plot_archive_boxplot(fig.add_subplot(gs[1, col_idx]), df_arch, op) + return fig + + +# --------------------------------------------------------------------------- +# Figure 4 – archive: extract_seq & extract_par +# --------------------------------------------------------------------------- + + +def plot_extract_archive_timeseries(ax, df_arch: pd.DataFrame, operation: str) -> None: + """ + Time-series for an extract operation over the full archive. + + Since extract has no single directory column, lines are keyed by the + combined (create_subdir, update_subdir) archive config pair. + Color = hpss mode; line style = create_subdir. + Outliers removed within each (date, create_subdir, update_subdir, hpss) group. + """ + df_op = df_arch[df_arch["operation"] == operation].copy() + df_op = _filter_df_outliers( + df_op, ["record_date", "create_subdir", "update_subdir", "hpss_label"] + ) + + all_pairs = sorted( + df_op[["create_subdir", "update_subdir"]] + .drop_duplicates() + .apply(tuple, axis=1) + .tolist(), + key=lambda p: (dir_sort_key(p[0]), dir_sort_key(p[1])), + ) + + markers = ["o", "s", "^", "D", "v", "P", "X", "*", "h"] + + for hpss in HPSS_ORDER: + color = HPSS_COLORS[hpss] + for p_idx, (create_sub, update_sub) in enumerate(all_pairs): + ls = SUBDIR_LINESTYLES.get(create_sub, "solid") + marker = markers[p_idx % len(markers)] + mask = ( + (df_op["hpss_label"] == hpss) + & (df_op["create_subdir"] == create_sub) + & (df_op["update_subdir"] == update_sub) + ) + df_line = ( + df_op[mask] + .groupby("record_date")["elapsed_seconds"] + .mean() + .reset_index() + .sort_values("record_date") + ) + if df_line.empty: + continue + ax.plot( + df_line["record_date"], + df_line["elapsed_seconds"], + color=color, + linestyle=ls, + linewidth=1.6, + marker=marker, + markersize=4, + label=f"{HPSS_LABELS[hpss]} – create:{create_sub}/ update:{update_sub}/", + zorder=3, + ) + + ax.set_title( + f"{OP_TITLES[operation]} – runtime over time", + fontsize=10, + fontweight="bold", + pad=6, + ) + ax.set_xlabel("Record date", fontsize=8) + ax.set_ylabel("Wall-clock time (s)", fontsize=8) + ax.xaxis.set_major_formatter(matplotlib.dates.DateFormatter("%Y-%m-%d")) + plt.setp(ax.get_xticklabels(), rotation=30, ha="right", fontsize=7) + ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) + ax.set_axisbelow(True) + color_handles = [ + matplotlib.lines.Line2D( + [], [], color=HPSS_COLORS[h], linewidth=2, label=HPSS_LABELS[h] + ) + for h in HPSS_ORDER + ] + style_handles = [ + matplotlib.lines.Line2D( + [], + [], + color="grey", + linewidth=2, + linestyle=SUBDIR_LINESTYLES.get(s, "solid"), + label=f"create: {s}/", + ) + for s in SUBDIR_ORDER + if any(p[0] == s for p in all_pairs) + ] + ax.legend( + handles=color_handles + style_handles, + fontsize=6.5, + loc="upper left", + ncol=2, + framealpha=0.8, + ) + + +def plot_extract_archive_boxplot(ax, df_arch: pd.DataFrame, operation: str) -> None: + """ + Box-and-whisker for an extract operation across the full archive. + + X-axis groups = (create_subdir, update_subdir) archive config pairs + (matching the x-axis used in Figures 1 and 2). + Within each group the three HPSS modes sit side by side. + Outliers removed per (archive config pair, hpss_label) group. + """ + df_op = df_arch[df_arch["operation"] == operation].copy() + df_op = _filter_df_outliers(df_op, ["create_subdir", "update_subdir", "hpss_label"]) + + all_pairs = sorted( + df_op[["create_subdir", "update_subdir"]] + .drop_duplicates() + .apply(tuple, axis=1) + .tolist(), + key=lambda p: (dir_sort_key(p[0]), dir_sort_key(p[1])), + ) + + n_hpss = len(HPSS_ORDER) + group_width = n_hpss * BAR_WIDTH + 0.10 + x_base = np.arange(len(all_pairs)) * group_width + offsets = np.linspace(0, (n_hpss - 1) * BAR_WIDTH, n_hpss) + + tick_positions, tick_labels = [], [] + + for p_idx, (create_sub, update_sub) in enumerate(all_pairs): + tick_positions.append(x_base[p_idx] + offsets.mean()) + tick_labels.append(_extract_tick_label(create_sub, update_sub)) + + for h_idx, hpss in enumerate(HPSS_ORDER): + mask = ( + (df_op["hpss_label"] == hpss) + & (df_op["create_subdir"] == create_sub) + & (df_op["update_subdir"] == update_sub) + ) + vals = df_op[mask]["elapsed_seconds"].dropna().values + x_pos = x_base[p_idx] + offsets[h_idx] + if len(vals) == 0: + continue + color = HPSS_COLORS[hpss] + ax.boxplot( + vals, + positions=[x_pos], + widths=BAR_WIDTH * 0.85, + patch_artist=True, + vert=True, + manage_ticks=False, + zorder=2, + boxprops=dict(facecolor=color, alpha=0.55, linewidth=0.8), + medianprops=dict(color="black", linewidth=1.5), + whiskerprops=dict(linewidth=0.8), + capprops=dict(linewidth=0.8), + flierprops=dict(marker="", linestyle="none"), + ) + jitter = np.random.uniform( + -BAR_WIDTH * 0.2, BAR_WIDTH * 0.2, size=len(vals) + ) + ax.scatter( + x_pos + jitter, + vals, + color="white", + edgecolors=color, + s=DOT_SIZE, + zorder=3, + alpha=DOT_ALPHA, + linewidths=1.2, + ) + + ax.set_xticks(tick_positions) + ax.set_xticklabels(tick_labels, fontsize=7) + ax.set_title( + f"{OP_TITLES[operation]} – runtime distribution (all records)", + fontsize=10, + fontweight="bold", + pad=6, + ) + ax.set_xlabel("Archive contents (create → update)", fontsize=8) + ax.set_ylabel("Wall-clock time (s)", fontsize=8) + ax.yaxis.grid(True, linestyle="--", alpha=0.5, zorder=0) + ax.set_axisbelow(True) + hpss_patches = [ + mpatches.Patch(color=HPSS_COLORS[h], alpha=0.75, label=HPSS_LABELS[h]) + for h in HPSS_ORDER + ] + ax.legend(handles=hpss_patches, fontsize=7, loc="upper right") + + +def build_extract_archive_figure(df_arch: pd.DataFrame) -> plt.Figure: + """ + Figure 4 – full archive overview for extract_seq & extract_par. + + Layout (2 rows × 2 cols): + [0,0] extract_seq time-series | [0,1] extract_par time-series + [1,0] extract_seq box plot | [1,1] extract_par box plot + + Only produced when the archive contains extract operation rows. + """ + fig = plt.figure(figsize=(15, 12)) + fig.suptitle( + "zstash Performance – Full Record Archive (extract_seq & extract_par)\n" + "Time series: color = HPSS mode · line style = create_subdir " + "(solid = build/, dashed = run/, dotted = init/)\n" + "Box plots: every recorded runtime per (archive config, HPSS) combination\n" + "Outliers removed via IQR method before plotting", + fontsize=11, + fontweight="bold", + y=0.995, + ) + gs = fig.add_gridspec( + 2, 2, hspace=0.55, wspace=0.32, top=0.90, bottom=0.10, left=0.07, right=0.97 + ) + for col_idx, op in enumerate(["extract_seq", "extract_par"]): + plot_extract_archive_timeseries(fig.add_subplot(gs[0, col_idx]), df_arch, op) + plot_extract_archive_boxplot(fig.add_subplot(gs[1, col_idx]), df_arch, op) return fig # --------------------------------------------------------------------------- -# Main +# Main – helpers # --------------------------------------------------------------------------- -def main(): +def _parse_args(): parser = argparse.ArgumentParser( description="Visualise zstash performance results." ) @@ -1176,60 +1418,45 @@ def main(): parser.add_argument( "--dpi", type=int, default=150, help="Output DPI (default: 150)" ) - args = parser.parse_args() + return parser.parse_args() - cfg_path = Path(args.cfg) - if not cfg_path.is_file(): - print(f"ERROR: config file not found: {cfg_path}", file=sys.stderr) - print( - f"Copy {_SCRIPT_DIR / 'perf.cfg'} and edit it for your run.", - file=sys.stderr, - ) - sys.exit(1) - cfg = _load_cfg(cfg_path) - RESULTS_CSV: str = _cfg_require(cfg, "results_csv", cfg_path) - BASELINE_RESULTS_CSV: Optional[str] = _cfg_optional(cfg, "baseline_results_csv") - OUTPUT_PATH: Optional[str] = _cfg_optional(cfg, "output_path") - ARCHIVE_DIR: Optional[str] = _cfg_optional(cfg, "performance_archive_dir") - - results_path = Path(RESULTS_CSV) - if not RESULTS_CSV or not results_path.is_file(): - print(f"ERROR: results_csv not found: {RESULTS_CSV!r}", file=sys.stderr) +def _load_results(results_csv: str) -> pd.DataFrame: + """Load and validate the primary results CSV; exit on error.""" + results_path = Path(results_csv) + if not results_path.is_file(): + print(f"ERROR: results_csv not found: {results_csv!r}", file=sys.stderr) sys.exit(1) df = load_data(str(results_path)) if df.empty: print( - f"ERROR: results_csv is empty or could not be parsed: {RESULTS_CSV!r}", - file=sys.stderr, + f"ERROR: results_csv empty or unparseable: {results_csv!r}", file=sys.stderr ) sys.exit(1) + return df - # Determine the sorted list of directories that actually appear in the data - all_dirs = sorted( - set(df["create_subdir"].dropna()) | set(df["update_subdir"].dropna()), - key=dir_sort_key, - ) - # ----------------------------------------------------------------------- - # Figure layout: 3 rows × 2 cols - # Row 0: create | update - # Row 1: extract_seq | extract_par - # Row 2: extract seq-vs-par comparison (spans both columns) - # ----------------------------------------------------------------------- +def build_overview_figure(df: pd.DataFrame, all_dirs: list) -> plt.Figure: + """ + Figure 1 – performance overview. + + Layout: 3 rows × 2 cols + Row 0: create | update + Row 1: extract_seq | extract_par + Row 2: extract seq-vs-par comparison (spans both columns) + """ fig = plt.figure(figsize=(15, 16)) fig.suptitle( "zstash Performance Profiling\n" - "(bars = mean over test configs; dots = individual runs)", + "(bars = mean over test configs; dots = individual runs; " + "outliers removed via IQR)", fontsize=13, fontweight="bold", y=0.98, ) - gs = fig.add_gridspec( 3, 2, hspace=0.55, wspace=0.35, top=0.93, bottom=0.07, left=0.07, right=0.97 ) - axes = { "create": fig.add_subplot(gs[0, 0]), "update": fig.add_subplot(gs[0, 1]), @@ -1238,20 +1465,13 @@ def main(): } ax_cmp = fig.add_subplot(gs[2, :]) - # ----------------------------------------------------------------------- - # Draw the four single-operation subplots - # ----------------------------------------------------------------------- legend_handles = None for op in OP_ORDER: ax = axes[op] if op in OP_DIR_COL: - df_op = df[df["operation"] == op] - plot_operation(ax, df_op, op, all_dirs) + plot_operation(ax, df[df["operation"] == op], op, all_dirs) else: - # extract_seq / extract_par each get a dedicated single-op view - # that still uses the combined archive config as the x-axis. _plot_extract_single_op(ax, df, op) - if legend_handles is None: legend_handles = [ mpatches.Patch(color=HPSS_COLORS[h], label=HPSS_LABELS[h]) @@ -1259,77 +1479,147 @@ def main(): ] ax.legend(handles=legend_handles, fontsize=7, loc="upper right") - # ----------------------------------------------------------------------- - # Draw the sequential vs parallel comparison subplot - # ----------------------------------------------------------------------- plot_extract_comparison(ax_cmp, df) + return fig - # ----------------------------------------------------------------------- - # Baseline comparison figure (Figure 2) - # ----------------------------------------------------------------------- - fig_cmp = None - if BASELINE_RESULTS_CSV: - bas_path = Path(BASELINE_RESULTS_CSV) - if not bas_path.exists(): - print( - f"WARNING: baseline_results_csv not found: {bas_path}", file=sys.stderr - ) - print("Skipping baseline comparison figure.", file=sys.stderr) - else: - df_bas = load_data(str(bas_path)) - # Derive a short label from the CSV path for titles, - # e.g. ".../performance_20260101/results.csv" → "performance_20260101" - bas_label = bas_path.parent.name - cur_label = Path(RESULTS_CSV).parent.name - fig_cmp = build_comparison_figure( - df, df_bas, all_dirs, cur_label, bas_label - ) - # ----------------------------------------------------------------------- - # Full archive figure (Figure 3) - # ----------------------------------------------------------------------- - fig_arch = None - if ARCHIVE_DIR: - df_arch = load_archive_data(ARCHIVE_DIR) - if not df_arch.empty: - fig_arch = build_archive_figure(df_arch) - else: - print( - "WARNING: no archive data found; skipping Figure 3.", - file=sys.stderr, - ) +def _try_build_comparison_figure( + df: pd.DataFrame, + all_dirs: list, + results_csv: str, + baseline_results_csv: Optional[str], +) -> Optional[plt.Figure]: + """Figure 2 – baseline comparison. Returns None when not applicable.""" + if not baseline_results_csv: + return None + bas_path = Path(baseline_results_csv) + if not bas_path.exists(): + print(f"WARNING: baseline_results_csv not found: {bas_path}", file=sys.stderr) + print("Skipping baseline comparison figure.", file=sys.stderr) + return None + df_bas = load_data(str(bas_path)) + bas_label = bas_path.parent.name + cur_label = Path(results_csv).parent.name + return build_comparison_figure(df, df_bas, all_dirs, cur_label, bas_label) - # ----------------------------------------------------------------------- - # Save or show - # ----------------------------------------------------------------------- - def save_or_show(figure, out_path_str, label): - if out_path_str: - out_path = Path(out_path_str) - out_path.parent.mkdir(parents=True, exist_ok=True) - figure.savefig(out_path, dpi=args.dpi, bbox_inches="tight") - print(f"{label} saved to: {out_path}") - try: - os.chmod(out_path, 0o644) - except OSError: - pass - web_path = str(out_path).replace( - "/global/cfs/cdirs/e3sm/www/", - "https://portal.nersc.gov/cfs/e3sm/", - ) - print(f" Accessible at: {web_path}") - else: - plt.show() - - if OUTPUT_PATH: - save_or_show(fig, OUTPUT_PATH, "Figure 1 (overview)") - if fig_cmp is not None: - p = Path(OUTPUT_PATH) - cmp_output = str(p.with_stem(p.stem + "_vs_baseline")) - save_or_show(fig_cmp, cmp_output, "Figure 2 (baseline comparison)") - if fig_arch is not None: - p = Path(OUTPUT_PATH) - arch_output = str(p.with_stem(p.stem + "_archive")) - save_or_show(fig_arch, arch_output, "Figure 3 (full archive)") + +def _try_build_archive_figures( + archive_dir: Optional[str], +) -> tuple: + """ + Figures 3 & 4 – full record archive. + + Returns a (fig_arch, fig_arch_extract) tuple; either element may be None + when the corresponding data is unavailable. + """ + if not archive_dir: + return None, None + df_arch = load_archive_data(archive_dir) + if df_arch.empty: + print( + "WARNING: no archive data found; skipping Figures 3 & 4.", file=sys.stderr + ) + return None, None + fig_arch = build_archive_figure(df_arch) + has_extract = df_arch["operation"].isin(["extract_seq", "extract_par"]).any() + if has_extract: + fig_arch_extract = build_extract_archive_figure(df_arch) + else: + print("INFO: no extract data in archive; skipping Figure 4.", file=sys.stderr) + fig_arch_extract = None + return fig_arch, fig_arch_extract + + +def _save_figure(figure: plt.Figure, out_path_str: str, label: str, dpi: int) -> None: + """Save *figure* to *out_path_str* and print the destination.""" + out_path = Path(out_path_str) + out_path.parent.mkdir(parents=True, exist_ok=True) + figure.savefig(out_path, dpi=dpi, bbox_inches="tight") + print(f"{label} saved to: {out_path}") + try: + os.chmod(out_path, 0o644) + except OSError: + pass + web_path = str(out_path).replace( + "/global/cfs/cdirs/e3sm/www/", + "https://portal.nersc.gov/cfs/e3sm/", + ) + print(f" Accessible at: {web_path}") + + +def _save_all_figures( + fig: plt.Figure, + fig_cmp: Optional[plt.Figure], + fig_arch: Optional[plt.Figure], + fig_arch_extract: Optional[plt.Figure], + output_path: str, + dpi: int, +) -> None: + """Save every non-None figure to a path derived from *output_path*.""" + p = Path(output_path) + _save_figure(fig, output_path, "Figure 1 (overview)", dpi) + if fig_cmp is not None: + _save_figure( + fig_cmp, + str(p.with_stem(p.stem + "_vs_baseline")), + "Figure 2 (baseline comparison)", + dpi, + ) + if fig_arch is not None: + _save_figure( + fig_arch, + str(p.with_stem(p.stem + "_archive")), + "Figure 3 (full archive: create & update)", + dpi, + ) + if fig_arch_extract is not None: + _save_figure( + fig_arch_extract, + str(p.with_stem(p.stem + "_archive_extract")), + "Figure 4 (full archive: extract_seq & extract_par)", + dpi, + ) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def main(): + args = _parse_args() + + cfg_path = Path(args.cfg) + if not cfg_path.is_file(): + print(f"ERROR: config file not found: {cfg_path}", file=sys.stderr) + print( + f"Copy {_SCRIPT_DIR / 'perf.cfg'} and edit it for your run.", + file=sys.stderr, + ) + sys.exit(1) + + cfg = _load_cfg(cfg_path) + results_csv: str = _cfg_require(cfg, "results_csv", cfg_path) + baseline_results_csv: Optional[str] = _cfg_optional(cfg, "baseline_results_csv") + output_path: Optional[str] = _cfg_optional(cfg, "output_path") + archive_dir: Optional[str] = _cfg_optional(cfg, "performance_archive_dir") + + df = _load_results(results_csv) + all_dirs = sorted( + set(df["create_subdir"].dropna()) | set(df["update_subdir"].dropna()), + key=dir_sort_key, + ) + + fig = build_overview_figure(df, all_dirs) + fig_cmp = _try_build_comparison_figure( + df, all_dirs, results_csv, baseline_results_csv + ) + fig_arch, fig_arch_extract = _try_build_archive_figures(archive_dir) + + if output_path: + _save_all_figures( + fig, fig_cmp, fig_arch, fig_arch_extract, output_path, args.dpi + ) else: plt.show() From e2ce94d1d2c8a5eb26716c7e42111a5c81cbe1cf Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 18 Jun 2026 17:35:07 -0700 Subject: [PATCH 23/27] Streamline developer interface --- tests/performance/README.md | 64 +++- tests/performance/develop_run.cfg | 99 ----- tests/performance/generate/developer_run.cfg | 19 + .../generate_performance_data.bash | 14 +- tests/performance/{ => generate}/perf.cfg | 24 +- tests/performance/visualize/developer_run.cfg | 14 + tests/performance/visualize/perf.cfg | 94 +++++ .../{ => visualize}/visualize_performance.py | 348 +++++++++++++++--- 8 files changed, 478 insertions(+), 198 deletions(-) delete mode 100644 tests/performance/develop_run.cfg create mode 100644 tests/performance/generate/developer_run.cfg rename tests/performance/{ => generate}/generate_performance_data.bash (97%) rename tests/performance/{ => generate}/perf.cfg (74%) create mode 100644 tests/performance/visualize/developer_run.cfg create mode 100644 tests/performance/visualize/perf.cfg rename tests/performance/{ => visualize}/visualize_performance.py (83%) diff --git a/tests/performance/README.md b/tests/performance/README.md index 6f8ea724..06e36019 100644 --- a/tests/performance/README.md +++ b/tests/performance/README.md @@ -34,7 +34,7 @@ Edit the run metadata section of your cfg file: # Use /pscratch since a lot of data will be transferred. # The results csv alone will be copied to a long-term directory at the end. work_dir=/pscratch/sd/u/username/zstash_performance/ -unique_id=performance_20260603 +gen_run_id=performance_20260603 # The environment that zstash will be run in. # Using Unified environment: @@ -76,11 +76,11 @@ cd tests/performance/ If no cfg file argument is given, the script looks for `perf.cfg` in the same directory. -Results will be saved to `${work_dir}${unique_id}/results.csv`. To keep all records together in a non-scratch space, the results csv is also copied to `${performance_archive_dir}/${unique_id}_results.csv`. +Results will be saved to `${work_dir}${gen_run_id}/results.csv`. To keep all records together in a non-scratch space, the results csv is also copied to `${performance_archive_dir}/${gen_run_id}_results.csv`. ## Visualize performance -Edit the visualizer section of your cfg file: +The visualizer lives in `tests/performance/visualize/`. Edit the visualizer section of your cfg file: ```ini # Path to the results CSV to show in Figure 1. @@ -95,13 +95,48 @@ baseline_results_csv=/pscratch/sd/u/username/zstash_performance/performance_2026 # Output path for the saved figures. # Leave blank to display interactively instead of saving. # Make sure to use the web server path, i.e., /global/cfs/cdirs/e3sm/www/... -output_path=/global/cfs/cdirs/e3sm/www/username/zstash_performance/performance_pr427_20260603.png +output_path=/global/cfs/cdirs/e3sm/www/username/zstash_performance/ +``` + +The following options are available for finer control over the visualizer output: + +```ini +# Subset of HPSS modes to include in every figure (comma-separated). +# Valid values: none, hpss, globus. Leave blank to include all three. +# Example: hpss_filter=none,hpss +hpss_filter= + +# Top-level subdirectory for all output files. +# When set (together with most_recent_gen_run_id), figures are placed under +# //. +# When blank, the existing behaviour (stem of output_path as filename stem) is used. +viz_run_id=pr427_20260603 + +# Identifier for the most recent generate run; used as the filename stem for +# Figures 1 & 2 and as a subdirectory under viz_run_id/. +# Requires viz_run_id to also be set. +most_recent_gen_run_id=performance_20260603 + +# Which figures to produce (comma-separated). Valid values: 1, 2, 3, 4. +# Leave blank to produce all applicable figures. +# Note: Figure 2 still requires baseline_results_csv; Figures 3/4 still require +# performance_archive_dir, regardless of this setting. +figures=1,2,3,4 +``` + +When both `viz_run_id` and `most_recent_gen_run_id` are set, output files are laid out as: + +``` +//.png +//_vs_baseline.png +//record_create_and_update.png +//record_extract.png ``` Once you have the parameters set up, run: ```bash -cd tests/performance/ +cd tests/performance/visualize/ python visualize_performance.py --cfg my_run.cfg ``` @@ -109,14 +144,25 @@ If `--cfg` is omitted, the script looks for `perf.cfg` in the same directory. The script will print both the file path and the URL to access the plots. +### Figures produced + +**Figure 1 – Performance overview** (`results_csv` required): A 2×2 grid of subplots (one per operation: create, update, extract_seq, extract_par) plus a 5th subplot comparing sequential vs parallel extract side-by-side. Bars represent HPSS mode (none / hpss / globus); individual data points are overlaid as dots when multiple test configs share the same directory. + +**Figure 2 – Baseline comparison** (`baseline_results_csv` required): Same layout as Figure 1, but each cell shows two bars (current = solid, baseline = hatched) with a current/baseline ratio annotation. Ratio > 1 indicates a regression (slower); ratio < 1 indicates an improvement (faster). + +**Figure 3 – Historical archive for create & update** (`performance_archive_dir` required): A 2×2 grid of time-series and box plots for create and update operations across all historical CSVs in the archive directory. + +**Figure 4 – Historical archive for extract** (`performance_archive_dir` required): Same layout as Figure 3, for extract_seq and extract_par operations. + ## For reference Records made before the long-term record space was made have been copied to it via: ```bash -SCRATCH_SPACE=/pscratch/sd/f/forsyth/zstash_performance -RECORDS_SPACE=/global/homes/f/forsyth/zstash_performance_records +SCRATCH_SPACE=/pscratch/sd/u/username/zstash_performance +RECORDS_SPACE=/global/homes/u/username/zstash_performance_records -for unique_id in \ +# Example: +for gen_run_id in \ performance_20260225 \ performance_20260226_pr402 \ performance_20260226_pr424 \ @@ -126,6 +172,6 @@ for unique_id in \ performance_pr416_20260403 \ performance_pr416_20260406 do - cp "${SCRATCH_SPACE}/${unique_id}/results.csv" "${RECORDS_SPACE}/${unique_id}_results.csv" + cp "${SCRATCH_SPACE}/${gen_run_id}/results.csv" "${RECORDS_SPACE}/${gen_run_id}_results.csv" done ``` diff --git a/tests/performance/develop_run.cfg b/tests/performance/develop_run.cfg deleted file mode 100644 index f48b2d20..00000000 --- a/tests/performance/develop_run.cfg +++ /dev/null @@ -1,99 +0,0 @@ -# This version of perf.cfg has filled-in username paths. - -# Performance profiling configuration – shared by both scripts: -# generate_performance_data.bash (pass as first argument, or name it perf.cfg) -# visualize_performance.py (pass via --cfg, or name it perf.cfg) -# Usage: -# ./generate_performance_data.bash [path/to/this/file] -# python visualize_performance.py [--cfg path/to/this/file] -# Default cfg file name (when no argument given): perf.cfg -# -# NOTE: Perlmutter home/scratch paths follow the pattern: -# /global/homes/u/username/... -# /pscratch/sd/u/username/... -# where u is the first letter of your username, e.g. user "forsyth" goes under "f". - -# --------------------------------------------------------------------------- -# Run metadata <- edit these for each new run -# --------------------------------------------------------------------------- - -# Scratch directory where intermediate data and logs are written. -# Use /pscratch since a lot of data will be transferred. -work_dir=/pscratch/sd/f/forsyth/zstash_performance/ - -# Unique identifier for this run (used as a sub-directory name and CSV prefix). -unique_id=performance_20260603 - -# Shell command(s) that activate the zstash environment. -# Separate multiple commands with ' ; ' (space-semicolon-space). -# Example - Unified environment: -# environment_commands=source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh -# Example - dev environment: -# environment_commands=source /global/homes/u/username/miniforge3/etc/profile.d/conda.sh ; conda activate zstash-pr427-20260603 -environment_commands=source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh - -# --------------------------------------------------------------------------- -# Data source <- usually no need to change these -# --------------------------------------------------------------------------- - -dir_to_copy_from=/global/cfs/cdirs/e3sm/forsyth/E3SMv2/v2.LR.historical_0201/ -subdir0=build/ -subdir1=run/ -subdir2=init/ - -# --------------------------------------------------------------------------- -# HPSS options -# --------------------------------------------------------------------------- - -# Space-separated list of HPSS modes to exercise. -# Valid values: none hpss globus -HPSS_OPTIONS=none hpss globus - -# Destination path on HPSS (used when hpss is in HPSS_OPTIONS). -dst_hpss_path=/home/f/forsyth/zstash_performance - -# --------------------------------------------------------------------------- -# Globus options <- used when globus is in HPSS_OPTIONS -# --------------------------------------------------------------------------- - -# Set to true to force a fresh Globus authentication at the start of the run. -# NOTE: This will delete your ~/.zstash.ini & ~/.zstash_globus_tokens.json files. -fresh_globus=true - -# UUID of the destination Globus endpoint. -# Common endpoints: -# LCRC Improv DTN 15288284-7006-4041-ba1a-6b52501e49f1 -# NERSC Perlmutter 6bdc7956-fc0f-4ad2-989c-7aa5ee643a79 -# NERSC HPSS 9cd89cfd-6d04-11e5-ba46-22000b92c6ec -# PIC Compy DTN 68fbd2fa-83d7-11e9-8e63-029d279f7e24 -dst_endpoint_uuid=15288284-7006-4041-ba1a-6b52501e49f1 - -# Destination directory on the Globus endpoint. -dst_endpoint_archive_dir=/lcrc/group/e3sm/ac.forsyth2/zstash_performance_dst_dir/ - -# --------------------------------------------------------------------------- -# Results archiving -# --------------------------------------------------------------------------- - -# Long-term (non-scratch) directory where the results CSV is copied at the end -# of a run. The file will be saved as: -# ${performance_archive_dir}/${unique_id}_results.csv -performance_archive_dir=/global/homes/f/forsyth/zstash_performance_records - -# --------------------------------------------------------------------------- -# visualize_performance.py options <- edit these for each new run -# --------------------------------------------------------------------------- - -# Path to the results CSV to show in Figure 1. -# This is the CSV produced by generate_performance_data.bash for this run. -results_csv=/pscratch/sd/f/forsyth/zstash_performance/performance_20260603/results.csv - -# Path to a baseline results CSV to compare against in Figure 2. -# Leave blank (or comment out) to skip Figure 2. -baseline_results_csv=/pscratch/sd/f/forsyth/zstash_performance/performance_20260414/results.csv - -# Output path for the saved figures. -# Leave blank (or comment out) to display interactively instead of saving. -# Make sure to use the web-server path so the URL is printed correctly, e.g.: -# /global/cfs/cdirs/e3sm/www/... -output_path=/global/cfs/cdirs/e3sm/www/forsyth/zstash_performance/performance_pr427_20260603.png diff --git a/tests/performance/generate/developer_run.cfg b/tests/performance/generate/developer_run.cfg new file mode 100644 index 00000000..d925eb7b --- /dev/null +++ b/tests/performance/generate/developer_run.cfg @@ -0,0 +1,19 @@ +# This version of perf.cfg has filled-in username paths. + +work_dir=/pscratch/sd/f/forsyth/zstash_performance/ +gen_run_id=performance_20260603 +environment_commands=source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh + +dir_to_copy_from=/global/cfs/cdirs/e3sm/forsyth/E3SMv2/v2.LR.historical_0201/ +subdir0=build/ +subdir1=run/ +subdir2=init/ + +HPSS_OPTIONS=none hpss globus +dst_hpss_path=/home/f/forsyth/zstash_performance + +fresh_globus=true +dst_endpoint_uuid=15288284-7006-4041-ba1a-6b52501e49f1 +dst_endpoint_archive_dir=/lcrc/group/e3sm/ac.forsyth2/zstash_performance_dst_dir/ + +performance_archive_dir=/global/homes/f/forsyth/zstash_performance_records diff --git a/tests/performance/generate_performance_data.bash b/tests/performance/generate/generate_performance_data.bash similarity index 97% rename from tests/performance/generate_performance_data.bash rename to tests/performance/generate/generate_performance_data.bash index 79957ba7..1271ba15 100755 --- a/tests/performance/generate_performance_data.bash +++ b/tests/performance/generate/generate_performance_data.bash @@ -64,7 +64,7 @@ cfg_get() { # a direct transfer to HPSS & a Globus transfer to Chrysalis work_dir="$(cfg_require work_dir)" work_dir="${work_dir%/}/" -unique_id="$(cfg_require unique_id)" +gen_run_id="$(cfg_require gen_run_id)" environment_commands="$(cfg_require environment_commands)" ############################################################################### @@ -116,7 +116,7 @@ dst_endpoint_archive_dir="$(cfg_get dst_endpoint_archive_dir "")" performance_archive_dir="$(cfg_require performance_archive_dir)" echo "[INFO] Loaded configuration from: ${CFG_FILE}" -echo "[INFO] work_dir=${work_dir} unique_id=${unique_id}" +echo "[INFO] work_dir=${work_dir} gen_run_id=${gen_run_id}" ############################################################################### # Utility functions @@ -338,7 +338,7 @@ run_extract() # Results tracking # CSV file to collect all runtimes for later visualization -results_csv="${work_dir}${unique_id}/results.csv" +results_csv="${work_dir}${gen_run_id}/results.csv" record_result() { @@ -369,7 +369,7 @@ if [ "${fresh_globus}" == "true" ] && [[ " ${HPSS_OPTIONS[*]} " == *" globus "* fi # Create the top-level results directory and CSV header -mkdir -p "${work_dir}${unique_id}" +mkdir -p "${work_dir}${gen_run_id}" echo "test_label,create_subdir,update_subdir,hpss_label,operation,elapsed_seconds" > "${results_csv}" print_info "Results CSV: ${results_csv}" @@ -416,13 +416,13 @@ for test_idx in 0 1 2 3 4 5; do print_step "==========================================" # Create unique work directories for this test - work_subdir="${work_dir}${unique_id}/test${test_label}/" + work_subdir="${work_dir}${gen_run_id}/test${test_label}/" mkdir -p "${work_subdir}" log_dir="${work_subdir}logs/" mkdir -p "${log_dir}" - dst_globus_path="globus://${dst_endpoint_uuid}/${dst_endpoint_archive_dir}${unique_id}/test${test_label}/" + dst_globus_path="globus://${dst_endpoint_uuid}/${dst_endpoint_archive_dir}${gen_run_id}/test${test_label}/" # Iterate over the three HPSS modes declare -A hpss_path_map=( @@ -476,7 +476,7 @@ done print_success "All tests completed. Results saved to: ${results_csv}" mkdir -p "${performance_archive_dir}" -performance_archive_path="${performance_archive_dir}/${unique_id}_results.csv" +performance_archive_path="${performance_archive_dir}/${gen_run_id}_results.csv" cp "${results_csv}" "${performance_archive_path}" print_success "Results copied to: ${performance_archive_path}" diff --git a/tests/performance/perf.cfg b/tests/performance/generate/perf.cfg similarity index 74% rename from tests/performance/perf.cfg rename to tests/performance/generate/perf.cfg index 01027b9f..06a58485 100644 --- a/tests/performance/perf.cfg +++ b/tests/performance/generate/perf.cfg @@ -1,9 +1,7 @@ # Performance profiling configuration – shared by both scripts: # generate_performance_data.bash (pass as first argument, or name it perf.cfg) -# visualize_performance.py (pass via --cfg, or name it perf.cfg) # Usage: # ./generate_performance_data.bash [path/to/this/file] -# python visualize_performance.py [--cfg path/to/this/file] # Default cfg file name (when no argument given): perf.cfg # # NOTE: Perlmutter home/scratch paths follow the pattern: @@ -20,7 +18,7 @@ work_dir=/pscratch/sd/u/username/zstash_performance/ # Unique identifier for this run (used as a sub-directory name and CSV prefix). -unique_id=performance_20260603 +gen_run_id=performance_20260603 # Shell command(s) that activate the zstash environment. # Separate multiple commands with ' ; ' (space-semicolon-space). @@ -75,23 +73,5 @@ dst_endpoint_archive_dir=/lcrc/group/e3sm/username/zstash_performance_dst_dir/ # Long-term (non-scratch) directory where the results CSV is copied at the end # of a run. The file will be saved as: -# ${performance_archive_dir}/${unique_id}_results.csv +# ${performance_archive_dir}/${gen_run_id}_results.csv performance_archive_dir=/global/homes/u/username/zstash_performance_records - -# --------------------------------------------------------------------------- -# visualize_performance.py options <- edit these for each new run -# --------------------------------------------------------------------------- - -# Path to the results CSV to show in Figure 1. -# This is the CSV produced by generate_performance_data.bash for this run. -results_csv=/pscratch/sd/u/username/zstash_performance/performance_20260603/results.csv - -# Path to a baseline results CSV to compare against in Figure 2. -# Leave blank (or comment out) to skip Figure 2. -baseline_results_csv=/pscratch/sd/u/username/zstash_performance/performance_20260414/results.csv - -# Output path for the saved figures. -# Leave blank (or comment out) to display interactively instead of saving. -# Make sure to use the web-server path so the URL is printed correctly, e.g.: -# /global/cfs/cdirs/e3sm/www/... -output_path=/global/cfs/cdirs/e3sm/www/username/zstash_performance/performance_pr427_20260603.png diff --git a/tests/performance/visualize/developer_run.cfg b/tests/performance/visualize/developer_run.cfg new file mode 100644 index 00000000..b690e695 --- /dev/null +++ b/tests/performance/visualize/developer_run.cfg @@ -0,0 +1,14 @@ +# This version of perf.cfg has filled-in username paths. + +performance_archive_dir=/global/homes/f/forsyth/zstash_performance_records + +results_csv=/pscratch/sd/f/forsyth/zstash_performance/performance_20260603/results.csv +baseline_results_csv=/pscratch/sd/f/forsyth/zstash_performance/performance_20260414/results.csv +output_path=/global/cfs/cdirs/e3sm/www/forsyth/zstash_performance/performance_pr427_20260603.png + +viz_run_id=viz_test_20260618_try2 +most_recent_gen_run_id=performance_20260603 + +hpss_filter=none,hpss,globus + +figures= diff --git a/tests/performance/visualize/perf.cfg b/tests/performance/visualize/perf.cfg new file mode 100644 index 00000000..17278fe4 --- /dev/null +++ b/tests/performance/visualize/perf.cfg @@ -0,0 +1,94 @@ +# Performance profiling configuration – shared by both scripts: +# visualize_performance.py (pass via --cfg, or name it perf.cfg) +# Usage: +# python visualize_performance.py [--cfg path/to/this/file] +# Default cfg file name (when no argument given): perf.cfg +# +# NOTE: Perlmutter home/scratch paths follow the pattern: +# /global/homes/u/username/... +# /pscratch/sd/u/username/... +# where u is the first letter of your username, e.g. user "forsyth" goes under "f". + +# --------------------------------------------------------------------------- +# Results archiving +# --------------------------------------------------------------------------- + +# Long-term (non-scratch) directory where results CSVs are copied at the end +# of a run. +performance_archive_dir=/global/homes/u/username/zstash_performance_records + +# --------------------------------------------------------------------------- +# visualize_performance.py options <- edit these for each new run +# --------------------------------------------------------------------------- + +# Path to the results CSV to show in Figure 1. +# This is the CSV produced by generate_performance_data.bash for this run. +results_csv=/pscratch/sd/u/username/zstash_performance/performance_20260603/results.csv + +# Path to a baseline results CSV to compare against in Figure 2. +# Leave blank (or comment out) to skip Figure 2. +baseline_results_csv=/pscratch/sd/u/username/zstash_performance/performance_20260414/results.csv + +# Output path for the saved figures. +# Leave blank (or comment out) to display interactively instead of saving. +# Make sure to use the web-server path so the URL is printed correctly, e.g.: +# /global/cfs/cdirs/e3sm/www/... +# When viz_run_id and most_recent_gen_run_id are both set, figures are saved +# under // using the filenames described below. +output_path=/global/cfs/cdirs/e3sm/www/username/zstash_performance/performance_pr427_20260603.png + +# --------------------------------------------------------------------------- +# Output filename control +# --------------------------------------------------------------------------- + +# Short identifier used as a top-level subdirectory for all output figures. +# When set together with most_recent_gen_run_id, figures are written to +# // +# and named as follows: +# .png +# _vs_baseline.png +# record_create_and_update.png +# record_extract.png +# Leave blank (or comment out) to use the stem of output_path (original behaviour). +# Example: viz_run_id=pr427_20260603 +viz_run_id= + +# Identifier for the most recent generate_performance_data.bash run. +# Used as the filename stem for Figures 1 & 2 and as a subdirectory under +# viz_run_id/. Requires viz_run_id to also be set; ignored when viz_run_id +# is blank. +# Example: most_recent_gen_run_id=performance_20260603 +most_recent_gen_run_id= + +# --------------------------------------------------------------------------- +# HPSS mode filter +# --------------------------------------------------------------------------- + +# Comma-separated list of HPSS modes to include in every figure. +# Valid values: none, hpss, globus (case-insensitive, any order). +# Leave blank (or comment out) to include all three modes. +# Examples: +# hpss_filter=none # local-only runs, no HPSS bars +# hpss_filter=none,hpss # skip globus +# hpss_filter=hpss,globus # skip the no-HPSS baseline +hpss_filter= + +# --------------------------------------------------------------------------- +# Figure selection +# --------------------------------------------------------------------------- + +# Comma-separated list of figure numbers to produce. +# Valid values: 1, 2, 3, 4 +# 1 = Performance overview (always available) +# 2 = Baseline comparison (requires baseline_results_csv) +# 3 = Archive: create & update (requires performance_archive_dir) +# 4 = Archive: extract_seq & extract_par (requires performance_archive_dir) +# Leave blank (or comment out) to produce all applicable figures. +# Note: listing a figure number does not bypass its data requirements — +# the data must still be present. Figures whose data is missing are skipped +# with a warning regardless of this setting. +# Examples: +# figures=1 # overview only, fast +# figures=1,2 # overview + baseline comparison +# figures=3,4 # archive history only +figures= diff --git a/tests/performance/visualize_performance.py b/tests/performance/visualize/visualize_performance.py similarity index 83% rename from tests/performance/visualize_performance.py rename to tests/performance/visualize/visualize_performance.py index 385ed9cd..0d24400f 100644 --- a/tests/performance/visualize_performance.py +++ b/tests/performance/visualize/visualize_performance.py @@ -34,7 +34,8 @@ to make the parallelism speed-up immediately visible. Figure 2 – Baseline comparison (current branch vs main): - Produced only when baseline_results_csv is set to a valid path in the cfg. + Produced only when baseline_results_csv is set to a valid path in the cfg + AND figure 2 is included in the figures list. Same 2×2 + comparison layout, but each cell shows two bars (current = solid, baseline = hatched) with a ratio annotation (current/baseline) above each pair. Ratio > 1 = regression (slower), @@ -43,7 +44,8 @@ Figure 3 – Full record archive for create & update (all historical CSVs in performance_archive_dir): Produced only when performance_archive_dir is set in the cfg and contains - *results*.csv files with YYYYMMDD in their names. + *results*.csv files with YYYYMMDD in their names, AND figure 3 is included + in the figures list. Layout: 2×2 grid (create | update) × (time-series | box plot). Time-series: x = record date, y = runtime, color = hpss mode, line style = subdir (solid=build, dashed=run, dotted=init). @@ -51,7 +53,8 @@ with individual data-point dots overlaid. Figure 4 – Full record archive for extract_seq & extract_par: - Produced only when performance_archive_dir is set and contains extract data. + Produced only when performance_archive_dir is set and contains extract data, + AND figure 4 is included in the figures list. Layout: 2×2 grid (extract_seq | extract_par) × (time-series | box plot). X-axis groups for box plots: (create_subdir, update_subdir) archive config pairs. Same color/line-style encoding as Figure 3. @@ -63,6 +66,44 @@ [Q1 - 1.5 * IQR, Q3 + 1.5 * IQR] are dropped silently. This prevents a single aberrant run from dominating axis scales while preserving legitimate spread. + +New cfg options +--------------- +hpss_filter + Comma-separated list of HPSS modes to include in every figure. + Valid values: none, hpss, globus (case-insensitive). + Leave blank or omit to include all three. + Example: hpss_filter=none,hpss + +viz_run_id + A short identifier string used as a top-level subdirectory for all output + files. When set (together with most_recent_gen_run_id), figures are placed + under //. + When blank, the existing behaviour (stem of output_path as filename stem, + parent directory of output_path as output directory) is used. + Example: viz_run_id=pr427_20260603 + +most_recent_gen_run_id + Identifier for the most recent generate_performance_data.bash run. + Used as the filename stem for Figures 1 & 2 and as a subdirectory under + viz_run_id/. Requires viz_run_id to also be set. + Example: most_recent_gen_run_id=performance_20260603 + +Output file layout when both viz_run_id and most_recent_gen_run_id are set: + //.png + //_vs_baseline.png + //record_create_and_update.png + //record_extract.png + +figures + Comma-separated list of figure numbers to produce. + Valid values: 1, 2, 3, 4. + Leave blank or omit to produce all applicable figures. + Specifying a figure number does not override data requirements: + Figure 2 still needs baseline_results_csv; Figures 3/4 still need + performance_archive_dir. But figures NOT in this list are skipped + entirely (data is not even loaded for them). + Example: figures=1,2 """ import argparse @@ -122,14 +163,76 @@ def _cfg_require(cfg: dict, key: str, cfg_path: Path) -> str: return v +# --------------------------------------------------------------------------- +# New cfg option parsers +# --------------------------------------------------------------------------- + +_ALL_HPSS = ["none", "hpss", "globus"] +_ALL_FIGURES = {1, 2, 3, 4} + + +def _parse_hpss_filter(raw: Optional[str]) -> list: + """ + Parse the hpss_filter cfg value into an ordered list of HPSS mode strings. + + Validates each token against the known set. Preserves the canonical order + (none → hpss → globus) regardless of the order given in the cfg. Returns + the full list when *raw* is None or blank. + """ + if not raw: + return list(_ALL_HPSS) + tokens = [t.strip().lower() for t in raw.split(",") if t.strip()] + invalid = [t for t in tokens if t not in _ALL_HPSS] + if invalid: + print( + f"ERROR: hpss_filter contains unknown mode(s): {invalid}\n" + f" Valid values: {_ALL_HPSS}", + file=sys.stderr, + ) + sys.exit(1) + # Return in canonical order so plots are always consistent. + return [h for h in _ALL_HPSS if h in tokens] + + +def _parse_figures(raw: Optional[str]) -> set: + """ + Parse the figures cfg value into a set of integer figure numbers. + + Validates each token. Returns the full set {1,2,3,4} when *raw* is None + or blank. + """ + if not raw: + return set(_ALL_FIGURES) + tokens = [t.strip() for t in raw.split(",") if t.strip()] + result = set() + for t in tokens: + if not t.isdigit() or int(t) not in _ALL_FIGURES: + print( + f"ERROR: figures contains invalid value: {t!r}\n" + f" Valid values: 1, 2, 3, 4", + file=sys.stderr, + ) + sys.exit(1) + result.add(int(t)) + return result + + # --------------------------------------------------------------------------- # Config (styling – not user-configurable) # --------------------------------------------------------------------------- +# HPSS_ORDER and HPSS_COLORS / HPSS_LABELS remain the full canonical sets. +# The active subset selected by hpss_filter is stored in the module-level +# variable ACTIVE_HPSS, set once in main() before any plotting begins. HPSS_ORDER = ["none", "hpss", "globus"] HPSS_COLORS = {"none": "#4C72B0", "hpss": "#DD8452", "globus": "#55A868"} HPSS_LABELS = {"none": "No HPSS", "hpss": "Direct HPSS", "globus": "Globus"} +# Module-level active HPSS list; overwritten in main() from cfg. +# All plotting functions reference ACTIVE_HPSS instead of HPSS_ORDER directly +# so a single assignment here propagates everywhere. +ACTIVE_HPSS: list = list(HPSS_ORDER) + OP_ORDER = ["create", "update", "extract_seq", "extract_par"] OP_TITLES = { "create": "zstash create", @@ -277,12 +380,12 @@ def plot_operation(ax, df_op: pd.DataFrame, operation: str, dirs: list): df_op = _filter_df_outliers(df_op.copy(), [dir_col, "hpss_label"]) n_dirs = len(dirs) - n_hpss = len(HPSS_ORDER) + n_hpss = len(ACTIVE_HPSS) x_base = np.arange(n_dirs) offsets = np.linspace(-(n_hpss - 1) / 2, (n_hpss - 1) / 2, n_hpss) * BAR_WIDTH - for h_idx, hpss in enumerate(HPSS_ORDER): + for h_idx, hpss in enumerate(ACTIVE_HPSS): df_h = df_op[df_op["hpss_label"] == hpss] means, all_vals, xs = [], [], [] @@ -375,12 +478,12 @@ def _plot_extract_single_op(ax, df: pd.DataFrame, operation: str): configs = _extract_configs(df) n_configs = len(configs) - n_hpss = len(HPSS_ORDER) + n_hpss = len(ACTIVE_HPSS) x_base = np.arange(n_configs, dtype=float) offsets = np.linspace(-(n_hpss - 1) / 2, (n_hpss - 1) / 2, n_hpss) * BAR_WIDTH - for h_idx, hpss in enumerate(HPSS_ORDER): + for h_idx, hpss in enumerate(ACTIVE_HPSS): means, all_vals, xs = [], [], [] for c_idx, (create_sub, update_sub) in enumerate(configs): vals = ( @@ -459,13 +562,13 @@ def plot_extract_comparison(ax, df: pd.DataFrame): n_configs = len(configs) ops = ["extract_seq", "extract_par"] hatches = {"extract_seq": "", "extract_par": "////"} - n_bars = len(HPSS_ORDER) * len(ops) + n_bars = len(ACTIVE_HPSS) * len(ops) group_width = n_bars * BAR_WIDTH + 0.15 x_base = np.arange(n_configs) * group_width for c_idx, (create_sub, update_sub) in enumerate(configs): - for h_idx, hpss in enumerate(HPSS_ORDER): + for h_idx, hpss in enumerate(ACTIVE_HPSS): for op_idx, op in enumerate(ops): df_cell = df_ext[ (df_ext["operation"] == op) @@ -504,7 +607,7 @@ def plot_extract_comparison(ax, df: pd.DataFrame): ax.set_axisbelow(True) hpss_patches = [ - mpatches.Patch(color=HPSS_COLORS[h], label=HPSS_LABELS[h]) for h in HPSS_ORDER + mpatches.Patch(color=HPSS_COLORS[h], label=HPSS_LABELS[h]) for h in ACTIVE_HPSS ] seq_patch = mpatches.Patch( facecolor="grey", hatch="", label="Sequential (1 worker)" @@ -556,14 +659,14 @@ def plot_comparison_operation( ) n_dirs = len(dirs) - n_hpss = len(HPSS_ORDER) + n_hpss = len(ACTIVE_HPSS) pair_width = BAR_WIDTH gap = BAR_WIDTH * 0.3 group_span = n_hpss * (2 * pair_width + gap) + 0.2 x_base = np.arange(n_dirs) * group_span - for h_idx, hpss in enumerate(HPSS_ORDER): + for h_idx, hpss in enumerate(ACTIVE_HPSS): color = HPSS_COLORS[hpss] pair_offset = h_idx * (2 * pair_width + gap) @@ -655,14 +758,14 @@ def _plot_comparison_extract_single_op( configs = _extract_configs(df_cur) n_configs = len(configs) - n_hpss = len(HPSS_ORDER) + n_hpss = len(ACTIVE_HPSS) pair_width = BAR_WIDTH gap = BAR_WIDTH * 0.3 group_span = n_hpss * (2 * pair_width + gap) + 0.2 x_base = np.arange(n_configs) * group_span - for h_idx, hpss in enumerate(HPSS_ORDER): + for h_idx, hpss in enumerate(ACTIVE_HPSS): color = HPSS_COLORS[hpss] pair_offset = h_idx * (2 * pair_width + gap) for c_idx, (create_sub, update_sub) in enumerate(configs): @@ -759,11 +862,11 @@ def plot_comparison_extract(ax, df_cur: pd.DataFrame, df_bas: pd.DataFrame): pair_span = 2 * pair_width + inner_gap hpss_group_span = 2 * pair_span + op_gap - group_span = len(HPSS_ORDER) * (hpss_group_span + hpss_gap) + 0.3 + group_span = len(ACTIVE_HPSS) * (hpss_group_span + hpss_gap) + 0.3 x_base = np.arange(n_configs) * group_span for c_idx, (create_sub, update_sub) in enumerate(configs): - for h_idx, hpss in enumerate(HPSS_ORDER): + for h_idx, hpss in enumerate(ACTIVE_HPSS): color = HPSS_COLORS[hpss] hpss_origin = x_base[c_idx] + h_idx * (hpss_group_span + hpss_gap) for op_idx, op in enumerate(ops): @@ -829,7 +932,7 @@ def mean_for(df, _op=op, _h=hpss, _cs=create_sub, _us=update_sub): zorder=4, ) - group_total_bar_span = len(HPSS_ORDER) * (hpss_group_span + hpss_gap) - hpss_gap + group_total_bar_span = len(ACTIVE_HPSS) * (hpss_group_span + hpss_gap) - hpss_gap x_ticks = x_base + group_total_bar_span / 2 ax.set_xticks(x_ticks) ax.set_xticklabels([_extract_tick_label(c, u) for c, u in configs], fontsize=7.5) @@ -848,7 +951,7 @@ def mean_for(df, _op=op, _h=hpss, _cs=create_sub, _us=update_sub): ax.set_axisbelow(True) hpss_patches = [ - mpatches.Patch(color=HPSS_COLORS[h], label=HPSS_LABELS[h]) for h in HPSS_ORDER + mpatches.Patch(color=HPSS_COLORS[h], label=HPSS_LABELS[h]) for h in ACTIVE_HPSS ] seq_patch = mpatches.Patch( facecolor="grey", hatch="", alpha=0.85, label="Sequential, current" @@ -913,7 +1016,7 @@ def build_comparison_figure( facecolor="grey", alpha=0.40, hatch="////", label="Baseline (main)" ) hpss_patches = [ - mpatches.Patch(color=HPSS_COLORS[h], label=HPSS_LABELS[h]) for h in HPSS_ORDER + mpatches.Patch(color=HPSS_COLORS[h], label=HPSS_LABELS[h]) for h in ACTIVE_HPSS ] axes["create"].legend( handles=[cur_patch, bas_patch] + hpss_patches, fontsize=7, loc="upper right" @@ -1015,7 +1118,7 @@ def plot_archive_timeseries(ax, df_arch: pd.DataFrame, operation: str) -> None: df_op = df_arch[df_arch["operation"] == operation].copy() df_op = _filter_df_outliers(df_op, ["record_date", dir_col, "hpss_label"]) - for hpss in HPSS_ORDER: + for hpss in ACTIVE_HPSS: color = HPSS_COLORS[hpss] for subdir in SUBDIR_ORDER: ls = SUBDIR_LINESTYLES.get(subdir, "solid") @@ -1058,7 +1161,7 @@ def plot_archive_timeseries(ax, df_arch: pd.DataFrame, operation: str) -> None: matplotlib.lines.Line2D( [], [], color=HPSS_COLORS[h], linewidth=2, label=HPSS_LABELS[h] ) - for h in HPSS_ORDER + for h in ACTIVE_HPSS ] style_handles = [ matplotlib.lines.Line2D( @@ -1089,7 +1192,7 @@ def plot_archive_boxplot(ax, df_arch: pd.DataFrame, operation: str) -> None: df_op = df_arch[df_arch["operation"] == operation].copy() df_op = _filter_df_outliers(df_op, [dir_col, "hpss_label"]) - n_hpss = len(HPSS_ORDER) + n_hpss = len(ACTIVE_HPSS) group_width = n_hpss * BAR_WIDTH + 0.10 x_base = np.arange(len(SUBDIR_ORDER)) * group_width offsets = np.linspace(0, (n_hpss - 1) * BAR_WIDTH, n_hpss) @@ -1100,7 +1203,7 @@ def plot_archive_boxplot(ax, df_arch: pd.DataFrame, operation: str) -> None: tick_positions.append(x_base[s_idx] + offsets.mean()) tick_labels.append(f"{subdir}/") - for h_idx, hpss in enumerate(HPSS_ORDER): + for h_idx, hpss in enumerate(ACTIVE_HPSS): mask = (df_op["hpss_label"] == hpss) & (df_op[dir_col] == subdir) vals = df_op[mask]["elapsed_seconds"].dropna().values x_pos = x_base[s_idx] + offsets[h_idx] @@ -1112,7 +1215,7 @@ def plot_archive_boxplot(ax, df_arch: pd.DataFrame, operation: str) -> None: positions=[x_pos], widths=BAR_WIDTH * 0.85, patch_artist=True, - vert=True, + orientation="vertical", manage_ticks=False, zorder=2, boxprops=dict(facecolor=color, alpha=0.55, linewidth=0.8), @@ -1149,7 +1252,7 @@ def plot_archive_boxplot(ax, df_arch: pd.DataFrame, operation: str) -> None: ax.set_axisbelow(True) hpss_patches = [ mpatches.Patch(color=HPSS_COLORS[h], alpha=0.75, label=HPSS_LABELS[h]) - for h in HPSS_ORDER + for h in ACTIVE_HPSS ] ax.legend(handles=hpss_patches, fontsize=7, loc="upper right") @@ -1211,7 +1314,7 @@ def plot_extract_archive_timeseries(ax, df_arch: pd.DataFrame, operation: str) - markers = ["o", "s", "^", "D", "v", "P", "X", "*", "h"] - for hpss in HPSS_ORDER: + for hpss in ACTIVE_HPSS: color = HPSS_COLORS[hpss] for p_idx, (create_sub, update_sub) in enumerate(all_pairs): ls = SUBDIR_LINESTYLES.get(create_sub, "solid") @@ -1259,7 +1362,7 @@ def plot_extract_archive_timeseries(ax, df_arch: pd.DataFrame, operation: str) - matplotlib.lines.Line2D( [], [], color=HPSS_COLORS[h], linewidth=2, label=HPSS_LABELS[h] ) - for h in HPSS_ORDER + for h in ACTIVE_HPSS ] style_handles = [ matplotlib.lines.Line2D( @@ -1302,7 +1405,7 @@ def plot_extract_archive_boxplot(ax, df_arch: pd.DataFrame, operation: str) -> N key=lambda p: (dir_sort_key(p[0]), dir_sort_key(p[1])), ) - n_hpss = len(HPSS_ORDER) + n_hpss = len(ACTIVE_HPSS) group_width = n_hpss * BAR_WIDTH + 0.10 x_base = np.arange(len(all_pairs)) * group_width offsets = np.linspace(0, (n_hpss - 1) * BAR_WIDTH, n_hpss) @@ -1313,7 +1416,7 @@ def plot_extract_archive_boxplot(ax, df_arch: pd.DataFrame, operation: str) -> N tick_positions.append(x_base[p_idx] + offsets.mean()) tick_labels.append(_extract_tick_label(create_sub, update_sub)) - for h_idx, hpss in enumerate(HPSS_ORDER): + for h_idx, hpss in enumerate(ACTIVE_HPSS): mask = ( (df_op["hpss_label"] == hpss) & (df_op["create_subdir"] == create_sub) @@ -1329,7 +1432,7 @@ def plot_extract_archive_boxplot(ax, df_arch: pd.DataFrame, operation: str) -> N positions=[x_pos], widths=BAR_WIDTH * 0.85, patch_artist=True, - vert=True, + orientation="vertical", manage_ticks=False, zorder=2, boxprops=dict(facecolor=color, alpha=0.55, linewidth=0.8), @@ -1366,7 +1469,7 @@ def plot_extract_archive_boxplot(ax, df_arch: pd.DataFrame, operation: str) -> N ax.set_axisbelow(True) hpss_patches = [ mpatches.Patch(color=HPSS_COLORS[h], alpha=0.75, label=HPSS_LABELS[h]) - for h in HPSS_ORDER + for h in ACTIVE_HPSS ] ax.legend(handles=hpss_patches, fontsize=7, loc="upper right") @@ -1475,7 +1578,7 @@ def build_overview_figure(df: pd.DataFrame, all_dirs: list) -> plt.Figure: if legend_handles is None: legend_handles = [ mpatches.Patch(color=HPSS_COLORS[h], label=HPSS_LABELS[h]) - for h in HPSS_ORDER + for h in ACTIVE_HPSS ] ax.legend(handles=legend_handles, fontsize=7, loc="upper right") @@ -1491,6 +1594,7 @@ def _try_build_comparison_figure( ) -> Optional[plt.Figure]: """Figure 2 – baseline comparison. Returns None when not applicable.""" if not baseline_results_csv: + print("INFO: baseline_results_csv not set; skipping Figure 2.", file=sys.stderr) return None bas_path = Path(baseline_results_csv) if not bas_path.exists(): @@ -1505,14 +1609,21 @@ def _try_build_comparison_figure( def _try_build_archive_figures( archive_dir: Optional[str], + want_fig3: bool, + want_fig4: bool, ) -> tuple: """ Figures 3 & 4 – full record archive. + *want_fig3* and *want_fig4* gate whether each figure is actually built. Returns a (fig_arch, fig_arch_extract) tuple; either element may be None - when the corresponding data is unavailable. + when the corresponding data is unavailable or the figure was not requested. """ if not archive_dir: + print( + "INFO: performance_archive_dir not set; skipping Figures 3 & 4.", + file=sys.stderr, + ) return None, None df_arch = load_archive_data(archive_dir) if df_arch.empty: @@ -1520,20 +1631,80 @@ def _try_build_archive_figures( "WARNING: no archive data found; skipping Figures 3 & 4.", file=sys.stderr ) return None, None - fig_arch = build_archive_figure(df_arch) - has_extract = df_arch["operation"].isin(["extract_seq", "extract_par"]).any() - if has_extract: - fig_arch_extract = build_extract_archive_figure(df_arch) + + fig_arch = build_archive_figure(df_arch) if want_fig3 else None + if not want_fig3: + print("INFO: Figure 3 not in figures list; skipping.", file=sys.stderr) + + fig_arch_extract = None + if want_fig4: + has_extract = df_arch["operation"].isin(["extract_seq", "extract_par"]).any() + if has_extract: + fig_arch_extract = build_extract_archive_figure(df_arch) + else: + print( + "INFO: no extract data in archive; skipping Figure 4.", file=sys.stderr + ) else: - print("INFO: no extract data in archive; skipping Figure 4.", file=sys.stderr) - fig_arch_extract = None + print("INFO: Figure 4 not in figures list; skipping.", file=sys.stderr) + return fig_arch, fig_arch_extract +def _output_paths( + output_path: str, + viz_run_id: Optional[str], + most_recent_gen_run_id: Optional[str], +) -> tuple: + """ + Return the four output file paths as a tuple: (fig1, fig2, fig3, fig4). + + When both *viz_run_id* and *most_recent_gen_run_id* are set, the new + directory-based layout is used: + //.png + //_vs_baseline.png + //record_create_and_update.png + //record_extract.png + + Otherwise the legacy flat layout is used (all files in the parent + directory of output_path, stem from viz_run_id or output_path): + /.png + /_vs_baseline.png + /_archive.png + /_archive_extract.png + """ + p = Path(output_path) + suffix = p.suffix or ".png" + + if viz_run_id and most_recent_gen_run_id: + out_dir = p.parent / viz_run_id + stem = most_recent_gen_run_id + return ( + str(out_dir / (stem + suffix)), + str(out_dir / (stem + "_vs_baseline" + suffix)), + str(out_dir / ("record_create_and_update" + suffix)), + str(out_dir / ("record_extract" + suffix)), + ) + + # Legacy behaviour: flat files in the parent directory of output_path. + out_dir = p.parent + stem = viz_run_id if viz_run_id else p.stem + return ( + str(out_dir / (stem + suffix)), + str(out_dir / (stem + "_vs_baseline" + suffix)), + str(out_dir / (stem + "_archive" + suffix)), + str(out_dir / (stem + "_archive_extract" + suffix)), + ) + + def _save_figure(figure: plt.Figure, out_path_str: str, label: str, dpi: int) -> None: """Save *figure* to *out_path_str* and print the destination.""" out_path = Path(out_path_str) out_path.parent.mkdir(parents=True, exist_ok=True) + try: + os.chmod(out_path.parent, 0o755) + except OSError: + pass figure.savefig(out_path, dpi=dpi, bbox_inches="tight") print(f"{label} saved to: {out_path}") try: @@ -1548,34 +1719,32 @@ def _save_figure(figure: plt.Figure, out_path_str: str, label: str, dpi: int) -> def _save_all_figures( - fig: plt.Figure, + fig: Optional[plt.Figure], fig_cmp: Optional[plt.Figure], fig_arch: Optional[plt.Figure], fig_arch_extract: Optional[plt.Figure], output_path: str, + viz_run_id: Optional[str], + most_recent_gen_run_id: Optional[str], dpi: int, ) -> None: - """Save every non-None figure to a path derived from *output_path*.""" - p = Path(output_path) - _save_figure(fig, output_path, "Figure 1 (overview)", dpi) + """ + Save every non-None figure using paths from _output_paths(). + """ + path1, path2, path3, path4 = _output_paths( + output_path, viz_run_id, most_recent_gen_run_id + ) + + if fig is not None: + _save_figure(fig, path1, "Figure 1 (overview)", dpi) if fig_cmp is not None: - _save_figure( - fig_cmp, - str(p.with_stem(p.stem + "_vs_baseline")), - "Figure 2 (baseline comparison)", - dpi, - ) + _save_figure(fig_cmp, path2, "Figure 2 (baseline comparison)", dpi) if fig_arch is not None: - _save_figure( - fig_arch, - str(p.with_stem(p.stem + "_archive")), - "Figure 3 (full archive: create & update)", - dpi, - ) + _save_figure(fig_arch, path3, "Figure 3 (full archive: create & update)", dpi) if fig_arch_extract is not None: _save_figure( fig_arch_extract, - str(p.with_stem(p.stem + "_archive_extract")), + path4, "Figure 4 (full archive: extract_seq & extract_par)", dpi, ) @@ -1604,21 +1773,78 @@ def main(): output_path: Optional[str] = _cfg_optional(cfg, "output_path") archive_dir: Optional[str] = _cfg_optional(cfg, "performance_archive_dir") + # --- New options -------------------------------------------------------- + # hpss_filter: subset of HPSS modes to plot (default: all three) + global ACTIVE_HPSS + ACTIVE_HPSS = _parse_hpss_filter(_cfg_optional(cfg, "hpss_filter")) + if ACTIVE_HPSS != HPSS_ORDER: + print( + f"INFO: hpss_filter active — plotting only: {ACTIVE_HPSS}", file=sys.stderr + ) + + # viz_run_id: top-level subdirectory for output files + viz_run_id: Optional[str] = _cfg_optional(cfg, "viz_run_id") + if viz_run_id: + print(f"INFO: viz_run_id = {viz_run_id!r}", file=sys.stderr) + + # most_recent_gen_run_id: stem for Figures 1 & 2, subdir under viz_run_id/ + most_recent_gen_run_id: Optional[str] = _cfg_optional(cfg, "most_recent_gen_run_id") + if most_recent_gen_run_id: + print( + f"INFO: most_recent_gen_run_id = {most_recent_gen_run_id!r}", + file=sys.stderr, + ) + if viz_run_id and not most_recent_gen_run_id: + print( + "WARNING: viz_run_id is set but most_recent_gen_run_id is not; " + "falling back to legacy flat filename layout.", + file=sys.stderr, + ) + + # figures: which figures to produce (default: all applicable) + figures_set: set = _parse_figures(_cfg_optional(cfg, "figures")) + if figures_set != _ALL_FIGURES: + print( + f"INFO: figures filter active — producing only: {sorted(figures_set)}", + file=sys.stderr, + ) + # ------------------------------------------------------------------------ + df = _load_results(results_csv) all_dirs = sorted( set(df["create_subdir"].dropna()) | set(df["update_subdir"].dropna()), key=dir_sort_key, ) - fig = build_overview_figure(df, all_dirs) - fig_cmp = _try_build_comparison_figure( - df, all_dirs, results_csv, baseline_results_csv + # Build only the requested figures. + fig = build_overview_figure(df, all_dirs) if 1 in figures_set else None + if 1 not in figures_set: + print("INFO: Figure 1 not in figures list; skipping.", file=sys.stderr) + + fig_cmp = ( + _try_build_comparison_figure(df, all_dirs, results_csv, baseline_results_csv) + if 2 in figures_set + else None + ) + if 2 not in figures_set: + print("INFO: Figure 2 not in figures list; skipping.", file=sys.stderr) + + fig_arch, fig_arch_extract = _try_build_archive_figures( + archive_dir, + want_fig3=3 in figures_set, + want_fig4=4 in figures_set, ) - fig_arch, fig_arch_extract = _try_build_archive_figures(archive_dir) if output_path: _save_all_figures( - fig, fig_cmp, fig_arch, fig_arch_extract, output_path, args.dpi + fig, + fig_cmp, + fig_arch, + fig_arch_extract, + output_path, + viz_run_id, + most_recent_gen_run_id, + args.dpi, ) else: plt.show() From b52eda862bb9e24b4109d295303ad996cedbf399 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 18 Jun 2026 17:57:19 -0700 Subject: [PATCH 24/27] Address Copilot review comments --- conda/dev.yml | 4 ++-- conda/perf.yml | 4 ++-- tests/performance/README.md | 14 ++++++++------ .../generate/generate_performance_data.bash | 2 +- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/conda/dev.yml b/conda/dev.yml index 85f64d32..492918c1 100644 --- a/conda/dev.yml +++ b/conda/dev.yml @@ -12,7 +12,7 @@ dependencies: - globus-sdk >=3.15.0,<4.0 # Developer Tools # ================= - # If versions are updated, also update 'rev' in `.pre-commit.config.yaml` + # If versions are updated, also update 'rev' in `.pre-commit-config.yaml` - black ==25.1.0 - flake8 ==7.3.0 - isort ==6.0.1 @@ -25,7 +25,7 @@ dependencies: - pytest-cov # Documentation # ================= - # If versions are updated, also update in `.github/workflows/workflow.yml` + # If versions are updated, also update in `.github/workflows/build_workflow.yml` and `.github/workflows/release_workflow.yml` - jinja2 <3.1 - sphinx >=5.2.0 - sphinx-multiversion >=0.2.4 diff --git a/conda/perf.yml b/conda/perf.yml index 7b447c74..f254e53e 100644 --- a/conda/perf.yml +++ b/conda/perf.yml @@ -12,7 +12,7 @@ dependencies: - globus-sdk >=3.15.0,<4.0 # Developer Tools # ================= - # If versions are updated, also update 'rev' in `.pre-commit.config.yaml` + # If versions are updated, also update 'rev' in `.pre-commit-config.yaml` - black ==25.1.0 - flake8 ==7.3.0 - isort ==6.0.1 @@ -25,7 +25,7 @@ dependencies: - pytest-cov # Documentation # ================= - # If versions are updated, also update in `.github/workflows/workflow.yml` + # If versions are updated, also update in `.github/workflows/build_workflow.yml` and `.github/workflows/release_workflow.yml` - jinja2 <3.1 - sphinx >=5.2.0 - sphinx-multiversion >=0.2.4 diff --git a/tests/performance/README.md b/tests/performance/README.md index 06e36019..77948106 100644 --- a/tests/performance/README.md +++ b/tests/performance/README.md @@ -12,15 +12,17 @@ conda activate zstash_perf python -m pip install . ``` -All parameters for both scripts live in a single config file (`perf.cfg`) so you never need to edit the scripts themselves. Copy the provided template and fill in your values: +All parameters for both scripts can live in a single config file (`key=value`) so you never need to edit the scripts themselves. Start by copying the generator template and fill in your values: ```bash -cd tests/performance/ +cd tests/performance/generate cp perf.cfg my_run.cfg # or just edit perf.cfg in place ``` The config file uses a simple `key=value` format (lines starting with `#` are comments). It is shared between the bash script and the Python visualizer. +To visualize, add the visualizer keys from `visualize/perf.cfg` into the same file. + > **Perlmutter path convention:** home and scratch directories follow the pattern > `/global/homes/u/username/...` and `/pscratch/sd/u/username/...` > where `u` is the first letter of your username. @@ -70,8 +72,8 @@ dst_endpoint_archive_dir=/lcrc/group/e3sm/username/zstash_performance_dst_dir/ Once you have the parameters set up, run: ```bash -cd tests/performance/ -./generate_performance_data.bash my_run.cfg +cd tests/performance/generate/ +./generate_performance_data.bash ../my_run.cfg ``` If no cfg file argument is given, the script looks for `perf.cfg` in the same directory. @@ -95,7 +97,7 @@ baseline_results_csv=/pscratch/sd/u/username/zstash_performance/performance_2026 # Output path for the saved figures. # Leave blank to display interactively instead of saving. # Make sure to use the web server path, i.e., /global/cfs/cdirs/e3sm/www/... -output_path=/global/cfs/cdirs/e3sm/www/username/zstash_performance/ +output_path=/global/cfs/cdirs/e3sm/www/username/zstash_performance/performance_20260603.png ``` The following options are available for finer control over the visualizer output: @@ -137,7 +139,7 @@ Once you have the parameters set up, run: ```bash cd tests/performance/visualize/ -python visualize_performance.py --cfg my_run.cfg +python visualize_performance.py --cfg ../my_run.cfg ``` If `--cfg` is omitted, the script looks for `perf.cfg` in the same directory. diff --git a/tests/performance/generate/generate_performance_data.bash b/tests/performance/generate/generate_performance_data.bash index 1271ba15..6b2bf78e 100755 --- a/tests/performance/generate/generate_performance_data.bash +++ b/tests/performance/generate/generate_performance_data.bash @@ -480,4 +480,4 @@ performance_archive_path="${performance_archive_dir}/${gen_run_id}_results.csv" cp "${results_csv}" "${performance_archive_path}" print_success "Results copied to: ${performance_archive_path}" -print_info "Now run: python visualize_performance.py --cfg ${CFG_FILE}" +print_info "Now run: python ${SCRIPT_DIR}/../visualize/visualize_performance.py --cfg ${CFG_FILE}" From c2df774a64883d3a3629d3b794591d2f83fa7005 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 18 Jun 2026 18:01:37 -0700 Subject: [PATCH 25/27] Update README to note 2 cfgs --- tests/performance/README.md | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/performance/README.md b/tests/performance/README.md index 77948106..9658cab4 100644 --- a/tests/performance/README.md +++ b/tests/performance/README.md @@ -12,16 +12,17 @@ conda activate zstash_perf python -m pip install . ``` -All parameters for both scripts can live in a single config file (`key=value`) so you never need to edit the scripts themselves. Start by copying the generator template and fill in your values: +Each script has its own config file (`key=value`) so you never need to edit the scripts themselves. Start by copying the templates and filling in your values: ```bash -cd tests/performance/generate -cp perf.cfg my_run.cfg # or just edit perf.cfg in place -``` +# Generator config +cp tests/performance/generate/perf.cfg tests/performance/generate/my_run.cfg -The config file uses a simple `key=value` format (lines starting with `#` are comments). It is shared between the bash script and the Python visualizer. +# Visualizer config +cp tests/performance/visualize/perf.cfg tests/performance/visualize/my_run.cfg +``` -To visualize, add the visualizer keys from `visualize/perf.cfg` into the same file. +The config files use a simple `key=value` format (lines starting with `#` are comments). > **Perlmutter path convention:** home and scratch directories follow the pattern > `/global/homes/u/username/...` and `/pscratch/sd/u/username/...` @@ -30,7 +31,7 @@ To visualize, add the visualizer keys from `visualize/perf.cfg` into the same fi ## Generate performance data -Edit the run metadata section of your cfg file: +Edit the run metadata section of `generate/my_run.cfg`: ```ini # Use /pscratch since a lot of data will be transferred. @@ -73,7 +74,7 @@ Once you have the parameters set up, run: ```bash cd tests/performance/generate/ -./generate_performance_data.bash ../my_run.cfg +./generate_performance_data.bash my_run.cfg ``` If no cfg file argument is given, the script looks for `perf.cfg` in the same directory. @@ -82,7 +83,7 @@ Results will be saved to `${work_dir}${gen_run_id}/results.csv`. To keep all rec ## Visualize performance -The visualizer lives in `tests/performance/visualize/`. Edit the visualizer section of your cfg file: +The visualizer lives in `tests/performance/visualize/`. Edit `visualize/my_run.cfg`: ```ini # Path to the results CSV to show in Figure 1. @@ -139,7 +140,7 @@ Once you have the parameters set up, run: ```bash cd tests/performance/visualize/ -python visualize_performance.py --cfg ../my_run.cfg +python visualize_performance.py --cfg my_run.cfg ``` If `--cfg` is omitted, the script looks for `perf.cfg` in the same directory. From 0820c3b6ae8e96edff522da6b0ddcf6897c3cdc7 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Fri, 19 Jun 2026 07:17:41 -0700 Subject: [PATCH 26/27] Update developer cfgs --- tests/performance/generate/developer_run.cfg | 2 +- tests/performance/visualize/developer_run.cfg | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/performance/generate/developer_run.cfg b/tests/performance/generate/developer_run.cfg index d925eb7b..18395d7e 100644 --- a/tests/performance/generate/developer_run.cfg +++ b/tests/performance/generate/developer_run.cfg @@ -1,7 +1,7 @@ # This version of perf.cfg has filled-in username paths. work_dir=/pscratch/sd/f/forsyth/zstash_performance/ -gen_run_id=performance_20260603 +gen_run_id=performance_20260618 environment_commands=source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh dir_to_copy_from=/global/cfs/cdirs/e3sm/forsyth/E3SMv2/v2.LR.historical_0201/ diff --git a/tests/performance/visualize/developer_run.cfg b/tests/performance/visualize/developer_run.cfg index b690e695..9d5d2b17 100644 --- a/tests/performance/visualize/developer_run.cfg +++ b/tests/performance/visualize/developer_run.cfg @@ -2,12 +2,12 @@ performance_archive_dir=/global/homes/f/forsyth/zstash_performance_records -results_csv=/pscratch/sd/f/forsyth/zstash_performance/performance_20260603/results.csv -baseline_results_csv=/pscratch/sd/f/forsyth/zstash_performance/performance_20260414/results.csv -output_path=/global/cfs/cdirs/e3sm/www/forsyth/zstash_performance/performance_pr427_20260603.png +results_csv=/pscratch/sd/f/forsyth/zstash_performance/performance_20260618/results.csv +baseline_results_csv=/pscratch/sd/f/forsyth/zstash_performance/performance_20260603/results.csv +output_path=/global/cfs/cdirs/e3sm/www/forsyth/zstash_performance/performance_20260618.png -viz_run_id=viz_test_20260618_try2 -most_recent_gen_run_id=performance_20260603 +viz_run_id=viz_test_20260618_try3 +most_recent_gen_run_id=performance_20260618 hpss_filter=none,hpss,globus From d001e451e8853181abd9a040fa76cf24083fedbc Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Fri, 19 Jun 2026 07:25:52 -0700 Subject: [PATCH 27/27] Address Copilot review comments --- tests/performance/generate/generate_performance_data.bash | 2 +- tests/performance/visualize/visualize_performance.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/performance/generate/generate_performance_data.bash b/tests/performance/generate/generate_performance_data.bash index 6b2bf78e..349d9ce1 100755 --- a/tests/performance/generate/generate_performance_data.bash +++ b/tests/performance/generate/generate_performance_data.bash @@ -480,4 +480,4 @@ performance_archive_path="${performance_archive_dir}/${gen_run_id}_results.csv" cp "${results_csv}" "${performance_archive_path}" print_success "Results copied to: ${performance_archive_path}" -print_info "Now run: python ${SCRIPT_DIR}/../visualize/visualize_performance.py --cfg ${CFG_FILE}" +print_info "Now run: python ${SCRIPT_DIR}/../visualize/visualize_performance.py --cfg ${SCRIPT_DIR}/../visualize/perf.cfg (copy/edit this cfg to point at ${results_csv})" diff --git a/tests/performance/visualize/visualize_performance.py b/tests/performance/visualize/visualize_performance.py index 0d24400f..eb7d8c3e 100644 --- a/tests/performance/visualize/visualize_performance.py +++ b/tests/performance/visualize/visualize_performance.py @@ -6,8 +6,8 @@ python visualize_performance.py [--cfg path/to/perf.cfg] [--dpi 150] Pass --cfg (default: perf.cfg next to this script) instead of editing -hard-coded constants. The cfg file uses the same key=value format as -generate_performance_data.bash and can be shared between both scripts. +hard-coded constants. The cfg file uses the same key=value *format* as +generate_performance_data.bash (see tests/performance/README.md for templates). The CSV is produced by generate_performance_data.bash and has columns: test_label, create_subdir, update_subdir, hpss_label, operation, elapsed_seconds