From 9d89db12b60b68892c14c72010a1009de96cd145 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Fri, 23 Jan 2026 14:14:22 -0600 Subject: [PATCH 01/38] Add main branch testing automation --- tests/main_branch_testing/README.md | 258 +++++++++ .../main_branch_testing/run_image_tests.bash | 56 ++ .../run_integration_test.bash | 502 ++++++++++++++++++ 3 files changed, 816 insertions(+) create mode 100644 tests/main_branch_testing/README.md create mode 100644 tests/main_branch_testing/run_image_tests.bash create mode 100644 tests/main_branch_testing/run_integration_test.bash diff --git a/tests/main_branch_testing/README.md b/tests/main_branch_testing/README.md new file mode 100644 index 00000000..9e1b8dd9 --- /dev/null +++ b/tests/main_branch_testing/README.md @@ -0,0 +1,258 @@ +# zppy Integration Test Automation + +This automation system streamlines the zppy integration testing workflow, reducing manual steps and wait time while maintaining quality control checkpoints. + +## Quick Start + +### Basic Usage (Interactive Mode) +```bash +./run_integration_test.bash +``` +This will: +- Use today's date as the test identifier +- Stop at checkpoints for you to verify +- Set up all environments from scratch +- Run the complete test suite + +### Fully Automated Mode +```bash +./run_integration_test.bash --auto +``` +Runs end-to-end without stopping (suitable for CI or overnight runs). + +### Custom Date Stamp +```bash +./run_integration_test.bash --date 20260123 +``` + +### Resume from Specific Phase +```bash +# If Phase 1 completed but you need to re-run Phase 2 +./run_integration_test.bash --phase 2 --date 20260123 +``` + +## Complete Options + +``` +./run_integration_test.bash [OPTIONS] + +Options: + --date YYYYMMDD Date stamp for test (default: today) + --auto Run fully automated (no checkpoints) + --phase N Start from phase N (1=setup, 2=bundles_part2, 3=validation) + --help Show help message +``` + +## Workflow Phases + +### Phase 1: Setup (~2-4 hours including SLURM wait) +- Sets up e3sm_diags conda environment +- Sets up zppy-interfaces conda environment +- Sets up zppy conda environment +- Applies optional cherry-pick +- Generates config files +- Submits initial SLURM jobs (6 configs) +- Waits for jobs to complete + +### Phase 2: Bundles Part 2 (~30-60 minutes including SLURM wait) +- Checks status of bundles runs +- Submits bundles part 2 jobs +- Waits for completion + +### Phase 3: Validation (~15 minutes, excluding image tests) +- Checks all status files +- Runs pytest integration tests: + - test_bash_generation.py + - test_campaign.py + - test_defaults.py + - test_last_year.py + - test_bundles.py +- Provides instructions for running test_images.py on compute node + +## Running Image Tests + +The image tests require a compute node allocation. Two options: + +### Option 1: Manual Allocation (Recommended) +```bash +salloc --nodes=1 --partition=debug --time=02:00:00 --account=e3sm +./run_image_tests.bash --date 20260123 +``` + +### Option 2: Automatic Allocation +```bash +./run_image_tests.bash --date 20260123 --auto +``` + +## Common Workflows + +### Full Test with Custom Date +```bash +# Interactive mode with checkpoints +./run_integration_test.bash --date 20260123 + +# When prompted, allocate compute node for images: +salloc --nodes=1 --partition=debug --time=02:00:00 --account=e3sm +./run_image_tests.bash --date 20260123 +``` + +### Overnight Automated Run +```bash +# Start before leaving for the day +nohup ./run_integration_test.bash --auto --date 20260123 > test_run.log 2>&1 & + +# Next morning, run image tests manually +salloc --nodes=1 --partition=debug --time=02:00:00 --account=e3sm +./run_image_tests.bash --date 20260123 +``` + +## Output and Logs + +The script provides color-coded output: +- 🔵 **Blue**: Informational messages +- 🟢 **Green**: Success messages +- 🟡 **Yellow**: Warnings and checkpoints +- 🔴 **Red**: Errors + +### SLURM Job Monitoring +The script automatically monitors SLURM jobs and shows: +``` +Jobs remaining: 42 (elapsed: 1234s) +``` + +### Status File Checking +Automatically checks for errors in status files: +``` +✓ v2: No errors found +✓ Legacy v2: No errors found +✓ v3: No errors found +... +``` + +## Environment Variables + +You can set these before running the script: + +```bash +export DATE_STAMP=20260123 +export UNIQUE_ID="custom_test_id" +./run_integration_test.bash +``` + +## Customization + +### Modify Test Configurations + +Edit the script to change which configs are run: + +```bash +# In the generated Python code section, modify: +"cfgs_to_run": [ + "weekly_bundles", + "weekly_comprehensive_v2", + # Add or remove configs here +], +``` + +### Adjust Timeouts + +```bash +# In phase_1_setup(), change max wait time: +wait_for_slurm_jobs 30 14400 # 30s interval, 4hr max + +# In phase_2_bundles_part2(): +wait_for_slurm_jobs 30 3600 # 30s interval, 1hr max +``` + +### Add Custom Checks + +Add your own validation in `phase_3_validation()`: + +```bash +# Custom validation example +log "Running custom checks..." +if [ -f "$ZPPY_DIR/my_custom_check.bash" ]; then + bash "$ZPPY_DIR/my_custom_check.bash" +fi +``` + +## Troubleshooting + +### Script Exits Early +Check the error message. The script uses `set -e`, so it exits on any error. + +### Jobs Don't Complete +- Check SLURM queue: `squeue -u ac.forsyth2` +- Check job logs in the output directories +- Increase timeout: edit `wait_for_slurm_jobs` calls + +### Environment Issues +```bash +# Clean and rebuild +conda remove --y --all --name test-diags-main-20260123 +conda remove --y --all --name test-zi-main-20260123 +conda remove --y --all --name test-zppy-main-20260123-env + +# Re-run +./run_integration_test.bash --date 20260123 +``` + +### Git Issues +```bash +# If git operations fail, manually clean up: +cd ~/ez/zppy +git reset --hard upstream/main +git clean -fd +./run_integration_test.bash --date 20260123 +``` + +### Checkpoint Issues in Auto Mode +The script will proceed automatically but log warnings. Review logs after completion. + +## Files Created + +``` +~/ez/zppy/ +├── tests/integration/generated/ +│ ├── test_weekly_bundles_chrysalis.cfg +│ ├── test_weekly_comprehensive_v2_chrysalis.cfg +│ ├── test_weekly_comprehensive_v3_chrysalis.cfg +│ ├── test_weekly_legacy_3.0.0_bundles_chrysalis.cfg +│ ├── test_weekly_legacy_3.0.0_comprehensive_v2_chrysalis.cfg +│ └── test_weekly_legacy_3.0.0_comprehensive_v3_chrysalis.cfg +└── test_images_summary.md + +/lcrc/group/e3sm/ac.forsyth2/ +├── zppy_weekly_bundles_output/zppy_main_branch_test_YYYYMMDD/ +├── zppy_weekly_comprehensive_v2_output/zppy_main_branch_test_YYYYMMDD/ +├── zppy_weekly_comprehensive_v3_output/zppy_main_branch_test_YYYYMMDD/ +└── zppy_weekly_legacy_3.0.0_*/zppy_main_branch_test_YYYYMMDD/ +``` + +## Tips + +1. **Use descriptive date stamps**: Instead of the current date, use a meaningful identifier like `20260123_bugfix` or `20260123_pr769` + +2. **Run overnight**: The full test takes 2-4 hours. Start it before leaving: + ```bash + nohup ./run_integration_test.bash --auto > test.log 2>&1 & + ``` + +3. **Keep logs**: Redirect output to files for later review: + ```bash + ./run_integration_test.bash --auto 2>&1 | tee test_$(date +%Y%m%d).log + ``` + +4. **Parallel testing**: Run different date stamps to test multiple branches: + ```bash + ./run_integration_test.bash --date 20260123_main & + ./run_integration_test.bash --date 20260123_feature --cherry-pick abc123 & + ``` + +## Support + +For issues or questions: +1. Check the troubleshooting section above +2. Review the script output for error messages +3. Check SLURM logs in the output directories +4. Contact the zppy development team diff --git a/tests/main_branch_testing/run_image_tests.bash b/tests/main_branch_testing/run_image_tests.bash new file mode 100644 index 00000000..3a414621 --- /dev/null +++ b/tests/main_branch_testing/run_image_tests.bash @@ -0,0 +1,56 @@ +#!/bin/bash +# Run image tests on compute node +# Usage: ./run_image_tests.bash [--date YYYYMMDD] [--auto] + +set -e + +DATE_STAMP="${DATE_STAMP:-$(date +%Y%m%d)}" +AUTO_ALLOCATE=false + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --date) + DATE_STAMP="$2" + shift 2 + ;; + --auto) + AUTO_ALLOCATE=true + shift + ;; + *) + echo "Unknown option: $1" + echo "Usage: $0 [--date YYYYMMDD] [--auto]" + exit 1 + ;; + esac +done + +ZPPY_ENV="test-zppy-main-${DATE_STAMP}-env" +ZPPY_DIR="$HOME/ez/zppy" +CONDA_PROFILE="$HOME/miniforge3/etc/profile.d/conda.sh" + +if [ "$AUTO_ALLOCATE" = true ]; then + echo "Auto-allocating compute node and running tests..." + salloc --nodes=1 --partition=debug --time=02:00:00 --account=e3sm << EOFALLOC +source ~/.bashrc +lcrc_conda # Run conda activation function defined in ~/.bashrc +conda activate $ZPPY_ENV +cd $ZPPY_DIR +pytest tests/integration/test_images.py +cat test_images_summary.md +EOFALLOC +else + # DEFAULT + # Assume we're already on a compute node or user will allocate manually + echo "Running image tests..." + echo "If not on compute node, first run:" + echo " salloc --nodes=1 --partition=debug --time=02:00:00 --account=e3sm" + echo "" + + source "$CONDA_PROFILE" + conda activate "$ZPPY_ENV" + cd "$ZPPY_DIR" + pytest tests/integration/test_images.py + cat test_images_summary.md +fi diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash new file mode 100644 index 00000000..d0abac8c --- /dev/null +++ b/tests/main_branch_testing/run_integration_test.bash @@ -0,0 +1,502 @@ +#!/bin/bash +# zppy Integration Test Automation Script +# Usage: +# 1. Copy this file and `run_image_tests.bash` out of the zppy repo. (This script will change the branch). +# 2. Edit configuration parameters below. +# 3. Run: ./run_integration_test.bash [OPTIONS] +# 4. Run: ./run_image_tests.bash +# +# Options: +# --date YYYYMMDD Date stamp for test (default: today) +# --auto Run fully automated (no checkpoints) +# --phase N Start from phase N (1=setup, 2=bundles_part2, 3=validation) +# --help Show this help message + +set -e # Exit on error +set -u # Exit on undefined variable + +# ============================================================================ +# Configuration +# ============================================================================ + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +DATE_STAMP="${DATE_STAMP:-$(date +%Y%m%d)}" +AUTO_MODE=false +START_PHASE=1 + +# Paths +HOME_DIR="$HOME" +EZ_DIR="$HOME_DIR/ez" +E3SM_DIAGS_DIR="$EZ_DIR/e3sm_diags" +ZPPY_INTERFACES_DIR="$EZ_DIR/zppy-interfaces" +ZPPY_DIR="$EZ_DIR/zppy" +CONDA_PROFILE="$HOME_DIR/miniforge3/etc/profile.d/conda.sh" +OUTPUT_WORKSPACE="/lcrc/group/e3sm/ac.forsyth2" + +# Environment names +DIAGS_ENV="test-diags-main-${DATE_STAMP}" +ZI_ENV="test-zi-main-${DATE_STAMP}" +ZPPY_ENV="test-zppy-main-${DATE_STAMP}-env" + +# Test configuration +UNIQUE_ID="zppy_main_branch_test_${DATE_STAMP}" + +# Cherry pick configuration +CHERRY_PICK_BRANCH="test-fixes" +CHERRY_PICK_COMMIT="" + +# Output directories +BUNDLES_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_bundles_output/${UNIQUE_ID}/v3.LR.historical_0051/post/scripts" +LEGACY_BUNDLES_OUTPUT="${OUTPUT_WORKSPACE}zppy_weekly_legacy_3.0.0_bundles_output/${UNIQUE_ID}/v3.LR.historical_0051/post/scripts" +V2_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_comprehensive_v2_output/${UNIQUE_ID}/v2.LR.historical_0201/post/scripts" +LEGACY_V2_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_legacy_3.0.0_comprehensive_v2_output/${UNIQUE_ID}/v2.LR.historical_0201/post/scripts" +V3_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_comprehensive_v3_output/${UNIQUE_ID}/v3.LR.historical_0051/post/scripts" +LEGACY_V3_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_legacy_3.0.0_comprehensive_v3_output/${UNIQUE_ID}/v3.LR.historical_0051/post/scripts" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# ============================================================================ +# Helper Functions +# ============================================================================ + +log() { + echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $*" +} + +log_success() { + echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] ✓${NC} $*" +} + +log_warning() { + echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')] ⚠${NC} $*" +} + +log_error() { + echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] ✗${NC} $*" +} + +checkpoint() { + local message="$1" + if [ "$AUTO_MODE" = false ]; then + log_warning "CHECKPOINT: $message" + read -p "Press Enter to continue or Ctrl+C to abort..." + else + log "AUTO MODE: $message" + fi +} + +show_help() { + grep "^#" "$0" | grep -v "#!/bin/bash" | sed 's/^# //' | sed 's/^#//' + exit 0 +} + +activate_conda() { + source ~/.bashrc + lcrc_conda # Run conda activation function defined in ~/.bashrc +} + +wait_for_slurm_jobs() { + local check_interval=${1:-600} # Check every 600 seconds (10 minutes) by default + local max_wait=${2:-14400} # Max wait 4 hours by default + + log "Waiting for SLURM jobs to complete..." + local elapsed=0 + local initial_count=$(squeue -u ac.forsyth2 | wc -l) + initial_count=$((initial_count - 1)) # Subtract header + + log "Initial job count: $initial_count" + + while true; do + local job_count=$(squeue -u ac.forsyth2 | wc -l) + job_count=$((job_count - 1)) # Subtract header + + if [ "$job_count" -eq 0 ]; then + log_success "All SLURM jobs completed!" + return 0 + fi + + if [ $elapsed -ge $max_wait ]; then + log_error "Timeout waiting for SLURM jobs after ${max_wait}s" + return 1 + fi + + echo -ne "\r${YELLOW}Jobs remaining: $job_count${NC} (elapsed: ${elapsed}s)" + sleep "$check_interval" + elapsed=$((elapsed + check_interval)) + done + echo "" # New line after progress indicator +} + +check_status_files() { + local dir="$1" + local name="$2" + + if [ ! -d "$dir" ]; then + log_warning "Directory not found: $dir" + return 1 + fi + + cd "$dir" + local errors=$(grep -v "OK" *status 2>/dev/null || true) + + if [ -z "$errors" ]; then + log_success "$name: No errors found" + return 0 + else + log_error "$name: Errors found!" + echo "$errors" + return 1 + fi +} + +# ============================================================================ +# Parse Arguments +# ============================================================================ + +while [[ $# -gt 0 ]]; do + case $1 in + --date) + DATE_STAMP="$2" + shift 2 + ;; + --auto) + AUTO_MODE=true + shift + ;; + --phase) + START_PHASE="$2" + shift 2 + ;; + --help) + show_help + ;; + *) + log_error "Unknown option: $1" + show_help + ;; + esac +done + +# ============================================================================ +# Phase 1: Setup +# ============================================================================ + +phase_1_setup() { + log "=========================================" + log "Phase 1: Setup" + log "Date: $DATE_STAMP" + log "Unique ID: $UNIQUE_ID" + log "=========================================" + + activate_conda + + # ==================================================================== + # Set up e3sm_diags environment + # ==================================================================== + log "Setting up e3sm_diags environment..." + cd "$E3SM_DIAGS_DIR" + + git status + git add -A + git commit -m "Auto-save before test" --no-verify || true + + git fetch upstream main + git checkout main + git reset --hard upstream/main + + log "Latest e3sm_diags commit (should match https://github.com/E3SM-Project/e3sm_diags/commits/main):" + git log -1 --oneline + + rm -rf build + conda clean --all --yes + conda env create -f conda-env/dev.yml -n "$DIAGS_ENV" + conda activate "$DIAGS_ENV" + python -m pip install . + log_success "e3sm_diags environment ready: $DIAGS_ENV" + + # ==================================================================== + # Set up zppy-interfaces environment + # ==================================================================== + log "Setting up zppy-interfaces environment..." + cd "$ZPPY_INTERFACES_DIR" + + git status + git add -A + git commit -m "Auto-save before test" --no-verify || true + + git fetch upstream main + git checkout main + git reset --hard upstream/main + + log "Latest zppy-interfaces commit (should match https://github.com/E3SM-Project/zppy-interfaces/commits/main):" + git log -1 --oneline + + rm -rf build + conda clean --all --yes + conda env create -f conda/dev.yml -n "$ZI_ENV" + conda activate "$ZI_ENV" + python -m pip install . + log_success "zppy-interfaces environment ready: $ZI_ENV" + + # Run unit tests + log "Running pytest unit tests..." + pytest /zppy-interfaces/tests/unit/global_time_series/test_*.py + pytest /zppy-interfaces/tests/unit/pcmdi_diags/test_*.py + log_success "zppy-interfaces unit tests passed" + + # ======================================================================== + # Set up zppy environment + # ======================================================================== + log "Setting up zppy environment..." + cd "$ZPPY_DIR" + + git status + git add -A + git commit -m "Auto-save before test" --no-verify || true + + git fetch upstream main + git checkout -b "test-zppy-main-${DATE_STAMP}" upstream/main + + log "Latest zppy commit:" + git log -1 --oneline + + # Cherry-pick if requested + if [ -n "$CHERRY_PICK_COMMIT" ]; then + log "Cherry-picking commit: $CHERRY_PICK_COMMIT" + git fetch upstream "$CHERRY_PICK_BRANCH" + git cherry-pick "$CHERRY_PICK_COMMIT" + log_success "Cherry-pick applied" + fi + + rm -rf build + conda clean --all --yes + conda env create -f conda/dev.yml -n "$ZPPY_ENV" + conda activate "$ZPPY_ENV" + python -m pip install . + log_success "zppy environment ready: $ZPPY_ENV" + + # Run unit tests + log "Running pytest unit tests..." + pytest tests/test_*.py + log_success "zppy unit tests passed" + + # ======================================================================== + # Generate config files + # ======================================================================== + log "Generating config files..." + + # Update utils.py with test specifics + UTILS_FILE="tests/integration/utils.py" + + # Create a temporary Python script to update TEST_SPECIFICS + cat > /tmp/update_utils.py << EOF +import re + +utils_file = "${UTILS_FILE}" + +with open(utils_file, 'r') as f: + content = f.read() + +# Find TEST_SPECIFICS dictionary and replace it +pattern = r'TEST_SPECIFICS: Dict\[str, Any\] = \{.*?\n\}' +replacement = '''TEST_SPECIFICS: Dict[str, Any] = { + "diags_environment_commands": "source ${CONDA_PROFILE}; conda activate ${DIAGS_ENV}", + "mpas_analysis_environment_commands": "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh", + "global_time_series_environment_commands": "source ${CONDA_PROFILE}; conda activate ${ZI_ENV}", + "pcmdi_diags_environment_commands": "source ${CONDA_PROFILE}; conda activate ${ZI_ENV}", + "environment_commands": "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh", + "cfgs_to_run": [ + "weekly_bundles", + "weekly_comprehensive_v2", + "weekly_comprehensive_v3", + "weekly_legacy_3.0.0_bundles", + "weekly_legacy_3.0.0_comprehensive_v2", + "weekly_legacy_3.0.0_comprehensive_v3", + ], + "tasks_to_run": [ + "e3sm_diags", + "mpas_analysis", + "global_time_series", + "ilamb", + "pcmdi_diags", + ], + "unique_id": "${UNIQUE_ID}", +}''' + +content = re.sub(pattern, replacement, content, flags=re.DOTALL) + +with open(utils_file, 'w') as f: + f.write(content) + +print("Updated utils.py") +EOF + + python /tmp/update_utils.py + + log "Running utils.py to generate configs..." + python tests/integration/utils.py + + log_success "Config files generated" + + # ======================================================================== + # Submit initial SLURM jobs + # ======================================================================== + log "Submitting initial SLURM jobs..." + + zppy -c tests/integration/generated/test_weekly_bundles_chrysalis.cfg + zppy -c tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg + zppy -c tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg + zppy -c tests/integration/generated/test_weekly_legacy_3.0.0_bundles_chrysalis.cfg + zppy -c tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v2_chrysalis.cfg + zppy -c tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v3_chrysalis.cfg + + local job_count=$(squeue -u ac.forsyth2 | wc -l) + job_count=$((job_count - 1)) # Don't count the header + log_success "Submitted jobs. Total in queue: $job_count" + + checkpoint "Phase 1 complete. Jobs submitted and running." + + # Wait for jobs to complete + wait_for_slurm_jobs 600 14400 # Check every 600sec (10min), max 4 hours + + log_success "Phase 1 complete!" +} + +# ============================================================================ +# Phase 2: Bundles Part 2 +# ============================================================================ + +phase_2_bundles_part2() { + log "=========================================" + log "Phase 2: Bundles Part 2" + log "=========================================" + + activate_conda + conda activate "$ZPPY_ENV" + cd "$ZPPY_DIR" + + # Check bundles status + log "Checking bundles status..." + check_status_files "$BUNDLES_OUTPUT" "Bundles" + check_status_files "$LEGACY_BUNDLES_OUTPUT" "Legacy Bundles" + + checkpoint "Bundles status checked. Ready to submit part 2." + + # Submit bundles part 2 + log "Submitting bundles part 2..." + zppy -c tests/integration/generated/test_weekly_bundles_chrysalis.cfg + zppy -c tests/integration/generated/test_weekly_legacy_3.0.0_bundles_chrysalis.cfg + + local job_count=$(squeue -u ac.forsyth2 | wc -l) + job_count=$((job_count - 1)) # Don't count the header + log_success "Submitted bundles part 2. Total in queue: $job_count" + + # Wait for jobs to complete + wait_for_slurm_jobs 600 3600 # # Check every 600sec (10min), max 1 hour + + log_success "Phase 2 complete!" +} + +# ============================================================================ +# Phase 3: Validation +# ============================================================================ + +phase_3_validation() { + log "=========================================" + log "Phase 3: Validation" + log "=========================================" + + activate_conda + conda activate "$ZPPY_ENV" + cd "$ZPPY_DIR" + + # Check all status files + log "Checking all status files..." + + local all_good=true + + check_status_files "$V2_OUTPUT" "v2" || all_good=false + check_status_files "$LEGACY_V2_OUTPUT" "Legacy v2" || all_good=false + check_status_files "$V3_OUTPUT" "v3" || all_good=false + check_status_files "$LEGACY_V3_OUTPUT" "Legacy v3" || all_good=false + check_status_files "$BUNDLES_OUTPUT" "Bundles" || all_good=false + check_status_files "$LEGACY_BUNDLES_OUTPUT" "Legacy Bundles" || all_good=false + + if [ "$all_good" = false ]; then + log_error "Some status checks failed!" + checkpoint "Errors found in status files. Continue anyway?" + else + log_success "All status files clean!" + fi + + # Run pytest tests + log "Running integration tests..." + + log "Running test_bash_generation.py..." + pytest tests/integration/test_bash_generation.py || log_warning "test_bash_generation.py had failures (may be expected)" + + log "Running test_campaign.py..." + pytest tests/integration/test_campaign.py || log_warning "test_campaign.py had failures (may be expected)" + + log "Running test_defaults.py..." || log_warning "test_defaults.py had failures (may be expected)" + pytest tests/integration/test_defaults.py + + log "Running test_last_year.py..." || log_warning "test_last_year.py had failures (may be expected)" + pytest tests/integration/test_last_year.py + + log "Running test_bundles.py..." || log_warning "test_bundles.py had failures (may be expected)" + pytest tests/integration/test_bundles.py + + checkpoint "Ready to run test_images.py (requires compute node allocation)" + + log "To run test_images.py, execute:" + log " salloc --nodes=1 --partition=debug --time=02:00:00 --account=e3sm" + log " source ${CONDA_PROFILE}" + log " conda activate ${ZPPY_ENV}" + log " cd ${ZPPY_DIR}" + log " pytest tests/integration/test_images.py" + log " cat test_images_summary.md" + log "Alternative: run ./run_integration_test.bash" + + log_success "Phase 3 complete!" + log_success "All automated tests finished successfully!" +} + +# ============================================================================ +# Main Execution +# ============================================================================ + +main() { + log "Starting zppy integration test automation" + log "Date stamp: $DATE_STAMP" + log "Auto mode: $AUTO_MODE" + log "Starting from phase: $START_PHASE" + + case $START_PHASE in + 1) + phase_1_setup + phase_2_bundles_part2 + phase_3_validation + ;; + 2) + phase_2_bundles_part2 + phase_3_validation + ;; + 3) + phase_3_validation + ;; + *) + log_error "Invalid phase: $START_PHASE" + exit 1 + ;; + esac + + log_success "Integration test automation complete!" +} + +main From 010c09c75d53a538960371807b5118f2e8000950 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Fri, 23 Jan 2026 19:22:35 -0600 Subject: [PATCH 02/38] Changes tested --- .../run_integration_test.bash | 130 +++++++++++------- 1 file changed, 77 insertions(+), 53 deletions(-) mode change 100644 => 100755 tests/main_branch_testing/run_integration_test.bash diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash old mode 100644 new mode 100755 index d0abac8c..b809a19b --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -36,14 +36,14 @@ OUTPUT_WORKSPACE="/lcrc/group/e3sm/ac.forsyth2" # Environment names DIAGS_ENV="test-diags-main-${DATE_STAMP}" ZI_ENV="test-zi-main-${DATE_STAMP}" -ZPPY_ENV="test-zppy-main-${DATE_STAMP}-env" +ZPPY_ENV="test-zppy-main-${DATE_STAMP}" # Test configuration UNIQUE_ID="zppy_main_branch_test_${DATE_STAMP}" # Cherry pick configuration CHERRY_PICK_BRANCH="test-fixes" -CHERRY_PICK_COMMIT="" +CHERRY_PICK_COMMIT="b56a38c6ae5b24a96bbc80a2dacbc6d1b3dd730b" # Output directories BUNDLES_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_bundles_output/${UNIQUE_ID}/v3.LR.historical_0051/post/scripts" @@ -95,9 +95,70 @@ show_help() { exit 0 } -activate_conda() { +activate_env() { + local env_name="${1:-}" # Default to empty string if not provided + set +u source ~/.bashrc lcrc_conda # Run conda activation function defined in ~/.bashrc + + # Only activate if an environment name was provided + if [ -n "$env_name" ]; then + conda activate "$env_name" + fi + set -u +} + +setup_conda_env() { + local conda_dir="$1" + local env_name="$2" + + # Check if environment already exists + if conda env list | grep -q "^${env_name} "; then + log "Environment '$env_name' already exists, skipping creation" + else + log "Creating new environment '$env_name'" + rm -rf build + conda clean --all --yes + conda env create -f "${conda_dir}/dev.yml" -n "$env_name" + fi + + activate_env "$env_name" + + # Always install/update the package + log "Installing package in '$env_name'" + python -m pip install . + log_success "Environment '$env_name' ready" +} + +ensure_test_branch() { + local test_branch="$1" + local current_branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null) + + # Check if we're already on the test branch + if [ "$current_branch" = "$test_branch" ]; then + log "Already on branch '$test_branch', skipping checkout" + return 0 + fi + + # Save current work before switching branches + log "Saving current work..." + git status + git add -A + git commit -m "Auto-save before test" --no-verify || true + + # Check if the test branch exists + if git show-ref --verify --quiet "refs/heads/$test_branch"; then + # Branch exists, just check it out + log "Checking out existing branch '$test_branch'" + git checkout "$test_branch" + log_success "Checked out existing branch '$test_branch'" + else + # Branch doesn't exist, create it from upstream/main + log "Creating new branch '$test_branch' from upstream/main" + git fetch upstream main + git checkout -b "$test_branch" upstream/main + log_success "Created and checked out new branch '$test_branch'" + fi } wait_for_slurm_jobs() { @@ -193,60 +254,34 @@ phase_1_setup() { log "Unique ID: $UNIQUE_ID" log "=========================================" - activate_conda + activate_env # ==================================================================== # Set up e3sm_diags environment # ==================================================================== log "Setting up e3sm_diags environment..." cd "$E3SM_DIAGS_DIR" - - git status - git add -A - git commit -m "Auto-save before test" --no-verify || true - - git fetch upstream main - git checkout main - git reset --hard upstream/main + ensure_test_branch test_e3sm_diags_${DATE_STAMP} log "Latest e3sm_diags commit (should match https://github.com/E3SM-Project/e3sm_diags/commits/main):" git log -1 --oneline - - rm -rf build - conda clean --all --yes - conda env create -f conda-env/dev.yml -n "$DIAGS_ENV" - conda activate "$DIAGS_ENV" - python -m pip install . - log_success "e3sm_diags environment ready: $DIAGS_ENV" + setup_conda_env "conda-env" "$DIAGS_ENV" # ==================================================================== # Set up zppy-interfaces environment # ==================================================================== log "Setting up zppy-interfaces environment..." cd "$ZPPY_INTERFACES_DIR" - - git status - git add -A - git commit -m "Auto-save before test" --no-verify || true - - git fetch upstream main - git checkout main - git reset --hard upstream/main + ensure_test_branch test_zi_${DATE_STAMP} log "Latest zppy-interfaces commit (should match https://github.com/E3SM-Project/zppy-interfaces/commits/main):" git log -1 --oneline - - rm -rf build - conda clean --all --yes - conda env create -f conda/dev.yml -n "$ZI_ENV" - conda activate "$ZI_ENV" - python -m pip install . - log_success "zppy-interfaces environment ready: $ZI_ENV" + setup_conda_env "conda" "$ZI_ENV" # Run unit tests log "Running pytest unit tests..." - pytest /zppy-interfaces/tests/unit/global_time_series/test_*.py - pytest /zppy-interfaces/tests/unit/pcmdi_diags/test_*.py + pytest tests/unit/global_time_series/test_*.py + pytest tests/unit/pcmdi_diags/test_*.py log_success "zppy-interfaces unit tests passed" # ======================================================================== @@ -254,13 +289,7 @@ phase_1_setup() { # ======================================================================== log "Setting up zppy environment..." cd "$ZPPY_DIR" - - git status - git add -A - git commit -m "Auto-save before test" --no-verify || true - - git fetch upstream main - git checkout -b "test-zppy-main-${DATE_STAMP}" upstream/main + ensure_test_branch test_zppy_${DATE_STAMP} log "Latest zppy commit:" git log -1 --oneline @@ -273,12 +302,7 @@ phase_1_setup() { log_success "Cherry-pick applied" fi - rm -rf build - conda clean --all --yes - conda env create -f conda/dev.yml -n "$ZPPY_ENV" - conda activate "$ZPPY_ENV" - python -m pip install . - log_success "zppy environment ready: $ZPPY_ENV" + setup_conda_env "conda" "$ZPPY_ENV" # Run unit tests log "Running pytest unit tests..." @@ -376,9 +400,9 @@ phase_2_bundles_part2() { log "Phase 2: Bundles Part 2" log "=========================================" - activate_conda - conda activate "$ZPPY_ENV" + activate_env "$ZPPY_ENV" cd "$ZPPY_DIR" + ensure_test_branch test_zppy_${DATE_STAMP} # Check bundles status log "Checking bundles status..." @@ -411,9 +435,9 @@ phase_3_validation() { log "Phase 3: Validation" log "=========================================" - activate_conda - conda activate "$ZPPY_ENV" + activate_env "$ZPPY_ENV" cd "$ZPPY_DIR" + ensure_test_branch test_zppy_${DATE_STAMP} # Check all status files log "Checking all status files..." From 0653e47a684e3763f712661ca79d8eae39585c24 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Fri, 23 Jan 2026 19:30:36 -0600 Subject: [PATCH 03/38] Further changes to test --- tests/integration/utils.py | 2 +- tests/main_branch_testing/README.md | 24 +++++++++++++++++++ .../run_integration_test.bash | 9 +++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 17519e47..dce4ef3e 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -72,7 +72,7 @@ def get_chyrsalis_expansions(config): "diags_walltime": "5:00:00", "expected_dir": "/lcrc/group/e3sm/public_html/zppy_test_resources/", "livvkit_mapping_file_path": f"{diagnostics_base_path}/maps", - "mpas_analysis_walltime": "00:30:00", + "mpas_analysis_walltime": "02:00:00", "partition_long": "compute", "partition_short": "debug", # This differs from the default path /lcrc/group/e3sm/diagnostics/observations/Atm/climatology diff --git a/tests/main_branch_testing/README.md b/tests/main_branch_testing/README.md index 9e1b8dd9..2eed21a9 100644 --- a/tests/main_branch_testing/README.md +++ b/tests/main_branch_testing/README.md @@ -256,3 +256,27 @@ For issues or questions: 2. Review the script output for error messages 3. Check SLURM logs in the output directories 4. Contact the zppy development team + +## Example run + +```bash +# For safety, confirm branches have no uncommitted changes beforehand: +cd ~/ez/e3sm_diags +git status +cd ~/ez/zppy-interfaces +git status +cd ~/ez/zppy +git status + +cd /home/ac.forsyth2/ez/zppy_main_branch_tests/test_20260123 +cp ~/ez/zppy/tests/main_branch_testing/* . # Copy, so the script isn't affected by the branch change +emacs run_integration_test.bash # Configure parameters + +screen # Run on screen +cd /home/ac.forsyth2/ez/zppy_main_branch_tests/test_20260123 # If not there already +# Bypass manual checkpoints, tee output: +time ./run_integration_test.bash --date 20260123_run1 --auto 2>&1 | tee integration_test.log +# CTRL A D to exit screen +screen -ls # Check which node the screen is on. +tail -f integration_test.log # Follow log updates +``` diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index b809a19b..9fb80726 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -176,6 +176,15 @@ wait_for_slurm_jobs() { local job_count=$(squeue -u ac.forsyth2 | wc -l) job_count=$((job_count - 1)) # Subtract header + # Check for failed dependencies + local failed_jobs=$(squeue -u ac.forsyth2 | grep "DependencyNeverSatisfied" || true) + if [ -n "$failed_jobs" ]; then + log_error "Jobs failed with DependencyNeverSatisfied!" + echo "$failed_jobs" + log_error "Check job logs for details" + return 1 + fi + if [ "$job_count" -eq 0 ]; then log_success "All SLURM jobs completed!" return 0 From 8c11fadd1b0535bc4723b9a361d542b8b2489f8d Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 26 Jan 2026 14:40:10 -0600 Subject: [PATCH 04/38] Changes to test 20260126 --- tests/integration/utils.py | 2 +- tests/main_branch_testing/README.md | 145 ++---------------- .../run_integration_test.bash | 123 +++++---------- 3 files changed, 56 insertions(+), 214 deletions(-) diff --git a/tests/integration/utils.py b/tests/integration/utils.py index dce4ef3e..17519e47 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -72,7 +72,7 @@ def get_chyrsalis_expansions(config): "diags_walltime": "5:00:00", "expected_dir": "/lcrc/group/e3sm/public_html/zppy_test_resources/", "livvkit_mapping_file_path": f"{diagnostics_base_path}/maps", - "mpas_analysis_walltime": "02:00:00", + "mpas_analysis_walltime": "00:30:00", "partition_long": "compute", "partition_short": "debug", # This differs from the default path /lcrc/group/e3sm/diagnostics/observations/Atm/climatology diff --git a/tests/main_branch_testing/README.md b/tests/main_branch_testing/README.md index 2eed21a9..e5906405 100644 --- a/tests/main_branch_testing/README.md +++ b/tests/main_branch_testing/README.md @@ -8,40 +8,6 @@ This automation system streamlines the zppy integration testing workflow, reduci ```bash ./run_integration_test.bash ``` -This will: -- Use today's date as the test identifier -- Stop at checkpoints for you to verify -- Set up all environments from scratch -- Run the complete test suite - -### Fully Automated Mode -```bash -./run_integration_test.bash --auto -``` -Runs end-to-end without stopping (suitable for CI or overnight runs). - -### Custom Date Stamp -```bash -./run_integration_test.bash --date 20260123 -``` - -### Resume from Specific Phase -```bash -# If Phase 1 completed but you need to re-run Phase 2 -./run_integration_test.bash --phase 2 --date 20260123 -``` - -## Complete Options - -``` -./run_integration_test.bash [OPTIONS] - -Options: - --date YYYYMMDD Date stamp for test (default: today) - --auto Run fully automated (no checkpoints) - --phase N Start from phase N (1=setup, 2=bundles_part2, 3=validation) - --help Show help message -``` ## Workflow Phases @@ -49,7 +15,6 @@ Options: - Sets up e3sm_diags conda environment - Sets up zppy-interfaces conda environment - Sets up zppy conda environment -- Applies optional cherry-pick - Generates config files - Submits initial SLURM jobs (6 configs) - Waits for jobs to complete @@ -84,28 +49,6 @@ salloc --nodes=1 --partition=debug --time=02:00:00 --account=e3sm ./run_image_tests.bash --date 20260123 --auto ``` -## Common Workflows - -### Full Test with Custom Date -```bash -# Interactive mode with checkpoints -./run_integration_test.bash --date 20260123 - -# When prompted, allocate compute node for images: -salloc --nodes=1 --partition=debug --time=02:00:00 --account=e3sm -./run_image_tests.bash --date 20260123 -``` - -### Overnight Automated Run -```bash -# Start before leaving for the day -nohup ./run_integration_test.bash --auto --date 20260123 > test_run.log 2>&1 & - -# Next morning, run image tests manually -salloc --nodes=1 --partition=debug --time=02:00:00 --account=e3sm -./run_image_tests.bash --date 20260123 -``` - ## Output and Logs The script provides color-coded output: @@ -129,16 +72,6 @@ Automatically checks for errors in status files: ... ``` -## Environment Variables - -You can set these before running the script: - -```bash -export DATE_STAMP=20260123 -export UNIQUE_ID="custom_test_id" -./run_integration_test.bash -``` - ## Customization ### Modify Test Configurations @@ -164,51 +97,27 @@ wait_for_slurm_jobs 30 14400 # 30s interval, 4hr max wait_for_slurm_jobs 30 3600 # 30s interval, 1hr max ``` -### Add Custom Checks - -Add your own validation in `phase_3_validation()`: - -```bash -# Custom validation example -log "Running custom checks..." -if [ -f "$ZPPY_DIR/my_custom_check.bash" ]; then - bash "$ZPPY_DIR/my_custom_check.bash" -fi -``` - ## Troubleshooting ### Script Exits Early Check the error message. The script uses `set -e`, so it exits on any error. ### Jobs Don't Complete -- Check SLURM queue: `squeue -u ac.forsyth2` +- Check SLURM queue: `squeue -u ` - Check job logs in the output directories - Increase timeout: edit `wait_for_slurm_jobs` calls ### Environment Issues ```bash # Clean and rebuild -conda remove --y --all --name test-diags-main-20260123 -conda remove --y --all --name test-zi-main-20260123 -conda remove --y --all --name test-zppy-main-20260123-env +conda remove --y --all --name test-diags-main-YYYYMMDD +conda remove --y --all --name test-zi-main-YYYYMMDD +conda remove --y --all --name test-zppy-main-YYYYMMDD -# Re-run -./run_integration_test.bash --date 20260123 -``` - -### Git Issues -```bash -# If git operations fail, manually clean up: -cd ~/ez/zppy -git reset --hard upstream/main -git clean -fd -./run_integration_test.bash --date 20260123 +# Check configurations and then re-run +./run_integration_test.bash ``` -### Checkpoint Issues in Auto Mode -The script will proceed automatically but log warnings. Review logs after completion. - ## Files Created ``` @@ -223,40 +132,12 @@ The script will proceed automatically but log warnings. Review logs after comple └── test_images_summary.md /lcrc/group/e3sm/ac.forsyth2/ -├── zppy_weekly_bundles_output/zppy_main_branch_test_YYYYMMDD/ -├── zppy_weekly_comprehensive_v2_output/zppy_main_branch_test_YYYYMMDD/ -├── zppy_weekly_comprehensive_v3_output/zppy_main_branch_test_YYYYMMDD/ -└── zppy_weekly_legacy_3.0.0_*/zppy_main_branch_test_YYYYMMDD/ +├── zppy_weekly_bundles_output/zppy_main_branch_test_YYYYMMDD_run1/ +├── zppy_weekly_comprehensive_v2_output/zppy_main_branch_test_YYYYMMDD_run1/ +├── zppy_weekly_comprehensive_v3_output/zppy_main_branch_test_YYYYMMDD_run1/ +└── zppy_weekly_legacy_3.0.0_*/zppy_main_branch_test_YYYYMMDD_run1/ ``` -## Tips - -1. **Use descriptive date stamps**: Instead of the current date, use a meaningful identifier like `20260123_bugfix` or `20260123_pr769` - -2. **Run overnight**: The full test takes 2-4 hours. Start it before leaving: - ```bash - nohup ./run_integration_test.bash --auto > test.log 2>&1 & - ``` - -3. **Keep logs**: Redirect output to files for later review: - ```bash - ./run_integration_test.bash --auto 2>&1 | tee test_$(date +%Y%m%d).log - ``` - -4. **Parallel testing**: Run different date stamps to test multiple branches: - ```bash - ./run_integration_test.bash --date 20260123_main & - ./run_integration_test.bash --date 20260123_feature --cherry-pick abc123 & - ``` - -## Support - -For issues or questions: -1. Check the troubleshooting section above -2. Review the script output for error messages -3. Check SLURM logs in the output directories -4. Contact the zppy development team - ## Example run ```bash @@ -268,14 +149,14 @@ git status cd ~/ez/zppy git status -cd /home/ac.forsyth2/ez/zppy_main_branch_tests/test_20260123 +cd /home/ac.forsyth2/ez/zppy_main_branch_tests/test_20260126 cp ~/ez/zppy/tests/main_branch_testing/* . # Copy, so the script isn't affected by the branch change emacs run_integration_test.bash # Configure parameters screen # Run on screen -cd /home/ac.forsyth2/ez/zppy_main_branch_tests/test_20260123 # If not there already +cd /home/ac.forsyth2/ez/zppy_main_branch_tests/test_20260126 # If not there already # Bypass manual checkpoints, tee output: -time ./run_integration_test.bash --date 20260123_run1 --auto 2>&1 | tee integration_test.log +time ./run_integration_test.bash --date 20260126_run1 --auto 2>&1 | tee integration_test.log # CTRL A D to exit screen screen -ls # Check which node the screen is on. tail -f integration_test.log # Follow log updates diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index 9fb80726..d5401e49 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -3,27 +3,35 @@ # Usage: # 1. Copy this file and `run_image_tests.bash` out of the zppy repo. (This script will change the branch). # 2. Edit configuration parameters below. -# 3. Run: ./run_integration_test.bash [OPTIONS] +# 3. Run: ./run_integration_test.bash # 4. Run: ./run_image_tests.bash -# -# Options: -# --date YYYYMMDD Date stamp for test (default: today) -# --auto Run fully automated (no checkpoints) -# --phase N Start from phase N (1=setup, 2=bundles_part2, 3=validation) -# --help Show this help message set -e # Exit on error set -u # Exit on undefined variable +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +DATE_STAMP="${DATE_STAMP:-$(date +%Y%m%d)}" +echo "User is: ${USER:-unknown}" # Pick this up from the environment + # ============================================================================ # Configuration # ============================================================================ -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -DATE_STAMP="${DATE_STAMP:-$(date +%Y%m%d)}" -AUTO_MODE=false -START_PHASE=1 +# Check these every time ##################################################### +RUN_NUMBER=1 +AUTO_MODE=true # By default, run automatically +START_PHASE=1 # By default, start at phase 1 (of 3) + +UNIQUE_ID="zppy_main_branch_test_${DATE_STAMP}_run${RUN_NUMBER}" + +# Base branches (THESE ARE WHAT WE'RE TESTING) +# Usually we test "main". +# If we need to test PRs or include test fixes, we may use a different branch. +DIAGS_BASE_BRANCH="main" +ZI_BASE_BRANCH="main" +ZPPY_BASE_BRANCH="test-fixes" # https://github.com/E3SM-Project/zppy/pull/769 +# Set these up once ########################################################### # Paths HOME_DIR="$HOME" EZ_DIR="$HOME_DIR/ez" @@ -31,19 +39,13 @@ E3SM_DIAGS_DIR="$EZ_DIR/e3sm_diags" ZPPY_INTERFACES_DIR="$EZ_DIR/zppy-interfaces" ZPPY_DIR="$EZ_DIR/zppy" CONDA_PROFILE="$HOME_DIR/miniforge3/etc/profile.d/conda.sh" -OUTPUT_WORKSPACE="/lcrc/group/e3sm/ac.forsyth2" +OUTPUT_WORKSPACE="/lcrc/group/e3sm/${USER}" +# Probably won't need to edit these ########################################### # Environment names -DIAGS_ENV="test-diags-main-${DATE_STAMP}" -ZI_ENV="test-zi-main-${DATE_STAMP}" -ZPPY_ENV="test-zppy-main-${DATE_STAMP}" - -# Test configuration -UNIQUE_ID="zppy_main_branch_test_${DATE_STAMP}" - -# Cherry pick configuration -CHERRY_PICK_BRANCH="test-fixes" -CHERRY_PICK_COMMIT="b56a38c6ae5b24a96bbc80a2dacbc6d1b3dd730b" +DIAGS_ENV="test-diags-${DIAGS_BASE_BRANCH}-${DATE_STAMP}" +ZI_ENV="test-zi-${ZI_BASE_BRANCH}-${DATE_STAMP}" +ZPPY_ENV="test-zppy-${ZPPY_BASE_BRANCH}-${DATE_STAMP}" # Output directories BUNDLES_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_bundles_output/${UNIQUE_ID}/v3.LR.historical_0051/post/scripts" @@ -90,11 +92,6 @@ checkpoint() { fi } -show_help() { - grep "^#" "$0" | grep -v "#!/bin/bash" | sed 's/^# //' | sed 's/^#//' - exit 0 -} - activate_env() { local env_name="${1:-}" # Default to empty string if not provided set +u @@ -132,6 +129,7 @@ setup_conda_env() { ensure_test_branch() { local test_branch="$1" + local base_branch="$2" local current_branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null) # Check if we're already on the test branch @@ -153,10 +151,10 @@ ensure_test_branch() { git checkout "$test_branch" log_success "Checked out existing branch '$test_branch'" else - # Branch doesn't exist, create it from upstream/main - log "Creating new branch '$test_branch' from upstream/main" - git fetch upstream main - git checkout -b "$test_branch" upstream/main + # Branch doesn't exist, create it from upstream/${base_branch} + log "Creating new branch '$test_branch' from upstream/${base_branch}" + git fetch upstream ${base_branch} + git checkout -b "$test_branch" upstream/${base_branch} log_success "Created and checked out new branch '$test_branch'" fi } @@ -167,17 +165,17 @@ wait_for_slurm_jobs() { log "Waiting for SLURM jobs to complete..." local elapsed=0 - local initial_count=$(squeue -u ac.forsyth2 | wc -l) + local initial_count=$(squeue -u ${USER} | wc -l) initial_count=$((initial_count - 1)) # Subtract header log "Initial job count: $initial_count" while true; do - local job_count=$(squeue -u ac.forsyth2 | wc -l) + local job_count=$(squeue -u ${USER} | wc -l) job_count=$((job_count - 1)) # Subtract header # Check for failed dependencies - local failed_jobs=$(squeue -u ac.forsyth2 | grep "DependencyNeverSatisfied" || true) + local failed_jobs=$(squeue -u ${USER} | grep "DependencyNeverSatisfied" || true) if [ -n "$failed_jobs" ]; then log_error "Jobs failed with DependencyNeverSatisfied!" echo "$failed_jobs" @@ -224,34 +222,6 @@ check_status_files() { fi } -# ============================================================================ -# Parse Arguments -# ============================================================================ - -while [[ $# -gt 0 ]]; do - case $1 in - --date) - DATE_STAMP="$2" - shift 2 - ;; - --auto) - AUTO_MODE=true - shift - ;; - --phase) - START_PHASE="$2" - shift 2 - ;; - --help) - show_help - ;; - *) - log_error "Unknown option: $1" - show_help - ;; - esac -done - # ============================================================================ # Phase 1: Setup # ============================================================================ @@ -270,9 +240,9 @@ phase_1_setup() { # ==================================================================== log "Setting up e3sm_diags environment..." cd "$E3SM_DIAGS_DIR" - ensure_test_branch test_e3sm_diags_${DATE_STAMP} + ensure_test_branch test_e3sm_diags_${DATE_STAMP} ${DIAGS_BASE_BRANCH} - log "Latest e3sm_diags commit (should match https://github.com/E3SM-Project/e3sm_diags/commits/main):" + log "Latest e3sm_diags commit (should match https://github.com/E3SM-Project/e3sm_diags/commits/${DIAGS_BASE_BRANCH}):" git log -1 --oneline setup_conda_env "conda-env" "$DIAGS_ENV" @@ -281,9 +251,9 @@ phase_1_setup() { # ==================================================================== log "Setting up zppy-interfaces environment..." cd "$ZPPY_INTERFACES_DIR" - ensure_test_branch test_zi_${DATE_STAMP} + ensure_test_branch test_zi_${DATE_STAMP} ${ZI_BASE_BRANCH} - log "Latest zppy-interfaces commit (should match https://github.com/E3SM-Project/zppy-interfaces/commits/main):" + log "Latest zppy-interfaces commit (should match https://github.com/E3SM-Project/zppy-interfaces/commits/${ZI_BASE_BRANCH}):" git log -1 --oneline setup_conda_env "conda" "$ZI_ENV" @@ -298,19 +268,10 @@ phase_1_setup() { # ======================================================================== log "Setting up zppy environment..." cd "$ZPPY_DIR" - ensure_test_branch test_zppy_${DATE_STAMP} + ensure_test_branch test_zppy_${DATE_STAMP} ${ZPPY_BASE_BRANCH} - log "Latest zppy commit:" + log "Latest zppy commit (should match https://github.com/E3SM-Project/zppy/commits/${ZPPY_BASE_BRANCH}):" git log -1 --oneline - - # Cherry-pick if requested - if [ -n "$CHERRY_PICK_COMMIT" ]; then - log "Cherry-picking commit: $CHERRY_PICK_COMMIT" - git fetch upstream "$CHERRY_PICK_BRANCH" - git cherry-pick "$CHERRY_PICK_COMMIT" - log_success "Cherry-pick applied" - fi - setup_conda_env "conda" "$ZPPY_ENV" # Run unit tests @@ -388,7 +349,7 @@ EOF zppy -c tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v2_chrysalis.cfg zppy -c tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v3_chrysalis.cfg - local job_count=$(squeue -u ac.forsyth2 | wc -l) + local job_count=$(squeue -u ${USER} | wc -l) job_count=$((job_count - 1)) # Don't count the header log_success "Submitted jobs. Total in queue: $job_count" @@ -411,7 +372,7 @@ phase_2_bundles_part2() { activate_env "$ZPPY_ENV" cd "$ZPPY_DIR" - ensure_test_branch test_zppy_${DATE_STAMP} + ensure_test_branch test_zppy_${DATE_STAMP} ${ZPPY_BASE_BRANCH} # Check bundles status log "Checking bundles status..." @@ -425,7 +386,7 @@ phase_2_bundles_part2() { zppy -c tests/integration/generated/test_weekly_bundles_chrysalis.cfg zppy -c tests/integration/generated/test_weekly_legacy_3.0.0_bundles_chrysalis.cfg - local job_count=$(squeue -u ac.forsyth2 | wc -l) + local job_count=$(squeue -u ${USER} | wc -l) job_count=$((job_count - 1)) # Don't count the header log_success "Submitted bundles part 2. Total in queue: $job_count" @@ -446,7 +407,7 @@ phase_3_validation() { activate_env "$ZPPY_ENV" cd "$ZPPY_DIR" - ensure_test_branch test_zppy_${DATE_STAMP} + ensure_test_branch test_zppy_${DATE_STAMP} ${ZPPY_BASE_BRANCH} # Check all status files log "Checking all status files..." From 55cffe152780699c30d9ff1377440a90f1360881 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 26 Jan 2026 16:29:34 -0600 Subject: [PATCH 05/38] Changes to test 20260126_v2 --- tests/main_branch_testing/README.md | 3 ++ .../run_integration_test.bash | 41 ++++++++++++------- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/tests/main_branch_testing/README.md b/tests/main_branch_testing/README.md index e5906405..043a3e3f 100644 --- a/tests/main_branch_testing/README.md +++ b/tests/main_branch_testing/README.md @@ -141,6 +141,7 @@ conda remove --y --all --name test-zppy-main-YYYYMMDD ## Example run ```bash +# The script WILL change branches and MAY cancel jobs. # For safety, confirm branches have no uncommitted changes beforehand: cd ~/ez/e3sm_diags git status @@ -148,6 +149,8 @@ cd ~/ez/zppy-interfaces git status cd ~/ez/zppy git status +# For safety, confirm no jobs are currently running: +squeue -u cd /home/ac.forsyth2/ez/zppy_main_branch_tests/test_20260126 cp ~/ez/zppy/tests/main_branch_testing/* . # Copy, so the script isn't affected by the branch change diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index d5401e49..51fe6131 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -18,12 +18,11 @@ echo "User is: ${USER:-unknown}" # Pick this up from the environment # ============================================================================ # Check these every time ##################################################### + RUN_NUMBER=1 AUTO_MODE=true # By default, run automatically START_PHASE=1 # By default, start at phase 1 (of 3) -UNIQUE_ID="zppy_main_branch_test_${DATE_STAMP}_run${RUN_NUMBER}" - # Base branches (THESE ARE WHAT WE'RE TESTING) # Usually we test "main". # If we need to test PRs or include test fixes, we may use a different branch. @@ -32,6 +31,7 @@ ZI_BASE_BRANCH="main" ZPPY_BASE_BRANCH="test-fixes" # https://github.com/E3SM-Project/zppy/pull/769 # Set these up once ########################################################### + # Paths HOME_DIR="$HOME" EZ_DIR="$HOME_DIR/ez" @@ -42,10 +42,15 @@ CONDA_PROFILE="$HOME_DIR/miniforge3/etc/profile.d/conda.sh" OUTPUT_WORKSPACE="/lcrc/group/e3sm/${USER}" # Probably won't need to edit these ########################################### + +# ID +TAG="${DATE_STAMP}_run${RUN_NUMBER}" +UNIQUE_ID="zppy_main_branch_test_${TAG}" + # Environment names -DIAGS_ENV="test-diags-${DIAGS_BASE_BRANCH}-${DATE_STAMP}" -ZI_ENV="test-zi-${ZI_BASE_BRANCH}-${DATE_STAMP}" -ZPPY_ENV="test-zppy-${ZPPY_BASE_BRANCH}-${DATE_STAMP}" +DIAGS_ENV="test-diags-${DIAGS_BASE_BRANCH}-${TAG}" +ZI_ENV="test-zi-${ZI_BASE_BRANCH}-${TAG}" +ZPPY_ENV="test-zppy-${ZPPY_BASE_BRANCH}-${TAG}" # Output directories BUNDLES_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_bundles_output/${UNIQUE_ID}/v3.LR.historical_0051/post/scripts" @@ -176,12 +181,20 @@ wait_for_slurm_jobs() { # Check for failed dependencies local failed_jobs=$(squeue -u ${USER} | grep "DependencyNeverSatisfied" || true) - if [ -n "$failed_jobs" ]; then - log_error "Jobs failed with DependencyNeverSatisfied!" + local failed_count=$(echo "$failed_jobs" | grep -c . || echo 0) + # prev_failed_count defaults to 0 + if [ "$failed_count" -gt "${prev_failed_count:-0}" ]; then + log_error "Jobs found with DependencyNeverSatisfied:" echo "$failed_jobs" - log_error "Check job logs for details" - return 1 + # Check if ALL jobs have DependencyNeverSatisfied + if [ "$job_count" -eq "$failed_count" ]; then + checkpoint "Some jobs can't run, because of DependencyNeverSatisfied" + log_error "All jobs have DependencyNeverSatisfied - cancelling all jobs" + scancel -u ${USER} + job_count=0 + fi fi + prev_failed_count=$failed_count if [ "$job_count" -eq 0 ]; then log_success "All SLURM jobs completed!" @@ -240,7 +253,7 @@ phase_1_setup() { # ==================================================================== log "Setting up e3sm_diags environment..." cd "$E3SM_DIAGS_DIR" - ensure_test_branch test_e3sm_diags_${DATE_STAMP} ${DIAGS_BASE_BRANCH} + ensure_test_branch test_e3sm_diags_${TAG} ${DIAGS_BASE_BRANCH} log "Latest e3sm_diags commit (should match https://github.com/E3SM-Project/e3sm_diags/commits/${DIAGS_BASE_BRANCH}):" git log -1 --oneline @@ -251,7 +264,7 @@ phase_1_setup() { # ==================================================================== log "Setting up zppy-interfaces environment..." cd "$ZPPY_INTERFACES_DIR" - ensure_test_branch test_zi_${DATE_STAMP} ${ZI_BASE_BRANCH} + ensure_test_branch test_zi_${TAG} ${ZI_BASE_BRANCH} log "Latest zppy-interfaces commit (should match https://github.com/E3SM-Project/zppy-interfaces/commits/${ZI_BASE_BRANCH}):" git log -1 --oneline @@ -268,7 +281,7 @@ phase_1_setup() { # ======================================================================== log "Setting up zppy environment..." cd "$ZPPY_DIR" - ensure_test_branch test_zppy_${DATE_STAMP} ${ZPPY_BASE_BRANCH} + ensure_test_branch test_zppy_${TAG} ${ZPPY_BASE_BRANCH} log "Latest zppy commit (should match https://github.com/E3SM-Project/zppy/commits/${ZPPY_BASE_BRANCH}):" git log -1 --oneline @@ -372,7 +385,7 @@ phase_2_bundles_part2() { activate_env "$ZPPY_ENV" cd "$ZPPY_DIR" - ensure_test_branch test_zppy_${DATE_STAMP} ${ZPPY_BASE_BRANCH} + ensure_test_branch test_zppy_${TAG} ${ZPPY_BASE_BRANCH} # Check bundles status log "Checking bundles status..." @@ -407,7 +420,7 @@ phase_3_validation() { activate_env "$ZPPY_ENV" cd "$ZPPY_DIR" - ensure_test_branch test_zppy_${DATE_STAMP} ${ZPPY_BASE_BRANCH} + ensure_test_branch test_zppy_${TAG} ${ZPPY_BASE_BRANCH} # Check all status files log "Checking all status files..." From 2a69af8946cc8940bb8e430584259061babe4e90 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 26 Jan 2026 17:11:41 -0600 Subject: [PATCH 06/38] Changes to test 20260126_v3 --- tests/main_branch_testing/run_integration_test.bash | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index 51fe6131..11f59318 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -181,7 +181,10 @@ wait_for_slurm_jobs() { # Check for failed dependencies local failed_jobs=$(squeue -u ${USER} | grep "DependencyNeverSatisfied" || true) - local failed_count=$(echo "$failed_jobs" | grep -c . || echo 0) + local failed_count=0 + if [ -n "$failed_jobs" ]; then + failed_count=$(echo "$failed_jobs" | wc -l) + fi # prev_failed_count defaults to 0 if [ "$failed_count" -gt "${prev_failed_count:-0}" ]; then log_error "Jobs found with DependencyNeverSatisfied:" From 9118ef78d6843f54c4dd5474bea7b201298c9753 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Fri, 30 Jan 2026 16:17:55 -0600 Subject: [PATCH 07/38] Updates as of 20260130 --- tests/main_branch_testing/README.md | 21 +------ .../main_branch_testing/run_image_tests.bash | 56 ------------------- .../run_integration_test.bash | 26 +++++---- 3 files changed, 17 insertions(+), 86 deletions(-) delete mode 100644 tests/main_branch_testing/run_image_tests.bash diff --git a/tests/main_branch_testing/README.md b/tests/main_branch_testing/README.md index 043a3e3f..02dba5b4 100644 --- a/tests/main_branch_testing/README.md +++ b/tests/main_branch_testing/README.md @@ -11,7 +11,7 @@ This automation system streamlines the zppy integration testing workflow, reduci ## Workflow Phases -### Phase 1: Setup (~2-4 hours including SLURM wait) +### Phase 1: Setup - Sets up e3sm_diags conda environment - Sets up zppy-interfaces conda environment - Sets up zppy conda environment @@ -19,12 +19,12 @@ This automation system streamlines the zppy integration testing workflow, reduci - Submits initial SLURM jobs (6 configs) - Waits for jobs to complete -### Phase 2: Bundles Part 2 (~30-60 minutes including SLURM wait) +### Phase 2: Bundles Part 2 - Checks status of bundles runs - Submits bundles part 2 jobs - Waits for completion -### Phase 3: Validation (~15 minutes, excluding image tests) +### Phase 3: Validation - Checks all status files - Runs pytest integration tests: - test_bash_generation.py @@ -34,21 +34,6 @@ This automation system streamlines the zppy integration testing workflow, reduci - test_bundles.py - Provides instructions for running test_images.py on compute node -## Running Image Tests - -The image tests require a compute node allocation. Two options: - -### Option 1: Manual Allocation (Recommended) -```bash -salloc --nodes=1 --partition=debug --time=02:00:00 --account=e3sm -./run_image_tests.bash --date 20260123 -``` - -### Option 2: Automatic Allocation -```bash -./run_image_tests.bash --date 20260123 --auto -``` - ## Output and Logs The script provides color-coded output: diff --git a/tests/main_branch_testing/run_image_tests.bash b/tests/main_branch_testing/run_image_tests.bash deleted file mode 100644 index 3a414621..00000000 --- a/tests/main_branch_testing/run_image_tests.bash +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -# Run image tests on compute node -# Usage: ./run_image_tests.bash [--date YYYYMMDD] [--auto] - -set -e - -DATE_STAMP="${DATE_STAMP:-$(date +%Y%m%d)}" -AUTO_ALLOCATE=false - -# Parse arguments -while [[ $# -gt 0 ]]; do - case $1 in - --date) - DATE_STAMP="$2" - shift 2 - ;; - --auto) - AUTO_ALLOCATE=true - shift - ;; - *) - echo "Unknown option: $1" - echo "Usage: $0 [--date YYYYMMDD] [--auto]" - exit 1 - ;; - esac -done - -ZPPY_ENV="test-zppy-main-${DATE_STAMP}-env" -ZPPY_DIR="$HOME/ez/zppy" -CONDA_PROFILE="$HOME/miniforge3/etc/profile.d/conda.sh" - -if [ "$AUTO_ALLOCATE" = true ]; then - echo "Auto-allocating compute node and running tests..." - salloc --nodes=1 --partition=debug --time=02:00:00 --account=e3sm << EOFALLOC -source ~/.bashrc -lcrc_conda # Run conda activation function defined in ~/.bashrc -conda activate $ZPPY_ENV -cd $ZPPY_DIR -pytest tests/integration/test_images.py -cat test_images_summary.md -EOFALLOC -else - # DEFAULT - # Assume we're already on a compute node or user will allocate manually - echo "Running image tests..." - echo "If not on compute node, first run:" - echo " salloc --nodes=1 --partition=debug --time=02:00:00 --account=e3sm" - echo "" - - source "$CONDA_PROFILE" - conda activate "$ZPPY_ENV" - cd "$ZPPY_DIR" - pytest tests/integration/test_images.py - cat test_images_summary.md -fi diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index 11f59318..a7ab30cf 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -54,7 +54,7 @@ ZPPY_ENV="test-zppy-${ZPPY_BASE_BRANCH}-${TAG}" # Output directories BUNDLES_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_bundles_output/${UNIQUE_ID}/v3.LR.historical_0051/post/scripts" -LEGACY_BUNDLES_OUTPUT="${OUTPUT_WORKSPACE}zppy_weekly_legacy_3.0.0_bundles_output/${UNIQUE_ID}/v3.LR.historical_0051/post/scripts" +LEGACY_BUNDLES_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_legacy_3.0.0_bundles_output/${UNIQUE_ID}/v3.LR.historical_0051/post/scripts" V2_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_comprehensive_v2_output/${UNIQUE_ID}/v2.LR.historical_0201/post/scripts" LEGACY_V2_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_legacy_3.0.0_comprehensive_v2_output/${UNIQUE_ID}/v2.LR.historical_0201/post/scripts" V3_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_comprehensive_v3_output/${UNIQUE_ID}/v3.LR.historical_0051/post/scripts" @@ -97,6 +97,7 @@ checkpoint() { fi } +# Use this to activate an existing environment and install zppy. activate_env() { local env_name="${1:-}" # Default to empty string if not provided set +u @@ -106,10 +107,14 @@ activate_env() { # Only activate if an environment name was provided if [ -n "$env_name" ]; then conda activate "$env_name" + # Always install/update the package + log "Installing package in '$env_name'" + python -m pip install . fi set -u } +# Use this to create a new conda environment AND activate it and install zppy. setup_conda_env() { local conda_dir="$1" local env_name="$2" @@ -125,10 +130,6 @@ setup_conda_env() { fi activate_env "$env_name" - - # Always install/update the package - log "Installing package in '$env_name'" - python -m pip install . log_success "Environment '$env_name' ready" } @@ -147,6 +148,8 @@ ensure_test_branch() { log "Saving current work..." git status git add -A + # We have to use --no-verify because we might not be in a conda env. + # (We need to be in a conda env that supports pre-commit checks to avoid this flag). git commit -m "Auto-save before test" --no-verify || true # Check if the test branch exists @@ -229,10 +232,10 @@ check_status_files() { local errors=$(grep -v "OK" *status 2>/dev/null || true) if [ -z "$errors" ]; then - log_success "$name: No errors found" + log_success "$name: No errors found in ${dir}" return 0 else - log_error "$name: Errors found!" + log_error "$name: Errors found in ${dir}!" echo "$errors" return 1 fi @@ -249,8 +252,6 @@ phase_1_setup() { log "Unique ID: $UNIQUE_ID" log "=========================================" - activate_env - # ==================================================================== # Set up e3sm_diags environment # ==================================================================== @@ -386,8 +387,8 @@ phase_2_bundles_part2() { log "Phase 2: Bundles Part 2" log "=========================================" - activate_env "$ZPPY_ENV" cd "$ZPPY_DIR" + activate_env "$ZPPY_ENV" ensure_test_branch test_zppy_${TAG} ${ZPPY_BASE_BRANCH} # Check bundles status @@ -421,8 +422,8 @@ phase_3_validation() { log "Phase 3: Validation" log "=========================================" - activate_env "$ZPPY_ENV" cd "$ZPPY_DIR" + activate_env "$ZPPY_ENV" ensure_test_branch test_zppy_${TAG} ${ZPPY_BASE_BRANCH} # Check all status files @@ -445,6 +446,7 @@ phase_3_validation() { fi # Run pytest tests + cd ${ZPPY_DIR} log "Running integration tests..." log "Running test_bash_generation.py..." @@ -471,10 +473,10 @@ phase_3_validation() { log " cd ${ZPPY_DIR}" log " pytest tests/integration/test_images.py" log " cat test_images_summary.md" - log "Alternative: run ./run_integration_test.bash" log_success "Phase 3 complete!" log_success "All automated tests finished successfully!" + log_success "Reminder: run test_images.py manually, as described above" } # ============================================================================ From 1695247c19454b83a7b7f4b594a6d585bc1062fe Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Fri, 30 Jan 2026 17:08:25 -0600 Subject: [PATCH 08/38] Add activation command --- tests/main_branch_testing/run_integration_test.bash | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index a7ab30cf..f120e9e4 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -119,6 +119,8 @@ setup_conda_env() { local conda_dir="$1" local env_name="$2" + activate_env # We need conda to exist + # Check if environment already exists if conda env list | grep -q "^${env_name} "; then log "Environment '$env_name' already exists, skipping creation" From 0e6d063b2a0fddc726c8302837fc6f10f4fce48b Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Tue, 9 Jun 2026 17:28:17 -0500 Subject: [PATCH 09/38] Claude rewrite of test automation --- tests/main_branch_testing/README.md | 182 +++++--- .../run_integration_test.bash | 430 ++++++++++-------- 2 files changed, 346 insertions(+), 266 deletions(-) diff --git a/tests/main_branch_testing/README.md b/tests/main_branch_testing/README.md index 02dba5b4..25ef6f44 100644 --- a/tests/main_branch_testing/README.md +++ b/tests/main_branch_testing/README.md @@ -9,30 +9,56 @@ This automation system streamlines the zppy integration testing workflow, reduci ./run_integration_test.bash ``` +### Skip Straight to a Later Phase +```bash +./run_integration_test.bash --phase 2 # Start at Bundles Part 2 +./run_integration_test.bash --phase 3 # Start at Validation only +``` + +### Non-Interactive (Auto) Mode +```bash +./run_integration_test.bash --auto +``` +In auto mode all checkpoints are bypassed and the script runs end-to-end without waiting for user input. + +## Configuration + +Open `run_integration_test.bash` and edit the **"Check these every time"** block at the top before each test run: + +| Variable | Description | +| --- | --- | +| `RUN_NUMBER` | Increment this if you run multiple tests on the same day | +| `DIAGS_BASE_BRANCH` | Branch to test for e3sm_diags (usually `main`) | +| `ZI_BASE_BRANCH` | Branch to test for zppy-interfaces (usually `main`) | +| `ZPPY_BASE_BRANCH` | Branch to test for zppy (usually `main`) | +| `SKIP_MPAS` | Set to `true` to omit mpas_analysis (workaround for known segfault) | + +The **"Set these up once"** block below that contains machine paths (`EZ_DIR`, `CONDA_PROFILE`, `OUTPUT_WORKSPACE`) which typically don't change between runs. + ## Workflow Phases ### Phase 1: Setup -- Sets up e3sm_diags conda environment -- Sets up zppy-interfaces conda environment -- Sets up zppy conda environment -- Generates config files -- Submits initial SLURM jobs (6 configs) -- Waits for jobs to complete +- Creates conda environments for e3sm_diags, zppy-interfaces, and zppy +- Runs unit tests for zppy-interfaces and zppy +- Patches `tests/integration/utils.py` with test-specific environment commands, config list, and unique ID +- Generates config files via `python tests/integration/utils.py` +- Submits all 9 initial SLURM jobs (3 current + 3 legacy 3.1.0 + 3 legacy 3.0.0) +- Waits for jobs to complete (polls every 10 min, 4-hour max) ### Phase 2: Bundles Part 2 -- Checks status of bundles runs -- Submits bundles part 2 jobs -- Waits for completion +- Checks status files for all three bundles output directories; warns if any are non-OK before proceeding (a non-OK status here is the likely cause of the historical KeyError) +- Submits bundles part 2 jobs (`weekly_bundles`, `legacy_3.1.0_bundles`, `legacy_3.0.0_bundles`) +- Waits for completion (polls every 10 min, 1-hour max) ### Phase 3: Validation -- Checks all status files +- Checks status files for all 9 output directories - Runs pytest integration tests: - - test_bash_generation.py - - test_campaign.py - - test_defaults.py - - test_last_year.py - - test_bundles.py -- Provides instructions for running test_images.py on compute node + - `test_last_year.py` + - `test_bash_generation.py` + - `test_campaign.py` + - `test_defaults.py` + - `test_bundles.py` +- Prints instructions for running `test_images.py` manually on a compute node ## Output and Logs @@ -45,61 +71,56 @@ The script provides color-coded output: ### SLURM Job Monitoring The script automatically monitors SLURM jobs and shows: ``` -Jobs remaining: 42 (elapsed: 1234s) +Jobs remaining: 42 (elapsed: 1234s / max: 14400s) ``` +If all remaining jobs enter `DependencyNeverSatisfied`, they are cancelled automatically and the script exits with an error. ### Status File Checking -Automatically checks for errors in status files: +Automatically checks for non-OK entries in all status directories: ``` -✓ v2: No errors found -✓ Legacy v2: No errors found -✓ v3: No errors found +✓ v2: All status files OK +✓ Legacy 3.1.0 v2: All status files OK +✓ Legacy 3.0.0 v2: All status files OK ... ``` ## Customization -### Modify Test Configurations - -Edit the script to change which configs are run: - -```bash -# In the generated Python code section, modify: -"cfgs_to_run": [ - "weekly_bundles", - "weekly_comprehensive_v2", - # Add or remove configs here -], -``` - ### Adjust Timeouts ```bash -# In phase_1_setup(), change max wait time: -wait_for_slurm_jobs 30 14400 # 30s interval, 4hr max +# In phase_1_setup(): +wait_for_slurm_jobs 600 14400 # Check every 10 min, max 4 hours # In phase_2_bundles_part2(): -wait_for_slurm_jobs 30 3600 # 30s interval, 1hr max +wait_for_slurm_jobs 600 3600 # Check every 10 min, max 1 hour ``` ## Troubleshooting ### Script Exits Early -Check the error message. The script uses `set -e`, so it exits on any error. +Check the error message. The script uses `set -e`, so it exits on any error. Common causes: +- A unit test failure during Phase 1 setup +- A SLURM timeout (increase the max-wait argument to `wait_for_slurm_jobs`) +- `DependencyNeverSatisfied` on all queued jobs (check your cfg files and SLURM account) + +### mpas_analysis Segfault +Set `SKIP_MPAS=true` in the configuration section. This removes `mpas_analysis` from `tasks_to_run` and sets a placeholder for `mpas_analysis_environment_commands` so it cannot be accidentally invoked. -### Jobs Don't Complete -- Check SLURM queue: `squeue -u ` -- Check job logs in the output directories -- Increase timeout: edit `wait_for_slurm_jobs` calls +### KeyError on Bundles Part 2 +Phase 2 now checks bundle status files before resubmitting and warns if any are non-OK. Resolve any failures in the Phase 1 bundle runs before proceeding. You can restart from Phase 2 once the underlying jobs are clean: +```bash +./run_integration_test.bash --phase 2 +``` ### Environment Issues ```bash -# Clean and rebuild -conda remove --y --all --name test-diags-main-YYYYMMDD -conda remove --y --all --name test-zi-main-YYYYMMDD -conda remove --y --all --name test-zppy-main-YYYYMMDD +# Remove and rebuild stale environments +conda remove --yes --all --name test-diags-main-YYYYMMDD_runN +conda remove --yes --all --name test-zi-main-YYYYMMDD_runN +conda remove --yes --all --name test-zppy-main-YYYYMMDD_runN -# Check configurations and then re-run +# Then re-run from Phase 1 ./run_integration_test.bash ``` @@ -111,41 +132,52 @@ conda remove --y --all --name test-zppy-main-YYYYMMDD │ ├── test_weekly_bundles_chrysalis.cfg │ ├── test_weekly_comprehensive_v2_chrysalis.cfg │ ├── test_weekly_comprehensive_v3_chrysalis.cfg +│ ├── test_weekly_legacy_3.1.0_bundles_chrysalis.cfg +│ ├── test_weekly_legacy_3.1.0_comprehensive_v2_chrysalis.cfg +│ ├── test_weekly_legacy_3.1.0_comprehensive_v3_chrysalis.cfg │ ├── test_weekly_legacy_3.0.0_bundles_chrysalis.cfg │ ├── test_weekly_legacy_3.0.0_comprehensive_v2_chrysalis.cfg │ └── test_weekly_legacy_3.0.0_comprehensive_v3_chrysalis.cfg └── test_images_summary.md -/lcrc/group/e3sm/ac.forsyth2/ -├── zppy_weekly_bundles_output/zppy_main_branch_test_YYYYMMDD_run1/ -├── zppy_weekly_comprehensive_v2_output/zppy_main_branch_test_YYYYMMDD_run1/ -├── zppy_weekly_comprehensive_v3_output/zppy_main_branch_test_YYYYMMDD_run1/ -└── zppy_weekly_legacy_3.0.0_*/zppy_main_branch_test_YYYYMMDD_run1/ +/lcrc/group/e3sm// +├── zppy_weekly_bundles_output/zppy_main_branch_test_YYYYMMDD_runN/ +├── zppy_weekly_comprehensive_v2_output/zppy_main_branch_test_YYYYMMDD_runN/ +├── zppy_weekly_comprehensive_v3_output/zppy_main_branch_test_YYYYMMDD_runN/ +├── zppy_weekly_legacy_3.1.0_bundles_output/zppy_main_branch_test_YYYYMMDD_runN/ +├── zppy_weekly_legacy_3.1.0_comprehensive_v2_output/zppy_main_branch_test_YYYYMMDD_runN/ +├── zppy_weekly_legacy_3.1.0_comprehensive_v3_output/zppy_main_branch_test_YYYYMMDD_runN/ +├── zppy_weekly_legacy_3.0.0_bundles_output/zppy_main_branch_test_YYYYMMDD_runN/ +├── zppy_weekly_legacy_3.0.0_comprehensive_v2_output/zppy_main_branch_test_YYYYMMDD_runN/ +└── zppy_weekly_legacy_3.0.0_comprehensive_v3_output/zppy_main_branch_test_YYYYMMDD_runN/ ``` -## Example run +## Example Run ```bash -# The script WILL change branches and MAY cancel jobs. -# For safety, confirm branches have no uncommitted changes beforehand: -cd ~/ez/e3sm_diags -git status -cd ~/ez/zppy-interfaces -git status -cd ~/ez/zppy -git status -# For safety, confirm no jobs are currently running: -squeue -u - -cd /home/ac.forsyth2/ez/zppy_main_branch_tests/test_20260126 -cp ~/ez/zppy/tests/main_branch_testing/* . # Copy, so the script isn't affected by the branch change -emacs run_integration_test.bash # Configure parameters - -screen # Run on screen -cd /home/ac.forsyth2/ez/zppy_main_branch_tests/test_20260126 # If not there already -# Bypass manual checkpoints, tee output: -time ./run_integration_test.bash --date 20260126_run1 --auto 2>&1 | tee integration_test.log -# CTRL A D to exit screen -screen -ls # Check which node the screen is on. -tail -f integration_test.log # Follow log updates +# Confirm repos have no uncommitted changes +cd ~/ez/e3sm_diags && git status +cd ~/ez/zppy-interfaces && git status +cd ~/ez/zppy && git status + +# Confirm no jobs are currently running +squeue -u $USER + +# Copy the script out of the repo (Phase 1 will change branches) +mkdir -p ~/ez/zppy_main_branch_tests/test_YYYYMMDD +cd ~/ez/zppy_main_branch_tests/test_YYYYMMDD +cp ~/ez/zppy/tests/main_branch_testing/* . + +# Edit configuration parameters +emacs run_integration_test.bash + +# Run inside a screen session so it survives disconnects +screen +cd ~/ez/zppy_main_branch_tests/test_YYYYMMDD +time ./run_integration_test.bash --auto 2>&1 | tee integration_test.log +# Ctrl-A D to detach from screen + +# Monitor progress from another terminal +screen -ls # Find the screen session +tail -f integration_test.log # Follow log output ``` diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index f120e9e4..daf9f4e7 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -1,71 +1,94 @@ #!/bin/bash # zppy Integration Test Automation Script +# # Usage: -# 1. Copy this file and `run_image_tests.bash` out of the zppy repo. (This script will change the branch). -# 2. Edit configuration parameters below. -# 3. Run: ./run_integration_test.bash -# 4. Run: ./run_image_tests.bash +# 1. Copy this file OUT of the zppy repo (this script will change branches). +# 2. Edit the "Check these every time" configuration section below. +# 3. Run: ./run_integration_test.bash [--phase N] [--auto] +# +# Phases: +# 1 - Full setup: build envs, run unit tests, generate configs, submit SLURM jobs +# 2 - Bundles Part 2 (run after Phase 1 jobs finish) +# 3 - Validation: status checks + pytest integration tests +# +# Notes: +# - test_images.py must be run manually from a compute node (see Phase 3 output). +# - mpas_analysis is skipped by default due to known segfault issues; see SKIP_MPAS below. +# - If Bundles Part 2 fails with a KeyError, re-run with --phase 2 after confirming +# that the bundle status files are all "OK". set -e # Exit on error set -u # Exit on undefined variable -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -DATE_STAMP="${DATE_STAMP:-$(date +%Y%m%d)}" -echo "User is: ${USER:-unknown}" # Pick this up from the environment +# ============================================================================ +# Parse arguments +# ============================================================================ + +AUTO_MODE=false +START_PHASE=1 + +while [[ $# -gt 0 ]]; do + case "$1" in + --auto) AUTO_MODE=true; shift ;; + --phase) START_PHASE="$2"; shift 2 ;; + *) echo "Unknown argument: $1"; exit 1 ;; + esac +done # ============================================================================ # Configuration # ============================================================================ -# Check these every time ##################################################### +# --- Check these every time -------------------------------------------------- RUN_NUMBER=1 -AUTO_MODE=true # By default, run automatically -START_PHASE=1 # By default, start at phase 1 (of 3) -# Base branches (THESE ARE WHAT WE'RE TESTING) -# Usually we test "main". -# If we need to test PRs or include test fixes, we may use a different branch. +# Base branches (what we're testing -- usually "main") DIAGS_BASE_BRANCH="main" ZI_BASE_BRANCH="main" -ZPPY_BASE_BRANCH="test-fixes" # https://github.com/E3SM-Project/zppy/pull/769 +ZPPY_BASE_BRANCH="main" + +# Set to true to skip mpas_analysis due to known segfault issues on some machines. +# When true, "mpas_analysis" is removed from tasks_to_run in utils.py. +SKIP_MPAS=false -# Set these up once ########################################################### +# --- Set these up once ------------------------------------------------------- -# Paths HOME_DIR="$HOME" -EZ_DIR="$HOME_DIR/ez" +EZ_DIR="$HOME_DIR/ez" # Parent dir for all repos E3SM_DIAGS_DIR="$EZ_DIR/e3sm_diags" ZPPY_INTERFACES_DIR="$EZ_DIR/zppy-interfaces" ZPPY_DIR="$EZ_DIR/zppy" CONDA_PROFILE="$HOME_DIR/miniforge3/etc/profile.d/conda.sh" -OUTPUT_WORKSPACE="/lcrc/group/e3sm/${USER}" +OUTPUT_WORKSPACE="/lcrc/group/e3sm/${USER}" # Chrysalis default; edit for Compy/Perlmutter -# Probably won't need to edit these ########################################### +# --- Derived (probably no edits needed) -------------------------------------- -# ID +DATE_STAMP="${DATE_STAMP:-$(date +%Y%m%d)}" TAG="${DATE_STAMP}_run${RUN_NUMBER}" UNIQUE_ID="zppy_main_branch_test_${TAG}" -# Environment names DIAGS_ENV="test-diags-${DIAGS_BASE_BRANCH}-${TAG}" ZI_ENV="test-zi-${ZI_BASE_BRANCH}-${TAG}" ZPPY_ENV="test-zppy-${ZPPY_BASE_BRANCH}-${TAG}" -# Output directories +# Output directories (status file locations) BUNDLES_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_bundles_output/${UNIQUE_ID}/v3.LR.historical_0051/post/scripts" -LEGACY_BUNDLES_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_legacy_3.0.0_bundles_output/${UNIQUE_ID}/v3.LR.historical_0051/post/scripts" +LEGACY_310_BUNDLES_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_legacy_3.1.0_bundles_output/${UNIQUE_ID}/v3.LR.historical_0051/post/scripts" +LEGACY_300_BUNDLES_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_legacy_3.0.0_bundles_output/${UNIQUE_ID}/v3.LR.historical_0051/post/scripts" V2_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_comprehensive_v2_output/${UNIQUE_ID}/v2.LR.historical_0201/post/scripts" -LEGACY_V2_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_legacy_3.0.0_comprehensive_v2_output/${UNIQUE_ID}/v2.LR.historical_0201/post/scripts" +LEGACY_310_V2_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_legacy_3.1.0_comprehensive_v2_output/${UNIQUE_ID}/v2.LR.historical_0201/post/scripts" +LEGACY_300_V2_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_legacy_3.0.0_comprehensive_v2_output/${UNIQUE_ID}/v2.LR.historical_0201/post/scripts" V3_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_comprehensive_v3_output/${UNIQUE_ID}/v3.LR.historical_0051/post/scripts" -LEGACY_V3_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_legacy_3.0.0_comprehensive_v3_output/${UNIQUE_ID}/v3.LR.historical_0051/post/scripts" +LEGACY_310_V3_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_legacy_3.1.0_comprehensive_v3_output/${UNIQUE_ID}/v3.LR.historical_0051/post/scripts" +LEGACY_300_V3_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_legacy_3.0.0_comprehensive_v3_output/${UNIQUE_ID}/v3.LR.historical_0051/post/scripts" -# Colors for output +# Colors RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' -NC='\033[0m' # No Color +NC='\033[0m' # ============================================================================ # Helper Functions @@ -91,41 +114,39 @@ checkpoint() { local message="$1" if [ "$AUTO_MODE" = false ]; then log_warning "CHECKPOINT: $message" - read -p "Press Enter to continue or Ctrl+C to abort..." + read -rp "Press Enter to continue or Ctrl+C to abort..." else - log "AUTO MODE: $message" + log "AUTO MODE: Passing checkpoint -- $message" fi } -# Use this to activate an existing environment and install zppy. +# Activate conda and (optionally) a named environment. activate_env() { - local env_name="${1:-}" # Default to empty string if not provided + local env_name="${1:-}" set +u + # shellcheck disable=SC1090 source ~/.bashrc - lcrc_conda # Run conda activation function defined in ~/.bashrc + lcrc_conda # Defined in ~/.bashrc for Chrysalis; adjust for Compy/Perlmutter - # Only activate if an environment name was provided if [ -n "$env_name" ]; then conda activate "$env_name" - # Always install/update the package - log "Installing package in '$env_name'" + log "Installing/updating package in '$env_name'..." python -m pip install . fi set -u } -# Use this to create a new conda environment AND activate it and install zppy. +# Create (if needed) and activate a conda environment. setup_conda_env() { - local conda_dir="$1" + local conda_dir="$1" # Directory containing dev.yml (e.g. "conda" or "conda-env") local env_name="$2" - activate_env # We need conda to exist + activate_env # Ensure conda itself is available - # Check if environment already exists if conda env list | grep -q "^${env_name} "; then log "Environment '$env_name' already exists, skipping creation" else - log "Creating new environment '$env_name'" + log "Creating environment '$env_name' from ${conda_dir}/dev.yml..." rm -rf build conda clean --all --yes conda env create -f "${conda_dir}/dev.yml" -n "$env_name" @@ -135,71 +156,63 @@ setup_conda_env() { log_success "Environment '$env_name' ready" } +# Checkout test branch, creating it from upstream/ if it doesn't exist. +# Stashes/commits any in-progress work first. ensure_test_branch() { local test_branch="$1" local base_branch="$2" - local current_branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null) + local current_branch + current_branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null) - # Check if we're already on the test branch if [ "$current_branch" = "$test_branch" ]; then - log "Already on branch '$test_branch', skipping checkout" + log "Already on branch '$test_branch'" return 0 fi - # Save current work before switching branches - log "Saving current work..." - git status + log "Saving current work before switching branches..." git add -A - # We have to use --no-verify because we might not be in a conda env. - # (We need to be in a conda env that supports pre-commit checks to avoid this flag). git commit -m "Auto-save before test" --no-verify || true - # Check if the test branch exists if git show-ref --verify --quiet "refs/heads/$test_branch"; then - # Branch exists, just check it out - log "Checking out existing branch '$test_branch'" + log "Checking out existing branch '$test_branch'..." git checkout "$test_branch" - log_success "Checked out existing branch '$test_branch'" else - # Branch doesn't exist, create it from upstream/${base_branch} - log "Creating new branch '$test_branch' from upstream/${base_branch}" - git fetch upstream ${base_branch} - git checkout -b "$test_branch" upstream/${base_branch} - log_success "Created and checked out new branch '$test_branch'" + log "Creating new branch '$test_branch' from upstream/${base_branch}..." + git fetch upstream "${base_branch}" + git checkout -b "$test_branch" "upstream/${base_branch}" fi + log_success "On branch '$test_branch'" } +# Poll squeue until no user jobs remain (or timeout). wait_for_slurm_jobs() { - local check_interval=${1:-600} # Check every 600 seconds (10 minutes) by default - local max_wait=${2:-14400} # Max wait 4 hours by default + local check_interval=${1:-600} # seconds between checks (default 10 min) + local max_wait=${2:-14400} # max total wait seconds (default 4 hours) - log "Waiting for SLURM jobs to complete..." + log "Waiting for SLURM jobs to complete (checking every ${check_interval}s, max ${max_wait}s)..." local elapsed=0 - local initial_count=$(squeue -u ${USER} | wc -l) - initial_count=$((initial_count - 1)) # Subtract header - - log "Initial job count: $initial_count" + local prev_failed_count=0 while true; do - local job_count=$(squeue -u ${USER} | wc -l) - job_count=$((job_count - 1)) # Subtract header + local job_count + job_count=$(squeue -u "${USER}" | wc -l) + job_count=$((job_count - 1)) # subtract header - # Check for failed dependencies - local failed_jobs=$(squeue -u ${USER} | grep "DependencyNeverSatisfied" || true) + # Detect DependencyNeverSatisfied + local failed_jobs + failed_jobs=$(squeue -u "${USER}" | grep "DependencyNeverSatisfied" || true) local failed_count=0 if [ -n "$failed_jobs" ]; then failed_count=$(echo "$failed_jobs" | wc -l) fi - # prev_failed_count defaults to 0 - if [ "$failed_count" -gt "${prev_failed_count:-0}" ]; then - log_error "Jobs found with DependencyNeverSatisfied:" + + if [ "$failed_count" -gt "$prev_failed_count" ]; then + log_error "Jobs with DependencyNeverSatisfied:" echo "$failed_jobs" - # Check if ALL jobs have DependencyNeverSatisfied if [ "$job_count" -eq "$failed_count" ]; then - checkpoint "Some jobs can't run, because of DependencyNeverSatisfied" - log_error "All jobs have DependencyNeverSatisfied - cancelling all jobs" - scancel -u ${USER} - job_count=0 + log_error "All remaining jobs have DependencyNeverSatisfied -- cancelling." + scancel -u "${USER}" + return 1 fi fi prev_failed_count=$failed_count @@ -209,173 +222,185 @@ wait_for_slurm_jobs() { return 0 fi - if [ $elapsed -ge $max_wait ]; then - log_error "Timeout waiting for SLURM jobs after ${max_wait}s" + if [ "$elapsed" -ge "$max_wait" ]; then + log_error "Timeout after ${max_wait}s waiting for SLURM jobs" return 1 fi - echo -ne "\r${YELLOW}Jobs remaining: $job_count${NC} (elapsed: ${elapsed}s)" + echo -ne "\r${YELLOW}Jobs remaining: $job_count${NC} (elapsed: ${elapsed}s / max: ${max_wait}s)" sleep "$check_interval" elapsed=$((elapsed + check_interval)) done - echo "" # New line after progress indicator + echo "" } +# Grep status files in a directory for any non-OK lines. +# Returns 0 if all OK, 1 if any failures found. check_status_files() { local dir="$1" local name="$2" if [ ! -d "$dir" ]; then - log_warning "Directory not found: $dir" + log_warning "$name: Directory not found: $dir" return 1 fi - cd "$dir" - local errors=$(grep -v "OK" *status 2>/dev/null || true) + local errors + errors=$(grep -v "OK" "${dir}"/*status 2>/dev/null || true) if [ -z "$errors" ]; then - log_success "$name: No errors found in ${dir}" + log_success "$name: All status files OK in ${dir}" return 0 else - log_error "$name: Errors found in ${dir}!" + log_error "$name: Non-OK statuses found in ${dir}:" echo "$errors" return 1 fi } # ============================================================================ -# Phase 1: Setup +# Phase 1: Environment Setup + Initial SLURM Jobs # ============================================================================ phase_1_setup() { log "=========================================" log "Phase 1: Setup" - log "Date: $DATE_STAMP" - log "Unique ID: $UNIQUE_ID" + log "Date stamp: $DATE_STAMP" + log "Unique ID: $UNIQUE_ID" + log "Skip mpas: $SKIP_MPAS" log "=========================================" - # ==================================================================== - # Set up e3sm_diags environment - # ==================================================================== + # ------------------------------------------------------------------ + # e3sm_diags + # ------------------------------------------------------------------ log "Setting up e3sm_diags environment..." cd "$E3SM_DIAGS_DIR" - ensure_test_branch test_e3sm_diags_${TAG} ${DIAGS_BASE_BRANCH} + ensure_test_branch "test_e3sm_diags_${TAG}" "$DIAGS_BASE_BRANCH" log "Latest e3sm_diags commit (should match https://github.com/E3SM-Project/e3sm_diags/commits/${DIAGS_BASE_BRANCH}):" git log -1 --oneline setup_conda_env "conda-env" "$DIAGS_ENV" - # ==================================================================== - # Set up zppy-interfaces environment - # ==================================================================== + # ------------------------------------------------------------------ + # zppy-interfaces (includes unit tests) + # ------------------------------------------------------------------ log "Setting up zppy-interfaces environment..." cd "$ZPPY_INTERFACES_DIR" - ensure_test_branch test_zi_${TAG} ${ZI_BASE_BRANCH} + ensure_test_branch "test_zi_${TAG}" "$ZI_BASE_BRANCH" log "Latest zppy-interfaces commit (should match https://github.com/E3SM-Project/zppy-interfaces/commits/${ZI_BASE_BRANCH}):" git log -1 --oneline setup_conda_env "conda" "$ZI_ENV" - # Run unit tests - log "Running pytest unit tests..." + log "Running zppy-interfaces unit tests..." pytest tests/unit/global_time_series/test_*.py pytest tests/unit/pcmdi_diags/test_*.py log_success "zppy-interfaces unit tests passed" - # ======================================================================== - # Set up zppy environment - # ======================================================================== + # ------------------------------------------------------------------ + # zppy (includes unit tests + config generation) + # ------------------------------------------------------------------ log "Setting up zppy environment..." cd "$ZPPY_DIR" - ensure_test_branch test_zppy_${TAG} ${ZPPY_BASE_BRANCH} + ensure_test_branch "test_zppy_${TAG}" "$ZPPY_BASE_BRANCH" log "Latest zppy commit (should match https://github.com/E3SM-Project/zppy/commits/${ZPPY_BASE_BRANCH}):" git log -1 --oneline setup_conda_env "conda" "$ZPPY_ENV" - # Run unit tests - log "Running pytest unit tests..." + log "Running zppy unit tests..." pytest tests/test_*.py log_success "zppy unit tests passed" - # ======================================================================== - # Generate config files - # ======================================================================== + # ------------------------------------------------------------------ + # Generate config files (update utils.py TEST_SPECIFICS, then run it) + # ------------------------------------------------------------------ log "Generating config files..." - # Update utils.py with test specifics UTILS_FILE="tests/integration/utils.py" - # Create a temporary Python script to update TEST_SPECIFICS - cat > /tmp/update_utils.py << EOF -import re + # Build tasks_to_run list, optionally skipping mpas_analysis + if [ "$SKIP_MPAS" = true ]; then + log_warning "SKIP_MPAS=true: omitting mpas_analysis from tasks_to_run" + TASKS_TO_RUN='["e3sm_diags", "global_time_series", "ilamb", "livvkit", "pcmdi_diags"]' + MPAS_ENV_CMD='# mpas_analysis skipped' + else + TASKS_TO_RUN='["e3sm_diags", "mpas_analysis", "global_time_series", "ilamb", "livvkit", "pcmdi_diags"]' + MPAS_ENV_CMD="source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" + fi -utils_file = "${UTILS_FILE}" + python - < Date: Wed, 10 Jun 2026 16:33:35 -0500 Subject: [PATCH 10/38] Undo skip mpas_analysis --- tests/main_branch_testing/README.md | 4 --- .../run_integration_test.bash | 26 +++---------------- 2 files changed, 3 insertions(+), 27 deletions(-) diff --git a/tests/main_branch_testing/README.md b/tests/main_branch_testing/README.md index 25ef6f44..6f4cdf82 100644 --- a/tests/main_branch_testing/README.md +++ b/tests/main_branch_testing/README.md @@ -31,7 +31,6 @@ Open `run_integration_test.bash` and edit the **"Check these every time"** block | `DIAGS_BASE_BRANCH` | Branch to test for e3sm_diags (usually `main`) | | `ZI_BASE_BRANCH` | Branch to test for zppy-interfaces (usually `main`) | | `ZPPY_BASE_BRANCH` | Branch to test for zppy (usually `main`) | -| `SKIP_MPAS` | Set to `true` to omit mpas_analysis (workaround for known segfault) | The **"Set these up once"** block below that contains machine paths (`EZ_DIR`, `CONDA_PROFILE`, `OUTPUT_WORKSPACE`) which typically don't change between runs. @@ -104,9 +103,6 @@ Check the error message. The script uses `set -e`, so it exits on any error. Com - A SLURM timeout (increase the max-wait argument to `wait_for_slurm_jobs`) - `DependencyNeverSatisfied` on all queued jobs (check your cfg files and SLURM account) -### mpas_analysis Segfault -Set `SKIP_MPAS=true` in the configuration section. This removes `mpas_analysis` from `tasks_to_run` and sets a placeholder for `mpas_analysis_environment_commands` so it cannot be accidentally invoked. - ### KeyError on Bundles Part 2 Phase 2 now checks bundle status files before resubmitting and warns if any are non-OK. Resolve any failures in the Phase 1 bundle runs before proceeding. You can restart from Phase 2 once the underlying jobs are clean: ```bash diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index daf9f4e7..98adcc7b 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -13,7 +13,6 @@ # # Notes: # - test_images.py must be run manually from a compute node (see Phase 3 output). -# - mpas_analysis is skipped by default due to known segfault issues; see SKIP_MPAS below. # - If Bundles Part 2 fails with a KeyError, re-run with --phase 2 after confirming # that the bundle status files are all "OK". @@ -48,10 +47,6 @@ DIAGS_BASE_BRANCH="main" ZI_BASE_BRANCH="main" ZPPY_BASE_BRANCH="main" -# Set to true to skip mpas_analysis due to known segfault issues on some machines. -# When true, "mpas_analysis" is removed from tasks_to_run in utils.py. -SKIP_MPAS=false - # --- Set these up once ------------------------------------------------------- HOME_DIR="$HOME" @@ -267,7 +262,6 @@ phase_1_setup() { log "Phase 1: Setup" log "Date stamp: $DATE_STAMP" log "Unique ID: $UNIQUE_ID" - log "Skip mpas: $SKIP_MPAS" log "=========================================" # ------------------------------------------------------------------ @@ -319,30 +313,17 @@ phase_1_setup() { UTILS_FILE="tests/integration/utils.py" - # Build tasks_to_run list, optionally skipping mpas_analysis - if [ "$SKIP_MPAS" = true ]; then - log_warning "SKIP_MPAS=true: omitting mpas_analysis from tasks_to_run" - TASKS_TO_RUN='["e3sm_diags", "global_time_series", "ilamb", "livvkit", "pcmdi_diags"]' - MPAS_ENV_CMD='# mpas_analysis skipped' - else - TASKS_TO_RUN='["e3sm_diags", "mpas_analysis", "global_time_series", "ilamb", "livvkit", "pcmdi_diags"]' - MPAS_ENV_CMD="source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" - fi - python - < Date: Wed, 10 Jun 2026 16:53:54 -0500 Subject: [PATCH 11/38] Make script machine independent --- tests/main_branch_testing/README.md | 12 ++-- .../run_integration_test.bash | 58 ++++++++++++++----- 2 files changed, 51 insertions(+), 19 deletions(-) diff --git a/tests/main_branch_testing/README.md b/tests/main_branch_testing/README.md index 6f4cdf82..b0da91be 100644 --- a/tests/main_branch_testing/README.md +++ b/tests/main_branch_testing/README.md @@ -6,18 +6,18 @@ This automation system streamlines the zppy integration testing workflow, reduci ### Basic Usage (Interactive Mode) ```bash -./run_integration_test.bash +./run_integration_test.bash --machine chrysalis ``` ### Skip Straight to a Later Phase ```bash -./run_integration_test.bash --phase 2 # Start at Bundles Part 2 -./run_integration_test.bash --phase 3 # Start at Validation only +./run_integration_test.bash --machine chrysalis --phase 2 +./run_integration_test.bash --machine chrysalis --phase 3 ``` ### Non-Interactive (Auto) Mode ```bash -./run_integration_test.bash --auto +./run_integration_test.bash --machine chrysalis --auto ``` In auto mode all checkpoints are bypassed and the script runs end-to-end without waiting for user input. @@ -32,7 +32,7 @@ Open `run_integration_test.bash` and edit the **"Check these every time"** block | `ZI_BASE_BRANCH` | Branch to test for zppy-interfaces (usually `main`) | | `ZPPY_BASE_BRANCH` | Branch to test for zppy (usually `main`) | -The **"Set these up once"** block below that contains machine paths (`EZ_DIR`, `CONDA_PROFILE`, `OUTPUT_WORKSPACE`) which typically don't change between runs. +The **"Set these up once"** block below that contains paths (`EZ_DIR`, `CONDA_PROFILE`) which typically don't change between runs. Machine-specific settings (`OUTPUT_WORKSPACE`, conda activation command, unified environment path, `salloc` command) are derived automatically from `--machine`. ## Workflow Phases @@ -170,7 +170,7 @@ emacs run_integration_test.bash # Run inside a screen session so it survives disconnects screen cd ~/ez/zppy_main_branch_tests/test_YYYYMMDD -time ./run_integration_test.bash --auto 2>&1 | tee integration_test.log +time ./run_integration_test.bash --machine chrysalis --auto 2>&1 | tee integration_test.log # Ctrl-A D to detach from screen # Monitor progress from another terminal diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index 98adcc7b..c5beb99f 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -4,7 +4,8 @@ # Usage: # 1. Copy this file OUT of the zppy repo (this script will change branches). # 2. Edit the "Check these every time" configuration section below. -# 3. Run: ./run_integration_test.bash [--phase N] [--auto] +# 3. Run: ./run_integration_test.bash --machine MACHINE [--phase N] [--auto] +# MACHINE: chrysalis | compy | perlmutter # # Phases: # 1 - Full setup: build envs, run unit tests, generate configs, submit SLURM jobs @@ -25,15 +26,22 @@ set -u # Exit on undefined variable AUTO_MODE=false START_PHASE=1 +MACHINE="" while [[ $# -gt 0 ]]; do case "$1" in - --auto) AUTO_MODE=true; shift ;; - --phase) START_PHASE="$2"; shift 2 ;; + --auto) AUTO_MODE=true; shift ;; + --phase) START_PHASE="$2"; shift 2 ;; + --machine) MACHINE="$2"; shift 2 ;; *) echo "Unknown argument: $1"; exit 1 ;; esac done +if [[ -z "$MACHINE" ]]; then + echo "Error: --machine is required. Valid values: chrysalis | compy | perlmutter" + exit 1 +fi + # ============================================================================ # Configuration # ============================================================================ @@ -55,7 +63,33 @@ E3SM_DIAGS_DIR="$EZ_DIR/e3sm_diags" ZPPY_INTERFACES_DIR="$EZ_DIR/zppy-interfaces" ZPPY_DIR="$EZ_DIR/zppy" CONDA_PROFILE="$HOME_DIR/miniforge3/etc/profile.d/conda.sh" -OUTPUT_WORKSPACE="/lcrc/group/e3sm/${USER}" # Chrysalis default; edit for Compy/Perlmutter + +# --- Machine-specific settings ----------------------------------------------- + +case "$MACHINE" in + chrysalis) + OUTPUT_WORKSPACE="/lcrc/group/e3sm/${USER}" + CONDA_ACTIVATION_CMD="lcrc_conda" + UNIFIED_ENV_CMD="source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" + SALLOC_CMD="salloc --nodes=1 --partition=debug --time=02:00:00 --account=e3sm" + ;; + compy) + OUTPUT_WORKSPACE="/compyfs/${USER}" + CONDA_ACTIVATION_CMD="compy_conda" + UNIFIED_ENV_CMD="source /share/apps/E3SM/conda_envs/load_latest_e3sm_unified_compy.sh" + SALLOC_CMD="salloc --nodes=1 --partition=short --time=01:00:00 --account=e3sm" + ;; + perlmutter) + OUTPUT_WORKSPACE="/global/cfs/cdirs/e3sm/${USER}" + CONDA_ACTIVATION_CMD="nersc_conda" + UNIFIED_ENV_CMD="source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh" + SALLOC_CMD="salloc --nodes=1 --qos=interactive --time=01:00:00 --constraint=cpu --account=e3sm" + ;; + *) + echo "Error: Unknown machine '$MACHINE'. Valid values: chrysalis | compy | perlmutter" + exit 1 + ;; +esac # --- Derived (probably no edits needed) -------------------------------------- @@ -121,7 +155,7 @@ activate_env() { set +u # shellcheck disable=SC1090 source ~/.bashrc - lcrc_conda # Defined in ~/.bashrc for Chrysalis; adjust for Compy/Perlmutter + $CONDA_ACTIVATION_CMD # Machine-specific conda init (lcrc_conda / compy_conda / nersc_conda) if [ -n "$env_name" ]; then conda activate "$env_name" @@ -323,11 +357,11 @@ with open(utils_file, 'r') as f: replacement = '''TEST_SPECIFICS: Dict[str, Any] = { "nco_path": "", "diags_environment_commands": "source ${CONDA_PROFILE}; conda activate ${DIAGS_ENV}", - "mpas_analysis_environment_commands": "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh", + "mpas_analysis_environment_commands": "${UNIFIED_ENV_CMD}", "global_time_series_environment_commands": "source ${CONDA_PROFILE}; conda activate ${ZI_ENV}", - "livvkit_environment_commands": "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh", + "livvkit_environment_commands": "${UNIFIED_ENV_CMD}", "pcmdi_diags_environment_commands": "source ${CONDA_PROFILE}; conda activate ${ZI_ENV}", - "environment_commands": "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh", + "environment_commands": "${UNIFIED_ENV_CMD}", "cfgs_to_run": [ "weekly_bundles", "weekly_comprehensive_v2", @@ -494,16 +528,13 @@ phase_3_validation() { # test_images.py -- must run from a compute node # ------------------------------------------------------------------ log_warning "test_images.py requires a compute node and must be run manually." - log "To run it on Chrysalis:" - log " salloc --nodes=1 --partition=debug --time=02:00:00 --account=e3sm" + log "To run it on ${MACHINE}:" + log " ${SALLOC_CMD}" log " source ${CONDA_PROFILE}" log " conda activate ${ZPPY_ENV}" log " cd ${ZPPY_DIR}" log " pytest tests/integration/test_images.py" log " cat test_images_summary.md" - log "" - log "On Compy: salloc --nodes=1 --partition=short --time=01:00:00 --account=e3sm" - log "On Perlmutter: salloc --nodes=1 --qos=interactive --time=01:00:00 --constraint=cpu --account=e3sm" log_success "Phase 3 automated tests complete!" log_success "Remember to run test_images.py manually from a compute node." @@ -515,6 +546,7 @@ phase_3_validation() { main() { log "Starting zppy integration test automation" + log "Machine: $MACHINE" log "Date stamp: $DATE_STAMP" log "Auto mode: $AUTO_MODE" log "Start phase: $START_PHASE" From 23788be805f9f5757b38979048a2dffd5e3367ce Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 10 Jun 2026 17:01:39 -0500 Subject: [PATCH 12/38] Revisions --- tests/main_branch_testing/README.md | 7 +++---- tests/main_branch_testing/run_integration_test.bash | 9 +++------ 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/tests/main_branch_testing/README.md b/tests/main_branch_testing/README.md index b0da91be..b3512971 100644 --- a/tests/main_branch_testing/README.md +++ b/tests/main_branch_testing/README.md @@ -45,7 +45,7 @@ The **"Set these up once"** block below that contains paths (`EZ_DIR`, `CONDA_PR - Waits for jobs to complete (polls every 10 min, 4-hour max) ### Phase 2: Bundles Part 2 -- Checks status files for all three bundles output directories; warns if any are non-OK before proceeding (a non-OK status here is the likely cause of the historical KeyError) +- Checks status files for all three bundles output directories; warns if any are non-OK before proceeding - Submits bundles part 2 jobs (`weekly_bundles`, `legacy_3.1.0_bundles`, `legacy_3.0.0_bundles`) - Waits for completion (polls every 10 min, 1-hour max) @@ -103,8 +103,7 @@ Check the error message. The script uses `set -e`, so it exits on any error. Com - A SLURM timeout (increase the max-wait argument to `wait_for_slurm_jobs`) - `DependencyNeverSatisfied` on all queued jobs (check your cfg files and SLURM account) -### KeyError on Bundles Part 2 -Phase 2 now checks bundle status files before resubmitting and warns if any are non-OK. Resolve any failures in the Phase 1 bundle runs before proceeding. You can restart from Phase 2 once the underlying jobs are clean: +Phase 2 checks bundle status files before resubmitting and warns if any are non-OK. Resolve any failures in the Phase 1 bundle runs before proceeding. You can restart from Phase 2 once the underlying jobs are clean: ```bash ./run_integration_test.bash --phase 2 ``` @@ -167,7 +166,7 @@ cp ~/ez/zppy/tests/main_branch_testing/* . # Edit configuration parameters emacs run_integration_test.bash -# Run inside a screen session so it survives disconnects +# Run inside a screen session so it will survive disconnections. screen cd ~/ez/zppy_main_branch_tests/test_YYYYMMDD time ./run_integration_test.bash --machine chrysalis --auto 2>&1 | tee integration_test.log diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index c5beb99f..8765c4cd 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -14,8 +14,6 @@ # # Notes: # - test_images.py must be run manually from a compute node (see Phase 3 output). -# - If Bundles Part 2 fails with a KeyError, re-run with --phase 2 after confirming -# that the bundle status files are all "OK". set -e # Exit on error set -u # Exit on undefined variable @@ -112,12 +110,12 @@ V3_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_comprehensive_v3_output/${UNIQUE_ID}/ LEGACY_310_V3_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_legacy_3.1.0_comprehensive_v3_output/${UNIQUE_ID}/v3.LR.historical_0051/post/scripts" LEGACY_300_V3_OUTPUT="${OUTPUT_WORKSPACE}/zppy_weekly_legacy_3.0.0_comprehensive_v3_output/${UNIQUE_ID}/v3.LR.historical_0051/post/scripts" -# Colors +# Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' -NC='\033[0m' +NC='\033[0m' # No Color # ============================================================================ # Helper Functions @@ -434,7 +432,6 @@ phase_2_bundles_part2() { ensure_test_branch "test_zppy_${TAG}" "$ZPPY_BASE_BRANCH" # Verify all bundle status files are clean before submitting part 2. - # A non-OK status here is the likely cause of the historical KeyError. log "Checking bundle status files before submitting part 2..." local all_ok=true check_status_files "$BUNDLES_OUTPUT" "Bundles" || all_ok=false @@ -443,7 +440,7 @@ phase_2_bundles_part2() { if [ "$all_ok" = false ]; then log_error "One or more bundle status files have non-OK entries." - checkpoint "Errors found. Proceeding may cause a KeyError in bundles part 2. Continue anyway?" + checkpoint "Errors found. Continue anyway?" else log_success "Bundle status files look clean -- safe to submit part 2." fi From db2b20355dbd102f14d94bd71a7b212711229f24 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 10 Jun 2026 17:35:52 -0500 Subject: [PATCH 13/38] Improve environment setup --- tests/main_branch_testing/README.md | 14 +- .../run_integration_test.bash | 122 +++++++++++++++--- 2 files changed, 113 insertions(+), 23 deletions(-) diff --git a/tests/main_branch_testing/README.md b/tests/main_branch_testing/README.md index b3512971..386d2752 100644 --- a/tests/main_branch_testing/README.md +++ b/tests/main_branch_testing/README.md @@ -29,15 +29,21 @@ Open `run_integration_test.bash` and edit the **"Check these every time"** block | --- | --- | | `RUN_NUMBER` | Increment this if you run multiple tests on the same day | | `DIAGS_BASE_BRANCH` | Branch to test for e3sm_diags (usually `main`) | +| `E3SM_TO_CMIP_BASE_BRANCH` | Branch to test for e3sm_to_cmip (usually `master`) | +| `MPAS_BASE_BRANCH` | Branch to test for MPAS-Analysis (usually `develop`) | | `ZI_BASE_BRANCH` | Branch to test for zppy-interfaces (usually `main`) | | `ZPPY_BASE_BRANCH` | Branch to test for zppy (usually `main`) | +| `DIAGS_ENV_TYPE` | `"dev"` to build a dedicated conda env; `"unified"` to use e3sm-unified | +| `E3SM_TO_CMIP_ENV_TYPE` | `"dev"` to build a dedicated conda env; `"unified"` to use e3sm-unified | +| `MPAS_ENV_TYPE` | `"dev"` to build a dedicated conda env; `"unified"` to use e3sm-unified | +| `ZI_ENV_TYPE` | `"dev"` to build a dedicated conda env; `"unified"` to use e3sm-unified | The **"Set these up once"** block below that contains paths (`EZ_DIR`, `CONDA_PROFILE`) which typically don't change between runs. Machine-specific settings (`OUTPUT_WORKSPACE`, conda activation command, unified environment path, `salloc` command) are derived automatically from `--machine`. ## Workflow Phases ### Phase 1: Setup -- Creates conda environments for e3sm_diags, zppy-interfaces, and zppy +- Creates conda environments for each component where `ENV_TYPE="dev"` (e3sm_to_cmip, e3sm_diags, MPAS-Analysis, zppy-interfaces, zppy); skips env creation and uses e3sm-unified for any component where `ENV_TYPE="unified"` - Runs unit tests for zppy-interfaces and zppy - Patches `tests/integration/utils.py` with test-specific environment commands, config list, and unique ID - Generates config files via `python tests/integration/utils.py` @@ -110,8 +116,10 @@ Phase 2 checks bundle status files before resubmitting and warns if any are non- ### Environment Issues ```bash -# Remove and rebuild stale environments +# Remove and rebuild stale environments (only applies to components with ENV_TYPE="dev") +conda remove --yes --all --name test-e3sm-to-cmip-master-YYYYMMDD_runN conda remove --yes --all --name test-diags-main-YYYYMMDD_runN +conda remove --yes --all --name test-mpas-develop-YYYYMMDD_runN conda remove --yes --all --name test-zi-main-YYYYMMDD_runN conda remove --yes --all --name test-zppy-main-YYYYMMDD_runN @@ -151,7 +159,9 @@ conda remove --yes --all --name test-zppy-main-YYYYMMDD_runN ```bash # Confirm repos have no uncommitted changes +cd ~/ez/e3sm_to_cmip && git status cd ~/ez/e3sm_diags && git status +cd ~/ez/MPAS-Analysis && git status cd ~/ez/zppy-interfaces && git status cd ~/ez/zppy && git status diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index 8765c4cd..d4218a10 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -48,16 +48,28 @@ fi RUN_NUMBER=1 -# Base branches (what we're testing -- usually "main") +# Base branches (what we're testing -- usually "main"/"master"/"develop") DIAGS_BASE_BRANCH="main" +E3SM_TO_CMIP_BASE_BRANCH="master" +MPAS_BASE_BRANCH="develop" ZI_BASE_BRANCH="main" ZPPY_BASE_BRANCH="main" +# Dev vs unified env per component. +# "dev" = build a dedicated conda env from the repo's dev.yml +# "unified" = use the machine's e3sm-unified env (UNIFIED_ENV_CMD) +DIAGS_ENV_TYPE="dev" +E3SM_TO_CMIP_ENV_TYPE="dev" +MPAS_ENV_TYPE="unified" +ZI_ENV_TYPE="dev" + # --- Set these up once ------------------------------------------------------- HOME_DIR="$HOME" -EZ_DIR="$HOME_DIR/ez" # Parent dir for all repos +EZ_DIR="$HOME_DIR/ez" # Parent dir for all repos E3SM_DIAGS_DIR="$EZ_DIR/e3sm_diags" +E3SM_TO_CMIP_DIR="$EZ_DIR/e3sm_to_cmip" +MPAS_ANALYSIS_DIR="$EZ_DIR/MPAS-Analysis" ZPPY_INTERFACES_DIR="$EZ_DIR/zppy-interfaces" ZPPY_DIR="$EZ_DIR/zppy" CONDA_PROFILE="$HOME_DIR/miniforge3/etc/profile.d/conda.sh" @@ -95,8 +107,6 @@ DATE_STAMP="${DATE_STAMP:-$(date +%Y%m%d)}" TAG="${DATE_STAMP}_run${RUN_NUMBER}" UNIQUE_ID="zppy_main_branch_test_${TAG}" -DIAGS_ENV="test-diags-${DIAGS_BASE_BRANCH}-${TAG}" -ZI_ENV="test-zi-${ZI_BASE_BRANCH}-${TAG}" ZPPY_ENV="test-zppy-${ZPPY_BASE_BRANCH}-${TAG}" # Output directories (status file locations) @@ -211,6 +221,19 @@ ensure_test_branch() { log_success "On branch '$test_branch'" } +# Return the environment_commands string for a component. +# Usage: get_env_cmd "dev" "$ENV_NAME" +# get_env_cmd "unified" "" +get_env_cmd() { + local env_type="$1" + local env_name="$2" + if [[ "$env_type" == "dev" ]]; then + echo "source ${CONDA_PROFILE}; conda activate ${env_name}" + else + echo "$UNIFIED_ENV_CMD" + fi +} + # Poll squeue until no user jobs remain (or timeout). wait_for_slurm_jobs() { local check_interval=${1:-600} # seconds between checks (default 10 min) @@ -296,27 +319,77 @@ phase_1_setup() { log "Unique ID: $UNIQUE_ID" log "=========================================" + # ------------------------------------------------------------------ + # e3sm_to_cmip + # ------------------------------------------------------------------ + log "Setting up e3sm_to_cmip..." + cd "$E3SM_TO_CMIP_DIR" + ensure_test_branch "test_e3sm_to_cmip_${TAG}" "$E3SM_TO_CMIP_BASE_BRANCH" + + log "Latest e3sm_to_cmip commit (should match https://github.com/E3SM-Project/e3sm_to_cmip/commits/${E3SM_TO_CMIP_BASE_BRANCH}):" + git log -1 --oneline + + local E3SM_TO_CMIP_ENV="" + if [[ "$E3SM_TO_CMIP_ENV_TYPE" == "dev" ]]; then + E3SM_TO_CMIP_ENV="test-e3sm-to-cmip-${E3SM_TO_CMIP_BASE_BRANCH}-${TAG}" + setup_conda_env "conda" "$E3SM_TO_CMIP_ENV" + else + log "Using unified env for e3sm_to_cmip (skipping conda env creation)" + fi + # ------------------------------------------------------------------ # e3sm_diags # ------------------------------------------------------------------ - log "Setting up e3sm_diags environment..." + log "Setting up e3sm_diags..." cd "$E3SM_DIAGS_DIR" ensure_test_branch "test_e3sm_diags_${TAG}" "$DIAGS_BASE_BRANCH" log "Latest e3sm_diags commit (should match https://github.com/E3SM-Project/e3sm_diags/commits/${DIAGS_BASE_BRANCH}):" git log -1 --oneline - setup_conda_env "conda-env" "$DIAGS_ENV" + + local DIAGS_ENV="" + if [[ "$DIAGS_ENV_TYPE" == "dev" ]]; then + DIAGS_ENV="test-diags-${DIAGS_BASE_BRANCH}-${TAG}" + setup_conda_env "conda-env" "$DIAGS_ENV" + else + log "Using unified env for e3sm_diags (skipping conda env creation)" + fi + + # ------------------------------------------------------------------ + # MPAS-Analysis + # ------------------------------------------------------------------ + log "Setting up MPAS-Analysis..." + cd "$MPAS_ANALYSIS_DIR" + ensure_test_branch "test_mpas_${TAG}" "$MPAS_BASE_BRANCH" + + log "Latest MPAS-Analysis commit (should match https://github.com/MPAS-Dev/MPAS-Analysis/commits/${MPAS_BASE_BRANCH}):" + git log -1 --oneline + + local MPAS_ENV="" + if [[ "$MPAS_ENV_TYPE" == "dev" ]]; then + MPAS_ENV="test-mpas-${MPAS_BASE_BRANCH}-${TAG}" + setup_conda_env "conda" "$MPAS_ENV" + else + log "Using unified env for MPAS-Analysis (skipping conda env creation)" + fi # ------------------------------------------------------------------ # zppy-interfaces (includes unit tests) # ------------------------------------------------------------------ - log "Setting up zppy-interfaces environment..." + log "Setting up zppy-interfaces..." cd "$ZPPY_INTERFACES_DIR" ensure_test_branch "test_zi_${TAG}" "$ZI_BASE_BRANCH" log "Latest zppy-interfaces commit (should match https://github.com/E3SM-Project/zppy-interfaces/commits/${ZI_BASE_BRANCH}):" git log -1 --oneline - setup_conda_env "conda" "$ZI_ENV" + + local ZI_ENV="" + if [[ "$ZI_ENV_TYPE" == "dev" ]]; then + ZI_ENV="test-zi-${ZI_BASE_BRANCH}-${TAG}" + setup_conda_env "conda" "$ZI_ENV" + else + log "Using unified env for zppy-interfaces (skipping conda env creation)" + fi log "Running zppy-interfaces unit tests..." pytest tests/unit/global_time_series/test_*.py @@ -326,7 +399,7 @@ phase_1_setup() { # ------------------------------------------------------------------ # zppy (includes unit tests + config generation) # ------------------------------------------------------------------ - log "Setting up zppy environment..." + log "Setting up zppy..." cd "$ZPPY_DIR" ensure_test_branch "test_zppy_${TAG}" "$ZPPY_BASE_BRANCH" @@ -343,6 +416,13 @@ phase_1_setup() { # ------------------------------------------------------------------ log "Generating config files..." + local DIAGS_CMD + local MPAS_CMD + local ZI_CMD + DIAGS_CMD=$(get_env_cmd "$DIAGS_ENV_TYPE" "$DIAGS_ENV") + MPAS_CMD=$(get_env_cmd "$MPAS_ENV_TYPE" "$MPAS_ENV") + ZI_CMD=$(get_env_cmd "$ZI_ENV_TYPE" "$ZI_ENV") + UTILS_FILE="tests/integration/utils.py" python - < Date: Fri, 12 Jun 2026 09:18:43 -0700 Subject: [PATCH 14/38] Fixes made while testing phase 1 --- .../run_integration_test.bash | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index d4218a10..ff4e661e 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -60,7 +60,7 @@ ZPPY_BASE_BRANCH="main" # "unified" = use the machine's e3sm-unified env (UNIFIED_ENV_CMD) DIAGS_ENV_TYPE="dev" E3SM_TO_CMIP_ENV_TYPE="dev" -MPAS_ENV_TYPE="unified" +MPAS_ENV_TYPE="dev" ZI_ENV_TYPE="dev" # --- Set these up once ------------------------------------------------------- @@ -186,7 +186,11 @@ setup_conda_env() { log "Creating environment '$env_name' from ${conda_dir}/dev.yml..." rm -rf build conda clean --all --yes - conda env create -f "${conda_dir}/dev.yml" -n "$env_name" + if [[ "$conda_dir" == "none" ]]; then + conda create --name "$env_name" --file dev-spec.txt --yes + else + conda env create -f "${conda_dir}/dev.yml" -n "$env_name" + fi fi activate_env "$env_name" @@ -274,6 +278,7 @@ wait_for_slurm_jobs() { if [ "$elapsed" -ge "$max_wait" ]; then log_error "Timeout after ${max_wait}s waiting for SLURM jobs" + log_error "This script is going to exit now. However, the jobs in the queue will NOT be terminated. Once they finish, you may re-invoke this script with --phase 2 or --phase 3 to continue." return 1 fi @@ -332,7 +337,7 @@ phase_1_setup() { local E3SM_TO_CMIP_ENV="" if [[ "$E3SM_TO_CMIP_ENV_TYPE" == "dev" ]]; then E3SM_TO_CMIP_ENV="test-e3sm-to-cmip-${E3SM_TO_CMIP_BASE_BRANCH}-${TAG}" - setup_conda_env "conda" "$E3SM_TO_CMIP_ENV" + setup_conda_env "conda-env" "$E3SM_TO_CMIP_ENV" else log "Using unified env for e3sm_to_cmip (skipping conda env creation)" fi @@ -368,7 +373,7 @@ phase_1_setup() { local MPAS_ENV="" if [[ "$MPAS_ENV_TYPE" == "dev" ]]; then MPAS_ENV="test-mpas-${MPAS_BASE_BRANCH}-${TAG}" - setup_conda_env "conda" "$MPAS_ENV" + setup_conda_env "none" "$MPAS_ENV" else log "Using unified env for MPAS-Analysis (skipping conda env creation)" fi @@ -416,9 +421,11 @@ phase_1_setup() { # ------------------------------------------------------------------ log "Generating config files..." + local E3SM_TO_CMIP_CMD local DIAGS_CMD local MPAS_CMD local ZI_CMD + E3SM_TO_CMIP_CMD=$(get_env_cmd "$E3SM_TO_CMIP_ENV_TYPE" "$E3SM_TO_CMIP_ENV") DIAGS_CMD=$(get_env_cmd "$DIAGS_ENV_TYPE" "$DIAGS_ENV") MPAS_CMD=$(get_env_cmd "$MPAS_ENV_TYPE" "$MPAS_ENV") ZI_CMD=$(get_env_cmd "$ZI_ENV_TYPE" "$ZI_ENV") @@ -434,6 +441,7 @@ with open(utils_file, 'r') as f: replacement = '''TEST_SPECIFICS: Dict[str, Any] = { "nco_path": "", + "e3sm_to_cmip_environment_commands": "${E3SM_TO_CMIP_CMD}", "diags_environment_commands": "${DIAGS_CMD}", "mpas_analysis_environment_commands": "${MPAS_CMD}", "global_time_series_environment_commands": "${ZI_CMD}", From a1e289d638500151759ddecb0abd3408ef876e91 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Fri, 12 Jun 2026 09:29:23 -0700 Subject: [PATCH 15/38] Fix tag logic --- .../run_integration_test.bash | 48 +++++++++++++++++-- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index ff4e661e..9fae4801 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -4,7 +4,7 @@ # Usage: # 1. Copy this file OUT of the zppy repo (this script will change branches). # 2. Edit the "Check these every time" configuration section below. -# 3. Run: ./run_integration_test.bash --machine MACHINE [--phase N] [--auto] +# 3. Run: ./run_integration_test.bash --machine MACHINE [--phase N] [--tag TAG] [--auto] # MACHINE: chrysalis | compy | perlmutter # # Phases: @@ -14,6 +14,8 @@ # # Notes: # - test_images.py must be run manually from a compute node (see Phase 3 output). +# - If you need to resume from Phase 2 or 3 on a later day, pass --tag with the +# TAG printed at the start of Phase 1 (or stored in ~/.zppy_test_tag). set -e # Exit on error set -u # Exit on undefined variable @@ -25,12 +27,14 @@ set -u # Exit on undefined variable AUTO_MODE=false START_PHASE=1 MACHINE="" +EXPLICIT_TAG="" while [[ $# -gt 0 ]]; do case "$1" in --auto) AUTO_MODE=true; shift ;; --phase) START_PHASE="$2"; shift 2 ;; --machine) MACHINE="$2"; shift 2 ;; + --tag) EXPLICIT_TAG="$2"; shift 2 ;; *) echo "Unknown argument: $1"; exit 1 ;; esac done @@ -74,6 +78,9 @@ ZPPY_INTERFACES_DIR="$EZ_DIR/zppy-interfaces" ZPPY_DIR="$EZ_DIR/zppy" CONDA_PROFILE="$HOME_DIR/miniforge3/etc/profile.d/conda.sh" +# File used to persist the TAG across separate invocations (e.g. phase 2/3 next day). +TAG_CACHE_FILE="$HOME_DIR/.zppy_test_tag" + # --- Machine-specific settings ----------------------------------------------- case "$MACHINE" in @@ -101,10 +108,31 @@ case "$MACHINE" in ;; esac +# --- Resolve TAG ------------------------------------------------------------- +# +# Priority: +# 1. --tag CLI argument (explicit, always wins) +# 2. $TAG_CACHE_FILE written by a prior Phase 1 run (auto-resume) +# 3. Fresh timestamp (Phase 1 first run) +# +# Phase 1 always writes the resolved TAG to $TAG_CACHE_FILE so later phases +# can pick it up automatically without needing --tag. + +if [[ -n "$EXPLICIT_TAG" ]]; then + TAG="$EXPLICIT_TAG" + DATE_STAMP="${TAG%%_run*}" # Extract date portion for display; best-effort. +elif [[ "$START_PHASE" -gt 1 && -f "$TAG_CACHE_FILE" ]]; then + TAG="$(cat "$TAG_CACHE_FILE")" + DATE_STAMP="${TAG%%_run*}" + echo "Loaded TAG from ${TAG_CACHE_FILE}: ${TAG}" + echo "(Pass --tag ${TAG} explicitly to override.)" +else + DATE_STAMP="${DATE_STAMP:-$(date +%Y%m%d)}" + TAG="${DATE_STAMP}_run${RUN_NUMBER}" +fi + # --- Derived (probably no edits needed) -------------------------------------- -DATE_STAMP="${DATE_STAMP:-$(date +%Y%m%d)}" -TAG="${DATE_STAMP}_run${RUN_NUMBER}" UNIQUE_ID="zppy_main_branch_test_${TAG}" ZPPY_ENV="test-zppy-${ZPPY_BASE_BRANCH}-${TAG}" @@ -279,6 +307,9 @@ wait_for_slurm_jobs() { if [ "$elapsed" -ge "$max_wait" ]; then log_error "Timeout after ${max_wait}s waiting for SLURM jobs" log_error "This script is going to exit now. However, the jobs in the queue will NOT be terminated. Once they finish, you may re-invoke this script with --phase 2 or --phase 3 to continue." + log_error " TAG for this run: ${TAG}" + log_error " Resume command: $0 --machine ${MACHINE} --phase 2 --tag ${TAG}" + log_error " (TAG is also saved in ${TAG_CACHE_FILE})" return 1 fi @@ -318,10 +349,17 @@ check_status_files() { # ============================================================================ phase_1_setup() { + # Save TAG immediately so later phases can find it even if the date changes. + echo "$TAG" > "$TAG_CACHE_FILE" + log "=========================================" log "Phase 1: Setup" log "Date stamp: $DATE_STAMP" + log "TAG: $TAG (saved to ${TAG_CACHE_FILE})" log "Unique ID: $UNIQUE_ID" + log "" + log "To resume from a later phase, run:" + log " $0 --machine ${MACHINE} --phase 2 --tag ${TAG}" log "=========================================" # ------------------------------------------------------------------ @@ -513,6 +551,7 @@ PYEOF phase_2_bundles_part2() { log "=========================================" log "Phase 2: Bundles Part 2" + log "TAG: $TAG" log "=========================================" cd "$ZPPY_DIR" @@ -555,6 +594,7 @@ phase_2_bundles_part2() { phase_3_validation() { log "=========================================" log "Phase 3: Validation" + log "TAG: $TAG" log "=========================================" cd "$ZPPY_DIR" @@ -632,7 +672,7 @@ phase_3_validation() { main() { log "Starting zppy integration test automation" log "Machine: $MACHINE" - log "Date stamp: $DATE_STAMP" + log "TAG: $TAG" log "Auto mode: $AUTO_MODE" log "Start phase: $START_PHASE" From 36453b0c4d2b32ade4520c2f812ca52f6acdf75a Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Fri, 12 Jun 2026 11:41:05 -0700 Subject: [PATCH 16/38] Add config file --- tests/main_branch_testing/README.md | 52 ++++--- .../run_integration_test.bash | 137 +++++++++--------- tests/main_branch_testing/zppy_test.cfg | 48 ++++++ 3 files changed, 152 insertions(+), 85 deletions(-) create mode 100644 tests/main_branch_testing/zppy_test.cfg diff --git a/tests/main_branch_testing/README.md b/tests/main_branch_testing/README.md index 386d2752..fe6a0555 100644 --- a/tests/main_branch_testing/README.md +++ b/tests/main_branch_testing/README.md @@ -6,28 +6,31 @@ This automation system streamlines the zppy integration testing workflow, reduci ### Basic Usage (Interactive Mode) ```bash -./run_integration_test.bash --machine chrysalis +./run_integration_test.bash --config zppy_test.cfg ``` ### Skip Straight to a Later Phase +Set `START_PHASE=2` or `START_PHASE=3` in your config file, then re-run: ```bash -./run_integration_test.bash --machine chrysalis --phase 2 -./run_integration_test.bash --machine chrysalis --phase 3 +./run_integration_test.bash --config zppy_test.cfg ``` ### Non-Interactive (Auto) Mode -```bash -./run_integration_test.bash --machine chrysalis --auto -``` -In auto mode all checkpoints are bypassed and the script runs end-to-end without waiting for user input. +Set `AUTO_MODE=true` in your config file. All checkpoints are bypassed and the script runs end-to-end without waiting for user input. ## Configuration -Open `run_integration_test.bash` and edit the **"Check these every time"** block at the top before each test run: +Copy `zppy_test.cfg` and edit it before each test run. It has three sections: + +### Runtime settings (update as needed each run) | Variable | Description | | --- | --- | -| `RUN_NUMBER` | Increment this if you run multiple tests on the same day | +| `MACHINE` | `chrysalis`, `compy`, or `perlmutter` | +| `START_PHASE` | `1`, `2`, or `3` | +| `AUTO_MODE` | `true` to skip all interactive checkpoints | +| `EXPLICIT_TAG` | Leave empty to auto-generate; set to a prior TAG to resume | +| `RUN_NUMBER` | Increment if you run multiple tests on the same day | | `DIAGS_BASE_BRANCH` | Branch to test for e3sm_diags (usually `main`) | | `E3SM_TO_CMIP_BASE_BRANCH` | Branch to test for e3sm_to_cmip (usually `master`) | | `MPAS_BASE_BRANCH` | Branch to test for MPAS-Analysis (usually `develop`) | @@ -38,7 +41,21 @@ Open `run_integration_test.bash` and edit the **"Check these every time"** block | `MPAS_ENV_TYPE` | `"dev"` to build a dedicated conda env; `"unified"` to use e3sm-unified | | `ZI_ENV_TYPE` | `"dev"` to build a dedicated conda env; `"unified"` to use e3sm-unified | -The **"Set these up once"** block below that contains paths (`EZ_DIR`, `CONDA_PROFILE`) which typically don't change between runs. Machine-specific settings (`OUTPUT_WORKSPACE`, conda activation command, unified environment path, `salloc` command) are derived automatically from `--machine`. +### One-time setup (paths that rarely change) + +| Variable | Description | +| --- | --- | +| `HOME_DIR` | Your home directory (default: `$HOME`) | +| `EZ_DIR` | Parent directory for all repos (default: `$HOME/ez`) | +| `E3SM_DIAGS_DIR` | Path to e3sm_diags repo | +| `E3SM_TO_CMIP_DIR` | Path to e3sm_to_cmip repo | +| `MPAS_ANALYSIS_DIR` | Path to MPAS-Analysis repo | +| `ZPPY_INTERFACES_DIR` | Path to zppy-interfaces repo | +| `ZPPY_DIR` | Path to zppy repo | +| `CONDA_PROFILE` | Path to your conda profile script | +| `TAG_CACHE_FILE` | Where the TAG is saved between phases (default: `~/.zppy_test_tag`) | + +Machine-specific settings (`OUTPUT_WORKSPACE`, conda activation command, unified environment path, `salloc` command) are derived automatically from `MACHINE` and do not appear in the config. ## Workflow Phases @@ -109,9 +126,9 @@ Check the error message. The script uses `set -e`, so it exits on any error. Com - A SLURM timeout (increase the max-wait argument to `wait_for_slurm_jobs`) - `DependencyNeverSatisfied` on all queued jobs (check your cfg files and SLURM account) -Phase 2 checks bundle status files before resubmitting and warns if any are non-OK. Resolve any failures in the Phase 1 bundle runs before proceeding. You can restart from Phase 2 once the underlying jobs are clean: +Phase 2 checks bundle status files before resubmitting and warns if any are non-OK. Resolve any failures in the Phase 1 bundle runs before proceeding. You can restart from Phase 2 by setting `START_PHASE=2` in your config (the TAG from Phase 1 is saved in `~/.zppy_test_tag` and picked up automatically, or set `EXPLICIT_TAG` to be explicit): ```bash -./run_integration_test.bash --phase 2 +./run_integration_test.bash --config zppy_test.cfg ``` ### Environment Issues @@ -124,7 +141,7 @@ conda remove --yes --all --name test-zi-main-YYYYMMDD_runN conda remove --yes --all --name test-zppy-main-YYYYMMDD_runN # Then re-run from Phase 1 -./run_integration_test.bash +./run_integration_test.bash --config zppy_test.cfg ``` ## Files Created @@ -168,18 +185,19 @@ cd ~/ez/zppy && git status # Confirm no jobs are currently running squeue -u $USER -# Copy the script out of the repo (Phase 1 will change branches) +# Copy the script and config out of the repo (Phase 1 will change branches) mkdir -p ~/ez/zppy_main_branch_tests/test_YYYYMMDD cd ~/ez/zppy_main_branch_tests/test_YYYYMMDD -cp ~/ez/zppy/tests/main_branch_testing/* . +cp ~/ez/zppy/tests/main_branch_testing/run_integration_test.bash . +cp ~/ez/zppy/tests/main_branch_testing/zppy_test.cfg . # Edit configuration parameters -emacs run_integration_test.bash +emacs zppy_test.cfg # Run inside a screen session so it will survive disconnections. screen cd ~/ez/zppy_main_branch_tests/test_YYYYMMDD -time ./run_integration_test.bash --machine chrysalis --auto 2>&1 | tee integration_test.log +time ./run_integration_test.bash --config zppy_test.cfg 2>&1 | tee integration_test.log # Ctrl-A D to detach from screen # Monitor progress from another terminal diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index 9fae4801..cda4f727 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -2,20 +2,21 @@ # zppy Integration Test Automation Script # # Usage: -# 1. Copy this file OUT of the zppy repo (this script will change branches). -# 2. Edit the "Check these every time" configuration section below. -# 3. Run: ./run_integration_test.bash --machine MACHINE [--phase N] [--tag TAG] [--auto] -# MACHINE: chrysalis | compy | perlmutter +# 1. Copy this file AND the sample config OUT of the zppy repo +# (this script will change branches). +# 2. Edit your config file (see zppy_test.cfg.sample). +# 3. Run: ./run_integration_test.bash --config path/to/your.cfg # -# Phases: +# Phases (set START_PHASE in your config): # 1 - Full setup: build envs, run unit tests, generate configs, submit SLURM jobs # 2 - Bundles Part 2 (run after Phase 1 jobs finish) # 3 - Validation: status checks + pytest integration tests # # Notes: # - test_images.py must be run manually from a compute node (see Phase 3 output). -# - If you need to resume from Phase 2 or 3 on a later day, pass --tag with the -# TAG printed at the start of Phase 1 (or stored in ~/.zppy_test_tag). +# - To resume from Phase 2 or 3 on a later day, set EXPLICIT_TAG in your config +# to the TAG printed at the start of Phase 1 (or stored in ~/.zppy_test_tag), +# and set START_PHASE accordingly. set -e # Exit on error set -u # Exit on undefined variable @@ -24,64 +25,61 @@ set -u # Exit on undefined variable # Parse arguments # ============================================================================ -AUTO_MODE=false -START_PHASE=1 -MACHINE="" -EXPLICIT_TAG="" +CONFIG_FILE="" while [[ $# -gt 0 ]]; do case "$1" in - --auto) AUTO_MODE=true; shift ;; - --phase) START_PHASE="$2"; shift 2 ;; - --machine) MACHINE="$2"; shift 2 ;; - --tag) EXPLICIT_TAG="$2"; shift 2 ;; + --config) CONFIG_FILE="$2"; shift 2 ;; *) echo "Unknown argument: $1"; exit 1 ;; esac done -if [[ -z "$MACHINE" ]]; then - echo "Error: --machine is required. Valid values: chrysalis | compy | perlmutter" +if [[ -z "$CONFIG_FILE" ]]; then + echo "Error: --config is required." + echo "Usage: $0 --config path/to/your.cfg" exit 1 fi -# ============================================================================ -# Configuration -# ============================================================================ - -# --- Check these every time -------------------------------------------------- - -RUN_NUMBER=1 - -# Base branches (what we're testing -- usually "main"/"master"/"develop") -DIAGS_BASE_BRANCH="main" -E3SM_TO_CMIP_BASE_BRANCH="master" -MPAS_BASE_BRANCH="develop" -ZI_BASE_BRANCH="main" -ZPPY_BASE_BRANCH="main" - -# Dev vs unified env per component. -# "dev" = build a dedicated conda env from the repo's dev.yml -# "unified" = use the machine's e3sm-unified env (UNIFIED_ENV_CMD) -DIAGS_ENV_TYPE="dev" -E3SM_TO_CMIP_ENV_TYPE="dev" -MPAS_ENV_TYPE="dev" -ZI_ENV_TYPE="dev" - -# --- Set these up once ------------------------------------------------------- +if [[ ! -f "$CONFIG_FILE" ]]; then + echo "Error: Config file not found: $CONFIG_FILE" + exit 1 +fi -HOME_DIR="$HOME" -EZ_DIR="$HOME_DIR/ez" # Parent dir for all repos -E3SM_DIAGS_DIR="$EZ_DIR/e3sm_diags" -E3SM_TO_CMIP_DIR="$EZ_DIR/e3sm_to_cmip" -MPAS_ANALYSIS_DIR="$EZ_DIR/MPAS-Analysis" -ZPPY_INTERFACES_DIR="$EZ_DIR/zppy-interfaces" -ZPPY_DIR="$EZ_DIR/zppy" -CONDA_PROFILE="$HOME_DIR/miniforge3/etc/profile.d/conda.sh" +# Source the config. Variables defined there become the script's environment. +# shellcheck disable=SC1090 +source "$CONFIG_FILE" + +# Validate required config keys. +_required_vars=( + MACHINE START_PHASE AUTO_MODE EXPLICIT_TAG + RUN_NUMBER + DIAGS_BASE_BRANCH E3SM_TO_CMIP_BASE_BRANCH MPAS_BASE_BRANCH ZI_BASE_BRANCH ZPPY_BASE_BRANCH + DIAGS_ENV_TYPE E3SM_TO_CMIP_ENV_TYPE MPAS_ENV_TYPE ZI_ENV_TYPE + HOME_DIR EZ_DIR + E3SM_DIAGS_DIR E3SM_TO_CMIP_DIR MPAS_ANALYSIS_DIR ZPPY_INTERFACES_DIR ZPPY_DIR + CONDA_PROFILE TAG_CACHE_FILE +) +_missing=() +for _var in "${_required_vars[@]}"; do + if [[ -z "${!_var+x}" ]]; then + _missing+=("$_var") + fi +done +if [[ ${#_missing[@]} -gt 0 ]]; then + echo "Error: The following required variables are missing from ${CONFIG_FILE}:" + printf ' %s\n' "${_missing[@]}" + exit 1 +fi -# File used to persist the TAG across separate invocations (e.g. phase 2/3 next day). -TAG_CACHE_FILE="$HOME_DIR/.zppy_test_tag" +# Validate MACHINE value. +case "$MACHINE" in + chrysalis|compy|perlmutter) ;; + *) echo "Error: Unknown MACHINE '${MACHINE}'. Valid values: chrysalis | compy | perlmutter"; exit 1 ;; +esac -# --- Machine-specific settings ----------------------------------------------- +# ============================================================================ +# Machine-specific settings +# ============================================================================ case "$MACHINE" in chrysalis) @@ -102,36 +100,36 @@ case "$MACHINE" in UNIFIED_ENV_CMD="source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh" SALLOC_CMD="salloc --nodes=1 --qos=interactive --time=01:00:00 --constraint=cpu --account=e3sm" ;; - *) - echo "Error: Unknown machine '$MACHINE'. Valid values: chrysalis | compy | perlmutter" - exit 1 - ;; esac -# --- Resolve TAG ------------------------------------------------------------- +# ============================================================================ +# Resolve TAG +# ============================================================================ # # Priority: -# 1. --tag CLI argument (explicit, always wins) +# 1. EXPLICIT_TAG from config (always wins when non-empty) # 2. $TAG_CACHE_FILE written by a prior Phase 1 run (auto-resume) # 3. Fresh timestamp (Phase 1 first run) # # Phase 1 always writes the resolved TAG to $TAG_CACHE_FILE so later phases -# can pick it up automatically without needing --tag. +# can pick it up automatically without needing EXPLICIT_TAG. if [[ -n "$EXPLICIT_TAG" ]]; then TAG="$EXPLICIT_TAG" - DATE_STAMP="${TAG%%_run*}" # Extract date portion for display; best-effort. + DATE_STAMP="${TAG%%_run*}" elif [[ "$START_PHASE" -gt 1 && -f "$TAG_CACHE_FILE" ]]; then TAG="$(cat "$TAG_CACHE_FILE")" DATE_STAMP="${TAG%%_run*}" echo "Loaded TAG from ${TAG_CACHE_FILE}: ${TAG}" - echo "(Pass --tag ${TAG} explicitly to override.)" + echo "(Set EXPLICIT_TAG in your config to override.)" else - DATE_STAMP="${DATE_STAMP:-$(date +%Y%m%d)}" + DATE_STAMP="$(date +%Y%m%d)" TAG="${DATE_STAMP}_run${RUN_NUMBER}" fi -# --- Derived (probably no edits needed) -------------------------------------- +# ============================================================================ +# Derived (probably no edits needed) +# ============================================================================ UNIQUE_ID="zppy_main_branch_test_${TAG}" @@ -306,9 +304,9 @@ wait_for_slurm_jobs() { if [ "$elapsed" -ge "$max_wait" ]; then log_error "Timeout after ${max_wait}s waiting for SLURM jobs" - log_error "This script is going to exit now. However, the jobs in the queue will NOT be terminated. Once they finish, you may re-invoke this script with --phase 2 or --phase 3 to continue." + log_error "This script is going to exit now. However, the jobs in the queue will NOT be terminated. Once they finish, you may re-invoke this script with START_PHASE=2 or START_PHASE=3 in your config to continue." log_error " TAG for this run: ${TAG}" - log_error " Resume command: $0 --machine ${MACHINE} --phase 2 --tag ${TAG}" + log_error " Resume: set START_PHASE=2 and EXPLICIT_TAG=${TAG} in your config, then re-run." log_error " (TAG is also saved in ${TAG_CACHE_FILE})" return 1 fi @@ -354,12 +352,14 @@ phase_1_setup() { log "=========================================" log "Phase 1: Setup" + log "Config file: $CONFIG_FILE" log "Date stamp: $DATE_STAMP" log "TAG: $TAG (saved to ${TAG_CACHE_FILE})" log "Unique ID: $UNIQUE_ID" log "" - log "To resume from a later phase, run:" - log " $0 --machine ${MACHINE} --phase 2 --tag ${TAG}" + log "To resume from a later phase, set in your config:" + log " START_PHASE=2" + log " EXPLICIT_TAG=${TAG}" log "=========================================" # ------------------------------------------------------------------ @@ -671,6 +671,7 @@ phase_3_validation() { main() { log "Starting zppy integration test automation" + log "Config file: $CONFIG_FILE" log "Machine: $MACHINE" log "TAG: $TAG" log "Auto mode: $AUTO_MODE" @@ -690,7 +691,7 @@ main() { phase_3_validation ;; *) - log_error "Invalid phase: $START_PHASE (must be 1, 2, or 3)" + log_error "Invalid START_PHASE: $START_PHASE (must be 1, 2, or 3)" exit 1 ;; esac diff --git a/tests/main_branch_testing/zppy_test.cfg b/tests/main_branch_testing/zppy_test.cfg new file mode 100644 index 00000000..8e2c9d2e --- /dev/null +++ b/tests/main_branch_testing/zppy_test.cfg @@ -0,0 +1,48 @@ +# zppy integration test configuration +# Source: run_integration_test.bash --config path/to/this.cfg +# +# Sections are comments only -- bash sources this file as KEY=VALUE pairs. + +# ---------------------------------------------------------------------------- +# Runtime (formerly "Parse arguments") +# ---------------------------------------------------------------------------- + +MACHINE=chrysalis # chrysalis | compy | perlmutter +START_PHASE=1 # 1 | 2 | 3 +AUTO_MODE=false # true = skip all interactive checkpoints +EXPLICIT_TAG="" # Leave empty to auto-generate; set to resume a prior run + +# ---------------------------------------------------------------------------- +# Per-run settings (formerly "Check these every time") +# ---------------------------------------------------------------------------- + +RUN_NUMBER=1 + +DIAGS_BASE_BRANCH="main" +E3SM_TO_CMIP_BASE_BRANCH="master" +MPAS_BASE_BRANCH="develop" +ZI_BASE_BRANCH="main" +ZPPY_BASE_BRANCH="main" + +# "dev" = build a dedicated conda env from the repo's dev.yml +# "unified" = use the machine's e3sm-unified env (UNIFIED_ENV_CMD) +DIAGS_ENV_TYPE="dev" +E3SM_TO_CMIP_ENV_TYPE="dev" +MPAS_ENV_TYPE="dev" +ZI_ENV_TYPE="dev" + +# ---------------------------------------------------------------------------- +# One-time setup (formerly "Set these up once") +# ---------------------------------------------------------------------------- + +HOME_DIR="$HOME" +EZ_DIR="$HOME_DIR/ez" + +E3SM_DIAGS_DIR="$EZ_DIR/e3sm_diags" +E3SM_TO_CMIP_DIR="$EZ_DIR/e3sm_to_cmip" +MPAS_ANALYSIS_DIR="$EZ_DIR/MPAS-Analysis" +ZPPY_INTERFACES_DIR="$EZ_DIR/zppy-interfaces" +ZPPY_DIR="$EZ_DIR/zppy" + +CONDA_PROFILE="$HOME_DIR/miniforge3/etc/profile.d/conda.sh" +TAG_CACHE_FILE="$HOME_DIR/.zppy_test_tag" From 1c463ef137ec0bfa808586633fcf04e09b55a007 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Fri, 12 Jun 2026 11:52:17 -0700 Subject: [PATCH 17/38] Enable cfg/task specification --- tests/main_branch_testing/README.md | 7 ++ .../run_integration_test.bash | 71 +++++++++++-------- tests/main_branch_testing/zppy_test.cfg | 8 +++ 3 files changed, 57 insertions(+), 29 deletions(-) diff --git a/tests/main_branch_testing/README.md b/tests/main_branch_testing/README.md index fe6a0555..6dd04475 100644 --- a/tests/main_branch_testing/README.md +++ b/tests/main_branch_testing/README.md @@ -40,6 +40,13 @@ Copy `zppy_test.cfg` and edit it before each test run. It has three sections: | `E3SM_TO_CMIP_ENV_TYPE` | `"dev"` to build a dedicated conda env; `"unified"` to use e3sm-unified | | `MPAS_ENV_TYPE` | `"dev"` to build a dedicated conda env; `"unified"` to use e3sm-unified | | `ZI_ENV_TYPE` | `"dev"` to build a dedicated conda env; `"unified"` to use e3sm-unified | +| `CFGS_TO_RUN` | Comma-separated list of zppy cfg names to generate and submit (see below) | +| `TASKS_TO_RUN` | Comma-separated list of tasks to enable (e.g. `e3sm_diags,mpas_analysis`) | + +`CFGS_TO_RUN` values correspond to generated filenames `test_weekly__.cfg`. Any name containing `bundle` is automatically treated as a bundle cfg and re-submitted in Phase 2. Example to run only the v3 comprehensive and bundle cfgs: +``` +CFGS_TO_RUN="weekly_bundles,weekly_comprehensive_v3,weekly_legacy_3.1.0_bundles,weekly_legacy_3.1.0_comprehensive_v3,weekly_legacy_3.0.0_bundles,weekly_legacy_3.0.0_comprehensive_v3" +``` ### One-time setup (paths that rarely change) diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index cda4f727..ef24ed31 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -55,6 +55,7 @@ _required_vars=( RUN_NUMBER DIAGS_BASE_BRANCH E3SM_TO_CMIP_BASE_BRANCH MPAS_BASE_BRANCH ZI_BASE_BRANCH ZPPY_BASE_BRANCH DIAGS_ENV_TYPE E3SM_TO_CMIP_ENV_TYPE MPAS_ENV_TYPE ZI_ENV_TYPE + CFGS_TO_RUN TASKS_TO_RUN HOME_DIR EZ_DIR E3SM_DIAGS_DIR E3SM_TO_CMIP_DIR MPAS_ANALYSIS_DIR ZPPY_INTERFACES_DIR ZPPY_DIR CONDA_PROFILE TAG_CACHE_FILE @@ -102,8 +103,17 @@ case "$MACHINE" in ;; esac -# ============================================================================ -# Resolve TAG +# Derive the filename suffix used by generated zppy cfg files. +case "$MACHINE" in + chrysalis) MACHINE_CFG_SUFFIX="chrysalis" ;; + compy) MACHINE_CFG_SUFFIX="compy" ;; + perlmutter) MACHINE_CFG_SUFFIX="pm-cpu" ;; +esac + +# Split comma-separated config lists into bash arrays. +# IFS = Internal Field Separator +IFS=',' read -ra CFGS_ARRAY <<< "$CFGS_TO_RUN" +IFS=',' read -ra TASKS_ARRAY <<< "$TASKS_TO_RUN" # ============================================================================ # # Priority: @@ -470,6 +480,20 @@ phase_1_setup() { UTILS_FILE="tests/integration/utils.py" + # Build Python list literals from the bash arrays for injection into the heredoc. + local CFGS_PY_LIST TASKS_PY_LIST cfg task + CFGS_PY_LIST="" + for cfg in "${CFGS_ARRAY[@]}"; do + cfg="${cfg// /}" # strip any accidental whitespace + CFGS_PY_LIST+=" \"${cfg}\","$'\n' + done + TASKS_PY_LIST="" + for task in "${TASKS_ARRAY[@]}"; do + task="${task// /}" + TASKS_PY_LIST+="\"${task}\", " + done + TASKS_PY_LIST="${TASKS_PY_LIST%, }" # strip trailing comma+space + python - <_.cfg +# Any name containing "bundle" is treated as a bundle cfg and re-submitted in Phase 2. +CFGS_TO_RUN="weekly_bundles,weekly_comprehensive_v2,weekly_comprehensive_v3,weekly_legacy_3.1.0_bundles,weekly_legacy_3.1.0_comprehensive_v2,weekly_legacy_3.1.0_comprehensive_v3,weekly_legacy_3.0.0_bundles,weekly_legacy_3.0.0_comprehensive_v2,weekly_legacy_3.0.0_comprehensive_v3" + +# Comma-separated list of tasks to enable in utils.py. +TASKS_TO_RUN="e3sm_diags,mpas_analysis,global_time_series,ilamb,livvkit,pcmdi_diags" + # ---------------------------------------------------------------------------- # One-time setup (formerly "Set these up once") # ---------------------------------------------------------------------------- From 0b46ee8c453c017ee0c6c837c05666f03b7427db Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 15 Jun 2026 12:26:59 -0700 Subject: [PATCH 18/38] Update test cfg --- tests/main_branch_testing/zppy_test.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/main_branch_testing/zppy_test.cfg b/tests/main_branch_testing/zppy_test.cfg index 9227388d..f595b683 100644 --- a/tests/main_branch_testing/zppy_test.cfg +++ b/tests/main_branch_testing/zppy_test.cfg @@ -9,11 +9,11 @@ MACHINE=chrysalis # chrysalis | compy | perlmutter START_PHASE=1 # 1 | 2 | 3 -AUTO_MODE=false # true = skip all interactive checkpoints +AUTO_MODE=true # true = skip all interactive checkpoints EXPLICIT_TAG="" # Leave empty to auto-generate; set to resume a prior run # ---------------------------------------------------------------------------- -# Per-run settings (formerly "Check these every time") +# Per-run settings # ---------------------------------------------------------------------------- RUN_NUMBER=1 @@ -40,7 +40,7 @@ CFGS_TO_RUN="weekly_bundles,weekly_comprehensive_v2,weekly_comprehensive_v3,week TASKS_TO_RUN="e3sm_diags,mpas_analysis,global_time_series,ilamb,livvkit,pcmdi_diags" # ---------------------------------------------------------------------------- -# One-time setup (formerly "Set these up once") +# One-time setup # ---------------------------------------------------------------------------- HOME_DIR="$HOME" From 56ff67e0eb671eb0bf55b3fda90de3937bf5a177 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 15 Jun 2026 12:35:20 -0700 Subject: [PATCH 19/38] Enable use of existing envs --- tests/main_branch_testing/README.md | 21 ++++- .../run_integration_test.bash | 79 ++++++++++++++++--- tests/main_branch_testing/zppy_test.cfg | 12 ++- 3 files changed, 100 insertions(+), 12 deletions(-) diff --git a/tests/main_branch_testing/README.md b/tests/main_branch_testing/README.md index 6dd04475..01029abe 100644 --- a/tests/main_branch_testing/README.md +++ b/tests/main_branch_testing/README.md @@ -40,9 +40,25 @@ Copy `zppy_test.cfg` and edit it before each test run. It has three sections: | `E3SM_TO_CMIP_ENV_TYPE` | `"dev"` to build a dedicated conda env; `"unified"` to use e3sm-unified | | `MPAS_ENV_TYPE` | `"dev"` to build a dedicated conda env; `"unified"` to use e3sm-unified | | `ZI_ENV_TYPE` | `"dev"` to build a dedicated conda env; `"unified"` to use e3sm-unified | +| `DIAGS_EXISTING_ENV` | *(optional)* Name of an existing conda env to reuse for e3sm_diags; skips creation | +| `E3SM_TO_CMIP_EXISTING_ENV` | *(optional)* Name of an existing conda env to reuse for e3sm_to_cmip; skips creation | +| `MPAS_EXISTING_ENV` | *(optional)* Name of an existing conda env to reuse for MPAS-Analysis; skips creation | +| `ZI_EXISTING_ENV` | *(optional)* Name of an existing conda env to reuse for zppy-interfaces; skips creation | +| `ZPPY_EXISTING_ENV` | *(optional)* Name of an existing conda env to reuse for zppy; skips creation | | `CFGS_TO_RUN` | Comma-separated list of zppy cfg names to generate and submit (see below) | | `TASKS_TO_RUN` | Comma-separated list of tasks to enable (e.g. `e3sm_diags,mpas_analysis`) | +The `*_EXISTING_ENV` variables only take effect when the corresponding `*_ENV_TYPE` is `"dev"`. When set, the script skips conda env creation entirely and activates the named env directly (still running `pip install .` to pick up any local changes). Leave them empty to let the script auto-name and create environments as usual. + +Example — reuse envs from a prior run on the same day: +``` +DIAGS_EXISTING_ENV="test-diags-main-20250601_run1" +E3SM_TO_CMIP_EXISTING_ENV="test-e3sm-to-cmip-master-20250601_run1" +MPAS_EXISTING_ENV="test-mpas-develop-20250601_run1" +ZI_EXISTING_ENV="test-zi-main-20250601_run1" +ZPPY_EXISTING_ENV="test-zppy-main-20250601_run1" +``` + `CFGS_TO_RUN` values correspond to generated filenames `test_weekly__.cfg`. Any name containing `bundle` is automatically treated as a bundle cfg and re-submitted in Phase 2. Example to run only the v3 comprehensive and bundle cfgs: ``` CFGS_TO_RUN="weekly_bundles,weekly_comprehensive_v3,weekly_legacy_3.1.0_bundles,weekly_legacy_3.1.0_comprehensive_v3,weekly_legacy_3.0.0_bundles,weekly_legacy_3.0.0_comprehensive_v3" @@ -67,7 +83,7 @@ Machine-specific settings (`OUTPUT_WORKSPACE`, conda activation command, unified ## Workflow Phases ### Phase 1: Setup -- Creates conda environments for each component where `ENV_TYPE="dev"` (e3sm_to_cmip, e3sm_diags, MPAS-Analysis, zppy-interfaces, zppy); skips env creation and uses e3sm-unified for any component where `ENV_TYPE="unified"` +- Creates conda environments for each component where `ENV_TYPE="dev"` and no `*_EXISTING_ENV` is set; reuses the named env when `*_EXISTING_ENV` is set; skips env handling entirely for any component where `ENV_TYPE="unified"` - Runs unit tests for zppy-interfaces and zppy - Patches `tests/integration/utils.py` with test-specific environment commands, config list, and unique ID - Generates config files via `python tests/integration/utils.py` @@ -140,7 +156,8 @@ Phase 2 checks bundle status files before resubmitting and warns if any are non- ### Environment Issues ```bash -# Remove and rebuild stale environments (only applies to components with ENV_TYPE="dev") +# Remove and rebuild stale environments (only applies to components with ENV_TYPE="dev" +# and no *_EXISTING_ENV set) conda remove --yes --all --name test-e3sm-to-cmip-master-YYYYMMDD_runN conda remove --yes --all --name test-diags-main-YYYYMMDD_runN conda remove --yes --all --name test-mpas-develop-YYYYMMDD_runN diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index ef24ed31..cb801012 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -72,6 +72,14 @@ if [[ ${#_missing[@]} -gt 0 ]]; then exit 1 fi +# Apply defaults for optional *_EXISTING_ENV variables so the rest of the +# script can reference them unconditionally. +DIAGS_EXISTING_ENV="${DIAGS_EXISTING_ENV:-}" +E3SM_TO_CMIP_EXISTING_ENV="${E3SM_TO_CMIP_EXISTING_ENV:-}" +MPAS_EXISTING_ENV="${MPAS_EXISTING_ENV:-}" +ZI_EXISTING_ENV="${ZI_EXISTING_ENV:-}" +ZPPY_EXISTING_ENV="${ZPPY_EXISTING_ENV:-}" + # Validate MACHINE value. case "$MACHINE" in chrysalis|compy|perlmutter) ;; @@ -233,6 +241,23 @@ setup_conda_env() { log_success "Environment '$env_name' ready" } +# Resolve the conda env name for a "dev"-type component. +# If an existing env name is provided, log that it will be reused and echo it. +# Otherwise, echo the auto-generated name. +# Usage: env_name=$(resolve_dev_env "e3sm_diags" "$DIAGS_EXISTING_ENV" "test-diags-main-${TAG}") +resolve_dev_env() { + local component="$1" + local existing_env="$2" + local auto_name="$3" + + if [[ -n "$existing_env" ]]; then + log "Reusing existing '$component' env: $existing_env (skipping creation)" + echo "$existing_env" + else + echo "$auto_name" + fi +} + # Checkout test branch, creating it from upstream/ if it doesn't exist. # Stashes/commits any in-progress work first. ensure_test_branch() { @@ -384,8 +409,15 @@ phase_1_setup() { local E3SM_TO_CMIP_ENV="" if [[ "$E3SM_TO_CMIP_ENV_TYPE" == "dev" ]]; then - E3SM_TO_CMIP_ENV="test-e3sm-to-cmip-${E3SM_TO_CMIP_BASE_BRANCH}-${TAG}" - setup_conda_env "conda-env" "$E3SM_TO_CMIP_ENV" + E3SM_TO_CMIP_ENV=$(resolve_dev_env \ + "e3sm_to_cmip" \ + "$E3SM_TO_CMIP_EXISTING_ENV" \ + "test-e3sm-to-cmip-${E3SM_TO_CMIP_BASE_BRANCH}-${TAG}") + if [[ -z "$E3SM_TO_CMIP_EXISTING_ENV" ]]; then + setup_conda_env "conda-env" "$E3SM_TO_CMIP_ENV" + else + activate_env "$E3SM_TO_CMIP_ENV" + fi else log "Using unified env for e3sm_to_cmip (skipping conda env creation)" fi @@ -402,8 +434,15 @@ phase_1_setup() { local DIAGS_ENV="" if [[ "$DIAGS_ENV_TYPE" == "dev" ]]; then - DIAGS_ENV="test-diags-${DIAGS_BASE_BRANCH}-${TAG}" - setup_conda_env "conda-env" "$DIAGS_ENV" + DIAGS_ENV=$(resolve_dev_env \ + "e3sm_diags" \ + "$DIAGS_EXISTING_ENV" \ + "test-diags-${DIAGS_BASE_BRANCH}-${TAG}") + if [[ -z "$DIAGS_EXISTING_ENV" ]]; then + setup_conda_env "conda-env" "$DIAGS_ENV" + else + activate_env "$DIAGS_ENV" + fi else log "Using unified env for e3sm_diags (skipping conda env creation)" fi @@ -420,8 +459,15 @@ phase_1_setup() { local MPAS_ENV="" if [[ "$MPAS_ENV_TYPE" == "dev" ]]; then - MPAS_ENV="test-mpas-${MPAS_BASE_BRANCH}-${TAG}" - setup_conda_env "none" "$MPAS_ENV" + MPAS_ENV=$(resolve_dev_env \ + "MPAS-Analysis" \ + "$MPAS_EXISTING_ENV" \ + "test-mpas-${MPAS_BASE_BRANCH}-${TAG}") + if [[ -z "$MPAS_EXISTING_ENV" ]]; then + setup_conda_env "none" "$MPAS_ENV" + else + activate_env "$MPAS_ENV" + fi else log "Using unified env for MPAS-Analysis (skipping conda env creation)" fi @@ -438,8 +484,15 @@ phase_1_setup() { local ZI_ENV="" if [[ "$ZI_ENV_TYPE" == "dev" ]]; then - ZI_ENV="test-zi-${ZI_BASE_BRANCH}-${TAG}" - setup_conda_env "conda" "$ZI_ENV" + ZI_ENV=$(resolve_dev_env \ + "zppy-interfaces" \ + "$ZI_EXISTING_ENV" \ + "test-zi-${ZI_BASE_BRANCH}-${TAG}") + if [[ -z "$ZI_EXISTING_ENV" ]]; then + setup_conda_env "conda" "$ZI_ENV" + else + activate_env "$ZI_ENV" + fi else log "Using unified env for zppy-interfaces (skipping conda env creation)" fi @@ -458,7 +511,15 @@ phase_1_setup() { log "Latest zppy commit (should match https://github.com/E3SM-Project/zppy/commits/${ZPPY_BASE_BRANCH}):" git log -1 --oneline - setup_conda_env "conda" "$ZPPY_ENV" + + # Resolve ZPPY_ENV: if an existing env is specified, use it; otherwise use + # the auto-generated name and create/update the env as normal. + if [[ -n "$ZPPY_EXISTING_ENV" ]]; then + ZPPY_ENV=$(resolve_dev_env "zppy" "$ZPPY_EXISTING_ENV" "$ZPPY_ENV") + activate_env "$ZPPY_ENV" + else + setup_conda_env "conda" "$ZPPY_ENV" + fi log "Running zppy unit tests..." pytest tests/test_*.py diff --git a/tests/main_branch_testing/zppy_test.cfg b/tests/main_branch_testing/zppy_test.cfg index f595b683..983564f7 100644 --- a/tests/main_branch_testing/zppy_test.cfg +++ b/tests/main_branch_testing/zppy_test.cfg @@ -9,7 +9,7 @@ MACHINE=chrysalis # chrysalis | compy | perlmutter START_PHASE=1 # 1 | 2 | 3 -AUTO_MODE=true # true = skip all interactive checkpoints +AUTO_MODE=true # true = skip all interactive checkpoints EXPLICIT_TAG="" # Leave empty to auto-generate; set to resume a prior run # ---------------------------------------------------------------------------- @@ -31,6 +31,16 @@ E3SM_TO_CMIP_ENV_TYPE="dev" MPAS_ENV_TYPE="dev" ZI_ENV_TYPE="dev" +# Optional: reuse an existing named conda env instead of creating a new one. +# When non-empty AND the corresponding ENV_TYPE is "dev", the script skips +# conda env creation and activates this env directly. +# Leave empty to let the script auto-name and create the env as usual. +DIAGS_EXISTING_ENV="" +E3SM_TO_CMIP_EXISTING_ENV="" +MPAS_EXISTING_ENV="" +ZI_EXISTING_ENV="" +ZPPY_EXISTING_ENV="" + # Comma-separated list of zppy cfg names to generate and submit. # These correspond to generated filenames: test_weekly__.cfg # Any name containing "bundle" is treated as a bundle cfg and re-submitted in Phase 2. From 6e9ec7a585db2e23bb409aa56a97915f44c6dbc9 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 15 Jun 2026 12:48:49 -0700 Subject: [PATCH 20/38] Fix env handling --- .../run_integration_test.bash | 71 +++++++------------ 1 file changed, 26 insertions(+), 45 deletions(-) diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index cb801012..72c1b81c 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -241,22 +241,6 @@ setup_conda_env() { log_success "Environment '$env_name' ready" } -# Resolve the conda env name for a "dev"-type component. -# If an existing env name is provided, log that it will be reused and echo it. -# Otherwise, echo the auto-generated name. -# Usage: env_name=$(resolve_dev_env "e3sm_diags" "$DIAGS_EXISTING_ENV" "test-diags-main-${TAG}") -resolve_dev_env() { - local component="$1" - local existing_env="$2" - local auto_name="$3" - - if [[ -n "$existing_env" ]]; then - log "Reusing existing '$component' env: $existing_env (skipping creation)" - echo "$existing_env" - else - echo "$auto_name" - fi -} # Checkout test branch, creating it from upstream/ if it doesn't exist. # Stashes/commits any in-progress work first. @@ -409,14 +393,13 @@ phase_1_setup() { local E3SM_TO_CMIP_ENV="" if [[ "$E3SM_TO_CMIP_ENV_TYPE" == "dev" ]]; then - E3SM_TO_CMIP_ENV=$(resolve_dev_env \ - "e3sm_to_cmip" \ - "$E3SM_TO_CMIP_EXISTING_ENV" \ - "test-e3sm-to-cmip-${E3SM_TO_CMIP_BASE_BRANCH}-${TAG}") - if [[ -z "$E3SM_TO_CMIP_EXISTING_ENV" ]]; then - setup_conda_env "conda-env" "$E3SM_TO_CMIP_ENV" - else + if [[ -n "$E3SM_TO_CMIP_EXISTING_ENV" ]]; then + log "Reusing existing 'e3sm_to_cmip' env: $E3SM_TO_CMIP_EXISTING_ENV (skipping creation)" + E3SM_TO_CMIP_ENV="$E3SM_TO_CMIP_EXISTING_ENV" activate_env "$E3SM_TO_CMIP_ENV" + else + E3SM_TO_CMIP_ENV="test-e3sm-to-cmip-${E3SM_TO_CMIP_BASE_BRANCH}-${TAG}" + setup_conda_env "conda-env" "$E3SM_TO_CMIP_ENV" fi else log "Using unified env for e3sm_to_cmip (skipping conda env creation)" @@ -434,14 +417,13 @@ phase_1_setup() { local DIAGS_ENV="" if [[ "$DIAGS_ENV_TYPE" == "dev" ]]; then - DIAGS_ENV=$(resolve_dev_env \ - "e3sm_diags" \ - "$DIAGS_EXISTING_ENV" \ - "test-diags-${DIAGS_BASE_BRANCH}-${TAG}") - if [[ -z "$DIAGS_EXISTING_ENV" ]]; then - setup_conda_env "conda-env" "$DIAGS_ENV" - else + if [[ -n "$DIAGS_EXISTING_ENV" ]]; then + log "Reusing existing 'e3sm_diags' env: $DIAGS_EXISTING_ENV (skipping creation)" + DIAGS_ENV="$DIAGS_EXISTING_ENV" activate_env "$DIAGS_ENV" + else + DIAGS_ENV="test-diags-${DIAGS_BASE_BRANCH}-${TAG}" + setup_conda_env "conda-env" "$DIAGS_ENV" fi else log "Using unified env for e3sm_diags (skipping conda env creation)" @@ -459,14 +441,13 @@ phase_1_setup() { local MPAS_ENV="" if [[ "$MPAS_ENV_TYPE" == "dev" ]]; then - MPAS_ENV=$(resolve_dev_env \ - "MPAS-Analysis" \ - "$MPAS_EXISTING_ENV" \ - "test-mpas-${MPAS_BASE_BRANCH}-${TAG}") - if [[ -z "$MPAS_EXISTING_ENV" ]]; then - setup_conda_env "none" "$MPAS_ENV" - else + if [[ -n "$MPAS_EXISTING_ENV" ]]; then + log "Reusing existing 'MPAS-Analysis' env: $MPAS_EXISTING_ENV (skipping creation)" + MPAS_ENV="$MPAS_EXISTING_ENV" activate_env "$MPAS_ENV" + else + MPAS_ENV="test-mpas-${MPAS_BASE_BRANCH}-${TAG}" + setup_conda_env "none" "$MPAS_ENV" fi else log "Using unified env for MPAS-Analysis (skipping conda env creation)" @@ -484,14 +465,13 @@ phase_1_setup() { local ZI_ENV="" if [[ "$ZI_ENV_TYPE" == "dev" ]]; then - ZI_ENV=$(resolve_dev_env \ - "zppy-interfaces" \ - "$ZI_EXISTING_ENV" \ - "test-zi-${ZI_BASE_BRANCH}-${TAG}") - if [[ -z "$ZI_EXISTING_ENV" ]]; then - setup_conda_env "conda" "$ZI_ENV" - else + if [[ -n "$ZI_EXISTING_ENV" ]]; then + log "Reusing existing 'zppy-interfaces' env: $ZI_EXISTING_ENV (skipping creation)" + ZI_ENV="$ZI_EXISTING_ENV" activate_env "$ZI_ENV" + else + ZI_ENV="test-zi-${ZI_BASE_BRANCH}-${TAG}" + setup_conda_env "conda" "$ZI_ENV" fi else log "Using unified env for zppy-interfaces (skipping conda env creation)" @@ -515,7 +495,8 @@ phase_1_setup() { # Resolve ZPPY_ENV: if an existing env is specified, use it; otherwise use # the auto-generated name and create/update the env as normal. if [[ -n "$ZPPY_EXISTING_ENV" ]]; then - ZPPY_ENV=$(resolve_dev_env "zppy" "$ZPPY_EXISTING_ENV" "$ZPPY_ENV") + log "Reusing existing 'zppy' env: $ZPPY_EXISTING_ENV (skipping creation)" + ZPPY_ENV="$ZPPY_EXISTING_ENV" activate_env "$ZPPY_ENV" else setup_conda_env "conda" "$ZPPY_ENV" From 5ec121cbf46c8f94c96418eafd7f51e76060019f Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 15 Jun 2026 13:12:22 -0700 Subject: [PATCH 21/38] Fix cfg file paths --- tests/main_branch_testing/README.md | 2 +- .../run_integration_test.bash | 22 +++++++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/tests/main_branch_testing/README.md b/tests/main_branch_testing/README.md index 01029abe..3331cbf2 100644 --- a/tests/main_branch_testing/README.md +++ b/tests/main_branch_testing/README.md @@ -59,7 +59,7 @@ ZI_EXISTING_ENV="test-zi-main-20250601_run1" ZPPY_EXISTING_ENV="test-zppy-main-20250601_run1" ``` -`CFGS_TO_RUN` values correspond to generated filenames `test_weekly__.cfg`. Any name containing `bundle` is automatically treated as a bundle cfg and re-submitted in Phase 2. Example to run only the v3 comprehensive and bundle cfgs: +`CFGS_TO_RUN` values correspond to generated filenames `test__.cfg`. Any name containing `bundle` is automatically treated as a bundle cfg and re-submitted in Phase 2. Example to run only the v3 comprehensive and bundle cfgs: ``` CFGS_TO_RUN="weekly_bundles,weekly_comprehensive_v3,weekly_legacy_3.1.0_bundles,weekly_legacy_3.1.0_comprehensive_v3,weekly_legacy_3.0.0_bundles,weekly_legacy_3.0.0_comprehensive_v3" ``` diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index 72c1b81c..7f9580c9 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -577,10 +577,17 @@ PYEOF # Submit initial SLURM jobs # ------------------------------------------------------------------ log "Submitting SLURM jobs..." - local cfg + local cfg cfg_path for cfg in "${CFGS_ARRAY[@]}"; do cfg="${cfg// /}" - zppy -c "tests/integration/generated/test_weekly_${cfg}_${MACHINE_CFG_SUFFIX}.cfg" + cfg_path="tests/integration/generated/test_${cfg}_${MACHINE_CFG_SUFFIX}.cfg" + if [[ ! -f "$cfg_path" ]]; then + log_error "Config file not found: $cfg_path" + log_error "Check that CFGS_TO_RUN names do not include the 'weekly_' prefix." + exit 1 + fi + log "Submitting: $cfg_path" + zppy -c "$cfg_path" done local job_count @@ -624,11 +631,18 @@ phase_2_bundles_part2() { fi log "Submitting bundles part 2..." - local cfg + local cfg cfg_path for cfg in "${CFGS_ARRAY[@]}"; do cfg="${cfg// /}" if [[ "$cfg" == *bundle* ]]; then - zppy -c "tests/integration/generated/test_weekly_${cfg}_${MACHINE_CFG_SUFFIX}.cfg" + cfg_path="tests/integration/generated/test_${cfg}_${MACHINE_CFG_SUFFIX}.cfg" + if [[ ! -f "$cfg_path" ]]; then + log_error "Config file not found: $cfg_path" + log_error "Check that CFGS_TO_RUN names do not include the 'weekly_' prefix." + exit 1 + fi + log "Submitting: $cfg_path" + zppy -c "$cfg_path" fi done From 8583de9b588a1e22dee793044baf2065bd21e53d Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 15 Jun 2026 15:37:14 -0700 Subject: [PATCH 22/38] Use bash subshells --- .../run_integration_test.bash | 166 ++++++++++++------ 1 file changed, 108 insertions(+), 58 deletions(-) diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index 7f9580c9..e1cc2294 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -385,126 +385,170 @@ phase_1_setup() { # e3sm_to_cmip # ------------------------------------------------------------------ log "Setting up e3sm_to_cmip..." - cd "$E3SM_TO_CMIP_DIR" - ensure_test_branch "test_e3sm_to_cmip_${TAG}" "$E3SM_TO_CMIP_BASE_BRANCH" - - log "Latest e3sm_to_cmip commit (should match https://github.com/E3SM-Project/e3sm_to_cmip/commits/${E3SM_TO_CMIP_BASE_BRANCH}):" - git log -1 --oneline local E3SM_TO_CMIP_ENV="" if [[ "$E3SM_TO_CMIP_ENV_TYPE" == "dev" ]]; then if [[ -n "$E3SM_TO_CMIP_EXISTING_ENV" ]]; then - log "Reusing existing 'e3sm_to_cmip' env: $E3SM_TO_CMIP_EXISTING_ENV (skipping creation)" E3SM_TO_CMIP_ENV="$E3SM_TO_CMIP_EXISTING_ENV" - activate_env "$E3SM_TO_CMIP_ENV" else E3SM_TO_CMIP_ENV="test-e3sm-to-cmip-${E3SM_TO_CMIP_BASE_BRANCH}-${TAG}" - setup_conda_env "conda-env" "$E3SM_TO_CMIP_ENV" fi - else - log "Using unified env for e3sm_to_cmip (skipping conda env creation)" fi + ( + cd "$E3SM_TO_CMIP_DIR" + ensure_test_branch "test_e3sm_to_cmip_${TAG}" "$E3SM_TO_CMIP_BASE_BRANCH" + + log "Latest e3sm_to_cmip commit (should match https://github.com/E3SM-Project/e3sm_to_cmip/commits/${E3SM_TO_CMIP_BASE_BRANCH}):" + git log -1 --oneline + + if [[ "$E3SM_TO_CMIP_ENV_TYPE" == "dev" ]]; then + if [[ -n "$E3SM_TO_CMIP_EXISTING_ENV" ]]; then + log "Reusing existing 'e3sm_to_cmip' env: $E3SM_TO_CMIP_EXISTING_ENV (skipping creation)" + activate_env "$E3SM_TO_CMIP_EXISTING_ENV" + else + setup_conda_env "conda-env" "$E3SM_TO_CMIP_ENV" + fi + else + log "Using unified env for e3sm_to_cmip (skipping conda env creation)" + fi + ) + # ------------------------------------------------------------------ # e3sm_diags # ------------------------------------------------------------------ log "Setting up e3sm_diags..." - cd "$E3SM_DIAGS_DIR" - ensure_test_branch "test_e3sm_diags_${TAG}" "$DIAGS_BASE_BRANCH" - - log "Latest e3sm_diags commit (should match https://github.com/E3SM-Project/e3sm_diags/commits/${DIAGS_BASE_BRANCH}):" - git log -1 --oneline local DIAGS_ENV="" if [[ "$DIAGS_ENV_TYPE" == "dev" ]]; then if [[ -n "$DIAGS_EXISTING_ENV" ]]; then - log "Reusing existing 'e3sm_diags' env: $DIAGS_EXISTING_ENV (skipping creation)" DIAGS_ENV="$DIAGS_EXISTING_ENV" - activate_env "$DIAGS_ENV" else DIAGS_ENV="test-diags-${DIAGS_BASE_BRANCH}-${TAG}" - setup_conda_env "conda-env" "$DIAGS_ENV" fi - else - log "Using unified env for e3sm_diags (skipping conda env creation)" fi + ( + cd "$E3SM_DIAGS_DIR" + ensure_test_branch "test_e3sm_diags_${TAG}" "$DIAGS_BASE_BRANCH" + + log "Latest e3sm_diags commit (should match https://github.com/E3SM-Project/e3sm_diags/commits/${DIAGS_BASE_BRANCH}):" + git log -1 --oneline + + if [[ "$DIAGS_ENV_TYPE" == "dev" ]]; then + if [[ -n "$DIAGS_EXISTING_ENV" ]]; then + log "Reusing existing 'e3sm_diags' env: $DIAGS_EXISTING_ENV (skipping creation)" + activate_env "$DIAGS_EXISTING_ENV" + else + setup_conda_env "conda-env" "$DIAGS_ENV" + fi + else + log "Using unified env for e3sm_diags (skipping conda env creation)" + fi + ) + # ------------------------------------------------------------------ # MPAS-Analysis # ------------------------------------------------------------------ log "Setting up MPAS-Analysis..." - cd "$MPAS_ANALYSIS_DIR" - ensure_test_branch "test_mpas_${TAG}" "$MPAS_BASE_BRANCH" - - log "Latest MPAS-Analysis commit (should match https://github.com/MPAS-Dev/MPAS-Analysis/commits/${MPAS_BASE_BRANCH}):" - git log -1 --oneline local MPAS_ENV="" if [[ "$MPAS_ENV_TYPE" == "dev" ]]; then if [[ -n "$MPAS_EXISTING_ENV" ]]; then - log "Reusing existing 'MPAS-Analysis' env: $MPAS_EXISTING_ENV (skipping creation)" MPAS_ENV="$MPAS_EXISTING_ENV" - activate_env "$MPAS_ENV" else MPAS_ENV="test-mpas-${MPAS_BASE_BRANCH}-${TAG}" - setup_conda_env "none" "$MPAS_ENV" fi - else - log "Using unified env for MPAS-Analysis (skipping conda env creation)" fi + ( + cd "$MPAS_ANALYSIS_DIR" + ensure_test_branch "test_mpas_${TAG}" "$MPAS_BASE_BRANCH" + + log "Latest MPAS-Analysis commit (should match https://github.com/MPAS-Dev/MPAS-Analysis/commits/${MPAS_BASE_BRANCH}):" + git log -1 --oneline + + if [[ "$MPAS_ENV_TYPE" == "dev" ]]; then + if [[ -n "$MPAS_EXISTING_ENV" ]]; then + log "Reusing existing 'MPAS-Analysis' env: $MPAS_EXISTING_ENV (skipping creation)" + activate_env "$MPAS_EXISTING_ENV" + else + setup_conda_env "none" "$MPAS_ENV" + fi + else + log "Using unified env for MPAS-Analysis (skipping conda env creation)" + fi + ) + # ------------------------------------------------------------------ # zppy-interfaces (includes unit tests) # ------------------------------------------------------------------ log "Setting up zppy-interfaces..." - cd "$ZPPY_INTERFACES_DIR" - ensure_test_branch "test_zi_${TAG}" "$ZI_BASE_BRANCH" - - log "Latest zppy-interfaces commit (should match https://github.com/E3SM-Project/zppy-interfaces/commits/${ZI_BASE_BRANCH}):" - git log -1 --oneline local ZI_ENV="" if [[ "$ZI_ENV_TYPE" == "dev" ]]; then if [[ -n "$ZI_EXISTING_ENV" ]]; then - log "Reusing existing 'zppy-interfaces' env: $ZI_EXISTING_ENV (skipping creation)" ZI_ENV="$ZI_EXISTING_ENV" - activate_env "$ZI_ENV" else ZI_ENV="test-zi-${ZI_BASE_BRANCH}-${TAG}" - setup_conda_env "conda" "$ZI_ENV" fi - else - log "Using unified env for zppy-interfaces (skipping conda env creation)" fi - log "Running zppy-interfaces unit tests..." - pytest tests/unit/global_time_series/test_*.py - pytest tests/unit/pcmdi_diags/test_*.py - log_success "zppy-interfaces unit tests passed" + ( + cd "$ZPPY_INTERFACES_DIR" + ensure_test_branch "test_zi_${TAG}" "$ZI_BASE_BRANCH" + + log "Latest zppy-interfaces commit (should match https://github.com/E3SM-Project/zppy-interfaces/commits/${ZI_BASE_BRANCH}):" + git log -1 --oneline + + if [[ "$ZI_ENV_TYPE" == "dev" ]]; then + if [[ -n "$ZI_EXISTING_ENV" ]]; then + log "Reusing existing 'zppy-interfaces' env: $ZI_EXISTING_ENV (skipping creation)" + activate_env "$ZI_EXISTING_ENV" + else + setup_conda_env "conda" "$ZI_ENV" + fi + else + log "Using unified env for zppy-interfaces (skipping conda env creation)" + fi + + log "Running zppy-interfaces unit tests..." + pytest tests/unit/global_time_series/test_*.py + pytest tests/unit/pcmdi_diags/test_*.py + log_success "zppy-interfaces unit tests passed" + ) # ------------------------------------------------------------------ # zppy (includes unit tests + config generation) # ------------------------------------------------------------------ log "Setting up zppy..." - cd "$ZPPY_DIR" - ensure_test_branch "test_zppy_${TAG}" "$ZPPY_BASE_BRANCH" - log "Latest zppy commit (should match https://github.com/E3SM-Project/zppy/commits/${ZPPY_BASE_BRANCH}):" - git log -1 --oneline - - # Resolve ZPPY_ENV: if an existing env is specified, use it; otherwise use - # the auto-generated name and create/update the env as normal. + # Resolve ZPPY_ENV name before the subshell so it's available for + # config generation and later phases. if [[ -n "$ZPPY_EXISTING_ENV" ]]; then - log "Reusing existing 'zppy' env: $ZPPY_EXISTING_ENV (skipping creation)" ZPPY_ENV="$ZPPY_EXISTING_ENV" - activate_env "$ZPPY_ENV" - else - setup_conda_env "conda" "$ZPPY_ENV" fi + # (If ZPPY_EXISTING_ENV is empty, ZPPY_ENV retains the auto-generated + # name set at the top of the script.) + + ( + cd "$ZPPY_DIR" + ensure_test_branch "test_zppy_${TAG}" "$ZPPY_BASE_BRANCH" - log "Running zppy unit tests..." - pytest tests/test_*.py - log_success "zppy unit tests passed" + log "Latest zppy commit (should match https://github.com/E3SM-Project/zppy/commits/${ZPPY_BASE_BRANCH}):" + git log -1 --oneline + + if [[ -n "$ZPPY_EXISTING_ENV" ]]; then + log "Reusing existing 'zppy' env: $ZPPY_EXISTING_ENV (skipping creation)" + activate_env "$ZPPY_ENV" + else + setup_conda_env "conda" "$ZPPY_ENV" + fi + + log "Running zppy unit tests..." + pytest tests/test_*.py + log_success "zppy unit tests passed" + ) # ------------------------------------------------------------------ # Generate config files (update utils.py TEST_SPECIFICS, then run it) @@ -520,6 +564,12 @@ phase_1_setup() { MPAS_CMD=$(get_env_cmd "$MPAS_ENV_TYPE" "$MPAS_ENV") ZI_CMD=$(get_env_cmd "$ZI_ENV_TYPE" "$ZI_ENV") + # Config generation and job submission run in the parent shell so that + # the zppy command is available and cd/env state is consistent. + cd "$ZPPY_DIR" + activate_env "$ZPPY_ENV" + ensure_test_branch "test_zppy_${TAG}" "$ZPPY_BASE_BRANCH" + UTILS_FILE="tests/integration/utils.py" # Build Python list literals from the bash arrays for injection into the heredoc. From 56d1c75eb2c98326c4188eef7d6f8487d6c14e6b Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 15 Jun 2026 17:34:28 -0700 Subject: [PATCH 23/38] Fix Unified env setup --- .../run_integration_test.bash | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index e1cc2294..7f0d2b30 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -217,6 +217,20 @@ activate_env() { set -u } +# Activate the machine-specific unified environment. +# UNIFIED_ENV_CMD is always "source /path/to/script.sh" (set in the +# machine-specific case block above), so we strip the leading "source " +# and source the path directly -- no eval required. +activate_unified_env() { + set +u + # shellcheck disable=SC1090 + source ~/.bashrc + $CONDA_ACTIVATION_CMD + # shellcheck disable=SC1090 + source "${UNIFIED_ENV_CMD#source }" + set -u +} + # Create (if needed) and activate a conda environment. setup_conda_env() { local conda_dir="$1" # Directory containing dev.yml (e.g. "conda" or "conda-env") @@ -509,7 +523,8 @@ phase_1_setup() { setup_conda_env "conda" "$ZI_ENV" fi else - log "Using unified env for zppy-interfaces (skipping conda env creation)" + log "Using unified env for zppy-interfaces..." + activate_unified_env fi log "Running zppy-interfaces unit tests..." From 7c970d34b6a73e5416542cef93129728b62bbdc9 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Tue, 16 Jun 2026 09:19:49 -0700 Subject: [PATCH 24/38] Log the environment --- tests/main_branch_testing/run_integration_test.bash | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index 7f0d2b30..dee0dff7 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -21,6 +21,8 @@ set -e # Exit on error set -u # Exit on undefined variable +SCRIPT_RUN_DIR="$PWD" + # ============================================================================ # Parse arguments # ============================================================================ @@ -642,6 +644,7 @@ PYEOF # Submit initial SLURM jobs # ------------------------------------------------------------------ log "Submitting SLURM jobs..." + env > "${SCRIPT_RUN_DIR}/env_${TAG}.txt" local cfg cfg_path for cfg in "${CFGS_ARRAY[@]}"; do cfg="${cfg// /}" From 44b8d9067320c13257cd85590bcfd43e8e9ac674 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Tue, 16 Jun 2026 11:18:41 -0700 Subject: [PATCH 25/38] Fix test cfg envs --- .../template_weekly_legacy_3.1.0_comprehensive_v2.cfg | 1 + .../template_weekly_legacy_3.1.0_comprehensive_v3.cfg | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/integration/template_weekly_legacy_3.1.0_comprehensive_v2.cfg b/tests/integration/template_weekly_legacy_3.1.0_comprehensive_v2.cfg index 6d570441..69bde1e8 100644 --- a/tests/integration/template_weekly_legacy_3.1.0_comprehensive_v2.cfg +++ b/tests/integration/template_weekly_legacy_3.1.0_comprehensive_v2.cfg @@ -174,6 +174,7 @@ active = #expand active_mpas_analysis# anomalyRefYear = 1980 climo_years ="1980-1984", "1985-1990", enso_years = "1980-1984", "1985-1990", +environment_commands = "#expand mpas_analysis_environment_commands#" mesh = "EC30to60E2r2" parallelTaskCount = 6 partition = "#expand partition_long#" diff --git a/tests/integration/template_weekly_legacy_3.1.0_comprehensive_v3.cfg b/tests/integration/template_weekly_legacy_3.1.0_comprehensive_v3.cfg index ad39a309..2f924d23 100644 --- a/tests/integration/template_weekly_legacy_3.1.0_comprehensive_v3.cfg +++ b/tests/integration/template_weekly_legacy_3.1.0_comprehensive_v3.cfg @@ -207,6 +207,7 @@ active = #expand active_mpas_analysis# anomalyRefYear = 1985 climo_years = "1985-1989", "1990-1995", enso_years = "1985-1989", "1990-1995", +environment_commands = "#expand mpas_analysis_environment_commands#" mesh = "IcoswISC30E3r5" parallelTaskCount = 6 partition = "#expand partition_long#" From 412db04c517fd26ca3b3876776cb389cd41d9db3 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Tue, 16 Jun 2026 13:02:44 -0700 Subject: [PATCH 26/38] Improve dependency granularity --- tests/integration/template_weekly_bundles.cfg | 15 ++-- .../template_weekly_comprehensive_v2.cfg | 21 +++--- .../template_weekly_comprehensive_v3.cfg | 38 +++++----- .../template_weekly_legacy_3.0.0_bundles.cfg | 15 ++-- ...e_weekly_legacy_3.0.0_comprehensive_v2.cfg | 21 +++--- ...e_weekly_legacy_3.0.0_comprehensive_v3.cfg | 21 +++--- .../template_weekly_legacy_3.1.0_bundles.cfg | 15 ++-- ...e_weekly_legacy_3.1.0_comprehensive_v2.cfg | 21 +++--- ...e_weekly_legacy_3.1.0_comprehensive_v3.cfg | 21 +++--- tests/integration/utils.py | 70 ++++++++++++++++--- 10 files changed, 178 insertions(+), 80 deletions(-) diff --git a/tests/integration/template_weekly_bundles.cfg b/tests/integration/template_weekly_bundles.cfg index ae94c1d8..9f164ebb 100644 --- a/tests/integration/template_weekly_bundles.cfg +++ b/tests/integration/template_weekly_bundles.cfg @@ -51,10 +51,11 @@ bundle = "bundle1" years = "1985:1989:2", [[ atm_monthly_180x360_aave ]] + active = #expand active_climo_month_atm# frequency = "monthly" [[ atm_monthly_diurnal_8xdaily_180x360_aave ]] - active = #expand active_e3sm_diags# + active = #expand active_climo_diurnal_atm# frequency = "diurnal_8xdaily" input_files = "eam.h3" input_subdir = "archive/atm/hist" @@ -66,12 +67,13 @@ bundle = "bundle1" years = "1985:1989:2", [[ atm_monthly_180x360_aave ]] + active = #expand active_ts_month_atm# frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" [[ atm_monthly_glb ]] - active = #expand active_global_time_series# + active = #expand active_ts_month_atm_glb# bundle = "bundle2" # Override bundle1 frequency = "monthly" input_files = "eam.h0" @@ -80,6 +82,7 @@ years = "1985:1989:2", years = "1985:1995:5", [[ land_monthly ]] + active = #expand active_ts_month_lnd# extra_vars = "landfrac" frequency = "monthly" input_files = "elm.h0" @@ -88,7 +91,7 @@ years = "1985:1989:2", vars = "LAISHA,LAISUN" [[ rof_monthly ]] - active = #expand active_e3sm_diags# + active = #expand active_ts_month_rof# bundle = "bundle3" # Override bundle1, let bundle1 finish first because "e3sm_diags: atm_monthly_180x360_aave_mvm" requires "ts: atm_monthly_180x360_aave" extra_vars = 'areatotal2' frequency = "monthly" @@ -98,7 +101,7 @@ years = "1985:1989:2", vars = "RIVER_DISCHARGE_OVER_LAND_LIQ" [e3sm_to_cmip] -active = #expand active_e3sm_to_cmip# +active = True bundle = "bundle1" environment_commands = "#expand e3sm_to_cmip_environment_commands#" frequency = "monthly" @@ -106,14 +109,16 @@ ts_num_years = 2 years = "1985:1989:2", [[ atm_monthly_180x360_aave ]] + active = #expand active_e3sm_to_cmip_month_atm# input_files = "eam.h0" [[ land_monthly ]] + active = #expand active_e3sm_to_cmip_month_lnd# input_files = "elm.h0" # TODO: Add "tc_analysis" back in after empty dat is resolved. # [tc_analysis] -# active = True +# active = #expand active_tc_analysis# # bundle = "bundle3" # Let bundle1 finish first because "e3sm_diags: atm_monthly_180x360_aave_mvm" requires "ts: atm_monthly_180x360_aave" # years = "1985:1989:2", diff --git a/tests/integration/template_weekly_comprehensive_v2.cfg b/tests/integration/template_weekly_comprehensive_v2.cfg index 7ba7aef8..9f3cec9b 100644 --- a/tests/integration/template_weekly_comprehensive_v2.cfg +++ b/tests/integration/template_weekly_comprehensive_v2.cfg @@ -19,20 +19,21 @@ active = True walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] + active = #expand active_climo_month_atm# frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" vars = "" [[ atm_monthly_diurnal_8xdaily_180x360_aave ]] - active = #expand active_e3sm_diags# + active = #expand active_climo_diurnal_atm# frequency = "diurnal_8xdaily" input_files = "eam.h3" input_subdir = "archive/atm/hist" vars = "PRECT" [[ land_monthly_climo ]] - active = #expand active_e3sm_diags# + active = #expand active_climo_month_lnd# frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" @@ -43,19 +44,20 @@ active = True walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] + active = #expand active_ts_month_atm# frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" [[ atm_daily_180x360_aave ]] - active = #expand active_e3sm_diags# + active = #expand active_ts_daily_atm# frequency = "daily" input_files = "eam.h1" input_subdir = "archive/atm/hist" vars = "PRECT" [[ rof_monthly ]] - active = #expand active_e3sm_diags# + active = #expand active_ts_month_rof# extra_vars = 'areatotal2' frequency = "monthly" input_files = "mosart.h0" @@ -65,7 +67,7 @@ walltime = "00:30:00" [[ atm_monthly_glb ]] # Note global average won't work for 3D variables. - active = #expand active_global_time_series# + active = #expand active_ts_month_atm_glb# frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" @@ -73,7 +75,7 @@ walltime = "00:30:00" years = "1980:1990:5", [[ lnd_monthly_glb ]] - active = #expand active_global_time_series# + active = #expand active_ts_month_lnd_glb# frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" @@ -82,6 +84,7 @@ walltime = "00:30:00" years = "1980:1990:5", [[ land_monthly ]] + active = #expand active_ts_month_lnd# extra_vars = "landfrac" frequency = "monthly" input_files = "elm.h0" @@ -89,20 +92,22 @@ walltime = "00:30:00" vars = "LAISHA,LAISUN" [e3sm_to_cmip] -active = #expand active_e3sm_to_cmip# +active = True environment_commands = "#expand e3sm_to_cmip_environment_commands#" frequency = "monthly" ts_num_years=2 walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] + active = #expand active_e3sm_to_cmip_month_atm# input_files = "eam.h0" [[ land_monthly ]] + active = #expand active_e3sm_to_cmip_month_lnd# input_files = "elm.h0" [tc_analysis] -active = #expand active_e3sm_diags# +active = #expand active_tc_analysis# walltime = "00:30:00" [e3sm_diags] diff --git a/tests/integration/template_weekly_comprehensive_v3.cfg b/tests/integration/template_weekly_comprehensive_v3.cfg index 5c3d1f0b..75535312 100644 --- a/tests/integration/template_weekly_comprehensive_v3.cfg +++ b/tests/integration/template_weekly_comprehensive_v3.cfg @@ -22,26 +22,27 @@ frequency = "monthly" walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] + active = #expand active_climo_month_atm# input_files = "eam.h0" input_subdir = "archive/atm/hist" vars = "" [[ atm_monthly_diurnal_8xdaily_180x360_aave ]] - active = #expand active_e3sm_diags# + active = #expand active_climo_diurnal_atm# frequency = "diurnal_8xdaily" input_files = "eam.h3" input_subdir = "archive/atm/hist" vars = "PRECT" [[ land_monthly_climo ]] - active = True + active = #expand active_climo_month_lnd# input_files = "elm.h0" input_subdir = "archive/lnd/hist" mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" vars = "" [[ land_monthly_180x360_traave ]] - active = #expand active_livvkit# + active = #expand active_climo_month_lnd_for_livvkit# input_files = "elm.h0" input_subdir = "archive/lnd/hist" mapping_file = "#expand livvkit_mapping_file_path#/map_r05_to_cmip6_180x360_traave.20231110.nc" @@ -49,7 +50,7 @@ walltime = "00:30:00" years = "1985:1994:10", [[ land_monthly_climo_native ]] - active = #expand active_livvkit# + active = #expand active_climo_month_lnd_for_livvkit# climo_jobs = 12 input_files = "elm.h0" input_subdir = "archive/lnd/hist" @@ -58,7 +59,7 @@ walltime = "00:30:00" years = "1985:1994:10" [[ land_monthly_climo_racmo_gis ]] - active = #expand active_livvkit# + active = #expand active_climo_month_lnd_for_livvkit# climo_jobs = 12 climo_subsection = "racmo_gis" grid = "racmo_gis" @@ -69,7 +70,7 @@ walltime = "00:30:00" years = "1985:1994:10" [[ land_monthly_climo_racmo_ais ]] - active = #expand active_livvkit# + active = #expand active_climo_month_lnd_for_livvkit# climo_jobs = 12 input_files = "elm.h0" climo_subsection = "racmo_ais" @@ -80,7 +81,7 @@ walltime = "00:30:00" years = "1985:1994:10" [[ land_monthly_climo_merra2 ]] - active = #expand active_livvkit# + active = #expand active_climo_month_lnd_for_livvkit# climo_jobs = 12 input_files = "elm.h0" climo_subsection = "merra2" @@ -91,7 +92,7 @@ walltime = "00:30:00" years = "1985:1994:10" [[ land_monthly_climo_era5 ]] - active = #expand active_livvkit# + active = #expand active_climo_month_lnd_for_livvkit# climo_jobs = 12 input_files = "elm.h0" climo_subsection = "era5" @@ -106,6 +107,7 @@ active = True walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] + active = #expand active_ts_month_atm# frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" @@ -115,14 +117,14 @@ walltime = "00:30:00" vars = "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,PSL,LANDFRAC,CLD_CAL_TMPICE,CLD_CAL_TMPLIQ,CLDLIQ,T" [[ atm_daily_180x360_aave ]] - active = #expand active_e3sm_diags# + active = #expand active_ts_daily_atm# frequency = "daily" input_files = "eam.h1" input_subdir = "archive/atm/hist" vars = "PRECT" [[ rof_monthly ]] - active = #expand active_e3sm_diags# + active = #expand active_ts_month_rof# extra_vars = 'areatotal2' frequency = "monthly" input_files = "mosart.h0" @@ -132,7 +134,7 @@ walltime = "00:30:00" [[ atm_monthly_glb ]] # Note global average won't work for 3D variables. - active = #expand active_global_time_series# + active = #expand active_ts_month_atm_glb# frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" @@ -140,7 +142,7 @@ walltime = "00:30:00" years = "1985:1995:5", [[ lnd_monthly_glb ]] - active = #expand active_global_time_series# + active = #expand active_ts_month_lnd_glb# frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" @@ -149,6 +151,7 @@ walltime = "00:30:00" years = "1985:1995:5", [[ land_monthly ]] + active = #expand active_ts_month_lnd# extra_vars = "landfrac" frequency = "monthly" input_files = "elm.h0" @@ -157,7 +160,7 @@ walltime = "00:30:00" vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" [[ land_monthly_energy ]] - active = #expand active_livvkit# + active = #expand active_ts_month_lnd_for_livvkit# frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" @@ -166,8 +169,7 @@ walltime = "00:30:00" years = "1985:1994:10" [[ land_monthly_smb ]] - # The data necessary for this is not found in the data input path. - active = False + active = False # The data necessary for this is not found in the data input path. frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" @@ -176,13 +178,14 @@ walltime = "00:30:00" years = "1985:1994:10" [e3sm_to_cmip] -active = #expand active_e3sm_to_cmip# +active = True environment_commands = "#expand e3sm_to_cmip_environment_commands#" frequency = "monthly" ts_num_years=2 walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] + active = #expand active_e3sm_to_cmip_month_atm# cmip_plevdata = "#expand diagnostics_base_path#/e3sm_to_cmip_data/maps/vrt_remap_plev19.nc" cmip_vars = "ua, va, ta, wap, zg, hur, tas, ts, psl, ps, sfcWind, huss, pr, prc, prsn, evspsbl, tauu, tauv, hfls, clt, rlus, rsds, rsus, hfss, clivi, clwvi, rlut, rsdt, rsuscs, rsut, rtmt, abs550aer, od550aer, rsdscs, tasmax, tasmin" input_files = "eam.h0" @@ -191,12 +194,13 @@ walltime = "00:30:00" years = "1985:1995:2", # Need 10 years for pcmdi_diags task [[ land_monthly ]] + active = #expand active_e3sm_to_cmip_month_lnd# input_files = "elm.h0" ts_subsection = "land_monthly" # TODO: Add "tc_analysis" back in after empty dat is resolved. # [tc_analysis] -# active = True +# active = #expand active_tc_analysis# # walltime = "00:30:00" [e3sm_diags] diff --git a/tests/integration/template_weekly_legacy_3.0.0_bundles.cfg b/tests/integration/template_weekly_legacy_3.0.0_bundles.cfg index fac3483b..800a3c8b 100644 --- a/tests/integration/template_weekly_legacy_3.0.0_bundles.cfg +++ b/tests/integration/template_weekly_legacy_3.0.0_bundles.cfg @@ -53,10 +53,11 @@ bundle = "bundle1" years = "1985:1989:2", [[ atm_monthly_180x360_aave ]] + active = #expand active_climo_month_atm# frequency = "monthly" [[ atm_monthly_diurnal_8xdaily_180x360_aave ]] - active = #expand active_e3sm_diags# + active = #expand active_climo_diurnal_atm# frequency = "diurnal_8xdaily" input_files = "eam.h3" input_subdir = "archive/atm/hist" @@ -68,12 +69,13 @@ bundle = "bundle1" years = "1985:1989:2", [[ atm_monthly_180x360_aave ]] + active = #expand active_ts_month_atm# frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" [[ atm_monthly_glb ]] - active = #expand active_global_time_series# + active = #expand active_ts_month_atm_glb# bundle = "bundle2" # Override bundle1 frequency = "monthly" input_files = "eam.h0" @@ -82,6 +84,7 @@ years = "1985:1989:2", years = "1985:1995:5", [[ land_monthly ]] + active = #expand active_ts_month_lnd# extra_vars = "landfrac" frequency = "monthly" input_files = "elm.h0" @@ -90,7 +93,7 @@ years = "1985:1989:2", vars = "LAISHA,LAISUN" [[ rof_monthly ]] - active = #expand active_e3sm_diags# + active = #expand active_ts_month_rof# bundle = "bundle3" # Override bundle1, let bundle1 finish first because "e3sm_diags: atm_monthly_180x360_aave_mvm" requires "ts: atm_monthly_180x360_aave" extra_vars = 'areatotal2' frequency = "monthly" @@ -100,7 +103,7 @@ years = "1985:1989:2", vars = "RIVER_DISCHARGE_OVER_LAND_LIQ" [e3sm_to_cmip] -active = #expand active_e3sm_to_cmip# +active = True bundle = "bundle1" environment_commands = "#expand e3sm_to_cmip_environment_commands#" frequency = "monthly" @@ -108,14 +111,16 @@ ts_num_years = 2 years = "1985:1989:2", [[ atm_monthly_180x360_aave ]] + active = #expand active_e3sm_to_cmip_month_atm# input_files = "eam.h0" [[ land_monthly ]] + active = #expand active_e3sm_to_cmip_month_lnd# input_files = "elm.h0" # TODO: Add "tc_analysis" back in after empty dat is resolved. # [tc_analysis] -# active = True +# active = #expand active_tc_analysis# # bundle = "bundle3" # Let bundle1 finish first because "e3sm_diags: atm_monthly_180x360_aave_mvm" requires "ts: atm_monthly_180x360_aave" # years = "1985:1989:2", diff --git a/tests/integration/template_weekly_legacy_3.0.0_comprehensive_v2.cfg b/tests/integration/template_weekly_legacy_3.0.0_comprehensive_v2.cfg index 2fd687c8..559e574f 100644 --- a/tests/integration/template_weekly_legacy_3.0.0_comprehensive_v2.cfg +++ b/tests/integration/template_weekly_legacy_3.0.0_comprehensive_v2.cfg @@ -21,20 +21,21 @@ active = True walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] + active = #expand active_climo_month_atm# frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" vars = "" [[ atm_monthly_diurnal_8xdaily_180x360_aave ]] - active = #expand active_e3sm_diags# + active = #expand active_climo_diurnal_atm# frequency = "diurnal_8xdaily" input_files = "eam.h3" input_subdir = "archive/atm/hist" vars = "PRECT" [[ land_monthly_climo ]] - active = #expand active_e3sm_diags# + active = #expand active_climo_month_lnd# frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" @@ -45,19 +46,20 @@ active = True walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] + active = #expand active_ts_month_atm# frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" [[ atm_daily_180x360_aave ]] - active = #expand active_e3sm_diags# + active = #expand active_ts_daily_atm# frequency = "daily" input_files = "eam.h1" input_subdir = "archive/atm/hist" vars = "PRECT" [[ rof_monthly ]] - active = #expand active_e3sm_diags# + active = #expand active_ts_month_rof# extra_vars = 'areatotal2' frequency = "monthly" input_files = "mosart.h0" @@ -67,7 +69,7 @@ walltime = "00:30:00" [[ atm_monthly_glb ]] # Note global average won't work for 3D variables. - active = #expand active_global_time_series# + active = #expand active_ts_month_atm_glb# frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" @@ -75,7 +77,7 @@ walltime = "00:30:00" years = "1980:1990:5", [[ lnd_monthly_glb ]] - active = #expand active_global_time_series# + active = #expand active_ts_month_lnd_glb# frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" @@ -84,6 +86,7 @@ walltime = "00:30:00" years = "1980:1990:5", [[ land_monthly ]] + active = #expand active_ts_month_lnd# extra_vars = "landfrac" frequency = "monthly" input_files = "elm.h0" @@ -91,20 +94,22 @@ walltime = "00:30:00" vars = "LAISHA,LAISUN" [e3sm_to_cmip] -active = #expand active_e3sm_to_cmip# +active = True environment_commands = "#expand e3sm_to_cmip_environment_commands#" frequency = "monthly" ts_num_years=2 walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] + active = #expand active_e3sm_to_cmip_month_atm# input_files = "eam.h0" [[ land_monthly ]] + active = #expand active_e3sm_to_cmip_month_lnd# input_files = "elm.h0" [tc_analysis] -active = #expand active_e3sm_diags# +active = #expand active_tc_analysis# walltime = "00:30:00" [e3sm_diags] diff --git a/tests/integration/template_weekly_legacy_3.0.0_comprehensive_v3.cfg b/tests/integration/template_weekly_legacy_3.0.0_comprehensive_v3.cfg index 8b57e400..b87057c0 100644 --- a/tests/integration/template_weekly_legacy_3.0.0_comprehensive_v3.cfg +++ b/tests/integration/template_weekly_legacy_3.0.0_comprehensive_v3.cfg @@ -28,20 +28,21 @@ active = True walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] + active = #expand active_climo_month_atm# frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" vars = "" [[ atm_monthly_diurnal_8xdaily_180x360_aave ]] - active = #expand active_e3sm_diags# + active = #expand active_climo_diurnal_atm# frequency = "diurnal_8xdaily" input_files = "eam.h3" input_subdir = "archive/atm/hist" vars = "PRECT" [[ land_monthly_climo ]] - active = #expand active_e3sm_diags# + active = #expand active_climo_month_lnd# frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" @@ -53,19 +54,20 @@ active = True walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] + active = #expand active_ts_month_atm# frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" [[ atm_daily_180x360_aave ]] - active = #expand active_e3sm_diags# + active = #expand active_ts_daily_atm# frequency = "daily" input_files = "eam.h1" input_subdir = "archive/atm/hist" vars = "PRECT" [[ rof_monthly ]] - active = #expand active_e3sm_diags# + active = #expand active_ts_month_rof# extra_vars = 'areatotal2' frequency = "monthly" input_files = "mosart.h0" @@ -75,7 +77,7 @@ walltime = "00:30:00" [[ atm_monthly_glb ]] # Note global average won't work for 3D variables. - active = #expand active_global_time_series# + active = #expand active_ts_month_atm_glb# frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" @@ -83,7 +85,7 @@ walltime = "00:30:00" years = "1985:1995:5", [[ lnd_monthly_glb ]] - active = #expand active_global_time_series# + active = #expand active_ts_month_lnd_glb# frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" @@ -92,6 +94,7 @@ walltime = "00:30:00" years = "1985:1995:5", [[ land_monthly ]] + active = #expand active_ts_month_lnd# extra_vars = "landfrac" frequency = "monthly" input_files = "elm.h0" @@ -100,23 +103,25 @@ walltime = "00:30:00" vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" [e3sm_to_cmip] -active = #expand active_e3sm_to_cmip# +active = True environment_commands = "#expand e3sm_to_cmip_environment_commands#" frequency = "monthly" ts_num_years=2 walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] + active = #expand active_e3sm_to_cmip_month_atm# input_files = "eam.h0" ts_subsection = "atm_monthly_180x360_aave" [[ land_monthly ]] + active = #expand active_e3sm_to_cmip_month_lnd# input_files = "elm.h0" ts_subsection = "land_monthly" # TODO: Add "tc_analysis" back in after empty dat is resolved. # [tc_analysis] -# active = True +# active = #expand active_tc_analysis# # walltime = "00:30:00" [e3sm_diags] diff --git a/tests/integration/template_weekly_legacy_3.1.0_bundles.cfg b/tests/integration/template_weekly_legacy_3.1.0_bundles.cfg index 368303dc..0aabba8d 100644 --- a/tests/integration/template_weekly_legacy_3.1.0_bundles.cfg +++ b/tests/integration/template_weekly_legacy_3.1.0_bundles.cfg @@ -53,10 +53,11 @@ bundle = "bundle1" years = "1985:1989:2", [[ atm_monthly_180x360_aave ]] + active = #expand active_climo_month_atm# frequency = "monthly" [[ atm_monthly_diurnal_8xdaily_180x360_aave ]] - active = #expand active_e3sm_diags# + active = #expand active_climo_diurnal_atm# frequency = "diurnal_8xdaily" input_files = "eam.h3" input_subdir = "archive/atm/hist" @@ -68,12 +69,13 @@ bundle = "bundle1" years = "1985:1989:2", [[ atm_monthly_180x360_aave ]] + active = #expand active_ts_month_atm# frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" [[ atm_monthly_glb ]] - active = #expand active_global_time_series# + active = #expand active_ts_month_atm_glb# bundle = "bundle2" # Override bundle1 frequency = "monthly" input_files = "eam.h0" @@ -82,6 +84,7 @@ years = "1985:1989:2", years = "1985:1995:5", [[ land_monthly ]] + active = #expand active_ts_month_lnd# extra_vars = "landfrac" frequency = "monthly" input_files = "elm.h0" @@ -90,7 +93,7 @@ years = "1985:1989:2", vars = "LAISHA,LAISUN" [[ rof_monthly ]] - active = #expand active_e3sm_diags# + active = #expand active_ts_month_rof# bundle = "bundle3" # Override bundle1, let bundle1 finish first because "e3sm_diags: atm_monthly_180x360_aave_mvm" requires "ts: atm_monthly_180x360_aave" extra_vars = 'areatotal2' frequency = "monthly" @@ -100,7 +103,7 @@ years = "1985:1989:2", vars = "RIVER_DISCHARGE_OVER_LAND_LIQ" [e3sm_to_cmip] -active = #expand active_e3sm_to_cmip# +active = True bundle = "bundle1" environment_commands = "#expand e3sm_to_cmip_environment_commands#" frequency = "monthly" @@ -108,14 +111,16 @@ ts_num_years = 2 years = "1985:1989:2", [[ atm_monthly_180x360_aave ]] + #expand active_e3sm_to_cmip_month_atm# input_files = "eam.h0" [[ land_monthly ]] + #expand active_e3sm_to_cmip_month_lnd# input_files = "elm.h0" # TODO: Add "tc_analysis" back in after empty dat is resolved. # [tc_analysis] -# active = True +# active = #expand active_tc_analysis# # bundle = "bundle3" # Let bundle1 finish first because "e3sm_diags: atm_monthly_180x360_aave_mvm" requires "ts: atm_monthly_180x360_aave" # years = "1985:1989:2", diff --git a/tests/integration/template_weekly_legacy_3.1.0_comprehensive_v2.cfg b/tests/integration/template_weekly_legacy_3.1.0_comprehensive_v2.cfg index 69bde1e8..ea171c41 100644 --- a/tests/integration/template_weekly_legacy_3.1.0_comprehensive_v2.cfg +++ b/tests/integration/template_weekly_legacy_3.1.0_comprehensive_v2.cfg @@ -21,20 +21,21 @@ active = True walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] + active = #expand active_climo_month_atm# frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" vars = "" [[ atm_monthly_diurnal_8xdaily_180x360_aave ]] - active = #expand active_e3sm_diags# + active = #expand active_climo_diurnal_atm# frequency = "diurnal_8xdaily" input_files = "eam.h3" input_subdir = "archive/atm/hist" vars = "PRECT" [[ land_monthly_climo ]] - active = #expand active_e3sm_diags# + active = #expand active_climo_month_lnd# frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" @@ -45,19 +46,20 @@ active = True walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] + active = #expand active_ts_month_atm# frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" [[ atm_daily_180x360_aave ]] - active = #expand active_e3sm_diags# + active = #expand active_ts_daily_atm# frequency = "daily" input_files = "eam.h1" input_subdir = "archive/atm/hist" vars = "PRECT" [[ rof_monthly ]] - active = #expand active_e3sm_diags# + active = #expand active_ts_month_rof# extra_vars = 'areatotal2' frequency = "monthly" input_files = "mosart.h0" @@ -67,7 +69,7 @@ walltime = "00:30:00" [[ atm_monthly_glb ]] # Note global average won't work for 3D variables. - active = #expand active_global_time_series# + active = #expand active_ts_month_atm_glb# frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" @@ -75,7 +77,7 @@ walltime = "00:30:00" years = "1980:1990:5", [[ lnd_monthly_glb ]] - active = #expand active_global_time_series# + active = #expand active_ts_month_lnd_glb# frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" @@ -84,6 +86,7 @@ walltime = "00:30:00" years = "1980:1990:5", [[ land_monthly ]] + active = #expand active_ts_month_lnd# extra_vars = "landfrac" frequency = "monthly" input_files = "elm.h0" @@ -91,20 +94,22 @@ walltime = "00:30:00" vars = "LAISHA,LAISUN" [e3sm_to_cmip] -active = #expand active_e3sm_to_cmip# +active = True environment_commands = "#expand e3sm_to_cmip_environment_commands#" frequency = "monthly" ts_num_years=2 walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] + active = #expand active_e3sm_to_cmip_month_atm# input_files = "eam.h0" [[ land_monthly ]] + active = #expand active_e3sm_to_cmip_month_lnd# input_files = "elm.h0" [tc_analysis] -active = #expand active_e3sm_diags# +active = #expand active_tc_analysis# walltime = "00:30:00" [e3sm_diags] diff --git a/tests/integration/template_weekly_legacy_3.1.0_comprehensive_v3.cfg b/tests/integration/template_weekly_legacy_3.1.0_comprehensive_v3.cfg index 2f924d23..2c1e65de 100644 --- a/tests/integration/template_weekly_legacy_3.1.0_comprehensive_v3.cfg +++ b/tests/integration/template_weekly_legacy_3.1.0_comprehensive_v3.cfg @@ -23,20 +23,21 @@ active = True walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] + active = #expand active_climo_month_atm# frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" vars = "" [[ atm_monthly_diurnal_8xdaily_180x360_aave ]] - active = #expand active_e3sm_diags# + active = #expand active_climo_diurnal_atm# frequency = "diurnal_8xdaily" input_files = "eam.h3" input_subdir = "archive/atm/hist" vars = "PRECT" [[ land_monthly_climo ]] - active = #expand active_e3sm_diags# + active = #expand active_climo_month_lnd# frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" @@ -48,20 +49,21 @@ active = True walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] + active = #expand active_ts_month_atm# frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" years = "1985:1995:2", # Need 10 years for pcmdi_diags task [[ atm_daily_180x360_aave ]] - active = #expand active_e3sm_diags# + active = #expand active_ts_daily_atm# frequency = "daily" input_files = "eam.h1" input_subdir = "archive/atm/hist" vars = "PRECT" [[ rof_monthly ]] - active = #expand active_e3sm_diags# + active = #expand active_ts_month_rof# extra_vars = 'areatotal2' frequency = "monthly" input_files = "mosart.h0" @@ -71,7 +73,7 @@ walltime = "00:30:00" [[ atm_monthly_glb ]] # Note global average won't work for 3D variables. - active = #expand active_global_time_series# + active = #expand active_ts_month_atm_glb# frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" @@ -79,7 +81,7 @@ walltime = "00:30:00" years = "1985:1995:5", [[ lnd_monthly_glb ]] - active = #expand active_global_time_series# + active = #expand active_ts_month_lnd_glb# frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" @@ -88,6 +90,7 @@ walltime = "00:30:00" years = "1985:1995:5", [[ land_monthly ]] + active = #expand active_ts_month_lnd# extra_vars = "landfrac" frequency = "monthly" input_files = "elm.h0" @@ -96,13 +99,14 @@ walltime = "00:30:00" vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" [e3sm_to_cmip] -active = #expand active_e3sm_to_cmip# +active = True environment_commands = "#expand e3sm_to_cmip_environment_commands#" frequency = "monthly" ts_num_years=2 walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] + active = #expand active_e3sm_to_cmip_month_atm# cmip_plevdata = "#expand diagnostics_base_path#/e3sm_to_cmip_data/maps/vrt_remap_plev19.nc" cmip_vars = "ua, va, ta, wap, zg, hur, tas, ts, psl, ps, sfcWind, huss, pr, prc, prsn, evspsbl, tauu, tauv, hfls, clt, rlus, rsds, rsus, hfss, clivi, clwvi, rlut, rsdt, rsuscs, rsut, rtmt, abs550aer, od550aer, rsdscs, tasmax, tasmin" input_files = "eam.h0" @@ -111,12 +115,13 @@ walltime = "00:30:00" years = "1985:1995:2", # Need 10 years for pcmdi_diags task [[ land_monthly ]] + active = #expand active_e3sm_to_cmip_month_lnd# input_files = "elm.h0" ts_subsection = "land_monthly" # TODO: Add "tc_analysis" back in after empty dat is resolved. # [tc_analysis] -# active = True +# active = #expand active_tc_analysis# # walltime = "00:30:00" [e3sm_diags] diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 17519e47..df65e34d 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -180,29 +180,83 @@ def get_expansions(): expansions["environment_commands"] = TEST_SPECIFICS["environment_commands"] # Activate requested tests - expansions["active_e3sm_to_cmip"] = "False" + + # Dependencies + expansions["active_climo_month_atm"] = "False" # For e3sm_diags + expansions["active_climo_month_lnd"] = "False" # For e3sm_diags + expansions["active_climo_month_lnd_for_livvkit"] = "False" # For livvkit + expansions["active_climo_diurnal_atm"] = "False" # For e3sm_diags + + expansions["active_ts_daily_atm"] = "False" # For e3sm_diags + expansions["active_ts_month_atm"] = "False" # For e3sm_diags, ilamb, pcmdi_diags + expansions["active_ts_month_atm_glb"] = "False" # For global_time_series + expansions["active_ts_month_lnd"] = "False" # For ilamb + expansions["active_ts_month_lnd_for_livvkit"] = "False" # For livvkit + expansions["active_ts_month_lnd_glb"] = "False" # For global_time_series + expansions["active_ts_month_rof"] = "False" # For e3sm_diags + + expansions["active_e3sm_to_cmip_month_atm"] = "False" # For ilamb, pcmdi_diags + expansions["active_e3sm_to_cmip_month_lnd"] = "False" # For ilamb + + expansions["active_tc_analysis"] = "False" # For e3sm_diags + + # Plotting packages expansions["active_e3sm_diags"] = "False" - expansions["active_mpas_analysis"] = "False" + expansions["active_mpas_analysis"] = "False" # Also used for global_time_series expansions["active_global_time_series"] = "False" expansions["active_ilamb"] = "False" expansions["active_livvkit"] = "False" expansions["active_pcmdi_diags"] = "False" + + # TODO: Continue adding dependencies to all tasks, update all 9 test cfgs + if "e3sm_diags" in TEST_SPECIFICS["tasks_to_run"]: expansions["active_e3sm_diags"] = "True" + + expansions["active_climo_month_atm"] = "True" + expansions["active_climo_month_lnd"] = "True" + expansions["active_climo_diurnal_atm"] = "True" + + expansions["active_ts_month_atm"] = "True" + expansions["active_ts_month_rof"] = "True" + expansions["active_ts_daily_atm"] = "True" + + expansions["active_tc_analysis"] = "True" + if "mpas_analysis" in TEST_SPECIFICS["tasks_to_run"]: expansions["active_mpas_analysis"] = "True" + if "global_time_series" in TEST_SPECIFICS["tasks_to_run"]: expansions["active_global_time_series"] = "True" - expansions["active_mpas_analysis"] = "True" # For ocn plots - expansions["active_e3sm_to_cmip"] = "True" # For lnd plots - if "livvkit" in TEST_SPECIFICS["tasks_to_run"]: - expansions["active_livvkit"] = "True" + + expansions["active_ts_month_atm_glb"] = "True" + expansions["active_ts_month_lnd_glb"] = "True" + + expansions["active_mpas_analysis"] = "True" + if "ilamb" in TEST_SPECIFICS["tasks_to_run"]: expansions["active_ilamb"] = "True" - expansions["active_e3sm_to_cmip"] = "True" + + expansions["active_ts_month_atm"] = "True" + expansions["active_ts_month_lnd"] = "True" + + expansions["active_e3sm_to_cmip_month_atm"] = "True" + expansions["active_e3sm_to_cmip_month_lnd"] = "True" + + if "livvkit" in TEST_SPECIFICS["tasks_to_run"]: + expansions["active_livvkit"] = "True" + + expansions["active_climo_month_lnd_for_livvkit"] = "True" + + expansions["active_ts_month_lnd_for_livvkit"] = "True" + if "pcmdi_diags" in TEST_SPECIFICS["tasks_to_run"]: expansions["active_pcmdi_diags"] = "True" - expansions["active_e3sm_to_cmip"] = "True" + + expansions["active_ts_month_atm"] = "True" + + expansions["active_e3sm_to_cmip_month_atm"] = "True" + expansions["cfgs_to_run"] = TEST_SPECIFICS["cfgs_to_run"] expansions["tasks_to_run"] = TEST_SPECIFICS["tasks_to_run"] From df8648906f1be2f9dff4f0f1e1a02c8b830fe47b Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Tue, 16 Jun 2026 13:20:54 -0700 Subject: [PATCH 27/38] Address comments --- .../integration/template_weekly_legacy_3.1.0_bundles.cfg | 4 ++-- tests/integration/utils.py | 2 -- tests/main_branch_testing/run_integration_test.bash | 9 +++++++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/integration/template_weekly_legacy_3.1.0_bundles.cfg b/tests/integration/template_weekly_legacy_3.1.0_bundles.cfg index 0aabba8d..60cdf6cb 100644 --- a/tests/integration/template_weekly_legacy_3.1.0_bundles.cfg +++ b/tests/integration/template_weekly_legacy_3.1.0_bundles.cfg @@ -111,11 +111,11 @@ ts_num_years = 2 years = "1985:1989:2", [[ atm_monthly_180x360_aave ]] - #expand active_e3sm_to_cmip_month_atm# + active = #expand active_e3sm_to_cmip_month_atm# input_files = "eam.h0" [[ land_monthly ]] - #expand active_e3sm_to_cmip_month_lnd# + active = #expand active_e3sm_to_cmip_month_lnd# input_files = "elm.h0" # TODO: Add "tc_analysis" back in after empty dat is resolved. diff --git a/tests/integration/utils.py b/tests/integration/utils.py index df65e34d..b82c9dec 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -208,8 +208,6 @@ def get_expansions(): expansions["active_livvkit"] = "False" expansions["active_pcmdi_diags"] = "False" - # TODO: Continue adding dependencies to all tasks, update all 9 test cfgs - if "e3sm_diags" in TEST_SPECIFICS["tasks_to_run"]: expansions["active_e3sm_diags"] = "True" diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index dee0dff7..f03371db 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -364,6 +364,11 @@ check_status_files() { return 1 fi + if ! compgen -G "${dir}/*status" > /dev/null; then + log_warning "$name: No status files found in ${dir}" + return 1 + fi + local errors errors=$(grep -v "OK" "${dir}"/*status 2>/dev/null || true) @@ -651,7 +656,7 @@ PYEOF cfg_path="tests/integration/generated/test_${cfg}_${MACHINE_CFG_SUFFIX}.cfg" if [[ ! -f "$cfg_path" ]]; then log_error "Config file not found: $cfg_path" - log_error "Check that CFGS_TO_RUN names do not include the 'weekly_' prefix." + log_error "Check that each CFGS_TO_RUN entry matches a cfg name generated by tests/integration/utils.py (e.g., weekly_comprehensive_v3)." exit 1 fi log "Submitting: $cfg_path" @@ -706,7 +711,7 @@ phase_2_bundles_part2() { cfg_path="tests/integration/generated/test_${cfg}_${MACHINE_CFG_SUFFIX}.cfg" if [[ ! -f "$cfg_path" ]]; then log_error "Config file not found: $cfg_path" - log_error "Check that CFGS_TO_RUN names do not include the 'weekly_' prefix." + log_error "Check that each CFGS_TO_RUN entry matches a cfg name generated by tests/integration/utils.py (e.g., weekly_bundles)." exit 1 fi log "Submitting: $cfg_path" From db664d3bfa8d085bfefe2e909175aaba8d81a0cd Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Fri, 19 Jun 2026 18:14:30 -0500 Subject: [PATCH 28/38] Update mpas shell --- tests/main_branch_testing/run_integration_test.bash | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index f03371db..5963ce62 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -483,6 +483,8 @@ phase_1_setup() { fi ( + conda deactivate 2>/dev/null || true + unset LD_LIBRARY_PATH PYTHONPATH LD_PRELOAD cd "$MPAS_ANALYSIS_DIR" ensure_test_branch "test_mpas_${TAG}" "$MPAS_BASE_BRANCH" From d5f4ac2db584f2f7c6648cfb58e1403a5817464a Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Fri, 19 Jun 2026 19:02:40 -0500 Subject: [PATCH 29/38] Undo mpas shell debugging --- tests/main_branch_testing/run_integration_test.bash | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index 5963ce62..f03371db 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -483,8 +483,6 @@ phase_1_setup() { fi ( - conda deactivate 2>/dev/null || true - unset LD_LIBRARY_PATH PYTHONPATH LD_PRELOAD cd "$MPAS_ANALYSIS_DIR" ensure_test_branch "test_mpas_${TAG}" "$MPAS_BASE_BRANCH" From 0aa5dc02ae3d31c62a7efbe42f8eb7d14dc73928 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 1 Jul 2026 11:50:30 -0500 Subject: [PATCH 30/38] Add debugging for mpas-analysis --- .../run_integration_test.bash | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index f03371db..e0814ef7 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -203,6 +203,71 @@ checkpoint() { fi } +# ============================================================================ +# Debug Helper Functions +# ============================================================================ + +# Capture a full environment snapshot to both stdout and a log file. +# Usage: debug_env_snapshot "label-string" +debug_env_snapshot() { + local label="$1" + local out_file="${SCRIPT_RUN_DIR}/debug_env_${TAG}.txt" + { + echo "========================================" + echo "ENV SNAPSHOT: $label" + echo "Timestamp: $(date)" + echo "----------------------------------------" + echo "--- conda info ---" + conda info 2>/dev/null || echo "(conda not available)" + echo "--- CONDA_DEFAULT_ENV: ${CONDA_DEFAULT_ENV:-}" + echo "--- CONDA_PREFIX: ${CONDA_PREFIX:-}" + echo "--- Active python: $(which python 2>/dev/null || echo '')" + echo "--- Python version: $(python --version 2>/dev/null || echo '')" + echo "--- LD_LIBRARY_PATH: ${LD_LIBRARY_PATH:-}" + echo "--- PYTHONPATH: ${PYTHONPATH:-}" + echo "--- PYTHONHOME: ${PYTHONHOME:-}" + echo "--- PATH (first 5 entries):" + echo "$PATH" | tr ':' '\n' | head -5 + echo "--- ulimit -s (stack size): $(ulimit -s)" + echo "--- ulimit -v (virtual mem): $(ulimit -v)" + echo "--- OMP_NUM_THREADS: ${OMP_NUM_THREADS:-}" + echo "--- MKL_NUM_THREADS: ${MKL_NUM_THREADS:-}" + echo "--- OPENBLAS_NUM_THREADS: ${OPENBLAS_NUM_THREADS:-}" + echo "========================================" + } | tee -a "$out_file" +} + +# Capture MPAS-specific preflight diagnostics to both stdout and a log file. +# Checks the mpas_analysis binary, shared library linkage, and key Python +# packages (numpy BLAS config, ESMF, cartopy) that are common segfault sources. +# Usage: debug_mpas_preflight "label-string" +debug_mpas_preflight() { + local label="$1" + local out_file="${SCRIPT_RUN_DIR}/debug_mpas_${TAG}.txt" + { + echo "========================================" + echo "MPAS PREFLIGHT: $label" + echo "Timestamp: $(date)" + echo "----------------------------------------" + echo "--- mpas_analysis binary: $(which mpas_analysis 2>/dev/null || echo '')" + echo "--- mpas_analysis version: $(mpas_analysis --version 2>/dev/null || echo '')" + echo "--- ldd on mpas_analysis binary (if applicable):" + _mpas_bin="$(which mpas_analysis 2>/dev/null || true)" + if [[ -n "$_mpas_bin" && -f "$_mpas_bin" ]]; then + ldd "$_mpas_bin" 2>/dev/null | grep -i "not found" && echo "(^ missing libs above)" || echo "(all libs found)" + else + echo "(mpas_analysis not a regular file or not found, skipping ldd)" + fi + echo "--- numpy config (segfaults often trace to BLAS/LAPACK mismatches):" + python -c "import numpy; numpy.show_config()" 2>/dev/null || echo "(numpy not importable)" + echo "--- ESMF version:" + python -c "import ESMF; print(ESMF.__version__)" 2>/dev/null || echo "(ESMF not importable)" + echo "--- cartopy version:" + python -c "import cartopy; print(cartopy.__version__)" 2>/dev/null || echo "(cartopy not importable)" + echo "========================================" + } | tee -a "$out_file" +} + # Activate conda and (optionally) a named environment. activate_env() { local env_name="${1:-}" @@ -213,6 +278,9 @@ activate_env() { if [ -n "$env_name" ]; then conda activate "$env_name" + # Log every activation so we can spot env-stack contamination between subshells. + echo "[DEBUG activate_env] $(date) | activated: $env_name | python: $(which python 2>/dev/null || echo '') | CONDA_PREFIX: ${CONDA_PREFIX:-} | LD_LIBRARY_PATH: ${LD_LIBRARY_PATH:-}" \ + >> "${SCRIPT_RUN_DIR}/debug_activate_log_${TAG}.txt" log "Installing/updating package in '$env_name'..." python -m pip install . fi @@ -402,6 +470,10 @@ phase_1_setup() { log " EXPLICIT_TAG=${TAG}" log "=========================================" + # Capture the baseline environment before any component setup so we have + # a clean reference to diff against later snapshots. + debug_env_snapshot "phase-1-start" + # ------------------------------------------------------------------ # e3sm_to_cmip # ------------------------------------------------------------------ @@ -473,6 +545,12 @@ phase_1_setup() { # ------------------------------------------------------------------ log "Setting up MPAS-Analysis..." + # Snapshot the environment immediately before entering the MPAS subshell. + # Compare this against the post-activate snapshot to spot any leakage + # from the e3sm_to_cmip or e3sm_diags subshells above (LD_LIBRARY_PATH, + # CONDA_PREFIX nesting, PYTHONPATH, etc.). + debug_env_snapshot "before-mpas-subshell" + local MPAS_ENV="" if [[ "$MPAS_ENV_TYPE" == "dev" ]]; then if [[ -n "$MPAS_EXISTING_ENV" ]]; then @@ -493,11 +571,25 @@ phase_1_setup() { if [[ -n "$MPAS_EXISTING_ENV" ]]; then log "Reusing existing 'MPAS-Analysis' env: $MPAS_EXISTING_ENV (skipping creation)" activate_env "$MPAS_EXISTING_ENV" + # Snapshot after activating the pre-existing env. A mismatch + # between this and the before-mpas-subshell snapshot (especially + # in LD_LIBRARY_PATH) is a likely segfault cause. + debug_env_snapshot "mpas-subshell-after-activate-existing" + debug_mpas_preflight "mpas-existing-env" else setup_conda_env "none" "$MPAS_ENV" + # Snapshot after a fresh env creation + activation. Check that + # CONDA_PREFIX is clean and LD_LIBRARY_PATH only contains paths + # from this env, not any prior one. + debug_env_snapshot "mpas-subshell-after-setup-conda" + debug_mpas_preflight "mpas-new-env" fi else log "Using unified env for MPAS-Analysis (skipping conda env creation)" + # Snapshot even for the unified-env path -- the unified env loader + # modifies LD_LIBRARY_PATH in ways that can conflict with prior envs. + debug_env_snapshot "mpas-subshell-unified-env" + debug_mpas_preflight "mpas-unified-env" fi ) From 4379fbbb6d346861b8846318da3fbe161f64e4bc Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 1 Jul 2026 13:20:25 -0500 Subject: [PATCH 31/38] Add esmpy for mpas_analysis --- tests/main_branch_testing/run_integration_test.bash | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index e0814ef7..baed63e2 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -316,6 +316,10 @@ setup_conda_env() { conda clean --all --yes if [[ "$conda_dir" == "none" ]]; then conda create --name "$env_name" --file dev-spec.txt --yes + # esmpy (Python bindings for esmf) is missing from MPAS-Analysis + # dev-spec.txt. Install manually until the upstream repo adds it. + # The *mpich* glob matches the MPI variant already in dev-spec.txt. + conda install -n "$env_name" -c conda-forge "esmpy=*=*mpich*" --yes else conda env create -f "${conda_dir}/dev.yml" -n "$env_name" fi From 13c2afb3d62c6067c2fe8821e8a2e37936436988 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 1 Jul 2026 13:42:33 -0500 Subject: [PATCH 32/38] Remove esmpy for mpas_analysis --- tests/main_branch_testing/run_integration_test.bash | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index baed63e2..e0814ef7 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -316,10 +316,6 @@ setup_conda_env() { conda clean --all --yes if [[ "$conda_dir" == "none" ]]; then conda create --name "$env_name" --file dev-spec.txt --yes - # esmpy (Python bindings for esmf) is missing from MPAS-Analysis - # dev-spec.txt. Install manually until the upstream repo adds it. - # The *mpich* glob matches the MPI variant already in dev-spec.txt. - conda install -n "$env_name" -c conda-forge "esmpy=*=*mpich*" --yes else conda env create -f "${conda_dir}/dev.yml" -n "$env_name" fi From 08606956b2291878f7cdc843dffc102ffccef2a3 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Tue, 4 Aug 2026 16:30:05 -0500 Subject: [PATCH 33/38] Add automated testing to documentation --- docs/source/dev_guide/index.rst | 5 +- .../source/dev_guide/tests/automated_test.rst | 346 ++++++++++++++++++ docs/source/dev_guide/tests/index.rst | 14 + .../{test.rst => tests/manual_test.rst} | 13 +- .../{ => tests}/update_expected_results.rst | 0 5 files changed, 371 insertions(+), 7 deletions(-) create mode 100644 docs/source/dev_guide/tests/automated_test.rst create mode 100644 docs/source/dev_guide/tests/index.rst rename docs/source/dev_guide/{test.rst => tests/manual_test.rst} (98%) rename docs/source/dev_guide/{ => tests}/update_expected_results.rst (100%) diff --git a/docs/source/dev_guide/index.rst b/docs/source/dev_guide/index.rst index 9ff5ade8..42b780ec 100644 --- a/docs/source/dev_guide/index.rst +++ b/docs/source/dev_guide/index.rst @@ -12,9 +12,8 @@ This guide covers everything needed to develop, test, and release ``zppy``. parameters provenance tasks/index - test - update_expected_results - releases/index.rst + tests/index + releases/index new_task new_diags_set archive/index diff --git a/docs/source/dev_guide/tests/automated_test.rst b/docs/source/dev_guide/tests/automated_test.rst new file mode 100644 index 00000000..7b3630bb --- /dev/null +++ b/docs/source/dev_guide/tests/automated_test.rst @@ -0,0 +1,346 @@ +.. _automated-testing-zppy: + +************************* +Automated testing of zppy +************************* + +Follow the steps below to test ``zppy``. As you do so, please produce a Markdown report summarizing your results. + +Step 1: Determine what the current expected results are +======================================================= + +Machine-specific setup +~~~~~~~~~~~~~~~~~~~~~~ + +Chrysalis: + +.. code-block:: bash + + expected_results_dir=/lcrc/group/e3sm/public_html/zppy_test_resources + expected_results_records_dir=/lcrc/group/e3sm/public_html/zppy_test_resources_previous + +Compy: + +.. code-block:: bash + + expected_results_dir=/compyfs/www/zppy_test_resources + expected_results_records_dir=/compyfs/fors729/zppy_test_resources_previous + +Note that Compy doesn't give write access to ``/compyfs/www/``, so we can't add a new directory there. That's why ``zppy_test_resources_previous`` is in a separate path. + +Perlmutter: + +.. code-block:: bash + + expected_results_dir=/global/cfs/cdirs/e3sm/www/zppy_test_resources + expected_results_records_dir=/global/cfs/cdirs/e3sm/www/zppy_test_resources_previous + +Process +~~~~~~~ + +.. code-block:: bash + + ls -lt ${expected_results_dir} + +In your Markdown report, note the date the expected results were last updated. + +Step 2: Review changes since expected results were updated +========================================================== + +Now that we know the date the expected results are from, we can review what changes we'll be testing. + +Review each of the following commit logs and note commits made since the date the expected results were updated: + +* For the ``e3sm_to_cmip`` task: `e3sm_to_cmip `_ +* For the ``e3sm_diags`` task: `e3sm_diags `_ +* For the ``mpas_analysis`` task: `MPAS-Analysis `_ +* For the ``global_time_series`` and ``pcmdi_diags`` tasks: `zppy-interfaces `_ +* For ``zppy`` itself: `zppy `_ + +For the remaining tasks (``climo``, ``ts``, ``tc_analysis``, ``ilamb``, ``livvkit``), we typically just use the associated package's latest release rather than making dev environments. As such, their latest development will have no impact on our tests unless we have started using one of their newer releases. + +In your Markdown report, make a table like: + +.. code-block:: + + | Package | Changes since expected results were updated | + | --- | --- | + | [package name](link to package's commit log) | Links to all PRs merged since the expected results were updated | + ... + +The automated test script +========================= + +The automated test script handles the following steps from the manual testing process: + +* Step 3: Set up environments for called packages +* Step 4: Set up zppy environment +* Step 5: Launch zppy jobs +* Step 6: Launch zppy jobs – bundles part 2 +* Step 7: Review finished returns +* Step 8: Run Python tests (excluding the final ``pytest tests/integration/test_images.py`` call from a compute node) + +A. Set up the test script +~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + cd ${repo_parent_dir}/zppy + git status # Check for uncommitted changes + + # If there are uncommitted changes, + # commit them so we can move cleanly to a new branch: + git add -A + git commit -m "Checkpoint" + + git fetch upstream main # This assumes you've named your remote for the main repo as "upstream" + git checkout -b test-zppy-yyyymmdd upstream/main # Use today's date + git log --oneline | head -n 1 + # Check that this matches the corresponding commit log: + # https://github.com/E3SM-Project/zppy/commits/main + + # Now, copy the test script and cfg from the zppy repo into the directory + # that you'll be running the test script from. + mkdir -p ${test_script_dir}/test_yyyymmdd_runN + cd ${test_script_dir}/test_yyyymmdd_runN + cp ${repo_parent_dir}/zppy/tests/main_branch_testing/run_integration_test.bash . + cp ${repo_parent_dir}/zppy/tests/main_branch_testing/zppy_test.cfg . + + # Now, edit the test cfg as needed + emacs zppy_test.cfg + +B. Set up the test cfg +~~~~~~~~~~~~~~~~~~~ + +Let's examine the parts of the test cfg. + +You'll likely just need to update the ``MACHINE`` name if you're not running on Chrysalis. + +.. code-block:: + + # For these, + + MACHINE=chrysalis # chrysalis | compy | perlmutter + START_PHASE=1 # 1 | 2 | 3 + AUTO_MODE=true # true = skip all interactive checkpoints + EXPLICIT_TAG="" # Leave empty to auto-generate; set to resume a prior run + +Update the ``RUN_NUMBER`` if you've already done a test run today. + +.. code-block:: + + RUN_NUMBER=1 + +Update the ``_BASE_BRANCH`` parameters if you plan to test new features or bug fixes that aren't yet included on whatever the repo calls its "official" branch. + +.. code-block:: + + DIAGS_BASE_BRANCH="main" + E3SM_TO_CMIP_BASE_BRANCH="master" + MPAS_BASE_BRANCH="develop" + ZI_BASE_BRANCH="main" + ZPPY_BASE_BRANCH="main" + +Update the ``_ENV_TYPE`` parameters if you want to use E3SM-Unified rather than a dev environment. If you plan to only run a subset of tasks, you can set the ones you aren't running to use E3SM-Unified, so that the script doesn't spend time building a dev environment that won't be used. + +.. code-block:: + + # "dev" = build a dedicated conda env from the repo's dev.yml + # "unified" = use the machine's e3sm-unified env (UNIFIED_ENV_CMD) + DIAGS_ENV_TYPE="dev" + E3SM_TO_CMIP_ENV_TYPE="dev" + MPAS_ENV_TYPE="dev" + ZI_ENV_TYPE="dev" + +Update the ``_EXISTING_ENV`` parameters if you already have an environment from a previous test run to use. + +.. code-block:: + + # Optional: reuse an existing named conda env instead of creating a new one. + # When non-empty AND the corresponding ENV_TYPE is "dev", the script skips + # conda env creation and activates this env directly. + # Leave empty to let the script auto-name and create the env as usual. + DIAGS_EXISTING_ENV="" + E3SM_TO_CMIP_EXISTING_ENV="" + MPAS_EXISTING_ENV="" + ZI_EXISTING_ENV="" + ZPPY_EXISTING_ENV="" + +Update these two parameters to configure which jobs run. + +.. code-block:: + + # Comma-separated list of zppy cfg names to generate and submit. + # These correspond to generated filenames: test_weekly__.cfg + # Any name containing "bundle" is treated as a bundle cfg and re-submitted in Phase 2. + CFGS_TO_RUN="weekly_bundles,weekly_comprehensive_v2,weekly_comprehensive_v3,weekly_legacy_3.1.0_bundles,weekly_legacy_3.1.0_comprehensive_v2,weekly_legacy_3.1.0_comprehensive_v3,weekly_legacy_3.0.0_bundles,weekly_legacy_3.0.0_comprehensive_v2,weekly_legacy_3.0.0_comprehensive_v3" + + # Comma-separated list of tasks to enable in utils.py. + TASKS_TO_RUN="e3sm_diags,mpas_analysis,global_time_series,ilamb,livvkit,pcmdi_diags" + + +These parameters are unlikely to change between runs. They just let the test script know where to find files in your particular workspace. It is recommended to clone a new copy of the repos and use that for each ``_DIR`` parameter listed below. The script will change branches, so using a distinct copy means you won't get your work overwritten. + +.. code-block:: + + HOME_DIR="$HOME" + EZ_DIR="$HOME_DIR/ez" + + E3SM_DIAGS_DIR="$EZ_DIR/e3sm_diags" + E3SM_TO_CMIP_DIR="$EZ_DIR/e3sm_to_cmip" + MPAS_ANALYSIS_DIR="$EZ_DIR/MPAS-Analysis" + ZPPY_INTERFACES_DIR="$EZ_DIR/zppy-interfaces" + ZPPY_DIR="$EZ_DIR/zppy" + + CONDA_PROFILE="$HOME_DIR/miniforge3/etc/profile.d/conda.sh" + TAG_CACHE_FILE="$HOME_DIR/.zppy_test_tag" + + +C. Run the test script +~~~~~~~~~~~~~~~~~~~~~~ + +Now that we have the test cfg set up, we can run it. + +.. code-block:: bash + + screen # Use `screen`` so that even if the terminal connection is interrupted, the script will keep running. + ulimit -s unlimited # This is necessary for MPAS-Analysis to work inside `screen` + cd ${test_script_dir}/test_yyyymmdd_runN + cat zppy_test.cfg # Make sure changes are there + time ./run_integration_test.bash --config zppy_test.cfg 2>&1 | tee integration_test_runN.log + # Ctrl-A D to detach from screen + screen -ls # See what screen sessions you have + tail -f integration_test_runN.log + +Follow the ``tail`` output until you get to: + +.. code-block:: + + ✓ Phase 3 automated tests complete! + ✓ Remember to run test_images.py manually from a compute node. + ✓ Integration test automation complete! + +D. Review the output +~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + # CTRL C # Exit tail + screen -R # The script should have finished and there should be ``time`` output: real, user, sys + exit # Exit screen + cd ${test_script_dir}/test_yyyymmdd_runN + cat integration_test_runN.log + +Let's review the test script's output log. + +First, the unit tests. There are two blocks, starting with: + +.. code-block:: + + Running zppy unit tests... + +and + +.. code-block:: + + Running zppy-interfaces unit tests... + +Second, the output directories status. It should look like the following: + +.. code-block:: + + Checking all status files... + ... + ✓ All status files clean! + +If some status files were unsuccessful, you'll want to run the following to review the errors: + +.. code-block:: bash + + cd ${dir_with_failures} + grep -v "OK" * status # See what jobs failed + # Review errors: + tail ${job_that_failed}.o${id_of_job_that_failed} + grep -i error ${job_that_failed}.o${id_of_job_that_failed} + +Third, the integration tests. + +.. code-block:: + + test_last_year.py + test_bash_generation.py + test_campaign.py + test_defaults.py + test_bundles.py + +Errors here may actually be expected if the expected results haven't been updated yet to reflect a recently merged pull request. Another reason for errors on ``test_bundles.py`` in particular is if you didn't run all the jobs necessary (i.e., if you're running a partial test). + +If all 3 pieces look good, you can proceed with the final integration test, the image checker. + +Step 8: Run Python tests +======================== + +Machine-specific setup +~~~~~~~~~~~~~~~~~~~~~~ + +Chrysalis: + +.. code-block:: bash + + launch_compute_node() + { + salloc --nodes=1 --partition=debug --time=02:00:00 --account=e3sm + } + +Compy: + +.. code-block:: bash + + launch_compute_node() + { + salloc --nodes=1 --partition=short --time=01:00:00 --account=e3sm + } + +Perlmutter: + +.. code-block:: bash + + launch_compute_node() + { + salloc --nodes=1 --qos=interactive --time=01:00:00 --constraint=cpu --account=e3sm + } + +Process +~~~~~~~ + +.. code-block:: bash + + cd ${repo_parent_dir}/zppy + git status + # You might have changed branches while you were waiting for jobs to finish. + # Make sure you're now back on the correct branch: test-zppy-yyyymmdd + # Also confirm you're back in the correct env: zppy-yyyymmdd or the Unified env + + # The image checker test, which we'll run from a compute node: + launch_compute_node + + start_bash_subshell + # EITHER: + # Activate EITHER a dev environment or the Unified env: + conda activate zppy-yyyymmdd + # OR: the command from `activate_unified_env` + + pytest tests/integration/test_images.py + # Typically takes between 10 and 20 minutes on Chrysalis and Perlmutter. + # Typically takes closer to 50 minutes on Compy. + cat test_images_summary.md + exit # Exit bash shell + exit # Exit compute note + +In your Markdown report: + +* From the ``pytest tests/integration/test_images.py `` command-line output, copy everything after ``Captured stdout call`` to a code block labeled "Output" +* Copy the results of ``cat test_images_summary.md`` to a section labeled "Complete summary table" +* Make a new section named "Summary table -- only failing image-check tests, sorted by task". For each task that has missing and/or mismatched images, copy the relevant rows from the summary table. Skip this section if there were no failing image-check tests. +* Note any test failures from the other Python tests. +* If there were no failures at all, print "All tests pass" diff --git a/docs/source/dev_guide/tests/index.rst b/docs/source/dev_guide/tests/index.rst new file mode 100644 index 00000000..f0119ea8 --- /dev/null +++ b/docs/source/dev_guide/tests/index.rst @@ -0,0 +1,14 @@ +.. _tests: + +******* +Tests +******* + +This page collects documentation on testing ``zppy``. + +.. toctree:: + :maxdepth: 1 + + manual_test + automated_test + update_expected_results diff --git a/docs/source/dev_guide/test.rst b/docs/source/dev_guide/tests/manual_test.rst similarity index 98% rename from docs/source/dev_guide/test.rst rename to docs/source/dev_guide/tests/manual_test.rst index ff57f678..c4d3ba02 100644 --- a/docs/source/dev_guide/test.rst +++ b/docs/source/dev_guide/tests/manual_test.rst @@ -1,8 +1,8 @@ -.. _testing-zppy: +.. _manually-testing-zppy: -************* -Testing zppy -************* +********************* +Manually testing zppy +********************* Follow the steps below to test ``zppy``. As you do so, please produce a Markdown report summarizing your results. @@ -93,6 +93,11 @@ Chrysalis: lcrc_conda # Or however you activate conda rm -rf build conda clean --all --y + # The dev.yml file may be in a different directory. + # e3sm_to_cmip, e3sm_diags: use conda-dev/ + # zppy-interfaces, zppy: use conda/ + # MPAS-Analysis: don't use this command, instead use: + # conda create --name ${env_name} --file dev-spec.txt --yes conda env create -f conda/dev.yml -n ${env_name} conda activate ${env_name} pre-commit run --all-files # Confirm this passes diff --git a/docs/source/dev_guide/update_expected_results.rst b/docs/source/dev_guide/tests/update_expected_results.rst similarity index 100% rename from docs/source/dev_guide/update_expected_results.rst rename to docs/source/dev_guide/tests/update_expected_results.rst From ac08dc422012cdb6bf406e40d3627c72e3cc296a Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Tue, 4 Aug 2026 16:40:29 -0500 Subject: [PATCH 34/38] Minor docs updates --- docs/source/dev_guide/tests/automated_test.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/dev_guide/tests/automated_test.rst b/docs/source/dev_guide/tests/automated_test.rst index 7b3630bb..b25c57b3 100644 --- a/docs/source/dev_guide/tests/automated_test.rst +++ b/docs/source/dev_guide/tests/automated_test.rst @@ -81,7 +81,7 @@ The automated test script handles the following steps from the manual testing pr * Step 8: Run Python tests (excluding the final ``pytest tests/integration/test_images.py`` call from a compute node) A. Set up the test script -~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~ .. code-block:: bash @@ -110,7 +110,7 @@ A. Set up the test script emacs zppy_test.cfg B. Set up the test cfg -~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~ Let's examine the parts of the test cfg. @@ -237,13 +237,13 @@ First, the unit tests. There are two blocks, starting with: .. code-block:: - Running zppy unit tests... + Running zppy-interfaces unit tests... and .. code-block:: - Running zppy-interfaces unit tests... + Running zppy unit tests... Second, the output directories status. It should look like the following: From 2a21a3e700b40feed435436b8623767082ef8ba0 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Tue, 4 Aug 2026 16:53:21 -0500 Subject: [PATCH 35/38] Address Review comment --- tests/main_branch_testing/run_integration_test.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index e0814ef7..e63ee48f 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -4,7 +4,7 @@ # Usage: # 1. Copy this file AND the sample config OUT of the zppy repo # (this script will change branches). -# 2. Edit your config file (see zppy_test.cfg.sample). +# 2. Edit your config file (see zppy_test.cfg). # 3. Run: ./run_integration_test.bash --config path/to/your.cfg # # Phases (set START_PHASE in your config): From 86c27ae28d1db2264b1b5d086ae706683b288387 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 10 Aug 2026 14:22:13 -0700 Subject: [PATCH 36/38] Claude-generated frozen env setup --- .../source/dev_guide/tests/automated_test.rst | 60 ++++++++ tests/main_branch_testing/README.md | 42 ++++- .../run_integration_test.bash | 143 +++++++++++++++++- tests/main_branch_testing/zppy_test.cfg | 56 +++++++ 4 files changed, 292 insertions(+), 9 deletions(-) diff --git a/docs/source/dev_guide/tests/automated_test.rst b/docs/source/dev_guide/tests/automated_test.rst index b25c57b3..739ce7a3 100644 --- a/docs/source/dev_guide/tests/automated_test.rst +++ b/docs/source/dev_guide/tests/automated_test.rst @@ -68,6 +68,30 @@ In your Markdown report, make a table like: | [package name](link to package's commit log) | Links to all PRs merged since the expected results were updated | ... +Step 2.5: Refresh frozen dependency lock files (if needed) +============================================================ + +The test script freezes dependencies for ``e3sm_to_cmip``, ``e3sm_diags``, ``mpas_analysis``, ``zppy-interfaces``, and ``zppy`` so that ``test_images.py`` diffs can be attributed to the package under test rather than to an unrelated dependency (e.g. ``matplotlib``) that happened to move between runs. Each of these five gets its own dedicated, fully-resolved conda env (a "frozen base"); the test env for a given run is created by *cloning* that frozen base and ``pip install``-ing the branch under test on top, so nothing else in the environment can drift. + +This means each of the five needs a lock file -- an exact-version ``conda list --explicit`` snapshot, not a ``dev.yml`` (a ``dev.yml`` re-solves and can drift between runs even unmodified). You only need to regenerate a component's lock file when: + +* You don't have one yet (first-time setup), or +* That component's ``dev.yml`` changed in a way that should be picked up (a new or updated dependency), or +* The test script warned that ``pip install .`` pulled in something beyond the frozen base for that component (see Step C below) and you've decided to bake that change in. + +To (re)generate a lock file for a component: + +.. code-block:: bash + + cd ${repo_parent_dir}/ # e.g. e3sm_diags, MPAS-Analysis, zppy-interfaces, zppy, e3sm_to_cmip + conda env create -f /dev.yml -n tmp-lock-gen # or --file dev-spec.txt for mpas_analysis + conda activate tmp-lock-gen + conda list --explicit > ${EZ_DIR}/frozen-base-.txt + conda deactivate + conda remove --yes --all --name tmp-lock-gen + +Repeat for each of the five components, using the exact filenames referenced by ``BASE_ENV_LOCK_FILE_`` in your ``zppy_test.cfg`` (see Section B below). If ``FREEZE_DEPENDENCIES=false`` in your config, you can skip this step entirely -- every component will solve its own ``dev.yml`` fresh, as before. + The automated test script ========================= @@ -178,6 +202,30 @@ Update these two parameters to configure which jobs run. # Comma-separated list of tasks to enable in utils.py. TASKS_TO_RUN="e3sm_diags,mpas_analysis,global_time_series,ilamb,livvkit,pcmdi_diags" +These parameters control the frozen dependency base discussed in Step 2.5 above. In most runs you won't need to touch these beyond making sure ``FREEZE_DEPENDENCIES=true`` and that a lock file exists for each component in ``FROZEN_BASE_COMPONENTS``. + +.. code-block:: + + # Master switch. false = every component solves its own dev.yml fresh, + # as before this feature existed. + FREEZE_DEPENDENCIES=true + + # Which components get their own dedicated frozen base. Defaults to all + # five dev-env components, since image-check diffs can come from any of + # them, not just e3sm_diags/pcmdi_diags. + FROZEN_BASE_COMPONENTS="e3sm_to_cmip,e3sm_diags,mpas_analysis,zppy_interfaces,zppy" + +Further down, after ``EZ_DIR`` is defined (see below), each frozen component's lock file path is set: + +.. code-block:: + + BASE_ENV_LOCK_FILE_E3SM_TO_CMIP="$EZ_DIR/frozen-base-e3sm_to_cmip.txt" + BASE_ENV_LOCK_FILE_E3SM_DIAGS="$EZ_DIR/frozen-base-e3sm_diags.txt" + BASE_ENV_LOCK_FILE_MPAS_ANALYSIS="$EZ_DIR/frozen-base-mpas_analysis.txt" + BASE_ENV_LOCK_FILE_ZPPY_INTERFACES="$EZ_DIR/frozen-base-zppy_interfaces.txt" + BASE_ENV_LOCK_FILE_ZPPY="$EZ_DIR/frozen-base-zppy.txt" + +If ``FREEZE_DEPENDENCIES=false``, or a component is removed from ``FROZEN_BASE_COMPONENTS``, its corresponding ``BASE_ENV_LOCK_FILE_*`` value is simply ignored. These parameters are unlikely to change between runs. They just let the test script know where to find files in your particular workspace. It is recommended to clone a new copy of the repos and use that for each ``_DIR`` parameter listed below. The script will change branches, so using a distinct copy means you won't get your work overwritten. @@ -220,6 +268,16 @@ Follow the ``tail`` output until you get to: ✓ Remember to run test_images.py manually from a compute node. ✓ Integration test automation complete! +If ``FREEZE_DEPENDENCIES=true``, watch for warnings like the following while Phase 1 sets up environments: + +.. code-block:: + + ⚠ 'test-diags-main-yyyymmdd_runN': package set changed beyond the frozen base after 'pip install .'. + ⚠ This is expected ONLY if 'e3sm_diags' itself added or bumped a dependency: + < some diff lines > + +This means the branch under test needed something beyond what's pinned in that component's lock file. It's not necessarily a problem -- just confirm the added/bumped package is one you'd expect that branch to need, and consider regenerating that component's lock file (Step 2.5) so future runs pick it up without the warning. The full diff is also saved as ``pre_install__.txt`` / ``post_install__.txt`` in your run directory. + D. Review the output ~~~~~~~~~~~~~~~~~~~~ @@ -337,6 +395,8 @@ Process exit # Exit bash shell exit # Exit compute note +If ``FREEZE_DEPENDENCIES=true`` was used for this run, any diffs reported here should trace back to the branches under test in ``FROZEN_BASE_COMPONENTS`` rather than to incidental dependency movement -- that's the isolation this feature is for. If a diff still looks like it could be dependency-related, double check the ``pre_install_*``/``post_install_*`` files and any warnings from Step C for that component. + In your Markdown report: * From the ``pytest tests/integration/test_images.py `` command-line output, copy everything after ``Captured stdout call`` to a code block labeled "Output" diff --git a/tests/main_branch_testing/README.md b/tests/main_branch_testing/README.md index 3331cbf2..0e9f2d8a 100644 --- a/tests/main_branch_testing/README.md +++ b/tests/main_branch_testing/README.md @@ -20,7 +20,7 @@ Set `AUTO_MODE=true` in your config file. All checkpoints are bypassed and the s ## Configuration -Copy `zppy_test.cfg` and edit it before each test run. It has three sections: +Copy `zppy_test.cfg` and edit it before each test run. It has four sections: ### Runtime settings (update as needed each run) @@ -64,6 +64,29 @@ ZPPY_EXISTING_ENV="test-zppy-main-20250601_run1" CFGS_TO_RUN="weekly_bundles,weekly_comprehensive_v3,weekly_legacy_3.1.0_bundles,weekly_legacy_3.1.0_comprehensive_v3,weekly_legacy_3.0.0_bundles,weekly_legacy_3.0.0_comprehensive_v3" ``` +### Frozen dependency base (isolates image-diff root causes) + +`test_images.py` can show diffs for `e3sm_diags` or `pcmdi_diags` plots that are actually caused by an unrelated dependency (e.g. `matplotlib` moving a plot a pixel to the right) rather than by the package under test. To isolate root causes, any component listed in `FROZEN_BASE_COMPONENTS` gets its **own dedicated, fully-resolved conda env**, built once per run from that component's lock file. A test env for that component is then created by **cloning** its frozen base (fast — no solve, no network) and `pip install`ing the branch under test on top, so every dependency other than that component's own code stays pinned to exactly what's in its lock file. + +Each component's frozen base is independent — `e3sm_diags`'s cdat-based stack, `mpas_analysis`'s ESMF/MPAS libs, and `e3sm_to_cmip`'s `cmor` never need to be resolved into one shared environment together. + +| Variable | Description | +| --- | --- | +| `FREEZE_DEPENDENCIES` | Master switch. `false` restores the original behavior for every component (solve its own `dev.yml` fresh each run) regardless of the settings below. | +| `FROZEN_BASE_COMPONENTS` | Comma-separated subset of `e3sm_to_cmip,e3sm_diags,mpas_analysis,zppy_interfaces,zppy` that should use a frozen base. Defaults to all five, since image diffs can come from any plotting-adjacent tool, not just `e3sm_diags`/`pcmdi_diags`. | +| `BASE_ENV_LOCK_FILE_` | Path to a fully-resolved environment spec (uppercase component name, e.g. `BASE_ENV_LOCK_FILE_E3SM_DIAGS`). Required for every component listed in `FROZEN_BASE_COMPONENTS`. | + +**Generating/updating a lock file** for a component (repeat per component in `FROZEN_BASE_COMPONENTS`): +```bash +conda env create -f -n tmp-lock-gen +conda activate tmp-lock-gen +conda list --explicit > $EZ_DIR/frozen-base-.txt +conda deactivate && conda remove --yes --all --name tmp-lock-gen +``` +A `dev.yml` is **not** sufficient on its own — it re-solves and can drift between runs even when unmodified, which is exactly the noise this feature exists to remove. Regenerate a component's lock file whenever that component's branch genuinely needs a new or updated dependency; the script will tell you when that's happened (see below). + +If a component's branch pulls in a dependency beyond what's frozen, `pip install .` still installs it — the script just makes that visible instead of letting it pass silently. It snapshots `conda list --explicit` before and after `pip install .` for every frozen component and logs a warning with the diff (also saved to `pre_install__.txt` / `post_install__.txt` in the run directory) whenever the two don't match. + ### One-time setup (paths that rarely change) | Variable | Description | @@ -84,6 +107,7 @@ Machine-specific settings (`OUTPUT_WORKSPACE`, conda activation command, unified ### Phase 1: Setup - Creates conda environments for each component where `ENV_TYPE="dev"` and no `*_EXISTING_ENV` is set; reuses the named env when `*_EXISTING_ENV` is set; skips env handling entirely for any component where `ENV_TYPE="unified"` + - For components in `FROZEN_BASE_COMPONENTS` (when `FREEZE_DEPENDENCIES=true`), the env is created by cloning that component's dedicated frozen base instead of solving `dev.yml` fresh - Runs unit tests for zppy-interfaces and zppy - Patches `tests/integration/utils.py` with test-specific environment commands, config list, and unique ID - Generates config files via `python tests/integration/utils.py` @@ -148,6 +172,7 @@ Check the error message. The script uses `set -e`, so it exits on any error. Com - A unit test failure during Phase 1 setup - A SLURM timeout (increase the max-wait argument to `wait_for_slurm_jobs`) - `DependencyNeverSatisfied` on all queued jobs (check your cfg files and SLURM account) +- `FREEZE_DEPENDENCIES=true` with a missing/not-yet-generated `BASE_ENV_LOCK_FILE_` for a component in `FROZEN_BASE_COMPONENTS` (the error message names the exact variable and gives the `conda list --explicit` command to generate it) Phase 2 checks bundle status files before resubmitting and warns if any are non-OK. Resolve any failures in the Phase 1 bundle runs before proceeding. You can restart from Phase 2 by setting `START_PHASE=2` in your config (the TAG from Phase 1 is saved in `~/.zppy_test_tag` and picked up automatically, or set `EXPLICIT_TAG` to be explicit): ```bash @@ -164,6 +189,14 @@ conda remove --yes --all --name test-mpas-develop-YYYYMMDD_runN conda remove --yes --all --name test-zi-main-YYYYMMDD_runN conda remove --yes --all --name test-zppy-main-YYYYMMDD_runN +# If using FREEZE_DEPENDENCIES=true, also remove that run's frozen base envs +# (one per component in FROZEN_BASE_COMPONENTS): +conda remove --yes --all --name test-frozen-base-e3sm_to_cmip-YYYYMMDD_runN +conda remove --yes --all --name test-frozen-base-e3sm_diags-YYYYMMDD_runN +conda remove --yes --all --name test-frozen-base-mpas_analysis-YYYYMMDD_runN +conda remove --yes --all --name test-frozen-base-zppy_interfaces-YYYYMMDD_runN +conda remove --yes --all --name test-frozen-base-zppy-YYYYMMDD_runN + # Then re-run from Phase 1 ./run_integration_test.bash --config zppy_test.cfg ``` @@ -196,6 +229,13 @@ conda remove --yes --all --name test-zppy-main-YYYYMMDD_runN └── zppy_weekly_legacy_3.0.0_comprehensive_v3_output/zppy_main_branch_test_YYYYMMDD_runN/ ``` +If `FREEZE_DEPENDENCIES=true`, each frozen component's run directory also gains: +``` +pre_install__.txt # `conda list --explicit` right after cloning the frozen base +post_install__.txt # `conda list --explicit` right after `pip install .` +``` +These only differ if the branch under test pulled in a new or updated dependency — check `integration_test.log` for a warning with the diff when that happens. + ## Example Run ```bash diff --git a/tests/main_branch_testing/run_integration_test.bash b/tests/main_branch_testing/run_integration_test.bash index e63ee48f..8b8f944a 100755 --- a/tests/main_branch_testing/run_integration_test.bash +++ b/tests/main_branch_testing/run_integration_test.bash @@ -17,6 +17,16 @@ # - To resume from Phase 2 or 3 on a later day, set EXPLICIT_TAG in your config # to the TAG printed at the start of Phase 1 (or stored in ~/.zppy_test_tag), # and set START_PHASE accordingly. +# +# Frozen dependency base (see FREEZE_DEPENDENCIES in the config): +# - Each component listed in FROZEN_BASE_COMPONENTS gets its OWN dedicated, +# fully-resolved conda env (BASE_ENV_LOCK_FILE_) instead of +# solving its dev.yml fresh every run. A test env for that component is +# then built by cloning its frozen base and `pip install`ing the branch +# under test on top, so unrelated dependency drift (e.g. matplotlib) +# can't produce false-positive diffs in test_images.py. See the README +# section "Regenerating a component's frozen base lock file" for how/when +# to update a BASE_ENV_LOCK_FILE_ entry. set -e # Exit on error set -u # Exit on undefined variable @@ -82,6 +92,10 @@ MPAS_EXISTING_ENV="${MPAS_EXISTING_ENV:-}" ZI_EXISTING_ENV="${ZI_EXISTING_ENV:-}" ZPPY_EXISTING_ENV="${ZPPY_EXISTING_ENV:-}" +# Apply defaults for the optional frozen-dependency-base variables. +FREEZE_DEPENDENCIES="${FREEZE_DEPENDENCIES:-false}" +FROZEN_BASE_COMPONENTS="${FROZEN_BASE_COMPONENTS:-}" + # Validate MACHINE value. case "$MACHINE" in chrysalis|compy|perlmutter) ;; @@ -124,6 +138,7 @@ esac # IFS = Internal Field Separator IFS=',' read -ra CFGS_ARRAY <<< "$CFGS_TO_RUN" IFS=',' read -ra TASKS_ARRAY <<< "$TASKS_TO_RUN" +IFS=',' read -ra FROZEN_BASE_ARRAY <<< "$FROZEN_BASE_COMPONENTS" # ============================================================================ # # Priority: @@ -301,15 +316,100 @@ activate_unified_env() { set -u } -# Create (if needed) and activate a conda environment. +# Return 0 (true) if the given component key uses a dedicated frozen base +# env (rather than solving its own dev.yml fresh each run). component_key is +# one of: e3sm_to_cmip, e3sm_diags, mpas_analysis, zppy_interfaces, zppy +uses_frozen_base() { + local component="$1" + [[ "$FREEZE_DEPENDENCIES" == true ]] || return 1 + local c + for c in "${FROZEN_BASE_ARRAY[@]}"; do + c="${c// /}" + if [[ "$c" == "$component" ]]; then + return 0 + fi + done + return 1 +} + +# Resolve the lock file for a component's dedicated frozen base. +# Each component in FROZEN_BASE_COMPONENTS must have its own +# BASE_ENV_LOCK_FILE_ (uppercase) set in the config -- there is no +# shared/fallback lock file. Each component gets its own resolved environment +# because their dependency stacks don't need to (and may not be able to) +# co-resolve into one universal env -- e3sm_diags's cdat-based stack, +# mpas_analysis's ESMF/MPAS libs, and e3sm_to_cmip's cmor can each be frozen +# independently without needing to agree with one another. +get_component_lock_file() { + local component="$1" + local var="BASE_ENV_LOCK_FILE_${component^^}" + echo "${!var:-}" +} + +# Build (once per TAG) the dedicated, pinned base env for one component. +# The lock file must be a FULLY RESOLVED spec (e.g. the output of +# `conda list --explicit` from a known-good env for that component), not a +# dev.yml -- a dev.yml re-solves and can silently drift between runs even +# unmodified. +# +# This is what makes "freeze everything else" possible: every test env +# cloned from base_env_name starts from byte-identical package versions, so +# any image diff that shows up after `pip install .`-ing the branch under +# test can be attributed to that branch rather than to incidental dependency +# movement. +setup_base_env() { + local component="$1" + local base_env_name="$2" + local lock_file="$3" + + activate_env # Ensure conda itself is available + + if conda env list | grep -q "^${base_env_name} "; then + log "Base env '$base_env_name' already exists, skipping creation" + return 0 + fi + + if [[ -z "$lock_file" || ! -f "$lock_file" ]]; then + log_error "FREEZE_DEPENDENCIES=true and '${component}' is in FROZEN_BASE_COMPONENTS, but BASE_ENV_LOCK_FILE_${component^^} is missing or not found: '${lock_file}'" + log_error "Generate one from a known-good env for this component, e.g.:" + log_error " conda env create -f <${component}'s dev.yml> -n tmp-lock-gen" + log_error " conda activate tmp-lock-gen" + log_error " conda list --explicit > /path/to/frozen-base-${component}.txt" + log_error "Then set BASE_ENV_LOCK_FILE_${component^^} in your config to that path." + exit 1 + fi + + log "Creating dedicated frozen base '$base_env_name' for '${component}' from ${lock_file}..." + conda create --name "$base_env_name" --file "$lock_file" --yes + log_success "Base env '$base_env_name' ready" +} + +# Create (if needed) and activate a conda environment for a component. +# +# component_key: one of e3sm_to_cmip, e3sm_diags, mpas_analysis, +# zppy_interfaces, zppy -- used to check FROZEN_BASE_COMPONENTS +# via uses_frozen_base() and to look up this component's own +# BASE_ENV_LOCK_FILE_. +# conda_dir: directory containing dev.yml (e.g. "conda" or "conda-env"), +# or "none" to use dev-spec.txt instead. Ignored when this +# component is using its frozen base. +# env_name: name of the environment to create/activate. setup_conda_env() { - local conda_dir="$1" # Directory containing dev.yml (e.g. "conda" or "conda-env") - local env_name="$2" + local component_key="$1" + local conda_dir="$2" + local env_name="$3" activate_env # Ensure conda itself is available if conda env list | grep -q "^${env_name} "; then log "Environment '$env_name' already exists, skipping creation" + elif uses_frozen_base "$component_key"; then + local base_env_name lock_file + base_env_name="test-frozen-base-${component_key}-${TAG}" + lock_file="$(get_component_lock_file "$component_key")" + setup_base_env "$component_key" "$base_env_name" "$lock_file" # no-op if it already exists this run + log "Cloning '$env_name' from '${component_key}'s frozen base '$base_env_name' (deps pinned)..." + conda create --name "$env_name" --clone "$base_env_name" --yes else log "Creating environment '$env_name' from ${conda_dir}/dev.yml..." rm -rf build @@ -321,8 +421,26 @@ setup_conda_env() { fi fi + # For frozen-base envs, snapshot the exact package set before the + # `pip install .` that activate_env() below performs, so we can tell + # whether the package under test pulled in anything beyond itself. + local _pre_install_file="${SCRIPT_RUN_DIR}/pre_install_${env_name}_${TAG}.txt" + if uses_frozen_base "$component_key"; then + conda list --explicit > "$_pre_install_file" 2>/dev/null || true + fi + activate_env "$env_name" log_success "Environment '$env_name' ready" + + if uses_frozen_base "$component_key"; then + local _post_install_file="${SCRIPT_RUN_DIR}/post_install_${env_name}_${TAG}.txt" + conda list --explicit > "$_post_install_file" 2>/dev/null || true + if [[ -f "$_pre_install_file" ]] && ! diff -q "$_pre_install_file" "$_post_install_file" > /dev/null 2>&1; then + log_warning "'$env_name': package set changed beyond the frozen base after 'pip install .'." + log_warning "This is expected ONLY if '${component_key}' itself added or bumped a dependency:" + diff "$_pre_install_file" "$_post_install_file" || true + fi + fi } @@ -464,6 +582,11 @@ phase_1_setup() { log "Date stamp: $DATE_STAMP" log "TAG: $TAG (saved to ${TAG_CACHE_FILE})" log "Unique ID: $UNIQUE_ID" + if [[ "$FREEZE_DEPENDENCIES" == true ]]; then + log "Frozen base: ENABLED for components: ${FROZEN_BASE_COMPONENTS} (each has its own dedicated frozen base env)" + else + log "Frozen base: disabled (each dev env solves its own dev.yml)" + fi log "" log "To resume from a later phase, set in your config:" log " START_PHASE=2" @@ -500,7 +623,7 @@ phase_1_setup() { log "Reusing existing 'e3sm_to_cmip' env: $E3SM_TO_CMIP_EXISTING_ENV (skipping creation)" activate_env "$E3SM_TO_CMIP_EXISTING_ENV" else - setup_conda_env "conda-env" "$E3SM_TO_CMIP_ENV" + setup_conda_env "e3sm_to_cmip" "conda-env" "$E3SM_TO_CMIP_ENV" fi else log "Using unified env for e3sm_to_cmip (skipping conda env creation)" @@ -533,7 +656,7 @@ phase_1_setup() { log "Reusing existing 'e3sm_diags' env: $DIAGS_EXISTING_ENV (skipping creation)" activate_env "$DIAGS_EXISTING_ENV" else - setup_conda_env "conda-env" "$DIAGS_ENV" + setup_conda_env "e3sm_diags" "conda-env" "$DIAGS_ENV" fi else log "Using unified env for e3sm_diags (skipping conda env creation)" @@ -577,7 +700,7 @@ phase_1_setup() { debug_env_snapshot "mpas-subshell-after-activate-existing" debug_mpas_preflight "mpas-existing-env" else - setup_conda_env "none" "$MPAS_ENV" + setup_conda_env "mpas_analysis" "none" "$MPAS_ENV" # Snapshot after a fresh env creation + activation. Check that # CONDA_PREFIX is clean and LD_LIBRARY_PATH only contains paths # from this env, not any prior one. @@ -619,7 +742,7 @@ phase_1_setup() { log "Reusing existing 'zppy-interfaces' env: $ZI_EXISTING_ENV (skipping creation)" activate_env "$ZI_EXISTING_ENV" else - setup_conda_env "conda" "$ZI_ENV" + setup_conda_env "zppy_interfaces" "conda" "$ZI_ENV" fi else log "Using unified env for zppy-interfaces..." @@ -656,7 +779,7 @@ phase_1_setup() { log "Reusing existing 'zppy' env: $ZPPY_EXISTING_ENV (skipping creation)" activate_env "$ZPPY_ENV" else - setup_conda_env "conda" "$ZPPY_ENV" + setup_conda_env "zppy" "conda" "$ZPPY_ENV" fi log "Running zppy unit tests..." @@ -894,6 +1017,10 @@ phase_3_validation() { log " cd ${ZPPY_DIR}" log " pytest tests/integration/test_images.py" log " cat test_images_summary.md" + if [[ "$FREEZE_DEPENDENCIES" == true ]]; then + log " (Frozen base was used for: ${FROZEN_BASE_COMPONENTS} -- any diffs here" + log " should trace back to those packages, not unrelated dependency drift.)" + fi log_success "Phase 3 automated tests complete!" log_success "Remember to run test_images.py manually from a compute node." diff --git a/tests/main_branch_testing/zppy_test.cfg b/tests/main_branch_testing/zppy_test.cfg index 983564f7..850884ac 100644 --- a/tests/main_branch_testing/zppy_test.cfg +++ b/tests/main_branch_testing/zppy_test.cfg @@ -49,6 +49,47 @@ CFGS_TO_RUN="weekly_bundles,weekly_comprehensive_v2,weekly_comprehensive_v3,week # Comma-separated list of tasks to enable in utils.py. TASKS_TO_RUN="e3sm_diags,mpas_analysis,global_time_series,ilamb,livvkit,pcmdi_diags" +# ---------------------------------------------------------------------------- +# Frozen dependency base (isolates image-diff root causes) +# ---------------------------------------------------------------------------- +# +# Problem this solves: test_images.py can show diffs for e3sm_diags or +# pcmdi_diags plots that are actually caused by an unrelated dependency +# (e.g. matplotlib moving a plot a pixel to the right) rather than by the +# package under test. Freezing a component's dependencies isolates root +# causes and skips redundant env solves/builds. +# +# How it works: each component listed in FROZEN_BASE_COMPONENTS gets its OWN +# dedicated, fully-resolved conda env, built once per run from that +# component's BASE_ENV_LOCK_FILE_. Test envs for that component +# are then created by CLONING its frozen base (fast, no solve) and +# `pip install`ing the branch under test on top -- so every dependency other +# than that component's own code stays pinned to exactly what's in its lock +# file. Each component's frozen base is independent: e3sm_diags's cdat-based +# stack, mpas_analysis's ESMF/MPAS libs, and e3sm_to_cmip's cmor never need to +# be resolved into a single shared environment together. +# +# Caveat: if a component's branch genuinely needs a new or newer dependency, +# `pip install .` will pull it in on top of its frozen base -- the script +# logs a diff when that happens (see pre_install_*/post_install_* files in +# the run directory) so it's a visible, deliberate exception rather than +# silent drift. When that happens, regenerate that component's lock file to +# bake the new dependency in for future runs. + +# Master switch. When false, every component below falls back to its +# original behavior (solve its own dev.yml), regardless of the other +# settings in this section. +FREEZE_DEPENDENCIES=true + +# Which components get their own dedicated frozen base instead of solving +# their dev.yml fresh each run. Comma-separated subset of: +# e3sm_to_cmip,e3sm_diags,mpas_analysis,zppy_interfaces,zppy +# Defaults to every component this script builds a dev env for -- image +# diffs can come from any plotting-adjacent tool (e3sm_diags, pcmdi_diags via +# zppy_interfaces, mpas_analysis), not just the ones flagged in the original +# bug report, so there's no reason to leave any of them unfrozen by default. +FROZEN_BASE_COMPONENTS="e3sm_to_cmip,e3sm_diags,mpas_analysis,zppy_interfaces,zppy" + # ---------------------------------------------------------------------------- # One-time setup # ---------------------------------------------------------------------------- @@ -64,3 +105,18 @@ ZPPY_DIR="$EZ_DIR/zppy" CONDA_PROFILE="$HOME_DIR/miniforge3/etc/profile.d/conda.sh" TAG_CACHE_FILE="$HOME_DIR/.zppy_test_tag" + +# One FULLY RESOLVED environment spec per frozen component -- required for +# each name listed in FROZEN_BASE_COMPONENTS above. Must be exact-version +# output, e.g.: +# conda env create -f -n tmp-lock-gen +# conda activate tmp-lock-gen +# conda list --explicit > /home/$USER/ez/frozen-base-.txt +# A dev.yml is NOT sufficient here -- it re-solves and can drift between +# runs even unmodified. Regenerate a component's file whenever that +# component needs a new/updated dependency baked into its frozen base. +BASE_ENV_LOCK_FILE_E3SM_TO_CMIP="$EZ_DIR/frozen-base-e3sm_to_cmip.txt" +BASE_ENV_LOCK_FILE_E3SM_DIAGS="$EZ_DIR/frozen-base-e3sm_diags.txt" +BASE_ENV_LOCK_FILE_MPAS_ANALYSIS="$EZ_DIR/frozen-base-mpas_analysis.txt" +BASE_ENV_LOCK_FILE_ZPPY_INTERFACES="$EZ_DIR/frozen-base-zppy_interfaces.txt" +BASE_ENV_LOCK_FILE_ZPPY="$EZ_DIR/frozen-base-zppy.txt" From fa55ab33901db4d9e452d4f0d53a96ca694c4455 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Tue, 11 Aug 2026 12:22:10 -0500 Subject: [PATCH 37/38] Add scripts to pin dev-env deps to Unified versions Add get_unified_versions.sh and pin_dev_env_to_unified.py to help generate frozen-base lock files that track E3SM-Unified's resolved dependency versions, narrowing dev-vs-Unified drift as a source of test_images.py ambiguity. - get_unified_versions.sh: sources a Unified load script and captures the resulting environment's package versions (pip list, with an importlib.metadata fallback). - pin_dev_env_to_unified.py: cross-references those versions against a component's dev.yml, sorting each dependency into pinned / forced deviation (dev.yml's own constraint rules out Unified's version) / flagged for manual review (ambiguous exact pins or non-numeric versions) / no Unified match, and writes a resolved dev.yml plus a Markdown report. Stdlib-only, preserves comments/formatting. Update automated_test.rst (Step 2.5) to document this workflow and to note that expected results may come from either Unified or a dev environment. --- .../source/dev_guide/tests/automated_test.rst | 74 ++- .../get_unified_versions.sh | 90 +++ .../pin_dev_env_to_unified.py | 600 ++++++++++++++++++ 3 files changed, 763 insertions(+), 1 deletion(-) create mode 100755 tests/main_branch_testing/get_unified_versions.sh create mode 100644 tests/main_branch_testing/pin_dev_env_to_unified.py diff --git a/docs/source/dev_guide/tests/automated_test.rst b/docs/source/dev_guide/tests/automated_test.rst index 739ce7a3..fdb6955f 100644 --- a/docs/source/dev_guide/tests/automated_test.rst +++ b/docs/source/dev_guide/tests/automated_test.rst @@ -42,7 +42,7 @@ Process ls -lt ${expected_results_dir} -In your Markdown report, note the date the expected results were last updated. +In your Markdown report, note the date the expected results were last updated, and whether they were generated using the E3SM-Unified environment or a dev environment (check the run's cfg/logs for the relevant ``_ENV_TYPE`` settings, or ask if it's unclear). This matters for Step 2.5 below. Step 2: Review changes since expected results were updated ========================================================== @@ -79,12 +79,84 @@ This means each of the five needs a lock file -- an exact-version ``conda list - * That component's ``dev.yml`` changed in a way that should be picked up (a new or updated dependency), or * The test script warned that ``pip install .`` pulled in something beyond the frozen base for that component (see Step C below) and you've decided to bake that change in. +Reviewing dependency-setup changes since expected results were updated +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The expected results may have been generated using either the E3SM-Unified environment or a dev environment -- see the note you made in Step 1. In practice, the first test run(s) after a new Unified release are likely to use Unified-produced results as the baseline "expected results," since a Unified release is usually the occasion for refreshing them; at other times, a dev environment may have been used instead. + +Regardless of which one produced the *current* expected results, the goal for the frozen base (below) is to track Unified's dependency versions as closely as possible -- Unified is the stable common target, independent of which source happens to be backing the expected results at any given moment. What's useful to check here, before getting to that, is narrower: whether each component's *own* dependency-setup file has changed since the expected results were updated. If it has, any image-check diffs in that component's task carry extra ambiguity (dependency version change vs. a code change in the branch under test) until dependencies have been reconciled with Unified as described in the next subsection. + +Review each of the following, comparing from the commit that was current on the date the expected results were last updated (Step 1) to the current commit on the relevant base branch: + +* ``e3sm_to_cmip``: `conda-env/dev.yml `__ +* ``e3sm_diags``: `conda-env/dev.yml `__ +* ``MPAS-Analysis``: `dev-spec.txt `__ +* ``zppy-interfaces``: `conda/dev.yml `__ +* ``zppy``: `conda/dev.yml `__ + +In your Markdown report, make a table like: + +.. code-block:: + + | Package | Dev setup changes since expected results were updated? | + | --- | --- | + | [package name](compare link, e.g. .../compare/.../) | "No changes" or a description of what changed | + ... + +Pinning dev-env dependencies to match the Unified environment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The frozen base for each component should default to matching Unified's resolved dependency versions as closely as possible -- treat Unified as the target state regardless of whether the *current* expected results happen to have come from Unified or from a dev environment. Only deviate from Unified's version for a given package when the dev environment genuinely requires it: for example, a new feature merged into one of our packages needs a dependency newer than what Unified currently ships. In that case, keep the ``dev.yml``'s own (newer) constraint for that package instead of overriding it with Unified's older version, and note the deviation explicitly in your report so it isn't mistaken for accidental drift later. + +This does not make the dev env identical to Unified -- the two cover different package sets, and conda's solver can still pick different transitive dependencies than Unified's solver did for the same top-level pin -- but it removes version drift in whatever packages they *do* share as an avoidable source of ambiguity in later image-check diffs. + +Two scripts automate the matching part (both are standard-library-only, so no ``pip install`` is needed to run them): + +* ``get_unified_versions.sh`` -- sources the Unified load script for the target machine and captures the resolved package versions of the resulting environment (via ``pip list --format=json``, with an ``importlib.metadata`` fallback, plus the interpreter's own version, since ``pip list`` doesn't report Python itself). +* ``pin_dev_env_to_unified.py`` -- cross-references those versions against a component's ``dev.yml``, sorting every dependency into one of four buckets: + + * **Pinned to Unified** -- no conflict (either the dep was unconstrained in dev.yml, or dev.yml's own constraint is satisfied by Unified's version). + * **Forced deviation** -- dev.yml has an explicit range constraint (e.g. ``>=0.23``) that Unified's version fails to satisfy. This is detected automatically and mechanically: the constraint is parsed and checked against Unified's version, so it isn't a guess -- dev.yml's own requirement rules Unified out. + * **Flagged for manual review** -- either an *exact* pin (e.g. ``numpy=1.24.3``) that differs from Unified's version, or a range constraint where the versions involved aren't plain dotted-numeric (e.g. a pre-release like ``1.11.0rc1``) and so can't be compared with confidence by a stdlib-only comparator. Both cases are genuinely ambiguous or unresolvable without more context, and the script deliberately doesn't guess -- it keeps dev.yml's version and leaves the decision to a human. + * **No Unified match** -- the package isn't in Unified at all; left as-is. + +.. code-block:: bash + + # 1. Capture Unified's resolved versions (once; reused for all five components). + # On Chrysalis, the Unified load script is: + # /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh + ./get_unified_versions.sh unified_versions.json + + # 2. For each of the five components, cross-reference and resolve: + python pin_dev_env_to_unified.py \ + --unified unified_versions.json \ + --devyml \ + --out-devyml pinned-dev-.yml \ + --out-report pin-report-.md \ + --component + +Use the resulting ``pinned-dev-.yml`` in place of the stock ``dev.yml`` in the "To (re)generate a lock file for a component" steps below. + +.. note:: + + Only the "flagged for manual review" bucket needs a human -- check ``pin-report-.md`` for those packages and decide by hand whether to accept Unified's version or keep the dev.yml pin, editing the pinned file directly if you keep it. "Forced deviation" packages need no action; the script already kept dev.yml's constraint because Unified's version provably fails it. + +.. important:: + + Confirm the Unified load script you point at is the exact release that generated the expected results (if they came from Unified) before trusting the pin. ``load_latest_...`` tracks whichever release is currently newest -- if Unified has moved on since the expected-results date, source an archived/dated load script for that specific release instead, if your site keeps one, or you'll be comparing against a newer Unified than the one that actually produced the expected-results images. + +In your Markdown report, include (or summarize) each component's ``pin-report-.md``, calling out any forced deviations and any packages still flagged for manual review -- both are relevant context for interpreting later image-check diffs for that component's task. + +If ``FREEZE_DEPENDENCIES=false`` in your config, you can skip this step (and the pinning step above) entirely -- every component will solve its own ``dev.yml`` fresh, as before. + To (re)generate a lock file for a component: .. code-block:: bash cd ${repo_parent_dir}/ # e.g. e3sm_diags, MPAS-Analysis, zppy-interfaces, zppy, e3sm_to_cmip conda env create -f /dev.yml -n tmp-lock-gen # or --file dev-spec.txt for mpas_analysis + # If you pinned dependencies to Unified above, use that file instead: + # conda env create -f pinned-dev-.yml -n tmp-lock-gen conda activate tmp-lock-gen conda list --explicit > ${EZ_DIR}/frozen-base-.txt conda deactivate diff --git a/tests/main_branch_testing/get_unified_versions.sh b/tests/main_branch_testing/get_unified_versions.sh new file mode 100755 index 00000000..f84cbcce --- /dev/null +++ b/tests/main_branch_testing/get_unified_versions.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# get_unified_versions.sh +# +# Captures the resolved package versions from an E3SM-Unified environment by +# sourcing its load script and introspecting the resulting Python +# environment directly (via `pip list --format=json`, falling back to +# `importlib.metadata`). This deliberately does NOT rely on `pixi list` or +# `conda list`: the Unified load script just puts a prebuilt environment on +# PATH, not a pixi project directory, and asking Python what's actually +# installed works no matter which tool (conda, mamba, pixi/rattler) built +# that environment. +# +# Usage: +# ./get_unified_versions.sh +# +# Example (Chrysalis): +# ./get_unified_versions.sh \ +# /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh \ +# unified_versions.json +# +# IMPORTANT: "load_latest_..." tracks whatever Unified release is currently +# newest. Make sure this is the SAME release that was active on the date +# the expected results were generated (Step 1) -- if your site keeps a +# dated/archived load script for that specific release, source that one +# instead, or you'll be comparing against a newer Unified than the one that +# actually produced the expected-results images. + +set -euo pipefail + +LOAD_SCRIPT="${1:?Usage: $0 }" +OUT_FILE="${2:?Usage: $0 }" + +if [ ! -f "$LOAD_SCRIPT" ]; then + echo "ERROR: load script not found: $LOAD_SCRIPT" >&2 + exit 1 +fi + +echo "Sourcing: $LOAD_SCRIPT" +# Third-party activation scripts (conda/pixi/etc.) are often not written to +# be safe under `set -u`/`set -e` -- e.g. they may reference environment +# variables that are only conditionally set. Relax our strict flags just +# for the source call so an unrelated unbound-variable check in someone +# else's script doesn't abort ours. +set +euo pipefail +# shellcheck disable=SC1090 +source "$LOAD_SCRIPT" +set -euo pipefail + +PY="$(command -v python || command -v python3 || true)" +if [ -z "$PY" ]; then + echo "ERROR: no python/python3 on PATH after sourcing the load script." >&2 + echo " Check that the load script actually activated an environment." >&2 + exit 1 +fi + +echo "Using interpreter: $PY" +"$PY" -c "import sys; print('sys.prefix:', sys.prefix)" + +if "$PY" -m pip list --format=json > "$OUT_FILE" 2>/dev/null && [ -s "$OUT_FILE" ]; then + echo "Wrote package list (via pip) -> $OUT_FILE" +else + echo "pip unavailable (or failed) in this env; falling back to importlib.metadata" + "$PY" -c " +import json, importlib.metadata as m +pkgs = [] +for d in m.distributions(): + try: + pkgs.append({'name': d.metadata['Name'], 'version': d.version}) + except Exception: + pass +print(json.dumps(pkgs)) +" > "$OUT_FILE" + echo "Wrote package list (via importlib.metadata) -> $OUT_FILE" +fi + +# `pip list` does not include the interpreter itself as a package, but +# dev.yml files almost always pin `python=`, so inject it explicitly. +"$PY" -c " +import json, sys +with open(sys.argv[1]) as f: + pkgs = json.load(f) +py_version = '.'.join(str(v) for v in sys.version_info[:3]) +pkgs = [p for p in pkgs if p.get('name', '').lower() != 'python'] +pkgs.append({'name': 'python', 'version': py_version}) +with open(sys.argv[1], 'w') as f: + json.dump(pkgs, f) +" "$OUT_FILE" + +COUNT=$("$PY" -c "import json,sys; print(len(json.load(open(sys.argv[1]))))" "$OUT_FILE" 2>/dev/null || echo "?") +echo "Captured $COUNT packages from the Unified environment (including python itself)." diff --git a/tests/main_branch_testing/pin_dev_env_to_unified.py b/tests/main_branch_testing/pin_dev_env_to_unified.py new file mode 100644 index 00000000..3592a3ed --- /dev/null +++ b/tests/main_branch_testing/pin_dev_env_to_unified.py @@ -0,0 +1,600 @@ +#!/usr/bin/env python3 +""" +pin_dev_env_to_unified.py + +Cross-references a component's dev.yml (or dev-spec.txt-style yaml) against +the resolved package versions from the E3SM-Unified environment used to +generate the expected results, and produces: + + 1. A modified copy of the dev env file, with each dependency resolved into + one of four buckets (see below). Only the version portion of a changed + line is rewritten -- comments, indentation, and every unrelated line + (name:, channels:, etc.) are left exactly as they were. + 2. A Markdown report explaining what was decided for every package and why. + +Standard library only -- no PyYAML, no `packaging`. The dev.yml files this +targets have a narrow, predictable shape (a `dependencies:` list, optionally +with a nested `pip:` sub-list), so a small line-based editor handles them +without pulling in a full YAML parser, and without the side effect a +YAML round-trip has of discarding every comment in the file. + +Resolution logic per package: + + - unconstrained in dev.yml (e.g. "numpy") + -> pinned to Unified's version. Nothing in dev.yml objects to this. + + - a range constraint in dev.yml (e.g. "numpy>=1.24", "numpy>=1.24,<2.0") + -> if Unified's version satisfies the constraint, pinned to Unified's + version. + -> if it does NOT satisfy the constraint, this is a genuine, provable + forced deviation: dev.yml's own requirement rules out Unified's + version, so the original dev.yml spec is kept, unchanged, and + called out as a forced deviation in the report. + -> if the versions involved aren't plain dotted-numeric (e.g. they + contain letters like "rc1" or "dev0"), the comparison can't be made + confidently without a real version-parsing library -- flagged for + manual review instead of guessing. + + - an exact pin in dev.yml (e.g. "numpy=1.24.3") that differs from + Unified's version + -> this is NOT automatically resolvable regardless of version format. + Nothing in the file says whether the exact pin is an intentional + "must be this version" requirement or just whatever conda happened + to solve last time the file was regenerated. The original dev.yml + spec is kept and the package is flagged for manual review. + + - not present in Unified at all + -> left as-is (nothing to reconcile against). + +This narrows -- but does not eliminate -- the dev-env-vs-Unified dependency +gap: even where versions are matched, conda and Unified's solver can still +resolve the same top-level pin to different transitive dependencies. + +Usage: + python pin_dev_env_to_unified.py \\ + --unified unified_versions.json \\ + --devyml /path/to/dev.yml \\ + --out-devyml pinned-dev.yml \\ + --out-report report.md \\ + --component e3sm_diags +""" + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Any, Dict, List + + +def normalize(name: str) -> str: + return name.strip().lower().replace("_", "-") + + +# --------------------------------------------------------------------------- +# Loading Unified's resolved versions (JSON from get_unified_versions.sh, or +# plain-text table output as a fallback) -- stdlib json only, no yaml needed +# here since this is not a yaml file. +# --------------------------------------------------------------------------- + + +def parse_unified_json(path: Path) -> dict: + data = json.loads(path.read_text()) + versions = {} + if isinstance(data, dict): + for key in ("packages", "data", "items"): + if key in data and isinstance(data[key], list): + data = data[key] + break + for entry in data: + if not isinstance(entry, dict): + continue + name = entry.get("name") or entry.get("Package") or entry.get("package") + version = entry.get("version") or entry.get("Version") + if name and version: + versions[normalize(str(name))] = str(version) + return versions + + +def parse_unified_text(path: Path) -> dict: + """Fallback parser for `pixi list` / similar plain-text table output.""" + versions = {} + for line in path.read_text().splitlines(): + line = line.strip() + if not line or line.lower().startswith("package"): + continue + if set(line) <= set("-+ "): + continue + parts = line.split() + if len(parts) < 2: + continue + name, version = parts[0], parts[1] + if not re.match(r"^[A-Za-z0-9]", version): + continue + versions[normalize(name)] = version + return versions + + +def load_unified_versions(path: Path) -> dict: + if path.suffix == ".json": + try: + return parse_unified_json(path) + except json.JSONDecodeError: + pass + return parse_unified_text(path) + + +# --------------------------------------------------------------------------- +# Minimal version comparison (stdlib only). Handles plain dotted-numeric +# versions (the vast majority of conda-forge packages: "1.26.4", "3.8.2", +# "2024.1.1", etc.). Anything else (letters, pre/post/dev suffixes, epochs) +# is deliberately NOT guessed at -- callers get None back and should treat +# that as "can't confidently determine", not as a pass/fail. +# --------------------------------------------------------------------------- + + +def parse_simple_version(v: str): + parts = v.strip().split(".") + try: + return tuple(int(p) for p in parts) + except ValueError: + return None + + +def _compare_tuples(a: tuple, b: tuple) -> int: + length = max(len(a), len(b)) + a = a + (0,) * (length - len(a)) + b = b + (0,) * (length - len(b)) + if a < b: + return -1 + if a > b: + return 1 + return 0 + + +CLAUSE_RE = re.compile(r"(==|!=|>=|<=|>|<)\s*([^,]+)") + + +def check_constraint(unified_version: str, constraint: str): + """ + Returns True/False if the constraint's clauses are all plain + dotted-numeric and thus confidently comparable; returns None if any + version involved can't be parsed that simply, meaning the caller should + not treat this as a resolved answer. + """ + uv = parse_simple_version(unified_version) + if uv is None: + return None + clauses = CLAUSE_RE.findall(constraint) + if not clauses: + return None + for op, verstr in clauses: + cv = parse_simple_version(verstr.strip()) + if cv is None: + return None + cmp = _compare_tuples(uv, cv) + if op == "==" and cmp != 0: + return False + if op == "!=" and cmp == 0: + return False + if op == ">=" and cmp < 0: + return False + if op == "<=" and cmp > 0: + return False + if op == ">" and cmp <= 0: + return False + if op == "<" and cmp >= 0: + return False + return True + + +# --------------------------------------------------------------------------- +# Parsing a single dev.yml dependency spec +# --------------------------------------------------------------------------- + +NAME_RE = re.compile(r"^([A-Za-z0-9_.\-]+)\s*(.*)$") + + +def parse_dep_spec(spec: str): + """ + Split a dependency spec into (name, kind, detail). + + kind is one of: + 'unconstrained' - bare package name, no version info detail=None + 'exact' - pinned to one exact version detail=version string + 'range' - a comparison constraint (>=, >, <=, <, !=, or a + comma-separated combination of these) detail=constraint string + 'unparseable' - didn't match a recognized shape detail=raw remainder + """ + spec = spec.strip() + m = NAME_RE.match(spec) + if not m: + return spec, "unparseable", None + name, rest = m.group(1), m.group(2).strip() + + if not rest: + return name, "unconstrained", None + + # Conda-style single '=' exact pin, possibly with a build string: + # e.g. "numpy=1.24.3" or "numpy=1.24.3=py311h1234abc_0" + if rest.startswith("=") and not rest.startswith("=="): + version = rest[1:].split("=")[0].strip() + return name, "exact", version + + if rest.startswith("=="): + version = rest[2:].split(",")[0].strip() + return name, "exact", version + + if rest[0] in "<>!": + return name, "range", rest + + return name, "unparseable", rest + + +# --------------------------------------------------------------------------- +# Resolving each package +# --------------------------------------------------------------------------- + + +def resolve_dependency(spec: str, unified_versions: dict, buckets: dict) -> str: + """ + Decide what to do with one dependency spec string (no surrounding + quotes/comments -- callers strip those). Appends a record to the + appropriate list in `buckets` and returns the spec string to use in the + output file (same as input if nothing changes). + """ + name, kind, detail = parse_dep_spec(spec) + key = normalize(name) + unified_version = unified_versions.get(key) + + if unified_version is None: + buckets["no_match"].append((name, spec)) + return spec + + if kind == "unconstrained": + buckets["pinned"].append((name, spec, unified_version, None)) + return f"{name}={unified_version}" + + if kind == "exact": + if detail == unified_version: + buckets["pinned"].append((name, spec, unified_version, "already matched")) + return f"{name}={unified_version}" + buckets["flagged"].append( + ( + name, + spec, + unified_version, + f"dev.yml pins exactly {detail}; Unified has {unified_version}. " + "Cannot tell from the file alone whether the exact pin is " + "required -- kept dev.yml's version pending manual review.", + ) + ) + return spec + + if kind == "range": + satisfied = check_constraint(unified_version, detail) + if satisfied is True: + buckets["pinned"].append( + ( + name, + spec, + unified_version, + f"Unified's version satisfies dev.yml's constraint '{detail}'.", + ) + ) + return f"{name}={unified_version}" + elif satisfied is False: + buckets["forced_deviation"].append( + ( + name, + spec, + unified_version, + f"Unified's version {unified_version} does NOT satisfy " + f"dev.yml's constraint '{detail}'. Kept dev.yml's own " + "constraint as a forced deviation.", + ) + ) + return spec + else: + buckets["flagged"].append( + ( + name, + spec, + unified_version, + f"Could not confidently compare constraint '{detail}' against " + f"version '{unified_version}' (non-numeric version component) " + "-- kept dev.yml's spec pending manual review.", + ) + ) + return spec + + # unparseable + buckets["flagged"].append( + ( + name, + spec, + unified_version, + "Could not parse this dependency spec's format -- kept dev.yml's " + "spec pending manual review.", + ) + ) + return spec + + +# --------------------------------------------------------------------------- +# Line-based dev.yml editing (no YAML library). Only rewrites the version +# portion of dependency lines under the `dependencies:` key; every other +# line, and every comment, is passed through unchanged. +# --------------------------------------------------------------------------- + +PIP_KEY_RE = re.compile(r"^pip\s*:\s*$") + + +def _line_indent(line: str) -> int: + return len(line) - len(line.lstrip(" ")) + + +def _strip_quotes(s: str): + """Return (unquoted, quote_char_or_None).""" + if len(s) >= 2 and s[0] == s[-1] and s[0] in "'\"": + return s[1:-1], s[0] + return s, None + + +def process_block(lines, start_idx, base_indent, unified_versions, buckets): + """ + Process a YAML list block (the items of `dependencies:` or a nested + `pip:` list) starting at start_idx, whose items are indented at + base_indent. Returns (rewritten_lines, index_after_block). + """ + out = [] + i = start_idx + while i < len(lines): + line = lines[i] + stripped = line.strip() + + if stripped == "": + out.append(line) + i += 1 + continue + + indent = _line_indent(line) + if indent < base_indent or not stripped.startswith("-"): + break + + item_text = stripped[1:].strip() + + if PIP_KEY_RE.match(item_text): + out.append(line) + i += 1 + # Skip/pass through any blank lines before the nested block. + while i < len(lines) and lines[i].strip() == "": + out.append(lines[i]) + i += 1 + if i < len(lines) and _line_indent(lines[i]) > base_indent: + nested_out, i = process_block( + lines, i, _line_indent(lines[i]), unified_versions, buckets + ) + out.extend(nested_out) + continue + + # Plain dependency spec line. Split off an inline comment if present. + comment = "" + spec_text = item_text + hash_idx = spec_text.find("#") + if hash_idx != -1: + comment = spec_text[hash_idx:] + spec_text = spec_text[:hash_idx].strip() + + spec_text, quote_char = _strip_quotes(spec_text) + new_spec = resolve_dependency(spec_text, unified_versions, buckets) + + dash_pos = line.index("-") + prefix = line[: dash_pos + 1] # leading whitespace + '-' + rendered_spec = ( + f"{quote_char}{new_spec}{quote_char}" if quote_char else new_spec + ) + rebuilt = f"{prefix} {rendered_spec}" + if comment: + rebuilt += f" {comment}" + out.append(rebuilt) + i += 1 + + return out, i + + +DEPENDENCIES_KEY_RE = re.compile(r"^dependencies\s*:\s*$") + + +def process_devyml_text(text: str, unified_versions: dict, buckets: dict) -> str: + had_trailing_newline = text.endswith("\n") + lines = text.splitlines() + + dep_idx = None + for idx, line in enumerate(lines): + if _line_indent(line) == 0 and DEPENDENCIES_KEY_RE.match(line.strip()): + dep_idx = idx + break + if dep_idx is None: + raise ValueError("Could not find a top-level 'dependencies:' key in this file.") + + # Find the indentation of the first list item after the key. + j = dep_idx + 1 + while j < len(lines) and lines[j].strip() == "": + j += 1 + if j >= len(lines) or not lines[j].strip().startswith("-"): + raise ValueError("'dependencies:' key has no list items after it.") + base_indent = _line_indent(lines[j]) + + body, end_idx = process_block(lines, j, base_indent, unified_versions, buckets) + + new_lines = lines[: dep_idx + 1] + body + lines[end_idx:] + result = "\n".join(new_lines) + if had_trailing_newline: + result += "\n" + return result + + +# --------------------------------------------------------------------------- +# Report generation +# --------------------------------------------------------------------------- + + +def build_report(component: str, buckets: dict, unified_versions: dict) -> str: + handled_keys = { + normalize(n) + for group in ("pinned", "forced_deviation", "flagged") + for (n, *_rest) in buckets[group] + } + no_match_keys = {normalize(n) for n, _ in buckets["no_match"]} + unified_only = sorted(set(unified_versions) - handled_keys - no_match_keys) + + lines = [f"# Dev-env vs Unified version pinning report: {component}", ""] + + lines.append("## Pinned to Unified's version (no conflict)") + lines.append("") + lines.append("| Package | Original dev.yml spec | Unified version | Note |") + lines.append("| --- | --- | --- | --- |") + for name, spec, uv, note in sorted(buckets["pinned"]): + lines.append(f"| {name} | {spec} | {uv} | {note or ''} |") + if not buckets["pinned"]: + lines.append("| _(none)_ | | | |") + lines.append("") + + lines.append( + "## Forced deviations (Unified's version fails dev.yml's own constraint)" + ) + lines.append("") + lines.append( + "These are kept at dev.yml's spec automatically -- dev.yml's own " + "range constraint rules out Unified's version, so this isn't a " + "guess." + ) + lines.append("") + lines.append("| Package | Original dev.yml spec | Unified version | Why |") + lines.append("| --- | --- | --- | --- |") + for name, spec, uv, note in sorted(buckets["forced_deviation"]): + lines.append(f"| {name} | {spec} | {uv} | {note} |") + if not buckets["forced_deviation"]: + lines.append("| _(none)_ | | | |") + lines.append("") + + lines.append("## Flagged for manual review (ambiguous -- please check by hand)") + lines.append("") + lines.append( + "Either an exact pin that differs from Unified's version (file alone " + "can't say whether that's intentional), or a version comparison that " + "couldn't be made confidently without a full version-parsing library " + "(e.g. a non-numeric version like a pre/dev release). Decide by hand, " + "then edit the pinned file directly if needed." + ) + lines.append("") + lines.append("| Package | Original dev.yml spec | Unified version | Why |") + lines.append("| --- | --- | --- | --- |") + for name, spec, uv, note in sorted(buckets["flagged"]): + lines.append(f"| {name} | {spec} | {uv} | {note} |") + if not buckets["flagged"]: + lines.append("| _(none)_ | | | |") + lines.append("") + + lines.append("## Left as-is (in dev.yml, not found in Unified)") + lines.append("") + lines.append("| Package | Original dev.yml spec |") + lines.append("| --- | --- |") + for name, spec in sorted(buckets["no_match"]): + lines.append(f"| {name} | {spec} |") + if not buckets["no_match"]: + lines.append("| _(none)_ | |") + lines.append("") + + lines.append("## In Unified but not in dev.yml (informational only, not acted on)") + lines.append("") + lines.append(", ".join(unified_only) if unified_only else "(none)") + lines.append("") + + lines.append( + "**Caveat:** matching shared top-level packages by version does not " + "guarantee identical transitive dependencies -- conda and Unified's " + "solver can resolve the same top-level pin to different " + "sub-dependencies or builds. This narrows the dev-vs-Unified gap, " + "it does not eliminate it." + ) + + return "\n".join(lines) + + +def main(): + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument( + "--unified", + required=True, + type=Path, + help="Output file from get_unified_versions.sh (.json or .txt)", + ) + ap.add_argument( + "--devyml", + required=True, + type=Path, + help="Path to the component's dev.yml / dev-spec.txt", + ) + ap.add_argument( + "--out-devyml", + required=True, + type=Path, + help="Where to write the resolved copy", + ) + ap.add_argument( + "--out-report", + required=True, + type=Path, + help="Where to write the Markdown report", + ) + ap.add_argument( + "--component", + default="component", + help="Name for the report header (e.g. e3sm_diags)", + ) + args = ap.parse_args() + + unified_versions = load_unified_versions(args.unified) + if not unified_versions: + sys.exit(f"Could not parse any package versions from {args.unified}") + + buckets: Dict[str, List[Any]] = { + "pinned": [], + "forced_deviation": [], + "flagged": [], + "no_match": [], + } + + original_text = args.devyml.read_text() + try: + new_text = process_devyml_text(original_text, unified_versions, buckets) + except ValueError as e: + sys.exit(f"Could not process {args.devyml}: {e}") + + header = ( + "# Lines below were resolved by pin_dev_env_to_unified.py.\n" + "# Packages with no conflicting dev.yml constraint were pinned to\n" + "# Unified's resolved version. Packages where dev.yml's own\n" + "# constraint rules out Unified's version, or where an exact pin's\n" + "# intent is ambiguous, were left as-is -- see the report for which\n" + "# ones need a manual look. All other lines/comments are untouched.\n" + ) + args.out_devyml.write_text(header + new_text) + + report = build_report(args.component, buckets, unified_versions) + args.out_report.write_text(report) + + print(f"Pinned to Unified: {len(buckets['pinned'])}") + print(f"Forced deviations: {len(buckets['forced_deviation'])}") + print(f"Flagged for manual review: {len(buckets['flagged'])}") + print(f"No Unified match: {len(buckets['no_match'])}") + print(f"Wrote resolved dev env file -> {args.out_devyml}") + print(f"Wrote report -> {args.out_report}") + + +if __name__ == "__main__": + main() From 0b9c354a1234613b40ad745aff1b6412f9ebe3fe Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Tue, 11 Aug 2026 14:41:09 -0500 Subject: [PATCH 38/38] Fixes for frozen env setup --- .gitignore | 3 + .../frozen_env_setup/create_frozen_envs.sh | 67 +++++++++++ .../get_unified_versions.sh | 0 .../pin_dev_env_to_unified.py | 109 ++++++++++++++++-- 4 files changed, 172 insertions(+), 7 deletions(-) create mode 100755 tests/main_branch_testing/frozen_env_setup/create_frozen_envs.sh rename tests/main_branch_testing/{ => frozen_env_setup}/get_unified_versions.sh (100%) rename tests/main_branch_testing/{ => frozen_env_setup}/pin_dev_env_to_unified.py (82%) diff --git a/.gitignore b/.gitignore index 56bc1cdb..e8eb2394 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,9 @@ test_images_summary.md test_*_output tests/*.cfg.txt tests/integration/image_check_failures* +tests/main_branch_testing/frozen_env_setup/pin-report-*.md +tests/main_branch_testing/frozen_env_setup/pinned-dev-*.txt +tests/main_branch_testing/frozen_env_setup/pinned-dev-*.yml # Sphinx documentation docs/_build/ diff --git a/tests/main_branch_testing/frozen_env_setup/create_frozen_envs.sh b/tests/main_branch_testing/frozen_env_setup/create_frozen_envs.sh new file mode 100755 index 00000000..d9bf6e6a --- /dev/null +++ b/tests/main_branch_testing/frozen_env_setup/create_frozen_envs.sh @@ -0,0 +1,67 @@ +# Run from zppy/tests/main_branch_testing/frozen_env_setup + +#!/usr/bin/env bash +set -euo pipefail + +# --- Config ----------------------------------------------------------- +UNIFIED_SCRIPT=/lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh +REPO_DIR=/lcrc/group/e3sm/ac.forsyth2/zppy_main_branch_test_dirs +CONDA_PROFILE="$HOME/miniforge3/etc/profile.d/conda.sh" + +# name -> path (relative to the repo dir) of that component's dev-env file. +declare -A DEVYML_PATHS=( + ["e3sm_to_cmip"]="conda-env/dev.yml" + ["e3sm_diags"]="conda-env/dev.yml" + ["MPAS-Analysis"]="dev-spec.txt" + ["zppy-interfaces"]="conda/dev.yml" + ["zppy"]="conda/dev.yml" +) + +# name -> the lowercase/underscore form zppy_test.cfg expects in +# BASE_ENV_LOCK_FILE_ (frozen-base-.txt). Doesn't always +# match the repo directory name (e.g. zppy-interfaces -> zppy_interfaces, +# MPAS-Analysis -> mpas_analysis). +declare -A CFG_NAMES=( + ["e3sm_to_cmip"]="e3sm_to_cmip" + ["e3sm_diags"]="e3sm_diags" + ["MPAS-Analysis"]="mpas_analysis" + ["zppy-interfaces"]="zppy_interfaces" + ["zppy"]="zppy" +) + +# Fixed order, since associative-array iteration order isn't guaranteed. +names_ordered=(e3sm_to_cmip e3sm_diags MPAS-Analysis zppy-interfaces zppy) + +# --- Step 1: capture Unified's resolved versions (once) --------------- +./get_unified_versions.sh "${UNIFIED_SCRIPT}" "${REPO_DIR}/unified_versions.json" +echo "Done creating unified_versions.json" + +# --- Step 2: pin each component's dev-env file, then build its frozen base +source "$CONDA_PROFILE" + +for name in "${names_ordered[@]}"; do + devyml_rel="${DEVYML_PATHS[$name]}" + ext="${devyml_rel##*.}" # "yml" for dev.yml, "txt" for dev-spec.txt + cfg_name="${CFG_NAMES[$name]}" + pinned_devyml="pinned-dev-${name}.${ext}" + + python pin_dev_env_to_unified.py \ + --unified "${REPO_DIR}/unified_versions.json" \ + --devyml "${REPO_DIR}/${name}/${devyml_rel}" \ + --out-devyml "${pinned_devyml}" \ + --out-report "pin-report-${name}.md" \ + --component "${name}" + echo "Done pinning dev env for ${name}" + + if [ "$ext" = "yml" ]; then + conda env create -f "${pinned_devyml}" -n tmp-lock-gen + else + conda create --name tmp-lock-gen --file "${pinned_devyml}" --yes + fi + conda activate tmp-lock-gen + conda list --explicit > "${REPO_DIR}/frozen-base-${cfg_name}.txt" + conda deactivate + conda remove --yes --all --name tmp-lock-gen + + echo "Done creating frozen env for ${name}" +done diff --git a/tests/main_branch_testing/get_unified_versions.sh b/tests/main_branch_testing/frozen_env_setup/get_unified_versions.sh similarity index 100% rename from tests/main_branch_testing/get_unified_versions.sh rename to tests/main_branch_testing/frozen_env_setup/get_unified_versions.sh diff --git a/tests/main_branch_testing/pin_dev_env_to_unified.py b/tests/main_branch_testing/frozen_env_setup/pin_dev_env_to_unified.py similarity index 82% rename from tests/main_branch_testing/pin_dev_env_to_unified.py rename to tests/main_branch_testing/frozen_env_setup/pin_dev_env_to_unified.py index 3592a3ed..4e6965d3 100644 --- a/tests/main_branch_testing/pin_dev_env_to_unified.py +++ b/tests/main_branch_testing/frozen_env_setup/pin_dev_env_to_unified.py @@ -204,6 +204,11 @@ def parse_dep_spec(spec: str): 'exact' - pinned to one exact version detail=version string 'range' - a comparison constraint (>=, >, <=, <, !=, or a comma-separated combination of these) detail=constraint string + 'wildcard' - a conda build-string selector where the version + itself is '*' (e.g. "esmf=*=mpi_mpich_*") -- this + is not a version pin at all, just a build-variant + selector, so there's nothing to compare against + Unified. detail=raw remainder 'unparseable' - didn't match a recognized shape detail=raw remainder """ spec = spec.strip() @@ -219,6 +224,10 @@ def parse_dep_spec(spec: str): # e.g. "numpy=1.24.3" or "numpy=1.24.3=py311h1234abc_0" if rest.startswith("=") and not rest.startswith("=="): version = rest[1:].split("=")[0].strip() + if version == "*": + # e.g. "esmf=*=mpi_mpich_*" -- selecting a build variant, not a + # version. Nothing to compare against Unified here. + return name, "wildcard", rest return name, "exact", version if rest.startswith("=="): @@ -308,6 +317,19 @@ def resolve_dependency(spec: str, unified_versions: dict, buckets: dict) -> str: ) return spec + if kind == "wildcard": + buckets["flagged"].append( + ( + name, + spec, + unified_version, + f"'{detail}' is a build-string selector (version is '*'), not a " + "version pin -- there's nothing to compare against Unified's " + "version. Left as-is.", + ) + ) + return spec + # unparseable buckets["flagged"].append( ( @@ -353,7 +375,12 @@ def process_block(lines, start_idx, base_indent, unified_versions, buckets): line = lines[i] stripped = line.strip() - if stripped == "": + # Blank lines and full-line comments (e.g. section-divider comments + # like "# Base" / "# ===...===", or a commented-out dependency like + # "# - somepkg 1.2.3") don't affect YAML structure -- pass them + # through and keep scanning for the next real item rather than + # treating them as the end of the list. + if stripped == "" or stripped.startswith("#"): out.append(line) i += 1 continue @@ -367,8 +394,11 @@ def process_block(lines, start_idx, base_indent, unified_versions, buckets): if PIP_KEY_RE.match(item_text): out.append(line) i += 1 - # Skip/pass through any blank lines before the nested block. - while i < len(lines) and lines[i].strip() == "": + # Skip/pass through any blank or comment-only lines before the + # nested block. + while i < len(lines) and ( + lines[i].strip() == "" or lines[i].strip().startswith("#") + ): out.append(lines[i]) i += 1 if i < len(lines) and _line_indent(lines[i]) > base_indent: @@ -418,9 +448,13 @@ def process_devyml_text(text: str, unified_versions: dict, buckets: dict) -> str if dep_idx is None: raise ValueError("Could not find a top-level 'dependencies:' key in this file.") - # Find the indentation of the first list item after the key. + # Find the indentation of the first list item after the key, skipping + # any blank or comment-only lines (e.g. a "# Base" section-divider + # comment right after "dependencies:"). j = dep_idx + 1 - while j < len(lines) and lines[j].strip() == "": + while j < len(lines) and ( + lines[j].strip() == "" or lines[j].strip().startswith("#") + ): j += 1 if j >= len(lines) or not lines[j].strip().startswith("-"): raise ValueError("'dependencies:' key has no list items after it.") @@ -428,13 +462,74 @@ def process_devyml_text(text: str, unified_versions: dict, buckets: dict) -> str body, end_idx = process_block(lines, j, base_indent, unified_versions, buckets) - new_lines = lines[: dep_idx + 1] + body + lines[end_idx:] + # lines[:j] includes the 'dependencies:' line itself plus any blank/ + # comment lines we skipped over while locating the first real item + # (e.g. a "# Base" section-divider comment) -- keep them. + new_lines = lines[:j] + body + lines[end_idx:] result = "\n".join(new_lines) if had_trailing_newline: result += "\n" return result +def process_flat_spec_text(text: str, unified_versions: dict, buckets: dict) -> str: + """ + Handle the other real dev-env format in use here: a plain spec-list file + consumed via `conda create --file dev-spec.txt` (as MPAS-Analysis does). + One package spec per line, comment lines start with '#', operators are + typically space-separated from the name (e.g. "python >=3.11"). No YAML + structure at all -- just rewrite each non-comment, non-blank line in + place. + """ + had_trailing_newline = text.endswith("\n") + lines = text.splitlines() + out = [] + for line in lines: + stripped = line.strip() + if stripped == "" or stripped.startswith("#"): + out.append(line) + continue + + indent_len = len(line) - len(line.lstrip(" ")) + leading_ws = line[:indent_len] + content = line[indent_len:] + + comment = "" + hash_idx = content.find("#") + if hash_idx != -1: + comment = content[hash_idx:] + content = content[:hash_idx].rstrip() + + new_spec = resolve_dependency(content.strip(), unified_versions, buckets) + rebuilt = f"{leading_ws}{new_spec}" + if comment: + rebuilt += f" {comment}" + out.append(rebuilt) + + result = "\n".join(out) + if had_trailing_newline: + result += "\n" + return result + + +def process_env_file_text(text: str, unified_versions: dict, buckets: dict) -> str: + """ + Dispatch to the right format handler. The two dev-env formats actually + in use here are a YAML `dependencies:` list (dev.yml, conda/dev.yml, + conda-env/dev.yml) and a flat one-spec-per-line file consumed via + `conda create --file ...` (dev-spec.txt). Detect which one this is by + checking for a top-level `dependencies:` key. + """ + lines = text.splitlines() + has_dependencies_key = any( + _line_indent(line) == 0 and DEPENDENCIES_KEY_RE.match(line.strip()) + for line in lines + ) + if has_dependencies_key: + return process_devyml_text(text, unified_versions, buckets) + return process_flat_spec_text(text, unified_versions, buckets) + + # --------------------------------------------------------------------------- # Report generation # --------------------------------------------------------------------------- @@ -571,7 +666,7 @@ def main(): original_text = args.devyml.read_text() try: - new_text = process_devyml_text(original_text, unified_versions, buckets) + new_text = process_env_file_text(original_text, unified_versions, buckets) except ValueError as e: sys.exit(f"Could not process {args.devyml}: {e}")