From f1955ce00b40f6f05931f5d1d554d2e635b35975 Mon Sep 17 00:00:00 2001 From: ShixuanZhang Date: Mon, 16 Sep 2024 17:55:18 -0500 Subject: [PATCH 01/47] Add PCMDI Diags to zppy --- conda/chrisalis.yml | 34 + zppy/__main__.py | 4 + zppy/defaults/default.ini | 229 +++ zppy/pcmdi_diags.py | 289 ++++ zppy/templates/coupled_global.py | 872 +++++++++++ zppy/templates/global_time_series.bash | 0 .../e3sm_to_cmip/default_metadata.json | 0 .../e3sm_to_cmip/vrt_remap_plev19.nc | Bin 0 -> 1260 bytes zppy/templates/pcmdi_diags.bash | 1308 +++++++++++++++++ zppy/templates/ts.bash | 86 ++ 10 files changed, 2822 insertions(+) create mode 100644 conda/chrisalis.yml mode change 100644 => 100755 zppy/defaults/default.ini create mode 100644 zppy/pcmdi_diags.py create mode 100644 zppy/templates/coupled_global.py mode change 100644 => 100755 zppy/templates/global_time_series.bash mode change 100644 => 100755 zppy/templates/inclusions/e3sm_to_cmip/default_metadata.json create mode 100755 zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc create mode 100755 zppy/templates/pcmdi_diags.bash mode change 100644 => 100755 zppy/templates/ts.bash diff --git a/conda/chrisalis.yml b/conda/chrisalis.yml new file mode 100644 index 00000000..cdefec7f --- /dev/null +++ b/conda/chrisalis.yml @@ -0,0 +1,34 @@ +name: zppy_dev +channels: + - conda-forge + - defaults +dependencies: + # Base + # ================= + - python=3.9.13 + - pip=22.2.2 + - configobj=5.0.6 + - jinja2=3.1.2 + - mache>=1.5.0 + - mpas_tools>=0.15.0 + - pillow=9.2.0 + # Developer Tools + # ================= + # If versions are updated, also update 'rev' in `.pre-commit-config.yaml` + - black=22.8.0 # version from https://anaconda.org/conda-forge/black + - flake8=5.0.4 # version from https://anaconda.org/conda-forge/flake8 + # This line also implicitly installs isort + - flake8-isort=4.2.0 # version from https://anaconda.org/conda-forge/flake8-isort + - mypy=0.982 # version from https://anaconda.org/conda-forge/mypy + - pre-commit=2.20.0 # version from https://anaconda.org/conda-forge/pre-commit + - tbump=6.9.0 + # Documentation + # If versions are updated, also update in `.github/workflows/build_workflow.yml` + # ================= + - sphinx=5.2.3 + - sphinx-multiversion=0.2.4 + - sphinx_rtd_theme=1.0.0 + # Need to pin docutils because 0.17 has a bug with unordered lists + # https://github.com/readthedocs/sphinx_rtd_theme/issues/1115 + - docutils=0.16 +prefix: /home/ac.szhang/.conda/envs/zppy_dev diff --git a/zppy/__main__.py b/zppy/__main__.py index fe2d05d3..366f5320 100644 --- a/zppy/__main__.py +++ b/zppy/__main__.py @@ -20,6 +20,7 @@ from zppy.ilamb import ilamb from zppy.logger import _setup_custom_logger from zppy.mpas_analysis import mpas_analysis +from zppy.pcmdi_diags import pcmdi_diags from zppy.tc_analysis import tc_analysis from zppy.ts import ts from zppy.utils import check_status, submit_script @@ -270,6 +271,9 @@ def _launch_scripts(config: ConfigObj, script_dir, job_ids_file, plugins) -> Non # ilamb tasks existing_bundles = ilamb(config, script_dir, existing_bundles, job_ids_file) + # pcmdi_diags tasks + existing_bundles = pcmdi_diags(config, script_dir, existing_bundles, job_ids_file) + # zppy external plugins for plugin in plugins: # Get plugin module function diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini old mode 100644 new mode 100755 index 454f68cb..f9304578 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -124,9 +124,12 @@ input_component = string(default="") [ts] area_nm = string(default="area") +cmip_metadata = string(default="inclusions/e3sm_to_cmip/default_metadata.json") +cmip_plevdata = string(default="inclusions/e3sm_to_cmip/vrt_remap_plev19.nc") # Days per file dpf = integer(default=30) extra_vars = string(default="") +interp_vars = string(default="U,V,T,Q,RELHUM,OMEGA,Z3") # Time-steps per day tpd = integer(default=1) # Model component having generated input files (eam, eamxx, elm, mosart, ...) @@ -134,6 +137,8 @@ input_component = string(default="") [[__many__]] area_nm = string(default=None) + cmip_metadata = string(default=None) + cmip_plevdata = string(default=None) dpf = integer(default=None) extra_vars = string(default=None) tpd = integer(default=None) @@ -151,6 +156,7 @@ input_component = string(default="") cmip_metadata = string(default=None) cmip_vars = string(default=None) input_component = string(default=None) + interp_vars = string(default=None) [tc_analysis] # NOTE: always overrides value in [default] @@ -159,6 +165,229 @@ input_files = string(default="eam.h2") # If not set, zppy will attempt to infer this value from the topography file. res = string(default="") +[pcmdi_diags] +backend = string(default="mpl") +cfg = string(default="") +# Title of zppy-pcmdi diagnostics +pcmdi_webtitle = string(default="E3SM-PMP Diagnostics") +# Version of zppy-pcmdi code +pcmdi_version = string(default="v3.8.2") +# File of specified regions for mean climate calculation +regions_specs = string(default="pcmdi_data/region/regions_specs.json") +# File of observation data name for mean climate calculation +reference_alias = string(default="pcmdi_data/reference/reference_alias.json") +# Utility file with functions for zppy-pcmdi data processing +pcmdi_zppy_util = string(default="pcmdi_data/utility/pcmdi_zppy_util.py") +# Utility file with functions for zppy-pcmdi viewer page processing +pcmdi_viewer_util = string(default="pcmdi_data/utility/pcmdi_viewer_util.py") +# Template files for zppy-pcmdi viewer page processing +pcmdi_viewer_template = string(default="pcmdi_data/viewer") +# File of variable list to generate synthetic metrics plot +synthetic_metrics = string(default="pcmdi_data/synthetic_metrics/synthetic_metrics_list.json") +# Path to observation time-series data +# Required for "mean_climate","variability_mode","enso" +obs_ts = string(default="") +# observational data sets (see reference_alias.json) +# observation data tag in reference_alias +obs_sets = string(default="default") +# options specific for constructing pcmdi preferred file name conventions +# required for "model_vs_obs" comparison +model_name = string(default="e3sm.historical.v3-LR.0051") +model_tableID = string(default="Amon") +# required for "model_vs_model" comparison +model_name_ref = string(default="ERA5") +model_tableID_ref = string(default="Amon") +# variables in the cmip6 table that can be potentially used by pcmdi +# this list depends on the definition of cmip variable +# required for "mean climate" diagnostics +cmip_vars = string(default="pr,prw,psl,rlds,rldscs,rlut,rlutcs,rsut,rsutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rtmt,sfcWind,tas,tauu,tauv,ts,ta,ua,va,zg") +# flag to process the land/sea mask within pcmdi +generate_sftlf = string(default=True) +# variables to be used by the pcmdi diagnostics +# needs to setup for each subsections, defalut setup is the mean climate metrics +vars = string(default="pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500") +# sets of diagnostics from pcmdi package +sets = string_list(default=list("mean_climate","variability_modes_atm","variability_modes_cpl","enso","synthetic_plots")) +# options to identify subset of pcmdi drivers ("mean_climate","variability_mode","enso") +subsets = string_list(default=list("mean_climate","variability_modes","enso")) +########################################################################################### +# Required for run_type="model_vs_model" runs, different from e3sm_diag, +# model_vs_model in pcmdi referred to the comparision of two model simulations +# with observations and cmip models. +########################################################################################### +#path for reference model data (time series) +reference_data_path = string(default="") +reference_data_path_ts = string(default="") +# pcmdi_diags.py will set to match `years` if not specified +ref_years = string_list(default=list("")) +# The years increment for reference data +ts_num_years_ref = integer(default=5) +# Final year (i.e., the last available year) for the reference data +ref_final_yr = string(default="") +# Start year for the reference data +ref_start_yr = string(default="") +# reference model name +ref_name = string(default="") +# The years range for test model data +ts_years = string_list(default=list("")) +# The years increment for test model data +ts_num_years = integer(default=5) +# Set to true to swap test and ref when run_type="model_vs_model" +swap_test_ref = boolean(default=False) +########################################################################################## +# below followed the setup in e3sm_diag but used for PCMDI workflow +########################################################################################## +# See url +multiprocessing = boolean(default=True) +# See url +num_workers = integer(default=24) +# See url +figure_format = string(default="png") +# comparision type (same as e3sm_diag) +run_type = string(default="model_vs_obs") +# Used to label the results directory +# Options are "model_vs_obs" and "model_vs_model" +tag = string(default="model_vs_obs") +########################################################################################## +# options for pcmdi mode varibility diagnostics +# vars = "psl" for atm_modes +# vars = "ts" for cpl_modes +######################################################################################### +#name of atmospheric modes varibility +atm_modes = string(default="NAM,NAO,PNA,NPO,SAM,PSA1,PSA2") +#name of coupled modes varibility +cpl_modes = string(default="PDO,NPGO,AMO") +#keywards for unit conversion in pcmdi (model) +ModUnitsAdjust = string(default="") +#keywards for unit conversion in pcmdi (observation) +ObsUnitsAdjust = string(default="") +#frequency of the model data +frequency = string(default="mo") +#options specific for mode varibility metrics in pcmdi +seasons = string(default="monthly") +landmask = string(default=False) +RmDomainMean = string(default=True) +EofScaling = string(default=False) +ConvEOF = string(default=True) +CBF = string(default=True) +mov_plot_obs = string(default=True) +mov_plot_model = string(default=True) +mov_nc_out_obs = string(default=True) +mov_nc_out_model = string(default=True) +########################################################################################## +# options for pcmdi enso diagnostics +# vars = "psl,pr,prsn,ts,tas,tauu,tauv,hflx,hfss,rlds,rsds,rlus,rlut,rsdt" +########################################################################################## +enso_groups = string(default="ENSO_perf,ENSO_proc,ENSO_tel") +########################################################################################## +# optional for mean climate diagnostics +# vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs, +# rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200, +# va-850,zg-500" +########################################################################################## +# model data grid after remapping +grid = string(default="180x360_aave") +#default regions for mean climate metrics data +#more options can be found at "regions_specs" +#regions = string(default="global,ocean,land,NHEX,SHEX,TROPICS,NHEX_ocean,SHEX_ocean,NHEX_land,SHEX_land,ocean_50S50N") +regions = string(default="global,ocean,land,NHEX,SHEX,TROPICS") +# save derived climatology data +save_test_clims = string(default=True) +# Regridding by pcmdi (required for mean climate) +# OPTIONS: '1x1' for e3sm, '2.5x2.5' for cmip, or an actual cdms2 grid object, +target_grid = string(default="1x1") +# OPTIONS: "1px1p" for e3sm, "2p5x2p5" for cmip, description on the selected grid +target_grid_string = string(default="1px1p") +# OPTIONS: 'regrid2','esmf' +regrid_tool = string(default="esmf") +# OPTIONS: 'linear','conservative', only if tool is esmf +regrid_method = string(default="regrid2") +# OPTIONS: 'linear','conservative', only if tool is esmf +regrid_method_ocn = string(default="conservative") +################################################################### +#options for synthetic plots with exisiting pcmdi cmip model metrics +################################################################### +synthetic_sets = string(default="portrait,parcoord") +# path to pcmdi generated cmip metrics data +cmip_enso_dir = string(default="") +cmip_clim_dir = string(default="") +cmip_movs_dir = string(default="") +#group of pcmdi generated cmip metrics data (mip.exp.version) +cmip_clim_set = string(default="cmip6.historical.v20250108") +cmip_movs_set = string(default="cmip6.historical.v20220825") +cmip_enso_set = string(default="cmip6.historical.v20210620") +# options shared by pcmdi +pcmdi_debug = string(default=False) + + [[__many__]] + backend = string(default=None) + cfg = string(default=None) + vars = string(default=None) + grid = string(default=None) + cmip_enso_set = string(default=None) + cmip_clim__set = string(default=None) + cmip_movs_set = string(default=None) + cmip_enso_dir = string(default=None) + cmip_clim_dir = string(default=None) + cmip_movs_dir = string(default=None) + synthetic_sets = string(default=None) + reference_alias = string(default=None) + regions_specs = string(default=None) + pcmdi_webtitle = string(default=None) + pcmdi_version = string(default=None) + pcmdi_zppy_util = string(default=None) + pcmdi_viewer_util = string(default=None) + pcmdi_viewer_template = string(default=None) + synthetic_metrics = string(default=None) + multiprocessing = boolean(default=None) + num_workers = integer(default=None) + obs_ts = string(default=None) + figure_format = string(default=None) + ref_name = string(default=None) + ref_years = string_list(default=None) + ref_start_yr = string(default=None) + ref_final_yr = string(default=None) + reference_data_path = string(default=None) + reference_data_path_ts = string(default=None) + run_type = string(default=None) + sets = string_list(default=None) + swap_test_ref = boolean(default=None) + tag = string(default=None) + ts_years = string_list(default=None) + ts_num_years = integer(default=None) + ts_num_years_ref = integer(default=None) + target_grid = string(default=None) + target_grid_string = string(default=None) + regrid_tool = string(default=None) + regrid_method = string(default=None) + regrid_method_ocn = string(default=None) + obs_sets = string(default=None) + regions = string(default=None) + save_test_clims = string(default=None) + seasons = string(default=None) + RmDomainMean = string(default=None) + EofScaling = string(default=None) + ConvEOF = string(default=None) + CBF = string(default=None) + subsets = string_list(default=None) + landmask = string(default=None) + frequency = string(default=None) + generate_sftlf = string(default=None) + atm_modes = string(default=None) + cpl_modes = string(default=None) + enso_groups = string(default=None) + ModUnitsAdjust = string(default=None) + ObsUnitsAdjust = string(default=None) + model_name = string(default=None) + model_name_ref = string(default=None) + model_tableID = string(default=None) + model_vars = string(default=None) + mov_nc_out_obs = string(default=None) + mov_nc_out_model = string(default=None) + mov_plot_obs = string(default=None) + mov_plot_model = string(default=None) + pcmdi_debug = string(default=None) + [e3sm_diags] # See https://e3sm-project.github.io/e3sm_diags/_build/html/master/available-parameters.html backend = string(default="mpl") diff --git a/zppy/pcmdi_diags.py b/zppy/pcmdi_diags.py new file mode 100644 index 00000000..6cbe9c13 --- /dev/null +++ b/zppy/pcmdi_diags.py @@ -0,0 +1,289 @@ +import os +from typing import Any, Dict, List, Set, Tuple + +from zppy.bundle import handle_bundles +from zppy.utils import ( + ParameterGuessType, + add_dependencies, + check_parameter_defined, + check_required_parameters, + check_status, + define_or_guess, + define_or_guess2, + get_file_names, + get_tasks, + get_years, + initialize_template, + make_executable, + print_url, + submit_script, + write_settings_file, +) + + +# ----------------------------------------------------------------------------- +def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): + + template, _ = initialize_template(config, "pcmdi_diags.bash") + + # --- List of pcmdi_diags tasks --- + tasks: List[Dict[str, Any]] = get_tasks(config, "pcmdi_diags") + if len(tasks) == 0: + return existing_bundles + + # --- Generate and submit pcmdi_diags scripts --- + for c in tasks: + dependencies: List[str] = [] + c["sub"] = define_or_guess( + c, "subsection", "sub", ParameterGuessType.SECTION_GUESS + ) + check_parameters_for_bash(c) + + c["scriptDir"] = script_dir + if "ts_num_years" in c.keys(): + c["ts_num_years"] = int(c["ts_num_years"]) + + # procedure type for e3sm_to_cmip + c["cmor_tables_prefix"] = c["diagnostics_base_path"] + + # check and set parameter for pcmdi + c["pcmdi_external_prefix"] = c["diagnostics_base_path"] + check_parameters_for_pcmdi(c) + + # Loop over year sets + year_sets: List[Tuple[int, int]] = get_years(c["ts_years"]) + ref_year_sets: List[Tuple[int, int]] + if ("ref_years" in c.keys()) and (c["ref_years"] != [""]): + ref_year_sets = get_years(c["ref_years"]) + else: + ref_year_sets = year_sets + + for i, (s, rs) in enumerate(zip(year_sets, ref_year_sets)): + c["year1"] = s[0] + c["year2"] = s[1] + if ("last_year" in c.keys()) and (c["year2"] > c["last_year"]): + continue # Skip this year set + + c["ref_year1"] = rs[0] + c["ref_year2"] = rs[1] + + if c["sub"] != "synthetic_plots": + check_and_define_parameters(c) + else: + prefix = f"pcmdi_diags_{c['sub']}_{c['run_type']}" + print(prefix) + c["prefix"] = prefix + + bash_file, settings_file, status_file = get_file_names( + script_dir, c["prefix"] + ) + skip: bool = check_status(status_file) + if skip: + continue + + # Create script + with open(bash_file, "w") as f: + f.write(template.render(**c)) + make_executable(bash_file) + + # List of dependencies + # Iterate from year1 to year2 incrementing by the number of years per time series file. + if "ts_num_years" in c.keys(): + for yr in range(c["year1"], c["year2"], c["ts_num_years"]): + add_ts_dependencies(c, dependencies, script_dir, yr) + + if c["sub"] == "synthetic_plots": + add_pcmdi_dependencies(c, dependencies, script_dir) + if i < len(year_sets) - 1: + continue + + c["dependencies"] = dependencies + write_settings_file(settings_file, c, s) + + export = "ALL" + existing_bundles = handle_bundles( + c, + bash_file, + export, + dependFiles=dependencies, + existing_bundles=existing_bundles, + ) + if not c["dry_run"]: + if c["bundle"] == "": + # Submit job + submit_script( + bash_file, + status_file, + export, + job_ids_file, + dependFiles=dependencies, + fail_on_dependency_skip=c["fail_on_dependency_skip"], + ) + else: + print(f"...adding to bundle {c['bundle']}") + print(f" environment_commands={c['environment_commands']}") + print_url(c, "pcmdi_diags") + + return existing_bundles + + +def check_parameters_for_bash(c: Dict[str, Any]) -> None: + if c["sub"] != "synthetic_plots": + check_required_parameters( + c, + set( + ["mean_climate", "variability_mode_cpl", "variability_mode_atm", "enso"] + ), + "ref_final_yr", + ) + check_required_parameters( + c, + set( + ["mean_climate", "variability_mode_cpl", "variability_mode_atm", "enso"] + ), + "ref_start_yr", + ) + check_required_parameters( + c, + set( + ["mean_climate", "variability_mode_cpl", "variability_mode_atm", "enso"] + ), + "ref_end_yr", + ) + + +def check_parameters_for_pcmdi(c: Dict[str, Any]) -> None: + # check and set up the external data needed by pcmdi + if c["sub"] == "synthetic_plots": + define_or_guess2( + c, + "cmip_enso_dir", + f"{c['diagnostics_base_path']}/pcmdi_data/metrics_data/enso_metric", + ParameterGuessType.PATH_GUESS, + ) + define_or_guess2( + c, + "cmip_clim_dir", + f"{c['diagnostics_base_path']}/pcmdi_data/metrics_data/mean_climate", + ParameterGuessType.PATH_GUESS, + ) + define_or_guess2( + c, + "cmip_movs_dir", + f"{c['diagnostics_base_path']}/pcmdi_data/metrics_data/variability_modes", + ParameterGuessType.PATH_GUESS, + ) + + +def check_mvm_only_parameters_for_bash(c: Dict[str, Any]) -> None: + check_parameter_defined(c, "reference_data_path_ts") + check_parameter_defined(c, "model_name_ref") + check_parameter_defined(c, "model_tableID_ref") + if c["sub"] != "synthetic_plots": + check_required_parameters( + c, + set( + ["mean_climate", "variability_mode_cpl", "variability_mode_atm", "enso"] + ), + "ref_start_yr", + ) + ts_sets = set( + [ + "mean_climate", + "variability_mode_cpl", + "variability_mode_atm", + "enso", + ] + ) + check_required_parameters(c, ts_sets, "ts_num_years_ref") + check_required_parameters(c, ts_sets, "ts_subsection") + + +def check_and_define_parameters(c: Dict[str, Any]) -> None: + # TODO: do this based on sets, rather than by relying on the user setting ts_num_years + if "ts_num_years" in c.keys(): + define_or_guess2( + c, + "obs_ts", + f"{c['diagnostics_base_path']}/observations/Atm/time-series/", + ParameterGuessType.PATH_GUESS, + ) + prefix: str + if c["run_type"] == "model_vs_obs": + prefix = ( + f"pcmdi_diags_{c['sub']}_{c['run_type']}_{c['year1']:04d}-{c['year2']:04d}" + ) + elif c["run_type"] == "model_vs_model": + check_mvm_only_parameters_for_bash(c) + prefix = f"pcmdi_diags_{c['sub']}_{c['run_type']}_{c['year1']:04d}-{c['year2']:04d}_vs_{c['ref_year1']:04d}-{c['ref_year2']:04d}" + reference_data_path = c["reference_data_path"].split("/post")[0] + "/post" + if set( + ["mean_climate", "variability_mode_cpl", "variability_mode_atm", "enso"] + ) & set(c["sets"]): + define_or_guess2( + c, + "reference_data_path_ts", + f"{reference_data_path}/atm/{c['grid']}/cmip_ts/monthly", + ParameterGuessType.PATH_GUESS, + ) + else: + raise ValueError(f"Invalid run_type={c['run_type']}") + print(prefix) + c["prefix"] = prefix + + +def add_ts_dependencies( + c: Dict[str, Any], dependencies: List[str], script_dir: str, yr: int +): + start_yr = yr + end_yr = yr + c["ts_num_years"] - 1 + depend_on_ts: Set[str] = set( + ["mean_climate", "variability_mode_atm", "variability_mode_cpl", "enso"] + ) + if depend_on_ts & set(c["sets"]): + add_dependencies( + dependencies, + script_dir, + "ts", + "atm_monthly_180x360_aave", + start_yr, + end_yr, + c["ts_num_years"], + ) + + +def add_pcmdi_dependencies( + c: Dict[str, Any], dependencies: List[str], script_dir: str +) -> None: + check_parameter_defined(c, "run_type") + if c["run_type"] == "model_vs_obs": + status_suffix = f"_{c['year1']:04d}-{c['year2']:04d}" + elif c["run_type"] == "model_vs_model": + status_suffix = f"_{c['year1']:04d}-{c['year2']:04d}_vs_{c['ref_year1']:04d}-{c['ref_year2']:04d}" + if "mean_climate" in c["sets"]: + status_file = os.path.join( + script_dir, + f"pcmdi_diags_mean_climate_{c['run_type']}{status_suffix}.status", + ) + if os.path.exists(status_file): + dependencies.append(status_file) + if "variability_modes_cpl" in c["sets"]: + status_file = os.path.join( + script_dir, + f"pcmdi_diags_variability_modes_cpl_{c['run_type']}{status_suffix}.status", + ) + if os.path.exists(status_file): + dependencies.append(status_file) + if "variability_modes_atm" in c["sets"]: + status_file = os.path.join( + script_dir, + f"pcmdi_diags_variability_modes_atm_{c['run_type']}{status_suffix}.status", + ) + if os.path.exists(status_file): + dependencies.append(status_file) + if "enso" in c["sets"]: + status_file = os.path.join( + script_dir, f"pcmdi_diags_enso_{c['run_type']}{status_suffix}.status" + ) + if os.path.exists(status_file): + dependencies.append(status_file) diff --git a/zppy/templates/coupled_global.py b/zppy/templates/coupled_global.py new file mode 100644 index 00000000..9fc41401 --- /dev/null +++ b/zppy/templates/coupled_global.py @@ -0,0 +1,872 @@ +# Script to plot some global atmosphere and ocean time series +import glob +import math +import sys +import traceback +from typing import Any, Dict, List, Optional, Tuple + +import cftime +import matplotlib as mpl +import matplotlib.backends.backend_pdf +import matplotlib.pyplot as plt +import numpy as np +import xarray +from netCDF4 import Dataset +from readTS import TS + +mpl.use("Agg") + + +# ---additional function to get moc time series +def getmoc(dir_in): + files = sorted(glob.glob(dir_in + "mocTimeSeries*.nc")) + nfiles = len(files) + print(dir_in, nfiles, "moc files in total") + var = np.array([]) + time = np.array([]) + for i in range(nfiles): + # Open input file + fin = Dataset(files[i], "r") + time0 = fin["year"][:] + var0 = fin["mocAtlantic26"][:] + for iyear in range(int(time0[0]), int(time0[-1]) + 1): + if i > 0 and iyear <= time[-1]: + print( + "the amoc value for year", + iyear, + "has been included in the moc time series from another moc file", + files[i - 1], + time[-1], + "Skipping...", + ) + else: + imon = np.where(time0 == iyear)[0] + if len(imon) == 12: + var = np.append(var, np.mean(var0[imon])) + time = np.append(time, iyear) + else: + print("error in input file :", files[i]) + + return time, var + + +# ----------------------------------------------------------------------------- +# Function to add horizontal line showing average value over a specified period +def add_line(year, var, year1, year2, ax, format="%4.2f", lw=1, color="b"): + + i1 = (np.abs(year - year1)).argmin() + i2 = (np.abs(year - year2)).argmin() + + tmp = np.average(var[i1 : i2 + 1]) + ax.plot((year[i1], year[i2]), (tmp, tmp), lw=lw, color=color, label="average") + ax.text(ax.get_xlim()[1] + 1, tmp, format % tmp, va="center", color=color) + + return + + +# ----------------------------------------------------------------------------- +# Function to add line showing linear trend over a specified period +def add_trend( + year, + var, + year1, + year2, + ax, + format="%4.2f", + lw=1, + color="b", + verbose=False, + ohc=False, + vol=False, +): + + i1 = (np.abs(year - year1)).argmin() + i2 = (np.abs(year - year2)).argmin() + x = year[i1 : i2 + 1] + y = var[i1 : i2 + 1] + + fit = np.polyfit(x, y, 1) + if verbose: + print(fit) + fit_fn = np.poly1d(fit) + ax.plot(x, fit_fn(x), lw=lw, ls="--", c=color, label="trend") + if ohc: + # Earth radius 6371229. from MPAS-O output files + heat_uptake = fit[0] / (4.0 * math.pi * (6371229.0) ** 2 * 365.0 * 86400.0) + ax.text( + ax.get_xlim()[1] + 1, + fit_fn(x[-1]), + "%+4.2f W m$^{-2}$" % (heat_uptake), + color=color, + ) + if vol: + # Earth radius 6371229. from MPAS-O output files + # sea_lvl = fit[0] / ( 4.0*math.pi*(6371229.)**2*0.7) #for oceanic portion of the Earth surface + ax.text( + ax.get_xlim()[1] + 1, + fit_fn(x[-1]), + "%+5.4f mm yr$^{-1}$" % (fit[0]), + color=color, + ) + + return + + +# ----------------------------------------------------------------------------- +# Function to get ylim +def get_ylim(standard_range, extreme_values): + if len(extreme_values) > 0: + has_extreme_values = True + extreme_min = np.amin(extreme_values) + extreme_max = np.amax(extreme_values) + else: + has_extreme_values = False + extreme_min = None + extreme_max = None + if len(standard_range) == 2: + has_standard_range = True + standard_min = standard_range[0] + standard_max = standard_range[1] + else: + has_standard_range = False + standard_min = None + standard_max = None + if has_extreme_values and has_standard_range: + # Use at least the standard range, + # perhaps a wider window to include extremes + if standard_min <= extreme_min: + ylim_min = standard_min + else: + ylim_min = extreme_min + if standard_max >= extreme_max: + ylim_max = standard_max + else: + ylim_max = extreme_max + elif has_extreme_values and not has_standard_range: + ylim_min = extreme_min + ylim_max = extreme_max + elif has_standard_range and not has_extreme_values: + ylim_min = standard_min + ylim_max = standard_max + else: + raise ValueError("Not enough range information supplied") + return [ylim_min, ylim_max] + + +# ----------------------------------------------------------------------------- +# Plotting functions + + +# 1 +def plot_net_toa_flux_restom(ax, xlim, exps, rgn): + print("Plot 1: plot_net_toa_flux_restom") + param_dict = { + "2nd_var": False, + "axhline_y": 0, + "check_exp_ocean": False, + "check_exp_vol": False, + "check_exp_year": True, + "default_ylim": [-1.5, 1.5], + "do_add_line": True, + "do_add_trend": True, + "format": "%4.2f", + "glb_only": False, + "lw": 1.0, + "ohc": False, + "set_axhline": True, + "set_legend": True, + "shorten_year": False, + "title": "Net TOA flux (restom)", + "use_getmoc": False, + "var": lambda exp: np.array(exp["annual"]["RESTOM"][0]), + "verbose": False, + "vol": False, + "ylabel": "W m-2", + } + plot(ax, xlim, exps, param_dict, rgn) + + +# 2 +def plot_global_surface_air_temperature(ax, xlim, exps, rgn): + print("Plot 2: plot_global_surface_air_temperature") + if rgn == "glb": + region_title = "Global" + elif rgn == "n": + region_title = "Northern Hemisphere" + elif rgn == "s": + region_title = "Southern Hemisphere" + else: + raise RuntimeError(f"Invalid rgn={rgn}") + param_dict = { + "2nd_var": False, + "axhline_y": None, + "check_exp_ocean": False, + "check_exp_vol": False, + "check_exp_year": True, + "default_ylim": [13, 15.5], + "do_add_line": True, + "do_add_trend": True, + "format": "%4.2f", + "glb_only": False, + "lw": 1.0, + "ohc": False, + "set_axhline": False, + "set_legend": True, + "shorten_year": False, + "title": f"{region_title} surface air temperature", + "use_getmoc": False, + "var": lambda exp: np.array(exp["annual"]["TREFHT"][0]) - 273.15, + "verbose": False, + "vol": False, + "ylabel": "degC", + } + plot(ax, xlim, exps, param_dict, rgn) + + +# 3 +def plot_toa_radiation(ax, xlim, exps, rgn): + print("Plot 3: plot_toa_radiation") + param_dict = { + "2nd_var": True, + "axhline_y": None, + "check_exp_ocean": False, + "check_exp_vol": False, + "check_exp_year": False, + "default_ylim": [235, 245], + "do_add_line": False, + "do_add_trend": False, + "format": None, + "glb_only": False, + "lw": 1.0, + "ohc": False, + "set_axhline": False, + "set_legend": False, + "shorten_year": False, + "title": "TOA radiation: SW (solid), LW (dashed)", + "use_getmoc": False, + "var": lambda exp: np.array(exp["annual"]["FSNTOA"][0]), + "verbose": None, + "vol": None, + "ylabel": "W m-2", + } + plot(ax, xlim, exps, param_dict, rgn) + + +# 4 +def plot_net_atm_energy_imbalance(ax, xlim, exps, rgn): + print("Plot 4: plot_net_atm_energy_imbalance") + param_dict = { + "2nd_var": False, + "axhline_y": None, + "check_exp_ocean": False, + "check_exp_vol": False, + "check_exp_year": True, + "default_ylim": [-0.3, 0.3], + "do_add_line": True, + "do_add_trend": False, + "format": "%4.2f", + "glb_only": False, + "lw": 1.0, + "ohc": False, + "set_axhline": False, + "set_legend": True, + "shorten_year": False, + "title": "Net atm energy imbalance (restom-ressurf)", + "use_getmoc": False, + "var": lambda exp: np.array(exp["annual"]["RESTOM"][0]) + - np.array(exp["annual"]["RESSURF"][0]), + "verbose": False, + "vol": False, + "ylabel": "W m-2", + } + plot(ax, xlim, exps, param_dict, rgn) + + +# 5 +def plot_change_ohc(ax, xlim, exps, rgn): + print("Plot 5: plot_change_ohc") + param_dict = { + "2nd_var": False, + "axhline_y": 0, + "check_exp_ocean": True, + "check_exp_vol": False, + "check_exp_year": False, + "default_ylim": [-0.3e24, 0.9e24], + "do_add_line": False, + "do_add_trend": True, + "format": "%4.2f", + "glb_only": True, + "lw": 1.5, + "ohc": True, + "set_axhline": True, + "set_legend": True, + "shorten_year": True, + "title": "Change in ocean heat content", + "use_getmoc": False, + "var": lambda exp: np.array(exp["annual"]["ohc"]), + "verbose": False, + "vol": False, + "ylabel": "J", + } + plot(ax, xlim, exps, param_dict, rgn) + + +# 6 +def plot_max_moc(ax, xlim, exps, rgn): + print("Plot 6: plot_max_moc") + param_dict = { + "2nd_var": False, + "axhline_y": 10, + "check_exp_ocean": False, + "check_exp_vol": False, + "check_exp_year": False, + "default_ylim": [4, 22], + "do_add_line": False, + "do_add_trend": True, + "format": "%4.2f", + "glb_only": True, + "lw": 1.5, + "ohc": False, + "set_axhline": True, + "set_legend": True, + "shorten_year": False, + "title": "Max MOC Atlantic streamfunction at 26.5N", + "use_getmoc": True, + "var": None, + "verbose": True, + "vol": None, + "ylabel": "Sv", + } + plot(ax, xlim, exps, param_dict, rgn) + + +# 7 +def plot_change_sea_level(ax, xlim, exps, rgn): + print("Plot 7: plot_change_sea_level") + param_dict = { + "2nd_var": False, + "axhline_y": None, + "check_exp_ocean": False, + "check_exp_vol": True, + "check_exp_year": True, + "default_ylim": [4, 22], + "do_add_line": False, + "do_add_trend": True, + "format": "%5.3f", + "glb_only": True, + "lw": 1.5, + "ohc": False, + "set_axhline": False, + "set_legend": True, + "shorten_year": True, + "title": "Change in sea level", + "use_getmoc": False, + "var": lambda exp: ( + 1e3 + * np.array(exp["annual"]["volume"]) + / (4.0 * math.pi * (6371229.0) ** 2 * 0.7) + ), + "verbose": True, + "vol": True, + "ylabel": "mm", + } + plot(ax, xlim, exps, param_dict, rgn) + + +# 8 +def plot_net_atm_water_imbalance(ax, xlim, exps, rgn): + print("Plot 8: plot_net_atm_water_imbalance") + param_dict = { + "2nd_var": False, + "axhline_y": None, + "check_exp_ocean": False, + "check_exp_vol": False, + "check_exp_year": False, + "default_ylim": [-1, 1], + "do_add_line": True, + "do_add_trend": False, + "format": "%5.4f", + "glb_only": False, + "lw": 1.0, + "ohc": False, + "set_axhline": False, + "set_legend": True, + "shorten_year": False, + "title": "Net atm water imbalance (evap-prec)", + "use_getmoc": False, + "var": lambda exp: ( + 365 + * 86400 + * ( + np.array(exp["annual"]["QFLX"][0]) + - 1e3 + * ( + np.array(exp["annual"]["PRECC"][0]) + + np.array(exp["annual"]["PRECL"][0]) + ) + ) + ), + "verbose": False, + "vol": False, + "ylabel": "mm yr-1", + } + plot(ax, xlim, exps, param_dict, rgn) + + +# Generic plot function +def plot_generic(ax, xlim, exps, var_name, rgn): + print(f"plot_generic for {var_name}") + param_dict = { + "2nd_var": False, + "axhline_y": 0, + "check_exp_ocean": False, + "check_exp_vol": False, + "check_exp_year": True, + "default_ylim": [], + "do_add_line": True, + "do_add_trend": True, + "format": "%4.2f", + "glb_only": False, + "lw": 1.0, + "ohc": False, + "set_axhline": False, + "set_legend": True, + "shorten_year": False, + "title": var_name, + "use_getmoc": False, + "var": lambda exp: np.array(exp["annual"][var_name][0]), + "verbose": False, + "vol": False, + "ylabel": lambda exp: np.array(exp["annual"][var_name][1]), + } + plot(ax, xlim, exps, param_dict, rgn) + + +# FIXME: C901 'plot' is too complex (19) +def plot(ax, xlim, exps, param_dict, rgn): # noqa: C901 + if param_dict["glb_only"] and (rgn != "glb"): + return + ax.set_xlim(xlim) + extreme_values = [] + for exp in exps: + # Relevant to "Plot 5: plot_change_ohc" + if param_dict["check_exp_ocean"] and (exp["ocean"] is None): + continue + # Relevant to "Plot 7: plot_change_sea_level" + # This must be checked before plot 6, + # otherwise, `param_dict["var"]` will be run, + # but `exp["annual"]["volume"]` won't exist. + if param_dict["check_exp_vol"] and (exp["vol"] is None): + continue + # Relevant to "Plot 6: plot_max_moc" + if param_dict["use_getmoc"]: + if exp["moc"]: + [year, var] = getmoc(exp["moc"]) + else: + continue + else: + year = np.array(exp["annual"]["year"]) + exp["yoffset"] + var = param_dict["var"](exp) + extreme_values.append(np.amax(var)) + extreme_values.append(np.amin(var)) + if param_dict["shorten_year"]: + year = year[: len(var)] + try: + ax.plot( + year, + var, + lw=param_dict["lw"], + marker=None, + c=exp["color"], + label=exp["name"], + ) + except Exception: + raise RuntimeError(f"{param_dict['title']} could not be plotted.") + if param_dict["2nd_var"]: + # Specifically for plot_toa_radiation + # TODO: if more plots require a 2nd variable, we can change `var` to be a list, + # but that will be a more significant refactoring. + var = np.array(exp["annual"]["FLUT"][0]) + ax.plot(year, var, lw=1.0, marker=None, ls=":", c=exp["color"]) + continue + if param_dict["check_exp_year"] and exp["yr"] is None: + continue + elif param_dict["do_add_line"] or param_dict["do_add_trend"]: + for yrs in exp["yr"]: + if param_dict["do_add_line"]: + add_line( + year, + var, + yrs[0], + yrs[1], + format=param_dict["format"], + ax=ax, + lw=2 * param_dict["lw"], + color=exp["color"], + ) + if param_dict["do_add_trend"]: + add_trend( + year, + var, + yrs[0], + yrs[1], + format=param_dict["format"], + ax=ax, + lw=2 * param_dict["lw"], + color=exp["color"], + ohc=param_dict["ohc"], + verbose=param_dict["verbose"], + vol=param_dict["vol"], + ) + ylim = get_ylim(param_dict["default_ylim"], extreme_values) + ax.set_ylim(ylim) + if param_dict["set_axhline"]: + ax.axhline(y=param_dict["axhline_y"], lw=1, c="0.5") + ax.set_title(param_dict["title"]) + ax.set_xlabel("Year") + units = param_dict["ylabel"] + c = callable(units) + if c: + units = units(exps[0]) + ax.set_ylabel(units) + if param_dict["set_legend"]: + ax.legend(loc="best") + + +PLOT_DICT = { + "net_toa_flux_restom": plot_net_toa_flux_restom, + "global_surface_air_temperature": plot_global_surface_air_temperature, + "toa_radiation": plot_toa_radiation, + "net_atm_energy_imbalance": plot_net_atm_energy_imbalance, + "change_ohc": plot_change_ohc, # only glb + "max_moc": plot_max_moc, # only glb + "change_sea_level": plot_change_sea_level, # only glb + "net_atm_water_imbalance": plot_net_atm_water_imbalance, +} + + +def param_get_list(param_value): + if param_value == "None": + return [] + else: + return param_value.split(",") + + +def set_var( + exp: Dict[str, Any], + exp_key: str, + var_list: List[str], + valid_vars: List[str], + invalid_vars: List[str], + rgn: str, +) -> None: + if exp[exp_key] is not None: + ts = TS(exp[exp_key]) + for var in var_list: + try: + v: xarray.core.dataarray.DataArray + units: Optional[str] + v, units = ts.globalAnnual(var) + valid_vars.append(str(var)) + except Exception as e: + print(e) + print(f"globalAnnual failed. Invalid var = {var}") + invalid_vars.append(str(var)) + continue + if v.sizes["rgn"] > 1: + # number of years x 3 regions = v.shape + # 3 regions = global, northern hemisphere, southern hemisphere + # We get here if we used the updated `ts` task + # (using `rgn_avg` rather than `glb_avg`). + if rgn == "glb": + n = 0 + elif rgn == "n": + n = 1 + elif rgn == "s": + n = 2 + else: + raise RuntimeError(f"Invalid rgn={rgn}") + v = v.isel(rgn=n) # Just use nth region + elif rgn != "glb": + # v only has one dimension -- glb. + # Therefore it is not possible to get n or s plots. + raise RuntimeError( + f"var={var} only has global data. Cannot process rgn={rgn}" + ) + exp["annual"][var] = (v, units) + if "year" not in exp["annual"]: + years: np.ndarray[cftime.DatetimeNoLeap] = v.coords["time"].values + exp["annual"]["year"] = [x.year for x in years] + del ts + + +def make_plot_pdfs( + figstr, rgn, component, xlim, exps, plot_list, valid_plots, invalid_plots +): + num_plots = len(plot_list) + if num_plots == 0: + return + nrows = 4 + ncols = 2 + plots_per_page = nrows * ncols + num_pages = math.ceil(num_plots / plots_per_page) + + counter = 0 + # https://stackoverflow.com/questions/58738992/save-multiple-figures-with-subplots-into-a-pdf-with-multiple-pages + pdf = matplotlib.backends.backend_pdf.PdfPages(f"{figstr}_{rgn}_{component}.pdf") + for page in range(num_pages): + fig = plt.figure(1, figsize=[13.5, 16.5]) + fig.suptitle(f"{figstr}_{rgn}_{component}") + for j in range(plots_per_page): + # The final page doesn't need to be filled out with plots. + if counter >= num_plots: + break + ax = plt.subplot(nrows, ncols, j + 1) + if component == "original": + try: + plot_function = PLOT_DICT[plot_list[counter]] + except KeyError: + raise KeyError(f"Invalid plot name: {plot_list[counter]}") + try: + plot_function(ax, xlim, exps, rgn) + valid_plots.append(plot_list[counter]) + except Exception: + traceback.print_exc() + plot_name = plot_list[counter] + required_vars = [] + if plot_name == "net_toa_flux_restom": + required_vars = ["RESTOM"] + elif plot_name == "net_atm_energy_imbalance": + required_vars = ["RESTOM", "RESSURF"] + elif plot_name == "global_surface_air_temperature": + required_vars = ["TREFHT"] + elif plot_name == "toa_radiation": + required_vars = ["FSNTOA", "FLUT"] + elif plot_name == "net_atm_water_imbalance": + required_vars = ["PRECC", "PRECL", "QFLX"] + print( + f"Failed plot_function for {plot_name}. Check that {required_vars} are available." + ) + invalid_plots.append(plot_name) + counter += 1 + else: + try: + plot_name = plot_list[counter] + plot_generic(ax, xlim, exps, plot_name, rgn) + valid_plots.append(plot_name) + except Exception: + traceback.print_exc() + print(f"plot_generic failed. Invalid plot={plot_name}") + invalid_plots.append(plot_name) + counter += 1 + + fig.tight_layout() + pdf.savefig(1) + if num_pages > 1: + fig.savefig(f"{figstr}_{rgn}_{component}_{page}.png", dpi=150) + else: + fig.savefig(f"{figstr}_{rgn}_{component}.png", dpi=150) + plt.clf() + pdf.close() + + +# ----------------------------------------------------------------------------- +# FIXME: C901 'run' is too complex (19) +def run(parameters, rgn): # noqa: C901 + # These are the "Tableau 20" colors as RGB. + t20: List[Tuple[float, float, float]] = [ + (31, 119, 180), + (174, 199, 232), + (255, 127, 14), + (255, 187, 120), + (44, 160, 44), + (152, 223, 138), + (214, 39, 40), + (255, 152, 150), + (148, 103, 189), + (197, 176, 213), + (140, 86, 75), + (196, 156, 148), + (227, 119, 194), + (247, 182, 210), + (127, 127, 127), + (199, 199, 199), + (188, 189, 34), + (219, 219, 141), + (23, 190, 207), + (158, 218, 229), + ] + # Scale the RGB values to the [0, 1] range, which is the format matplotlib accepts. + for i in range(len(t20)): + r, g, b = t20[i] + t20[i] = (r / 255.0, g / 255.0, b / 255.0) + + # "Tableau 10" uses every other color + t10 = [] + for i in range(0, len(t20), 2): + t10.append(t20[i]) + + # ----------------------------------------------------------------------------- + # --- Atmos data --- + + # Experiments + case_dir = parameters[1] + experiment_name = parameters[2] + figstr = parameters[3] + year1 = int(parameters[4]) + year2 = int(parameters[5]) + color = parameters[6] + ts_num_years = parameters[7] + plots_original = param_get_list(parameters[8]) + if parameters[9].lower() == "false": + atmosphere_only = False + else: + atmosphere_only = True + plots_atm = param_get_list(parameters[10]) + plots_ice = param_get_list(parameters[11]) + plots_lnd = param_get_list(parameters[12]) + plots_ocn = param_get_list(parameters[13]) + vars_original = [] + if "net_toa_flux_restom" or "net_atm_energy_imbalance" in plots_original: + vars_original.append("RESTOM") + if "net_atm_energy_imbalance" in plots_original: + vars_original.append("RESSURF") + if "global_surface_air_temperature" in plots_original: + vars_original.append("TREFHT") + if "toa_radiation" in plots_original: + vars_original.append("FSNTOA") + vars_original.append("FLUT") + if "net_atm_water_imbalance" in plots_original: + vars_original.append("PRECC") + vars_original.append("PRECL") + vars_original.append("QFLX") + use_atmos = plots_atm or plots_original + has_original_ocn_plots = ( + ("change_ohc" in plots_original) + or ("max_moc" in plots_original) + or ("change_sea_level" in plots_original) + ) + use_ocn = plots_ocn or (not atmosphere_only and has_original_ocn_plots) + exps: List[Dict[str, Any]] = [ + { + "atmos": ( + f"{case_dir}/post/atm/glb/ts/monthly/{ts_num_years}yr/" + if use_atmos + else None + ), + "ice": ( + f"{case_dir}/post/ice/glb/ts/monthly/{ts_num_years}yr/" + if plots_ice + else None + ), + "land": ( + f"{case_dir}/post/lnd/glb/ts/monthly/{ts_num_years}yr/" + if plots_lnd + else None + ), + "ocean": ( + f"{case_dir}/post/ocn/glb/ts/monthly/{ts_num_years}yr/" + if use_ocn + else None + ), + "moc": ( + f"{case_dir}/post/ocn/glb/ts/monthly/{ts_num_years}yr/" + if use_ocn + else None + ), + "vol": ( + f"{case_dir}/post/ocn/glb/ts/monthly/{ts_num_years}yr/" + if use_ocn + else None + ), + "name": experiment_name, + "yoffset": 0.0, + "yr": ([year1, year2],), + "color": f"{color}", + } + ] + + valid_vars: List[str] = [] + invalid_vars: List[str] = [] + + # Read data + exp: Dict[str, Any] + for exp in exps: + exp["annual"] = {} + + # Use vars_original rather than plots_original, + # since the plots have different names than the variables + set_var(exp, "atmos", vars_original, valid_vars, invalid_vars, rgn) + set_var(exp, "atmos", plots_atm, valid_vars, invalid_vars, rgn) + set_var(exp, "ice", plots_ice, valid_vars, invalid_vars, rgn) + set_var(exp, "land", plots_lnd, valid_vars, invalid_vars, rgn) + set_var(exp, "ocean", plots_ocn, valid_vars, invalid_vars, rgn) + + # Optionally read ohc + if exp["ocean"] is not None: + ts = TS(exp["ocean"]) + exp["annual"]["ohc"], _ = ts.globalAnnual("ohc") + # annomalies with respect to first year + exp["annual"]["ohc"][:] = exp["annual"]["ohc"][:] - exp["annual"]["ohc"][0] + + if exp["vol"] is not None: + ts = TS(exp["vol"]) + exp["annual"]["volume"], _ = ts.globalAnnual("volume") + # annomalies with respect to first year + exp["annual"]["volume"][:] = ( + exp["annual"]["volume"][:] - exp["annual"]["volume"][0] + ) + + print( + f"{rgn} region globalAnnual was computed successfully for these variables: {valid_vars}" + ) + print( + f"{rgn} region globalAnnual could not be computed for these variables: {invalid_vars}" + ) + + # ----------------------------------------------------------------------------- + # --- Generate plots --- + + xlim = [float(year1), float(year2)] + + valid_plots: List[str] = [] + invalid_plots: List[str] = [] + + make_plot_pdfs( + figstr, rgn, "original", xlim, exps, plots_original, valid_plots, invalid_plots + ) + make_plot_pdfs( + figstr, rgn, "atm", xlim, exps, plots_atm, valid_plots, invalid_plots + ) + make_plot_pdfs( + figstr, rgn, "ice", xlim, exps, plots_ice, valid_plots, invalid_plots + ) + make_plot_pdfs( + figstr, rgn, "lnd", xlim, exps, plots_lnd, valid_plots, invalid_plots + ) + make_plot_pdfs( + figstr, rgn, "ocn", xlim, exps, plots_ocn, valid_plots, invalid_plots + ) + + print(f"These {rgn} region plots generated successfully: {valid_plots}") + print( + f"These {rgn} region plots could not be generated successfully: {invalid_plots}" + ) + + +def run_by_region(parameters): + regions = parameters[14].split(",") + for rgn in regions: + if rgn.lower() in ["glb", "global"]: + rgn = "glb" + elif rgn.lower() in ["n", "north", "northern"]: + rgn = "n" + elif rgn.lower() in ["s", "south", "southern"]: + rgn = "s" + else: + raise RuntimeError(f"Invalid rgn={rgn}") + run(parameters, rgn) + + +if __name__ == "__main__": + run_by_region(sys.argv) diff --git a/zppy/templates/global_time_series.bash b/zppy/templates/global_time_series.bash old mode 100644 new mode 100755 diff --git a/zppy/templates/inclusions/e3sm_to_cmip/default_metadata.json b/zppy/templates/inclusions/e3sm_to_cmip/default_metadata.json old mode 100644 new mode 100755 diff --git a/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc b/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc new file mode 100755 index 0000000000000000000000000000000000000000..2b71cc296ff2f207b80961668af618cecf911c7e GIT binary patch literal 1260 zcma)6QE${Z5cZaBizpJDctk=Lssx-iah$ZkwN@t*Mpq+L{V6^u3af0v<-VB`cL(xUBVs*tzuH z0qVP%tWb-(pW}#cM?4;cJG<#GHJ^x5gIn+HJTxkagxyU;mFvnHJW7_m_Vd-UM4Q%iQwCt(YJ@xiP%r~ zPM;HIvIun;B?%9r_7}kynhac#OWu}t>SQ@PISbLmpRm%|XDah&JlC#bGAk6*rd;@3 zddZ|O7@3f+Tug2%9NsF!8oxMtWT6?B0vp4WsW_`S`~NN9^vEm<;fmNHP@Mm` zT~_F_FKX$uoXl0lYUhPhg)9Z8ln>6zO3odoN2m27PmS{F&WIjCnI#mmqILUcpc~;< zIG{K0+d`Fcs^}d$^S&H1W-=RA$I#HeM^kv;cVw#lY%vKlTc8orEsV1U^y8dC`Nt3E z$b(kV{hM^&0qIAi^WZ$``~(x{-=~E9^N^4~?-H_rsUklRa!3go{z1r%%Y {{ prefix }}.status + +# Basic definitions +case="{{ case }}" +www="{{ www }}" +run_type="{{ run_type }}" +results_dir="{{ run_type }}" +ref_name={{ ref_name }} +{% if "synthetic_plots" not in subsection %} + +# Input variables +y1={{ year1 }} +y2={{ year2 }} +ref_y1={{ ref_year1 }} +ref_y2={{ ref_year2 }} +ref_start_yr={{ ref_start_yr }} +ref_final_yr={{ ref_final_yr }} + +# Formatted versions +Y1="$(printf "%04d" ${y1})" +Y2="$(printf "%04d" ${y2})" +ref_Y1="$(printf "%04d" ${ref_y1})" +ref_Y2="$(printf "%04d" ${ref_y2})" + +# Refine reference range +if [[ ${ref_y1} -lt ${ref_start_yr} ]]; then + ref_y1=${ref_start_yr} + ref_Y1="$(printf "%04d" ${ref_y1})" +fi + +num_years=$((y2 - y1 + 1)) +ref_end_yr=$((ref_y1 + num_years - 1)) + +if [[ ${ref_y2} -gt ${ref_end_yr} ]]; then + ref_y2=${ref_end_yr} + ref_Y2="$(printf "%04d" ${ref_y2})" +fi + +if [[ ${ref_y2} -gt ${ref_final_yr} ]]; then + ref_y2=${ref_final_yr} + ref_Y2="$(printf "%04d" ${ref_y2})" +fi + +{% endif %} + +# Top-level directory +web_dir=${www}/${case}/pcmdi_diags + +################################################## +# info to construct pcmdi-preferred data convention +################################################## +model_name='{{ model_name }}' +tableID='{{ model_tableID }}' +{% if run_type == "model_vs_obs" %} +model_name_ref='obs.historical.%(model).00' +tableID_ref=${tableID} +{% elif run_type == "model_vs_model" %} +model_name_ref='{{ model_name_ref }}' +tableID_ref='{{ model_tableID_ref }}' +{%- endif %} +case_id=v$(date '+%Y%m%d') + +# Create temporary workdir +workdir=`mktemp -d tmp.${id}.XXXX` +cd ${workdir} + +# utility file for pcmdi-zppy workflow +cp -r '{{pcmdi_external_prefix}}/{{pcmdi_zppy_util}}' . + +# files for definition of regions for regional mean +cp -r '{{pcmdi_external_prefix}}/{{regions_specs}}' . + +# file for aliases of observation datasets +cp -r '{{pcmdi_external_prefix}}/{{reference_alias}}' . + +# file for list of variables for synthetic_metrics metric plots +cp -r '{{pcmdi_external_prefix}}/{{synthetic_metrics}}' . + +# utility file for pcmdi-zppy viewer +cp -r '{{pcmdi_external_prefix}}/{{pcmdi_viewer_util}}' . + +{% if "mean_climate" in subsection %} +create_links_acyc_climo() { + local ts_dir_source="$1" + local ts_dir_destination="$2" + local begin_year="$3" + local end_year="$4" + local name_key="$5" + local error_num="$6" + + mkdir -p "${ts_dir_destination}" + cd "${ts_dir_destination}" || exit + + local variables="{{ cmip_vars }}" + local script_dir="{{ scriptDir }}" + local prefix="{{ prefix }}" + local ts_step="{{ ts_num_years }}" + local dofm=(15.5 45 74.5 105 125.5 166 196.5 227.5 258 288.5 319 349.5) + + for v in ${variables//,/ }; do + > "${v}_files.txt" # Start fresh + + shopt -s nullglob + for year in $(seq "${begin_year}" "${ts_step}" "${end_year}"); do + local YYYY + YYYY=$(printf "%04d" "${year}") + for file in ${ts_dir_source}/${v}_*_${YYYY}*.nc; do + [[ -f "${file}" ]] && echo "${file}" >> "${v}_files.txt" + done + done + shopt -u nullglob + + # Derive monthly climatology files + for month in $(seq 1 12); do + local MM + MM=$(printf "%02d" "${month}") + ncra -O -h -F -d time,"${month}",,12 $(< "${v}_files.txt") "${v}_clm_${MM}.nc" + done + + # Combine to form full annual cycle file + local combined_name="${name_key}.${v}.${begin_year}01-${end_year}12.AC.${case_id}.nc" + ncrcat -O -d time,0, "${v}_clm_"*.nc "${combined_name}" + + # Adjust time metadata for PCMDI diagnostics + local cmdfix1='time[time]={15.5, 45, 74.5, 105, 125.5, 166, 196.5, 227.5, 258, 288.5,319, 349.5}' + local cmdfix2='time_bnds[time,bnds]={0,31,31,59,59,90,90,120,120,151,151,181,181,212,212,243,243,273,273,304,304,334,334,365.}' + local cmdfix3='time@units="days since 1850-01-01 00:00:00"' + local cmdfix4='time@calendar="noleap"' + local cmdfix5='time@bounds="time_bnds"' + ncap2 -O -h -s "${cmdfix1};${cmdfix2};${cmdfix3};${cmdfix4};${cmdfix5}" "${combined_name}" "${combined_name}" + + rm -vf "${v}_clm_"*.nc + + if [[ $? -ne 0 ]]; then + cd "${script_dir}" || exit + echo "ERROR (${error_num})" > "${prefix}.status" + exit "${error_num}" + fi + done + + cd .. +} + +{% if run_type == "model_vs_obs" %} +create_links_acyc_climo_obs() { + local ts_dir_source="$1" + local ts_dir_destination="$2" + local begin_year="$3" + local end_year="$4" + local error_num="$5" + + local script_dir="{{ scriptDir }}" + local prefix="{{ prefix }}" + local dofm=(15.5 45 74.5 105 125.5 166 196.5 227.5 258 288.5 319 349.5) + + mkdir -p "${ts_dir_destination}" + cd "${ts_dir_destination}" || exit + + for file in ${ts_dir_source}/*.nc; do + local fname + local YYYYS YYYYE + local PREFIX + local ttag tmp_file MM combined_name + + fname=$(basename "${file}") + + # Match two date patterns (YYYYMM or YYYYMMDD) separated by _ or - + if [[ ${fname} =~ ([0-9]{6,8})[_-]([0-9]{6,8}) ]]; then + YYYYS="${BASH_REMATCH[1]}" + YYYYE="${BASH_REMATCH[2]}" + else + echo "Warning: Could not extract dates from ${fname}" + continue + fi + + # Clip to specified year range + if [[ ${YYYYS} -lt ${begin_year} ]]; then YYYYS=${begin_year}; fi + if [[ ${YYYYE} -gt ${end_year} ]]; then YYYYE=${end_year}; fi + + # Extract prefix before the date range (removes from .${YYYYS} or -${YYYYS}) + PREFIX="${fname%%[._-]${YYYYS}*}" + + ttag="$(printf "%04d" "${YYYYS}")01-$(printf "%04d" "${YYYYE}")12" + tmp_file="tmp_combine_${ttag}.nc" + + ncrcat -O -d time,"${YYYYS}-01-01","${YYYYE}-12-31" "${file}" "${tmp_file}" + + # Derive monthly climatology + for month in $(seq 1 12); do + MM=$(printf "%02d" ${month}) + ncra -O -h -F -d time,"${month}",,12 "${tmp_file}" "tmp_clm_${MM}.nc" + done + + combined_name="${PREFIX}.${ttag}.AC.${case_id}.nc" + ncrcat -O tmp_clm_*.nc "${combined_name}" + + # Adjust time metadata + local cmdfix1='time[time]={15.5, 45, 74.5, 105, 125.5, 166, 196.5, 227.5, 258, 288.5,319, 349.5}' + local cmdfix2='time@units="days since 1850-01-01 00:00:00"' + local cmdfix3='time@calendar="noleap"' + ncap2 -O -h -s "${cmdfix1};${cmdfix2};${cmdfix3}" "${combined_name}" "${combined_name}" + + local cmdfix4='defdim("bnds",2)' + local cmdfix5='time_bnds=make_bounds(time,$bnds,"time_bnds")' + local cmdfix6='time_bnds@units=time@units' + local cmdfix7='time_bnds@calendar=time@calendar' + ncap2 -O -h -s "${cmdfix4};${cmdfix5};${cmdfix6};${cmdfix7}" "${combined_name}" "${combined_name}" + + rm -vf tmp_*.nc + + if [[ $? -ne 0 ]]; then + cd "${script_dir}" || exit + echo "ERROR (${error_num})" > "${prefix}.status" + exit "${error_num}" + fi + done + + cd .. +} +{% endif %} +{% endif %} + +{% if "variability_modes_cpl" in subsection + or "variability_modes_atm" in subsection + or "enso" in subsection %} +create_links_ts() { + local ts_dir_source="$1" + local ts_dir_destination="$2" + local begin_year="$3" + local end_year="$4" + local subname="$5" + local error_num="$6" + + local script_dir="{{ scriptDir }}" + local prefix="{{ prefix }}" + local vars="{{ vars }}" + local ts_step="{{ ts_num_years }}" + + mkdir -p "${ts_dir_destination}" + cd "${ts_dir_destination}" || exit + + local v file YYYY combined_name + + # Convert comma-separated list to array + IFS=',' read -ra var_array <<< "${vars}" + for v in "${var_array[@]}"; do + > "${v}_files.txt" # Reset file list + + shopt -s nullglob + for year in $(seq "${begin_year}" "${ts_step}" "${end_year}"); do + YYYY=$(printf "%04d" "${year}") + for file in ${ts_dir_source}/${v}_*_${YYYY}*.nc; do + [[ -f "${file}" ]] && echo "${file}" >> "${v}_files.txt" + done + done + shopt -u nullglob + + combined_name="${subname}.${v}.${begin_year}01-${end_year}12.nc" + if [[ -s "${v}_files.txt" ]]; then + ncrcat -O -v "${v}" -d time,"${begin_year}-01-01","${end_year}-12-31" $(< "${v}_files.txt") "${combined_name}" + + # Add calendar attribute if missing + if ! ncks -m "${combined_name}" | grep -q "calendar"; then + echo "Adding missing calendar attribute to time..." + ncatted -a calendar,time,o,c,"standard" "${combined_name}" + fi + + # Add time bounds + local cmdfix1='defdim("bnds",2)' + local cmdfix2='time_bnds=make_bounds(time,$bnds,"time_bnds")' + local cmdfix3='time_bnds@units=time@units' + local cmdfix4='time_bnds@calendar=time@calendar' + ncap2 -O -h -s "${cmdfix1};${cmdfix2};${cmdfix3};${cmdfix4}" "${combined_name}" "${combined_name}" + + if [[ $? -ne 0 ]]; then + cd "${script_dir}" || exit + echo "ERROR (${error_num})" > "${prefix}.status" + exit "${error_num}" + fi + else + echo "Warning: No input files found for variable ${v}. Skipping." + fi + done + + cd .. +} + +{% if run_type == "model_vs_obs" %} +create_links_ts_obs() { + local ts_dir_source="$1" + local ts_dir_destination="$2" + local begin_year="$3" + local end_year="$4" + local error_num="$5" + + local script_dir="{{ scriptDir }}" + local prefix="{{ prefix }}" + + mkdir -p "${ts_dir_destination}" + cd "${ts_dir_destination}" || exit + + local file fname PREFIX YYYYS YYYYE ttag combined_name + + for file in ${ts_dir_source}/*.nc; do + fname=$(basename "$file") + # Match two time patterns (YYYYMM or YYYYMMDD) separated by _ or - + if [[ ${fname} =~ ([0-9]{6,8})[_-]([0-9]{6,8}) ]]; then + YYYYS="${BASH_REMATCH[1]}" + YYYYE="${BASH_REMATCH[2]}" + else + echo "Warning: Could not extract dates from ${fname}" + continue + fi + + # Optional: clip years if needed + if [[ ${YYYYS} -lt ${begin_year} ]]; then + YYYYS="${begin_year}" + fi + + if [[ ${YYYYE} -gt ${end_year} ]]; then + YYYYE="${end_year}" + fi + + # Extract prefix (before the date range, ignoring separator) + PREFIX="${fname%%[._-]${YYYYS}*}" + + ttag="$(printf "%04d" "${YYYYS}")01-$(printf "%04d" "${YYYYE}")12" + combined_name="${PREFIX}.${ttag}.nc" + + # Extract subset of time series + ncrcat -O -d time,"${YYYYS}-01-01","${YYYYE}-12-31" "${file}" "${combined_name}" + + # Ensure time has calendar attribute + if ! ncks -m "${combined_name}" | grep -q "calendar"; then + echo "Adding missing calendar attribute to time..." + ncatted -a calendar,time,o,c,"standard" "${combined_name}" + fi + + # Add time bounds + local cmdfix1='defdim("bnds",2)' + local cmdfix2='time_bnds=make_bounds(time,$bnds,"time_bnds")' + local cmdfix3='time_bnds@units=time@units' + local cmdfix4='time_bnds@calendar=time@calendar' + ncap2 -O -h -s "${cmdfix1};${cmdfix2};${cmdfix3};${cmdfix4}" "${combined_name}" "${combined_name}" + + if [[ $? -ne 0 ]]; then + cd "${script_dir}" || exit + echo "ERROR (${error_num})" > "${prefix}.status" + exit "${error_num}" + fi + done + + cd .. +} +{% endif %} +{% endif %} + +######################## +# Prepare the model data +######################## +{% if "mean_climate" in subsection %} +# Define output directory for climatology files +climo_dir_primary="climo" +# Path to model's monthly climatology files +climo_dir_source="{{ output }}/post/atm/{{ grid }}/cmip_ts/monthly" +# Link and process primary model climo data +create_links_acyc_climo "${climo_dir_source}" "${climo_dir_primary}" "${Y1}" "${Y2}" "${model_name}.${tableID}" 1 +{% if run_type == "model_vs_model" %} +# Path to reference model's climatology files +climo_dir_source_ref="{{ reference_data_path_ts }}" +climo_dir_ref="climo_ref" +# Link and process reference model climo data +create_links_acyc_climo "${climo_dir_source_ref}" "${climo_dir_ref}" "${ref_Y1}" "${ref_Y2}" "${model_name_ref}.${tableID_ref}" 2 +{% endif %} +{% endif %} + +{% if "variability_modes_cpl" in subsection or "variability_modes_atm" in subsection or "enso" in subsection %} +# All diagnostics in this subsection use time series (ts) data +# Define output directory for primary model time series +ts_dir_primary="ts" +ts_dir_source="{{ output }}/post/atm/{{ grid }}/cmip_ts/monthly" +# Create local links and combine time series NetCDF files for the primary model +create_links_ts "${ts_dir_source}" "${ts_dir_primary}" "${Y1}" "${Y2}" "${model_name}.${tableID}" 3 +{% if run_type == "model_vs_model" %} +# Define time series path for reference model (adjust for different year spans) +ts_dir_source_ref="{{ reference_data_path_ts }}" +ts_dir_ref="ts_ref" +# Create local links and combine ts files for the reference model +create_links_ts "${ts_dir_source_ref}" "${ts_dir_ref}" "${ref_Y1}" "${ref_Y2}" "${model_name_ref}.${tableID_ref}" 4 +{% endif %} +{% endif %} + +{% if run_type == "model_vs_obs" and "synthetic_plots" not in subsection %} +########################################################################### +# Prepare the observation data +# Observation datasets vary by diagnostic, so we use an external +# Python utility to handle linking and remapping to standard names. +########################################################################### +obstmp_dir="obs_link" +mkdir -p "${obstmp_dir}" +echo "Linking observational data into ${obstmp_dir}..." + +cat > link_observation.py << EOF +import os +import re +import glob +import json + +try: + from pcmdi_zppy_util import ObservationLinker +except ImportError as e: + raise ImportError("Module 'pcmdi_zppy_util' not found. Make sure it's installed and accessible.") from e + +# Inputs populated via templating +MODEL_NAME = '${model_name_ref}.${tableID_ref}' +VARIABLES = '{{ vars }}'.split(",") +OBS_SETS = '{{ obs_sets }}'.split(",") +OBS_TS_DIR = '{{ obs_ts }}' +OBS_TMP_DIR = '${obstmp_dir}' +OBS_ALIAS_FILE = "reference_alias.json" + +# Mapping from observational variable names to CMIP-standard +ALT_OBS_MAP = { + "pr": "PRECT", + "sst": "ts", + "sfcWind": "si10", + "taux": "tauu", + "tauy": "tauv", + "rltcre": "toa_cre_lw_mon", + "rstcre": "toa_cre_sw_mon", + "rtmt": "toa_net_all_mon" +} + +linker = ObservationLinker( + model_name=MODEL_NAME, + variables=VARIABLES, + obs_sets=OBS_SETS, + ts_dir_ref_source=OBS_TS_DIR, + obstmp_dir=OBS_TMP_DIR, + altobs_dic=ALT_OBS_MAP, + obs_alias_file=OBS_ALIAS_FILE +) + +linker.link_obs_data() +linker.process_derived_variables() +EOF + +################### +# Run process job +################### +echo "Linking observational data using SLURM..." + +command="srun -N 1 python -u link_observation.py" +echo "Running: ${command}" +time eval "${command}" + +if [ $? -ne 0 ]; then + cd {{ scriptDir }} + echo "ERROR (6)" > {{ prefix }}.status + exit 6 +fi + +####################################################### +# Now create obs climo and time series for PCMDI diags +# Use same period as test model when possible +####################################################### +ts_dir_ref_source="{{ scriptDir }}/${workdir}/${obstmp_dir}" + +{% if "mean_climate" in subsection %} +climo_dir_ref=climo_ref +create_links_acyc_climo_obs "${ts_dir_ref_source}" "${climo_dir_ref}" ${ref_Y1} ${ref_Y2} 7 +{% elif "variability_modes_cpl" in subsection or "variability_modes_atm" in subsection or "enso" in subsection %} +ts_dir_ref=ts_ref +create_links_ts_obs "${ts_dir_ref_source}" "${ts_dir_ref}" ${ref_Y1} ${ref_Y2} 8 +{% endif %} + +{% endif %} + +{% if "synthetic_plots" not in subsection %} +######################################################## +# generate basic parameter file for pcmdi metrics driver +######################################################## +cat > parameterfile.py << EOF +import os +import sys +import json +##################### +# Basic Information +##################### + +start_yr = int('${Y1}') +end_yr = int('${Y2}') +num_years = end_yr - start_yr + 1 +period = f"{start_yr:04d}01-{end_yr:04d}12" + +model_parts = '${model_name}'.split('.') +mip, exp, product, realm = model_parts[:4] + +############################################## +# Configuration Shared with PCMDI Diagnostics +############################################## + +# Whether to generate NetCDF outputs for observations and model results +nc_out_obs = {{ mov_nc_out_obs }} +nc_out_model = {{ mov_nc_out_model }} + +# Output file extension: use .nc if either output is enabled, +# otherwise default to .xml +ext = ".nc" if nc_out_model or nc_out_obs else ".xml" + +# User annotation and debug flag +user_notes = 'Provenance and results' +debug = {{ pcmdi_debug }} + +# Enable plot generation for model and observation +plot_model = {{ mov_plot_model }} +plot_obs = {{ mov_plot_obs }} # optional + +# Execution mode and output format +run_type = '{{ run_type }}' +figure_format = '{{ figure_format }}' + +# Save interpolated model climatologies? +save_test_clims = {{ save_test_clims }} + +# Save all metrics results in a single file? +# Set to 'n' as metrics are computed per variable +metrics_in_single_file = 'n' + +# Custom values for land/sea masking +regions_values = { + "land": 100.0, + "ocean": 0.0 +} + +# Template path for land/sea mask file (fixed input) +modpath_lf = os.path.join( + 'fixed', + 'sftlf.%(model).nc' +) + +{% if "mean_climate" in subsection %} + +############################################ +# Setup Specific for Mean Climate Metrics +############################################ +modver = "${case_id}" +parallel = False +generate_sftlf = False +sftlf_filename_template = modpath_lf + +# Target grid: can be '2.5x2.5' or a CDMS2 grid object string +target_grid = '{{ target_grid }}' +targetGrid = target_grid # for backward compatibility +target_grid_string = '{{ target_grid_string }}' + +# Regridding tool and method (general use) +# OPTIONS: 'regrid2' or 'esmf' +regrid_tool = '{{ regrid_tool }}' +# OPTIONS: 'linear' or 'conservative' (only for 'esmf') +regrid_method = '{{ regrid_method }}' + +# Regridding tool and method for ocean diagnostics +regrid_tool_ocn = '{{ regrid_tool_ocn }}' # 'regrid2' or 'esmf' +regrid_method_ocn = ('{{ regrid_method_ocn }}') # 'linear' or 'conservative' + +# Model realization(s) to consider +realization = "*" + +# Model product name from input +test_data_set = [product] + +# Path to model climatology files +test_data_path = '${climo_dir_primary}' + +# Template for model climatology filenames +filename_template = '.'.join([ + mip, + exp, + '%(model)', + '%(realization)', + '${tableID}', + '%(variable)', + period, + 'AC', + '${case_id}', + 'nc' +]) + +# Path to reference climatology files +reference_data_path = '${climo_dir_ref}' + +# Observation catalogue file (dynamic by subsection) +custom_observations = os.path.join( + 'pcmdi_diags', + '{}_{}_catalogue.json'.format( + '${climo_dir_ref}', + '{{subsection}}' + ) +) + +# Load variable-specific region definitions +regions = json.load(open('regions.json')) + +# Load predefined region specifications and normalize domain lat/lon as tuples +regions_specs = json.load(open('regions_specs.json')) +for key in regions_specs: + domain = regions_specs[key].get('domain', {}) + if 'latitude' in domain: + domain['latitude'] = tuple(domain['latitude']) + regions_specs[key]['domain']['latitude'] = domain['latitude'] + if 'longitude' in domain: + domain['longitude'] = tuple(domain['longitude']) + regions_specs[key]['domain']['longitude'] = domain['longitude'] + +# METRICS OUTPUT +metrics_output_path = os.path.join( + 'pcmdi_diags', + 'metrics_results', + 'mean_climate', + mip, + exp, + '%(case_id)' +) + +#INTERPOLATED MODELS' CLIMATOLOGIES +diagnostics_output_path = os.path.join( + 'pcmdi_diags', + 'diagnostic_results', + 'mean_climate', + mip, + exp, + '%(case_id)' +) + +test_clims_interpolated_output = diagnostics_output_path + +{% endif %} + +{% if "variability_modes" in subsection %} +# Setup for Mode Variability Diagnostics +msyear = int(start_yr) +meyear = int(end_yr) + +# Seasons to analyze (comma-separated string to list) +seasons = '{{ seasons }}'.split(",") + +# Data frequency (e.g., monthly, seasonal) +frequency = '{{ frequency }}' + +# Variables to analyze (comma-separated string or space-separated) +varModel = '{{ vars }}' + +# Unit conversion flags for model and observations +ModUnitsAdjust = {{ ModUnitsAdjust }} +ObsUnitsAdjust = {{ ObsUnitsAdjust }} + +# Mask out land regions (consider ocean-only if True) +landmask = {{ landmask }} + +# If True, remove domain mean from each time step +RmDomainMean = {{ RmDomainMean }} + +# If True, normalize EOFs to unit variance +EofScaling = {{ EofScaling }} + +# Conduct Combined EOF/CBF analysis (if True) +CBF = {{ CBF }} + +# Conduct Conventional EOF analysis (if True) +ConvEOF = {{ ConvEOF }} + +# Skip CMEC output (hardcoded for now) +cmec = False + +# Whether to overwrite existing diagnostic output +update_json = False + +# Template for model input file paths +modnames = [product] +realization = "*" +modpath = os.path.join( + '${ts_dir_primary}', + '{}.{}.%(model).%(realization).{}.%(variable).{}.nc'.format( + mip, exp, '${tableID}', period + ) +) + +# Output results directory +results_dir = os.path.join( + 'pcmdi_diags', + '%(output_type)', + 'variability_modes', + '%(mip)', + '%(exp)', + '${case_id}', + '%(variability_mode)', + '%(reference_data_name)', +) +{% endif %} + +{% if "enso" in subsection %} +# Parameter Setup for ENSO Metrics +modnames = [product] +realization = realm + +modpath = os.path.join( + '${ts_dir_primary}', + '{}.{}.%(model).%(realization).{}.%(variable).{}.nc'.format( + mip, exp, '${tableID}', period + ) +) + +# Observation/Reference settings +obs_cmor = True +obs_cmor_path = '${ts_dir_ref}' +obs_catalogue = 'obs_catalogue.json' + +# Land/Sea mask for reference data +reference_data_lf_path = json.load(open('obs_landmask.json')) + +# Metrics collection type (e.g., ENSO_perf, ENSO_tel, ENSO_proc) +# Defined externally via metricsCollection + +# Output directory structure +results_dir = os.path.join( + 'pcmdi_diags', + '%(output_type)', + 'enso_metric', + '%(mip)', + '%(exp)', + '${case_id}', + '%(metricsCollection)', +) + +# Output filenames for JSON and NetCDF +json_name = "%(mip)_%(exp)_%(metricsCollection)_${case_id}_%(model)_%(realization)" + +netcdf_name = json_name + +{% endif %} + +EOF +{% endif %} + +################################################################ +# Run PCMDI Diags +echo +echo ===== RUN PCMDI DIAGS ===== +echo +# Prepare configuration file +cat > pcmdi.py << EOF +import os +import glob +import re +import json +import time +import datetime +import xcdat as xc +import numpy as np +import pandas as pd +import shutil + +import collections +from collections import OrderedDict + +import psutil +import subprocess +from itertools import chain +from subprocess import Popen, PIPE, call + +import pcmdi_metrics + +from pcmdi_metrics.graphics import ( + normalize_by_median, +) + +from pcmdi_zppy_util import( + count_child_processes, + run_serial_jobs, + run_parallel_jobs, + derive_missing_variable, + save_variable_regions, + generate_mean_clim_cmds, + generate_varmode_cmds, + build_enso_obsvar_catalog, + build_enso_obsvar_landmask, + generate_enso_cmds, + shift_row_to_bottom, + check_badvals, + archive_data, + drop_vars, + enso_plot_driver, + variability_modes_plot_driver, + mean_climate_plot_driver, + parcoord_metric_plot, + portrait_metric_plot, + ObservationLinker, + DataCatalogueBuilder, + LandSeaMaskGenerator, + ClimMetricsReader, + ClimMetricsMerger, + MeanClimateMetricsCollector, + VariabilityMetricsCollector, + EnsoDiagnosticsCollector, + SyntheticMetricsPlotter +) + +from pcmdi_viewer_util import( + collect_config, + setup_jinja_env, + create_section, + add_section, + generate_methodology_html, + generate_data_html, + generate_viewer_html +) + +# Determine multiprocessing usage +num_workers = {{ num_workers }} +multiprocessing = {{ multiprocessing }} if num_workers >= 2 else False + +{% if "synthetic_plots" not in subsection %} + +# Time range +start_yr = int('${Y1}') +end_yr = int('${Y2}') +num_years = end_yr - start_yr + 1 + +# Set data paths based on diagnostic type +{% if "mean_climate" in subsection %} +test_data_path = '${climo_dir_primary}' +reference_data_path = '${climo_dir_ref}' +{% elif "variability_modes" in subsection or "enso" in subsection %} +test_data_path = '${ts_dir_primary}' +reference_data_path = '${ts_dir_ref}' +{% endif %} + +# Dataset identifiers +test_data_set = ['${model_name}'.split(".")[1]] +{% if run_type == "model_vs_obs" %} +reference_data_set = '{{ obs_sets }}'.split(",") +{% elif run_type == "model_vs_model" %} +reference_data_set = ['${model_name_ref}'.split(".")[1]] +{% endif %} + +variables = '{{ vars }}'.split(",") + +############################################################### +#check and process derived quantities, these quantities are +#likely not included as default in e3sm_to_cmip module +############################################################### +for var in variables: + varin = re.split(r"[_-]", var)[0] if "_" in var or "-" in var else var + + test_fpaths = sorted(glob.glob(os.path.join(test_data_path, f"*.{var}.*.nc"))) + if not test_fpaths: + derive_missing_variable(varin, test_data_path, '${model_name}.${tableID}') + +{% if run_type == "model_vs_model" %} + ref_fpaths = sorted(glob.glob(os.path.join(reference_data_path, f"*.{var}.*.nc"))) + if not ref_fpaths: + derive_missing_variable(varin, reference_data_path, '${model_name_ref}.${tableID_ref}') +{% endif %} + +####################################################### +#collect and document data info in a dictionary +# for convenience of pcmdi processing +####################################################### +builder = DataCatalogueBuilder( + test_data_path, test_data_set, + reference_data_path, reference_data_set, + variables, '{{subsection}}', 'pcmdi_diags' +) +test_dic, obs_dic = builder.build_catalogues() + +########################################################## +# land/sea mask is needed in PCMDI diagnostics, check and +# generate it here as these data are not always available +# for model or observations +########################################################## +# Whether to generate the land/sea mask +generate_flag = {{ generate_sftlf }} +# Instantiate and run +mask_generator = LandSeaMaskGenerator( + test_path=test_data_path, + ref_path=reference_data_path, + subsection='{{subsection}}', + fixed_dir='fixed' +) +mask_generator.run(generate_flag) + +# Diagnostic input file templates +input_template = os.path.join( + 'pcmdi_diags', + '%(output_type)', + '%(metric_type)', + '${model_name}'.split(".")[0], + '${model_name}'.split(".")[1], + '${case_id}' +) + + +# Diagnostic output path templates +out_path = os.path.join('${results_dir}', '%(group_type)') + +{% endif %} + +{% if "mean_climate" in subsection %} +regions = '{{regions}}'.split(",") + +#assiagn region to each variable +save_variable_regions(variables, regions) + +# generate the command list +lstcmd = generate_mean_clim_cmds( + variables=variables, + obs_dic=obs_dic, + case_id='${case_id}' +) + +#################################################### +# call pcmdi mean climate diagnostics +#################################################### +if (len(lstcmd) > 0 ) and multiprocessing: + try: + results = run_parallel_jobs(lstcmd, num_workers) + for i, (stdout, stderr, return_code) in enumerate(results): + print(f"\nCommand {i+1} finished:") + print(f"STDOUT: {stdout}") + print(f"STDERR: {stderr}") + print(f"Return code: {return_code}") + except RuntimeError as e: + print(f"Execution failed: {e}") +elif (len(lstcmd) > 0 ): + try: + results = run_serial_jobs(lstcmd) + for i, (stdout, stderr, return_code) in enumerate(results): + print(f"\nCommand {i+1} finished:") + print(f"STDOUT: {stdout}") + print(f"STDERR: {stderr}") + print(f"Return code: {return_code}") + except RuntimeError as e: + print(f"Execution failed: {e}") +else: + print("no jobs to run,continue....") + +print("successfully finish all jobs....") +#time delay to ensure process completely finished +time.sleep(5) + +#orgnize diagnostic output +collector = MeanClimateMetricsCollector( + regions=regions, + variables=variables, + fig_format='{{figure_format}}', + model_info=tuple('${model_name}'.split(".")), # (mip, exp, model, relm) + case_id='${case_id}', + input_template=input_template, + output_dir=out_path +) +collector.collect() + +{% endif %} + +{% if "variability_modes" in subsection %} +########################################## +# call pcmdi mode variability diagnostics +########################################## +print("calculate mode variability metrics") + +{% if subsection == "variability_modes_atm" %} +var_modes = '{{ atm_modes }}'.split(",") +{% elif subsection == "variability_modes_cpl" %} +var_modes = '{{ cpl_modes }}'.split(",") +{% endif %} + +#from configuration file +varOBS = '{{vars}}' +refset = obs_dic[varOBS]['set'] +refname = obs_dic[varOBS][refset] +refpath = obs_dic[varOBS][refname]['file_path'] +reftyrs = int(str(obs_dic[varOBS][refname]['yymms'])[0:4]) +reftyre = int(str(obs_dic[varOBS][refname]['yymme'])[0:4]) + +# Call the function +lstcmd = generate_varmode_cmds( + modes=var_modes, + varOBS=varOBS, + reftyrs=reftyrs, + reftyre=reftyre, + refname=refname, + refpath=refpath, + case_id='${case_id}' +) + +if (len(lstcmd) > 0 ) and multiprocessing: + try: + results = run_parallel_jobs(lstcmd, num_workers) + for i, (stdout, stderr, return_code) in enumerate(results): + print(f"\nCommand {i+1} finished:") + print(f"STDOUT: {stdout}") + print(f"STDERR: {stderr}") + print(f"Return code: {return_code}") + except RuntimeError as e: + print(f"Execution failed: {e}") +elif (len(lstcmd) > 0 ): + try: + results = run_serial_jobs(lstcmd) + for i, (stdout, stderr, return_code) in enumerate(results): + print(f"\nCommand {i+1} finished:") + print(f"STDOUT: {stdout}") + print(f"STDERR: {stderr}") + print(f"Return code: {return_code}") + except RuntimeError as e: + print(f"Execution failed: {e}") +else: + print("no jobs to run,continue...") + +print("successfully finish all jobs....") +#time delay to ensure process completely finished +time.sleep(5) + +# Create the collector instance +collector = VariabilityMetricsCollector( + modes=var_modes, + fig_format='{{figure_format}}', + mip='${model_name}'.split(".")[0], + exp='${model_name}'.split(".")[1], + model='${model_name}'.split(".")[2], + relm='${model_name}'.split(".")[3], + case_id='${case_id}', + input_dir=input_template, + output_dir=out_path +) + +# Run the collection process +collector.collect() + +{% endif %} + +{% if "enso" in subsection %} +############################################# +# call enso_driver.py to process diagnostics +############################################# +build_enso_obsvar_catalog(obs_dic, variables) +build_enso_obsvar_landmask(obs_dic, variables) + +#now start enso driver +lstcmd = generate_enso_cmds('{{ enso_groups }}', '${case_id}') +if (len(lstcmd) > 0 ) and multiprocessing: + try: + results = run_parallel_jobs(lstcmd, num_workers) + for i, (stdout, stderr, return_code) in enumerate(results): + print(f"\nCommand {i+1} finished:") + print(f"STDOUT: {stdout}") + print(f"STDERR: {stderr}") + print(f"Return code: {return_code}") + except RuntimeError as e: + print(f"Execution failed: {e}") +elif (len(lstcmd) > 0 ) and not multiprocessing: + try: + results = run_serial_jobs(lstcmd) + for i, (stdout, stderr, return_code) in enumerate(results): + print(f"\nCommand {i+1} finished:") + print(f"STDOUT: {stdout}") + print(f"STDERR: {stderr}") + print(f"Return code: {return_code}") + except RuntimeError as e: + print(f"Execution failed: {e}") +else: + print("no jobs to run...") + +print("successfully finish all jobs....") +#time delay to ensure process completely finished +time.sleep(5) + +# Initialize and run collector +obs_dict = json.load(open('obs_catalogue.json')) +obs_name = list(obs_dict.keys())[0] +collector = EnsoDiagnosticsCollector( + fig_format='{{figure_format}}', + refname=obs_name, + model_name_parts='${model_name}'.split("."), + case_id='${case_id}', + input_dir=input_template, + output_dir=out_path +) + +enso_groups = '{{ enso_groups }}'.split(",") +collector.run(enso_groups) + +{% endif %} + +{% if "synthetic_plots" in subsection %} +######################################### +#plot synthetic figures for pcmdi metrics +######################################### +print("generate synthetic metrics plot ...") +metric_sets = '{{sub_sets}}'.split(",") +figure_sets = '{{synthetic_sets}}'.split(",") +figure_format = '{{figure_format}}' +test_input_path = os.path.join( + '${www}', + '%(model_name)', + 'pcmdi_diags', + '${results_dir}', + 'metrics_data', + '%(group_type)' +) + +metric_dict = json.load(open('synthetic_metrics_list.json')) + +plotter = SyntheticMetricsPlotter( + case_name='{{case}}', + test_name='{{model_name}}', + table_id='{{model_tableID}}', + figure_format=figure_format, + figure_sets=figure_sets, + metric_dict=metric_dict, + save_data=True, + base_test_input_path=test_input_path, + results_dir='${web_dir}/${results_dir}', + cmip_clim_dir='{{cmip_clim_dir}}', + cmip_clim_set='{{cmip_clim_set}}', + cmip_movs_dir='{{cmip_movs_dir}}', + cmip_movs_set='{{cmip_movs_set}}', + atm_modes='{{ atm_modes }}', + cpl_modes='{{ cpl_modes }}', + cmip_enso_dir='{{cmip_enso_dir}}', + cmip_enso_set='{{cmip_enso_set}}' +) + +# Generate Summary Metrics plots +# e.g., "climatology,enso,variability" +groups = '{{sub_sets}}'.split(',') +plotter.generate(groups) + +print("Generating viewer page for diagnostics...") + +# Extract template values (assumes substitution happens before execution) +title = "{{pcmdi_webtitle}}" +version = "{{pcmdi_version}}" +subtitle = "${run_type}".replace('_', ' ').capitalize() +case_id = "${case}" +model_name = "{{model_name}}" +table_id = "{{model_tableID}}" + +# ts_years is assumed to be a list via string_list(default=list("")) +ts_years = {{ts_years}} +ts_periods = ts_years if isinstance(ts_years, list) else [] + +# Validate and unpack periods +if len(ts_periods) == 3: + clim_period, emov_period, enso_period = [p.strip() for p in ts_periods] +else: + raise ValueError( + f"Expected 3 periods (climatology, EMoV, ENSO), " + f"but got {len(ts_periods)}: {ts_periods}" + ) + +# Set up paths +obs_dir = os.path.join('{{pcmdi_external_prefix}}', 'observations', 'Atm', 'time-series') +pmp_dir = os.path.join('{{pcmdi_external_prefix}}', 'pcmdi_data') +out_dir = os.path.join("${web_dir}", "${results_dir}", "viewer") +os.makedirs(out_dir, exist_ok=True) + +# Copy logo +web_logo_src = os.path.join( + '{{pcmdi_external_prefix}}', + '{{pcmdi_viewer_template}}', + 'e3sm_pmp_logo.png' +) +web_logo_dst = os.path.join(out_dir, 'e3sm_pmp_logo.png') +shutil.copy(web_logo_src, web_logo_dst) + +# Build config +config = collect_config( + title=title, + subtitle=subtitle, + version=version, + case_id=case_id, + diag_dir="${web_dir}", + obs_dir=obs_dir, + pmp_dir=pmp_dir, + out_dir=out_dir, + clim_period=clim_period, + emov_period=emov_period, + enso_period=enso_period +) + +# Render viewer +generate_methodology_html(config) +generate_data_html(config) +generate_viewer_html(config) + +{% endif %} + +EOF +################################ +# Run diagnostics +mkdir -p pcmdi_diags +command="srun -N 1 python -u pcmdi.py" +# Run diagnostics +time ${command} +if [ $? != 0 ]; then + cd {{ scriptDir }} + echo 'ERROR (11)' > {{ prefix }}.status + exit 11 +fi + +################################# +# Copy output to web server +echo +echo ===== COPY FILES TO WEB SERVER ===== +echo + +# Create top-level directory +mkdir -p ${web_dir} +if [ $? != 0 ]; then + cd {{ scriptDir }} + echo 'ERROR (13)' > {{ prefix }}.status + exit 13 +fi + +{% if machine in ['pm-cpu', 'pm-gpu'] %} +# For NERSC, make sure it is world readable +f=`realpath ${web_dir}` +while [[ $f != "/" ]] +do + owner=`stat --format '%U' $f` + if [ "${owner}" = "${USER}" ]; then + chgrp e3sm $f + chmod go+rx $f + fi + f=$(dirname $f) +done +{% endif %} + +############################################ +# Copy files +#rsync -a --delete ${results_dir} ${web_dir}/ +{% if "synthetic_plots" not in subsection %} +rsync -a ${results_dir} ${web_dir}/ +if [ $? != 0 ]; then + cd {{ scriptDir }} + echo 'ERROR (14)' > {{ prefix }}.status + exit 14 +fi +{% endif %} + +{% if machine in ['pm-cpu', 'pm-gpu'] %} +# For NERSC, change permissions of new files +pushd ${web_dir}/ +chgrp -R e3sm ${results_dir} +chmod -R go+rX,go-w ${results_dir} +popd +{% endif %} + +{% if machine in ['anvil', 'chrysalis'] %} +# For LCRC, change permissions of new files +pushd ${web_dir}/ +chmod -R go+rX,go-w ${results_dir} +popd +{% endif %} + +# Delete temporary workdir +cd .. +if [[ "${debug,,}" != "true" ]]; then + rm -rf ${workdir} +fi + +# Update status file and exit +{% raw %} +ENDTIME=$(date +%s) +ELAPSEDTIME=$(($ENDTIME - $STARTTIME)) +{% endraw %} +echo ============================================== +echo "Elapsed time: $ELAPSEDTIME seconds" +echo ============================================== +rm -f {{ prefix }}.status +echo 'OK' > {{ prefix }}.status +exit 0 diff --git a/zppy/templates/ts.bash b/zppy/templates/ts.bash old mode 100644 new mode 100755 index d5a1b71c..0f991457 --- a/zppy/templates/ts.bash +++ b/zppy/templates/ts.bash @@ -115,6 +115,92 @@ if [ $? != 0 ]; then exit 3 fi +{%- if ts_fmt != 'ts_only' %} +tmp_dir=tmp_{{ prefix }} + +# Generate CMIP ts +cat > default_metadata.json << EOF +{% include cmip_metadata %} +EOF +{ + export cmortables_dir={{ cmor_tables_prefix }}/cmip6-cmor-tables/Tables + #input_dir={{ output }}/post/{{ component }}/{{ grid }}/ts/{{ frequency }}/{{ '%dyr' % (ypf) }} + input_dir=${dest}/{{ '%04d' % (yr_start) }}_{{ '%04d' % (yr_end) }} + mkdir -p $input_dir + + cp -s $dest/*_{{ '%04d' % (yr_start) }}??_{{ '%04d' % (yr_end) }}??.nc $input_dir + dest_cmip={{ output }}/post/{{ component }}/{{ grid }}/cmip_ts/{{ frequency }} + mkdir -p ${dest_cmip} + + {{ e3sm_to_cmip_environment_commands }} + + {% if input_files.split(".")[0] == 'cam' or input_files.split(".")[0] == 'eam' -%} + #add code to do vertical interpolation variables at model levels before e3sm_to_cmip + IFS=',' read -ra mlvars <<< "{{ interp_vars }}" + for var in "${mlvars[@]}" + do + for file in ${input_dir}/${var}_{{ '%04d' % (yr_start) }}??_{{ '%04d' % (yr_end) }}??.nc + do + if [ -f ${file} ]; then + #ncks --rgr xtr_mth=mss_val --vrt_fl='{{cmip_plevdata}}' ${file} ${file}.plev + ncremap -p mpi --vrt_ntp=log --vrt_xtr=mss_val --vrt_out='{{cmip_plevdata}}' ${file} ${file}.plev + #overwrite the model level data + mv ${file}.plev ${file} + fi + done + done + {% endif -%} + + #call e3sm_to_cmip + srun -N 1 e3sm_to_cmip \ + --output-path \ + ${dest_cmip}/${tmp_dir} \ + {% if input_files.split(".")[0] == 'clm2' or input_files.split(".")[0] == 'elm' -%} + --var-list \ + 'snd, mrsos, mrso, mrfso, mrros, mrro, prveg, evspsblveg, evspsblsoi, tran, tsl, lai, cLitter, cProduct, cSoilFast, cSoilMedium, cSoilSlow, fFire, fHarvest, cVeg, nbp, gpp, ra, rh' \ + --realm \ + lnd \ + {% endif -%} + {% if input_files.split(".")[0] == 'cam' or input_files.split(".")[0] == 'eam' -%} + --var-list \ + 'ua, va, ta, wa, zg, hur, pr, prc, prsn, ts, tas, prw, psl, sfcWind, tasmax, tasmin, tauu, tauv, rtmt, rsdt, rsds, rsdscs, rlds, rldscs, rsus, rsuscs, rsut, rsutcs, rlus, rlut, rlutcs, clivi, clwvi, clt, evspsbl, hfls, hfss, huss' \ + --realm \ + atm \ + {% endif -%} + --input-path \ + ${input_dir} \ + --user-metadata \ + {{ scriptDir }}/${workdir}/default_metadata.json \ + --num-proc \ + 12 \ + --tables-path \ + ${cmortables_dir} + + if [ $? != 0 ]; then + cd {{ scriptDir }} + echo 'ERROR (4)' > {{ prefix }}.status + exit 4 + fi + {{ environment_commands }} + + # Move output ts files to final destination + mv ${dest_cmip}/${tmp_dir}/CMIP6/CMIP/*/*/*/*/*/*/*/*/*.nc ${dest_cmip} + if [ $? != 0 ]; then + cd {{ scriptDir }} + echo 'ERROR (5)' > {{ prefix }}.status + exit 5 + fi + + rm -r ${dest_cmip}/${tmp_dir} + +} +if [ $? != 0 ]; then + cd {{ scriptDir }} + echo 'ERROR (6)' > {{ prefix }}.status + exit 6 +fi +{%- endif %} + # Delete temporary workdir cd .. if [[ "${debug,,}" != "true" ]]; then From ced018b78b891082f26d7dc381efde049609fcfa Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 22 May 2025 13:48:31 -0500 Subject: [PATCH 02/47] Refactor and fixes --- conda/chrisalis.yml | 34 - examples/post.v3.LR.amip.0101.cfg | 178 ++++ examples/remove_duplicate_log_lines.py | 18 + setup.py | 34 + zppy/defaults/default.ini | 445 +++++---- zppy/e3sm_diags.py | 29 +- zppy/pcmdi_diags.py | 59 +- zppy/templates/coupled_global.py | 872 ------------------ zppy/templates/e3sm_to_cmip.bash | 34 +- zppy/templates/global_time_series.bash | 0 .../e3sm_to_cmip/default_metadata.json | 0 .../e3sm_to_cmip/vrt_remap_plev19.nc | Bin 1260 -> 0 bytes zppy/templates/pcmdi_diags.bash | 591 ++---------- zppy/templates/ts.bash | 86 -- zppy/utils.py | 28 +- 15 files changed, 643 insertions(+), 1765 deletions(-) delete mode 100644 conda/chrisalis.yml create mode 100644 examples/post.v3.LR.amip.0101.cfg create mode 100644 examples/remove_duplicate_log_lines.py create mode 100644 setup.py delete mode 100644 zppy/templates/coupled_global.py mode change 100755 => 100644 zppy/templates/global_time_series.bash mode change 100755 => 100644 zppy/templates/inclusions/e3sm_to_cmip/default_metadata.json delete mode 100755 zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc mode change 100755 => 100644 zppy/templates/ts.bash diff --git a/conda/chrisalis.yml b/conda/chrisalis.yml deleted file mode 100644 index cdefec7f..00000000 --- a/conda/chrisalis.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: zppy_dev -channels: - - conda-forge - - defaults -dependencies: - # Base - # ================= - - python=3.9.13 - - pip=22.2.2 - - configobj=5.0.6 - - jinja2=3.1.2 - - mache>=1.5.0 - - mpas_tools>=0.15.0 - - pillow=9.2.0 - # Developer Tools - # ================= - # If versions are updated, also update 'rev' in `.pre-commit-config.yaml` - - black=22.8.0 # version from https://anaconda.org/conda-forge/black - - flake8=5.0.4 # version from https://anaconda.org/conda-forge/flake8 - # This line also implicitly installs isort - - flake8-isort=4.2.0 # version from https://anaconda.org/conda-forge/flake8-isort - - mypy=0.982 # version from https://anaconda.org/conda-forge/mypy - - pre-commit=2.20.0 # version from https://anaconda.org/conda-forge/pre-commit - - tbump=6.9.0 - # Documentation - # If versions are updated, also update in `.github/workflows/build_workflow.yml` - # ================= - - sphinx=5.2.3 - - sphinx-multiversion=0.2.4 - - sphinx_rtd_theme=1.0.0 - # Need to pin docutils because 0.17 has a bug with unordered lists - # https://github.com/readthedocs/sphinx_rtd_theme/issues/1115 - - docutils=0.16 -prefix: /home/ac.szhang/.conda/envs/zppy_dev diff --git a/examples/post.v3.LR.amip.0101.cfg b/examples/post.v3.LR.amip.0101.cfg new file mode 100644 index 00000000..1de1fba4 --- /dev/null +++ b/examples/post.v3.LR.amip.0101.cfg @@ -0,0 +1,178 @@ +# To run: +# `pip install .` the zppy-interfaces changes into the `zi-pcmdi-20250529` environment +# Update unique_id in output, www below +# Be in the zppy-pr719-dev-20250612 environment +# > pip install . && zppy -c examples/post.v3.LR.amip.0101.cfg + +[default] +input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/AMIP/v3.LR.amip_0101 +output = /lcrc/group/e3sm/ac.forsyth2/zppy_pr719_output/unique_id_16/v3.LR.amip_0101 +www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pr719_output/unique_id_16 +campaign = "water_cycle" +case = v3.LR.amip_0101 +environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" +partition = compute + +[ts] +active = True +walltime = "00:10:00" +years = "2005:2015:5" + + [[ atm_monthly_180x360_aave ]] + input_subdir = "archive/atm/hist" + input_files = "eam.h0" + frequency = "monthly" + mapping_file = /home/ac.zender/data/maps/map_ne30pg2_to_cmip6_180x360_aave.20200201.nc + # list for pcmdi diag, note: PHIS,hyam,hybm,hyai,hybi need to be included to process the 3D fields + vars = "ICEFRAC,LANDFRAC,OCNFRAC,PSL,FSNTC,FSNTOAC,SWCF,LWCF,FLUT,FSNT,FSNTOA,FLNT,FLNTC,FSNS,FLNS,FSNS,SHFLX,QFLX,LHFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,U10,QREFHT,TMQ,CLDTOT,CLDHGH,CLDMED,CLDLOW,FLDS,FSDS,TGCLDIWP,TGCLDCWP,TGCLDLWP,FLNSC,FLUTC,FSDSC,SOLIN,FSNSC,AODABS,AODVIS,AODDUST,AREL,TREFMNAV,TREFMXAV,PS,PHIS,U,V,T,Z3" + +[e3sm_to_cmip] +active = True +frequency = "monthly" +grid = "180x360_aave" +ts_grid = "180x360_aave" +ts_num_years=5 +walltime = "00:10:00" +years = "2005:2015:5" + + [[ atm_monthly_180x360_aave ]] + # list of variables to be interpolated to pressure level; see e3sm_to_cmip/vrt_remap_plev19.nc + cmip_plevdata = "inclusions/e3sm_to_cmip/vrt_remap_plev19.nc" + input_files = "eam.h0" + interp_vars = "U,V,T,Z3" + ts_subsection = "atm_monthly_180x360_aave" + vars = "ICEFRAC,LANDFRAC,OCNFRAC,PSL,FSNTC,FSNTOAC,SWCF,LWCF,FLUT,FSNT,FSNTOA,FLNT,FLNTC,FSNS,FLNS,FSNS,SHFLX,QFLX,LHFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,U10,QREFHT,TMQ,CLDTOT,CLDHGH,CLDMED,CLDLOW,FLDS,FSDS,TGCLDIWP,TGCLDCWP,TGCLDLWP,FLNSC,FLUTC,FSDSC,SOLIN,FSNSC,AODABS,AODVIS,AODDUST,AREL,TREFMNAV,TREFMXAV,PS,PHIS,U,V,T,Z3" + +[pcmdi_diags] +active = True +multiprocessing = True +nodes = 1 +num_workers = 24 +#observational data +obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series' +pcmdi_debug = False +run_type = "model_vs_obs" +reference_data_path = "" +reference_data_path_ts = "" +ts_grid = "180x360_aave" +walltime = "4:00:00" +environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" + + [[ mean_climate ]] + active = True + ################################################################################## + #create rules to construct cmip-like datasets with a string constructed as + #"mip.exp.product.realization", where "."is used as the delimiter (mandatary) + ts_years = "2005-2014", + ts_num_years = 5 + ref_years = "2005-2014", + ref_start_yr = 1979 + ref_final_yr = 2018 + model_name = 'e3sm.amip.v3-LR.0101' + model_tableID = 'Amon' + cmip_clim_set = "cmip6.amip.v20241029" + ################################################################################## + #variables in current pcmdi-cmip list + #note: rltcre and rstcre are derived variables as they are not included in cmip_ts + vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rsus,rsuscs,rlus,rlut,rlutcs,rsds,rsdscs,rsdt,rsut,rsutcs,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" + #default for all variables or specify source for each variable separately. + obs_sets = "default" + #see zppy/templates/pcmdi_diags/regions_specs.json for options of all regions + #below is default setup for global/regional mean climate metrics + regions = "global,ocean,land,NHEX,SHEX,TROPICS" + target_grid = "2.5x2.5" + target_grid_string = "2p5x2p5" + + [[ variability_modes_cpl ]] + active = True + ################################################################################## + ts_years = "2005-2014", + ts_num_years = 5 + ref_years = "2005-2014", + ref_start_yr = 1870 + ref_final_yr = 2019 + #create rules to construct cmip-like datasets with a string constructed as + #"mip.exp.product.realization", where "."is used as the delimiter (mandatary) + model_name = 'e3sm.amip.v3-LR.0101' + model_tableID = 'Amon' + cmip_movs_set = "cmip6.amip.v20210119" + ################################################################################## + grid = '180x360_aave' + vars = "ts" + frequency = 'mo' + ModUnitsAdjust = '(True,"subtract",273.15)' + landmask = True + #see zppy/templates/pcmdi_diags/reference_alias.json for options + obs_sets = "alternate1" #"default" + ObsUnitsAdjust = '(True,"subtract",273.15)' + #name of modes varibility + cpl_modes = "PDO,NPGO,AMO" + #options specific for mode varibility metrics in pcmdi + seasons = "yearly,monthly" + figure_format = "png" + + [[ variability_modes_atm ]] + active = True + ################################################################################## + ts_years = "2005-2014", + ts_num_years = 5 + ref_years = "2005-2014", + ref_start_yr = 1853 + ref_final_yr = 2015 + #create rules to construct cmip-like datasets with a string constructed as + #"mip.exp.product.realization", where "." is used as the delimiter (mandatary) + model_name = 'e3sm.amip.v3-LR.0101' + model_tableID = 'Amon' + cmip_movs_set = "cmip6.amip.v20210119" + ################################################################################## + grid = '180x360_aave' + vars = "psl" + landmask = False + frequency = 'mo' + ModUnitsAdjust = '(True,"divide",100.0)' + #observational data + obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series' + #see zppy/templates/pcmdi_diags/reference_alias.json for options + #default for all variables + obs_sets = "alternate2" # "default" + ObsUnitsAdjust = '(True,"divide",100.0)' + #varibility modes + atm_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" + #options specific for mode varibility metrics in pcmdi + seasons = "DJF,MAM,JJA,SON,yearly,monthly" + figure_format = "png" + + [[ enso ]] + active = True + ################################################################################## + ts_years = "2005-2015", + ts_num_years = 5 + ref_years = "2005-2015", + ref_start_yr = 1979 + ref_final_yr = 2018 + #create rules to construct cmip-like datasets with a string constructed as + #"mip.exp.product.realization", where "."is used as the delimiter (mandatary) + model_name = 'e3sm.amip.v3-LR.0101' + model_tableID = 'Amon' + cmip_enso_set = "cmip6.amip.v20210620" + ################################################################################## + #see zppy/templates/pcmdi_diags/reference_alias.json for options + customized_cmor_obs = True + vars = "psl,pr,prsn,ts,tas,tauu,tauv,hfls,hfss,rlds,rsds,rlus,rlut,rsdt" + # For enso, the obs_sets is explicitely set for each variable to ensure that + # they are from the same source, below are setup to used ERA5 reanalysis. + obs_sets = "default,alternate2,default,default,default,default,default,default,default,alternate2,alternate2,alternate2,alternate2,alternate2" + #see zppy/templates/pcmdi_diags/reference_alias.json for options + enso_groups = "ENSO_perf,ENSO_proc,ENSO_tel" + figure_format = "png" + + [[ synthetic_plots ]] + active = True + model_name = 'e3sm.amip.v3-LR.0101' + model_tableID = 'Amon' + #sub_sets = "mean_climate,variability_modes,enso_metric" + sub_sets = "mean_climate,variability_modes" + ts_years = "2005-2014", + ref_years = "2005-2014", + synthetic_sets = "portrait,parcoord" + figure_format = "png" diff --git a/examples/remove_duplicate_log_lines.py b/examples/remove_duplicate_log_lines.py new file mode 100644 index 00000000..9819fef2 --- /dev/null +++ b/examples/remove_duplicate_log_lines.py @@ -0,0 +1,18 @@ +# Written with LivChat +def remove_consecutive_duplicates(input_file_path): + output_file_path = input_file_path + ".duplicates_removed" + last_written_line = None + + with open(input_file_path, "r", encoding="utf-8") as infile, open( + output_file_path, "w", encoding="utf-8" + ) as outfile: + for line in infile: + if line != last_written_line: + outfile.write(line) + last_written_line = line + return output_file_path + + +if __name__ == "__main__": + f = "/lcrc/group/e3sm/ac.forsyth2/zppy_pr719_output/unique_id_16/v3.LR.amip_0101/post/scripts/pcmdi_diags_enso_model_vs_obs_2005-2015.o785792" + remove_consecutive_duplicates(f) diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..36d4982f --- /dev/null +++ b/setup.py @@ -0,0 +1,34 @@ +import os + +from setuptools import find_packages, setup + + +# From https://github.com/MPAS-Dev/compass/blob/master/setup.py +def package_files(directory, prefixes, extensions): + """based on https://stackoverflow.com/a/36693250/7728169""" + paths = [] + for path, directories, filenames in os.walk(directory): + for filename in filenames: + parts = filename.split(".") + prefix = parts[0] + extension = parts[-1] + if prefix in prefixes or extension in extensions: + paths.append(os.path.join("..", path, filename)) + return paths + + +data_files = package_files( + "zppy", prefixes=[], extensions=["bash", "csh", "cfg", "ini", "sh", "json"] +) + +setup( + name="zppy", + version="3.0.0", + author="Ryan Forsyth, Chris Golaz", + author_email="forsyth2@llnl.gov, golaz1@llnl.gov", + description="Post-processing software for E3SM", + python_requires=">=3.9", + packages=find_packages(include=["zppy", "zppy.*"]), + package_data={"": data_files}, + entry_points={"console_scripts": ["zppy=zppy.__main__:main"]}, +) diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index f9304578..14762bfb 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -124,12 +124,9 @@ input_component = string(default="") [ts] area_nm = string(default="area") -cmip_metadata = string(default="inclusions/e3sm_to_cmip/default_metadata.json") -cmip_plevdata = string(default="inclusions/e3sm_to_cmip/vrt_remap_plev19.nc") # Days per file dpf = integer(default=30) extra_vars = string(default="") -interp_vars = string(default="U,V,T,Q,RELHUM,OMEGA,Z3") # Time-steps per day tpd = integer(default=1) # Model component having generated input files (eam, eamxx, elm, mosart, ...) @@ -137,8 +134,6 @@ input_component = string(default="") [[__many__]] area_nm = string(default=None) - cmip_metadata = string(default=None) - cmip_plevdata = string(default=None) dpf = integer(default=None) extra_vars = string(default=None) tpd = integer(default=None) @@ -147,13 +142,18 @@ input_component = string(default="") [e3sm_to_cmip] # Metadata json file cmip_metadata = string(default="inclusions/e3sm_to_cmip/default_metadata.json") +# Pressure level nc file +cmip_plevdata = string(default="inclusions/e3sm_to_cmip/vrt_remap_plev19.nc") # variables to run e3sm_to_cmip on cmip_vars = string(default="") # Model component having generated input files (eam, eamxx, elm, mosart, ...) input_component = string(default="") +# TBD: description here +interp_vars = string(default="U,V,T,Q,RELHUM,OMEGA,Z3") [[__many__]] cmip_metadata = string(default=None) + cmip_plevdata = string(default=None) cmip_vars = string(default=None) input_component = string(default=None) interp_vars = string(default=None) @@ -165,229 +165,6 @@ input_files = string(default="eam.h2") # If not set, zppy will attempt to infer this value from the topography file. res = string(default="") -[pcmdi_diags] -backend = string(default="mpl") -cfg = string(default="") -# Title of zppy-pcmdi diagnostics -pcmdi_webtitle = string(default="E3SM-PMP Diagnostics") -# Version of zppy-pcmdi code -pcmdi_version = string(default="v3.8.2") -# File of specified regions for mean climate calculation -regions_specs = string(default="pcmdi_data/region/regions_specs.json") -# File of observation data name for mean climate calculation -reference_alias = string(default="pcmdi_data/reference/reference_alias.json") -# Utility file with functions for zppy-pcmdi data processing -pcmdi_zppy_util = string(default="pcmdi_data/utility/pcmdi_zppy_util.py") -# Utility file with functions for zppy-pcmdi viewer page processing -pcmdi_viewer_util = string(default="pcmdi_data/utility/pcmdi_viewer_util.py") -# Template files for zppy-pcmdi viewer page processing -pcmdi_viewer_template = string(default="pcmdi_data/viewer") -# File of variable list to generate synthetic metrics plot -synthetic_metrics = string(default="pcmdi_data/synthetic_metrics/synthetic_metrics_list.json") -# Path to observation time-series data -# Required for "mean_climate","variability_mode","enso" -obs_ts = string(default="") -# observational data sets (see reference_alias.json) -# observation data tag in reference_alias -obs_sets = string(default="default") -# options specific for constructing pcmdi preferred file name conventions -# required for "model_vs_obs" comparison -model_name = string(default="e3sm.historical.v3-LR.0051") -model_tableID = string(default="Amon") -# required for "model_vs_model" comparison -model_name_ref = string(default="ERA5") -model_tableID_ref = string(default="Amon") -# variables in the cmip6 table that can be potentially used by pcmdi -# this list depends on the definition of cmip variable -# required for "mean climate" diagnostics -cmip_vars = string(default="pr,prw,psl,rlds,rldscs,rlut,rlutcs,rsut,rsutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rtmt,sfcWind,tas,tauu,tauv,ts,ta,ua,va,zg") -# flag to process the land/sea mask within pcmdi -generate_sftlf = string(default=True) -# variables to be used by the pcmdi diagnostics -# needs to setup for each subsections, defalut setup is the mean climate metrics -vars = string(default="pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500") -# sets of diagnostics from pcmdi package -sets = string_list(default=list("mean_climate","variability_modes_atm","variability_modes_cpl","enso","synthetic_plots")) -# options to identify subset of pcmdi drivers ("mean_climate","variability_mode","enso") -subsets = string_list(default=list("mean_climate","variability_modes","enso")) -########################################################################################### -# Required for run_type="model_vs_model" runs, different from e3sm_diag, -# model_vs_model in pcmdi referred to the comparision of two model simulations -# with observations and cmip models. -########################################################################################### -#path for reference model data (time series) -reference_data_path = string(default="") -reference_data_path_ts = string(default="") -# pcmdi_diags.py will set to match `years` if not specified -ref_years = string_list(default=list("")) -# The years increment for reference data -ts_num_years_ref = integer(default=5) -# Final year (i.e., the last available year) for the reference data -ref_final_yr = string(default="") -# Start year for the reference data -ref_start_yr = string(default="") -# reference model name -ref_name = string(default="") -# The years range for test model data -ts_years = string_list(default=list("")) -# The years increment for test model data -ts_num_years = integer(default=5) -# Set to true to swap test and ref when run_type="model_vs_model" -swap_test_ref = boolean(default=False) -########################################################################################## -# below followed the setup in e3sm_diag but used for PCMDI workflow -########################################################################################## -# See url -multiprocessing = boolean(default=True) -# See url -num_workers = integer(default=24) -# See url -figure_format = string(default="png") -# comparision type (same as e3sm_diag) -run_type = string(default="model_vs_obs") -# Used to label the results directory -# Options are "model_vs_obs" and "model_vs_model" -tag = string(default="model_vs_obs") -########################################################################################## -# options for pcmdi mode varibility diagnostics -# vars = "psl" for atm_modes -# vars = "ts" for cpl_modes -######################################################################################### -#name of atmospheric modes varibility -atm_modes = string(default="NAM,NAO,PNA,NPO,SAM,PSA1,PSA2") -#name of coupled modes varibility -cpl_modes = string(default="PDO,NPGO,AMO") -#keywards for unit conversion in pcmdi (model) -ModUnitsAdjust = string(default="") -#keywards for unit conversion in pcmdi (observation) -ObsUnitsAdjust = string(default="") -#frequency of the model data -frequency = string(default="mo") -#options specific for mode varibility metrics in pcmdi -seasons = string(default="monthly") -landmask = string(default=False) -RmDomainMean = string(default=True) -EofScaling = string(default=False) -ConvEOF = string(default=True) -CBF = string(default=True) -mov_plot_obs = string(default=True) -mov_plot_model = string(default=True) -mov_nc_out_obs = string(default=True) -mov_nc_out_model = string(default=True) -########################################################################################## -# options for pcmdi enso diagnostics -# vars = "psl,pr,prsn,ts,tas,tauu,tauv,hflx,hfss,rlds,rsds,rlus,rlut,rsdt" -########################################################################################## -enso_groups = string(default="ENSO_perf,ENSO_proc,ENSO_tel") -########################################################################################## -# optional for mean climate diagnostics -# vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs, -# rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200, -# va-850,zg-500" -########################################################################################## -# model data grid after remapping -grid = string(default="180x360_aave") -#default regions for mean climate metrics data -#more options can be found at "regions_specs" -#regions = string(default="global,ocean,land,NHEX,SHEX,TROPICS,NHEX_ocean,SHEX_ocean,NHEX_land,SHEX_land,ocean_50S50N") -regions = string(default="global,ocean,land,NHEX,SHEX,TROPICS") -# save derived climatology data -save_test_clims = string(default=True) -# Regridding by pcmdi (required for mean climate) -# OPTIONS: '1x1' for e3sm, '2.5x2.5' for cmip, or an actual cdms2 grid object, -target_grid = string(default="1x1") -# OPTIONS: "1px1p" for e3sm, "2p5x2p5" for cmip, description on the selected grid -target_grid_string = string(default="1px1p") -# OPTIONS: 'regrid2','esmf' -regrid_tool = string(default="esmf") -# OPTIONS: 'linear','conservative', only if tool is esmf -regrid_method = string(default="regrid2") -# OPTIONS: 'linear','conservative', only if tool is esmf -regrid_method_ocn = string(default="conservative") -################################################################### -#options for synthetic plots with exisiting pcmdi cmip model metrics -################################################################### -synthetic_sets = string(default="portrait,parcoord") -# path to pcmdi generated cmip metrics data -cmip_enso_dir = string(default="") -cmip_clim_dir = string(default="") -cmip_movs_dir = string(default="") -#group of pcmdi generated cmip metrics data (mip.exp.version) -cmip_clim_set = string(default="cmip6.historical.v20250108") -cmip_movs_set = string(default="cmip6.historical.v20220825") -cmip_enso_set = string(default="cmip6.historical.v20210620") -# options shared by pcmdi -pcmdi_debug = string(default=False) - - [[__many__]] - backend = string(default=None) - cfg = string(default=None) - vars = string(default=None) - grid = string(default=None) - cmip_enso_set = string(default=None) - cmip_clim__set = string(default=None) - cmip_movs_set = string(default=None) - cmip_enso_dir = string(default=None) - cmip_clim_dir = string(default=None) - cmip_movs_dir = string(default=None) - synthetic_sets = string(default=None) - reference_alias = string(default=None) - regions_specs = string(default=None) - pcmdi_webtitle = string(default=None) - pcmdi_version = string(default=None) - pcmdi_zppy_util = string(default=None) - pcmdi_viewer_util = string(default=None) - pcmdi_viewer_template = string(default=None) - synthetic_metrics = string(default=None) - multiprocessing = boolean(default=None) - num_workers = integer(default=None) - obs_ts = string(default=None) - figure_format = string(default=None) - ref_name = string(default=None) - ref_years = string_list(default=None) - ref_start_yr = string(default=None) - ref_final_yr = string(default=None) - reference_data_path = string(default=None) - reference_data_path_ts = string(default=None) - run_type = string(default=None) - sets = string_list(default=None) - swap_test_ref = boolean(default=None) - tag = string(default=None) - ts_years = string_list(default=None) - ts_num_years = integer(default=None) - ts_num_years_ref = integer(default=None) - target_grid = string(default=None) - target_grid_string = string(default=None) - regrid_tool = string(default=None) - regrid_method = string(default=None) - regrid_method_ocn = string(default=None) - obs_sets = string(default=None) - regions = string(default=None) - save_test_clims = string(default=None) - seasons = string(default=None) - RmDomainMean = string(default=None) - EofScaling = string(default=None) - ConvEOF = string(default=None) - CBF = string(default=None) - subsets = string_list(default=None) - landmask = string(default=None) - frequency = string(default=None) - generate_sftlf = string(default=None) - atm_modes = string(default=None) - cpl_modes = string(default=None) - enso_groups = string(default=None) - ModUnitsAdjust = string(default=None) - ObsUnitsAdjust = string(default=None) - model_name = string(default=None) - model_name_ref = string(default=None) - model_tableID = string(default=None) - model_vars = string(default=None) - mov_nc_out_obs = string(default=None) - mov_nc_out_model = string(default=None) - mov_plot_obs = string(default=None) - mov_plot_model = string(default=None) - pcmdi_debug = string(default=None) - [e3sm_diags] # See https://e3sm-project.github.io/e3sm_diags/_build/html/master/available-parameters.html backend = string(default="mpl") @@ -613,3 +390,215 @@ e3sm_to_cmip_land_subsection = string(default="") ilamb_obs = string(default="") # for land_only run land_only = boolean(default=False) + +[pcmdi_diags] +# Title of zppy-pcmdi diagnostics +pcmdi_webtitle = string(default="E3SM-PMP-Diagnostics") # Cannot have spaces! +# Version of zppy-pcmdi code +pcmdi_version = string(default="v3.8.2") +# File of specified regions for mean climate calculation +regions_specs = string(default="pcmdi_data/region/regions_specs.json") +# File of observation data name for mean climate calculation +reference_alias = string(default="pcmdi_data/reference/reference_alias.json") +# Utility file with functions for zppy-pcmdi data processing +pcmdi_zppy_util = string(default="pcmdi_data/utility/pcmdi_zppy_util.py") +# Utility file with functions for zppy-pcmdi viewer page processing +pcmdi_viewer_util = string(default="pcmdi_data/utility/pcmdi_viewer_util.py") +# Template files for zppy-pcmdi viewer page processing +pcmdi_viewer_template = string(default="pcmdi_data/viewer") +# File of variable list to generate synthetic metrics plot +synthetic_metrics = string(default="pcmdi_data/synthetic_metrics/synthetic_metrics_list.json") +# Path to observation time-series data +# Required for "mean_climate","variability_mode","enso" +obs_ts = string(default="") +# observational data sets (see reference_alias.json) +# observation data tag in reference_alias +obs_sets = string(default="default") +# options specific for constructing pcmdi preferred file name conventions +# required for "model_vs_obs" comparison +model_name = string(default="e3sm.historical.v3-LR.0051") +model_tableID = string(default="Amon") +# required for "model_vs_model" comparison +model_name_ref = string(default="ERA5") +model_tableID_ref = string(default="Amon") +# variables in the cmip6 table that can be potentially used by pcmdi +# this list depends on the definition of cmip variable +# required for "mean climate" diagnostics +cmip_vars = string(default="pr,prw,psl,rlds,rldscs,rlut,rlutcs,rsut,rsutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rtmt,sfcWind,tas,tauu,tauv,ts,ta,ua,va,zg") +# flag to process the land/sea mask within pcmdi +generate_sftlf = string(default=True) +# variables to be used by the pcmdi diagnostics +# needs to setup for each subsections, defalut setup is the mean climate metrics +vars = string(default="pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500") +# sets of diagnostics from pcmdi package +sets = string_list(default=list("mean_climate","variability_modes_atm","variability_modes_cpl","enso","synthetic_plots")) +########################################################################################### +# Required for run_type="model_vs_model" runs, different from e3sm_diag, +# model_vs_model in pcmdi referred to the comparision of two model simulations +# with observations and cmip models. +########################################################################################### +#path for reference model data (time series) +reference_data_path = string(default="") +reference_data_path_ts = string(default="") +# pcmdi_diags.py will set to match `years` if not specified +ref_years = string_list(default=list("")) +# The years increment for reference data +ts_num_years_ref = integer(default=5) +# Final year (i.e., the last available year) for the reference data +ref_final_yr = string(default="") +# Start year for the reference data +ref_start_yr = string(default="") +# reference model name +ref_name = string(default="") +# The years range for test model data +ts_years = string_list(default=list("")) +# The years increment for test model data +ts_num_years = integer(default=5) +########################################################################################## +# below followed the setup in e3sm_diag but used for PCMDI workflow +########################################################################################## +# See url +multiprocessing = boolean(default=True) +# See url +num_workers = integer(default=24) +# See url +figure_format = string(default="png") +# comparision type (same as e3sm_diag) +run_type = string(default="model_vs_obs") +########################################################################################## +# options for pcmdi mode varibility diagnostics +# vars = "psl" for atm_modes +# vars = "ts" for cpl_modes +######################################################################################### +#name of atmospheric modes varibility +atm_modes = string(default="NAM,NAO,PNA,NPO,SAM,PSA1,PSA2") +#name of coupled modes varibility +cpl_modes = string(default="PDO,NPGO,AMO") +#keywards for unit conversion in pcmdi (model) +ModUnitsAdjust = string(default="") +#keywards for unit conversion in pcmdi (observation) +ObsUnitsAdjust = string(default="") +#frequency of the model data +frequency = string(default="mo") +#options specific for mode varibility metrics in pcmdi +seasons = string(default="monthly") +landmask = string(default=False) +RmDomainMean = string(default=True) +EofScaling = string(default=False) +ConvEOF = string(default=True) +CBF = string(default=True) +mov_plot_obs = string(default=True) +mov_plot_model = string(default=True) +mov_nc_out_obs = string(default=True) +mov_nc_out_model = string(default=True) +########################################################################################## +# options for pcmdi enso diagnostics +# vars = "psl,pr,prsn,ts,tas,tauu,tauv,hflx,hfss,rlds,rsds,rlus,rlut,rsdt" +########################################################################################## +enso_groups = string(default="ENSO_perf,ENSO_proc,ENSO_tel") +########################################################################################## +# optional for mean climate diagnostics +# vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs, +# rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200, +# va-850,zg-500" +########################################################################################## +# model data grid after remapping +grid = string(default="180x360_aave") +#default regions for mean climate metrics data +#more options can be found at "regions_specs" +#regions = string(default="global,ocean,land,NHEX,SHEX,TROPICS,NHEX_ocean,SHEX_ocean,NHEX_land,SHEX_land,ocean_50S50N") +regions = string(default="global,ocean,land,NHEX,SHEX,TROPICS") +# save derived climatology data +save_test_clims = string(default=True) +# Regridding by pcmdi (required for mean climate) +# OPTIONS: '1x1' for e3sm, '2.5x2.5' for cmip, or an actual cdms2 grid object, +target_grid = string(default="1x1") +# OPTIONS: "1px1p" for e3sm, "2p5x2p5" for cmip, description on the selected grid +target_grid_string = string(default="1px1p") +# OPTIONS: 'regrid2','esmf' +regrid_tool = string(default="esmf") +# OPTIONS: 'linear','conservative', only if tool is esmf +regrid_method = string(default="regrid2") +# OPTIONS: 'linear','conservative', only if tool is esmf +regrid_method_ocn = string(default="conservative") +################################################################### +#options for synthetic plots with exisiting pcmdi cmip model metrics +################################################################### +synthetic_sets = string(default="portrait,parcoord") +# path to pcmdi generated cmip metrics data +cmip_enso_dir = string(default="") +cmip_clim_dir = string(default="") +cmip_movs_dir = string(default="") +#group of pcmdi generated cmip metrics data (mip.exp.version) +cmip_clim_set = string(default="cmip6.historical.v20250108") +cmip_movs_set = string(default="cmip6.historical.v20220825") +cmip_enso_set = string(default="cmip6.historical.v20210620") +# options shared by pcmdi +pcmdi_debug = string(default=False) +# Name of the `[e3sm_to_cmip]` atm subtask to depend on +e3sm_to_cmip_atm_subsection = string(default="") + + [[__many__]] + vars = string(default=None) + grid = string(default=None) + cmip_enso_set = string(default=None) + cmip_clim__set = string(default=None) + cmip_movs_set = string(default=None) + cmip_enso_dir = string(default=None) + cmip_clim_dir = string(default=None) + cmip_movs_dir = string(default=None) + synthetic_sets = string(default=None) + reference_alias = string(default=None) + regions_specs = string(default=None) + pcmdi_webtitle = string(default=None) + pcmdi_version = string(default=None) + pcmdi_zppy_util = string(default=None) + pcmdi_viewer_util = string(default=None) + pcmdi_viewer_template = string(default=None) + synthetic_metrics = string(default=None) + multiprocessing = boolean(default=None) + num_workers = integer(default=None) + obs_ts = string(default=None) + figure_format = string(default=None) + ref_name = string(default=None) + ref_years = string_list(default=None) + ref_start_yr = string(default=None) + ref_final_yr = string(default=None) + reference_data_path = string(default=None) + reference_data_path_ts = string(default=None) + run_type = string(default=None) + sets = string_list(default=None) + ts_years = string_list(default=None) + ts_num_years = integer(default=None) + ts_num_years_ref = integer(default=None) + target_grid = string(default=None) + target_grid_string = string(default=None) + regrid_tool = string(default=None) + regrid_method = string(default=None) + regrid_method_ocn = string(default=None) + obs_sets = string(default=None) + regions = string(default=None) + save_test_clims = string(default=None) + seasons = string(default=None) + RmDomainMean = string(default=None) + EofScaling = string(default=None) + ConvEOF = string(default=None) + CBF = string(default=None) + landmask = string(default=None) + frequency = string(default=None) + generate_sftlf = string(default=None) + atm_modes = string(default=None) + cpl_modes = string(default=None) + enso_groups = string(default=None) + ModUnitsAdjust = string(default=None) + ObsUnitsAdjust = string(default=None) + model_name = string(default=None) + model_name_ref = string(default=None) + model_tableID = string(default=None) + model_vars = string(default=None) + mov_nc_out_obs = string(default=None) + mov_nc_out_model = string(default=None) + mov_plot_obs = string(default=None) + mov_plot_model = string(default=None) + pcmdi_debug = string(default=None) + e3sm_to_cmip_atm_subsection = string(default=None) diff --git a/zppy/e3sm_diags.py b/zppy/e3sm_diags.py index c6b690ba..0095d66f 100644 --- a/zppy/e3sm_diags.py +++ b/zppy/e3sm_diags.py @@ -6,8 +6,9 @@ from zppy.bundle import handle_bundles from zppy.utils import ( ParameterInferenceType, - ParameterNotProvidedError, add_dependencies, + check_parameter_defined, + check_set_specific_parameter, check_status, get_file_names, get_tasks, @@ -104,32 +105,6 @@ def e3sm_diags(config: ConfigObj, script_dir: str, existing_bundles, job_ids_fil return existing_bundles -def check_parameter_defined( - c: Dict[str, Any], relevant_parameter: str, explanation: str = "" -) -> None: - if (relevant_parameter not in c.keys()) or (c[relevant_parameter] == ""): - if explanation: - message = f"{relevant_parameter} is needed because {explanation}" - else: - message = f"{relevant_parameter} is not defined." - raise ParameterNotProvidedError(message) - - -def check_set_specific_parameter( - c: Dict[str, Any], sets_with_requirement: Set[str], relevant_parameter: str -) -> None: - requested_sets = set(c["sets"]) - intersection = sets_with_requirement & requested_sets - if ( - intersection - and (relevant_parameter in c.keys()) - and (c[relevant_parameter] == "") - ): - raise ParameterNotProvidedError( - f"{relevant_parameter} is required because the sets {intersection} were requested." - ) - - def check_parameters_for_bash(c: Dict[str, Any]) -> None: # Check parameters that aren't used until e3sm_diags.bash is run check_set_specific_parameter(c, set(["qbo"]), "ref_final_yr") diff --git a/zppy/pcmdi_diags.py b/zppy/pcmdi_diags.py index 6cbe9c13..5aab9e9b 100644 --- a/zppy/pcmdi_diags.py +++ b/zppy/pcmdi_diags.py @@ -3,19 +3,19 @@ from zppy.bundle import handle_bundles from zppy.utils import ( - ParameterGuessType, + ParameterInferenceType, add_dependencies, check_parameter_defined, - check_required_parameters, + check_set_specific_parameter, check_status, - define_or_guess, - define_or_guess2, get_file_names, get_tasks, + get_value_from_parameter, get_years, initialize_template, make_executable, print_url, + set_value_of_parameter_if_undefined, submit_script, write_settings_file, ) @@ -34,8 +34,8 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): # --- Generate and submit pcmdi_diags scripts --- for c in tasks: dependencies: List[str] = [] - c["sub"] = define_or_guess( - c, "subsection", "sub", ParameterGuessType.SECTION_GUESS + c["sub"] = get_value_from_parameter( + c, "subsection", "sub", ParameterInferenceType.SECTION_INFERENCE ) check_parameters_for_bash(c) @@ -91,6 +91,21 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): if "ts_num_years" in c.keys(): for yr in range(c["year1"], c["year2"], c["ts_num_years"]): add_ts_dependencies(c, dependencies, script_dir, yr) + set_value_of_parameter_if_undefined( + c, + "e3sm_to_cmip_atm_subsection", + "atm_monthly_180x360_aave", + ParameterInferenceType.SECTION_INFERENCE, + ) + add_dependencies( + dependencies, + script_dir, + "e3sm_to_cmip", + c["e3sm_to_cmip_atm_subsection"], + c["year1"], + c["year2"], + c["ts_num_years"], + ) if c["sub"] == "synthetic_plots": add_pcmdi_dependencies(c, dependencies, script_dir) @@ -129,21 +144,21 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): def check_parameters_for_bash(c: Dict[str, Any]) -> None: if c["sub"] != "synthetic_plots": - check_required_parameters( + check_set_specific_parameter( c, set( ["mean_climate", "variability_mode_cpl", "variability_mode_atm", "enso"] ), "ref_final_yr", ) - check_required_parameters( + check_set_specific_parameter( c, set( ["mean_climate", "variability_mode_cpl", "variability_mode_atm", "enso"] ), "ref_start_yr", ) - check_required_parameters( + check_set_specific_parameter( c, set( ["mean_climate", "variability_mode_cpl", "variability_mode_atm", "enso"] @@ -155,23 +170,23 @@ def check_parameters_for_bash(c: Dict[str, Any]) -> None: def check_parameters_for_pcmdi(c: Dict[str, Any]) -> None: # check and set up the external data needed by pcmdi if c["sub"] == "synthetic_plots": - define_or_guess2( + set_value_of_parameter_if_undefined( c, "cmip_enso_dir", f"{c['diagnostics_base_path']}/pcmdi_data/metrics_data/enso_metric", - ParameterGuessType.PATH_GUESS, + ParameterInferenceType.PATH_INFERENCE, ) - define_or_guess2( + set_value_of_parameter_if_undefined( c, "cmip_clim_dir", f"{c['diagnostics_base_path']}/pcmdi_data/metrics_data/mean_climate", - ParameterGuessType.PATH_GUESS, + ParameterInferenceType.PATH_INFERENCE, ) - define_or_guess2( + set_value_of_parameter_if_undefined( c, "cmip_movs_dir", f"{c['diagnostics_base_path']}/pcmdi_data/metrics_data/variability_modes", - ParameterGuessType.PATH_GUESS, + ParameterInferenceType.PATH_INFERENCE, ) @@ -180,7 +195,7 @@ def check_mvm_only_parameters_for_bash(c: Dict[str, Any]) -> None: check_parameter_defined(c, "model_name_ref") check_parameter_defined(c, "model_tableID_ref") if c["sub"] != "synthetic_plots": - check_required_parameters( + check_set_specific_parameter( c, set( ["mean_climate", "variability_mode_cpl", "variability_mode_atm", "enso"] @@ -195,18 +210,18 @@ def check_mvm_only_parameters_for_bash(c: Dict[str, Any]) -> None: "enso", ] ) - check_required_parameters(c, ts_sets, "ts_num_years_ref") - check_required_parameters(c, ts_sets, "ts_subsection") + check_set_specific_parameter(c, ts_sets, "ts_num_years_ref") + check_set_specific_parameter(c, ts_sets, "ts_subsection") def check_and_define_parameters(c: Dict[str, Any]) -> None: # TODO: do this based on sets, rather than by relying on the user setting ts_num_years if "ts_num_years" in c.keys(): - define_or_guess2( + set_value_of_parameter_if_undefined( c, "obs_ts", f"{c['diagnostics_base_path']}/observations/Atm/time-series/", - ParameterGuessType.PATH_GUESS, + ParameterInferenceType.PATH_INFERENCE, ) prefix: str if c["run_type"] == "model_vs_obs": @@ -220,11 +235,11 @@ def check_and_define_parameters(c: Dict[str, Any]) -> None: if set( ["mean_climate", "variability_mode_cpl", "variability_mode_atm", "enso"] ) & set(c["sets"]): - define_or_guess2( + set_value_of_parameter_if_undefined( c, "reference_data_path_ts", f"{reference_data_path}/atm/{c['grid']}/cmip_ts/monthly", - ParameterGuessType.PATH_GUESS, + ParameterInferenceType.PATH_INFERENCE, ) else: raise ValueError(f"Invalid run_type={c['run_type']}") diff --git a/zppy/templates/coupled_global.py b/zppy/templates/coupled_global.py deleted file mode 100644 index 9fc41401..00000000 --- a/zppy/templates/coupled_global.py +++ /dev/null @@ -1,872 +0,0 @@ -# Script to plot some global atmosphere and ocean time series -import glob -import math -import sys -import traceback -from typing import Any, Dict, List, Optional, Tuple - -import cftime -import matplotlib as mpl -import matplotlib.backends.backend_pdf -import matplotlib.pyplot as plt -import numpy as np -import xarray -from netCDF4 import Dataset -from readTS import TS - -mpl.use("Agg") - - -# ---additional function to get moc time series -def getmoc(dir_in): - files = sorted(glob.glob(dir_in + "mocTimeSeries*.nc")) - nfiles = len(files) - print(dir_in, nfiles, "moc files in total") - var = np.array([]) - time = np.array([]) - for i in range(nfiles): - # Open input file - fin = Dataset(files[i], "r") - time0 = fin["year"][:] - var0 = fin["mocAtlantic26"][:] - for iyear in range(int(time0[0]), int(time0[-1]) + 1): - if i > 0 and iyear <= time[-1]: - print( - "the amoc value for year", - iyear, - "has been included in the moc time series from another moc file", - files[i - 1], - time[-1], - "Skipping...", - ) - else: - imon = np.where(time0 == iyear)[0] - if len(imon) == 12: - var = np.append(var, np.mean(var0[imon])) - time = np.append(time, iyear) - else: - print("error in input file :", files[i]) - - return time, var - - -# ----------------------------------------------------------------------------- -# Function to add horizontal line showing average value over a specified period -def add_line(year, var, year1, year2, ax, format="%4.2f", lw=1, color="b"): - - i1 = (np.abs(year - year1)).argmin() - i2 = (np.abs(year - year2)).argmin() - - tmp = np.average(var[i1 : i2 + 1]) - ax.plot((year[i1], year[i2]), (tmp, tmp), lw=lw, color=color, label="average") - ax.text(ax.get_xlim()[1] + 1, tmp, format % tmp, va="center", color=color) - - return - - -# ----------------------------------------------------------------------------- -# Function to add line showing linear trend over a specified period -def add_trend( - year, - var, - year1, - year2, - ax, - format="%4.2f", - lw=1, - color="b", - verbose=False, - ohc=False, - vol=False, -): - - i1 = (np.abs(year - year1)).argmin() - i2 = (np.abs(year - year2)).argmin() - x = year[i1 : i2 + 1] - y = var[i1 : i2 + 1] - - fit = np.polyfit(x, y, 1) - if verbose: - print(fit) - fit_fn = np.poly1d(fit) - ax.plot(x, fit_fn(x), lw=lw, ls="--", c=color, label="trend") - if ohc: - # Earth radius 6371229. from MPAS-O output files - heat_uptake = fit[0] / (4.0 * math.pi * (6371229.0) ** 2 * 365.0 * 86400.0) - ax.text( - ax.get_xlim()[1] + 1, - fit_fn(x[-1]), - "%+4.2f W m$^{-2}$" % (heat_uptake), - color=color, - ) - if vol: - # Earth radius 6371229. from MPAS-O output files - # sea_lvl = fit[0] / ( 4.0*math.pi*(6371229.)**2*0.7) #for oceanic portion of the Earth surface - ax.text( - ax.get_xlim()[1] + 1, - fit_fn(x[-1]), - "%+5.4f mm yr$^{-1}$" % (fit[0]), - color=color, - ) - - return - - -# ----------------------------------------------------------------------------- -# Function to get ylim -def get_ylim(standard_range, extreme_values): - if len(extreme_values) > 0: - has_extreme_values = True - extreme_min = np.amin(extreme_values) - extreme_max = np.amax(extreme_values) - else: - has_extreme_values = False - extreme_min = None - extreme_max = None - if len(standard_range) == 2: - has_standard_range = True - standard_min = standard_range[0] - standard_max = standard_range[1] - else: - has_standard_range = False - standard_min = None - standard_max = None - if has_extreme_values and has_standard_range: - # Use at least the standard range, - # perhaps a wider window to include extremes - if standard_min <= extreme_min: - ylim_min = standard_min - else: - ylim_min = extreme_min - if standard_max >= extreme_max: - ylim_max = standard_max - else: - ylim_max = extreme_max - elif has_extreme_values and not has_standard_range: - ylim_min = extreme_min - ylim_max = extreme_max - elif has_standard_range and not has_extreme_values: - ylim_min = standard_min - ylim_max = standard_max - else: - raise ValueError("Not enough range information supplied") - return [ylim_min, ylim_max] - - -# ----------------------------------------------------------------------------- -# Plotting functions - - -# 1 -def plot_net_toa_flux_restom(ax, xlim, exps, rgn): - print("Plot 1: plot_net_toa_flux_restom") - param_dict = { - "2nd_var": False, - "axhline_y": 0, - "check_exp_ocean": False, - "check_exp_vol": False, - "check_exp_year": True, - "default_ylim": [-1.5, 1.5], - "do_add_line": True, - "do_add_trend": True, - "format": "%4.2f", - "glb_only": False, - "lw": 1.0, - "ohc": False, - "set_axhline": True, - "set_legend": True, - "shorten_year": False, - "title": "Net TOA flux (restom)", - "use_getmoc": False, - "var": lambda exp: np.array(exp["annual"]["RESTOM"][0]), - "verbose": False, - "vol": False, - "ylabel": "W m-2", - } - plot(ax, xlim, exps, param_dict, rgn) - - -# 2 -def plot_global_surface_air_temperature(ax, xlim, exps, rgn): - print("Plot 2: plot_global_surface_air_temperature") - if rgn == "glb": - region_title = "Global" - elif rgn == "n": - region_title = "Northern Hemisphere" - elif rgn == "s": - region_title = "Southern Hemisphere" - else: - raise RuntimeError(f"Invalid rgn={rgn}") - param_dict = { - "2nd_var": False, - "axhline_y": None, - "check_exp_ocean": False, - "check_exp_vol": False, - "check_exp_year": True, - "default_ylim": [13, 15.5], - "do_add_line": True, - "do_add_trend": True, - "format": "%4.2f", - "glb_only": False, - "lw": 1.0, - "ohc": False, - "set_axhline": False, - "set_legend": True, - "shorten_year": False, - "title": f"{region_title} surface air temperature", - "use_getmoc": False, - "var": lambda exp: np.array(exp["annual"]["TREFHT"][0]) - 273.15, - "verbose": False, - "vol": False, - "ylabel": "degC", - } - plot(ax, xlim, exps, param_dict, rgn) - - -# 3 -def plot_toa_radiation(ax, xlim, exps, rgn): - print("Plot 3: plot_toa_radiation") - param_dict = { - "2nd_var": True, - "axhline_y": None, - "check_exp_ocean": False, - "check_exp_vol": False, - "check_exp_year": False, - "default_ylim": [235, 245], - "do_add_line": False, - "do_add_trend": False, - "format": None, - "glb_only": False, - "lw": 1.0, - "ohc": False, - "set_axhline": False, - "set_legend": False, - "shorten_year": False, - "title": "TOA radiation: SW (solid), LW (dashed)", - "use_getmoc": False, - "var": lambda exp: np.array(exp["annual"]["FSNTOA"][0]), - "verbose": None, - "vol": None, - "ylabel": "W m-2", - } - plot(ax, xlim, exps, param_dict, rgn) - - -# 4 -def plot_net_atm_energy_imbalance(ax, xlim, exps, rgn): - print("Plot 4: plot_net_atm_energy_imbalance") - param_dict = { - "2nd_var": False, - "axhline_y": None, - "check_exp_ocean": False, - "check_exp_vol": False, - "check_exp_year": True, - "default_ylim": [-0.3, 0.3], - "do_add_line": True, - "do_add_trend": False, - "format": "%4.2f", - "glb_only": False, - "lw": 1.0, - "ohc": False, - "set_axhline": False, - "set_legend": True, - "shorten_year": False, - "title": "Net atm energy imbalance (restom-ressurf)", - "use_getmoc": False, - "var": lambda exp: np.array(exp["annual"]["RESTOM"][0]) - - np.array(exp["annual"]["RESSURF"][0]), - "verbose": False, - "vol": False, - "ylabel": "W m-2", - } - plot(ax, xlim, exps, param_dict, rgn) - - -# 5 -def plot_change_ohc(ax, xlim, exps, rgn): - print("Plot 5: plot_change_ohc") - param_dict = { - "2nd_var": False, - "axhline_y": 0, - "check_exp_ocean": True, - "check_exp_vol": False, - "check_exp_year": False, - "default_ylim": [-0.3e24, 0.9e24], - "do_add_line": False, - "do_add_trend": True, - "format": "%4.2f", - "glb_only": True, - "lw": 1.5, - "ohc": True, - "set_axhline": True, - "set_legend": True, - "shorten_year": True, - "title": "Change in ocean heat content", - "use_getmoc": False, - "var": lambda exp: np.array(exp["annual"]["ohc"]), - "verbose": False, - "vol": False, - "ylabel": "J", - } - plot(ax, xlim, exps, param_dict, rgn) - - -# 6 -def plot_max_moc(ax, xlim, exps, rgn): - print("Plot 6: plot_max_moc") - param_dict = { - "2nd_var": False, - "axhline_y": 10, - "check_exp_ocean": False, - "check_exp_vol": False, - "check_exp_year": False, - "default_ylim": [4, 22], - "do_add_line": False, - "do_add_trend": True, - "format": "%4.2f", - "glb_only": True, - "lw": 1.5, - "ohc": False, - "set_axhline": True, - "set_legend": True, - "shorten_year": False, - "title": "Max MOC Atlantic streamfunction at 26.5N", - "use_getmoc": True, - "var": None, - "verbose": True, - "vol": None, - "ylabel": "Sv", - } - plot(ax, xlim, exps, param_dict, rgn) - - -# 7 -def plot_change_sea_level(ax, xlim, exps, rgn): - print("Plot 7: plot_change_sea_level") - param_dict = { - "2nd_var": False, - "axhline_y": None, - "check_exp_ocean": False, - "check_exp_vol": True, - "check_exp_year": True, - "default_ylim": [4, 22], - "do_add_line": False, - "do_add_trend": True, - "format": "%5.3f", - "glb_only": True, - "lw": 1.5, - "ohc": False, - "set_axhline": False, - "set_legend": True, - "shorten_year": True, - "title": "Change in sea level", - "use_getmoc": False, - "var": lambda exp: ( - 1e3 - * np.array(exp["annual"]["volume"]) - / (4.0 * math.pi * (6371229.0) ** 2 * 0.7) - ), - "verbose": True, - "vol": True, - "ylabel": "mm", - } - plot(ax, xlim, exps, param_dict, rgn) - - -# 8 -def plot_net_atm_water_imbalance(ax, xlim, exps, rgn): - print("Plot 8: plot_net_atm_water_imbalance") - param_dict = { - "2nd_var": False, - "axhline_y": None, - "check_exp_ocean": False, - "check_exp_vol": False, - "check_exp_year": False, - "default_ylim": [-1, 1], - "do_add_line": True, - "do_add_trend": False, - "format": "%5.4f", - "glb_only": False, - "lw": 1.0, - "ohc": False, - "set_axhline": False, - "set_legend": True, - "shorten_year": False, - "title": "Net atm water imbalance (evap-prec)", - "use_getmoc": False, - "var": lambda exp: ( - 365 - * 86400 - * ( - np.array(exp["annual"]["QFLX"][0]) - - 1e3 - * ( - np.array(exp["annual"]["PRECC"][0]) - + np.array(exp["annual"]["PRECL"][0]) - ) - ) - ), - "verbose": False, - "vol": False, - "ylabel": "mm yr-1", - } - plot(ax, xlim, exps, param_dict, rgn) - - -# Generic plot function -def plot_generic(ax, xlim, exps, var_name, rgn): - print(f"plot_generic for {var_name}") - param_dict = { - "2nd_var": False, - "axhline_y": 0, - "check_exp_ocean": False, - "check_exp_vol": False, - "check_exp_year": True, - "default_ylim": [], - "do_add_line": True, - "do_add_trend": True, - "format": "%4.2f", - "glb_only": False, - "lw": 1.0, - "ohc": False, - "set_axhline": False, - "set_legend": True, - "shorten_year": False, - "title": var_name, - "use_getmoc": False, - "var": lambda exp: np.array(exp["annual"][var_name][0]), - "verbose": False, - "vol": False, - "ylabel": lambda exp: np.array(exp["annual"][var_name][1]), - } - plot(ax, xlim, exps, param_dict, rgn) - - -# FIXME: C901 'plot' is too complex (19) -def plot(ax, xlim, exps, param_dict, rgn): # noqa: C901 - if param_dict["glb_only"] and (rgn != "glb"): - return - ax.set_xlim(xlim) - extreme_values = [] - for exp in exps: - # Relevant to "Plot 5: plot_change_ohc" - if param_dict["check_exp_ocean"] and (exp["ocean"] is None): - continue - # Relevant to "Plot 7: plot_change_sea_level" - # This must be checked before plot 6, - # otherwise, `param_dict["var"]` will be run, - # but `exp["annual"]["volume"]` won't exist. - if param_dict["check_exp_vol"] and (exp["vol"] is None): - continue - # Relevant to "Plot 6: plot_max_moc" - if param_dict["use_getmoc"]: - if exp["moc"]: - [year, var] = getmoc(exp["moc"]) - else: - continue - else: - year = np.array(exp["annual"]["year"]) + exp["yoffset"] - var = param_dict["var"](exp) - extreme_values.append(np.amax(var)) - extreme_values.append(np.amin(var)) - if param_dict["shorten_year"]: - year = year[: len(var)] - try: - ax.plot( - year, - var, - lw=param_dict["lw"], - marker=None, - c=exp["color"], - label=exp["name"], - ) - except Exception: - raise RuntimeError(f"{param_dict['title']} could not be plotted.") - if param_dict["2nd_var"]: - # Specifically for plot_toa_radiation - # TODO: if more plots require a 2nd variable, we can change `var` to be a list, - # but that will be a more significant refactoring. - var = np.array(exp["annual"]["FLUT"][0]) - ax.plot(year, var, lw=1.0, marker=None, ls=":", c=exp["color"]) - continue - if param_dict["check_exp_year"] and exp["yr"] is None: - continue - elif param_dict["do_add_line"] or param_dict["do_add_trend"]: - for yrs in exp["yr"]: - if param_dict["do_add_line"]: - add_line( - year, - var, - yrs[0], - yrs[1], - format=param_dict["format"], - ax=ax, - lw=2 * param_dict["lw"], - color=exp["color"], - ) - if param_dict["do_add_trend"]: - add_trend( - year, - var, - yrs[0], - yrs[1], - format=param_dict["format"], - ax=ax, - lw=2 * param_dict["lw"], - color=exp["color"], - ohc=param_dict["ohc"], - verbose=param_dict["verbose"], - vol=param_dict["vol"], - ) - ylim = get_ylim(param_dict["default_ylim"], extreme_values) - ax.set_ylim(ylim) - if param_dict["set_axhline"]: - ax.axhline(y=param_dict["axhline_y"], lw=1, c="0.5") - ax.set_title(param_dict["title"]) - ax.set_xlabel("Year") - units = param_dict["ylabel"] - c = callable(units) - if c: - units = units(exps[0]) - ax.set_ylabel(units) - if param_dict["set_legend"]: - ax.legend(loc="best") - - -PLOT_DICT = { - "net_toa_flux_restom": plot_net_toa_flux_restom, - "global_surface_air_temperature": plot_global_surface_air_temperature, - "toa_radiation": plot_toa_radiation, - "net_atm_energy_imbalance": plot_net_atm_energy_imbalance, - "change_ohc": plot_change_ohc, # only glb - "max_moc": plot_max_moc, # only glb - "change_sea_level": plot_change_sea_level, # only glb - "net_atm_water_imbalance": plot_net_atm_water_imbalance, -} - - -def param_get_list(param_value): - if param_value == "None": - return [] - else: - return param_value.split(",") - - -def set_var( - exp: Dict[str, Any], - exp_key: str, - var_list: List[str], - valid_vars: List[str], - invalid_vars: List[str], - rgn: str, -) -> None: - if exp[exp_key] is not None: - ts = TS(exp[exp_key]) - for var in var_list: - try: - v: xarray.core.dataarray.DataArray - units: Optional[str] - v, units = ts.globalAnnual(var) - valid_vars.append(str(var)) - except Exception as e: - print(e) - print(f"globalAnnual failed. Invalid var = {var}") - invalid_vars.append(str(var)) - continue - if v.sizes["rgn"] > 1: - # number of years x 3 regions = v.shape - # 3 regions = global, northern hemisphere, southern hemisphere - # We get here if we used the updated `ts` task - # (using `rgn_avg` rather than `glb_avg`). - if rgn == "glb": - n = 0 - elif rgn == "n": - n = 1 - elif rgn == "s": - n = 2 - else: - raise RuntimeError(f"Invalid rgn={rgn}") - v = v.isel(rgn=n) # Just use nth region - elif rgn != "glb": - # v only has one dimension -- glb. - # Therefore it is not possible to get n or s plots. - raise RuntimeError( - f"var={var} only has global data. Cannot process rgn={rgn}" - ) - exp["annual"][var] = (v, units) - if "year" not in exp["annual"]: - years: np.ndarray[cftime.DatetimeNoLeap] = v.coords["time"].values - exp["annual"]["year"] = [x.year for x in years] - del ts - - -def make_plot_pdfs( - figstr, rgn, component, xlim, exps, plot_list, valid_plots, invalid_plots -): - num_plots = len(plot_list) - if num_plots == 0: - return - nrows = 4 - ncols = 2 - plots_per_page = nrows * ncols - num_pages = math.ceil(num_plots / plots_per_page) - - counter = 0 - # https://stackoverflow.com/questions/58738992/save-multiple-figures-with-subplots-into-a-pdf-with-multiple-pages - pdf = matplotlib.backends.backend_pdf.PdfPages(f"{figstr}_{rgn}_{component}.pdf") - for page in range(num_pages): - fig = plt.figure(1, figsize=[13.5, 16.5]) - fig.suptitle(f"{figstr}_{rgn}_{component}") - for j in range(plots_per_page): - # The final page doesn't need to be filled out with plots. - if counter >= num_plots: - break - ax = plt.subplot(nrows, ncols, j + 1) - if component == "original": - try: - plot_function = PLOT_DICT[plot_list[counter]] - except KeyError: - raise KeyError(f"Invalid plot name: {plot_list[counter]}") - try: - plot_function(ax, xlim, exps, rgn) - valid_plots.append(plot_list[counter]) - except Exception: - traceback.print_exc() - plot_name = plot_list[counter] - required_vars = [] - if plot_name == "net_toa_flux_restom": - required_vars = ["RESTOM"] - elif plot_name == "net_atm_energy_imbalance": - required_vars = ["RESTOM", "RESSURF"] - elif plot_name == "global_surface_air_temperature": - required_vars = ["TREFHT"] - elif plot_name == "toa_radiation": - required_vars = ["FSNTOA", "FLUT"] - elif plot_name == "net_atm_water_imbalance": - required_vars = ["PRECC", "PRECL", "QFLX"] - print( - f"Failed plot_function for {plot_name}. Check that {required_vars} are available." - ) - invalid_plots.append(plot_name) - counter += 1 - else: - try: - plot_name = plot_list[counter] - plot_generic(ax, xlim, exps, plot_name, rgn) - valid_plots.append(plot_name) - except Exception: - traceback.print_exc() - print(f"plot_generic failed. Invalid plot={plot_name}") - invalid_plots.append(plot_name) - counter += 1 - - fig.tight_layout() - pdf.savefig(1) - if num_pages > 1: - fig.savefig(f"{figstr}_{rgn}_{component}_{page}.png", dpi=150) - else: - fig.savefig(f"{figstr}_{rgn}_{component}.png", dpi=150) - plt.clf() - pdf.close() - - -# ----------------------------------------------------------------------------- -# FIXME: C901 'run' is too complex (19) -def run(parameters, rgn): # noqa: C901 - # These are the "Tableau 20" colors as RGB. - t20: List[Tuple[float, float, float]] = [ - (31, 119, 180), - (174, 199, 232), - (255, 127, 14), - (255, 187, 120), - (44, 160, 44), - (152, 223, 138), - (214, 39, 40), - (255, 152, 150), - (148, 103, 189), - (197, 176, 213), - (140, 86, 75), - (196, 156, 148), - (227, 119, 194), - (247, 182, 210), - (127, 127, 127), - (199, 199, 199), - (188, 189, 34), - (219, 219, 141), - (23, 190, 207), - (158, 218, 229), - ] - # Scale the RGB values to the [0, 1] range, which is the format matplotlib accepts. - for i in range(len(t20)): - r, g, b = t20[i] - t20[i] = (r / 255.0, g / 255.0, b / 255.0) - - # "Tableau 10" uses every other color - t10 = [] - for i in range(0, len(t20), 2): - t10.append(t20[i]) - - # ----------------------------------------------------------------------------- - # --- Atmos data --- - - # Experiments - case_dir = parameters[1] - experiment_name = parameters[2] - figstr = parameters[3] - year1 = int(parameters[4]) - year2 = int(parameters[5]) - color = parameters[6] - ts_num_years = parameters[7] - plots_original = param_get_list(parameters[8]) - if parameters[9].lower() == "false": - atmosphere_only = False - else: - atmosphere_only = True - plots_atm = param_get_list(parameters[10]) - plots_ice = param_get_list(parameters[11]) - plots_lnd = param_get_list(parameters[12]) - plots_ocn = param_get_list(parameters[13]) - vars_original = [] - if "net_toa_flux_restom" or "net_atm_energy_imbalance" in plots_original: - vars_original.append("RESTOM") - if "net_atm_energy_imbalance" in plots_original: - vars_original.append("RESSURF") - if "global_surface_air_temperature" in plots_original: - vars_original.append("TREFHT") - if "toa_radiation" in plots_original: - vars_original.append("FSNTOA") - vars_original.append("FLUT") - if "net_atm_water_imbalance" in plots_original: - vars_original.append("PRECC") - vars_original.append("PRECL") - vars_original.append("QFLX") - use_atmos = plots_atm or plots_original - has_original_ocn_plots = ( - ("change_ohc" in plots_original) - or ("max_moc" in plots_original) - or ("change_sea_level" in plots_original) - ) - use_ocn = plots_ocn or (not atmosphere_only and has_original_ocn_plots) - exps: List[Dict[str, Any]] = [ - { - "atmos": ( - f"{case_dir}/post/atm/glb/ts/monthly/{ts_num_years}yr/" - if use_atmos - else None - ), - "ice": ( - f"{case_dir}/post/ice/glb/ts/monthly/{ts_num_years}yr/" - if plots_ice - else None - ), - "land": ( - f"{case_dir}/post/lnd/glb/ts/monthly/{ts_num_years}yr/" - if plots_lnd - else None - ), - "ocean": ( - f"{case_dir}/post/ocn/glb/ts/monthly/{ts_num_years}yr/" - if use_ocn - else None - ), - "moc": ( - f"{case_dir}/post/ocn/glb/ts/monthly/{ts_num_years}yr/" - if use_ocn - else None - ), - "vol": ( - f"{case_dir}/post/ocn/glb/ts/monthly/{ts_num_years}yr/" - if use_ocn - else None - ), - "name": experiment_name, - "yoffset": 0.0, - "yr": ([year1, year2],), - "color": f"{color}", - } - ] - - valid_vars: List[str] = [] - invalid_vars: List[str] = [] - - # Read data - exp: Dict[str, Any] - for exp in exps: - exp["annual"] = {} - - # Use vars_original rather than plots_original, - # since the plots have different names than the variables - set_var(exp, "atmos", vars_original, valid_vars, invalid_vars, rgn) - set_var(exp, "atmos", plots_atm, valid_vars, invalid_vars, rgn) - set_var(exp, "ice", plots_ice, valid_vars, invalid_vars, rgn) - set_var(exp, "land", plots_lnd, valid_vars, invalid_vars, rgn) - set_var(exp, "ocean", plots_ocn, valid_vars, invalid_vars, rgn) - - # Optionally read ohc - if exp["ocean"] is not None: - ts = TS(exp["ocean"]) - exp["annual"]["ohc"], _ = ts.globalAnnual("ohc") - # annomalies with respect to first year - exp["annual"]["ohc"][:] = exp["annual"]["ohc"][:] - exp["annual"]["ohc"][0] - - if exp["vol"] is not None: - ts = TS(exp["vol"]) - exp["annual"]["volume"], _ = ts.globalAnnual("volume") - # annomalies with respect to first year - exp["annual"]["volume"][:] = ( - exp["annual"]["volume"][:] - exp["annual"]["volume"][0] - ) - - print( - f"{rgn} region globalAnnual was computed successfully for these variables: {valid_vars}" - ) - print( - f"{rgn} region globalAnnual could not be computed for these variables: {invalid_vars}" - ) - - # ----------------------------------------------------------------------------- - # --- Generate plots --- - - xlim = [float(year1), float(year2)] - - valid_plots: List[str] = [] - invalid_plots: List[str] = [] - - make_plot_pdfs( - figstr, rgn, "original", xlim, exps, plots_original, valid_plots, invalid_plots - ) - make_plot_pdfs( - figstr, rgn, "atm", xlim, exps, plots_atm, valid_plots, invalid_plots - ) - make_plot_pdfs( - figstr, rgn, "ice", xlim, exps, plots_ice, valid_plots, invalid_plots - ) - make_plot_pdfs( - figstr, rgn, "lnd", xlim, exps, plots_lnd, valid_plots, invalid_plots - ) - make_plot_pdfs( - figstr, rgn, "ocn", xlim, exps, plots_ocn, valid_plots, invalid_plots - ) - - print(f"These {rgn} region plots generated successfully: {valid_plots}") - print( - f"These {rgn} region plots could not be generated successfully: {invalid_plots}" - ) - - -def run_by_region(parameters): - regions = parameters[14].split(",") - for rgn in regions: - if rgn.lower() in ["glb", "global"]: - rgn = "glb" - elif rgn.lower() in ["n", "north", "northern"]: - rgn = "n" - elif rgn.lower() in ["s", "south", "southern"]: - rgn = "s" - else: - raise RuntimeError(f"Invalid rgn={rgn}") - run(parameters, rgn) - - -if __name__ == "__main__": - run_by_region(sys.argv) diff --git a/zppy/templates/e3sm_to_cmip.bash b/zppy/templates/e3sm_to_cmip.bash index ac30d992..7e0638f7 100644 --- a/zppy/templates/e3sm_to_cmip.bash +++ b/zppy/templates/e3sm_to_cmip.bash @@ -19,21 +19,47 @@ cat > default_metadata.json << EOF {% include cmip_metadata %} EOF { - export cmortables_dir={{ cmor_tables_prefix }}/e3sm_to_cmip_data/cmip6-cmor-tables/Tables + export cmortables_dir={{ cmor_tables_prefix }}/cmip6-cmor-tables/Tables input_dir=${dest}/{{ '%04d' % (yr_start) }}_{{ '%04d' % (yr_end) }} mkdir -p $input_dir cp -s $dest/*_{{ '%04d' % (yr_start) }}??_{{ '%04d' % (yr_end) }}??.nc $input_dir - dest_cmip={{ output }}/post/{{ component }}/{{ ts_grid }}/cmip_ts/{{ frequency }} + dest_cmip={{ output }}/post/{{ component }}/{{ grid }}/cmip_ts/{{ frequency }} mkdir -p ${dest_cmip} + {% if input_files.split(".")[0] == 'cam' or input_files.split(".")[0] == 'eam' -%} + #add code to do vertical interpolation variables at model levels before e3sm_to_cmip + IFS=',' read -ra mlvars <<< "{{ interp_vars }}" + for var in "${mlvars[@]}" + do + for file in ${input_dir}/${var}_{{ '%04d' % (yr_start) }}??_{{ '%04d' % (yr_end) }}??.nc + do + if [ -f ${file} ]; then + #ncks --rgr xtr_mth=mss_val --vrt_fl='{{cmip_plevdata}}' ${file} ${file}.plev + ncremap -p mpi --vrt_ntp=log --vrt_xtr=mss_val --vrt_out='{{cmip_plevdata}}' ${file} ${file}.plev + #overwrite the model level data + mv ${file}.plev ${file} + fi + done + done + {% endif -%} + + #call e3sm_to_cmip srun -N 1 e3sm_to_cmip \ --output-path \ ${dest_cmip}/${tmp_dir} \ + {% if input_files.split(".")[0] == 'clm2' or input_files.split(".")[0] == 'elm' -%} + --var-list \ + 'snd, mrsos, mrso, mrfso, mrros, mrro, prveg, evspsblveg, evspsblsoi, tran, tsl, lai, cLitter, cProduct, cSoilFast, cSoilMedium, cSoilSlow, fFire, fHarvest, cVeg, nbp, gpp, ra, rh' \ + --realm \ + lnd \ + {% endif -%} + {% if input_files.split(".")[0] == 'cam' or input_files.split(".")[0] == 'eam' -%} --var-list \ - {{ cmip_vars }} \ + 'ua, va, ta, wa, zg, hur, pr, prc, prsn, ts, tas, prw, psl, sfcWind, tasmax, tasmin, tauu, tauv, rtmt, rsdt, rsds, rsdscs, rlds, rldscs, rsus, rsuscs, rsut, rsutcs, rlus, rlut, rlutcs, clivi, clwvi, clt, evspsbl, hfls, hfss, huss' \ --realm \ - {{ component }} \ + atm \ + {% endif -%} --input-path \ ${input_dir} \ --user-metadata \ diff --git a/zppy/templates/global_time_series.bash b/zppy/templates/global_time_series.bash old mode 100755 new mode 100644 diff --git a/zppy/templates/inclusions/e3sm_to_cmip/default_metadata.json b/zppy/templates/inclusions/e3sm_to_cmip/default_metadata.json old mode 100755 new mode 100644 diff --git a/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc b/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc deleted file mode 100755 index 2b71cc296ff2f207b80961668af618cecf911c7e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1260 zcma)6QE${Z5cZaBizpJDctk=Lssx-iah$ZkwN@t*Mpq+L{V6^u3af0v<-VB`cL(xUBVs*tzuH z0qVP%tWb-(pW}#cM?4;cJG<#GHJ^x5gIn+HJTxkagxyU;mFvnHJW7_m_Vd-UM4Q%iQwCt(YJ@xiP%r~ zPM;HIvIun;B?%9r_7}kynhac#OWu}t>SQ@PISbLmpRm%|XDah&JlC#bGAk6*rd;@3 zddZ|O7@3f+Tug2%9NsF!8oxMtWT6?B0vp4WsW_`S`~NN9^vEm<;fmNHP@Mm` zT~_F_FKX$uoXl0lYUhPhg)9Z8ln>6zO3odoN2m27PmS{F&WIjCnI#mmqILUcpc~;< zIG{K0+d`Fcs^}d$^S&H1W-=RA$I#HeM^kv;cVw#lY%vKlTc8orEsV1U^y8dC`Nt3E z$b(kV{hM^&0qIAi^WZ$``~(x{-=~E9^N^4~?-H_rsUklRa!3go{z1r%%Y "${prefix}.status" + exit "${error_num}" + fi + echo "ncrcat successful" # Ensure time has calendar attribute if ! ncks -m "${combined_name}" | grep -q "calendar"; then echo "Adding missing calendar attribute to time..." ncatted -a calendar,time,o,c,"standard" "${combined_name}" + if [[ $? -ne 0 ]]; then + cd "${script_dir}" || exit + echo "ERROR (${error_num})" > "${prefix}.status" + exit "${error_num}" + fi + echo "ncatted successful" fi # Add time bounds @@ -367,14 +415,21 @@ create_links_ts_obs() { local cmdfix3='time_bnds@units=time@units' local cmdfix4='time_bnds@calendar=time@calendar' ncap2 -O -h -s "${cmdfix1};${cmdfix2};${cmdfix3};${cmdfix4}" "${combined_name}" "${combined_name}" - if [[ $? -ne 0 ]]; then cd "${script_dir}" || exit echo "ERROR (${error_num})" > "${prefix}.status" exit "${error_num}" fi + echo "ncap2 successful" done + if [ -z "$( ls . )" ]; then + echo "create_links_ts_obs: ${ts_dir_destination} was not updated!" + cd "${script_dir}" || exit + echo "ERROR (${error_num})" > "${prefix}.status" + exit "${error_num}" + fi + cd .. } {% endif %} @@ -387,6 +442,7 @@ create_links_ts_obs() { # Define output directory for climatology files climo_dir_primary="climo" # Path to model's monthly climatology files +# TODO: how do we have cmip_ts under climo?? climo_dir_source="{{ output }}/post/atm/{{ grid }}/cmip_ts/monthly" # Link and process primary model climo data create_links_acyc_climo "${climo_dir_source}" "${climo_dir_primary}" "${Y1}" "${Y2}" "${model_name}.${tableID}" 1 @@ -403,7 +459,7 @@ create_links_acyc_climo "${climo_dir_source_ref}" "${climo_dir_ref}" "${ref_Y1}" # All diagnostics in this subsection use time series (ts) data # Define output directory for primary model time series ts_dir_primary="ts" -ts_dir_source="{{ output }}/post/atm/{{ grid }}/cmip_ts/monthly" +ts_dir_source="{{ output }}/post/atm/{{ ts_grid }}/cmip_ts/monthly" # Create local links and combine time series NetCDF files for the primary model create_links_ts "${ts_dir_source}" "${ts_dir_primary}" "${Y1}" "${Y2}" "${model_name}.${tableID}" 3 {% if run_type == "model_vs_model" %} @@ -425,59 +481,17 @@ obstmp_dir="obs_link" mkdir -p "${obstmp_dir}" echo "Linking observational data into ${obstmp_dir}..." -cat > link_observation.py << EOF -import os -import re -import glob -import json - -try: - from pcmdi_zppy_util import ObservationLinker -except ImportError as e: - raise ImportError("Module 'pcmdi_zppy_util' not found. Make sure it's installed and accessible.") from e - -# Inputs populated via templating -MODEL_NAME = '${model_name_ref}.${tableID_ref}' -VARIABLES = '{{ vars }}'.split(",") -OBS_SETS = '{{ obs_sets }}'.split(",") -OBS_TS_DIR = '{{ obs_ts }}' -OBS_TMP_DIR = '${obstmp_dir}' -OBS_ALIAS_FILE = "reference_alias.json" - -# Mapping from observational variable names to CMIP-standard -ALT_OBS_MAP = { - "pr": "PRECT", - "sst": "ts", - "sfcWind": "si10", - "taux": "tauu", - "tauy": "tauv", - "rltcre": "toa_cre_lw_mon", - "rstcre": "toa_cre_sw_mon", - "rtmt": "toa_net_all_mon" -} - -linker = ObservationLinker( - model_name=MODEL_NAME, - variables=VARIABLES, - obs_sets=OBS_SETS, - ts_dir_ref_source=OBS_TS_DIR, - obstmp_dir=OBS_TMP_DIR, - altobs_dic=ALT_OBS_MAP, - obs_alias_file=OBS_ALIAS_FILE -) - -linker.link_obs_data() -linker.process_derived_variables() -EOF - ################### # Run process job ################### echo "Linking observational data using SLURM..." -command="srun -N 1 python -u link_observation.py" +command="zi-pcmdi-link-observation --model_name_ref ${model_name_ref} --tableID_ref ${tableID_ref} --vars={{ vars }} --obs_sets {{ obs_sets }} --obs_ts {{ obs_ts }} --obstmp_dir ${obstmp_dir}" echo "Running: ${command}" + +source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-20250529 time eval "${command}" +source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh if [ $? -ne 0 ]; then cd {{ scriptDir }} @@ -505,6 +519,9 @@ create_links_ts_obs "${ts_dir_ref_source}" "${ts_dir_ref}" ${ref_Y1} ${ref_Y2} 8 ######################################################## # generate basic parameter file for pcmdi metrics driver ######################################################## +echo "About to create parameterfile.py, which will be passed in with -p" +echo "The current directory is: $PWD" # This will be of the form .../post/scripts/tmpDir + cat > parameterfile.py << EOF import os import sys @@ -561,7 +578,7 @@ regions_values = { # Template path for land/sea mask file (fixed input) modpath_lf = os.path.join( 'fixed', - 'sftlf.%(model).nc' + 'sftlf.put_model_here.nc' ) {% if "mean_climate" in subsection %} @@ -602,7 +619,7 @@ test_data_path = '${climo_dir_primary}' filename_template = '.'.join([ mip, exp, - '%(model)', + 'put_model_here', '%(realization)', '${tableID}', '%(variable)', @@ -706,7 +723,7 @@ modnames = [product] realization = "*" modpath = os.path.join( '${ts_dir_primary}', - '{}.{}.%(model).%(realization).{}.%(variable).{}.nc'.format( + '{}.{}.put_model_here.%(realization).{}.%(variable).{}.nc'.format( mip, exp, '${tableID}', period ) ) @@ -731,7 +748,7 @@ realization = realm modpath = os.path.join( '${ts_dir_primary}', - '{}.{}.%(model).%(realization).{}.%(variable).{}.nc'.format( + '{}.{}.put_model_here.%(realization).{}.%(variable).{}.nc'.format( mip, exp, '${tableID}', period ) ) @@ -759,13 +776,14 @@ results_dir = os.path.join( ) # Output filenames for JSON and NetCDF -json_name = "%(mip)_%(exp)_%(metricsCollection)_${case_id}_%(model)_%(realization)" +json_name = "%(mip)_%(exp)_%(metricsCollection)_${case_id}_put_model_here_%(realization)" netcdf_name = json_name {% endif %} EOF + {% endif %} ################################################################ @@ -773,466 +791,57 @@ EOF echo echo ===== RUN PCMDI DIAGS ===== echo -# Prepare configuration file -cat > pcmdi.py << EOF -import os -import glob -import re -import json -import time -import datetime -import xcdat as xc -import numpy as np -import pandas as pd -import shutil - -import collections -from collections import OrderedDict - -import psutil -import subprocess -from itertools import chain -from subprocess import Popen, PIPE, call - -import pcmdi_metrics - -from pcmdi_metrics.graphics import ( - normalize_by_median, -) - -from pcmdi_zppy_util import( - count_child_processes, - run_serial_jobs, - run_parallel_jobs, - derive_missing_variable, - save_variable_regions, - generate_mean_clim_cmds, - generate_varmode_cmds, - build_enso_obsvar_catalog, - build_enso_obsvar_landmask, - generate_enso_cmds, - shift_row_to_bottom, - check_badvals, - archive_data, - drop_vars, - enso_plot_driver, - variability_modes_plot_driver, - mean_climate_plot_driver, - parcoord_metric_plot, - portrait_metric_plot, - ObservationLinker, - DataCatalogueBuilder, - LandSeaMaskGenerator, - ClimMetricsReader, - ClimMetricsMerger, - MeanClimateMetricsCollector, - VariabilityMetricsCollector, - EnsoDiagnosticsCollector, - SyntheticMetricsPlotter -) +################################ -from pcmdi_viewer_util import( - collect_config, - setup_jinja_env, - create_section, - add_section, - generate_methodology_html, - generate_data_html, - generate_viewer_html -) +echo "About to set up a zi-pcmdi command" +echo "The current directory is: $PWD" # This will be of the form .../post/scripts/tmpDir -# Determine multiprocessing usage -num_workers = {{ num_workers }} -multiprocessing = {{ multiprocessing }} if num_workers >= 2 else False +# Run diagnostics +mkdir -p pcmdi_diags {% if "synthetic_plots" not in subsection %} - -# Time range -start_yr = int('${Y1}') -end_yr = int('${Y2}') -num_years = end_yr - start_yr + 1 - -# Set data paths based on diagnostic type {% if "mean_climate" in subsection %} -test_data_path = '${climo_dir_primary}' -reference_data_path = '${climo_dir_ref}' +source_dirs="--climo_ts_dir_primary ${climo_dir_primary} --climo_ts_dir_ref ${climo_dir_ref}" {% elif "variability_modes" in subsection or "enso" in subsection %} -test_data_path = '${ts_dir_primary}' -reference_data_path = '${ts_dir_ref}' +source_dirs="--climo_ts_dir_primary ${ts_dir_primary} --climo_ts_dir_ref ${ts_dir_ref}" {% endif %} - -# Dataset identifiers -test_data_set = ['${model_name}'.split(".")[1]] -{% if run_type == "model_vs_obs" %} -reference_data_set = '{{ obs_sets }}'.split(",") -{% elif run_type == "model_vs_model" %} -reference_data_set = ['${model_name_ref}'.split(".")[1]] -{% endif %} - -variables = '{{ vars }}'.split(",") - -############################################################### -#check and process derived quantities, these quantities are -#likely not included as default in e3sm_to_cmip module -############################################################### -for var in variables: - varin = re.split(r"[_-]", var)[0] if "_" in var or "-" in var else var - - test_fpaths = sorted(glob.glob(os.path.join(test_data_path, f"*.{var}.*.nc"))) - if not test_fpaths: - derive_missing_variable(varin, test_data_path, '${model_name}.${tableID}') - -{% if run_type == "model_vs_model" %} - ref_fpaths = sorted(glob.glob(os.path.join(reference_data_path, f"*.{var}.*.nc"))) - if not ref_fpaths: - derive_missing_variable(varin, reference_data_path, '${model_name_ref}.${tableID_ref}') -{% endif %} - -####################################################### -#collect and document data info in a dictionary -# for convenience of pcmdi processing -####################################################### -builder = DataCatalogueBuilder( - test_data_path, test_data_set, - reference_data_path, reference_data_set, - variables, '{{subsection}}', 'pcmdi_diags' -) -test_dic, obs_dic = builder.build_catalogues() - -########################################################## -# land/sea mask is needed in PCMDI diagnostics, check and -# generate it here as these data are not always available -# for model or observations -########################################################## -# Whether to generate the land/sea mask -generate_flag = {{ generate_sftlf }} -# Instantiate and run -mask_generator = LandSeaMaskGenerator( - test_path=test_data_path, - ref_path=reference_data_path, - subsection='{{subsection}}', - fixed_dir='fixed' -) -mask_generator.run(generate_flag) - -# Diagnostic input file templates -input_template = os.path.join( - 'pcmdi_diags', - '%(output_type)', - '%(metric_type)', - '${model_name}'.split(".")[0], - '${model_name}'.split(".")[1], - '${case_id}' -) - - -# Diagnostic output path templates -out_path = os.path.join('${results_dir}', '%(group_type)') - +# Parameter passing can encounter errors if parameters are empty. +# So, it's a good idea to make sure they can't be (or at least are unlikely to be) empty. +# run_type == "model_vs_obs" only: obs_sets (default value is NOT "") +# run_type == "model_vs_model" only: model_name_ref, tableID_ref (default values are NOT "") +core_parameters="--num_workers {{ num_workers }} --multiprocessing {{ multiprocessing }} --subsection {{ subsection }} ${source_dirs} --model_name ${model_name} --model_tableID {{model_tableID }} --figure_format {{ figure_format }} --run_type {{ run_type }} --obs_sets {{ obs_sets }} --model_name_ref ${model_name_ref} --vars {{ vars }} --tableID_ref ${tableID_ref} --generate_sftlf {{ generate_sftlf }} --case_id ${case_id} --results_dir ${results_dir}" {% endif %} {% if "mean_climate" in subsection %} -regions = '{{regions}}'.split(",") - -#assiagn region to each variable -save_variable_regions(variables, regions) - -# generate the command list -lstcmd = generate_mean_clim_cmds( - variables=variables, - obs_dic=obs_dic, - case_id='${case_id}' -) - -#################################################### -# call pcmdi mean climate diagnostics -#################################################### -if (len(lstcmd) > 0 ) and multiprocessing: - try: - results = run_parallel_jobs(lstcmd, num_workers) - for i, (stdout, stderr, return_code) in enumerate(results): - print(f"\nCommand {i+1} finished:") - print(f"STDOUT: {stdout}") - print(f"STDERR: {stderr}") - print(f"Return code: {return_code}") - except RuntimeError as e: - print(f"Execution failed: {e}") -elif (len(lstcmd) > 0 ): - try: - results = run_serial_jobs(lstcmd) - for i, (stdout, stderr, return_code) in enumerate(results): - print(f"\nCommand {i+1} finished:") - print(f"STDOUT: {stdout}") - print(f"STDERR: {stderr}") - print(f"Return code: {return_code}") - except RuntimeError as e: - print(f"Execution failed: {e}") -else: - print("no jobs to run,continue....") - -print("successfully finish all jobs....") -#time delay to ensure process completely finished -time.sleep(5) - -#orgnize diagnostic output -collector = MeanClimateMetricsCollector( - regions=regions, - variables=variables, - fig_format='{{figure_format}}', - model_info=tuple('${model_name}'.split(".")), # (mip, exp, model, relm) - case_id='${case_id}', - input_template=input_template, - output_dir=out_path -) -collector.collect() - +command="zi-pcmdi-mean-climate ${core_parameters} --regions {{ regions }}" {% endif %} - {% if "variability_modes" in subsection %} -########################################## -# call pcmdi mode variability diagnostics -########################################## -print("calculate mode variability metrics") - {% if subsection == "variability_modes_atm" %} -var_modes = '{{ atm_modes }}'.split(",") +var_modes={{ atm_modes }} {% elif subsection == "variability_modes_cpl" %} -var_modes = '{{ cpl_modes }}'.split(",") +var_modes={{ cpl_modes }} {% endif %} - -#from configuration file -varOBS = '{{vars}}' -refset = obs_dic[varOBS]['set'] -refname = obs_dic[varOBS][refset] -refpath = obs_dic[varOBS][refname]['file_path'] -reftyrs = int(str(obs_dic[varOBS][refname]['yymms'])[0:4]) -reftyre = int(str(obs_dic[varOBS][refname]['yymme'])[0:4]) - -# Call the function -lstcmd = generate_varmode_cmds( - modes=var_modes, - varOBS=varOBS, - reftyrs=reftyrs, - reftyre=reftyre, - refname=refname, - refpath=refpath, - case_id='${case_id}' -) - -if (len(lstcmd) > 0 ) and multiprocessing: - try: - results = run_parallel_jobs(lstcmd, num_workers) - for i, (stdout, stderr, return_code) in enumerate(results): - print(f"\nCommand {i+1} finished:") - print(f"STDOUT: {stdout}") - print(f"STDERR: {stderr}") - print(f"Return code: {return_code}") - except RuntimeError as e: - print(f"Execution failed: {e}") -elif (len(lstcmd) > 0 ): - try: - results = run_serial_jobs(lstcmd) - for i, (stdout, stderr, return_code) in enumerate(results): - print(f"\nCommand {i+1} finished:") - print(f"STDOUT: {stdout}") - print(f"STDERR: {stderr}") - print(f"Return code: {return_code}") - except RuntimeError as e: - print(f"Execution failed: {e}") -else: - print("no jobs to run,continue...") - -print("successfully finish all jobs....") -#time delay to ensure process completely finished -time.sleep(5) - -# Create the collector instance -collector = VariabilityMetricsCollector( - modes=var_modes, - fig_format='{{figure_format}}', - mip='${model_name}'.split(".")[0], - exp='${model_name}'.split(".")[1], - model='${model_name}'.split(".")[2], - relm='${model_name}'.split(".")[3], - case_id='${case_id}', - input_dir=input_template, - output_dir=out_path -) - -# Run the collection process -collector.collect() - +command="zi-pcmdi-variability-modes ${core_parameters} --var_modes ${var_modes}" {% endif %} - {% if "enso" in subsection %} -############################################# -# call enso_driver.py to process diagnostics -############################################# -build_enso_obsvar_catalog(obs_dic, variables) -build_enso_obsvar_landmask(obs_dic, variables) - -#now start enso driver -lstcmd = generate_enso_cmds('{{ enso_groups }}', '${case_id}') -if (len(lstcmd) > 0 ) and multiprocessing: - try: - results = run_parallel_jobs(lstcmd, num_workers) - for i, (stdout, stderr, return_code) in enumerate(results): - print(f"\nCommand {i+1} finished:") - print(f"STDOUT: {stdout}") - print(f"STDERR: {stderr}") - print(f"Return code: {return_code}") - except RuntimeError as e: - print(f"Execution failed: {e}") -elif (len(lstcmd) > 0 ) and not multiprocessing: - try: - results = run_serial_jobs(lstcmd) - for i, (stdout, stderr, return_code) in enumerate(results): - print(f"\nCommand {i+1} finished:") - print(f"STDOUT: {stdout}") - print(f"STDERR: {stderr}") - print(f"Return code: {return_code}") - except RuntimeError as e: - print(f"Execution failed: {e}") -else: - print("no jobs to run...") - -print("successfully finish all jobs....") -#time delay to ensure process completely finished -time.sleep(5) - -# Initialize and run collector -obs_dict = json.load(open('obs_catalogue.json')) -obs_name = list(obs_dict.keys())[0] -collector = EnsoDiagnosticsCollector( - fig_format='{{figure_format}}', - refname=obs_name, - model_name_parts='${model_name}'.split("."), - case_id='${case_id}', - input_dir=input_template, - output_dir=out_path -) - -enso_groups = '{{ enso_groups }}'.split(",") -collector.run(enso_groups) - +command="zi-pcmdi-enso ${core_parameters} --enso_groups {{ enso_groups }}" {% endif %} - {% if "synthetic_plots" in subsection %} -######################################### -#plot synthetic figures for pcmdi metrics -######################################### -print("generate synthetic metrics plot ...") -metric_sets = '{{sub_sets}}'.split(",") -figure_sets = '{{synthetic_sets}}'.split(",") -figure_format = '{{figure_format}}' -test_input_path = os.path.join( - '${www}', - '%(model_name)', - 'pcmdi_diags', - '${results_dir}', - 'metrics_data', - '%(group_type)' -) - -metric_dict = json.load(open('synthetic_metrics_list.json')) - -plotter = SyntheticMetricsPlotter( - case_name='{{case}}', - test_name='{{model_name}}', - table_id='{{model_tableID}}', - figure_format=figure_format, - figure_sets=figure_sets, - metric_dict=metric_dict, - save_data=True, - base_test_input_path=test_input_path, - results_dir='${web_dir}/${results_dir}', - cmip_clim_dir='{{cmip_clim_dir}}', - cmip_clim_set='{{cmip_clim_set}}', - cmip_movs_dir='{{cmip_movs_dir}}', - cmip_movs_set='{{cmip_movs_set}}', - atm_modes='{{ atm_modes }}', - cpl_modes='{{ cpl_modes }}', - cmip_enso_dir='{{cmip_enso_dir}}', - cmip_enso_set='{{cmip_enso_set}}' -) - -# Generate Summary Metrics plots -# e.g., "climatology,enso,variability" -groups = '{{sub_sets}}'.split(',') -plotter.generate(groups) - -print("Generating viewer page for diagnostics...") - -# Extract template values (assumes substitution happens before execution) -title = "{{pcmdi_webtitle}}" -version = "{{pcmdi_version}}" -subtitle = "${run_type}".replace('_', ' ').capitalize() -case_id = "${case}" -model_name = "{{model_name}}" -table_id = "{{model_tableID}}" - -# ts_years is assumed to be a list via string_list(default=list("")) -ts_years = {{ts_years}} -ts_periods = ts_years if isinstance(ts_years, list) else [] - -# Validate and unpack periods -if len(ts_periods) == 3: - clim_period, emov_period, enso_period = [p.strip() for p in ts_periods] -else: - raise ValueError( - f"Expected 3 periods (climatology, EMoV, ENSO), " - f"but got {len(ts_periods)}: {ts_periods}" - ) - -# Set up paths -obs_dir = os.path.join('{{pcmdi_external_prefix}}', 'observations', 'Atm', 'time-series') -pmp_dir = os.path.join('{{pcmdi_external_prefix}}', 'pcmdi_data') -out_dir = os.path.join("${web_dir}", "${results_dir}", "viewer") -os.makedirs(out_dir, exist_ok=True) - -# Copy logo -web_logo_src = os.path.join( - '{{pcmdi_external_prefix}}', - '{{pcmdi_viewer_template}}', - 'e3sm_pmp_logo.png' -) -web_logo_dst = os.path.join(out_dir, 'e3sm_pmp_logo.png') -shutil.copy(web_logo_src, web_logo_dst) - -# Build config -config = collect_config( - title=title, - subtitle=subtitle, - version=version, - case_id=case_id, - diag_dir="${web_dir}", - obs_dir=obs_dir, - pmp_dir=pmp_dir, - out_dir=out_dir, - clim_period=clim_period, - emov_period=emov_period, - enso_period=enso_period -) - -# Render viewer -generate_methodology_html(config) -generate_data_html(config) -generate_viewer_html(config) - +# Note: ts_years may be List[str] +command="zi-pcmdi-synthetic-plots --synthetic_sets {{ synthetic_sets }} --figure_format {{ figure_format }} --www ${www} --results_dir ${results_dir} --case {{ case }} --model_name {{ model_name }} --model_tableID {{model_tableID }} --web_dir=${web_dir} --cmip_clim_dir {{ cmip_clim_dir }} --cmip_clim_set {{ cmip_clim_set }} --cmip_movs_dir {{ cmip_movs_dir }} --cmip_movs_set {{ cmip_movs_set }} --atm_modes {{ atm_modes }} --cpl_modes {{ cpl_modes }} --cmip_enso_dir {{ cmip_enso_dir }} --cmip_enso_set {{ cmip_enso_set }} --sub_sets {{ sub_sets }} --pcmdi_webtitle {{ pcmdi_webtitle }} --pcmdi_version {{ pcmdi_version }} --run_type ${run_type} --ts_years {{ ts_years }} --pcmdi_external_prefix {{ pcmdi_external_prefix }} --pcmdi_viewer_template {{ pcmdi_viewer_template }}" {% endif %} -EOF -################################ -# Run diagnostics -mkdir -p pcmdi_diags -command="srun -N 1 python -u pcmdi.py" # Run diagnostics +source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-20250529 +echo "About to run a zi-pcmdi command" +echo "The current directory is: $PWD" # This will be of the form .../post/scripts/tmpDir time ${command} if [ $? != 0 ]; then cd {{ scriptDir }} echo 'ERROR (11)' > {{ prefix }}.status exit 11 fi +source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh ################################# # Copy output to web server diff --git a/zppy/templates/ts.bash b/zppy/templates/ts.bash old mode 100755 new mode 100644 index 0f991457..d5a1b71c --- a/zppy/templates/ts.bash +++ b/zppy/templates/ts.bash @@ -115,92 +115,6 @@ if [ $? != 0 ]; then exit 3 fi -{%- if ts_fmt != 'ts_only' %} -tmp_dir=tmp_{{ prefix }} - -# Generate CMIP ts -cat > default_metadata.json << EOF -{% include cmip_metadata %} -EOF -{ - export cmortables_dir={{ cmor_tables_prefix }}/cmip6-cmor-tables/Tables - #input_dir={{ output }}/post/{{ component }}/{{ grid }}/ts/{{ frequency }}/{{ '%dyr' % (ypf) }} - input_dir=${dest}/{{ '%04d' % (yr_start) }}_{{ '%04d' % (yr_end) }} - mkdir -p $input_dir - - cp -s $dest/*_{{ '%04d' % (yr_start) }}??_{{ '%04d' % (yr_end) }}??.nc $input_dir - dest_cmip={{ output }}/post/{{ component }}/{{ grid }}/cmip_ts/{{ frequency }} - mkdir -p ${dest_cmip} - - {{ e3sm_to_cmip_environment_commands }} - - {% if input_files.split(".")[0] == 'cam' or input_files.split(".")[0] == 'eam' -%} - #add code to do vertical interpolation variables at model levels before e3sm_to_cmip - IFS=',' read -ra mlvars <<< "{{ interp_vars }}" - for var in "${mlvars[@]}" - do - for file in ${input_dir}/${var}_{{ '%04d' % (yr_start) }}??_{{ '%04d' % (yr_end) }}??.nc - do - if [ -f ${file} ]; then - #ncks --rgr xtr_mth=mss_val --vrt_fl='{{cmip_plevdata}}' ${file} ${file}.plev - ncremap -p mpi --vrt_ntp=log --vrt_xtr=mss_val --vrt_out='{{cmip_plevdata}}' ${file} ${file}.plev - #overwrite the model level data - mv ${file}.plev ${file} - fi - done - done - {% endif -%} - - #call e3sm_to_cmip - srun -N 1 e3sm_to_cmip \ - --output-path \ - ${dest_cmip}/${tmp_dir} \ - {% if input_files.split(".")[0] == 'clm2' or input_files.split(".")[0] == 'elm' -%} - --var-list \ - 'snd, mrsos, mrso, mrfso, mrros, mrro, prveg, evspsblveg, evspsblsoi, tran, tsl, lai, cLitter, cProduct, cSoilFast, cSoilMedium, cSoilSlow, fFire, fHarvest, cVeg, nbp, gpp, ra, rh' \ - --realm \ - lnd \ - {% endif -%} - {% if input_files.split(".")[0] == 'cam' or input_files.split(".")[0] == 'eam' -%} - --var-list \ - 'ua, va, ta, wa, zg, hur, pr, prc, prsn, ts, tas, prw, psl, sfcWind, tasmax, tasmin, tauu, tauv, rtmt, rsdt, rsds, rsdscs, rlds, rldscs, rsus, rsuscs, rsut, rsutcs, rlus, rlut, rlutcs, clivi, clwvi, clt, evspsbl, hfls, hfss, huss' \ - --realm \ - atm \ - {% endif -%} - --input-path \ - ${input_dir} \ - --user-metadata \ - {{ scriptDir }}/${workdir}/default_metadata.json \ - --num-proc \ - 12 \ - --tables-path \ - ${cmortables_dir} - - if [ $? != 0 ]; then - cd {{ scriptDir }} - echo 'ERROR (4)' > {{ prefix }}.status - exit 4 - fi - {{ environment_commands }} - - # Move output ts files to final destination - mv ${dest_cmip}/${tmp_dir}/CMIP6/CMIP/*/*/*/*/*/*/*/*/*.nc ${dest_cmip} - if [ $? != 0 ]; then - cd {{ scriptDir }} - echo 'ERROR (5)' > {{ prefix }}.status - exit 5 - fi - - rm -r ${dest_cmip}/${tmp_dir} - -} -if [ $? != 0 ]; then - cd {{ scriptDir }} - echo 'ERROR (6)' > {{ prefix }}.status - exit 6 -fi -{%- endif %} - # Delete temporary workdir cd .. if [[ "${debug,,}" != "true" ]]; then diff --git a/zppy/utils.py b/zppy/utils.py index 0f4fa563..2812b252 100644 --- a/zppy/utils.py +++ b/zppy/utils.py @@ -7,7 +7,7 @@ import time from enum import Enum from subprocess import PIPE, Popen -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, List, Set, Tuple import jinja2 from configobj import ConfigObj @@ -228,6 +228,21 @@ def set_component_and_prc_typ(c: Dict[str, Any]) -> None: c["prc_typ"] = prc_typ +def check_set_specific_parameter( + c: Dict[str, Any], sets_with_requirement: Set[str], relevant_parameter: str +) -> None: + requested_sets = set(c["sets"]) + intersection = sets_with_requirement & requested_sets + if ( + intersection + and (relevant_parameter in c.keys()) + and (c[relevant_parameter] == "") + ): + raise ParameterNotProvidedError( + f"{relevant_parameter} is required because the sets {intersection} were requested." + ) + + # Return all year sets from a configuration given by a list of strings # "year_begin:year_end:year_freq" # "year_begin-year_end" @@ -309,6 +324,17 @@ def set_value_of_parameter_if_undefined( ) +def check_parameter_defined( + c: Dict[str, Any], relevant_parameter: str, explanation: str = "" +) -> None: + if (relevant_parameter not in c.keys()) or (c[relevant_parameter] == ""): + if explanation: + message = f"{relevant_parameter} is needed because {explanation}" + else: + message = f"{relevant_parameter} is not defined." + raise ParameterNotProvidedError(message) + + def get_file_names(script_dir: str, prefix: str): return tuple( [ From fc228c178edd734df2d62586ee80fbc2919cd79c Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 30 Jul 2025 13:59:12 -0500 Subject: [PATCH 03/47] Minor updates --- examples/post.v3.LR.amip.0101.cfg | 5 +++-- zppy/templates/pcmdi_diags.bash | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/post.v3.LR.amip.0101.cfg b/examples/post.v3.LR.amip.0101.cfg index 1de1fba4..413a0811 100644 --- a/examples/post.v3.LR.amip.0101.cfg +++ b/examples/post.v3.LR.amip.0101.cfg @@ -6,10 +6,11 @@ [default] input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/AMIP/v3.LR.amip_0101 -output = /lcrc/group/e3sm/ac.forsyth2/zppy_pr719_output/unique_id_16/v3.LR.amip_0101 -www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pr719_output/unique_id_16 +output = /lcrc/group/e3sm/ac.forsyth2/zppy_pr719_output/unique_id_21/v3.LR.amip_0101 +www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pr719_output/unique_id_21 campaign = "water_cycle" case = v3.LR.amip_0101 +debug = True environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" partition = compute diff --git a/zppy/templates/pcmdi_diags.bash b/zppy/templates/pcmdi_diags.bash index 6bbd3d04..37542f94 100755 --- a/zppy/templates/pcmdi_diags.bash +++ b/zppy/templates/pcmdi_diags.bash @@ -1,5 +1,5 @@ #!/bin/bash -{% include 'inclusions/slurm_header.sh' %} +{% include 'inclusions/slurm_header.bash' %} {{ environment_commands }} From c6d3b1a7254199d23fa83956020f6b0fdc52a672 Mon Sep 17 00:00:00 2001 From: ShixuanZhang Date: Thu, 4 Sep 2025 02:17:53 -0500 Subject: [PATCH 04/47] This commits contains bug fixes and update to for zppy-pcmdi diagnostics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Key changes are described below: 1. Mean Climate: revise the e3sm_to_cmip to handle the 3-D variable including U, V, T, and Z3. The "vrt_remap_plev19.nc" was added to "zppy/templates/inclusions/e3sm_to_cmip" directory to facillitate the interpolation from model level to standard pressure level 2. Modes of Variability: correction on the naming convention of the model file template so that it can be identified by the PCMDI diagnostic drivers. Also fix the flag change "plot_model = True" to "plot = True" following the changes made in the PCMDI code base 3. ENSO Diagnostics: revised the yml file to use NumPy < 2.1.0 for the PCMDI conda environment so that the “CLIVAR-PRP/ENSO_metrics” package can be called correctly for sucess diagnostics of ENSO --- examples/post.v3.LR.amip.0101.cfg | 18 ++++++------- zppy/defaults/default.ini | 2 ++ zppy/e3sm_to_cmip.py | 2 +- zppy/pcmdi_diags.py | 6 ++++- zppy/templates/pcmdi_diags.bash | 42 +++++++++++++++---------------- 5 files changed, 37 insertions(+), 33 deletions(-) diff --git a/examples/post.v3.LR.amip.0101.cfg b/examples/post.v3.LR.amip.0101.cfg index 413a0811..1101c54f 100644 --- a/examples/post.v3.LR.amip.0101.cfg +++ b/examples/post.v3.LR.amip.0101.cfg @@ -6,13 +6,13 @@ [default] input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/AMIP/v3.LR.amip_0101 -output = /lcrc/group/e3sm/ac.forsyth2/zppy_pr719_output/unique_id_21/v3.LR.amip_0101 -www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pr719_output/unique_id_21 +output = /lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy_pr719_output/unique_id_21/v3.LR.amip_0101 +www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.szhang/zppy_pr719_output/unique_id_21 campaign = "water_cycle" case = v3.LR.amip_0101 debug = True environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" -partition = compute +partition = debug #compute [ts] active = True @@ -38,7 +38,7 @@ years = "2005:2015:5" [[ atm_monthly_180x360_aave ]] # list of variables to be interpolated to pressure level; see e3sm_to_cmip/vrt_remap_plev19.nc - cmip_plevdata = "inclusions/e3sm_to_cmip/vrt_remap_plev19.nc" + cmip_plevdata = /lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc input_files = "eam.h0" interp_vars = "U,V,T,Z3" ts_subsection = "atm_monthly_180x360_aave" @@ -146,9 +146,9 @@ environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_ [[ enso ]] active = True ################################################################################## - ts_years = "2005-2015", + ts_years = "2005-2014", ts_num_years = 5 - ref_years = "2005-2015", + ref_years = "2005-2014", ref_start_yr = 1979 ref_final_yr = 2018 #create rules to construct cmip-like datasets with a string constructed as @@ -171,9 +171,7 @@ environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_ active = True model_name = 'e3sm.amip.v3-LR.0101' model_tableID = 'Amon' - #sub_sets = "mean_climate,variability_modes,enso_metric" - sub_sets = "mean_climate,variability_modes" - ts_years = "2005-2014", - ref_years = "2005-2014", synthetic_sets = "portrait,parcoord" + figure_sets = "mean_climate,variability_modes,enso_metric" + figure_sets_period = "2005-2014,2005-2014,2005-2014" figure_format = "png" diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index 14762bfb..49ff7ea1 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -559,6 +559,8 @@ e3sm_to_cmip_atm_subsection = string(default="") multiprocessing = boolean(default=None) num_workers = integer(default=None) obs_ts = string(default=None) + figure_sets = string(default=None) + figure_sets_period = string(default=None) figure_format = string(default=None) ref_name = string(default=None) ref_years = string_list(default=None) diff --git a/zppy/e3sm_to_cmip.py b/zppy/e3sm_to_cmip.py index 0ee71b30..8ea1b151 100644 --- a/zppy/e3sm_to_cmip.py +++ b/zppy/e3sm_to_cmip.py @@ -56,7 +56,7 @@ def e3sm_to_cmip(config: ConfigObj, script_dir: str, existing_bundles, job_ids_f if c["cmip_vars"] == "": if c["component"] == "atm": c["cmip_vars"] = ( - "tas, ts, psl, ps, sfcWind, huss, pr, prc, prsn, evspsbl, tauu, tauv, hfls, clt, rlds, rlus, rsds, rsus, hfss, clivi, clwvi, prw, rldscs, rlut, rlutcs, rsdt, rsuscs, rsut, rsutcs, rtmt, abs550aer, od550aer, rsdscs, tasmax, tasmin" + "ua, va, ta, wa, zg, hur, tas, ts, psl, ps, sfcWind, huss, pr, prc, prsn, evspsbl, tauu, tauv, hfls, clt, rlds, rlus, rsds, rsus, hfss, clivi, clwvi, prw, rldscs, rlut, rlutcs, rsdt, rsuscs, rsut, rsutcs, rtmt, abs550aer, od550aer, rsdscs, tasmax, tasmin" ) elif c["component"] == "lnd": c["cmip_vars"] = ( diff --git a/zppy/pcmdi_diags.py b/zppy/pcmdi_diags.py index 5aab9e9b..5c14f6d7 100644 --- a/zppy/pcmdi_diags.py +++ b/zppy/pcmdi_diags.py @@ -51,7 +51,11 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): check_parameters_for_pcmdi(c) # Loop over year sets - year_sets: List[Tuple[int, int]] = get_years(c["ts_years"]) + if c["sub"] != "synthetic_plots": + year_sets: List[Tuple[int, int]] = get_years(c["ts_years"]) + else: + year_sets: List[Tuple[int, int]] = get_years(c["figure_sets_period"].split(",")) + ref_year_sets: List[Tuple[int, int]] if ("ref_years" in c.keys()) and (c["ref_years"] != [""]): ref_year_sets = get_years(c["ref_years"]) diff --git a/zppy/templates/pcmdi_diags.bash b/zppy/templates/pcmdi_diags.bash index 37542f94..3f0cc60a 100755 --- a/zppy/templates/pcmdi_diags.bash +++ b/zppy/templates/pcmdi_diags.bash @@ -198,7 +198,7 @@ create_links_acyc_climo_obs() { for file in ${ts_dir_source}/*; do local fname local YYYYS YYYYE - local PREFIX + local SUBSTR local ttag tmp_file MM combined_name fname=$(basename "${file}") @@ -221,7 +221,7 @@ create_links_acyc_climo_obs() { if [[ ${YYYYE} -gt ${end_year} ]]; then YYYYE=${end_year}; fi # Extract prefix before the date range (removes from .${YYYYS} or -${YYYYS}) - PREFIX="${fname%%[._-]${YYYYS}*}" + SUBSTR="${fname%%[._-]${YYYYS}*}" ttag="$(printf "%04d" "${YYYYS}")01-$(printf "%04d" "${YYYYE}")12" tmp_file="tmp_combine_${ttag}.nc" @@ -234,7 +234,7 @@ create_links_acyc_climo_obs() { ncra -O -h -F -d time,"${month}",,12 "${tmp_file}" "tmp_clm_${MM}.nc" done - combined_name="${PREFIX}.${ttag}.AC.${case_id}.nc" + combined_name="${SUBSTR}.${ttag}.AC.${case_id}.nc" ncrcat -O tmp_clm_*.nc "${combined_name}" # Adjust time metadata @@ -354,7 +354,7 @@ create_links_ts_obs() { mkdir -p "${ts_dir_destination}" cd "${ts_dir_destination}" || exit - local file fname PREFIX YYYYS YYYYE ttag combined_name + local file fname SUBSTR YYYYS YYYYE ttag combined_name for file in ${ts_dir_source}/*; do fname=$(basename "$file") @@ -365,10 +365,12 @@ create_links_ts_obs() { fi echo "create_links_ts_obs: processing ${file}" # Match two time patterns (YYYYMM or YYYYMMDD) separated by _ or - - if [[ ${fname} =~ ([0-9]{6,8})[_-]([0-9]{6,8}) ]]; then - YYYYS="${BASH_REMATCH[1]}" - YYYYE="${BASH_REMATCH[2]}" + if [[ $fname =~ ^(.+)\.([0-9]{4})([0-9]{2}){1,2}[-_]([0-9]{4})([0-9]{2}){1,2}\.nc$ ]]; then + SUBSTR="${BASH_REMATCH[1]}" # everything before the .YYYY... + YYYYS="${BASH_REMATCH[2]}" # start year + YYYYE="${BASH_REMATCH[4]}" # end year else + echo "Warning: Could not extract dates from ${fname}, basename of ${file}" continue fi @@ -382,11 +384,8 @@ create_links_ts_obs() { YYYYE="${end_year}" fi - # Extract prefix (before the date range, ignoring separator) - PREFIX="${fname%%[._-]${YYYYS}*}" - ttag="$(printf "%04d" "${YYYYS}")01-$(printf "%04d" "${YYYYE}")12" - combined_name="${PREFIX}.${ttag}.nc" + combined_name="${SUBSTR}.${ttag}.nc" # Extract subset of time series ncrcat -O -d time,"${YYYYS}-01-01","${YYYYE}-12-31" "${file}" "${combined_name}" @@ -489,7 +488,7 @@ echo "Linking observational data using SLURM..." command="zi-pcmdi-link-observation --model_name_ref ${model_name_ref} --tableID_ref ${tableID_ref} --vars={{ vars }} --obs_sets {{ obs_sets }} --obs_ts {{ obs_ts }} --obstmp_dir ${obstmp_dir}" echo "Running: ${command}" -source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-20250529 +source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20250730 time eval "${command}" source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh @@ -555,7 +554,7 @@ user_notes = 'Provenance and results' debug = {{ pcmdi_debug }} # Enable plot generation for model and observation -plot_model = {{ mov_plot_model }} +plot = {{ mov_plot_model }} plot_obs = {{ mov_plot_obs }} # optional # Execution mode and output format @@ -578,7 +577,7 @@ regions_values = { # Template path for land/sea mask file (fixed input) modpath_lf = os.path.join( 'fixed', - 'sftlf.put_model_here.nc' + 'sftlf.%(model).nc' ) {% if "mean_climate" in subsection %} @@ -619,7 +618,7 @@ test_data_path = '${climo_dir_primary}' filename_template = '.'.join([ mip, exp, - 'put_model_here', + '%(model)', '%(realization)', '${tableID}', '%(variable)', @@ -723,7 +722,7 @@ modnames = [product] realization = "*" modpath = os.path.join( '${ts_dir_primary}', - '{}.{}.put_model_here.%(realization).{}.%(variable).{}.nc'.format( + '{}.{}.%(model).%(realization).{}.%(variable).{}.nc'.format( mip, exp, '${tableID}', period ) ) @@ -748,7 +747,7 @@ realization = realm modpath = os.path.join( '${ts_dir_primary}', - '{}.{}.put_model_here.%(realization).{}.%(variable).{}.nc'.format( + '{}.{}.%(model).%(realization).{}.%(variable).{}.nc'.format( mip, exp, '${tableID}', period ) ) @@ -776,7 +775,7 @@ results_dir = os.path.join( ) # Output filenames for JSON and NetCDF -json_name = "%(mip)_%(exp)_%(metricsCollection)_${case_id}_put_model_here_%(realization)" +json_name = "%(mip)_%(exp)_%(metricsCollection)_${case_id}_%(model)_%(realization)" netcdf_name = json_name @@ -827,12 +826,13 @@ command="zi-pcmdi-variability-modes ${core_parameters} --var_modes ${var_modes}" command="zi-pcmdi-enso ${core_parameters} --enso_groups {{ enso_groups }}" {% endif %} {% if "synthetic_plots" in subsection %} -# Note: ts_years may be List[str] -command="zi-pcmdi-synthetic-plots --synthetic_sets {{ synthetic_sets }} --figure_format {{ figure_format }} --www ${www} --results_dir ${results_dir} --case {{ case }} --model_name {{ model_name }} --model_tableID {{model_tableID }} --web_dir=${web_dir} --cmip_clim_dir {{ cmip_clim_dir }} --cmip_clim_set {{ cmip_clim_set }} --cmip_movs_dir {{ cmip_movs_dir }} --cmip_movs_set {{ cmip_movs_set }} --atm_modes {{ atm_modes }} --cpl_modes {{ cpl_modes }} --cmip_enso_dir {{ cmip_enso_dir }} --cmip_enso_set {{ cmip_enso_set }} --sub_sets {{ sub_sets }} --pcmdi_webtitle {{ pcmdi_webtitle }} --pcmdi_version {{ pcmdi_version }} --run_type ${run_type} --ts_years {{ ts_years }} --pcmdi_external_prefix {{ pcmdi_external_prefix }} --pcmdi_viewer_template {{ pcmdi_viewer_template }}" +# Note: figure_sets_period changed to a string +# Note: sub_sets was renamed to figure_sets to be clearer +command="zi-pcmdi-synthetic-plots --synthetic_sets {{ synthetic_sets }} --figure_format {{ figure_format }} --www ${www} --results_dir ${results_dir} --case {{ case }} --model_name {{ model_name }} --model_tableID {{model_tableID }} --web_dir=${web_dir} --cmip_clim_dir {{ cmip_clim_dir }} --cmip_clim_set {{ cmip_clim_set }} --cmip_movs_dir {{ cmip_movs_dir }} --cmip_movs_set {{ cmip_movs_set }} --atm_modes {{ atm_modes }} --cpl_modes {{ cpl_modes }} --cmip_enso_dir {{ cmip_enso_dir }} --cmip_enso_set {{ cmip_enso_set }} --figure_sets {{ figure_sets }} --pcmdi_webtitle {{ pcmdi_webtitle }} --pcmdi_version {{ pcmdi_version }} --run_type ${run_type} --figure_sets_period {{ figure_sets_period }} --pcmdi_external_prefix {{ pcmdi_external_prefix }} --pcmdi_viewer_template {{ pcmdi_viewer_template }}" {% endif %} # Run diagnostics -source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-20250529 +source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20250730 echo "About to run a zi-pcmdi command" echo "The current directory is: $PWD" # This will be of the form .../post/scripts/tmpDir time ${command} From 158d3a076f6f61bd44095a1e5de63622a28edd40 Mon Sep 17 00:00:00 2001 From: ShixuanZhang Date: Thu, 4 Sep 2025 02:25:55 -0500 Subject: [PATCH 05/47] Add vrt_remap_plev19.nc for vertical interpolation --- .../inclusions/e3sm_to_cmip/vrt_remap_plev19.nc | Bin 0 -> 1260 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100755 zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc diff --git a/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc b/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc new file mode 100755 index 0000000000000000000000000000000000000000..2b71cc296ff2f207b80961668af618cecf911c7e GIT binary patch literal 1260 zcma)6QE${Z5cZaBizpJDctk=Lssx-iah$ZkwN@t*Mpq+L{V6^u3af0v<-VB`cL(xUBVs*tzuH z0qVP%tWb-(pW}#cM?4;cJG<#GHJ^x5gIn+HJTxkagxyU;mFvnHJW7_m_Vd-UM4Q%iQwCt(YJ@xiP%r~ zPM;HIvIun;B?%9r_7}kynhac#OWu}t>SQ@PISbLmpRm%|XDah&JlC#bGAk6*rd;@3 zddZ|O7@3f+Tug2%9NsF!8oxMtWT6?B0vp4WsW_`S`~NN9^vEm<;fmNHP@Mm` zT~_F_FKX$uoXl0lYUhPhg)9Z8ln>6zO3odoN2m27PmS{F&WIjCnI#mmqILUcpc~;< zIG{K0+d`Fcs^}d$^S&H1W-=RA$I#HeM^kv;cVw#lY%vKlTc8orEsV1U^y8dC`Nt3E z$b(kV{hM^&0qIAi^WZ$``~(x{-=~E9^N^4~?-H_rsUklRa!3go{z1r%%Y Date: Thu, 4 Sep 2025 11:35:49 -0500 Subject: [PATCH 06/47] Format fix --- zppy/pcmdi_diags.py | 4 +++- zppy/templates/pcmdi_diags.bash | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/zppy/pcmdi_diags.py b/zppy/pcmdi_diags.py index 5c14f6d7..66f30186 100644 --- a/zppy/pcmdi_diags.py +++ b/zppy/pcmdi_diags.py @@ -54,7 +54,9 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): if c["sub"] != "synthetic_plots": year_sets: List[Tuple[int, int]] = get_years(c["ts_years"]) else: - year_sets: List[Tuple[int, int]] = get_years(c["figure_sets_period"].split(",")) + year_sets: List[Tuple[int, int]] = get_years( + c["figure_sets_period"].split(",") + ) ref_year_sets: List[Tuple[int, int]] if ("ref_years" in c.keys()) and (c["ref_years"] != [""]): diff --git a/zppy/templates/pcmdi_diags.bash b/zppy/templates/pcmdi_diags.bash index 3f0cc60a..663ef5e6 100755 --- a/zppy/templates/pcmdi_diags.bash +++ b/zppy/templates/pcmdi_diags.bash @@ -826,7 +826,7 @@ command="zi-pcmdi-variability-modes ${core_parameters} --var_modes ${var_modes}" command="zi-pcmdi-enso ${core_parameters} --enso_groups {{ enso_groups }}" {% endif %} {% if "synthetic_plots" in subsection %} -# Note: figure_sets_period changed to a string +# Note: figure_sets_period changed to a string # Note: sub_sets was renamed to figure_sets to be clearer command="zi-pcmdi-synthetic-plots --synthetic_sets {{ synthetic_sets }} --figure_format {{ figure_format }} --www ${www} --results_dir ${results_dir} --case {{ case }} --model_name {{ model_name }} --model_tableID {{model_tableID }} --web_dir=${web_dir} --cmip_clim_dir {{ cmip_clim_dir }} --cmip_clim_set {{ cmip_clim_set }} --cmip_movs_dir {{ cmip_movs_dir }} --cmip_movs_set {{ cmip_movs_set }} --atm_modes {{ atm_modes }} --cpl_modes {{ cpl_modes }} --cmip_enso_dir {{ cmip_enso_dir }} --cmip_enso_set {{ cmip_enso_set }} --figure_sets {{ figure_sets }} --pcmdi_webtitle {{ pcmdi_webtitle }} --pcmdi_version {{ pcmdi_version }} --run_type ${run_type} --figure_sets_period {{ figure_sets_period }} --pcmdi_external_prefix {{ pcmdi_external_prefix }} --pcmdi_viewer_template {{ pcmdi_viewer_template }}" {% endif %} From a9ebd9419603e2313eeb8f8562db18a6d11e73c2 Mon Sep 17 00:00:00 2001 From: ShixuanZhang Date: Thu, 4 Sep 2025 11:39:42 -0500 Subject: [PATCH 07/47] bug fix --- zppy/pcmdi_diags.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/zppy/pcmdi_diags.py b/zppy/pcmdi_diags.py index 66f30186..bdeeaaa4 100644 --- a/zppy/pcmdi_diags.py +++ b/zppy/pcmdi_diags.py @@ -51,12 +51,11 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): check_parameters_for_pcmdi(c) # Loop over year sets + year_sets: List[Tuple[int, int]] if c["sub"] != "synthetic_plots": - year_sets: List[Tuple[int, int]] = get_years(c["ts_years"]) + year_sets = get_years(c["ts_years"]) else: - year_sets: List[Tuple[int, int]] = get_years( - c["figure_sets_period"].split(",") - ) + year_sets = get_years(c["figure_sets_period"].split(",")) ref_year_sets: List[Tuple[int, int]] if ("ref_years" in c.keys()) and (c["ref_years"] != [""]): From 110c39220296df403f8ecac60837950218f3d56f Mon Sep 17 00:00:00 2001 From: ShixuanZhang Date: Thu, 4 Sep 2025 13:38:03 -0500 Subject: [PATCH 08/47] bug fix --- examples/post.v3.LR.amip.0101.cfg | 13 +- examples/post.v3.LR.historical.0101.cfg | 185 ++++++++++++++++++++++++ zppy/defaults/default.ini | 2 +- 3 files changed, 197 insertions(+), 3 deletions(-) create mode 100644 examples/post.v3.LR.historical.0101.cfg diff --git a/examples/post.v3.LR.amip.0101.cfg b/examples/post.v3.LR.amip.0101.cfg index 1101c54f..8fd8f171 100644 --- a/examples/post.v3.LR.amip.0101.cfg +++ b/examples/post.v3.LR.amip.0101.cfg @@ -71,7 +71,9 @@ environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_ ref_final_yr = 2018 model_name = 'e3sm.amip.v3-LR.0101' model_tableID = 'Amon' - cmip_clim_set = "cmip6.amip.v20241029" + ################################################################################## + #location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/mean_climate/cmip6/amip + cmip_clim_set = "cmip6.amip.v20250702" ################################################################################## #variables in current pcmdi-cmip list #note: rltcre and rstcre are derived variables as they are not included in cmip_ts @@ -96,6 +98,8 @@ environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_ #"mip.exp.product.realization", where "."is used as the delimiter (mandatary) model_name = 'e3sm.amip.v3-LR.0101' model_tableID = 'Amon' + ################################################################################## + #location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes/cmip6/amip cmip_movs_set = "cmip6.amip.v20210119" ################################################################################## grid = '180x360_aave' @@ -124,6 +128,8 @@ environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_ #"mip.exp.product.realization", where "." is used as the delimiter (mandatary) model_name = 'e3sm.amip.v3-LR.0101' model_tableID = 'Amon' + ################################################################################## + #location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes/cmip6/amip cmip_movs_set = "cmip6.amip.v20210119" ################################################################################## grid = '180x360_aave' @@ -155,7 +161,10 @@ environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_ #"mip.exp.product.realization", where "."is used as the delimiter (mandatary) model_name = 'e3sm.amip.v3-LR.0101' model_tableID = 'Amon' - cmip_enso_set = "cmip6.amip.v20210620" + ################################################################################## + #location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/enso_metric/cmip6/historical + #Note: there is no enso metric for CMIP6 AMIP, so use historical instead + cmip_enso_set = "cmip6.historical.v20210620" ################################################################################## #see zppy/templates/pcmdi_diags/reference_alias.json for options customized_cmor_obs = True diff --git a/examples/post.v3.LR.historical.0101.cfg b/examples/post.v3.LR.historical.0101.cfg new file mode 100644 index 00000000..9bd89ffd --- /dev/null +++ b/examples/post.v3.LR.historical.0101.cfg @@ -0,0 +1,185 @@ +# To run: +# `pip install .` the zppy-interfaces changes into the `zi-pcmdi-20250529` environment +# Update unique_id in output, www below +# Be in the zppy-pr719-dev-20250612 environment +# > pip install . && zppy -c examples/post.v3.LR.historical.0101.cfg + +[default] +input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/AMIP/v3.LR.historical_0101 +output = /lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy_pr719_output/unique_id_21/v3.LR.historical_0101 +www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.szhang/zppy_pr719_output/unique_id_21 +campaign = "water_cycle" +case = v3.LR.historical_0101 +debug = True +environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" +partition = debug #compute + +[ts] +active = True +walltime = "00:10:00" +years = "2005:2015:5" + + [[ atm_monthly_180x360_aave ]] + input_subdir = "archive/atm/hist" + input_files = "eam.h0" + frequency = "monthly" + mapping_file = /home/ac.zender/data/maps/map_ne30pg2_to_cmip6_180x360_aave.20200201.nc + # list for pcmdi diag, note: PHIS,hyam,hybm,hyai,hybi need to be included to process the 3D fields + vars = "ICEFRAC,LANDFRAC,OCNFRAC,PSL,FSNTC,FSNTOAC,SWCF,LWCF,FLUT,FSNT,FSNTOA,FLNT,FLNTC,FSNS,FLNS,FSNS,SHFLX,QFLX,LHFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,U10,QREFHT,TMQ,CLDTOT,CLDHGH,CLDMED,CLDLOW,FLDS,FSDS,TGCLDIWP,TGCLDCWP,TGCLDLWP,FLNSC,FLUTC,FSDSC,SOLIN,FSNSC,AODABS,AODVIS,AODDUST,AREL,TREFMNAV,TREFMXAV,PS,PHIS,U,V,T,Z3" + +[e3sm_to_cmip] +active = True +frequency = "monthly" +grid = "180x360_aave" +ts_grid = "180x360_aave" +ts_num_years=5 +walltime = "00:10:00" +years = "2005:2015:5" + + [[ atm_monthly_180x360_aave ]] + # list of variables to be interpolated to pressure level; see e3sm_to_cmip/vrt_remap_plev19.nc + cmip_plevdata = /lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc + input_files = "eam.h0" + interp_vars = "U,V,T,Z3" + ts_subsection = "atm_monthly_180x360_aave" + vars = "ICEFRAC,LANDFRAC,OCNFRAC,PSL,FSNTC,FSNTOAC,SWCF,LWCF,FLUT,FSNT,FSNTOA,FLNT,FLNTC,FSNS,FLNS,FSNS,SHFLX,QFLX,LHFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,U10,QREFHT,TMQ,CLDTOT,CLDHGH,CLDMED,CLDLOW,FLDS,FSDS,TGCLDIWP,TGCLDCWP,TGCLDLWP,FLNSC,FLUTC,FSDSC,SOLIN,FSNSC,AODABS,AODVIS,AODDUST,AREL,TREFMNAV,TREFMXAV,PS,PHIS,U,V,T,Z3" + +[pcmdi_diags] +active = True +multiprocessing = True +nodes = 1 +num_workers = 24 +#observational data +obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series' +pcmdi_debug = False +run_type = "model_vs_obs" +reference_data_path = "" +reference_data_path_ts = "" +ts_grid = "180x360_aave" +walltime = "4:00:00" +environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" + + [[ mean_climate ]] + active = True + ################################################################################## + #create rules to construct cmip-like datasets with a string constructed as + #"mip.exp.product.realization", where "."is used as the delimiter (mandatary) + ts_years = "2005-2014", + ts_num_years = 5 + ref_years = "2005-2014", + ref_start_yr = 1979 + ref_final_yr = 2018 + model_name = 'e3sm.historical.v3-LR.0101' + model_tableID = 'Amon' + ################################################################################## + #location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/mean_climate/cmip6/historical + cmip_clim_set = "cmip6.historical.v20250707" + ################################################################################## + #variables in current pcmdi-cmip list + #note: rltcre and rstcre are derived variables as they are not included in cmip_ts + vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rsus,rsuscs,rlus,rlut,rlutcs,rsds,rsdscs,rsdt,rsut,rsutcs,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" + #default for all variables or specify source for each variable separately. + obs_sets = "default" + #see zppy/templates/pcmdi_diags/regions_specs.json for options of all regions + #below is default setup for global/regional mean climate metrics + regions = "global,ocean,land,NHEX,SHEX,TROPICS" + target_grid = "2.5x2.5" + target_grid_string = "2p5x2p5" + + [[ variability_modes_cpl ]] + active = True + ################################################################################## + ts_years = "2005-2014", + ts_num_years = 5 + ref_years = "2005-2014", + ref_start_yr = 1870 + ref_final_yr = 2019 + #create rules to construct cmip-like datasets with a string constructed as + #"mip.exp.product.realization", where "."is used as the delimiter (mandatary) + model_name = 'e3sm.historical.v3-LR.0101' + model_tableID = 'Amon' + ################################################################################## + #location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes/cmip6/historical + cmip_movs_set = "cmip6.historical.v20220825" + ################################################################################## + grid = '180x360_aave' + vars = "ts" + frequency = 'mo' + ModUnitsAdjust = '(True,"subtract",273.15)' + landmask = True + #see zppy/templates/pcmdi_diags/reference_alias.json for options + obs_sets = "alternate1" #"default" + ObsUnitsAdjust = '(True,"subtract",273.15)' + #name of modes varibility + cpl_modes = "PDO,NPGO,AMO" + #options specific for mode varibility metrics in pcmdi + seasons = "yearly,monthly" + figure_format = "png" + + [[ variability_modes_atm ]] + active = True + ################################################################################## + ts_years = "2005-2014", + ts_num_years = 5 + ref_years = "2005-2014", + ref_start_yr = 1853 + ref_final_yr = 2015 + #create rules to construct cmip-like datasets with a string constructed as + #"mip.exp.product.realization", where "." is used as the delimiter (mandatary) + model_name = 'e3sm.historical.v3-LR.0101' + model_tableID = 'Amon' + ################################################################################## + #location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes/cmip6/historical + cmip_movs_set = "cmip6.historical.v20220825" + ################################################################################## + grid = '180x360_aave' + vars = "psl" + landmask = False + frequency = 'mo' + ModUnitsAdjust = '(True,"divide",100.0)' + #observational data + obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series' + #see zppy/templates/pcmdi_diags/reference_alias.json for options + #default for all variables + obs_sets = "alternate2" # "default" + ObsUnitsAdjust = '(True,"divide",100.0)' + #varibility modes + atm_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" + #options specific for mode varibility metrics in pcmdi + seasons = "DJF,MAM,JJA,SON,yearly,monthly" + figure_format = "png" + + [[ enso ]] + active = True + ################################################################################## + ts_years = "2005-2014", + ts_num_years = 5 + ref_years = "2005-2014", + ref_start_yr = 1979 + ref_final_yr = 2018 + #create rules to construct cmip-like datasets with a string constructed as + #"mip.exp.product.realization", where "."is used as the delimiter (mandatary) + model_name = 'e3sm.historical.v3-LR.0101' + model_tableID = 'Amon' + ################################################################################## + #location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/enso_metric/cmip6/historical + cmip_enso_set = "cmip6.historical.v20210620" + ################################################################################## + #see zppy/templates/pcmdi_diags/reference_alias.json for options + customized_cmor_obs = True + vars = "psl,pr,prsn,ts,tas,tauu,tauv,hfls,hfss,rlds,rsds,rlus,rlut,rsdt" + # For enso, the obs_sets is explicitely set for each variable to ensure that + # they are from the same source, below are setup to used ERA5 reanalysis. + obs_sets = "default,alternate2,default,default,default,default,default,default,default,alternate2,alternate2,alternate2,alternate2,alternate2" + #see zppy/templates/pcmdi_diags/reference_alias.json for options + enso_groups = "ENSO_perf,ENSO_proc,ENSO_tel" + figure_format = "png" + + [[ synthetic_plots ]] + active = True + model_name = 'e3sm.historical.v3-LR.0101' + model_tableID = 'Amon' + synthetic_sets = "portrait,parcoord" + figure_sets = "mean_climate,variability_modes,enso_metric" + figure_sets_period = "2005-2014,2005-2014,2005-2014" + figure_format = "png" diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index 49ff7ea1..ba496610 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -530,7 +530,7 @@ cmip_enso_dir = string(default="") cmip_clim_dir = string(default="") cmip_movs_dir = string(default="") #group of pcmdi generated cmip metrics data (mip.exp.version) -cmip_clim_set = string(default="cmip6.historical.v20250108") +cmip_clim_set = string(default="cmip6.historical.v20250707") cmip_movs_set = string(default="cmip6.historical.v20220825") cmip_enso_set = string(default="cmip6.historical.v20210620") # options shared by pcmdi From 56ee4d0db16ccd079351674215789826691480f9 Mon Sep 17 00:00:00 2001 From: ShixuanZhang Date: Tue, 16 Sep 2025 14:50:46 -0500 Subject: [PATCH 09/47] Modifications to reproduce the modes varibility in initial implementation --- zppy/defaults/default.ini | 4 +++- zppy/templates/pcmdi_diags.bash | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index ba496610..902e7198 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -487,6 +487,7 @@ RmDomainMean = string(default=True) EofScaling = string(default=False) ConvEOF = string(default=True) CBF = string(default=True) +eofn_mod_max = integer(default=3) mov_plot_obs = string(default=True) mov_plot_model = string(default=True) mov_nc_out_obs = string(default=True) @@ -541,6 +542,7 @@ e3sm_to_cmip_atm_subsection = string(default="") [[__many__]] vars = string(default=None) grid = string(default=None) + num_workers = integer(default=None) cmip_enso_set = string(default=None) cmip_clim__set = string(default=None) cmip_movs_set = string(default=None) @@ -557,7 +559,6 @@ e3sm_to_cmip_atm_subsection = string(default="") pcmdi_viewer_template = string(default=None) synthetic_metrics = string(default=None) multiprocessing = boolean(default=None) - num_workers = integer(default=None) obs_ts = string(default=None) figure_sets = string(default=None) figure_sets_period = string(default=None) @@ -586,6 +587,7 @@ e3sm_to_cmip_atm_subsection = string(default="") EofScaling = string(default=None) ConvEOF = string(default=None) CBF = string(default=None) + eofn_mod_max = integer(default=None) landmask = string(default=None) frequency = string(default=None) generate_sftlf = string(default=None) diff --git a/zppy/templates/pcmdi_diags.bash b/zppy/templates/pcmdi_diags.bash index 663ef5e6..7c7aacd7 100755 --- a/zppy/templates/pcmdi_diags.bash +++ b/zppy/templates/pcmdi_diags.bash @@ -699,6 +699,9 @@ ObsUnitsAdjust = {{ ObsUnitsAdjust }} # Mask out land regions (consider ocean-only if True) landmask = {{ landmask }} +# Maximum number of eof modes +eofn_mod_max = {{ eofn_mod_max }} + # If True, remove domain mean from each time step RmDomainMean = {{ RmDomainMean }} From 56b83cddb93bf2c11978185ce899b36657594e4f Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 17 Sep 2025 15:45:52 -0500 Subject: [PATCH 10/47] Testing changes --- examples/post.v3.LR.amip.0101.cfg | 14 +++++--------- zppy/defaults/default.ini | 1 + zppy/templates/pcmdi_diags.bash | 10 +++++----- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/examples/post.v3.LR.amip.0101.cfg b/examples/post.v3.LR.amip.0101.cfg index 8fd8f171..054a8290 100644 --- a/examples/post.v3.LR.amip.0101.cfg +++ b/examples/post.v3.LR.amip.0101.cfg @@ -1,13 +1,7 @@ -# To run: -# `pip install .` the zppy-interfaces changes into the `zi-pcmdi-20250529` environment -# Update unique_id in output, www below -# Be in the zppy-pr719-dev-20250612 environment -# > pip install . && zppy -c examples/post.v3.LR.amip.0101.cfg - [default] input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/AMIP/v3.LR.amip_0101 -output = /lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy_pr719_output/unique_id_21/v3.LR.amip_0101 -www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.szhang/zppy_pr719_output/unique_id_21 +output = /lcrc/group/e3sm/ac.forsyth2/zppy_pr719_output/unique_id_22/v3.LR.amip_0101 +www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pr719_output/unique_id_22 campaign = "water_cycle" case = v3.LR.amip_0101 debug = True @@ -38,7 +32,8 @@ years = "2005:2015:5" [[ atm_monthly_180x360_aave ]] # list of variables to be interpolated to pressure level; see e3sm_to_cmip/vrt_remap_plev19.nc - cmip_plevdata = /lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc + # This defaults to cmip_plevdata = string(default="inclusions/e3sm_to_cmip/vrt_remap_plev19.nc") + # cmip_plevdata = /lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc input_files = "eam.h0" interp_vars = "U,V,T,Z3" ts_subsection = "atm_monthly_180x360_aave" @@ -58,6 +53,7 @@ reference_data_path_ts = "" ts_grid = "180x360_aave" walltime = "4:00:00" environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" +environment_commands_secondary = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20250917" [[ mean_climate ]] active = True diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index 902e7198..b9d9b2fd 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -20,6 +20,7 @@ dry_run = boolean(default=False) # Set up the environment -- this is where you can tell zppy to use a custom conda environment. # To use a custom conda environment, you can set `environment_commands="source ; conda activate "`. environment_commands = string(default="") +environment_commands_secondary = string(default="") # If set to True, zppy will fail as soon as a job is unable to launch because of a missing dependency. # If set to False, zppy will launch other jobs, if possible. fail_on_dependency_skip = boolean(default=False) diff --git a/zppy/templates/pcmdi_diags.bash b/zppy/templates/pcmdi_diags.bash index 7c7aacd7..4194cb10 100755 --- a/zppy/templates/pcmdi_diags.bash +++ b/zppy/templates/pcmdi_diags.bash @@ -488,9 +488,9 @@ echo "Linking observational data using SLURM..." command="zi-pcmdi-link-observation --model_name_ref ${model_name_ref} --tableID_ref ${tableID_ref} --vars={{ vars }} --obs_sets {{ obs_sets }} --obs_ts {{ obs_ts }} --obstmp_dir ${obstmp_dir}" echo "Running: ${command}" -source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20250730 +{{ environment_commands_secondary }} time eval "${command}" -source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh +{{ environment_commands }} if [ $? -ne 0 ]; then cd {{ scriptDir }} @@ -699,7 +699,7 @@ ObsUnitsAdjust = {{ ObsUnitsAdjust }} # Mask out land regions (consider ocean-only if True) landmask = {{ landmask }} -# Maximum number of eof modes +# Maximum number of eof modes eofn_mod_max = {{ eofn_mod_max }} # If True, remove domain mean from each time step @@ -835,7 +835,7 @@ command="zi-pcmdi-synthetic-plots --synthetic_sets {{ synthetic_sets }} --figure {% endif %} # Run diagnostics -source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20250730 +{{ environment_commands_secondary }} echo "About to run a zi-pcmdi command" echo "The current directory is: $PWD" # This will be of the form .../post/scripts/tmpDir time ${command} @@ -844,7 +844,7 @@ if [ $? != 0 ]; then echo 'ERROR (11)' > {{ prefix }}.status exit 11 fi -source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh +{{ environment_commands }} ################################# # Copy output to web server From 607ad1fdeb50a509ca4aafeb85e0362154d58d47 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 17 Sep 2025 18:01:33 -0500 Subject: [PATCH 11/47] Update parameters --- zppy/defaults/default.ini | 14 ++------------ zppy/pcmdi_diags.py | 1 - zppy/templates/pcmdi_diags.bash | 10 +--------- 3 files changed, 3 insertions(+), 22 deletions(-) diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index b9d9b2fd..f36e1bbc 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -401,10 +401,6 @@ pcmdi_version = string(default="v3.8.2") regions_specs = string(default="pcmdi_data/region/regions_specs.json") # File of observation data name for mean climate calculation reference_alias = string(default="pcmdi_data/reference/reference_alias.json") -# Utility file with functions for zppy-pcmdi data processing -pcmdi_zppy_util = string(default="pcmdi_data/utility/pcmdi_zppy_util.py") -# Utility file with functions for zppy-pcmdi viewer page processing -pcmdi_viewer_util = string(default="pcmdi_data/utility/pcmdi_viewer_util.py") # Template files for zppy-pcmdi viewer page processing pcmdi_viewer_template = string(default="pcmdi_data/viewer") # File of variable list to generate synthetic metrics plot @@ -443,14 +439,10 @@ reference_data_path = string(default="") reference_data_path_ts = string(default="") # pcmdi_diags.py will set to match `years` if not specified ref_years = string_list(default=list("")) -# The years increment for reference data -ts_num_years_ref = integer(default=5) # Final year (i.e., the last available year) for the reference data ref_final_yr = string(default="") # Start year for the reference data ref_start_yr = string(default="") -# reference model name -ref_name = string(default="") # The years range for test model data ts_years = string_list(default=list("")) # The years increment for test model data @@ -519,6 +511,7 @@ target_grid = string(default="1x1") target_grid_string = string(default="1px1p") # OPTIONS: 'regrid2','esmf' regrid_tool = string(default="esmf") +regrid_tool_ocn = string(default="esmf") # OPTIONS: 'linear','conservative', only if tool is esmf regrid_method = string(default="regrid2") # OPTIONS: 'linear','conservative', only if tool is esmf @@ -555,8 +548,6 @@ e3sm_to_cmip_atm_subsection = string(default="") regions_specs = string(default=None) pcmdi_webtitle = string(default=None) pcmdi_version = string(default=None) - pcmdi_zppy_util = string(default=None) - pcmdi_viewer_util = string(default=None) pcmdi_viewer_template = string(default=None) synthetic_metrics = string(default=None) multiprocessing = boolean(default=None) @@ -564,7 +555,6 @@ e3sm_to_cmip_atm_subsection = string(default="") figure_sets = string(default=None) figure_sets_period = string(default=None) figure_format = string(default=None) - ref_name = string(default=None) ref_years = string_list(default=None) ref_start_yr = string(default=None) ref_final_yr = string(default=None) @@ -574,10 +564,10 @@ e3sm_to_cmip_atm_subsection = string(default="") sets = string_list(default=None) ts_years = string_list(default=None) ts_num_years = integer(default=None) - ts_num_years_ref = integer(default=None) target_grid = string(default=None) target_grid_string = string(default=None) regrid_tool = string(default=None) + regrid_tool_ocn = string(default=None) regrid_method = string(default=None) regrid_method_ocn = string(default=None) obs_sets = string(default=None) diff --git a/zppy/pcmdi_diags.py b/zppy/pcmdi_diags.py index bdeeaaa4..dbff0e3f 100644 --- a/zppy/pcmdi_diags.py +++ b/zppy/pcmdi_diags.py @@ -215,7 +215,6 @@ def check_mvm_only_parameters_for_bash(c: Dict[str, Any]) -> None: "enso", ] ) - check_set_specific_parameter(c, ts_sets, "ts_num_years_ref") check_set_specific_parameter(c, ts_sets, "ts_subsection") diff --git a/zppy/templates/pcmdi_diags.bash b/zppy/templates/pcmdi_diags.bash index 4194cb10..23faa82f 100755 --- a/zppy/templates/pcmdi_diags.bash +++ b/zppy/templates/pcmdi_diags.bash @@ -30,7 +30,6 @@ case="{{ case }}" www="{{ www }}" run_type="{{ run_type }}" results_dir="{{ run_type }}" -ref_name={{ ref_name }} {% if "synthetic_plots" not in subsection %} # Input variables @@ -92,11 +91,7 @@ workdir=`mktemp -d tmp.${id}.XXXX` cd ${workdir} -# TODO: Move anything in {{pcmdi_external_prefix}} == /lcrc/group/e3sm/diagnostics/ to the inclusions subdir -# or to zppy-interfaces - -# utility file for pcmdi-zppy workflow -cp -r '{{pcmdi_external_prefix}}/{{pcmdi_zppy_util}}' . +# TODO: Move anything in {{pcmdi_external_prefix}} == /lcrc/group/e3sm/diagnostics/ to the inclusions subdir or to zppy-interfaces. In theory, we would then not need to copy these files around. # files for definition of regions for regional mean cp -r '{{pcmdi_external_prefix}}/{{regions_specs}}' . @@ -107,9 +102,6 @@ cp -r '{{pcmdi_external_prefix}}/{{reference_alias}}' . # file for list of variables for synthetic_metrics metric plots cp -r '{{pcmdi_external_prefix}}/{{synthetic_metrics}}' . -# utility file for pcmdi-zppy viewer -cp -r '{{pcmdi_external_prefix}}/{{pcmdi_viewer_util}}' . - {% if "mean_climate" in subsection %} create_links_acyc_climo() { local ts_dir_source="$1" From 3de9d6e4ba9fae35b48dee5e1db954ca84ef9b49 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 18 Sep 2025 11:51:03 -0500 Subject: [PATCH 12/47] e3sm_to_cmip fixes --- examples/post.v3.LR.amip.0101.cfg | 1 - zppy/e3sm_to_cmip.py | 2 +- zppy/templates/e3sm_to_cmip.bash | 12 ++---------- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/examples/post.v3.LR.amip.0101.cfg b/examples/post.v3.LR.amip.0101.cfg index 054a8290..6b4a6a4a 100644 --- a/examples/post.v3.LR.amip.0101.cfg +++ b/examples/post.v3.LR.amip.0101.cfg @@ -24,7 +24,6 @@ years = "2005:2015:5" [e3sm_to_cmip] active = True frequency = "monthly" -grid = "180x360_aave" ts_grid = "180x360_aave" ts_num_years=5 walltime = "00:10:00" diff --git a/zppy/e3sm_to_cmip.py b/zppy/e3sm_to_cmip.py index 8ea1b151..6de0a28e 100644 --- a/zppy/e3sm_to_cmip.py +++ b/zppy/e3sm_to_cmip.py @@ -60,7 +60,7 @@ def e3sm_to_cmip(config: ConfigObj, script_dir: str, existing_bundles, job_ids_f ) elif c["component"] == "lnd": c["cmip_vars"] = ( - "mrsos, mrso, mrfso, mrros, mrro, prveg, evspsblveg, evspsblsoi, tran, tsl, lai, cLitter, cProduct, cSoilFast, cSoilMedium, cSoilSlow, fFire, fHarvest, cVeg, nbp, gpp, ra, rh" + "snd, mrsos, mrso, mrfso, mrros, mrro, prveg, evspsblveg, evspsblsoi, tran, tsl, lai, cLitter, cProduct, cSoilFast, cSoilMedium, cSoilSlow, fFire, fHarvest, cVeg, nbp, gpp, ra, rh" ) prefix = f"e3sm_to_cmip_{sub}_{c['yr_start']:04d}-{c['yr_end']:04d}-{c['ypf']:04d}" print(prefix) diff --git a/zppy/templates/e3sm_to_cmip.bash b/zppy/templates/e3sm_to_cmip.bash index 7e0638f7..91f85063 100644 --- a/zppy/templates/e3sm_to_cmip.bash +++ b/zppy/templates/e3sm_to_cmip.bash @@ -48,18 +48,10 @@ EOF srun -N 1 e3sm_to_cmip \ --output-path \ ${dest_cmip}/${tmp_dir} \ - {% if input_files.split(".")[0] == 'clm2' or input_files.split(".")[0] == 'elm' -%} --var-list \ - 'snd, mrsos, mrso, mrfso, mrros, mrro, prveg, evspsblveg, evspsblsoi, tran, tsl, lai, cLitter, cProduct, cSoilFast, cSoilMedium, cSoilSlow, fFire, fHarvest, cVeg, nbp, gpp, ra, rh' \ + {{ cmip_vars }} \ --realm \ - lnd \ - {% endif -%} - {% if input_files.split(".")[0] == 'cam' or input_files.split(".")[0] == 'eam' -%} - --var-list \ - 'ua, va, ta, wa, zg, hur, pr, prc, prsn, ts, tas, prw, psl, sfcWind, tasmax, tasmin, tauu, tauv, rtmt, rsdt, rsds, rsdscs, rlds, rldscs, rsus, rsuscs, rsut, rsutcs, rlus, rlut, rlutcs, clivi, clwvi, clt, evspsbl, hfls, hfss, huss' \ - --realm \ - atm \ - {% endif -%} + {{ component }} \ --input-path \ ${input_dir} \ --user-metadata \ From b3acce20b25664ae8974173e8fcf198ee1153dd3 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 18 Sep 2025 21:09:11 -0500 Subject: [PATCH 13/47] e3sm_to_cmip fixes follow-up --- examples/post.v3.LR.amip.0101.cfg | 4 ++-- zppy/templates/e3sm_to_cmip.bash | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/post.v3.LR.amip.0101.cfg b/examples/post.v3.LR.amip.0101.cfg index 6b4a6a4a..2ee4c407 100644 --- a/examples/post.v3.LR.amip.0101.cfg +++ b/examples/post.v3.LR.amip.0101.cfg @@ -1,7 +1,7 @@ [default] input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/AMIP/v3.LR.amip_0101 -output = /lcrc/group/e3sm/ac.forsyth2/zppy_pr719_output/unique_id_22/v3.LR.amip_0101 -www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pr719_output/unique_id_22 +output = /lcrc/group/e3sm/ac.forsyth2/zppy_pr719_output/unique_id_32/v3.LR.amip_0101 +www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pr719_output/unique_id_32 campaign = "water_cycle" case = v3.LR.amip_0101 debug = True diff --git a/zppy/templates/e3sm_to_cmip.bash b/zppy/templates/e3sm_to_cmip.bash index 91f85063..e5f37bde 100644 --- a/zppy/templates/e3sm_to_cmip.bash +++ b/zppy/templates/e3sm_to_cmip.bash @@ -24,7 +24,7 @@ EOF mkdir -p $input_dir cp -s $dest/*_{{ '%04d' % (yr_start) }}??_{{ '%04d' % (yr_end) }}??.nc $input_dir - dest_cmip={{ output }}/post/{{ component }}/{{ grid }}/cmip_ts/{{ frequency }} + dest_cmip={{ output }}/post/{{ component }}/{{ ts_grid }}/cmip_ts/{{ frequency }} mkdir -p ${dest_cmip} {% if input_files.split(".")[0] == 'cam' or input_files.split(".")[0] == 'eam' -%} @@ -49,7 +49,7 @@ EOF --output-path \ ${dest_cmip}/${tmp_dir} \ --var-list \ - {{ cmip_vars }} \ + '{{ cmip_vars }}' \ --realm \ {{ component }} \ --input-path \ From feb629637239babd1da30d720a29a212181cc231 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 18 Sep 2025 13:43:09 -0500 Subject: [PATCH 14/47] Add json to inclusions --- zppy/defaults/default.ini | 6 +- zppy/pcmdi_diags.py | 5 +- .../pcmdi_diags/reference_alias.json | 393 ++++++++++++++++++ .../inclusions/pcmdi_diags/regions_specs.json | 263 ++++++++++++ .../pcmdi_diags/synthetic_metrics_list.json | 58 +++ zppy/templates/pcmdi_diags.bash | 21 +- 6 files changed, 732 insertions(+), 14 deletions(-) create mode 100755 zppy/templates/inclusions/pcmdi_diags/reference_alias.json create mode 100755 zppy/templates/inclusions/pcmdi_diags/regions_specs.json create mode 100755 zppy/templates/inclusions/pcmdi_diags/synthetic_metrics_list.json diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index f36e1bbc..1a6c3c8f 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -398,13 +398,13 @@ pcmdi_webtitle = string(default="E3SM-PMP-Diagnostics") # Cannot have spaces! # Version of zppy-pcmdi code pcmdi_version = string(default="v3.8.2") # File of specified regions for mean climate calculation -regions_specs = string(default="pcmdi_data/region/regions_specs.json") +regions_specs = string(default="inclusions/pcmdi_diags/regions_specs.json") # File of observation data name for mean climate calculation -reference_alias = string(default="pcmdi_data/reference/reference_alias.json") +reference_alias = string(default="inclusions/pcmdi_diags/reference_alias.json") # Template files for zppy-pcmdi viewer page processing pcmdi_viewer_template = string(default="pcmdi_data/viewer") # File of variable list to generate synthetic metrics plot -synthetic_metrics = string(default="pcmdi_data/synthetic_metrics/synthetic_metrics_list.json") +synthetic_metrics = string(default="inclusions/pcmdi_diags/synthetic_metrics_list.json") # Path to observation time-series data # Required for "mean_climate","variability_mode","enso" obs_ts = string(default="") diff --git a/zppy/pcmdi_diags.py b/zppy/pcmdi_diags.py index dbff0e3f..46c8a0da 100644 --- a/zppy/pcmdi_diags.py +++ b/zppy/pcmdi_diags.py @@ -47,7 +47,6 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): c["cmor_tables_prefix"] = c["diagnostics_base_path"] # check and set parameter for pcmdi - c["pcmdi_external_prefix"] = c["diagnostics_base_path"] check_parameters_for_pcmdi(c) # Loop over year sets @@ -219,7 +218,9 @@ def check_mvm_only_parameters_for_bash(c: Dict[str, Any]) -> None: def check_and_define_parameters(c: Dict[str, Any]) -> None: - # TODO: do this based on sets, rather than by relying on the user setting ts_num_years + # TODO, future PR: do this based on sets, rather than by relying on the user setting ts_num_years + # This is the same item as for e3sm_diags. + # Here, we'd have to determine which sets of sets "mean_climate","variability_modes_atm","variability_modes_cpl","enso","synthetic_plots" require `ts` if "ts_num_years" in c.keys(): set_value_of_parameter_if_undefined( c, diff --git a/zppy/templates/inclusions/pcmdi_diags/reference_alias.json b/zppy/templates/inclusions/pcmdi_diags/reference_alias.json new file mode 100755 index 00000000..48cd6ebb --- /dev/null +++ b/zppy/templates/inclusions/pcmdi_diags/reference_alias.json @@ -0,0 +1,393 @@ +{ + "rlds" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "rldscs" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "rlus" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "rsds" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "rsdscs" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + + "rsus" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "rsuscs": { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "rstcre" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "rltcre" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "rlut" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "rlutcs" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "rsdt" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "rsut" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "rsutcs" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "rtmt" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "pr" : { + "defaultpi" : "NOAA-20C", + "default" : "GPCP_v2.3", + "alternate" : "GPCP_v2.2", + "alternate1" : "GPCP_1DD", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C", + "alternate6" : "GPCP_v3.2" + }, + "prc" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "NOAA-20C" + }, + "prsn" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "NOAA-20C" + }, + "prw" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "MERRA2", + "alternate2" : "ERA-Interim", + "alternate3" : "NOAA-20C" + }, + "psl" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C" + }, + "ps" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate " : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C" + }, + "huss" : { + "defaultpi" : "NOAA-20C", + "default" : "MERRA2", + "alternate" : "NOAA-20C", + "alternate1" : "ERA5", + "alternate2" : "ERA-Interim" + }, + "ta" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C" + }, + "ua" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C" + }, + "va" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C" + }, + "hur" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C" + }, + "wap" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C" + }, + "zg" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C" + }, + "o3" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C" + }, + "hus" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C" + }, + "uas" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C" + }, + "vas" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C" + }, + "tauu" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C", + "alternate3" : "COREv2-Flux" + }, + "taux" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C", + "alternate3" : "COREv2-Flux" + }, + "tauv" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C", + "alternate3" : "COREv2-Flux" + }, + "tauy" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C", + "alternate3" : "COREv2-Flux" + }, + "tas" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C" + }, + "ts" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "NOAA-20C", + "alternate1" : "HadISST2" + }, + "sst" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "NOAA-20C", + "alternate1" : "HadISST2" + }, + "sfcWind" : { + "defaultpi" : "NOAA-20C", + "default" : "NOAA-20C", + "alternate" : "ERA5", + "alternate1" : "MERRA2", + "alternate2" : "ERA-Interim" + }, + "hfls" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C", + "alternate3" : "OAFlux" + }, + "hfss" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C", + "alternate3" : "OAFlux" + }, + "evspsbl" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "NOAA-20C" + }, + "clt" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate3" : "NOAA-20C" + }, + "clwvi" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "NOAA-20C" + }, + "clivi" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "NOAA-20C" + }, + "tasmin" : { + "defaultpi" : "NOAA-20C", + "default" : "MERRA2" + }, + "tasmax" : { + "defaultpi" : "NOAA-20C", + "default" : "MERRA2" + }, + "sic" : { + "defaultpi" : "HadSST2", + "default" : "HadSST2" + }, + "tos" : { + "defaultpi" : "HadSST2", + "default" : "HadSST2" + }, + "zos" : { + "defaultpi" : "AVISO", + "default" : "AVISO", + "alternate" : "HadISST" + }, + "sos" : { + "defaultpi" : "Aquarius", + "default" : "Aquarius", + "alternate" : "HadISST" + } +} diff --git a/zppy/templates/inclusions/pcmdi_diags/regions_specs.json b/zppy/templates/inclusions/pcmdi_diags/regions_specs.json new file mode 100755 index 00000000..811eb1e9 --- /dev/null +++ b/zppy/templates/inclusions/pcmdi_diags/regions_specs.json @@ -0,0 +1,263 @@ +{ + "global": { + "domain": { "latitude":[-90.0, 90.0]} + }, + "NH": { + "domain": { "latitude":[0.0, 90.0]} + }, + "SH": { + "domain": { "latitude":[-90.0, 0]} + }, + "NHEX": { + "domain": { "latitude":[30.0, 90.0]} + }, + "SHEX": { + "domain": { "latitude":[-90.0, -30.0]} + }, + "TROPICS": { + "domain": { "latitude":[-30.0, 30.0]} + }, + "90S50S": { + "domain": { "latitude":[-90.0, -50.0]} + }, + "50S20S": { + "domain": { "latitude":[-50.0, -20.0]} + }, + "20S20N": { + "domain": { "latitude":[-20.0, 20.0]} + }, + "20N50N": { + "domain": { "latitude":[20.0, 50.0]} + }, + "50N90N": { + "domain": { "latitude":[50.0, 90.0]} + }, + "ocean_NH": { + "value": 0.0, + "domain": { "latitude":[0.0, 90.0]} + }, + "ocean_SH": { + "value": 0.0, + "domain": { "latitude":[-90.0, 0.0]} + }, + "land_NH": { + "value": 100, + "domain": { "latitude":[0.0, 90.0]} + }, + "land_SH": { + "value": 100, + "domain": { "latitude":[-90.0, 0.0]} + }, + "land_NHEX": { + "value": 100, + "domain": { "latitude":[30.0, 90.0]} + }, + "land_SHEX": { + "value": 100, + "domain": { "latitude":[-90.0, -30.0]} + }, + "land_TROPICS": { + "value": 100, + "domain": { "latitude":[-30.0, 30.0]} + }, + "land": { + "value": 100 + }, + "ocean_NHEX": { + "value": 0, + "domain": { "latitude":[30.0, 90.0]} + }, + "ocean_SHEX": { + "value": 0, + "domain": { "latitude":[-90.0, -30.0]} + }, + "ocean_TROPICS": { + "value": 0, + "domain": { "latitude":[30.0, 30.0]} + }, + "ocean": { + "value": 0 + }, + "ocean_50S50N": { + "value": 0.0, + "domain": { "latitude":[-50.0, 50.0]} + }, + "ocean_50S20S": { + "value": 0.0, + "domain": { "latitude":[-50.0, -20.0]} + }, + "ocean_20S20N": { + "value": 0.0, + "domain": { "latitude":[-20.0, 20.0]} + }, + "ocean_20N50N": { + "value": 0.0, + "domain": { "latitude":[20.0, 50.0]} + }, + "ocean_50N90N": { + "value": 0.0, + "domain": { "latitude":[50.0, 90.0]} + }, + "ocean_90S50S": { + "value": 0.0, + "domain": { "latitude":[-90.0, -50.0]} + }, + "NAM": { + "domain": { "latitude":[20.0, 90], + "longitude":[-180, 180]} + }, + "NAO": { + "domain": { "latitude":[20.0, 80], + "longitude":[-90, 40]} + }, + "SAM": { + "domain": { "latitude":[-20.0, -90], + "longitude":[0, 360]} + }, + "PSA1": { + "domain": { "latitude":[-20.0, -90], + "longitude":[0, 360]} + }, + "PSA2": { + "domain": { "latitude":[-20.0, -90], + "longitude":[0, 360]} + }, + "PNA": { + "domain": { "latitude":[20.0, 85], + "longitude":[120, 240]} + }, + "PDO": { + "domain": { "latitude":[20.0, 70], + "longitude":[110, 260]} + }, + "AMO": { + "domain": { "latitude":[0.0, 70], + "longitude":[-80, 0]} + }, + "AllMW": { + "domain": { "latitude":[-40.0, 45.0], + "longitude":[0.0, 360.0]} + }, + "AllM": { + "domain": { "latitude":[-45.0, 45.0], + "longitude":[0.0, 360.0]} + }, + "NAMM": { + "domain": { "latitude":[0.0, 45.0], + "longitude":[210.0, 310.0]} + }, + "SAMM": { + "domain": { "latitude":[-45.0, 0.0], + "longitude":[240.0, 330.0]} + }, + "NAFM": { + "domain": { "latitude":[0.0, 45.0], + "longitude":[310.0, 60.0]} + }, + "SAFM": { + "domain": { "latitude":[-45.0, 0.0], + "longitude":[0.0, 90.0]} + }, + "ASM": { + "domain": { "latitude":[0.0, 45.0], + "longitude":[60.0, 180.0]} + }, + "AUSM": { + "domain": { "latitude":[-45.0, 0.0], + "longitude":[90.0, 160.0]} + }, + "AIR": { + "domain": { "latitude":[7.0, 25.0], + "longitude":[65.0, 85.0]} + }, + "AUS": { + "domain": { "latitude":[-20.0, -10.0], + "longitude":[120.0, 150.0]} + }, + "Sahel": { + "domain": { "latitude":[13.0, 18.0], + "longitude":[-10.0, 10.0]} + }, + "GoG": { + "domain": { "latitude":[0.0, 5.0], + "longitude":[-10.0, 10.0]} + }, + "NAmo": { + "domain": { "latitude":[20.0, 37.0], + "longitude":[-112.0, -103.0]} + }, + "SAmo": { + "domain": { "latitude":[-20.0, 2.5], + "longitude":[-65.0, -40.0]} + }, + "Nino34": { + "value": 0.0, + "domain": { "latitude":[-5.0, 5.0], + "longitude":[190.0, 240.0]} + }, + "Nino3": { + "value": 0.0, + "domain": { "latitude":[-5.0, 5.0], + "longitude":[210.0, 270.0]} + }, + "Nino4": { + "value": 0.0, + "domain": { "latitude":[-5.0, 5.0], + "longitude":[160.0, 210.0]} + }, + "ONI": { + "value": 0.0, + "domain": { "latitude":[-5.0, 5.0], + "longitude":[190.0, 240.0]} + }, + "Nino12": { + "value": 0.0, + "domain": { "latitude":[-10.0, 0.0], + "longitude":[270.0, 280.0]} + }, + "AMMS": { + "value": 0.0, + "domain": { "latitude":[-15.0, -5.0], + "longitude":[-20.0, 10.0]} + }, + "AMMN": { + "value": 0.0, + "domain": { "latitude":[5.0, 15.0], + "longitude":[-50.0, -20.0]} + }, + "ATL3": { + "value": 0.0, + "domain": { "latitude":[-3.0, 3.0], + "longitude":[-20.0, 0.0]} + }, + "TSA": { + "value": 0.0, + "domain": { "latitude":[-20.0, 0.0], + "longitude":[-30.0, 10.0]} + }, + "TNA": { + "value": 0.0, + "domain": { "latitude":[5.5, 23.5], + "longitude":[302.5, 345.0]} + }, + "TIO": { + "value": 0.0, + "domain": { "latitude":[-15.0, 15.0], + "longitude":[40.0, 115.0]} + }, + "IODE": { + "value": 0.0, + "domain": { "latitude":[-10.0, 10.0], + "longitude":[50.0, 70.0]} + }, + "IODW": { + "value": 0.0, + "domain": { "latitude":[-10.0, 0.0], + "longitude":[90.0, 110.0]} + }, + "SOCN": { + "value": 0.0, + "domain": { "latitude":[-70.0, -50.0], + "longitude":[0.0, 360.0]} + } +} diff --git a/zppy/templates/inclusions/pcmdi_diags/synthetic_metrics_list.json b/zppy/templates/inclusions/pcmdi_diags/synthetic_metrics_list.json new file mode 100755 index 00000000..e9262d2a --- /dev/null +++ b/zppy/templates/inclusions/pcmdi_diags/synthetic_metrics_list.json @@ -0,0 +1,58 @@ +{ + "mean_climate" : { + "mae_xy" : { + "name" : "Mean Absolute Error Normalized by Median", + "type" : ["portrait"], + "region" : ["global","ocean","land","NHEX","SHEX","TROPICS"], + "season" : ["djf", "mam", "jja", "son"] + }, + "cor_xy" : { + "name" : "Pattern Corr.", + "type" : ["portrait"], + "region" : ["global","ocean","land","NHEX","SHEX","TROPICS"], + "season" : ["djf", "mam", "jja", "son"] + }, + "rms_xy" : { + "name" : "RMSE Normalized by Median", + "type" : ["portrait"], + "region" : ["global","ocean","land","NHEX","SHEX","TROPICS"], + "season" : ["djf", "mam", "jja", "son"] + }, + "bias_xy" : { + "name" : "Annual Mean Bias", + "type" : ["parcoord"], + "region" : ["global","ocean","land","NHEX","SHEX","TROPICS"], + "season" : ["ann"] + }, + "rms_xyt" : { + "name" : "Spatiotemporal RMSE", + "type" : ["parcoord"], + "region" : ["global","ocean","land","NHEX","SHEX","TROPICS"], + "season" : ["ann"] + } + }, + "variability_modes" : { + "rms" : { + "name" : "RMSE Normalized by Median", + "type" : ["portrait","parcoord"], + "season" : [] + }, + "rmsc" : { + "name" : "CenteredRMSE Normalized by Median", + "type" : ["portrait","parcoord"], + "season" : [] + }, + "stdv_pc_ratio_to_obs" : { + "name" : "Amplitude Ratio to Reference", + "type" : ["portrait","parcoord"], + "season" : [] + } + }, + "enso_metric" : { + "skill" : { + "collection" : ["ENSO_perf", "ENSO_tel", "ENSO_proc"], + "type" : ["portrait"], + "season" : [] + } + } +} diff --git a/zppy/templates/pcmdi_diags.bash b/zppy/templates/pcmdi_diags.bash index 23faa82f..53e9e51c 100755 --- a/zppy/templates/pcmdi_diags.bash +++ b/zppy/templates/pcmdi_diags.bash @@ -87,20 +87,23 @@ tableID_ref='{{ model_tableID_ref }}' case_id=v$(date '+%Y%m%d') # Create temporary workdir -workdir=`mktemp -d tmp.${id}.XXXX` +workdir=`mktemp -d tmp.{{ prefix }}.${id}.XXXX` cd ${workdir} - -# TODO: Move anything in {{pcmdi_external_prefix}} == /lcrc/group/e3sm/diagnostics/ to the inclusions subdir or to zppy-interfaces. In theory, we would then not need to copy these files around. - # files for definition of regions for regional mean -cp -r '{{pcmdi_external_prefix}}/{{regions_specs}}' . +cat > regions_specs.json << EOF +{{ regions_specs }} +EOF # file for aliases of observation datasets -cp -r '{{pcmdi_external_prefix}}/{{reference_alias}}' . +cat > reference_alias.json << EOF +{{ reference_alias }} +EOF # file for list of variables for synthetic_metrics metric plots -cp -r '{{pcmdi_external_prefix}}/{{synthetic_metrics}}' . +cat > synthetic_metrics.json << EOF +{{ synthetic_metrics }} +EOF {% if "mean_climate" in subsection %} create_links_acyc_climo() { @@ -433,7 +436,7 @@ create_links_ts_obs() { # Define output directory for climatology files climo_dir_primary="climo" # Path to model's monthly climatology files -# TODO: how do we have cmip_ts under climo?? +# This cmip_ts path is created via zppy/templates/e3sm_to_cmip.bash climo_dir_source="{{ output }}/post/atm/{{ grid }}/cmip_ts/monthly" # Link and process primary model climo data create_links_acyc_climo "${climo_dir_source}" "${climo_dir_primary}" "${Y1}" "${Y2}" "${model_name}.${tableID}" 1 @@ -823,7 +826,7 @@ command="zi-pcmdi-enso ${core_parameters} --enso_groups {{ enso_groups }}" {% if "synthetic_plots" in subsection %} # Note: figure_sets_period changed to a string # Note: sub_sets was renamed to figure_sets to be clearer -command="zi-pcmdi-synthetic-plots --synthetic_sets {{ synthetic_sets }} --figure_format {{ figure_format }} --www ${www} --results_dir ${results_dir} --case {{ case }} --model_name {{ model_name }} --model_tableID {{model_tableID }} --web_dir=${web_dir} --cmip_clim_dir {{ cmip_clim_dir }} --cmip_clim_set {{ cmip_clim_set }} --cmip_movs_dir {{ cmip_movs_dir }} --cmip_movs_set {{ cmip_movs_set }} --atm_modes {{ atm_modes }} --cpl_modes {{ cpl_modes }} --cmip_enso_dir {{ cmip_enso_dir }} --cmip_enso_set {{ cmip_enso_set }} --figure_sets {{ figure_sets }} --pcmdi_webtitle {{ pcmdi_webtitle }} --pcmdi_version {{ pcmdi_version }} --run_type ${run_type} --figure_sets_period {{ figure_sets_period }} --pcmdi_external_prefix {{ pcmdi_external_prefix }} --pcmdi_viewer_template {{ pcmdi_viewer_template }}" +command="zi-pcmdi-synthetic-plots --synthetic_sets {{ synthetic_sets }} --figure_format {{ figure_format }} --www ${www} --results_dir ${results_dir} --case {{ case }} --model_name {{ model_name }} --model_tableID {{model_tableID }} --web_dir=${web_dir} --cmip_clim_dir {{ cmip_clim_dir }} --cmip_clim_set {{ cmip_clim_set }} --cmip_movs_dir {{ cmip_movs_dir }} --cmip_movs_set {{ cmip_movs_set }} --atm_modes {{ atm_modes }} --cpl_modes {{ cpl_modes }} --cmip_enso_dir {{ cmip_enso_dir }} --cmip_enso_set {{ cmip_enso_set }} --figure_sets {{ figure_sets }} --pcmdi_webtitle {{ pcmdi_webtitle }} --pcmdi_version {{ pcmdi_version }} --run_type ${run_type} --figure_sets_period {{ figure_sets_period }} --pcmdi_external_prefix {{ diagnostics_base_path }} --pcmdi_viewer_template {{ pcmdi_viewer_template }}" {% endif %} # Run diagnostics From 1a3e5f75c80d2d755ced14f35186fab9dcd03fc5 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 18 Sep 2025 23:03:43 -0500 Subject: [PATCH 15/47] Add json to inclusions follow-up --- examples/post.v3.LR.amip.0101.cfg | 4 ++-- zppy/defaults/default.ini | 4 ++-- zppy/templates/pcmdi_diags.bash | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/post.v3.LR.amip.0101.cfg b/examples/post.v3.LR.amip.0101.cfg index 2ee4c407..672ce69b 100644 --- a/examples/post.v3.LR.amip.0101.cfg +++ b/examples/post.v3.LR.amip.0101.cfg @@ -1,7 +1,7 @@ [default] input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/AMIP/v3.LR.amip_0101 -output = /lcrc/group/e3sm/ac.forsyth2/zppy_pr719_output/unique_id_32/v3.LR.amip_0101 -www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pr719_output/unique_id_32 +output = /lcrc/group/e3sm/ac.forsyth2/zppy_pr719_output/unique_id_34/v3.LR.amip_0101 +www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pr719_output/unique_id_34 campaign = "water_cycle" case = v3.LR.amip_0101 debug = True diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index 1a6c3c8f..e6a6ec59 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -404,7 +404,7 @@ reference_alias = string(default="inclusions/pcmdi_diags/reference_alias.json") # Template files for zppy-pcmdi viewer page processing pcmdi_viewer_template = string(default="pcmdi_data/viewer") # File of variable list to generate synthetic metrics plot -synthetic_metrics = string(default="inclusions/pcmdi_diags/synthetic_metrics_list.json") +synthetic_metrics_list = string(default="inclusions/pcmdi_diags/synthetic_metrics_list.json") # Path to observation time-series data # Required for "mean_climate","variability_mode","enso" obs_ts = string(default="") @@ -549,7 +549,7 @@ e3sm_to_cmip_atm_subsection = string(default="") pcmdi_webtitle = string(default=None) pcmdi_version = string(default=None) pcmdi_viewer_template = string(default=None) - synthetic_metrics = string(default=None) + synthetic_metrics_list = string(default=None) multiprocessing = boolean(default=None) obs_ts = string(default=None) figure_sets = string(default=None) diff --git a/zppy/templates/pcmdi_diags.bash b/zppy/templates/pcmdi_diags.bash index 53e9e51c..7fac49af 100755 --- a/zppy/templates/pcmdi_diags.bash +++ b/zppy/templates/pcmdi_diags.bash @@ -92,17 +92,17 @@ cd ${workdir} # files for definition of regions for regional mean cat > regions_specs.json << EOF -{{ regions_specs }} +{% include regions_specs %} EOF # file for aliases of observation datasets cat > reference_alias.json << EOF -{{ reference_alias }} +{% include reference_alias %} EOF # file for list of variables for synthetic_metrics metric plots -cat > synthetic_metrics.json << EOF -{{ synthetic_metrics }} +cat > synthetic_metrics_list.json << EOF +{% include synthetic_metrics_list %} EOF {% if "mean_climate" in subsection %} From 3439a789a0ab25d4b0e866ba51899f630fc34ed5 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 18 Sep 2025 13:59:22 -0500 Subject: [PATCH 16/47] Fix unit tests --- tests/test_sections.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_sections.py b/tests/test_sections.py index 218ecc66..1a430340 100644 --- a/tests/test_sections.py +++ b/tests/test_sections.py @@ -73,6 +73,7 @@ def test_sections(): "debug": False, "dry_run": False, "environment_commands": "", + "environment_commands_secondary": "", "fail_on_dependency_skip": False, "frequency": "monthly", "grid": "", @@ -134,6 +135,7 @@ def test_sections(): "dpf": 30, "dry_run": False, "environment_commands": "", + "environment_commands_secondary": "", "extra_vars": "", "fail_on_dependency_skip": False, "frequency": "monthly", @@ -196,6 +198,7 @@ def test_sections(): "debug": False, "dry_run": False, "environment_commands": "", + "environment_commands_secondary": "", "exclude": False, "fail_on_dependency_skip": False, "frequency": "monthly", @@ -260,6 +263,7 @@ def test_subsections(): "debug": False, "dry_run": False, "environment_commands": "", + "environment_commands_secondary": "", "fail_on_dependency_skip": False, "frequency": "monthly", "grid": "", @@ -337,6 +341,7 @@ def test_subsections(): "dpf": 30, "dry_run": False, "environment_commands": "", + "environment_commands_secondary": "", "extra_vars": "", "fail_on_dependency_skip": False, "frequency": "monthly", @@ -384,6 +389,7 @@ def test_subsections(): "dpf": 30, "dry_run": False, "environment_commands": "", + "environment_commands_secondary": "", "extra_vars": "", "fail_on_dependency_skip": False, "frequency": "monthly", @@ -462,6 +468,7 @@ def test_subsections(): "debug": False, "dry_run": False, "environment_commands": "", + "environment_commands_secondary": "", "exclude": False, "fail_on_dependency_skip": False, "frequency": "monthly", @@ -506,6 +513,7 @@ def test_subsections(): "debug": False, "dry_run": False, "environment_commands": "", + "environment_commands_secondary": "", "exclude": False, "fail_on_dependency_skip": False, "frequency": "monthly", From 6a67d02b99b6971a20140c2a711794650bccfb6d Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 22 Sep 2025 19:52:38 -0500 Subject: [PATCH 17/47] Disable ENSO --- zppy/pcmdi_diags.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/zppy/pcmdi_diags.py b/zppy/pcmdi_diags.py index 46c8a0da..ae357413 100644 --- a/zppy/pcmdi_diags.py +++ b/zppy/pcmdi_diags.py @@ -2,6 +2,7 @@ from typing import Any, Dict, List, Set, Tuple from zppy.bundle import handle_bundles +from zppy.logger import _setup_custom_logger from zppy.utils import ( ParameterInferenceType, add_dependencies, @@ -20,6 +21,8 @@ write_settings_file, ) +logger = _setup_custom_logger(__name__) + # ----------------------------------------------------------------------------- def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): @@ -34,6 +37,16 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): # --- Generate and submit pcmdi_diags scripts --- for c in tasks: dependencies: List[str] = [] + if "enso" in c["subsection"]: + logger.warning( + "The 'enso' set is not yet supported. Skipping the enso task." + ) + break # Skip this task; we can't complete this subsection. + if "enso" in c["sets"]: + logger.warning( + "The 'enso' set is not yet supported. Skipping the enso set in synthetic_plots." + ) + c["sets"].remove("enso") c["sub"] = get_value_from_parameter( c, "subsection", "sub", ParameterInferenceType.SECTION_INFERENCE ) From 9b75692a6c6e4f96380c4be4d8c76aaebb14ba01 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Tue, 23 Sep 2025 14:08:25 -0500 Subject: [PATCH 18/47] Replace sets and subsection with current_set --- zppy/defaults/default.ini | 7 +- zppy/pcmdi_diags.py | 110 ++++++++++++++++++-------------- zppy/templates/pcmdi_diags.bash | 48 +++++++------- 3 files changed, 90 insertions(+), 75 deletions(-) diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index e6a6ec59..b6f8b422 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -393,6 +393,9 @@ ilamb_obs = string(default="") land_only = boolean(default=False) [pcmdi_diags] +# Set to one of ["mean_climate", "variability_modes_cpl", "variability_modes_atm", "enso"] +# If not set, zppy can try to infer this value from the subsection. +current_set = string(default="") # Title of zppy-pcmdi diagnostics pcmdi_webtitle = string(default="E3SM-PMP-Diagnostics") # Cannot have spaces! # Version of zppy-pcmdi code @@ -427,8 +430,6 @@ generate_sftlf = string(default=True) # variables to be used by the pcmdi diagnostics # needs to setup for each subsections, defalut setup is the mean climate metrics vars = string(default="pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500") -# sets of diagnostics from pcmdi package -sets = string_list(default=list("mean_climate","variability_modes_atm","variability_modes_cpl","enso","synthetic_plots")) ########################################################################################### # Required for run_type="model_vs_model" runs, different from e3sm_diag, # model_vs_model in pcmdi referred to the comparision of two model simulations @@ -534,6 +535,7 @@ pcmdi_debug = string(default=False) e3sm_to_cmip_atm_subsection = string(default="") [[__many__]] + current_set = string(default=None) vars = string(default=None) grid = string(default=None) num_workers = integer(default=None) @@ -561,7 +563,6 @@ e3sm_to_cmip_atm_subsection = string(default="") reference_data_path = string(default=None) reference_data_path_ts = string(default=None) run_type = string(default=None) - sets = string_list(default=None) ts_years = string_list(default=None) ts_num_years = integer(default=None) target_grid = string(default=None) diff --git a/zppy/pcmdi_diags.py b/zppy/pcmdi_diags.py index ae357413..e948630d 100644 --- a/zppy/pcmdi_diags.py +++ b/zppy/pcmdi_diags.py @@ -5,6 +5,7 @@ from zppy.logger import _setup_custom_logger from zppy.utils import ( ParameterInferenceType, + ParameterNotProvidedError, add_dependencies, check_parameter_defined, check_set_specific_parameter, @@ -23,6 +24,19 @@ logger = _setup_custom_logger(__name__) +VALID_PCMDI_SETS: Set[str] = set( + [ + "mean_climate", + "variability_modes_atm", + "variability_modes_cpl", + "enso", + "synthetic_plots", + ] +) +BASE_PCMDI_SETS: Set[str] = set( + ["mean_climate", "variability_modes_atm", "variability_modes_cpl", "enso"] +) + # ----------------------------------------------------------------------------- def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): @@ -37,16 +51,12 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): # --- Generate and submit pcmdi_diags scripts --- for c in tasks: dependencies: List[str] = [] - if "enso" in c["subsection"]: + define_current_set(c) + if c["current_set"] == "enso": logger.warning( - "The 'enso' set is not yet supported. Skipping the enso task." + "The 'enso' set is not yet supported in PCMDI Diags. Skipping launching of associated jobs." ) - break # Skip this task; we can't complete this subsection. - if "enso" in c["sets"]: - logger.warning( - "The 'enso' set is not yet supported. Skipping the enso set in synthetic_plots." - ) - c["sets"].remove("enso") + break # Skip this task c["sub"] = get_value_from_parameter( c, "subsection", "sub", ParameterInferenceType.SECTION_INFERENCE ) @@ -64,7 +74,7 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): # Loop over year sets year_sets: List[Tuple[int, int]] - if c["sub"] != "synthetic_plots": + if c["current_set"] != "synthetic_plots": year_sets = get_years(c["ts_years"]) else: year_sets = get_years(c["figure_sets_period"].split(",")) @@ -84,7 +94,7 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): c["ref_year1"] = rs[0] c["ref_year2"] = rs[1] - if c["sub"] != "synthetic_plots": + if c["current_set"] != "synthetic_plots": check_and_define_parameters(c) else: prefix = f"pcmdi_diags_{c['sub']}_{c['run_type']}" @@ -159,34 +169,57 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): return existing_bundles +def define_current_set(c: Dict[str, Any]): + if c["current_set"] != "": + current_set = c["current_set"] + if current_set not in VALID_PCMDI_SETS: + raise ValueError( + f"Invalid set '{current_set}'. Must be one of {VALID_PCMDI_SETS}" + ) + elif c["infer_path_parameters"]: + if "mean_climate" in c["subsection"]: + current_set = "mean_climate" + elif "variability_modes_cpl" in c["subsection"]: + current_set = "variability_modes_cpl" + elif "variability_modes_atm" in c["susbsection"]: + current_set = "variability_modes_atm" + elif "enso" in c["subsection"]: + current_set = "enso" + else: + raise ValueError(f"Could not determine set from {c['subsection']}") + if current_set not in VALID_PCMDI_SETS: + raise ValueError( + f"Invalid set: '{current_set}' inferred from {c['subsection']}. Must be one of {VALID_PCMDI_SETS}" + ) + else: + raise ParameterNotProvidedError( + "current_set was not provided, and inferring is turned off. Turn on inferring by setting infer_path_parameters to True." + ) + c["current_set"] = current_set + + def check_parameters_for_bash(c: Dict[str, Any]) -> None: if c["sub"] != "synthetic_plots": check_set_specific_parameter( c, - set( - ["mean_climate", "variability_mode_cpl", "variability_mode_atm", "enso"] - ), + BASE_PCMDI_SETS, "ref_final_yr", ) check_set_specific_parameter( c, - set( - ["mean_climate", "variability_mode_cpl", "variability_mode_atm", "enso"] - ), + BASE_PCMDI_SETS, "ref_start_yr", ) check_set_specific_parameter( c, - set( - ["mean_climate", "variability_mode_cpl", "variability_mode_atm", "enso"] - ), + BASE_PCMDI_SETS, "ref_end_yr", ) def check_parameters_for_pcmdi(c: Dict[str, Any]) -> None: # check and set up the external data needed by pcmdi - if c["sub"] == "synthetic_plots": + if c["current_set"] == "synthetic_plots": set_value_of_parameter_if_undefined( c, "cmip_enso_dir", @@ -211,29 +244,16 @@ def check_mvm_only_parameters_for_bash(c: Dict[str, Any]) -> None: check_parameter_defined(c, "reference_data_path_ts") check_parameter_defined(c, "model_name_ref") check_parameter_defined(c, "model_tableID_ref") - if c["sub"] != "synthetic_plots": + if c["current_set"] != "synthetic_plots": check_set_specific_parameter( c, - set( - ["mean_climate", "variability_mode_cpl", "variability_mode_atm", "enso"] - ), + BASE_PCMDI_SETS, "ref_start_yr", ) - ts_sets = set( - [ - "mean_climate", - "variability_mode_cpl", - "variability_mode_atm", - "enso", - ] - ) - check_set_specific_parameter(c, ts_sets, "ts_subsection") + check_set_specific_parameter(c, BASE_PCMDI_SETS, "ts_subsection") def check_and_define_parameters(c: Dict[str, Any]) -> None: - # TODO, future PR: do this based on sets, rather than by relying on the user setting ts_num_years - # This is the same item as for e3sm_diags. - # Here, we'd have to determine which sets of sets "mean_climate","variability_modes_atm","variability_modes_cpl","enso","synthetic_plots" require `ts` if "ts_num_years" in c.keys(): set_value_of_parameter_if_undefined( c, @@ -250,9 +270,7 @@ def check_and_define_parameters(c: Dict[str, Any]) -> None: check_mvm_only_parameters_for_bash(c) prefix = f"pcmdi_diags_{c['sub']}_{c['run_type']}_{c['year1']:04d}-{c['year2']:04d}_vs_{c['ref_year1']:04d}-{c['ref_year2']:04d}" reference_data_path = c["reference_data_path"].split("/post")[0] + "/post" - if set( - ["mean_climate", "variability_mode_cpl", "variability_mode_atm", "enso"] - ) & set(c["sets"]): + if c["current_set"] in BASE_PCMDI_SETS: set_value_of_parameter_if_undefined( c, "reference_data_path_ts", @@ -270,10 +288,7 @@ def add_ts_dependencies( ): start_yr = yr end_yr = yr + c["ts_num_years"] - 1 - depend_on_ts: Set[str] = set( - ["mean_climate", "variability_mode_atm", "variability_mode_cpl", "enso"] - ) - if depend_on_ts & set(c["sets"]): + if c["current_set"] in BASE_PCMDI_SETS: add_dependencies( dependencies, script_dir, @@ -293,28 +308,29 @@ def add_pcmdi_dependencies( status_suffix = f"_{c['year1']:04d}-{c['year2']:04d}" elif c["run_type"] == "model_vs_model": status_suffix = f"_{c['year1']:04d}-{c['year2']:04d}_vs_{c['ref_year1']:04d}-{c['ref_year2']:04d}" - if "mean_climate" in c["sets"]: + if "mean_climate" in c["figure_sets"]: status_file = os.path.join( script_dir, f"pcmdi_diags_mean_climate_{c['run_type']}{status_suffix}.status", ) if os.path.exists(status_file): dependencies.append(status_file) - if "variability_modes_cpl" in c["sets"]: + if "variability_modes" in c["figure_sets"]: + # cpl status_file = os.path.join( script_dir, f"pcmdi_diags_variability_modes_cpl_{c['run_type']}{status_suffix}.status", ) if os.path.exists(status_file): dependencies.append(status_file) - if "variability_modes_atm" in c["sets"]: + # atm status_file = os.path.join( script_dir, f"pcmdi_diags_variability_modes_atm_{c['run_type']}{status_suffix}.status", ) if os.path.exists(status_file): dependencies.append(status_file) - if "enso" in c["sets"]: + if "enso_metric" in c["figure_sets"]: status_file = os.path.join( script_dir, f"pcmdi_diags_enso_{c['run_type']}{status_suffix}.status" ) diff --git a/zppy/templates/pcmdi_diags.bash b/zppy/templates/pcmdi_diags.bash index 7fac49af..9d86d6fc 100755 --- a/zppy/templates/pcmdi_diags.bash +++ b/zppy/templates/pcmdi_diags.bash @@ -30,7 +30,7 @@ case="{{ case }}" www="{{ www }}" run_type="{{ run_type }}" results_dir="{{ run_type }}" -{% if "synthetic_plots" not in subsection %} +{% if current_set != "synthetic_plots" %} # Input variables y1={{ year1 }} @@ -105,7 +105,7 @@ cat > synthetic_metrics_list.json << EOF {% include synthetic_metrics_list %} EOF -{% if "mean_climate" in subsection %} +{% if current_set == "mean_climate" %} create_links_acyc_climo() { local ts_dir_source="$1" local ts_dir_destination="$2" @@ -262,9 +262,7 @@ create_links_acyc_climo_obs() { {% endif %} {% endif %} -{% if "variability_modes_cpl" in subsection - or "variability_modes_atm" in subsection - or "enso" in subsection %} +{% if current_set in ["variability_modes_cpl", "variability_modes_atm", "enso"] %} create_links_ts() { local ts_dir_source="$1" local ts_dir_destination="$2" @@ -432,7 +430,7 @@ create_links_ts_obs() { ######################## # Prepare the model data ######################## -{% if "mean_climate" in subsection %} +{% if current_set == "mean_climate" %} # Define output directory for climatology files climo_dir_primary="climo" # Path to model's monthly climatology files @@ -449,8 +447,8 @@ create_links_acyc_climo "${climo_dir_source_ref}" "${climo_dir_ref}" "${ref_Y1}" {% endif %} {% endif %} -{% if "variability_modes_cpl" in subsection or "variability_modes_atm" in subsection or "enso" in subsection %} -# All diagnostics in this subsection use time series (ts) data +{% if current_set in ["variability_modes_cpl", "variability_modes_atm", "enso"] %} +# All these diagnostics use time series (ts) data # Define output directory for primary model time series ts_dir_primary="ts" ts_dir_source="{{ output }}/post/atm/{{ ts_grid }}/cmip_ts/monthly" @@ -465,7 +463,7 @@ create_links_ts "${ts_dir_source_ref}" "${ts_dir_ref}" "${ref_Y1}" "${ref_Y2}" " {% endif %} {% endif %} -{% if run_type == "model_vs_obs" and "synthetic_plots" not in subsection %} +{% if run_type == "model_vs_obs" and current_set != "synthetic_plots" %} ########################################################################### # Prepare the observation data # Observation datasets vary by diagnostic, so we use an external @@ -499,17 +497,17 @@ fi ####################################################### ts_dir_ref_source="{{ scriptDir }}/${workdir}/${obstmp_dir}" -{% if "mean_climate" in subsection %} +{% if current_set == "mean_climate" %} climo_dir_ref=climo_ref create_links_acyc_climo_obs "${ts_dir_ref_source}" "${climo_dir_ref}" ${ref_Y1} ${ref_Y2} 7 -{% elif "variability_modes_cpl" in subsection or "variability_modes_atm" in subsection or "enso" in subsection %} +{% elif current_set in ["variability_modes_cpl", "variability_modes_atm", "enso"] %} ts_dir_ref=ts_ref create_links_ts_obs "${ts_dir_ref_source}" "${ts_dir_ref}" ${ref_Y1} ${ref_Y2} 8 {% endif %} {% endif %} -{% if "synthetic_plots" not in subsection %} +{% if current_set != "synthetic_plots" %} ######################################################## # generate basic parameter file for pcmdi metrics driver ######################################################## @@ -575,7 +573,7 @@ modpath_lf = os.path.join( 'sftlf.%(model).nc' ) -{% if "mean_climate" in subsection %} +{% if current_set == "mean_climate" %} ############################################ # Setup Specific for Mean Climate Metrics @@ -673,7 +671,7 @@ test_clims_interpolated_output = diagnostics_output_path {% endif %} -{% if "variability_modes" in subsection %} +{% if current_set in ["variability_modes_cpl", "variability_modes_atm"] %} # Setup for Mode Variability Diagnostics msyear = int(start_yr) meyear = int(end_yr) @@ -738,7 +736,7 @@ results_dir = os.path.join( ) {% endif %} -{% if "enso" in subsection %} +{% if current_set == "enso" %} # Parameter Setup for ENSO Metrics modnames = [product] realization = realm @@ -796,10 +794,10 @@ echo "The current directory is: $PWD" # This will be of the form .../post/script # Run diagnostics mkdir -p pcmdi_diags -{% if "synthetic_plots" not in subsection %} -{% if "mean_climate" in subsection %} +{% if current_set != "synthetic_plots" %} +{% if current_set == "mean_climate"%} source_dirs="--climo_ts_dir_primary ${climo_dir_primary} --climo_ts_dir_ref ${climo_dir_ref}" -{% elif "variability_modes" in subsection or "enso" in subsection %} +{% elif current_set in ["variability_modes_cpl", "variability_modes_atm", "enso"] %} source_dirs="--climo_ts_dir_primary ${ts_dir_primary} --climo_ts_dir_ref ${ts_dir_ref}" {% endif %} # Parameter passing can encounter errors if parameters are empty. @@ -809,21 +807,21 @@ source_dirs="--climo_ts_dir_primary ${ts_dir_primary} --climo_ts_dir_ref ${ts_di core_parameters="--num_workers {{ num_workers }} --multiprocessing {{ multiprocessing }} --subsection {{ subsection }} ${source_dirs} --model_name ${model_name} --model_tableID {{model_tableID }} --figure_format {{ figure_format }} --run_type {{ run_type }} --obs_sets {{ obs_sets }} --model_name_ref ${model_name_ref} --vars {{ vars }} --tableID_ref ${tableID_ref} --generate_sftlf {{ generate_sftlf }} --case_id ${case_id} --results_dir ${results_dir}" {% endif %} -{% if "mean_climate" in subsection %} +{% if current_set == "mean_climate" %} command="zi-pcmdi-mean-climate ${core_parameters} --regions {{ regions }}" {% endif %} -{% if "variability_modes" in subsection %} -{% if subsection == "variability_modes_atm" %} +{% if current_set in ["variability_modes_cpl", "variability_modes_atm"] %} +{% if current_set == "variability_modes_atm" %} var_modes={{ atm_modes }} -{% elif subsection == "variability_modes_cpl" %} +{% elif current_set == "variability_modes_cpl" %} var_modes={{ cpl_modes }} {% endif %} command="zi-pcmdi-variability-modes ${core_parameters} --var_modes ${var_modes}" {% endif %} -{% if "enso" in subsection %} +{% if current_set == "enso"n %} command="zi-pcmdi-enso ${core_parameters} --enso_groups {{ enso_groups }}" {% endif %} -{% if "synthetic_plots" in subsection %} +{% if current_set == "synthetic_plots" %} # Note: figure_sets_period changed to a string # Note: sub_sets was renamed to figure_sets to be clearer command="zi-pcmdi-synthetic-plots --synthetic_sets {{ synthetic_sets }} --figure_format {{ figure_format }} --www ${www} --results_dir ${results_dir} --case {{ case }} --model_name {{ model_name }} --model_tableID {{model_tableID }} --web_dir=${web_dir} --cmip_clim_dir {{ cmip_clim_dir }} --cmip_clim_set {{ cmip_clim_set }} --cmip_movs_dir {{ cmip_movs_dir }} --cmip_movs_set {{ cmip_movs_set }} --atm_modes {{ atm_modes }} --cpl_modes {{ cpl_modes }} --cmip_enso_dir {{ cmip_enso_dir }} --cmip_enso_set {{ cmip_enso_set }} --figure_sets {{ figure_sets }} --pcmdi_webtitle {{ pcmdi_webtitle }} --pcmdi_version {{ pcmdi_version }} --run_type ${run_type} --figure_sets_period {{ figure_sets_period }} --pcmdi_external_prefix {{ diagnostics_base_path }} --pcmdi_viewer_template {{ pcmdi_viewer_template }}" @@ -872,7 +870,7 @@ done ############################################ # Copy files #rsync -a --delete ${results_dir} ${web_dir}/ -{% if "synthetic_plots" not in subsection %} +{% if current_set != "synthetic_plots" %} rsync -a ${results_dir} ${web_dir}/ if [ $? != 0 ]; then cd {{ scriptDir }} From b504f5bc55cf9be93ce87ffc501eded37cb755f5 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Tue, 23 Sep 2025 17:38:28 -0500 Subject: [PATCH 19/47] Organize parameters --- examples/post.v3.LR.amip.0101.cfg | 144 +++++--------- zppy/defaults/default.ini | 307 +++++++++++++++--------------- zppy/templates/pcmdi_diags.bash | 4 +- 3 files changed, 202 insertions(+), 253 deletions(-) diff --git a/examples/post.v3.LR.amip.0101.cfg b/examples/post.v3.LR.amip.0101.cfg index 672ce69b..ae24dc41 100644 --- a/examples/post.v3.LR.amip.0101.cfg +++ b/examples/post.v3.LR.amip.0101.cfg @@ -40,142 +40,88 @@ years = "2005:2015:5" [pcmdi_diags] active = True -multiprocessing = True +environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" +environment_commands_secondary = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20250917" nodes = 1 +walltime = "4:00:00" +figure_format = "png" +run_type = "model_vs_obs" +# Create rules to construct cmip-like datasets with a string constructed as +# "mip.exp.product.realization", where "."is used as the delimiter (mandatory) +model_name = 'e3sm.amip.v3-LR.0101' +model_tableID = 'Amon' +multiprocessing = True num_workers = 24 -#observational data +# Observational data obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series' pcmdi_debug = False -run_type = "model_vs_obs" -reference_data_path = "" -reference_data_path_ts = "" -ts_grid = "180x360_aave" -walltime = "4:00:00" -environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" -environment_commands_secondary = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20250917" +ref_years = "2005-2014", +ts_years = "2005-2014", [[ mean_climate ]] active = True - ################################################################################## - #create rules to construct cmip-like datasets with a string constructed as - #"mip.exp.product.realization", where "."is used as the delimiter (mandatary) - ts_years = "2005-2014", - ts_num_years = 5 - ref_years = "2005-2014", - ref_start_yr = 1979 + # Default for all variables or specify source for each variable separately. + obs_sets = "default" ref_final_yr = 2018 - model_name = 'e3sm.amip.v3-LR.0101' - model_tableID = 'Amon' - ################################################################################## - #location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/mean_climate/cmip6/amip - cmip_clim_set = "cmip6.amip.v20250702" - ################################################################################## - #variables in current pcmdi-cmip list - #note: rltcre and rstcre are derived variables as they are not included in cmip_ts + ref_start_yr = 1979 + # Variables in current pcmdi-cmip list + # rltcre and rstcre are derived variables as they are not included in cmip_ts vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rsus,rsuscs,rlus,rlut,rlutcs,rsds,rsdscs,rsdt,rsut,rsutcs,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" - #default for all variables or specify source for each variable separately. - obs_sets = "default" - #see zppy/templates/pcmdi_diags/regions_specs.json for options of all regions - #below is default setup for global/regional mean climate metrics + # See zppy/templates/pcmdi_diags/regions_specs.json for options of all regions + # Below is default setup for global/regional mean climate metrics regions = "global,ocean,land,NHEX,SHEX,TROPICS" target_grid = "2.5x2.5" target_grid_string = "2p5x2p5" [[ variability_modes_cpl ]] active = True - ################################################################################## - ts_years = "2005-2014", - ts_num_years = 5 - ref_years = "2005-2014", - ref_start_yr = 1870 + # See zppy/templates/pcmdi_diags/reference_alias.json for options + obs_sets = "alternate1" #"default" ref_final_yr = 2019 - #create rules to construct cmip-like datasets with a string constructed as - #"mip.exp.product.realization", where "."is used as the delimiter (mandatary) - model_name = 'e3sm.amip.v3-LR.0101' - model_tableID = 'Amon' - ################################################################################## - #location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes/cmip6/amip - cmip_movs_set = "cmip6.amip.v20210119" - ################################################################################## - grid = '180x360_aave' + ref_start_yr = 1870 vars = "ts" + # Name of modes varibility + cpl_modes = "PDO,NPGO,AMO" frequency = 'mo' - ModUnitsAdjust = '(True,"subtract",273.15)' landmask = True - #see zppy/templates/pcmdi_diags/reference_alias.json for options - obs_sets = "alternate1" #"default" + ModUnitsAdjust = '(True,"subtract",273.15)' ObsUnitsAdjust = '(True,"subtract",273.15)' - #name of modes varibility - cpl_modes = "PDO,NPGO,AMO" - #options specific for mode varibility metrics in pcmdi seasons = "yearly,monthly" - figure_format = "png" [[ variability_modes_atm ]] active = True - ################################################################################## - ts_years = "2005-2014", - ts_num_years = 5 - ref_years = "2005-2014", - ref_start_yr = 1853 + # See zppy/templates/pcmdi_diags/reference_alias.json for options + # Default for all variables + obs_sets = "alternate2" # "default" ref_final_yr = 2015 - #create rules to construct cmip-like datasets with a string constructed as - #"mip.exp.product.realization", where "." is used as the delimiter (mandatary) - model_name = 'e3sm.amip.v3-LR.0101' - model_tableID = 'Amon' - ################################################################################## - #location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes/cmip6/amip - cmip_movs_set = "cmip6.amip.v20210119" - ################################################################################## - grid = '180x360_aave' + ref_start_yr = 1853 vars = "psl" - landmask = False + # Varibility modes + atm_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" frequency = 'mo' + landmask = False ModUnitsAdjust = '(True,"divide",100.0)' - #observational data - obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series' - #see zppy/templates/pcmdi_diags/reference_alias.json for options - #default for all variables - obs_sets = "alternate2" # "default" ObsUnitsAdjust = '(True,"divide",100.0)' - #varibility modes - atm_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" - #options specific for mode varibility metrics in pcmdi seasons = "DJF,MAM,JJA,SON,yearly,monthly" - figure_format = "png" [[ enso ]] - active = True - ################################################################################## - ts_years = "2005-2014", - ts_num_years = 5 - ref_years = "2005-2014", - ref_start_yr = 1979 - ref_final_yr = 2018 - #create rules to construct cmip-like datasets with a string constructed as - #"mip.exp.product.realization", where "."is used as the delimiter (mandatary) - model_name = 'e3sm.amip.v3-LR.0101' - model_tableID = 'Amon' - ################################################################################## - #location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/enso_metric/cmip6/historical - #Note: there is no enso metric for CMIP6 AMIP, so use historical instead - cmip_enso_set = "cmip6.historical.v20210620" - ################################################################################## - #see zppy/templates/pcmdi_diags/reference_alias.json for options - customized_cmor_obs = True - vars = "psl,pr,prsn,ts,tas,tauu,tauv,hfls,hfss,rlds,rsds,rlus,rlut,rsdt" + active = False # For enso, the obs_sets is explicitely set for each variable to ensure that # they are from the same source, below are setup to used ERA5 reanalysis. obs_sets = "default,alternate2,default,default,default,default,default,default,default,alternate2,alternate2,alternate2,alternate2,alternate2" - #see zppy/templates/pcmdi_diags/reference_alias.json for options + ref_final_yr = 2018 + ref_start_yr = 1979 + vars = "psl,pr,prsn,ts,tas,tauu,tauv,hfls,hfss,rlds,rsds,rlus,rlut,rsdt" + # See zppy/templates/pcmdi_diags/reference_alias.json for options enso_groups = "ENSO_perf,ENSO_proc,ENSO_tel" - figure_format = "png" [[ synthetic_plots ]] active = True - model_name = 'e3sm.amip.v3-LR.0101' - model_tableID = 'Amon' - synthetic_sets = "portrait,parcoord" - figure_sets = "mean_climate,variability_modes,enso_metric" figure_sets_period = "2005-2014,2005-2014,2005-2014" - figure_format = "png" + # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/mean_climate/cmip6/amip + cmip_clim_set = "cmip6.amip.v20250702" + # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes/cmip6/amip + cmip_movs_set = "cmip6.amip.v20210119" + # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/enso_metric/cmip6/historical + # There is no enso metric for CMIP6 AMIP, so use historical instead + cmip_enso_set = "cmip6.historical.v20210620" diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index b6f8b422..96ca2b41 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -393,118 +393,85 @@ ilamb_obs = string(default="") land_only = boolean(default=False) [pcmdi_diags] -# Set to one of ["mean_climate", "variability_modes_cpl", "variability_modes_atm", "enso"] -# If not set, zppy can try to infer this value from the subsection. -current_set = string(default="") -# Title of zppy-pcmdi diagnostics -pcmdi_webtitle = string(default="E3SM-PMP-Diagnostics") # Cannot have spaces! -# Version of zppy-pcmdi code -pcmdi_version = string(default="v3.8.2") -# File of specified regions for mean climate calculation -regions_specs = string(default="inclusions/pcmdi_diags/regions_specs.json") -# File of observation data name for mean climate calculation -reference_alias = string(default="inclusions/pcmdi_diags/reference_alias.json") -# Template files for zppy-pcmdi viewer page processing -pcmdi_viewer_template = string(default="pcmdi_data/viewer") -# File of variable list to generate synthetic metrics plot -synthetic_metrics_list = string(default="inclusions/pcmdi_diags/synthetic_metrics_list.json") -# Path to observation time-series data -# Required for "mean_climate","variability_mode","enso" -obs_ts = string(default="") -# observational data sets (see reference_alias.json) -# observation data tag in reference_alias -obs_sets = string(default="default") -# options specific for constructing pcmdi preferred file name conventions -# required for "model_vs_obs" comparison +########################################################################################### +# Most useful at task-level; used by all sets +########################################################################################### +# See url +figure_format = string(default="png") +# Comparision type (same as e3sm_diag) +run_type = string(default="model_vs_obs") +### 4 options specific for constructing pcmdi preferred file name conventions: +### 2 options required for "model_vs_obs" comparison: model_name = string(default="e3sm.historical.v3-LR.0051") model_tableID = string(default="Amon") -# required for "model_vs_model" comparison +### 2 options required for "model_vs_model" comparison: model_name_ref = string(default="ERA5") model_tableID_ref = string(default="Amon") -# variables in the cmip6 table that can be potentially used by pcmdi -# this list depends on the definition of cmip variable -# required for "mean climate" diagnostics -cmip_vars = string(default="pr,prw,psl,rlds,rldscs,rlut,rlutcs,rsut,rsutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rtmt,sfcWind,tas,tauu,tauv,ts,ta,ua,va,zg") -# flag to process the land/sea mask within pcmdi -generate_sftlf = string(default=True) -# variables to be used by the pcmdi diagnostics -# needs to setup for each subsections, defalut setup is the mean climate metrics -vars = string(default="pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500") ########################################################################################### -# Required for run_type="model_vs_model" runs, different from e3sm_diag, -# model_vs_model in pcmdi referred to the comparision of two model simulations -# with observations and cmip models. +# Most useful at task-level; used by all sets except synthetic_plots ########################################################################################### -#path for reference model data (time series) +# Name of the `[e3sm_to_cmip]` atm subtask to depend on +e3sm_to_cmip_atm_subsection = string(default="") +# Flag to process the land/sea mask within pcmdi +generate_sftlf = string(default=True) +# model data grid after remapping +grid = string(default="180x360_aave") +mov_plot_obs = string(default=True) +mov_plot_model = string(default=True) +mov_nc_out_obs = string(default=True) +mov_nc_out_model = string(default=True) +# See url +multiprocessing = boolean(default=True) +# See url +num_workers = integer(default=24) +# Path to observation time-series data +# Required for "mean_climate","variability_mode","enso" +obs_ts = string(default="") +pcmdi_debug = string(default=False) +# Save derived climatology data +save_test_clims = string(default=True) +### 7 options required for run_type="model_vs_model" runs: +# Unlike in e3sm_diags, model_vs_model in pcmdi refers to the comparision of two model simulations with observations and cmip models. +### 2 options for path for reference model data (time series): reference_data_path = string(default="") reference_data_path_ts = string(default="") # pcmdi_diags.py will set to match `years` if not specified ref_years = string_list(default=list("")) +# The years increment for test model data +ts_num_years = integer(default=5) +# The years range for test model data +ts_years = string_list(default=list("")) +########################################################################################### +# Specify these for each subtask +########################################################################################### +# Set to one of ["mean_climate", "variability_modes_cpl", "variability_modes_atm", "enso"] +# If not set, zppy can try to infer this value from the subsection. +current_set = string(default="") +# Observational data sets (see reference_alias.json) +# Observation data tag in reference_alias +obs_sets = string(default="default") # Final year (i.e., the last available year) for the reference data ref_final_yr = string(default="") # Start year for the reference data ref_start_yr = string(default="") -# The years range for test model data -ts_years = string_list(default=list("")) -# The years increment for test model data -ts_num_years = integer(default=5) -########################################################################################## -# below followed the setup in e3sm_diag but used for PCMDI workflow -########################################################################################## -# See url -multiprocessing = boolean(default=True) -# See url -num_workers = integer(default=24) -# See url -figure_format = string(default="png") -# comparision type (same as e3sm_diag) -run_type = string(default="model_vs_obs") -########################################################################################## -# options for pcmdi mode varibility diagnostics -# vars = "psl" for atm_modes -# vars = "ts" for cpl_modes -######################################################################################### -#name of atmospheric modes varibility -atm_modes = string(default="NAM,NAO,PNA,NPO,SAM,PSA1,PSA2") -#name of coupled modes varibility -cpl_modes = string(default="PDO,NPGO,AMO") -#keywards for unit conversion in pcmdi (model) -ModUnitsAdjust = string(default="") -#keywards for unit conversion in pcmdi (observation) -ObsUnitsAdjust = string(default="") -#frequency of the model data -frequency = string(default="mo") -#options specific for mode varibility metrics in pcmdi -seasons = string(default="monthly") -landmask = string(default=False) -RmDomainMean = string(default=True) -EofScaling = string(default=False) -ConvEOF = string(default=True) -CBF = string(default=True) -eofn_mod_max = integer(default=3) -mov_plot_obs = string(default=True) -mov_plot_model = string(default=True) -mov_nc_out_obs = string(default=True) -mov_nc_out_model = string(default=True) -########################################################################################## -# options for pcmdi enso diagnostics -# vars = "psl,pr,prsn,ts,tas,tauu,tauv,hflx,hfss,rlds,rsds,rlus,rlut,rsdt" -########################################################################################## -enso_groups = string(default="ENSO_perf,ENSO_proc,ENSO_tel") -########################################################################################## -# optional for mean climate diagnostics -# vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs, -# rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200, -# va-850,zg-500" -########################################################################################## -# model data grid after remapping -grid = string(default="180x360_aave") +# Variables to be used by the pcmdi diagnostics +# mean_climate: vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" +# variability_modes_atm: vars = "psl" +# variability_modes_cpl: vars = "ts" +# enso: vars = "psl,pr,prsn,ts,tas,tauu,tauv,hflx,hfss,rlds,rsds,rlus,rlut,rsdt" +# Default setup is for mean_climate: +vars = string(default="pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500") +########################################################################################### +# Specify these for the mean_climate subtask +########################################################################################### +# variables in the cmip6 table that can be potentially used by pcmdi +# this list depends on the definition of cmip variable +# required for "mean climate" diagnostics +cmip_vars = string(default="pr,prw,psl,rlds,rldscs,rlut,rlutcs,rsut,rsutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rtmt,sfcWind,tas,tauu,tauv,ts,ta,ua,va,zg") #default regions for mean climate metrics data #more options can be found at "regions_specs" #regions = string(default="global,ocean,land,NHEX,SHEX,TROPICS,NHEX_ocean,SHEX_ocean,NHEX_land,SHEX_land,ocean_50S50N") regions = string(default="global,ocean,land,NHEX,SHEX,TROPICS") -# save derived climatology data -save_test_clims = string(default=True) # Regridding by pcmdi (required for mean climate) # OPTIONS: '1x1' for e3sm, '2.5x2.5' for cmip, or an actual cdms2 grid object, target_grid = string(default="1x1") @@ -517,84 +484,122 @@ regrid_tool_ocn = string(default="esmf") regrid_method = string(default="regrid2") # OPTIONS: 'linear','conservative', only if tool is esmf regrid_method_ocn = string(default="conservative") -################################################################### -#options for synthetic plots with exisiting pcmdi cmip model metrics -################################################################### +########################################################################################### +# Specify these for the variability_modes_cpl and variability_modes_atm subtasks +########################################################################################### +### 2 options for names of atm, cpl modes varibility: +atm_modes = string(default="NAM,NAO,PNA,NPO,SAM,PSA1,PSA2") +cpl_modes = string(default="PDO,NPGO,AMO") +CBF = string(default=True) +ConvEOF = string(default=True) +eofn_mod_max = integer(default=3) +EofScaling = string(default=False) +# Frequency of the model data +frequency = string(default="mo") +landmask = string(default=False) +# Keywords for unit conversion in pcmdi (model) +ModUnitsAdjust = string(default="") +# Keywords for unit conversion in pcmdi (observation) +ObsUnitsAdjust = string(default="") +RmDomainMean = string(default=True) +seasons = string(default="monthly") +########################################################################################### +# Specify these for the enso subtasks (not currently enabled) +########################################################################################### +enso_groups = string(default="ENSO_perf,ENSO_proc,ENSO_tel") +########################################################################################### +# Specify these for the synthetic_plots subtask +########################################################################################### +figure_sets = string(default="mean_climate,variability_modes") +figure_sets_period = string(default="") +# File of observation data name for mean climate calculation +reference_alias = string(default="inclusions/pcmdi_diags/reference_alias.json") +# File of specified regions for mean climate calculation +regions_specs = string(default="inclusions/pcmdi_diags/regions_specs.json") +# Version of zppy-pcmdi code +pcmdi_version = string(default="v3.8.2") +# Template files for zppy-pcmdi viewer page processing +pcmdi_viewer_template = string(default="pcmdi_data/viewer") +# Title of zppy-pcmdi diagnostics +pcmdi_webtitle = string(default="E3SM-PMP-Diagnostics") # Cannot have spaces! +# File of variable list to generate synthetic metrics plot +synthetic_metrics_list = string(default="inclusions/pcmdi_diags/synthetic_metrics_list.json") synthetic_sets = string(default="portrait,parcoord") -# path to pcmdi generated cmip metrics data -cmip_enso_dir = string(default="") +### 3 options for paths to pcmdi generated cmip metrics data: cmip_clim_dir = string(default="") +cmip_enso_dir = string(default="") cmip_movs_dir = string(default="") -#group of pcmdi generated cmip metrics data (mip.exp.version) +### 3 options for groups of pcmdi generated cmip metrics data (mip.exp.version): cmip_clim_set = string(default="cmip6.historical.v20250707") -cmip_movs_set = string(default="cmip6.historical.v20220825") cmip_enso_set = string(default="cmip6.historical.v20210620") -# options shared by pcmdi -pcmdi_debug = string(default=False) -# Name of the `[e3sm_to_cmip]` atm subtask to depend on -e3sm_to_cmip_atm_subsection = string(default="") +cmip_movs_set = string(default="cmip6.historical.v20220825") +# TODO: Match "many" section to above +# TODO: Update example cfg accordingly +# TODO: Commit +# TODO: Run example cfg [[__many__]] - current_set = string(default=None) - vars = string(default=None) + figure_format = string(default=None) + run_type = string(default=None) + model_name = string(default=None) + model_tableID = string(default=None) + model_name_ref = string(default=None) + model_tableID_ref = string(default=None) + e3sm_to_cmip_atm_subsection = string(default=None) + generate_sftlf = string(default=None) grid = string(default=None) - num_workers = integer(default=None) - cmip_enso_set = string(default=None) - cmip_clim__set = string(default=None) - cmip_movs_set = string(default=None) - cmip_enso_dir = string(default=None) - cmip_clim_dir = string(default=None) - cmip_movs_dir = string(default=None) - synthetic_sets = string(default=None) - reference_alias = string(default=None) - regions_specs = string(default=None) - pcmdi_webtitle = string(default=None) - pcmdi_version = string(default=None) - pcmdi_viewer_template = string(default=None) - synthetic_metrics_list = string(default=None) + mov_plot_obs = string(default=None) + mov_plot_model = string(default=None) + mov_nc_out_obs = string(default=None) + mov_nc_out_model = string(default=None) multiprocessing = boolean(default=None) + num_workers = integer(default=None) obs_ts = string(default=None) - figure_sets = string(default=None) - figure_sets_period = string(default=None) - figure_format = string(default=None) - ref_years = string_list(default=None) - ref_start_yr = string(default=None) - ref_final_yr = string(default=None) + pcmdi_debug = string(default=None) + save_test_clims = string(default=None) reference_data_path = string(default=None) reference_data_path_ts = string(default=None) - run_type = string(default=None) - ts_years = string_list(default=None) + ref_years = string_list(default=None) ts_num_years = integer(default=None) + ts_years = string_list(default=None) + current_set = string(default=None) + obs_sets = string(default=None) + ref_final_yr = string(default=None) + ref_start_yr = string(default=None) + vars = string(default=None) + cmip_vars = string(default=None) + regions = string(default=None) target_grid = string(default=None) target_grid_string = string(default=None) regrid_tool = string(default=None) regrid_tool_ocn = string(default=None) regrid_method = string(default=None) regrid_method_ocn = string(default=None) - obs_sets = string(default=None) - regions = string(default=None) - save_test_clims = string(default=None) - seasons = string(default=None) - RmDomainMean = string(default=None) - EofScaling = string(default=None) - ConvEOF = string(default=None) + atm_modes = string(default=None) + cpl_modes = string(default=None) CBF = string(default=None) + ConvEOF = string(default=None) eofn_mod_max = integer(default=None) - landmask = string(default=None) + EofScaling = string(default=None) frequency = string(default=None) - generate_sftlf = string(default=None) - atm_modes = string(default=None) - cpl_modes = string(default=None) - enso_groups = string(default=None) + landmask = string(default=None) ModUnitsAdjust = string(default=None) ObsUnitsAdjust = string(default=None) - model_name = string(default=None) - model_name_ref = string(default=None) - model_tableID = string(default=None) - model_vars = string(default=None) - mov_nc_out_obs = string(default=None) - mov_nc_out_model = string(default=None) - mov_plot_obs = string(default=None) - mov_plot_model = string(default=None) - pcmdi_debug = string(default=None) - e3sm_to_cmip_atm_subsection = string(default=None) + RmDomainMean = string(default=None) + seasons = string(default=None) + enso_groups = string(default=None) + figure_sets = string(default=None) + figure_sets_period = string(default=None) + reference_alias = string(default=None) + regions_specs = string(default=None) + pcmdi_version = string(default=None) + pcmdi_viewer_template = string(default=None) + pcmdi_webtitle = string(default=None) + cmip_clim_dir = string(default=None) + cmip_enso_dir = string(default=None) + cmip_movs_dir = string(default=None) + cmip_clim__set = string(default=None) + cmip_enso_set = string(default=None) + cmip_movs_set = string(default=None) + synthetic_metrics_list = string(default=None) + synthetic_sets = string(default=None) diff --git a/zppy/templates/pcmdi_diags.bash b/zppy/templates/pcmdi_diags.bash index 9d86d6fc..650f617b 100755 --- a/zppy/templates/pcmdi_diags.bash +++ b/zppy/templates/pcmdi_diags.bash @@ -818,12 +818,10 @@ var_modes={{ cpl_modes }} {% endif %} command="zi-pcmdi-variability-modes ${core_parameters} --var_modes ${var_modes}" {% endif %} -{% if current_set == "enso"n %} +{% if current_set == "enso" %} command="zi-pcmdi-enso ${core_parameters} --enso_groups {{ enso_groups }}" {% endif %} {% if current_set == "synthetic_plots" %} -# Note: figure_sets_period changed to a string -# Note: sub_sets was renamed to figure_sets to be clearer command="zi-pcmdi-synthetic-plots --synthetic_sets {{ synthetic_sets }} --figure_format {{ figure_format }} --www ${www} --results_dir ${results_dir} --case {{ case }} --model_name {{ model_name }} --model_tableID {{model_tableID }} --web_dir=${web_dir} --cmip_clim_dir {{ cmip_clim_dir }} --cmip_clim_set {{ cmip_clim_set }} --cmip_movs_dir {{ cmip_movs_dir }} --cmip_movs_set {{ cmip_movs_set }} --atm_modes {{ atm_modes }} --cpl_modes {{ cpl_modes }} --cmip_enso_dir {{ cmip_enso_dir }} --cmip_enso_set {{ cmip_enso_set }} --figure_sets {{ figure_sets }} --pcmdi_webtitle {{ pcmdi_webtitle }} --pcmdi_version {{ pcmdi_version }} --run_type ${run_type} --figure_sets_period {{ figure_sets_period }} --pcmdi_external_prefix {{ diagnostics_base_path }} --pcmdi_viewer_template {{ pcmdi_viewer_template }}" {% endif %} From eba7114f9c40c589a44197f1c02564e4efba7350 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Tue, 23 Sep 2025 20:32:40 -0500 Subject: [PATCH 20/47] Fixes to get example cfg working --- examples/post.v3.LR.amip.0101.cfg | 12 +++++--- zppy/defaults/default.ini | 10 +++++-- zppy/pcmdi_diags.py | 50 ++++++++++++++++++------------- zppy/utils.py | 10 ++++++- 4 files changed, 54 insertions(+), 28 deletions(-) diff --git a/examples/post.v3.LR.amip.0101.cfg b/examples/post.v3.LR.amip.0101.cfg index ae24dc41..cc0858b6 100644 --- a/examples/post.v3.LR.amip.0101.cfg +++ b/examples/post.v3.LR.amip.0101.cfg @@ -1,7 +1,7 @@ [default] input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/AMIP/v3.LR.amip_0101 -output = /lcrc/group/e3sm/ac.forsyth2/zppy_pr719_output/unique_id_34/v3.LR.amip_0101 -www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pr719_output/unique_id_34 +output = /lcrc/group/e3sm/ac.forsyth2/zppy_pr719_output/unique_id_48/v3.LR.amip_0101 +www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pr719_output/unique_id_48 campaign = "water_cycle" case = v3.LR.amip_0101 debug = True @@ -41,7 +41,7 @@ years = "2005:2015:5" [pcmdi_diags] active = True environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" -environment_commands_secondary = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20250917" +environment_commands_secondary = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20250923" nodes = 1 walltime = "4:00:00" figure_format = "png" @@ -55,7 +55,6 @@ num_workers = 24 # Observational data obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series' pcmdi_debug = False -ref_years = "2005-2014", ts_years = "2005-2014", [[ mean_climate ]] @@ -64,6 +63,7 @@ ts_years = "2005-2014", obs_sets = "default" ref_final_yr = 2018 ref_start_yr = 1979 + ref_years = "2005-2014", # Variables in current pcmdi-cmip list # rltcre and rstcre are derived variables as they are not included in cmip_ts vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rsus,rsuscs,rlus,rlut,rlutcs,rsds,rsdscs,rsdt,rsut,rsutcs,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" @@ -79,6 +79,7 @@ ts_years = "2005-2014", obs_sets = "alternate1" #"default" ref_final_yr = 2019 ref_start_yr = 1870 + ref_years = "2005-2014", vars = "ts" # Name of modes varibility cpl_modes = "PDO,NPGO,AMO" @@ -95,6 +96,7 @@ ts_years = "2005-2014", obs_sets = "alternate2" # "default" ref_final_yr = 2015 ref_start_yr = 1853 + ref_years = "2005-2014", vars = "psl" # Varibility modes atm_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" @@ -111,12 +113,14 @@ ts_years = "2005-2014", obs_sets = "default,alternate2,default,default,default,default,default,default,default,alternate2,alternate2,alternate2,alternate2,alternate2" ref_final_yr = 2018 ref_start_yr = 1979 + ref_years = "2005-2014", vars = "psl,pr,prsn,ts,tas,tauu,tauv,hfls,hfss,rlds,rsds,rlus,rlut,rsdt" # See zppy/templates/pcmdi_diags/reference_alias.json for options enso_groups = "ENSO_perf,ENSO_proc,ENSO_tel" [[ synthetic_plots ]] active = True + # clim_period, emov_period, enso_period: figure_sets_period = "2005-2014,2005-2014,2005-2014" # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/mean_climate/cmip6/amip cmip_clim_set = "cmip6.amip.v20250702" diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index 96ca2b41..5d997959 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -435,8 +435,6 @@ save_test_clims = string(default=True) ### 2 options for path for reference model data (time series): reference_data_path = string(default="") reference_data_path_ts = string(default="") -# pcmdi_diags.py will set to match `years` if not specified -ref_years = string_list(default=list("")) # The years increment for test model data ts_num_years = integer(default=5) # The years range for test model data @@ -454,6 +452,10 @@ obs_sets = string(default="default") ref_final_yr = string(default="") # Start year for the reference data ref_start_yr = string(default="") +# This should be kept as "" for synthetic plots; +# zppy will then set it to match `figure_sets_period` +# For other plots, zppy will set it to match `years`, if not specified +ref_years = string_list(default=list("")) # Variables to be used by the pcmdi diagnostics # mean_climate: vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" # variability_modes_atm: vars = "psl" @@ -511,6 +513,8 @@ enso_groups = string(default="ENSO_perf,ENSO_proc,ENSO_tel") # Specify these for the synthetic_plots subtask ########################################################################################### figure_sets = string(default="mean_climate,variability_modes") +# Should be a list of length 3: +# clim_period, emov_period, enso_period figure_sets_period = string(default="") # File of observation data name for mean climate calculation reference_alias = string(default="inclusions/pcmdi_diags/reference_alias.json") @@ -559,13 +563,13 @@ cmip_movs_set = string(default="cmip6.historical.v20220825") save_test_clims = string(default=None) reference_data_path = string(default=None) reference_data_path_ts = string(default=None) - ref_years = string_list(default=None) ts_num_years = integer(default=None) ts_years = string_list(default=None) current_set = string(default=None) obs_sets = string(default=None) ref_final_yr = string(default=None) ref_start_yr = string(default=None) + ref_years = string_list(default=None) vars = string(default=None) cmip_vars = string(default=None) regions = string(default=None) diff --git a/zppy/pcmdi_diags.py b/zppy/pcmdi_diags.py index e948630d..8de37362 100644 --- a/zppy/pcmdi_diags.py +++ b/zppy/pcmdi_diags.py @@ -96,9 +96,15 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): if c["current_set"] != "synthetic_plots": check_and_define_parameters(c) + print(c["prefix"]) else: prefix = f"pcmdi_diags_{c['sub']}_{c['run_type']}" - print(prefix) + if i < len(year_sets) - 1: + logger.debug( + f"Not printing prefix={prefix} because we will only submit the job on the final iteration." + ) + else: + print(prefix) c["prefix"] = prefix bash_file, settings_file, status_file = get_file_names( @@ -115,7 +121,7 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): # List of dependencies # Iterate from year1 to year2 incrementing by the number of years per time series file. - if "ts_num_years" in c.keys(): + if c["current_set"] != "synthetic_plots": for yr in range(c["year1"], c["year2"], c["ts_num_years"]): add_ts_dependencies(c, dependencies, script_dir, yr) set_value_of_parameter_if_undefined( @@ -124,17 +130,16 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): "atm_monthly_180x360_aave", ParameterInferenceType.SECTION_INFERENCE, ) - add_dependencies( - dependencies, - script_dir, - "e3sm_to_cmip", - c["e3sm_to_cmip_atm_subsection"], - c["year1"], - c["year2"], - c["ts_num_years"], - ) - - if c["sub"] == "synthetic_plots": + add_dependencies( + dependencies, + script_dir, + "e3sm_to_cmip", + c["e3sm_to_cmip_atm_subsection"], + c["year1"], + c["year2"], + c["ts_num_years"], + ) + else: add_pcmdi_dependencies(c, dependencies, script_dir) if i < len(year_sets) - 1: continue @@ -177,14 +182,20 @@ def define_current_set(c: Dict[str, Any]): f"Invalid set '{current_set}'. Must be one of {VALID_PCMDI_SETS}" ) elif c["infer_path_parameters"]: - if "mean_climate" in c["subsection"]: + if "subsection" not in c: + raise ParameterNotProvidedError( + "subsection must be available if current_set is to be inferred." + ) + elif "mean_climate" in c["subsection"]: current_set = "mean_climate" elif "variability_modes_cpl" in c["subsection"]: current_set = "variability_modes_cpl" - elif "variability_modes_atm" in c["susbsection"]: + elif "variability_modes_atm" in c["subsection"]: current_set = "variability_modes_atm" elif "enso" in c["subsection"]: current_set = "enso" + elif "synthetic_plots" in c["subsection"]: + current_set = "synthetic_plots" else: raise ValueError(f"Could not determine set from {c['subsection']}") if current_set not in VALID_PCMDI_SETS: @@ -279,7 +290,6 @@ def check_and_define_parameters(c: Dict[str, Any]) -> None: ) else: raise ValueError(f"Invalid run_type={c['run_type']}") - print(prefix) c["prefix"] = prefix @@ -313,7 +323,7 @@ def add_pcmdi_dependencies( script_dir, f"pcmdi_diags_mean_climate_{c['run_type']}{status_suffix}.status", ) - if os.path.exists(status_file): + if os.path.exists(status_file) and (status_file not in dependencies): dependencies.append(status_file) if "variability_modes" in c["figure_sets"]: # cpl @@ -321,18 +331,18 @@ def add_pcmdi_dependencies( script_dir, f"pcmdi_diags_variability_modes_cpl_{c['run_type']}{status_suffix}.status", ) - if os.path.exists(status_file): + if os.path.exists(status_file) and (status_file not in dependencies): dependencies.append(status_file) # atm status_file = os.path.join( script_dir, f"pcmdi_diags_variability_modes_atm_{c['run_type']}{status_suffix}.status", ) - if os.path.exists(status_file): + if os.path.exists(status_file) and (status_file not in dependencies): dependencies.append(status_file) if "enso_metric" in c["figure_sets"]: status_file = os.path.join( script_dir, f"pcmdi_diags_enso_{c['run_type']}{status_suffix}.status" ) - if os.path.exists(status_file): + if os.path.exists(status_file) and (status_file not in dependencies): dependencies.append(status_file) diff --git a/zppy/utils.py b/zppy/utils.py index 2812b252..4120cae7 100644 --- a/zppy/utils.py +++ b/zppy/utils.py @@ -231,7 +231,15 @@ def set_component_and_prc_typ(c: Dict[str, Any]) -> None: def check_set_specific_parameter( c: Dict[str, Any], sets_with_requirement: Set[str], relevant_parameter: str ) -> None: - requested_sets = set(c["sets"]) + requested_sets: Set[str] + if ("sets" in c) and c["sets"]: + # For tasks that permit multiple sets + requested_sets = set(c["sets"]) + elif ("current_set" in c) and c["current_set"]: + # For tasks that permit only a single set + requested_sets = {c["current_set"]} + else: + requested_sets = set() intersection = sets_with_requirement & requested_sets if ( intersection From 0f57ddc7e66c278c7d60416ef4c10cf08fcf0e12 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 24 Sep 2025 10:53:29 -0500 Subject: [PATCH 21/47] Add pcmdi_diags to v3 test cfg --- .../template_weekly_comprehensive_v3.cfg | 72 ++++++++++++++++++- tests/integration/utils.py | 16 ++++- 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/tests/integration/template_weekly_comprehensive_v3.cfg b/tests/integration/template_weekly_comprehensive_v3.cfg index ffba6f10..45d7a7bf 100644 --- a/tests/integration/template_weekly_comprehensive_v3.cfg +++ b/tests/integration/template_weekly_comprehensive_v3.cfg @@ -194,7 +194,6 @@ tc_obs = "#expand diagnostics_base_path#/observations/Atm/tc-analysis/" # Reference paths reference_data_path = "#expand user_output#zppy_weekly_comprehensive_v3_output/#expand unique_id#/#expand case_name#/post/lnd/180x360_aave/clim" - [mpas_analysis] active = #expand active_mpas_analysis# anomalyRefYear = 1985 @@ -307,3 +306,74 @@ ts_land_subsection = "land_monthly" ts_num_years = 2 walltime = "2:00:00" years = "1985:1989:4" + +[pcmdi_diags] +active = True +environment_commands_secondary = "#expand pcmdi_diags_environment_commands#" +walltime = "2:00:00" +figure_format = "png" +run_type = "model_vs_obs" +# Create rules to construct cmip-like datasets with a string constructed as +# "mip.exp.product.realization", where "."is used as the delimiter (mandatory) +model_name = '#expand case_name#' +model_tableID = 'Amon' +multiprocessing = True +num_workers = 24 +# Observational data +obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series' +pcmdi_debug = False +ts_years = "1985-1986", + + [[ mean_climate ]] + active = True + # Default for all variables or specify source for each variable separately. + obs_sets = "default" + ref_final_yr = 1986 + ref_start_yr = 1985 + ref_years = "1985-1986", + # Variables in current pcmdi-cmip list + # rltcre and rstcre are derived variables as they are not included in cmip_ts + vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rsus,rsuscs,rlus,rlut,rlutcs,rsds,rsdscs,rsdt,rsut,rsutcs,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" + # See zppy/templates/pcmdi_diags/regions_specs.json for options of all regions + # Below is default setup for global/regional mean climate metrics + regions = "global,ocean,land,NHEX,SHEX,TROPICS" + target_grid = "2.5x2.5" + target_grid_string = "2p5x2p5" + + [[ variability_modes_cpl ]] + active = True + # See zppy/templates/pcmdi_diags/reference_alias.json for options + obs_sets = "alternate1" #"default" + ref_final_yr = 1986 + ref_start_yr = 1985 + ref_years = "1985-1986", + vars = "ts" + # Name of modes varibility + cpl_modes = "PDO,NPGO,AMO" + frequency = 'mo' + landmask = True + ModUnitsAdjust = '(True,"subtract",273.15)' + ObsUnitsAdjust = '(True,"subtract",273.15)' + seasons = "yearly,monthly" + + [[ variability_modes_atm ]] + active = True + # See zppy/templates/pcmdi_diags/reference_alias.json for options + # Default for all variables + obs_sets = "alternate2" # "default" + ref_final_yr = 1986 + ref_start_yr = 1985 + ref_years = "1985-1986", + vars = "psl" + # Varibility modes + atm_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" + frequency = 'mo' + landmask = False + ModUnitsAdjust = '(True,"divide",100.0)' + ObsUnitsAdjust = '(True,"divide",100.0)' + seasons = "DJF,MAM,JJA,SON,yearly,monthly" + + [[ synthetic_plots ]] + active = True + # clim_period, emov_period, enso_period: + figure_sets_period = "1985-1986,1985-1986,1985-1986" diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 6187408d..96f477d0 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -14,6 +14,7 @@ TEST_SPECIFICS: Dict[str, Any] = { "diags_environment_commands": "source /conda.sh; conda activate ", "global_time_series_environment_commands": "source /conda.sh; conda activate ", + "pcmdi_diags_environment_commands": "source /conda.sh; conda activate ", "cfgs_to_run": [ "weekly_bundles", "weekly_comprehensive_v2", @@ -22,7 +23,13 @@ "weekly_legacy_3.0.0_comprehensive_v2", "weekly_legacy_3.0.0_comprehensive_v3", ], - "tasks_to_run": ["e3sm_diags", "mpas_analysis", "global_time_series", "ilamb"], + "tasks_to_run": [ + "e3sm_diags", + "mpas_analysis", + "global_time_series", + "ilamb", + "pcmdi_diags", + ], "unique_id": "unique_id", } @@ -132,6 +139,9 @@ def get_expansions(): expansions["global_time_series_environment_commands"] = TEST_SPECIFICS[ "global_time_series_environment_commands" ] + expansions["pcmdi_diags_environment_commands"] = TEST_SPECIFICS[ + "pcmdi_diags_environment_commands" + ] # Activate requested tests expansions["active_e3sm_to_cmip"] = "False" @@ -139,6 +149,7 @@ def get_expansions(): expansions["active_mpas_analysis"] = "False" expansions["active_global_time_series"] = "False" expansions["active_ilamb"] = "False" + expansions["active_pcmdi_diags"] = "False" if "e3sm_diags" in TEST_SPECIFICS["tasks_to_run"]: expansions["active_e3sm_diags"] = "True" if "mpas_analysis" in TEST_SPECIFICS["tasks_to_run"]: @@ -150,6 +161,9 @@ def get_expansions(): if "ilamb" in TEST_SPECIFICS["tasks_to_run"]: expansions["active_ilamb"] = "True" expansions["active_e3sm_to_cmip"] = "True" + if "pcmdi_diags" in TEST_SPECIFICS["tasks_to_run"]: + expansions["active_pcmdi_diags"] = "True" + expansions["active_e3sm_to_cmip"] = "True" expansions["cfgs_to_run"] = TEST_SPECIFICS["cfgs_to_run"] expansions["tasks_to_run"] = TEST_SPECIFICS["tasks_to_run"] From d8cd7682585287bb25aca3923d2cb27a1d643972 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 24 Sep 2025 11:54:17 -0500 Subject: [PATCH 22/47] pcmdi_diags test cfg fixes --- ...test_weekly_comprehensive_v3_chrysalis.cfg | 80 ++++++++++++++++++- .../template_weekly_comprehensive_v3.cfg | 8 ++ tests/integration/utils.py | 3 + zppy/pcmdi_diags.py | 39 ++++----- 4 files changed, 111 insertions(+), 19 deletions(-) diff --git a/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg b/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg index d823964b..598b14d4 100644 --- a/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg +++ b/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg @@ -194,7 +194,6 @@ tc_obs = "/lcrc/group/e3sm/diagnostics/observations/Atm/tc-analysis/" # Reference paths reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/lnd/180x360_aave/clim" - [mpas_analysis] active = True anomalyRefYear = 1985 @@ -307,3 +306,82 @@ ts_land_subsection = "land_monthly" ts_num_years = 2 walltime = "2:00:00" years = "1985:1989:4" + +[pcmdi_diags] +active = True +environment_commands_secondary = "source /conda.sh; conda activate " +walltime = "2:00:00" +figure_format = "png" +run_type = "model_vs_obs" +# Create rules to construct cmip-like datasets with a string constructed as +# "mip.exp.product.realization", where "."is used as the delimiter (mandatory) +model_name = 'v3.LR.historical_0051' +model_tableID = 'Amon' +e3sm_to_cmip_atm_subsection = "atm_monthly_180x360_aave" # Can be inferred if needed +multiprocessing = True +num_workers = 24 +# Observational data +obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series' +pcmdi_debug = False +ts_num_years = 2 +ts_years = "1985-1986", + + [[ mean_climate ]] + active = True + current_set = "mean_climate" # Can be inferred if needed + # Default for all variables or specify source for each variable separately. + obs_sets = "default" + ref_final_yr = 1986 + ref_start_yr = 1985 + ref_years = "1985-1986", + # Variables in current pcmdi-cmip list + # rltcre and rstcre are derived variables as they are not included in cmip_ts + vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rsus,rsuscs,rlus,rlut,rlutcs,rsds,rsdscs,rsdt,rsut,rsutcs,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" + # See zppy/templates/pcmdi_diags/regions_specs.json for options of all regions + # Below is default setup for global/regional mean climate metrics + regions = "global,ocean,land,NHEX,SHEX,TROPICS" + target_grid = "2.5x2.5" + target_grid_string = "2p5x2p5" + + [[ variability_modes_cpl ]] + active = True + current_set = "variability_modes_cpl" # Can be inferred if needed + # See zppy/templates/pcmdi_diags/reference_alias.json for options + obs_sets = "alternate1" #"default" + ref_final_yr = 1986 + ref_start_yr = 1985 + ref_years = "1985-1986", + vars = "ts" + # Name of modes varibility + cpl_modes = "PDO,NPGO,AMO" + frequency = 'mo' + landmask = True + ModUnitsAdjust = '(True,"subtract",273.15)' + ObsUnitsAdjust = '(True,"subtract",273.15)' + seasons = "yearly,monthly" + + [[ variability_modes_atm ]] + active = True + current_set = "variability_modes_atm" # Can be inferred if needed + # See zppy/templates/pcmdi_diags/reference_alias.json for options + # Default for all variables + obs_sets = "alternate2" # "default" + ref_final_yr = 1986 + ref_start_yr = 1985 + ref_years = "1985-1986", + vars = "psl" + # Varibility modes + atm_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" + frequency = 'mo' + landmask = False + ModUnitsAdjust = '(True,"divide",100.0)' + ObsUnitsAdjust = '(True,"divide",100.0)' + seasons = "DJF,MAM,JJA,SON,yearly,monthly" + + [[ synthetic_plots ]] + active = True + current_set = "synthetic_plots" # Can be inferred if needed + # clim_period, emov_period, enso_period: + figure_sets_period = "1985-1986,1985-1986,1985-1986" + cmip_clim_dir = /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/mean_climate + cmip_movs_dir = /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes diff --git a/tests/integration/template_weekly_comprehensive_v3.cfg b/tests/integration/template_weekly_comprehensive_v3.cfg index 45d7a7bf..58f9a8e3 100644 --- a/tests/integration/template_weekly_comprehensive_v3.cfg +++ b/tests/integration/template_weekly_comprehensive_v3.cfg @@ -317,15 +317,18 @@ run_type = "model_vs_obs" # "mip.exp.product.realization", where "."is used as the delimiter (mandatory) model_name = '#expand case_name#' model_tableID = 'Amon' +e3sm_to_cmip_atm_subsection = "atm_monthly_180x360_aave" # Can be inferred if needed multiprocessing = True num_workers = 24 # Observational data obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series' pcmdi_debug = False +ts_num_years = 2 ts_years = "1985-1986", [[ mean_climate ]] active = True + current_set = "mean_climate" # Can be inferred if needed # Default for all variables or specify source for each variable separately. obs_sets = "default" ref_final_yr = 1986 @@ -342,6 +345,7 @@ ts_years = "1985-1986", [[ variability_modes_cpl ]] active = True + current_set = "variability_modes_cpl" # Can be inferred if needed # See zppy/templates/pcmdi_diags/reference_alias.json for options obs_sets = "alternate1" #"default" ref_final_yr = 1986 @@ -358,6 +362,7 @@ ts_years = "1985-1986", [[ variability_modes_atm ]] active = True + current_set = "variability_modes_atm" # Can be inferred if needed # See zppy/templates/pcmdi_diags/reference_alias.json for options # Default for all variables obs_sets = "alternate2" # "default" @@ -375,5 +380,8 @@ ts_years = "1985-1986", [[ synthetic_plots ]] active = True + current_set = "synthetic_plots" # Can be inferred if needed # clim_period, emov_period, enso_period: figure_sets_period = "1985-1986,1985-1986,1985-1986" + cmip_clim_dir = #expand diagnostics_base_path#/pcmdi_data/metrics_data/mean_climate + cmip_movs_dir = #expand diagnostics_base_path#/pcmdi_data/metrics_data/variability_modes diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 96f477d0..72f3e297 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -316,6 +316,9 @@ def generate_cfgs(unified_testing=False, dry_run=False): print( f"global_time_series_environment_commands={expansions['global_time_series_environment_commands']}" ) + print( + f"pcmdi_diags_environment_commands={expansions['pcmdi_diags_environment_commands']}" + ) print(f"environment_commands={expansions['environment_commands']}") print( "Reminder: `environment_commands=''` => the latest E3SM Unified environment will be used" diff --git a/zppy/pcmdi_diags.py b/zppy/pcmdi_diags.py index 8de37362..9612f5ff 100644 --- a/zppy/pcmdi_diags.py +++ b/zppy/pcmdi_diags.py @@ -231,24 +231,27 @@ def check_parameters_for_bash(c: Dict[str, Any]) -> None: def check_parameters_for_pcmdi(c: Dict[str, Any]) -> None: # check and set up the external data needed by pcmdi if c["current_set"] == "synthetic_plots": - set_value_of_parameter_if_undefined( - c, - "cmip_enso_dir", - f"{c['diagnostics_base_path']}/pcmdi_data/metrics_data/enso_metric", - ParameterInferenceType.PATH_INFERENCE, - ) - set_value_of_parameter_if_undefined( - c, - "cmip_clim_dir", - f"{c['diagnostics_base_path']}/pcmdi_data/metrics_data/mean_climate", - ParameterInferenceType.PATH_INFERENCE, - ) - set_value_of_parameter_if_undefined( - c, - "cmip_movs_dir", - f"{c['diagnostics_base_path']}/pcmdi_data/metrics_data/variability_modes", - ParameterInferenceType.PATH_INFERENCE, - ) + if "enso_metric" in c["figure_sets"]: + set_value_of_parameter_if_undefined( + c, + "cmip_enso_dir", + f"{c['diagnostics_base_path']}/pcmdi_data/metrics_data/enso_metric", + ParameterInferenceType.PATH_INFERENCE, + ) + if "mean_climate" in c["figure_sets"]: + set_value_of_parameter_if_undefined( + c, + "cmip_clim_dir", + f"{c['diagnostics_base_path']}/pcmdi_data/metrics_data/mean_climate", + ParameterInferenceType.PATH_INFERENCE, + ) + if "variability_modes" in c["figure_sets"]: + set_value_of_parameter_if_undefined( + c, + "cmip_movs_dir", + f"{c['diagnostics_base_path']}/pcmdi_data/metrics_data/variability_modes", + ParameterInferenceType.PATH_INFERENCE, + ) def check_mvm_only_parameters_for_bash(c: Dict[str, Any]) -> None: From 58d5ec679f7eba3ac2c9bfe2011d29dfa83945bc Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 24 Sep 2025 18:21:04 -0500 Subject: [PATCH 23/47] Add some unit tests for non-IO functions --- tests/test_zppy_e3sm_diags.py | 29 ------ tests/test_zppy_pcmdi_diags.py | 162 +++++++++++++++++++++++++++++++++ tests/test_zppy_utils.py | 29 ++++++ zppy/pcmdi_diags.py | 10 +- zppy/utils.py | 1 + 5 files changed, 193 insertions(+), 38 deletions(-) create mode 100644 tests/test_zppy_pcmdi_diags.py diff --git a/tests/test_zppy_e3sm_diags.py b/tests/test_zppy_e3sm_diags.py index f69f7cc4..cba17877 100644 --- a/tests/test_zppy_e3sm_diags.py +++ b/tests/test_zppy_e3sm_diags.py @@ -7,40 +7,11 @@ add_ts_dependencies, check_and_define_parameters, check_mvm_only_parameters_for_bash, - check_parameter_defined, check_parameters_for_bash, - check_set_specific_parameter, ) from zppy.utils import ParameterNotProvidedError -def test_check_parameter_defined(): - c = {"a": 1, "b": 2, "c": ""} - check_parameter_defined(c, "a") - with pytest.raises(ParameterNotProvidedError): - check_parameter_defined(c, "c") - with pytest.raises(ParameterNotProvidedError): - check_parameter_defined(c, "d") - - -def test_check_set_specific_parameter(): - # Parameter is required - # a, b need parameter p, and we want sets a, b, c - c = {"sets": ["a", "b", "c"], "p": "exists"} - check_set_specific_parameter(c, set(["a", "b"]), "p") - - # Parameter isn't required based on the sets we want - # z needs parameter p, but we only want sets a, b, c - c = {"sets": ["a", "b", "c"], "p": ""} - check_set_specific_parameter(c, set(["z"]), "p") - - # Parameter is required - # a, b need parameter p, and we want sets a, b, c - c = {"sets": ["a", "b", "c"], "p": ""} - with pytest.raises(ParameterNotProvidedError): - check_set_specific_parameter(c, set(["a", "b"]), "p") - - def test_check_parameters_for_bash(): # diurnal_cycle c = {"sets": ["diurnal_cycle"], "climo_diurnal_frequency": "diurnal_8xdaily"} diff --git a/tests/test_zppy_pcmdi_diags.py b/tests/test_zppy_pcmdi_diags.py new file mode 100644 index 00000000..3d0354d2 --- /dev/null +++ b/tests/test_zppy_pcmdi_diags.py @@ -0,0 +1,162 @@ +import pytest + +from zppy.pcmdi_diags import ( + check_parameters_for_bash, + check_parameters_for_pcmdi, + define_current_set, +) +from zppy.utils import ParameterNotProvidedError + + +def test_define_current_set(): + c = {"current_set": "mean_climate", "infer_path_parameters": False} + define_current_set(c) + assert c["current_set"] == "mean_climate" + c = {"current_set": "variability_modes_cpl", "infer_path_parameters": False} + define_current_set(c) + assert c["current_set"] == "variability_modes_cpl" + c = {"current_set": "variability_modes_atm", "infer_path_parameters": False} + define_current_set(c) + assert c["current_set"] == "variability_modes_atm" + c = {"current_set": "enso", "infer_path_parameters": False} + define_current_set(c) + assert c["current_set"] == "enso" + c = {"current_set": "synthetic_plots", "infer_path_parameters": False} + define_current_set(c) + assert c["current_set"] == "synthetic_plots" + c = {"current_set": "invalid_set", "infer_path_parameters": False} + with pytest.raises(ValueError): + define_current_set(c) + + c = {"current_set": "", "subsection": "mean_climate", "infer_path_parameters": True} + define_current_set(c) + assert c["current_set"] == "mean_climate" + c = { + "current_set": "", + "subsection": "variability_modes_cpl", + "infer_path_parameters": True, + } + define_current_set(c) + assert c["current_set"] == "variability_modes_cpl" + c = { + "current_set": "", + "subsection": "variability_modes_atm", + "infer_path_parameters": True, + } + define_current_set(c) + assert c["current_set"] == "variability_modes_atm" + c = {"current_set": "", "subsection": "enso", "infer_path_parameters": True} + define_current_set(c) + assert c["current_set"] == "enso" + c = { + "current_set": "", + "subsection": "synthetic_plots", + "infer_path_parameters": True, + } + define_current_set(c) + assert c["current_set"] == "synthetic_plots" + c = {"current_set": "", "subsection": "invalid_set", "infer_path_parameters": True} + with pytest.raises(ValueError): + define_current_set(c) + + c = { + "current_set": "", + "subsection": "mean_climate", + "infer_path_parameters": False, + } + with pytest.raises(ParameterNotProvidedError): + define_current_set(c) + + +def test_check_parameters_for_bash(): + c = {"current_set": "mean_climate", "ref_final_yr": "2000", "ref_start_yr": "1850"} + check_parameters_for_bash(c) + + c = {"current_set": "mean_climate", "ref_final_yr": "", "ref_start_yr": "1850"} + with pytest.raises(ParameterNotProvidedError): + check_parameters_for_bash(c) + + c = {"current_set": "mean_climate", "ref_final_yr": "2000", "ref_start_yr": ""} + with pytest.raises(ParameterNotProvidedError): + check_parameters_for_bash(c) + + c = { + "current_set": "synthetic_plots", + "ref_final_yr": "2000", + "ref_start_yr": "1850", + } + check_parameters_for_bash(c) + + c = {"current_set": "synthetic_plots", "ref_final_yr": "", "ref_start_yr": "1850"} + check_parameters_for_bash(c) + + c = {"current_set": "synthetic_plots", "ref_final_yr": "2000", "ref_start_yr": ""} + check_parameters_for_bash(c) + + +def test_check_parameters_for_pcmdi(): + c = { + "current_set": "synthetic_plots", + "figure_sets": [ + "mean_climate", + "variability_modes", + ], + "cmip_enso_dir": "", + "cmip_clim_dir": "", + "cmip_movs_dir": "", + "diagnostics_base_path": "diags/post", + "infer_path_parameters": True, + } + check_parameters_for_pcmdi(c) + assert c["cmip_clim_dir"] == "diags/post/pcmdi_data/metrics_data/mean_climate" + assert c["cmip_movs_dir"] == "diags/post/pcmdi_data/metrics_data/variability_modes" + assert c["cmip_enso_dir"] == "" + + c = { + "current_set": "mean_climate", + "figure_sets": [ + "mean_climate", + "variability_modes", + ], + "cmip_enso_dir": "", + "cmip_clim_dir": "", + "cmip_movs_dir": "", + "diagnostics_base_path": "diags/post", + "infer_path_parameters": True, + } + check_parameters_for_pcmdi(c) + assert c["cmip_clim_dir"] == "" + assert c["cmip_movs_dir"] == "" + assert c["cmip_enso_dir"] == "" + + c = { + "current_set": "synthetic_plots", + "figure_sets": [ + "mean_climate", + "variability_modes", + ], + "cmip_enso_dir": "", + "cmip_clim_dir": "", + "cmip_movs_dir": "", + "diagnostics_base_path": "diags/post", + "infer_path_parameters": False, + } + with pytest.raises(ParameterNotProvidedError): + check_parameters_for_pcmdi(c) + + c = { + "current_set": "mean_climate", + "figure_sets": [ + "mean_climate", + "variability_modes", + ], + "cmip_enso_dir": "", + "cmip_clim_dir": "", + "cmip_movs_dir": "", + "diagnostics_base_path": "diags/post", + "infer_path_parameters": False, + } + check_parameters_for_pcmdi(c) + assert c["cmip_clim_dir"] == "" + assert c["cmip_movs_dir"] == "" + assert c["cmip_enso_dir"] == "" diff --git a/tests/test_zppy_utils.py b/tests/test_zppy_utils.py index 1e67ca8e..10035177 100644 --- a/tests/test_zppy_utils.py +++ b/tests/test_zppy_utils.py @@ -6,6 +6,8 @@ ParameterInferenceType, ParameterNotProvidedError, add_dependencies, + check_parameter_defined, + check_set_specific_parameter, get_active_status, get_file_names, get_inference_type_parameter, @@ -187,6 +189,24 @@ def test_set_component_and_prc_typ(): set_component_and_prc_typ(c) +def test_check_set_specific_parameter(): + # Parameter is required + # a, b need parameter p, and we want sets a, b, c + c = {"sets": ["a", "b", "c"], "p": "exists"} + check_set_specific_parameter(c, set(["a", "b"]), "p") + + # Parameter isn't required based on the sets we want + # z needs parameter p, but we only want sets a, b, c + c = {"sets": ["a", "b", "c"], "p": ""} + check_set_specific_parameter(c, set(["z"]), "p") + + # Parameter is required + # a, b need parameter p, and we want sets a, b, c + c = {"sets": ["a", "b", "c"], "p": ""} + with pytest.raises(ParameterNotProvidedError): + check_set_specific_parameter(c, set(["a", "b"]), "p") + + def test_get_years(): assert get_years("1980:1990:05") == [(1980, 1984), (1985, 1989)] assert get_years("1980-1990") == [(1980, 1990)] @@ -478,6 +498,15 @@ def test_set_value_of_parameter_if_undefined(): assert c["required_parameter"] == "backup_option" +def test_check_parameter_defined(): + c = {"a": 1, "b": 2, "c": ""} + check_parameter_defined(c, "a") + with pytest.raises(ParameterNotProvidedError): + check_parameter_defined(c, "c") + with pytest.raises(ParameterNotProvidedError): + check_parameter_defined(c, "d") + + def test_get_file_names(): bash, settings, status = get_file_names("script_dir", "prefix") assert bash == "script_dir/prefix.bash" diff --git a/zppy/pcmdi_diags.py b/zppy/pcmdi_diags.py index 9612f5ff..213966ca 100644 --- a/zppy/pcmdi_diags.py +++ b/zppy/pcmdi_diags.py @@ -66,9 +66,6 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): if "ts_num_years" in c.keys(): c["ts_num_years"] = int(c["ts_num_years"]) - # procedure type for e3sm_to_cmip - c["cmor_tables_prefix"] = c["diagnostics_base_path"] - # check and set parameter for pcmdi check_parameters_for_pcmdi(c) @@ -210,7 +207,7 @@ def define_current_set(c: Dict[str, Any]): def check_parameters_for_bash(c: Dict[str, Any]) -> None: - if c["sub"] != "synthetic_plots": + if c["current_set"] != "synthetic_plots": check_set_specific_parameter( c, BASE_PCMDI_SETS, @@ -221,11 +218,6 @@ def check_parameters_for_bash(c: Dict[str, Any]) -> None: BASE_PCMDI_SETS, "ref_start_yr", ) - check_set_specific_parameter( - c, - BASE_PCMDI_SETS, - "ref_end_yr", - ) def check_parameters_for_pcmdi(c: Dict[str, Any]) -> None: diff --git a/zppy/utils.py b/zppy/utils.py index 4120cae7..0b3ce877 100644 --- a/zppy/utils.py +++ b/zppy/utils.py @@ -330,6 +330,7 @@ def set_value_of_parameter_if_undefined( raise ParameterNotProvidedError( f"{parameter} was not provided, and inferring is turned off. Turn on inferring by setting {inference_type_parameter} to True." ) + # If parameter is already defined, do nothing. def check_parameter_defined( From d81063035d00b63d3a3b0dafeb7e760817afff10 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 24 Sep 2025 18:48:38 -0500 Subject: [PATCH 24/47] Add more unit tests with Claude --- tests/test_zppy_pcmdi_diags.py | 352 ++++++++++++++++++++++++++++++++- 1 file changed, 349 insertions(+), 3 deletions(-) diff --git a/tests/test_zppy_pcmdi_diags.py b/tests/test_zppy_pcmdi_diags.py index 3d0354d2..0b73648c 100644 --- a/tests/test_zppy_pcmdi_diags.py +++ b/tests/test_zppy_pcmdi_diags.py @@ -1,6 +1,13 @@ +from typing import List +from unittest.mock import patch + import pytest from zppy.pcmdi_diags import ( + add_pcmdi_dependencies, + add_ts_dependencies, + check_and_define_parameters, + check_mvm_only_parameters_for_bash, check_parameters_for_bash, check_parameters_for_pcmdi, define_current_set, @@ -9,28 +16,36 @@ def test_define_current_set(): + # Set current_set directly c = {"current_set": "mean_climate", "infer_path_parameters": False} define_current_set(c) assert c["current_set"] == "mean_climate" + c = {"current_set": "variability_modes_cpl", "infer_path_parameters": False} define_current_set(c) assert c["current_set"] == "variability_modes_cpl" + c = {"current_set": "variability_modes_atm", "infer_path_parameters": False} define_current_set(c) assert c["current_set"] == "variability_modes_atm" + c = {"current_set": "enso", "infer_path_parameters": False} define_current_set(c) assert c["current_set"] == "enso" + c = {"current_set": "synthetic_plots", "infer_path_parameters": False} define_current_set(c) assert c["current_set"] == "synthetic_plots" + c = {"current_set": "invalid_set", "infer_path_parameters": False} with pytest.raises(ValueError): define_current_set(c) + # Infer current_set from subsection c = {"current_set": "", "subsection": "mean_climate", "infer_path_parameters": True} define_current_set(c) assert c["current_set"] == "mean_climate" + c = { "current_set": "", "subsection": "variability_modes_cpl", @@ -38,6 +53,7 @@ def test_define_current_set(): } define_current_set(c) assert c["current_set"] == "variability_modes_cpl" + c = { "current_set": "", "subsection": "variability_modes_atm", @@ -45,9 +61,11 @@ def test_define_current_set(): } define_current_set(c) assert c["current_set"] == "variability_modes_atm" + c = {"current_set": "", "subsection": "enso", "infer_path_parameters": True} define_current_set(c) assert c["current_set"] == "enso" + c = { "current_set": "", "subsection": "synthetic_plots", @@ -55,6 +73,7 @@ def test_define_current_set(): } define_current_set(c) assert c["current_set"] == "synthetic_plots" + c = {"current_set": "", "subsection": "invalid_set", "infer_path_parameters": True} with pytest.raises(ValueError): define_current_set(c) @@ -67,8 +86,28 @@ def test_define_current_set(): with pytest.raises(ParameterNotProvidedError): define_current_set(c) + # Test case where subsection is missing with infer_path_parameters=True + c = {"current_set": "", "infer_path_parameters": True} + with pytest.raises(ParameterNotProvidedError): + define_current_set(c) + def test_check_parameters_for_bash(): + # Test synthetic_plots (should not require ref_start_yr and ref_end_yr) + c = { + "current_set": "synthetic_plots", + "ref_final_yr": "2000", + "ref_start_yr": "1850", + } + check_parameters_for_bash(c) + + c = {"current_set": "synthetic_plots", "ref_final_yr": "", "ref_start_yr": "1850"} + check_parameters_for_bash(c) + + c = {"current_set": "synthetic_plots", "ref_final_yr": "2000", "ref_start_yr": ""} + check_parameters_for_bash(c) + + # Test BASE_PCMDI_SETS c = {"current_set": "mean_climate", "ref_final_yr": "2000", "ref_start_yr": "1850"} check_parameters_for_bash(c) @@ -81,16 +120,36 @@ def test_check_parameters_for_bash(): check_parameters_for_bash(c) c = { - "current_set": "synthetic_plots", + "current_set": "variability_modes_atm", "ref_final_yr": "2000", "ref_start_yr": "1850", } check_parameters_for_bash(c) - c = {"current_set": "synthetic_plots", "ref_final_yr": "", "ref_start_yr": "1850"} + c = { + "current_set": "variability_modes_atm", + "ref_final_yr": "", + "ref_start_yr": "1850", + } + with pytest.raises(ParameterNotProvidedError): + check_parameters_for_bash(c) + + c = { + "current_set": "variability_modes_cpl", + "ref_final_yr": "2000", + "ref_start_yr": "1850", + } check_parameters_for_bash(c) - c = {"current_set": "synthetic_plots", "ref_final_yr": "2000", "ref_start_yr": ""} + c = { + "current_set": "variability_modes_cpl", + "ref_final_yr": "2000", + "ref_start_yr": "", + } + with pytest.raises(ParameterNotProvidedError): + check_parameters_for_bash(c) + + c = {"current_set": "enso", "ref_final_yr": "2000", "ref_start_yr": "1850"} check_parameters_for_bash(c) @@ -160,3 +219,290 @@ def test_check_parameters_for_pcmdi(): assert c["cmip_clim_dir"] == "" assert c["cmip_movs_dir"] == "" assert c["cmip_enso_dir"] == "" + + # Test enso_metric figure set + c = { + "current_set": "synthetic_plots", + "figure_sets": ["enso_metric"], + "cmip_enso_dir": "", + "cmip_clim_dir": "", + "cmip_movs_dir": "", + "diagnostics_base_path": "diags/post", + "infer_path_parameters": True, + } + check_parameters_for_pcmdi(c) + assert c["cmip_enso_dir"] == "diags/post/pcmdi_data/metrics_data/enso_metric" + + # Test when parameters are already defined + c = { + "current_set": "synthetic_plots", + "figure_sets": ["mean_climate"], + "cmip_enso_dir": "/custom/enso", + "cmip_clim_dir": "/custom/clim", + "cmip_movs_dir": "/custom/movs", + "diagnostics_base_path": "diags/post", + "infer_path_parameters": True, + } + check_parameters_for_pcmdi(c) + assert c["cmip_clim_dir"] == "/custom/clim" # Should not change + + +def test_check_mvm_only_parameters_for_bash(): + # Test with all required parameters present + c = { + "current_set": "mean_climate", + "reference_data_path_ts": "/path/to/ref/ts", + "model_name_ref": "CESM2", + "model_tableID_ref": "Amon", + "ref_start_yr": "1850", + "ts_subsection": "atm_monthly_180x360_aave", + } + check_mvm_only_parameters_for_bash(c) + + # Test missing reference_data_path_ts + c = { + "current_set": "mean_climate", + "model_name_ref": "CESM2", + "model_tableID_ref": "Amon", + "ref_start_yr": "1850", + "ts_subsection": "atm_monthly_180x360_aave", + } + with pytest.raises(ParameterNotProvidedError): + check_mvm_only_parameters_for_bash(c) + + # Test missing model_name_ref + c = { + "current_set": "mean_climate", + "reference_data_path_ts": "/path/to/ref/ts", + "model_tableID_ref": "Amon", + "ref_start_yr": "1850", + "ts_subsection": "atm_monthly_180x360_aave", + } + with pytest.raises(ParameterNotProvidedError): + check_mvm_only_parameters_for_bash(c) + + # Test missing model_tableID_ref + c = { + "current_set": "mean_climate", + "reference_data_path_ts": "/path/to/ref/ts", + "model_name_ref": "CESM2", + "ref_start_yr": "1850", + "ts_subsection": "atm_monthly_180x360_aave", + } + with pytest.raises(ParameterNotProvidedError): + check_mvm_only_parameters_for_bash(c) + + # Test missing ref_start_yr for BASE_PCMDI_SETS + c = { + "current_set": "mean_climate", + "reference_data_path_ts": "/path/to/ref/ts", + "model_name_ref": "CESM2", + "model_tableID_ref": "Amon", + "ref_start_yr": "", + "ts_subsection": "atm_monthly_180x360_aave", + } + with pytest.raises(ParameterNotProvidedError): + check_mvm_only_parameters_for_bash(c) + + # Test missing ts_subsection for BASE_PCMDI_SETS + c = { + "current_set": "mean_climate", + "reference_data_path_ts": "/path/to/ref/ts", + "model_name_ref": "CESM2", + "model_tableID_ref": "Amon", + "ref_start_yr": "1850", + "ts_subsection": "", + } + with pytest.raises(ParameterNotProvidedError): + check_mvm_only_parameters_for_bash(c) + + # Test synthetic_plots (should not require ref_start_yr and ts_subsection) + c = { + "current_set": "synthetic_plots", + "reference_data_path_ts": "/path/to/ref/ts", + "model_name_ref": "CESM2", + "model_tableID_ref": "Amon", + } + check_mvm_only_parameters_for_bash(c) + + +def test_check_and_define_parameters(): + # Test model_vs_obs run_type + c = { + "run_type": "model_vs_obs", + "sub": "mean_climate", + "year1": 2000, + "year2": 2010, + "ts_num_years": 1, + "obs_ts": "", + "diagnostics_base_path": "/diags", + "infer_path_parameters": True, + } + check_and_define_parameters(c) + assert c["prefix"] == "pcmdi_diags_mean_climate_model_vs_obs_2000-2010" + assert c["obs_ts"] == "/diags/observations/Atm/time-series/" + + # Test model_vs_model run_type + c = { + "run_type": "model_vs_model", + "sub": "variability_modes_atm", + "year1": 2000, + "year2": 2010, + "ref_year1": 1850, + "ref_year2": 1900, + "current_set": "variability_modes_atm", + "reference_data_path_ts": "/ref/path/ts", + "model_name_ref": "CESM2", + "model_tableID_ref": "Amon", + "ref_start_yr": "1850", + "ts_subsection": "atm_monthly_180x360_aave", + "reference_data_path": "/ref/path/post/analysis", + "grid": "180x360_aave", + "infer_path_parameters": True, + } + check_and_define_parameters(c) + assert ( + c["prefix"] + == "pcmdi_diags_variability_modes_atm_model_vs_model_2000-2010_vs_1850-1900" + ) + assert ( + c["reference_data_path_ts"] + == "/ref/path/ts" # Value is set, so won't be changed + ) + + # Test invalid run_type + c = { + "run_type": "invalid_type", + "sub": "mean_climate", + "year1": 2000, + "year2": 2010, + } + with pytest.raises(ValueError): + check_and_define_parameters(c) + + # Test model_vs_obs without ts_num_years (should not set obs_ts) + c = { + "run_type": "model_vs_obs", + "sub": "mean_climate", + "year1": 2000, + "year2": 2010, + "obs_ts": "", + } + check_and_define_parameters(c) + assert c["prefix"] == "pcmdi_diags_mean_climate_model_vs_obs_2000-2010" + assert c["obs_ts"] == "" + + +def test_add_ts_dependencies(): + dependencies: List[str] = [] + c = { + "current_set": "mean_climate", + "ts_num_years": 5, + } + script_dir = "/scripts" + yr = 2000 + + with patch("zppy.pcmdi_diags.add_dependencies") as mock_add_dependencies: + add_ts_dependencies(c, dependencies, script_dir, yr) + mock_add_dependencies.assert_called_once_with( + dependencies, + script_dir, + "ts", + "atm_monthly_180x360_aave", + 2000, + 2004, + 5, + ) + + # Test with non-BASE_PCMDI_SETS (should not call add_dependencies) + dependencies = [] + c = { + "current_set": "synthetic_plots", + "ts_num_years": 5, + } + + with patch("zppy.pcmdi_diags.add_dependencies") as mock_add_dependencies: + add_ts_dependencies(c, dependencies, script_dir, yr) + mock_add_dependencies.assert_not_called() + + +@patch("os.path.exists") +def test_add_pcmdi_dependencies(mock_exists): + dependencies: List[str] = [] + c = { + "run_type": "model_vs_obs", + "year1": 2000, + "year2": 2010, + "figure_sets": ["mean_climate", "variability_modes", "enso_metric"], + } + script_dir = "/scripts" + + # Mock all status files as existing + mock_exists.return_value = True + + add_pcmdi_dependencies(c, dependencies, script_dir) + + expected_dependencies = [ + "/scripts/pcmdi_diags_mean_climate_model_vs_obs_2000-2010.status", + "/scripts/pcmdi_diags_variability_modes_cpl_model_vs_obs_2000-2010.status", + "/scripts/pcmdi_diags_variability_modes_atm_model_vs_obs_2000-2010.status", + "/scripts/pcmdi_diags_enso_model_vs_obs_2000-2010.status", + ] + assert set(dependencies) == set(expected_dependencies) + + # Test model_vs_model run_type + dependencies = [] + c = { + "run_type": "model_vs_model", + "year1": 2000, + "year2": 2010, + "ref_year1": 1850, + "ref_year2": 1900, + "figure_sets": ["mean_climate"], + } + + add_pcmdi_dependencies(c, dependencies, script_dir) + expected_dependencies = [ + "/scripts/pcmdi_diags_mean_climate_model_vs_model_2000-2010_vs_1850-1900.status", + ] + assert dependencies == expected_dependencies + + # Test with non-existing status files + dependencies = [] + c = { + "run_type": "model_vs_obs", + "year1": 2000, + "year2": 2010, + "figure_sets": ["mean_climate"], + } + mock_exists.return_value = False + + add_pcmdi_dependencies(c, dependencies, script_dir) + assert dependencies == [] + + # Test missing run_type + c = { + "year1": 2000, + "year2": 2010, + "figure_sets": ["mean_climate"], + } + with pytest.raises(ParameterNotProvidedError): + add_pcmdi_dependencies(c, dependencies, script_dir) + + # Test duplicate dependencies are not added + dependencies = ["/scripts/pcmdi_diags_mean_climate_model_vs_obs_2000-2010.status"] + c = { + "run_type": "model_vs_obs", + "year1": 2000, + "year2": 2010, + "figure_sets": ["mean_climate"], + } + mock_exists.return_value = True + + add_pcmdi_dependencies(c, dependencies, script_dir) + # Should still only have one instance + assert len(dependencies) == 1 + assert ( + dependencies[0] + == "/scripts/pcmdi_diags_mean_climate_model_vs_obs_2000-2010.status" + ) From 765adce6912f23a295de15726352dce1f95be689 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 24 Sep 2025 19:51:01 -0500 Subject: [PATCH 25/47] v3 test cfg fixes --- .../generated/test_weekly_comprehensive_v3_chrysalis.cfg | 5 +++-- tests/integration/template_weekly_comprehensive_v3.cfg | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg b/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg index 598b14d4..a0cd3ea6 100644 --- a/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg +++ b/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg @@ -315,7 +315,7 @@ figure_format = "png" run_type = "model_vs_obs" # Create rules to construct cmip-like datasets with a string constructed as # "mip.exp.product.realization", where "."is used as the delimiter (mandatory) -model_name = 'v3.LR.historical_0051' +model_name = 'e3sm.historical.v3-LR.0051' model_tableID = 'Amon' e3sm_to_cmip_atm_subsection = "atm_monthly_180x360_aave" # Can be inferred if needed multiprocessing = True @@ -336,7 +336,8 @@ ts_years = "1985-1986", ref_years = "1985-1986", # Variables in current pcmdi-cmip list # rltcre and rstcre are derived variables as they are not included in cmip_ts - vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rsus,rsuscs,rlus,rlut,rlutcs,rsds,rsdscs,rsdt,rsut,rsutcs,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" + vars = "pr,rlut,rtmt,tas,tauu,tauv,ts" + #vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rsus,rsuscs,rlus,rlut,rlutcs,rsds,rsdscs,rsdt,rsut,rsutcs,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" # See zppy/templates/pcmdi_diags/regions_specs.json for options of all regions # Below is default setup for global/regional mean climate metrics regions = "global,ocean,land,NHEX,SHEX,TROPICS" diff --git a/tests/integration/template_weekly_comprehensive_v3.cfg b/tests/integration/template_weekly_comprehensive_v3.cfg index 58f9a8e3..a7a3566e 100644 --- a/tests/integration/template_weekly_comprehensive_v3.cfg +++ b/tests/integration/template_weekly_comprehensive_v3.cfg @@ -315,7 +315,7 @@ figure_format = "png" run_type = "model_vs_obs" # Create rules to construct cmip-like datasets with a string constructed as # "mip.exp.product.realization", where "."is used as the delimiter (mandatory) -model_name = '#expand case_name#' +model_name = 'e3sm.historical.v3-LR.0051' model_tableID = 'Amon' e3sm_to_cmip_atm_subsection = "atm_monthly_180x360_aave" # Can be inferred if needed multiprocessing = True @@ -336,7 +336,8 @@ ts_years = "1985-1986", ref_years = "1985-1986", # Variables in current pcmdi-cmip list # rltcre and rstcre are derived variables as they are not included in cmip_ts - vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rsus,rsuscs,rlus,rlut,rlutcs,rsds,rsdscs,rsdt,rsut,rsutcs,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" + vars = "pr,rlut,rtmt,tas,tauu,tauv,ts" + #vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rsus,rsuscs,rlus,rlut,rlutcs,rsds,rsdscs,rsdt,rsut,rsutcs,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" # See zppy/templates/pcmdi_diags/regions_specs.json for options of all regions # Below is default setup for global/regional mean climate metrics regions = "global,ocean,land,NHEX,SHEX,TROPICS" From b5c144b4d32513c771efde3f694bcc7550e763cf Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 25 Sep 2025 21:26:54 -0500 Subject: [PATCH 26/47] Incomplete integration test fixes --- examples/post.v3.LR.amip.0101.cfg | 8 ++-- ...test_weekly_comprehensive_v3_chrysalis.cfg | 44 +++++++++---------- .../update_weekly_expected_files_chrysalis.sh | 6 +-- .../template_weekly_comprehensive_v3.cfg | 2 +- tests/integration/utils.py | 17 ++----- zppy/defaults/default.ini | 2 +- zppy/pcmdi_diags.py | 6 +++ zppy/templates/pcmdi_diags.bash | 6 +++ 8 files changed, 47 insertions(+), 44 deletions(-) diff --git a/examples/post.v3.LR.amip.0101.cfg b/examples/post.v3.LR.amip.0101.cfg index cc0858b6..a6fa6248 100644 --- a/examples/post.v3.LR.amip.0101.cfg +++ b/examples/post.v3.LR.amip.0101.cfg @@ -1,10 +1,10 @@ [default] input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/AMIP/v3.LR.amip_0101 -output = /lcrc/group/e3sm/ac.forsyth2/zppy_pr719_output/unique_id_48/v3.LR.amip_0101 -www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pr719_output/unique_id_48 +output = /lcrc/group/e3sm/ac.forsyth2/zppy_pr719_output/unique_id_51/v3.LR.amip_0101 +www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pr719_output/unique_id_51 campaign = "water_cycle" case = v3.LR.amip_0101 -debug = True +debug = False environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" partition = debug #compute @@ -41,7 +41,7 @@ years = "2005:2015:5" [pcmdi_diags] active = True environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" -environment_commands_secondary = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20250923" +environment_commands_secondary = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20250925" nodes = 1 walltime = "4:00:00" figure_format = "png" diff --git a/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg b/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg index a0cd3ea6..417559b6 100644 --- a/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg +++ b/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg @@ -9,10 +9,10 @@ infer_section_parameters = False input = /lcrc/group/e3sm2/ac.wlin//E3SMv3/v3.LR.historical_0051 input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051" partition = "debug" qos = "regular" -www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_comprehensive_v3_www/unique_id" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_comprehensive_v3_www/unique_id_pcmdi_diags_20250925_try4" years = "1985:1989:2", [climo] @@ -26,14 +26,14 @@ walltime = "00:30:00" vars = "" [[ atm_monthly_diurnal_8xdaily_180x360_aave ]] - active = True + active = False frequency = "diurnal_8xdaily" input_files = "eam.h3" input_subdir = "archive/atm/hist" vars = "PRECT" [[ land_monthly_climo ]] - active = True + active = False frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" @@ -50,14 +50,14 @@ walltime = "00:30:00" input_subdir = "archive/atm/hist" [[ atm_daily_180x360_aave ]] - active = True + active = False frequency = "daily" input_files = "eam.h1" input_subdir = "archive/atm/hist" vars = "PRECT" [[ rof_monthly ]] - active = True + active = False extra_vars = 'areatotal2' frequency = "monthly" input_files = "mosart.h0" @@ -67,7 +67,7 @@ walltime = "00:30:00" [[ atm_monthly_glb ]] # Note global average won't work for 3D variables. - active = True + active = False frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" @@ -75,7 +75,7 @@ walltime = "00:30:00" years = "1985:1995:5", [[ lnd_monthly_glb ]] - active = True + active = False frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" @@ -111,11 +111,11 @@ walltime = "00:30:00" # walltime = "00:30:00" [e3sm_diags] -active = True +active = False climo_diurnal_frequency = "diurnal_8xdaily" climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave" climo_subsection = "atm_monthly_180x360_aave" -environment_commands = "source /conda.sh; conda activate " +environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" grid = '180x360_aave' multiprocessing = True num_workers = 8 @@ -167,16 +167,16 @@ tc_obs = "/lcrc/group/e3sm/diagnostics/observations/Atm/tc-analysis/" tag = "model_vs_model" ts_num_years_ref = 2 # Reference paths - reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/atm/180x360_aave/clim" + reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051/post/atm/180x360_aave/clim" # mvm streamflow only gauges_path = "/lcrc/group/e3sm/diagnostics/observations/Atm/time-series/GSIM/GSIM_catchment_characteristics_all_1km2.csv" - reference_data_path_ts_rof = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/rof/native/ts/monthly" + reference_data_path_ts_rof = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051/post/rof/native/ts/monthly" # mvm diurnal_cycle only - reference_data_path_climo_diurnal = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/atm/180x360_aave/clim_diurnal_8xdaily" + reference_data_path_climo_diurnal = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051/post/atm/180x360_aave/clim_diurnal_8xdaily" # mvm "enso_diags", "qbo", "area_mean_time_series" - reference_data_path_ts = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/atm/180x360_aave/ts/monthly" + reference_data_path_ts = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051/post/atm/180x360_aave/ts/monthly" # mvm tropical_subseasonal only - reference_data_path_ts_daily = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/atm/180x360_aave/ts/daily" + reference_data_path_ts_daily = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051/post/atm/180x360_aave/ts/daily" [[ lnd_monthly_mvm_lnd ]] # Test model-vs-model using the same files as the reference @@ -192,10 +192,10 @@ tc_obs = "/lcrc/group/e3sm/diagnostics/observations/Atm/tc-analysis/" tag = "model_vs_model" ts_num_years_ref = 2 # Reference paths - reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/lnd/180x360_aave/clim" + reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051/post/lnd/180x360_aave/clim" [mpas_analysis] -active = True +active = False anomalyRefYear = 1985 climo_years = "1985-1989", "1990-1995", enso_years = "1985-1989", "1990-1995", @@ -208,9 +208,9 @@ ts_years = "1985-1989", "1985-1995", walltime = "02:00:00" [global_time_series] -active = True +active = False climo_years = "1985-1989", "1990-1995", -environment_commands = "source /conda.sh; conda activate " +environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" experiment_name = "v3.LR.historical_0051" figstr = "v3.LR.historical_0051" #moc_file=mocTimeSeries_1985-1995.nc @@ -294,7 +294,7 @@ years = "1985-1995", [ilamb] -active = True +active = False e3sm_to_cmip_atm_subsection = "atm_monthly_180x360_aave" e3sm_to_cmip_land_subsection = "land_monthly" ilamb_obs = "/lcrc/group/e3sm/diagnostics/ilamb_data" @@ -309,7 +309,7 @@ years = "1985:1989:4" [pcmdi_diags] active = True -environment_commands_secondary = "source /conda.sh; conda activate " +environment_commands_secondary = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20250925" walltime = "2:00:00" figure_format = "png" run_type = "model_vs_obs" @@ -372,7 +372,7 @@ ts_years = "1985-1986", ref_years = "1985-1986", vars = "psl" # Varibility modes - atm_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" + atm_modes = "NAM,NAO,PNA,NPO,SAM,PSA1" # PSA2 requires 3 years of data frequency = 'mo' landmask = False ModUnitsAdjust = '(True,"divide",100.0)' diff --git a/tests/integration/generated/update_weekly_expected_files_chrysalis.sh b/tests/integration/generated/update_weekly_expected_files_chrysalis.sh index a1676a23..46ec2dff 100755 --- a/tests/integration/generated/update_weekly_expected_files_chrysalis.sh +++ b/tests/integration/generated/update_weekly_expected_files_chrysalis.sh @@ -35,17 +35,17 @@ do # Copy output so you don't have to rerun zppy to generate the output. if [[ "${test_name,,}" =~ "v2" ]]; then # We need the v2 case name - cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name} + cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id_pcmdi_diags_20250925_try4/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name} else # We need the v3 case name - cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id/v3.LR.historical_0051 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name} + cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name} fi # test_bundles.py also needs the bash files transferred. # Note that for legacy cfgs, we're only testing test_images.py if [[ "${test_name,,}" == "bundles" ]]; then mkdir -p /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files - cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_weekly_bundles_output/unique_id/v3.LR.historical_0051/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files + cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_weekly_bundles_output/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files fi zppy_top_level=$(pwd) diff --git a/tests/integration/template_weekly_comprehensive_v3.cfg b/tests/integration/template_weekly_comprehensive_v3.cfg index a7a3566e..f6f97cf8 100644 --- a/tests/integration/template_weekly_comprehensive_v3.cfg +++ b/tests/integration/template_weekly_comprehensive_v3.cfg @@ -372,7 +372,7 @@ ts_years = "1985-1986", ref_years = "1985-1986", vars = "psl" # Varibility modes - atm_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" + atm_modes = "NAM,NAO,PNA,NPO,SAM,PSA1" # PSA2 requires 3 years of data frequency = 'mo' landmask = False ModUnitsAdjust = '(True,"divide",100.0)' diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 72f3e297..b7e69eba 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -12,25 +12,16 @@ # pytest tests/integration/test_*.py TEST_SPECIFICS: Dict[str, Any] = { - "diags_environment_commands": "source /conda.sh; conda activate ", - "global_time_series_environment_commands": "source /conda.sh; conda activate ", - "pcmdi_diags_environment_commands": "source /conda.sh; conda activate ", + "diags_environment_commands": "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh", + "global_time_series_environment_commands": "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh", + "pcmdi_diags_environment_commands": "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20250925", "cfgs_to_run": [ - "weekly_bundles", - "weekly_comprehensive_v2", "weekly_comprehensive_v3", - "weekly_legacy_3.0.0_bundles", - "weekly_legacy_3.0.0_comprehensive_v2", - "weekly_legacy_3.0.0_comprehensive_v3", ], "tasks_to_run": [ - "e3sm_diags", - "mpas_analysis", - "global_time_series", - "ilamb", "pcmdi_diags", ], - "unique_id": "unique_id", + "unique_id": "unique_id_pcmdi_diags_20250925_try4", } # Multi-machine testing ######################################################### diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index 5d997959..aa87877e 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -85,7 +85,7 @@ ts_subsection = string(default="") # The variables to process # If `vars` is set to "", then climo and ts will process *all* variables. # This is accomplished by not setting `-v`/`--vars` for `ncclimo`. -vars = string(default="FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U") +vars = string(default="FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,PSL") # DEPRECATED: vars_exclude # This is the list of variables to exclude. It is only used if `vars` is set to "". # vars_exclude = string(default="H2OSOI,LAKEICEFRAC,O_SCALAR,PCT_LANDUNIT,PCT_NAT_PFT,SOILICE,SOILICE_ICE,SOILLIQ,SOILLIQ_ICE,SOILPSI,T_SCALAR,TLAKE,TSOI,TSOI_ICE,W_SCALAR") diff --git a/zppy/pcmdi_diags.py b/zppy/pcmdi_diags.py index 213966ca..f49a6a94 100644 --- a/zppy/pcmdi_diags.py +++ b/zppy/pcmdi_diags.py @@ -230,6 +230,8 @@ def check_parameters_for_pcmdi(c: Dict[str, Any]) -> None: f"{c['diagnostics_base_path']}/pcmdi_data/metrics_data/enso_metric", ParameterInferenceType.PATH_INFERENCE, ) + else: + c["cmip_enso_dir"] = "placeholder_dir" if "mean_climate" in c["figure_sets"]: set_value_of_parameter_if_undefined( c, @@ -237,6 +239,8 @@ def check_parameters_for_pcmdi(c: Dict[str, Any]) -> None: f"{c['diagnostics_base_path']}/pcmdi_data/metrics_data/mean_climate", ParameterInferenceType.PATH_INFERENCE, ) + else: + c["cmip_clim_dir"] = "placeholder_dir" if "variability_modes" in c["figure_sets"]: set_value_of_parameter_if_undefined( c, @@ -244,6 +248,8 @@ def check_parameters_for_pcmdi(c: Dict[str, Any]) -> None: f"{c['diagnostics_base_path']}/pcmdi_data/metrics_data/variability_modes", ParameterInferenceType.PATH_INFERENCE, ) + else: + c["cmip_movs_dir"] = "placeholder_dir" def check_mvm_only_parameters_for_bash(c: Dict[str, Any]) -> None: diff --git a/zppy/templates/pcmdi_diags.bash b/zppy/templates/pcmdi_diags.bash index 650f617b..dcada66e 100755 --- a/zppy/templates/pcmdi_diags.bash +++ b/zppy/templates/pcmdi_diags.bash @@ -822,6 +822,12 @@ command="zi-pcmdi-variability-modes ${core_parameters} --var_modes ${var_modes}" command="zi-pcmdi-enso ${core_parameters} --enso_groups {{ enso_groups }}" {% endif %} {% if current_set == "synthetic_plots" %} + +echo "Checking if var_mode_*.json files exist in variability_modes_dir:" +variability_modes_dir=${web_dir}/model_vs_obs/metrics_data/variability_modes/ +find ${variability_modes_dir} -name "var_mode_*.json" +echo "Done checking. There should be a list of files above, if they exist." + command="zi-pcmdi-synthetic-plots --synthetic_sets {{ synthetic_sets }} --figure_format {{ figure_format }} --www ${www} --results_dir ${results_dir} --case {{ case }} --model_name {{ model_name }} --model_tableID {{model_tableID }} --web_dir=${web_dir} --cmip_clim_dir {{ cmip_clim_dir }} --cmip_clim_set {{ cmip_clim_set }} --cmip_movs_dir {{ cmip_movs_dir }} --cmip_movs_set {{ cmip_movs_set }} --atm_modes {{ atm_modes }} --cpl_modes {{ cpl_modes }} --cmip_enso_dir {{ cmip_enso_dir }} --cmip_enso_set {{ cmip_enso_set }} --figure_sets {{ figure_sets }} --pcmdi_webtitle {{ pcmdi_webtitle }} --pcmdi_version {{ pcmdi_version }} --run_type ${run_type} --figure_sets_period {{ figure_sets_period }} --pcmdi_external_prefix {{ diagnostics_base_path }} --pcmdi_viewer_template {{ pcmdi_viewer_template }}" {% endif %} From b8d99f6b88a89c10b96e80139cdf6ccff226785d Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Fri, 26 Sep 2025 14:29:17 -0500 Subject: [PATCH 27/47] Working v3 test cfg --- ...test_weekly_comprehensive_v3_chrysalis.cfg | 37 ++++++++++--------- .../update_weekly_expected_files_chrysalis.sh | 6 +-- .../template_weekly_comprehensive_v3.cfg | 21 ++++++----- tests/integration/utils.py | 2 +- zppy/defaults/default.ini | 2 +- 5 files changed, 37 insertions(+), 31 deletions(-) diff --git a/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg b/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg index 417559b6..d8696317 100644 --- a/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg +++ b/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg @@ -9,10 +9,10 @@ infer_section_parameters = False input = /lcrc/group/e3sm2/ac.wlin//E3SMv3/v3.LR.historical_0051 input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051" partition = "debug" qos = "regular" -www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_comprehensive_v3_www/unique_id_pcmdi_diags_20250925_try4" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_comprehensive_v3_www/unique_id_pcmdi_diags_20250926" years = "1985:1989:2", [climo] @@ -48,6 +48,7 @@ walltime = "00:30:00" frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" + years = "1985:1995:2", # Need 10 years for pcmdi_diags task [[ atm_daily_180x360_aave ]] active = False @@ -100,6 +101,7 @@ walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] input_files = "eam.h0" ts_subsection = "atm_monthly_180x360_aave" + years = "1985:1995:2", # Need 10 years for pcmdi_diags task [[ land_monthly ]] input_files = "elm.h0" @@ -167,16 +169,16 @@ tc_obs = "/lcrc/group/e3sm/diagnostics/observations/Atm/tc-analysis/" tag = "model_vs_model" ts_num_years_ref = 2 # Reference paths - reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051/post/atm/180x360_aave/clim" + reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051/post/atm/180x360_aave/clim" # mvm streamflow only gauges_path = "/lcrc/group/e3sm/diagnostics/observations/Atm/time-series/GSIM/GSIM_catchment_characteristics_all_1km2.csv" - reference_data_path_ts_rof = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051/post/rof/native/ts/monthly" + reference_data_path_ts_rof = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051/post/rof/native/ts/monthly" # mvm diurnal_cycle only - reference_data_path_climo_diurnal = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051/post/atm/180x360_aave/clim_diurnal_8xdaily" + reference_data_path_climo_diurnal = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051/post/atm/180x360_aave/clim_diurnal_8xdaily" # mvm "enso_diags", "qbo", "area_mean_time_series" - reference_data_path_ts = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051/post/atm/180x360_aave/ts/monthly" + reference_data_path_ts = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051/post/atm/180x360_aave/ts/monthly" # mvm tropical_subseasonal only - reference_data_path_ts_daily = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051/post/atm/180x360_aave/ts/daily" + reference_data_path_ts_daily = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051/post/atm/180x360_aave/ts/daily" [[ lnd_monthly_mvm_lnd ]] # Test model-vs-model using the same files as the reference @@ -192,7 +194,7 @@ tc_obs = "/lcrc/group/e3sm/diagnostics/observations/Atm/tc-analysis/" tag = "model_vs_model" ts_num_years_ref = 2 # Reference paths - reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051/post/lnd/180x360_aave/clim" + reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051/post/lnd/180x360_aave/clim" [mpas_analysis] active = False @@ -324,16 +326,16 @@ num_workers = 24 obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series' pcmdi_debug = False ts_num_years = 2 -ts_years = "1985-1986", +ts_years = "1985-1994", [[ mean_climate ]] active = True current_set = "mean_climate" # Can be inferred if needed # Default for all variables or specify source for each variable separately. obs_sets = "default" - ref_final_yr = 1986 + ref_final_yr = 1994 ref_start_yr = 1985 - ref_years = "1985-1986", + ref_years = "1985-1994", # Variables in current pcmdi-cmip list # rltcre and rstcre are derived variables as they are not included in cmip_ts vars = "pr,rlut,rtmt,tas,tauu,tauv,ts" @@ -349,9 +351,9 @@ ts_years = "1985-1986", current_set = "variability_modes_cpl" # Can be inferred if needed # See zppy/templates/pcmdi_diags/reference_alias.json for options obs_sets = "alternate1" #"default" - ref_final_yr = 1986 + ref_final_yr = 1994 ref_start_yr = 1985 - ref_years = "1985-1986", + ref_years = "1985-1994", vars = "ts" # Name of modes varibility cpl_modes = "PDO,NPGO,AMO" @@ -367,12 +369,12 @@ ts_years = "1985-1986", # See zppy/templates/pcmdi_diags/reference_alias.json for options # Default for all variables obs_sets = "alternate2" # "default" - ref_final_yr = 1986 + ref_final_yr = 1994 ref_start_yr = 1985 - ref_years = "1985-1986", + ref_years = "1985-1994", vars = "psl" # Varibility modes - atm_modes = "NAM,NAO,PNA,NPO,SAM,PSA1" # PSA2 requires 3 years of data + atm_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" frequency = 'mo' landmask = False ModUnitsAdjust = '(True,"divide",100.0)' @@ -382,7 +384,8 @@ ts_years = "1985-1986", [[ synthetic_plots ]] active = True current_set = "synthetic_plots" # Can be inferred if needed + vars = "pr,rlut,rtmt,tas,tauu,tauv,ts" # MUST match mean_climate's vars # clim_period, emov_period, enso_period: - figure_sets_period = "1985-1986,1985-1986,1985-1986" + figure_sets_period = "1985-1994,1985-1994,1985-1994" cmip_clim_dir = /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/mean_climate cmip_movs_dir = /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes diff --git a/tests/integration/generated/update_weekly_expected_files_chrysalis.sh b/tests/integration/generated/update_weekly_expected_files_chrysalis.sh index 46ec2dff..a776ace3 100755 --- a/tests/integration/generated/update_weekly_expected_files_chrysalis.sh +++ b/tests/integration/generated/update_weekly_expected_files_chrysalis.sh @@ -35,17 +35,17 @@ do # Copy output so you don't have to rerun zppy to generate the output. if [[ "${test_name,,}" =~ "v2" ]]; then # We need the v2 case name - cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id_pcmdi_diags_20250925_try4/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name} + cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id_pcmdi_diags_20250926/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name} else # We need the v3 case name - cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name} + cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name} fi # test_bundles.py also needs the bash files transferred. # Note that for legacy cfgs, we're only testing test_images.py if [[ "${test_name,,}" == "bundles" ]]; then mkdir -p /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files - cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_weekly_bundles_output/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files + cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_weekly_bundles_output/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files fi zppy_top_level=$(pwd) diff --git a/tests/integration/template_weekly_comprehensive_v3.cfg b/tests/integration/template_weekly_comprehensive_v3.cfg index f6f97cf8..06bd64f5 100644 --- a/tests/integration/template_weekly_comprehensive_v3.cfg +++ b/tests/integration/template_weekly_comprehensive_v3.cfg @@ -48,6 +48,7 @@ walltime = "00:30:00" frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" + years = "1985:1995:2", # Need 10 years for pcmdi_diags task [[ atm_daily_180x360_aave ]] active = #expand active_e3sm_diags# @@ -100,6 +101,7 @@ walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] input_files = "eam.h0" ts_subsection = "atm_monthly_180x360_aave" + years = "1985:1995:2", # Need 10 years for pcmdi_diags task [[ land_monthly ]] input_files = "elm.h0" @@ -324,16 +326,16 @@ num_workers = 24 obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series' pcmdi_debug = False ts_num_years = 2 -ts_years = "1985-1986", +ts_years = "1985-1994", [[ mean_climate ]] active = True current_set = "mean_climate" # Can be inferred if needed # Default for all variables or specify source for each variable separately. obs_sets = "default" - ref_final_yr = 1986 + ref_final_yr = 1994 ref_start_yr = 1985 - ref_years = "1985-1986", + ref_years = "1985-1994", # Variables in current pcmdi-cmip list # rltcre and rstcre are derived variables as they are not included in cmip_ts vars = "pr,rlut,rtmt,tas,tauu,tauv,ts" @@ -349,9 +351,9 @@ ts_years = "1985-1986", current_set = "variability_modes_cpl" # Can be inferred if needed # See zppy/templates/pcmdi_diags/reference_alias.json for options obs_sets = "alternate1" #"default" - ref_final_yr = 1986 + ref_final_yr = 1994 ref_start_yr = 1985 - ref_years = "1985-1986", + ref_years = "1985-1994", vars = "ts" # Name of modes varibility cpl_modes = "PDO,NPGO,AMO" @@ -367,12 +369,12 @@ ts_years = "1985-1986", # See zppy/templates/pcmdi_diags/reference_alias.json for options # Default for all variables obs_sets = "alternate2" # "default" - ref_final_yr = 1986 + ref_final_yr = 1994 ref_start_yr = 1985 - ref_years = "1985-1986", + ref_years = "1985-1994", vars = "psl" # Varibility modes - atm_modes = "NAM,NAO,PNA,NPO,SAM,PSA1" # PSA2 requires 3 years of data + atm_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" frequency = 'mo' landmask = False ModUnitsAdjust = '(True,"divide",100.0)' @@ -382,7 +384,8 @@ ts_years = "1985-1986", [[ synthetic_plots ]] active = True current_set = "synthetic_plots" # Can be inferred if needed + vars = "pr,rlut,rtmt,tas,tauu,tauv,ts" # MUST match mean_climate's vars # clim_period, emov_period, enso_period: - figure_sets_period = "1985-1986,1985-1986,1985-1986" + figure_sets_period = "1985-1994,1985-1994,1985-1994" cmip_clim_dir = #expand diagnostics_base_path#/pcmdi_data/metrics_data/mean_climate cmip_movs_dir = #expand diagnostics_base_path#/pcmdi_data/metrics_data/variability_modes diff --git a/tests/integration/utils.py b/tests/integration/utils.py index b7e69eba..20f7bfdd 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -21,7 +21,7 @@ "tasks_to_run": [ "pcmdi_diags", ], - "unique_id": "unique_id_pcmdi_diags_20250925_try4", + "unique_id": "unique_id_pcmdi_diags_20250926", } # Multi-machine testing ######################################################### diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index aa87877e..11df42f7 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -457,7 +457,7 @@ ref_start_yr = string(default="") # For other plots, zppy will set it to match `years`, if not specified ref_years = string_list(default=list("")) # Variables to be used by the pcmdi diagnostics -# mean_climate: vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" +# mean_climate AND synthetic_plots: vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" # variability_modes_atm: vars = "psl" # variability_modes_cpl: vars = "ts" # enso: vars = "psl,pr,prsn,ts,tas,tauu,tauv,hflx,hfss,rlds,rsds,rlus,rlut,rsdt" From 58924e92909084c0d7ec627fad120489e3d30b20 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Fri, 26 Sep 2025 15:14:20 -0500 Subject: [PATCH 28/47] Clean up code --- ...test_weekly_comprehensive_v3_chrysalis.cfg | 42 +++++++++---------- .../update_weekly_expected_files_chrysalis.sh | 6 +-- tests/integration/utils.py | 17 ++++++-- tests/test_sections.py | 4 +- tests/test_zppy_pcmdi_diags.py | 2 +- 5 files changed, 40 insertions(+), 31 deletions(-) diff --git a/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg b/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg index d8696317..e7085143 100644 --- a/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg +++ b/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg @@ -9,10 +9,10 @@ infer_section_parameters = False input = /lcrc/group/e3sm2/ac.wlin//E3SMv3/v3.LR.historical_0051 input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051" partition = "debug" qos = "regular" -www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_comprehensive_v3_www/unique_id_pcmdi_diags_20250926" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_comprehensive_v3_www/unique_id" years = "1985:1989:2", [climo] @@ -26,14 +26,14 @@ walltime = "00:30:00" vars = "" [[ atm_monthly_diurnal_8xdaily_180x360_aave ]] - active = False + active = True frequency = "diurnal_8xdaily" input_files = "eam.h3" input_subdir = "archive/atm/hist" vars = "PRECT" [[ land_monthly_climo ]] - active = False + active = True frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" @@ -51,14 +51,14 @@ walltime = "00:30:00" years = "1985:1995:2", # Need 10 years for pcmdi_diags task [[ atm_daily_180x360_aave ]] - active = False + active = True frequency = "daily" input_files = "eam.h1" input_subdir = "archive/atm/hist" vars = "PRECT" [[ rof_monthly ]] - active = False + active = True extra_vars = 'areatotal2' frequency = "monthly" input_files = "mosart.h0" @@ -68,7 +68,7 @@ walltime = "00:30:00" [[ atm_monthly_glb ]] # Note global average won't work for 3D variables. - active = False + active = True frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" @@ -76,7 +76,7 @@ walltime = "00:30:00" years = "1985:1995:5", [[ lnd_monthly_glb ]] - active = False + active = True frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" @@ -113,11 +113,11 @@ walltime = "00:30:00" # walltime = "00:30:00" [e3sm_diags] -active = False +active = True climo_diurnal_frequency = "diurnal_8xdaily" climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave" climo_subsection = "atm_monthly_180x360_aave" -environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" +environment_commands = "source /conda.sh; conda activate " grid = '180x360_aave' multiprocessing = True num_workers = 8 @@ -169,16 +169,16 @@ tc_obs = "/lcrc/group/e3sm/diagnostics/observations/Atm/tc-analysis/" tag = "model_vs_model" ts_num_years_ref = 2 # Reference paths - reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051/post/atm/180x360_aave/clim" + reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/atm/180x360_aave/clim" # mvm streamflow only gauges_path = "/lcrc/group/e3sm/diagnostics/observations/Atm/time-series/GSIM/GSIM_catchment_characteristics_all_1km2.csv" - reference_data_path_ts_rof = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051/post/rof/native/ts/monthly" + reference_data_path_ts_rof = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/rof/native/ts/monthly" # mvm diurnal_cycle only - reference_data_path_climo_diurnal = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051/post/atm/180x360_aave/clim_diurnal_8xdaily" + reference_data_path_climo_diurnal = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/atm/180x360_aave/clim_diurnal_8xdaily" # mvm "enso_diags", "qbo", "area_mean_time_series" - reference_data_path_ts = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051/post/atm/180x360_aave/ts/monthly" + reference_data_path_ts = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/atm/180x360_aave/ts/monthly" # mvm tropical_subseasonal only - reference_data_path_ts_daily = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051/post/atm/180x360_aave/ts/daily" + reference_data_path_ts_daily = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/atm/180x360_aave/ts/daily" [[ lnd_monthly_mvm_lnd ]] # Test model-vs-model using the same files as the reference @@ -194,10 +194,10 @@ tc_obs = "/lcrc/group/e3sm/diagnostics/observations/Atm/tc-analysis/" tag = "model_vs_model" ts_num_years_ref = 2 # Reference paths - reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051/post/lnd/180x360_aave/clim" + reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/lnd/180x360_aave/clim" [mpas_analysis] -active = False +active = True anomalyRefYear = 1985 climo_years = "1985-1989", "1990-1995", enso_years = "1985-1989", "1990-1995", @@ -210,9 +210,9 @@ ts_years = "1985-1989", "1985-1995", walltime = "02:00:00" [global_time_series] -active = False +active = True climo_years = "1985-1989", "1990-1995", -environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" +environment_commands = "source /conda.sh; conda activate " experiment_name = "v3.LR.historical_0051" figstr = "v3.LR.historical_0051" #moc_file=mocTimeSeries_1985-1995.nc @@ -296,7 +296,7 @@ years = "1985-1995", [ilamb] -active = False +active = True e3sm_to_cmip_atm_subsection = "atm_monthly_180x360_aave" e3sm_to_cmip_land_subsection = "land_monthly" ilamb_obs = "/lcrc/group/e3sm/diagnostics/ilamb_data" @@ -311,7 +311,7 @@ years = "1985:1989:4" [pcmdi_diags] active = True -environment_commands_secondary = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20250925" +environment_commands_secondary = "source /conda.sh; conda activate " walltime = "2:00:00" figure_format = "png" run_type = "model_vs_obs" diff --git a/tests/integration/generated/update_weekly_expected_files_chrysalis.sh b/tests/integration/generated/update_weekly_expected_files_chrysalis.sh index a776ace3..a1676a23 100755 --- a/tests/integration/generated/update_weekly_expected_files_chrysalis.sh +++ b/tests/integration/generated/update_weekly_expected_files_chrysalis.sh @@ -35,17 +35,17 @@ do # Copy output so you don't have to rerun zppy to generate the output. if [[ "${test_name,,}" =~ "v2" ]]; then # We need the v2 case name - cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id_pcmdi_diags_20250926/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name} + cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name} else # We need the v3 case name - cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name} + cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id/v3.LR.historical_0051 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name} fi # test_bundles.py also needs the bash files transferred. # Note that for legacy cfgs, we're only testing test_images.py if [[ "${test_name,,}" == "bundles" ]]; then mkdir -p /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files - cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_weekly_bundles_output/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files + cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_weekly_bundles_output/unique_id/v3.LR.historical_0051/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files fi zppy_top_level=$(pwd) diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 20f7bfdd..72f3e297 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -12,16 +12,25 @@ # pytest tests/integration/test_*.py TEST_SPECIFICS: Dict[str, Any] = { - "diags_environment_commands": "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh", - "global_time_series_environment_commands": "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh", - "pcmdi_diags_environment_commands": "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20250925", + "diags_environment_commands": "source /conda.sh; conda activate ", + "global_time_series_environment_commands": "source /conda.sh; conda activate ", + "pcmdi_diags_environment_commands": "source /conda.sh; conda activate ", "cfgs_to_run": [ + "weekly_bundles", + "weekly_comprehensive_v2", "weekly_comprehensive_v3", + "weekly_legacy_3.0.0_bundles", + "weekly_legacy_3.0.0_comprehensive_v2", + "weekly_legacy_3.0.0_comprehensive_v3", ], "tasks_to_run": [ + "e3sm_diags", + "mpas_analysis", + "global_time_series", + "ilamb", "pcmdi_diags", ], - "unique_id": "unique_id_pcmdi_diags_20250926", + "unique_id": "unique_id", } # Multi-machine testing ######################################################### diff --git a/tests/test_sections.py b/tests/test_sections.py index 1a430340..018244d7 100644 --- a/tests/test_sections.py +++ b/tests/test_sections.py @@ -98,7 +98,7 @@ def test_sections(): "ts_land_subsection": "", "ts_num_years": 5, "ts_subsection": "", - "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U", + "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,PSL", "walltime": "02:00:00", "www": "WWWW", "years": [""], @@ -288,7 +288,7 @@ def test_subsections(): "ts_land_subsection": "", "ts_num_years": 5, "ts_subsection": "", - "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U", + "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,PSL", "walltime": "02:00:00", "www": "WWWW", "years": [""], diff --git a/tests/test_zppy_pcmdi_diags.py b/tests/test_zppy_pcmdi_diags.py index 0b73648c..127e46c6 100644 --- a/tests/test_zppy_pcmdi_diags.py +++ b/tests/test_zppy_pcmdi_diags.py @@ -169,7 +169,7 @@ def test_check_parameters_for_pcmdi(): check_parameters_for_pcmdi(c) assert c["cmip_clim_dir"] == "diags/post/pcmdi_data/metrics_data/mean_climate" assert c["cmip_movs_dir"] == "diags/post/pcmdi_data/metrics_data/variability_modes" - assert c["cmip_enso_dir"] == "" + assert c["cmip_enso_dir"] == "placeholder_dir" c = { "current_set": "mean_climate", From 23f956245e51bb24eebcf41951e11b3b6b153a72 Mon Sep 17 00:00:00 2001 From: ShixuanZhang Date: Sun, 28 Sep 2025 12:51:35 -0500 Subject: [PATCH 29/47] Improve robustness of reference year range handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added input validation to ensure all year variables are integers and within a valid order. Normalized start/end pairs so reversed inputs (e.g. y1 > y2, ref_start_yr > ref_final_yr) are automatically swapped. Recorded the original custom reference range (ref_y1, ref_y2) and clamped it to the available reference window for safe fallback. Refined the adjustment logic so: If availability fully covers the analysis period, use [y1, y2]. Otherwise, prefer the clamped custom range and trim only if it extends beyond the analysis window. If custom and analysis don’t overlap, fall back to the overlap of availability ∩ analysis. If no overlap exists, fall back to the clamped custom, and finally the full available window. Ensured final ref_y1 ≤ ref_y2 and applied zero-padding formatting once at the end. --- zppy/templates/pcmdi_diags.bash | 47 ++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/zppy/templates/pcmdi_diags.bash b/zppy/templates/pcmdi_diags.bash index dcada66e..3d1069e5 100755 --- a/zppy/templates/pcmdi_diags.bash +++ b/zppy/templates/pcmdi_diags.bash @@ -46,23 +46,44 @@ Y2="$(printf "%04d" ${y2})" ref_Y1="$(printf "%04d" ${ref_y1})" ref_Y2="$(printf "%04d" ${ref_y2})" -# Refine reference range -if [[ ${ref_y1} -lt ${ref_start_yr} ]]; then - ref_y1=${ref_start_yr} - ref_Y1="$(printf "%04d" ${ref_y1})" -fi - num_years=$((y2 - y1 + 1)) ref_end_yr=$((ref_y1 + num_years - 1)) -if [[ ${ref_y2} -gt ${ref_end_yr} ]]; then - ref_y2=${ref_end_yr} - ref_Y2="$(printf "%04d" ${ref_y2})" +# Keep originals for fallback +orig_ref_y1=${ref_y1} +orig_ref_y2=${ref_y2} +if [[ ${orig_ref_y1} -lt ${ref_start_yr} ]]; then + orig_ref_y1=${ref_start_yr} fi - -if [[ ${ref_y2} -gt ${ref_final_yr} ]]; then - ref_y2=${ref_final_yr} - ref_Y2="$(printf "%04d" ${ref_y2})" +if [[ ${orig_ref_y2} -gt ${ref_final_yr} ]]; then + orig_ref_y2=${ref_final_yr} +fi +# Refine reference range +if [[ ${ref_start_yr} -le ${y1} && ${ref_final_yr} -ge ${y2} ]]; then + # Availability fully covers analysis: use [y1, y2] + ref_y1=${y1} + ref_y2=${y2} +else + # Case 2: Prefer the user's clamped custom window; only tweak if needed + # 2a) If clamped custom overlaps the analysis window, use the intersection + if [[ ${orig_ref_y1} -le ${y2} && ${orig_ref_y2} -ge ${y1} ]]; then + # minimal trimming to fit inside [y1, y2] + ref_y1=$(( orig_ref_y1 > y1 ? orig_ref_y1 : y1 )) + ref_y2=$(( orig_ref_y2 < y2 ? orig_ref_y2 : y2 )) + else + # 2b) No overlap between custom and analysis: + # try availability i∩ analysis (minimal change wrt both) + ovl_start=$(( ref_start_yr > y1 ? ref_start_yr : y1 )) + ovl_end=$(( ref_final_yr < y2 ? ref_final_yr : y2 )) + if [[ ${ovl_start} -le ${ovl_end} ]]; then + ref_y1=${ovl_start} + ref_y2=${ovl_end} + else + # 2c) Truly no overlap possible; fall back to clamped custom window + ref_y1=${orig_ref_y1} + ref_y2=${orig_ref_y2} + fi + fi fi {% endif %} From 872e8c431244ebf164d0a0fdb21769b6479827b4 Mon Sep 17 00:00:00 2001 From: ShixuanZhang Date: Thu, 2 Oct 2025 18:52:06 -0500 Subject: [PATCH 30/47] update the definition of the variable list to be flexible for ploting and viewer This comment contains update in the default.ini interms of the definition of "vars" in pcmdi diagnostic sections. I now explicitly defined "clim_vars", "mova_vars", "movc_vars" and "enso_vars" to explicitly pass the needed variables for corresponding diagnostics in pcmdi. This will be later on used to process the viewer page The corresponding changes were also made in pcmdi_diags.bash to accept the new variables defined in .cfg files and default.ini --- examples/post.v3.LR.amip.0101.cfg | 17 +++++++++-------- zppy/defaults/default.ini | 21 +++++++++++++-------- zppy/templates/pcmdi_diags.bash | 25 ++++++++++++++++++++----- 3 files changed, 42 insertions(+), 21 deletions(-) diff --git a/examples/post.v3.LR.amip.0101.cfg b/examples/post.v3.LR.amip.0101.cfg index a6fa6248..6b915365 100644 --- a/examples/post.v3.LR.amip.0101.cfg +++ b/examples/post.v3.LR.amip.0101.cfg @@ -1,7 +1,7 @@ [default] input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/AMIP/v3.LR.amip_0101 -output = /lcrc/group/e3sm/ac.forsyth2/zppy_pr719_output/unique_id_51/v3.LR.amip_0101 -www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pr719_output/unique_id_51 +output = /lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy_pr719_output/unique_id_51/v3.LR.amip_0101 +www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.szhang/zppy_pr719_output/unique_id_53 campaign = "water_cycle" case = v3.LR.amip_0101 debug = False @@ -32,7 +32,7 @@ years = "2005:2015:5" [[ atm_monthly_180x360_aave ]] # list of variables to be interpolated to pressure level; see e3sm_to_cmip/vrt_remap_plev19.nc # This defaults to cmip_plevdata = string(default="inclusions/e3sm_to_cmip/vrt_remap_plev19.nc") - # cmip_plevdata = /lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc + #cmip_plevdata = /lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc input_files = "eam.h0" interp_vars = "U,V,T,Z3" ts_subsection = "atm_monthly_180x360_aave" @@ -41,7 +41,7 @@ years = "2005:2015:5" [pcmdi_diags] active = True environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" -environment_commands_secondary = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20250925" +environment_commands_secondary = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh; conda activate zi-pcmdi-diags-20250730" nodes = 1 walltime = "4:00:00" figure_format = "png" @@ -66,7 +66,8 @@ ts_years = "2005-2014", ref_years = "2005-2014", # Variables in current pcmdi-cmip list # rltcre and rstcre are derived variables as they are not included in cmip_ts - vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rsus,rsuscs,rlus,rlut,rlutcs,rsds,rsdscs,rsdt,rsut,rsutcs,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" + # define a new variable name to be flexible + clim_vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rsus,rsuscs,rlus,rlut,rlutcs,rsds,rsdscs,rsdt,rsut,rsutcs,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" # See zppy/templates/pcmdi_diags/regions_specs.json for options of all regions # Below is default setup for global/regional mean climate metrics regions = "global,ocean,land,NHEX,SHEX,TROPICS" @@ -80,7 +81,7 @@ ts_years = "2005-2014", ref_final_yr = 2019 ref_start_yr = 1870 ref_years = "2005-2014", - vars = "ts" + movc_vars = "ts" # Name of modes varibility cpl_modes = "PDO,NPGO,AMO" frequency = 'mo' @@ -97,7 +98,7 @@ ts_years = "2005-2014", ref_final_yr = 2015 ref_start_yr = 1853 ref_years = "2005-2014", - vars = "psl" + mova_vars = "psl" # Varibility modes atm_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" frequency = 'mo' @@ -114,7 +115,7 @@ ts_years = "2005-2014", ref_final_yr = 2018 ref_start_yr = 1979 ref_years = "2005-2014", - vars = "psl,pr,prsn,ts,tas,tauu,tauv,hfls,hfss,rlds,rsds,rlus,rlut,rsdt" + enso_vars = "psl,pr,prsn,ts,tas,tauu,tauv,hfls,hfss,rlds,rsds,rlus,rlut,rsdt" # See zppy/templates/pcmdi_diags/reference_alias.json for options enso_groups = "ENSO_perf,ENSO_proc,ENSO_tel" diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index 11df42f7..111fcf79 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -456,16 +456,12 @@ ref_start_yr = string(default="") # zppy will then set it to match `figure_sets_period` # For other plots, zppy will set it to match `years`, if not specified ref_years = string_list(default=list("")) -# Variables to be used by the pcmdi diagnostics -# mean_climate AND synthetic_plots: vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" -# variability_modes_atm: vars = "psl" -# variability_modes_cpl: vars = "ts" -# enso: vars = "psl,pr,prsn,ts,tas,tauu,tauv,hflx,hfss,rlds,rsds,rlus,rlut,rsdt" -# Default setup is for mean_climate: -vars = string(default="pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500") ########################################################################################### # Specify these for the mean_climate subtask ########################################################################################### +# Variables to be used by the pcmdi diagnostics +# mean_climate AND synthetic_plots: clim_vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" +clim_vars = string(default="pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500") # variables in the cmip6 table that can be potentially used by pcmdi # this list depends on the definition of cmip variable # required for "mean climate" diagnostics @@ -490,7 +486,11 @@ regrid_method_ocn = string(default="conservative") # Specify these for the variability_modes_cpl and variability_modes_atm subtasks ########################################################################################### ### 2 options for names of atm, cpl modes varibility: +# variability_modes_atm: vars = "psl" +atm_vars = string(default="psl") atm_modes = string(default="NAM,NAO,PNA,NPO,SAM,PSA1,PSA2") +# variability_modes_cpl: vars = "ts" +cpl_vars = string(default="ts") cpl_modes = string(default="PDO,NPGO,AMO") CBF = string(default=True) ConvEOF = string(default=True) @@ -508,6 +508,8 @@ seasons = string(default="monthly") ########################################################################################### # Specify these for the enso subtasks (not currently enabled) ########################################################################################### +# enso: vars = "psl,pr,prsn,ts,tas,tauu,tauv,hflx,hfss,rlds,rsds,rlus,rlut,rsdt" +enso_vars = string(default="psl,pr,prsn,ts,tas,tauu,tauv,hflx,hfss,rlds,rsds,rlus,rlut,rsdt") enso_groups = string(default="ENSO_perf,ENSO_proc,ENSO_tel") ########################################################################################### # Specify these for the synthetic_plots subtask @@ -570,8 +572,11 @@ cmip_movs_set = string(default="cmip6.historical.v20220825") ref_final_yr = string(default=None) ref_start_yr = string(default=None) ref_years = string_list(default=None) - vars = string(default=None) + clim_vars = string(default=None) cmip_vars = string(default=None) + mova_vars = string(default=None) + movc_vars = string(default=None) + enso_vars = string(default=None) regions = string(default=None) target_grid = string(default=None) target_grid_string = string(default=None) diff --git a/zppy/templates/pcmdi_diags.bash b/zppy/templates/pcmdi_diags.bash index 3d1069e5..1674a8b2 100755 --- a/zppy/templates/pcmdi_diags.bash +++ b/zppy/templates/pcmdi_diags.bash @@ -30,9 +30,10 @@ case="{{ case }}" www="{{ www }}" run_type="{{ run_type }}" results_dir="{{ run_type }}" + {% if current_set != "synthetic_plots" %} -# Input variables +# Input time range y1={{ year1 }} y2={{ year2 }} ref_y1={{ ref_year1 }} @@ -86,6 +87,16 @@ else fi fi +{% if current_set == "mean_climate"%} +source_vars="{{ clim_vars }}" +{% elif current_set == "variability_modes_cpl"%} +source_vars="{{ movc_vars }}" +{% elif current_set == "variability_modes_atm"%} +source_vars="{{ mova_vars }}" +{% elif current_set == "enso"%} +source_vars="{{ enso_vars }}" +{% endif %} + {% endif %} # Top-level directory @@ -294,8 +305,8 @@ create_links_ts() { local script_dir="{{ scriptDir }}" local prefix="{{ prefix }}" - local vars="{{ vars }}" local ts_step="{{ ts_num_years }}" + local vars="${source_vars}" echo "create_links_ts: linking from ${ts_dir_source} to ${ts_dir_destination}" @@ -499,7 +510,7 @@ echo "Linking observational data into ${obstmp_dir}..." ################### echo "Linking observational data using SLURM..." -command="zi-pcmdi-link-observation --model_name_ref ${model_name_ref} --tableID_ref ${tableID_ref} --vars={{ vars }} --obs_sets {{ obs_sets }} --obs_ts {{ obs_ts }} --obstmp_dir ${obstmp_dir}" +command="zi-pcmdi-link-observation --model_name_ref ${model_name_ref} --tableID_ref ${tableID_ref} --vars=${source_vars} --obs_sets {{ obs_sets }} --obs_ts {{ obs_ts }} --obstmp_dir ${obstmp_dir}" echo "Running: ${command}" {{ environment_commands_secondary }} @@ -704,7 +715,11 @@ seasons = '{{ seasons }}'.split(",") frequency = '{{ frequency }}' # Variables to analyze (comma-separated string or space-separated) -varModel = '{{ vars }}' +{% if current_set == "variability_modes_cpl"%} +varModel = '{{ movc_vars }}' +{% elif current_set == "variability_modes_atm"%} +varModel = '{{ mova_vars }}' +{% endif %} # Unit conversion flags for model and observations ModUnitsAdjust = {{ ModUnitsAdjust }} @@ -825,7 +840,7 @@ source_dirs="--climo_ts_dir_primary ${ts_dir_primary} --climo_ts_dir_ref ${ts_di # So, it's a good idea to make sure they can't be (or at least are unlikely to be) empty. # run_type == "model_vs_obs" only: obs_sets (default value is NOT "") # run_type == "model_vs_model" only: model_name_ref, tableID_ref (default values are NOT "") -core_parameters="--num_workers {{ num_workers }} --multiprocessing {{ multiprocessing }} --subsection {{ subsection }} ${source_dirs} --model_name ${model_name} --model_tableID {{model_tableID }} --figure_format {{ figure_format }} --run_type {{ run_type }} --obs_sets {{ obs_sets }} --model_name_ref ${model_name_ref} --vars {{ vars }} --tableID_ref ${tableID_ref} --generate_sftlf {{ generate_sftlf }} --case_id ${case_id} --results_dir ${results_dir}" +core_parameters="--num_workers {{ num_workers }} --multiprocessing {{ multiprocessing }} --subsection {{ subsection }} ${source_dirs} --model_name ${model_name} --model_tableID {{model_tableID }} --figure_format {{ figure_format }} --run_type {{ run_type }} --obs_sets {{ obs_sets }} --model_name_ref ${model_name_ref} --vars ${source_vars} --tableID_ref ${tableID_ref} --generate_sftlf {{ generate_sftlf }} --case_id ${case_id} --results_dir ${results_dir}" {% endif %} {% if current_set == "mean_climate" %} From 66f325d1085430b4188f6cc4095e53bb6f812995 Mon Sep 17 00:00:00 2001 From: ShixuanZhang Date: Sun, 5 Oct 2025 02:33:18 -0500 Subject: [PATCH 31/47] Improve robustness of viewer and workflow config flow Refactored the PCMDI diagnostics workflow to strengthen viewer robustness and configuration handling. Key updates ------------ - default.ini: * Added explicit defaults for viewer-related options. * Improved parameter consistency with zppy config interface. - pcmdi_diags.py: * Refactored configuration parsing to ensure safe propagation of viewer options. * Enhanced logging for viewer setup and diagnostic dispatch. - templates/pcmdi_diags.bash: * Updated environment variable exports for new viewer flags. * Improved argument quoting and error handling. --- zppy/defaults/default.ini | 46 ++++++++++++++++++++++----------- zppy/pcmdi_diags.py | 45 +++++++++++++++++++++++++------- zppy/templates/pcmdi_diags.bash | 33 +++++++++++++++++++---- 3 files changed, 95 insertions(+), 29 deletions(-) diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index 111fcf79..1b1db260 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -469,7 +469,7 @@ cmip_vars = string(default="pr,prw,psl,rlds,rldscs,rlut,rlutcs,rsut,rsutcs,rsds, #default regions for mean climate metrics data #more options can be found at "regions_specs" #regions = string(default="global,ocean,land,NHEX,SHEX,TROPICS,NHEX_ocean,SHEX_ocean,NHEX_land,SHEX_land,ocean_50S50N") -regions = string(default="global,ocean,land,NHEX,SHEX,TROPICS") +clim_regions = string(default="global,ocean,land") # Regridding by pcmdi (required for mean climate) # OPTIONS: '1x1' for e3sm, '2.5x2.5' for cmip, or an actual cdms2 grid object, target_grid = string(default="1x1") @@ -487,11 +487,11 @@ regrid_method_ocn = string(default="conservative") ########################################################################################### ### 2 options for names of atm, cpl modes varibility: # variability_modes_atm: vars = "psl" -atm_vars = string(default="psl") -atm_modes = string(default="NAM,NAO,PNA,NPO,SAM,PSA1,PSA2") +mova_vars = string(default="psl") +mova_modes = string(default="NAM,NAO,PNA,NPO,SAM,PSA1,PSA2") # variability_modes_cpl: vars = "ts" -cpl_vars = string(default="ts") -cpl_modes = string(default="PDO,NPGO,AMO") +movc_vars = string(default="ts") +movc_modes = string(default="PDO,NPGO,AMO") CBF = string(default=True) ConvEOF = string(default=True) eofn_mod_max = integer(default=3) @@ -514,10 +514,20 @@ enso_groups = string(default="ENSO_perf,ENSO_proc,ENSO_tel") ########################################################################################### # Specify these for the synthetic_plots subtask ########################################################################################### -figure_sets = string(default="mean_climate,variability_modes") -# Should be a list of length 3: -# clim_period, emov_period, enso_period -figure_sets_period = string(default="") +# add mean climate to viewer page (default is False) +clim_viewer = string(default=True) +clim_years = string(default="") +# add extratropic modes to viewer page (default is True) +mova_viewer = string(default=True) +mova_years = string(default="") +# add coupled modes to viewer page (default is True) +movc_viewer = string(default=True) +movc_years = string(default="") +# add enso to viewer page (default is False, not currently enabled) +enso_viewer = string(default=False) +enso_years = string(default="") +# Save all intemediate diagnostic data +save_all_data = string(default=True) # File of observation data name for mean climate calculation reference_alias = string(default="inclusions/pcmdi_diags/reference_alias.json") # File of specified regions for mean climate calculation @@ -572,20 +582,29 @@ cmip_movs_set = string(default="cmip6.historical.v20220825") ref_final_yr = string(default=None) ref_start_yr = string(default=None) ref_years = string_list(default=None) + clim_viewer = string(default=None) clim_vars = string(default=None) + clim_regions = string(default=None) + clim_years = string(default=None) cmip_vars = string(default=None) mova_vars = string(default=None) movc_vars = string(default=None) + mova_years = string(default=None) + movc_years = string(default=None) + mova_modes = string(default=None) + movc_modes = string(default=None) + mova_viewer = string(default=None) + movc_viewer = string(default=None) + enso_viewer = string(default=None) enso_vars = string(default=None) - regions = string(default=None) + enso_years = string(default=None) + enso_groups = string(default=None) target_grid = string(default=None) target_grid_string = string(default=None) regrid_tool = string(default=None) regrid_tool_ocn = string(default=None) regrid_method = string(default=None) regrid_method_ocn = string(default=None) - atm_modes = string(default=None) - cpl_modes = string(default=None) CBF = string(default=None) ConvEOF = string(default=None) eofn_mod_max = integer(default=None) @@ -596,9 +615,6 @@ cmip_movs_set = string(default="cmip6.historical.v20220825") ObsUnitsAdjust = string(default=None) RmDomainMean = string(default=None) seasons = string(default=None) - enso_groups = string(default=None) - figure_sets = string(default=None) - figure_sets_period = string(default=None) reference_alias = string(default=None) regions_specs = string(default=None) pcmdi_version = string(default=None) diff --git a/zppy/pcmdi_diags.py b/zppy/pcmdi_diags.py index f49a6a94..15aa032b 100644 --- a/zppy/pcmdi_diags.py +++ b/zppy/pcmdi_diags.py @@ -73,8 +73,16 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): year_sets: List[Tuple[int, int]] if c["current_set"] != "synthetic_plots": year_sets = get_years(c["ts_years"]) + elif ("clim_years" in c.keys()) and (c["clim_years"] != ""): + year_sets = get_years(c["clim_years"]) + elif ("mova_years" in c.keys()) and (c["mova_years"] != ""): + year_sets = get_years(c["mova_years"]) + elif ("movc_years" in c.keys()) and (c["movc_years"] != ""): + year_sets = get_years(c["movc_years"]) + elif ("enso_years" in c.keys()) and (c["enso_years"] != ""): + year_sets = get_years(c["enso_years"]) else: - year_sets = get_years(c["figure_sets_period"].split(",")) + year_sets = get_years(c["ts_years"]) ref_year_sets: List[Tuple[int, int]] if ("ref_years" in c.keys()) and (c["ref_years"] != [""]): @@ -90,10 +98,18 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): c["ref_year1"] = rs[0] c["ref_year2"] = rs[1] - + if c["current_set"] != "synthetic_plots": check_and_define_parameters(c) print(c["prefix"]) + if c["current_set"] == "mean_climate": + c["clim_years"] = f"{c['year1']}-{c['year2']}" + elif c["current_set"] == "variability_modes_cpl": + c["movc_years"] = f"{c['year1']}-{c['year2']}" + elif c["current_set"] == "variability_modes_atm": + c["mova_years"] = f"{c['year1']}-{c['year2']}" + elif c["current_set"] == "enso": + c["enso_years"] = f"{c['year1']}-{c['year2']}" else: prefix = f"pcmdi_diags_{c['sub']}_{c['run_type']}" if i < len(year_sets) - 1: @@ -103,7 +119,16 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): else: print(prefix) c["prefix"] = prefix - + # assign period for set if empty + if c["clim_years"] == "": + c["clim_years"] = f"{c['year1']}-{c['year2']}" + if c["mova_years"] == "": + c["mova_years"] = f"{c['year1']}-{c['year2']}" + if c["movc_years"] == "": + c["movc_years"] = f"{c['year1']}-{c['year2']}" + if c["enso_years"] == "": + c["enso_years"] = f"{c['year1']}-{c['year2']}" + bash_file, settings_file, status_file = get_file_names( script_dir, c["prefix"] ) @@ -223,7 +248,7 @@ def check_parameters_for_bash(c: Dict[str, Any]) -> None: def check_parameters_for_pcmdi(c: Dict[str, Any]) -> None: # check and set up the external data needed by pcmdi if c["current_set"] == "synthetic_plots": - if "enso_metric" in c["figure_sets"]: + if c["enso_viewer"]: set_value_of_parameter_if_undefined( c, "cmip_enso_dir", @@ -232,7 +257,7 @@ def check_parameters_for_pcmdi(c: Dict[str, Any]) -> None: ) else: c["cmip_enso_dir"] = "placeholder_dir" - if "mean_climate" in c["figure_sets"]: + if c["clim_viewer"]: set_value_of_parameter_if_undefined( c, "cmip_clim_dir", @@ -241,7 +266,7 @@ def check_parameters_for_pcmdi(c: Dict[str, Any]) -> None: ) else: c["cmip_clim_dir"] = "placeholder_dir" - if "variability_modes" in c["figure_sets"]: + if c["mova_viewer"] or c["movc_viewer"]: set_value_of_parameter_if_undefined( c, "cmip_movs_dir", @@ -319,14 +344,15 @@ def add_pcmdi_dependencies( status_suffix = f"_{c['year1']:04d}-{c['year2']:04d}" elif c["run_type"] == "model_vs_model": status_suffix = f"_{c['year1']:04d}-{c['year2']:04d}_vs_{c['ref_year1']:04d}-{c['ref_year2']:04d}" - if "mean_climate" in c["figure_sets"]: + + if c["clim_viewer"]: status_file = os.path.join( script_dir, f"pcmdi_diags_mean_climate_{c['run_type']}{status_suffix}.status", ) if os.path.exists(status_file) and (status_file not in dependencies): dependencies.append(status_file) - if "variability_modes" in c["figure_sets"]: + if c["movc_viewer"]: # cpl status_file = os.path.join( script_dir, @@ -334,6 +360,7 @@ def add_pcmdi_dependencies( ) if os.path.exists(status_file) and (status_file not in dependencies): dependencies.append(status_file) + if c["mova_viewer"]: # atm status_file = os.path.join( script_dir, @@ -341,7 +368,7 @@ def add_pcmdi_dependencies( ) if os.path.exists(status_file) and (status_file not in dependencies): dependencies.append(status_file) - if "enso_metric" in c["figure_sets"]: + if c["enso_viewer"]: status_file = os.path.join( script_dir, f"pcmdi_diags_enso_{c['run_type']}{status_suffix}.status" ) diff --git a/zppy/templates/pcmdi_diags.bash b/zppy/templates/pcmdi_diags.bash index 1674a8b2..b9578d12 100755 --- a/zppy/templates/pcmdi_diags.bash +++ b/zppy/templates/pcmdi_diags.bash @@ -844,13 +844,13 @@ core_parameters="--num_workers {{ num_workers }} --multiprocessing {{ multiproce {% endif %} {% if current_set == "mean_climate" %} -command="zi-pcmdi-mean-climate ${core_parameters} --regions {{ regions }}" +command="zi-pcmdi-mean-climate ${core_parameters} --regions {{ clim_regions }}" {% endif %} {% if current_set in ["variability_modes_cpl", "variability_modes_atm"] %} {% if current_set == "variability_modes_atm" %} -var_modes={{ atm_modes }} +var_modes={{ mova_modes }} {% elif current_set == "variability_modes_cpl" %} -var_modes={{ cpl_modes }} +var_modes={{ movc_modes }} {% endif %} command="zi-pcmdi-variability-modes ${core_parameters} --var_modes ${var_modes}" {% endif %} @@ -859,12 +859,35 @@ command="zi-pcmdi-enso ${core_parameters} --enso_groups {{ enso_groups }}" {% endif %} {% if current_set == "synthetic_plots" %} +#add command for mean climate viewer +clim_keys="--clim_viewer {{clim_viewer}} --cmip_clim_dir {{ cmip_clim_dir }} --cmip_clim_set {{ cmip_clim_set }} --clim_vars {{clim_vars}} --clim_years {{ clim_years }} --clim_regions {{ clim_regions }}" +{% if clim_viewer %} +echo "Checking if *.${case_id}.json files exist in clim_dir:" +clim_dir=${web_dir}/model_vs_obs/metrics_data/mean_climate/ +find ${clim_dir} -name "*.${case_id}*.json" +echo "Done checking. There should be a list of files above, if they exist." +{% endif %} + +#add command for modes variability viewer +movs_keys="--mova_viewer {{mova_viewer}} --movc_viewer {{movc_viewer}} --cmip_movs_dir {{ cmip_movs_dir }} --cmip_movs_set {{ cmip_movs_set }} --mova_modes {{ mova_modes }} --mova_vars {{mova_vars}} --mova_years {{mova_years}} --movc_modes {{ movc_modes }} --movc_vars {{movc_vars}} --movc_years {{movc_years}}" +{% if movc_viewer or mova_viewer %} echo "Checking if var_mode_*.json files exist in variability_modes_dir:" variability_modes_dir=${web_dir}/model_vs_obs/metrics_data/variability_modes/ -find ${variability_modes_dir} -name "var_mode_*.json" +find ${variability_modes_dir} -name "var_mode_*${case_id}*.json" echo "Done checking. There should be a list of files above, if they exist." +{% endif %} + +#add command for enso viewer +enso_keys="--enso_viewer {{enso_viewer}} --cmip_enso_dir {{ cmip_enso_dir }} --cmip_enso_set {{ cmip_enso_set }} --enso_vars {{enso_vars}} --enso_years {{enso_years}}" +{% if enso_viewer %} +echo "Checking if var_mode_*.json files exist in variability_modes_dir:" +enso_dir=${web_dir}/model_vs_obs/metrics_data/enso_metric/* +find ${enso_dir} -name "*.${case_id}*.json" +echo "Done checking. There should be a list of files above, if they exist." +{% endif %} + +command="zi-pcmdi-synthetic-plots --synthetic_sets {{ synthetic_sets }} --figure_format {{ figure_format }} --www ${www} --results_dir ${results_dir} --case {{ case }} --model_name {{ model_name }} --model_tableID {{model_tableID }} --web_dir=${web_dir} --pcmdi_webtitle {{ pcmdi_webtitle }} --pcmdi_version {{ pcmdi_version }} --run_type ${run_type} --pcmdi_external_prefix {{ diagnostics_base_path }} --pcmdi_viewer_template {{ pcmdi_viewer_template }} --save_all_data {{save_all_data}} ${clim_keys} ${movs_keys} ${enso_keys}" -command="zi-pcmdi-synthetic-plots --synthetic_sets {{ synthetic_sets }} --figure_format {{ figure_format }} --www ${www} --results_dir ${results_dir} --case {{ case }} --model_name {{ model_name }} --model_tableID {{model_tableID }} --web_dir=${web_dir} --cmip_clim_dir {{ cmip_clim_dir }} --cmip_clim_set {{ cmip_clim_set }} --cmip_movs_dir {{ cmip_movs_dir }} --cmip_movs_set {{ cmip_movs_set }} --atm_modes {{ atm_modes }} --cpl_modes {{ cpl_modes }} --cmip_enso_dir {{ cmip_enso_dir }} --cmip_enso_set {{ cmip_enso_set }} --figure_sets {{ figure_sets }} --pcmdi_webtitle {{ pcmdi_webtitle }} --pcmdi_version {{ pcmdi_version }} --run_type ${run_type} --figure_sets_period {{ figure_sets_period }} --pcmdi_external_prefix {{ diagnostics_base_path }} --pcmdi_viewer_template {{ pcmdi_viewer_template }}" {% endif %} # Run diagnostics From 4a1bd042ab9e5b217a262ff1461d56a733a0f28e Mon Sep 17 00:00:00 2001 From: ShixuanZhang Date: Sun, 5 Oct 2025 03:03:35 -0500 Subject: [PATCH 32/47] Add sample cfg (for AMIP simulation) file to test subset of plots and viewer page --- examples/post.v3.LR.amip.0101.cfg | 57 ++++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/examples/post.v3.LR.amip.0101.cfg b/examples/post.v3.LR.amip.0101.cfg index 6b915365..fea62cf0 100644 --- a/examples/post.v3.LR.amip.0101.cfg +++ b/examples/post.v3.LR.amip.0101.cfg @@ -1,12 +1,14 @@ [default] input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/AMIP/v3.LR.amip_0101 -output = /lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy_pr719_output/unique_id_51/v3.LR.amip_0101 -www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.szhang/zppy_pr719_output/unique_id_53 +output = /lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy_pr719_output/unique_id_56/v3.LR.amip_0101 +www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.szhang/zppy_pr719_output/unique_id_56 campaign = "water_cycle" case = v3.LR.amip_0101 debug = False environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" -partition = debug #compute +#partition = debug #compute +partition = priority +account = priority [ts] active = True @@ -66,11 +68,11 @@ ts_years = "2005-2014", ref_years = "2005-2014", # Variables in current pcmdi-cmip list # rltcre and rstcre are derived variables as they are not included in cmip_ts - # define a new variable name to be flexible + # define a new variable name to be flexible clim_vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rsus,rsuscs,rlus,rlut,rlutcs,rsds,rsdscs,rsdt,rsut,rsutcs,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" # See zppy/templates/pcmdi_diags/regions_specs.json for options of all regions # Below is default setup for global/regional mean climate metrics - regions = "global,ocean,land,NHEX,SHEX,TROPICS" + clim_regions = "global,ocean,land,Tropics,NHEX,SHEX" target_grid = "2.5x2.5" target_grid_string = "2p5x2p5" @@ -82,8 +84,9 @@ ts_years = "2005-2014", ref_start_yr = 1870 ref_years = "2005-2014", movc_vars = "ts" + movc_years = "2005-2014" # Name of modes varibility - cpl_modes = "PDO,NPGO,AMO" + movc_modes = "PDO,NPGO,AMO" frequency = 'mo' landmask = True ModUnitsAdjust = '(True,"subtract",273.15)' @@ -99,8 +102,9 @@ ts_years = "2005-2014", ref_start_yr = 1853 ref_years = "2005-2014", mova_vars = "psl" + mova_years = "2005-2014" # Varibility modes - atm_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" + mova_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" frequency = 'mo' landmask = False ModUnitsAdjust = '(True,"divide",100.0)' @@ -116,17 +120,52 @@ ts_years = "2005-2014", ref_start_yr = 1979 ref_years = "2005-2014", enso_vars = "psl,pr,prsn,ts,tas,tauu,tauv,hfls,hfss,rlds,rsds,rlus,rlut,rsdt" + enso_years = "2005-2014" # See zppy/templates/pcmdi_diags/reference_alias.json for options enso_groups = "ENSO_perf,ENSO_proc,ENSO_tel" [[ synthetic_plots ]] active = True - # clim_period, emov_period, enso_period: - figure_sets_period = "2005-2014,2005-2014,2005-2014" + + # mean climate + clim_viewer = True + # use subset regions for viewer + clim_regions = "global,ocean,land" + # use subset variables for viewer + clim_vars = "pr,prw,psl,rlds,rldscs,ua-200,zg-500" + # period to be plotted and shown on webpage + # note that the previous sections allows multiple ts_years periods for processing, while + # the viewer page focused on period customized by user + clim_years = "2005-2014" + # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/mean_climate/cmip6/amip cmip_clim_set = "cmip6.amip.v20250702" + + # modes variability (atmosphere) + mova_viewer = True + # use subset modes for viewer + mova_modes = "NAM,NAO,PNA,NPO,SAM" + # period to be plotted and shown on webpage + # note that the previous sections allows multiple ts_years periods for processing, while + # the viewer page focused on period customized by user + mova_years = "2005-2014" + + # modes variability (coupled) + movc_viewer = True + # use subset modes for viewer + movc_modes = "PDO,NPGO" + # period to be plotted and shown on webpage + # note that the previous sections allows multiple ts_years periods for processing, while + # the viewer page focused on period customized by user + movc_years = "2005-2014" + # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes/cmip6/amip cmip_movs_set = "cmip6.amip.v20210119" + + # enso + enso_viewer = False + enso_years = "" + # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/enso_metric/cmip6/historical # There is no enso metric for CMIP6 AMIP, so use historical instead cmip_enso_set = "cmip6.historical.v20210620" From 63f9b31ccebb34111e7e6874c378663e4add8d94 Mon Sep 17 00:00:00 2001 From: ShixuanZhang Date: Sun, 5 Oct 2025 03:42:07 -0500 Subject: [PATCH 33/47] add vertical interpolation file to avoid failure --- examples/post.v3.LR.amip.0101.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/post.v3.LR.amip.0101.cfg b/examples/post.v3.LR.amip.0101.cfg index fea62cf0..ba72982f 100644 --- a/examples/post.v3.LR.amip.0101.cfg +++ b/examples/post.v3.LR.amip.0101.cfg @@ -34,7 +34,7 @@ years = "2005:2015:5" [[ atm_monthly_180x360_aave ]] # list of variables to be interpolated to pressure level; see e3sm_to_cmip/vrt_remap_plev19.nc # This defaults to cmip_plevdata = string(default="inclusions/e3sm_to_cmip/vrt_remap_plev19.nc") - #cmip_plevdata = /lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc + cmip_plevdata = /lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc input_files = "eam.h0" interp_vars = "U,V,T,Z3" ts_subsection = "atm_monthly_180x360_aave" From 8b7541d6e382bcf3705ca2c87b7550f4555b50f9 Mon Sep 17 00:00:00 2001 From: ShixuanZhang Date: Sun, 5 Oct 2025 03:42:34 -0500 Subject: [PATCH 34/47] Second test for the hostical simulations --- examples/post.v3.LR.historical.0101.cfg | 214 +++++++++++------------- 1 file changed, 100 insertions(+), 114 deletions(-) diff --git a/examples/post.v3.LR.historical.0101.cfg b/examples/post.v3.LR.historical.0101.cfg index 9bd89ffd..c9095e0b 100644 --- a/examples/post.v3.LR.historical.0101.cfg +++ b/examples/post.v3.LR.historical.0101.cfg @@ -1,18 +1,14 @@ -# To run: -# `pip install .` the zppy-interfaces changes into the `zi-pcmdi-20250529` environment -# Update unique_id in output, www below -# Be in the zppy-pr719-dev-20250612 environment -# > pip install . && zppy -c examples/post.v3.LR.historical.0101.cfg - [default] -input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/AMIP/v3.LR.historical_0101 -output = /lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy_pr719_output/unique_id_21/v3.LR.historical_0101 -www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.szhang/zppy_pr719_output/unique_id_21 +input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/v3.LR.historical_0101 +output = /lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy_pr719_output/unique_id_56/v3.LR.historical_0101 +www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.szhang/zppy_pr719_output/unique_id_56 campaign = "water_cycle" case = v3.LR.historical_0101 -debug = True +debug = False environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" -partition = debug #compute +#partition = debug #compute +partition = priority +account = priority [ts] active = True @@ -30,7 +26,6 @@ years = "2005:2015:5" [e3sm_to_cmip] active = True frequency = "monthly" -grid = "180x360_aave" ts_grid = "180x360_aave" ts_num_years=5 walltime = "00:10:00" @@ -38,6 +33,7 @@ years = "2005:2015:5" [[ atm_monthly_180x360_aave ]] # list of variables to be interpolated to pressure level; see e3sm_to_cmip/vrt_remap_plev19.nc + # This defaults to cmip_plevdata = string(default="inclusions/e3sm_to_cmip/vrt_remap_plev19.nc") cmip_plevdata = /lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc input_files = "eam.h0" interp_vars = "U,V,T,Z3" @@ -46,140 +42,130 @@ years = "2005:2015:5" [pcmdi_diags] active = True -multiprocessing = True +environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" +environment_commands_secondary = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh; conda activate zi-pcmdi-diags-20250730" nodes = 1 +walltime = "4:00:00" +figure_format = "png" +run_type = "model_vs_obs" +# Create rules to construct cmip-like datasets with a string constructed as +# "mip.exp.product.realization", where "."is used as the delimiter (mandatory) +model_name = 'e3sm.historical.v3-LR.0101' +model_tableID = 'Amon' +multiprocessing = True num_workers = 24 -#observational data +# Observational data obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series' pcmdi_debug = False -run_type = "model_vs_obs" -reference_data_path = "" -reference_data_path_ts = "" -ts_grid = "180x360_aave" -walltime = "4:00:00" -environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" +ts_years = "2005-2014", [[ mean_climate ]] active = True - ################################################################################## - #create rules to construct cmip-like datasets with a string constructed as - #"mip.exp.product.realization", where "."is used as the delimiter (mandatary) - ts_years = "2005-2014", - ts_num_years = 5 - ref_years = "2005-2014", - ref_start_yr = 1979 - ref_final_yr = 2018 - model_name = 'e3sm.historical.v3-LR.0101' - model_tableID = 'Amon' - ################################################################################## - #location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/mean_climate/cmip6/historical - cmip_clim_set = "cmip6.historical.v20250707" - ################################################################################## - #variables in current pcmdi-cmip list - #note: rltcre and rstcre are derived variables as they are not included in cmip_ts - vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rsus,rsuscs,rlus,rlut,rlutcs,rsds,rsdscs,rsdt,rsut,rsutcs,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" - #default for all variables or specify source for each variable separately. + # Default for all variables or specify source for each variable separately. obs_sets = "default" - #see zppy/templates/pcmdi_diags/regions_specs.json for options of all regions - #below is default setup for global/regional mean climate metrics - regions = "global,ocean,land,NHEX,SHEX,TROPICS" + ref_final_yr = 2018 + ref_start_yr = 1979 + ref_years = "2005-2014", + # Variables in current pcmdi-cmip list + # rltcre and rstcre are derived variables as they are not included in cmip_ts + # define a new variable name to be flexible + clim_vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rsus,rsuscs,rlus,rlut,rlutcs,rsds,rsdscs,rsdt,rsut,rsutcs,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" + # See zppy/templates/pcmdi_diags/regions_specs.json for options of all regions + # Below is default setup for global/regional mean climate metrics + clim_regions = "global,ocean,land,Tropics,NHEX,SHEX" target_grid = "2.5x2.5" target_grid_string = "2p5x2p5" [[ variability_modes_cpl ]] active = True - ################################################################################## - ts_years = "2005-2014", - ts_num_years = 5 - ref_years = "2005-2014", - ref_start_yr = 1870 + # See zppy/templates/pcmdi_diags/reference_alias.json for options + obs_sets = "alternate1" #"default" ref_final_yr = 2019 - #create rules to construct cmip-like datasets with a string constructed as - #"mip.exp.product.realization", where "."is used as the delimiter (mandatary) - model_name = 'e3sm.historical.v3-LR.0101' - model_tableID = 'Amon' - ################################################################################## - #location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes/cmip6/historical - cmip_movs_set = "cmip6.historical.v20220825" - ################################################################################## - grid = '180x360_aave' - vars = "ts" + ref_start_yr = 1870 + ref_years = "2005-2014", + movc_vars = "ts" + movc_years = "2005-2014" + # Name of modes varibility + movc_modes = "PDO,NPGO,AMO" frequency = 'mo' - ModUnitsAdjust = '(True,"subtract",273.15)' landmask = True - #see zppy/templates/pcmdi_diags/reference_alias.json for options - obs_sets = "alternate1" #"default" + ModUnitsAdjust = '(True,"subtract",273.15)' ObsUnitsAdjust = '(True,"subtract",273.15)' - #name of modes varibility - cpl_modes = "PDO,NPGO,AMO" - #options specific for mode varibility metrics in pcmdi seasons = "yearly,monthly" - figure_format = "png" [[ variability_modes_atm ]] active = True - ################################################################################## - ts_years = "2005-2014", - ts_num_years = 5 - ref_years = "2005-2014", - ref_start_yr = 1853 + # See zppy/templates/pcmdi_diags/reference_alias.json for options + # Default for all variables + obs_sets = "alternate2" # "default" ref_final_yr = 2015 - #create rules to construct cmip-like datasets with a string constructed as - #"mip.exp.product.realization", where "." is used as the delimiter (mandatary) - model_name = 'e3sm.historical.v3-LR.0101' - model_tableID = 'Amon' - ################################################################################## - #location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes/cmip6/historical - cmip_movs_set = "cmip6.historical.v20220825" - ################################################################################## - grid = '180x360_aave' - vars = "psl" - landmask = False + ref_start_yr = 1853 + ref_years = "2005-2014", + mova_vars = "psl" + mova_years = "2005-2014" + # Varibility modes + mova_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" frequency = 'mo' + landmask = False ModUnitsAdjust = '(True,"divide",100.0)' - #observational data - obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series' - #see zppy/templates/pcmdi_diags/reference_alias.json for options - #default for all variables - obs_sets = "alternate2" # "default" ObsUnitsAdjust = '(True,"divide",100.0)' - #varibility modes - atm_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" - #options specific for mode varibility metrics in pcmdi seasons = "DJF,MAM,JJA,SON,yearly,monthly" - figure_format = "png" [[ enso ]] - active = True - ################################################################################## - ts_years = "2005-2014", - ts_num_years = 5 - ref_years = "2005-2014", - ref_start_yr = 1979 - ref_final_yr = 2018 - #create rules to construct cmip-like datasets with a string constructed as - #"mip.exp.product.realization", where "."is used as the delimiter (mandatary) - model_name = 'e3sm.historical.v3-LR.0101' - model_tableID = 'Amon' - ################################################################################## - #location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/enso_metric/cmip6/historical - cmip_enso_set = "cmip6.historical.v20210620" - ################################################################################## - #see zppy/templates/pcmdi_diags/reference_alias.json for options - customized_cmor_obs = True - vars = "psl,pr,prsn,ts,tas,tauu,tauv,hfls,hfss,rlds,rsds,rlus,rlut,rsdt" + active = False # For enso, the obs_sets is explicitely set for each variable to ensure that # they are from the same source, below are setup to used ERA5 reanalysis. obs_sets = "default,alternate2,default,default,default,default,default,default,default,alternate2,alternate2,alternate2,alternate2,alternate2" - #see zppy/templates/pcmdi_diags/reference_alias.json for options + ref_final_yr = 2018 + ref_start_yr = 1979 + ref_years = "2005-2014", + enso_vars = "psl,pr,prsn,ts,tas,tauu,tauv,hfls,hfss,rlds,rsds,rlus,rlut,rsdt" + enso_years = "2005-2014" + # See zppy/templates/pcmdi_diags/reference_alias.json for options enso_groups = "ENSO_perf,ENSO_proc,ENSO_tel" - figure_format = "png" [[ synthetic_plots ]] active = True - model_name = 'e3sm.historical.v3-LR.0101' - model_tableID = 'Amon' - synthetic_sets = "portrait,parcoord" - figure_sets = "mean_climate,variability_modes,enso_metric" - figure_sets_period = "2005-2014,2005-2014,2005-2014" - figure_format = "png" + + # mean climate + clim_viewer = True + # use subset regions for viewer + clim_regions = "global,ocean,land" + # use subset variables for viewer + clim_vars = "pr,prw,psl,rlds,rldscs,ua-200,zg-500" + # period to be plotted and shown on webpage + # note that the previous sections allows multiple ts_years periods for processing, while + # the viewer page focused on period customized by user + clim_years = "2005-2014" + + # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/mean_climate/cmip6/historical + cmip_clim_set = "cmip6.historical.v20250707" + + # modes variability (atmosphere) + mova_viewer = True + # use subset modes for viewer + mova_modes = "NAM,NAO,PNA,NPO,SAM" + # period to be plotted and shown on webpage + # note that the previous sections allows multiple ts_years periods for processing, while + # the viewer page focused on period customized by user + mova_years = "2005-2014" + + # modes variability (coupled) + movc_viewer = True + # use subset modes for viewer + movc_modes = "PDO,NPGO" + # period to be plotted and shown on webpage + # note that the previous sections allows multiple ts_years periods for processing, while + # the viewer page focused on period customized by user + movc_years = "2005-2014" + + # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes/cmip6/historical + cmip_movs_set = "cmip6.historical.v20220825" + + # enso + enso_viewer = False + enso_years = "" + + # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/enso_metric/cmip6/historical + # There is no enso metric for CMIP6 AMIP, so use historical instead + cmip_enso_set = "cmip6.historical.v20210620" From 0eb5e5eb07d1d657203f354486406d728f82d104 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 1 Oct 2025 10:40:02 -0500 Subject: [PATCH 35/47] Fix image checker --- tests/integration/image_checker.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/image_checker.py b/tests/integration/image_checker.py index 53ce8fdb..f6a8da7b 100644 --- a/tests/integration/image_checker.py +++ b/tests/integration/image_checker.py @@ -76,6 +76,9 @@ def set_up_and_run_image_checker( print(f"{key}: {d[key]}") parameters: Parameters = Parameters(d) for task in task_list: + if task == "pcmdi_diags": + print(f"{task} hs no expected results yet, skipping.") + continue test_results = check_images(parameters, task) test_results_dict[f"{cfg_specifier}_{task}"] = test_results From a54c9c0a36b63653250151036bc7d682b7b3097a Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 6 Oct 2025 12:13:49 -0500 Subject: [PATCH 36/47] pre-commit fixes --- examples/post.v3.LR.amip.0101.cfg | 30 ++++++++++++------------- examples/post.v3.LR.historical.0101.cfg | 30 ++++++++++++------------- examples/remove_duplicate_log_lines.py | 7 +++--- zppy/defaults/default.ini | 6 ++--- zppy/pcmdi_diags.py | 17 +++++++------- zppy/templates/pcmdi_diags.bash | 8 +++---- 6 files changed, 50 insertions(+), 48 deletions(-) diff --git a/examples/post.v3.LR.amip.0101.cfg b/examples/post.v3.LR.amip.0101.cfg index ba72982f..a7c1405e 100644 --- a/examples/post.v3.LR.amip.0101.cfg +++ b/examples/post.v3.LR.amip.0101.cfg @@ -129,13 +129,13 @@ ts_years = "2005-2014", # mean climate clim_viewer = True - # use subset regions for viewer + # use subset regions for viewer clim_regions = "global,ocean,land" - # use subset variables for viewer + # use subset variables for viewer clim_vars = "pr,prw,psl,rlds,rldscs,ua-200,zg-500" - # period to be plotted and shown on webpage - # note that the previous sections allows multiple ts_years periods for processing, while - # the viewer page focused on period customized by user + # period to be plotted and shown on webpage + # note that the previous sections allows multiple ts_years periods for processing, while + # the viewer page focused on period customized by user clim_years = "2005-2014" # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/mean_climate/cmip6/amip @@ -143,26 +143,26 @@ ts_years = "2005-2014", # modes variability (atmosphere) mova_viewer = True - # use subset modes for viewer + # use subset modes for viewer mova_modes = "NAM,NAO,PNA,NPO,SAM" - # period to be plotted and shown on webpage - # note that the previous sections allows multiple ts_years periods for processing, while - # the viewer page focused on period customized by user + # period to be plotted and shown on webpage + # note that the previous sections allows multiple ts_years periods for processing, while + # the viewer page focused on period customized by user mova_years = "2005-2014" # modes variability (coupled) movc_viewer = True - # use subset modes for viewer + # use subset modes for viewer movc_modes = "PDO,NPGO" - # period to be plotted and shown on webpage - # note that the previous sections allows multiple ts_years periods for processing, while - # the viewer page focused on period customized by user + # period to be plotted and shown on webpage + # note that the previous sections allows multiple ts_years periods for processing, while + # the viewer page focused on period customized by user movc_years = "2005-2014" # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes/cmip6/amip cmip_movs_set = "cmip6.amip.v20210119" - - # enso + + # enso enso_viewer = False enso_years = "" diff --git a/examples/post.v3.LR.historical.0101.cfg b/examples/post.v3.LR.historical.0101.cfg index c9095e0b..e1ec99b6 100644 --- a/examples/post.v3.LR.historical.0101.cfg +++ b/examples/post.v3.LR.historical.0101.cfg @@ -129,13 +129,13 @@ ts_years = "2005-2014", # mean climate clim_viewer = True - # use subset regions for viewer + # use subset regions for viewer clim_regions = "global,ocean,land" - # use subset variables for viewer + # use subset variables for viewer clim_vars = "pr,prw,psl,rlds,rldscs,ua-200,zg-500" - # period to be plotted and shown on webpage - # note that the previous sections allows multiple ts_years periods for processing, while - # the viewer page focused on period customized by user + # period to be plotted and shown on webpage + # note that the previous sections allows multiple ts_years periods for processing, while + # the viewer page focused on period customized by user clim_years = "2005-2014" # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/mean_climate/cmip6/historical @@ -143,26 +143,26 @@ ts_years = "2005-2014", # modes variability (atmosphere) mova_viewer = True - # use subset modes for viewer + # use subset modes for viewer mova_modes = "NAM,NAO,PNA,NPO,SAM" - # period to be plotted and shown on webpage - # note that the previous sections allows multiple ts_years periods for processing, while - # the viewer page focused on period customized by user + # period to be plotted and shown on webpage + # note that the previous sections allows multiple ts_years periods for processing, while + # the viewer page focused on period customized by user mova_years = "2005-2014" # modes variability (coupled) movc_viewer = True - # use subset modes for viewer + # use subset modes for viewer movc_modes = "PDO,NPGO" - # period to be plotted and shown on webpage - # note that the previous sections allows multiple ts_years periods for processing, while - # the viewer page focused on period customized by user + # period to be plotted and shown on webpage + # note that the previous sections allows multiple ts_years periods for processing, while + # the viewer page focused on period customized by user movc_years = "2005-2014" # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes/cmip6/historical cmip_movs_set = "cmip6.historical.v20220825" - - # enso + + # enso enso_viewer = False enso_years = "" diff --git a/examples/remove_duplicate_log_lines.py b/examples/remove_duplicate_log_lines.py index 9819fef2..272ebddd 100644 --- a/examples/remove_duplicate_log_lines.py +++ b/examples/remove_duplicate_log_lines.py @@ -3,9 +3,10 @@ def remove_consecutive_duplicates(input_file_path): output_file_path = input_file_path + ".duplicates_removed" last_written_line = None - with open(input_file_path, "r", encoding="utf-8") as infile, open( - output_file_path, "w", encoding="utf-8" - ) as outfile: + with ( + open(input_file_path, "r", encoding="utf-8") as infile, + open(output_file_path, "w", encoding="utf-8") as outfile, + ): for line in infile: if line != last_written_line: outfile.write(line) diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index 1b1db260..844111ca 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -524,10 +524,10 @@ mova_years = string(default="") movc_viewer = string(default=True) movc_years = string(default="") # add enso to viewer page (default is False, not currently enabled) -enso_viewer = string(default=False) +enso_viewer = string(default=False) enso_years = string(default="") -# Save all intemediate diagnostic data -save_all_data = string(default=True) +# Save all intemediate diagnostic data +save_all_data = string(default=True) # File of observation data name for mean climate calculation reference_alias = string(default="inclusions/pcmdi_diags/reference_alias.json") # File of specified regions for mean climate calculation diff --git a/zppy/pcmdi_diags.py b/zppy/pcmdi_diags.py index 15aa032b..1030d7e5 100644 --- a/zppy/pcmdi_diags.py +++ b/zppy/pcmdi_diags.py @@ -39,7 +39,8 @@ # ----------------------------------------------------------------------------- -def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): +# C901 'pcmdi_diags' is too complex (29) +def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): # noqa: C901 template, _ = initialize_template(config, "pcmdi_diags.bash") @@ -74,13 +75,13 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): if c["current_set"] != "synthetic_plots": year_sets = get_years(c["ts_years"]) elif ("clim_years" in c.keys()) and (c["clim_years"] != ""): - year_sets = get_years(c["clim_years"]) + year_sets = get_years(c["clim_years"]) elif ("mova_years" in c.keys()) and (c["mova_years"] != ""): - year_sets = get_years(c["mova_years"]) + year_sets = get_years(c["mova_years"]) elif ("movc_years" in c.keys()) and (c["movc_years"] != ""): - year_sets = get_years(c["movc_years"]) + year_sets = get_years(c["movc_years"]) elif ("enso_years" in c.keys()) and (c["enso_years"] != ""): - year_sets = get_years(c["enso_years"]) + year_sets = get_years(c["enso_years"]) else: year_sets = get_years(c["ts_years"]) @@ -98,7 +99,7 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): c["ref_year1"] = rs[0] c["ref_year2"] = rs[1] - + if c["current_set"] != "synthetic_plots": check_and_define_parameters(c) print(c["prefix"]) @@ -128,7 +129,7 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): c["movc_years"] = f"{c['year1']}-{c['year2']}" if c["enso_years"] == "": c["enso_years"] = f"{c['year1']}-{c['year2']}" - + bash_file, settings_file, status_file = get_file_names( script_dir, c["prefix"] ) @@ -344,7 +345,7 @@ def add_pcmdi_dependencies( status_suffix = f"_{c['year1']:04d}-{c['year2']:04d}" elif c["run_type"] == "model_vs_model": status_suffix = f"_{c['year1']:04d}-{c['year2']:04d}_vs_{c['ref_year1']:04d}-{c['ref_year2']:04d}" - + if c["clim_viewer"]: status_file = os.path.join( script_dir, diff --git a/zppy/templates/pcmdi_diags.bash b/zppy/templates/pcmdi_diags.bash index b9578d12..69d349ee 100755 --- a/zppy/templates/pcmdi_diags.bash +++ b/zppy/templates/pcmdi_diags.bash @@ -33,7 +33,7 @@ results_dir="{{ run_type }}" {% if current_set != "synthetic_plots" %} -# Input time range +# Input time range y1={{ year1 }} y2={{ year2 }} ref_y1={{ ref_year1 }} @@ -859,7 +859,7 @@ command="zi-pcmdi-enso ${core_parameters} --enso_groups {{ enso_groups }}" {% endif %} {% if current_set == "synthetic_plots" %} -#add command for mean climate viewer +#add command for mean climate viewer clim_keys="--clim_viewer {{clim_viewer}} --cmip_clim_dir {{ cmip_clim_dir }} --cmip_clim_set {{ cmip_clim_set }} --clim_vars {{clim_vars}} --clim_years {{ clim_years }} --clim_regions {{ clim_regions }}" {% if clim_viewer %} echo "Checking if *.${case_id}.json files exist in clim_dir:" @@ -868,7 +868,7 @@ find ${clim_dir} -name "*.${case_id}*.json" echo "Done checking. There should be a list of files above, if they exist." {% endif %} -#add command for modes variability viewer +#add command for modes variability viewer movs_keys="--mova_viewer {{mova_viewer}} --movc_viewer {{movc_viewer}} --cmip_movs_dir {{ cmip_movs_dir }} --cmip_movs_set {{ cmip_movs_set }} --mova_modes {{ mova_modes }} --mova_vars {{mova_vars}} --mova_years {{mova_years}} --movc_modes {{ movc_modes }} --movc_vars {{movc_vars}} --movc_years {{movc_years}}" {% if movc_viewer or mova_viewer %} echo "Checking if var_mode_*.json files exist in variability_modes_dir:" @@ -877,7 +877,7 @@ find ${variability_modes_dir} -name "var_mode_*${case_id}*.json" echo "Done checking. There should be a list of files above, if they exist." {% endif %} -#add command for enso viewer +#add command for enso viewer enso_keys="--enso_viewer {{enso_viewer}} --cmip_enso_dir {{ cmip_enso_dir }} --cmip_enso_set {{ cmip_enso_set }} --enso_vars {{enso_vars}} --enso_years {{enso_years}}" {% if enso_viewer %} echo "Checking if var_mode_*.json files exist in variability_modes_dir:" From 6bbf05528752b5b1afa035513304f4148195f534 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 1 Oct 2025 09:06:43 -0500 Subject: [PATCH 37/47] Standalone zi testing --- examples/post.v3.LR.amip.0101.zi_setup.cfg | 137 +++++++++++++++++++++ zppy/templates/pcmdi_diags.bash | 4 +- 2 files changed, 139 insertions(+), 2 deletions(-) create mode 100644 examples/post.v3.LR.amip.0101.zi_setup.cfg diff --git a/examples/post.v3.LR.amip.0101.zi_setup.cfg b/examples/post.v3.LR.amip.0101.zi_setup.cfg new file mode 100644 index 00000000..fc389344 --- /dev/null +++ b/examples/post.v3.LR.amip.0101.zi_setup.cfg @@ -0,0 +1,137 @@ +# Modify this cfg to generate the necessary data for stand-alone tests of zppy-interfaces +# 1. Update , , below. +# 2. Run with `zppy -c examples/post.v3.LR.amip.0101.zi_setup.cfg`. +# 3. Once the jobs finish, `cd /post/scripts`. +# 4. Run `grep -n "About to run:" pcmdi_diags*.o*` to find the pcmdi_diags commands. +# 5. Then, you can run those lines stand-alone. +[default] +input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/AMIP/v3.LR.amip_0101 +output = /lcrc/group/e3sm/ac.forsyth2/zi_pcmdi_stand_alone_setup/unique_id_1/v3.LR.amip_0101 +www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zi_pcmdi_stand_alone_setup/unique_id_1 +campaign = "water_cycle" +case = v3.LR.amip_0101 +debug = False +environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" +partition = debug #compute + +[ts] +active = True +walltime = "00:10:00" +years = "2005:2015:5" + + [[ atm_monthly_180x360_aave ]] + input_subdir = "archive/atm/hist" + input_files = "eam.h0" + frequency = "monthly" + mapping_file = /home/ac.zender/data/maps/map_ne30pg2_to_cmip6_180x360_aave.20200201.nc + # list for pcmdi diag, note: PHIS,hyam,hybm,hyai,hybi need to be included to process the 3D fields + vars = "ICEFRAC,LANDFRAC,OCNFRAC,PSL,FSNTC,FSNTOAC,SWCF,LWCF,FLUT,FSNT,FSNTOA,FLNT,FLNTC,FSNS,FLNS,FSNS,SHFLX,QFLX,LHFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,U10,QREFHT,TMQ,CLDTOT,CLDHGH,CLDMED,CLDLOW,FLDS,FSDS,TGCLDIWP,TGCLDCWP,TGCLDLWP,FLNSC,FLUTC,FSDSC,SOLIN,FSNSC,AODABS,AODVIS,AODDUST,AREL,TREFMNAV,TREFMXAV,PS,PHIS,U,V,T,Z3" + +[e3sm_to_cmip] +active = True +frequency = "monthly" +ts_grid = "180x360_aave" +ts_num_years=5 +walltime = "00:10:00" +years = "2005:2015:5" + + [[ atm_monthly_180x360_aave ]] + # list of variables to be interpolated to pressure level; see e3sm_to_cmip/vrt_remap_plev19.nc + # This defaults to cmip_plevdata = string(default="inclusions/e3sm_to_cmip/vrt_remap_plev19.nc") + # cmip_plevdata = /lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc + input_files = "eam.h0" + interp_vars = "U,V,T,Z3" + ts_subsection = "atm_monthly_180x360_aave" + vars = "ICEFRAC,LANDFRAC,OCNFRAC,PSL,FSNTC,FSNTOAC,SWCF,LWCF,FLUT,FSNT,FSNTOA,FLNT,FLNTC,FSNS,FLNS,FSNS,SHFLX,QFLX,LHFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,U10,QREFHT,TMQ,CLDTOT,CLDHGH,CLDMED,CLDLOW,FLDS,FSDS,TGCLDIWP,TGCLDCWP,TGCLDLWP,FLNSC,FLUTC,FSDSC,SOLIN,FSNSC,AODABS,AODVIS,AODDUST,AREL,TREFMNAV,TREFMXAV,PS,PHIS,U,V,T,Z3" + +[pcmdi_diags] +active = True +environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" +environment_commands_secondary = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20250930" +nodes = 1 +walltime = "4:00:00" +figure_format = "png" +run_type = "model_vs_obs" +# Create rules to construct cmip-like datasets with a string constructed as +# "mip.exp.product.realization", where "."is used as the delimiter (mandatory) +model_name = 'e3sm.amip.v3-LR.0101' +model_tableID = 'Amon' +multiprocessing = True +num_workers = 24 +# Observational data +obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series' +pcmdi_debug = False +ts_years = "2005-2014", + + [[ mean_climate ]] + active = True + # Default for all variables or specify source for each variable separately. + obs_sets = "default" + ref_final_yr = 2018 + ref_start_yr = 1979 + ref_years = "2005-2014", + # Variables in current pcmdi-cmip list + # rltcre and rstcre are derived variables as they are not included in cmip_ts + vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rsus,rsuscs,rlus,rlut,rlutcs,rsds,rsdscs,rsdt,rsut,rsutcs,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" + # See zppy/templates/pcmdi_diags/regions_specs.json for options of all regions + # Below is default setup for global/regional mean climate metrics + regions = "global,ocean,land,NHEX,SHEX,TROPICS" + target_grid = "2.5x2.5" + target_grid_string = "2p5x2p5" + + [[ variability_modes_cpl ]] + active = True + # See zppy/templates/pcmdi_diags/reference_alias.json for options + obs_sets = "alternate1" #"default" + ref_final_yr = 2019 + ref_start_yr = 1870 + ref_years = "2005-2014", + vars = "ts" + # Name of modes varibility + cpl_modes = "PDO,NPGO,AMO" + frequency = 'mo' + landmask = True + ModUnitsAdjust = '(True,"subtract",273.15)' + ObsUnitsAdjust = '(True,"subtract",273.15)' + seasons = "yearly,monthly" + + [[ variability_modes_atm ]] + active = True + # See zppy/templates/pcmdi_diags/reference_alias.json for options + # Default for all variables + obs_sets = "alternate2" # "default" + ref_final_yr = 2015 + ref_start_yr = 1853 + ref_years = "2005-2014", + vars = "psl" + # Varibility modes + atm_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" + frequency = 'mo' + landmask = False + ModUnitsAdjust = '(True,"divide",100.0)' + ObsUnitsAdjust = '(True,"divide",100.0)' + seasons = "DJF,MAM,JJA,SON,yearly,monthly" + + [[ enso ]] + active = False + # For enso, the obs_sets is explicitely set for each variable to ensure that + # they are from the same source, below are setup to used ERA5 reanalysis. + obs_sets = "default,alternate2,default,default,default,default,default,default,default,alternate2,alternate2,alternate2,alternate2,alternate2" + ref_final_yr = 2018 + ref_start_yr = 1979 + ref_years = "2005-2014", + vars = "psl,pr,prsn,ts,tas,tauu,tauv,hfls,hfss,rlds,rsds,rlus,rlut,rsdt" + # See zppy/templates/pcmdi_diags/reference_alias.json for options + enso_groups = "ENSO_perf,ENSO_proc,ENSO_tel" + + [[ synthetic_plots ]] + active = True + # clim_period, emov_period, enso_period: + figure_sets_period = "2005-2014,2005-2014,2005-2014" + # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/mean_climate/cmip6/amip + cmip_clim_set = "cmip6.amip.v20250702" + # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes/cmip6/amip + cmip_movs_set = "cmip6.amip.v20210119" + # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/enso_metric/cmip6/historical + # There is no enso metric for CMIP6 AMIP, so use historical instead + cmip_enso_set = "cmip6.historical.v20210620" diff --git a/zppy/templates/pcmdi_diags.bash b/zppy/templates/pcmdi_diags.bash index 69d349ee..3d8913e9 100755 --- a/zppy/templates/pcmdi_diags.bash +++ b/zppy/templates/pcmdi_diags.bash @@ -511,7 +511,7 @@ echo "Linking observational data into ${obstmp_dir}..." echo "Linking observational data using SLURM..." command="zi-pcmdi-link-observation --model_name_ref ${model_name_ref} --tableID_ref ${tableID_ref} --vars=${source_vars} --obs_sets {{ obs_sets }} --obs_ts {{ obs_ts }} --obstmp_dir ${obstmp_dir}" -echo "Running: ${command}" +echo "Running a zi-pcmdi command: ${command}" {{ environment_commands_secondary }} time eval "${command}" @@ -892,7 +892,7 @@ command="zi-pcmdi-synthetic-plots --synthetic_sets {{ synthetic_sets }} --figure # Run diagnostics {{ environment_commands_secondary }} -echo "About to run a zi-pcmdi command" +echo "Running a zi-pcmdi command: ${command}" echo "The current directory is: $PWD" # This will be of the form .../post/scripts/tmpDir time ${command} if [ $? != 0 ]; then From 9e399f46423c8ea1f8d41ea465bbb091c77a1d67 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 6 Oct 2025 12:50:54 -0500 Subject: [PATCH 38/47] Get unit tests passing --- tests/test_zppy_pcmdi_diags.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/tests/test_zppy_pcmdi_diags.py b/tests/test_zppy_pcmdi_diags.py index 127e46c6..98b41240 100644 --- a/tests/test_zppy_pcmdi_diags.py +++ b/tests/test_zppy_pcmdi_diags.py @@ -163,6 +163,9 @@ def test_check_parameters_for_pcmdi(): "cmip_enso_dir": "", "cmip_clim_dir": "", "cmip_movs_dir": "", + "enso_viewer": False, + "clim_viewer": True, + "mova_viewer": True, "diagnostics_base_path": "diags/post", "infer_path_parameters": True, } @@ -197,6 +200,8 @@ def test_check_parameters_for_pcmdi(): "cmip_enso_dir": "", "cmip_clim_dir": "", "cmip_movs_dir": "", + "enso_viewer": False, + "clim_viewer": True, "diagnostics_base_path": "diags/post", "infer_path_parameters": False, } @@ -227,11 +232,14 @@ def test_check_parameters_for_pcmdi(): "cmip_enso_dir": "", "cmip_clim_dir": "", "cmip_movs_dir": "", + "enso_viewer": False, + "clim_viewer": True, + "mova_viewer": True, "diagnostics_base_path": "diags/post", "infer_path_parameters": True, } check_parameters_for_pcmdi(c) - assert c["cmip_enso_dir"] == "diags/post/pcmdi_data/metrics_data/enso_metric" + assert c["cmip_enso_dir"] == "placeholder_dir" # Test when parameters are already defined c = { @@ -240,6 +248,9 @@ def test_check_parameters_for_pcmdi(): "cmip_enso_dir": "/custom/enso", "cmip_clim_dir": "/custom/clim", "cmip_movs_dir": "/custom/movs", + "enso_viewer": False, + "clim_viewer": True, + "mova_viewer": True, "diagnostics_base_path": "diags/post", "infer_path_parameters": True, } @@ -359,6 +370,7 @@ def test_check_and_define_parameters(): "reference_data_path": "/ref/path/post/analysis", "grid": "180x360_aave", "infer_path_parameters": True, + "clim_viewer": True, } check_and_define_parameters(c) assert ( @@ -434,6 +446,10 @@ def test_add_pcmdi_dependencies(mock_exists): "year1": 2000, "year2": 2010, "figure_sets": ["mean_climate", "variability_modes", "enso_metric"], + "clim_viewer": True, + "mova_viewer": True, + "movc_viewer": True, + "enso_viewer": False, } script_dir = "/scripts" @@ -446,7 +462,6 @@ def test_add_pcmdi_dependencies(mock_exists): "/scripts/pcmdi_diags_mean_climate_model_vs_obs_2000-2010.status", "/scripts/pcmdi_diags_variability_modes_cpl_model_vs_obs_2000-2010.status", "/scripts/pcmdi_diags_variability_modes_atm_model_vs_obs_2000-2010.status", - "/scripts/pcmdi_diags_enso_model_vs_obs_2000-2010.status", ] assert set(dependencies) == set(expected_dependencies) @@ -459,6 +474,10 @@ def test_add_pcmdi_dependencies(mock_exists): "ref_year1": 1850, "ref_year2": 1900, "figure_sets": ["mean_climate"], + "clim_viewer": True, + "enso_viewer": False, + "mova_viewer": False, + "movc_viewer": False, } add_pcmdi_dependencies(c, dependencies, script_dir) @@ -474,6 +493,10 @@ def test_add_pcmdi_dependencies(mock_exists): "year1": 2000, "year2": 2010, "figure_sets": ["mean_climate"], + "clim_viewer": True, + "enso_viewer": False, + "mova_viewer": False, + "movc_viewer": False, } mock_exists.return_value = False @@ -496,6 +519,10 @@ def test_add_pcmdi_dependencies(mock_exists): "year1": 2000, "year2": 2010, "figure_sets": ["mean_climate"], + "clim_viewer": True, + "enso_viewer": False, + "mova_viewer": False, + "movc_viewer": False, } mock_exists.return_value = True From 34f61e2cccbd032fbb18ac1e61668371a0764c81 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 6 Oct 2025 13:32:04 -0500 Subject: [PATCH 39/47] Organize default.ini --- zppy/defaults/default.ini | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index 844111ca..1945615e 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -582,29 +582,19 @@ cmip_movs_set = string(default="cmip6.historical.v20220825") ref_final_yr = string(default=None) ref_start_yr = string(default=None) ref_years = string_list(default=None) - clim_viewer = string(default=None) clim_vars = string(default=None) - clim_regions = string(default=None) - clim_years = string(default=None) cmip_vars = string(default=None) - mova_vars = string(default=None) - movc_vars = string(default=None) - mova_years = string(default=None) - movc_years = string(default=None) - mova_modes = string(default=None) - movc_modes = string(default=None) - mova_viewer = string(default=None) - movc_viewer = string(default=None) - enso_viewer = string(default=None) - enso_vars = string(default=None) - enso_years = string(default=None) - enso_groups = string(default=None) + clim_regions = string(default=None) target_grid = string(default=None) target_grid_string = string(default=None) regrid_tool = string(default=None) regrid_tool_ocn = string(default=None) regrid_method = string(default=None) regrid_method_ocn = string(default=None) + mova_vars = string(default=None) + mova_modes = string(default=None) + movc_vars = string(default=None) + movc_modes = string(default=None) CBF = string(default=None) ConvEOF = string(default=None) eofn_mod_max = integer(default=None) @@ -615,6 +605,17 @@ cmip_movs_set = string(default="cmip6.historical.v20220825") ObsUnitsAdjust = string(default=None) RmDomainMean = string(default=None) seasons = string(default=None) + enso_vars = string(default=None) + enso_groups = string(default=None) + clim_viewer = string(default=None) + clim_years = string(default=None) + mova_viewer = string(default=None) + mova_years = string(default=None) + movc_viewer = string(default=None) + movc_years = string(default=None) + enso_viewer = string(default=None) + enso_years = string(default=None) + save_all_data = string(default=None) reference_alias = string(default=None) regions_specs = string(default=None) pcmdi_version = string(default=None) From d3618b1202c173031f13556e9415d344c48d4adb Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 6 Oct 2025 13:56:27 -0500 Subject: [PATCH 40/47] Better organize default.ini --- zppy/defaults/default.ini | 61 ++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index 1945615e..e7f2ac6e 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -459,17 +459,10 @@ ref_years = string_list(default=list("")) ########################################################################################### # Specify these for the mean_climate subtask ########################################################################################### -# Variables to be used by the pcmdi diagnostics -# mean_climate AND synthetic_plots: clim_vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" -clim_vars = string(default="pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500") # variables in the cmip6 table that can be potentially used by pcmdi # this list depends on the definition of cmip variable # required for "mean climate" diagnostics cmip_vars = string(default="pr,prw,psl,rlds,rldscs,rlut,rlutcs,rsut,rsutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rtmt,sfcWind,tas,tauu,tauv,ts,ta,ua,va,zg") -#default regions for mean climate metrics data -#more options can be found at "regions_specs" -#regions = string(default="global,ocean,land,NHEX,SHEX,TROPICS,NHEX_ocean,SHEX_ocean,NHEX_land,SHEX_land,ocean_50S50N") -clim_regions = string(default="global,ocean,land") # Regridding by pcmdi (required for mean climate) # OPTIONS: '1x1' for e3sm, '2.5x2.5' for cmip, or an actual cdms2 grid object, target_grid = string(default="1x1") @@ -483,15 +476,18 @@ regrid_method = string(default="regrid2") # OPTIONS: 'linear','conservative', only if tool is esmf regrid_method_ocn = string(default="conservative") ########################################################################################### +# Specify these for the mean_climate subtask AND the synthetic_plots subtask +########################################################################################### +# Variables to be used by the pcmdi diagnostics +# mean_climate AND synthetic_plots: clim_vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" +clim_vars = string(default="pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500") +#default regions for mean climate metrics data +#more options can be found at "regions_specs" +#regions = string(default="global,ocean,land,NHEX,SHEX,TROPICS,NHEX_ocean,SHEX_ocean,NHEX_land,SHEX_land,ocean_50S50N") +clim_regions = string(default="global,ocean,land") +########################################################################################### # Specify these for the variability_modes_cpl and variability_modes_atm subtasks ########################################################################################### -### 2 options for names of atm, cpl modes varibility: -# variability_modes_atm: vars = "psl" -mova_vars = string(default="psl") -mova_modes = string(default="NAM,NAO,PNA,NPO,SAM,PSA1,PSA2") -# variability_modes_cpl: vars = "ts" -movc_vars = string(default="ts") -movc_modes = string(default="PDO,NPGO,AMO") CBF = string(default=True) ConvEOF = string(default=True) eofn_mod_max = integer(default=3) @@ -506,11 +502,26 @@ ObsUnitsAdjust = string(default="") RmDomainMean = string(default=True) seasons = string(default="monthly") ########################################################################################### -# Specify these for the enso subtasks (not currently enabled) +# Specify these for the variability_modes_atm subtask AND the synthetic_plots subtask +########################################################################################### +# variability_modes_atm: vars = "psl" +mova_vars = string(default="psl") +mova_modes = string(default="NAM,NAO,PNA,NPO,SAM,PSA1,PSA2") +########################################################################################### +# Specify these for the variability_modes_cpl subtask AND the synthetic_plots subtask +########################################################################################### +# variability_modes_cpl: vars = "ts" +movc_vars = string(default="ts") +movc_modes = string(default="PDO,NPGO,AMO") +########################################################################################### +# Specify these for the enso subtask (not currently enabled) +########################################################################################### +enso_groups = string(default="ENSO_perf,ENSO_proc,ENSO_tel") +########################################################################################### +# Specify these for the enso subtask AND the synthetic_plots subtask ########################################################################################### # enso: vars = "psl,pr,prsn,ts,tas,tauu,tauv,hflx,hfss,rlds,rsds,rlus,rlut,rsdt" enso_vars = string(default="psl,pr,prsn,ts,tas,tauu,tauv,hflx,hfss,rlds,rsds,rlus,rlut,rsdt") -enso_groups = string(default="ENSO_perf,ENSO_proc,ENSO_tel") ########################################################################################### # Specify these for the synthetic_plots subtask ########################################################################################### @@ -549,10 +560,6 @@ cmip_movs_dir = string(default="") cmip_clim_set = string(default="cmip6.historical.v20250707") cmip_enso_set = string(default="cmip6.historical.v20210620") cmip_movs_set = string(default="cmip6.historical.v20220825") -# TODO: Match "many" section to above -# TODO: Update example cfg accordingly -# TODO: Commit -# TODO: Run example cfg [[__many__]] figure_format = string(default=None) @@ -583,18 +590,14 @@ cmip_movs_set = string(default="cmip6.historical.v20220825") ref_start_yr = string(default=None) ref_years = string_list(default=None) clim_vars = string(default=None) - cmip_vars = string(default=None) - clim_regions = string(default=None) target_grid = string(default=None) target_grid_string = string(default=None) regrid_tool = string(default=None) regrid_tool_ocn = string(default=None) regrid_method = string(default=None) regrid_method_ocn = string(default=None) - mova_vars = string(default=None) - mova_modes = string(default=None) - movc_vars = string(default=None) - movc_modes = string(default=None) + cmip_vars = string(default=None) + clim_regions = string(default=None) CBF = string(default=None) ConvEOF = string(default=None) eofn_mod_max = integer(default=None) @@ -605,8 +608,12 @@ cmip_movs_set = string(default="cmip6.historical.v20220825") ObsUnitsAdjust = string(default=None) RmDomainMean = string(default=None) seasons = string(default=None) - enso_vars = string(default=None) + mova_vars = string(default=None) + mova_modes = string(default=None) + movc_vars = string(default=None) + movc_modes = string(default=None) enso_groups = string(default=None) + enso_vars = string(default=None) clim_viewer = string(default=None) clim_years = string(default=None) mova_viewer = string(default=None) From 7473405d5d29ef422e9ed53dbab2d04e83377ea1 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 6 Oct 2025 14:43:20 -0500 Subject: [PATCH 41/47] Organize parameters in cfgs --- examples/post.v3.LR.amip.0101.cfg | 72 ++++----- examples/post.v3.LR.amip.0101.zi_setup.cfg | 137 ------------------ examples/post.v3.LR.historical.0101.cfg | 65 +++------ ...test_weekly_comprehensive_v2_chrysalis.cfg | 2 +- ...test_weekly_comprehensive_v3_chrysalis.cfg | 49 +++---- ...egacy_3.0.0_comprehensive_v2_chrysalis.cfg | 2 +- ...egacy_3.0.0_comprehensive_v3_chrysalis.cfg | 2 +- .../template_weekly_comprehensive_v3.cfg | 47 +++--- zppy/defaults/default.ini | 1 + 9 files changed, 100 insertions(+), 277 deletions(-) delete mode 100644 examples/post.v3.LR.amip.0101.zi_setup.cfg diff --git a/examples/post.v3.LR.amip.0101.cfg b/examples/post.v3.LR.amip.0101.cfg index a7c1405e..3a07a1ac 100644 --- a/examples/post.v3.LR.amip.0101.cfg +++ b/examples/post.v3.LR.amip.0101.cfg @@ -1,14 +1,18 @@ +# Directions to run: +# 1. Update , , below. +# 2. Run with `zppy -c examples/post.v3.LR.amip.0101.cfg`. +# Direction to create stand-alone test data for zppy-interfaces: +# 3. Once the jobs finish, `cd /post/scripts`. +# 4. Run `grep -n "Running a zi-pcmdi command" pcmdi_diags*.o*` to find the pcmdi_diags commands. +# 5. Then, you can run those lines stand-alone. [default] input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/AMIP/v3.LR.amip_0101 -output = /lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy_pr719_output/unique_id_56/v3.LR.amip_0101 -www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.szhang/zppy_pr719_output/unique_id_56 +output = /lcrc/group/e3sm/ac.forsyth2/zppy_pmp_amip/unique_id_20251006_1/v3.LR.amip_0101 +www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pmp_amip/unique_id_20251006_1 campaign = "water_cycle" case = v3.LR.amip_0101 debug = False environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" -#partition = debug #compute -partition = priority -account = priority [ts] active = True @@ -42,12 +46,8 @@ years = "2005:2015:5" [pcmdi_diags] active = True -environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" -environment_commands_secondary = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh; conda activate zi-pcmdi-diags-20250730" -nodes = 1 +environment_commands_secondary = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20251006" walltime = "4:00:00" -figure_format = "png" -run_type = "model_vs_obs" # Create rules to construct cmip-like datasets with a string constructed as # "mip.exp.product.realization", where "."is used as the delimiter (mandatory) model_name = 'e3sm.amip.v3-LR.0101' @@ -66,6 +66,8 @@ ts_years = "2005-2014", ref_final_yr = 2018 ref_start_yr = 1979 ref_years = "2005-2014", + target_grid = "2.5x2.5" + target_grid_string = "2p5x2p5" # Variables in current pcmdi-cmip list # rltcre and rstcre are derived variables as they are not included in cmip_ts # define a new variable name to be flexible @@ -73,8 +75,6 @@ ts_years = "2005-2014", # See zppy/templates/pcmdi_diags/regions_specs.json for options of all regions # Below is default setup for global/regional mean climate metrics clim_regions = "global,ocean,land,Tropics,NHEX,SHEX" - target_grid = "2.5x2.5" - target_grid_string = "2p5x2p5" [[ variability_modes_cpl ]] active = True @@ -83,15 +83,14 @@ ts_years = "2005-2014", ref_final_yr = 2019 ref_start_yr = 1870 ref_years = "2005-2014", - movc_vars = "ts" - movc_years = "2005-2014" - # Name of modes varibility - movc_modes = "PDO,NPGO,AMO" frequency = 'mo' landmask = True ModUnitsAdjust = '(True,"subtract",273.15)' ObsUnitsAdjust = '(True,"subtract",273.15)' seasons = "yearly,monthly" + movc_vars = "ts" + # Name of modes varibility + movc_modes = "PDO,NPGO,AMO" [[ variability_modes_atm ]] active = True @@ -101,15 +100,14 @@ ts_years = "2005-2014", ref_final_yr = 2015 ref_start_yr = 1853 ref_years = "2005-2014", - mova_vars = "psl" - mova_years = "2005-2014" - # Varibility modes - mova_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" frequency = 'mo' landmask = False ModUnitsAdjust = '(True,"divide",100.0)' ObsUnitsAdjust = '(True,"divide",100.0)' seasons = "DJF,MAM,JJA,SON,yearly,monthly" + mova_vars = "psl" + # Varibility modes + mova_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" [[ enso ]] active = False @@ -119,53 +117,45 @@ ts_years = "2005-2014", ref_final_yr = 2018 ref_start_yr = 1979 ref_years = "2005-2014", - enso_vars = "psl,pr,prsn,ts,tas,tauu,tauv,hfls,hfss,rlds,rsds,rlus,rlut,rsdt" - enso_years = "2005-2014" # See zppy/templates/pcmdi_diags/reference_alias.json for options enso_groups = "ENSO_perf,ENSO_proc,ENSO_tel" + enso_vars = "psl,pr,prsn,ts,tas,tauu,tauv,hfls,hfss,rlds,rsds,rlus,rlut,rsdt" [[ synthetic_plots ]] active = True - - # mean climate - clim_viewer = True - # use subset regions for viewer - clim_regions = "global,ocean,land" # use subset variables for viewer clim_vars = "pr,prw,psl,rlds,rldscs,ua-200,zg-500" + # use subset regions for viewer + clim_regions = "global,ocean,land" + # use subset modes for viewer + mova_modes = "NAM,NAO,PNA,NPO,SAM" + # use subset modes for viewer + movc_modes = "PDO,NPGO" + # mean climate + clim_viewer = True # period to be plotted and shown on webpage # note that the previous sections allows multiple ts_years periods for processing, while # the viewer page focused on period customized by user clim_years = "2005-2014" - - # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/mean_climate/cmip6/amip - cmip_clim_set = "cmip6.amip.v20250702" - # modes variability (atmosphere) mova_viewer = True - # use subset modes for viewer - mova_modes = "NAM,NAO,PNA,NPO,SAM" # period to be plotted and shown on webpage # note that the previous sections allows multiple ts_years periods for processing, while # the viewer page focused on period customized by user mova_years = "2005-2014" - # modes variability (coupled) movc_viewer = True - # use subset modes for viewer - movc_modes = "PDO,NPGO" # period to be plotted and shown on webpage # note that the previous sections allows multiple ts_years periods for processing, while # the viewer page focused on period customized by user movc_years = "2005-2014" - - # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes/cmip6/amip - cmip_movs_set = "cmip6.amip.v20210119" - # enso enso_viewer = False enso_years = "" - + # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/mean_climate/cmip6/amip + cmip_clim_set = "cmip6.amip.v20250702" + # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes/cmip6/amip + cmip_movs_set = "cmip6.amip.v20210119" # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/enso_metric/cmip6/historical # There is no enso metric for CMIP6 AMIP, so use historical instead cmip_enso_set = "cmip6.historical.v20210620" diff --git a/examples/post.v3.LR.amip.0101.zi_setup.cfg b/examples/post.v3.LR.amip.0101.zi_setup.cfg deleted file mode 100644 index fc389344..00000000 --- a/examples/post.v3.LR.amip.0101.zi_setup.cfg +++ /dev/null @@ -1,137 +0,0 @@ -# Modify this cfg to generate the necessary data for stand-alone tests of zppy-interfaces -# 1. Update , , below. -# 2. Run with `zppy -c examples/post.v3.LR.amip.0101.zi_setup.cfg`. -# 3. Once the jobs finish, `cd /post/scripts`. -# 4. Run `grep -n "About to run:" pcmdi_diags*.o*` to find the pcmdi_diags commands. -# 5. Then, you can run those lines stand-alone. -[default] -input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/AMIP/v3.LR.amip_0101 -output = /lcrc/group/e3sm/ac.forsyth2/zi_pcmdi_stand_alone_setup/unique_id_1/v3.LR.amip_0101 -www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zi_pcmdi_stand_alone_setup/unique_id_1 -campaign = "water_cycle" -case = v3.LR.amip_0101 -debug = False -environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" -partition = debug #compute - -[ts] -active = True -walltime = "00:10:00" -years = "2005:2015:5" - - [[ atm_monthly_180x360_aave ]] - input_subdir = "archive/atm/hist" - input_files = "eam.h0" - frequency = "monthly" - mapping_file = /home/ac.zender/data/maps/map_ne30pg2_to_cmip6_180x360_aave.20200201.nc - # list for pcmdi diag, note: PHIS,hyam,hybm,hyai,hybi need to be included to process the 3D fields - vars = "ICEFRAC,LANDFRAC,OCNFRAC,PSL,FSNTC,FSNTOAC,SWCF,LWCF,FLUT,FSNT,FSNTOA,FLNT,FLNTC,FSNS,FLNS,FSNS,SHFLX,QFLX,LHFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,U10,QREFHT,TMQ,CLDTOT,CLDHGH,CLDMED,CLDLOW,FLDS,FSDS,TGCLDIWP,TGCLDCWP,TGCLDLWP,FLNSC,FLUTC,FSDSC,SOLIN,FSNSC,AODABS,AODVIS,AODDUST,AREL,TREFMNAV,TREFMXAV,PS,PHIS,U,V,T,Z3" - -[e3sm_to_cmip] -active = True -frequency = "monthly" -ts_grid = "180x360_aave" -ts_num_years=5 -walltime = "00:10:00" -years = "2005:2015:5" - - [[ atm_monthly_180x360_aave ]] - # list of variables to be interpolated to pressure level; see e3sm_to_cmip/vrt_remap_plev19.nc - # This defaults to cmip_plevdata = string(default="inclusions/e3sm_to_cmip/vrt_remap_plev19.nc") - # cmip_plevdata = /lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc - input_files = "eam.h0" - interp_vars = "U,V,T,Z3" - ts_subsection = "atm_monthly_180x360_aave" - vars = "ICEFRAC,LANDFRAC,OCNFRAC,PSL,FSNTC,FSNTOAC,SWCF,LWCF,FLUT,FSNT,FSNTOA,FLNT,FLNTC,FSNS,FLNS,FSNS,SHFLX,QFLX,LHFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,U10,QREFHT,TMQ,CLDTOT,CLDHGH,CLDMED,CLDLOW,FLDS,FSDS,TGCLDIWP,TGCLDCWP,TGCLDLWP,FLNSC,FLUTC,FSDSC,SOLIN,FSNSC,AODABS,AODVIS,AODDUST,AREL,TREFMNAV,TREFMXAV,PS,PHIS,U,V,T,Z3" - -[pcmdi_diags] -active = True -environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" -environment_commands_secondary = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20250930" -nodes = 1 -walltime = "4:00:00" -figure_format = "png" -run_type = "model_vs_obs" -# Create rules to construct cmip-like datasets with a string constructed as -# "mip.exp.product.realization", where "."is used as the delimiter (mandatory) -model_name = 'e3sm.amip.v3-LR.0101' -model_tableID = 'Amon' -multiprocessing = True -num_workers = 24 -# Observational data -obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series' -pcmdi_debug = False -ts_years = "2005-2014", - - [[ mean_climate ]] - active = True - # Default for all variables or specify source for each variable separately. - obs_sets = "default" - ref_final_yr = 2018 - ref_start_yr = 1979 - ref_years = "2005-2014", - # Variables in current pcmdi-cmip list - # rltcre and rstcre are derived variables as they are not included in cmip_ts - vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rsus,rsuscs,rlus,rlut,rlutcs,rsds,rsdscs,rsdt,rsut,rsutcs,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" - # See zppy/templates/pcmdi_diags/regions_specs.json for options of all regions - # Below is default setup for global/regional mean climate metrics - regions = "global,ocean,land,NHEX,SHEX,TROPICS" - target_grid = "2.5x2.5" - target_grid_string = "2p5x2p5" - - [[ variability_modes_cpl ]] - active = True - # See zppy/templates/pcmdi_diags/reference_alias.json for options - obs_sets = "alternate1" #"default" - ref_final_yr = 2019 - ref_start_yr = 1870 - ref_years = "2005-2014", - vars = "ts" - # Name of modes varibility - cpl_modes = "PDO,NPGO,AMO" - frequency = 'mo' - landmask = True - ModUnitsAdjust = '(True,"subtract",273.15)' - ObsUnitsAdjust = '(True,"subtract",273.15)' - seasons = "yearly,monthly" - - [[ variability_modes_atm ]] - active = True - # See zppy/templates/pcmdi_diags/reference_alias.json for options - # Default for all variables - obs_sets = "alternate2" # "default" - ref_final_yr = 2015 - ref_start_yr = 1853 - ref_years = "2005-2014", - vars = "psl" - # Varibility modes - atm_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" - frequency = 'mo' - landmask = False - ModUnitsAdjust = '(True,"divide",100.0)' - ObsUnitsAdjust = '(True,"divide",100.0)' - seasons = "DJF,MAM,JJA,SON,yearly,monthly" - - [[ enso ]] - active = False - # For enso, the obs_sets is explicitely set for each variable to ensure that - # they are from the same source, below are setup to used ERA5 reanalysis. - obs_sets = "default,alternate2,default,default,default,default,default,default,default,alternate2,alternate2,alternate2,alternate2,alternate2" - ref_final_yr = 2018 - ref_start_yr = 1979 - ref_years = "2005-2014", - vars = "psl,pr,prsn,ts,tas,tauu,tauv,hfls,hfss,rlds,rsds,rlus,rlut,rsdt" - # See zppy/templates/pcmdi_diags/reference_alias.json for options - enso_groups = "ENSO_perf,ENSO_proc,ENSO_tel" - - [[ synthetic_plots ]] - active = True - # clim_period, emov_period, enso_period: - figure_sets_period = "2005-2014,2005-2014,2005-2014" - # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/mean_climate/cmip6/amip - cmip_clim_set = "cmip6.amip.v20250702" - # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes/cmip6/amip - cmip_movs_set = "cmip6.amip.v20210119" - # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/enso_metric/cmip6/historical - # There is no enso metric for CMIP6 AMIP, so use historical instead - cmip_enso_set = "cmip6.historical.v20210620" diff --git a/examples/post.v3.LR.historical.0101.cfg b/examples/post.v3.LR.historical.0101.cfg index e1ec99b6..ed261a76 100644 --- a/examples/post.v3.LR.historical.0101.cfg +++ b/examples/post.v3.LR.historical.0101.cfg @@ -1,14 +1,11 @@ [default] input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/v3.LR.historical_0101 -output = /lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy_pr719_output/unique_id_56/v3.LR.historical_0101 -www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.szhang/zppy_pr719_output/unique_id_56 +output = /lcrc/group/e3sm/ac.forsyth2/zppy_pmp_hist/unique_id_20251006_1/v3.LR.amip_0101 +www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pmp_hist/unique_id_20251006_1 campaign = "water_cycle" case = v3.LR.historical_0101 debug = False environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" -#partition = debug #compute -partition = priority -account = priority [ts] active = True @@ -42,12 +39,8 @@ years = "2005:2015:5" [pcmdi_diags] active = True -environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" -environment_commands_secondary = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh; conda activate zi-pcmdi-diags-20250730" -nodes = 1 +environment_commands_secondary = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20251006" walltime = "4:00:00" -figure_format = "png" -run_type = "model_vs_obs" # Create rules to construct cmip-like datasets with a string constructed as # "mip.exp.product.realization", where "."is used as the delimiter (mandatory) model_name = 'e3sm.historical.v3-LR.0101' @@ -66,6 +59,8 @@ ts_years = "2005-2014", ref_final_yr = 2018 ref_start_yr = 1979 ref_years = "2005-2014", + target_grid = "2.5x2.5" + target_grid_string = "2p5x2p5" # Variables in current pcmdi-cmip list # rltcre and rstcre are derived variables as they are not included in cmip_ts # define a new variable name to be flexible @@ -73,8 +68,6 @@ ts_years = "2005-2014", # See zppy/templates/pcmdi_diags/regions_specs.json for options of all regions # Below is default setup for global/regional mean climate metrics clim_regions = "global,ocean,land,Tropics,NHEX,SHEX" - target_grid = "2.5x2.5" - target_grid_string = "2p5x2p5" [[ variability_modes_cpl ]] active = True @@ -83,15 +76,14 @@ ts_years = "2005-2014", ref_final_yr = 2019 ref_start_yr = 1870 ref_years = "2005-2014", - movc_vars = "ts" - movc_years = "2005-2014" - # Name of modes varibility - movc_modes = "PDO,NPGO,AMO" frequency = 'mo' landmask = True ModUnitsAdjust = '(True,"subtract",273.15)' ObsUnitsAdjust = '(True,"subtract",273.15)' seasons = "yearly,monthly" + movc_vars = "ts" + # Name of modes varibility + movc_modes = "PDO,NPGO,AMO" [[ variability_modes_atm ]] active = True @@ -101,15 +93,14 @@ ts_years = "2005-2014", ref_final_yr = 2015 ref_start_yr = 1853 ref_years = "2005-2014", - mova_vars = "psl" - mova_years = "2005-2014" - # Varibility modes - mova_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" frequency = 'mo' landmask = False ModUnitsAdjust = '(True,"divide",100.0)' ObsUnitsAdjust = '(True,"divide",100.0)' seasons = "DJF,MAM,JJA,SON,yearly,monthly" + mova_vars = "psl" + # Varibility modes + mova_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" [[ enso ]] active = False @@ -119,53 +110,45 @@ ts_years = "2005-2014", ref_final_yr = 2018 ref_start_yr = 1979 ref_years = "2005-2014", - enso_vars = "psl,pr,prsn,ts,tas,tauu,tauv,hfls,hfss,rlds,rsds,rlus,rlut,rsdt" - enso_years = "2005-2014" # See zppy/templates/pcmdi_diags/reference_alias.json for options enso_groups = "ENSO_perf,ENSO_proc,ENSO_tel" + enso_vars = "psl,pr,prsn,ts,tas,tauu,tauv,hfls,hfss,rlds,rsds,rlus,rlut,rsdt" [[ synthetic_plots ]] active = True - - # mean climate - clim_viewer = True - # use subset regions for viewer - clim_regions = "global,ocean,land" # use subset variables for viewer clim_vars = "pr,prw,psl,rlds,rldscs,ua-200,zg-500" + # use subset regions for viewer + clim_regions = "global,ocean,land" + # use subset modes for viewer + mova_modes = "NAM,NAO,PNA,NPO,SAM" + # use subset modes for viewer + movc_modes = "PDO,NPGO" + # mean climate + clim_viewer = True # period to be plotted and shown on webpage # note that the previous sections allows multiple ts_years periods for processing, while # the viewer page focused on period customized by user clim_years = "2005-2014" - - # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/mean_climate/cmip6/historical - cmip_clim_set = "cmip6.historical.v20250707" - # modes variability (atmosphere) mova_viewer = True - # use subset modes for viewer - mova_modes = "NAM,NAO,PNA,NPO,SAM" # period to be plotted and shown on webpage # note that the previous sections allows multiple ts_years periods for processing, while # the viewer page focused on period customized by user mova_years = "2005-2014" - # modes variability (coupled) movc_viewer = True - # use subset modes for viewer - movc_modes = "PDO,NPGO" # period to be plotted and shown on webpage # note that the previous sections allows multiple ts_years periods for processing, while # the viewer page focused on period customized by user movc_years = "2005-2014" - - # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes/cmip6/historical - cmip_movs_set = "cmip6.historical.v20220825" - # enso enso_viewer = False enso_years = "" - + # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/mean_climate/cmip6/historical + cmip_clim_set = "cmip6.historical.v20250707" + # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes/cmip6/historical + cmip_movs_set = "cmip6.historical.v20220825" # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/enso_metric/cmip6/historical # There is no enso metric for CMIP6 AMIP, so use historical instead cmip_enso_set = "cmip6.historical.v20210620" diff --git a/tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg b/tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg index 9c7ae047..12fa232c 100644 --- a/tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg +++ b/tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg @@ -176,7 +176,7 @@ partition = "compute" qos = "regular" shortTermArchive = True ts_years = "1980-1984", "1980-1990", -walltime = "02:00:00" +walltime = "00:30:00" [global_time_series] active = True diff --git a/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg b/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg index e7085143..94fd6fd9 100644 --- a/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg +++ b/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg @@ -207,7 +207,7 @@ partition = "compute" qos = "regular" shortTermArchive = True ts_years = "1985-1989", "1985-1995", -walltime = "02:00:00" +walltime = "00:30:00" [global_time_series] active = True @@ -313,16 +313,11 @@ years = "1985:1989:4" active = True environment_commands_secondary = "source /conda.sh; conda activate " walltime = "2:00:00" -figure_format = "png" -run_type = "model_vs_obs" -# Create rules to construct cmip-like datasets with a string constructed as -# "mip.exp.product.realization", where "."is used as the delimiter (mandatory) model_name = 'e3sm.historical.v3-LR.0051' model_tableID = 'Amon' e3sm_to_cmip_atm_subsection = "atm_monthly_180x360_aave" # Can be inferred if needed multiprocessing = True num_workers = 24 -# Observational data obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series' pcmdi_debug = False ts_num_years = 2 @@ -331,61 +326,59 @@ ts_years = "1985-1994", [[ mean_climate ]] active = True current_set = "mean_climate" # Can be inferred if needed - # Default for all variables or specify source for each variable separately. obs_sets = "default" ref_final_yr = 1994 ref_start_yr = 1985 ref_years = "1985-1994", - # Variables in current pcmdi-cmip list - # rltcre and rstcre are derived variables as they are not included in cmip_ts - vars = "pr,rlut,rtmt,tas,tauu,tauv,ts" - #vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rsus,rsuscs,rlus,rlut,rlutcs,rsds,rsdscs,rsdt,rsut,rsutcs,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" - # See zppy/templates/pcmdi_diags/regions_specs.json for options of all regions - # Below is default setup for global/regional mean climate metrics - regions = "global,ocean,land,NHEX,SHEX,TROPICS" target_grid = "2.5x2.5" target_grid_string = "2p5x2p5" + clim_vars = "pr,rlut,rtmt,tas,tauu,tauv,ts" + clim_regions = "global,ocean,land,NHEX,SHEX,TROPICS" [[ variability_modes_cpl ]] active = True current_set = "variability_modes_cpl" # Can be inferred if needed - # See zppy/templates/pcmdi_diags/reference_alias.json for options - obs_sets = "alternate1" #"default" + obs_sets = "alternate1" ref_final_yr = 1994 ref_start_yr = 1985 ref_years = "1985-1994", - vars = "ts" - # Name of modes varibility - cpl_modes = "PDO,NPGO,AMO" frequency = 'mo' landmask = True ModUnitsAdjust = '(True,"subtract",273.15)' ObsUnitsAdjust = '(True,"subtract",273.15)' seasons = "yearly,monthly" + movc_vars = "ts" + movc_modes = "PDO,NPGO,AMO" [[ variability_modes_atm ]] active = True current_set = "variability_modes_atm" # Can be inferred if needed - # See zppy/templates/pcmdi_diags/reference_alias.json for options - # Default for all variables - obs_sets = "alternate2" # "default" + obs_sets = "alternate2" ref_final_yr = 1994 ref_start_yr = 1985 ref_years = "1985-1994", - vars = "psl" - # Varibility modes - atm_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" frequency = 'mo' landmask = False ModUnitsAdjust = '(True,"divide",100.0)' ObsUnitsAdjust = '(True,"divide",100.0)' seasons = "DJF,MAM,JJA,SON,yearly,monthly" + mova_vars = "psl" + mova_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" [[ synthetic_plots ]] active = True current_set = "synthetic_plots" # Can be inferred if needed - vars = "pr,rlut,rtmt,tas,tauu,tauv,ts" # MUST match mean_climate's vars - # clim_period, emov_period, enso_period: - figure_sets_period = "1985-1994,1985-1994,1985-1994" + clim_vars = "pr,rlut,rtmt,tas,tauu,tauv,ts" + clim_regions = "global,ocean,land,NHEX,SHEX,TROPICS" + mova_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" + movc_modes = "PDO,NPGO,AMO" + clim_viewer = True + clim_years = "2005-2014" + mova_viewer = True + mova_years = "2005-2014" + movc_viewer = True + movc_years = "2005-2014" + enso_viewer = False + enso_years = "" cmip_clim_dir = /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/mean_climate cmip_movs_dir = /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes diff --git a/tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v2_chrysalis.cfg b/tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v2_chrysalis.cfg index 60568c98..955ac524 100644 --- a/tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v2_chrysalis.cfg +++ b/tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v2_chrysalis.cfg @@ -176,7 +176,7 @@ partition = "compute" qos = "regular" shortTermArchive = True ts_years = "1980-1984", "1980-1990", -walltime = "02:00:00" +walltime = "00:30:00" [global_time_series] active = True diff --git a/tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v3_chrysalis.cfg b/tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v3_chrysalis.cfg index 0fb66fbd..94fe4054 100644 --- a/tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v3_chrysalis.cfg +++ b/tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v3_chrysalis.cfg @@ -206,7 +206,7 @@ partition = "compute" qos = "regular" shortTermArchive = True ts_years = "1985-1989", "1985-1995", -walltime = "02:00:00" +walltime = "00:30:00" [global_time_series] active = True diff --git a/tests/integration/template_weekly_comprehensive_v3.cfg b/tests/integration/template_weekly_comprehensive_v3.cfg index 06bd64f5..7ace3d3c 100644 --- a/tests/integration/template_weekly_comprehensive_v3.cfg +++ b/tests/integration/template_weekly_comprehensive_v3.cfg @@ -313,16 +313,11 @@ years = "1985:1989:4" active = True environment_commands_secondary = "#expand pcmdi_diags_environment_commands#" walltime = "2:00:00" -figure_format = "png" -run_type = "model_vs_obs" -# Create rules to construct cmip-like datasets with a string constructed as -# "mip.exp.product.realization", where "."is used as the delimiter (mandatory) model_name = 'e3sm.historical.v3-LR.0051' model_tableID = 'Amon' e3sm_to_cmip_atm_subsection = "atm_monthly_180x360_aave" # Can be inferred if needed multiprocessing = True num_workers = 24 -# Observational data obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series' pcmdi_debug = False ts_num_years = 2 @@ -331,61 +326,59 @@ ts_years = "1985-1994", [[ mean_climate ]] active = True current_set = "mean_climate" # Can be inferred if needed - # Default for all variables or specify source for each variable separately. obs_sets = "default" ref_final_yr = 1994 ref_start_yr = 1985 ref_years = "1985-1994", - # Variables in current pcmdi-cmip list - # rltcre and rstcre are derived variables as they are not included in cmip_ts - vars = "pr,rlut,rtmt,tas,tauu,tauv,ts" - #vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rsus,rsuscs,rlus,rlut,rlutcs,rsds,rsdscs,rsdt,rsut,rsutcs,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" - # See zppy/templates/pcmdi_diags/regions_specs.json for options of all regions - # Below is default setup for global/regional mean climate metrics - regions = "global,ocean,land,NHEX,SHEX,TROPICS" target_grid = "2.5x2.5" target_grid_string = "2p5x2p5" + clim_vars = "pr,rlut,rtmt,tas,tauu,tauv,ts" + clim_regions = "global,ocean,land,NHEX,SHEX,TROPICS" [[ variability_modes_cpl ]] active = True current_set = "variability_modes_cpl" # Can be inferred if needed - # See zppy/templates/pcmdi_diags/reference_alias.json for options - obs_sets = "alternate1" #"default" + obs_sets = "alternate1" ref_final_yr = 1994 ref_start_yr = 1985 ref_years = "1985-1994", - vars = "ts" - # Name of modes varibility - cpl_modes = "PDO,NPGO,AMO" frequency = 'mo' landmask = True ModUnitsAdjust = '(True,"subtract",273.15)' ObsUnitsAdjust = '(True,"subtract",273.15)' seasons = "yearly,monthly" + movc_vars = "ts" + movc_modes = "PDO,NPGO,AMO" [[ variability_modes_atm ]] active = True current_set = "variability_modes_atm" # Can be inferred if needed - # See zppy/templates/pcmdi_diags/reference_alias.json for options - # Default for all variables - obs_sets = "alternate2" # "default" + obs_sets = "alternate2" ref_final_yr = 1994 ref_start_yr = 1985 ref_years = "1985-1994", - vars = "psl" - # Varibility modes - atm_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" frequency = 'mo' landmask = False ModUnitsAdjust = '(True,"divide",100.0)' ObsUnitsAdjust = '(True,"divide",100.0)' seasons = "DJF,MAM,JJA,SON,yearly,monthly" + mova_vars = "psl" + mova_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" [[ synthetic_plots ]] active = True current_set = "synthetic_plots" # Can be inferred if needed - vars = "pr,rlut,rtmt,tas,tauu,tauv,ts" # MUST match mean_climate's vars - # clim_period, emov_period, enso_period: - figure_sets_period = "1985-1994,1985-1994,1985-1994" + clim_vars = "pr,rlut,rtmt,tas,tauu,tauv,ts" + clim_regions = "global,ocean,land,NHEX,SHEX,TROPICS" + mova_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" + movc_modes = "PDO,NPGO,AMO" + clim_viewer = True + clim_years = "2005-2014" + mova_viewer = True + mova_years = "2005-2014" + movc_viewer = True + movc_years = "2005-2014" + enso_viewer = False + enso_years = "" cmip_clim_dir = #expand diagnostics_base_path#/pcmdi_data/metrics_data/mean_climate cmip_movs_dir = #expand diagnostics_base_path#/pcmdi_data/metrics_data/variability_modes diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index e7f2ac6e..8d2e5e31 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -525,6 +525,7 @@ enso_vars = string(default="psl,pr,prsn,ts,tas,tauu,tauv,hflx,hfss,rlds,rsds,rlu ########################################################################################### # Specify these for the synthetic_plots subtask ########################################################################################### +# The _years parameters are also defined for the other subtasks, but internally in pcmdi_diags.py. # add mean climate to viewer page (default is False) clim_viewer = string(default=True) clim_years = string(default="") From a9a54c81f987ebd57fa2e79ae7958476e67efc1f Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 6 Oct 2025 14:44:21 -0500 Subject: [PATCH 42/47] Remove duplicate log line file --- examples/remove_duplicate_log_lines.py | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 examples/remove_duplicate_log_lines.py diff --git a/examples/remove_duplicate_log_lines.py b/examples/remove_duplicate_log_lines.py deleted file mode 100644 index 272ebddd..00000000 --- a/examples/remove_duplicate_log_lines.py +++ /dev/null @@ -1,19 +0,0 @@ -# Written with LivChat -def remove_consecutive_duplicates(input_file_path): - output_file_path = input_file_path + ".duplicates_removed" - last_written_line = None - - with ( - open(input_file_path, "r", encoding="utf-8") as infile, - open(output_file_path, "w", encoding="utf-8") as outfile, - ): - for line in infile: - if line != last_written_line: - outfile.write(line) - last_written_line = line - return output_file_path - - -if __name__ == "__main__": - f = "/lcrc/group/e3sm/ac.forsyth2/zppy_pr719_output/unique_id_16/v3.LR.amip_0101/post/scripts/pcmdi_diags_enso_model_vs_obs_2005-2015.o785792" - remove_consecutive_duplicates(f) From 43525929391792859220fd5eb77d64d62050dec6 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 6 Oct 2025 15:53:00 -0500 Subject: [PATCH 43/47] Reduce function complexity --- tests/test_zppy_pcmdi_diags.py | 135 ++++++++++++++++++++++++++++++++- zppy/pcmdi_diags.py | 87 +++++++++++++-------- 2 files changed, 189 insertions(+), 33 deletions(-) diff --git a/tests/test_zppy_pcmdi_diags.py b/tests/test_zppy_pcmdi_diags.py index 98b41240..ef8656b9 100644 --- a/tests/test_zppy_pcmdi_diags.py +++ b/tests/test_zppy_pcmdi_diags.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Tuple from unittest.mock import patch import pytest @@ -11,6 +11,9 @@ check_parameters_for_bash, check_parameters_for_pcmdi, define_current_set, + define_relevant_years, + define_relevant_years_for_synthetic_plots, + define_year_sets, ) from zppy.utils import ParameterNotProvidedError @@ -258,6 +261,49 @@ def test_check_parameters_for_pcmdi(): assert c["cmip_clim_dir"] == "/custom/clim" # Should not change +def test_define_year_sets(): + # Should use ts_years + c = { + "current_set": "mean_climate", + "ts_years": "2000:2010:5", + } + actual: List[Tuple[int, int]] = define_year_sets(c) + expected = [(2000, 2004), (2005, 2009)] + assert actual == expected + + # Should use available year set + c = { + "current_set": "synthetic_plots", + "ts_years": "2000:2010:5", + "enso_years": "2001-2004", + } + actual = define_year_sets(c) + expected = [(2001, 2004)] + assert actual == expected + + # Should use the first available year set + c = { + "current_set": "synthetic_plots", + "ts_years": "2000:2010:5", + "clim_years": "2001-2004", + "mova_years": "2002-2003", + "movc_years": "2003-2004", + "enso_years": "2004-2005", + } + actual = define_year_sets(c) + expected = [(2001, 2004)] + assert actual == expected + + # Should default to ts_years + c = { + "current_set": "synthetic_plots", + "ts_years": "2000:2010:5", + } + actual = define_year_sets(c) + expected = [(2000, 2004), (2005, 2009)] + assert actual == expected + + def test_check_mvm_only_parameters_for_bash(): # Test with all required parameters present c = { @@ -405,6 +451,93 @@ def test_check_and_define_parameters(): assert c["obs_ts"] == "" +def test_define_relevant_years(): + c = { + "current_set": "mean_climate", + "year1": 2000, + "year2": 2010, + } + define_relevant_years(c) + assert c["clim_years"] == "2000-2010" + + c = { + "current_set": "variability_modes_cpl", + "year1": 1990, + "year2": 2000, + } + define_relevant_years(c) + assert c["movc_years"] == "1990-2000" + + c = { + "current_set": "variability_modes_atm", + "year1": 1980, + "year2": 1990, + } + define_relevant_years(c) + assert c["mova_years"] == "1980-1990" + + c = { + "current_set": "enso", + "year1": 1970, + "year2": 1980, + } + define_relevant_years(c) + assert c["enso_years"] == "1970-1980" + + # Test synthetic_plots (should not set any years) + c = { + "current_set": "synthetic_plots", + "year1": 2000, + "year2": 2010, + } + with pytest.raises(ValueError): + define_relevant_years(c) + + +def test_define_relevant_years_for_synthetic_plots(): + c = { + "current_set": "synthetic_plots", + "year1": 2000, + "year2": 2010, + "clim_years": "", + "mova_years": "", + "movc_years": "", + "enso_years": "", + } + define_relevant_years_for_synthetic_plots(c) + assert c["clim_years"] == "2000-2010" + assert c["mova_years"] == "2000-2010" + assert c["movc_years"] == "2000-2010" + assert c["enso_years"] == "2000-2010" + + c = { + "current_set": "synthetic_plots", + "year1": 1990, + "year2": 2000, + "clim_years": "1991-1995", + "mova_years": "", + "movc_years": "", + "enso_years": "", + } + define_relevant_years_for_synthetic_plots(c) + assert c["clim_years"] == "1991-1995" # Should not change + assert c["mova_years"] == "1990-2000" + assert c["movc_years"] == "1990-2000" + assert c["enso_years"] == "1990-2000" + + c = { + "current_set": "mean_climate", + "year1": 2000, + "year2": 2010, + "clim_years": "", + "mova_years": "", + "movc_years": "", + "enso_years": "", + } + with pytest.raises(ValueError): + define_relevant_years_for_synthetic_plots(c) + + def test_add_ts_dependencies(): dependencies: List[str] = [] c = { diff --git a/zppy/pcmdi_diags.py b/zppy/pcmdi_diags.py index 1030d7e5..2ca7ee6f 100644 --- a/zppy/pcmdi_diags.py +++ b/zppy/pcmdi_diags.py @@ -39,8 +39,7 @@ # ----------------------------------------------------------------------------- -# C901 'pcmdi_diags' is too complex (29) -def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): # noqa: C901 +def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): template, _ = initialize_template(config, "pcmdi_diags.bash") @@ -71,19 +70,7 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): # noqa: C9 check_parameters_for_pcmdi(c) # Loop over year sets - year_sets: List[Tuple[int, int]] - if c["current_set"] != "synthetic_plots": - year_sets = get_years(c["ts_years"]) - elif ("clim_years" in c.keys()) and (c["clim_years"] != ""): - year_sets = get_years(c["clim_years"]) - elif ("mova_years" in c.keys()) and (c["mova_years"] != ""): - year_sets = get_years(c["mova_years"]) - elif ("movc_years" in c.keys()) and (c["movc_years"] != ""): - year_sets = get_years(c["movc_years"]) - elif ("enso_years" in c.keys()) and (c["enso_years"] != ""): - year_sets = get_years(c["enso_years"]) - else: - year_sets = get_years(c["ts_years"]) + year_sets: List[Tuple[int, int]] = define_year_sets(c) ref_year_sets: List[Tuple[int, int]] if ("ref_years" in c.keys()) and (c["ref_years"] != [""]): @@ -103,14 +90,7 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): # noqa: C9 if c["current_set"] != "synthetic_plots": check_and_define_parameters(c) print(c["prefix"]) - if c["current_set"] == "mean_climate": - c["clim_years"] = f"{c['year1']}-{c['year2']}" - elif c["current_set"] == "variability_modes_cpl": - c["movc_years"] = f"{c['year1']}-{c['year2']}" - elif c["current_set"] == "variability_modes_atm": - c["mova_years"] = f"{c['year1']}-{c['year2']}" - elif c["current_set"] == "enso": - c["enso_years"] = f"{c['year1']}-{c['year2']}" + define_relevant_years(c) else: prefix = f"pcmdi_diags_{c['sub']}_{c['run_type']}" if i < len(year_sets) - 1: @@ -120,15 +100,7 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): # noqa: C9 else: print(prefix) c["prefix"] = prefix - # assign period for set if empty - if c["clim_years"] == "": - c["clim_years"] = f"{c['year1']}-{c['year2']}" - if c["mova_years"] == "": - c["mova_years"] = f"{c['year1']}-{c['year2']}" - if c["movc_years"] == "": - c["movc_years"] = f"{c['year1']}-{c['year2']}" - if c["enso_years"] == "": - c["enso_years"] = f"{c['year1']}-{c['year2']}" + define_relevant_years_for_synthetic_plots(c) bash_file, settings_file, status_file = get_file_names( script_dir, c["prefix"] @@ -278,6 +250,24 @@ def check_parameters_for_pcmdi(c: Dict[str, Any]) -> None: c["cmip_movs_dir"] = "placeholder_dir" +def define_year_sets(c: Dict[str, Any]) -> List[Tuple[int, int]]: + # Use ts_years unless we are in synthetic plots, in which case, use the appropriate set if defined + year_sets: List[Tuple[int, int]] + if c["current_set"] != "synthetic_plots": + year_sets = get_years(c["ts_years"]) + elif ("clim_years" in c.keys()) and (c["clim_years"] != ""): + year_sets = get_years(c["clim_years"]) + elif ("mova_years" in c.keys()) and (c["mova_years"] != ""): + year_sets = get_years(c["mova_years"]) + elif ("movc_years" in c.keys()) and (c["movc_years"] != ""): + year_sets = get_years(c["movc_years"]) + elif ("enso_years" in c.keys()) and (c["enso_years"] != ""): + year_sets = get_years(c["enso_years"]) + else: + year_sets = get_years(c["ts_years"]) + return year_sets + + def check_mvm_only_parameters_for_bash(c: Dict[str, Any]) -> None: check_parameter_defined(c, "reference_data_path_ts") check_parameter_defined(c, "model_name_ref") @@ -320,6 +310,39 @@ def check_and_define_parameters(c: Dict[str, Any]) -> None: c["prefix"] = prefix +def define_relevant_years(c: Dict[str, Any]) -> None: + if c["current_set"] == "synthetic_plots": + raise ValueError( + "define_relevant_years should not be called for synthetic_plots." + ) + year_str: str = f"{c['year1']}-{c['year2']}" + if c["current_set"] == "mean_climate": + c["clim_years"] = year_str + elif c["current_set"] == "variability_modes_cpl": + c["movc_years"] = year_str + elif c["current_set"] == "variability_modes_atm": + c["mova_years"] = year_str + elif c["current_set"] == "enso": + c["enso_years"] = year_str + + +def define_relevant_years_for_synthetic_plots(c: Dict[str, Any]) -> None: + if c["current_set"] != "synthetic_plots": + raise ValueError( + "define_relevant_years_for_synthetic_plots should only be called for synthetic_plots." + ) + # assign period for set if empty + year_str: str = f"{c['year1']}-{c['year2']}" + if c["clim_years"] == "": + c["clim_years"] = year_str + if c["mova_years"] == "": + c["mova_years"] = year_str + if c["movc_years"] == "": + c["movc_years"] = year_str + if c["enso_years"] == "": + c["enso_years"] = year_str + + def add_ts_dependencies( c: Dict[str, Any], dependencies: List[str], script_dir: str, yr: int ): From cf25104bd9dc77f6147e9f379a1c03f5d17b37b9 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 6 Oct 2025 16:13:08 -0500 Subject: [PATCH 44/47] Use cmip_plevdata from inclusions dir --- examples/post.v3.LR.amip.0101.cfg | 3 --- examples/post.v3.LR.historical.0101.cfg | 3 --- zppy/templates/e3sm_to_cmip.bash | 8 +++++++- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/post.v3.LR.amip.0101.cfg b/examples/post.v3.LR.amip.0101.cfg index 3a07a1ac..a69c3128 100644 --- a/examples/post.v3.LR.amip.0101.cfg +++ b/examples/post.v3.LR.amip.0101.cfg @@ -36,9 +36,6 @@ walltime = "00:10:00" years = "2005:2015:5" [[ atm_monthly_180x360_aave ]] - # list of variables to be interpolated to pressure level; see e3sm_to_cmip/vrt_remap_plev19.nc - # This defaults to cmip_plevdata = string(default="inclusions/e3sm_to_cmip/vrt_remap_plev19.nc") - cmip_plevdata = /lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc input_files = "eam.h0" interp_vars = "U,V,T,Z3" ts_subsection = "atm_monthly_180x360_aave" diff --git a/examples/post.v3.LR.historical.0101.cfg b/examples/post.v3.LR.historical.0101.cfg index ed261a76..451289cb 100644 --- a/examples/post.v3.LR.historical.0101.cfg +++ b/examples/post.v3.LR.historical.0101.cfg @@ -29,9 +29,6 @@ walltime = "00:10:00" years = "2005:2015:5" [[ atm_monthly_180x360_aave ]] - # list of variables to be interpolated to pressure level; see e3sm_to_cmip/vrt_remap_plev19.nc - # This defaults to cmip_plevdata = string(default="inclusions/e3sm_to_cmip/vrt_remap_plev19.nc") - cmip_plevdata = /lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc input_files = "eam.h0" interp_vars = "U,V,T,Z3" ts_subsection = "atm_monthly_180x360_aave" diff --git a/zppy/templates/e3sm_to_cmip.bash b/zppy/templates/e3sm_to_cmip.bash index e5f37bde..e2a981cd 100644 --- a/zppy/templates/e3sm_to_cmip.bash +++ b/zppy/templates/e3sm_to_cmip.bash @@ -14,6 +14,12 @@ dest={{ output }}/post/{{ component }}/{{ ts_grid }}/ts/{{ frequency }}/{{ '%dyr tmp_dir=tmp_{{ prefix }} +# Copy cmip_plevdata file +# Alternatively, this could be placed in diagnostics_base_path and reached from there +cat > cmip_plevdata.nc << EOF +{% include cmip_plevdata %} +EOF + # Generate CMIP ts cat > default_metadata.json << EOF {% include cmip_metadata %} @@ -36,7 +42,7 @@ EOF do if [ -f ${file} ]; then #ncks --rgr xtr_mth=mss_val --vrt_fl='{{cmip_plevdata}}' ${file} ${file}.plev - ncremap -p mpi --vrt_ntp=log --vrt_xtr=mss_val --vrt_out='{{cmip_plevdata}}' ${file} ${file}.plev + ncremap -p mpi --vrt_ntp=log --vrt_xtr=mss_val --vrt_out='cmip_plevdata.nc' ${file} ${file}.plev #overwrite the model level data mv ${file}.plev ${file} fi From f2fccc9c4ea22c0d70e95857689c357c8e71baf5 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 6 Oct 2025 16:24:28 -0500 Subject: [PATCH 45/47] Pass along debug parameter --- zppy/templates/pcmdi_diags.bash | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zppy/templates/pcmdi_diags.bash b/zppy/templates/pcmdi_diags.bash index 3d8913e9..1b244de9 100755 --- a/zppy/templates/pcmdi_diags.bash +++ b/zppy/templates/pcmdi_diags.bash @@ -510,7 +510,7 @@ echo "Linking observational data into ${obstmp_dir}..." ################### echo "Linking observational data using SLURM..." -command="zi-pcmdi-link-observation --model_name_ref ${model_name_ref} --tableID_ref ${tableID_ref} --vars=${source_vars} --obs_sets {{ obs_sets }} --obs_ts {{ obs_ts }} --obstmp_dir ${obstmp_dir}" +command="zi-pcmdi-link-observation --model_name_ref ${model_name_ref} --tableID_ref ${tableID_ref} --vars=${source_vars} --obs_sets {{ obs_sets }} --obs_ts {{ obs_ts }} --obstmp_dir ${obstmp_dir} --debug ${debug,,}" echo "Running a zi-pcmdi command: ${command}" {{ environment_commands_secondary }} @@ -840,7 +840,7 @@ source_dirs="--climo_ts_dir_primary ${ts_dir_primary} --climo_ts_dir_ref ${ts_di # So, it's a good idea to make sure they can't be (or at least are unlikely to be) empty. # run_type == "model_vs_obs" only: obs_sets (default value is NOT "") # run_type == "model_vs_model" only: model_name_ref, tableID_ref (default values are NOT "") -core_parameters="--num_workers {{ num_workers }} --multiprocessing {{ multiprocessing }} --subsection {{ subsection }} ${source_dirs} --model_name ${model_name} --model_tableID {{model_tableID }} --figure_format {{ figure_format }} --run_type {{ run_type }} --obs_sets {{ obs_sets }} --model_name_ref ${model_name_ref} --vars ${source_vars} --tableID_ref ${tableID_ref} --generate_sftlf {{ generate_sftlf }} --case_id ${case_id} --results_dir ${results_dir}" +core_parameters="--num_workers {{ num_workers }} --multiprocessing {{ multiprocessing }} --subsection {{ subsection }} ${source_dirs} --model_name ${model_name} --model_tableID {{model_tableID }} --figure_format {{ figure_format }} --run_type {{ run_type }} --obs_sets {{ obs_sets }} --model_name_ref ${model_name_ref} --vars ${source_vars} --tableID_ref ${tableID_ref} --generate_sftlf {{ generate_sftlf }} --case_id ${case_id} --results_dir ${results_dir} --debug ${debug,,}" {% endif %} {% if current_set == "mean_climate" %} @@ -886,7 +886,7 @@ find ${enso_dir} -name "*.${case_id}*.json" echo "Done checking. There should be a list of files above, if they exist." {% endif %} -command="zi-pcmdi-synthetic-plots --synthetic_sets {{ synthetic_sets }} --figure_format {{ figure_format }} --www ${www} --results_dir ${results_dir} --case {{ case }} --model_name {{ model_name }} --model_tableID {{model_tableID }} --web_dir=${web_dir} --pcmdi_webtitle {{ pcmdi_webtitle }} --pcmdi_version {{ pcmdi_version }} --run_type ${run_type} --pcmdi_external_prefix {{ diagnostics_base_path }} --pcmdi_viewer_template {{ pcmdi_viewer_template }} --save_all_data {{save_all_data}} ${clim_keys} ${movs_keys} ${enso_keys}" +command="zi-pcmdi-synthetic-plots --synthetic_sets {{ synthetic_sets }} --figure_format {{ figure_format }} --www ${www} --results_dir ${results_dir} --case {{ case }} --model_name {{ model_name }} --model_tableID {{model_tableID }} --web_dir=${web_dir} --pcmdi_webtitle {{ pcmdi_webtitle }} --pcmdi_version {{ pcmdi_version }} --run_type ${run_type} --pcmdi_external_prefix {{ diagnostics_base_path }} --pcmdi_viewer_template {{ pcmdi_viewer_template }} --save_all_data {{save_all_data}} ${clim_keys} ${movs_keys} ${enso_keys} --debug ${debug,,}" {% endif %} From 9127193004a943829e218e9f8351e4c5a33f0598 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Tue, 7 Oct 2025 18:14:33 -0500 Subject: [PATCH 46/47] Attempted fixes and failing integration tests --- examples/post.v3.LR.amip.0101.cfg | 7 +-- examples/post.v3.LR.historical.0101.cfg | 7 +-- ...test_weekly_comprehensive_v3_chrysalis.cfg | 43 +++++++++--------- .../update_weekly_expected_files_chrysalis.sh | 6 +-- .../template_weekly_comprehensive_v3.cfg | 1 + tests/integration/utils.py | 17 ++----- zppy/defaults/default.ini | 19 ++++---- zppy/e3sm_to_cmip.py | 8 ++++ zppy/templates/e3sm_to_cmip.bash | 8 +--- .../e3sm_to_cmip/vrt_remap_plev19.nc | Bin 1260 -> 0 bytes 10 files changed, 57 insertions(+), 59 deletions(-) delete mode 100755 zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc diff --git a/examples/post.v3.LR.amip.0101.cfg b/examples/post.v3.LR.amip.0101.cfg index a69c3128..c5ac9493 100644 --- a/examples/post.v3.LR.amip.0101.cfg +++ b/examples/post.v3.LR.amip.0101.cfg @@ -7,8 +7,8 @@ # 5. Then, you can run those lines stand-alone. [default] input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/AMIP/v3.LR.amip_0101 -output = /lcrc/group/e3sm/ac.forsyth2/zppy_pmp_amip/unique_id_20251006_1/v3.LR.amip_0101 -www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pmp_amip/unique_id_20251006_1 +output = /lcrc/group/e3sm/ac.forsyth2/zppy_pmp_amip/unique_id_20251007_3/v3.LR.amip_0101 +www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pmp_amip/unique_id_20251007_3 campaign = "water_cycle" case = v3.LR.amip_0101 debug = False @@ -36,6 +36,7 @@ walltime = "00:10:00" years = "2005:2015:5" [[ atm_monthly_180x360_aave ]] + cmip_plevdata = "/lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc" input_files = "eam.h0" interp_vars = "U,V,T,Z3" ts_subsection = "atm_monthly_180x360_aave" @@ -43,7 +44,7 @@ years = "2005:2015:5" [pcmdi_diags] active = True -environment_commands_secondary = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20251006" +environment_commands_secondary = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20251007-test1" walltime = "4:00:00" # Create rules to construct cmip-like datasets with a string constructed as # "mip.exp.product.realization", where "."is used as the delimiter (mandatory) diff --git a/examples/post.v3.LR.historical.0101.cfg b/examples/post.v3.LR.historical.0101.cfg index 451289cb..a6bee610 100644 --- a/examples/post.v3.LR.historical.0101.cfg +++ b/examples/post.v3.LR.historical.0101.cfg @@ -1,7 +1,7 @@ [default] input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/v3.LR.historical_0101 -output = /lcrc/group/e3sm/ac.forsyth2/zppy_pmp_hist/unique_id_20251006_1/v3.LR.amip_0101 -www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pmp_hist/unique_id_20251006_1 +output = /lcrc/group/e3sm/ac.forsyth2/zppy_pmp_hist/unique_id_20251007_3/v3.LR.amip_0101 +www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pmp_hist/unique_id_20251007_3 campaign = "water_cycle" case = v3.LR.historical_0101 debug = False @@ -29,6 +29,7 @@ walltime = "00:10:00" years = "2005:2015:5" [[ atm_monthly_180x360_aave ]] + cmip_plevdata = "/lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc" input_files = "eam.h0" interp_vars = "U,V,T,Z3" ts_subsection = "atm_monthly_180x360_aave" @@ -36,7 +37,7 @@ years = "2005:2015:5" [pcmdi_diags] active = True -environment_commands_secondary = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20251006" +environment_commands_secondary = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20251007-test1" walltime = "4:00:00" # Create rules to construct cmip-like datasets with a string constructed as # "mip.exp.product.realization", where "."is used as the delimiter (mandatory) diff --git a/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg b/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg index 94fd6fd9..6d9530f5 100644 --- a/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg +++ b/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg @@ -9,10 +9,10 @@ infer_section_parameters = False input = /lcrc/group/e3sm2/ac.wlin//E3SMv3/v3.LR.historical_0051 input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_3/v3.LR.historical_0051" partition = "debug" qos = "regular" -www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_comprehensive_v3_www/unique_id" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_comprehensive_v3_www/unique_id_test_20251007_3" years = "1985:1989:2", [climo] @@ -26,14 +26,14 @@ walltime = "00:30:00" vars = "" [[ atm_monthly_diurnal_8xdaily_180x360_aave ]] - active = True + active = False frequency = "diurnal_8xdaily" input_files = "eam.h3" input_subdir = "archive/atm/hist" vars = "PRECT" [[ land_monthly_climo ]] - active = True + active = False frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" @@ -51,14 +51,14 @@ walltime = "00:30:00" years = "1985:1995:2", # Need 10 years for pcmdi_diags task [[ atm_daily_180x360_aave ]] - active = True + active = False frequency = "daily" input_files = "eam.h1" input_subdir = "archive/atm/hist" vars = "PRECT" [[ rof_monthly ]] - active = True + active = False extra_vars = 'areatotal2' frequency = "monthly" input_files = "mosart.h0" @@ -68,7 +68,7 @@ walltime = "00:30:00" [[ atm_monthly_glb ]] # Note global average won't work for 3D variables. - active = True + active = False frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" @@ -76,7 +76,7 @@ walltime = "00:30:00" years = "1985:1995:5", [[ lnd_monthly_glb ]] - active = True + active = False frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" @@ -99,6 +99,7 @@ ts_num_years=2 walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] + cmip_plevdata = "/lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc" input_files = "eam.h0" ts_subsection = "atm_monthly_180x360_aave" years = "1985:1995:2", # Need 10 years for pcmdi_diags task @@ -113,11 +114,11 @@ walltime = "00:30:00" # walltime = "00:30:00" [e3sm_diags] -active = True +active = False climo_diurnal_frequency = "diurnal_8xdaily" climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave" climo_subsection = "atm_monthly_180x360_aave" -environment_commands = "source /conda.sh; conda activate " +environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" grid = '180x360_aave' multiprocessing = True num_workers = 8 @@ -169,16 +170,16 @@ tc_obs = "/lcrc/group/e3sm/diagnostics/observations/Atm/tc-analysis/" tag = "model_vs_model" ts_num_years_ref = 2 # Reference paths - reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/atm/180x360_aave/clim" + reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_3/v3.LR.historical_0051/post/atm/180x360_aave/clim" # mvm streamflow only gauges_path = "/lcrc/group/e3sm/diagnostics/observations/Atm/time-series/GSIM/GSIM_catchment_characteristics_all_1km2.csv" - reference_data_path_ts_rof = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/rof/native/ts/monthly" + reference_data_path_ts_rof = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_3/v3.LR.historical_0051/post/rof/native/ts/monthly" # mvm diurnal_cycle only - reference_data_path_climo_diurnal = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/atm/180x360_aave/clim_diurnal_8xdaily" + reference_data_path_climo_diurnal = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_3/v3.LR.historical_0051/post/atm/180x360_aave/clim_diurnal_8xdaily" # mvm "enso_diags", "qbo", "area_mean_time_series" - reference_data_path_ts = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/atm/180x360_aave/ts/monthly" + reference_data_path_ts = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_3/v3.LR.historical_0051/post/atm/180x360_aave/ts/monthly" # mvm tropical_subseasonal only - reference_data_path_ts_daily = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/atm/180x360_aave/ts/daily" + reference_data_path_ts_daily = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_3/v3.LR.historical_0051/post/atm/180x360_aave/ts/daily" [[ lnd_monthly_mvm_lnd ]] # Test model-vs-model using the same files as the reference @@ -194,10 +195,10 @@ tc_obs = "/lcrc/group/e3sm/diagnostics/observations/Atm/tc-analysis/" tag = "model_vs_model" ts_num_years_ref = 2 # Reference paths - reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/lnd/180x360_aave/clim" + reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_3/v3.LR.historical_0051/post/lnd/180x360_aave/clim" [mpas_analysis] -active = True +active = False anomalyRefYear = 1985 climo_years = "1985-1989", "1990-1995", enso_years = "1985-1989", "1990-1995", @@ -210,9 +211,9 @@ ts_years = "1985-1989", "1985-1995", walltime = "00:30:00" [global_time_series] -active = True +active = False climo_years = "1985-1989", "1990-1995", -environment_commands = "source /conda.sh; conda activate " +environment_commands = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20251007-test1" experiment_name = "v3.LR.historical_0051" figstr = "v3.LR.historical_0051" #moc_file=mocTimeSeries_1985-1995.nc @@ -296,7 +297,7 @@ years = "1985-1995", [ilamb] -active = True +active = False e3sm_to_cmip_atm_subsection = "atm_monthly_180x360_aave" e3sm_to_cmip_land_subsection = "land_monthly" ilamb_obs = "/lcrc/group/e3sm/diagnostics/ilamb_data" @@ -311,7 +312,7 @@ years = "1985:1989:4" [pcmdi_diags] active = True -environment_commands_secondary = "source /conda.sh; conda activate " +environment_commands_secondary = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20251007-test1" walltime = "2:00:00" model_name = 'e3sm.historical.v3-LR.0051' model_tableID = 'Amon' diff --git a/tests/integration/generated/update_weekly_expected_files_chrysalis.sh b/tests/integration/generated/update_weekly_expected_files_chrysalis.sh index a1676a23..d265e223 100755 --- a/tests/integration/generated/update_weekly_expected_files_chrysalis.sh +++ b/tests/integration/generated/update_weekly_expected_files_chrysalis.sh @@ -35,17 +35,17 @@ do # Copy output so you don't have to rerun zppy to generate the output. if [[ "${test_name,,}" =~ "v2" ]]; then # We need the v2 case name - cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name} + cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id_test_20251007_3/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name} else # We need the v3 case name - cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id/v3.LR.historical_0051 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name} + cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id_test_20251007_3/v3.LR.historical_0051 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name} fi # test_bundles.py also needs the bash files transferred. # Note that for legacy cfgs, we're only testing test_images.py if [[ "${test_name,,}" == "bundles" ]]; then mkdir -p /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files - cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_weekly_bundles_output/unique_id/v3.LR.historical_0051/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files + cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_weekly_bundles_output/unique_id_test_20251007_3/v3.LR.historical_0051/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files fi zppy_top_level=$(pwd) diff --git a/tests/integration/template_weekly_comprehensive_v3.cfg b/tests/integration/template_weekly_comprehensive_v3.cfg index 7ace3d3c..9a2cdf53 100644 --- a/tests/integration/template_weekly_comprehensive_v3.cfg +++ b/tests/integration/template_weekly_comprehensive_v3.cfg @@ -99,6 +99,7 @@ ts_num_years=2 walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] + cmip_plevdata = "/lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc" input_files = "eam.h0" ts_subsection = "atm_monthly_180x360_aave" years = "1985:1995:2", # Need 10 years for pcmdi_diags task diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 72f3e297..d8a8bf18 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -12,25 +12,16 @@ # pytest tests/integration/test_*.py TEST_SPECIFICS: Dict[str, Any] = { - "diags_environment_commands": "source /conda.sh; conda activate ", - "global_time_series_environment_commands": "source /conda.sh; conda activate ", - "pcmdi_diags_environment_commands": "source /conda.sh; conda activate ", + "diags_environment_commands": "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh", + "global_time_series_environment_commands": "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20251007-test1", + "pcmdi_diags_environment_commands": "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20251007-test1", "cfgs_to_run": [ - "weekly_bundles", - "weekly_comprehensive_v2", "weekly_comprehensive_v3", - "weekly_legacy_3.0.0_bundles", - "weekly_legacy_3.0.0_comprehensive_v2", - "weekly_legacy_3.0.0_comprehensive_v3", ], "tasks_to_run": [ - "e3sm_diags", - "mpas_analysis", - "global_time_series", - "ilamb", "pcmdi_diags", ], - "unique_id": "unique_id", + "unique_id": "unique_id_test_20251007_3", } # Multi-machine testing ######################################################### diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index 8d2e5e31..8dfa0810 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -144,7 +144,8 @@ input_component = string(default="") # Metadata json file cmip_metadata = string(default="inclusions/e3sm_to_cmip/default_metadata.json") # Pressure level nc file -cmip_plevdata = string(default="inclusions/e3sm_to_cmip/vrt_remap_plev19.nc") +# Defaults to a file in diagnostics_base_path if not set +cmip_plevdata = string(default="") # variables to run e3sm_to_cmip on cmip_vars = string(default="") # Model component having generated input files (eam, eamxx, elm, mosart, ...) @@ -527,16 +528,16 @@ enso_vars = string(default="psl,pr,prsn,ts,tas,tauu,tauv,hflx,hfss,rlds,rsds,rlu ########################################################################################### # The _years parameters are also defined for the other subtasks, but internally in pcmdi_diags.py. # add mean climate to viewer page (default is False) -clim_viewer = string(default=True) +clim_viewer = boolean(default=True) clim_years = string(default="") # add extratropic modes to viewer page (default is True) -mova_viewer = string(default=True) +mova_viewer = boolean(default=True) mova_years = string(default="") # add coupled modes to viewer page (default is True) -movc_viewer = string(default=True) +movc_viewer = boolean(default=True) movc_years = string(default="") # add enso to viewer page (default is False, not currently enabled) -enso_viewer = string(default=False) +enso_viewer = boolean(default=False) enso_years = string(default="") # Save all intemediate diagnostic data save_all_data = string(default=True) @@ -615,13 +616,13 @@ cmip_movs_set = string(default="cmip6.historical.v20220825") movc_modes = string(default=None) enso_groups = string(default=None) enso_vars = string(default=None) - clim_viewer = string(default=None) + clim_viewer = boolean(default=None) clim_years = string(default=None) - mova_viewer = string(default=None) + mova_viewer = boolean(default=None) mova_years = string(default=None) - movc_viewer = string(default=None) + movc_viewer = boolean(default=None) movc_years = string(default=None) - enso_viewer = string(default=None) + enso_viewer = boolean(default=None) enso_years = string(default=None) save_all_data = string(default=None) reference_alias = string(default=None) diff --git a/zppy/e3sm_to_cmip.py b/zppy/e3sm_to_cmip.py index 6de0a28e..4b6a7aa5 100644 --- a/zppy/e3sm_to_cmip.py +++ b/zppy/e3sm_to_cmip.py @@ -38,6 +38,14 @@ def e3sm_to_cmip(config: ConfigObj, script_dir: str, existing_bundles, job_ids_f set_component_and_prc_typ(c) check_parameters_for_bash(c) c["cmor_tables_prefix"] = c["diagnostics_base_path"] + if c["component"] == "atm": + default_cmip_plevdata = f"{c['diagnostics_base_path']}/vrt_remap_plev19.nc" + set_value_of_parameter_if_undefined( + c, + "cmip_plevdata", + default_cmip_plevdata, + ParameterInferenceType.PATH_INFERENCE, + ) year_sets: List[Tuple[int, int]] = get_years(c["years"]) # Loop over year sets for s in year_sets: diff --git a/zppy/templates/e3sm_to_cmip.bash b/zppy/templates/e3sm_to_cmip.bash index e2a981cd..e5f37bde 100644 --- a/zppy/templates/e3sm_to_cmip.bash +++ b/zppy/templates/e3sm_to_cmip.bash @@ -14,12 +14,6 @@ dest={{ output }}/post/{{ component }}/{{ ts_grid }}/ts/{{ frequency }}/{{ '%dyr tmp_dir=tmp_{{ prefix }} -# Copy cmip_plevdata file -# Alternatively, this could be placed in diagnostics_base_path and reached from there -cat > cmip_plevdata.nc << EOF -{% include cmip_plevdata %} -EOF - # Generate CMIP ts cat > default_metadata.json << EOF {% include cmip_metadata %} @@ -42,7 +36,7 @@ EOF do if [ -f ${file} ]; then #ncks --rgr xtr_mth=mss_val --vrt_fl='{{cmip_plevdata}}' ${file} ${file}.plev - ncremap -p mpi --vrt_ntp=log --vrt_xtr=mss_val --vrt_out='cmip_plevdata.nc' ${file} ${file}.plev + ncremap -p mpi --vrt_ntp=log --vrt_xtr=mss_val --vrt_out='{{cmip_plevdata}}' ${file} ${file}.plev #overwrite the model level data mv ${file}.plev ${file} fi diff --git a/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc b/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc deleted file mode 100755 index 2b71cc296ff2f207b80961668af618cecf911c7e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1260 zcma)6QE${Z5cZaBizpJDctk=Lssx-iah$ZkwN@t*Mpq+L{V6^u3af0v<-VB`cL(xUBVs*tzuH z0qVP%tWb-(pW}#cM?4;cJG<#GHJ^x5gIn+HJTxkagxyU;mFvnHJW7_m_Vd-UM4Q%iQwCt(YJ@xiP%r~ zPM;HIvIun;B?%9r_7}kynhac#OWu}t>SQ@PISbLmpRm%|XDah&JlC#bGAk6*rd;@3 zddZ|O7@3f+Tug2%9NsF!8oxMtWT6?B0vp4WsW_`S`~NN9^vEm<;fmNHP@Mm` zT~_F_FKX$uoXl0lYUhPhg)9Z8ln>6zO3odoN2m27PmS{F&WIjCnI#mmqILUcpc~;< zIG{K0+d`Fcs^}d$^S&H1W-=RA$I#HeM^kv;cVw#lY%vKlTc8orEsV1U^y8dC`Nt3E z$b(kV{hM^&0qIAi^WZ$``~(x{-=~E9^N^4~?-H_rsUklRa!3go{z1r%%Y Date: Tue, 7 Oct 2025 20:55:38 -0500 Subject: [PATCH 47/47] Examples work but test cfg fails --- examples/post.v3.LR.amip.0101.cfg | 4 ++-- examples/post.v3.LR.historical.0101.cfg | 4 ++-- .../test_weekly_comprehensive_v3_chrysalis.cfg | 16 ++++++++-------- .../update_weekly_expected_files_chrysalis.sh | 6 +++--- tests/integration/utils.py | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/examples/post.v3.LR.amip.0101.cfg b/examples/post.v3.LR.amip.0101.cfg index c5ac9493..dd6d7268 100644 --- a/examples/post.v3.LR.amip.0101.cfg +++ b/examples/post.v3.LR.amip.0101.cfg @@ -7,8 +7,8 @@ # 5. Then, you can run those lines stand-alone. [default] input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/AMIP/v3.LR.amip_0101 -output = /lcrc/group/e3sm/ac.forsyth2/zppy_pmp_amip/unique_id_20251007_3/v3.LR.amip_0101 -www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pmp_amip/unique_id_20251007_3 +output = /lcrc/group/e3sm/ac.forsyth2/zppy_pmp_amip/unique_id_20251007_4/v3.LR.amip_0101 +www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pmp_amip/unique_id_20251007_4 campaign = "water_cycle" case = v3.LR.amip_0101 debug = False diff --git a/examples/post.v3.LR.historical.0101.cfg b/examples/post.v3.LR.historical.0101.cfg index a6bee610..432049f2 100644 --- a/examples/post.v3.LR.historical.0101.cfg +++ b/examples/post.v3.LR.historical.0101.cfg @@ -1,7 +1,7 @@ [default] input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/v3.LR.historical_0101 -output = /lcrc/group/e3sm/ac.forsyth2/zppy_pmp_hist/unique_id_20251007_3/v3.LR.amip_0101 -www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pmp_hist/unique_id_20251007_3 +output = /lcrc/group/e3sm/ac.forsyth2/zppy_pmp_hist/unique_id_20251007_4/v3.LR.amip_0101 +www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pmp_hist/unique_id_20251007_4 campaign = "water_cycle" case = v3.LR.historical_0101 debug = False diff --git a/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg b/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg index 6d9530f5..eb2d6c71 100644 --- a/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg +++ b/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg @@ -9,10 +9,10 @@ infer_section_parameters = False input = /lcrc/group/e3sm2/ac.wlin//E3SMv3/v3.LR.historical_0051 input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_3/v3.LR.historical_0051" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_4/v3.LR.historical_0051" partition = "debug" qos = "regular" -www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_comprehensive_v3_www/unique_id_test_20251007_3" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_comprehensive_v3_www/unique_id_test_20251007_4" years = "1985:1989:2", [climo] @@ -170,16 +170,16 @@ tc_obs = "/lcrc/group/e3sm/diagnostics/observations/Atm/tc-analysis/" tag = "model_vs_model" ts_num_years_ref = 2 # Reference paths - reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_3/v3.LR.historical_0051/post/atm/180x360_aave/clim" + reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_4/v3.LR.historical_0051/post/atm/180x360_aave/clim" # mvm streamflow only gauges_path = "/lcrc/group/e3sm/diagnostics/observations/Atm/time-series/GSIM/GSIM_catchment_characteristics_all_1km2.csv" - reference_data_path_ts_rof = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_3/v3.LR.historical_0051/post/rof/native/ts/monthly" + reference_data_path_ts_rof = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_4/v3.LR.historical_0051/post/rof/native/ts/monthly" # mvm diurnal_cycle only - reference_data_path_climo_diurnal = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_3/v3.LR.historical_0051/post/atm/180x360_aave/clim_diurnal_8xdaily" + reference_data_path_climo_diurnal = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_4/v3.LR.historical_0051/post/atm/180x360_aave/clim_diurnal_8xdaily" # mvm "enso_diags", "qbo", "area_mean_time_series" - reference_data_path_ts = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_3/v3.LR.historical_0051/post/atm/180x360_aave/ts/monthly" + reference_data_path_ts = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_4/v3.LR.historical_0051/post/atm/180x360_aave/ts/monthly" # mvm tropical_subseasonal only - reference_data_path_ts_daily = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_3/v3.LR.historical_0051/post/atm/180x360_aave/ts/daily" + reference_data_path_ts_daily = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_4/v3.LR.historical_0051/post/atm/180x360_aave/ts/daily" [[ lnd_monthly_mvm_lnd ]] # Test model-vs-model using the same files as the reference @@ -195,7 +195,7 @@ tc_obs = "/lcrc/group/e3sm/diagnostics/observations/Atm/tc-analysis/" tag = "model_vs_model" ts_num_years_ref = 2 # Reference paths - reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_3/v3.LR.historical_0051/post/lnd/180x360_aave/clim" + reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_4/v3.LR.historical_0051/post/lnd/180x360_aave/clim" [mpas_analysis] active = False diff --git a/tests/integration/generated/update_weekly_expected_files_chrysalis.sh b/tests/integration/generated/update_weekly_expected_files_chrysalis.sh index d265e223..73973b26 100755 --- a/tests/integration/generated/update_weekly_expected_files_chrysalis.sh +++ b/tests/integration/generated/update_weekly_expected_files_chrysalis.sh @@ -35,17 +35,17 @@ do # Copy output so you don't have to rerun zppy to generate the output. if [[ "${test_name,,}" =~ "v2" ]]; then # We need the v2 case name - cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id_test_20251007_3/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name} + cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id_test_20251007_4/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name} else # We need the v3 case name - cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id_test_20251007_3/v3.LR.historical_0051 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name} + cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id_test_20251007_4/v3.LR.historical_0051 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name} fi # test_bundles.py also needs the bash files transferred. # Note that for legacy cfgs, we're only testing test_images.py if [[ "${test_name,,}" == "bundles" ]]; then mkdir -p /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files - cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_weekly_bundles_output/unique_id_test_20251007_3/v3.LR.historical_0051/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files + cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_weekly_bundles_output/unique_id_test_20251007_4/v3.LR.historical_0051/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files fi zppy_top_level=$(pwd) diff --git a/tests/integration/utils.py b/tests/integration/utils.py index d8a8bf18..927b6a91 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -21,7 +21,7 @@ "tasks_to_run": [ "pcmdi_diags", ], - "unique_id": "unique_id_test_20251007_3", + "unique_id": "unique_id_test_20251007_4", } # Multi-machine testing #########################################################