Zppy pcmdi 202505 rev

examples/post.v3.LR.amip.0101.cfg

@@ -0,0 +1,159 @@
+# Directions to run:
+# 1. Update <output>, <www>, <environment_commands_secondary> below.
+# 2. Run with `zppy -c examples/post.v3.LR.amip.0101.cfg`.
+# Direction to create stand-alone test data for zppy-interfaces:
+# 3. Once the jobs finish, `cd <output>/post/scripts`.
+# 4. Run `grep -n "Running a zi-pcmdi command" pcmdi_diags*.o*` to find the pcmdi_diags commands.
+# 5. Then, you can run those lines stand-alone.
+[default]
+input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/AMIP/v3.LR.amip_0101
+output = /lcrc/group/e3sm/ac.forsyth2/zppy_pmp_amip/unique_id_20251007_4/v3.LR.amip_0101
+www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pmp_amip/unique_id_20251007_4
+campaign = "water_cycle"
+case = v3.LR.amip_0101
+debug = False
+environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh"
+
+[ts]
+active = True
+walltime = "00:10:00"
+years = "2005:2015:5"
+
+  [[ atm_monthly_180x360_aave ]]
+  input_subdir = "archive/atm/hist"
+  input_files = "eam.h0"
+  frequency = "monthly"
+  mapping_file = /home/ac.zender/data/maps/map_ne30pg2_to_cmip6_180x360_aave.20200201.nc
+  # list for pcmdi diag, note: PHIS,hyam,hybm,hyai,hybi need to be included to process the 3D fields
+  vars = "ICEFRAC,LANDFRAC,OCNFRAC,PSL,FSNTC,FSNTOAC,SWCF,LWCF,FLUT,FSNT,FSNTOA,FLNT,FLNTC,FSNS,FLNS,FSNS,SHFLX,QFLX,LHFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,U10,QREFHT,TMQ,CLDTOT,CLDHGH,CLDMED,CLDLOW,FLDS,FSDS,TGCLDIWP,TGCLDCWP,TGCLDLWP,FLNSC,FLUTC,FSDSC,SOLIN,FSNSC,AODABS,AODVIS,AODDUST,AREL,TREFMNAV,TREFMXAV,PS,PHIS,U,V,T,Z3"
+
+[e3sm_to_cmip]
+active = True
+frequency = "monthly"
+ts_grid = "180x360_aave"
+ts_num_years=5
+walltime = "00:10:00"
+years = "2005:2015:5"
+
+  [[ atm_monthly_180x360_aave ]]
+  cmip_plevdata = "/lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc"
+  input_files = "eam.h0"
+  interp_vars = "U,V,T,Z3"
+  ts_subsection = "atm_monthly_180x360_aave"
+  vars = "ICEFRAC,LANDFRAC,OCNFRAC,PSL,FSNTC,FSNTOAC,SWCF,LWCF,FLUT,FSNT,FSNTOA,FLNT,FLNTC,FSNS,FLNS,FSNS,SHFLX,QFLX,LHFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,U10,QREFHT,TMQ,CLDTOT,CLDHGH,CLDMED,CLDLOW,FLDS,FSDS,TGCLDIWP,TGCLDCWP,TGCLDLWP,FLNSC,FLUTC,FSDSC,SOLIN,FSNSC,AODABS,AODVIS,AODDUST,AREL,TREFMNAV,TREFMXAV,PS,PHIS,U,V,T,Z3"
+
+[pcmdi_diags]
+active = True
+environment_commands_secondary = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20251007-test1"
+walltime = "4:00:00"
+# Create rules to construct cmip-like datasets with a string constructed as
+# "mip.exp.product.realization", where "."is used as the delimiter (mandatory)
+model_name = 'e3sm.amip.v3-LR.0101'
+model_tableID = 'Amon'
+multiprocessing = True
+num_workers = 24
+# Observational data
+obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series'
+pcmdi_debug = False
+ts_years = "2005-2014",
+
+  [[ mean_climate ]]
+  active = True
+  # Default for all variables or specify source for each variable separately.
+  obs_sets = "default"
+  ref_final_yr = 2018
+  ref_start_yr = 1979
+  ref_years = "2005-2014",
+  target_grid = "2.5x2.5"
+  target_grid_string = "2p5x2p5"
+  # Variables in current pcmdi-cmip list
+  # rltcre and rstcre are derived variables as they are not included in cmip_ts
+  # define a new variable name to be flexible
+  clim_vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rsus,rsuscs,rlus,rlut,rlutcs,rsds,rsdscs,rsdt,rsut,rsutcs,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500"
+  # See zppy/templates/pcmdi_diags/regions_specs.json for options of all regions
+  # Below is default setup for global/regional mean climate metrics
+  clim_regions = "global,ocean,land,Tropics,NHEX,SHEX"
+
+  [[ variability_modes_cpl ]]
+  active = True
+  # See zppy/templates/pcmdi_diags/reference_alias.json for options
+  obs_sets = "alternate1" #"default"
+  ref_final_yr = 2019
+  ref_start_yr = 1870
+  ref_years = "2005-2014",
+  frequency = 'mo'
+  landmask = True
+  ModUnitsAdjust = '(True,"subtract",273.15)'
+  ObsUnitsAdjust = '(True,"subtract",273.15)'
+  seasons = "yearly,monthly"
+  movc_vars = "ts"
+  # Name of modes varibility
+  movc_modes = "PDO,NPGO,AMO"
+
+  [[ variability_modes_atm ]]
+  active = True
+  # See zppy/templates/pcmdi_diags/reference_alias.json for options
+  # Default for all variables
+  obs_sets = "alternate2" # "default"
+  ref_final_yr = 2015
+  ref_start_yr = 1853
+  ref_years = "2005-2014",
+  frequency = 'mo'
+  landmask = False
+  ModUnitsAdjust = '(True,"divide",100.0)'
+  ObsUnitsAdjust = '(True,"divide",100.0)'
+  seasons = "DJF,MAM,JJA,SON,yearly,monthly"
+  mova_vars = "psl"
+  # Varibility modes
+  mova_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2"
+
+  [[ enso ]]
+  active = False
+  # For enso, the obs_sets is explicitely set for each variable to ensure that
+  # they are from the same source, below are setup to used ERA5 reanalysis.
+  obs_sets = "default,alternate2,default,default,default,default,default,default,default,alternate2,alternate2,alternate2,alternate2,alternate2"
+  ref_final_yr = 2018
+  ref_start_yr = 1979
+  ref_years = "2005-2014",
+  # See zppy/templates/pcmdi_diags/reference_alias.json for options
+  enso_groups = "ENSO_perf,ENSO_proc,ENSO_tel"
+  enso_vars = "psl,pr,prsn,ts,tas,tauu,tauv,hfls,hfss,rlds,rsds,rlus,rlut,rsdt"
+
+  [[ synthetic_plots ]]
+  active = True
+  # use subset variables for viewer
+  clim_vars = "pr,prw,psl,rlds,rldscs,ua-200,zg-500"
+  # use subset regions for viewer
+  clim_regions = "global,ocean,land"
+  # use subset modes for viewer
+  mova_modes = "NAM,NAO,PNA,NPO,SAM"
+  # use subset modes for viewer
+  movc_modes  = "PDO,NPGO"
+  # mean climate
+  clim_viewer = True
+  # period to be plotted and shown on webpage
+  # note that the previous sections allows multiple ts_years periods for processing, while
+  # the viewer page focused on period customized by user
+  clim_years = "2005-2014"
+  # modes variability (atmosphere)
+  mova_viewer = True
+  # period to be plotted and shown on webpage
+  # note that the previous sections allows multiple ts_years periods for processing, while
+  # the viewer page focused on period customized by user
+  mova_years = "2005-2014"
+  # modes variability (coupled)
+  movc_viewer = True
+  # period to be plotted and shown on webpage
+  # note that the previous sections allows multiple ts_years periods for processing, while
+  # the viewer page focused on period customized by user
+  movc_years = "2005-2014"
+  # enso
+  enso_viewer = False
+  enso_years = ""
+  # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/mean_climate/cmip6/amip
+  cmip_clim_set = "cmip6.amip.v20250702"
+  # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes/cmip6/amip
+  cmip_movs_set = "cmip6.amip.v20210119"
+  # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/enso_metric/cmip6/historical
+  # There is no enso metric for CMIP6 AMIP, so use historical instead
+  cmip_enso_set = "cmip6.historical.v20210620"

examples/post.v3.LR.historical.0101.cfg

@@ -0,0 +1,152 @@
+[default]
+input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/v3.LR.historical_0101
+output = /lcrc/group/e3sm/ac.forsyth2/zppy_pmp_hist/unique_id_20251007_4/v3.LR.amip_0101
+www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pmp_hist/unique_id_20251007_4
+campaign = "water_cycle"
+case = v3.LR.historical_0101
+debug = False
+environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh"
+
+[ts]
+active = True
+walltime = "00:10:00"
+years = "2005:2015:5"
+
+  [[ atm_monthly_180x360_aave ]]
+  input_subdir = "archive/atm/hist"
+  input_files = "eam.h0"
+  frequency = "monthly"
+  mapping_file = /home/ac.zender/data/maps/map_ne30pg2_to_cmip6_180x360_aave.20200201.nc
+  # list for pcmdi diag, note: PHIS,hyam,hybm,hyai,hybi need to be included to process the 3D fields
+  vars = "ICEFRAC,LANDFRAC,OCNFRAC,PSL,FSNTC,FSNTOAC,SWCF,LWCF,FLUT,FSNT,FSNTOA,FLNT,FLNTC,FSNS,FLNS,FSNS,SHFLX,QFLX,LHFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,U10,QREFHT,TMQ,CLDTOT,CLDHGH,CLDMED,CLDLOW,FLDS,FSDS,TGCLDIWP,TGCLDCWP,TGCLDLWP,FLNSC,FLUTC,FSDSC,SOLIN,FSNSC,AODABS,AODVIS,AODDUST,AREL,TREFMNAV,TREFMXAV,PS,PHIS,U,V,T,Z3"
+
+[e3sm_to_cmip]
+active = True
+frequency = "monthly"
+ts_grid = "180x360_aave"
+ts_num_years=5
+walltime = "00:10:00"
+years = "2005:2015:5"
+
+  [[ atm_monthly_180x360_aave ]]
+  cmip_plevdata = "/lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc"
+  input_files = "eam.h0"
+  interp_vars = "U,V,T,Z3"
+  ts_subsection = "atm_monthly_180x360_aave"
+  vars = "ICEFRAC,LANDFRAC,OCNFRAC,PSL,FSNTC,FSNTOAC,SWCF,LWCF,FLUT,FSNT,FSNTOA,FLNT,FLNTC,FSNS,FLNS,FSNS,SHFLX,QFLX,LHFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,U10,QREFHT,TMQ,CLDTOT,CLDHGH,CLDMED,CLDLOW,FLDS,FSDS,TGCLDIWP,TGCLDCWP,TGCLDLWP,FLNSC,FLUTC,FSDSC,SOLIN,FSNSC,AODABS,AODVIS,AODDUST,AREL,TREFMNAV,TREFMXAV,PS,PHIS,U,V,T,Z3"
+
+[pcmdi_diags]
+active = True
+environment_commands_secondary = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20251007-test1"
+walltime = "4:00:00"
+# Create rules to construct cmip-like datasets with a string constructed as
+# "mip.exp.product.realization", where "."is used as the delimiter (mandatory)
+model_name = 'e3sm.historical.v3-LR.0101'
+model_tableID = 'Amon'
+multiprocessing = True
+num_workers = 24
+# Observational data
+obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series'
+pcmdi_debug = False
+ts_years = "2005-2014",
+
+  [[ mean_climate ]]
+  active = True
+  # Default for all variables or specify source for each variable separately.
+  obs_sets = "default"
+  ref_final_yr = 2018
+  ref_start_yr = 1979
+  ref_years = "2005-2014",
+  target_grid = "2.5x2.5"
+  target_grid_string = "2p5x2p5"
+  # Variables in current pcmdi-cmip list
+  # rltcre and rstcre are derived variables as they are not included in cmip_ts
+  # define a new variable name to be flexible
+  clim_vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rsus,rsuscs,rlus,rlut,rlutcs,rsds,rsdscs,rsdt,rsut,rsutcs,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500"
+  # See zppy/templates/pcmdi_diags/regions_specs.json for options of all regions
+  # Below is default setup for global/regional mean climate metrics
+  clim_regions = "global,ocean,land,Tropics,NHEX,SHEX"
+
+  [[ variability_modes_cpl ]]
+  active = True
+  # See zppy/templates/pcmdi_diags/reference_alias.json for options
+  obs_sets = "alternate1" #"default"
+  ref_final_yr = 2019
+  ref_start_yr = 1870
+  ref_years = "2005-2014",
+  frequency = 'mo'
+  landmask = True
+  ModUnitsAdjust = '(True,"subtract",273.15)'
+  ObsUnitsAdjust = '(True,"subtract",273.15)'
+  seasons = "yearly,monthly"
+  movc_vars = "ts"
+  # Name of modes varibility
+  movc_modes = "PDO,NPGO,AMO"
+
+  [[ variability_modes_atm ]]
+  active = True
+  # See zppy/templates/pcmdi_diags/reference_alias.json for options
+  # Default for all variables
+  obs_sets = "alternate2" # "default"
+  ref_final_yr = 2015
+  ref_start_yr = 1853
+  ref_years = "2005-2014",
+  frequency = 'mo'
+  landmask = False
+  ModUnitsAdjust = '(True,"divide",100.0)'
+  ObsUnitsAdjust = '(True,"divide",100.0)'
+  seasons = "DJF,MAM,JJA,SON,yearly,monthly"
+  mova_vars = "psl"
+  # Varibility modes
+  mova_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2"
+
+  [[ enso ]]
+  active = False
+  # For enso, the obs_sets is explicitely set for each variable to ensure that
+  # they are from the same source, below are setup to used ERA5 reanalysis.
+  obs_sets = "default,alternate2,default,default,default,default,default,default,default,alternate2,alternate2,alternate2,alternate2,alternate2"
+  ref_final_yr = 2018
+  ref_start_yr = 1979
+  ref_years = "2005-2014",
+  # See zppy/templates/pcmdi_diags/reference_alias.json for options
+  enso_groups = "ENSO_perf,ENSO_proc,ENSO_tel"
+  enso_vars = "psl,pr,prsn,ts,tas,tauu,tauv,hfls,hfss,rlds,rsds,rlus,rlut,rsdt"
+
+  [[ synthetic_plots ]]
+  active = True
+  # use subset variables for viewer
+  clim_vars = "pr,prw,psl,rlds,rldscs,ua-200,zg-500"
+  # use subset regions for viewer
+  clim_regions = "global,ocean,land"
+  # use subset modes for viewer
+  mova_modes = "NAM,NAO,PNA,NPO,SAM"
+  # use subset modes for viewer
+  movc_modes  = "PDO,NPGO"
+  # mean climate
+  clim_viewer = True
+  # period to be plotted and shown on webpage
+  # note that the previous sections allows multiple ts_years periods for processing, while
+  # the viewer page focused on period customized by user
+  clim_years = "2005-2014"
+  # modes variability (atmosphere)
+  mova_viewer = True
+  # period to be plotted and shown on webpage
+  # note that the previous sections allows multiple ts_years periods for processing, while
+  # the viewer page focused on period customized by user
+  mova_years = "2005-2014"
+  # modes variability (coupled)
+  movc_viewer = True
+  # period to be plotted and shown on webpage
+  # note that the previous sections allows multiple ts_years periods for processing, while
+  # the viewer page focused on period customized by user
+  movc_years = "2005-2014"
+  # enso
+  enso_viewer = False
+  enso_years = ""
+  # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/mean_climate/cmip6/historical
+  cmip_clim_set = "cmip6.historical.v20250707"
+  # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes/cmip6/historical
+  cmip_movs_set = "cmip6.historical.v20220825"
+  # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/enso_metric/cmip6/historical
+  # There is no enso metric for CMIP6 AMIP, so use historical instead
+  cmip_enso_set = "cmip6.historical.v20210620"

setup.py

@@ -0,0 +1,34 @@
+import os
+
+from setuptools import find_packages, setup
+
+
+# From https://github.com/MPAS-Dev/compass/blob/master/setup.py
+def package_files(directory, prefixes, extensions):
+    """based on https://stackoverflow.com/a/36693250/7728169"""
+    paths = []
+    for path, directories, filenames in os.walk(directory):
+        for filename in filenames:
+            parts = filename.split(".")
+            prefix = parts[0]
+            extension = parts[-1]
+            if prefix in prefixes or extension in extensions:
+                paths.append(os.path.join("..", path, filename))
+    return paths
+
+
+data_files = package_files(
+    "zppy", prefixes=[], extensions=["bash", "csh", "cfg", "ini", "sh", "json"]
+)
+
+setup(
+    name="zppy",
+    version="3.0.0",
+    author="Ryan Forsyth, Chris Golaz",
+    author_email="forsyth2@llnl.gov, golaz1@llnl.gov",
+    description="Post-processing software for E3SM",
+    python_requires=">=3.9",
+    packages=find_packages(include=["zppy", "zppy.*"]),
+    package_data={"": data_files},
+    entry_points={"console_scripts": ["zppy=zppy.__main__:main"]},
+)

tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg

@@ -176,7 +176,7 @@ partition = "compute"
 qos = "regular"
 shortTermArchive = True
 ts_years = "1980-1984", "1980-1990",
-walltime = "02:00:00"
+walltime = "00:30:00"
 
 [global_time_series]
 active = True