diff --git a/examples/post.v3.LR.amip.0101.cfg b/examples/post.v3.LR.amip.0101.cfg new file mode 100644 index 00000000..dd6d7268 --- /dev/null +++ b/examples/post.v3.LR.amip.0101.cfg @@ -0,0 +1,159 @@ +# Directions to run: +# 1. Update , , below. +# 2. Run with `zppy -c examples/post.v3.LR.amip.0101.cfg`. +# Direction to create stand-alone test data for zppy-interfaces: +# 3. Once the jobs finish, `cd /post/scripts`. +# 4. Run `grep -n "Running a zi-pcmdi command" pcmdi_diags*.o*` to find the pcmdi_diags commands. +# 5. Then, you can run those lines stand-alone. +[default] +input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/AMIP/v3.LR.amip_0101 +output = /lcrc/group/e3sm/ac.forsyth2/zppy_pmp_amip/unique_id_20251007_4/v3.LR.amip_0101 +www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pmp_amip/unique_id_20251007_4 +campaign = "water_cycle" +case = v3.LR.amip_0101 +debug = False +environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" + +[ts] +active = True +walltime = "00:10:00" +years = "2005:2015:5" + + [[ atm_monthly_180x360_aave ]] + input_subdir = "archive/atm/hist" + input_files = "eam.h0" + frequency = "monthly" + mapping_file = /home/ac.zender/data/maps/map_ne30pg2_to_cmip6_180x360_aave.20200201.nc + # list for pcmdi diag, note: PHIS,hyam,hybm,hyai,hybi need to be included to process the 3D fields + vars = "ICEFRAC,LANDFRAC,OCNFRAC,PSL,FSNTC,FSNTOAC,SWCF,LWCF,FLUT,FSNT,FSNTOA,FLNT,FLNTC,FSNS,FLNS,FSNS,SHFLX,QFLX,LHFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,U10,QREFHT,TMQ,CLDTOT,CLDHGH,CLDMED,CLDLOW,FLDS,FSDS,TGCLDIWP,TGCLDCWP,TGCLDLWP,FLNSC,FLUTC,FSDSC,SOLIN,FSNSC,AODABS,AODVIS,AODDUST,AREL,TREFMNAV,TREFMXAV,PS,PHIS,U,V,T,Z3" + +[e3sm_to_cmip] +active = True +frequency = "monthly" +ts_grid = "180x360_aave" +ts_num_years=5 +walltime = "00:10:00" +years = "2005:2015:5" + + [[ atm_monthly_180x360_aave ]] + cmip_plevdata = "/lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc" + input_files = "eam.h0" + interp_vars = "U,V,T,Z3" + ts_subsection = "atm_monthly_180x360_aave" + vars = "ICEFRAC,LANDFRAC,OCNFRAC,PSL,FSNTC,FSNTOAC,SWCF,LWCF,FLUT,FSNT,FSNTOA,FLNT,FLNTC,FSNS,FLNS,FSNS,SHFLX,QFLX,LHFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,U10,QREFHT,TMQ,CLDTOT,CLDHGH,CLDMED,CLDLOW,FLDS,FSDS,TGCLDIWP,TGCLDCWP,TGCLDLWP,FLNSC,FLUTC,FSDSC,SOLIN,FSNSC,AODABS,AODVIS,AODDUST,AREL,TREFMNAV,TREFMXAV,PS,PHIS,U,V,T,Z3" + +[pcmdi_diags] +active = True +environment_commands_secondary = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20251007-test1" +walltime = "4:00:00" +# Create rules to construct cmip-like datasets with a string constructed as +# "mip.exp.product.realization", where "."is used as the delimiter (mandatory) +model_name = 'e3sm.amip.v3-LR.0101' +model_tableID = 'Amon' +multiprocessing = True +num_workers = 24 +# Observational data +obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series' +pcmdi_debug = False +ts_years = "2005-2014", + + [[ mean_climate ]] + active = True + # Default for all variables or specify source for each variable separately. + obs_sets = "default" + ref_final_yr = 2018 + ref_start_yr = 1979 + ref_years = "2005-2014", + target_grid = "2.5x2.5" + target_grid_string = "2p5x2p5" + # Variables in current pcmdi-cmip list + # rltcre and rstcre are derived variables as they are not included in cmip_ts + # define a new variable name to be flexible + clim_vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rsus,rsuscs,rlus,rlut,rlutcs,rsds,rsdscs,rsdt,rsut,rsutcs,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" + # See zppy/templates/pcmdi_diags/regions_specs.json for options of all regions + # Below is default setup for global/regional mean climate metrics + clim_regions = "global,ocean,land,Tropics,NHEX,SHEX" + + [[ variability_modes_cpl ]] + active = True + # See zppy/templates/pcmdi_diags/reference_alias.json for options + obs_sets = "alternate1" #"default" + ref_final_yr = 2019 + ref_start_yr = 1870 + ref_years = "2005-2014", + frequency = 'mo' + landmask = True + ModUnitsAdjust = '(True,"subtract",273.15)' + ObsUnitsAdjust = '(True,"subtract",273.15)' + seasons = "yearly,monthly" + movc_vars = "ts" + # Name of modes varibility + movc_modes = "PDO,NPGO,AMO" + + [[ variability_modes_atm ]] + active = True + # See zppy/templates/pcmdi_diags/reference_alias.json for options + # Default for all variables + obs_sets = "alternate2" # "default" + ref_final_yr = 2015 + ref_start_yr = 1853 + ref_years = "2005-2014", + frequency = 'mo' + landmask = False + ModUnitsAdjust = '(True,"divide",100.0)' + ObsUnitsAdjust = '(True,"divide",100.0)' + seasons = "DJF,MAM,JJA,SON,yearly,monthly" + mova_vars = "psl" + # Varibility modes + mova_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" + + [[ enso ]] + active = False + # For enso, the obs_sets is explicitely set for each variable to ensure that + # they are from the same source, below are setup to used ERA5 reanalysis. + obs_sets = "default,alternate2,default,default,default,default,default,default,default,alternate2,alternate2,alternate2,alternate2,alternate2" + ref_final_yr = 2018 + ref_start_yr = 1979 + ref_years = "2005-2014", + # See zppy/templates/pcmdi_diags/reference_alias.json for options + enso_groups = "ENSO_perf,ENSO_proc,ENSO_tel" + enso_vars = "psl,pr,prsn,ts,tas,tauu,tauv,hfls,hfss,rlds,rsds,rlus,rlut,rsdt" + + [[ synthetic_plots ]] + active = True + # use subset variables for viewer + clim_vars = "pr,prw,psl,rlds,rldscs,ua-200,zg-500" + # use subset regions for viewer + clim_regions = "global,ocean,land" + # use subset modes for viewer + mova_modes = "NAM,NAO,PNA,NPO,SAM" + # use subset modes for viewer + movc_modes = "PDO,NPGO" + # mean climate + clim_viewer = True + # period to be plotted and shown on webpage + # note that the previous sections allows multiple ts_years periods for processing, while + # the viewer page focused on period customized by user + clim_years = "2005-2014" + # modes variability (atmosphere) + mova_viewer = True + # period to be plotted and shown on webpage + # note that the previous sections allows multiple ts_years periods for processing, while + # the viewer page focused on period customized by user + mova_years = "2005-2014" + # modes variability (coupled) + movc_viewer = True + # period to be plotted and shown on webpage + # note that the previous sections allows multiple ts_years periods for processing, while + # the viewer page focused on period customized by user + movc_years = "2005-2014" + # enso + enso_viewer = False + enso_years = "" + # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/mean_climate/cmip6/amip + cmip_clim_set = "cmip6.amip.v20250702" + # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes/cmip6/amip + cmip_movs_set = "cmip6.amip.v20210119" + # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/enso_metric/cmip6/historical + # There is no enso metric for CMIP6 AMIP, so use historical instead + cmip_enso_set = "cmip6.historical.v20210620" diff --git a/examples/post.v3.LR.historical.0101.cfg b/examples/post.v3.LR.historical.0101.cfg new file mode 100644 index 00000000..432049f2 --- /dev/null +++ b/examples/post.v3.LR.historical.0101.cfg @@ -0,0 +1,152 @@ +[default] +input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/v3.LR.historical_0101 +output = /lcrc/group/e3sm/ac.forsyth2/zppy_pmp_hist/unique_id_20251007_4/v3.LR.amip_0101 +www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_pmp_hist/unique_id_20251007_4 +campaign = "water_cycle" +case = v3.LR.historical_0101 +debug = False +environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" + +[ts] +active = True +walltime = "00:10:00" +years = "2005:2015:5" + + [[ atm_monthly_180x360_aave ]] + input_subdir = "archive/atm/hist" + input_files = "eam.h0" + frequency = "monthly" + mapping_file = /home/ac.zender/data/maps/map_ne30pg2_to_cmip6_180x360_aave.20200201.nc + # list for pcmdi diag, note: PHIS,hyam,hybm,hyai,hybi need to be included to process the 3D fields + vars = "ICEFRAC,LANDFRAC,OCNFRAC,PSL,FSNTC,FSNTOAC,SWCF,LWCF,FLUT,FSNT,FSNTOA,FLNT,FLNTC,FSNS,FLNS,FSNS,SHFLX,QFLX,LHFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,U10,QREFHT,TMQ,CLDTOT,CLDHGH,CLDMED,CLDLOW,FLDS,FSDS,TGCLDIWP,TGCLDCWP,TGCLDLWP,FLNSC,FLUTC,FSDSC,SOLIN,FSNSC,AODABS,AODVIS,AODDUST,AREL,TREFMNAV,TREFMXAV,PS,PHIS,U,V,T,Z3" + +[e3sm_to_cmip] +active = True +frequency = "monthly" +ts_grid = "180x360_aave" +ts_num_years=5 +walltime = "00:10:00" +years = "2005:2015:5" + + [[ atm_monthly_180x360_aave ]] + cmip_plevdata = "/lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc" + input_files = "eam.h0" + interp_vars = "U,V,T,Z3" + ts_subsection = "atm_monthly_180x360_aave" + vars = "ICEFRAC,LANDFRAC,OCNFRAC,PSL,FSNTC,FSNTOAC,SWCF,LWCF,FLUT,FSNT,FSNTOA,FLNT,FLNTC,FSNS,FLNS,FSNS,SHFLX,QFLX,LHFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,U10,QREFHT,TMQ,CLDTOT,CLDHGH,CLDMED,CLDLOW,FLDS,FSDS,TGCLDIWP,TGCLDCWP,TGCLDLWP,FLNSC,FLUTC,FSDSC,SOLIN,FSNSC,AODABS,AODVIS,AODDUST,AREL,TREFMNAV,TREFMXAV,PS,PHIS,U,V,T,Z3" + +[pcmdi_diags] +active = True +environment_commands_secondary = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20251007-test1" +walltime = "4:00:00" +# Create rules to construct cmip-like datasets with a string constructed as +# "mip.exp.product.realization", where "."is used as the delimiter (mandatory) +model_name = 'e3sm.historical.v3-LR.0101' +model_tableID = 'Amon' +multiprocessing = True +num_workers = 24 +# Observational data +obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series' +pcmdi_debug = False +ts_years = "2005-2014", + + [[ mean_climate ]] + active = True + # Default for all variables or specify source for each variable separately. + obs_sets = "default" + ref_final_yr = 2018 + ref_start_yr = 1979 + ref_years = "2005-2014", + target_grid = "2.5x2.5" + target_grid_string = "2p5x2p5" + # Variables in current pcmdi-cmip list + # rltcre and rstcre are derived variables as they are not included in cmip_ts + # define a new variable name to be flexible + clim_vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rsus,rsuscs,rlus,rlut,rlutcs,rsds,rsdscs,rsdt,rsut,rsutcs,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" + # See zppy/templates/pcmdi_diags/regions_specs.json for options of all regions + # Below is default setup for global/regional mean climate metrics + clim_regions = "global,ocean,land,Tropics,NHEX,SHEX" + + [[ variability_modes_cpl ]] + active = True + # See zppy/templates/pcmdi_diags/reference_alias.json for options + obs_sets = "alternate1" #"default" + ref_final_yr = 2019 + ref_start_yr = 1870 + ref_years = "2005-2014", + frequency = 'mo' + landmask = True + ModUnitsAdjust = '(True,"subtract",273.15)' + ObsUnitsAdjust = '(True,"subtract",273.15)' + seasons = "yearly,monthly" + movc_vars = "ts" + # Name of modes varibility + movc_modes = "PDO,NPGO,AMO" + + [[ variability_modes_atm ]] + active = True + # See zppy/templates/pcmdi_diags/reference_alias.json for options + # Default for all variables + obs_sets = "alternate2" # "default" + ref_final_yr = 2015 + ref_start_yr = 1853 + ref_years = "2005-2014", + frequency = 'mo' + landmask = False + ModUnitsAdjust = '(True,"divide",100.0)' + ObsUnitsAdjust = '(True,"divide",100.0)' + seasons = "DJF,MAM,JJA,SON,yearly,monthly" + mova_vars = "psl" + # Varibility modes + mova_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" + + [[ enso ]] + active = False + # For enso, the obs_sets is explicitely set for each variable to ensure that + # they are from the same source, below are setup to used ERA5 reanalysis. + obs_sets = "default,alternate2,default,default,default,default,default,default,default,alternate2,alternate2,alternate2,alternate2,alternate2" + ref_final_yr = 2018 + ref_start_yr = 1979 + ref_years = "2005-2014", + # See zppy/templates/pcmdi_diags/reference_alias.json for options + enso_groups = "ENSO_perf,ENSO_proc,ENSO_tel" + enso_vars = "psl,pr,prsn,ts,tas,tauu,tauv,hfls,hfss,rlds,rsds,rlus,rlut,rsdt" + + [[ synthetic_plots ]] + active = True + # use subset variables for viewer + clim_vars = "pr,prw,psl,rlds,rldscs,ua-200,zg-500" + # use subset regions for viewer + clim_regions = "global,ocean,land" + # use subset modes for viewer + mova_modes = "NAM,NAO,PNA,NPO,SAM" + # use subset modes for viewer + movc_modes = "PDO,NPGO" + # mean climate + clim_viewer = True + # period to be plotted and shown on webpage + # note that the previous sections allows multiple ts_years periods for processing, while + # the viewer page focused on period customized by user + clim_years = "2005-2014" + # modes variability (atmosphere) + mova_viewer = True + # period to be plotted and shown on webpage + # note that the previous sections allows multiple ts_years periods for processing, while + # the viewer page focused on period customized by user + mova_years = "2005-2014" + # modes variability (coupled) + movc_viewer = True + # period to be plotted and shown on webpage + # note that the previous sections allows multiple ts_years periods for processing, while + # the viewer page focused on period customized by user + movc_years = "2005-2014" + # enso + enso_viewer = False + enso_years = "" + # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/mean_climate/cmip6/historical + cmip_clim_set = "cmip6.historical.v20250707" + # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes/cmip6/historical + cmip_movs_set = "cmip6.historical.v20220825" + # Location: /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/enso_metric/cmip6/historical + # There is no enso metric for CMIP6 AMIP, so use historical instead + cmip_enso_set = "cmip6.historical.v20210620" diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..36d4982f --- /dev/null +++ b/setup.py @@ -0,0 +1,34 @@ +import os + +from setuptools import find_packages, setup + + +# From https://github.com/MPAS-Dev/compass/blob/master/setup.py +def package_files(directory, prefixes, extensions): + """based on https://stackoverflow.com/a/36693250/7728169""" + paths = [] + for path, directories, filenames in os.walk(directory): + for filename in filenames: + parts = filename.split(".") + prefix = parts[0] + extension = parts[-1] + if prefix in prefixes or extension in extensions: + paths.append(os.path.join("..", path, filename)) + return paths + + +data_files = package_files( + "zppy", prefixes=[], extensions=["bash", "csh", "cfg", "ini", "sh", "json"] +) + +setup( + name="zppy", + version="3.0.0", + author="Ryan Forsyth, Chris Golaz", + author_email="forsyth2@llnl.gov, golaz1@llnl.gov", + description="Post-processing software for E3SM", + python_requires=">=3.9", + packages=find_packages(include=["zppy", "zppy.*"]), + package_data={"": data_files}, + entry_points={"console_scripts": ["zppy=zppy.__main__:main"]}, +) diff --git a/tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg b/tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg index 9c7ae047..12fa232c 100644 --- a/tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg +++ b/tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg @@ -176,7 +176,7 @@ partition = "compute" qos = "regular" shortTermArchive = True ts_years = "1980-1984", "1980-1990", -walltime = "02:00:00" +walltime = "00:30:00" [global_time_series] active = True diff --git a/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg b/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg index d823964b..eb2d6c71 100644 --- a/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg +++ b/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg @@ -9,10 +9,10 @@ infer_section_parameters = False input = /lcrc/group/e3sm2/ac.wlin//E3SMv3/v3.LR.historical_0051 input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_4/v3.LR.historical_0051" partition = "debug" qos = "regular" -www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_comprehensive_v3_www/unique_id" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_comprehensive_v3_www/unique_id_test_20251007_4" years = "1985:1989:2", [climo] @@ -26,14 +26,14 @@ walltime = "00:30:00" vars = "" [[ atm_monthly_diurnal_8xdaily_180x360_aave ]] - active = True + active = False frequency = "diurnal_8xdaily" input_files = "eam.h3" input_subdir = "archive/atm/hist" vars = "PRECT" [[ land_monthly_climo ]] - active = True + active = False frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" @@ -48,16 +48,17 @@ walltime = "00:30:00" frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" + years = "1985:1995:2", # Need 10 years for pcmdi_diags task [[ atm_daily_180x360_aave ]] - active = True + active = False frequency = "daily" input_files = "eam.h1" input_subdir = "archive/atm/hist" vars = "PRECT" [[ rof_monthly ]] - active = True + active = False extra_vars = 'areatotal2' frequency = "monthly" input_files = "mosart.h0" @@ -67,7 +68,7 @@ walltime = "00:30:00" [[ atm_monthly_glb ]] # Note global average won't work for 3D variables. - active = True + active = False frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" @@ -75,7 +76,7 @@ walltime = "00:30:00" years = "1985:1995:5", [[ lnd_monthly_glb ]] - active = True + active = False frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" @@ -98,8 +99,10 @@ ts_num_years=2 walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] + cmip_plevdata = "/lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc" input_files = "eam.h0" ts_subsection = "atm_monthly_180x360_aave" + years = "1985:1995:2", # Need 10 years for pcmdi_diags task [[ land_monthly ]] input_files = "elm.h0" @@ -111,11 +114,11 @@ walltime = "00:30:00" # walltime = "00:30:00" [e3sm_diags] -active = True +active = False climo_diurnal_frequency = "diurnal_8xdaily" climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave" climo_subsection = "atm_monthly_180x360_aave" -environment_commands = "source /conda.sh; conda activate " +environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" grid = '180x360_aave' multiprocessing = True num_workers = 8 @@ -167,16 +170,16 @@ tc_obs = "/lcrc/group/e3sm/diagnostics/observations/Atm/tc-analysis/" tag = "model_vs_model" ts_num_years_ref = 2 # Reference paths - reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/atm/180x360_aave/clim" + reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_4/v3.LR.historical_0051/post/atm/180x360_aave/clim" # mvm streamflow only gauges_path = "/lcrc/group/e3sm/diagnostics/observations/Atm/time-series/GSIM/GSIM_catchment_characteristics_all_1km2.csv" - reference_data_path_ts_rof = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/rof/native/ts/monthly" + reference_data_path_ts_rof = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_4/v3.LR.historical_0051/post/rof/native/ts/monthly" # mvm diurnal_cycle only - reference_data_path_climo_diurnal = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/atm/180x360_aave/clim_diurnal_8xdaily" + reference_data_path_climo_diurnal = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_4/v3.LR.historical_0051/post/atm/180x360_aave/clim_diurnal_8xdaily" # mvm "enso_diags", "qbo", "area_mean_time_series" - reference_data_path_ts = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/atm/180x360_aave/ts/monthly" + reference_data_path_ts = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_4/v3.LR.historical_0051/post/atm/180x360_aave/ts/monthly" # mvm tropical_subseasonal only - reference_data_path_ts_daily = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/atm/180x360_aave/ts/daily" + reference_data_path_ts_daily = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_4/v3.LR.historical_0051/post/atm/180x360_aave/ts/daily" [[ lnd_monthly_mvm_lnd ]] # Test model-vs-model using the same files as the reference @@ -192,11 +195,10 @@ tc_obs = "/lcrc/group/e3sm/diagnostics/observations/Atm/tc-analysis/" tag = "model_vs_model" ts_num_years_ref = 2 # Reference paths - reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/lnd/180x360_aave/clim" - + reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_test_20251007_4/v3.LR.historical_0051/post/lnd/180x360_aave/clim" [mpas_analysis] -active = True +active = False anomalyRefYear = 1985 climo_years = "1985-1989", "1990-1995", enso_years = "1985-1989", "1990-1995", @@ -206,12 +208,12 @@ partition = "compute" qos = "regular" shortTermArchive = True ts_years = "1985-1989", "1985-1995", -walltime = "02:00:00" +walltime = "00:30:00" [global_time_series] -active = True +active = False climo_years = "1985-1989", "1990-1995", -environment_commands = "source /conda.sh; conda activate " +environment_commands = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20251007-test1" experiment_name = "v3.LR.historical_0051" figstr = "v3.LR.historical_0051" #moc_file=mocTimeSeries_1985-1995.nc @@ -295,7 +297,7 @@ years = "1985-1995", [ilamb] -active = True +active = False e3sm_to_cmip_atm_subsection = "atm_monthly_180x360_aave" e3sm_to_cmip_land_subsection = "land_monthly" ilamb_obs = "/lcrc/group/e3sm/diagnostics/ilamb_data" @@ -307,3 +309,77 @@ ts_land_subsection = "land_monthly" ts_num_years = 2 walltime = "2:00:00" years = "1985:1989:4" + +[pcmdi_diags] +active = True +environment_commands_secondary = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20251007-test1" +walltime = "2:00:00" +model_name = 'e3sm.historical.v3-LR.0051' +model_tableID = 'Amon' +e3sm_to_cmip_atm_subsection = "atm_monthly_180x360_aave" # Can be inferred if needed +multiprocessing = True +num_workers = 24 +obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series' +pcmdi_debug = False +ts_num_years = 2 +ts_years = "1985-1994", + + [[ mean_climate ]] + active = True + current_set = "mean_climate" # Can be inferred if needed + obs_sets = "default" + ref_final_yr = 1994 + ref_start_yr = 1985 + ref_years = "1985-1994", + target_grid = "2.5x2.5" + target_grid_string = "2p5x2p5" + clim_vars = "pr,rlut,rtmt,tas,tauu,tauv,ts" + clim_regions = "global,ocean,land,NHEX,SHEX,TROPICS" + + [[ variability_modes_cpl ]] + active = True + current_set = "variability_modes_cpl" # Can be inferred if needed + obs_sets = "alternate1" + ref_final_yr = 1994 + ref_start_yr = 1985 + ref_years = "1985-1994", + frequency = 'mo' + landmask = True + ModUnitsAdjust = '(True,"subtract",273.15)' + ObsUnitsAdjust = '(True,"subtract",273.15)' + seasons = "yearly,monthly" + movc_vars = "ts" + movc_modes = "PDO,NPGO,AMO" + + [[ variability_modes_atm ]] + active = True + current_set = "variability_modes_atm" # Can be inferred if needed + obs_sets = "alternate2" + ref_final_yr = 1994 + ref_start_yr = 1985 + ref_years = "1985-1994", + frequency = 'mo' + landmask = False + ModUnitsAdjust = '(True,"divide",100.0)' + ObsUnitsAdjust = '(True,"divide",100.0)' + seasons = "DJF,MAM,JJA,SON,yearly,monthly" + mova_vars = "psl" + mova_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" + + [[ synthetic_plots ]] + active = True + current_set = "synthetic_plots" # Can be inferred if needed + clim_vars = "pr,rlut,rtmt,tas,tauu,tauv,ts" + clim_regions = "global,ocean,land,NHEX,SHEX,TROPICS" + mova_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" + movc_modes = "PDO,NPGO,AMO" + clim_viewer = True + clim_years = "2005-2014" + mova_viewer = True + mova_years = "2005-2014" + movc_viewer = True + movc_years = "2005-2014" + enso_viewer = False + enso_years = "" + cmip_clim_dir = /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/mean_climate + cmip_movs_dir = /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes diff --git a/tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v2_chrysalis.cfg b/tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v2_chrysalis.cfg index 60568c98..955ac524 100644 --- a/tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v2_chrysalis.cfg +++ b/tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v2_chrysalis.cfg @@ -176,7 +176,7 @@ partition = "compute" qos = "regular" shortTermArchive = True ts_years = "1980-1984", "1980-1990", -walltime = "02:00:00" +walltime = "00:30:00" [global_time_series] active = True diff --git a/tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v3_chrysalis.cfg b/tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v3_chrysalis.cfg index 0fb66fbd..94fe4054 100644 --- a/tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v3_chrysalis.cfg +++ b/tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v3_chrysalis.cfg @@ -206,7 +206,7 @@ partition = "compute" qos = "regular" shortTermArchive = True ts_years = "1985-1989", "1985-1995", -walltime = "02:00:00" +walltime = "00:30:00" [global_time_series] active = True diff --git a/tests/integration/generated/update_weekly_expected_files_chrysalis.sh b/tests/integration/generated/update_weekly_expected_files_chrysalis.sh index a1676a23..73973b26 100755 --- a/tests/integration/generated/update_weekly_expected_files_chrysalis.sh +++ b/tests/integration/generated/update_weekly_expected_files_chrysalis.sh @@ -35,17 +35,17 @@ do # Copy output so you don't have to rerun zppy to generate the output. if [[ "${test_name,,}" =~ "v2" ]]; then # We need the v2 case name - cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name} + cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id_test_20251007_4/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name} else # We need the v3 case name - cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id/v3.LR.historical_0051 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name} + cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id_test_20251007_4/v3.LR.historical_0051 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name} fi # test_bundles.py also needs the bash files transferred. # Note that for legacy cfgs, we're only testing test_images.py if [[ "${test_name,,}" == "bundles" ]]; then mkdir -p /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files - cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_weekly_bundles_output/unique_id/v3.LR.historical_0051/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files + cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_weekly_bundles_output/unique_id_test_20251007_4/v3.LR.historical_0051/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files fi zppy_top_level=$(pwd) diff --git a/tests/integration/image_checker.py b/tests/integration/image_checker.py index 53ce8fdb..f6a8da7b 100644 --- a/tests/integration/image_checker.py +++ b/tests/integration/image_checker.py @@ -76,6 +76,9 @@ def set_up_and_run_image_checker( print(f"{key}: {d[key]}") parameters: Parameters = Parameters(d) for task in task_list: + if task == "pcmdi_diags": + print(f"{task} hs no expected results yet, skipping.") + continue test_results = check_images(parameters, task) test_results_dict[f"{cfg_specifier}_{task}"] = test_results diff --git a/tests/integration/template_weekly_comprehensive_v3.cfg b/tests/integration/template_weekly_comprehensive_v3.cfg index ffba6f10..9a2cdf53 100644 --- a/tests/integration/template_weekly_comprehensive_v3.cfg +++ b/tests/integration/template_weekly_comprehensive_v3.cfg @@ -48,6 +48,7 @@ walltime = "00:30:00" frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" + years = "1985:1995:2", # Need 10 years for pcmdi_diags task [[ atm_daily_180x360_aave ]] active = #expand active_e3sm_diags# @@ -98,8 +99,10 @@ ts_num_years=2 walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] + cmip_plevdata = "/lcrc/group/e3sm/ac.szhang/acme_scratch/e3sm_project/zppy/zppy/templates/inclusions/e3sm_to_cmip/vrt_remap_plev19.nc" input_files = "eam.h0" ts_subsection = "atm_monthly_180x360_aave" + years = "1985:1995:2", # Need 10 years for pcmdi_diags task [[ land_monthly ]] input_files = "elm.h0" @@ -194,7 +197,6 @@ tc_obs = "#expand diagnostics_base_path#/observations/Atm/tc-analysis/" # Reference paths reference_data_path = "#expand user_output#zppy_weekly_comprehensive_v3_output/#expand unique_id#/#expand case_name#/post/lnd/180x360_aave/clim" - [mpas_analysis] active = #expand active_mpas_analysis# anomalyRefYear = 1985 @@ -307,3 +309,77 @@ ts_land_subsection = "land_monthly" ts_num_years = 2 walltime = "2:00:00" years = "1985:1989:4" + +[pcmdi_diags] +active = True +environment_commands_secondary = "#expand pcmdi_diags_environment_commands#" +walltime = "2:00:00" +model_name = 'e3sm.historical.v3-LR.0051' +model_tableID = 'Amon' +e3sm_to_cmip_atm_subsection = "atm_monthly_180x360_aave" # Can be inferred if needed +multiprocessing = True +num_workers = 24 +obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series' +pcmdi_debug = False +ts_num_years = 2 +ts_years = "1985-1994", + + [[ mean_climate ]] + active = True + current_set = "mean_climate" # Can be inferred if needed + obs_sets = "default" + ref_final_yr = 1994 + ref_start_yr = 1985 + ref_years = "1985-1994", + target_grid = "2.5x2.5" + target_grid_string = "2p5x2p5" + clim_vars = "pr,rlut,rtmt,tas,tauu,tauv,ts" + clim_regions = "global,ocean,land,NHEX,SHEX,TROPICS" + + [[ variability_modes_cpl ]] + active = True + current_set = "variability_modes_cpl" # Can be inferred if needed + obs_sets = "alternate1" + ref_final_yr = 1994 + ref_start_yr = 1985 + ref_years = "1985-1994", + frequency = 'mo' + landmask = True + ModUnitsAdjust = '(True,"subtract",273.15)' + ObsUnitsAdjust = '(True,"subtract",273.15)' + seasons = "yearly,monthly" + movc_vars = "ts" + movc_modes = "PDO,NPGO,AMO" + + [[ variability_modes_atm ]] + active = True + current_set = "variability_modes_atm" # Can be inferred if needed + obs_sets = "alternate2" + ref_final_yr = 1994 + ref_start_yr = 1985 + ref_years = "1985-1994", + frequency = 'mo' + landmask = False + ModUnitsAdjust = '(True,"divide",100.0)' + ObsUnitsAdjust = '(True,"divide",100.0)' + seasons = "DJF,MAM,JJA,SON,yearly,monthly" + mova_vars = "psl" + mova_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" + + [[ synthetic_plots ]] + active = True + current_set = "synthetic_plots" # Can be inferred if needed + clim_vars = "pr,rlut,rtmt,tas,tauu,tauv,ts" + clim_regions = "global,ocean,land,NHEX,SHEX,TROPICS" + mova_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2" + movc_modes = "PDO,NPGO,AMO" + clim_viewer = True + clim_years = "2005-2014" + mova_viewer = True + mova_years = "2005-2014" + movc_viewer = True + movc_years = "2005-2014" + enso_viewer = False + enso_years = "" + cmip_clim_dir = #expand diagnostics_base_path#/pcmdi_data/metrics_data/mean_climate + cmip_movs_dir = #expand diagnostics_base_path#/pcmdi_data/metrics_data/variability_modes diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 6187408d..927b6a91 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -12,18 +12,16 @@ # pytest tests/integration/test_*.py TEST_SPECIFICS: Dict[str, Any] = { - "diags_environment_commands": "source /conda.sh; conda activate ", - "global_time_series_environment_commands": "source /conda.sh; conda activate ", + "diags_environment_commands": "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh", + "global_time_series_environment_commands": "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20251007-test1", + "pcmdi_diags_environment_commands": "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-pcmdi-diags-20251007-test1", "cfgs_to_run": [ - "weekly_bundles", - "weekly_comprehensive_v2", "weekly_comprehensive_v3", - "weekly_legacy_3.0.0_bundles", - "weekly_legacy_3.0.0_comprehensive_v2", - "weekly_legacy_3.0.0_comprehensive_v3", ], - "tasks_to_run": ["e3sm_diags", "mpas_analysis", "global_time_series", "ilamb"], - "unique_id": "unique_id", + "tasks_to_run": [ + "pcmdi_diags", + ], + "unique_id": "unique_id_test_20251007_4", } # Multi-machine testing ######################################################### @@ -132,6 +130,9 @@ def get_expansions(): expansions["global_time_series_environment_commands"] = TEST_SPECIFICS[ "global_time_series_environment_commands" ] + expansions["pcmdi_diags_environment_commands"] = TEST_SPECIFICS[ + "pcmdi_diags_environment_commands" + ] # Activate requested tests expansions["active_e3sm_to_cmip"] = "False" @@ -139,6 +140,7 @@ def get_expansions(): expansions["active_mpas_analysis"] = "False" expansions["active_global_time_series"] = "False" expansions["active_ilamb"] = "False" + expansions["active_pcmdi_diags"] = "False" if "e3sm_diags" in TEST_SPECIFICS["tasks_to_run"]: expansions["active_e3sm_diags"] = "True" if "mpas_analysis" in TEST_SPECIFICS["tasks_to_run"]: @@ -150,6 +152,9 @@ def get_expansions(): if "ilamb" in TEST_SPECIFICS["tasks_to_run"]: expansions["active_ilamb"] = "True" expansions["active_e3sm_to_cmip"] = "True" + if "pcmdi_diags" in TEST_SPECIFICS["tasks_to_run"]: + expansions["active_pcmdi_diags"] = "True" + expansions["active_e3sm_to_cmip"] = "True" expansions["cfgs_to_run"] = TEST_SPECIFICS["cfgs_to_run"] expansions["tasks_to_run"] = TEST_SPECIFICS["tasks_to_run"] @@ -302,6 +307,9 @@ def generate_cfgs(unified_testing=False, dry_run=False): print( f"global_time_series_environment_commands={expansions['global_time_series_environment_commands']}" ) + print( + f"pcmdi_diags_environment_commands={expansions['pcmdi_diags_environment_commands']}" + ) print(f"environment_commands={expansions['environment_commands']}") print( "Reminder: `environment_commands=''` => the latest E3SM Unified environment will be used" diff --git a/tests/test_sections.py b/tests/test_sections.py index 218ecc66..018244d7 100644 --- a/tests/test_sections.py +++ b/tests/test_sections.py @@ -73,6 +73,7 @@ def test_sections(): "debug": False, "dry_run": False, "environment_commands": "", + "environment_commands_secondary": "", "fail_on_dependency_skip": False, "frequency": "monthly", "grid": "", @@ -97,7 +98,7 @@ def test_sections(): "ts_land_subsection": "", "ts_num_years": 5, "ts_subsection": "", - "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U", + "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,PSL", "walltime": "02:00:00", "www": "WWWW", "years": [""], @@ -134,6 +135,7 @@ def test_sections(): "dpf": 30, "dry_run": False, "environment_commands": "", + "environment_commands_secondary": "", "extra_vars": "", "fail_on_dependency_skip": False, "frequency": "monthly", @@ -196,6 +198,7 @@ def test_sections(): "debug": False, "dry_run": False, "environment_commands": "", + "environment_commands_secondary": "", "exclude": False, "fail_on_dependency_skip": False, "frequency": "monthly", @@ -260,6 +263,7 @@ def test_subsections(): "debug": False, "dry_run": False, "environment_commands": "", + "environment_commands_secondary": "", "fail_on_dependency_skip": False, "frequency": "monthly", "grid": "", @@ -284,7 +288,7 @@ def test_subsections(): "ts_land_subsection": "", "ts_num_years": 5, "ts_subsection": "", - "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U", + "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,PSL", "walltime": "02:00:00", "www": "WWWW", "years": [""], @@ -337,6 +341,7 @@ def test_subsections(): "dpf": 30, "dry_run": False, "environment_commands": "", + "environment_commands_secondary": "", "extra_vars": "", "fail_on_dependency_skip": False, "frequency": "monthly", @@ -384,6 +389,7 @@ def test_subsections(): "dpf": 30, "dry_run": False, "environment_commands": "", + "environment_commands_secondary": "", "extra_vars": "", "fail_on_dependency_skip": False, "frequency": "monthly", @@ -462,6 +468,7 @@ def test_subsections(): "debug": False, "dry_run": False, "environment_commands": "", + "environment_commands_secondary": "", "exclude": False, "fail_on_dependency_skip": False, "frequency": "monthly", @@ -506,6 +513,7 @@ def test_subsections(): "debug": False, "dry_run": False, "environment_commands": "", + "environment_commands_secondary": "", "exclude": False, "fail_on_dependency_skip": False, "frequency": "monthly", diff --git a/tests/test_zppy_e3sm_diags.py b/tests/test_zppy_e3sm_diags.py index f69f7cc4..cba17877 100644 --- a/tests/test_zppy_e3sm_diags.py +++ b/tests/test_zppy_e3sm_diags.py @@ -7,40 +7,11 @@ add_ts_dependencies, check_and_define_parameters, check_mvm_only_parameters_for_bash, - check_parameter_defined, check_parameters_for_bash, - check_set_specific_parameter, ) from zppy.utils import ParameterNotProvidedError -def test_check_parameter_defined(): - c = {"a": 1, "b": 2, "c": ""} - check_parameter_defined(c, "a") - with pytest.raises(ParameterNotProvidedError): - check_parameter_defined(c, "c") - with pytest.raises(ParameterNotProvidedError): - check_parameter_defined(c, "d") - - -def test_check_set_specific_parameter(): - # Parameter is required - # a, b need parameter p, and we want sets a, b, c - c = {"sets": ["a", "b", "c"], "p": "exists"} - check_set_specific_parameter(c, set(["a", "b"]), "p") - - # Parameter isn't required based on the sets we want - # z needs parameter p, but we only want sets a, b, c - c = {"sets": ["a", "b", "c"], "p": ""} - check_set_specific_parameter(c, set(["z"]), "p") - - # Parameter is required - # a, b need parameter p, and we want sets a, b, c - c = {"sets": ["a", "b", "c"], "p": ""} - with pytest.raises(ParameterNotProvidedError): - check_set_specific_parameter(c, set(["a", "b"]), "p") - - def test_check_parameters_for_bash(): # diurnal_cycle c = {"sets": ["diurnal_cycle"], "climo_diurnal_frequency": "diurnal_8xdaily"} diff --git a/tests/test_zppy_pcmdi_diags.py b/tests/test_zppy_pcmdi_diags.py new file mode 100644 index 00000000..ef8656b9 --- /dev/null +++ b/tests/test_zppy_pcmdi_diags.py @@ -0,0 +1,668 @@ +from typing import List, Tuple +from unittest.mock import patch + +import pytest + +from zppy.pcmdi_diags import ( + add_pcmdi_dependencies, + add_ts_dependencies, + check_and_define_parameters, + check_mvm_only_parameters_for_bash, + check_parameters_for_bash, + check_parameters_for_pcmdi, + define_current_set, + define_relevant_years, + define_relevant_years_for_synthetic_plots, + define_year_sets, +) +from zppy.utils import ParameterNotProvidedError + + +def test_define_current_set(): + # Set current_set directly + c = {"current_set": "mean_climate", "infer_path_parameters": False} + define_current_set(c) + assert c["current_set"] == "mean_climate" + + c = {"current_set": "variability_modes_cpl", "infer_path_parameters": False} + define_current_set(c) + assert c["current_set"] == "variability_modes_cpl" + + c = {"current_set": "variability_modes_atm", "infer_path_parameters": False} + define_current_set(c) + assert c["current_set"] == "variability_modes_atm" + + c = {"current_set": "enso", "infer_path_parameters": False} + define_current_set(c) + assert c["current_set"] == "enso" + + c = {"current_set": "synthetic_plots", "infer_path_parameters": False} + define_current_set(c) + assert c["current_set"] == "synthetic_plots" + + c = {"current_set": "invalid_set", "infer_path_parameters": False} + with pytest.raises(ValueError): + define_current_set(c) + + # Infer current_set from subsection + c = {"current_set": "", "subsection": "mean_climate", "infer_path_parameters": True} + define_current_set(c) + assert c["current_set"] == "mean_climate" + + c = { + "current_set": "", + "subsection": "variability_modes_cpl", + "infer_path_parameters": True, + } + define_current_set(c) + assert c["current_set"] == "variability_modes_cpl" + + c = { + "current_set": "", + "subsection": "variability_modes_atm", + "infer_path_parameters": True, + } + define_current_set(c) + assert c["current_set"] == "variability_modes_atm" + + c = {"current_set": "", "subsection": "enso", "infer_path_parameters": True} + define_current_set(c) + assert c["current_set"] == "enso" + + c = { + "current_set": "", + "subsection": "synthetic_plots", + "infer_path_parameters": True, + } + define_current_set(c) + assert c["current_set"] == "synthetic_plots" + + c = {"current_set": "", "subsection": "invalid_set", "infer_path_parameters": True} + with pytest.raises(ValueError): + define_current_set(c) + + c = { + "current_set": "", + "subsection": "mean_climate", + "infer_path_parameters": False, + } + with pytest.raises(ParameterNotProvidedError): + define_current_set(c) + + # Test case where subsection is missing with infer_path_parameters=True + c = {"current_set": "", "infer_path_parameters": True} + with pytest.raises(ParameterNotProvidedError): + define_current_set(c) + + +def test_check_parameters_for_bash(): + # Test synthetic_plots (should not require ref_start_yr and ref_end_yr) + c = { + "current_set": "synthetic_plots", + "ref_final_yr": "2000", + "ref_start_yr": "1850", + } + check_parameters_for_bash(c) + + c = {"current_set": "synthetic_plots", "ref_final_yr": "", "ref_start_yr": "1850"} + check_parameters_for_bash(c) + + c = {"current_set": "synthetic_plots", "ref_final_yr": "2000", "ref_start_yr": ""} + check_parameters_for_bash(c) + + # Test BASE_PCMDI_SETS + c = {"current_set": "mean_climate", "ref_final_yr": "2000", "ref_start_yr": "1850"} + check_parameters_for_bash(c) + + c = {"current_set": "mean_climate", "ref_final_yr": "", "ref_start_yr": "1850"} + with pytest.raises(ParameterNotProvidedError): + check_parameters_for_bash(c) + + c = {"current_set": "mean_climate", "ref_final_yr": "2000", "ref_start_yr": ""} + with pytest.raises(ParameterNotProvidedError): + check_parameters_for_bash(c) + + c = { + "current_set": "variability_modes_atm", + "ref_final_yr": "2000", + "ref_start_yr": "1850", + } + check_parameters_for_bash(c) + + c = { + "current_set": "variability_modes_atm", + "ref_final_yr": "", + "ref_start_yr": "1850", + } + with pytest.raises(ParameterNotProvidedError): + check_parameters_for_bash(c) + + c = { + "current_set": "variability_modes_cpl", + "ref_final_yr": "2000", + "ref_start_yr": "1850", + } + check_parameters_for_bash(c) + + c = { + "current_set": "variability_modes_cpl", + "ref_final_yr": "2000", + "ref_start_yr": "", + } + with pytest.raises(ParameterNotProvidedError): + check_parameters_for_bash(c) + + c = {"current_set": "enso", "ref_final_yr": "2000", "ref_start_yr": "1850"} + check_parameters_for_bash(c) + + +def test_check_parameters_for_pcmdi(): + c = { + "current_set": "synthetic_plots", + "figure_sets": [ + "mean_climate", + "variability_modes", + ], + "cmip_enso_dir": "", + "cmip_clim_dir": "", + "cmip_movs_dir": "", + "enso_viewer": False, + "clim_viewer": True, + "mova_viewer": True, + "diagnostics_base_path": "diags/post", + "infer_path_parameters": True, + } + check_parameters_for_pcmdi(c) + assert c["cmip_clim_dir"] == "diags/post/pcmdi_data/metrics_data/mean_climate" + assert c["cmip_movs_dir"] == "diags/post/pcmdi_data/metrics_data/variability_modes" + assert c["cmip_enso_dir"] == "placeholder_dir" + + c = { + "current_set": "mean_climate", + "figure_sets": [ + "mean_climate", + "variability_modes", + ], + "cmip_enso_dir": "", + "cmip_clim_dir": "", + "cmip_movs_dir": "", + "diagnostics_base_path": "diags/post", + "infer_path_parameters": True, + } + check_parameters_for_pcmdi(c) + assert c["cmip_clim_dir"] == "" + assert c["cmip_movs_dir"] == "" + assert c["cmip_enso_dir"] == "" + + c = { + "current_set": "synthetic_plots", + "figure_sets": [ + "mean_climate", + "variability_modes", + ], + "cmip_enso_dir": "", + "cmip_clim_dir": "", + "cmip_movs_dir": "", + "enso_viewer": False, + "clim_viewer": True, + "diagnostics_base_path": "diags/post", + "infer_path_parameters": False, + } + with pytest.raises(ParameterNotProvidedError): + check_parameters_for_pcmdi(c) + + c = { + "current_set": "mean_climate", + "figure_sets": [ + "mean_climate", + "variability_modes", + ], + "cmip_enso_dir": "", + "cmip_clim_dir": "", + "cmip_movs_dir": "", + "diagnostics_base_path": "diags/post", + "infer_path_parameters": False, + } + check_parameters_for_pcmdi(c) + assert c["cmip_clim_dir"] == "" + assert c["cmip_movs_dir"] == "" + assert c["cmip_enso_dir"] == "" + + # Test enso_metric figure set + c = { + "current_set": "synthetic_plots", + "figure_sets": ["enso_metric"], + "cmip_enso_dir": "", + "cmip_clim_dir": "", + "cmip_movs_dir": "", + "enso_viewer": False, + "clim_viewer": True, + "mova_viewer": True, + "diagnostics_base_path": "diags/post", + "infer_path_parameters": True, + } + check_parameters_for_pcmdi(c) + assert c["cmip_enso_dir"] == "placeholder_dir" + + # Test when parameters are already defined + c = { + "current_set": "synthetic_plots", + "figure_sets": ["mean_climate"], + "cmip_enso_dir": "/custom/enso", + "cmip_clim_dir": "/custom/clim", + "cmip_movs_dir": "/custom/movs", + "enso_viewer": False, + "clim_viewer": True, + "mova_viewer": True, + "diagnostics_base_path": "diags/post", + "infer_path_parameters": True, + } + check_parameters_for_pcmdi(c) + assert c["cmip_clim_dir"] == "/custom/clim" # Should not change + + +def test_define_year_sets(): + # Should use ts_years + c = { + "current_set": "mean_climate", + "ts_years": "2000:2010:5", + } + actual: List[Tuple[int, int]] = define_year_sets(c) + expected = [(2000, 2004), (2005, 2009)] + assert actual == expected + + # Should use available year set + c = { + "current_set": "synthetic_plots", + "ts_years": "2000:2010:5", + "enso_years": "2001-2004", + } + actual = define_year_sets(c) + expected = [(2001, 2004)] + assert actual == expected + + # Should use the first available year set + c = { + "current_set": "synthetic_plots", + "ts_years": "2000:2010:5", + "clim_years": "2001-2004", + "mova_years": "2002-2003", + "movc_years": "2003-2004", + "enso_years": "2004-2005", + } + actual = define_year_sets(c) + expected = [(2001, 2004)] + assert actual == expected + + # Should default to ts_years + c = { + "current_set": "synthetic_plots", + "ts_years": "2000:2010:5", + } + actual = define_year_sets(c) + expected = [(2000, 2004), (2005, 2009)] + assert actual == expected + + +def test_check_mvm_only_parameters_for_bash(): + # Test with all required parameters present + c = { + "current_set": "mean_climate", + "reference_data_path_ts": "/path/to/ref/ts", + "model_name_ref": "CESM2", + "model_tableID_ref": "Amon", + "ref_start_yr": "1850", + "ts_subsection": "atm_monthly_180x360_aave", + } + check_mvm_only_parameters_for_bash(c) + + # Test missing reference_data_path_ts + c = { + "current_set": "mean_climate", + "model_name_ref": "CESM2", + "model_tableID_ref": "Amon", + "ref_start_yr": "1850", + "ts_subsection": "atm_monthly_180x360_aave", + } + with pytest.raises(ParameterNotProvidedError): + check_mvm_only_parameters_for_bash(c) + + # Test missing model_name_ref + c = { + "current_set": "mean_climate", + "reference_data_path_ts": "/path/to/ref/ts", + "model_tableID_ref": "Amon", + "ref_start_yr": "1850", + "ts_subsection": "atm_monthly_180x360_aave", + } + with pytest.raises(ParameterNotProvidedError): + check_mvm_only_parameters_for_bash(c) + + # Test missing model_tableID_ref + c = { + "current_set": "mean_climate", + "reference_data_path_ts": "/path/to/ref/ts", + "model_name_ref": "CESM2", + "ref_start_yr": "1850", + "ts_subsection": "atm_monthly_180x360_aave", + } + with pytest.raises(ParameterNotProvidedError): + check_mvm_only_parameters_for_bash(c) + + # Test missing ref_start_yr for BASE_PCMDI_SETS + c = { + "current_set": "mean_climate", + "reference_data_path_ts": "/path/to/ref/ts", + "model_name_ref": "CESM2", + "model_tableID_ref": "Amon", + "ref_start_yr": "", + "ts_subsection": "atm_monthly_180x360_aave", + } + with pytest.raises(ParameterNotProvidedError): + check_mvm_only_parameters_for_bash(c) + + # Test missing ts_subsection for BASE_PCMDI_SETS + c = { + "current_set": "mean_climate", + "reference_data_path_ts": "/path/to/ref/ts", + "model_name_ref": "CESM2", + "model_tableID_ref": "Amon", + "ref_start_yr": "1850", + "ts_subsection": "", + } + with pytest.raises(ParameterNotProvidedError): + check_mvm_only_parameters_for_bash(c) + + # Test synthetic_plots (should not require ref_start_yr and ts_subsection) + c = { + "current_set": "synthetic_plots", + "reference_data_path_ts": "/path/to/ref/ts", + "model_name_ref": "CESM2", + "model_tableID_ref": "Amon", + } + check_mvm_only_parameters_for_bash(c) + + +def test_check_and_define_parameters(): + # Test model_vs_obs run_type + c = { + "run_type": "model_vs_obs", + "sub": "mean_climate", + "year1": 2000, + "year2": 2010, + "ts_num_years": 1, + "obs_ts": "", + "diagnostics_base_path": "/diags", + "infer_path_parameters": True, + } + check_and_define_parameters(c) + assert c["prefix"] == "pcmdi_diags_mean_climate_model_vs_obs_2000-2010" + assert c["obs_ts"] == "/diags/observations/Atm/time-series/" + + # Test model_vs_model run_type + c = { + "run_type": "model_vs_model", + "sub": "variability_modes_atm", + "year1": 2000, + "year2": 2010, + "ref_year1": 1850, + "ref_year2": 1900, + "current_set": "variability_modes_atm", + "reference_data_path_ts": "/ref/path/ts", + "model_name_ref": "CESM2", + "model_tableID_ref": "Amon", + "ref_start_yr": "1850", + "ts_subsection": "atm_monthly_180x360_aave", + "reference_data_path": "/ref/path/post/analysis", + "grid": "180x360_aave", + "infer_path_parameters": True, + "clim_viewer": True, + } + check_and_define_parameters(c) + assert ( + c["prefix"] + == "pcmdi_diags_variability_modes_atm_model_vs_model_2000-2010_vs_1850-1900" + ) + assert ( + c["reference_data_path_ts"] + == "/ref/path/ts" # Value is set, so won't be changed + ) + + # Test invalid run_type + c = { + "run_type": "invalid_type", + "sub": "mean_climate", + "year1": 2000, + "year2": 2010, + } + with pytest.raises(ValueError): + check_and_define_parameters(c) + + # Test model_vs_obs without ts_num_years (should not set obs_ts) + c = { + "run_type": "model_vs_obs", + "sub": "mean_climate", + "year1": 2000, + "year2": 2010, + "obs_ts": "", + } + check_and_define_parameters(c) + assert c["prefix"] == "pcmdi_diags_mean_climate_model_vs_obs_2000-2010" + assert c["obs_ts"] == "" + + +def test_define_relevant_years(): + c = { + "current_set": "mean_climate", + "year1": 2000, + "year2": 2010, + } + define_relevant_years(c) + assert c["clim_years"] == "2000-2010" + + c = { + "current_set": "variability_modes_cpl", + "year1": 1990, + "year2": 2000, + } + define_relevant_years(c) + assert c["movc_years"] == "1990-2000" + + c = { + "current_set": "variability_modes_atm", + "year1": 1980, + "year2": 1990, + } + define_relevant_years(c) + assert c["mova_years"] == "1980-1990" + + c = { + "current_set": "enso", + "year1": 1970, + "year2": 1980, + } + define_relevant_years(c) + assert c["enso_years"] == "1970-1980" + + # Test synthetic_plots (should not set any years) + c = { + "current_set": "synthetic_plots", + "year1": 2000, + "year2": 2010, + } + with pytest.raises(ValueError): + define_relevant_years(c) + + +def test_define_relevant_years_for_synthetic_plots(): + c = { + "current_set": "synthetic_plots", + "year1": 2000, + "year2": 2010, + "clim_years": "", + "mova_years": "", + "movc_years": "", + "enso_years": "", + } + define_relevant_years_for_synthetic_plots(c) + assert c["clim_years"] == "2000-2010" + assert c["mova_years"] == "2000-2010" + assert c["movc_years"] == "2000-2010" + assert c["enso_years"] == "2000-2010" + + c = { + "current_set": "synthetic_plots", + "year1": 1990, + "year2": 2000, + "clim_years": "1991-1995", + "mova_years": "", + "movc_years": "", + "enso_years": "", + } + define_relevant_years_for_synthetic_plots(c) + assert c["clim_years"] == "1991-1995" # Should not change + assert c["mova_years"] == "1990-2000" + assert c["movc_years"] == "1990-2000" + assert c["enso_years"] == "1990-2000" + + c = { + "current_set": "mean_climate", + "year1": 2000, + "year2": 2010, + "clim_years": "", + "mova_years": "", + "movc_years": "", + "enso_years": "", + } + with pytest.raises(ValueError): + define_relevant_years_for_synthetic_plots(c) + + +def test_add_ts_dependencies(): + dependencies: List[str] = [] + c = { + "current_set": "mean_climate", + "ts_num_years": 5, + } + script_dir = "/scripts" + yr = 2000 + + with patch("zppy.pcmdi_diags.add_dependencies") as mock_add_dependencies: + add_ts_dependencies(c, dependencies, script_dir, yr) + mock_add_dependencies.assert_called_once_with( + dependencies, + script_dir, + "ts", + "atm_monthly_180x360_aave", + 2000, + 2004, + 5, + ) + + # Test with non-BASE_PCMDI_SETS (should not call add_dependencies) + dependencies = [] + c = { + "current_set": "synthetic_plots", + "ts_num_years": 5, + } + + with patch("zppy.pcmdi_diags.add_dependencies") as mock_add_dependencies: + add_ts_dependencies(c, dependencies, script_dir, yr) + mock_add_dependencies.assert_not_called() + + +@patch("os.path.exists") +def test_add_pcmdi_dependencies(mock_exists): + dependencies: List[str] = [] + c = { + "run_type": "model_vs_obs", + "year1": 2000, + "year2": 2010, + "figure_sets": ["mean_climate", "variability_modes", "enso_metric"], + "clim_viewer": True, + "mova_viewer": True, + "movc_viewer": True, + "enso_viewer": False, + } + script_dir = "/scripts" + + # Mock all status files as existing + mock_exists.return_value = True + + add_pcmdi_dependencies(c, dependencies, script_dir) + + expected_dependencies = [ + "/scripts/pcmdi_diags_mean_climate_model_vs_obs_2000-2010.status", + "/scripts/pcmdi_diags_variability_modes_cpl_model_vs_obs_2000-2010.status", + "/scripts/pcmdi_diags_variability_modes_atm_model_vs_obs_2000-2010.status", + ] + assert set(dependencies) == set(expected_dependencies) + + # Test model_vs_model run_type + dependencies = [] + c = { + "run_type": "model_vs_model", + "year1": 2000, + "year2": 2010, + "ref_year1": 1850, + "ref_year2": 1900, + "figure_sets": ["mean_climate"], + "clim_viewer": True, + "enso_viewer": False, + "mova_viewer": False, + "movc_viewer": False, + } + + add_pcmdi_dependencies(c, dependencies, script_dir) + expected_dependencies = [ + "/scripts/pcmdi_diags_mean_climate_model_vs_model_2000-2010_vs_1850-1900.status", + ] + assert dependencies == expected_dependencies + + # Test with non-existing status files + dependencies = [] + c = { + "run_type": "model_vs_obs", + "year1": 2000, + "year2": 2010, + "figure_sets": ["mean_climate"], + "clim_viewer": True, + "enso_viewer": False, + "mova_viewer": False, + "movc_viewer": False, + } + mock_exists.return_value = False + + add_pcmdi_dependencies(c, dependencies, script_dir) + assert dependencies == [] + + # Test missing run_type + c = { + "year1": 2000, + "year2": 2010, + "figure_sets": ["mean_climate"], + } + with pytest.raises(ParameterNotProvidedError): + add_pcmdi_dependencies(c, dependencies, script_dir) + + # Test duplicate dependencies are not added + dependencies = ["/scripts/pcmdi_diags_mean_climate_model_vs_obs_2000-2010.status"] + c = { + "run_type": "model_vs_obs", + "year1": 2000, + "year2": 2010, + "figure_sets": ["mean_climate"], + "clim_viewer": True, + "enso_viewer": False, + "mova_viewer": False, + "movc_viewer": False, + } + mock_exists.return_value = True + + add_pcmdi_dependencies(c, dependencies, script_dir) + # Should still only have one instance + assert len(dependencies) == 1 + assert ( + dependencies[0] + == "/scripts/pcmdi_diags_mean_climate_model_vs_obs_2000-2010.status" + ) diff --git a/tests/test_zppy_utils.py b/tests/test_zppy_utils.py index 1e67ca8e..10035177 100644 --- a/tests/test_zppy_utils.py +++ b/tests/test_zppy_utils.py @@ -6,6 +6,8 @@ ParameterInferenceType, ParameterNotProvidedError, add_dependencies, + check_parameter_defined, + check_set_specific_parameter, get_active_status, get_file_names, get_inference_type_parameter, @@ -187,6 +189,24 @@ def test_set_component_and_prc_typ(): set_component_and_prc_typ(c) +def test_check_set_specific_parameter(): + # Parameter is required + # a, b need parameter p, and we want sets a, b, c + c = {"sets": ["a", "b", "c"], "p": "exists"} + check_set_specific_parameter(c, set(["a", "b"]), "p") + + # Parameter isn't required based on the sets we want + # z needs parameter p, but we only want sets a, b, c + c = {"sets": ["a", "b", "c"], "p": ""} + check_set_specific_parameter(c, set(["z"]), "p") + + # Parameter is required + # a, b need parameter p, and we want sets a, b, c + c = {"sets": ["a", "b", "c"], "p": ""} + with pytest.raises(ParameterNotProvidedError): + check_set_specific_parameter(c, set(["a", "b"]), "p") + + def test_get_years(): assert get_years("1980:1990:05") == [(1980, 1984), (1985, 1989)] assert get_years("1980-1990") == [(1980, 1990)] @@ -478,6 +498,15 @@ def test_set_value_of_parameter_if_undefined(): assert c["required_parameter"] == "backup_option" +def test_check_parameter_defined(): + c = {"a": 1, "b": 2, "c": ""} + check_parameter_defined(c, "a") + with pytest.raises(ParameterNotProvidedError): + check_parameter_defined(c, "c") + with pytest.raises(ParameterNotProvidedError): + check_parameter_defined(c, "d") + + def test_get_file_names(): bash, settings, status = get_file_names("script_dir", "prefix") assert bash == "script_dir/prefix.bash" diff --git a/zppy/__main__.py b/zppy/__main__.py index fe2d05d3..366f5320 100644 --- a/zppy/__main__.py +++ b/zppy/__main__.py @@ -20,6 +20,7 @@ from zppy.ilamb import ilamb from zppy.logger import _setup_custom_logger from zppy.mpas_analysis import mpas_analysis +from zppy.pcmdi_diags import pcmdi_diags from zppy.tc_analysis import tc_analysis from zppy.ts import ts from zppy.utils import check_status, submit_script @@ -270,6 +271,9 @@ def _launch_scripts(config: ConfigObj, script_dir, job_ids_file, plugins) -> Non # ilamb tasks existing_bundles = ilamb(config, script_dir, existing_bundles, job_ids_file) + # pcmdi_diags tasks + existing_bundles = pcmdi_diags(config, script_dir, existing_bundles, job_ids_file) + # zppy external plugins for plugin in plugins: # Get plugin module function diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini old mode 100644 new mode 100755 index 454f68cb..8dfa0810 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -20,6 +20,7 @@ dry_run = boolean(default=False) # Set up the environment -- this is where you can tell zppy to use a custom conda environment. # To use a custom conda environment, you can set `environment_commands="source ; conda activate "`. environment_commands = string(default="") +environment_commands_secondary = string(default="") # If set to True, zppy will fail as soon as a job is unable to launch because of a missing dependency. # If set to False, zppy will launch other jobs, if possible. fail_on_dependency_skip = boolean(default=False) @@ -84,7 +85,7 @@ ts_subsection = string(default="") # The variables to process # If `vars` is set to "", then climo and ts will process *all* variables. # This is accomplished by not setting `-v`/`--vars` for `ncclimo`. -vars = string(default="FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U") +vars = string(default="FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,PSL") # DEPRECATED: vars_exclude # This is the list of variables to exclude. It is only used if `vars` is set to "". # vars_exclude = string(default="H2OSOI,LAKEICEFRAC,O_SCALAR,PCT_LANDUNIT,PCT_NAT_PFT,SOILICE,SOILICE_ICE,SOILLIQ,SOILLIQ_ICE,SOILPSI,T_SCALAR,TLAKE,TSOI,TSOI_ICE,W_SCALAR") @@ -142,15 +143,22 @@ input_component = string(default="") [e3sm_to_cmip] # Metadata json file cmip_metadata = string(default="inclusions/e3sm_to_cmip/default_metadata.json") +# Pressure level nc file +# Defaults to a file in diagnostics_base_path if not set +cmip_plevdata = string(default="") # variables to run e3sm_to_cmip on cmip_vars = string(default="") # Model component having generated input files (eam, eamxx, elm, mosart, ...) input_component = string(default="") +# TBD: description here +interp_vars = string(default="U,V,T,Q,RELHUM,OMEGA,Z3") [[__many__]] cmip_metadata = string(default=None) + cmip_plevdata = string(default=None) cmip_vars = string(default=None) input_component = string(default=None) + interp_vars = string(default=None) [tc_analysis] # NOTE: always overrides value in [default] @@ -384,3 +392,249 @@ e3sm_to_cmip_land_subsection = string(default="") ilamb_obs = string(default="") # for land_only run land_only = boolean(default=False) + +[pcmdi_diags] +########################################################################################### +# Most useful at task-level; used by all sets +########################################################################################### +# See url +figure_format = string(default="png") +# Comparision type (same as e3sm_diag) +run_type = string(default="model_vs_obs") +### 4 options specific for constructing pcmdi preferred file name conventions: +### 2 options required for "model_vs_obs" comparison: +model_name = string(default="e3sm.historical.v3-LR.0051") +model_tableID = string(default="Amon") +### 2 options required for "model_vs_model" comparison: +model_name_ref = string(default="ERA5") +model_tableID_ref = string(default="Amon") +########################################################################################### +# Most useful at task-level; used by all sets except synthetic_plots +########################################################################################### +# Name of the `[e3sm_to_cmip]` atm subtask to depend on +e3sm_to_cmip_atm_subsection = string(default="") +# Flag to process the land/sea mask within pcmdi +generate_sftlf = string(default=True) +# model data grid after remapping +grid = string(default="180x360_aave") +mov_plot_obs = string(default=True) +mov_plot_model = string(default=True) +mov_nc_out_obs = string(default=True) +mov_nc_out_model = string(default=True) +# See url +multiprocessing = boolean(default=True) +# See url +num_workers = integer(default=24) +# Path to observation time-series data +# Required for "mean_climate","variability_mode","enso" +obs_ts = string(default="") +pcmdi_debug = string(default=False) +# Save derived climatology data +save_test_clims = string(default=True) +### 7 options required for run_type="model_vs_model" runs: +# Unlike in e3sm_diags, model_vs_model in pcmdi refers to the comparision of two model simulations with observations and cmip models. +### 2 options for path for reference model data (time series): +reference_data_path = string(default="") +reference_data_path_ts = string(default="") +# The years increment for test model data +ts_num_years = integer(default=5) +# The years range for test model data +ts_years = string_list(default=list("")) +########################################################################################### +# Specify these for each subtask +########################################################################################### +# Set to one of ["mean_climate", "variability_modes_cpl", "variability_modes_atm", "enso"] +# If not set, zppy can try to infer this value from the subsection. +current_set = string(default="") +# Observational data sets (see reference_alias.json) +# Observation data tag in reference_alias +obs_sets = string(default="default") +# Final year (i.e., the last available year) for the reference data +ref_final_yr = string(default="") +# Start year for the reference data +ref_start_yr = string(default="") +# This should be kept as "" for synthetic plots; +# zppy will then set it to match `figure_sets_period` +# For other plots, zppy will set it to match `years`, if not specified +ref_years = string_list(default=list("")) +########################################################################################### +# Specify these for the mean_climate subtask +########################################################################################### +# variables in the cmip6 table that can be potentially used by pcmdi +# this list depends on the definition of cmip variable +# required for "mean climate" diagnostics +cmip_vars = string(default="pr,prw,psl,rlds,rldscs,rlut,rlutcs,rsut,rsutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rtmt,sfcWind,tas,tauu,tauv,ts,ta,ua,va,zg") +# Regridding by pcmdi (required for mean climate) +# OPTIONS: '1x1' for e3sm, '2.5x2.5' for cmip, or an actual cdms2 grid object, +target_grid = string(default="1x1") +# OPTIONS: "1px1p" for e3sm, "2p5x2p5" for cmip, description on the selected grid +target_grid_string = string(default="1px1p") +# OPTIONS: 'regrid2','esmf' +regrid_tool = string(default="esmf") +regrid_tool_ocn = string(default="esmf") +# OPTIONS: 'linear','conservative', only if tool is esmf +regrid_method = string(default="regrid2") +# OPTIONS: 'linear','conservative', only if tool is esmf +regrid_method_ocn = string(default="conservative") +########################################################################################### +# Specify these for the mean_climate subtask AND the synthetic_plots subtask +########################################################################################### +# Variables to be used by the pcmdi diagnostics +# mean_climate AND synthetic_plots: clim_vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500" +clim_vars = string(default="pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500") +#default regions for mean climate metrics data +#more options can be found at "regions_specs" +#regions = string(default="global,ocean,land,NHEX,SHEX,TROPICS,NHEX_ocean,SHEX_ocean,NHEX_land,SHEX_land,ocean_50S50N") +clim_regions = string(default="global,ocean,land") +########################################################################################### +# Specify these for the variability_modes_cpl and variability_modes_atm subtasks +########################################################################################### +CBF = string(default=True) +ConvEOF = string(default=True) +eofn_mod_max = integer(default=3) +EofScaling = string(default=False) +# Frequency of the model data +frequency = string(default="mo") +landmask = string(default=False) +# Keywords for unit conversion in pcmdi (model) +ModUnitsAdjust = string(default="") +# Keywords for unit conversion in pcmdi (observation) +ObsUnitsAdjust = string(default="") +RmDomainMean = string(default=True) +seasons = string(default="monthly") +########################################################################################### +# Specify these for the variability_modes_atm subtask AND the synthetic_plots subtask +########################################################################################### +# variability_modes_atm: vars = "psl" +mova_vars = string(default="psl") +mova_modes = string(default="NAM,NAO,PNA,NPO,SAM,PSA1,PSA2") +########################################################################################### +# Specify these for the variability_modes_cpl subtask AND the synthetic_plots subtask +########################################################################################### +# variability_modes_cpl: vars = "ts" +movc_vars = string(default="ts") +movc_modes = string(default="PDO,NPGO,AMO") +########################################################################################### +# Specify these for the enso subtask (not currently enabled) +########################################################################################### +enso_groups = string(default="ENSO_perf,ENSO_proc,ENSO_tel") +########################################################################################### +# Specify these for the enso subtask AND the synthetic_plots subtask +########################################################################################### +# enso: vars = "psl,pr,prsn,ts,tas,tauu,tauv,hflx,hfss,rlds,rsds,rlus,rlut,rsdt" +enso_vars = string(default="psl,pr,prsn,ts,tas,tauu,tauv,hflx,hfss,rlds,rsds,rlus,rlut,rsdt") +########################################################################################### +# Specify these for the synthetic_plots subtask +########################################################################################### +# The _years parameters are also defined for the other subtasks, but internally in pcmdi_diags.py. +# add mean climate to viewer page (default is False) +clim_viewer = boolean(default=True) +clim_years = string(default="") +# add extratropic modes to viewer page (default is True) +mova_viewer = boolean(default=True) +mova_years = string(default="") +# add coupled modes to viewer page (default is True) +movc_viewer = boolean(default=True) +movc_years = string(default="") +# add enso to viewer page (default is False, not currently enabled) +enso_viewer = boolean(default=False) +enso_years = string(default="") +# Save all intemediate diagnostic data +save_all_data = string(default=True) +# File of observation data name for mean climate calculation +reference_alias = string(default="inclusions/pcmdi_diags/reference_alias.json") +# File of specified regions for mean climate calculation +regions_specs = string(default="inclusions/pcmdi_diags/regions_specs.json") +# Version of zppy-pcmdi code +pcmdi_version = string(default="v3.8.2") +# Template files for zppy-pcmdi viewer page processing +pcmdi_viewer_template = string(default="pcmdi_data/viewer") +# Title of zppy-pcmdi diagnostics +pcmdi_webtitle = string(default="E3SM-PMP-Diagnostics") # Cannot have spaces! +# File of variable list to generate synthetic metrics plot +synthetic_metrics_list = string(default="inclusions/pcmdi_diags/synthetic_metrics_list.json") +synthetic_sets = string(default="portrait,parcoord") +### 3 options for paths to pcmdi generated cmip metrics data: +cmip_clim_dir = string(default="") +cmip_enso_dir = string(default="") +cmip_movs_dir = string(default="") +### 3 options for groups of pcmdi generated cmip metrics data (mip.exp.version): +cmip_clim_set = string(default="cmip6.historical.v20250707") +cmip_enso_set = string(default="cmip6.historical.v20210620") +cmip_movs_set = string(default="cmip6.historical.v20220825") + + [[__many__]] + figure_format = string(default=None) + run_type = string(default=None) + model_name = string(default=None) + model_tableID = string(default=None) + model_name_ref = string(default=None) + model_tableID_ref = string(default=None) + e3sm_to_cmip_atm_subsection = string(default=None) + generate_sftlf = string(default=None) + grid = string(default=None) + mov_plot_obs = string(default=None) + mov_plot_model = string(default=None) + mov_nc_out_obs = string(default=None) + mov_nc_out_model = string(default=None) + multiprocessing = boolean(default=None) + num_workers = integer(default=None) + obs_ts = string(default=None) + pcmdi_debug = string(default=None) + save_test_clims = string(default=None) + reference_data_path = string(default=None) + reference_data_path_ts = string(default=None) + ts_num_years = integer(default=None) + ts_years = string_list(default=None) + current_set = string(default=None) + obs_sets = string(default=None) + ref_final_yr = string(default=None) + ref_start_yr = string(default=None) + ref_years = string_list(default=None) + clim_vars = string(default=None) + target_grid = string(default=None) + target_grid_string = string(default=None) + regrid_tool = string(default=None) + regrid_tool_ocn = string(default=None) + regrid_method = string(default=None) + regrid_method_ocn = string(default=None) + cmip_vars = string(default=None) + clim_regions = string(default=None) + CBF = string(default=None) + ConvEOF = string(default=None) + eofn_mod_max = integer(default=None) + EofScaling = string(default=None) + frequency = string(default=None) + landmask = string(default=None) + ModUnitsAdjust = string(default=None) + ObsUnitsAdjust = string(default=None) + RmDomainMean = string(default=None) + seasons = string(default=None) + mova_vars = string(default=None) + mova_modes = string(default=None) + movc_vars = string(default=None) + movc_modes = string(default=None) + enso_groups = string(default=None) + enso_vars = string(default=None) + clim_viewer = boolean(default=None) + clim_years = string(default=None) + mova_viewer = boolean(default=None) + mova_years = string(default=None) + movc_viewer = boolean(default=None) + movc_years = string(default=None) + enso_viewer = boolean(default=None) + enso_years = string(default=None) + save_all_data = string(default=None) + reference_alias = string(default=None) + regions_specs = string(default=None) + pcmdi_version = string(default=None) + pcmdi_viewer_template = string(default=None) + pcmdi_webtitle = string(default=None) + cmip_clim_dir = string(default=None) + cmip_enso_dir = string(default=None) + cmip_movs_dir = string(default=None) + cmip_clim__set = string(default=None) + cmip_enso_set = string(default=None) + cmip_movs_set = string(default=None) + synthetic_metrics_list = string(default=None) + synthetic_sets = string(default=None) diff --git a/zppy/e3sm_diags.py b/zppy/e3sm_diags.py index c6b690ba..0095d66f 100644 --- a/zppy/e3sm_diags.py +++ b/zppy/e3sm_diags.py @@ -6,8 +6,9 @@ from zppy.bundle import handle_bundles from zppy.utils import ( ParameterInferenceType, - ParameterNotProvidedError, add_dependencies, + check_parameter_defined, + check_set_specific_parameter, check_status, get_file_names, get_tasks, @@ -104,32 +105,6 @@ def e3sm_diags(config: ConfigObj, script_dir: str, existing_bundles, job_ids_fil return existing_bundles -def check_parameter_defined( - c: Dict[str, Any], relevant_parameter: str, explanation: str = "" -) -> None: - if (relevant_parameter not in c.keys()) or (c[relevant_parameter] == ""): - if explanation: - message = f"{relevant_parameter} is needed because {explanation}" - else: - message = f"{relevant_parameter} is not defined." - raise ParameterNotProvidedError(message) - - -def check_set_specific_parameter( - c: Dict[str, Any], sets_with_requirement: Set[str], relevant_parameter: str -) -> None: - requested_sets = set(c["sets"]) - intersection = sets_with_requirement & requested_sets - if ( - intersection - and (relevant_parameter in c.keys()) - and (c[relevant_parameter] == "") - ): - raise ParameterNotProvidedError( - f"{relevant_parameter} is required because the sets {intersection} were requested." - ) - - def check_parameters_for_bash(c: Dict[str, Any]) -> None: # Check parameters that aren't used until e3sm_diags.bash is run check_set_specific_parameter(c, set(["qbo"]), "ref_final_yr") diff --git a/zppy/e3sm_to_cmip.py b/zppy/e3sm_to_cmip.py index 0ee71b30..4b6a7aa5 100644 --- a/zppy/e3sm_to_cmip.py +++ b/zppy/e3sm_to_cmip.py @@ -38,6 +38,14 @@ def e3sm_to_cmip(config: ConfigObj, script_dir: str, existing_bundles, job_ids_f set_component_and_prc_typ(c) check_parameters_for_bash(c) c["cmor_tables_prefix"] = c["diagnostics_base_path"] + if c["component"] == "atm": + default_cmip_plevdata = f"{c['diagnostics_base_path']}/vrt_remap_plev19.nc" + set_value_of_parameter_if_undefined( + c, + "cmip_plevdata", + default_cmip_plevdata, + ParameterInferenceType.PATH_INFERENCE, + ) year_sets: List[Tuple[int, int]] = get_years(c["years"]) # Loop over year sets for s in year_sets: @@ -56,11 +64,11 @@ def e3sm_to_cmip(config: ConfigObj, script_dir: str, existing_bundles, job_ids_f if c["cmip_vars"] == "": if c["component"] == "atm": c["cmip_vars"] = ( - "tas, ts, psl, ps, sfcWind, huss, pr, prc, prsn, evspsbl, tauu, tauv, hfls, clt, rlds, rlus, rsds, rsus, hfss, clivi, clwvi, prw, rldscs, rlut, rlutcs, rsdt, rsuscs, rsut, rsutcs, rtmt, abs550aer, od550aer, rsdscs, tasmax, tasmin" + "ua, va, ta, wa, zg, hur, tas, ts, psl, ps, sfcWind, huss, pr, prc, prsn, evspsbl, tauu, tauv, hfls, clt, rlds, rlus, rsds, rsus, hfss, clivi, clwvi, prw, rldscs, rlut, rlutcs, rsdt, rsuscs, rsut, rsutcs, rtmt, abs550aer, od550aer, rsdscs, tasmax, tasmin" ) elif c["component"] == "lnd": c["cmip_vars"] = ( - "mrsos, mrso, mrfso, mrros, mrro, prveg, evspsblveg, evspsblsoi, tran, tsl, lai, cLitter, cProduct, cSoilFast, cSoilMedium, cSoilSlow, fFire, fHarvest, cVeg, nbp, gpp, ra, rh" + "snd, mrsos, mrso, mrfso, mrros, mrro, prveg, evspsblveg, evspsblsoi, tran, tsl, lai, cLitter, cProduct, cSoilFast, cSoilMedium, cSoilSlow, fFire, fHarvest, cVeg, nbp, gpp, ra, rh" ) prefix = f"e3sm_to_cmip_{sub}_{c['yr_start']:04d}-{c['yr_end']:04d}-{c['ypf']:04d}" print(prefix) diff --git a/zppy/pcmdi_diags.py b/zppy/pcmdi_diags.py new file mode 100644 index 00000000..2ca7ee6f --- /dev/null +++ b/zppy/pcmdi_diags.py @@ -0,0 +1,400 @@ +import os +from typing import Any, Dict, List, Set, Tuple + +from zppy.bundle import handle_bundles +from zppy.logger import _setup_custom_logger +from zppy.utils import ( + ParameterInferenceType, + ParameterNotProvidedError, + add_dependencies, + check_parameter_defined, + check_set_specific_parameter, + check_status, + get_file_names, + get_tasks, + get_value_from_parameter, + get_years, + initialize_template, + make_executable, + print_url, + set_value_of_parameter_if_undefined, + submit_script, + write_settings_file, +) + +logger = _setup_custom_logger(__name__) + +VALID_PCMDI_SETS: Set[str] = set( + [ + "mean_climate", + "variability_modes_atm", + "variability_modes_cpl", + "enso", + "synthetic_plots", + ] +) +BASE_PCMDI_SETS: Set[str] = set( + ["mean_climate", "variability_modes_atm", "variability_modes_cpl", "enso"] +) + + +# ----------------------------------------------------------------------------- +def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): + + template, _ = initialize_template(config, "pcmdi_diags.bash") + + # --- List of pcmdi_diags tasks --- + tasks: List[Dict[str, Any]] = get_tasks(config, "pcmdi_diags") + if len(tasks) == 0: + return existing_bundles + + # --- Generate and submit pcmdi_diags scripts --- + for c in tasks: + dependencies: List[str] = [] + define_current_set(c) + if c["current_set"] == "enso": + logger.warning( + "The 'enso' set is not yet supported in PCMDI Diags. Skipping launching of associated jobs." + ) + break # Skip this task + c["sub"] = get_value_from_parameter( + c, "subsection", "sub", ParameterInferenceType.SECTION_INFERENCE + ) + check_parameters_for_bash(c) + + c["scriptDir"] = script_dir + if "ts_num_years" in c.keys(): + c["ts_num_years"] = int(c["ts_num_years"]) + + # check and set parameter for pcmdi + check_parameters_for_pcmdi(c) + + # Loop over year sets + year_sets: List[Tuple[int, int]] = define_year_sets(c) + + ref_year_sets: List[Tuple[int, int]] + if ("ref_years" in c.keys()) and (c["ref_years"] != [""]): + ref_year_sets = get_years(c["ref_years"]) + else: + ref_year_sets = year_sets + + for i, (s, rs) in enumerate(zip(year_sets, ref_year_sets)): + c["year1"] = s[0] + c["year2"] = s[1] + if ("last_year" in c.keys()) and (c["year2"] > c["last_year"]): + continue # Skip this year set + + c["ref_year1"] = rs[0] + c["ref_year2"] = rs[1] + + if c["current_set"] != "synthetic_plots": + check_and_define_parameters(c) + print(c["prefix"]) + define_relevant_years(c) + else: + prefix = f"pcmdi_diags_{c['sub']}_{c['run_type']}" + if i < len(year_sets) - 1: + logger.debug( + f"Not printing prefix={prefix} because we will only submit the job on the final iteration." + ) + else: + print(prefix) + c["prefix"] = prefix + define_relevant_years_for_synthetic_plots(c) + + bash_file, settings_file, status_file = get_file_names( + script_dir, c["prefix"] + ) + skip: bool = check_status(status_file) + if skip: + continue + + # Create script + with open(bash_file, "w") as f: + f.write(template.render(**c)) + make_executable(bash_file) + + # List of dependencies + # Iterate from year1 to year2 incrementing by the number of years per time series file. + if c["current_set"] != "synthetic_plots": + for yr in range(c["year1"], c["year2"], c["ts_num_years"]): + add_ts_dependencies(c, dependencies, script_dir, yr) + set_value_of_parameter_if_undefined( + c, + "e3sm_to_cmip_atm_subsection", + "atm_monthly_180x360_aave", + ParameterInferenceType.SECTION_INFERENCE, + ) + add_dependencies( + dependencies, + script_dir, + "e3sm_to_cmip", + c["e3sm_to_cmip_atm_subsection"], + c["year1"], + c["year2"], + c["ts_num_years"], + ) + else: + add_pcmdi_dependencies(c, dependencies, script_dir) + if i < len(year_sets) - 1: + continue + + c["dependencies"] = dependencies + write_settings_file(settings_file, c, s) + + export = "ALL" + existing_bundles = handle_bundles( + c, + bash_file, + export, + dependFiles=dependencies, + existing_bundles=existing_bundles, + ) + if not c["dry_run"]: + if c["bundle"] == "": + # Submit job + submit_script( + bash_file, + status_file, + export, + job_ids_file, + dependFiles=dependencies, + fail_on_dependency_skip=c["fail_on_dependency_skip"], + ) + else: + print(f"...adding to bundle {c['bundle']}") + print(f" environment_commands={c['environment_commands']}") + print_url(c, "pcmdi_diags") + + return existing_bundles + + +def define_current_set(c: Dict[str, Any]): + if c["current_set"] != "": + current_set = c["current_set"] + if current_set not in VALID_PCMDI_SETS: + raise ValueError( + f"Invalid set '{current_set}'. Must be one of {VALID_PCMDI_SETS}" + ) + elif c["infer_path_parameters"]: + if "subsection" not in c: + raise ParameterNotProvidedError( + "subsection must be available if current_set is to be inferred." + ) + elif "mean_climate" in c["subsection"]: + current_set = "mean_climate" + elif "variability_modes_cpl" in c["subsection"]: + current_set = "variability_modes_cpl" + elif "variability_modes_atm" in c["subsection"]: + current_set = "variability_modes_atm" + elif "enso" in c["subsection"]: + current_set = "enso" + elif "synthetic_plots" in c["subsection"]: + current_set = "synthetic_plots" + else: + raise ValueError(f"Could not determine set from {c['subsection']}") + if current_set not in VALID_PCMDI_SETS: + raise ValueError( + f"Invalid set: '{current_set}' inferred from {c['subsection']}. Must be one of {VALID_PCMDI_SETS}" + ) + else: + raise ParameterNotProvidedError( + "current_set was not provided, and inferring is turned off. Turn on inferring by setting infer_path_parameters to True." + ) + c["current_set"] = current_set + + +def check_parameters_for_bash(c: Dict[str, Any]) -> None: + if c["current_set"] != "synthetic_plots": + check_set_specific_parameter( + c, + BASE_PCMDI_SETS, + "ref_final_yr", + ) + check_set_specific_parameter( + c, + BASE_PCMDI_SETS, + "ref_start_yr", + ) + + +def check_parameters_for_pcmdi(c: Dict[str, Any]) -> None: + # check and set up the external data needed by pcmdi + if c["current_set"] == "synthetic_plots": + if c["enso_viewer"]: + set_value_of_parameter_if_undefined( + c, + "cmip_enso_dir", + f"{c['diagnostics_base_path']}/pcmdi_data/metrics_data/enso_metric", + ParameterInferenceType.PATH_INFERENCE, + ) + else: + c["cmip_enso_dir"] = "placeholder_dir" + if c["clim_viewer"]: + set_value_of_parameter_if_undefined( + c, + "cmip_clim_dir", + f"{c['diagnostics_base_path']}/pcmdi_data/metrics_data/mean_climate", + ParameterInferenceType.PATH_INFERENCE, + ) + else: + c["cmip_clim_dir"] = "placeholder_dir" + if c["mova_viewer"] or c["movc_viewer"]: + set_value_of_parameter_if_undefined( + c, + "cmip_movs_dir", + f"{c['diagnostics_base_path']}/pcmdi_data/metrics_data/variability_modes", + ParameterInferenceType.PATH_INFERENCE, + ) + else: + c["cmip_movs_dir"] = "placeholder_dir" + + +def define_year_sets(c: Dict[str, Any]) -> List[Tuple[int, int]]: + # Use ts_years unless we are in synthetic plots, in which case, use the appropriate set if defined + year_sets: List[Tuple[int, int]] + if c["current_set"] != "synthetic_plots": + year_sets = get_years(c["ts_years"]) + elif ("clim_years" in c.keys()) and (c["clim_years"] != ""): + year_sets = get_years(c["clim_years"]) + elif ("mova_years" in c.keys()) and (c["mova_years"] != ""): + year_sets = get_years(c["mova_years"]) + elif ("movc_years" in c.keys()) and (c["movc_years"] != ""): + year_sets = get_years(c["movc_years"]) + elif ("enso_years" in c.keys()) and (c["enso_years"] != ""): + year_sets = get_years(c["enso_years"]) + else: + year_sets = get_years(c["ts_years"]) + return year_sets + + +def check_mvm_only_parameters_for_bash(c: Dict[str, Any]) -> None: + check_parameter_defined(c, "reference_data_path_ts") + check_parameter_defined(c, "model_name_ref") + check_parameter_defined(c, "model_tableID_ref") + if c["current_set"] != "synthetic_plots": + check_set_specific_parameter( + c, + BASE_PCMDI_SETS, + "ref_start_yr", + ) + check_set_specific_parameter(c, BASE_PCMDI_SETS, "ts_subsection") + + +def check_and_define_parameters(c: Dict[str, Any]) -> None: + if "ts_num_years" in c.keys(): + set_value_of_parameter_if_undefined( + c, + "obs_ts", + f"{c['diagnostics_base_path']}/observations/Atm/time-series/", + ParameterInferenceType.PATH_INFERENCE, + ) + prefix: str + if c["run_type"] == "model_vs_obs": + prefix = ( + f"pcmdi_diags_{c['sub']}_{c['run_type']}_{c['year1']:04d}-{c['year2']:04d}" + ) + elif c["run_type"] == "model_vs_model": + check_mvm_only_parameters_for_bash(c) + prefix = f"pcmdi_diags_{c['sub']}_{c['run_type']}_{c['year1']:04d}-{c['year2']:04d}_vs_{c['ref_year1']:04d}-{c['ref_year2']:04d}" + reference_data_path = c["reference_data_path"].split("/post")[0] + "/post" + if c["current_set"] in BASE_PCMDI_SETS: + set_value_of_parameter_if_undefined( + c, + "reference_data_path_ts", + f"{reference_data_path}/atm/{c['grid']}/cmip_ts/monthly", + ParameterInferenceType.PATH_INFERENCE, + ) + else: + raise ValueError(f"Invalid run_type={c['run_type']}") + c["prefix"] = prefix + + +def define_relevant_years(c: Dict[str, Any]) -> None: + if c["current_set"] == "synthetic_plots": + raise ValueError( + "define_relevant_years should not be called for synthetic_plots." + ) + year_str: str = f"{c['year1']}-{c['year2']}" + if c["current_set"] == "mean_climate": + c["clim_years"] = year_str + elif c["current_set"] == "variability_modes_cpl": + c["movc_years"] = year_str + elif c["current_set"] == "variability_modes_atm": + c["mova_years"] = year_str + elif c["current_set"] == "enso": + c["enso_years"] = year_str + + +def define_relevant_years_for_synthetic_plots(c: Dict[str, Any]) -> None: + if c["current_set"] != "synthetic_plots": + raise ValueError( + "define_relevant_years_for_synthetic_plots should only be called for synthetic_plots." + ) + # assign period for set if empty + year_str: str = f"{c['year1']}-{c['year2']}" + if c["clim_years"] == "": + c["clim_years"] = year_str + if c["mova_years"] == "": + c["mova_years"] = year_str + if c["movc_years"] == "": + c["movc_years"] = year_str + if c["enso_years"] == "": + c["enso_years"] = year_str + + +def add_ts_dependencies( + c: Dict[str, Any], dependencies: List[str], script_dir: str, yr: int +): + start_yr = yr + end_yr = yr + c["ts_num_years"] - 1 + if c["current_set"] in BASE_PCMDI_SETS: + add_dependencies( + dependencies, + script_dir, + "ts", + "atm_monthly_180x360_aave", + start_yr, + end_yr, + c["ts_num_years"], + ) + + +def add_pcmdi_dependencies( + c: Dict[str, Any], dependencies: List[str], script_dir: str +) -> None: + check_parameter_defined(c, "run_type") + if c["run_type"] == "model_vs_obs": + status_suffix = f"_{c['year1']:04d}-{c['year2']:04d}" + elif c["run_type"] == "model_vs_model": + status_suffix = f"_{c['year1']:04d}-{c['year2']:04d}_vs_{c['ref_year1']:04d}-{c['ref_year2']:04d}" + + if c["clim_viewer"]: + status_file = os.path.join( + script_dir, + f"pcmdi_diags_mean_climate_{c['run_type']}{status_suffix}.status", + ) + if os.path.exists(status_file) and (status_file not in dependencies): + dependencies.append(status_file) + if c["movc_viewer"]: + # cpl + status_file = os.path.join( + script_dir, + f"pcmdi_diags_variability_modes_cpl_{c['run_type']}{status_suffix}.status", + ) + if os.path.exists(status_file) and (status_file not in dependencies): + dependencies.append(status_file) + if c["mova_viewer"]: + # atm + status_file = os.path.join( + script_dir, + f"pcmdi_diags_variability_modes_atm_{c['run_type']}{status_suffix}.status", + ) + if os.path.exists(status_file) and (status_file not in dependencies): + dependencies.append(status_file) + if c["enso_viewer"]: + status_file = os.path.join( + script_dir, f"pcmdi_diags_enso_{c['run_type']}{status_suffix}.status" + ) + if os.path.exists(status_file) and (status_file not in dependencies): + dependencies.append(status_file) diff --git a/zppy/templates/e3sm_to_cmip.bash b/zppy/templates/e3sm_to_cmip.bash index ac30d992..e5f37bde 100644 --- a/zppy/templates/e3sm_to_cmip.bash +++ b/zppy/templates/e3sm_to_cmip.bash @@ -19,7 +19,7 @@ cat > default_metadata.json << EOF {% include cmip_metadata %} EOF { - export cmortables_dir={{ cmor_tables_prefix }}/e3sm_to_cmip_data/cmip6-cmor-tables/Tables + export cmortables_dir={{ cmor_tables_prefix }}/cmip6-cmor-tables/Tables input_dir=${dest}/{{ '%04d' % (yr_start) }}_{{ '%04d' % (yr_end) }} mkdir -p $input_dir @@ -27,11 +27,29 @@ EOF dest_cmip={{ output }}/post/{{ component }}/{{ ts_grid }}/cmip_ts/{{ frequency }} mkdir -p ${dest_cmip} + {% if input_files.split(".")[0] == 'cam' or input_files.split(".")[0] == 'eam' -%} + #add code to do vertical interpolation variables at model levels before e3sm_to_cmip + IFS=',' read -ra mlvars <<< "{{ interp_vars }}" + for var in "${mlvars[@]}" + do + for file in ${input_dir}/${var}_{{ '%04d' % (yr_start) }}??_{{ '%04d' % (yr_end) }}??.nc + do + if [ -f ${file} ]; then + #ncks --rgr xtr_mth=mss_val --vrt_fl='{{cmip_plevdata}}' ${file} ${file}.plev + ncremap -p mpi --vrt_ntp=log --vrt_xtr=mss_val --vrt_out='{{cmip_plevdata}}' ${file} ${file}.plev + #overwrite the model level data + mv ${file}.plev ${file} + fi + done + done + {% endif -%} + + #call e3sm_to_cmip srun -N 1 e3sm_to_cmip \ --output-path \ ${dest_cmip}/${tmp_dir} \ --var-list \ - {{ cmip_vars }} \ + '{{ cmip_vars }}' \ --realm \ {{ component }} \ --input-path \ diff --git a/zppy/templates/inclusions/pcmdi_diags/reference_alias.json b/zppy/templates/inclusions/pcmdi_diags/reference_alias.json new file mode 100755 index 00000000..48cd6ebb --- /dev/null +++ b/zppy/templates/inclusions/pcmdi_diags/reference_alias.json @@ -0,0 +1,393 @@ +{ + "rlds" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "rldscs" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "rlus" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "rsds" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "rsdscs" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + + "rsus" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "rsuscs": { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "rstcre" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "rltcre" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "rlut" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "rlutcs" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "rsdt" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "rsut" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "rsutcs" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "rtmt" : { + "defaultpi" : "NOAA-20C", + "default" : "ceres_ebaf_v4.1", + "alternate" : "ceres_ebaf_v4.0", + "alternate1" : "ceres_ebaf_v2.8", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C" + }, + "pr" : { + "defaultpi" : "NOAA-20C", + "default" : "GPCP_v2.3", + "alternate" : "GPCP_v2.2", + "alternate1" : "GPCP_1DD", + "alternate2" : "ERA5", + "alternate3" : "MERRA2", + "alternate4" : "ERA-Interim", + "alternate5" : "NOAA-20C", + "alternate6" : "GPCP_v3.2" + }, + "prc" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "NOAA-20C" + }, + "prsn" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "NOAA-20C" + }, + "prw" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "MERRA2", + "alternate2" : "ERA-Interim", + "alternate3" : "NOAA-20C" + }, + "psl" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C" + }, + "ps" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate " : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C" + }, + "huss" : { + "defaultpi" : "NOAA-20C", + "default" : "MERRA2", + "alternate" : "NOAA-20C", + "alternate1" : "ERA5", + "alternate2" : "ERA-Interim" + }, + "ta" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C" + }, + "ua" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C" + }, + "va" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C" + }, + "hur" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C" + }, + "wap" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C" + }, + "zg" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C" + }, + "o3" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C" + }, + "hus" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C" + }, + "uas" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C" + }, + "vas" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C" + }, + "tauu" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C", + "alternate3" : "COREv2-Flux" + }, + "taux" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C", + "alternate3" : "COREv2-Flux" + }, + "tauv" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C", + "alternate3" : "COREv2-Flux" + }, + "tauy" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C", + "alternate3" : "COREv2-Flux" + }, + "tas" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C" + }, + "ts" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "NOAA-20C", + "alternate1" : "HadISST2" + }, + "sst" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "NOAA-20C", + "alternate1" : "HadISST2" + }, + "sfcWind" : { + "defaultpi" : "NOAA-20C", + "default" : "NOAA-20C", + "alternate" : "ERA5", + "alternate1" : "MERRA2", + "alternate2" : "ERA-Interim" + }, + "hfls" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C", + "alternate3" : "OAFlux" + }, + "hfss" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "MERRA2", + "alternate1" : "ERA-Interim", + "alternate2" : "NOAA-20C", + "alternate3" : "OAFlux" + }, + "evspsbl" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "NOAA-20C" + }, + "clt" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate3" : "NOAA-20C" + }, + "clwvi" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "NOAA-20C" + }, + "clivi" : { + "defaultpi" : "NOAA-20C", + "default" : "ERA5", + "alternate" : "NOAA-20C" + }, + "tasmin" : { + "defaultpi" : "NOAA-20C", + "default" : "MERRA2" + }, + "tasmax" : { + "defaultpi" : "NOAA-20C", + "default" : "MERRA2" + }, + "sic" : { + "defaultpi" : "HadSST2", + "default" : "HadSST2" + }, + "tos" : { + "defaultpi" : "HadSST2", + "default" : "HadSST2" + }, + "zos" : { + "defaultpi" : "AVISO", + "default" : "AVISO", + "alternate" : "HadISST" + }, + "sos" : { + "defaultpi" : "Aquarius", + "default" : "Aquarius", + "alternate" : "HadISST" + } +} diff --git a/zppy/templates/inclusions/pcmdi_diags/regions_specs.json b/zppy/templates/inclusions/pcmdi_diags/regions_specs.json new file mode 100755 index 00000000..811eb1e9 --- /dev/null +++ b/zppy/templates/inclusions/pcmdi_diags/regions_specs.json @@ -0,0 +1,263 @@ +{ + "global": { + "domain": { "latitude":[-90.0, 90.0]} + }, + "NH": { + "domain": { "latitude":[0.0, 90.0]} + }, + "SH": { + "domain": { "latitude":[-90.0, 0]} + }, + "NHEX": { + "domain": { "latitude":[30.0, 90.0]} + }, + "SHEX": { + "domain": { "latitude":[-90.0, -30.0]} + }, + "TROPICS": { + "domain": { "latitude":[-30.0, 30.0]} + }, + "90S50S": { + "domain": { "latitude":[-90.0, -50.0]} + }, + "50S20S": { + "domain": { "latitude":[-50.0, -20.0]} + }, + "20S20N": { + "domain": { "latitude":[-20.0, 20.0]} + }, + "20N50N": { + "domain": { "latitude":[20.0, 50.0]} + }, + "50N90N": { + "domain": { "latitude":[50.0, 90.0]} + }, + "ocean_NH": { + "value": 0.0, + "domain": { "latitude":[0.0, 90.0]} + }, + "ocean_SH": { + "value": 0.0, + "domain": { "latitude":[-90.0, 0.0]} + }, + "land_NH": { + "value": 100, + "domain": { "latitude":[0.0, 90.0]} + }, + "land_SH": { + "value": 100, + "domain": { "latitude":[-90.0, 0.0]} + }, + "land_NHEX": { + "value": 100, + "domain": { "latitude":[30.0, 90.0]} + }, + "land_SHEX": { + "value": 100, + "domain": { "latitude":[-90.0, -30.0]} + }, + "land_TROPICS": { + "value": 100, + "domain": { "latitude":[-30.0, 30.0]} + }, + "land": { + "value": 100 + }, + "ocean_NHEX": { + "value": 0, + "domain": { "latitude":[30.0, 90.0]} + }, + "ocean_SHEX": { + "value": 0, + "domain": { "latitude":[-90.0, -30.0]} + }, + "ocean_TROPICS": { + "value": 0, + "domain": { "latitude":[30.0, 30.0]} + }, + "ocean": { + "value": 0 + }, + "ocean_50S50N": { + "value": 0.0, + "domain": { "latitude":[-50.0, 50.0]} + }, + "ocean_50S20S": { + "value": 0.0, + "domain": { "latitude":[-50.0, -20.0]} + }, + "ocean_20S20N": { + "value": 0.0, + "domain": { "latitude":[-20.0, 20.0]} + }, + "ocean_20N50N": { + "value": 0.0, + "domain": { "latitude":[20.0, 50.0]} + }, + "ocean_50N90N": { + "value": 0.0, + "domain": { "latitude":[50.0, 90.0]} + }, + "ocean_90S50S": { + "value": 0.0, + "domain": { "latitude":[-90.0, -50.0]} + }, + "NAM": { + "domain": { "latitude":[20.0, 90], + "longitude":[-180, 180]} + }, + "NAO": { + "domain": { "latitude":[20.0, 80], + "longitude":[-90, 40]} + }, + "SAM": { + "domain": { "latitude":[-20.0, -90], + "longitude":[0, 360]} + }, + "PSA1": { + "domain": { "latitude":[-20.0, -90], + "longitude":[0, 360]} + }, + "PSA2": { + "domain": { "latitude":[-20.0, -90], + "longitude":[0, 360]} + }, + "PNA": { + "domain": { "latitude":[20.0, 85], + "longitude":[120, 240]} + }, + "PDO": { + "domain": { "latitude":[20.0, 70], + "longitude":[110, 260]} + }, + "AMO": { + "domain": { "latitude":[0.0, 70], + "longitude":[-80, 0]} + }, + "AllMW": { + "domain": { "latitude":[-40.0, 45.0], + "longitude":[0.0, 360.0]} + }, + "AllM": { + "domain": { "latitude":[-45.0, 45.0], + "longitude":[0.0, 360.0]} + }, + "NAMM": { + "domain": { "latitude":[0.0, 45.0], + "longitude":[210.0, 310.0]} + }, + "SAMM": { + "domain": { "latitude":[-45.0, 0.0], + "longitude":[240.0, 330.0]} + }, + "NAFM": { + "domain": { "latitude":[0.0, 45.0], + "longitude":[310.0, 60.0]} + }, + "SAFM": { + "domain": { "latitude":[-45.0, 0.0], + "longitude":[0.0, 90.0]} + }, + "ASM": { + "domain": { "latitude":[0.0, 45.0], + "longitude":[60.0, 180.0]} + }, + "AUSM": { + "domain": { "latitude":[-45.0, 0.0], + "longitude":[90.0, 160.0]} + }, + "AIR": { + "domain": { "latitude":[7.0, 25.0], + "longitude":[65.0, 85.0]} + }, + "AUS": { + "domain": { "latitude":[-20.0, -10.0], + "longitude":[120.0, 150.0]} + }, + "Sahel": { + "domain": { "latitude":[13.0, 18.0], + "longitude":[-10.0, 10.0]} + }, + "GoG": { + "domain": { "latitude":[0.0, 5.0], + "longitude":[-10.0, 10.0]} + }, + "NAmo": { + "domain": { "latitude":[20.0, 37.0], + "longitude":[-112.0, -103.0]} + }, + "SAmo": { + "domain": { "latitude":[-20.0, 2.5], + "longitude":[-65.0, -40.0]} + }, + "Nino34": { + "value": 0.0, + "domain": { "latitude":[-5.0, 5.0], + "longitude":[190.0, 240.0]} + }, + "Nino3": { + "value": 0.0, + "domain": { "latitude":[-5.0, 5.0], + "longitude":[210.0, 270.0]} + }, + "Nino4": { + "value": 0.0, + "domain": { "latitude":[-5.0, 5.0], + "longitude":[160.0, 210.0]} + }, + "ONI": { + "value": 0.0, + "domain": { "latitude":[-5.0, 5.0], + "longitude":[190.0, 240.0]} + }, + "Nino12": { + "value": 0.0, + "domain": { "latitude":[-10.0, 0.0], + "longitude":[270.0, 280.0]} + }, + "AMMS": { + "value": 0.0, + "domain": { "latitude":[-15.0, -5.0], + "longitude":[-20.0, 10.0]} + }, + "AMMN": { + "value": 0.0, + "domain": { "latitude":[5.0, 15.0], + "longitude":[-50.0, -20.0]} + }, + "ATL3": { + "value": 0.0, + "domain": { "latitude":[-3.0, 3.0], + "longitude":[-20.0, 0.0]} + }, + "TSA": { + "value": 0.0, + "domain": { "latitude":[-20.0, 0.0], + "longitude":[-30.0, 10.0]} + }, + "TNA": { + "value": 0.0, + "domain": { "latitude":[5.5, 23.5], + "longitude":[302.5, 345.0]} + }, + "TIO": { + "value": 0.0, + "domain": { "latitude":[-15.0, 15.0], + "longitude":[40.0, 115.0]} + }, + "IODE": { + "value": 0.0, + "domain": { "latitude":[-10.0, 10.0], + "longitude":[50.0, 70.0]} + }, + "IODW": { + "value": 0.0, + "domain": { "latitude":[-10.0, 0.0], + "longitude":[90.0, 110.0]} + }, + "SOCN": { + "value": 0.0, + "domain": { "latitude":[-70.0, -50.0], + "longitude":[0.0, 360.0]} + } +} diff --git a/zppy/templates/inclusions/pcmdi_diags/synthetic_metrics_list.json b/zppy/templates/inclusions/pcmdi_diags/synthetic_metrics_list.json new file mode 100755 index 00000000..e9262d2a --- /dev/null +++ b/zppy/templates/inclusions/pcmdi_diags/synthetic_metrics_list.json @@ -0,0 +1,58 @@ +{ + "mean_climate" : { + "mae_xy" : { + "name" : "Mean Absolute Error Normalized by Median", + "type" : ["portrait"], + "region" : ["global","ocean","land","NHEX","SHEX","TROPICS"], + "season" : ["djf", "mam", "jja", "son"] + }, + "cor_xy" : { + "name" : "Pattern Corr.", + "type" : ["portrait"], + "region" : ["global","ocean","land","NHEX","SHEX","TROPICS"], + "season" : ["djf", "mam", "jja", "son"] + }, + "rms_xy" : { + "name" : "RMSE Normalized by Median", + "type" : ["portrait"], + "region" : ["global","ocean","land","NHEX","SHEX","TROPICS"], + "season" : ["djf", "mam", "jja", "son"] + }, + "bias_xy" : { + "name" : "Annual Mean Bias", + "type" : ["parcoord"], + "region" : ["global","ocean","land","NHEX","SHEX","TROPICS"], + "season" : ["ann"] + }, + "rms_xyt" : { + "name" : "Spatiotemporal RMSE", + "type" : ["parcoord"], + "region" : ["global","ocean","land","NHEX","SHEX","TROPICS"], + "season" : ["ann"] + } + }, + "variability_modes" : { + "rms" : { + "name" : "RMSE Normalized by Median", + "type" : ["portrait","parcoord"], + "season" : [] + }, + "rmsc" : { + "name" : "CenteredRMSE Normalized by Median", + "type" : ["portrait","parcoord"], + "season" : [] + }, + "stdv_pc_ratio_to_obs" : { + "name" : "Amplitude Ratio to Reference", + "type" : ["portrait","parcoord"], + "season" : [] + } + }, + "enso_metric" : { + "skill" : { + "collection" : ["ENSO_perf", "ENSO_tel", "ENSO_proc"], + "type" : ["portrait"], + "season" : [] + } + } +} diff --git a/zppy/templates/pcmdi_diags.bash b/zppy/templates/pcmdi_diags.bash new file mode 100755 index 00000000..1b244de9 --- /dev/null +++ b/zppy/templates/pcmdi_diags.bash @@ -0,0 +1,976 @@ +#!/bin/bash +{% include 'inclusions/slurm_header.bash' %} + +{{ environment_commands }} + +# Turn on debug output if needed +debug={{ debug }} +if [[ "${debug,,}" == "true" ]]; then + set -x +fi + +# Need this setup as otherwise can not generate diagnostics +export UCX_SHM_DEVICES=all # or not set UCX_NET_DEVICES at all + +# Make sure UVCDAT doesn't prompt us about anonymous logging +export UVCDAT_ANONYMOUS_LOG=False + +# Script dir +cd {{ scriptDir }} + +# Get jobid +id=${SLURM_JOBID} + +# Update status file +STARTTIME=$(date +%s) +echo "RUNNING ${id}" > {{ prefix }}.status + +# Basic definitions +case="{{ case }}" +www="{{ www }}" +run_type="{{ run_type }}" +results_dir="{{ run_type }}" + +{% if current_set != "synthetic_plots" %} + +# Input time range +y1={{ year1 }} +y2={{ year2 }} +ref_y1={{ ref_year1 }} +ref_y2={{ ref_year2 }} +ref_start_yr={{ ref_start_yr }} +ref_final_yr={{ ref_final_yr }} + +# Formatted versions +Y1="$(printf "%04d" ${y1})" +Y2="$(printf "%04d" ${y2})" +ref_Y1="$(printf "%04d" ${ref_y1})" +ref_Y2="$(printf "%04d" ${ref_y2})" + +num_years=$((y2 - y1 + 1)) +ref_end_yr=$((ref_y1 + num_years - 1)) + +# Keep originals for fallback +orig_ref_y1=${ref_y1} +orig_ref_y2=${ref_y2} +if [[ ${orig_ref_y1} -lt ${ref_start_yr} ]]; then + orig_ref_y1=${ref_start_yr} +fi +if [[ ${orig_ref_y2} -gt ${ref_final_yr} ]]; then + orig_ref_y2=${ref_final_yr} +fi +# Refine reference range +if [[ ${ref_start_yr} -le ${y1} && ${ref_final_yr} -ge ${y2} ]]; then + # Availability fully covers analysis: use [y1, y2] + ref_y1=${y1} + ref_y2=${y2} +else + # Case 2: Prefer the user's clamped custom window; only tweak if needed + # 2a) If clamped custom overlaps the analysis window, use the intersection + if [[ ${orig_ref_y1} -le ${y2} && ${orig_ref_y2} -ge ${y1} ]]; then + # minimal trimming to fit inside [y1, y2] + ref_y1=$(( orig_ref_y1 > y1 ? orig_ref_y1 : y1 )) + ref_y2=$(( orig_ref_y2 < y2 ? orig_ref_y2 : y2 )) + else + # 2b) No overlap between custom and analysis: + # try availability i∩ analysis (minimal change wrt both) + ovl_start=$(( ref_start_yr > y1 ? ref_start_yr : y1 )) + ovl_end=$(( ref_final_yr < y2 ? ref_final_yr : y2 )) + if [[ ${ovl_start} -le ${ovl_end} ]]; then + ref_y1=${ovl_start} + ref_y2=${ovl_end} + else + # 2c) Truly no overlap possible; fall back to clamped custom window + ref_y1=${orig_ref_y1} + ref_y2=${orig_ref_y2} + fi + fi +fi + +{% if current_set == "mean_climate"%} +source_vars="{{ clim_vars }}" +{% elif current_set == "variability_modes_cpl"%} +source_vars="{{ movc_vars }}" +{% elif current_set == "variability_modes_atm"%} +source_vars="{{ mova_vars }}" +{% elif current_set == "enso"%} +source_vars="{{ enso_vars }}" +{% endif %} + +{% endif %} + +# Top-level directory +web_dir=${www}/${case}/pcmdi_diags + +################################################## +# info to construct pcmdi-preferred data convention +################################################## +model_name='{{ model_name }}' +tableID='{{ model_tableID }}' +{% if run_type == "model_vs_obs" %} +# The put_model_here gets replaced is passed into Python as a string literal +# And then replaced inside the Python code. +model_name_ref='obs.historical.put_model_here.00' +tableID_ref=${tableID} +{% elif run_type == "model_vs_model" %} +model_name_ref='{{ model_name_ref }}' +tableID_ref='{{ model_tableID_ref }}' +{%- endif %} +case_id=v$(date '+%Y%m%d') + +# Create temporary workdir +workdir=`mktemp -d tmp.{{ prefix }}.${id}.XXXX` +cd ${workdir} + +# files for definition of regions for regional mean +cat > regions_specs.json << EOF +{% include regions_specs %} +EOF + +# file for aliases of observation datasets +cat > reference_alias.json << EOF +{% include reference_alias %} +EOF + +# file for list of variables for synthetic_metrics metric plots +cat > synthetic_metrics_list.json << EOF +{% include synthetic_metrics_list %} +EOF + +{% if current_set == "mean_climate" %} +create_links_acyc_climo() { + local ts_dir_source="$1" + local ts_dir_destination="$2" + local begin_year="$3" + local end_year="$4" + local name_key="$5" + local error_num="$6" + + echo "create_links_acyc_climo: linking from ${ts_dir_source} to ${ts_dir_destination}" + + mkdir -p "${ts_dir_destination}" + cd "${ts_dir_destination}" || exit + + local variables="{{ cmip_vars }}" + local script_dir="{{ scriptDir }}" + local prefix="{{ prefix }}" + local ts_step="{{ ts_num_years }}" + local dofm=(15.5 45 74.5 105 125.5 166 196.5 227.5 258 288.5 319 349.5) + + for v in ${variables//,/ }; do + > "${v}_files.txt" # Start fresh + + shopt -s nullglob + for year in $(seq "${begin_year}" "${ts_step}" "${end_year}"); do + local YYYY + YYYY=$(printf "%04d" "${year}") + for file in ${ts_dir_source}/${v}_*_${YYYY}*.nc; do + [[ -f "${file}" ]] && echo "${file}" >> "${v}_files.txt" + done + done + shopt -u nullglob + + # Derive monthly climatology files + for month in $(seq 1 12); do + local MM + MM=$(printf "%02d" "${month}") + ncra -O -h -F -d time,"${month}",,12 $(< "${v}_files.txt") "${v}_clm_${MM}.nc" + done + + # Combine to form full annual cycle file + local combined_name="${name_key}.${v}.${begin_year}01-${end_year}12.AC.${case_id}.nc" + ncrcat -O -d time,0, "${v}_clm_"*.nc "${combined_name}" + + # Adjust time metadata for PCMDI diagnostics + local cmdfix1='time[time]={15.5, 45, 74.5, 105, 125.5, 166, 196.5, 227.5, 258, 288.5,319, 349.5}' + local cmdfix2='time_bnds[time,bnds]={0,31,31,59,59,90,90,120,120,151,151,181,181,212,212,243,243,273,273,304,304,334,334,365.}' + local cmdfix3='time@units="days since 1850-01-01 00:00:00"' + local cmdfix4='time@calendar="noleap"' + local cmdfix5='time@bounds="time_bnds"' + ncap2 -O -h -s "${cmdfix1};${cmdfix2};${cmdfix3};${cmdfix4};${cmdfix5}" "${combined_name}" "${combined_name}" + + rm -vf "${v}_clm_"*.nc + + if [[ $? -ne 0 ]]; then + cd "${script_dir}" || exit + echo "ERROR (${error_num})" > "${prefix}.status" + exit "${error_num}" + fi + done + + if [ -z "$( ls . )" ]; then + echo "create_links_acyc_climo: ${ts_dir_destination} was not updated!" + fi + + cd .. +} + +{% if run_type == "model_vs_obs" %} +create_links_acyc_climo_obs() { + local ts_dir_source="$1" + local ts_dir_destination="$2" + local begin_year="$3" + local end_year="$4" + local error_num="$5" + + local script_dir="{{ scriptDir }}" + local prefix="{{ prefix }}" + local dofm=(15.5 45 74.5 105 125.5 166 196.5 227.5 258 288.5 319 349.5) + + echo "create_links_acyc_climo_obs: linking from ${ts_dir_source} to ${ts_dir_destination}" + + mkdir -p "${ts_dir_destination}" + cd "${ts_dir_destination}" || exit + + for file in ${ts_dir_source}/*; do + local fname + local YYYYS YYYYE + local SUBSTR + local ttag tmp_file MM combined_name + + fname=$(basename "${file}") + if [[ ! ${fname} =~ ".nc" ]]; then + # Skip non-.nc files + continue + fi + + # Match two date patterns (YYYYMM or YYYYMMDD) separated by _ or - + if [[ ${fname} =~ ([0-9]{6,8})[_-]([0-9]{6,8}) ]]; then + YYYYS="${BASH_REMATCH[1]}" + YYYYE="${BASH_REMATCH[2]}" + else + echo "Warning: Could not extract dates from ${fname}, basename of ${file}" + continue + fi + + # Clip to specified year range + if [[ ${YYYYS} -lt ${begin_year} ]]; then YYYYS=${begin_year}; fi + if [[ ${YYYYE} -gt ${end_year} ]]; then YYYYE=${end_year}; fi + + # Extract prefix before the date range (removes from .${YYYYS} or -${YYYYS}) + SUBSTR="${fname%%[._-]${YYYYS}*}" + + ttag="$(printf "%04d" "${YYYYS}")01-$(printf "%04d" "${YYYYE}")12" + tmp_file="tmp_combine_${ttag}.nc" + + ncrcat -O -d time,"${YYYYS}-01-01","${YYYYE}-12-31" "${file}" "${tmp_file}" + + # Derive monthly climatology + for month in $(seq 1 12); do + MM=$(printf "%02d" ${month}) + ncra -O -h -F -d time,"${month}",,12 "${tmp_file}" "tmp_clm_${MM}.nc" + done + + combined_name="${SUBSTR}.${ttag}.AC.${case_id}.nc" + ncrcat -O tmp_clm_*.nc "${combined_name}" + + # Adjust time metadata + local cmdfix1='time[time]={15.5, 45, 74.5, 105, 125.5, 166, 196.5, 227.5, 258, 288.5,319, 349.5}' + local cmdfix2='time@units="days since 1850-01-01 00:00:00"' + local cmdfix3='time@calendar="noleap"' + ncap2 -O -h -s "${cmdfix1};${cmdfix2};${cmdfix3}" "${combined_name}" "${combined_name}" + + local cmdfix4='defdim("bnds",2)' + local cmdfix5='time_bnds=make_bounds(time,$bnds,"time_bnds")' + local cmdfix6='time_bnds@units=time@units' + local cmdfix7='time_bnds@calendar=time@calendar' + ncap2 -O -h -s "${cmdfix4};${cmdfix5};${cmdfix6};${cmdfix7}" "${combined_name}" "${combined_name}" + + rm -vf tmp_*.nc + + if [[ $? -ne 0 ]]; then + cd "${script_dir}" || exit + echo "ERROR (${error_num})" > "${prefix}.status" + exit "${error_num}" + fi + done + + if [ -z "$( ls . )" ]; then + echo "create_links_acyc_climo_obs: ${ts_dir_destination} was not updated!" + fi + + cd .. +} +{% endif %} +{% endif %} + +{% if current_set in ["variability_modes_cpl", "variability_modes_atm", "enso"] %} +create_links_ts() { + local ts_dir_source="$1" + local ts_dir_destination="$2" + local begin_year="$3" + local end_year="$4" + local subname="$5" + local error_num="$6" + + local script_dir="{{ scriptDir }}" + local prefix="{{ prefix }}" + local ts_step="{{ ts_num_years }}" + local vars="${source_vars}" + + echo "create_links_ts: linking from ${ts_dir_source} to ${ts_dir_destination}" + + mkdir -p "${ts_dir_destination}" + cd "${ts_dir_destination}" || exit + + local v file YYYY combined_name + + # Convert comma-separated list to array + IFS=',' read -ra var_array <<< "${vars}" + for v in "${var_array[@]}"; do + > "${v}_files.txt" # Reset file list + + shopt -s nullglob + for year in $(seq "${begin_year}" "${ts_step}" "${end_year}"); do + YYYY=$(printf "%04d" "${year}") + for file in ${ts_dir_source}/${v}_*_${YYYY}*.nc; do + [[ -f "${file}" ]] && echo "${file}" >> "${v}_files.txt" + done + done + shopt -u nullglob + + combined_name="${subname}.${v}.${begin_year}01-${end_year}12.nc" + if [[ -s "${v}_files.txt" ]]; then + ncrcat -O -v "${v}" -d time,"${begin_year}-01-01","${end_year}-12-31" $(< "${v}_files.txt") "${combined_name}" + + # Add calendar attribute if missing + if ! ncks -m "${combined_name}" | grep -q "calendar"; then + echo "Adding missing calendar attribute to time..." + ncatted -a calendar,time,o,c,"standard" "${combined_name}" + fi + + # Add time bounds + local cmdfix1='defdim("bnds",2)' + local cmdfix2='time_bnds=make_bounds(time,$bnds,"time_bnds")' + local cmdfix3='time_bnds@units=time@units' + local cmdfix4='time_bnds@calendar=time@calendar' + ncap2 -O -h -s "${cmdfix1};${cmdfix2};${cmdfix3};${cmdfix4}" "${combined_name}" "${combined_name}" + + if [[ $? -ne 0 ]]; then + cd "${script_dir}" || exit + echo "ERROR (${error_num})" > "${prefix}.status" + exit "${error_num}" + fi + else + echo "Warning: No input files found for variable ${v} in ${ts_dir_source}. Skipping." + fi + done + + if [ -z "$( ls . )" ]; then + echo "create_links_ts: ${ts_dir_destination} was not updated!" + fi + + cd .. +} + +{% if run_type == "model_vs_obs" %} +create_links_ts_obs() { + local ts_dir_source="$1" + local ts_dir_destination="$2" + local begin_year="$3" + local end_year="$4" + local error_num="$5" + + local script_dir="{{ scriptDir }}" + local prefix="{{ prefix }}" + + echo "create_links_ts_obs: linking from ${ts_dir_source} to ${ts_dir_destination}" + + mkdir -p "${ts_dir_destination}" + cd "${ts_dir_destination}" || exit + + local file fname SUBSTR YYYYS YYYYE ttag combined_name + + for file in ${ts_dir_source}/*; do + fname=$(basename "$file") + echo "create_links_ts_obs: checking if nc file: ${fname}" + if [[ ! ${fname} =~ ".nc" ]]; then + # Skip non-.nc files + continue + fi + echo "create_links_ts_obs: processing ${file}" + # Match two time patterns (YYYYMM or YYYYMMDD) separated by _ or - + if [[ $fname =~ ^(.+)\.([0-9]{4})([0-9]{2}){1,2}[-_]([0-9]{4})([0-9]{2}){1,2}\.nc$ ]]; then + SUBSTR="${BASH_REMATCH[1]}" # everything before the .YYYY... + YYYYS="${BASH_REMATCH[2]}" # start year + YYYYE="${BASH_REMATCH[4]}" # end year + else + + echo "Warning: Could not extract dates from ${fname}, basename of ${file}" + continue + fi + + # Optional: clip years if needed + if [[ ${YYYYS} -lt ${begin_year} ]]; then + YYYYS="${begin_year}" + fi + + if [[ ${YYYYE} -gt ${end_year} ]]; then + YYYYE="${end_year}" + fi + + ttag="$(printf "%04d" "${YYYYS}")01-$(printf "%04d" "${YYYYE}")12" + combined_name="${SUBSTR}.${ttag}.nc" + + # Extract subset of time series + ncrcat -O -d time,"${YYYYS}-01-01","${YYYYE}-12-31" "${file}" "${combined_name}" + if [[ $? -ne 0 ]]; then + cd "${script_dir}" || exit + echo "ERROR (${error_num})" > "${prefix}.status" + exit "${error_num}" + fi + echo "ncrcat successful" + + # Ensure time has calendar attribute + if ! ncks -m "${combined_name}" | grep -q "calendar"; then + echo "Adding missing calendar attribute to time..." + ncatted -a calendar,time,o,c,"standard" "${combined_name}" + if [[ $? -ne 0 ]]; then + cd "${script_dir}" || exit + echo "ERROR (${error_num})" > "${prefix}.status" + exit "${error_num}" + fi + echo "ncatted successful" + fi + + # Add time bounds + local cmdfix1='defdim("bnds",2)' + local cmdfix2='time_bnds=make_bounds(time,$bnds,"time_bnds")' + local cmdfix3='time_bnds@units=time@units' + local cmdfix4='time_bnds@calendar=time@calendar' + ncap2 -O -h -s "${cmdfix1};${cmdfix2};${cmdfix3};${cmdfix4}" "${combined_name}" "${combined_name}" + if [[ $? -ne 0 ]]; then + cd "${script_dir}" || exit + echo "ERROR (${error_num})" > "${prefix}.status" + exit "${error_num}" + fi + echo "ncap2 successful" + done + + if [ -z "$( ls . )" ]; then + echo "create_links_ts_obs: ${ts_dir_destination} was not updated!" + cd "${script_dir}" || exit + echo "ERROR (${error_num})" > "${prefix}.status" + exit "${error_num}" + fi + + cd .. +} +{% endif %} +{% endif %} + +######################## +# Prepare the model data +######################## +{% if current_set == "mean_climate" %} +# Define output directory for climatology files +climo_dir_primary="climo" +# Path to model's monthly climatology files +# This cmip_ts path is created via zppy/templates/e3sm_to_cmip.bash +climo_dir_source="{{ output }}/post/atm/{{ grid }}/cmip_ts/monthly" +# Link and process primary model climo data +create_links_acyc_climo "${climo_dir_source}" "${climo_dir_primary}" "${Y1}" "${Y2}" "${model_name}.${tableID}" 1 +{% if run_type == "model_vs_model" %} +# Path to reference model's climatology files +climo_dir_source_ref="{{ reference_data_path_ts }}" +climo_dir_ref="climo_ref" +# Link and process reference model climo data +create_links_acyc_climo "${climo_dir_source_ref}" "${climo_dir_ref}" "${ref_Y1}" "${ref_Y2}" "${model_name_ref}.${tableID_ref}" 2 +{% endif %} +{% endif %} + +{% if current_set in ["variability_modes_cpl", "variability_modes_atm", "enso"] %} +# All these diagnostics use time series (ts) data +# Define output directory for primary model time series +ts_dir_primary="ts" +ts_dir_source="{{ output }}/post/atm/{{ ts_grid }}/cmip_ts/monthly" +# Create local links and combine time series NetCDF files for the primary model +create_links_ts "${ts_dir_source}" "${ts_dir_primary}" "${Y1}" "${Y2}" "${model_name}.${tableID}" 3 +{% if run_type == "model_vs_model" %} +# Define time series path for reference model (adjust for different year spans) +ts_dir_source_ref="{{ reference_data_path_ts }}" +ts_dir_ref="ts_ref" +# Create local links and combine ts files for the reference model +create_links_ts "${ts_dir_source_ref}" "${ts_dir_ref}" "${ref_Y1}" "${ref_Y2}" "${model_name_ref}.${tableID_ref}" 4 +{% endif %} +{% endif %} + +{% if run_type == "model_vs_obs" and current_set != "synthetic_plots" %} +########################################################################### +# Prepare the observation data +# Observation datasets vary by diagnostic, so we use an external +# Python utility to handle linking and remapping to standard names. +########################################################################### +obstmp_dir="obs_link" +mkdir -p "${obstmp_dir}" +echo "Linking observational data into ${obstmp_dir}..." + +################### +# Run process job +################### +echo "Linking observational data using SLURM..." + +command="zi-pcmdi-link-observation --model_name_ref ${model_name_ref} --tableID_ref ${tableID_ref} --vars=${source_vars} --obs_sets {{ obs_sets }} --obs_ts {{ obs_ts }} --obstmp_dir ${obstmp_dir} --debug ${debug,,}" +echo "Running a zi-pcmdi command: ${command}" + +{{ environment_commands_secondary }} +time eval "${command}" +{{ environment_commands }} + +if [ $? -ne 0 ]; then + cd {{ scriptDir }} + echo "ERROR (6)" > {{ prefix }}.status + exit 6 +fi + +####################################################### +# Now create obs climo and time series for PCMDI diags +# Use same period as test model when possible +####################################################### +ts_dir_ref_source="{{ scriptDir }}/${workdir}/${obstmp_dir}" + +{% if current_set == "mean_climate" %} +climo_dir_ref=climo_ref +create_links_acyc_climo_obs "${ts_dir_ref_source}" "${climo_dir_ref}" ${ref_Y1} ${ref_Y2} 7 +{% elif current_set in ["variability_modes_cpl", "variability_modes_atm", "enso"] %} +ts_dir_ref=ts_ref +create_links_ts_obs "${ts_dir_ref_source}" "${ts_dir_ref}" ${ref_Y1} ${ref_Y2} 8 +{% endif %} + +{% endif %} + +{% if current_set != "synthetic_plots" %} +######################################################## +# generate basic parameter file for pcmdi metrics driver +######################################################## +echo "About to create parameterfile.py, which will be passed in with -p" +echo "The current directory is: $PWD" # This will be of the form .../post/scripts/tmpDir + +cat > parameterfile.py << EOF +import os +import sys +import json +##################### +# Basic Information +##################### + +start_yr = int('${Y1}') +end_yr = int('${Y2}') +num_years = end_yr - start_yr + 1 +period = f"{start_yr:04d}01-{end_yr:04d}12" + +model_parts = '${model_name}'.split('.') +mip, exp, product, realm = model_parts[:4] + +############################################## +# Configuration Shared with PCMDI Diagnostics +############################################## + +# Whether to generate NetCDF outputs for observations and model results +nc_out_obs = {{ mov_nc_out_obs }} +nc_out_model = {{ mov_nc_out_model }} + +# Output file extension: use .nc if either output is enabled, +# otherwise default to .xml +ext = ".nc" if nc_out_model or nc_out_obs else ".xml" + +# User annotation and debug flag +user_notes = 'Provenance and results' +debug = {{ pcmdi_debug }} + +# Enable plot generation for model and observation +plot = {{ mov_plot_model }} +plot_obs = {{ mov_plot_obs }} # optional + +# Execution mode and output format +run_type = '{{ run_type }}' +figure_format = '{{ figure_format }}' + +# Save interpolated model climatologies? +save_test_clims = {{ save_test_clims }} + +# Save all metrics results in a single file? +# Set to 'n' as metrics are computed per variable +metrics_in_single_file = 'n' + +# Custom values for land/sea masking +regions_values = { + "land": 100.0, + "ocean": 0.0 +} + +# Template path for land/sea mask file (fixed input) +modpath_lf = os.path.join( + 'fixed', + 'sftlf.%(model).nc' +) + +{% if current_set == "mean_climate" %} + +############################################ +# Setup Specific for Mean Climate Metrics +############################################ +modver = "${case_id}" +parallel = False +generate_sftlf = False +sftlf_filename_template = modpath_lf + +# Target grid: can be '2.5x2.5' or a CDMS2 grid object string +target_grid = '{{ target_grid }}' +targetGrid = target_grid # for backward compatibility +target_grid_string = '{{ target_grid_string }}' + +# Regridding tool and method (general use) +# OPTIONS: 'regrid2' or 'esmf' +regrid_tool = '{{ regrid_tool }}' +# OPTIONS: 'linear' or 'conservative' (only for 'esmf') +regrid_method = '{{ regrid_method }}' + +# Regridding tool and method for ocean diagnostics +regrid_tool_ocn = '{{ regrid_tool_ocn }}' # 'regrid2' or 'esmf' +regrid_method_ocn = ('{{ regrid_method_ocn }}') # 'linear' or 'conservative' + +# Model realization(s) to consider +realization = "*" + +# Model product name from input +test_data_set = [product] + +# Path to model climatology files +test_data_path = '${climo_dir_primary}' + +# Template for model climatology filenames +filename_template = '.'.join([ + mip, + exp, + '%(model)', + '%(realization)', + '${tableID}', + '%(variable)', + period, + 'AC', + '${case_id}', + 'nc' +]) + +# Path to reference climatology files +reference_data_path = '${climo_dir_ref}' + +# Observation catalogue file (dynamic by subsection) +custom_observations = os.path.join( + 'pcmdi_diags', + '{}_{}_catalogue.json'.format( + '${climo_dir_ref}', + '{{subsection}}' + ) +) + +# Load variable-specific region definitions +regions = json.load(open('regions.json')) + +# Load predefined region specifications and normalize domain lat/lon as tuples +regions_specs = json.load(open('regions_specs.json')) +for key in regions_specs: + domain = regions_specs[key].get('domain', {}) + if 'latitude' in domain: + domain['latitude'] = tuple(domain['latitude']) + regions_specs[key]['domain']['latitude'] = domain['latitude'] + if 'longitude' in domain: + domain['longitude'] = tuple(domain['longitude']) + regions_specs[key]['domain']['longitude'] = domain['longitude'] + +# METRICS OUTPUT +metrics_output_path = os.path.join( + 'pcmdi_diags', + 'metrics_results', + 'mean_climate', + mip, + exp, + '%(case_id)' +) + +#INTERPOLATED MODELS' CLIMATOLOGIES +diagnostics_output_path = os.path.join( + 'pcmdi_diags', + 'diagnostic_results', + 'mean_climate', + mip, + exp, + '%(case_id)' +) + +test_clims_interpolated_output = diagnostics_output_path + +{% endif %} + +{% if current_set in ["variability_modes_cpl", "variability_modes_atm"] %} +# Setup for Mode Variability Diagnostics +msyear = int(start_yr) +meyear = int(end_yr) + +# Seasons to analyze (comma-separated string to list) +seasons = '{{ seasons }}'.split(",") + +# Data frequency (e.g., monthly, seasonal) +frequency = '{{ frequency }}' + +# Variables to analyze (comma-separated string or space-separated) +{% if current_set == "variability_modes_cpl"%} +varModel = '{{ movc_vars }}' +{% elif current_set == "variability_modes_atm"%} +varModel = '{{ mova_vars }}' +{% endif %} + +# Unit conversion flags for model and observations +ModUnitsAdjust = {{ ModUnitsAdjust }} +ObsUnitsAdjust = {{ ObsUnitsAdjust }} + +# Mask out land regions (consider ocean-only if True) +landmask = {{ landmask }} + +# Maximum number of eof modes +eofn_mod_max = {{ eofn_mod_max }} + +# If True, remove domain mean from each time step +RmDomainMean = {{ RmDomainMean }} + +# If True, normalize EOFs to unit variance +EofScaling = {{ EofScaling }} + +# Conduct Combined EOF/CBF analysis (if True) +CBF = {{ CBF }} + +# Conduct Conventional EOF analysis (if True) +ConvEOF = {{ ConvEOF }} + +# Skip CMEC output (hardcoded for now) +cmec = False + +# Whether to overwrite existing diagnostic output +update_json = False + +# Template for model input file paths +modnames = [product] +realization = "*" +modpath = os.path.join( + '${ts_dir_primary}', + '{}.{}.%(model).%(realization).{}.%(variable).{}.nc'.format( + mip, exp, '${tableID}', period + ) +) + +# Output results directory +results_dir = os.path.join( + 'pcmdi_diags', + '%(output_type)', + 'variability_modes', + '%(mip)', + '%(exp)', + '${case_id}', + '%(variability_mode)', + '%(reference_data_name)', +) +{% endif %} + +{% if current_set == "enso" %} +# Parameter Setup for ENSO Metrics +modnames = [product] +realization = realm + +modpath = os.path.join( + '${ts_dir_primary}', + '{}.{}.%(model).%(realization).{}.%(variable).{}.nc'.format( + mip, exp, '${tableID}', period + ) +) + +# Observation/Reference settings +obs_cmor = True +obs_cmor_path = '${ts_dir_ref}' +obs_catalogue = 'obs_catalogue.json' + +# Land/Sea mask for reference data +reference_data_lf_path = json.load(open('obs_landmask.json')) + +# Metrics collection type (e.g., ENSO_perf, ENSO_tel, ENSO_proc) +# Defined externally via metricsCollection + +# Output directory structure +results_dir = os.path.join( + 'pcmdi_diags', + '%(output_type)', + 'enso_metric', + '%(mip)', + '%(exp)', + '${case_id}', + '%(metricsCollection)', +) + +# Output filenames for JSON and NetCDF +json_name = "%(mip)_%(exp)_%(metricsCollection)_${case_id}_%(model)_%(realization)" + +netcdf_name = json_name + +{% endif %} + +EOF + +{% endif %} + +################################################################ +# Run PCMDI Diags +echo +echo ===== RUN PCMDI DIAGS ===== +echo +################################ + +echo "About to set up a zi-pcmdi command" +echo "The current directory is: $PWD" # This will be of the form .../post/scripts/tmpDir + +# Run diagnostics +mkdir -p pcmdi_diags + +{% if current_set != "synthetic_plots" %} +{% if current_set == "mean_climate"%} +source_dirs="--climo_ts_dir_primary ${climo_dir_primary} --climo_ts_dir_ref ${climo_dir_ref}" +{% elif current_set in ["variability_modes_cpl", "variability_modes_atm", "enso"] %} +source_dirs="--climo_ts_dir_primary ${ts_dir_primary} --climo_ts_dir_ref ${ts_dir_ref}" +{% endif %} +# Parameter passing can encounter errors if parameters are empty. +# So, it's a good idea to make sure they can't be (or at least are unlikely to be) empty. +# run_type == "model_vs_obs" only: obs_sets (default value is NOT "") +# run_type == "model_vs_model" only: model_name_ref, tableID_ref (default values are NOT "") +core_parameters="--num_workers {{ num_workers }} --multiprocessing {{ multiprocessing }} --subsection {{ subsection }} ${source_dirs} --model_name ${model_name} --model_tableID {{model_tableID }} --figure_format {{ figure_format }} --run_type {{ run_type }} --obs_sets {{ obs_sets }} --model_name_ref ${model_name_ref} --vars ${source_vars} --tableID_ref ${tableID_ref} --generate_sftlf {{ generate_sftlf }} --case_id ${case_id} --results_dir ${results_dir} --debug ${debug,,}" +{% endif %} + +{% if current_set == "mean_climate" %} +command="zi-pcmdi-mean-climate ${core_parameters} --regions {{ clim_regions }}" +{% endif %} +{% if current_set in ["variability_modes_cpl", "variability_modes_atm"] %} +{% if current_set == "variability_modes_atm" %} +var_modes={{ mova_modes }} +{% elif current_set == "variability_modes_cpl" %} +var_modes={{ movc_modes }} +{% endif %} +command="zi-pcmdi-variability-modes ${core_parameters} --var_modes ${var_modes}" +{% endif %} +{% if current_set == "enso" %} +command="zi-pcmdi-enso ${core_parameters} --enso_groups {{ enso_groups }}" +{% endif %} +{% if current_set == "synthetic_plots" %} + +#add command for mean climate viewer +clim_keys="--clim_viewer {{clim_viewer}} --cmip_clim_dir {{ cmip_clim_dir }} --cmip_clim_set {{ cmip_clim_set }} --clim_vars {{clim_vars}} --clim_years {{ clim_years }} --clim_regions {{ clim_regions }}" +{% if clim_viewer %} +echo "Checking if *.${case_id}.json files exist in clim_dir:" +clim_dir=${web_dir}/model_vs_obs/metrics_data/mean_climate/ +find ${clim_dir} -name "*.${case_id}*.json" +echo "Done checking. There should be a list of files above, if they exist." +{% endif %} + +#add command for modes variability viewer +movs_keys="--mova_viewer {{mova_viewer}} --movc_viewer {{movc_viewer}} --cmip_movs_dir {{ cmip_movs_dir }} --cmip_movs_set {{ cmip_movs_set }} --mova_modes {{ mova_modes }} --mova_vars {{mova_vars}} --mova_years {{mova_years}} --movc_modes {{ movc_modes }} --movc_vars {{movc_vars}} --movc_years {{movc_years}}" +{% if movc_viewer or mova_viewer %} +echo "Checking if var_mode_*.json files exist in variability_modes_dir:" +variability_modes_dir=${web_dir}/model_vs_obs/metrics_data/variability_modes/ +find ${variability_modes_dir} -name "var_mode_*${case_id}*.json" +echo "Done checking. There should be a list of files above, if they exist." +{% endif %} + +#add command for enso viewer +enso_keys="--enso_viewer {{enso_viewer}} --cmip_enso_dir {{ cmip_enso_dir }} --cmip_enso_set {{ cmip_enso_set }} --enso_vars {{enso_vars}} --enso_years {{enso_years}}" +{% if enso_viewer %} +echo "Checking if var_mode_*.json files exist in variability_modes_dir:" +enso_dir=${web_dir}/model_vs_obs/metrics_data/enso_metric/* +find ${enso_dir} -name "*.${case_id}*.json" +echo "Done checking. There should be a list of files above, if they exist." +{% endif %} + +command="zi-pcmdi-synthetic-plots --synthetic_sets {{ synthetic_sets }} --figure_format {{ figure_format }} --www ${www} --results_dir ${results_dir} --case {{ case }} --model_name {{ model_name }} --model_tableID {{model_tableID }} --web_dir=${web_dir} --pcmdi_webtitle {{ pcmdi_webtitle }} --pcmdi_version {{ pcmdi_version }} --run_type ${run_type} --pcmdi_external_prefix {{ diagnostics_base_path }} --pcmdi_viewer_template {{ pcmdi_viewer_template }} --save_all_data {{save_all_data}} ${clim_keys} ${movs_keys} ${enso_keys} --debug ${debug,,}" + +{% endif %} + +# Run diagnostics +{{ environment_commands_secondary }} +echo "Running a zi-pcmdi command: ${command}" +echo "The current directory is: $PWD" # This will be of the form .../post/scripts/tmpDir +time ${command} +if [ $? != 0 ]; then + cd {{ scriptDir }} + echo 'ERROR (11)' > {{ prefix }}.status + exit 11 +fi +{{ environment_commands }} + +################################# +# Copy output to web server +echo +echo ===== COPY FILES TO WEB SERVER ===== +echo + +# Create top-level directory +mkdir -p ${web_dir} +if [ $? != 0 ]; then + cd {{ scriptDir }} + echo 'ERROR (13)' > {{ prefix }}.status + exit 13 +fi + +{% if machine in ['pm-cpu', 'pm-gpu'] %} +# For NERSC, make sure it is world readable +f=`realpath ${web_dir}` +while [[ $f != "/" ]] +do + owner=`stat --format '%U' $f` + if [ "${owner}" = "${USER}" ]; then + chgrp e3sm $f + chmod go+rx $f + fi + f=$(dirname $f) +done +{% endif %} + +############################################ +# Copy files +#rsync -a --delete ${results_dir} ${web_dir}/ +{% if current_set != "synthetic_plots" %} +rsync -a ${results_dir} ${web_dir}/ +if [ $? != 0 ]; then + cd {{ scriptDir }} + echo 'ERROR (14)' > {{ prefix }}.status + exit 14 +fi +{% endif %} + +{% if machine in ['pm-cpu', 'pm-gpu'] %} +# For NERSC, change permissions of new files +pushd ${web_dir}/ +chgrp -R e3sm ${results_dir} +chmod -R go+rX,go-w ${results_dir} +popd +{% endif %} + +{% if machine in ['anvil', 'chrysalis'] %} +# For LCRC, change permissions of new files +pushd ${web_dir}/ +chmod -R go+rX,go-w ${results_dir} +popd +{% endif %} + +# Delete temporary workdir +cd .. +if [[ "${debug,,}" != "true" ]]; then + rm -rf ${workdir} +fi + +# Update status file and exit +{% raw %} +ENDTIME=$(date +%s) +ELAPSEDTIME=$(($ENDTIME - $STARTTIME)) +{% endraw %} +echo ============================================== +echo "Elapsed time: $ELAPSEDTIME seconds" +echo ============================================== +rm -f {{ prefix }}.status +echo 'OK' > {{ prefix }}.status +exit 0 diff --git a/zppy/utils.py b/zppy/utils.py index 0f4fa563..0b3ce877 100644 --- a/zppy/utils.py +++ b/zppy/utils.py @@ -7,7 +7,7 @@ import time from enum import Enum from subprocess import PIPE, Popen -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, List, Set, Tuple import jinja2 from configobj import ConfigObj @@ -228,6 +228,29 @@ def set_component_and_prc_typ(c: Dict[str, Any]) -> None: c["prc_typ"] = prc_typ +def check_set_specific_parameter( + c: Dict[str, Any], sets_with_requirement: Set[str], relevant_parameter: str +) -> None: + requested_sets: Set[str] + if ("sets" in c) and c["sets"]: + # For tasks that permit multiple sets + requested_sets = set(c["sets"]) + elif ("current_set" in c) and c["current_set"]: + # For tasks that permit only a single set + requested_sets = {c["current_set"]} + else: + requested_sets = set() + intersection = sets_with_requirement & requested_sets + if ( + intersection + and (relevant_parameter in c.keys()) + and (c[relevant_parameter] == "") + ): + raise ParameterNotProvidedError( + f"{relevant_parameter} is required because the sets {intersection} were requested." + ) + + # Return all year sets from a configuration given by a list of strings # "year_begin:year_end:year_freq" # "year_begin-year_end" @@ -307,6 +330,18 @@ def set_value_of_parameter_if_undefined( raise ParameterNotProvidedError( f"{parameter} was not provided, and inferring is turned off. Turn on inferring by setting {inference_type_parameter} to True." ) + # If parameter is already defined, do nothing. + + +def check_parameter_defined( + c: Dict[str, Any], relevant_parameter: str, explanation: str = "" +) -> None: + if (relevant_parameter not in c.keys()) or (c[relevant_parameter] == ""): + if explanation: + message = f"{relevant_parameter} is needed because {explanation}" + else: + message = f"{relevant_parameter} is not defined." + raise ParameterNotProvidedError(message) def get_file_names(script_dir: str, prefix: str):