From b88785a39065d69a66a50f15040e962d8f41defa Mon Sep 17 00:00:00 2001 From: Felicity Chun Date: Tue, 31 Mar 2026 14:34:22 +1100 Subject: [PATCH 1/2] new config for v2.14 --- config/config-developer.yml | 194 ----------------------- config/config-user.yml | 229 ++------------------------- config/data-esmvalcore-esgf.yml | 19 +++ config/data-hpc-nci.yml | 125 +++++++++++++++ config/data-native-access.yml | 14 ++ config/extra_facets_access.yml | 137 ++++++++++++++++ config/extra_facets_native6-era5.yml | 179 +++++++++++++++++++++ 7 files changed, 486 insertions(+), 411 deletions(-) delete mode 100644 config/config-developer.yml create mode 100644 config/data-esmvalcore-esgf.yml create mode 100644 config/data-hpc-nci.yml create mode 100644 config/data-native-access.yml create mode 100644 config/extra_facets_access.yml create mode 100644 config/extra_facets_native6-era5.yml diff --git a/config/config-developer.yml b/config/config-developer.yml deleted file mode 100644 index 62efe94..0000000 --- a/config/config-developer.yml +++ /dev/null @@ -1,194 +0,0 @@ -############################################################################### -# Developer's configuration file for the ESMValTool -############################################################################### -# This file retains the project- and machine-dependent directory and file name -# definitions of the input and output data. -# Each dictionary is structured as follows: -# -# PROJECT: -# input_dir: -# default: -# drs1: -# drs2: -# etc: -# input_file: -# output_file: -# -# Only the default drs is mandatory, the others are optional. -# -# In addition, an entry for the custom tables can be given. For this, only the -# option 'cmor_path' is considered, which specifies the directory from which -# custom CMOR tables are loaded. 'cmor_path' can be a relative path (relative -# to ESMValCore/esmvalcore/cmor/tables) or an absolute path. By default, uses -# ESMValCore/esmvalcore/cmor/tables/custom. -# -# custom: -# cmor_path: ~/my/own/custom_tables -############################################################################### ---- - -CMIP6: - cmor_strict: true - input_dir: - default: '/' - BADC: '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}' - DKRZ: '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}' - ESGF: '{project}/{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}' - ETHZ: '{exp}/{mip}/{short_name}/{dataset}/{ensemble}/{grid}/' - SYNDA: '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}' - NCI: '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}' - input_file: '{short_name}_{mip}_{dataset}_{exp}_{ensemble}_{grid}*.nc' - output_file: '{project}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}_{grid}' - cmor_type: 'CMIP6' - -CMIP5: - cmor_strict: true - input_dir: - default: '/' - BADC: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{version}/{short_name}' - NCI: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{version}/{short_name}' - BSC: '{type}/{project}/{exp}/{dataset.lower}' - CP4CDS: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{short_name}/latest/' - DKRZ: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{version}/{short_name}' - ETHZ: '{exp}/{mip}/{short_name}/{dataset}/{ensemble}/' - ESGF: '{project.lower}/{product}/{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{version}' - RCAST: '{exp}/{mip}/{short_name}/{dataset}/{ensemble}/' - SMHI: '{dataset}/{ensemble}/{exp}/{frequency}' - SYNDA: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{version}' - input_file: '{short_name}_{mip}_{dataset}_{exp}_{ensemble}*.nc' - output_file: '{project}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}' - -CMIP3: - cmor_strict: true - input_dir: - default: '/' - BADC: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{short_name}/{ensemble}/{version}' - DKRZ: '{exp}/{modeling_realm}/{frequency}/{short_name}/{dataset}/{ensemble}' - ESGF: '{project.lower}/{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{ensemble}/{short_name}/{version}' - IPSL: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{ensemble}/{short_name}/{version}/{short_name}' - NCI: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{ensemble}/{short_name}/{latestversion}' - input_file: '{short_name}_*.nc' - output_file: '{project}_{institute}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}' - cmor_type: 'CMIP3' - -OBS: - cmor_strict: false - input_dir: - default: 'Tier{tier}/{dataset}' - BSC: '{type}/{institute.lower}/{dataset.lower}/{freq_folder}/{short_name}{freq_base}' - RCAST: '{dataset}' - input_file: - default: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}[_.]*nc' - BSC: '{short_name}_*.nc' - RCAST: '{short_name}_{mip}_{type}_{dataset}_*.nc' - output_file: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}' - cmor_type: 'CMIP5' - -OBS6: - cmor_strict: false - input_dir: - default: 'Tier{tier}/{dataset}' - BSC: '{type}/{institute.lower}/{dataset.lower}/{freq_folder}/{short_name}{freq_base}' - input_file: - default: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}[_.]*nc' - BSC: '{short_name}_*.nc' - output_file: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}' - cmor_type: 'CMIP6' - -native6: - cmor_strict: false - input_dir: - default: 'Tier{tier}/{dataset}/{version}/{frequency}/{short_name}' - input_file: - default: '*.nc' - output_file: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}' - cmor_type: 'CMIP6' - cmor_default_table_prefix: 'CMIP6_' - -obs4MIPs: - cmor_strict: false - input_dir: - default: 'Tier{tier}/{dataset}' - ESGF: '{project}/{dataset}/{version}' - RCAST: '/' - IPSL: '{realm}/{short_name}/{freq}/{grid}/{institute}/{dataset}/{latest_version}' - input_file: - default: '{short_name}_*.nc' - ESGF: '{short_name}_*.nc' - output_file: '{project}_{dataset}_{short_name}' - cmor_type: 'CMIP6' - cmor_path: 'obs4mips' - cmor_default_table_prefix: 'obs4MIPs_' - -ana4mips: - cmor_strict: false - input_dir: - default: 'Tier{tier}/{dataset}' - RCAST: '/' - input_file: '{short_name}_{mip}_{type}_{dataset}_*.nc' - output_file: '{project}_{mip}_{type}_{dataset}_{short_name}' - cmor_type: 'CMIP5' - # TODO: add cmor_path and table and set cmor_strict to true - -EMAC: - cmor_strict: false - input_dir: - default: '{exp}/{channel}' - input_file: - default: '{exp}*{channel}{postproc_flag}.nc' - output_file: '{project}_{dataset}_{exp}_{channel}_{mip}_{short_name}' - cmor_type: 'CMIP6' - ignore_warnings: - load: - - {message: 'Ignored formula of unrecognised type: .*', module: iris} - - {message: 'Ignoring formula terms variable .* referenced by data variable .* via variable .*', module: iris} - - {message: 'Missing CF-netCDF formula term variable .*, referenced by netCDF variable .*', module: iris} - - {message: 'NetCDF variable .* contains unknown cell method .*', module: iris} - -CORDEX: - input_dir: - default: '/' - spec: '{domain}/{institute}/{driver}/{exp}/{ensemble}/{dataset}/{rcm_version}/{mip}/{short_name}' - BADC: '{domain}/{institute}/{driver}/{exp}/{ensemble}/{dataset}/{rcm_version}/{mip}/{short_name}/{version}' - ESGF: '{project.lower}/output/{domain}/{institute}/{driver}/{exp}/{ensemble}/{dataset}/{rcm_version}/{frequency}/{short_name}/{version}' - input_file: '{short_name}_{domain}_{driver}_{exp}_{ensemble}_{dataset}_{rcm_version}_{mip}*.nc' - output_file: '{project}_{dataset}_{rcm_version}_{driver}_{domain}_{mip}_{exp}_{ensemble}_{short_name}' - cmor_type: 'CMIP5' - cmor_path: 'cordex' - -IPSLCM: - cmor_strict: false - input_dir: - default: '{root}/{account}/{model}/{status}/{exp}/{simulation}/{dir}/{out}/{freq}' - input_file: - default: - - '{simulation}_*_{ipsl_varname}.nc' - - '{simulation}_*_{group}.nc' - output_file: '{dataset}_{account}_{model}_{status}_{exp}_{simulation}_{freq}_{short_name}' - cmor_type: 'CMIP6' - cmor_default_table_prefix: 'CMIP6_' - -ICON: - cmor_strict: false - input_dir: - default: - - '{exp}' - - '{exp}/outdata' - input_file: - default: '{exp}_{var_type}*.nc' - output_file: '{project}_{dataset}_{exp}_{var_type}_{mip}_{short_name}' - cmor_type: 'CMIP6' - cmor_default_table_prefix: 'CMIP6_' - -CESM: - cmor_strict: false - input_dir: - default: - - '/' # run directory - - '{case}/{gcomp}/hist' # short-term archiving - - '{case}/{gcomp}/proc/{tdir}/{tperiod}' # postprocessed data - input_file: - default: '{case}.{scomp}.{type}.{string}*nc' - output_file: '{project}_{dataset}_{case}_{gcomp}_{scomp}_{type}_{mip}_{short_name}' - cmor_type: 'CMIP6' - cmor_default_table_prefix: 'CMIP6_' diff --git a/config/config-user.yml b/config/config-user.yml index a2e29c2..0e2d2a0 100644 --- a/config/config-user.yml +++ b/config/config-user.yml @@ -1,28 +1,12 @@ -############################################################################### -# Example user configuration file for ESMValTool -############################################################################### -# +# Default top level options. + # Note for users: # -------------- -# Site-specific entries for different HPC centers are given at the bottom of -# this file. Comment out/replace as needed. This default version of the file -# can be used in combination with the command line argument -# ``search_esgf=when_missing``. If only certain values are allowed for an -# option, these are listed after ``---``. The option in square brackets is the -# default value, i.e., the one that is used if this option is omitted in the -# file. -# -############################################################################### -# -# Note for developers: -# ------------------- -# Two identical copies of this file (``ESMValTool/config-user-example.yml`` and -# ``ESMValCore/esmvalcore/config-user.yml``) exist. If you change one of it, -# make sure to apply the changes to the other. -# -############################################################################### ---- +# If only certain values are allowed for an option, these are listed after +# ``---``. The option in square brackets is the default value, i.e., the one +# that is used if this option is omitted in the file. +--- # Destination directory where all output will be written # Includes log files and performance stats. output_dir: esmvaltool_output @@ -31,21 +15,10 @@ output_dir: esmvaltool_output # Used by some recipes to look for additional datasets. auxiliary_data_dir: /g/data/xp65/public/apps/cartopy-data -# Automatic data download from ESGF --- [never]/when_missing/always -# Use automatic download of missing CMIP3, CMIP5, CMIP6, CORDEX, and obs4MIPs -# data from ESGF. ``never`` disables this feature, which is useful if you are -# working on a computer without an internet connection, or if you have limited -# disk space. ``when_missing`` enables the automatic download for files that -# are not available locally. ``always`` will always check ESGF for the latest -# version of a file, and will only use local files if they correspond to that -# latest version. -search_esgf: when_missing - -# Directory for storing downloaded climate data -# Make sure to use a directory where you can store multiple GBs of data. Your -# home directory on a HPC is usually not suited for this purpose, so please -# change the default value in this case! -download_dir: climate_data +# Data search mode --- [quick]/complete +# Use ``quick`` to stop searching as soon as a file is found for a dataset. Use +# ``complete`` to search all configured data sources. +search_data: quick # Run at most this many tasks in parallel --- [null]/1/2/3/4/... # Set to ``null`` to use the number of available CPUs. If you run out of @@ -71,7 +44,7 @@ output_file_type: png # By default this option is set to ``true``, so all preprocessor output files # will be removed after a successful run. Set to ``false`` if you need those # files. -remove_preproc_dir: true +remove_preproc_dir: false # Use netCDF compression --- true/[false] compress_netcdf: false @@ -84,187 +57,9 @@ save_intermediary_cubes: false # Path to custom ``config-developer.yml`` file # This can be used to customise project configurations. See # ``config-developer.yml`` for an example. Set to ``null`` to use the default. -config_developer_file: null +# config_developer_file: null # Use a profiling tool for the diagnostic run --- [false]/true # A profiler tells you which functions in your code take most time to run. # Only available for Python diagnostics. profile_diagnostic: false - -# Rootpaths to the data from different projects -# This default setting will work if files have been downloaded by ESMValTool -# via ``search_esgf``. Lists are also possible. For site-specific entries and -# more examples, see below. Comment out these when using a site-specific path. -#rootpath: -# default: ~/climate_data - -# Directory structure for input data --- [default]/ESGF/BADC/DKRZ/ETHZ/etc. -# This default setting will work if files have been downloaded by ESMValTool -# via ``search_esgf``. See ``config-developer.yml`` for definitions. Comment -# out/replace as per needed. -#drs: -# CMIP3: ESGF -# CMIP5: ESGF -# CMIP6: ESGF -# CORDEX: ESGF -# obs4MIPs: ESGF - -# Example rootpaths and directory structure that showcases the different -# projects and also the use of lists -# For site-specific entries, see below. -#rootpath: -# CMIP3: [~/cmip3_inputpath1, ~/cmip3_inputpath2] -# CMIP5: [~/cmip5_inputpath1, ~/cmip5_inputpath2] -# CMIP6: [~/cmip6_inputpath1, ~/cmip6_inputpath2] -# OBS: ~/obs_inputpath -# OBS6: ~/obs6_inputpath -# obs4MIPs: ~/obs4mips_inputpath -# ana4mips: ~/ana4mips_inputpath -# native6: ~/native6_inputpath -# RAWOBS: ~/rawobs_inputpath -# default: ~/default_inputpath -#drs: -# CMIP3: default -# CMIP5: default -# CMIP6: default -# CORDEX: default -# obs4MIPs: default - -# Directory tree created by automatically downloading from ESGF -# Uncomment the lines below to locate data that has been automatically -# downloaded from ESGF (using ``search_esgf``). -#rootpath: -# CMIP3: ~/climate_data -# CMIP5: ~/climate_data -# CMIP6: ~/climate_data -# CORDEX: ~/climate_data -# obs4MIPs: ~/climate_data -#drs: -# CMIP3: ESGF -# CMIP5: ESGF -# CMIP6: ESGF -# CORDEX: ESGF -# obs4MIPs: ESGF - -# Site-specific entries: JASMIN -# Uncomment the lines below to locate data on JASMIN. -#auxiliary_data_dir: /gws/nopw/j04/esmeval/aux_data/AUX -#rootpath: -# CMIP6: /badc/cmip6/data/CMIP6 -# CMIP5: /badc/cmip5/data/cmip5/output1 -# CMIP3: /badc/cmip3_drs/data/cmip3/output -# OBS: /gws/nopw/j04/esmeval/obsdata-v2 -# OBS6: /gws/nopw/j04/esmeval/obsdata-v2 -# obs4MIPs: /gws/nopw/j04/esmeval/obsdata-v2 -# ana4mips: /gws/nopw/j04/esmeval/obsdata-v2 -# CORDEX: /badc/cordex/data/CORDEX/output -#drs: -# CMIP6: BADC -# CMIP5: BADC -# CMIP3: BADC -# CORDEX: BADC -# OBS: default -# OBS6: default -# obs4MIPs: default -# ana4mips: default - -# Site-specific entries: DKRZ-Levante -# For bd0854 members a shared download directory is available -#search_esgf: when_missing -#download_dir: /work/bd0854/DATA/ESMValTool2/download -# Uncomment the lines below to locate data on Levante at DKRZ. -#auxiliary_data_dir: /work/bd0854/DATA/ESMValTool2/AUX -#rootpath: -# CMIP6: /work/bd0854/DATA/ESMValTool2/CMIP6_DKRZ -# CMIP5: /work/bd0854/DATA/ESMValTool2/CMIP5_DKRZ -# CMIP3: /work/bd0854/DATA/ESMValTool2/CMIP3 -# CORDEX: /work/ik1017/C3SCORDEX/data/c3s-cordex/output -# OBS: /work/bd0854/DATA/ESMValTool2/OBS -# OBS6: /work/bd0854/DATA/ESMValTool2/OBS -# obs4MIPs: /work/bd0854/DATA/ESMValTool2/OBS -# ana4mips: /work/bd0854/DATA/ESMValTool2/OBS -# native6: /work/bd0854/DATA/ESMValTool2/RAWOBS -# RAWOBS: /work/bd0854/DATA/ESMValTool2/RAWOBS -#drs: -# CMIP6: DKRZ -# CMIP5: DKRZ -# CMIP3: DKRZ -# CORDEX: BADC -# obs4MIPs: default -# ana4mips: default -# OBS: default -# OBS6: default -# native6: default - -# Site-specific entries: ETHZ -# Uncomment the lines below to locate data at ETHZ. -#rootpath: -# CMIP6: /net/atmos/data/cmip6 -# CMIP5: /net/atmos/data/cmip5 -# CMIP3: /net/atmos/data/cmip3 -# OBS: /net/exo/landclim/PROJECTS/C3S/datadir/obsdir/ -#drs: -# CMIP6: ETHZ -# CMIP5: ETHZ -# CMIP3: ETHZ - -# Site-specific entries: IPSL -# Uncomment the lines below to locate data on Ciclad at IPSL. -#rootpath: -# IPSLCM: / -# CMIP5: /bdd/CMIP5/output -# CMIP6: /bdd/CMIP6 -# CMIP3: /bdd/CMIP3 -# CORDEX: /bdd/CORDEX/output -# obs4MIPs: /bdd/obs4MIPS/obs-CFMIP/observations -# ana4mips: /not_yet -# OBS: /not_yet -# OBS6: /not_yet -# RAWOBS: /not_yet -#drs: -# CMIP6: DKRZ -# CMIP5: DKRZ -# CMIP3: IPSL -# CORDEX: BADC -# obs4MIPs: IPSL -# ana4mips: default -# OBS: not_yet -# OBS6: not_yet - -# Site-specific entries: Met Office -# Uncomment the lines below to locate data at the Met Office. -#rootpath: -# CMIP5: /project/champ/data/cmip5/output1 -# CMIP6: /project/champ/data/CMIP6 -# CORDEX: /project/champ/data/cordex/output -# OBS: /data/users/esmval/ESMValTool/temporary/obs/ -# obs4MIPs: /data/users/esmval/ESMValTool/temporary/obs/ -# ana4mips: /project/champ/data/ana4MIPs -#drs: -# CMIP5: BADC -# CMIP6: BADC -# CORDEX: BADC -# OBS: default -# obs4MIPs: default -# ana4mips: BADC - -# Site-specific entries: NCI ACCESS-NRI -# Uncomment the lines below to locate data on NCI ACCESS-NRI. -rootpath: - CMIP6: [/g/data/oi10/replicas/CMIP6, /g/data/fs38/publications/CMIP6, /g/data/xp65/public/apps/esmvaltool/replicas/CMIP6] - CMIP5: [/g/data/r87/DRSv3/CMIP5, /g/data/al33/replicas/CMIP5/combined, /g/data/rr3/publications/CMIP5/output1, /g/data/xp65/public/apps/esmvaltool/replicas/cmip5/output1] - CMIP3: /g/data/r87/DRSv3/CMIP3 - OBS: [/g/data/ct11/access-nri/replicas/esmvaltool/obsdata-v2, /g/data/kj13/datasets/esmvaltool/obsdata-v2] - OBS6: [/g/data/ct11/access-nri/replicas/esmvaltool/obsdata-v2, /g/data/kj13/datasets/esmvaltool/obsdata-v2] - obs4MIPs: [/g/data/ct11/access-nri/replicas/esmvaltool/obsdata-v2, /g/data/kj13/datasets/esmvaltool/obsdata-v2] - ana4mips: [/g/data/ct11/access-nri/replicas/esmvaltool/obsdata-v2, /g/data/kj13/datasets/esmvaltool/obsdata-v2] - native6: [/g/data/xp65/public/apps/esmvaltool/native6, /g/data/kj13/datasets/esmvaltool/raw-data] - RAWOBS: /g/data/kj13/datasets/esmvaltool/raw-data - -drs: - CMIP6: NCI - CMIP5: NCI - CMIP3: NCI - CORDEX: ESGF - obs4MIPs: default - ana4mips: default diff --git a/config/data-esmvalcore-esgf.yml b/config/data-esmvalcore-esgf.yml new file mode 100644 index 0000000..b65d975 --- /dev/null +++ b/config/data-esmvalcore-esgf.yml @@ -0,0 +1,19 @@ +# Download CMIP, CORDEX, and obs4MIPs data from ESGF using the `esmvalcore.io.esgf` +# module, which uses the legacy ESGF search interface. +projects: + CMIP6: &esgf-pyclient-data + data: + esgf-pyclient: + type: esmvalcore.io.esgf.ESGFDataSource + download_dir: ~/climate_data + # Use a lower priority than for esmvalcore.io.local.LocalDataSource + # to avoid searching ESGF with the setting `search_esgf: when_missing`. + priority: 10 + CMIP5: + <<: *esgf-pyclient-data + CMIP3: + <<: *esgf-pyclient-data + CORDEX: + <<: *esgf-pyclient-data + # obs4MIPs: + # <<: *esgf-pyclient-data diff --git a/config/data-hpc-nci.yml b/config/data-hpc-nci.yml new file mode 100644 index 0000000..9e99ca2 --- /dev/null +++ b/config/data-hpc-nci.yml @@ -0,0 +1,125 @@ +# Read CMIP, obs4MIPs, native6, OBS6, and OBS data from the filesystem at NCI. +projects: + CMIP6: + data: + oi10: &cmip6 + type: "esmvalcore.local.LocalDataSource" + rootpath: /g/data/oi10/replicas + dirname_template: "{project}/{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}" + filename_template: "{short_name}_{mip}_{dataset}_{exp}_{ensemble}_{grid}*.nc" + fs38: + <<: *cmip6 + rootpath: /g/data/fs38/publications + + CMIP5: + data: + r87: &cmip5 + type: "esmvalcore.local.LocalDataSource" + rootpath: /g/data/r87/DRSv3/CMIP5 + dirname_template: "{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{version}/{short_name}" + filename_template: "{short_name}_{mip}_{dataset}_{exp}_{ensemble}*.nc" + al33: + <<: *cmip5 + rootpath: /g/data/al33/replicas/CMIP5/combined + rr3: &cmip5-default + <<: *cmip5 + rootpath: /g/data/rr3/publications + dirname_template: "{project}/{product}/{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{version}/{short_name}" + + CMIP3: + data: + local: + type: "esmvalcore.local.LocalDataSource" + rootpath: /g/data/r87/DRSv3/CMIP3 + dirname_template: "{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{ensemble}/{short_name}/{latestversion}" + filename_template: "{short_name}_*.nc" + + obs4MIPs: + data: + local: + type: "esmvalcore.local.LocalDataSource" + rootpath: /g/data/qv56/replicas/obs4MIPs + dirname_template: "{institute}/{dataset}/{frequency}/{short_name}/{grid}/{version}" + # - '{institute}/{dataset}/{short_name}/{freq}/{version}' + # - '{institute}/{dataset}/{realm}/{freq}/{version}/{short_name}' + # - '{institute}/{dataset}/{realm}/{freq}/{version}' + # - '{institute}/{dataset}/{realm}/{short_name}/{freq}/{version}' + filename_template: "{short_name}_*.nc" + + native6: + data: + local: # NCI native6 ERA5 data + type: "esmvalcore.local.LocalDataSource" + rootpath: /g/data/rt52/era5 + dirname_template: "{level}/{nci_type}/{era5_shortname}/*" + filename_template: "*.nc" + OBS6: + data: + local: + type: "esmvalcore.local.LocalDataSource" + rootpath: /g/data/ct11/access-nri/replicas/esmvaltool/obsdata-v2 + dirname_template: "Tier{tier}/{dataset}" + filename_template: "{project}_{dataset}_{type}_{version}_{mip}_{short_name}[_.]*nc" + # OBS6: + # data: + # local: + # type: "esmvalcore.local.LocalDataSource" + # rootpath: /g/data/kj13/datasets/esmvaltool/obsdata-v2 + # dirname_template: "Tier{tier}/{dataset}" + # filename_template: "{project}_{dataset}_{type}_{version}_{mip}_{short_name}[_.]*nc" + OBS: + data: + local: + type: "esmvalcore.local.LocalDataSource" + rootpath: /g/data/ct11/access-nri/replicas/esmvaltool/obsdata-v2 + dirname_template: "Tier{tier}/{dataset}" + filename_template: "{project}_{dataset}_{type}_{version}_{mip}_{short_name}[_.]*nc" + + ZV30: + data: + local: + type: "esmvalcore.local.LocalDataSource" + rootpath: /g/data/zv30/cmip/CMIP6 + dirname_template: "{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}" + filename_template: "{short_name}_{mip}_{dataset}_{exp}_{ensemble}_{grid}*.nc" + cmor_table: + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip6/Tables + - cmip6-custom + strict: false + + CORDEX-CMIP6: + data: + CCAM: &cordex-cmip6 + type: "esmvalcore.local.LocalDataSource" + rootpath: /g/data/hq89/CCAM/output/ + dirname_template: "CMIP6/DD/{domain}/{institute}/{driver}/{exp}/{ensemble}/{dataset}/{rcm_version}/{mip}/{short_name}/{version}" + filename_template: "{short_name}_{domain}_{driver}_{exp}_{ensemble}_{institute}_{dataset}_{rcm_version}_{mip}*.nc" + BARPA: + <<: *cordex-cmip6 + rootpath: /g/data/py18/BARPA/output/ + NARClim2: + <<: *cordex-cmip6 + rootpath: /g/data/zz63/NARCliM2-0/output/ + QldFCP: + <<: *cordex-cmip6 + rootpath: /g/data/ig45/QldFCP-2/output/ + cmor_table: + type: esmvalcore.cmor.table.CMIP6Info + paths: + - /g/data/xp65/public/apps/esmvaltool/utils/extras/cordex-cmip6-cmor-tables + strict: false + + BARRA2: + data: + local: + type: "esmvalcore.local.LocalDataSource" + rootpath: /g/data/ob53/BARRA2/output/reanalysis + dirname_template: "{domain}/{institute}/{driver}/{exp}/{ensemble}/{dataset}/{rcm_version}/{mip}/{short_name}/{version}" + filename_template: "{short_name}_{domain}_{driver}_{exp}_{ensemble}_{institute}_{dataset}_{rcm_version}_{mip}*.nc" + cmor_table: + type: esmvalcore.cmor.table.CMIP6Info + paths: + - /g/data/xp65/public/apps/esmvaltool/utils/extras/cordex-cmip6-cmor-tables + strict: false diff --git a/config/data-native-access.yml b/config/data-native-access.yml new file mode 100644 index 0000000..84aea2c --- /dev/null +++ b/config/data-native-access.yml @@ -0,0 +1,14 @@ +# Read data from the ACCESS model in its native format. +projects: + ACCESS: + data: + access-sub-dataset: + type: esmvalcore.io.local.LocalDataSource + rootpath: ~/climate_data + dirname_template: "{dataset}/{sub_dataset}/{exp}/{modeling_realm}/netCDF" + filename_template: "{sub_dataset}.{freq_attribute}-*.nc" + access-ocean: + type: esmvalcore.io.local.LocalDataSource + rootpath: ~/climate_data + dirname_template: "{dataset}/{sub_dataset}/{exp}/{modeling_realm}/netCDF" + filename_template: "ocean_{freq_attribute}.nc-*" diff --git a/config/extra_facets_access.yml b/config/extra_facets_access.yml new file mode 100644 index 0000000..371b2fb --- /dev/null +++ b/config/extra_facets_access.yml @@ -0,0 +1,137 @@ +# Extra facets for native ACCESS model output. + +# A complete list of supported keys is given in the documentation (see +# ESMValCore/doc/quickstart/find_data.rst). +--- + +projects: + ACCESS: + extra_facets: + ACCESS-ESM1-5: + '*': + # Atmosphere + tas: + raw_name: fld_s03i236 + modeling_realm: atm + pr: + raw_name: fld_s05i216 + modeling_realm: atm + ps: + raw_name: fld_s00i409 + modeling_realm: atm + clt: + raw_name: fld_s02i204 + modeling_realm: atm + psl: + raw_name: fld_s16i222 + modeling_realm: atm + hus: + raw_name: fld_s30i205 + modeling_realm: atm + zg: + raw_name: fld_s30i207 + modeling_realm: atm + va: + raw_name: fld_s30i202 + modeling_realm: atm + ua: + raw_name: fld_s30i201 + modeling_realm: atm + ta: + raw_name: fld_s30i204 + modeling_realm: atm + rlus: + raw_name: + - fld_s02i207 + - fld_s02i201 + - fld_s03i332 + - fld_s02i205 + modeling_realm: atm + rlds: + raw_name: fld_s02i207 + modeling_realm: atm + rsus: + raw_name: + - fld_s01i235 + - fld_s01i201 + modeling_realm: atm + clivi: + raw_name: fld_s30i406 + modeling_realm: atm + evspsbl: + raw_name: fld_s03i223 + modeling_realm: atm + hfls: + raw_name: fld_s03i234 + modeling_realm: atm + hfss: + raw_name: fld_s03i217 + modeling_realm: atm + hur: + raw_name: fld_s30i206 + modeling_realm: atm + hurs: + raw_name: fld_s03i245 + modeling_realm: atm + huss: + raw_name: fld_s03i237 + modeling_realm: atm + prsn: + raw_name: fld_s05i215 + modeling_realm: atm + rldscs: + raw_name: fld_s02i208 + modeling_realm: atm + rsdt: + raw_name: fld_s01i207 + modeling_realm: atm + rsuscs: + raw_name: fld_s01i211 + modeling_realm: atm + rsut: + raw_name: fld_s01i208 + modeling_realm: atm + rsutcs: + raw_name: fld_s01i209 + modeling_realm: atm + sci: + raw_name: fld_s05i270 + modeling_realm: atm + sfcWind: + raw_name: fld_s03i230 + modeling_realm: atm + sfcWindmax: + raw_name: fld_s03i227_max + modeling_realm: atm + tauu: + raw_name: fld_s03i460 + modeling_realm: atm + tauv: + raw_name: fld_s03i461 + modeling_realm: atm + ts: + raw_name: fld_s00i024 + modeling_realm: atm + uas: + raw_name: fld_s03i209 + modeling_realm: atm + vas: + raw_name: fld_s03i210 + modeling_realm: atm + wap: + raw_name: fld_s30i208 + modeling_realm: atm + rls: + raw_name: fld_s02i201 + modeling_realm: atm + rss: + raw_name: fld_s01i201 + modeling_realm: atm + + # Ocean + tos: + raw_name: sst + modeling_realm: ocn + so: + raw_name: salt + modeling_realm: ocn diff --git a/config/extra_facets_native6-era5.yml b/config/extra_facets_native6-era5.yml new file mode 100644 index 0000000..efe3156 --- /dev/null +++ b/config/extra_facets_native6-era5.yml @@ -0,0 +1,179 @@ +# Extra facets for native6 ERA5 data in NCI Gadi +# + +# A complete list of supported keys is given in the documentation (see +# ESMValCore/doc/quickstart/find_data.rst). + +# Updated for new configuration v2.12/2.13 +--- + +projects: + native6: + extra_facets: + ERA5: + # Settings for all variables of all MIPs + '*': + '*': + regrid: true + type: an + version: v1 + + # Variable-specific settings + pr: + era5_shortname: tp + level: single-levels + prw: + era5_shortname: tcwv + level: single-levels + cli: + era5_shortname: ciwc + level: pressure-levels + clivi: + era5_shortname: ciwc + level: pressure-levels + clt: + era5_shortname: tcc + level: single-levels + clw: + era5_shortname: clwc + level: pressure-levels + clwvi: + era5_shortname: clwc + level: pressure-levels + evspsbl: + era5_shortname: e + level: single-levels + evspsblpot: + era5_shortname: pev + level: single-levels + hur: + era5_shortname: r + level: pressure-levels + hus: + era5_shortname: q + level: pressure-levels + mrro: + era5_shortname: ro + level: single-levels + o3: + era5_shortname: o3 + level: single-levels + prsn: + era5_shortname: sf + level: single-levels + prw: + era5_shortname: tcwv + level: single-levels + ps: + era5_shortname: sp + level: single-levels + psl: + era5_shortname: msl + level: single-levels + #ptype: + # era5_shortname: 2t + # level: single-levels + rainmxrat27: + era5_shortname: crwc + level: pressure-levels + rlds: + era5_shortname: strd + level: single-levels + rls: + era5_shortname: msnlwrf + level: single-levels + rsds: + era5_shortname: ssrd + level: single-levels + rsdt: + era5_shortname: tisr + level: single-levels + rss: + era5_shortname: ssr + level: single-levels + snowmxrat27: + era5_shortname: cswc + level: pressure-levels + ta: + era5_shortname: t + level: pressure-levels + tas: + era5_shortname: 2t + level: single-levels + tasmax: + era5_shortname: mx2t + level: single-levels + tasmin: + era5_shortname: mn2t + level: single-levels + tdps: + era5_shortname: 2d + level: single-levels + ts: + era5_shortname: skt + level: single-levels + tsn: + era5_shortname: tsn + level: single-levels + ua: + era5_shortname: u + level: pressure-levels + uas: + era5_shortname: 10u + level: single-levels + va: + era5_shortname: v + level: pressure-levels + vas: + era5_shortname: 10v + level: single-levels + wap: + era5_shortname: w + level: pressure-levels + zg: + era5_shortname: z + level: pressure-levels + + # MIP-specific settings + E1hr: + '*': + nci_type: reanalysis + AERhr: + '*': + nci_type: reanalysis + AERmon: + '*': + nci_type: monthly-averaged + AERmonZ: + '*': + nci_type: monthly-averaged + Amon: + '*': + nci_type: monthly-averaged + CFmon: + '*': + nci_type: monthly-averaged + Emon: + '*': + nci_type: monthly-averaged + EmonZ: + '*': + nci_type: monthly-averaged + ImonAnt: + '*': + nci_type: monthly-averaged + ImonGre: + '*': + nci_type: monthly-averaged + LImon: + '*': + nci_type: monthly-averaged + Lmon: + '*': + nci_type: monthly-averaged + Omon: + '*': + nci_type: monthly-averaged + SImon: + '*': + nci_type: monthly-averaged From 9afdc3a83dbd77007e92d770fe3f7d8993678cdb Mon Sep 17 00:00:00 2001 From: Felicity Chun Date: Tue, 31 Mar 2026 15:00:41 +1100 Subject: [PATCH 2/2] custom cmor tables --- config/data-hpc-nci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/data-hpc-nci.yml b/config/data-hpc-nci.yml index 9e99ca2..91b7e6f 100644 --- a/config/data-hpc-nci.yml +++ b/config/data-hpc-nci.yml @@ -109,6 +109,7 @@ projects: type: esmvalcore.cmor.table.CMIP6Info paths: - /g/data/xp65/public/apps/esmvaltool/utils/extras/cordex-cmip6-cmor-tables + - cmip6-custom strict: false BARRA2: @@ -122,4 +123,5 @@ projects: type: esmvalcore.cmor.table.CMIP6Info paths: - /g/data/xp65/public/apps/esmvaltool/utils/extras/cordex-cmip6-cmor-tables + - cmip6-custom strict: false