diff --git a/tests/test_sections.py b/tests/test_sections.py index 8b34fd42..c3e6364b 100644 --- a/tests/test_sections.py +++ b/tests/test_sections.py @@ -118,6 +118,8 @@ def test_sections(): "input_component": "", "job_nbr": 0, "mapping_file": "MAPPING_FILE_TS", + "mpas_calendar": "noleap", + "mpas_start_time": "0001-01-01 00:00:00", "tpd": 1, "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT", "vrt_remap_vars": "", @@ -155,6 +157,8 @@ def test_sections(): "input_subdir": "INPUT_SUBDIR", "job_nbr": 0, "mapping_file": "MAPPING_FILE_TS", + "mpas_calendar": "noleap", + "mpas_start_time": "0001-01-01 00:00:00", "ncclimo_cmd": "ncclimo", "nco_path": "", "nodes": 1, @@ -333,6 +337,8 @@ def test_subsections(): "input_component": None, "job_nbr": None, "mapping_file": "MAPPING_FILE_TS_GRID1", + "mpas_calendar": None, + "mpas_start_time": None, "tpd": None, "vrt_remap_vars": None, "vrt_in_file": None, @@ -346,6 +352,8 @@ def test_subsections(): "input_component": None, "job_nbr": None, "mapping_file": "MAPPING_FILE_TS_GRID2", + "mpas_calendar": None, + "mpas_start_time": None, "tpd": None, "vrt_remap_vars": None, "vrt_in_file": None, @@ -353,6 +361,8 @@ def test_subsections(): "years": ["0001:0020:10"], }, "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT", + "mpas_calendar": "noleap", + "mpas_start_time": "0001-01-01 00:00:00", "vrt_remap_vars": "", "vrt_in_file": "", "vrt_remap_file": "", @@ -387,6 +397,8 @@ def test_subsections(): "input_subdir": "INPUT_SUBDIR", "job_nbr": 0, "mapping_file": "MAPPING_FILE_TS_GRID1", + "mpas_calendar": "noleap", + "mpas_start_time": "0001-01-01 00:00:00", "ncclimo_cmd": "ncclimo", "nco_path": "", "nodes": 1, @@ -442,6 +454,8 @@ def test_subsections(): "input_subdir": "INPUT_SUBDIR", "job_nbr": 0, "mapping_file": "MAPPING_FILE_TS_GRID2", + "mpas_calendar": "noleap", + "mpas_start_time": "0001-01-01 00:00:00", "ncclimo_cmd": "ncclimo", "nco_path": "", "nodes": 1, diff --git a/tests/test_zppy_pcmdi_diags.py b/tests/test_zppy_pcmdi_diags.py index ef8656b9..74e5f878 100644 --- a/tests/test_zppy_pcmdi_diags.py +++ b/tests/test_zppy_pcmdi_diags.py @@ -542,6 +542,7 @@ def test_add_ts_dependencies(): dependencies: List[str] = [] c = { "current_set": "mean_climate", + "infer_section_parameters": True, "ts_num_years": 5, } script_dir = "/scripts" diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index 948fadbb..290a9429 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -151,6 +151,15 @@ job_nbr = integer(default=0) tpd = integer(default=1) # Model component having generated input files (eam, eamxx, elm, mosart, ...) input_component = string(default="") +# MPAS calendar used to construct a CF-style numeric time coordinate +# from timeMonthly_avg_daysSinceStartOfSim when native MPAS time-series +# files do not contain a standard time variable. +mpas_calendar = string(default="noleap") +# MPAS simulation start time used as the reference time for constructing +# a CF-style numeric time coordinate: +# time:units = "days since " +# This should match the MPAS/E3SM config_start_time when possible. +mpas_start_time = string(default="0001-01-01 00:00:00") # Vertical regrid (model levels → pressure levels) — applied after ncclimo splits # files into per-var time-series. Vars listed here will additionally be regridded # and written to a sibling directory `ts_vrt_remap/` parallel to `ts/`, so users @@ -171,6 +180,8 @@ vrt_in_file = string(default="") job_nbr = integer(default=None) tpd = integer(default=None) input_component = string(default=None) + mpas_calendar = string(default=None) + mpas_start_time = string(default=None) vrt_remap_vars = string(default=None) vrt_remap_file = string(default=None) vrt_in_file = string(default=None) @@ -647,7 +658,7 @@ enso_groups = string(default="ENSO_perf,ENSO_proc,ENSO_tel") # Specify these for the enso subtask AND the synthetic_plots subtask ########################################################################################### # enso: vars = "psl,pr,prsn,ts,tas,tauu,tauv,hflx,hfss,rlds,rsds,rlus,rlut,rsdt" -enso_vars = string(default="psl,pr,prsn,ts,tas,tauu,tauv,hflx,hfss,rlds,rsds,rlus,rlut,rsdt") +enso_vars = string(default="psl,pr,ts,tas,tauu,tauv,hfls,hfss,rlds,rsds,rlus,rlut,rsdt") ########################################################################################### # Specify these for the synthetic_plots subtask ########################################################################################### diff --git a/zppy/pcmdi_diags.py b/zppy/pcmdi_diags.py index 4d109bee..a4ccfe4b 100644 --- a/zppy/pcmdi_diags.py +++ b/zppy/pcmdi_diags.py @@ -54,9 +54,9 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): define_current_set(c) if c["current_set"] == "enso": logger.warning( - "The 'enso' set is not yet supported in PCMDI Diags. Skipping launching of associated jobs." + "The 'enso' set is currently a testing mode in PCMDI Diags. Turn on it with care." ) - break # Skip this task + # continue # Skip this task c["sub"] = get_value_from_parameter( c, "subsection", "sub", ParameterInferenceType.SECTION_INFERENCE ) @@ -89,7 +89,7 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): if c["current_set"] != "synthetic_plots": check_and_define_parameters(c) - print(c["prefix"]) + logger.debug(c["prefix"]) define_relevant_years(c) else: prefix = f"pcmdi_diags_{c['sub']}_{c['run_type']}" @@ -119,21 +119,8 @@ def pcmdi_diags(config, script_dir, existing_bundles, job_ids_file): if c["current_set"] != "synthetic_plots": for yr in range(c["year1"], c["year2"], c["ts_num_years"]): add_ts_dependencies(c, dependencies, script_dir, yr) - set_value_of_parameter_if_undefined( - c, - "e3sm_to_cmip_atm_subsection", - "atm_monthly_180x360_aave", - ParameterInferenceType.SECTION_INFERENCE, - ) - add_dependencies( - dependencies, - script_dir, - "e3sm_to_cmip", - c["e3sm_to_cmip_atm_subsection"], - c["year1"], - c["year2"], - c["ts_num_years"], - ) + for yr in range(c["year1"], c["year2"], c["ts_num_years"]): + add_e3sm_to_cmip_dependencies(c, dependencies, script_dir, yr) else: if i < len(year_sets) - 1: continue @@ -345,22 +332,79 @@ def define_relevant_years_for_synthetic_plots(c: Dict[str, Any]) -> None: c["enso_years"] = year_str +def _as_subsection_list(subsection: str) -> List[str]: + return [item.strip() for item in subsection.split(",") if item.strip()] + + def add_ts_dependencies( c: Dict[str, Any], dependencies: List[str], script_dir: str, yr: int ): start_yr = yr end_yr = yr + c["ts_num_years"] - 1 + if c["current_set"] in BASE_PCMDI_SETS: - add_dependencies( - dependencies, - script_dir, - "ts", + # Ensure the key exists because set_value_of_parameter_if_undefined() + # only fills values for existing keys with an empty string. + if "ts_subsection" not in c: + c["ts_subsection"] = "" + + set_value_of_parameter_if_undefined( + c, + "ts_subsection", + "atm_monthly_180x360_aave", + ParameterInferenceType.SECTION_INFERENCE, + ) + + ts_subsections = [ + item.strip() for item in c["ts_subsection"].split(",") if item.strip() + ] + + for ts_subsection in ts_subsections: + add_dependencies( + dependencies, + script_dir, + "ts", + ts_subsection, + start_yr, + end_yr, + c["ts_num_years"], + ) + + +def add_e3sm_to_cmip_dependencies( + c: Dict[str, Any], dependencies: List[str], script_dir: str, yr: int +): + start_yr = yr + end_yr = yr + c["ts_num_years"] - 1 + + if c["current_set"] in BASE_PCMDI_SETS: + if "e3sm_to_cmip_atm_subsection" not in c: + c["e3sm_to_cmip_atm_subsection"] = "" + + set_value_of_parameter_if_undefined( + c, + "e3sm_to_cmip_atm_subsection", "atm_monthly_180x360_aave", - start_yr, - end_yr, - c["ts_num_years"], + ParameterInferenceType.SECTION_INFERENCE, ) + e3sm_to_cmip_atm_subsections = [ + item.strip() + for item in c["e3sm_to_cmip_atm_subsection"].split(",") + if item.strip() + ] + + for e3sm_to_cmip_atm_subsection in e3sm_to_cmip_atm_subsections: + add_dependencies( + dependencies, + script_dir, + "e3sm_to_cmip", + e3sm_to_cmip_atm_subsection, + start_yr, + end_yr, + c["ts_num_years"], + ) + def add_pcmdi_dependencies( c: Dict[str, Any], dependencies: List[str], script_dir: str @@ -370,6 +414,8 @@ def add_pcmdi_dependencies( status_suffix = f"_{c['year1']:04d}-{c['year2']:04d}" elif c["run_type"] == "model_vs_model": status_suffix = f"_{c['year1']:04d}-{c['year2']:04d}_vs_{c['ref_year1']:04d}-{c['ref_year2']:04d}" + else: + raise ValueError(f"Invalid run_type={c['run_type']}") if c["clim_viewer"]: status_file = os.path.join( diff --git a/zppy/templates/pcmdi_diags.bash b/zppy/templates/pcmdi_diags.bash index a049b5e9..e45e2527 100755 --- a/zppy/templates/pcmdi_diags.bash +++ b/zppy/templates/pcmdi_diags.bash @@ -18,11 +18,27 @@ export UCX_SHM_DEVICES=all # or not set UCX_NET_DEVICES at all # Make sure UVCDAT doesn't prompt us about anonymous logging export UVCDAT_ANONYMOUS_LOG=False +# Use a non-interactive Matplotlib backend for batch diagnostics. +# Safe for workflows that save figures to files and avoids GUI/Tkinter errors +# on systems without a display. +export MPLBACKEND=Agg + +# Script dir +cd {{ scriptDir }} + +# Get jobid +id=${SLURM_JOBID} + +# Update status file +STARTTIME=$(date +%s) +echo "RUNNING ${id}" > {{ prefix }}.status + # Basic definitions case="{{ case }}" www="{{ www }}" run_type="{{ run_type }}" results_dir="{{ run_type }}" +error_key=0 {% if current_set != "synthetic_plots" %} @@ -34,14 +50,9 @@ ref_y2={{ ref_year2 }} ref_start_yr={{ ref_start_yr }} ref_final_yr={{ ref_final_yr }} -# Formatted versions +# Formatted versions (analysis window only; ref formatted after refinement below) Y1="$(printf "%04d" ${y1})" Y2="$(printf "%04d" ${y2})" -ref_Y1="$(printf "%04d" ${ref_y1})" -ref_Y2="$(printf "%04d" ${ref_y2})" - -num_years=$((y2 - y1 + 1)) -ref_end_yr=$((ref_y1 + num_years - 1)) # Keep originals for fallback orig_ref_y1=${ref_y1} @@ -80,6 +91,10 @@ else fi fi +# Format reference years after refinement +ref_Y1="$(printf "%04d" ${ref_y1})" +ref_Y2="$(printf "%04d" ${ref_y2})" + {% if current_set == "mean_climate"%} source_vars="{{ clim_vars }}" {% elif current_set == "variability_modes_cpl"%} @@ -124,7 +139,7 @@ case_id=v$(date '+%Y%m%d') {% endif %} # Create temporary workdir -workdir=`mktemp -d tmp.{{ prefix }}.${id}.XXXX` +workdir=$(mktemp -d tmp.{{ prefix }}.${id}.XXXX) cd ${workdir} # files for definition of regions for regional mean @@ -149,7 +164,7 @@ create_links_acyc_climo() { local begin_year="$3" local end_year="$4" local name_key="$5" - local error_num="$6" + local error_num="$6" # Will use error_num +0,+1,+2 echo "create_links_acyc_climo: linking from ${ts_dir_source} to ${ts_dir_destination}" @@ -160,7 +175,6 @@ create_links_acyc_climo() { local script_dir="{{ scriptDir }}" local prefix="{{ prefix }}" local ts_step="{{ ts_num_years }}" - local dofm=(15.5 45 74.5 105 125.5 166 196.5 227.5 258 288.5 319 349.5) for v in ${variables//,/ }; do > "${v}_files.txt" # Start fresh @@ -175,16 +189,51 @@ create_links_acyc_climo() { done shopt -u nullglob + if [[ ! -s "${v}_files.txt" ]]; then + echo "WARNING: No input files found for variable ${v}, skipping climatology. \ + This variable may not have been processed or output by e3sm_to_cmip." + continue + fi + + first_file=$(head -n 1 "${v}_files.txt") + if ! run_nco ncks -m "${first_file}" >/dev/null 2>&1; then + echo "WARNING: First file is not a valid NetCDF for ${v}: ${first_file}, skipping climatology. \ + This variable may not have been processed or output by e3sm_to_cmip." + continue + fi + # Derive monthly climatology files for month in $(seq 1 12); do local MM MM=$(printf "%02d" "${month}") run_nco ncra -O -h -F -d time,"${month}",,12 $(< "${v}_files.txt") "${v}_clm_${MM}.nc" + if [[ $? -ne 0 ]]; then + cd "${script_dir}" || exit + echo "ERROR ($((error_num + 1)))" > "${prefix}.status" + exit "$((error_num + 1))" + fi done # Combine to form full annual cycle file local combined_name="${name_key}.${v}.${begin_year}01-${end_year}12.AC.${case_id}.nc" - run_nco ncrcat -O -d time,0, "${v}_clm_"*.nc "${combined_name}" + run_nco ncrcat -O -h -d time,0, "${v}_clm_"*.nc "${combined_name}" + if [[ $? -ne 0 ]]; then + cd "${script_dir}" || exit + echo "ERROR ($((error_num + 2)))" > "${prefix}.status" + exit "$((error_num + 2))" + fi + + # Add bnds dimension if missing + if ! run_nco ncks -m "${combined_name}" | grep -q "bnds = 2"; then + echo "Adding missing bnds dimension..." + run_nco ncap2 -O -h -s 'defdim("bnds",2)' \ + "${combined_name}" "${combined_name}" + if [[ $? -ne 0 ]]; then + cd "${script_dir}" || exit + echo "ERROR ($((error_num + 3)))" > "${prefix}.status" + exit "$((error_num + 3))" + fi + fi # Adjust time metadata for PCMDI diagnostics local cmdfix1='time[time]={15.5, 45, 74.5, 105, 125.5, 166, 196.5, 227.5, 258, 288.5,319, 349.5}' @@ -192,19 +241,25 @@ create_links_acyc_climo() { local cmdfix3='time@units="days since 1850-01-01 00:00:00"' local cmdfix4='time@calendar="noleap"' local cmdfix5='time@bounds="time_bnds"' - run_nco ncap2 -O -h -s "${cmdfix1};${cmdfix2};${cmdfix3};${cmdfix4};${cmdfix5}" "${combined_name}" "${combined_name}" - - rm -vf "${v}_clm_"*.nc + local cmdfix6='time_bnds@units=time@units' + local cmdfix7='time_bnds@calendar=time@calendar' + run_nco ncap2 -O -h -s "${cmdfix1};${cmdfix2};${cmdfix3};${cmdfix4};${cmdfix5};${cmdfix6};${cmdfix7}" \ + "${combined_name}" "${combined_name}" if [[ $? -ne 0 ]]; then cd "${script_dir}" || exit - echo "ERROR (${error_num})" > "${prefix}.status" - exit "${error_num}" + echo "ERROR ($((error_num + 4)))" > "${prefix}.status" + exit "$((error_num + 4))" fi + + rm -vf "${v}_clm_"*.nc done - if [ -z "$( ls . )" ]; then + if [ -z "$(ls ./*.nc 2>/dev/null)" ]; then echo "create_links_acyc_climo: ${ts_dir_destination} was not updated!" + cd "${script_dir}" || exit + echo "ERROR ($((error_num + 5)))" > "${prefix}.status" + exit "$((error_num + 5))" fi cd .. @@ -220,21 +275,20 @@ create_links_acyc_climo_obs() { local script_dir="{{ scriptDir }}" local prefix="{{ prefix }}" - local dofm=(15.5 45 74.5 105 125.5 166 196.5 227.5 258 288.5 319 349.5) echo "create_links_acyc_climo_obs: linking from ${ts_dir_source} to ${ts_dir_destination}" mkdir -p "${ts_dir_destination}" cd "${ts_dir_destination}" || exit - for file in ${ts_dir_source}/*; do + for file in "${ts_dir_source}"/*; do local fname local YYYYS YYYYE local SUBSTR local ttag tmp_file MM combined_name fname=$(basename "${file}") - if [[ ! ${fname} =~ ".nc" ]]; then + if [[ ${fname} != *.nc ]]; then # Skip non-.nc files continue fi @@ -248,31 +302,47 @@ create_links_acyc_climo_obs() { continue fi - # Clip to specified year range - if [[ ${YYYYS} -lt ${begin_year} ]]; then YYYYS=${begin_year}; fi - if [[ ${YYYYE} -gt ${end_year} ]]; then YYYYE=${end_year}; fi + # Clip to specified year range if no overlap use full range covered by file + orig_YYYYS="${YYYYS}" + file_start_year=${YYYYS:0:4} + file_end_year=${YYYYE:0:4} + + if (( file_end_year < begin_year || file_start_year > end_year )); then + echo "WARNING: ${fname} does not overlap requested range ${begin_year}-${end_year}; using file range ${file_start_year}-${file_end_year}." + YYYYS="${file_start_year}" + YYYYE="${file_end_year}" + else + # Clip to specified year range + (( file_start_year < begin_year )) && YYYYS="${begin_year}" || YYYYS="${file_start_year}" + (( file_end_year > end_year )) && YYYYE="${end_year}" || YYYYE="${file_end_year}" + fi # Extract prefix before the date range (removes from .${YYYYS} or -${YYYYS}) - SUBSTR="${fname%%[._-]${YYYYS}*}" + SUBSTR="${fname%%[._-]${orig_YYYYS}*}" ttag="$(printf "%04d" "${YYYYS}")01-$(printf "%04d" "${YYYYE}")12" tmp_file="tmp_combine_${ttag}.nc" - run_nco ncrcat -O -d time,"${YYYYS}-01-01","${YYYYE}-12-31" "${file}" "${tmp_file}" + run_nco ncrcat -O -h -d time,"${YYYYS}-01-01","${YYYYE}-12-31" "${file}" "${tmp_file}" if [[ $? -ne 0 ]]; then - echo "WARNING: ncrcat failed for ${fname} (date range ${YYYYS}-${YYYYE}), skipping." + echo "ERROR: ncrcat failed for ${fname} (date range ${YYYYS}-${YYYYE})" rm -f "${tmp_file}" - continue + cd "${script_dir}" || exit + echo "ERROR ($((error_num + 1)))" > "${prefix}.status" + exit "$((error_num + 1))" fi # Derive monthly climatology for month in $(seq 1 12); do + local MM MM=$(printf "%02d" ${month}) run_nco ncra -O -h -F -d time,"${month}",,12 "${tmp_file}" "tmp_clm_${MM}.nc" if [[ $? -ne 0 ]]; then - echo "WARNING: ncra failed for ${fname} month ${MM}, skipping file." + echo "ERROR: ncra failed for ${fname} month ${MM}." rm -f tmp_clm_*.nc "${tmp_file}" - break # exit the month loop + cd "${script_dir}" || exit + echo "ERROR ($((error_num + 2)))" > "${prefix}.status" + exit "$((error_num + 2))" fi done # If any month failed, the tmp_clm files were cleaned up — skip to next file @@ -281,11 +351,11 @@ create_links_acyc_climo_obs() { fi combined_name="${SUBSTR}.${ttag}.AC.${case_id}.nc" - run_nco ncrcat -O tmp_clm_*.nc "${combined_name}" + run_nco ncrcat -O -h tmp_clm_*.nc "${combined_name}" if [[ $? -ne 0 ]]; then cd "${script_dir}" || exit - echo "ERROR (${error_num})" > "${prefix}.status" - exit "${error_num}" + echo "ERROR ($((error_num + 3)))" > "${prefix}.status" + exit "$((error_num + 3))" fi # Adjust time metadata @@ -295,19 +365,31 @@ create_links_acyc_climo_obs() { run_nco ncap2 -O -h -s "${cmdfix1};${cmdfix2};${cmdfix3}" "${combined_name}" "${combined_name}" if [[ $? -ne 0 ]]; then cd "${script_dir}" || exit - echo "ERROR (${error_num})" > "${prefix}.status" - exit "$((error_num + 1))" + echo "ERROR ($((error_num + 4)))" > "${prefix}.status" + exit "$((error_num + 4))" fi - local cmdfix4='defdim("bnds",2)' - local cmdfix5='time_bnds=make_bounds(time,$bnds,"time_bnds")' - local cmdfix6='time_bnds@units=time@units' - local cmdfix7='time_bnds@calendar=time@calendar' + # Add bnds dimension if missing + if ! run_nco ncks -m "${combined_name}" | grep -q "bnds = 2"; then + echo "Adding missing bnds dimension..." + run_nco ncap2 -O -h -s 'defdim("bnds",2)' \ + "${combined_name}" "${combined_name}" + if [[ $? -ne 0 ]]; then + cd "${script_dir}" || exit + echo "ERROR ($((error_num + 5)))" > "${prefix}.status" + exit "$((error_num + 5))" + fi + fi + + local cmdfix4='time_bnds=make_bounds(time,$bnds,"time_bnds")' + local cmdfix5='time_bnds@units=time@units' + local cmdfix6='time_bnds@calendar=time@calendar' + local cmdfix7='time@bounds="time_bnds"' run_nco ncap2 -O -h -s "${cmdfix4};${cmdfix5};${cmdfix6};${cmdfix7}" "${combined_name}" "${combined_name}" if [[ $? -ne 0 ]]; then cd "${script_dir}" || exit - echo "ERROR (${error_num})" > "${prefix}.status" - exit "$((error_num + 2))" + echo "ERROR ($((error_num + 6)))" > "${prefix}.status" + exit "$((error_num + 6))" fi rm -vf tmp_*.nc @@ -315,6 +397,9 @@ create_links_acyc_climo_obs() { if [ -z "$( ls . )" ]; then echo "create_links_acyc_climo_obs: ${ts_dir_destination} was not updated!" + cd "${script_dir}" || exit + echo "ERROR (${error_num})" > "${prefix}.status" + exit "${error_num}" fi cd .. @@ -359,33 +444,80 @@ create_links_ts() { combined_name="${subname}.${v}.${begin_year}01-${end_year}12.nc" if [[ -s "${v}_files.txt" ]]; then - run_nco ncrcat -O -v "${v}" -d time,"${begin_year}-01-01","${end_year}-12-31" $(< "${v}_files.txt") "${combined_name}" + # Ensure input files have time calendar before string-based subsetting + while IFS= read -r file; do + [[ -z "${file}" ]] && continue + + if ! run_nco ncks -m "${file}" | grep -q "time:calendar"; then + echo "Adding missing calendar attribute to input time: ${file}" + run_nco ncatted -O -h -a calendar,time,o,c,"standard" "${file}" + if [[ $? -ne 0 ]]; then + cd "${script_dir}" || exit + echo "ERROR ($((error_num + 1)))" > "${prefix}.status" + exit "$((error_num + 1))" + fi + fi + done < "${v}_files.txt" + + # Extract subset of time series + run_nco ncrcat -O -h \ + -v "${v},time" \ + -d time,"${begin_year}-01-01","${end_year}-12-31" $(< "${v}_files.txt") "${combined_name}" - # Add calendar attribute if missing - if ! run_nco ncks -m "${combined_name}" | grep -q "calendar"; then - echo "Adding missing calendar attribute to time..." - run_nco ncatted -a calendar,time,o,c,"standard" "${combined_name}" + if [[ $? -ne 0 ]]; then + cd "${script_dir}" || exit + echo "ERROR ($((error_num + 2)))" > "${prefix}.status" + exit "$((error_num + 2))" + fi + echo "ncrcat subset successful" + + # Add bnds dimension if missing + if ! run_nco ncks -m "${combined_name}" | grep -q "bnds = 2"; then + echo "Adding missing bnds dimension..." + run_nco ncap2 -O -h -s 'defdim("bnds",2)' "${combined_name}" "${combined_name}" + if [[ $? -ne 0 ]]; then + cd "${script_dir}" || exit + echo "ERROR ($((error_num + 3)))" > "${prefix}.status" + exit "$((error_num + 3))" + fi + echo "ncap2 defdim successful" fi - # Add time bounds - local cmdfix1='defdim("bnds",2)' - local cmdfix2='time_bnds=make_bounds(time,$bnds,"time_bnds")' - local cmdfix3='time_bnds@units=time@units' - local cmdfix4='time_bnds@calendar=time@calendar' - run_nco ncap2 -O -h -s "${cmdfix1};${cmdfix2};${cmdfix3};${cmdfix4}" "${combined_name}" "${combined_name}" + # Always overwrite time bounds + local cmdfix1='time_bnds=make_bounds(time,$bnds,"time_bnds")' + local cmdfix2='time_bnds@units=time@units' + local cmdfix3='time_bnds@calendar=time@calendar' + local cmdfix4='time@bounds="time_bnds"' + run_nco ncap2 -O -h -s "${cmdfix1};${cmdfix2};${cmdfix3};${cmdfix4}" \ + "${combined_name}" "${combined_name}" + if [[ $? -ne 0 ]]; then + cd "${script_dir}" || exit + echo "ERROR ($((error_num + 4)))" > "${prefix}.status" + exit "$((error_num + 4))" + fi + echo "ncap2 time_bnds successful" + # Add CF metadata + run_nco ncatted -O \ + -a axis,time,o,c,"T" \ + -a standard_name,time,o,c,"time" \ + "${combined_name}" "${combined_name}" if [[ $? -ne 0 ]]; then cd "${script_dir}" || exit - echo "ERROR (${error_num})" > "${prefix}.status" - exit "${error_num}" + echo "ERROR ($((error_num + 5)))" > "${prefix}.status" + exit "$((error_num + 5))" fi + echo "ncatted CF metadata successful" else echo "Warning: No input files found for variable ${v} in ${ts_dir_source}. Skipping." fi done - if [ -z "$( ls . )" ]; then + if [ -z "$(ls ./*.nc 2>/dev/null)" ]; then echo "create_links_ts: ${ts_dir_destination} was not updated!" + cd "${script_dir}" || exit + echo "ERROR ($((error_num + 6)))" > "${prefix}.status" + exit "$((error_num + 6))" fi cd .. @@ -409,77 +541,102 @@ create_links_ts_obs() { local file fname SUBSTR YYYYS YYYYE ttag combined_name - for file in ${ts_dir_source}/*; do + for file in "${ts_dir_source}"/*; do fname=$(basename "$file") echo "create_links_ts_obs: checking if nc file: ${fname}" - if [[ ! ${fname} =~ ".nc" ]]; then + if [[ ${fname} != *.nc ]]; then # Skip non-.nc files continue fi echo "create_links_ts_obs: processing ${file}" + # Match two time patterns (YYYYMM or YYYYMMDD) separated by _ or - if [[ $fname =~ ^(.+)\.([0-9]{4})([0-9]{2}){1,2}[-_]([0-9]{4})([0-9]{2}){1,2}\.nc$ ]]; then SUBSTR="${BASH_REMATCH[1]}" # everything before the .YYYY... - YYYYS="${BASH_REMATCH[2]}" # start year - YYYYE="${BASH_REMATCH[4]}" # end year + file_start_year="${BASH_REMATCH[2]}" # file start year + file_end_year="${BASH_REMATCH[4]}" # file end year else - echo "Warning: Could not extract dates from ${fname}, basename of ${file}" continue fi - # Optional: clip years if needed - if [[ ${YYYYS} -lt ${begin_year} ]]; then - YYYYS="${begin_year}" - fi - - if [[ ${YYYYE} -gt ${end_year} ]]; then - YYYYE="${end_year}" + # Clip years to be in the selected range + if (( file_end_year < begin_year || file_start_year > end_year )); then + echo "WARNING: ${fname} does not overlap requested range ${begin_year}-${end_year}; using file range ${file_start_year}-${file_end_year}." + YYYYS="${file_start_year}" + YYYYE="${file_end_year}" + else + (( file_start_year < begin_year )) && YYYYS="${begin_year}" || YYYYS="${file_start_year}" + (( file_end_year > end_year )) && YYYYE="${end_year}" || YYYYE="${file_end_year}" fi ttag="$(printf "%04d" "${YYYYS}")01-$(printf "%04d" "${YYYYE}")12" combined_name="${SUBSTR}.${ttag}.nc" + # Ensure input time has calendar attribute before string-based time subsetting + if ! run_nco ncks -m "${file}" | grep -q "time:calendar"; then + echo "Adding missing calendar attribute to input time..." + run_nco ncatted -O -h -a calendar,time,o,c,"standard" "${file}" + if [[ $? -ne 0 ]]; then + cd "${script_dir}" || exit + echo "ERROR ($((error_num + 1)))" > "${prefix}.status" + exit "$((error_num + 1))" + fi + echo "ncatted successful" + fi + # Extract subset of time series - run_nco ncrcat -O -d time,"${YYYYS}-01-01","${YYYYE}-12-31" "${file}" "${combined_name}" + run_nco ncrcat -O -h -d time,"${YYYYS}-01-01","${YYYYE}-12-31" "${file}" "${combined_name}" if [[ $? -ne 0 ]]; then cd "${script_dir}" || exit - echo "ERROR (${error_num})" > "${prefix}.status" - exit "${error_num}" + echo "ERROR ($((error_num + 2)))" > "${prefix}.status" + exit "$((error_num + 2))" fi - echo "ncrcat successful" + echo "ncrcat subset successful" - # Ensure time has calendar attribute - if ! run_nco ncks -m "${combined_name}" | grep -q "calendar"; then - echo "Adding missing calendar attribute to time..." - run_nco ncatted -a calendar,time,o,c,"standard" "${combined_name}" + # Add bnds dimension if missing + if ! run_nco ncks -m "${combined_name}" | grep -q "bnds = 2"; then + echo "Adding missing bnds dimension..." + run_nco ncap2 -O -h -s 'defdim("bnds",2)' "${combined_name}" "${combined_name}" if [[ $? -ne 0 ]]; then cd "${script_dir}" || exit - echo "ERROR (${error_num})" > "${prefix}.status" - exit "$((error_num + 1))" + echo "ERROR ($((error_num + 3)))" > "${prefix}.status" + exit "$((error_num + 3))" fi - echo "ncatted successful" + echo "ncap2 defdim successful" fi # Add time bounds - local cmdfix1='defdim("bnds",2)' - local cmdfix2='time_bnds=make_bounds(time,$bnds,"time_bnds")' - local cmdfix3='time_bnds@units=time@units' - local cmdfix4='time_bnds@calendar=time@calendar' + local cmdfix1='time_bnds=make_bounds(time,$bnds,"time_bnds")' + local cmdfix2='time_bnds@units=time@units' + local cmdfix3='time_bnds@calendar=time@calendar' + local cmdfix4='time@bounds="time_bnds"' run_nco ncap2 -O -h -s "${cmdfix1};${cmdfix2};${cmdfix3};${cmdfix4}" "${combined_name}" "${combined_name}" if [[ $? -ne 0 ]]; then cd "${script_dir}" || exit - echo "ERROR (${error_num})" > "${prefix}.status" - exit "$((error_num + 2))" + echo "ERROR ($((error_num + 4)))" > "${prefix}.status" + exit "$((error_num + 4))" fi echo "ncap2 successful" + + # Add CF metadata + run_nco ncatted -O \ + -a axis,time,o,c,"T" \ + -a standard_name,time,o,c,"time" \ + "${combined_name}" "${combined_name}" + if [[ $? -ne 0 ]]; then + cd "${script_dir}" || exit + echo "ERROR ($((error_num + 5)))" > "${prefix}.status" + exit "$((error_num + 5))" + fi + echo "ncatted CF metadata successful" done if [ -z "$( ls . )" ]; then echo "create_links_ts_obs: ${ts_dir_destination} was not updated!" cd "${script_dir}" || exit - echo "ERROR (${error_num})" > "${prefix}.status" - exit "${error_num}" + echo "ERROR ($((error_num + 6)))" > "${prefix}.status" + exit "$((error_num + 6))" fi cd .. @@ -497,13 +654,13 @@ climo_dir_primary="climo" # This cmip_ts path is created via zppy/templates/e3sm_to_cmip.bash climo_dir_source="{{ output }}/post/atm/{{ grid }}/cmip_ts/monthly" # Link and process primary model climo data -create_links_acyc_climo "${climo_dir_source}" "${climo_dir_primary}" "${Y1}" "${Y2}" "${model_name}.${tableID}" 1 +create_links_acyc_climo "${climo_dir_source}" "${climo_dir_primary}" "${Y1}" "${Y2}" "${model_name}.${tableID}" 10 {% if run_type == "model_vs_model" %} # Path to reference model's climatology files climo_dir_source_ref="{{ reference_data_path_ts }}" climo_dir_ref="climo_ref" # Link and process reference model climo data -create_links_acyc_climo "${climo_dir_source_ref}" "${climo_dir_ref}" "${ref_Y1}" "${ref_Y2}" "${model_name_ref}.${tableID_ref}" 2 +create_links_acyc_climo "${climo_dir_source_ref}" "${climo_dir_ref}" "${ref_Y1}" "${ref_Y2}" "${model_name_ref}.${tableID_ref}" 20 {% endif %} {% endif %} @@ -513,13 +670,13 @@ create_links_acyc_climo "${climo_dir_source_ref}" "${climo_dir_ref}" "${ref_Y1}" ts_dir_primary="ts" ts_dir_source="{{ output }}/post/atm/{{ ts_grid }}/cmip_ts/monthly" # Create local links and combine time series NetCDF files for the primary model -create_links_ts "${ts_dir_source}" "${ts_dir_primary}" "${Y1}" "${Y2}" "${model_name}.${tableID}" 3 +create_links_ts "${ts_dir_source}" "${ts_dir_primary}" "${Y1}" "${Y2}" "${model_name}.${tableID}" 30 {% if run_type == "model_vs_model" %} # Define time series path for reference model (adjust for different year spans) ts_dir_source_ref="{{ reference_data_path_ts }}" ts_dir_ref="ts_ref" # Create local links and combine ts files for the reference model -create_links_ts "${ts_dir_source_ref}" "${ts_dir_ref}" "${ref_Y1}" "${ref_Y2}" "${model_name_ref}.${tableID_ref}" 4 +create_links_ts "${ts_dir_source_ref}" "${ts_dir_ref}" "${ref_Y1}" "${ref_Y2}" "${model_name_ref}.${tableID_ref}" 40 {% endif %} {% endif %} @@ -545,12 +702,12 @@ set -e {{ environment_commands_secondary }} set +e -time eval "${command}" +time ${command} if [ $? -ne 0 ]; then cd {{ scriptDir }} - echo "ERROR (5)" > {{ prefix }}.status - exit 5 + echo "ERROR (50)" > {{ prefix }}.status + exit 50 fi set -e @@ -565,12 +722,12 @@ ts_dir_ref_source="{{ scriptDir }}/${workdir}/${obstmp_dir}" {% if current_set == "mean_climate" %} climo_dir_ref=climo_ref -# This will use errors code 6,7,8: -create_links_acyc_climo_obs "${ts_dir_ref_source}" "${climo_dir_ref}" ${ref_Y1} ${ref_Y2} 6 +# This will use errors code 61,62,63...: +create_links_acyc_climo_obs "${ts_dir_ref_source}" "${climo_dir_ref}" ${ref_Y1} ${ref_Y2} 60 {% elif current_set in ["variability_modes_cpl", "variability_modes_atm", "enso"] %} ts_dir_ref=ts_ref -# This will use error codes 9,10,11: -create_links_ts_obs "${ts_dir_ref_source}" "${ts_dir_ref}" ${ref_Y1} ${ref_Y2} 9 +# This will use error codes 71,72,73...: +create_links_ts_obs "${ts_dir_ref_source}" "${ts_dir_ref}" ${ref_Y1} ${ref_Y2} 70 {% endif %} {% endif %} @@ -872,11 +1029,54 @@ source_dirs="--climo_ts_dir_primary ${climo_dir_primary} --climo_ts_dir_ref ${cl {% elif current_set in ["variability_modes_cpl", "variability_modes_atm", "enso"] %} source_dirs="--climo_ts_dir_primary ${ts_dir_primary} --climo_ts_dir_ref ${ts_dir_ref}" {% endif %} + +# ------------------------------------------------------- +# Adaptive safeguard for resource-intensive diagnostics +# ------------------------------------------------------- +requested_workers={{ num_workers }} +effective_workers=${requested_workers} +subsection="{{ subsection }}" + +# Each zppy subset is a separate SLURM job, so num_workers spans the entire +# job allocation. The relevant CPU budget is the total across all nodes. +# {{ nodes }} is the compile-time node count (same value as --nodes={{ nodes }}), +# so multiplying by it is simpler and more portable than relying on the +# runtime variable SLURM_JOB_NUM_NODES. +if [[ -n "${SLURM_CPUS_ON_NODE}" ]]; then + allocated_cpus=$(( SLURM_CPUS_ON_NODE * {{ nodes }} )) +elif command -v nproc >/dev/null 2>&1; then + allocated_cpus=$(nproc) +else + allocated_cpus=1 +fi + +# Guard against zero or empty (should not happen, but be safe) +if [[ -z "${allocated_cpus}" || "${allocated_cpus}" -lt 1 ]]; then + allocated_cpus=1 +fi + +# EOF / variability modes are heavy because each subprocess may call +# threaded LAPACK/BLAS routines and allocate large arrays. +if [[ "${subsection}" == "variability_modes_atm" || "${subsection}" == "variability_modes_cpl" ]]; then + safe_cap=$(( allocated_cpus / 4 )) + + if [[ ${safe_cap} -lt 1 ]]; then + safe_cap=1 + fi + + if [[ ${requested_workers} -gt ${safe_cap} ]]; then + echo "WARNING: Reducing num_workers from ${requested_workers} to ${safe_cap} for ${subsection} because EOF diagnostics are resource-intensive." + effective_workers=${safe_cap} + fi +fi + +echo "PCMDI worker policy: requested_workers=${requested_workers}, allocated_cpus=${allocated_cpus}, effective_workers=${effective_workers}, subsection=${subsection}" + # Parameter passing can encounter errors if parameters are empty. # So, it's a good idea to make sure they can't be (or at least are unlikely to be) empty. # run_type == "model_vs_obs" only: obs_sets (default value is NOT "") # run_type == "model_vs_model" only: model_name_ref, tableID_ref (default values are NOT "") -core_parameters="--num_workers {{ num_workers }} --multiprocessing {{ multiprocessing }} --subsection {{ subsection }} ${source_dirs} --model_name ${model_name} --model_tableID {{model_tableID }} --figure_format {{ figure_format }} --run_type {{ run_type }} --obs_sets {{ obs_sets }} --model_name_ref ${model_name_ref} --vars ${source_vars} --tableID_ref ${tableID_ref} --generate_sftlf {{ generate_sftlf }} --case_id ${case_id} --results_dir ${results_dir} --debug ${debug,,}" +core_parameters="--num_workers ${effective_workers} --multiprocessing {{ multiprocessing }} --subsection {{ subsection }} ${source_dirs} --model_name ${model_name} --model_tableID {{ model_tableID }} --figure_format {{ figure_format }} --run_type {{ run_type }} --obs_sets {{ obs_sets }} --model_name_ref ${model_name_ref} --vars ${source_vars} --tableID_ref ${tableID_ref} --generate_sftlf {{ generate_sftlf }} --case_id ${case_id} --results_dir ${results_dir} --debug ${debug,,}" {% endif %} {% if current_set == "mean_climate" %} @@ -891,7 +1091,7 @@ var_modes={{ movc_modes }} command="zi-pcmdi-variability-modes ${core_parameters} --var_modes ${var_modes}" {% endif %} {% if current_set == "enso" %} -command="zi-pcmdi-enso ${core_parameters} --enso_groups {{ enso_groups }}" +command="zi-pcmdi-enso ${core_parameters} --enso_groups {{ enso_groups }}" {% endif %} {% if current_set == "synthetic_plots" %} @@ -914,7 +1114,7 @@ echo "Done checking. There should be a list of files above, if they exist." {% endif %} #add command for enso viewer -enso_keys="--enso_viewer {{enso_viewer}} --cmip_enso_dir {{ cmip_enso_dir }} --cmip_enso_set {{ cmip_enso_set }} --enso_vars {{enso_vars}} --enso_years {{enso_years}}" +enso_keys="--enso_viewer {{ enso_viewer }} --cmip_enso_dir {{ cmip_enso_dir }} --cmip_enso_set {{ cmip_enso_set }} --enso_vars {{ enso_vars }} --enso_years {{ enso_years }}" {% if enso_viewer %} echo "Checking if var_mode_*.json files exist in variability_modes_dir:" enso_dir=${web_dir}/model_vs_obs/metrics_data/enso_metric/* @@ -922,7 +1122,7 @@ find ${enso_dir} -name "*.${case_id}*.json" echo "Done checking. There should be a list of files above, if they exist." {% endif %} -command="zi-pcmdi-synthetic-plots --synthetic_sets {{ synthetic_sets }} --figure_format {{ figure_format }} --www ${www} --results_dir ${results_dir} --case {{ case }} --model_name {{ model_name }} --model_tableID {{model_tableID }} --web_dir=${web_dir} --pcmdi_webtitle {{ pcmdi_webtitle }} --pcmdi_version {{ pcmdi_version }} --run_type ${run_type} --pcmdi_external_prefix {{ diagnostics_base_path }} --pcmdi_viewer_template {{ pcmdi_viewer_template }} --save_all_data {{save_all_data}} ${clim_keys} ${movs_keys} ${enso_keys} --debug ${debug,,}" +command="zi-pcmdi-synthetic-plots --synthetic_sets {{ synthetic_sets }} --figure_format {{ figure_format }} --www ${www} --results_dir ${results_dir} --case {{ case }} --model_name {{ model_name }} --model_tableID {{ model_tableID }} --web_dir=${web_dir} --pcmdi_webtitle {{ pcmdi_webtitle }} --pcmdi_version {{ pcmdi_version }} --run_type ${run_type} --pcmdi_external_prefix {{ diagnostics_base_path }} --pcmdi_viewer_template {{ pcmdi_viewer_template }} --save_all_data {{ save_all_data }} ${clim_keys} ${movs_keys} ${enso_keys} --debug ${debug,,}" {% endif %} @@ -936,8 +1136,8 @@ echo "The current directory is: $PWD" # This will be of the form .../post/script time ${command} if [ $? != 0 ]; then cd {{ scriptDir }} - echo 'ERROR (12)' > {{ prefix }}.status - exit 12 + echo 'ERROR (80)' > {{ prefix }}.status + exit 80 fi set -e @@ -954,8 +1154,8 @@ echo mkdir -p ${web_dir} if [ $? != 0 ]; then cd {{ scriptDir }} - echo 'ERROR (13)' > {{ prefix }}.status - exit 13 + echo 'ERROR (90)' > {{ prefix }}.status + exit 90 fi {% if machine in ['pm-cpu', 'pm-gpu'] %} @@ -979,16 +1179,44 @@ done rsync -a ${results_dir} ${web_dir}/ if [ $? != 0 ]; then cd {{ scriptDir }} - echo 'ERROR (14)' > {{ prefix }}.status - exit 14 + echo 'ERROR (100)' > {{ prefix }}.status + exit 100 fi {% endif %} +{% if current_set == "mean_climate" %} +# optional, when save_all_data = True, also copy the mean climate nc data +{% if save_all_data|lower == "true" %} +cd ${workdir} + +clim_dir=${web_dir}/${run_type}/metrics_data/mean_climate/ +mkdir -p ${clim_dir} + +rsync -a climo ${clim_dir}/ +rc1=$? + +rsync -a climo_ref ${clim_dir}/ +rc2=$? + +if [ ${rc1} != 0 ] || [ ${rc2} != 0 ]; then + cd {{ scriptDir }} + echo 'ERROR (110)' > {{ prefix }}.status + exit 110 +fi +{% endif %} +{% endif %} + {% if machine in ['pm-cpu', 'pm-gpu'] %} # For NERSC, change permissions of new files pushd ${web_dir}/ chgrp -R e3sm ${results_dir} chmod -R go+rX,go-w ${results_dir} +{% if current_set == "mean_climate" %} +{% if save_all_data|lower == "true" %} +chgrp -R e3sm ${run_type}/metrics_data/mean_climate +chmod -R go+rX,go-w ${run_type}/metrics_data/mean_climate +{% endif %} +{% endif %} popd {% endif %} @@ -996,6 +1224,11 @@ popd # For LCRC, change permissions of new files pushd ${web_dir}/ chmod -R go+rX,go-w ${results_dir} +{% if current_set == "mean_climate" %} +{% if save_all_data|lower == "true" %} +chmod -R go+rX,go-w ${run_type}/metrics_data/mean_climate +{% endif %} +{% endif %} popd {% endif %} diff --git a/zppy/templates/ts.bash b/zppy/templates/ts.bash index e49b9a99..9bba3434 100644 --- a/zppy/templates/ts.bash +++ b/zppy/templates/ts.bash @@ -54,7 +54,7 @@ vars={{ vars }} # https://unix.stackexchange.com/questions/237297/the-fastest-way-to-remove-a-string-in-a-variable # https://stackoverflow.com/questions/26457052/remove-a-substring-from-a-bash-variable # Remove U, since it is a 3D variable and thus will not work with rgn_avg -vars=${vars//,U} +# vars=${vars//,U} {%- else %} vars={{ vars }} {%- endif %} @@ -66,7 +66,7 @@ if grep -q "*" input.txt; then echo 'ERROR (1)' > {{ prefix }}.status exit 1 fi -# Generate time series files +# Generate time series files. # If the user-defined parameter "vars" is "", then ${vars}, defined above, will be too. cat input.txt | run_nco ncclimo \ -c {{ case }} \ @@ -86,17 +86,21 @@ cat input.txt | run_nco ncclimo \ --yr_srt={{ yr_start }} \ --yr_end={{ yr_end }} \ --ypf={{ ypf }} \ -{% if mapping_file == '' -%} +{%- if mapping_file == '' %} -o output \ -{%- elif mapping_file == 'glb' -%} +{%- elif mapping_file == 'glb' %} -o output \ --rgn_avg \ --area={{ area_nm }} \ -{%- else -%} +{%- else %} +{%- if prc_typ in ['mpasocean', 'mpasseaice'] %} +-o trash \ +{%- else %} --map={{ mapping_file }} \ -o trash \ -O output \ {%- endif %} +{%- endif %} {%- if frequency != 'monthly' %} --clm_md=hfs \ --dpf={{ dpf }} \ @@ -104,14 +108,262 @@ cat input.txt | run_nco ncclimo \ {%- endif %} --prc_typ={{ prc_typ }} - - +# Check ncclimo status before any optional separate regridding. if [ $? != 0 ]; then cd {{ scriptDir }} echo 'ERROR (2)' > {{ prefix }}.status exit 2 fi +{%- if mapping_file != '' and mapping_file != 'glb' and prc_typ in ['mpasocean', 'mpasseaice'] %} + +# Regrid MPAS-Ocean and MPAS-Seaice files separately. +# +# For MPAS components, generate native-grid time-series files first, then +# regrid them with ncremap. This lets us restore MPAS-native time metadata, +# construct a CF-style numeric time coordinate, and repair missing_value +# attributes after regridding if needed. +# +# For MPAS-Seaice, also provide valid spatial-only SGS helper variables for +# regridding. MPAS-Seaice monthly files may include time-dependent SGS +# diagnostics such as: +# +# timeMonthly_avg_iceAreaCell(Time,nCells) +# +# This variable should remain in the final time-series output. However, +# NCO's regridder requires sgs_frc_nm and sgs_msk_nm to be spatial-only +# fields matching the horizontal source grid size, i.e., nCells. Therefore, +# create temporary spatial-only helper variables with separate names and use +# them only for regridding. +mkdir -p output + +mpas_calendar="{{ mpas_calendar }}" +mpas_start_time="{{ mpas_start_time }}" +mpas_start_time="${mpas_start_time/_/ }" + +for f in trash/*.nc ; do + bname=$(basename "${f}") + tmp_f="tmp_regrid_${bname}" + tmp_frc="tmp_sgs_frc_${bname}" + tmp_msk="tmp_sgs_msk_${bname}" + out_f="output/${bname}" + + cp "${f}" "${tmp_f}" + + # Detect whether the time dimension is named Time or time. + # MPAS native ncclimo files often have a Time dimension but no + # coordinate variable named Time/time. + if run_nco ncks -m "${tmp_f}" | grep -qE "^[[:space:]]*Time[[:space:]]*=|\(Time[,)]|, Time[,)]"; then + time_dim="Time" + elif run_nco ncks -m "${tmp_f}" | grep -qE "^[[:space:]]*time[[:space:]]*=|\(time[,)]|, time[,)]"; then + time_dim="time" + else + echo "ERROR: Cannot find Time/time dimension in ${tmp_f}" + cd {{ scriptDir }} + echo 'ERROR (3)' > {{ prefix }}.status + exit 3 + fi + + # Infer the main variable name from the output filename. + # Example: + # timeMonthly_avg_seaSurfaceTemperature_000101_000512.nc + # becomes: + # timeMonthly_avg_seaSurfaceTemperature + main_var=$(echo "${bname}" | sed -E 's/_[0-9]{6}_[0-9]{6}\.nc$//') + + use_sgs_helpers="false" + + {%- if prc_typ == 'mpasseaice' %} + + # Construct explicit static SGS helpers for MPAS-Seaice. + # + # ncremap -P mpasseaice automatically tries to use + # timeMonthly_avg_iceAreaCell as sgs_frc_nm. However, that variable is + # time-dependent, with shape Time x nCells, while NCO requires sgs_frc_nm + # to be spatial-only, with shape nCells. + # + # Therefore, whenever timeMonthly_avg_iceAreaCell is available, construct: + # + # sgs_frc_static(nCells) = time mean of timeMonthly_avg_iceAreaCell + # sgs_msk_static(nCells) = 1 where sgs_frc_static > 0, else 0 + # + # Do not fall back to implicit ncremap SGS handling when this helper can be + # constructed, because the implicit path may reuse the time-dependent field. + if run_nco ncks -m "${tmp_f}" | grep -q "timeMonthly_avg_iceAreaCell"; then + + # Construct spatial-only SGS fraction helper. + run_nco ncwa -O -a "${time_dim}" -y avg \ + -v timeMonthly_avg_iceAreaCell \ + "${tmp_f}" "${tmp_frc}" + + run_nco ncrename -O \ + -v timeMonthly_avg_iceAreaCell,sgs_frc_static \ + "${tmp_frc}" + + # Construct a spatial-only binary SGS mask from the static fraction. + # This avoids relying on timeMonthly_avg_icePresentIntMask, which may + # not be present in every split file. + run_nco ncap2 -O \ + -s 'sgs_msk_static=(sgs_frc_static > 0.0)' \ + "${tmp_frc}" "${tmp_msk}" + + run_nco ncks -O -x -v sgs_frc_static "${tmp_msk}" "${tmp_msk}.only_msk" + mv "${tmp_msk}.only_msk" "${tmp_msk}" + + # Append helper variables to the temporary file. + # This does not remove or overwrite the original time-dependent variables. + run_nco ncks -A "${tmp_frc}" "${tmp_f}" + run_nco ncks -A "${tmp_msk}" "${tmp_f}" + + use_sgs_helpers="true" + else + echo "ERROR: timeMonthly_avg_iceAreaCell is missing in ${tmp_f}; cannot construct static SGS helpers for MPAS-Seaice." + cd {{ scriptDir }} + echo 'ERROR (3)' > {{ prefix }}.status + exit 3 + fi + + {%- endif %} + + if [ "${use_sgs_helpers}" == "true" ]; then + run_nco ncremap \ + -P {{ prc_typ }} \ + --d2f \ + --no_stagger \ + -t 1 \ + --nco_opt='--no_tmp_fl --hdr_pad=10000' \ + --map={{ mapping_file }} \ + --sgs_frc=sgs_frc_static \ + --sgs_msk=sgs_msk_static \ + --sgs_nrm=1.0 \ + "${tmp_f}" \ + "${out_f}" + else + run_nco ncremap \ + -P {{ prc_typ }} \ + --d2f \ + --no_stagger \ + -t 1 \ + --nco_opt='--no_tmp_fl --hdr_pad=10000' \ + --map={{ mapping_file }} \ + "${tmp_f}" \ + "${out_f}" + fi + + rc=$? + + # Remove temporary/helper variables from the final regridded output. + # Keep timeMonthly_avg_iceAreaCell only when it is the requested main variable. + if [ ${rc} == 0 ] && [ "{{ prc_typ }}" == "mpasseaice" ]; then + vars_to_remove="sgs_frc_static,sgs_msk_static" + + if [ "${main_var}" != "timeMonthly_avg_iceAreaCell" ]; then + vars_to_remove="${vars_to_remove},timeMonthly_avg_iceAreaCell" + fi + + # Remove helper variables only if at least one is present in the output. + remove_present="false" + for rm_var in $(echo "${vars_to_remove}" | tr ',' ' '); do + if run_nco ncks -m "${out_f}" | grep -qE "^[[:space:]]*[^:]+[[:space:]]+${rm_var}(\(|\[)"; then + remove_present="true" + break + fi + done + + if [ "${remove_present}" == "true" ]; then + run_nco ncks -O -x -v "${vars_to_remove}" "${out_f}" "${out_f}.clean" + mv "${out_f}.clean" "${out_f}" + rc=$? + fi + fi + + # Restore MPAS-native time metadata variables if needed. + # MPAS files often have a Time dimension and xtime_* variables, but no + # standard numeric coordinate variable named Time/time. + if [ ${rc} == 0 ]; then + for tvar in xtime_startMonthly xtime_endMonthly timeMonthly_avg_daysSinceStartOfSim; do + if run_nco ncks -m "${tmp_f}" | grep -q "${tvar}"; then + if ! run_nco ncks -m "${out_f}" | grep -q "${tvar}"; then + run_nco ncks -A -v "${tvar}" "${tmp_f}" "${out_f}" + rc=$? + if [ ${rc} != 0 ]; then + break + fi + fi + fi + done + fi + + # Construct a CF-style numeric time coordinate if it is missing. + # Use MPAS days-since-start metadata: + # + # timeMonthly_avg_daysSinceStartOfSim(Time) + # + # to create: + # + # time(Time) + # units = "days since ${mpas_start_time}" + # calendar = "${mpas_calendar}" + # + # This improves compatibility with downstream tools that expect a + # standard numeric time coordinate. + if [ ${rc} == 0 ]; then + if ! run_nco ncks -m "${out_f}" | grep -qE "^[[:space:]]*(float|double)[[:space:]]+time(\(|\[)"; then + if run_nco ncks -m "${tmp_f}" | grep -q "timeMonthly_avg_daysSinceStartOfSim"; then + tmp_time="tmp_time_${bname}" + + run_nco ncks -O \ + -v timeMonthly_avg_daysSinceStartOfSim \ + "${tmp_f}" "${tmp_time}" + + run_nco ncrename -O \ + -v timeMonthly_avg_daysSinceStartOfSim,time \ + "${tmp_time}" + + run_nco ncatted -O \ + -a units,time,o,c,"days since ${mpas_start_time}" \ + -a calendar,time,o,c,"${mpas_calendar}" \ + -a long_name,time,o,c,time \ + -a standard_name,time,o,c,time \ + "${tmp_time}" + + run_nco ncks -A -v time "${tmp_time}" "${out_f}" + rc=$? + + rm -f "${tmp_time}" + fi + fi + fi + + # Add missing_value metadata only for the main floating-point data variable + # if it is missing. Avoid modifying coordinate or integer mask variables. + # Do not force-create _FillValue here, because some NetCDF workflows are + # sensitive to adding _FillValue after variable creation. Existing _FillValue + # attributes are preserved. + if [ ${rc} == 0 ]; then + if run_nco ncks -m "${out_f}" | grep -qE "^[[:space:]]*(float|double)[[:space:]]+${main_var}(\(|\[)"; then + if ! run_nco ncks -m "${out_f}" | grep -A8 -E "^[[:space:]]*(float|double)[[:space:]]+${main_var}(\(|\[)" | grep -q "missing_value"; then + run_nco ncatted -O \ + -a missing_value,"${main_var}",o,f,1.0e36 \ + "${out_f}" + rc=$? + fi + fi + fi + + rm -f "${tmp_f}" "${tmp_frc}" "${tmp_msk}" "${tmp_msk}.only_msk" "${out_f}.clean" + + if [ ${rc} != 0 ]; then + echo "ERROR: MPAS component regridding failed for ${bname}" + cd {{ scriptDir }} + echo 'ERROR (4)' > {{ prefix }}.status + exit 4 + fi + +done + +{%- endif %} + {%- if vrt_remap_vars != '' %} # Vertical regrid (model levels → pressure levels) @@ -156,8 +408,8 @@ done } if [ $? != 0 ]; then cd {{ scriptDir }} - echo 'ERROR (3)' > {{ prefix }}.status - exit 3 + echo 'ERROR (5)' > {{ prefix }}.status + exit 5 fi {%- if vrt_remap_vars != '' %} diff --git a/zppy/utils.py b/zppy/utils.py index 4afff338..ed9c0272 100644 --- a/zppy/utils.py +++ b/zppy/utils.py @@ -227,6 +227,12 @@ def set_component_and_prc_typ(c: Dict[str, Any]) -> None: elif tmp in ("mosart",): component = "rof" prc_typ = tmp + elif tmp in ("mpaso",): + component = "ocn" + prc_typ = "mpasocean" + elif tmp in ("mpassi",): + component = "ice" + prc_typ = "mpasseaice" else: raise ValueError( f"Cannot extract output component name from {c['input_component']} or {c['input_files']}."