From 0dbed084315b1fbbde38b36c582fe3ee1c177361 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 13 Jan 2026 08:32:44 -0600 Subject: [PATCH 1/9] Add MPAS-Analysis model vs. model support This merge adds zppy config options: * `reference_data_path` -- the base path to a previously run zppy "control" (reference) run (if any) * `test_data_path` -- the base path to a previously run zppy "main" (test) run (if any) * `ref_ts_years`, `ref_climo_years`, `ref_enso_years` -- the range of years for time series, climatologies and ENSO indices if different from the "main" run. Thise are needed to be able to find the config file for the control (reference) run zppy finds the MPAS-Analysis config file use for the "control" and optionally the "main" runs, using them in the MPAS-Analsyis config file for the new zppy run. --- zppy/defaults/default.ini | 44 ++++++ zppy/mpas_analysis.py | 224 +++++++++++++++++++++++++++--- zppy/templates/mpas_analysis.bash | 9 ++ 3 files changed, 259 insertions(+), 18 deletions(-) diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index 01e12c3d..a23ad579 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -314,6 +314,25 @@ ts_daily_subsection = string(default="") [mpas_analysis] anomalyRefYear = integer(default=1) cache = boolean(default=True) +# Year ranges for the MPAS-Analysis sub-runs. These are used to form identifiers like: +# ts_1850-2014_climo_1985-2014 +ts_years = string_list(default=list("")) +climo_years = string_list(default=list("")) +enso_years = string_list(default=list("")) +# Optional: enable MPAS-Analysis test-vs-reference ("main vs. control") mode. +# Set these to the output directory of a prior zppy run (or the post/ dir itself). +# zppy will auto-locate: +# post/analysis/mpas_analysis/cfg/mpas_analysis_.cfg +# where matches each MPAS-Analysis sub-run (e.g. ts_1850-2014_climo_1985-2014). +reference_data_path = string(default="") +test_data_path = string(default="") +# Optional: allow reference run years to differ from test run years. +# If these are left empty, zppy uses the test run's corresponding year sets. +# If a single range is provided, it will be replicated for all test year sets. +ref_ts_years = string_list(default=list("")) +ref_climo_years = string_list(default=list("")) +ref_enso_years = string_list(default=list("")) + # Note that environment_commands needs to be the same for all related runs of [mpas_analysis]. # For example, if years 1-50 are run using one environment and years 51-100 are run using another, MPAS-Analysis may fail. generate = string_list(default=list('all', 'no_landIceCavities', 'no_BGC', 'no_icebergs', 'no_min', 'no_max', 'no_sose', 'no_waves', 'no_eke', 'no_climatologyMapAntarcticMelt', 'no_regionalTSDiagrams', 'no_timeSeriesAntarcticMelt', 'no_timeSeriesOceanRegions', 'no_climatologyMapSose', 'no_woceTransects', 'no_soseTransects', 'no_geojsonTransects', 'no_oceanRegionalProfiles', 'no_hovmollerOceanRegions', 'no_oceanConservation')) @@ -331,6 +350,31 @@ stream_ocn = string(default="streams.ocean") # NOTE: always overrides value in [default] walltime = string(default="06:00:00") + [[__many__]] + anomalyRefYear = integer(default=None) + cache = boolean(default=None) + ts_years = string_list(default=None) + climo_years = string_list(default=None) + enso_years = string_list(default=None) + reference_data_path = string(default=None) + test_data_path = string(default=None) + ref_ts_years = string_list(default=None) + ref_climo_years = string_list(default=None) + ref_enso_years = string_list(default=None) + generate = string_list(default=None) + mapMpiTasks = integer(default=None) + mpaso_nml = string(default=None) + mpassi_nml = string(default=None) + ncclimoThreads = integer(default=None) + ncclimoParallelMode = string(default=None) + parallelTaskCount = integer(default=None) + PostMOC = boolean(default=None) + purge = boolean(default=None) + shortTermArchive = boolean(default=None) + stream_ice = string(default=None) + stream_ocn = string(default=None) + walltime = string(default=None) + [global_time_series] climo_years = string_list(default=list("")) # The color to be used for the graphs. diff --git a/zppy/mpas_analysis.py b/zppy/mpas_analysis.py index 8355a13c..163cbe43 100644 --- a/zppy/mpas_analysis.py +++ b/zppy/mpas_analysis.py @@ -1,3 +1,5 @@ +import os +from pathlib import Path from typing import Any, Dict, List, Tuple from configobj import ConfigObj @@ -41,33 +43,113 @@ def mpas_analysis(config: ConfigObj, script_dir: str, existing_bundles, job_ids_ set_subdirs(config, c) # Loop over year sets ts_year_sets: List[Tuple[int, int]] = get_years(c["ts_years"]) - climo_year_sets: List[Tuple[int, int]] - enso_year_sets: List[Tuple[int, int]] - if c["climo_years"] != [""]: - climo_year_sets = get_years(c["climo_years"]) - else: - climo_year_sets = ts_year_sets - if c["enso_years"] != [""]: - enso_year_sets = get_years(c["enso_years"]) - else: - enso_year_sets = ts_year_sets - for s, rs, es in zip(ts_year_sets, climo_year_sets, enso_year_sets): - c["ts_year1"] = s[0] - c["ts_year2"] = s[1] + + # Main climo/index year sets default to time series years. + climo_year_sets = _resolve_year_sets( + c.get("climo_years", [""]), + fallback=ts_year_sets, + target_len=len(ts_year_sets), + label="climo_years", + ) + enso_year_sets = _resolve_year_sets( + c.get("enso_years", [""]), + fallback=ts_year_sets, + target_len=len(ts_year_sets), + label="enso_years", + ) + + # Reference year sets for test-vs-reference (main-vs-control) mode. + # If not provided, default to the main run's corresponding year sets. + ref_ts_year_sets = _resolve_year_sets( + c.get("ref_ts_years", [""]), + fallback=ts_year_sets, + target_len=len(ts_year_sets), + label="ref_ts_years", + ) + ref_climo_year_sets = _resolve_year_sets( + c.get("ref_climo_years", [""]), + fallback=ref_ts_year_sets, + target_len=len(ts_year_sets), + label="ref_climo_years", + ) + ref_enso_year_sets = _resolve_year_sets( + c.get("ref_enso_years", [""]), + fallback=ref_ts_year_sets, + target_len=len(ts_year_sets), + label="ref_enso_years", + ) + + for ts, climo, enso, ctrl_ts, ctrl_climo, ctrl_enso in zip( + ts_year_sets, + climo_year_sets, + enso_year_sets, + ref_ts_year_sets, + ref_climo_year_sets, + ref_enso_year_sets, + ): + c["ts_year1"] = ts[0] + c["ts_year2"] = ts[1] if ("last_year" in c.keys()) and (c["ts_year2"] > c["last_year"]): continue # Skip this year set - c["climo_year1"] = rs[0] - c["climo_year2"] = rs[1] + c["climo_year1"] = climo[0] + c["climo_year2"] = climo[1] if ("last_year" in c.keys()) and (c["climo_year2"] > c["last_year"]): continue # Skip this year set - c["enso_year1"] = es[0] - c["enso_year2"] = es[1] + c["enso_year1"] = enso[0] + c["enso_year2"] = enso[1] if ("last_year" in c.keys()) and (c["enso_year2"] > c["last_year"]): continue # Skip this year set c["scriptDir"] = script_dir + identifier: str = _get_identifier( + ts_year1=c["ts_year1"], + ts_year2=c["ts_year2"], + climo_year1=c["climo_year1"], + climo_year2=c["climo_year2"], + ) + c["identifier"] = identifier + + ref_identifier: str = _get_identifier( + ts_year1=ctrl_ts[0], + ts_year2=ctrl_ts[1], + climo_year1=ctrl_climo[0], + climo_year2=ctrl_climo[1], + ) + c["ref_identifier"] = ref_identifier + + # Optional: point MPAS-Analysis at already-completed MPAS-Analysis runs + # so it can do a "test vs. reference" ("main vs. control") comparison. + # + # These zppy options are expected to point to either: + # - a prior zppy run's output directory (containing a post/ directory), + # - the post/ directory itself, + # - an MPAS-Analysis directory (analysis/mpas_analysis), + # - the cfg/ directory, or + # - directly to the specific mpas_analysis_*.cfg file. + # + # For user consistency with other zppy tasks, prefer *_data_path naming. + # These point to a prior zppy run's output directory (or its post/ dir), + # and zppy will locate the MPAS-Analysis config file within it. + # + # We keep *_run_output_dir as backwards-compatible aliases. + reference_data_path = c.get("reference_data_path", "") + test_data_path = c.get("test_data_path", "") + c["controlRunConfigFile"] = ( + _resolve_mpas_analysis_config_file(reference_data_path, ref_identifier) + if reference_data_path + else "" + ) + c["mainRunConfigFile"] = ( + _resolve_mpas_analysis_config_file(test_data_path, identifier) + if test_data_path + else "" + ) prefix_suffix: str = ( f"_ts_{c['ts_year1']:04d}-{c['ts_year2']:04d}_climo_{c['climo_year1']:04d}-{c['climo_year2']:04d}" ) + + # If reference years differ, include them in the script/status prefix to avoid collisions. + if c["controlRunConfigFile"] and (ref_identifier != identifier): + prefix_suffix = f"{prefix_suffix}_vs_ref_{ref_identifier}" prefix: str if c["subsection"]: prefix = f"mpas_analysis_{c['subsection']}{prefix_suffix}" @@ -87,7 +169,7 @@ def mpas_analysis(config: ConfigObj, script_dir: str, existing_bundles, job_ids_ f.write(template.render(**c)) make_executable(bash_file) c["dependencies"] = dependencies - write_settings_file(settings_file, c, s) + write_settings_file(settings_file, c, ts) export = "ALL" existing_bundles = handle_bundles( c, @@ -122,6 +204,112 @@ def mpas_analysis(config: ConfigObj, script_dir: str, existing_bundles, job_ids_ return existing_bundles +def _get_identifier( + *, ts_year1: int, ts_year2: int, climo_year1: int, climo_year2: int +) -> str: + # Must match identifier in zppy/templates/mpas_analysis.bash + ts_y1 = f"{ts_year1:04d}" + ts_y2 = f"{ts_year2:04d}" + clim_y1 = f"{climo_year1:04d}" + clim_y2 = f"{climo_year2:04d}" + return f"ts_{ts_y1}-{ts_y2}_climo_{clim_y1}-{clim_y2}" + + +def _resolve_year_sets( + years_value: Any, + *, + fallback: List[Tuple[int, int]], + target_len: int, + label: str, +) -> List[Tuple[int, int]]: + """Parse and normalize year sets. + + Behavior: + - If the parsed year sets are empty, return fallback. + - If there is a single year set and target_len > 1, replicate to match target_len. + - If the length is neither 1 nor target_len (and non-empty), raise ValueError. + """ + + parsed = get_years(years_value) + if len(parsed) == 0: + return fallback + if (len(parsed) == 1) and (target_len > 1): + return parsed * target_len + if (target_len > 0) and (len(parsed) != target_len): + raise ValueError( + f"{label} has {len(parsed)} ranges but expected 1 or {target_len} to match ts_years." + ) + return parsed + + +def _resolve_mpas_analysis_config_file(run_output_dir: str, identifier: str) -> str: + """Resolve the MPAS-Analysis config file for a prior run. + + The resolved file is expected to be named: + mpas_analysis_.cfg + + with identifier like: + ts_1850-2014_climo_1985-2014 + + The input may be a directory (various plausible roots) or a direct .cfg path. + """ + + path = Path(os.path.expandvars(os.path.expanduser(run_output_dir))).resolve() + + # Direct file path + if path.is_file(): + return str(path) + + file_name = f"mpas_analysis_{identifier}.cfg" + + if not path.is_dir(): + raise FileNotFoundError( + f"MPAS-Analysis run path does not exist: {run_output_dir}" + ) + + candidate_dirs = [ + # User points at zppy output root + path / "post" / "analysis" / "mpas_analysis" / "cfg", + # User points at post/ + path / "analysis" / "mpas_analysis" / "cfg", + # User points at mpas_analysis/ + path / "cfg", + # Fallbacks + path, + ] + + tried: List[str] = [] + for cfg_dir in candidate_dirs: + candidate = cfg_dir / file_name + tried.append(str(candidate)) + if candidate.is_file(): + return str(candidate) + + # Last resort: small bounded search (avoids an expensive full tree walk) + max_depth = 5 + matches: List[Path] = [] + root_depth = len(path.parts) + for root, dirs, files in os.walk(path): + current_depth = len(Path(root).parts) - root_depth + if current_depth > max_depth: + dirs[:] = [] + continue + if file_name in files: + matches.append(Path(root) / file_name) + break + + if matches: + return str(matches[0]) + + raise FileNotFoundError( + "Could not find prior MPAS-Analysis config file for model-vs-model run. " + f"Searched for {file_name} under {run_output_dir}. " + "Common fix: set [mpas_analysis]/reference_data_path to the prior run's " + "zppy output directory (the one containing post/analysis/mpas_analysis/cfg/). " + f"Tried: {tried}" + ) + + def set_subdirs(config: ConfigObj, c: Dict[str, Any]) -> None: if config["mpas_analysis"]["shortTermArchive"]: c["subdir_ocean"] = "/archive/ocn/hist" diff --git a/zppy/templates/mpas_analysis.bash b/zppy/templates/mpas_analysis.bash index a34653fc..ee4c2dde 100644 --- a/zppy/templates/mpas_analysis.bash +++ b/zppy/templates/mpas_analysis.bash @@ -98,6 +98,15 @@ cat > cfg/mpas_analysis_${identifier}.cfg << EOF # mainRunName is a name that identifies the simulation being analyzed. mainRunName = {{ case }} +{% if controlRunConfigFile %} +# config file for a control run to which this run will be compared. +controlRunConfigFile = {{ controlRunConfigFile }} +{% endif %} +{% if mainRunConfigFile %} +# config file for a main run on which the analysis was already run to completion. +mainRunConfigFile = {{ mainRunConfigFile }} +{% endif %} + [execute] ## options related to executing parallel tasks From cc926a5c94992d75ca522ca83f16e312b3f0ab72 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 13 Jan 2026 08:54:06 -0600 Subject: [PATCH 2/9] Update the docs --- docs/source/parameters.rst | 35 +++++++++++++++-- .../post.mpas_analysis_model_vs_model.cfg | 38 +++++++++++++++++++ docs/source/tutorial.rst | 21 ++++++++++ 3 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 docs/source/post.mpas_analysis_model_vs_model.cfg diff --git a/docs/source/parameters.rst b/docs/source/parameters.rst index 5de040ba..b97d126a 100644 --- a/docs/source/parameters.rst +++ b/docs/source/parameters.rst @@ -12,8 +12,8 @@ be overridden by parameters set in a ``[[subsection]``. Note that some parameters will be overriden by defaults if you define them too high up in the inheritance hierarchy. See `this release's parameter defaults `_ -on GitHub for a complete list of parameters and their default values. -You can also view the most up-to-date, +on GitHub for a complete list of parameters and their default values. +You can also view the most up-to-date, `unreleased parameter defaults `_. Deprecated parameters @@ -61,6 +61,33 @@ For the ``e3sm_diags`` task: * If ``reference_data_path`` (the path to the reference data) is undefined, assume it is the ``diagnostics_base_path`` from Mache plus ``/observations/Atm/climatology/``. (So, it is important to change this for model-vs-model runs). +For the ``mpas_analysis`` task: + +* ``reference_data_path`` and ``test_data_path`` are optional and are only used for model-vs-model comparisons. + If provided, ``zppy`` uses them to locate the MPAS-Analysis config files from a *previous* MPAS-Analysis run and passes those through to MPAS-Analysis as ``controlRunConfigFile`` (reference) and ``mainRunConfigFile`` (test). + + .. note:: + These parameter names are intentionally consistent with the terminology used by ``e3sm_diags`` for model-vs-model runs: in both cases, ``reference_data_path`` identifies the *reference simulation's zppy-generated outputs*. + + The practical difference is what each downstream tool consumes: + ``e3sm_diags`` needs ``reference_data_path`` to be the specific directory containing the reference climatology files (typically under the reference run's ``post/.../clim`` tree), whereas ``mpas_analysis`` needs to find the reference MPAS-Analysis config file. + For MPAS-Analysis, ``zppy`` can resolve that config file when ``reference_data_path`` points to the prior run's zppy output directory (the one containing ``post/``), the ``post/`` directory itself, or directly to an ``mpas_analysis_*.cfg`` file. + + ``reference_data_path`` is intended to point to the prior run's zppy output directory (the one containing ``post/``) but zppy will also find the MPAS-Analysis config file if ``reference_data_path`` points to the ``post/`` directory itself, the MPAS-Analysis directory (``analysis/mpas_analysis``), the ``cfg/`` directory, or directly to an ``mpas_analysis_*.cfg`` file. + + +**MPAS-Analysis model-vs-model year ranges** + +MPAS-Analysis comparisons are configured by year ranges, similar to other ``zppy`` tasks. +For model-vs-model comparisons, ``zppy`` supports separate year ranges for the test and reference runs: + +* ``ts_years``, ``climo_years``, ``enso_years`` define the test run year ranges. +* ``ref_ts_years``, ``ref_climo_years``, ``ref_enso_years`` optionally override the reference run year ranges. + +If a ``ref_*_years`` parameter is not provided, it defaults to the corresponding test year ranges. +If a ``ref_*_years`` parameter contains a single range and multiple test ranges are requested, the single reference range is used for each test range. + + For the ``ilamb`` task: * If ``ilamb_obs`` (the path to observation data for ``ilamb``) is undefined, assume it is the ``diagnostics_base_path`` from Mache plus ``/ilamb_data``. @@ -76,7 +103,7 @@ There are many parameter-handling functions. In ``utils.py``: -* ``get_value_from_parameter``: check if parameter is in the configuration dictionary. If not, if inference is turned on (the default), then just use the value of ``second_choice_parameter``. If inferenceis turned off, raise a ``ParameterNotProvidedError``. Use this function if the backup option +* ``get_value_from_parameter``: check if parameter is in the configuration dictionary. If not, if inference is turned on (the default), then just use the value of ``second_choice_parameter``. If inferenceis turned off, raise a ``ParameterNotProvidedError``. Use this function if the backup option * ``set_value_of_parameter_if_undefined``: check if parameter is in the configuration dictionary. If not, if inferenceis turned on (the default), then just set the parameter's value to the ``backup_option``. If inferenceis turned off, raise a ``ParameterNotProvidedError``. In ``e3sm_diags.py``: @@ -85,6 +112,6 @@ In ``e3sm_diags.py``: * ``check_set_specific_parameter``: if any requested ``e3sm_diags`` sets require this parameter, make sure it is present. If not, raise a ``ParameterNotProvidedError``. * ``check_parameters_for_bash``: use ``check_set_specific_parameter`` to check the existence of parameters that aren't used until the bash script. * ``check_mvm_only_parameters_for_bash``: similar, but these are specifically parameters used for model-vs-model runs. Uses ``check_parameter_defined`` in addition to ``check_set_specific_parameter``. -* ``check_and_define_parameters``: make sure all parameters are defined, using ``utils.py get_value_from_parameter``, ``utils.py set_value_of_parameter_if_undefined``, and ``check_mvm_only_parameters_for_bash``. +* ``check_and_define_parameters``: make sure all parameters are defined, using ``utils.py get_value_from_parameter``, ``utils.py set_value_of_parameter_if_undefined``, and ``check_mvm_only_parameters_for_bash``. ``check_parameters_for_bash`` can be run immediately for each subtask because it has very few conditions. Other checks are included in ``check_and_define_parameters`` later on in the code. \ No newline at end of file diff --git a/docs/source/post.mpas_analysis_model_vs_model.cfg b/docs/source/post.mpas_analysis_model_vs_model.cfg new file mode 100644 index 00000000..43e31998 --- /dev/null +++ b/docs/source/post.mpas_analysis_model_vs_model.cfg @@ -0,0 +1,38 @@ +[default] +input = +# Where the `post/` directory will be written +output = +case = +www = +partition = + +# MPAS-Analysis: model vs. model ("test" vs. "reference") +[mpas_analysis] +active = True +walltime = "4:00:00" +parallelTaskCount = 6 +mesh = "EC30to60E2r2" +shortTermArchive = True + +# Test run year ranges (these determine the identifier used for the generated +# MPAS-Analysis cfg: mpas_analysis_ts__climo_.cfg) +ts_years = "1850-2014", +climo_years = "1985-2014", +enso_years = "1850-2014", + +# Point at a *previous zppy run output directory* for the reference simulation. +# zppy will locate the matching MPAS-Analysis cfg file under (typical): +# /post/analysis/mpas_analysis/cfg/ +# You may also point at the `post/` directory, `analysis/mpas_analysis`, the `cfg/` +# directory, or directly at an mpas_analysis_*.cfg file. +reference_data_path = + +# Optional: point at a previous zppy output directory for the test simulation, +# if you want MPAS-Analysis to reuse a completed test run as well. +# test_data_path = + +# Optional: override the reference year ranges (defaults to the test ranges). +# If you provide a single range, it will be reused for each test range. +# ref_ts_years = "451-500", +# ref_climo_years = "451-500", +# ref_enso_years = "451-500", diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index f19dbea3..536dfc1c 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -58,6 +58,27 @@ This is another example of a configuration file, this time using a RRM simulatio :language: cfg :linenos: + +Example 3 +========= + +MPAS-Analysis model vs. model +----------------------------- + +MPAS-Analysis supports "main vs. control" (model-vs-model) comparisons. +In ``zppy``, this is configured in the ``[mpas_analysis]`` section using +``reference_data_path`` (and optionally ``test_data_path``), consistent with the +terminology used for ``e3sm_diags`` model-vs-model runs. + +Unlike ``e3sm_diags`` (where ``run_type = "model_vs_model"`` and ``reference_data_path`` +points directly at reference climatology output), MPAS-Analysis comparisons are driven +by MPAS-Analysis config files. For model-vs-model mode, ``zppy`` locates the matching +config file(s) from prior MPAS-Analysis output and passes them to MPAS-Analysis. + +.. literalinclude:: post.mpas_analysis_model_vs_model.cfg + :language: cfg + :linenos: + Debugging failures ================== From 44aa21c32f898fe9ccc8a92694322ed255d740b2 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 3 Feb 2026 15:16:48 +0100 Subject: [PATCH 3/9] Allow `reference_data_path` to point to previous subsection And the same for `test_data_path`. --- zppy/defaults/default.ini | 3 ++- zppy/mpas_analysis.py | 45 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index a23ad579..8d5554c7 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -320,7 +320,8 @@ ts_years = string_list(default=list("")) climo_years = string_list(default=list("")) enso_years = string_list(default=list("")) # Optional: enable MPAS-Analysis test-vs-reference ("main vs. control") mode. -# Set these to the output directory of a prior zppy run (or the post/ dir itself). +# Set these to the output directory of a prior zppy run (or the post/ dir itself), +# or to [[ subsection ]] to refer to a previous mpas_analysis subsection in this workflow. # zppy will auto-locate: # post/analysis/mpas_analysis/cfg/mpas_analysis_.cfg # where matches each MPAS-Analysis sub-run (e.g. ts_1850-2014_climo_1985-2014). diff --git a/zppy/mpas_analysis.py b/zppy/mpas_analysis.py index 163cbe43..bced72ca 100644 --- a/zppy/mpas_analysis.py +++ b/zppy/mpas_analysis.py @@ -1,4 +1,5 @@ import os +import re from pathlib import Path from typing import Any, Dict, List, Tuple @@ -37,10 +38,25 @@ def mpas_analysis(config: ConfigObj, script_dir: str, existing_bundles, job_ids_ # Dependencies carried over from previous task. carried_over_dependencies: List[str] = [] + # Track base output directories for previously defined mpas_analysis subsections. + prior_subsection_outputs: Dict[str, str] = {} + for c in tasks: dependencies: List[str] = carried_over_dependencies set_subdirs(config, c) + reference_data_path = _resolve_subsection_reference( + c.get("reference_data_path", ""), + prior_subsection_outputs, + "reference_data_path", + ) + test_data_path = _resolve_subsection_reference( + c.get("test_data_path", ""), + prior_subsection_outputs, + "test_data_path", + ) + c["reference_data_path"] = reference_data_path + c["test_data_path"] = test_data_path # Loop over year sets ts_year_sets: List[Tuple[int, int]] = get_years(c["ts_years"]) @@ -131,8 +147,6 @@ def mpas_analysis(config: ConfigObj, script_dir: str, existing_bundles, job_ids_ # and zppy will locate the MPAS-Analysis config file within it. # # We keep *_run_output_dir as backwards-compatible aliases. - reference_data_path = c.get("reference_data_path", "") - test_data_path = c.get("test_data_path", "") c["controlRunConfigFile"] = ( _resolve_mpas_analysis_config_file(reference_data_path, ref_identifier) if reference_data_path @@ -201,6 +215,12 @@ def mpas_analysis(config: ConfigObj, script_dir: str, existing_bundles, job_ids_ print(f" environment_commands={c['environment_commands']}") print_url(c, "mpas_analysis") + if c.get("subsection"): + output_dir = os.path.abspath( + os.path.expandvars(os.path.expanduser(c["output"])) + ) + prior_subsection_outputs[c["subsection"]] = output_dir + return existing_bundles @@ -242,6 +262,27 @@ def _resolve_year_sets( return parsed +def _resolve_subsection_reference( + value: str, + prior_subsection_outputs: Dict[str, str], + parameter_name: str, +) -> str: + if not value or not isinstance(value, str): + return value + + match = re.match(r"^\s*\[\[\s*(.+?)\s*\]\]\s*$", value) + if not match: + return value + + subsection = match.group(1).strip() + if subsection not in prior_subsection_outputs: + raise ValueError( + f"{parameter_name} refers to mpas_analysis subsection '{subsection}', " + "but it has not been defined earlier in [mpas_analysis]." + ) + return prior_subsection_outputs[subsection] + + def _resolve_mpas_analysis_config_file(run_output_dir: str, identifier: str) -> str: """Resolve the MPAS-Analysis config file for a prior run. From 866469e0e0d4734a3923802b7d8649d10585cb7c Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 3 Feb 2026 15:28:25 +0100 Subject: [PATCH 4/9] Require `reference_data_path` to be the root of a zppy run Same with `test_data_path`. No longer check if the config file exists, as it may be generated at runtime. --- docs/source/parameters.rst | 5 +- .../post.mpas_analysis_model_vs_model.cfg | 4 +- zppy/defaults/default.ini | 6 +- zppy/mpas_analysis.py | 74 ++++--------------- 4 files changed, 22 insertions(+), 67 deletions(-) diff --git a/docs/source/parameters.rst b/docs/source/parameters.rst index b97d126a..d803d1c4 100644 --- a/docs/source/parameters.rst +++ b/docs/source/parameters.rst @@ -71,9 +71,10 @@ For the ``mpas_analysis`` task: The practical difference is what each downstream tool consumes: ``e3sm_diags`` needs ``reference_data_path`` to be the specific directory containing the reference climatology files (typically under the reference run's ``post/.../clim`` tree), whereas ``mpas_analysis`` needs to find the reference MPAS-Analysis config file. - For MPAS-Analysis, ``zppy`` can resolve that config file when ``reference_data_path`` points to the prior run's zppy output directory (the one containing ``post/``), the ``post/`` directory itself, or directly to an ``mpas_analysis_*.cfg`` file. + For MPAS-Analysis, ``zppy`` resolves the config file when ``reference_data_path`` points to the prior run's zppy output directory (the one containing ``post/``). - ``reference_data_path`` is intended to point to the prior run's zppy output directory (the one containing ``post/``) but zppy will also find the MPAS-Analysis config file if ``reference_data_path`` points to the ``post/`` directory itself, the MPAS-Analysis directory (``analysis/mpas_analysis``), the ``cfg/`` directory, or directly to an ``mpas_analysis_*.cfg`` file. + ``reference_data_path`` is intended to point to the prior run's zppy output directory (the one containing ``post/``). ``zppy`` will then use: + ``/post/analysis/mpas_analysis/cfg/mpas_analysis_.cfg``. **MPAS-Analysis model-vs-model year ranges** diff --git a/docs/source/post.mpas_analysis_model_vs_model.cfg b/docs/source/post.mpas_analysis_model_vs_model.cfg index 43e31998..6f0b240c 100644 --- a/docs/source/post.mpas_analysis_model_vs_model.cfg +++ b/docs/source/post.mpas_analysis_model_vs_model.cfg @@ -21,10 +21,8 @@ climo_years = "1985-2014", enso_years = "1850-2014", # Point at a *previous zppy run output directory* for the reference simulation. -# zppy will locate the matching MPAS-Analysis cfg file under (typical): +# zppy will locate the matching MPAS-Analysis cfg file under: # /post/analysis/mpas_analysis/cfg/ -# You may also point at the `post/` directory, `analysis/mpas_analysis`, the `cfg/` -# directory, or directly at an mpas_analysis_*.cfg file. reference_data_path = # Optional: point at a previous zppy output directory for the test simulation, diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index 8d5554c7..ab533e71 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -320,10 +320,10 @@ ts_years = string_list(default=list("")) climo_years = string_list(default=list("")) enso_years = string_list(default=list("")) # Optional: enable MPAS-Analysis test-vs-reference ("main vs. control") mode. -# Set these to the output directory of a prior zppy run (or the post/ dir itself), +# Set these to the output directory of a prior zppy run (the one containing post/), # or to [[ subsection ]] to refer to a previous mpas_analysis subsection in this workflow. -# zppy will auto-locate: -# post/analysis/mpas_analysis/cfg/mpas_analysis_.cfg +# zppy will use: +# /post/analysis/mpas_analysis/cfg/mpas_analysis_.cfg # where matches each MPAS-Analysis sub-run (e.g. ts_1850-2014_climo_1985-2014). reference_data_path = string(default="") test_data_path = string(default="") diff --git a/zppy/mpas_analysis.py b/zppy/mpas_analysis.py index bced72ca..7481c998 100644 --- a/zppy/mpas_analysis.py +++ b/zppy/mpas_analysis.py @@ -135,12 +135,8 @@ def mpas_analysis(config: ConfigObj, script_dir: str, existing_bundles, job_ids_ # Optional: point MPAS-Analysis at already-completed MPAS-Analysis runs # so it can do a "test vs. reference" ("main vs. control") comparison. # - # These zppy options are expected to point to either: - # - a prior zppy run's output directory (containing a post/ directory), - # - the post/ directory itself, - # - an MPAS-Analysis directory (analysis/mpas_analysis), - # - the cfg/ directory, or - # - directly to the specific mpas_analysis_*.cfg file. + # These zppy options are expected to point to a prior zppy run's + # output directory (containing a post/ directory). # # For user consistency with other zppy tasks, prefer *_data_path naming. # These point to a prior zppy run's output directory (or its post/ dir), @@ -242,7 +238,8 @@ def _resolve_year_sets( target_len: int, label: str, ) -> List[Tuple[int, int]]: - """Parse and normalize year sets. + """ + Parse and normalize year sets. Behavior: - If the parsed year sets are empty, return fallback. @@ -284,7 +281,8 @@ def _resolve_subsection_reference( def _resolve_mpas_analysis_config_file(run_output_dir: str, identifier: str) -> str: - """Resolve the MPAS-Analysis config file for a prior run. + """ + Resolve the MPAS-Analysis config file path for a prior run. The resolved file is expected to be named: mpas_analysis_.cfg @@ -292,63 +290,21 @@ def _resolve_mpas_analysis_config_file(run_output_dir: str, identifier: str) -> with identifier like: ts_1850-2014_climo_1985-2014 - The input may be a directory (various plausible roots) or a direct .cfg path. + The input is expected to be the prior run's zppy output directory, which + contains a post/ directory. + + Note: We intentionally do not check for filesystem existence here. This allows + zppy workflows where the referenced MPAS-Analysis run is produced later in the + same workflow. MPAS-Analysis will raise an error at runtime if the config file + is missing. """ path = Path(os.path.expandvars(os.path.expanduser(run_output_dir))).resolve() - # Direct file path - if path.is_file(): - return str(path) - file_name = f"mpas_analysis_{identifier}.cfg" - if not path.is_dir(): - raise FileNotFoundError( - f"MPAS-Analysis run path does not exist: {run_output_dir}" - ) - - candidate_dirs = [ - # User points at zppy output root - path / "post" / "analysis" / "mpas_analysis" / "cfg", - # User points at post/ - path / "analysis" / "mpas_analysis" / "cfg", - # User points at mpas_analysis/ - path / "cfg", - # Fallbacks - path, - ] - - tried: List[str] = [] - for cfg_dir in candidate_dirs: - candidate = cfg_dir / file_name - tried.append(str(candidate)) - if candidate.is_file(): - return str(candidate) - - # Last resort: small bounded search (avoids an expensive full tree walk) - max_depth = 5 - matches: List[Path] = [] - root_depth = len(path.parts) - for root, dirs, files in os.walk(path): - current_depth = len(Path(root).parts) - root_depth - if current_depth > max_depth: - dirs[:] = [] - continue - if file_name in files: - matches.append(Path(root) / file_name) - break - - if matches: - return str(matches[0]) - - raise FileNotFoundError( - "Could not find prior MPAS-Analysis config file for model-vs-model run. " - f"Searched for {file_name} under {run_output_dir}. " - "Common fix: set [mpas_analysis]/reference_data_path to the prior run's " - "zppy output directory (the one containing post/analysis/mpas_analysis/cfg/). " - f"Tried: {tried}" - ) + cfg_dir = path / "post" / "analysis" / "mpas_analysis" / "cfg" + return str(cfg_dir / file_name) def set_subdirs(config: ConfigObj, c: Dict[str, Any]) -> None: From d5dc0b54ffa321ad9335f0a3a29d4de3988050d6 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 4 Feb 2026 05:52:43 -0600 Subject: [PATCH 5/9] Default to using years from previous subsection for ref and test --- docs/source/parameters.rst | 5 ++ .../post.mpas_analysis_model_vs_model.cfg | 4 + zppy/defaults/default.ini | 2 + zppy/mpas_analysis.py | 88 ++++++++++++++++--- 4 files changed, 86 insertions(+), 13 deletions(-) diff --git a/docs/source/parameters.rst b/docs/source/parameters.rst index d803d1c4..3bce30f3 100644 --- a/docs/source/parameters.rst +++ b/docs/source/parameters.rst @@ -83,9 +83,14 @@ MPAS-Analysis comparisons are configured by year ranges, similar to other ``zppy For model-vs-model comparisons, ``zppy`` supports separate year ranges for the test and reference runs: * ``ts_years``, ``climo_years``, ``enso_years`` define the test run year ranges. + If ``test_data_path`` references a prior ``[mpas_analysis]`` subsection using + ``[[subsection]]``, and these values are not provided, zppy uses that + subsection's year ranges instead. * ``ref_ts_years``, ``ref_climo_years``, ``ref_enso_years`` optionally override the reference run year ranges. If a ``ref_*_years`` parameter is not provided, it defaults to the corresponding test year ranges. +If ``reference_data_path`` references a prior ``[mpas_analysis]`` subsection using ``[[subsection]]``, +the defaults come from that subsection's year ranges instead. If a ``ref_*_years`` parameter contains a single range and multiple test ranges are requested, the single reference range is used for each test range. diff --git a/docs/source/post.mpas_analysis_model_vs_model.cfg b/docs/source/post.mpas_analysis_model_vs_model.cfg index 6f0b240c..4363fe42 100644 --- a/docs/source/post.mpas_analysis_model_vs_model.cfg +++ b/docs/source/post.mpas_analysis_model_vs_model.cfg @@ -27,9 +27,13 @@ reference_data_path = # Optional: point at a previous zppy output directory for the test simulation, # if you want MPAS-Analysis to reuse a completed test run as well. +# If set to [[subsection]], zppy will use that subsection's year ranges +# when ts_years/climo_years/enso_years are not provided. # test_data_path = # Optional: override the reference year ranges (defaults to the test ranges). +# If reference_data_path points to a prior [mpas_analysis] subsection using +# the form [[subsection]], these default to that subsection's year ranges. # If you provide a single range, it will be reused for each test range. # ref_ts_years = "451-500", # ref_climo_years = "451-500", diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index ab533e71..69018a5a 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -329,6 +329,8 @@ reference_data_path = string(default="") test_data_path = string(default="") # Optional: allow reference run years to differ from test run years. # If these are left empty, zppy uses the test run's corresponding year sets. +# If reference_data_path points to [[ subsection ]], zppy uses that subsection's +# year sets when these are left empty. # If a single range is provided, it will be replicated for all test year sets. ref_ts_years = string_list(default=list("")) ref_climo_years = string_list(default=list("")) diff --git a/zppy/mpas_analysis.py b/zppy/mpas_analysis.py index 7481c998..2204904d 100644 --- a/zppy/mpas_analysis.py +++ b/zppy/mpas_analysis.py @@ -40,57 +40,103 @@ def mpas_analysis(config: ConfigObj, script_dir: str, existing_bundles, job_ids_ # Track base output directories for previously defined mpas_analysis subsections. prior_subsection_outputs: Dict[str, str] = {} + # Track year sets for previously defined subsections so later tasks can reference them. + prior_subsection_year_sets: Dict[str, Dict[str, List[Tuple[int, int]]]] = {} for c in tasks: dependencies: List[str] = carried_over_dependencies set_subdirs(config, c) + reference_data_path_value = c.get("reference_data_path", "") + reference_subsection = _parse_subsection_reference(reference_data_path_value) reference_data_path = _resolve_subsection_reference( - c.get("reference_data_path", ""), + reference_data_path_value, prior_subsection_outputs, "reference_data_path", ) + test_data_path_value = c.get("test_data_path", "") + test_subsection = _parse_subsection_reference(test_data_path_value) test_data_path = _resolve_subsection_reference( - c.get("test_data_path", ""), + test_data_path_value, prior_subsection_outputs, "test_data_path", ) c["reference_data_path"] = reference_data_path c["test_data_path"] = test_data_path # Loop over year sets - ts_year_sets: List[Tuple[int, int]] = get_years(c["ts_years"]) + ts_years_value = c.get("ts_years", [""]) + ts_year_sets: List[Tuple[int, int]] = get_years(ts_years_value) + if (len(ts_year_sets) == 0) and (test_subsection in prior_subsection_year_sets): + ts_year_sets = prior_subsection_year_sets[test_subsection]["ts"] # Main climo/index year sets default to time series years. + climo_years_value = c.get("climo_years", [""]) + climo_fallback = ts_year_sets + if test_subsection in prior_subsection_year_sets: + if len(get_years(climo_years_value)) == 0: + climo_fallback = prior_subsection_year_sets[test_subsection]["climo"] + climo_year_sets = _resolve_year_sets( - c.get("climo_years", [""]), - fallback=ts_year_sets, + climo_years_value, + fallback=climo_fallback, target_len=len(ts_year_sets), label="climo_years", ) + + enso_years_value = c.get("enso_years", [""]) + enso_fallback = ts_year_sets + if test_subsection in prior_subsection_year_sets: + if len(get_years(enso_years_value)) == 0: + enso_fallback = prior_subsection_year_sets[test_subsection]["enso"] + enso_year_sets = _resolve_year_sets( - c.get("enso_years", [""]), - fallback=ts_year_sets, + enso_years_value, + fallback=enso_fallback, target_len=len(ts_year_sets), label="enso_years", ) # Reference year sets for test-vs-reference (main-vs-control) mode. # If not provided, default to the main run's corresponding year sets. + ref_ts_years_value = c.get("ref_ts_years", [""]) + ref_ts_fallback = ts_year_sets + if reference_subsection in prior_subsection_year_sets: + if len(get_years(ref_ts_years_value)) == 0: + ref_ts_fallback = prior_subsection_year_sets[reference_subsection]["ts"] + ref_ts_year_sets = _resolve_year_sets( - c.get("ref_ts_years", [""]), - fallback=ts_year_sets, + ref_ts_years_value, + fallback=ref_ts_fallback, target_len=len(ts_year_sets), label="ref_ts_years", ) + + ref_climo_years_value = c.get("ref_climo_years", [""]) + ref_climo_fallback = ref_ts_year_sets + if reference_subsection in prior_subsection_year_sets: + if len(get_years(ref_climo_years_value)) == 0: + ref_climo_fallback = prior_subsection_year_sets[reference_subsection][ + "climo" + ] + ref_climo_year_sets = _resolve_year_sets( - c.get("ref_climo_years", [""]), - fallback=ref_ts_year_sets, + ref_climo_years_value, + fallback=ref_climo_fallback, target_len=len(ts_year_sets), label="ref_climo_years", ) + + ref_enso_years_value = c.get("ref_enso_years", [""]) + ref_enso_fallback = ref_ts_year_sets + if reference_subsection in prior_subsection_year_sets: + if len(get_years(ref_enso_years_value)) == 0: + ref_enso_fallback = prior_subsection_year_sets[reference_subsection][ + "enso" + ] + ref_enso_year_sets = _resolve_year_sets( - c.get("ref_enso_years", [""]), - fallback=ref_ts_year_sets, + ref_enso_years_value, + fallback=ref_enso_fallback, target_len=len(ts_year_sets), label="ref_enso_years", ) @@ -216,6 +262,11 @@ def mpas_analysis(config: ConfigObj, script_dir: str, existing_bundles, job_ids_ os.path.expandvars(os.path.expanduser(c["output"])) ) prior_subsection_outputs[c["subsection"]] = output_dir + prior_subsection_year_sets[c["subsection"]] = { + "ts": ts_year_sets, + "climo": climo_year_sets, + "enso": enso_year_sets, + } return existing_bundles @@ -280,6 +331,17 @@ def _resolve_subsection_reference( return prior_subsection_outputs[subsection] +def _parse_subsection_reference(value: str) -> str: + if not value or not isinstance(value, str): + return "" + + match = re.match(r"^\s*\[\[\s*(.+?)\s*\]\]\s*$", value) + if not match: + return "" + + return match.group(1).strip() + + def _resolve_mpas_analysis_config_file(run_output_dir: str, identifier: str) -> str: """ Resolve the MPAS-Analysis config file path for a prior run. From 3294c5619632bf680d48388655db17bc55d08810 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 4 Feb 2026 05:58:00 -0600 Subject: [PATCH 6/9] Reduce complexity of mpas_analysis() This function has been broken into 7 helper functions. --- zppy/mpas_analysis.py | 364 ++++++++++++++++++++++++------------------ 1 file changed, 212 insertions(+), 152 deletions(-) diff --git a/zppy/mpas_analysis.py b/zppy/mpas_analysis.py index 2204904d..b0733604 100644 --- a/zppy/mpas_analysis.py +++ b/zppy/mpas_analysis.py @@ -47,98 +47,19 @@ def mpas_analysis(config: ConfigObj, script_dir: str, existing_bundles, job_ids_ dependencies: List[str] = carried_over_dependencies set_subdirs(config, c) - reference_data_path_value = c.get("reference_data_path", "") - reference_subsection = _parse_subsection_reference(reference_data_path_value) - reference_data_path = _resolve_subsection_reference( - reference_data_path_value, - prior_subsection_outputs, - "reference_data_path", + ( + reference_data_path, + test_data_path, + reference_subsection, + test_subsection, + ) = _resolve_subsection_paths(c, prior_subsection_outputs) + ts_year_sets, climo_year_sets, enso_year_sets = _resolve_test_year_sets( + c, test_subsection, prior_subsection_year_sets ) - test_data_path_value = c.get("test_data_path", "") - test_subsection = _parse_subsection_reference(test_data_path_value) - test_data_path = _resolve_subsection_reference( - test_data_path_value, - prior_subsection_outputs, - "test_data_path", - ) - c["reference_data_path"] = reference_data_path - c["test_data_path"] = test_data_path - # Loop over year sets - ts_years_value = c.get("ts_years", [""]) - ts_year_sets: List[Tuple[int, int]] = get_years(ts_years_value) - if (len(ts_year_sets) == 0) and (test_subsection in prior_subsection_year_sets): - ts_year_sets = prior_subsection_year_sets[test_subsection]["ts"] - - # Main climo/index year sets default to time series years. - climo_years_value = c.get("climo_years", [""]) - climo_fallback = ts_year_sets - if test_subsection in prior_subsection_year_sets: - if len(get_years(climo_years_value)) == 0: - climo_fallback = prior_subsection_year_sets[test_subsection]["climo"] - - climo_year_sets = _resolve_year_sets( - climo_years_value, - fallback=climo_fallback, - target_len=len(ts_year_sets), - label="climo_years", - ) - - enso_years_value = c.get("enso_years", [""]) - enso_fallback = ts_year_sets - if test_subsection in prior_subsection_year_sets: - if len(get_years(enso_years_value)) == 0: - enso_fallback = prior_subsection_year_sets[test_subsection]["enso"] - - enso_year_sets = _resolve_year_sets( - enso_years_value, - fallback=enso_fallback, - target_len=len(ts_year_sets), - label="enso_years", - ) - - # Reference year sets for test-vs-reference (main-vs-control) mode. - # If not provided, default to the main run's corresponding year sets. - ref_ts_years_value = c.get("ref_ts_years", [""]) - ref_ts_fallback = ts_year_sets - if reference_subsection in prior_subsection_year_sets: - if len(get_years(ref_ts_years_value)) == 0: - ref_ts_fallback = prior_subsection_year_sets[reference_subsection]["ts"] - - ref_ts_year_sets = _resolve_year_sets( - ref_ts_years_value, - fallback=ref_ts_fallback, - target_len=len(ts_year_sets), - label="ref_ts_years", - ) - - ref_climo_years_value = c.get("ref_climo_years", [""]) - ref_climo_fallback = ref_ts_year_sets - if reference_subsection in prior_subsection_year_sets: - if len(get_years(ref_climo_years_value)) == 0: - ref_climo_fallback = prior_subsection_year_sets[reference_subsection][ - "climo" - ] - - ref_climo_year_sets = _resolve_year_sets( - ref_climo_years_value, - fallback=ref_climo_fallback, - target_len=len(ts_year_sets), - label="ref_climo_years", - ) - - ref_enso_years_value = c.get("ref_enso_years", [""]) - ref_enso_fallback = ref_ts_year_sets - if reference_subsection in prior_subsection_year_sets: - if len(get_years(ref_enso_years_value)) == 0: - ref_enso_fallback = prior_subsection_year_sets[reference_subsection][ - "enso" - ] - - ref_enso_year_sets = _resolve_year_sets( - ref_enso_years_value, - fallback=ref_enso_fallback, - target_len=len(ts_year_sets), - label="ref_enso_years", + ref_ts_year_sets, ref_climo_year_sets, ref_enso_year_sets = ( + _resolve_reference_year_sets( + c, reference_subsection, prior_subsection_year_sets, ts_year_sets + ) ) for ts, climo, enso, ctrl_ts, ctrl_climo, ctrl_enso in zip( @@ -149,70 +70,16 @@ def mpas_analysis(config: ConfigObj, script_dir: str, existing_bundles, job_ids_ ref_climo_year_sets, ref_enso_year_sets, ): - c["ts_year1"] = ts[0] - c["ts_year2"] = ts[1] - if ("last_year" in c.keys()) and (c["ts_year2"] > c["last_year"]): - continue # Skip this year set - c["climo_year1"] = climo[0] - c["climo_year2"] = climo[1] - if ("last_year" in c.keys()) and (c["climo_year2"] > c["last_year"]): - continue # Skip this year set - c["enso_year1"] = enso[0] - c["enso_year2"] = enso[1] - if ("last_year" in c.keys()) and (c["enso_year2"] > c["last_year"]): - continue # Skip this year set - c["scriptDir"] = script_dir - identifier: str = _get_identifier( - ts_year1=c["ts_year1"], - ts_year2=c["ts_year2"], - climo_year1=c["climo_year1"], - climo_year2=c["climo_year2"], - ) - c["identifier"] = identifier - - ref_identifier: str = _get_identifier( - ts_year1=ctrl_ts[0], - ts_year2=ctrl_ts[1], - climo_year1=ctrl_climo[0], - climo_year2=ctrl_climo[1], - ) - c["ref_identifier"] = ref_identifier - - # Optional: point MPAS-Analysis at already-completed MPAS-Analysis runs - # so it can do a "test vs. reference" ("main vs. control") comparison. - # - # These zppy options are expected to point to a prior zppy run's - # output directory (containing a post/ directory). - # - # For user consistency with other zppy tasks, prefer *_data_path naming. - # These point to a prior zppy run's output directory (or its post/ dir), - # and zppy will locate the MPAS-Analysis config file within it. - # - # We keep *_run_output_dir as backwards-compatible aliases. - c["controlRunConfigFile"] = ( - _resolve_mpas_analysis_config_file(reference_data_path, ref_identifier) - if reference_data_path - else "" - ) - c["mainRunConfigFile"] = ( - _resolve_mpas_analysis_config_file(test_data_path, identifier) - if test_data_path - else "" + if _set_run_years(c, ts, climo, enso): + continue + identifier, ref_identifier = _set_identifiers( + c, script_dir, ctrl_ts, ctrl_climo ) - prefix_suffix: str = ( - f"_ts_{c['ts_year1']:04d}-{c['ts_year2']:04d}_climo_{c['climo_year1']:04d}-{c['climo_year2']:04d}" + _set_run_config_files( + c, reference_data_path, test_data_path, ref_identifier, identifier ) - - # If reference years differ, include them in the script/status prefix to avoid collisions. - if c["controlRunConfigFile"] and (ref_identifier != identifier): - prefix_suffix = f"{prefix_suffix}_vs_ref_{ref_identifier}" - prefix: str - if c["subsection"]: - prefix = f"mpas_analysis_{c['subsection']}{prefix_suffix}" - else: - prefix = f"mpas_analysis{prefix_suffix}" + prefix = _build_prefix(c, ref_identifier, identifier) print(prefix) - c["prefix"] = prefix bash_file, settings_file, status_file = get_file_names(script_dir, prefix) # Check if we can skip because it completed successfully before skip: bool = check_status(status_file) @@ -282,6 +149,199 @@ def _get_identifier( return f"ts_{ts_y1}-{ts_y2}_climo_{clim_y1}-{clim_y2}" +def _resolve_subsection_paths( + c: Dict[str, Any], + prior_subsection_outputs: Dict[str, str], +) -> Tuple[str, str, str, str]: + reference_data_path_value = c.get("reference_data_path", "") + reference_subsection = _parse_subsection_reference(reference_data_path_value) + reference_data_path = _resolve_subsection_reference( + reference_data_path_value, + prior_subsection_outputs, + "reference_data_path", + ) + test_data_path_value = c.get("test_data_path", "") + test_subsection = _parse_subsection_reference(test_data_path_value) + test_data_path = _resolve_subsection_reference( + test_data_path_value, + prior_subsection_outputs, + "test_data_path", + ) + c["reference_data_path"] = reference_data_path + c["test_data_path"] = test_data_path + return reference_data_path, test_data_path, reference_subsection, test_subsection + + +def _resolve_test_year_sets( + c: Dict[str, Any], + test_subsection: str, + prior_subsection_year_sets: Dict[str, Dict[str, List[Tuple[int, int]]]], +) -> Tuple[List[Tuple[int, int]], List[Tuple[int, int]], List[Tuple[int, int]]]: + ts_years_value = c.get("ts_years", [""]) + ts_year_sets: List[Tuple[int, int]] = get_years(ts_years_value) + if (len(ts_year_sets) == 0) and (test_subsection in prior_subsection_year_sets): + ts_year_sets = prior_subsection_year_sets[test_subsection]["ts"] + + climo_years_value = c.get("climo_years", [""]) + climo_fallback = ts_year_sets + if test_subsection in prior_subsection_year_sets: + if len(get_years(climo_years_value)) == 0: + climo_fallback = prior_subsection_year_sets[test_subsection]["climo"] + + climo_year_sets = _resolve_year_sets( + climo_years_value, + fallback=climo_fallback, + target_len=len(ts_year_sets), + label="climo_years", + ) + + enso_years_value = c.get("enso_years", [""]) + enso_fallback = ts_year_sets + if test_subsection in prior_subsection_year_sets: + if len(get_years(enso_years_value)) == 0: + enso_fallback = prior_subsection_year_sets[test_subsection]["enso"] + + enso_year_sets = _resolve_year_sets( + enso_years_value, + fallback=enso_fallback, + target_len=len(ts_year_sets), + label="enso_years", + ) + + return ts_year_sets, climo_year_sets, enso_year_sets + + +def _resolve_reference_year_sets( + c: Dict[str, Any], + reference_subsection: str, + prior_subsection_year_sets: Dict[str, Dict[str, List[Tuple[int, int]]]], + ts_year_sets: List[Tuple[int, int]], +) -> Tuple[List[Tuple[int, int]], List[Tuple[int, int]], List[Tuple[int, int]]]: + ref_ts_years_value = c.get("ref_ts_years", [""]) + ref_ts_fallback = ts_year_sets + if reference_subsection in prior_subsection_year_sets: + if len(get_years(ref_ts_years_value)) == 0: + ref_ts_fallback = prior_subsection_year_sets[reference_subsection]["ts"] + + ref_ts_year_sets = _resolve_year_sets( + ref_ts_years_value, + fallback=ref_ts_fallback, + target_len=len(ts_year_sets), + label="ref_ts_years", + ) + + ref_climo_years_value = c.get("ref_climo_years", [""]) + ref_climo_fallback = ref_ts_year_sets + if reference_subsection in prior_subsection_year_sets: + if len(get_years(ref_climo_years_value)) == 0: + ref_climo_fallback = prior_subsection_year_sets[reference_subsection][ + "climo" + ] + + ref_climo_year_sets = _resolve_year_sets( + ref_climo_years_value, + fallback=ref_climo_fallback, + target_len=len(ts_year_sets), + label="ref_climo_years", + ) + + ref_enso_years_value = c.get("ref_enso_years", [""]) + ref_enso_fallback = ref_ts_year_sets + if reference_subsection in prior_subsection_year_sets: + if len(get_years(ref_enso_years_value)) == 0: + ref_enso_fallback = prior_subsection_year_sets[reference_subsection]["enso"] + + ref_enso_year_sets = _resolve_year_sets( + ref_enso_years_value, + fallback=ref_enso_fallback, + target_len=len(ts_year_sets), + label="ref_enso_years", + ) + + return ref_ts_year_sets, ref_climo_year_sets, ref_enso_year_sets + + +def _set_run_years( + c: Dict[str, Any], + ts: Tuple[int, int], + climo: Tuple[int, int], + enso: Tuple[int, int], +) -> bool: + c["ts_year1"] = ts[0] + c["ts_year2"] = ts[1] + if ("last_year" in c.keys()) and (c["ts_year2"] > c["last_year"]): + return True + c["climo_year1"] = climo[0] + c["climo_year2"] = climo[1] + if ("last_year" in c.keys()) and (c["climo_year2"] > c["last_year"]): + return True + c["enso_year1"] = enso[0] + c["enso_year2"] = enso[1] + if ("last_year" in c.keys()) and (c["enso_year2"] > c["last_year"]): + return True + return False + + +def _set_identifiers( + c: Dict[str, Any], + script_dir: str, + ctrl_ts: Tuple[int, int], + ctrl_climo: Tuple[int, int], +) -> Tuple[str, str]: + c["scriptDir"] = script_dir + identifier = _get_identifier( + ts_year1=c["ts_year1"], + ts_year2=c["ts_year2"], + climo_year1=c["climo_year1"], + climo_year2=c["climo_year2"], + ) + c["identifier"] = identifier + + ref_identifier = _get_identifier( + ts_year1=ctrl_ts[0], + ts_year2=ctrl_ts[1], + climo_year1=ctrl_climo[0], + climo_year2=ctrl_climo[1], + ) + c["ref_identifier"] = ref_identifier + return identifier, ref_identifier + + +def _set_run_config_files( + c: Dict[str, Any], + reference_data_path: str, + test_data_path: str, + ref_identifier: str, + identifier: str, +) -> None: + c["controlRunConfigFile"] = ( + _resolve_mpas_analysis_config_file(reference_data_path, ref_identifier) + if reference_data_path + else "" + ) + c["mainRunConfigFile"] = ( + _resolve_mpas_analysis_config_file(test_data_path, identifier) + if test_data_path + else "" + ) + + +def _build_prefix(c: Dict[str, Any], ref_identifier: str, identifier: str) -> str: + prefix_suffix = ( + f"_ts_{c['ts_year1']:04d}-{c['ts_year2']:04d}" + f"_climo_{c['climo_year1']:04d}-{c['climo_year2']:04d}" + ) + + if c["controlRunConfigFile"] and (ref_identifier != identifier): + prefix_suffix = f"{prefix_suffix}_vs_ref_{ref_identifier}" + if c["subsection"]: + prefix = f"mpas_analysis_{c['subsection']}{prefix_suffix}" + else: + prefix = f"mpas_analysis{prefix_suffix}" + c["prefix"] = prefix + return prefix + + def _resolve_year_sets( years_value: Any, *, From b2ce62a815e4d122fbfb4ec8599f6bf3fc3f5e75 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 4 Feb 2026 06:40:53 -0600 Subject: [PATCH 7/9] Put model vs. model analysis in mpas_analysis_mvm Within that subdirectory, each gets a unique subdirectory. --- docs/source/parameters.rst | 6 +- .../post.mpas_analysis_model_vs_model.cfg | 4 + zppy/defaults/default.ini | 6 + zppy/mpas_analysis.py | 128 +++++++++++++++++- zppy/templates/mpas_analysis.bash | 39 +++--- 5 files changed, 156 insertions(+), 27 deletions(-) diff --git a/docs/source/parameters.rst b/docs/source/parameters.rst index 3bce30f3..f7bc146b 100644 --- a/docs/source/parameters.rst +++ b/docs/source/parameters.rst @@ -73,8 +73,10 @@ For the ``mpas_analysis`` task: ``e3sm_diags`` needs ``reference_data_path`` to be the specific directory containing the reference climatology files (typically under the reference run's ``post/.../clim`` tree), whereas ``mpas_analysis`` needs to find the reference MPAS-Analysis config file. For MPAS-Analysis, ``zppy`` resolves the config file when ``reference_data_path`` points to the prior run's zppy output directory (the one containing ``post/``). - ``reference_data_path`` is intended to point to the prior run's zppy output directory (the one containing ``post/``). ``zppy`` will then use: - ``/post/analysis/mpas_analysis/cfg/mpas_analysis_.cfg``. + ``reference_data_path`` is intended to point to the prior run's zppy output directory (the one containing ``post/``). ``zppy`` will then use: + ``/post/analysis/mpas_analysis/cfg/mpas_analysis_.cfg`` (or ``mpas_analysis_mvm`` if the referenced run was MVM). + + When ``reference_data_path`` is set to a non-subsection path, ``reference_case`` is required so the MVM output directory can include the reference case name. If ``reference_data_path`` is set to ``[[subsection]]``, ``reference_case`` is inferred to be the same as the current ``case``. **MPAS-Analysis model-vs-model year ranges** diff --git a/docs/source/post.mpas_analysis_model_vs_model.cfg b/docs/source/post.mpas_analysis_model_vs_model.cfg index 4363fe42..fe9569cc 100644 --- a/docs/source/post.mpas_analysis_model_vs_model.cfg +++ b/docs/source/post.mpas_analysis_model_vs_model.cfg @@ -25,6 +25,10 @@ enso_years = "1850-2014", # /post/analysis/mpas_analysis/cfg/ reference_data_path = +# Required when reference_data_path is not a [[subsection]]. +# Used to build the MVM output directory name. +reference_case = + # Optional: point at a previous zppy output directory for the test simulation, # if you want MPAS-Analysis to reuse a completed test run as well. # If set to [[subsection]], zppy will use that subsection's year ranges diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index 69018a5a..768a3ce7 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -324,9 +324,14 @@ enso_years = string_list(default=list("")) # or to [[ subsection ]] to refer to a previous mpas_analysis subsection in this workflow. # zppy will use: # /post/analysis/mpas_analysis/cfg/mpas_analysis_.cfg +# (or /post/analysis/mpas_analysis_mvm/cfg/ if the referenced run was MVM) # where matches each MPAS-Analysis sub-run (e.g. ts_1850-2014_climo_1985-2014). reference_data_path = string(default="") test_data_path = string(default="") +# Required when reference_data_path is set to a non-subsection path. +# If reference_data_path is set to [[subsection]], reference_case is inferred +# to be the same as the current case. +reference_case = string(default="") # Optional: allow reference run years to differ from test run years. # If these are left empty, zppy uses the test run's corresponding year sets. # If reference_data_path points to [[ subsection ]], zppy uses that subsection's @@ -361,6 +366,7 @@ walltime = string(default="06:00:00") enso_years = string_list(default=None) reference_data_path = string(default=None) test_data_path = string(default=None) + reference_case = string(default=None) ref_ts_years = string_list(default=None) ref_climo_years = string_list(default=None) ref_enso_years = string_list(default=None) diff --git a/zppy/mpas_analysis.py b/zppy/mpas_analysis.py index b0733604..21e76f56 100644 --- a/zppy/mpas_analysis.py +++ b/zppy/mpas_analysis.py @@ -42,6 +42,10 @@ def mpas_analysis(config: ConfigObj, script_dir: str, existing_bundles, job_ids_ prior_subsection_outputs: Dict[str, str] = {} # Track year sets for previously defined subsections so later tasks can reference them. prior_subsection_year_sets: Dict[str, Dict[str, List[Tuple[int, int]]]] = {} + # Track analysis subdirectories (mpas_analysis vs mpas_analysis_mvm) for subsections. + prior_subsection_analysis_subdirs: Dict[str, str] = {} + # Track identifiers used by MVM runs to avoid cfg name collisions. + mvm_identifiers: Dict[str, str] = {} for c in tasks: @@ -75,8 +79,33 @@ def mpas_analysis(config: ConfigObj, script_dir: str, existing_bundles, job_ids_ identifier, ref_identifier = _set_identifiers( c, script_dir, ctrl_ts, ctrl_climo ) + reference_analysis_subdir = _get_subsection_analysis_subdir( + reference_subsection, prior_subsection_analysis_subdirs + ) + test_analysis_subdir = _get_subsection_analysis_subdir( + test_subsection, prior_subsection_analysis_subdirs + ) _set_run_config_files( - c, reference_data_path, test_data_path, ref_identifier, identifier + c, + reference_data_path, + test_data_path, + ref_identifier, + identifier, + reference_analysis_subdir, + test_analysis_subdir, + ) + _set_output_paths( + c, + reference_data_path, + reference_subsection, + identifier, + ref_identifier, + ) + _check_mvm_identifier_collision( + c, + reference_data_path, + identifier, + mvm_identifiers, ) prefix = _build_prefix(c, ref_identifier, identifier) print(prefix) @@ -122,7 +151,7 @@ def mpas_analysis(config: ConfigObj, script_dir: str, existing_bundles, job_ids_ print(f"...adding to bundle {c['bundle']}") print(f" environment_commands={c['environment_commands']}") - print_url(c, "mpas_analysis") + print_url(c, c.get("analysis_task_name", "mpas_analysis")) if c.get("subsection"): output_dir = os.path.abspath( @@ -134,6 +163,9 @@ def mpas_analysis(config: ConfigObj, script_dir: str, existing_bundles, job_ids_ "climo": climo_year_sets, "enso": enso_year_sets, } + prior_subsection_analysis_subdirs[c["subsection"]] = c.get( + "analysis_subdir", "mpas_analysis" + ) return existing_bundles @@ -313,14 +345,22 @@ def _set_run_config_files( test_data_path: str, ref_identifier: str, identifier: str, + reference_analysis_subdir: str, + test_analysis_subdir: str, ) -> None: c["controlRunConfigFile"] = ( - _resolve_mpas_analysis_config_file(reference_data_path, ref_identifier) + _resolve_mpas_analysis_config_file( + reference_data_path, + ref_identifier, + analysis_subdir=reference_analysis_subdir, + ) if reference_data_path else "" ) c["mainRunConfigFile"] = ( - _resolve_mpas_analysis_config_file(test_data_path, identifier) + _resolve_mpas_analysis_config_file( + test_data_path, identifier, analysis_subdir=test_analysis_subdir + ) if test_data_path else "" ) @@ -342,6 +382,80 @@ def _build_prefix(c: Dict[str, Any], ref_identifier: str, identifier: str) -> st return prefix +def _get_subsection_analysis_subdir( + subsection: str, prior_subsection_analysis_subdirs: Dict[str, str] +) -> str: + if subsection and (subsection in prior_subsection_analysis_subdirs): + return prior_subsection_analysis_subdirs[subsection] + return "mpas_analysis" + + +def _set_output_paths( + c: Dict[str, Any], + reference_data_path: str, + reference_subsection: str, + identifier: str, + ref_identifier: str, +) -> None: + is_mvm = bool(reference_data_path) + if not is_mvm: + c["analysis_subdir"] = "mpas_analysis" + c["analysis_task_name"] = "mpas_analysis" + c["output_dir_name"] = identifier + return + + c["analysis_subdir"] = "mpas_analysis_mvm" + c["analysis_task_name"] = "mpas_analysis_mvm" + + if reference_subsection: + c["reference_case"] = c["case"] + else: + reference_case = c.get("reference_case", "") + if not reference_case: + raise ValueError( + "reference_case must be set when reference_data_path is provided and is not a subsection reference." + ) + c["reference_case"] = reference_case + + case = c["case"] + reference_case = c["reference_case"] + if case == reference_case: + output_dir_name = f"{identifier}_vs_{ref_identifier}" + else: + if identifier == ref_identifier: + output_dir_name = f"{case}_vs_{reference_case}" + else: + output_dir_name = ( + f"{case}_{identifier}_vs_{reference_case}_{ref_identifier}" + ) + + c["output_dir_name"] = output_dir_name + + +def _check_mvm_identifier_collision( + c: Dict[str, Any], + reference_data_path: str, + identifier: str, + mvm_identifiers: Dict[str, str], +) -> None: + if not reference_data_path: + return + + if identifier in mvm_identifiers: + raise ValueError( + "Multiple MPAS-Analysis MVM runs would overwrite the same config file " + f"(mpas_analysis_{identifier}.cfg). This identifier was already used by " + f"{mvm_identifiers[identifier]}. Adjust ts/climo years or split runs." + ) + + ref_case = c.get("reference_case", "") + if ref_case: + label = f"case={c['case']} ref_case={ref_case}" + else: + label = f"case={c['case']}" + mvm_identifiers[identifier] = label + + def _resolve_year_sets( years_value: Any, *, @@ -402,7 +516,9 @@ def _parse_subsection_reference(value: str) -> str: return match.group(1).strip() -def _resolve_mpas_analysis_config_file(run_output_dir: str, identifier: str) -> str: +def _resolve_mpas_analysis_config_file( + run_output_dir: str, identifier: str, *, analysis_subdir: str = "mpas_analysis" +) -> str: """ Resolve the MPAS-Analysis config file path for a prior run. @@ -425,7 +541,7 @@ def _resolve_mpas_analysis_config_file(run_output_dir: str, identifier: str) -> file_name = f"mpas_analysis_{identifier}.cfg" - cfg_dir = path / "post" / "analysis" / "mpas_analysis" / "cfg" + cfg_dir = path / "post" / "analysis" / analysis_subdir / "cfg" return str(cfg_dir / file_name) diff --git a/zppy/templates/mpas_analysis.bash b/zppy/templates/mpas_analysis.bash index ee4c2dde..4fe55fde 100644 --- a/zppy/templates/mpas_analysis.bash +++ b/zppy/templates/mpas_analysis.bash @@ -21,22 +21,23 @@ climY2="{{ '%04d' % (climo_year2) }}" # Job identifier identifier=ts_${tsY1}-${tsY2}_climo_${climY1}-${climY2} +output_dir_name="{{ output_dir_name }}" # Set-up work directory structure echo echo ===== SET UP MPAS-ANALYSIS DIRECTORY STRUCTURE ===== echo -workdir='../analysis/mpas_analysis' +workdir="../analysis/{{ analysis_subdir }}" mkdir -p ${workdir} cd ${workdir} {% if purge == true %} # If purge is on, delete previous directory - rm -rf ${identifier} + rm -rf ${output_dir_name} {% endif %} -mkdir -p ${identifier} +mkdir -p ${output_dir_name} mkdir -p cfg {% if cache == true %} @@ -45,13 +46,13 @@ cached=( "timeseries/moc" "timeseries/OceanBasins" "timeseries/transport" ) mkdir -p cache for subdir in "${cached[@]}" do - mkdir -p cache/${subdir} ${identifier}/${subdir} - rsync -av cache/${subdir}/ ${identifier}/${subdir}/ + mkdir -p cache/${subdir} ${output_dir_name}/${subdir} + rsync -av cache/${subdir}/ ${output_dir_name}/${subdir}/ done files=( "mpasIndexOcean.nc" "mpasTimeSeriesOcean.nc" "seaIceAreaVolNH.nc" "seaIceAreaVolSH.nc") for file in "${files[@]}" do - cp cache/timeseries/${file} ${identifier}/timeseries/${file} + cp cache/timeseries/${file} ${output_dir_name}/timeseries/${file} done {% endif %} @@ -198,7 +199,7 @@ mpasMeshName = {{ mesh }} # directory where analysis should be written # NOTE: This directory path must be specific to each test case. -baseDirectory = {{ scriptDir }}/${workdir}/${identifier} +baseDirectory = {{ scriptDir }}/${workdir}/${output_dir_name} # provide an absolute path to put HTML in an alternative location (e.g. a web # portal) @@ -299,8 +300,8 @@ if [ $? != 0 ]; then fi # Check master log for obvious errors -size=`wc -c ${identifier}/logs/taskProgress.log | awk '{print $1}'` -error=`grep ERROR ${identifier}/logs/taskProgress.log | wc -l` +size=`wc -c ${output_dir_name}/logs/taskProgress.log | awk '{print $1}'` +error=`grep ERROR ${output_dir_name}/logs/taskProgress.log | wc -l` if [ "${size}" = "" ] || [ "${size}" = "0" ] || [ "${error}" != "0" ];then echo 'ERROR (2)' > {{ scriptDir }}/{{ prefix }}.status exit 2 @@ -313,14 +314,14 @@ echo ===== CACHE OUTPUT FILES ===== echo for file in "${files[@]}" do - cp ${identifier}/timeseries/${file} cache/timeseries/${file} + cp ${output_dir_name}/timeseries/${file} cache/timeseries/${file} done for subdir in "${cached[@]}" do - rsync -av ${identifier}/${subdir}/ cache/${subdir}/ + rsync -av ${output_dir_name}/${subdir}/ cache/${subdir}/ done # Remove one particularly large file which does not need to be cached -rm ${identifier}/timeseries/mpasTimeSeriesSeaIce.nc +rm ${output_dir_name}/timeseries/mpasTimeSeriesSeaIce.nc {% endif %} # Copy output to web server @@ -329,7 +330,7 @@ echo ===== COPY FILES TO WEB SERVER ===== echo # Create top-level directory -f=${www}/${case}/mpas_analysis/${identifier}/ +f=${www}/${case}/{{ analysis_subdir }}/${output_dir_name}/ mkdir -p ${f} if [ $? != 0 ]; then echo 'ERROR (3)' > {{ scriptDir }}/{{ prefix }}.status @@ -351,7 +352,7 @@ done {% endif %} # Copy files -rsync -a --delete ${identifier}/html/ ${www}/${case}/mpas_analysis/${identifier}/ +rsync -a --delete ${output_dir_name}/html/ ${www}/${case}/{{ analysis_subdir }}/${output_dir_name}/ if [ $? != 0 ]; then echo 'ERROR (4)' > {{ scriptDir }}/{{ prefix }}.status exit 4 @@ -359,16 +360,16 @@ fi {% if machine in ['pm-cpu', 'pm-gpu'] %} # For NERSC, change permissions of new files -pushd ${www}/${case}/mpas_analysis/ -chgrp -R e3sm ${identifier} -chmod -R go+rX,go-w ${identifier} +pushd ${www}/${case}/{{ analysis_subdir }}/ +chgrp -R e3sm ${output_dir_name} +chmod -R go+rX,go-w ${output_dir_name} popd {% endif %} {% if machine in ['anvil', 'chrysalis'] %} # For LCRC, change permissions of new files -pushd ${www}/${case}/mpas_analysis/ -chmod -R go+rX,go-w ${identifier} +pushd ${www}/${case}/{{ analysis_subdir }}/ +chmod -R go+rX,go-w ${output_dir_name} popd {% endif %} From 86a1a2e8de35009e4b17e3ee8315f1782378bbd0 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 4 Feb 2026 06:52:17 -0600 Subject: [PATCH 8/9] Use same cache directory for model vs obs and model vs model --- zppy/templates/mpas_analysis.bash | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/zppy/templates/mpas_analysis.bash b/zppy/templates/mpas_analysis.bash index 4fe55fde..d37d9004 100644 --- a/zppy/templates/mpas_analysis.bash +++ b/zppy/templates/mpas_analysis.bash @@ -43,16 +43,17 @@ mkdir -p cfg {% if cache == true %} # Restore cached copies of pre-computed files cached=( "timeseries/moc" "timeseries/OceanBasins" "timeseries/transport" ) -mkdir -p cache +cache_dir="{{ scriptDir }}/../analysis/mpas_analysis/cache" +mkdir -p ${cache_dir} for subdir in "${cached[@]}" do - mkdir -p cache/${subdir} ${output_dir_name}/${subdir} - rsync -av cache/${subdir}/ ${output_dir_name}/${subdir}/ + mkdir -p ${cache_dir}/${subdir} ${output_dir_name}/${subdir} + rsync -av ${cache_dir}/${subdir}/ ${output_dir_name}/${subdir}/ done files=( "mpasIndexOcean.nc" "mpasTimeSeriesOcean.nc" "seaIceAreaVolNH.nc" "seaIceAreaVolSH.nc") for file in "${files[@]}" do - cp cache/timeseries/${file} ${output_dir_name}/timeseries/${file} + cp ${cache_dir}/timeseries/${file} ${output_dir_name}/timeseries/${file} done {% endif %} @@ -314,11 +315,11 @@ echo ===== CACHE OUTPUT FILES ===== echo for file in "${files[@]}" do - cp ${output_dir_name}/timeseries/${file} cache/timeseries/${file} + cp ${output_dir_name}/timeseries/${file} ${cache_dir}/timeseries/${file} done for subdir in "${cached[@]}" do - rsync -av ${output_dir_name}/${subdir}/ cache/${subdir}/ + rsync -av ${output_dir_name}/${subdir}/ ${cache_dir}/${subdir}/ done # Remove one particularly large file which does not need to be cached rm ${output_dir_name}/timeseries/mpasTimeSeriesSeaIce.nc From 188af0d2d3cedc4b8ec6dfe3135c61d8fd229014 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 5 Mar 2026 13:07:58 -0600 Subject: [PATCH 9/9] Add mpas_analysis mvm to test cfg --- .../template_weekly_comprehensive_v3.cfg | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/integration/template_weekly_comprehensive_v3.cfg b/tests/integration/template_weekly_comprehensive_v3.cfg index f22a3a93..fac286cb 100644 --- a/tests/integration/template_weekly_comprehensive_v3.cfg +++ b/tests/integration/template_weekly_comprehensive_v3.cfg @@ -201,17 +201,28 @@ tc_obs = "#expand diagnostics_base_path#/observations/Atm/tc-analysis/" [mpas_analysis] active = #expand active_mpas_analysis# anomalyRefYear = 1985 -climo_years = "1985-1989", "1990-1995", -enso_years = "1985-1989", "1990-1995", environment_commands = "#expand mpas_analysis_environment_commands#" mesh = "IcoswISC30E3r5" parallelTaskCount = 6 partition = "#expand partition_long#" qos = "#expand qos_long#" shortTermArchive = True -ts_years = "1985-1989", "1985-1995", walltime = "#expand mpas_analysis_walltime#" + [[ reference ]] + ts_years = "1985-1989", + climo_years = "1985-1989", + enso_years = "1985-1989", + + [[ test ]] + ts_years = "1985-1995", + climo_years = "1990-1995", + enso_years = "1990-1995", + + [[ mvm ]] + reference_data_path = [[ reference ]] + test_data_path = [[ test ]] + [global_time_series] active = #expand active_global_time_series# climo_years = "1985-1989", "1990-1995",