feat(compare_namelists): add TOML support for compare_namelists and mizuRoute baseline comparison

CIME/case/case_cmpgen_namelists.py

@@ -17,6 +17,10 @@
 
 
 def _do_full_nl_comp(case, test, compare_name, baseline_root=None):
+    """Compare nearly all files between test case dir and baseline dir.
+    Special-case comparators for runconfig, namelists, and toml; the rest fall back to simple line-by-line diffing, see CIME/simple_compare.py.
+    If baseline_root is None, the root is extracted from the case BASELINE_ROOT setting.
+    """
     test_dir = case.get_value("CASEROOT")
     casedoc_dir = os.path.join(test_dir, "CaseDocs")
     baseline_root = (

CIME/compare_namelists.py

@@ -813,6 +813,24 @@ def _compare_yamls(gold_file, compare_file, case):
     return normalized_dict_compare("top-level", data1, data2, case)
 
 
+###############################################################################
+def _compare_tomls(gold_file, compare_file, case):
+    ###############################################################################
+    """
+    Compare contents of two TOML files
+    """
+    try:
+        import tomllib
+    except ImportError:
+        import tomli as tomllib
+
+    with open(gold_file, "rb") as f1, open(compare_file, "rb") as f2:
+        data1 = tomllib.load(f1)
+        data2 = tomllib.load(f2)
+
+    return normalized_dict_compare("top-level", data1, data2, case)
+
+
 ###############################################################################
 def compare_namelist_files(gold_file, compare_file, case=None):
     ###############################################################################
@@ -824,6 +842,8 @@ def compare_namelist_files(gold_file, compare_file, case=None):
 
     if gold_file.endswith(".yaml") or gold_file.endswith(".yml"):
         comments = _compare_yamls(gold_file, compare_file, case)
+    elif gold_file.endswith("_toml"):
+        comments = _compare_tomls(gold_file, compare_file, case)
     else:
         gold_namelists = _parse_namelists(open(gold_file, "r").readlines(), gold_file)
         comp_namelists = _parse_namelists(

CIME/simple_compare.py

@@ -59,9 +59,29 @@ def _skip_comments_and_whitespace(lines, idx):
 
 
 ###############################################################################
-def _compare_data(gold_lines, comp_lines, case, offset_method=False):
+def compare_lines_generic(gold_lines, comp_lines, case_id, try_realign=False):
     ###############################################################################
     """
+    Compare lines in order, filtering #! comment lines and leading/trailing whitepace.
+
+    Expected Input Format:
+        gold_lines (list[str]): Lines of text from the gold/baseline file (e.g. readlines()).
+        comp_lines (list[str]): Lines of text from the file being compared against gold.
+        case_id (str | None): Optional case base ID string used to normalize case IDs/timestamps.
+        try_realign (bool): If True, advances the index of the longer line list on mismatch 
+                            to attempt realigning shifted lines. Default is False.
+
+    Returns:
+        tuple[str, int]: A tuple of (diff_desc, mismatch_count), where diff_desc contains diff 
+                         output describing inequivalent, extra, or missing lines, and 
+                         mismatch_count is the integer count of line mismatches.
+
+    Note:
+        This comparison only strips leading and trailing whitespace per line (.strip()) and skips 
+        lines that start with '#' or '!'. It is NOT fully whitespace-agnostic: internal spacing 
+        differences (e.g., between key and value), inline comments at line ends, and line-order 
+        changes will trigger mismatch errors.
+
     >>> teststr = '''
     ... data1
     ... data2 data3
@@ -70,7 +90,7 @@ def _compare_data(gold_lines, comp_lines, case, offset_method=False):
     ... # Comment
     ... data7 data8 data9 data10
     ... '''
-    >>> _compare_data(teststr.splitlines(), teststr.splitlines(), None)
+    >>> compare_lines_generic(teststr.splitlines(), teststr.splitlines(), None)
     ('', 0)
 
     >>> teststr2 = '''
@@ -80,7 +100,7 @@ def _compare_data(gold_lines, comp_lines, case, offset_method=False):
     ... data7 data8 data9 data10
     ... data00
     ... '''
-    >>> results,_ = _compare_data(teststr.splitlines(), teststr2.splitlines(), None)
+    >>> results,_ = compare_lines_generic(teststr.splitlines(), teststr2.splitlines(), None)
     >>> print(results)
     Inequivalent lines data2 data3 != data2 data30
       NORMALIZED: data2 data3 != data2 data30
@@ -93,7 +113,7 @@ def _compare_data(gold_lines, comp_lines, case, offset_method=False):
     ... data7 data8 data9 data10
     ... data00
     ... '''
-    >>> results,_ = _compare_data(teststr3.splitlines(), teststr2.splitlines(), None, offset_method=True)
+    >>> results,_ = compare_lines_generic(teststr3.splitlines(), teststr2.splitlines(), None, try_realign=True)
     >>> print(results)
     Inequivalent lines data4 data5 data6 != data2 data30
       NORMALIZED: data4 data5 data6 != data2 data30
@@ -124,15 +144,15 @@ def _compare_data(gold_lines, comp_lines, case, offset_method=False):
         comp_value = comp_lines[cidx].strip()
         comp_value = comp_value.replace('"', "'")
 
-        norm_gold_value = _normalize_string_value(gold_value, case)
-        norm_comp_value = _normalize_string_value(comp_value, case)
+        norm_gold_value = _normalize_string_value(gold_value, case_id)
+        norm_comp_value = _normalize_string_value(comp_value, case_id)
         if norm_gold_value != norm_comp_value:
             comments += "Inequivalent lines {} != {}\n".format(gold_value, comp_value)
             comments += "  NORMALIZED: {} != {}\n".format(
                 norm_gold_value, norm_comp_value
             )
             cnt += 1
-        if offset_method and (norm_gold_value != norm_comp_value):
+        if try_realign and (norm_gold_value != norm_comp_value):
             if gnum > cnum:
                 gidx += 1
             else:
@@ -148,22 +168,22 @@ def _compare_data(gold_lines, comp_lines, case, offset_method=False):
 def compare_files(gold_file, compare_file, case=None):
     ###############################################################################
     """
-    Returns true if files are the same, comments are returned too:
-    (success, comments)
+    Compare two text files with compare_lines_generic algorithm.
     """
     expect(os.path.exists(gold_file), "File not found: {}".format(gold_file))
     expect(os.path.exists(compare_file), "File not found: {}".format(compare_file))
 
-    comments, cnt = _compare_data(
+    comments, cnt = compare_lines_generic(
         open(gold_file, "r").readlines(), open(compare_file, "r").readlines(), case
     )
 
+    # If initial comparison finds mismatches, retry with offset realignment to see if line shifting yields fewer errors.
     if cnt > 0:
-        comments2, cnt2 = _compare_data(
+        comments2, cnt2 = compare_lines_generic(
             open(gold_file, "r").readlines(),
             open(compare_file, "r").readlines(),
             case,
-            offset_method=True,
+            try_realign=True,
         )
         if cnt2 < cnt:
             comments = comments2

CIME/tests/test_unit_compare_namelists.py

@@ -0,0 +1,85 @@
+# Unit tests for CIME.compare_namelists TOML comparison functions.
+import unittest
+import tempfile
+import os
+
+from CIME.compare_namelists import compare_namelist_files, is_namelist_file
+
+class TestCompareNamelists(unittest.TestCase):
+    def test_toml_whitespace_handling(self):
+        """Verify that TOML comparison ignores arbitrary whitespace and comments when matching key-value pairs."""
+        gold_toml = """
+[route_opt]
+value = 5
+# a comment
+"""
+        compare_toml = """
+   [ route_opt ]    
+        value      =       5
+"""
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix="_toml") as f1, \
+             tempfile.NamedTemporaryFile(mode="w", delete=False, suffix="_toml") as f2:
+            f1.write(gold_toml)
+            f2.write(compare_toml)
+            f1_name = f1.name
+            f2_name = f2.name
+
+        try:
+            self.assertTrue(is_namelist_file(f1_name))
+            match, comments = compare_namelist_files(f1_name, f2_name)
+            self.assertTrue(match)
+            self.assertEqual(comments, "")
+        finally:
+            os.remove(f1_name)
+            os.remove(f2_name)
+
+    def test_toml_compare_diff(self):
+        """Verify that TOML comparison correctly detects and reports mismatched key-value values."""
+        gold_toml = """
+[route_opt]
+value = 5
+
+[physics]
+method = "IRF"
+"""
+        compare_toml = """
+[route_opt]
+value = 5
+
+[physics]
+method = "MC"
+"""
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix="_toml") as f1, \
+             tempfile.NamedTemporaryFile(mode="w", delete=False, suffix="_toml") as f2:
+            f1.write(gold_toml)
+            f2.write(compare_toml)
+            f1_name = f1.name
+            f2_name = f2.name
+
+        try:
+            match, comments = compare_namelist_files(f1_name, f2_name)
+            self.assertFalse(match)
+            self.assertIn("method had mismatched values", comments)
+        finally:
+            os.remove(f1_name)
+            os.remove(f2_name)
+
+    def test_underscore_toml_extension(self):
+        """Verify TOML files ending in _toml (such as mizuroute_toml) are correctly parsed as TOML namelists."""
+        content = "[route_opt]\nvalue = 5\n"
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix="_toml") as f1, \
+             tempfile.NamedTemporaryFile(mode="w", delete=False, suffix="_toml") as f2:
+            f1.write(content)
+            f2.write(content)
+            f1_name = f1.name
+            f2_name = f2.name
+
+        try:
+            self.assertTrue(is_namelist_file(f1_name))
+            match, comments = compare_namelist_files(f1_name, f2_name)
+            self.assertTrue(match)
+        finally:
+            os.remove(f1_name)
+            os.remove(f2_name)
+
+

setup.py

@@ -13,6 +13,8 @@
     include_package_data=True,
     name="CIME",
     packages=find_packages(),
+    # Python 3.10 will end-of-life in Oct 2026; tomli is required for python_version < '3.11'
+    install_requires=["tomli; python_version < '3.11'"],
     test_suite="CIME.tests",
     tests_requires=["pytest"],
     url="https://github.com/ESMCI/cime",