feat(config): migrate mizuRoute configuration parsing to native TOML format

.gitmodules

@@ -32,3 +32,9 @@ fxtag = pio2_6_6
 fxrequired = ToplevelRequired
 # Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/NCAR/ParallelIO
+
+[submodule "toml-f"]
+path = externals/toml-f
+url = https://github.com/toml-f/toml-f.git
+fxtag = v0.5.2
+fxrequired = Required

cime_config/buildlib

@@ -3,7 +3,7 @@
 """
 build mizuRoute library
 """
-import sys, os
+import sys, os, glob, shutil
 
 _CIMEROOT = os.environ.get("CIMEROOT")
 if _CIMEROOT is None:
@@ -36,7 +36,70 @@ def _main_func():
 
         expect( driver == "nuopc", "mizuRoute only has a nuopc COMP_INTERFACE" )
         #-------------------------------------------------------
-        # create Filepath file
+        # Pre-build external toml-f static library via CMake
+        #-------------------------------------------------------
+        tomlf_lib = os.path.join(libroot, "libtomlf.a")
+        tomlf_bld = os.path.join(bldroot, "tomlf_bld")
+        tomlf_inst = os.path.join(bldroot, "tomlf_inst")
+
+        if not os.path.isfile(tomlf_lib):
+            incroot = case.get_value("INCROOT")
+            srcroot = case.get_value("SRCROOT")
+
+            tomlf_src = os.path.join(rof_root, "externals", "toml-f")
+            if not os.path.exists(os.path.join(tomlf_src, "CMakeLists.txt")):
+                tomlf_src = os.path.join(rof_root, "..", "externals", "toml-f")
+            if not os.path.exists(os.path.join(tomlf_src, "CMakeLists.txt")):
+                tomlf_src = os.path.join(srcroot, "externals", "toml-f")
+
+            expect(os.path.exists(os.path.join(tomlf_src, "CMakeLists.txt")),
+                   "Could not find toml-f CMakeLists.txt at %s" % tomlf_src)
+
+            if not os.path.isdir(tomlf_bld):
+                os.makedirs(tomlf_bld)
+
+            fc = os.environ.get("FC") or os.environ.get("MPIFC") or os.environ.get("SFC")
+            fflags = os.environ.get("FFLAGS", "")
+            macfile = os.path.join(caseroot, "Macros.make")
+            if os.path.isfile(macfile):
+                with open(macfile, "r") as mf:
+                    for line in mf:
+                        sline = line.strip()
+                        if sline.startswith("MPIFC :=") and not fc:
+                            fc = sline.split(":=")[-1].strip()
+                        elif sline.startswith("SFC :=") and not fc:
+                            fc = sline.split(":=")[-1].strip()
+                        elif sline.startswith("FFLAGS :=") and not fflags:
+                            fflags = sline.split(":=")[-1].strip()
+            if not fc:
+                fc = "ftn"
+
+            if not os.path.isfile(os.path.join(tomlf_bld, "Makefile")):
+                cmake_cmd = 'cmake -DCMAKE_Fortran_COMPILER="{}" -DCMAKE_Fortran_FLAGS="{}" -DCMAKE_INSTALL_PREFIX="{}" "{}"'.format(
+                    fc, fflags, tomlf_inst, tomlf_src
+                )
+                rc_cmake, out_cmake, err_cmake = run_cmd(cmake_cmd, from_dir=tomlf_bld)
+                logger.info("CMake configure toml-f:\n output:\n%s\n err:\n%s\n" % (out_cmake, err_cmake))
+                expect(rc_cmake == 0, "CMake configure for toml-f failed with rc=%s:\n%s" % (rc_cmake, err_cmake))
+
+            make_cmd = "{} -j {} && {} install".format(gmake, gmake_j, gmake)
+            rc_make, out_make, err_make = run_cmd(make_cmd, from_dir=tomlf_bld)
+            logger.info("Make toml-f:\n output:\n%s\n err:\n%s\n" % (out_make, err_make))
+            expect(rc_make == 0, "Make for toml-f failed with rc=%s:\n%s" % (rc_make, err_make))
+
+            mod_files = glob.glob(os.path.join(tomlf_bld, "**", "*.mod"), recursive=True) + \
+                        glob.glob(os.path.join(tomlf_inst, "**", "*.mod"), recursive=True)
+            for mod in mod_files:
+                shutil.copy(mod, incroot)
+                shutil.copy(mod, libroot)
+
+            compiled_libs = glob.glob(os.path.join(tomlf_bld, "**", "libtoml*.a"), recursive=True) + \
+                            glob.glob(os.path.join(tomlf_inst, "**", "libtoml*.a"), recursive=True)
+            expect(len(compiled_libs) > 0, "Could not find compiled toml-f library in %s or %s" % (tomlf_bld, tomlf_inst))
+            shutil.copy(compiled_libs[0], tomlf_lib)
+
+        #-------------------------------------------------------
+        # create Filepath file for mizuRoute
         #-------------------------------------------------------
         filepath_file = os.path.join(bldroot,"Filepath")
         if not os.path.isfile(filepath_file):
@@ -63,6 +126,23 @@ def _main_func():
         logger.info("%s: \n\n output:\n %s \n\n err:\n\n%s\n"%(cmd,out,err))
         expect(rc == 0, "Command %s failed with rc=%s" % (cmd, rc))
 
+        #-------------------------------------------------------
+        # Merge libtomlf.a object files into librof.a via MRI script
+        #-------------------------------------------------------
+        ar = case.get_value("AR") or "ar"
+        mri_script = os.path.join(tomlf_bld, "merge.mri")
+        with open(mri_script, "w") as f:
+            f.write("CREATE {}\n".format(complib))
+            f.write("ADDLIB {}\n".format(complib))
+            f.write("ADDLIB {}\n".format(tomlf_lib))
+            f.write("SAVE\n")
+            f.write("END\n")
+
+        ar_cmd = "{} -M < {}".format(ar, mri_script)
+        rc_ar, out_ar, err_ar = run_cmd(ar_cmd)
+        logger.info("Merging libtomlf.a into librof.a via MRI script:\n output:\n%s\n err:\n%s\n" % (out_ar, err_ar))
+        expect(rc_ar == 0, "Merge of libtomlf.a into librof.a failed with rc=%s:\n%s" % (rc_ar, err_ar))
+
 ###############################################################################
 
 if __name__ == "__main__":

cime_config/buildnml

@@ -242,25 +242,51 @@ def _create_control_files(case, caseroot, srcroot, confdir, inst_string, infile,
             fname_state_in = "empty"
 
     ctl.set( "fname_state_in", fname_state_in )
-    if fname_state_in is not "empty":
+    if fname_state_in != "empty":
         nmlgen.set_value( "fname_state_in", value=os.path.join( ancil_dir, fname_ntopOld ) )
 
-    # Read in the user control file for the case and change settings to it
-    file_src = "user_nl_mizuroute_control"
-    user_ctl_file = os.path.join(caseroot, file_src + inst_string)
-    if ( not os.path.exists( user_ctl_file ) ):
-       safe_copy( os.path.join( srcroot, "cime_config", file_src), user_ctl_file )
-    usrctl = mizuRoute_control()
-    usrctl.read( user_ctl_file, allowEmpty=True )
-    for element in usrctl.get_elmList():
-       value = ctl.get( element )
-       expect( value != "UNSET", "Element in the user_nl_mizuroute_control file is NOT in the control file: "+element )
-       ctl.set( element, usrctl.get( element ) )
+    # Check component defaults file in cime_config
+    default_file_toml = os.path.join(srcroot, "cime_config", "user_nl_mizuroute_toml")
+    default_file_control = os.path.join(srcroot, "cime_config", "user_nl_mizuroute_control")
+    default_file_plain = os.path.join(srcroot, "cime_config", "user_nl_mizuroute")
+
+    has_def_toml = os.path.exists(default_file_toml) or os.path.exists(default_file_plain)
+    has_def_control = os.path.exists(default_file_control)
+
+    expect(not (has_def_toml and has_def_control),
+           "Both TOML and control format default template files exist in cime_config. Exactly one must be present.")
+    expect(has_def_toml or has_def_control,
+           "Neither user_nl_mizuroute_toml nor user_nl_mizuroute_control default file exists in cime_config.")
+
+    # Check user override files in caseroot
+    user_file_toml = os.path.join(caseroot, "user_nl_mizuroute_toml" + inst_string)
+    user_file_control = os.path.join(caseroot, "user_nl_mizuroute_control" + inst_string)
+    user_file_plain = os.path.join(caseroot, "user_nl_mizuroute" + inst_string)
+
+    has_user_toml = os.path.exists(user_file_toml)
+    has_user_control = os.path.exists(user_file_control)
+
+    expect(not (has_user_toml and has_user_control),
+           "Both TOML and control format user override files exist in CASEROOT. Please use only one format.")
+
+    usrctl = None
+    if has_user_toml:
+        usrctl = mizuRoute_control.from_toml( user_file_toml, allowEmpty=True )
+    elif has_user_control:
+        usrctl = mizuRoute_control.from_control( user_file_control, allowEmpty=True )
+    elif os.path.exists(user_file_plain):
+        usrctl = mizuRoute_control.from_toml( user_file_plain, allowEmpty=True )
+
+    if usrctl is not None:
+        for element in usrctl.get_elmList():
+            value = usrctl.get( element )
+            expect( ctl.get(element) != "UNSET", "Element in the user control file is NOT in the template control file: "+element )
+            ctl.set( element, value )
 
     #----------------------------------------------------
     # Write output files
     #----------------------------------------------------
-    control_file = os.path.join(confdir, "mizuRoute.control")
+    control_file = os.path.join(confdir, "mizuroute_toml")
     nml_file = os.path.join(confdir, "mizuRoute_in")
     write_nml_in_file(case, nmlgen, confdir, nml_file )
     ctl.write( control_file )
@@ -296,9 +322,8 @@ def buildnml(case, caseroot, compname):
     #----------------------------------------------------
     # Construct the control file generator
     #----------------------------------------------------
-    sampleFile = srcroot + "/route/settings/SAMPLE-coupled.control"
-    ctl = mizuRoute_control()
-    ctl.read( sampleFile )
+    sampleFile = srcroot + "/route/settings/SAMPLE-coupled_toml"
+    ctl = mizuRoute_control.from_toml( sampleFile )
 
     #----------------------------------------------------
     # Do some checking
@@ -334,7 +359,7 @@ def buildnml(case, caseroot, compname):
     # copy control files to rundir
     if os.path.isdir(rundir):
        for destdir in [rundir, caseroot+"/CaseDocs"]:
-          for nfile in ["mizuRoute.control", "mizuRoute_in" ]:
+          for nfile in ["mizuroute_toml", "mizuRoute_in" ]:
              file_src = os.path.join(confdir, nfile )
              file_dest = os.path.join(destdir, nfile )
              if inst_string:

cime_config/test/runbuildnml

@@ -1,5 +1,6 @@
 #!/bin/bash
-# Run the buildnmal for mizuRoute, assing it's under a CTSM or CESM checkout
+# Run buildnml for mizuRoute under a CTSM or CESM checkout
+
 cd ../../../../cime >& /dev/null
 if [ $? != 0 ]; then
    echo "cime directory does not exist where expected"
@@ -9,24 +10,48 @@ export CIMEROOT=`pwd`
 echo "CIMEROOT = $CIMEROOT"
 
 cd -
-cp ../user_nl_* .
-mkdir CaseDocs
-echo "Run the help option"
+mkdir -p CaseDocs
+
+echo "1. Run the help option"
 ../buildnml --help > /dev/null
 if [ $? != 0 ] ; then
-   echo "test FAIL"
+   echo "help test FAIL"
    exit -1
 fi
-echo "Try a simple test"
+
+echo "2. Test TOML user control file (user_nl_mizuroute_toml)"
+rm -rf user_* Buildconf/mizurouteconf/* CaseDocs/*
+touch user_nl_mizuroute
+cp ../user_nl_mizuroute_toml user_nl_mizuroute_toml
+echo "newFileFrequency = \"monthly\"" >> user_nl_mizuroute_toml
 ../buildnml `pwd` --verbose
 if [ $? != 0 ] ; then
-   echo "test FAIL"
+   echo "TOML test FAIL"
+   exit -1
+fi
+
+echo "3. Test Legacy user control file (user_nl_mizuroute_control)"
+rm -rf user_* Buildconf/mizurouteconf/* CaseDocs/*
+touch user_nl_mizuroute
+echo "<newFileFrequency>     monthly  ! Test comment" > user_nl_mizuroute_control
+../buildnml `pwd` --verbose
+if [ $? != 0 ] ; then
+   echo "Legacy control test FAIL"
+   exit -1
+fi
+
+echo "4. Test Dual-Format conflict detection (expect failure when both exist)"
+rm -rf user_* Buildconf/mizurouteconf/* CaseDocs/*
+touch user_nl_mizuroute
+cp ../user_nl_mizuroute_toml user_nl_mizuroute_toml
+echo "<newFileFrequency>     monthly  ! Test comment" > user_nl_mizuroute_control
+../buildnml `pwd` --verbose >& /dev/null
+if [ $? == 0 ] ; then
+   echo "Dual-format conflict test FAIL (should have thrown error when both formats exist)"
    exit -1
 else
-   echo "Cat the results...."
-   cat Buildconf/mizurouteconf/mizuRoute*
-   echo "input_data_list..."
-   cat Buildconf/mizuroute.input_data_list
+   echo "Dual-format conflict test PASSED (correctly rejected ambiguous dual formats)"
 fi
+
 rm -rf user_* run/* Buildconf/mizurouteconf/* Buildconf/* CaseDocs
-echo "Successfully ran test"
+echo "Successfully ran all buildnml tests"

cime_config/user_nl_mizuroute_toml

@@ -0,0 +1,13 @@
+#----------------------------------------------------------------------------------
+# This is for changes to the mizuRoute control file only
+# Changes to the namelist file need to go in the user_nl_mizuRoute file
+#
+# See ../route/settings/SAMPLE-coupled_toml
+#
+# Here are some examples (remove the leading #):
+#
+# newFileFrequency = "monthly" # Ending comment
+# route_opt = 2 # Ending comment
+# doesAccumRunoff = 1 # Ending comment
+#
+#----------------------------------------------------------------------------------

route/build/Makefile

@@ -155,8 +155,8 @@ MOD_PATH = $(F_MASTER)build/
 EXE_PATH = $(F_MASTER)bin
 
 # External libraries (if used)
-EXTLIBS =
-EXTINCLUDES =
+EXTLIBS = $(F_MASTER)../externals/toml-f/_install/lib/libtoml-f.a
+EXTINCLUDES = -I$(F_MASTER)../externals/toml-f/_install/include/toml-f/modules
 
 #========================================================================
 # Assemble all of the sub-routines
@@ -274,6 +274,14 @@ ifdef PNETCDF_PATH
         LDFLAGS    += -L$(PNETCDF_PATH)/lib -lpnetcdf
 endif
 
+TOMLLIBDIR = $(F_MASTER)../externals/toml-f/_install
+
+$(TOMLLIBDIR)/lib/libtoml-f.a:
+	@mkdir -p $(F_MASTER)../externals/toml-f/_build
+	cd $(F_MASTER)../externals/toml-f/_build && \
+	cmake -DCMAKE_Fortran_COMPILER=$(FC_EXE) -DCMAKE_INSTALL_PREFIX=$(TOMLLIBDIR) .. && \
+	$(MAKE) && $(MAKE) install
+
 $(PIOLIB):
 	cd $(LIBDIR); \
 	$(MAKE) $(MFLAGS) F_MASTER=$(F_MASTER) FC=$(FC_EXE) FC_EXE=$(FC_EXE) FLAGS="$(FLAGS)" \

route/build/cpl/RtmVar.F90

@@ -40,7 +40,7 @@ MODULE RtmVar
   integer,           public            :: rtmhist_ndens  = 1                        ! namelist: output density of netcdf history files
   integer,           public            :: rtmhist_mfilt  = 30                       ! namelist: number of time samples per tape
   integer,           public            :: rtmhist_nhtfrq = 0                        ! namelist: history write freq(0=monthly)
-  character(len=256),public            :: cfile_name     = 'mizuRoute.control'
+  character(len=256),public            :: cfile_name     = 'mizuroute_toml'
   character(len=256),public            :: para_xxxx      = 'mizuRoute_in'
   ! Miscellaneous variables
   logical,           public            :: barrier_timers = .false.       ! barrier timers

route/build/lib/Makefile

@@ -91,6 +91,7 @@ cleanpiolib:
 .PHONY : cleanpiolib
 
 $(PIOLIBMAKE):
+	mkdir -p $(PIOLIBDIR); \
 	cd $(PIOLIBDIR); \
 	$(CMAKE_ENV_VARS) cmake $(CMAKE_OPTS) $(MODEARGS) $(PIO2DIR)