Add Coweeta watershed example and workflow utilities

.gitmodules

@@ -10,3 +10,7 @@
 	path = inputdata
 	url = git@github.com:rfiorella/pt-e3sm-inputdata
 	branch = compass-glm
+[submodule "watershed_workflow"]
+	path = watershed_workflow
+	url = git@github.com:environmental-modeling-workflows/watershed-workflow
+	branch = ecoon/elm_ats

Docker/Dockerfile-run

@@ -59,5 +59,4 @@ ENV CIMEROOT=/home/amanzi_user/E3SM/cime \
     E3SM_WORK_DIR=/home/amanzi_user/work \
     INPUTDATA_DIR=/home/amanzi_user/compass/inputdata \
     MACHINE_NAME=docker-ats \
-    COMPILER_NAME=gnu \
-    GITHUB_ACTIONS=TRUE
+    COMPILER_NAME=gnu

README.md

@@ -1,3 +1,42 @@
+# Assumptions
+
+This repo is a meta-repo which stores submodules to all the software
+stacks needed to run coupled ELM+ATS simulations.  It is important for
+users to understand the assumptions that go into ELM+ATS.
+
+
+## ELM assumptions
+
+* Only natural / vegetated land surface types are allowed
+* One land type per grid cell, one topo unit per land type, one water column per topo unit -- the heirarchy is collapsed through water columns.
+* Currently one PFT with area fraction 1, all others 0 per water column.  This could eventually be relaxed.
+
+## ATS assumptions
+
+* Meshes are extruded with a uniform number of grid cells.
+* Meshes are pre-partitioned and written with 'one block' option of Watershed Workflow.
+
+## Shared assumptions
+
+* ELM's surfdata and domain files are organized in the same ordering
+  as ATS's columns.  This is satisfied if the above ATS assumptions
+  are met (using WW with pre-partitioning and 'one block').
+* ATS's mesh is the same vertical structure as ELM's mesh -- zi, dz, nlevgrnd
+
+# Who owns what -- design
+
+## ELM owns:
+
+* meteorologic data
+* all vegetation and land surface properties
+* currently (may change) porosity, sand/silt/clay
+
+## ATS owns:
+
+* domain decomposition and mesh
+* permeability and WRMs
+
+
 # Building prototype ELM-ATS:
 
 [![Build Docker Image](https://github.com/amanzi/COMPASS-ELM-ATS/actions/workflows/docker-ci.yml/badge.svg)](https://github.com/amanzi/COMPASS-ELM-ATS/actions/workflows/docker-ci.yml)
@@ -115,6 +154,22 @@ Follow the examples:
   - `USE_ATS=TRUE ./build_example.sh` Runs ELM + ATS
 3) Follow the on-screen instructions to run the case: `cd ${CASE_DIR} && ./case.submit`
 
+# Directory Structure and Input File Workflow
+
+This repo uses multiple directories to track input files for a given `CASE_NAME`. This is convoluted and probably needs to be rethought and streamlined at some point.
+
+1. `./watershed_workflow/examples/CASE_NAME` may include scripts for Watershed Workflow used to generate input files. This is the workflow tool that sets up runs — most "real" runs will use this tool.
+
+2. Once the input data is created, it is typically moved into an `./examples/CASE_NAME` directory. This directory stages the input for E3SM and includes a `build_example.sh` script that runs E3SM/CIME's `create_newcase` and related scripts. Run-specific Watershed Workflow output from step 1 is copied into this directory (ATS xml files, ATS exo files, `user_nl` files — all non-data files).
+
+3. `./inputdata` is E3SM's repo for storing input files. Once met data, surfdata, and domain files are created in step 1, they are moved to subdirectories here.
+
+4. Once `create_newcase` is run (in the `build_example.sh` script), a `CASE_DIR` — typically something like `${CASE_ROOT}/CASE_NAME.npX` — is created. This is E3SM's staging directory for repeated runs. All files from step 2 are copied here and sometimes modified (e.g. via `sed`) to inject mesh names etc. **This is the authoritative location for ATS input XML files used in a run.**
+
+5. `create_newcase` also creates a run directory where the actual runs execute. Few or no input files live there (perhaps namelist files); it is primarily for run output.
+
+**Note:** A file `work/CASE_NAME.ats/run/CASE_NAME.xml` may exist but is **not** used — the XML in the `CASE_DIR` (step 4) is what ATS reads. Do not edit the run-directory copy.
+
 # Examples
 
 This documents and describes the sequence of examples developed in this repo.

examples/build_example.sh

@@ -80,9 +80,19 @@ if [ -z "${GITHUB_ACTIONS}" ]; then
     export GITHUB_ACTIONS=FALSE
 fi
 
+# Debug mode?
+if [ -z "${DEBUG_MODE}" ]; then
+    export DEBUG_MODE=FALSE
+fi
+
 CASE_DIR="${E3SM_WORK_DIR}/cases/${CASE_NAME}.${CASE_SUFFIX}"
 E3SM_SRC_DIR="${ELM_ATS_SRC_DIR}/E3SM"
-SED=gsed
+# Use GNU sed if available (gsed on macOS), otherwise plain sed
+if command -v gsed &> /dev/null; then
+    SED=gsed
+else
+    SED=sed
+fi
 
 # create the case
 echo "Creating case"
@@ -125,6 +135,11 @@ cd ${CASE_DIR}
 # make sure there is a clean endline -- an extra doesn't hurt
 echo "" >> user_nl_elm
 
+# override surfdata if specified
+if [ -n "${SURF_DATA_FILE}" ]; then
+    echo " fsurdat = '\${DIN_LOC_ROOT}/lnd/clm2/surfdata_map/${SURF_DATA_FILE}'" >> user_nl_elm
+fi
+
 # ATS-specific
 if [ "${USE_ATS}" != "FALSE" ]; then
     ${SED} -i "s^MESH_FILENAME^${CASE_DIR}/${DOMAIN_NAME}^g" ${ATS_CASE_NAME}.xml
@@ -136,14 +151,12 @@ if [ "${USE_ATS}" != "FALSE" ]; then
     fi
     echo " ats_inputdir = '${CASE_DIR}'" >> user_nl_elm
     echo " ats_inputfile = '${ATS_CASE_NAME}.xml'" >> user_nl_elm
+    echo " domain_decomp_type = 'ats'" >> user_nl_elm
 fi
 
 
 # ELM
-if [ -z "${INPUTDATA_DIR}" ]; then
-    INPUTDATA_DIR=${E3SM_WORK_DIR}/inputdata
-fi
-./xmlchange MOSART_MODE=NULL,DOUT_S=FALSE,DIN_LOC_ROOT=${INPUTDATA_DIR}
+./xmlchange MOSART_MODE=NULL,DOUT_S=FALSE,DIN_LOC_ROOT=${ELM_ATS_SRC_DIR}/inputdata
 ./xmlchange DIN_LOC_ROOT_CLMFORC=\$DIN_LOC_ROOT/atm/datm7
 ./xmlchange ELM_USRDAT_NAME=${INPUTDATA_NAME}
 
@@ -156,7 +169,7 @@ if [ -z "${DOMAIN_FILE}" ]; then
         if (( ${#matches[@]} == 1 )); then
             file="${matches[0]}"
             DOMAIN_FILE="${file##*/}"
-        elif (( ${#matches[@]} == 0))
+        elif (( ${#matches[@]} == 0)); then
             echo "Cannot find domain file for ${INPUTDATA_NAME} in ${E3SM_WORK_DIR}/inputdata/share/domains/domain.clm"
             exit 1
         else 
@@ -172,7 +185,7 @@ fi
 ./xmlchange LND_DOMAIN_FILE=${DOMAIN_FILE}
 
 # set the number of tasks
-if [ ! -v NTASKS ]; then
+if [ -z "${NTASKS}" ]; then
     NTASKS=1
 fi
 ./xmlchange NTASKS=${NTASKS}
@@ -182,7 +195,7 @@ fi
 ./xmlchange RUN_STARTDATE=2000-07-15
 ./xmlchange STOP_OPTION=nyears,STOP_N=2
 ./xmlchange BATCH_SYSTEM=none
-./xmlchange DEBUG=TRUE
+./xmlchange DEBUG=${DEBUG_MODE}
 
 # setup the case
 echo ""

examples/coweeta/build_example.sh

@@ -0,0 +1,133 @@
+#!/usr/bin/env bash
+
+set -e
+shopt -s nullglob
+
+if [ -z "${USE_ATS}" ]; then
+    echo "Set USE_ATS prior to running the build_example script"
+    exit 1
+fi
+
+if [ ! -v NTASKS ]; then
+    echo "Setting NTASKS = 1"
+    NTASKS=1
+fi
+
+# ----
+# process directory structure
+if [ -z "${ELM_ATS_SRC_DIR}" ]; then
+    echo "Set ELM_ATS_SRC_DIR before running."
+    exit 1
+fi
+if [ -z "${E3SM_WORK_DIR}" ]; then
+    echo "Set E3SM_WORK_DIR before running."
+    exit 1
+fi
+
+NAME=coweeta
+export CASE_NAME=coweeta.np${NTASKS}
+export DOMAIN_FILE=coweeta_domain.nc
+export INPUTDATA_FILE=coweeta_surfdata.nc
+export COMPSET="ICB20TRCNPRDCTCBC"
+
+
+# use ats?
+CASE_SUFFIX=
+if [ -z "${USE_ATS}" ]; then
+    echo "Set USE_ATS before running."
+    exit 1
+elif [ "${USE_ATS}" == "TRUE" ]; then
+    CASE_SUFFIX=ats
+elif [ "${USE_ATS}" == "IC_ONLY" ]; then
+    CASE_SUFFIX=ic_only
+elif [ "${USE_ATS}" == "FALSE" ]; then
+    CASE_SUFFIX=elm
+else
+    echo "USE_ATS must be one of TRUE, IC_ONLY, or FALSE"
+fi
+
+
+
+CASE_DIR="${E3SM_CASE_DIR}/${CASE_NAME}.${CASE_SUFFIX}"
+E3SM_SRC_DIR="${ELM_ATS_SRC_DIR}/E3SM"
+
+
+# create the case
+echo "Creating case"
+echo "----------------------"
+echo "${E3SM_SRC_DIR}/cime/scripts/create_newcase --case ${CASE_DIR} --res ELM_USRDAT --mach ${MACHINE_NAME} --compiler ${COMPILER_NAME} --compset ${COMPSET}"
+echo "----------------------"
+${E3SM_SRC_DIR}/cime/scripts/create_newcase --case ${CASE_DIR} --res ELM_USRDAT --mach ${MACHINE_NAME} --compiler ${COMPILER_NAME} --compset ${COMPSET}
+
+# cp over example files to the case directory
+cp -r ./* ${CASE_DIR}/
+cd ${CASE_DIR}
+
+# fix the exo directory
+gsed -i "s^elm_output_data^${CASE_DIR}^g" ${NAME}.xml
+
+echo " fsurdat = '${CASE_DIR}/${INPUTDATA_FILE}'" >> user_nl_elm
+
+# ATS-specific
+if [ "${USE_ATS}" != "FALSE" ]; then
+    if [ "${USE_ATS}" == "TRUE" ]; then
+	echo " use_ats = .true." >> user_nl_elm
+    else
+	echo " use_ats_ic = .true." >> user_nl_elm
+    fi
+    echo " domain_decomp_type = 'ats'" >> user_nl_elm
+    echo " ats_inputdir = '${CASE_DIR}'" >> user_nl_elm
+    echo " ats_inputfile = '${NAME}.xml'" >> user_nl_elm
+fi
+
+# make sure there is a clean endline -- an extra doesn't hurt
+echo "" >> user_nl_elm
+
+
+
+# ELM
+./xmlchange MOSART_MODE=NULL,DOUT_S=FALSE,DIN_LOC_ROOT=${ELM_ATS_SRC_DIR}/inputdata
+./xmlchange DIN_LOC_ROOT_CLMFORC=${ELM_ATS_SRC_DIR}/inputdata/atm/datm7
+./xmlchange ELM_USRDAT_NAME=${NAME}
+
+# try to find the domain.nc file: these are now example specific
+./xmlchange ATM_DOMAIN_PATH=${CASE_DIR}
+./xmlchange LND_DOMAIN_PATH=${CASE_DIR}
+./xmlchange ATM_DOMAIN_FILE=${DOMAIN_FILE}
+./xmlchange LND_DOMAIN_FILE=${DOMAIN_FILE}
+
+# set the number of tasks
+./xmlchange NTASKS=${NTASKS}
+./xmlchange NTASKS_PER_INST=${NTASKS}
+
+./xmlchange PIO_TYPENAME=netcdf
+./xmlchange RUN_STARTDATE=2020-10-01
+./xmlchange STOP_OPTION=nyears,STOP_N=2
+./xmlchange BATCH_SYSTEM=none
+./xmlchange DEBUG=TRUE
+
+# setup the case
+echo ""
+echo "Running case.setup"
+echo "----------------------"
+./case.setup
+echo -e '\nstring(APPEND CPPDEFS " -DCPL_BYPASS -DUSE_ATS_LIB")' >> cmake_macros/universal.cmake
+
+# build
+echo ""
+echo "Running case.build"
+echo "----------------------"
+./case.build
+
+
+echo ""
+echo "Run the case yourself:"
+echo "----------------------"
+echo "pushd ${CASE_DIR} && ./case.submit"
+if [ "$GITHUB_ACTIONS" = "TRUE" ]; then
+  ./case.submit --no-batch 
+fi 
+
+
+
+