diff --git a/.github/actions/build-mpas/action.yml b/.github/actions/build-mpas/action.yml new file mode 100644 index 0000000000..0db8a88847 --- /dev/null +++ b/.github/actions/build-mpas/action.yml @@ -0,0 +1,141 @@ +name: 'Build MPAS' +description: 'Build MPAS-Atmosphere with specified compiler configuration' + +inputs: + compiler: + description: 'Compiler family (gcc, nvhpc, oneapi, etc.)' + required: true + use-pio: + description: 'Use PIO library (true/false)' + required: false + default: 'false' + openacc: + description: 'Enable OpenACC (true/false)' + required: false + default: 'false' + precision: + description: 'Floating-point precision (single or double)' + required: false + default: 'single' + build-timeout: + description: 'Build timeout in minutes' + required: false + default: '20' + +outputs: + executable: + description: 'Path to built executable' + value: ${{ steps.build.outputs.executable }} + +runs: + using: 'composite' + steps: + - name: Build MPAS-A + id: build + shell: bash + run: | + # Source container environment if available + # This sets NETCDF, PNETCDF, PIO, and other library paths + if [ -f /container/config_env.sh ]; then + echo "Sourcing container environment from /container/config_env.sh" + source /container/config_env.sh + fi + + # MPAS Makefile uses `cpp` for Registry preprocessing. + # Some containers (e.g. openSUSE Leap) don't ship it separately. + if ! command -v cpp &>/dev/null; then + echo "cpp not found, installing..." + if command -v zypper &>/dev/null; then + zypper install -y --no-recommends cpp + elif command -v dnf &>/dev/null; then + dnf install -y cpp + elif command -v apt-get &>/dev/null; then + apt-get update && apt-get install -y cpp + fi + fi + + # Map compiler input to COMPILER_FAMILY if not already set + # The container should set this, but we provide a fallback + if [ -z "${COMPILER_FAMILY}" ]; then + case "${{ inputs.compiler }}" in + gcc|gfortran|gnu) export COMPILER_FAMILY="gcc" ;; + nvhpc|nvfortran) export COMPILER_FAMILY="nvhpc" ;; + oneapi|intel|ifx) export COMPILER_FAMILY="oneapi" ;; + llvm|flang|clang) export COMPILER_FAMILY="clang" ;; + *) export COMPILER_FAMILY="${{ inputs.compiler }}" ;; + esac + fi + + # Set up I/O configuration + if [ "${{ inputs.use-pio }}" = "true" ]; then + export PIO_ROOT=${PIO_ROOT:-/container/pio} + export USE_PIO2=true + else + unset PIO + export USE_PIO2=false + fi + + # Set up accelerator configuration + if [ "${{ inputs.openacc }}" = "true" ]; then + export OPENACC=true + fi + + echo "Build configuration:" + echo " Compiler input: ${{ inputs.compiler }}" + echo " COMPILER_FAMILY: ${COMPILER_FAMILY}" + echo " NETCDF: ${NETCDF:-not set}" + echo " PNETCDF: ${PNETCDF:-not set}" + echo " USE_PIO2: ${USE_PIO2}" + echo " OPENACC: ${OPENACC:-false}" + echo " PRECISION: ${{ inputs.precision }}" + + # Settings from .github/ci-config.env — edit that file to change targets/workarounds + CI_CONFIG="${GITHUB_WORKSPACE}/.github/ci-config.env" + if [ -f "${CI_CONFIG}" ]; then + source "${CI_CONFIG}" + fi + + # Map COMPILER_FAMILY to make target via ci-config.env lookup + VARNAME="MAKE_TARGET_${COMPILER_FAMILY}" + MAKE_TARGET="${!VARNAME}" + if [ -z "${MAKE_TARGET}" ]; then + echo "::error::No make target for COMPILER_FAMILY=${COMPILER_FAMILY}. Add MAKE_TARGET_${COMPILER_FAMILY}= to .github/ci-config.env" + exit 1 + fi + + EXTRA_MAKE_FLAGS="" + if [ "${COMPILER_FAMILY}" = "nvhpc" ]; then + ARCH="${NVHPC_TARGET_ARCH:--tp=px}" + sed -i "/^nvhpc:/,/^[a-z]/ s/\"FFLAGS_OPT = /\"FFLAGS_OPT = ${ARCH} /" Makefile + sed -i "/^nvhpc:/,/^[a-z]/ s/\"CFLAGS_OPT = /\"CFLAGS_OPT = ${ARCH} /" Makefile + sed -i "/^nvhpc:/,/^[a-z]/ s/\"CXXFLAGS_OPT = /\"CXXFLAGS_OPT = ${ARCH} /" Makefile + sed -i "/^nvhpc:/,/^[a-z]/ s/\"LDFLAGS_OPT = /\"LDFLAGS_OPT = ${ARCH} /" Makefile + EXTRA_MAKE_FLAGS="${NVHPC_EXTRA_MAKE_FLAGS}" + elif [ "${COMPILER_FAMILY}" = "oneapi" ]; then + EXTRA_MAKE_FLAGS="${ONEAPI_EXTRA_MAKE_FLAGS}" + fi + + echo " Make target: ${MAKE_TARGET}" + echo " Extra make flags: ${EXTRA_MAKE_FLAGS:-}" + echo " Parallel jobs: ${MAKE_J_PROCS:-$(nproc)}" + + PRECISION_FLAG="" + if [ "${{ inputs.precision }}" = "double" ]; then + PRECISION_FLAG="PRECISION=double" + fi + + timeout ${{ inputs.build-timeout }}m \ + make ${MAKE_TARGET} CORE=atmosphere ${EXTRA_MAKE_FLAGS} ${PRECISION_FLAG} --jobs ${MAKE_J_PROCS:-$(nproc)} + + # Set output + echo "executable=$(pwd)/atmosphere_model" >> $GITHUB_OUTPUT + + - name: Verify executable + shell: bash + run: | + if [ ! -f atmosphere_model ]; then + echo "ERROR: atmosphere_model not found!" + exit 1 + fi + ls -la atmosphere_model + file atmosphere_model diff --git a/.github/actions/download-testdata/action.yml b/.github/actions/download-testdata/action.yml new file mode 100644 index 0000000000..86d1457943 --- /dev/null +++ b/.github/actions/download-testdata/action.yml @@ -0,0 +1,97 @@ +name: 'Download Test Data' +description: 'Download and extract an MPAS test case archive from GitHub releases' + +inputs: + resolution: + description: 'Test case resolution (e.g., 240km, 120km). Used to look up RELEASE_TESTDATA_{RES} in ci-config.env.' + required: true + dest-dir: + description: 'Destination directory name for the extracted test case' + required: false + default: '' + +outputs: + case-dir: + description: 'Path to the extracted test case directory' + value: ${{ steps.extract.outputs.case-dir }} + +runs: + using: composite + steps: + - name: Resolve release tag and archive + id: resolve + shell: bash + run: | + RESOLUTION="${{ inputs.resolution }}" + ARCHIVE="${RESOLUTION}.tar.gz" + + CI_CONFIG="${GITHUB_WORKSPACE}/.github/ci-config.env" + if [ ! -f "${CI_CONFIG}" ]; then + echo "::error::ci-config.env not found at ${CI_CONFIG}" + exit 1 + fi + source "${CI_CONFIG}" + + RES_UPPER=$(echo "${RESOLUTION}" | tr '[:lower:]' '[:upper:]' | tr '-' '_') + TAG_VAR="RELEASE_TESTDATA_${RES_UPPER}" + TAG="${!TAG_VAR}" + + if [ -z "${TAG}" ]; then + echo "::error::No release tag for resolution '${RESOLUTION}'. Add ${TAG_VAR}= to ci-config.env." + exit 1 + fi + + REPO="${DATA_REPOSITORY:-${GITHUB_REPOSITORY}}" + URL="https://github.com/${REPO}/releases/download/${TAG}/${ARCHIVE}" + + echo "release-tag=${TAG}" >> $GITHUB_OUTPUT + echo "archive=${ARCHIVE}" >> $GITHUB_OUTPUT + echo "url=${URL}" >> $GITHUB_OUTPUT + echo "Resolved: ${TAG_VAR}=${TAG} → ${URL}" + + - name: Cache test case archive + id: cache + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: ${{ steps.resolve.outputs.archive }} + key: testdata-${{ steps.resolve.outputs.release-tag }} + + - name: Download test case archive + if: steps.cache.outputs.cache-hit != 'true' + shell: bash + run: | + URL="${{ steps.resolve.outputs.url }}" + ARCHIVE="${{ steps.resolve.outputs.archive }}" + echo "Downloading ${ARCHIVE} from ${URL}..." + curl -fsSL --retry 5 --retry-delay 5 "${URL}" -o "${ARCHIVE}" + echo "Downloaded $(du -h "${ARCHIVE}" | cut -f1)" + + - name: Extract test case + id: extract + shell: bash + run: | + ARCHIVE="${{ steps.resolve.outputs.archive }}" + DEST="${{ inputs.dest-dir }}" + + echo "Cache hit: ${{ steps.cache.outputs.cache-hit }}" + echo "Archive: ${ARCHIVE} ($(du -h "${ARCHIVE}" | cut -f1))" + + tar xzf "${ARCHIVE}" + + CASE_DIR=$(tar tzf "${ARCHIVE}" 2>/dev/null | head -1 | cut -d/ -f1 || true) + if [ -z "${CASE_DIR}" ]; then + CASE_DIR=$(ls -td */ 2>/dev/null | head -1 | tr -d '/') + fi + + if [ -z "${CASE_DIR}" ] || [ ! -d "${CASE_DIR}" ]; then + echo "::error::Failed to extract test case from ${ARCHIVE}" + exit 1 + fi + + if [ -n "${DEST}" ] && [ "${DEST}" != "${CASE_DIR}" ]; then + mv "${CASE_DIR}" "${DEST}" + CASE_DIR="${DEST}" + fi + + echo "case-dir=${CASE_DIR}" >> $GITHUB_OUTPUT + echo "Extracted test case to: ${CASE_DIR}" diff --git a/.github/actions/ect-summary/action.yml b/.github/actions/ect-summary/action.yml new file mode 100644 index 0000000000..6175f5a33a --- /dev/null +++ b/.github/actions/ect-summary/action.yml @@ -0,0 +1,100 @@ +name: 'ECT Summary' +description: > + Generate a consolidated Ensemble Consistency Test results table from + enriched result files produced by the validate-ect action. Writes a + Markdown table to $GITHUB_STEP_SUMMARY with auto-discovered columns. + +inputs: + results-path: + description: 'Directory containing downloaded ect-result-* artifact subdirectories' + required: true + +runs: + using: 'composite' + steps: + - name: Generate summary table + shell: bash + run: | + RESULTS_PATH="${{ inputs.results-path }}" + + # Collect all result files + RESULT_FILES=() + for f in "${RESULTS_PATH}"/ect-result-*/ect-result.txt; do + [ -f "$f" ] && RESULT_FILES+=("$f") + done + + if [ ${#RESULT_FILES[@]} -eq 0 ]; then + echo "::warning::No ECT result files found in ${RESULTS_PATH}" + echo "## Ensemble Consistency Test (ECT) Results" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "No results available." >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + # Discover column names from the first result file (all keys except "result") + COLUMNS=() + while IFS='=' read -r key value; do + [ -z "$key" ] && continue + if [ "$key" != "result" ]; then + COLUMNS+=("$key") + fi + done < "${RESULT_FILES[0]}" + + # Build header row + HEADER="| " + SEPARATOR="| " + for col in "${COLUMNS[@]}"; do + COL_TITLE=$(echo "$col" | sed 's/.*/\u&/') + HEADER+="${COL_TITLE} | " + SEPARATOR+="--- | " + done + HEADER+="Result |" + SEPARATOR+="--- |" + + # Build data rows + PASS=0 FAIL=0 ERROR=0 SKIP=0 TOTAL=0 + ROWS="" + for f in "${RESULT_FILES[@]}"; do + TOTAL=$((TOTAL + 1)) + + # Parse key=value pairs + declare -A DATA=() + while IFS='=' read -r key value; do + [ -z "$key" ] && continue + DATA["$key"]="$value" + done < "$f" + + RESULT="${DATA[result]}" + case "$RESULT" in + PASSED) ICON="PASSED"; PASS=$((PASS + 1)) ;; + FAILED) ICON="**FAILED**"; FAIL=$((FAIL + 1)) ;; + SKIPPED) ICON="SKIPPED"; SKIP=$((SKIP + 1)) ;; + *) ICON="ERROR"; ERROR=$((ERROR + 1)) ;; + esac + + ROW="| " + for col in "${COLUMNS[@]}"; do + ROW+="${DATA[$col]:-—} | " + done + ROW+="${ICON} |" + ROWS+="${ROW}"$'\n' + + unset DATA + done + + SORTED_ROWS=$(echo "$ROWS" | sort) + + { + echo "## Ensemble Consistency Test (ECT) Results" + echo "" + echo "$HEADER" + echo "$SEPARATOR" + echo "$SORTED_ROWS" + echo "" + echo "**Total: ${TOTAL}** — ${PASS} passed, ${FAIL} failed, ${ERROR} error, ${SKIP} skipped" + + if [ ${FAIL} -gt 0 ] || [ ${ERROR} -gt 0 ]; then + echo "" + echo "> One or more ECT validations failed. Check individual ECT Validate job logs for PyCECT details." + fi + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/actions/mpas-version/action.yml b/.github/actions/mpas-version/action.yml new file mode 100644 index 0000000000..73c3b3e47a --- /dev/null +++ b/.github/actions/mpas-version/action.yml @@ -0,0 +1,47 @@ +name: 'Get MPAS Version' +description: > + Read the MPAS version string from src/core_atmosphere/Registry.xml. + Strict — fails the workflow if the file is missing or the version + attribute cannot be parsed (no silent "unknown" fallback). + +inputs: + registry-path: + description: 'Path to Registry.xml relative to GITHUB_WORKSPACE' + required: false + default: 'src/core_atmosphere/Registry.xml' + +outputs: + version: + description: 'MPAS version string (e.g., 8.4.0)' + value: ${{ steps.extract.outputs.version }} + +runs: + using: 'composite' + steps: + - id: extract + shell: bash + run: | + python3 - "${{ inputs.registry-path }}" <<'PYEOF' + import os + import sys + import xml.etree.ElementTree as ET + + path = sys.argv[1] + if not os.path.isfile(path): + print(f"::error::MPAS Registry not found at {path}") + sys.exit(1) + try: + root = ET.parse(path).getroot() + except ET.ParseError as e: + print(f"::error::Could not parse {path}: {e}") + sys.exit(1) + + version = root.attrib.get('version') + if not version: + print(f"::error::No version attribute on in {path}") + sys.exit(1) + + with open(os.environ['GITHUB_OUTPUT'], 'a') as f: + f.write(f"version={version}\n") + print(f"MPAS version: {version}") + PYEOF diff --git a/.github/actions/print-mpas-logs/action.yml b/.github/actions/print-mpas-logs/action.yml new file mode 100644 index 0000000000..53ed7e33b1 --- /dev/null +++ b/.github/actions/print-mpas-logs/action.yml @@ -0,0 +1,64 @@ +name: 'Print MPAS Logs' +description: > + Print MPAS per-rank log files (log.atmosphere..out / .err) to the + workflow log inside collapsible ::group:: blocks. Read-only; never fails + on its own (use if: always() at the call site). + +inputs: + log-dir: + description: 'Directory to scan for log files' + required: true + pattern: + description: > + Glob (relative to log-dir) of files to print. Default catches both + .out and .err per rank. + required: false + default: 'log.atmosphere.*' + max-lines: + description: > + If set, print only the last N lines of each file via tail. Empty (default) + prints the whole file. + required: false + default: '' + +runs: + using: 'composite' + steps: + - name: Print logs + shell: bash + run: | + LOG_DIR="${{ inputs.log-dir }}" + PATTERN="${{ inputs.pattern }}" + MAX_LINES="${{ inputs.max-lines }}" + + if [ ! -d "${LOG_DIR}" ]; then + echo "print-mpas-logs: directory '${LOG_DIR}' does not exist; nothing to print." + exit 0 + fi + + cd "${LOG_DIR}" + shopt -s nullglob + + # Sort so .out comes before .err per rank (lexical order does this). + FILES=( $(ls -1 ${PATTERN} 2>/dev/null | sort) ) + + if [ ${#FILES[@]} -eq 0 ]; then + echo "print-mpas-logs: no files matching '${PATTERN}' in ${LOG_DIR}." + exit 0 + fi + + echo "print-mpas-logs: ${#FILES[@]} file(s) from ${LOG_DIR}" + # Close the implicit group GitHub opens for the run: script source so + # the per-file groups below render at the step's top level (collapsible + # at normal font size) instead of nested inside the script-source group. + echo "::endgroup::" + for f in "${FILES[@]}"; do + echo "::group::${f}" + if [ -n "${MAX_LINES}" ]; then + echo "(last ${MAX_LINES} lines)" + tail -n "${MAX_LINES}" "${f}" || true + else + cat "${f}" || true + fi + echo "::endgroup::" + done diff --git a/.github/actions/resolve-container/action.yml b/.github/actions/resolve-container/action.yml new file mode 100644 index 0000000000..fccafffd76 --- /dev/null +++ b/.github/actions/resolve-container/action.yml @@ -0,0 +1,65 @@ +name: 'Resolve Container Image' +description: 'Assemble a container image name from ci-config.env templates' + +inputs: + compiler: + description: 'Compiler family (gcc, nvhpc, oneapi)' + required: true + mpi: + description: 'MPI implementation (openmpi, mpich)' + required: true + gpu: + description: 'GPU variant (empty string, or "cuda")' + required: false + default: '' + +outputs: + image: + description: 'Fully qualified container image name' + value: ${{ steps.resolve.outputs.image }} + +runs: + using: 'composite' + steps: + - name: Resolve container image + id: resolve + shell: bash + run: | + CI_CONFIG="${GITHUB_WORKSPACE}/.github/ci-config.env" + if [ ! -f "${CI_CONFIG}" ]; then + echo "::error::ci-config.env not found at ${CI_CONFIG}" + exit 1 + fi + source "${CI_CONFIG}" + + COMPILER="${{ inputs.compiler }}" + MPI="${{ inputs.mpi }}" + GPU="${{ inputs.gpu }}" + + # Resolve compiler name (fallback to raw value if no mapping) + comp_var="CONTAINER_COMPILER_${COMPILER}" + COMPILER_NAME="${!comp_var:-$COMPILER}" + + # Resolve MPI name (fallback to raw value if no mapping) + mpi_var="CONTAINER_MPI_${MPI}" + MPI_NAME="${!mpi_var:-$MPI}" + + # Pick template: GPU vs CPU, then check for per-compiler override + if [ -n "${GPU}" ]; then + gpu_override="CONTAINER_IMAGE_GPU_${COMPILER}" + TEMPLATE="${!gpu_override:-$CONTAINER_IMAGE_GPU}" + else + cpu_override="CONTAINER_IMAGE_${COMPILER}" + TEMPLATE="${!cpu_override:-$CONTAINER_IMAGE}" + fi + + # Substitute placeholders + IMAGE="${TEMPLATE//\{compiler\}/$COMPILER_NAME}" + IMAGE="${IMAGE//\{mpi\}/$MPI_NAME}" + + echo "Resolved container image: ${IMAGE}" + echo " Compiler: ${COMPILER} → ${COMPILER_NAME}" + echo " MPI: ${MPI} → ${MPI_NAME}" + echo " GPU: ${GPU:-none}" + + echo "image=${IMAGE}" >> $GITHUB_OUTPUT diff --git a/.github/actions/run-mpas/action.yml b/.github/actions/run-mpas/action.yml new file mode 100644 index 0000000000..063697a90b --- /dev/null +++ b/.github/actions/run-mpas/action.yml @@ -0,0 +1,195 @@ +name: 'Run MPAS' +description: 'Run MPAS-Atmosphere test case' + +inputs: + executable: + description: 'Path to atmosphere_model executable' + required: false + default: './atmosphere_model' + num-procs: + description: 'Number of MPI processes' + required: false + default: '1' + run-duration: + description: 'Run duration (format: D_HH:MM:SS). If empty, uses the namelist default from the test case archive.' + required: false + default: '' + restart-interval: + description: 'Restart output interval (format: D_HH:MM:SS). If empty, uses the streams.atmosphere default.' + required: false + default: '' + resolution: + description: 'Test case resolution name (e.g., 240km). Used to download the archive and name the working directory.' + required: false + default: '240km' + mpi-impl: + description: 'MPI implementation (openmpi, mpich)' + required: false + default: '' + run-timeout: + description: 'Run timeout in minutes' + required: false + default: '20' + working-dir: + description: 'Working directory name for the run' + required: false + default: '' + strict-exit-check: + description: 'Fail on non-zero exit code (default true). Set false when gfortran IEEE warnings produce non-zero exit but model output is valid.' + required: false + default: 'true' + +outputs: + log-dir: + description: 'Directory containing log files' + value: ${{ steps.run.outputs.log-dir }} + status: + description: 'Run status (success/failed)' + value: ${{ steps.run.outputs.status }} + +runs: + using: 'composite' + steps: + - name: Resolve configuration + id: config + shell: bash + run: | + RESOLUTION="${{ inputs.resolution }}" + WORKDIR="${{ inputs.working-dir }}" + if [ -z "${WORKDIR}" ]; then + WORKDIR="run-${RESOLUTION}" + fi + + echo "workdir=${WORKDIR}" >> $GITHUB_OUTPUT + echo "resolution=${RESOLUTION}" >> $GITHUB_OUTPUT + + echo "=== Run configuration ===" + echo " Resolution: ${RESOLUTION}" + echo " Work dir: ${WORKDIR}" + echo " Timeout: ${{ inputs.run-timeout }}m" + if [ -n "${{ inputs.run-duration }}" ]; then + echo " Duration: ${{ inputs.run-duration }} (override)" + else + echo " Duration: (namelist default)" + fi + + - name: Download test case + uses: ./.github/actions/download-testdata + id: download + with: + resolution: ${{ inputs.resolution }} + dest-dir: ${{ steps.config.outputs.workdir }} + + - name: Link executable + shell: bash + run: | + WORKDIR="${{ steps.config.outputs.workdir }}" + chmod +x ${{ inputs.executable }} + ln -sf $(realpath ${{ inputs.executable }}) "${WORKDIR}/atmosphere_model" + + - name: Configure namelist overrides + shell: bash + working-directory: ${{ steps.config.outputs.workdir }} + run: | + if [ -n "${{ inputs.run-duration }}" ]; then + DURATION="${{ inputs.run-duration }}" + sed -i "s/config_run_duration = '[^']*'/config_run_duration = '${DURATION}'/" namelist.atmosphere + echo "Overrode config_run_duration = '${DURATION}'" + fi + + if [ -n "${{ inputs.restart-interval }}" ]; then + RESTART="${{ inputs.restart-interval }}" + sed -i '// s/output_interval="[^"]*"/output_interval="'"${RESTART}"'"/' streams.atmosphere + echo "Overrode restart output_interval = '${RESTART}'" + fi + + echo "=== Namelist ===" + grep config_run_duration namelist.atmosphere + + - name: Run MPAS-A + id: run + shell: bash + working-directory: ${{ steps.config.outputs.workdir }} + run: | + TIMEOUT="${{ inputs.run-timeout }}" + + if [ -f /container/config_env.sh ]; then + source /container/config_env.sh + fi + + # Workaround: some containers omit LD_LIBRARY_PATH from config_env.sh + if [ -z "${LD_LIBRARY_PATH}" ]; then + export LD_LIBRARY_PATH="/usr/lib64:/usr/lib" + fi + + if [ -n "${{ inputs.mpi-impl }}" ]; then + export MPI_IMPL="${{ inputs.mpi-impl }}" + fi + + CI_CONFIG="${GITHUB_WORKSPACE}/.github/ci-config.env" + if [ -f "${CI_CONFIG}" ]; then + source "${CI_CONFIG}" + fi + + MPI_FLAGS="" + if [ "${MPI_IMPL}" = "openmpi" ]; then + MPI_FLAGS="${OPENMPI_RUN_FLAGS:---allow-run-as-root --oversubscribe}" + fi + + ulimit -s unlimited 2>/dev/null || echo "Warning: Could not set unlimited stack size" + + echo "=== Run configuration ===" + echo " Resolution: ${{ steps.config.outputs.resolution }}" + echo " Processors: ${{ inputs.num-procs }}" + echo " MPI_IMPL: ${MPI_IMPL:-auto}" + echo " MPI_FLAGS: ${MPI_FLAGS}" + echo " Stack limit: $(ulimit -s)" + echo " Available CPUs: $(nproc 2>/dev/null || echo unknown)" + echo " Available RAM: $(free -m 2>/dev/null | awk '/^Mem:/{print $2 "MB"}' || echo unknown)" + echo " LD_LIBRARY_PATH: ${LD_LIBRARY_PATH:-not set}" + + set +e + timeout ${TIMEOUT}m mpirun -n ${{ inputs.num-procs }} ${MPI_FLAGS} ./atmosphere_model + RUN_STATUS=$? + set -e + + echo "log-dir=$(pwd)" >> $GITHUB_OUTPUT + echo "run-exit-code=${RUN_STATUS}" >> $GITHUB_OUTPUT + if [ $RUN_STATUS -eq 0 ]; then + echo "status=success" >> $GITHUB_OUTPUT + elif [ "${{ inputs.strict-exit-check }}" = "false" ]; then + echo "status=success" >> $GITHUB_OUTPUT + echo "::warning::Model exited with status ${RUN_STATUS} (non-strict mode, treating as success)" + else + echo "status=failed" >> $GITHUB_OUTPUT + echo "::warning::Model run exited with status $RUN_STATUS" + fi + + - name: List output files + shell: bash + if: always() + run: | + WORKDIR="${{ steps.config.outputs.workdir }}" + echo "=== Output files ===" + if [ -d "${WORKDIR}" ]; then + cd "${WORKDIR}" + ls -la log.* 2>/dev/null || echo "No log files found" + ls -la *.nc 2>/dev/null || echo "No NetCDF files found" + else + echo "Working directory ${WORKDIR} does not exist" + fi + + - name: Print MPAS logs + if: always() + uses: ./.github/actions/print-mpas-logs + with: + log-dir: ${{ steps.config.outputs.workdir }} + + - name: Check run status + shell: bash + if: always() + run: | + if [ "${{ steps.run.outputs.status }}" = "failed" ]; then + echo "::error::MPAS model run failed with exit code ${{ steps.run.outputs.run-exit-code }}" + exit 1 + fi diff --git a/.github/actions/run-perturb-mpas/action.yml b/.github/actions/run-perturb-mpas/action.yml new file mode 100644 index 0000000000..9af1660708 --- /dev/null +++ b/.github/actions/run-perturb-mpas/action.yml @@ -0,0 +1,284 @@ +name: 'Run Perturbed MPAS' +description: > + Run one or more perturbed MPAS-A ensemble members for ECT. + Handles IC perturbation, namelist/stream configuration, model execution, + and optional history file trimming. Supports both single-member + (ect-test) and batched (ect-ensemble-gen) use cases. + +inputs: + base-dir: + description: 'Path to the base test case directory (copied per member)' + required: true + executable: + description: 'Path to atmosphere_model executable' + required: true + member-start: + description: 'First ensemble member index' + required: true + member-end: + description: 'Last ensemble member index (same as member-start for single member)' + required: true + run-duration: + description: 'Model run duration per member (format: D_HH:MM:SS)' + required: true + run-timeout: + description: 'Per-member timeout in minutes' + required: false + default: '45' + num-ranks: + description: 'Number of MPI ranks per member' + required: false + default: '1' + mpi-impl: + description: 'MPI implementation (openmpi, mpich)' + required: false + default: 'openmpi' + output-dir: + description: 'Directory to collect trimmed history files' + required: false + default: 'history-output' + trim: + description: 'Trim history files after run (true/false)' + required: false + default: 'true' + restart-file: + description: 'Path to a spun-up restart file. When set, each member starts from this restart (with config_do_restart=.true.) instead of init.nc.' + required: false + default: '' + verbose: + description: 'Print detailed diagnostics for each member (namelist dumps, theta stats)' + required: false + default: 'false' + +outputs: + status: + description: 'Overall status (success/partial/failed)' + value: ${{ steps.run-members.outputs.status }} + members-completed: + description: 'Number of members that produced history files' + value: ${{ steps.run-members.outputs.members-completed }} + +runs: + using: 'composite' + steps: + - name: Run perturbed members + id: run-members + shell: bash + run: | + source /container/config_env.sh + + if command -v conda &>/dev/null; then + eval "$(conda shell.bash hook)" 2>/dev/null + conda activate base 2>/dev/null || true + fi + + if ! python3 -c "import netCDF4, numpy" 2>/dev/null; then + if python3 -m pip install --quiet netCDF4 numpy 2>/dev/null; then + echo "Installed netCDF4/numpy via pip" + elif command -v conda &>/dev/null; then + conda install -y -q netCDF4 numpy + echo "Installed netCDF4/numpy via conda" + else + echo "::error::Could not install netCDF4 (no pip, no conda)" + exit 1 + fi + fi + echo "python3: $(which python3) — $(python3 --version)" + + if [ -z "${LD_LIBRARY_PATH}" ]; then + export LD_LIBRARY_PATH="/usr/lib64:/usr/lib" + fi + + CI_CONFIG="${GITHUB_WORKSPACE}/.github/ci-config.env" + if [ -f "${CI_CONFIG}" ]; then + source "${CI_CONFIG}" + fi + ulimit -s unlimited 2>/dev/null || true + + BASEDIR="${{ inputs.base-dir }}" + EXE="${{ inputs.executable }}" + MEMBER_START=${{ inputs.member-start }} + MEMBER_END=${{ inputs.member-end }} + NRANKS=${{ inputs.num-ranks }} + MPI_IMPL="${{ inputs.mpi-impl }}" + OUTDIR="${{ inputs.output-dir }}" + TRIM="${{ inputs.trim }}" + RESTART_FILE="${{ inputs.restart-file }}" + VERBOSE="${{ inputs.verbose }}" + RUN_DURATION="${{ inputs.run-duration }}" + RUN_TIMEOUT="${{ inputs.run-timeout }}" + + MPI_FLAGS="" + if [ "${MPI_IMPL}" = "openmpi" ]; then + MPI_FLAGS="${OPENMPI_RUN_FLAGS:---allow-run-as-root --oversubscribe}" + fi + + EXCLUDE_FILE="" + if [ -n "${ECT_EXCLUDED_VARS}" ]; then + EXCLUDE_FILE="${GITHUB_WORKSPACE}/${ECT_EXCLUDED_VARS}" + fi + + mkdir -p "${OUTDIR}" + + TOTAL=0 + COMPLETED=0 + + for MEMBER in $(seq ${MEMBER_START} ${MEMBER_END}); do + TOTAL=$((TOTAL + 1)) + MEMBER_ID=$(printf "%04d" ${MEMBER}) + RUNDIR="run-ect-${MEMBER_ID}" + echo "" + echo "==========================================" + echo " Ensemble member ${MEMBER_ID}" + echo "==========================================" + + cp -r "${BASEDIR}" "${RUNDIR}" + chmod +x "${EXE}" + ln -sf $(realpath "${EXE}") "${RUNDIR}/atmosphere_model" + + if [ -n "${RESTART_FILE}" ]; then + RESTART_TIME=$(python3 -c " + import netCDF4, sys + ds = netCDF4.Dataset(sys.argv[1]) + print(ds.variables['xtime'][0].tobytes().decode().strip()) + ds.close() + " "${RESTART_FILE}") + RESTART_FNAME="restart.$(echo ${RESTART_TIME} | tr ':' '.').nc" + cp "${RESTART_FILE}" "${RUNDIR}/${RESTART_FNAME}" + echo "${RESTART_TIME}" > "${RUNDIR}/restart_timestamp" + PERTURB_FILE="${RUNDIR}/${RESTART_FNAME}" + echo "[$(date +%H:%M:%S)] Restart mode: ${RESTART_FNAME} (time=${RESTART_TIME})" + else + PERTURB_FILE="${RUNDIR}/init.nc" + if [ ! -f "${PERTURB_FILE}" ]; then + PERTURB_FILE=$(ls ${RUNDIR}/*.init*.nc 2>/dev/null | head -1) + fi + fi + + if [ -z "${PERTURB_FILE}" ] || [ ! -f "${PERTURB_FILE}" ]; then + echo "::error::Member ${MEMBER_ID}: could not find file to perturb" + rm -rf "${RUNDIR}" + continue + fi + + echo "[$(date +%H:%M:%S)] Perturbing theta (seed=${MEMBER})..." + python3 ${GITHUB_WORKSPACE}/.github/actions/run-perturb-mpas/perturb_theta.py \ + "${PERTURB_FILE}" --seed ${MEMBER} --magnitude ${ECT_PERTURB_MAGNITUDE} + + if [ "${VERBOSE}" = "true" ]; then + echo " Restart MD5: $(md5sum "${PERTURB_FILE}" | cut -d' ' -f1)" + fi + + cd "${RUNDIR}" + + if [ -n "${RESTART_FILE}" ]; then + sed -i "s/config_do_restart.*/config_do_restart = .true./" namelist.atmosphere + sed -i "s/config_start_time.*/config_start_time = 'file'/" namelist.atmosphere + sed -i '/&restart/a\ config_do_DAcycling = .true.' namelist.atmosphere + fi + + sed -i "s/config_run_duration = '[^']*'/config_run_duration = '${RUN_DURATION}'/" namelist.atmosphere + sed -i '// s/output_interval="[^"]*"/output_interval="none"/' streams.atmosphere + sed -i '// s/output_interval="[^"]*"/output_interval="'"${RUN_DURATION}"'"/' streams.atmosphere + + echo "[$(date +%H:%M:%S)] Running MPAS-A (${RUN_DURATION}, ${NRANKS} ranks)..." + set +e + timeout ${RUN_TIMEOUT}m mpirun -n ${NRANKS} ${MPI_FLAGS} ./atmosphere_model + RUN_STATUS=$? + set -e + echo "[$(date +%H:%M:%S)] Model finished (exit code ${RUN_STATUS})" + + for LOGFILE in log.atmosphere.*.out log.atmosphere.*.err; do + [ -f "${LOGFILE}" ] || continue + EXT="${LOGFILE##*.}" + BASE="${LOGFILE%.*}" + cp "${LOGFILE}" "../${OUTDIR}/${BASE}.member${MEMBER_ID}.${EXT}" + done + + if [ "${VERBOSE}" = "true" ]; then + echo "[$(date +%H:%M:%S)] === Namelist config ===" + grep -E 'config_do_restart|config_start_time|config_run_duration' namelist.atmosphere || true + echo "[$(date +%H:%M:%S)] === Restart stream ===" + grep -A2 'immutable_stream name="restart"' streams.atmosphere || true + echo "[$(date +%H:%M:%S)] === Output stream ===" + grep -A2 'stream name="output"' streams.atmosphere || true + + RESTART_NC=$(ls restart.*.nc 2>/dev/null | head -1) + if [ -n "${RESTART_NC}" ]; then + echo "[$(date +%H:%M:%S)] === Restart theta check ===" + python3 -c " + import netCDF4 as nc, numpy as np, sys + ds = nc.Dataset(sys.argv[1]) + th = ds.variables['theta'][:] + print(f' File: {sys.argv[1]}') + print(f' theta dtype={th.dtype} shape={th.shape} mean={np.mean(th):.15e}') + ds.close() + " "${RESTART_NC}" + fi + fi + + echo "[$(date +%H:%M:%S)] History files produced:" + ls -la history.*.nc 2>/dev/null || echo " (none)" + HIST_FILE=$(ls -t history.*.nc 2>/dev/null | head -1 || true) + if [ -n "${HIST_FILE}" ]; then + if [ "${VERBOSE}" = "true" ]; then + echo "[$(date +%H:%M:%S)] === History theta check ===" + python3 -c " + import netCDF4 as nc, numpy as np, sys + ds = nc.Dataset(sys.argv[1]) + if 'theta' in ds.variables: + th = ds.variables['theta'] + data = th[:] + print(f' theta dtype={data.dtype} shape={data.shape} mean={np.mean(data):.15e} min={np.min(data):.10e} max={np.max(data):.10e}') + else: + print(' theta NOT in history variables') + print(f' Available: {list(ds.variables.keys())[:15]}...') + ds.close() + " "${HIST_FILE}" + fi + TSLICE=$(python3 -c " + import netCDF4, sys + ds = netCDF4.Dataset(sys.argv[1]) + print(ds.dimensions['Time'].size - 1) + ds.close() + " "${HIST_FILE}") + echo "[$(date +%H:%M:%S)] Using tslice=${TSLICE} (last time slice)" + if [ "${TRIM}" = "true" ] && [ -n "${EXCLUDE_FILE}" ] && [ -f "${EXCLUDE_FILE}" ]; then + python3 ${GITHUB_WORKSPACE}/.github/actions/run-perturb-mpas/trim_history.py \ + "${HIST_FILE}" "../${OUTDIR}/history.${MEMBER_ID}.nc" \ + --tslice ${TSLICE} \ + --exclude-file "${EXCLUDE_FILE}" + echo "[$(date +%H:%M:%S)] Saved trimmed history for member ${MEMBER_ID}" + else + cp "${HIST_FILE}" "../${OUTDIR}/history.${MEMBER_ID}.nc" + echo "[$(date +%H:%M:%S)] Saved history for member ${MEMBER_ID}" + fi + COMPLETED=$((COMPLETED + 1)) + else + echo "::warning::Member ${MEMBER_ID} failed (exit ${RUN_STATUS}, no history file)" + fi + + cd .. + rm -rf "${RUNDIR}" + done + + echo "" + echo "=== Completed ${COMPLETED}/${TOTAL} members ===" + ls -la "${OUTDIR}/" + + echo "members-completed=${COMPLETED}" >> $GITHUB_OUTPUT + if [ ${COMPLETED} -eq ${TOTAL} ]; then + echo "status=success" >> $GITHUB_OUTPUT + elif [ ${COMPLETED} -gt 0 ]; then + echo "status=partial" >> $GITHUB_OUTPUT + else + echo "status=failed" >> $GITHUB_OUTPUT + echo "::error::No ensemble members produced history files" + exit 1 + fi + + - name: Print MPAS logs + if: always() + uses: ./.github/actions/print-mpas-logs + with: + log-dir: ${{ inputs.output-dir }} diff --git a/.github/actions/run-perturb-mpas/perturb_theta.py b/.github/actions/run-perturb-mpas/perturb_theta.py new file mode 100644 index 0000000000..62ab4520f7 --- /dev/null +++ b/.github/actions/run-perturb-mpas/perturb_theta.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +""" +Apply O(10^-14) perturbation to the theta (potential temperature) field +in an MPAS initial conditions file to generate ensemble diversity. + +This is a placeholder until native perturbation support is added to MPAS-A. +Each ensemble member uses a unique seed for reproducible perturbations. +""" + +import argparse +import sys + +import numpy as np + +try: + from netCDF4 import Dataset +except ImportError: + print("ERROR: netCDF4 is required. Install with: pip install netCDF4") + sys.exit(1) + + +def perturb_theta(ic_file, seed, magnitude=1e-14): + rng = np.random.default_rng(seed) + + with Dataset(ic_file, "r+") as ds: + if "theta" not in ds.variables: + print(f"ERROR: 'theta' variable not found in {ic_file}") + print(f" Available variables: {list(ds.variables.keys())[:20]}...") + sys.exit(1) + + theta = ds.variables["theta"] + data = theta[:] + original_mean = float(np.mean(data)) + + perturbation = rng.uniform(-magnitude, magnitude, size=data.shape) + theta[:] = data * (1.0 + perturbation) + + actual_max = np.max(np.abs(perturbation)) + print(f"Applied perturbation to theta field:") + print(f" File: {ic_file}") + print(f" Format: {ds.data_model}") + print(f" Seed: {seed}") + print(f" Magnitude: +/- {magnitude:.0e}") + print(f" Max |eps|: {actual_max:.2e}") + print(f" Shape: {data.shape}") + print(f" Var dtype: {theta.dtype}") + print(f" Original mean: {original_mean:.15e}") + + # Read-back verification: reopen and confirm perturbation persisted + with Dataset(ic_file, "r") as ds: + verify = ds.variables["theta"][:] + verify_mean = float(np.mean(verify)) + diff = verify.astype(np.float64) - data.astype(np.float64) + n_changed = int(np.count_nonzero(diff)) + max_diff = float(np.max(np.abs(diff))) + print(f" Verify mean: {verify_mean:.15e}") + print(f" Changed cells: {n_changed}/{diff.size}") + print(f" Max |diff|: {max_diff:.6e}") + if n_changed == 0: + print(f"ERROR: Perturbation did NOT persist in file!") + print(f" On-disk dtype: {ds.variables['theta'].dtype}") + print(f" If dtype is float32, perturbations below ~1.2e-7 will be rounded away.") + sys.exit(1) + + +def main(): + parser = argparse.ArgumentParser( + description="Perturb MPAS theta field for ensemble generation" + ) + parser.add_argument("ic_file", help="Path to MPAS initial conditions NetCDF file") + parser.add_argument("--seed", type=int, required=True, + help="Random seed for reproducible perturbation") + parser.add_argument("--magnitude", type=float, default=1e-14, + help="Perturbation magnitude (default: 1e-14)") + args = parser.parse_args() + + perturb_theta(args.ic_file, args.seed, args.magnitude) + + +if __name__ == "__main__": + main() diff --git a/.github/actions/run-perturb-mpas/trim_history.py b/.github/actions/run-perturb-mpas/trim_history.py new file mode 100644 index 0000000000..706b4268ef --- /dev/null +++ b/.github/actions/run-perturb-mpas/trim_history.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +""" +Trim MPAS history files for ECT ensemble processing. + +Extracts a single time slice, removes excluded variables, strips most +static mesh geometry, and applies lossless NetCDF4 deflation (zlib). + +PyCECT requires three area-weighting variables (areaCell, dvEdge, +areaTriangle) to compute global means. These static variables are +preserved even though they lack a Time dimension. + +The ECT exclusion list path is configured via ECT_EXCLUDED_VARS in ci-config.env +and passed via --exclude-file. + +Usage: + python3 trim_history.py input.nc output.nc --tslice -1 --exclude-file excluded_vars.txt +""" + +import argparse +import os +import sys + +import netCDF4 as nc + +# Static variables PyCECT reads for area-weighted global means +# (see pyEnsLib.py generate_global_mean_for_summary_MPAS, lines 745-758) +PYCECT_REQUIRED_STATIC = {'areaCell', 'dvEdge', 'areaTriangle'} + + +def trim_history(infile, outfile, tslice, exclude_vars=None): + exclude = set(exclude_vars or []) + + with nc.Dataset(infile, 'r') as src, nc.Dataset(outfile, 'w', format='NETCDF4') as dst: + ntime = src.dimensions['Time'].size + if tslice < 0: + tslice = ntime + tslice + if tslice < 0 or tslice >= ntime: + print(f"ERROR: tslice={tslice} out of range for {ntime} time slice(s) in {infile}") + sys.exit(1) + dst.setncatts({k: src.getncattr(k) for k in src.ncattrs()}) + + # Copy ALL dimensions — PyCECT checks nCells/nEdges/nVertices + for dname, dim in src.dimensions.items(): + if dname == 'Time': + dst.createDimension(dname, 1) + else: + dst.createDimension(dname, len(dim)) + + # Identify variables to keep: + # 1. Time-varying variables not in the exclude list + # 2. Static variables required by PyCECT for area weighting + keep_dynamic = {} + keep_static = {} + for name, var in src.variables.items(): + if name in exclude: + continue + if 'Time' in var.dimensions: + keep_dynamic[name] = var + elif name in PYCECT_REQUIRED_STATIC: + keep_static[name] = var + + kept = 0 + + # Write static variables (no time slicing needed) + for name, var in keep_static.items(): + outvar = dst.createVariable(name, var.dtype, var.dimensions) + outvar.setncatts({k: var.getncattr(k) for k in var.ncattrs()}) + outvar[:] = var[:] + kept += 1 + + # Write time-varying variables (extract single time slice, compress) + for name, var in keep_dynamic.items(): + dims = var.dimensions + use_zlib = var.size > 1000 + outvar = dst.createVariable( + name, var.dtype, dims, + zlib=use_zlib, complevel=1) + outvar.setncatts({k: var.getncattr(k) for k in var.ncattrs()}) + + tidx = dims.index('Time') + slices = [slice(None)] * len(dims) + slices[tidx] = slice(tslice, tslice + 1) + outvar[:] = var[tuple(slices)] + kept += 1 + + skipped = len(src.variables) - kept + + in_size = os.path.getsize(infile) / 1048576 + out_size = os.path.getsize(outfile) / 1048576 + print(f"Kept {kept} variables ({len(keep_dynamic)} dynamic + " + f"{len(keep_static)} static), dropped {skipped}, " + f"tslice={tslice}, {in_size:.0f}MB -> {out_size:.0f}MB") + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Trim MPAS history files for ECT') + parser.add_argument('input', help='Input history file') + parser.add_argument('output', help='Output trimmed file') + parser.add_argument('--tslice', type=int, default=-1, + help='Time slice index to extract (negative counts from end, e.g. -1 = last)') + parser.add_argument('--exclude-file', + help='File listing variable names to exclude') + args = parser.parse_args() + + exclude = [] + if args.exclude_file: + with open(args.exclude_file) as f: + exclude = [line.strip() for line in f + if line.strip() and not line.startswith('#')] + + trim_history(args.input, args.output, args.tslice, exclude) diff --git a/.github/actions/setup-nsight-systems/action.yml b/.github/actions/setup-nsight-systems/action.yml new file mode 100644 index 0000000000..9d28e73d33 --- /dev/null +++ b/.github/actions/setup-nsight-systems/action.yml @@ -0,0 +1,56 @@ +name: 'Setup Nsight Systems CLI' +description: | + Ensure a working `nsys` is available for GPU profiling. If the container image + does not provide Nsight Systems (or only an NVHPC stub), installs the + `nsight-systems-cli` RPM set from NVIDIA's devtools repository (RHEL/Alma/Rocky). + Caches downloaded RPMs under `.cache/nsight-systems-rpms` for faster reruns. + +runs: + using: composite + steps: + - id: nsys_ver + shell: bash + run: | + V=1 + if [ -f .github/ci-config.env ]; then + L=$(grep '^NSYS_CLI_CACHE_VERSION=' .github/ci-config.env | head -1 || true) + if [ -n "${L}" ]; then + V="${L#*=}" + V="${V%%#*}" + V="${V%%[[:space:]]*}" + fi + fi + echo "version=${V}" >> "${GITHUB_OUTPUT}" + echo "NSYS_CLI_CACHE_VERSION=${V}" >> "${GITHUB_ENV}" + + - name: Cache nsight-systems-cli RPMs + id: cache-nsys + uses: actions/cache@v5 + with: + path: ${{ github.workspace }}/.cache/nsight-systems-rpms + # Prefix is what you search for under repo Settings → Actions → Caches (not "nsys build"; we cache .rpm files) + key: nsight-systems-cli-rpms-v${{ steps.nsys_ver.outputs.version }}-rhel9-${{ runner.arch }}-${{ hashFiles('.github/scripts/install-nsight-systems-cli.sh', '.github/scripts/resolve-nsys.sh') }} + + - name: Install nsight-systems-cli + shell: bash + env: + NSYS_RPM_CACHE_DIR: ${{ github.workspace }}/.cache/nsight-systems-rpms + run: | + chmod +x .github/scripts/install-nsight-systems-cli.sh + bash .github/scripts/install-nsight-systems-cli.sh + + - name: Nsight RPM cache (for Actions cache UI) + shell: bash + run: | + { + echo "### Nsight Systems CLI — RPM cache" + echo "" + echo "This caches **downloaded NVIDIA RPMs** in \`.cache/nsight-systems-rpms/\`, not a standalone \`nsys\` build. After \`dnf install\`, \`nsys\` is on the container filesystem." + echo "" + echo "In **Settings → Actions → Caches**, search for keys starting with \`nsight-systems-cli-rpms-\`." + echo "" + echo "- **cache-hit** (this run): \`${{ steps.cache-nsys.outputs.cache-hit }}\`" + } >> "${GITHUB_STEP_SUMMARY}" + if [ -d .cache/nsight-systems-rpms ]; then + echo "- **RPM dir size:** \`$(du -sh .cache/nsight-systems-rpms 2>/dev/null || echo unknown)\`" >> "${GITHUB_STEP_SUMMARY}" + fi diff --git a/.github/actions/validate-ect/action.yml b/.github/actions/validate-ect/action.yml new file mode 100644 index 0000000000..c6037eaa89 --- /dev/null +++ b/.github/actions/validate-ect/action.yml @@ -0,0 +1,210 @@ +name: 'Validate ECT' +description: > + Run PyCECT against an ensemble summary file to validate history output. + Installs PyCECT at the pinned version from ci-config.env, downloads the + ensemble summary from a GitHub release, runs pyCECT, and writes an + enriched result file with dimension metadata for the summary action. + +inputs: + history-dir: + description: 'Path to directory containing history .nc files to validate' + required: true + label: + description: 'Human-readable label for log annotations (e.g. gcc/mpich3/smiol/4proc)' + required: true + mpas-version: + description: 'MPAS version string (used to build the ect-v{version} release tag)' + required: true + dimensions: + description: 'Multi-line key=value pairs describing this test combination (written into result file)' + required: false + default: '' + +outputs: + result: + description: 'ECT result: PASSED, FAILED, SKIPPED, or ERROR' + value: ${{ steps.outcome.outputs.result }} + available: + description: 'Whether the ensemble summary file was found (true/false)' + value: ${{ steps.summary.outputs.available }} + +runs: + using: 'composite' + steps: + - name: Load ECT configuration + id: config + shell: bash + run: | + CI_CONFIG=".github/ci-config.env" + if [ ! -f "${CI_CONFIG}" ]; then + echo "::error::ci-config.env not found" + exit 1 + fi + source "${CI_CONFIG}" + + # ECT release tag is derived from the MPAS version (passed in), + # not stored in ci-config.env. See .github/actions/mpas-version. + echo "summary-file=${ECT_SUMMARY_FILE}" >> $GITHUB_OUTPUT + echo "pycect-tag=${PYCECT_TAG}" >> $GITHUB_OUTPUT + echo "pycect-commit=${PYCECT_COMMIT}" >> $GITHUB_OUTPUT + echo "release-tag=ect-v${{ inputs.mpas-version }}" >> $GITHUB_OUTPUT + + - name: Install PyCECT dependencies + shell: bash + run: pip install "numpy<2" scipy netCDF4 + + - name: Clone PyCECT + shell: bash + run: | + TAG="${{ steps.config.outputs.pycect-tag }}" + COMMIT="${{ steps.config.outputs.pycect-commit }}" + echo "Cloning PyCECT at tag ${TAG} (commit ${COMMIT})..." + git clone --branch "${TAG}" https://github.com/NCAR/PyCECT.git pycect + cd pycect + if [ -n "${COMMIT}" ]; then + ACTUAL=$(git rev-parse HEAD) + if [ "${ACTUAL}" != "${COMMIT}" ]; then + echo "::error::PyCECT commit mismatch: expected ${COMMIT}, got ${ACTUAL}" + exit 1 + fi + fi + + - name: Download ensemble summary file + id: summary + shell: bash + run: | + source .github/ci-config.env + SUMMARY="${{ steps.config.outputs.summary-file }}" + TAG="${{ steps.config.outputs.release-tag }}" + REPO="${DATA_REPOSITORY:-${GITHUB_REPOSITORY}}" + URL="https://github.com/${REPO}/releases/download/${TAG}/${SUMMARY}" + + echo "Downloading ${SUMMARY} from release ${TAG}..." + HTTP_CODE=$(curl --retry 5 --retry-delay 5 -sL -w "%{http_code}" \ + "${URL}" -o "${SUMMARY}") + if [ "${HTTP_CODE}" != "200" ]; then + echo "::warning::Ensemble summary not available at ${URL} (HTTP ${HTTP_CODE})" + echo "available=false" >> $GITHUB_OUTPUT + exit 0 + fi + echo "Downloaded summary file: $(du -h ${SUMMARY})" + echo "available=true" >> $GITHUB_OUTPUT + echo "summary-file=${SUMMARY}" >> $GITHUB_OUTPUT + + - name: List test files + if: steps.summary.outputs.available == 'true' + shell: bash + run: | + echo "=== ECT test history files ===" + ls -la ${{ inputs.history-dir }}/ 2>/dev/null || echo "No history files found" + + - name: Run PyCECT + id: pycect + if: steps.summary.outputs.available == 'true' + shell: bash + run: | + LABEL="${{ inputs.label }}" + # --tslice 0 is invariant: history files are pre-trimmed to a single + # time slice by run-perturb-mpas (trim_history.py creates Time=1). + set +e + python pycect/pyCECT.py \ + --sumfile ${{ steps.summary.outputs.summary-file }} \ + --indir ${{ inputs.history-dir }} \ + --tslice 0 \ + --mpas \ + --verbose \ + --printStdMean \ + 2>&1 | tee pycect_output.txt + PYCECT_STATUS=$? + set -e + + if [ $PYCECT_STATUS -ne 0 ]; then + echo "::error::pyCECT crashed with exit code ${PYCECT_STATUS}" + echo "result=ERROR" >> $GITHUB_OUTPUT + exit 1 + fi + + if grep -qF '****PASSED****' pycect_output.txt; then + echo "::notice::ECT (${LABEL}): PASSED" + echo "result=PASSED" >> $GITHUB_OUTPUT + elif grep -qF '****FAILED****' pycect_output.txt; then + echo "::error::ECT (${LABEL}): FAILED" + echo "result=FAILED" >> $GITHUB_OUTPUT + exit 1 + else + echo "::warning::Could not determine ECT result from pyCECT output" + echo "result=ERROR" >> $GITHUB_OUTPUT + exit 1 + fi + + - name: ECT Results + if: steps.pycect.outputs.result != '' + shell: bash + run: | + LABEL="${{ inputs.label }}" + RESULT="${{ steps.pycect.outputs.result }}" + + if [ "${RESULT}" = "PASSED" ]; then + ICON=":white_check_mark:" + else + ICON=":x:" + fi + + { + echo "## ECT Result: ${ICON} ${RESULT}" + echo "" + echo "**Configuration:** \`${LABEL}\`" + echo "" + echo '```' + grep -E '(PASSED|FAILED|global|regional|Overall)' pycect_output.txt || true + echo '```' + if grep -q 'standardized mean' pycect_output.txt; then + echo "" + echo "
Standardized Mean Summary" + echo "" + echo '```' + grep 'standardized mean' pycect_output.txt || true + echo '```' + echo "
" + fi + } >> "$GITHUB_STEP_SUMMARY" + + echo "================================" + echo " ECT Result: ${RESULT}" + echo " Label: ${LABEL}" + echo "================================" + + - name: Fail if summary unavailable + if: steps.summary.outputs.available != 'true' + shell: bash + run: | + { + echo "## ECT Result: :warning: SKIPPED" + echo "" + echo "Ensemble summary file not found. Run \`ect-ensemble-gen.yml\` first." + } >> "$GITHUB_STEP_SUMMARY" + echo "::error::ECT validation cannot run — ensemble summary file not found. Run ect-ensemble-gen.yml first." + exit 1 + + - name: Write result file + id: outcome + if: always() + shell: bash + run: | + RESULT="${{ steps.pycect.outputs.result }}" + if [ -z "${RESULT}" ]; then + if [ "${{ steps.summary.outputs.available }}" != "true" ]; then + RESULT="SKIPPED" + else + RESULT="NO_DATA" + fi + fi + echo "result=${RESULT}" >> $GITHUB_OUTPUT + + { + echo "result=${RESULT}" + DIMS="${{ inputs.dimensions }}" + if [ -n "${DIMS}" ]; then + echo "${DIMS}" + fi + } > ect-result.txt diff --git a/.github/ci-config.env b/.github/ci-config.env new file mode 100644 index 0000000000..d7a13a6a07 --- /dev/null +++ b/.github/ci-config.env @@ -0,0 +1,138 @@ +# .github/ci-config.env — Central CI configuration +# ────────────────────────────────────────────────── +# Workflows and composite actions source this file. +# Edit here to change containers, compiler targets, MPI settings, +# test data versions, and test-specific parameters. + + +# ── Container images ────────────────────────────── +# CI jobs run inside Docker containers from ncarcisl/hpcdev with +# pre-installed compilers, MPI libraries, and I/O libraries. +# +# Image names are assembled from the templates below, replacing +# {compiler} and {mpi} placeholders at workflow start. + +# CPU container template. +CONTAINER_IMAGE="docker.io/ncarcisl/hpcdev-x86_64:almalinux9-{compiler}-{mpi}-26.02" + +# GPU (CUDA) container template. +CONTAINER_IMAGE_GPU="docker.io/ncarcisl/hpcdev-x86_64:almalinux9-{compiler}-{mpi}-cuda-26.02" + +# Per-compiler image overrides (optional). +# Format: CONTAINER_IMAGE_{compiler}="template" + +# ── Compiler / MPI name mappings ───────────────── +# Workflows use short family names (gcc, nvhpc, oneapi) for the +# {compiler} placeholder. If the Docker Hub image tag uses a different +# string, add a mapping here. Otherwise the family name is used as-is. +# +# Check available tags at: +# https://hub.docker.com/r/ncarcisl/hpcdev-x86_64/tags +# +# Example: workflow says "gcc" but the image tag is +# almalinux9-gcc14-mpich-26.02 (not almalinux9-gcc-mpich-26.02) +# so we need: +CONTAINER_COMPILER_gcc=gcc14 + +# Uncomment if an MPI mapping is ever needed: +# CONTAINER_MPI_mpich=mpich3 + + +# ── Compiler → Makefile target mapping ──────────── +# MPAS Makefile targets don't always match CI compiler names. +# Format: MAKE_TARGET_{compiler}={makefile_target} + +MAKE_TARGET_gcc=gfortran +MAKE_TARGET_nvhpc=nvhpc +MAKE_TARGET_oneapi=intel +MAKE_TARGET_clang=llvm + + +# ── Compiler workarounds ───────────────────────── + +# NVHPC: portable target architecture (CI builds and runs on different hosts). +NVHPC_TARGET_ARCH=-tp=px + +# NVHPC: disable MPI Fortran 2008 bindings (broken with MPICH4 CFI support). +# https://github.com/pmodels/mpich/issues/6505 +NVHPC_EXTRA_MAKE_FLAGS="MPAS_MPI_F08=0" + +# Intel/OneAPI: same F08 binding issue with hpcdev MPI libraries. +ONEAPI_EXTRA_MAKE_FLAGS="MPAS_MPI_F08=0" + + +# ── MPI runtime flags ──────────────────────────── +# Required because CI runs inside containers as root. + +# OpenMPI: allow root execution and oversubscribe cores. +# Consumed by .github/actions/run-mpas, .github/actions/run-perturb-mpas, +# and .github/scripts/run-nsys-profile.sh (add a parallel `MPICH_RUN_FLAGS` +# in those same files if MPICH ever needs runtime flags). +OPENMPI_RUN_FLAGS="--allow-run-as-root --oversubscribe" + + +# ── Test data releases ──────────────────────────── +# Test data is stored as GitHub release assets on this repository. +# Each asset is independently versioned. The download-testdata action +# resolves release tags from these variables. +# +# Adding a new test case: +# 1. Create a test case archive (namelist + init.nc + streams, etc.) +# 2. gh release create testdata-{resolution}-v1 {resolution}.tar.gz \ +# --repo NCAR/MPAS-Model-CI +# 3. Add RELEASE_TESTDATA_{RESOLUTION}=testdata-{resolution}-v1 below + +# Repository that hosts release assets (test data, ECT summaries, restarts). +# Forks: leave this as-is to pull data from the upstream NCAR repo. +DATA_REPOSITORY=NCAR/MPAS-Model-CI + +RELEASE_TESTDATA_240KM=testdata-240km-v3 +RELEASE_TESTDATA_120KM=testdata-120km-v2 + +# ── ECT release tag ────────────────────────────── +# The ECT release tag (ect-v{MPAS_VERSION}) is derived at runtime from +# src/core_atmosphere/Registry.xml via the .github/actions/mpas-version +# composite action. There is no manual tag here — that ensures readers +# (validate-ect, _test-compiler, _test-gpu, ect-test) and the writer +# (ect-ensemble-gen) cannot drift out of sync. +# +# To publish ECT data for a new MPAS version, run ect-ensemble-gen.yml. + + +# ── ECT configuration ──────────────────────────── +# Parameters for the Ensemble Consistency Test (PyCECT). +# These are CI-specific settings not carried in the model namelist. +# Reference: Price-Broncucia et al. (2025), doi:10.5194/gmd-18-2349-2025 + +ECT_RESOLUTION=120km +ECT_ENSEMBLE_SIZE=200 +ECT_PERTURB_MAGNITUDE=1e-14 +ECT_PERTURB_VARIABLE=theta +ECT_SUMMARY_FILE=mpas_ect_summary_120km.nc +ECT_RESTART_FILE=120km-spinup-restart.nc +ECT_EXCLUDED_VARS=.github/data/ect_excluded_vars.txt +PYCECT_TAG=3.3.1 +PYCECT_COMMIT=b3c36a9d72ee211f396d1bc7078f6d5466916b0a + + +# ── BFB test configuration ─────────────────────── +# Used by BFB workflows (feature-ci-bfb branch, _test-bfb.yml). +# Bit-for-bit reproducibility tests using the 240km single-precision +# test case with 3 timesteps (config_dt=1200s). + +BFB_RESOLUTION=240km +BFB_PRECISION=single +BFB_RUN_DURATION=0_01:00:00 +BFB_RUN_TIMEOUT=10 + + +# ── Nsight Systems CLI (profile-gpu-nsight workflow) ─ +# Bump NSYS_CLI_CACHE_VERSION to invalidate GitHub Actions cache of downloaded RPMs +# when NVIDIA updates packages in the devtools repo. +NSYS_CLI_CACHE_VERSION=1 + + +# ── KNOWN ISSUES ───────────────────────────────── +# NVHPC+OpenMPI ECT may fail on GitHub-hosted runners (SIGABRT); subset +# workflows use continue-on-error for that combination. +# NVHPC and OneAPI builds set MPAS_MPI_F08=0 (see ONEAPI/NVHPC extra flags above). diff --git a/.github/data/ect_excluded_vars.txt b/.github/data/ect_excluded_vars.txt new file mode 100644 index 0000000000..478c6e786a --- /dev/null +++ b/.github/data/ect_excluded_vars.txt @@ -0,0 +1,35 @@ +# Variables excluded from ECT history files to reduce artifact size. +# These are either not analyzed by PyCECT or are expensive research +# diagnostics not needed for consistency testing. +# +# Ertel PV diagnostics (3D cell fields, ~12MB each on 120km mesh) +ertel_pv +u_pv +v_pv +theta_pv +vort_pv +iLev_DT +# +# PV tendency terms (3D cell fields, only present with physics enabled) +depv_dt_lw +depv_dt_sw +depv_dt_bl +depv_dt_cu +depv_dt_mix +dtheta_dt_mp +depv_dt_mp +depv_dt_diab +depv_dt_fric +depv_dt_diab_pv +depv_dt_fric_pv +# +# Edge velocity — PyCECT recommends uReconstructZonal/Meridional instead +u +# +# Integer variables — PyCECT excludes these automatically +i_rainnc +i_rainc +kpbl +# +# Time metadata string +xtime diff --git a/.github/scripts/compare-bfb-nc.py b/.github/scripts/compare-bfb-nc.py new file mode 100644 index 0000000000..b7f56a9cef --- /dev/null +++ b/.github/scripts/compare-bfb-nc.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Compare two NetCDF files for bitwise-identical variable *data*. + +NetCDF files from PIO vs SMIOL or different MPI layouts often differ in headers, +attributes, or chunking while variable arrays remain identical — `cmp` is too strict. +""" +from __future__ import annotations + +import sys + +import netCDF4 as nc +import numpy as np + + +def as_array(x): + if np.ma.isMaskedArray(x): + return np.ma.filled(x) + return np.asarray(x) + + +def main() -> int: + if len(sys.argv) != 3: + print("usage: compare-bfb-nc.py ", file=sys.stderr) + return 2 + path_ref, path_test = sys.argv[1], sys.argv[2] + diffs: list[str] = [] + with nc.Dataset(path_ref) as ds_ref, nc.Dataset(path_test) as ds_test: + vr = set(ds_ref.variables) + vt = set(ds_test.variables) + if vr != vt: + only_r = sorted(vr - vt) + only_t = sorted(vt - vr) + print( + "FAIL\t" + f"variable set mismatch: only in reference {only_r}, only in candidate {only_t}" + ) + return 1 + for name in sorted(vr): + a = as_array(ds_ref.variables[name][:]) + b = as_array(ds_test.variables[name][:]) + if a.shape != b.shape: + diffs.append(f"{name}: shape {a.shape} vs {b.shape}") + continue + if not np.array_equal(a, b): + if np.issubdtype(a.dtype, np.floating) or np.issubdtype( + a.dtype, np.complexfloating + ): + maxdiff = float(np.max(np.abs(a - b))) + diffs.append(f"{name} (max|diff|={maxdiff:.6e})") + else: + diffs.append(f"{name} (data differs)") + if diffs: + msg = "; ".join(diffs[:25]) + print(f"FAIL\t{msg}") + if len(diffs) > 25: + print(f"... and {len(diffs) - 25} more", file=sys.stderr) + return 1 + print("OK\tall variables bitwise-identical (NetCDF container/metadata may differ)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/install-nsight-systems-cli.sh b/.github/scripts/install-nsight-systems-cli.sh new file mode 100644 index 0000000000..a6f58e582f --- /dev/null +++ b/.github/scripts/install-nsight-systems-cli.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# Install NVIDIA Nsight Systems CLI (nsys) on RHEL/Alma/Rocky via the devtools repo. +# Idempotent: skips if resolve-nsys.sh already finds a working nsys. +# Optional: NSYS_RPM_CACHE_DIR — directory to store .rpm files for actions/cache. +# +# shellcheck shell=bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/resolve-nsys.sh" + +_emit_github_env() { + if [ -n "${GITHUB_ENV:-}" ] && [ -n "${NSYS_BIN:-}" ]; then + { + echo "NSYS_BIN=${NSYS_BIN}" + echo "PATH=${PATH}" + } >> "${GITHUB_ENV}" + fi +} + +if resolve_nsys; then + echo "=== nsys already usable: ${NSYS_BIN} ===" + "${NSYS_BIN}" --version + _emit_github_env + exit 0 +fi + +echo "=== Installing nsight-systems-cli (NVIDIA devtools repo) ===" + +if ! command -v dnf &>/dev/null; then + echo "::error::dnf not found; this installer supports RHEL-family GPU images only." + exit 1 +fi + +# GPG key used by NVIDIA CUDA / devtools RPM repos (RHEL 9) +if [ -f /etc/os-release ]; then + # shellcheck source=/dev/null + source /etc/os-release +else + echo "::error::Cannot read /etc/os-release" + exit 1 +fi + +RHEL_VER="${VERSION_ID%%.*}" +ARCH_DIR="$(rpm --eval '%{_arch}' | sed 's/aarch/arm/')" +REPO_BASE="https://developer.download.nvidia.com/devtools/repos/rhel${RHEL_VER}/${ARCH_DIR}/" + +for key in \ + "https://developer.download.nvidia.com/compute/cuda/repos/rhel${RHEL_VER}/x86_64/D42D0685.pub" \ + "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub" +do + if rpm --import "${key}" 2>/dev/null; then + break + fi +done + +# Same repo layout as Nsight Systems Installation Guide (RHEL / Alma / Rocky). +cat > /etc/yum.repos.d/nvidia-devtools-ci.repo </dev/null; then + while IFS= read -r cand; do + if [ -x "${cand}" ] && "${cand}" --version &>/dev/null; then + export NSYS_BIN="${cand}" + export PATH="$(dirname "${cand}"):${PATH}" + echo "=== Pinned NSYS_BIN to RPM path: ${NSYS_BIN} ===" + break + fi + done < <(rpm -ql nsight-systems-cli 2>/dev/null | grep -E '/nsys$' || true) +fi + +if ! resolve_nsys; then + echo "::error::nsight-systems-cli did not yield a working nsys. Check NVIDIA repo and image OS version." + exit 1 +fi + +echo "=== nsys ready: ${NSYS_BIN} ===" +"${NSYS_BIN}" --version +_emit_github_env diff --git a/.github/scripts/resolve-nsys.sh b/.github/scripts/resolve-nsys.sh new file mode 100644 index 0000000000..188b36f7b4 --- /dev/null +++ b/.github/scripts/resolve-nsys.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Pick a working `nsys` binary. NVHPC may put a stub first on PATH that fails with +# "nsys-Error-Version ... Nsight_Systems/bin is not available in this installation"; +# we prefer /opt/nvidia/nsight-systems or CUDA toolkit paths and test with `nsys --version`. +# +# Usage: source this file from bash, then call resolve_nsys. +# On success: exports NSYS_BIN to the chosen executable. +# shellcheck shell=bash + +resolve_nsys() { + if [ -n "${NSYS_BIN:-}" ] && [ -x "${NSYS_BIN}" ] && "${NSYS_BIN}" --version &>/dev/null; then + export NSYS_BIN + return 0 + fi + + local prepend="" d dir + local -a dirs + + shopt -s nullglob + for d in /opt/nvidia/nsight-systems/*/bin /usr/local/cuda/bin /usr/local/cuda-*/bin; do + [ -d "$d" ] && prepend="${prepend}${d}:" + done + shopt -u nullglob + + export PATH="${prepend}${PATH}" + + IFS=':' read -ra dirs <<< "${PATH}" + for dir in "${dirs[@]}"; do + [ -z "$dir" ] && continue + [ -x "${dir}/nsys" ] || continue + if "${dir}/nsys" --version &>/dev/null; then + NSYS_BIN="${dir}/nsys" + export NSYS_BIN + return 0 + fi + done + + # Explicit paths (some images omit standard entries from PATH) + local c + shopt -s nullglob + for c in /opt/nvidia/nsight-systems/*/bin/nsys /usr/local/cuda/bin/nsys /usr/local/cuda-*/bin/nsys; do + [ -x "$c" ] || continue + if "$c" --version &>/dev/null; then + NSYS_BIN="$c" + export NSYS_BIN + shopt -u nullglob + return 0 + fi + done + shopt -u nullglob + return 1 +} diff --git a/.github/scripts/run-nsys-profile.sh b/.github/scripts/run-nsys-profile.sh new file mode 100644 index 0000000000..ce20625455 --- /dev/null +++ b/.github/scripts/run-nsys-profile.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Run MPAS-A under Nsight Systems (nsys profile). Intended for CIRRUS GPU CI. +# Usage: run-nsys-profile.sh +set -euo pipefail + +WORKDIR="${1:?workdir required}" +NUM_PROCS="${2:?num-procs required}" +MPI_IMPL="${3:?mpi-impl required}" +TIMEOUT="${4:?timeout minutes required}" +NSYS_BASENAME="${5:?nsys output basename required}" + +if [ -f /container/config_env.sh ]; then + # shellcheck source=/dev/null + source /container/config_env.sh +fi + +if [ -z "${LD_LIBRARY_PATH:-}" ]; then + export LD_LIBRARY_PATH="/usr/lib64:/usr/lib" +fi + +REPO_ROOT="${GITHUB_WORKSPACE:-$(git rev-parse --show-toplevel 2>/dev/null || pwd)}" +CI_CONFIG="${REPO_ROOT}/.github/ci-config.env" +if [ -f "${CI_CONFIG}" ]; then + # shellcheck source=/dev/null + source "${CI_CONFIG}" +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/resolve-nsys.sh" + +if ! resolve_nsys; then + echo "::error::No working nsys found. NVHPC may expose a stub that fails with Nsight version errors;" + echo "::error::install a full Nsight Systems build under /opt/nvidia/nsight-systems or ensure CUDA toolkit nsys is on PATH." + exit 1 +fi + +echo "=== nsys (${NSYS_BIN}) ===" +"${NSYS_BIN}" --version + +if [ -n "${GITHUB_ENV:-}" ]; then + { + echo "NSYS_BIN=${NSYS_BIN}" + } >> "${GITHUB_ENV}" +fi + +MPI_FLAGS="" +if [ "${MPI_IMPL}" = "openmpi" ]; then + MPI_FLAGS="${OPENMPI_RUN_FLAGS:---allow-run-as-root --oversubscribe}" +fi + +ulimit -s unlimited 2>/dev/null || true + +cd "${WORKDIR}" + +OUT_ABS="${PWD}/${NSYS_BASENAME}" +echo "=== Nsight profile ===" +echo " workdir: ${WORKDIR}" +echo " ranks: ${NUM_PROCS}" +echo " mpi: ${MPI_IMPL}" +echo " output: ${OUT_ABS}" +echo " timeout: ${TIMEOUT}m" + +set +e +timeout "${TIMEOUT}"m "${NSYS_BIN}" profile \ + --trace=cuda,nvtx,osrt \ + --stats=true \ + -o "${OUT_ABS}" \ + mpirun -n "${NUM_PROCS}" ${MPI_FLAGS} ./atmosphere_model +RUN_STATUS=$? +set -e + +if [ "${RUN_STATUS}" -ne 0 ]; then + echo "::warning::Profiled run exited with status ${RUN_STATUS}" + exit "${RUN_STATUS}" +fi + +echo "=== nsys profile finished ===" +ls -la "${NSYS_BASENAME}".* 2>/dev/null || ls -la ./*.nsys-rep 2>/dev/null || true diff --git a/.github/workflows/_test-bfb.yml b/.github/workflows/_test-bfb.yml new file mode 100644 index 0000000000..aa5ff3732c --- /dev/null +++ b/.github/workflows/_test-bfb.yml @@ -0,0 +1,427 @@ +# Reusable workflow: bit-for-bit reproducibility tests. +# +# Callers pass a JSON array of *variants*: independent run configurations that must +# produce bitwise-identical history variable data. The first variant (or +# reference_index) is the reference; every other variant is compared against it. +# +# Optional input `gpu: 'true'`: all variants use CUDA image + OpenACC + CIRRUS (nvhpc only). +# Alternatively set `gpu: 'false'` and add per-variant `"openacc": true|false` to mix +# CPU (GitHub-hosted) and GPU (CIRRUS) NVHPC builds in one comparison. +# +# MPI rank count and PIO vs SMIOL are common dimensions but not special — add any +# new scenario by appending another variant object (and a small caller workflow). + +name: _test-bfb + +permissions: + contents: read + actions: write + +on: + workflow_call: + inputs: + compiler: + description: 'Compiler family for the container image (gcc, nvhpc, oneapi)' + required: true + type: string + mpi: + description: 'MPI implementation for the container image (mpich, openmpi)' + required: true + type: string + variants: + description: > + JSON array of at least two variant objects. Required per variant: id (unique slug), + ranks (MPI processes). Optional: use_pio (default false), openacc (default follows gpu input), + label (summary text), resolution, run_duration (override workflow defaults for that variant only). + required: true + type: string + resolution: + description: 'Default test case resolution when a variant omits resolution' + required: false + type: string + default: '240km' + precision: + description: 'Floating-point precision for all builds (single or double)' + required: false + type: string + default: 'single' + run-duration: + description: 'Default model run duration (D_HH:MM:SS) when a variant omits run_duration' + required: false + type: string + default: '0_01:00:00' + run-timeout: + description: 'Run step timeout in minutes' + required: false + type: string + default: '10' + reference_index: + description: 'Which variant (0-based) is the reference for comparison' + required: false + type: number + default: 0 + gpu: + description: 'If true, CUDA image + OpenACC build + CIRRUS GPU runners (requires compiler nvhpc). Same security model as _test-gpu — use workflow_dispatch only.' + required: false + type: string + default: 'false' + +jobs: + config: + name: Resolve Config + runs-on: ubuntu-latest + outputs: + build_matrix: ${{ steps.matrices.outputs.build_matrix }} + run_variants: ${{ steps.matrices.outputs.run_variants }} + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: .github + + - name: Validate GPU BFB inputs + if: ${{ inputs.gpu == 'true' }} + shell: bash + run: | + set -euo pipefail + if [ "${{ inputs.compiler }}" != "nvhpc" ]; then + echo "::error::gpu=true requires compiler=nvhpc (got ${{ inputs.compiler }})" + exit 1 + fi + + - name: Parse variants + id: parse + shell: bash + env: + VARIANTS_JSON: ${{ inputs.variants }} + REF_IDX: ${{ inputs.reference_index }} + run: | + set -euo pipefail + RI="${REF_IDX:-0}" + echo "${VARIANTS_JSON}" | jq -e 'type == "array" and length >= 2' >/dev/null \ + || { echo "::error::variants must be a JSON array with at least two entries"; exit 1; } + echo "${VARIANTS_JSON}" | jq -e 'all(.[]; has("id") and has("ranks"))' >/dev/null \ + || { echo "::error::each variant must include id and ranks"; exit 1; } + echo "${VARIANTS_JSON}" | jq -e 'all(.[]; .id | test("^[a-zA-Z0-9._-]+$"))' >/dev/null \ + || { echo "::error::variant id must match ^[a-zA-Z0-9._-]+$"; exit 1; } + N=$(echo "${VARIANTS_JSON}" | jq 'length') + if [ "${RI}" -lt 0 ] || [ "${RI}" -ge "${N}" ]; then + echo "::error::reference_index must be between 0 and $((N - 1))" + exit 1 + fi + + GDEF_JSON='false' + [ "${{ inputs.gpu }}" = "true" ] && GDEF_JSON='true' + NEED_CUDA=$(echo "${VARIANTS_JSON}" | jq -r --argjson gdef "${GDEF_JSON}" 'any(.[]; (.openacc // $gdef))') + echo "need_cuda=${NEED_CUDA}" >> "$GITHUB_OUTPUT" + + - name: Require nvhpc for OpenACC variants + if: ${{ steps.parse.outputs.need_cuda == 'true' }} + shell: bash + run: | + set -euo pipefail + if [ "${{ inputs.compiler }}" != "nvhpc" ]; then + echo "::error::OpenACC variants require compiler=nvhpc (got ${{ inputs.compiler }})" + exit 1 + fi + + - uses: ./.github/actions/resolve-container + id: container_cpu + with: + compiler: ${{ inputs.compiler }} + mpi: ${{ inputs.mpi }} + gpu: '' + + - uses: ./.github/actions/resolve-container + id: container_cuda + if: ${{ steps.parse.outputs.need_cuda == 'true' }} + with: + compiler: ${{ inputs.compiler }} + mpi: ${{ inputs.mpi }} + gpu: cuda + + - name: Compute build and run matrices + id: matrices + shell: bash + env: + VARIANTS_JSON: ${{ inputs.variants }} + DEF_RES: ${{ inputs.resolution }} + DEF_DUR: ${{ inputs.run-duration }} + CPU_IMG: ${{ steps.container_cpu.outputs.image }} + CUDA_IMG: ${{ steps.container_cuda.outputs.image }} + run: | + set -euo pipefail + GDEF_JSON='false' + [ "${{ inputs.gpu }}" = "true" ] && GDEF_JSON='true' + + if [ "${{ steps.parse.outputs.need_cuda }}" = "true" ] && [ -z "${CUDA_IMG}" ]; then + echo "::error::CUDA container image missing but OpenACC variants are present" + exit 1 + fi + + MIXED=$(echo "${VARIANTS_JSON}" | jq -c --argjson gdef "${GDEF_JSON}" \ + '([.[] | (.openacc // $gdef)] | unique | length) > 1') + + BUILD_MATRIX=$(echo "${VARIANTS_JSON}" | jq -c --argjson gdef "${GDEF_JSON}" \ + --argjson mixed "${MIXED}" --arg cpuimg "${CPU_IMG}" --arg cudaimg "${CUDA_IMG}" ' + [.[] | { + build_tag: (if (.use_pio // false) then "pio" else "smiol" end), + openacc: (.openacc // $gdef) + }] + | unique_by("\(.build_tag)-\(.openacc)") + | map(. + { + image: (if .openacc then $cudaimg else $cpuimg end), + artifact_name: ( + if $mixed then + ("exe-bfb-" + .build_tag + "-" + (if .openacc then "openacc" else "cpu" end)) + else + ("exe-bfb-" + .build_tag) + end + ) + }) + ') + + RUN_VARIANTS=$(echo "${VARIANTS_JSON}" | jq -c --argjson gdef "${GDEF_JSON}" \ + --argjson mixed "${MIXED}" --arg cpuimg "${CPU_IMG}" --arg cudaimg "${CUDA_IMG}" \ + --arg defres "${DEF_RES}" --arg defdur "${DEF_DUR}" \ + '[.[] | + (if (.use_pio // false) then "pio" else "smiol" end) as $bt | + (.openacc // $gdef) as $oa | + { + id, + build_tag: $bt, + openacc: $oa, + ranks: (.ranks | tonumber), + resolution: (.resolution // $defres), + run_duration: (.run_duration // $defdur), + label: (.label // .id), + image: (if $oa then $cudaimg else $cpuimg end), + artifact_name: ( + if $mixed then + ("exe-bfb-" + $bt + "-" + (if $oa then "openacc" else "cpu" end)) + else + ("exe-bfb-" + $bt) + end + ) + } + ]') + + echo "build_matrix=${BUILD_MATRIX}" >> "$GITHUB_OUTPUT" + echo "run_variants=${RUN_VARIANTS}" >> "$GITHUB_OUTPUT" + echo "mixed_openacc_modes=${MIXED}" >> "$GITHUB_OUTPUT" + echo "Build matrix: ${BUILD_MATRIX}" + echo "Run matrix: ${RUN_VARIANTS}" + + build: + needs: config + strategy: + fail-fast: true + matrix: + include: ${{ fromJSON(needs.config.outputs.build_matrix) }} + name: Build (${{ matrix.build_tag }}, ${{ matrix.openacc && 'openacc' || 'cpu' }}) + runs-on: ${{ matrix.openacc && fromJSON('{"group":"CIRRUS-4x8-gpu"}') || 'ubuntu-latest' }} + container: + image: ${{ matrix.image }} + steps: + - uses: actions/checkout@v5 + with: + submodules: 'true' + + - name: Build MPAS-A + uses: ./.github/actions/build-mpas + with: + compiler: ${{ inputs.compiler }} + use-pio: ${{ matrix.build_tag == 'pio' && 'true' || 'false' }} + openacc: ${{ matrix.openacc && 'true' || 'false' }} + precision: ${{ inputs.precision }} + + - name: Upload executable + uses: actions/upload-artifact@v6 + with: + name: ${{ matrix.artifact_name }} + path: atmosphere_model + retention-days: 1 + + run: + needs: [config, build] + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.config.outputs.run_variants) }} + name: Run (${{ matrix.id }}) + runs-on: ${{ matrix.openacc && fromJSON('{"group":"CIRRUS-4x8-gpu"}') || 'ubuntu-latest' }} + container: + image: ${{ matrix.image }} + steps: + - uses: actions/checkout@v5 + + - name: Download executable + uses: actions/download-artifact@v7 + with: + name: ${{ matrix.artifact_name }} + + - name: Check GPU availability + if: ${{ matrix.openacc }} + run: | + echo "=== GPU Information ===" + nvidia-smi || echo "WARNING: nvidia-smi failed" + + - name: Run MPAS-A + uses: ./.github/actions/run-mpas + with: + executable: ./atmosphere_model + num-procs: '${{ matrix.ranks }}' + resolution: ${{ matrix.resolution }} + run-duration: ${{ matrix.run_duration }} + run-timeout: ${{ inputs.run-timeout }} + mpi-impl: ${{ inputs.mpi }} + working-dir: run-${{ matrix.id }} + strict-exit-check: 'false' + + - name: Upload history output + uses: actions/upload-artifact@v6 + with: + name: bfb-history-${{ matrix.id }} + path: run-${{ matrix.id }}/history.*.nc + retention-days: 1 + + compare: + needs: [config, run] + if: ${{ !cancelled() && needs.run.result == 'success' }} + name: Compare BFB + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: .github + + - name: Download all history files + uses: actions/download-artifact@v7 + with: + pattern: bfb-history-* + path: bfb-results + merge-multiple: false + + - name: Compare outputs + env: + VARIANTS_JSON: ${{ inputs.variants }} + REFERENCE_INDEX: ${{ inputs.reference_index }} + shell: bash + run: | + pip install --quiet netCDF4 numpy + + RI="${REFERENCE_INDEX:-0}" + + MERGED=$(echo "${VARIANTS_JSON}" | jq -c \ + --arg defres "${{ inputs.resolution }}" \ + --arg defdur "${{ inputs.run-duration }}" \ + '[.[] | { + id, + build_tag: (if (.use_pio // false) then "pio" else "smiol" end), + ranks: (.ranks | tonumber), + resolution: (.resolution // $defres), + run_duration: (.run_duration // $defdur), + label: (.label // .id) + }]') + + REF_ID=$(echo "${MERGED}" | jq -r --argjson idx "${RI}" '.[$idx].id') + REF_LABEL=$(echo "${MERGED}" | jq -r --argjson idx "${RI}" '.[$idx].label') + + echo "## Bit-for-Bit Comparison" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "Reference variant: **${REF_LABEL}** (\`${REF_ID}\`)" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + + PASS=0 + FAIL=0 + ERRORS="" + + REF_DIR="bfb-results/bfb-history-${REF_ID}" + REF_FILE=$(ls ${REF_DIR}/history.*.nc 2>/dev/null | head -1) + if [ -z "${REF_FILE}" ]; then + echo "::error::Reference history file not found in ${REF_DIR}" + find bfb-results -name '*.nc' -type f + exit 1 + fi + echo "Reference file: ${REF_FILE}" + + compare_files() { + local label="$1" + local test_file="$2" + local ref_file="$3" + + if [ ! -f "${test_file}" ]; then + echo "::error::${label}: history file missing" + ERRORS="${ERRORS}\n- **${label}**: :x: history file missing" + FAIL=$((FAIL + 1)) + return + fi + + if cmp -s "${ref_file}" "${test_file}"; then + echo "${label}: byte-identical" + echo "- **${label}**: :white_check_mark: byte-identical" >> "$GITHUB_STEP_SUMMARY" + PASS=$((PASS + 1)) + return + fi + + # compare-bfb-nc.py exits 1 on mismatch; with `set -e`, command substitution + # would abort before STATUS/DETAIL are handled — temporarily disable -e. + set +e + RESULT=$(python3 "${GITHUB_WORKSPACE}/.github/scripts/compare-bfb-nc.py" "${ref_file}" "${test_file}") + STATUS=$? + set -e + IFS=$'\t' read -r CODE DETAIL <<< "${RESULT}" + + if [ "${STATUS}" -eq 0 ] && [ "${CODE}" = "OK" ]; then + echo "${label}: data-identical (${DETAIL})" + echo "- **${label}**: :white_check_mark: data-identical (${DETAIL})" >> "$GITHUB_STEP_SUMMARY" + PASS=$((PASS + 1)) + else + echo "${label}: DIFFER — ${DETAIL}" + FAIL=$((FAIL + 1)) + ERRORS="${ERRORS}\n- **${label}**: :x: ${DETAIL}" + fi + } + + N=$(echo "${MERGED}" | jq 'length') + for ((i = 0; i < N; i++)); do + VID=$(echo "${MERGED}" | jq -r --argjson idx "$i" '.[$idx].id') + if [ "${VID}" = "${REF_ID}" ]; then + continue + fi + VLABEL=$(echo "${MERGED}" | jq -r --argjson idx "$i" '.[$idx].label') + DIR="bfb-results/bfb-history-${VID}" + TEST_FILE=$(ls ${DIR}/history.*.nc 2>/dev/null | head -1) + compare_files "${VLABEL} (\`${VID}\`)" "${TEST_FILE}" "${REF_FILE}" + done + + echo "" >> "$GITHUB_STEP_SUMMARY" + if [ -n "${ERRORS}" ]; then + echo -e "${ERRORS}" >> "$GITHUB_STEP_SUMMARY" + fi + + echo "" >> "$GITHUB_STEP_SUMMARY" + TOTAL=$((PASS + FAIL)) + echo "**Result: ${PASS}/${TOTAL} comparisons identical**" >> "$GITHUB_STEP_SUMMARY" + + if [ ${FAIL} -gt 0 ]; then + echo "::error::BFB comparison failed: ${FAIL} of ${TOTAL} comparisons differ" + exit 1 + fi + echo "All ${TOTAL} comparisons passed (byte-identical files or bitwise-identical variable data)" + + cleanup: + needs: [run, compare] + if: always() + runs-on: ubuntu-latest + name: Cleanup + steps: + - name: Delete temporary artifacts + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts \ + --paginate --jq '.artifacts[] | select(.name | startswith("exe-bfb-") or startswith("bfb-history-")) | .id' \ + | while read id; do + gh api -X DELETE repos/${{ github.repository }}/actions/artifacts/$id || true + done || true diff --git a/.github/workflows/_test-compiler.yml b/.github/workflows/_test-compiler.yml new file mode 100644 index 0000000000..bd132eb139 --- /dev/null +++ b/.github/workflows/_test-compiler.yml @@ -0,0 +1,258 @@ +# Reusable workflow: build and validate MPAS-A for a single compiler+MPI using ECT. +# Called by per-compiler per-MPI subset workflows (e.g. test-gcc-mpich.yml). +# +# Runs ECT with 4 MPI ranks. Validation uses the Ensemble Consistency Test +# (PyCECT) — perturbed runs compared against a pre-built ensemble summary — +# instead of bit-for-bit log comparison. + +name: _test-compiler + +permissions: + contents: read + +on: + workflow_call: + inputs: + compiler: + description: 'Compiler family (gcc, nvhpc, oneapi)' + required: true + type: string + mpi: + description: 'MPI implementation (mpich, openmpi)' + required: true + type: string + mpas-repository: + description: 'MPAS source repo (e.g. MPAS-Dev/MPAS-Model). Empty = this repo.' + required: false + type: string + default: '' + mpas-ref: + description: 'Git ref in the MPAS source repo (branch, tag, SHA)' + required: false + type: string + default: '' + +jobs: + config: + name: Resolve Config + runs-on: ubuntu-latest + outputs: + image: ${{ steps.container.outputs.image }} + mpas-version: ${{ steps.version.outputs.version }} + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + sparse-checkout: | + .github + src/core_atmosphere/Registry.xml + sparse-checkout-cone-mode: false + persist-credentials: false + - uses: ./.github/actions/resolve-container + id: container + with: + compiler: ${{ inputs.compiler }} + mpi: ${{ inputs.mpi }} + - uses: ./.github/actions/mpas-version + id: version + + build: + needs: config + name: Build (${{ inputs.compiler }}, ${{ inputs.mpi }}, smiol) + runs-on: ubuntu-latest + container: + image: ${{ needs.config.outputs.image }} + + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + repository: ${{ inputs.mpas-repository || github.repository }} + ref: ${{ inputs.mpas-ref || '' }} + submodules: 'true' + persist-credentials: false + + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + if: ${{ inputs.mpas-repository != '' }} + with: + path: _ci + sparse-checkout: .github + persist-credentials: false + + - name: Overlay CI infrastructure + if: ${{ inputs.mpas-repository != '' }} + shell: bash + run: | + cp -r _ci/.github . && rm -rf _ci + echo "## Source: ${{ inputs.mpas-repository }}@${{ inputs.mpas-ref }}" >> "$GITHUB_STEP_SUMMARY" + + - name: Build MPAS-A (double precision) + uses: ./.github/actions/build-mpas + with: + compiler: ${{ inputs.compiler }} + use-pio: 'false' + precision: double + + - name: Upload executable + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: exe-${{ inputs.compiler }}-${{ inputs.mpi }}-smiol + path: atmosphere_model + retention-days: 1 + + ect-run: + needs: [config, build] + if: ${{ needs.build.result == 'success' }} + strategy: + fail-fast: false + matrix: + member: [0, 1, 2] + + name: ECT member ${{ matrix.member }} (${{ inputs.compiler }}, ${{ inputs.mpi }}) + runs-on: ubuntu-latest + container: + image: ${{ needs.config.outputs.image }} + + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + persist-credentials: false + + - name: Download executable + id: download + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 + continue-on-error: true + with: + name: exe-${{ inputs.compiler }}-${{ inputs.mpi }}-smiol + + - name: Download test case + if: steps.download.outcome == 'success' + uses: ./.github/actions/download-testdata + with: + resolution: 120km + dest-dir: base-case + + - name: Restore cached restart + if: steps.download.outcome == 'success' + id: cache-restart + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: spinup-restart.nc + key: ect-spinup-restart-${{ hashFiles('.github/ci-config.env') }} + + - name: Download spin-up restart + if: steps.download.outcome == 'success' + id: restart + shell: bash + run: | + source .github/ci-config.env + RESTART="${ECT_RESTART_FILE}" + RELEASE_TAG="ect-v${{ needs.config.outputs.mpas-version }}" + + if [ -f "spinup-restart.nc" ]; then + echo "Using cached restart file" + mv spinup-restart.nc "${RESTART}" + echo "available=true" >> $GITHUB_OUTPUT + echo "file=${RESTART}" >> $GITHUB_OUTPUT + exit 0 + fi + + DATA_REPO="${DATA_REPOSITORY:-${GITHUB_REPOSITORY}}" + echo "Downloading ${RESTART}.gz from ${DATA_REPO} release ${RELEASE_TAG}..." + HTTP_CODE=$(curl -sL --retry 5 --retry-delay 5 -w "%{http_code}" \ + "https://github.com/${DATA_REPO}/releases/download/${RELEASE_TAG}/${RESTART}.gz" \ + -o "${RESTART}.gz") + if [ "${HTTP_CODE}" = "200" ]; then + gunzip "${RESTART}.gz" + echo "Downloaded restart: $(du -h ${RESTART})" + echo "available=true" >> $GITHUB_OUTPUT + echo "file=${RESTART}" >> $GITHUB_OUTPUT + else + echo "::warning::Spin-up restart not available (HTTP ${HTTP_CODE}), running from cold-start init.nc" + echo "available=false" >> $GITHUB_OUTPUT + fi + + - name: Run perturbed MPAS-A (member ${{ matrix.member }}) + if: steps.download.outcome == 'success' + uses: ./.github/actions/run-perturb-mpas + with: + base-dir: base-case + executable: ./atmosphere_model + member-start: '${{ matrix.member }}' + member-end: '${{ matrix.member }}' + num-ranks: '4' + mpi-impl: ${{ inputs.mpi }} + run-duration: '0_02:36:00' + run-timeout: '45' + restart-file: ${{ steps.restart.outputs.available == 'true' && steps.restart.outputs.file || '' }} + + - name: Upload history file + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + if: always() && steps.download.outcome == 'success' + with: + name: ect-history-${{ inputs.compiler }}-${{ inputs.mpi }}-member${{ matrix.member }} + path: history-output/history.*.nc + retention-days: 1 + + ect-validate: + needs: [config, ect-run] + if: ${{ needs.ect-run.result == 'success' }} + name: ECT Validate (${{ inputs.compiler }}, ${{ inputs.mpi }}) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + sparse-checkout: .github + persist-credentials: false + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + with: + python-version: '3.11' + + - name: Download history files + id: download + continue-on-error: true + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 + with: + pattern: ect-history-${{ inputs.compiler }}-${{ inputs.mpi }}-* + path: ect-test-files + merge-multiple: true + + - name: Run ECT validation + if: steps.download.outcome == 'success' + uses: ./.github/actions/validate-ect + with: + history-dir: ect-test-files + label: ${{ inputs.compiler }}/${{ inputs.mpi }}/smiol + mpas-version: ${{ needs.config.outputs.mpas-version }} + dimensions: | + compiler=${{ inputs.compiler }} + mpi=${{ inputs.mpi }} + io=smiol + + - name: Upload result + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: ect-result-${{ inputs.compiler }}-${{ inputs.mpi }} + path: ect-result.txt + retention-days: 1 + + cleanup: + needs: [ect-run, ect-validate] + if: always() + runs-on: ubuntu-latest + name: Cleanup + permissions: + actions: write + + steps: + - name: Delete temporary artifacts + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts \ + --paginate --jq '.artifacts[] | select(.name | startswith("exe-") or startswith("ect-history-")) | .id' \ + | while read id; do + gh api -X DELETE repos/${{ github.repository }}/actions/artifacts/$id || true + done || true diff --git a/.github/workflows/_test-gpu.yml b/.github/workflows/_test-gpu.yml new file mode 100644 index 0000000000..42735c0bfb --- /dev/null +++ b/.github/workflows/_test-gpu.yml @@ -0,0 +1,272 @@ +# Reusable workflow: GPU (CUDA) validation via ECT (NVHPC). +# Called by test-gpu-mpich.yml and test-gpu-openmpi.yml. +# +# Builds MPAS-A with OpenACC/CUDA on CIRRUS self-hosted runners, then +# validates via ECT (PyCECT) — the same statistical test used for CPU +# subsets. GPU results are not expected to be bit-for-bit identical to CPU. +# +# Security: Runs on self-hosted CIRRUS runners. Callers must use workflow_dispatch +# only — never push or pull_request triggers (fork / unreviewed code risk on CIRRUS). + +name: _test-gpu + +on: + workflow_call: + inputs: + mpi: + description: 'MPI implementation (mpich, openmpi)' + required: true + type: string + # Cross-repo inputs disabled until NCAR security review of running + # external code on self-hosted CIRRUS runners is complete. + # mpas-repository: + # description: 'MPAS source repo (e.g. MPAS-Dev/MPAS-Model). Empty = this repo.' + # required: false + # type: string + # default: '' + # mpas-ref: + # description: 'Git ref in the MPAS source repo (branch, tag, SHA)' + # required: false + # type: string + # default: '' + +jobs: + config: + name: Resolve Config + runs-on: ubuntu-latest + outputs: + image: ${{ steps.container.outputs.image }} + image-gpu: ${{ steps.gpu-container.outputs.image }} + mpas-version: ${{ steps.version.outputs.version }} + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: | + .github + src/core_atmosphere/Registry.xml + sparse-checkout-cone-mode: false + - uses: ./.github/actions/resolve-container + id: container + with: + compiler: nvhpc + mpi: ${{ inputs.mpi }} + - uses: ./.github/actions/resolve-container + id: gpu-container + with: + compiler: nvhpc + mpi: ${{ inputs.mpi }} + gpu: cuda + - uses: ./.github/actions/mpas-version + id: version + + build: + needs: config + name: Build (cuda, ${{ inputs.mpi }}) + runs-on: + group: CIRRUS-4x8-gpu + container: + image: ${{ needs.config.outputs.image-gpu }} + + steps: + - uses: actions/checkout@v5 + with: + submodules: 'true' + + # Cross-repo checkout disabled until NCAR security review is complete. + # To re-enable, restore mpas-repository/mpas-ref inputs above and + # replace the checkout step with: + # + # - uses: actions/checkout@v5 + # with: + # repository: ${{ inputs.mpas-repository || github.repository }} + # ref: ${{ inputs.mpas-ref || '' }} + # submodules: 'true' + # + # - uses: actions/checkout@v5 + # if: ${{ inputs.mpas-repository != '' }} + # with: + # path: _ci + # sparse-checkout: .github + # + # - name: Overlay CI infrastructure + # if: ${{ inputs.mpas-repository != '' }} + # shell: bash + # run: | + # cp -r _ci/.github . && rm -rf _ci + # echo "## Source: ${{ inputs.mpas-repository }}@${{ inputs.mpas-ref }}" >> "$GITHUB_STEP_SUMMARY" + + - name: Build MPAS-A (double precision, OpenACC) + uses: ./.github/actions/build-mpas + with: + compiler: nvhpc + use-pio: 'false' + openacc: 'true' + precision: double + + - name: Upload executable + uses: actions/upload-artifact@v6 + with: + name: exe-gpu-cuda-${{ inputs.mpi }} + path: atmosphere_model + retention-days: 1 + + ect-run: + needs: [config, build] + if: ${{ needs.build.result == 'success' }} + strategy: + fail-fast: false + matrix: + member: [0, 1, 2] + + name: ECT member ${{ matrix.member }} (cuda, ${{ inputs.mpi }}) + runs-on: + group: CIRRUS-4x8-gpu + container: + image: ${{ needs.config.outputs.image-gpu }} + + steps: + - uses: actions/checkout@v5 + + - name: Check GPU availability + run: | + echo "=== GPU Information ===" + nvidia-smi || echo "WARNING: nvidia-smi failed" + + - name: Download executable + id: download + uses: actions/download-artifact@v7 + continue-on-error: true + with: + name: exe-gpu-cuda-${{ inputs.mpi }} + + - name: Download test case + if: steps.download.outcome == 'success' + uses: ./.github/actions/download-testdata + with: + resolution: 120km + dest-dir: base-case + + - name: Restore cached restart + if: steps.download.outcome == 'success' + id: cache-restart + uses: actions/cache/restore@v5 + with: + path: spinup-restart.nc + key: ect-spinup-restart-${{ hashFiles('.github/ci-config.env') }} + + - name: Download spin-up restart + if: steps.download.outcome == 'success' + id: restart + shell: bash + run: | + source .github/ci-config.env + RESTART="${ECT_RESTART_FILE}" + RELEASE_TAG="ect-v${{ needs.config.outputs.mpas-version }}" + + if [ -f "spinup-restart.nc" ]; then + echo "Using cached restart file" + mv spinup-restart.nc "${RESTART}" + echo "available=true" >> $GITHUB_OUTPUT + echo "file=${RESTART}" >> $GITHUB_OUTPUT + exit 0 + fi + + DATA_REPO="${DATA_REPOSITORY:-${GITHUB_REPOSITORY}}" + echo "Downloading ${RESTART}.gz from ${DATA_REPO} release ${RELEASE_TAG}..." + HTTP_CODE=$(curl -sL --retry 5 --retry-delay 5 -w "%{http_code}" \ + "https://github.com/${DATA_REPO}/releases/download/${RELEASE_TAG}/${RESTART}.gz" \ + -o "${RESTART}.gz") + if [ "${HTTP_CODE}" = "200" ]; then + gunzip "${RESTART}.gz" + echo "Downloaded restart: $(du -h ${RESTART})" + echo "available=true" >> $GITHUB_OUTPUT + echo "file=${RESTART}" >> $GITHUB_OUTPUT + else + echo "::warning::Spin-up restart not available (HTTP ${HTTP_CODE}), running from cold-start init.nc" + echo "available=false" >> $GITHUB_OUTPUT + fi + + - name: Run perturbed MPAS-A (member ${{ matrix.member }}) + if: steps.download.outcome == 'success' + uses: ./.github/actions/run-perturb-mpas + with: + base-dir: base-case + executable: ./atmosphere_model + member-start: '${{ matrix.member }}' + member-end: '${{ matrix.member }}' + num-ranks: '4' + mpi-impl: ${{ inputs.mpi }} + run-duration: '0_02:36:00' + run-timeout: '45' + restart-file: ${{ steps.restart.outputs.available == 'true' && steps.restart.outputs.file || '' }} + + - name: Upload history file + uses: actions/upload-artifact@v6 + if: always() && steps.download.outcome == 'success' + with: + name: ect-history-gpu-cuda-${{ inputs.mpi }}-member${{ matrix.member }} + path: history-output/history.*.nc + retention-days: 1 + + ect-validate: + needs: [config, ect-run] + if: ${{ needs.ect-run.result == 'success' }} + name: ECT Validate (cuda, ${{ inputs.mpi }}) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: .github + + - uses: actions/setup-python@v6 + with: + python-version: '3.11' + + - name: Download history files + id: download + continue-on-error: true + uses: actions/download-artifact@v7 + with: + pattern: ect-history-gpu-cuda-${{ inputs.mpi }}-* + path: ect-test-files + merge-multiple: true + + - name: Run ECT validation + if: steps.download.outcome == 'success' + uses: ./.github/actions/validate-ect + with: + history-dir: ect-test-files + label: nvhpc/${{ inputs.mpi }}/cuda/smiol + mpas-version: ${{ needs.config.outputs.mpas-version }} + dimensions: | + compiler=nvhpc + mpi=${{ inputs.mpi }} + gpu=cuda + io=smiol + + - name: Upload result + if: always() + uses: actions/upload-artifact@v6 + with: + name: ect-result-gpu-cuda-${{ inputs.mpi }} + path: ect-result.txt + retention-days: 1 + + cleanup: + needs: [ect-run, ect-validate] + if: always() + runs-on: ubuntu-latest + name: Cleanup + + steps: + - name: Delete temporary artifacts + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts \ + --paginate --jq '.artifacts[] | select(.name | startswith("exe-") or startswith("ect-history-")) | .id' \ + | while read id; do + gh api -X DELETE repos/${{ github.repository }}/actions/artifacts/$id || true + done || true diff --git a/.github/workflows/bfb-decomp-gpu.yml b/.github/workflows/bfb-decomp-gpu.yml new file mode 100644 index 0000000000..a3bd6e45d7 --- /dev/null +++ b/.github/workflows/bfb-decomp-gpu.yml @@ -0,0 +1,20 @@ +# BFB on GPU: same OpenACC/CUDA build, different MPI rank counts (decomposition). +# CIRRUS self-hosted runners — trigger via workflow_dispatch only (see _test-gpu.yml). + +name: "BFB: Decomposition GPU (1 vs 4 ranks)" + +on: + workflow_dispatch: + +jobs: + test: + uses: ./.github/workflows/_test-bfb.yml + with: + compiler: nvhpc + mpi: mpich + gpu: 'true' + precision: single + run-timeout: '20' + variants: >- + [{"id":"r1","ranks":1,"label":"1 rank"}, + {"id":"r4","ranks":4,"label":"4 ranks"}] diff --git a/.github/workflows/bfb-decomp.yml b/.github/workflows/bfb-decomp.yml new file mode 100644 index 0000000000..438dea26d5 --- /dev/null +++ b/.github/workflows/bfb-decomp.yml @@ -0,0 +1,19 @@ +# Example BFB caller: same I/O build, different MPI rank counts (decomposition). +# Add new scenarios by copying this file and editing `variants` (see `_test-bfb.yml` and `.github/ci-config.env` BFB_*). + +name: "BFB: Decomposition (1 vs 4 ranks)" + +on: + workflow_dispatch: + push: + branches: [hackathon, 'hackathon/**', 'hackathon-*', feature-ci-bfb] + +jobs: + test: + uses: ./.github/workflows/_test-bfb.yml + with: + compiler: gcc + mpi: mpich + variants: >- + [{"id":"r1","ranks":1,"label":"1 rank"}, + {"id":"r4","ranks":4,"label":"4 ranks"}] diff --git a/.github/workflows/bfb-io-gpu.yml b/.github/workflows/bfb-io-gpu.yml new file mode 100644 index 0000000000..34fe2967cb --- /dev/null +++ b/.github/workflows/bfb-io-gpu.yml @@ -0,0 +1,20 @@ +# BFB on GPU: two I/O builds (SMIOL vs PIO), same MPI rank count. +# CIRRUS self-hosted runners — trigger via workflow_dispatch only (see _test-gpu.yml). + +name: "BFB: I/O GPU (SMIOL vs PIO)" + +on: + workflow_dispatch: + +jobs: + test: + uses: ./.github/workflows/_test-bfb.yml + with: + compiler: nvhpc + mpi: mpich + gpu: 'true' + precision: single + run-timeout: '20' + variants: >- + [{"id":"smiol-r4","ranks":4,"label":"SMIOL, 4 ranks"}, + {"id":"pio-r4","use_pio":true,"ranks":4,"label":"PIO, 4 ranks"}] diff --git a/.github/workflows/bfb-io.yml b/.github/workflows/bfb-io.yml new file mode 100644 index 0000000000..9eb2242f5a --- /dev/null +++ b/.github/workflows/bfb-io.yml @@ -0,0 +1,19 @@ +# Example BFB caller: two I/O builds (SMIOL vs PIO), same rank count. +# Add new scenarios by copying this file and editing `variants` (see `_test-bfb.yml` and `.github/ci-config.env` BFB_*). + +name: "BFB: I/O (SMIOL vs PIO)" + +on: + workflow_dispatch: + push: + branches: [hackathon, 'hackathon/**', 'hackathon-*', feature-ci-bfb] + +jobs: + test: + uses: ./.github/workflows/_test-bfb.yml + with: + compiler: gcc + mpi: mpich + variants: >- + [{"id":"smiol-r4","ranks":4,"label":"SMIOL, 4 ranks"}, + {"id":"pio-r4","use_pio":true,"ranks":4,"label":"PIO, 4 ranks"}] diff --git a/.github/workflows/bfb-nvhpc-cpu-vs-gpu.yml b/.github/workflows/bfb-nvhpc-cpu-vs-gpu.yml new file mode 100644 index 0000000000..78d477a7e4 --- /dev/null +++ b/.github/workflows/bfb-nvhpc-cpu-vs-gpu.yml @@ -0,0 +1,24 @@ +# BFB: NVHPC without OpenACC (GitHub-hosted) vs NVHPC + OpenACC (CIRRUS GPU). +# Same MPI, rank count, resolution, and single precision — compares history variable +# data bitwise. CPU vs GPU OpenACC often differs; this workflow makes that visible. +# +# workflow_dispatch for routine use (after this file is on the default branch). +# GPU/OpenACC comparisons remain dispatch-only so PR code cannot run on CIRRUS. + +name: "BFB: NVHPC CPU vs GPU (OpenACC)" + +on: + workflow_dispatch: + +jobs: + test: + uses: ./.github/workflows/_test-bfb.yml + with: + compiler: nvhpc + mpi: mpich + gpu: 'false' + precision: single + run-timeout: '20' + variants: >- + [{"id":"cpu","ranks":4,"openacc":false,"label":"NVHPC CPU (no OpenACC)"}, + {"id":"gpu","ranks":4,"openacc":true,"label":"NVHPC GPU (OpenACC)"}] diff --git a/.github/workflows/compile-nvhpc-cuda-mpich.yml b/.github/workflows/compile-nvhpc-cuda-mpich.yml new file mode 100644 index 0000000000..6bdffda53b --- /dev/null +++ b/.github/workflows/compile-nvhpc-cuda-mpich.yml @@ -0,0 +1,58 @@ +# NVHPC + MPICH + CUDA: compile-only on GitHub-hosted runners. +# Validates the OpenACC/CUDA toolchain (no GPU present; no model run). + +name: "NVHPC+MPICH+CUDA (compile-only)" + +permissions: + contents: read + +on: + workflow_dispatch: + push: + branches: [master, 'hackathon-*'] + pull_request: + branches: [master, 'hackathon-*'] + +jobs: + config: + name: Resolve CUDA container + runs-on: ubuntu-latest + outputs: + image: ${{ steps.gpu.outputs.image }} + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: .github + - uses: ./.github/actions/resolve-container + id: gpu + with: + compiler: nvhpc + mpi: mpich + gpu: cuda + + compile: + needs: config + name: Compile (OpenACC, MPICH) + runs-on: ubuntu-latest + container: + image: ${{ needs.config.outputs.image }} + + steps: + - uses: actions/checkout@v5 + with: + submodules: 'true' + + - name: Build MPAS-A (double precision, OpenACC) + uses: ./.github/actions/build-mpas + with: + compiler: nvhpc + use-pio: 'false' + openacc: 'true' + precision: double + build-timeout: '45' + + - name: Summary + shell: bash + run: | + echo "## NVHPC + MPICH + CUDA compile-only" >> "$GITHUB_STEP_SUMMARY" + echo "Built \`atmosphere_model\` with OpenACC in ${{ needs.config.outputs.image }}." >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 0000000000..0acd7c9981 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,125 @@ +# Code Coverage workflow +# Builds MPAS-A with GCC coverage instrumentation, runs the 240km test case, +# generates an lcov report, and uploads to Codecov. +# +name: Code Coverage + +on: + workflow_dispatch: + inputs: + mpas-repository: + description: 'MPAS source repo (e.g. MPAS-Dev/MPAS-Model). Leave empty to use this repo.' + required: false + default: '' + mpas-ref: + description: 'Git ref (branch, tag, or SHA) in the MPAS source repo' + required: false + default: '' + push: + branches: [master] + +jobs: + config: + name: Resolve Config + runs-on: ubuntu-latest + outputs: + image: ${{ steps.container.outputs.image }} + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: .github + - uses: ./.github/actions/resolve-container + id: container + with: + compiler: gcc + mpi: mpich + + coverage: + name: Build, Run, and Generate Coverage + needs: config + runs-on: ubuntu-latest + container: + image: ${{ needs.config.outputs.image }} + + steps: + - uses: actions/checkout@v5 + with: + repository: ${{ inputs.mpas-repository || github.repository }} + ref: ${{ inputs.mpas-ref || '' }} + submodules: 'true' + + - uses: actions/checkout@v5 + if: ${{ inputs.mpas-repository != '' }} + with: + path: _ci + sparse-checkout: .github + + - name: Overlay CI infrastructure + if: ${{ inputs.mpas-repository != '' }} + shell: bash + run: cp -r _ci/.github . && rm -rf _ci + + - name: Install lcov + run: | + dnf install -y epel-release + dnf install -y lcov + + - name: Build MPAS-A with coverage flags + run: | + source /container/config_env.sh + + # Patch gfortran target: swap -O3 for -O0 --coverage -g + sed -i 's/"FFLAGS_OPT = -O3 -ffree-line-length-none/"FFLAGS_OPT = -O0 --coverage -g -ffree-line-length-none/' Makefile + sed -i 's/"CFLAGS_OPT = -O3"/"CFLAGS_OPT = -O0 --coverage -g"/' Makefile + sed -i 's/"CXXFLAGS_OPT = -O3"/"CXXFLAGS_OPT = -O0 --coverage -g"/' Makefile + sed -i 's/"LDFLAGS_OPT = -O3"/"LDFLAGS_OPT = --coverage"/' Makefile + + make gfortran CORE=atmosphere --jobs $(nproc) + + - name: Download test data + uses: ./.github/actions/download-testdata + with: + resolution: 240km + + - name: Run 240km test case + run: | + source /container/config_env.sh + cd 240km + ln -sf ../atmosphere_model . + sed -i "s/config_run_duration = '[^']*'/config_run_duration = '0_00:30:00'/" namelist.atmosphere + ulimit -s unlimited 2>/dev/null || true + mpirun -n 1 ./atmosphere_model + + - name: Generate coverage report + run: | + source /container/config_env.sh + + lcov --capture \ + --directory src \ + --output-file coverage.info \ + --ignore-errors source,gcov + + # Remove external/system files from coverage + lcov --remove coverage.info \ + '/usr/*' \ + '*/external/*' \ + --output-file coverage.info \ + --ignore-errors unused + + - name: Upload to Codecov + uses: codecov/codecov-action@v6 + with: + files: coverage.info + flags: fortran + name: mpas-atmosphere + fail_ci_if_error: false + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + - name: Upload coverage artifact + uses: actions/upload-artifact@v6 + if: always() + with: + name: coverage-report + path: coverage.info + retention-days: 30 diff --git a/.github/workflows/ect-ensemble-gen.yml b/.github/workflows/ect-ensemble-gen.yml new file mode 100644 index 0000000000..0b7cc41e92 --- /dev/null +++ b/.github/workflows/ect-ensemble-gen.yml @@ -0,0 +1,590 @@ +# Ensemble Consistency Test — Ensemble Generation +# Generates 200 perturbed MPAS-A simulations on the 120km mesh and produces +# an ensemble summary file using pyEnsSumMPAS for use with PyCECT. +# +# This is expensive (~200 model runs) and should only be triggered manually +# when there are major version or science changes requiring a new reference ensemble. +# +# Reference: Price-Broncucia et al. (2025), doi:10.5194/gmd-18-2349-2025 + +name: ECT Ensemble Generation + +on: + workflow_dispatch: + inputs: + mpas-repository: + description: 'MPAS source repo (e.g. MPAS-Dev/MPAS-Model). Leave empty to use this repo.' + required: false + default: '' + mpas-ref: + description: 'Git ref (branch, tag, or SHA) in the MPAS source repo' + required: false + default: '' + ensemble-size: + description: 'Number of ensemble members to generate' + type: number + required: false + default: 200 + members-per-job: + description: 'Number of ensemble members to run per job (batch size)' + type: number + required: false + default: 10 + num-ranks: + description: 'Number of MPI ranks per ensemble member' + type: number + required: false + default: 4 + compiler: + description: 'Compiler suite (gcc or nvhpc)' + type: choice + options: [gcc, nvhpc] + required: false + default: 'nvhpc' + mpi-impl: + description: 'MPI implementation (mpich, openmpi)' + type: choice + options: [mpich, openmpi] + required: false + default: 'mpich' + +jobs: + #=========================================================================== + # PREPARE: Compute batch matrix + #=========================================================================== + prepare: + runs-on: ubuntu-latest + outputs: + batches: ${{ steps.compute.outputs.batches }} + ensemble-size: ${{ steps.compute.outputs.ensemble-size }} + container-image: ${{ steps.container.outputs.image }} + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: .github + + - uses: ./.github/actions/resolve-container + id: container + with: + compiler: ${{ inputs.compiler || 'nvhpc' }} + mpi: ${{ inputs.mpi-impl || 'mpich' }} + + - name: Compute batch ranges + id: compute + run: | + ESIZE=${{ inputs.ensemble-size || 200 }} + BATCH=${{ inputs.members-per-job || 10 }} + echo "ensemble-size=${ESIZE}" >> $GITHUB_OUTPUT + + BATCHES="[" + for (( start=0; start> $GITHUB_OUTPUT + echo "Generated batches: ${BATCHES}" + + #=========================================================================== + # BUILD + #=========================================================================== + build: + needs: prepare + name: Build MPAS-A (${{ inputs.compiler || 'nvhpc' }}/${{ inputs.mpi-impl || 'mpich' }}) + runs-on: ubuntu-latest + outputs: + mpas-version: ${{ steps.version.outputs.mpas-version }} + mpas-commit: ${{ steps.version.outputs.mpas-commit }} + container: + image: ${{ needs.prepare.outputs.container-image }} + + steps: + - uses: actions/checkout@v5 + with: + repository: ${{ inputs.mpas-repository || github.repository }} + ref: ${{ inputs.mpas-ref || '' }} + submodules: 'true' + + - uses: actions/checkout@v5 + if: ${{ inputs.mpas-repository != '' }} + with: + path: _ci + sparse-checkout: .github + + - name: Overlay CI infrastructure + if: ${{ inputs.mpas-repository != '' }} + shell: bash + run: cp -r _ci/.github . && rm -rf _ci + + - name: Extract MPAS version + id: mpas_version + uses: ./.github/actions/mpas-version + + - name: Resolve commit and re-export version + id: version + shell: bash + run: | + git config --global --add safe.directory "${GITHUB_WORKSPACE}" + COMMIT=$(git rev-parse --short=8 HEAD) + echo "mpas-version=${{ steps.mpas_version.outputs.version }}" >> $GITHUB_OUTPUT + echo "mpas-commit=${COMMIT}" >> $GITHUB_OUTPUT + + - name: Build MPAS-A (double precision) + uses: ./.github/actions/build-mpas + with: + compiler: ${{ inputs.compiler || 'nvhpc' }} + precision: double + + - name: Upload executable + uses: actions/upload-artifact@v6 + with: + name: exe-ect-ensemble + path: atmosphere_model + retention-days: 1 + + #=========================================================================== + # SPIN-UP: 24-hour unperturbed run to generate spun-up restart file + # Hydrometeor fields (rain, snow, cloud water) are zero in cold-start + # init.nc and need time to develop realistic values. + # See Price-Broncucia et al. (2025), Section 3.2. + #=========================================================================== + spinup: + needs: [prepare, build] + name: 24h Spin-up + runs-on: ubuntu-latest + container: + image: ${{ needs.prepare.outputs.container-image }} + + steps: + - uses: actions/checkout@v5 + + - name: Restore cached restart + id: cache + uses: actions/cache/restore@v5 + with: + path: spinup-restart.nc + key: ect-spinup-restart-${{ hashFiles('.github/ci-config.env') }} + + - name: Download executable + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/download-artifact@v7 + with: + name: exe-ect-ensemble + + - name: Make executable + if: steps.cache.outputs.cache-hit != 'true' + run: chmod +x atmosphere_model + + - name: Run 24h spin-up + if: steps.cache.outputs.cache-hit != 'true' + uses: ./.github/actions/run-mpas + with: + executable: ./atmosphere_model + resolution: 120km + num-procs: '4' + mpi-impl: ${{ inputs.mpi-impl || 'mpich' }} + run-duration: '1_00:00:00' + restart-interval: '1_00:00:00' + run-timeout: '350' + strict-exit-check: 'false' + + - name: Verify restart file + if: steps.cache.outputs.cache-hit != 'true' + run: | + RESTART=$(ls run-120km/restart.*.nc 2>/dev/null | head -1) + if [ -z "${RESTART}" ]; then + echo "::error::Spin-up did not produce a restart file" + ls -la run-120km/*.nc 2>/dev/null || true + exit 1 + fi + echo "Spin-up restart: ${RESTART} ($(du -h ${RESTART} | cut -f1))" + cp "${RESTART}" spinup-restart.nc + + - name: Cache restart file + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v5 + with: + path: spinup-restart.nc + key: ect-spinup-restart-${{ hashFiles('.github/ci-config.env') }} + + - name: Upload restart artifact + uses: actions/upload-artifact@v6 + with: + name: spinup-restart + path: spinup-restart.nc + retention-days: 1 + + #=========================================================================== + # RUN: Batched ensemble members + #=========================================================================== + run-ensemble: + needs: [prepare, build, spinup] + strategy: + fail-fast: false + matrix: + batch: ${{ fromJson(needs.prepare.outputs.batches) }} + + name: Members ${{ matrix.batch.start }}-${{ matrix.batch.end }} + runs-on: ubuntu-latest + container: + image: ${{ needs.prepare.outputs.container-image }} + + steps: + - uses: actions/checkout@v5 + + - name: Download executable + uses: actions/download-artifact@v7 + with: + name: exe-ect-ensemble + + - name: Make executable + run: chmod +x atmosphere_model + + - name: Download test case + uses: ./.github/actions/download-testdata + with: + resolution: 120km + dest-dir: base-case + + - name: Download spin-up restart + uses: actions/download-artifact@v7 + with: + name: spinup-restart + + - name: Run ensemble members ${{ matrix.batch.start }}-${{ matrix.batch.end }} + uses: ./.github/actions/run-perturb-mpas + with: + base-dir: base-case + executable: ./atmosphere_model + member-start: ${{ matrix.batch.start }} + member-end: ${{ matrix.batch.end }} + num-ranks: ${{ inputs.num-ranks || 4 }} + mpi-impl: ${{ inputs.mpi-impl || 'mpich' }} + run-duration: '0_02:36:00' + run-timeout: '45' + restart-file: spinup-restart.nc + + - name: Upload history files + uses: actions/upload-artifact@v6 + with: + name: ect-ensemble-${{ matrix.batch.start }}-${{ matrix.batch.end }} + path: history-output/history.*.nc + retention-days: 1 + + #=========================================================================== + # SUMMARY: Generate ensemble summary file with pyEnsSumMPAS + # Security: This job only checks out THIS repo (for config/tooling). + # The MPAS version and commit come from the build job outputs, so no + # external repo code is checked out or executed here. + #=========================================================================== + generate-summary: + needs: [prepare, build, run-ensemble] + if: always() && needs.run-ensemble.result != 'cancelled' + name: Generate Summary + runs-on: ubuntu-latest + outputs: + summary-name: ${{ steps.meta.outputs.summary-name }} + summary-prefix: ${{ steps.meta.outputs.summary-prefix }} + mpas-version: ${{ steps.meta.outputs.mpas-version }} + mpas-repo: ${{ steps.meta.outputs.mpas-repo }} + branch: ${{ steps.meta.outputs.branch }} + commit: ${{ steps.meta.outputs.commit }} + date: ${{ steps.meta.outputs.date }} + resolution: ${{ steps.meta.outputs.resolution }} + ensemble-size: ${{ steps.meta.outputs.ensemble-size }} + compiler: ${{ steps.meta.outputs.compiler }} + mpi-impl: ${{ steps.meta.outputs.mpi-impl }} + precision: ${{ steps.meta.outputs.precision }} + run-duration: ${{ steps.meta.outputs.run-duration }} + perturb-magnitude: ${{ steps.meta.outputs.perturb-magnitude }} + perturb-variable: ${{ steps.meta.outputs.perturb-variable }} + + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: .github + + - uses: actions/setup-python@v6 + with: + python-version: '3.11' + + - name: Collect metadata + id: meta + run: | + source .github/ci-config.env + ECT_SUMMARY_PREFIX="${ECT_SUMMARY_FILE%.nc}" + RUN_DURATION=0_02:36:00 + VERSION="${{ needs.build.outputs.mpas-version }}" + DATE=$(date -u +%Y-%m-%d) + SUMMARY_NAME="${ECT_SUMMARY_PREFIX}_v${VERSION}_${DATE//-/}.nc" + + MPAS_REPO="${{ inputs.mpas-repository || github.repository }}" + MPAS_BRANCH="${{ inputs.mpas-ref || github.ref_name }}" + MPAS_COMMIT="${{ needs.build.outputs.mpas-commit }}" + + echo "mpas-version=${VERSION}" >> $GITHUB_OUTPUT + echo "mpas-repo=${MPAS_REPO}" >> $GITHUB_OUTPUT + echo "branch=${MPAS_BRANCH}" >> $GITHUB_OUTPUT + echo "commit=${MPAS_COMMIT}" >> $GITHUB_OUTPUT + echo "date=${DATE}" >> $GITHUB_OUTPUT + echo "resolution=${ECT_RESOLUTION}" >> $GITHUB_OUTPUT + echo "ensemble-size=${{ needs.prepare.outputs.ensemble-size }}" >> $GITHUB_OUTPUT + echo "summary-prefix=${ECT_SUMMARY_PREFIX}" >> $GITHUB_OUTPUT + echo "summary-name=${SUMMARY_NAME}" >> $GITHUB_OUTPUT + echo "compiler=${{ inputs.compiler || 'nvhpc' }}" >> $GITHUB_OUTPUT + echo "mpi-impl=${{ inputs.mpi-impl || 'mpich' }}" >> $GITHUB_OUTPUT + echo "precision=double" >> $GITHUB_OUTPUT + echo "run-duration=${RUN_DURATION}" >> $GITHUB_OUTPUT + echo "perturb-magnitude=${ECT_PERTURB_MAGNITUDE}" >> $GITHUB_OUTPUT + echo "perturb-variable=${ECT_PERTURB_VARIABLE}" >> $GITHUB_OUTPUT + + echo "=== ECT Metadata ===" + echo " MPAS repo: ${MPAS_REPO}" + echo " MPAS version: ${VERSION}" + echo " Branch: ${MPAS_BRANCH}" + echo " Commit: ${MPAS_COMMIT}" + echo " Resolution: ${ECT_RESOLUTION}" + echo " Compiler: ${{ inputs.compiler || 'nvhpc' }}" + echo " MPI: ${{ inputs.mpi-impl || 'mpich' }}" + echo " Precision: double" + echo " Run duration: ${RUN_DURATION}" + echo " Perturbation: ${ECT_PERTURB_MAGNITUDE} (${ECT_PERTURB_VARIABLE})" + echo " Ensemble: ${{ needs.prepare.outputs.ensemble-size }} members" + echo " Summary: ${SUMMARY_NAME}" + + - name: Install dependencies + run: pip install "numpy<2" scipy netCDF4 + + - name: Clone PyCECT + run: | + source .github/ci-config.env + git clone --depth 1 --branch "${PYCECT_TAG}" https://github.com/NCAR/PyCECT.git pycect + + - name: Download all history files + uses: actions/download-artifact@v7 + with: + pattern: ect-ensemble-* + path: ensemble-files + merge-multiple: true + + - name: Validate ensemble files + run: | + echo "=== Validating ensemble history files ===" + GOOD=0 + BAD=0 + for f in ensemble-files/history.*.nc; do + if python3 -c "import netCDF4; netCDF4.Dataset('$f').close()" 2>/dev/null; then + GOOD=$((GOOD + 1)) + else + echo "::warning::Removing corrupted file: $f" + rm -f "$f" + BAD=$((BAD + 1)) + fi + done + echo "Valid: ${GOOD}, Corrupted: ${BAD}" + if [ "${GOOD}" -lt 48 ]; then + echo "::error::Too few valid ensemble members (${GOOD}). PyCECT needs more members than output variables (~47 after trimming). Recommended: 200." + exit 1 + fi + + - name: Generate ensemble summary + run: | + SUMMARY_NAME="${{ steps.meta.outputs.summary-name }}" + ESIZE=${{ needs.prepare.outputs.ensemble-size }} + NFILES=$(ls ensemble-files/history.*.nc 2>/dev/null | wc -l) + if [ "${NFILES}" -lt "${ESIZE}" ]; then + echo "::warning::Only ${NFILES} of ${ESIZE} requested members available (some may have been removed due to corruption)" + ESIZE=${NFILES} + fi + + JSONFILE=ect_pycect_exclude.json + echo '{"ExcludedVar": []}' > "${JSONFILE}" + + # --tslice 0 is invariant: ensemble files are pre-trimmed to a single + # time slice by run-perturb-mpas (trim_history.py creates Time=1). + python pycect/pyEnsSumMPAS.py \ + --esize ${ESIZE} \ + --indir ensemble-files \ + --sumfile "${SUMMARY_NAME}" \ + --tslice 0 \ + --tag $(date +%Y%m%d) \ + --model mpas \ + --mach github-actions \ + --verbose \ + --jsonfile ${JSONFILE} \ + --mpi_disable + + if [ ! -f "${SUMMARY_NAME}" ]; then + echo "::error::pyEnsSumMPAS did not produce ${SUMMARY_NAME}." + echo " PyCECT requires ensemble size >= number of output variables (~47 after trimming)." + echo " Current ensemble size: ${ESIZE}. Increase to at least 50 (recommended: 200)." + exit 1 + fi + + - name: Upload summary artifact + uses: actions/upload-artifact@v6 + if: always() + with: + name: mpas-ect-summary + path: ${{ steps.meta.outputs.summary-name }} + retention-days: 90 + if-no-files-found: warn + + - name: Summary + if: always() + run: | + SUMMARY_NAME="${{ steps.meta.outputs.summary-name }}" + echo "==========================================" + echo " ECT Ensemble Summary Generated" + echo "==========================================" + echo "" + echo " MPAS repo: ${{ steps.meta.outputs.mpas-repo }}" + echo " MPAS version: ${{ steps.meta.outputs.mpas-version }}" + echo " Branch: ${{ steps.meta.outputs.branch }}" + echo " Commit: ${{ steps.meta.outputs.commit }}" + echo " Resolution: ${{ steps.meta.outputs.resolution }}" + echo " Compiler: ${{ steps.meta.outputs.compiler }}/${{ steps.meta.outputs.mpi-impl }}" + echo " Precision: ${{ steps.meta.outputs.precision }}" + echo " Duration: ${{ steps.meta.outputs.run-duration }}" + echo " Perturbation: ${{ steps.meta.outputs.perturb-magnitude }} (${{ steps.meta.outputs.perturb-variable }})" + echo " Ensemble size: ${{ steps.meta.outputs.ensemble-size }}" + echo " Date: ${{ steps.meta.outputs.date }}" + echo "" + if [ -f "${SUMMARY_NAME}" ]; then + echo " File: ${SUMMARY_NAME}" + echo " Size: $(du -h "${SUMMARY_NAME}" | cut -f1)" + fi + + #=========================================================================== + # PUBLISH RESTART: Upload spin-up restart to ect-v{MPAS_VERSION} + # Runs after spinup regardless of ensemble/summary outcome. + # Auto-creates the release (tagged by MPAS version) if it doesn't exist. + #=========================================================================== + publish-restart: + needs: [build, spinup] + name: Publish Restart + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: .github/ci-config.env + sparse-checkout-cone-mode: false + + - name: Download restart artifact + uses: actions/download-artifact@v7 + with: + name: spinup-restart + + - name: Upload to GitHub release + env: + GH_TOKEN: ${{ github.token }} + run: | + source .github/ci-config.env + VERSION="${{ needs.build.outputs.mpas-version }}" + TAG="ect-v${VERSION}" + + # Create release if it doesn't exist + if ! gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" >/dev/null 2>&1; then + echo "Creating release ${TAG}..." + gh release create "${TAG}" --repo "${GITHUB_REPOSITORY}" \ + --title "ECT data (MPAS v${VERSION})" \ + --notes "ECT ensemble summary and spin-up restart for MPAS v${VERSION}. Updated automatically by the ect-ensemble-gen workflow." + fi + + RESTART_NAME="${ECT_RESTART_FILE}" + mv spinup-restart.nc "${RESTART_NAME}" + + echo "Compressing ${RESTART_NAME} ($(du -h "${RESTART_NAME}" | cut -f1))..." + gzip -1 "${RESTART_NAME}" + + gh release upload "${TAG}" "${RESTART_NAME}.gz" \ + --repo "${GITHUB_REPOSITORY}" --clobber + + echo "=== Published restart to release ${TAG} ===" + echo " File: ${RESTART_NAME}.gz ($(du -h "${RESTART_NAME}.gz" | cut -f1))" + + #=========================================================================== + # PUBLISH SUMMARY: Upload ensemble summary to ect-v{MPAS_VERSION} + # Only runs when summary generation succeeds. + # Auto-creates the release (tagged by MPAS version) if it doesn't exist. + #=========================================================================== + publish-summary: + needs: [build, generate-summary] + if: needs.generate-summary.result == 'success' + name: Publish Summary + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: .github/ci-config.env + sparse-checkout-cone-mode: false + + - name: Download summary artifact + uses: actions/download-artifact@v7 + with: + name: mpas-ect-summary + + - name: Upload to GitHub release + env: + GH_TOKEN: ${{ github.token }} + run: | + source .github/ci-config.env + VERSION="${{ needs.build.outputs.mpas-version }}" + TAG="ect-v${VERSION}" + + # Create release if it doesn't exist + if ! gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" >/dev/null 2>&1; then + echo "Creating release ${TAG}..." + gh release create "${TAG}" --repo "${GITHUB_REPOSITORY}" \ + --title "ECT data (MPAS v${VERSION})" \ + --notes "ECT ensemble summary and spin-up restart for MPAS v${VERSION}. Updated automatically by the ect-ensemble-gen workflow." + fi + + SUMMARY_NAME="${{ needs.generate-summary.outputs.summary-name }}" + SUMMARY_PREFIX="${{ needs.generate-summary.outputs.summary-prefix }}" + CURRENT_SUMMARY="${SUMMARY_PREFIX}.nc" + cp "${SUMMARY_NAME}" "${CURRENT_SUMMARY}" + + gh release upload "${TAG}" "${CURRENT_SUMMARY}" \ + --repo "${GITHUB_REPOSITORY}" --clobber + + echo "=== Published summary to release ${TAG} ===" + echo " File: ${CURRENT_SUMMARY}" + + #=========================================================================== + # CLEANUP + # Only runs on a fully successful workflow. On any failure (batch, summary, + # or publish), every artifact is retained until retention-days expires so + # `gh run rerun --failed` can find the executable, spin-up restart, and + # per-batch history files it needs to recover. + #=========================================================================== + cleanup: + needs: [run-ensemble, generate-summary, publish-restart, publish-summary] + if: needs.publish-summary.result == 'success' + runs-on: ubuntu-latest + name: Cleanup + + steps: + - name: Delete build and ensemble artifacts + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts \ + --paginate --jq '.artifacts[] | select(.name == "exe-ect-ensemble" or .name == "spinup-restart" or (.name | startswith("ect-ensemble-"))) | .id' \ + | while read id; do + gh api -X DELETE repos/${{ github.repository }}/actions/artifacts/$id || true + done || true diff --git a/.github/workflows/ect-test.yml b/.github/workflows/ect-test.yml new file mode 100644 index 0000000000..fa0b325f2b --- /dev/null +++ b/.github/workflows/ect-test.yml @@ -0,0 +1,237 @@ +# Ensemble Consistency Test (ECT) +# Runs 3 perturbed MPAS-A simulations on the 120km mesh and compares +# against a pre-built ensemble summary file using PyCECT. +# +# This test detects statistically meaningful changes in model output +# that go beyond internal variability (non-BFB but scientifically equivalent +# changes will pass). Run this on code changes NOT expected to change science. +# +# Reference: Price-Broncucia et al. (2025), doi:10.5194/gmd-18-2349-2025 + +name: Ensemble Consistency Test (ECT) + +on: + push: + branches: [master] + workflow_dispatch: + inputs: + mpas-repository: + description: 'MPAS source repo (e.g. MPAS-Dev/MPAS-Model). Leave empty to use this repo.' + required: false + default: '' + mpas-ref: + description: 'Git ref (branch, tag, or SHA) in the MPAS source repo' + required: false + default: '' + +jobs: + #=========================================================================== + # CONFIG: Resolve container image from ci-config.env + #=========================================================================== + config: + name: Resolve Config + runs-on: ubuntu-latest + outputs: + image: ${{ steps.container.outputs.image }} + mpas-version: ${{ steps.version.outputs.version }} + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: | + .github + src/core_atmosphere/Registry.xml + sparse-checkout-cone-mode: false + - uses: ./.github/actions/resolve-container + id: container + with: + compiler: gcc + mpi: openmpi + - uses: ./.github/actions/mpas-version + id: version + + #=========================================================================== + # BUILD + #=========================================================================== + build: + needs: config + name: Build MPAS-A + runs-on: ubuntu-latest + container: + image: ${{ needs.config.outputs.image }} + + steps: + - uses: actions/checkout@v5 + with: + repository: ${{ inputs.mpas-repository || github.repository }} + ref: ${{ inputs.mpas-ref || '' }} + submodules: 'true' + + - uses: actions/checkout@v5 + if: ${{ inputs.mpas-repository != '' }} + with: + path: _ci + sparse-checkout: .github + + - name: Overlay CI infrastructure + if: ${{ inputs.mpas-repository != '' }} + shell: bash + run: cp -r _ci/.github . && rm -rf _ci + + - name: Build MPAS-A (double precision) + uses: ./.github/actions/build-mpas + with: + compiler: gcc + precision: double + + - name: Upload executable + uses: actions/upload-artifact@v6 + with: + name: exe-ect + path: atmosphere_model + retention-days: 1 + + #=========================================================================== + # RUN: 3 perturbed ensemble members + #=========================================================================== + run: + needs: [config, build] + strategy: + fail-fast: false + matrix: + member: [0, 1, 2] + + name: ECT member ${{ matrix.member }} + runs-on: ubuntu-latest + container: + image: ${{ needs.config.outputs.image }} + + steps: + - uses: actions/checkout@v5 + + - name: Download executable + uses: actions/download-artifact@v7 + with: + name: exe-ect + + - name: Download test case + uses: ./.github/actions/download-testdata + with: + resolution: 120km + dest-dir: base-case + + - name: Restore cached restart + id: cache-restart + uses: actions/cache/restore@v5 + with: + path: spinup-restart.nc + key: ect-spinup-restart-${{ hashFiles('.github/ci-config.env') }} + + - name: Download spin-up restart + id: restart + shell: bash + run: | + source .github/ci-config.env + RESTART="${ECT_RESTART_FILE}" + RELEASE_TAG="ect-v${{ needs.config.outputs.mpas-version }}" + + if [ -f "spinup-restart.nc" ]; then + echo "Using cached restart file" + mv spinup-restart.nc "${RESTART}" + echo "available=true" >> $GITHUB_OUTPUT + echo "file=${RESTART}" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "Downloading ${RESTART}.gz from release ${RELEASE_TAG}..." + HTTP_CODE=$(curl -sL --retry 3 --retry-delay 5 -w "%{http_code}" \ + "https://github.com/${GITHUB_REPOSITORY}/releases/download/${RELEASE_TAG}/${RESTART}.gz" \ + -o "${RESTART}.gz") + if [ "${HTTP_CODE}" = "200" ]; then + gunzip "${RESTART}.gz" + echo "Downloaded restart: $(du -h ${RESTART})" + echo "available=true" >> $GITHUB_OUTPUT + echo "file=${RESTART}" >> $GITHUB_OUTPUT + else + echo "::warning::Spin-up restart not available (HTTP ${HTTP_CODE}), running from cold-start init.nc" + echo "available=false" >> $GITHUB_OUTPUT + fi + + - name: Run perturbed MPAS-A + uses: ./.github/actions/run-perturb-mpas + with: + base-dir: base-case + executable: ./atmosphere_model + member-start: ${{ matrix.member }} + member-end: ${{ matrix.member }} + run-duration: '0_02:36:00' + run-timeout: '45' + restart-file: ${{ steps.restart.outputs.available == 'true' && steps.restart.outputs.file || '' }} + + - name: Upload history file + uses: actions/upload-artifact@v6 + if: always() + with: + name: ect-history-${{ matrix.member }} + path: history-output/history.*.nc + retention-days: 1 + + - name: Upload log files + uses: actions/upload-artifact@v6 + if: always() + with: + name: ect-logs-${{ matrix.member }} + path: history-output/log.*.out + retention-days: 5 + + #=========================================================================== + # VALIDATE: Run PyCECT against ensemble summary + #=========================================================================== + validate: + needs: [config, run] + if: always() + name: PyCECT Validation + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: .github + + - uses: actions/setup-python@v6 + with: + python-version: '3.11' + + - name: Download history files + uses: actions/download-artifact@v7 + with: + pattern: ect-history-* + path: ect-test-files + merge-multiple: true + + - name: Run ECT validation + uses: ./.github/actions/validate-ect + with: + history-dir: ect-test-files + label: gcc/openmpi/ect-standalone + mpas-version: ${{ needs.config.outputs.mpas-version }} + + #=========================================================================== + # CLEANUP + #=========================================================================== + cleanup: + needs: [run, validate] + if: always() + runs-on: ubuntu-latest + name: Cleanup + + steps: + - name: Delete build artifacts + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts \ + --paginate --jq '.artifacts[] | select(.name == "exe-ect") | .id' \ + | while read id; do + gh api -X DELETE repos/${{ github.repository }}/actions/artifacts/$id || true + done || true diff --git a/.github/workflows/profile-gpu-nsight.yml b/.github/workflows/profile-gpu-nsight.yml new file mode 100644 index 0000000000..92ed3b7969 --- /dev/null +++ b/.github/workflows/profile-gpu-nsight.yml @@ -0,0 +1,194 @@ +# Nsight Systems GPU profiling on CIRRUS (keep to very short runs) + +name: GPU Nsight (profile) + +permissions: + contents: read + actions: write + +on: + workflow_dispatch: + inputs: + mpi: + description: MPI implementation + required: true + default: mpich + type: choice + options: + - mpich + - openmpi + resolution: + description: Test case resolution (see RELEASE_TESTDATA_* in ci-config.env) + required: false + default: '240km' + type: string + num_procs: + description: MPI rank count + required: false + default: '4' + type: string + run_duration: + description: config_run_duration + required: false + default: '0_01:00:00' + type: string + run_timeout_minutes: + description: Wall-clock timeout for mpirun + nsys + required: false + default: '40' + type: string + +jobs: + config: + name: Resolve CUDA container + runs-on: ubuntu-latest + outputs: + image-gpu: ${{ steps.gpu-container.outputs.image }} + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: .github + - uses: ./.github/actions/resolve-container + id: gpu-container + with: + compiler: nvhpc + mpi: ${{ inputs.mpi }} + gpu: cuda + + build: + needs: config + name: Build (cuda, ${{ inputs.mpi }}) + runs-on: + group: CIRRUS-4x8-gpu + container: + image: ${{ needs.config.outputs.image-gpu }} + + steps: + - uses: actions/checkout@v5 + with: + submodules: 'true' + + - name: Build MPAS-A (double precision, OpenACC) + uses: ./.github/actions/build-mpas + with: + compiler: nvhpc + use-pio: 'false' + openacc: 'true' + precision: double + build-timeout: '45' + + - name: Upload executable + uses: actions/upload-artifact@v6 + with: + name: exe-profile-nsight-${{ inputs.mpi }} + path: atmosphere_model + retention-days: 1 + + profile: + needs: [config, build] + if: ${{ needs.build.result == 'success' }} + name: Nsight profile (${{ inputs.mpi }}) + runs-on: + group: CIRRUS-4x8-gpu + container: + image: ${{ needs.config.outputs.image-gpu }} + + steps: + - uses: actions/checkout@v5 + + - name: GPU check + shell: bash + run: | + echo "## GPU" >> "$GITHUB_STEP_SUMMARY" + nvidia-smi || echo "::warning::nvidia-smi failed" + nvidia-smi || true + + - name: Setup Nsight Systems CLI (install + RPM cache) + uses: ./.github/actions/setup-nsight-systems + + - name: Download executable + uses: actions/download-artifact@v7 + with: + name: exe-profile-nsight-${{ inputs.mpi }} + + - name: Download test case + uses: ./.github/actions/download-testdata + with: + resolution: ${{ inputs.resolution }} + dest-dir: nsight-case + + - name: Link executable and config_run_duration (dt unchanged) + shell: bash + run: | + chmod +x atmosphere_model + ln -sf "$(pwd)/atmosphere_model" nsight-case/atmosphere_model + DURATION="${{ inputs.run_duration }}" + sed -i "s/config_run_duration = '[^']*'/config_run_duration = '${DURATION}'/" nsight-case/namelist.atmosphere + echo "config_run_duration -> ${DURATION} (config_dt left as in test case)" + grep -E 'config_dt|config_run_duration' nsight-case/namelist.atmosphere | head -n 5 || true + + - name: Run nsys profile + shell: bash + run: | + chmod +x .github/scripts/run-nsys-profile.sh + bash .github/scripts/run-nsys-profile.sh \ + nsight-case \ + "${{ inputs.num_procs }}" \ + "${{ inputs.mpi }}" \ + "${{ inputs.run_timeout_minutes }}" \ + nsys-profile + + - name: nsys stats (text) + shell: bash + working-directory: nsight-case + run: | + if [ -f /container/config_env.sh ]; then + source /container/config_env.sh + fi + source "${GITHUB_WORKSPACE}/.github/scripts/resolve-nsys.sh" + if ! resolve_nsys; then + echo "::warning::No working nsys; skipping nsys stats generation" + exit 0 + fi + echo "Using nsys: ${NSYS_BIN}" + shopt -s nullglob + REP="" + for f in nsys-profile.nsys-rep *.nsys-rep *.qdrep; do + if [ -f "$f" ]; then + REP="$f" + break + fi + done + if [ -z "${REP}" ]; then + echo "::warning::No Nsight session file found for nsys stats" + ls -la + exit 0 + fi + echo "=== nsys stats (full log → nsys-profile-stats.txt; first 200 lines below) ===" + "${NSYS_BIN}" stats "${REP}" > "nsys-profile-stats.txt" 2>&1 || true + head -n 200 nsys-profile-stats.txt + + - name: Job summary + if: always() + shell: bash + run: | + { + echo "## Nsight profile" + echo "- MPI: ${{ inputs.mpi }}" + echo "- Resolution: ${{ inputs.resolution }}" + echo "- Ranks: ${{ inputs.num_procs }}" + echo "- config_run_duration: ${{ inputs.run_duration }}" + echo "" + echo "Session files and \`nsys stats\` text are uploaded as artifacts (short retention)." + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload Nsight artifacts + if: always() + uses: actions/upload-artifact@v6 + with: + name: nsys-profile-${{ github.run_id }}-${{ inputs.mpi }} + path: | + nsight-case/nsys-profile.* + nsight-case/nsys-profile-stats.txt + retention-days: 3 + if-no-files-found: warn diff --git a/.github/workflows/test-cross-repo.yml b/.github/workflows/test-cross-repo.yml new file mode 100644 index 0000000000..7cc5e5ab32 --- /dev/null +++ b/.github/workflows/test-cross-repo.yml @@ -0,0 +1,80 @@ +# Cross-repo testing: run CI against an external MPAS source repo. +# Dispatch-only — does not affect badge status on master. + +name: "Cross-Repo Test" + +on: + workflow_dispatch: + inputs: + mpas-repository: + description: 'MPAS source repo (e.g. MPAS-Dev/MPAS-Model)' + required: true + mpas-ref: + description: 'Git ref (branch, tag, or SHA)' + required: true + compilers: + description: 'Compilers to test (comma-separated: gcc,nvhpc,oneapi)' + required: false + default: 'gcc,nvhpc,oneapi' + mpi: + description: 'MPI implementation' + required: false + default: 'mpich' + # GPU cross-repo testing disabled until NCAR security review of running + # external code on self-hosted CIRRUS runners is complete. + # include-gpu: + # description: 'Include GPU (CUDA) test' + # required: false + # type: boolean + # default: false + +jobs: + config: + runs-on: ubuntu-latest + outputs: + compilers: ${{ steps.set.outputs.compilers }} + mpi: ${{ steps.set.outputs.mpi }} + repo: ${{ steps.set.outputs.repo }} + ref: ${{ steps.set.outputs.ref }} + steps: + - id: set + shell: bash + run: | + COMPILERS="${{ inputs.compilers }}" + REPO="${{ inputs.mpas-repository || 'MPAS-Dev/MPAS-Model' }}" + REF="${{ inputs.mpas-ref || 'develop' }}" + MPI="${{ inputs.mpi || 'mpich' }}" + + echo "compilers=$(printf '%s' "${COMPILERS:-gcc}" | jq -Rc 'split(",") | map(gsub("^\\s+|\\s+$";""))')" >> "$GITHUB_OUTPUT" + echo "mpi=${MPI}" >> "$GITHUB_OUTPUT" + echo "repo=${REPO}" >> "$GITHUB_OUTPUT" + echo "ref=${REF}" >> "$GITHUB_OUTPUT" + + echo "## Cross-Repo Test" >> "$GITHUB_STEP_SUMMARY" + echo "| | |" >> "$GITHUB_STEP_SUMMARY" + echo "|---|---|" >> "$GITHUB_STEP_SUMMARY" + echo "| **Source** | [\`${REPO}@${REF}\`](https://github.com/${REPO}/tree/${REF}) |" >> "$GITHUB_STEP_SUMMARY" + echo "| **Compilers** | \`${COMPILERS:-gcc}\` |" >> "$GITHUB_STEP_SUMMARY" + echo "| **MPI** | \`${MPI}\` |" >> "$GITHUB_STEP_SUMMARY" + cpu-matrix: + needs: config + strategy: + fail-fast: false + matrix: + compiler: ${{ fromJSON(needs.config.outputs.compilers) }} + uses: ./.github/workflows/_test-compiler.yml + with: + compiler: ${{ matrix.compiler }} + mpi: ${{ needs.config.outputs.mpi }} + mpas-repository: ${{ needs.config.outputs.repo }} + mpas-ref: ${{ needs.config.outputs.ref }} + + # GPU cross-repo testing disabled until NCAR security review is complete. + # gpu: + # needs: config + # if: ${{ inputs.include-gpu }} + # uses: ./.github/workflows/_test-gpu.yml + # with: + # mpi: ${{ needs.config.outputs.mpi }} + # mpas-repository: ${{ needs.config.outputs.repo }} + # mpas-ref: ${{ needs.config.outputs.ref }} diff --git a/.github/workflows/test-gcc-mpich.yml b/.github/workflows/test-gcc-mpich.yml new file mode 100644 index 0000000000..fcefd8db1e --- /dev/null +++ b/.github/workflows/test-gcc-mpich.yml @@ -0,0 +1,19 @@ +# Subset CI: GNU + MPICH (CPU) +# Quick ECT validation with 4 MPI ranks. + +name: "GNU+MPICH (CPU)" + +on: + workflow_dispatch: + push: + branches: [master, 'hackathon-*'] + pull_request: + branches: [master, 'hackathon-*'] + +jobs: + test: + uses: ./.github/workflows/_test-compiler.yml + with: + compiler: gcc + mpi: mpich + diff --git a/.github/workflows/test-gcc-openmpi.yml b/.github/workflows/test-gcc-openmpi.yml new file mode 100644 index 0000000000..ca7965a80e --- /dev/null +++ b/.github/workflows/test-gcc-openmpi.yml @@ -0,0 +1,15 @@ +# Subset CI: GNU + OpenMPI (CPU) +# Quick ECT validation with 4 MPI ranks. + +name: "GNU+OpenMPI (CPU)" + +on: + workflow_dispatch: + +jobs: + test: + uses: ./.github/workflows/_test-compiler.yml + with: + compiler: gcc + mpi: openmpi + diff --git a/.github/workflows/test-gpu-mpich.yml b/.github/workflows/test-gpu-mpich.yml new file mode 100644 index 0000000000..efd1097fa6 --- /dev/null +++ b/.github/workflows/test-gpu-mpich.yml @@ -0,0 +1,15 @@ +# Subset CI: NVHPC + MPICH (GPU vs CPU) +# Runs on CIRRUS self-hosted runners with GPU access. +# workflow_dispatch only — no push/pull_request (self-hosted security). + +name: "NVHPC+MPICH (GPU)" + +on: + workflow_dispatch: + +jobs: + test: + uses: ./.github/workflows/_test-gpu.yml + with: + mpi: mpich + diff --git a/.github/workflows/test-gpu-openmpi.yml b/.github/workflows/test-gpu-openmpi.yml new file mode 100644 index 0000000000..fff0259dcf --- /dev/null +++ b/.github/workflows/test-gpu-openmpi.yml @@ -0,0 +1,16 @@ +# Subset CI: NVHPC + OpenMPI (GPU vs CPU) +# Runs on CIRRUS self-hosted runners with GPU access. +# Known issue: NVHPC+OpenMPI SIGABRT. +# workflow_dispatch only — no push/pull_request (self-hosted security). + +name: "NVHPC+OpenMPI (GPU)" + +on: + workflow_dispatch: + +jobs: + test: + uses: ./.github/workflows/_test-gpu.yml + with: + mpi: openmpi + diff --git a/.github/workflows/test-intel-mpich.yml b/.github/workflows/test-intel-mpich.yml new file mode 100644 index 0000000000..a73195efd3 --- /dev/null +++ b/.github/workflows/test-intel-mpich.yml @@ -0,0 +1,19 @@ +# Subset CI: Intel + MPICH (CPU) +# Quick ECT validation with 4 MPI ranks. + +name: "Intel+MPICH (CPU)" + +on: + workflow_dispatch: + push: + branches: [master, 'hackathon-*'] + pull_request: + branches: [master, 'hackathon-*'] + +jobs: + test: + uses: ./.github/workflows/_test-compiler.yml + with: + compiler: oneapi + mpi: mpich + diff --git a/.github/workflows/test-intel-openmpi.yml b/.github/workflows/test-intel-openmpi.yml new file mode 100644 index 0000000000..3050baf238 --- /dev/null +++ b/.github/workflows/test-intel-openmpi.yml @@ -0,0 +1,15 @@ +# Subset CI: Intel + OpenMPI (CPU) +# Quick ECT validation with 4 MPI ranks. + +name: "Intel+OpenMPI (CPU)" + +on: + workflow_dispatch: + +jobs: + test: + uses: ./.github/workflows/_test-compiler.yml + with: + compiler: oneapi + mpi: openmpi + diff --git a/.github/workflows/test-nvhpc-mpich.yml b/.github/workflows/test-nvhpc-mpich.yml new file mode 100644 index 0000000000..ac898e9822 --- /dev/null +++ b/.github/workflows/test-nvhpc-mpich.yml @@ -0,0 +1,19 @@ +# Subset CI: NVHPC + MPICH (CPU) +# Quick ECT validation with 4 MPI ranks. + +name: "NVHPC+MPICH (CPU)" + +on: + workflow_dispatch: + push: + branches: [master, 'hackathon-*'] + pull_request: + branches: [master, 'hackathon-*'] + +jobs: + test: + uses: ./.github/workflows/_test-compiler.yml + with: + compiler: nvhpc + mpi: mpich + diff --git a/.github/workflows/test-nvhpc-openmpi.yml b/.github/workflows/test-nvhpc-openmpi.yml new file mode 100644 index 0000000000..31297a0ede --- /dev/null +++ b/.github/workflows/test-nvhpc-openmpi.yml @@ -0,0 +1,17 @@ +# Subset CI: NVHPC + OpenMPI (CPU) +# Quick ECT validation with 4 MPI ranks. +# Known issue: model exits 134 (SIGABRT) on GA runners with 4 ranks. +# MPICH works. + +name: "NVHPC+OpenMPI (CPU)" + +on: + workflow_dispatch: + +jobs: + test: + uses: ./.github/workflows/_test-compiler.yml + with: + compiler: nvhpc + mpi: openmpi + diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 0000000000..10c2dc8426 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,102 @@ +# Unit Tests +# Builds and runs pFUnit-based unit tests for standalone MPAS procedures. +# Tests mathematical/computational routines that don't require the full +# MPAS framework (interpolation, geometry, sorting, etc.). +# +# Adapted from ESCOMP/CAM-SIMA MPAS dynamical core CI. + +name: Unit Tests + +permissions: + contents: read + +on: + pull_request: + paths: + - 'src/**' + - 'tests/**' + - '.github/workflows/unit-tests.yml' + push: + branches: [master] + paths: + - 'src/**' + - 'tests/**' + - '.github/workflows/unit-tests.yml' + workflow_dispatch: + +jobs: + unit-tests: + name: pFUnit (GCC ${{ matrix.gcc-version }}) + timeout-minutes: 15 + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + gcc-version: ['12', '13', '14'] + + env: + CC: gcc-${{ matrix.gcc-version }} + CXX: g++-${{ matrix.gcc-version }} + FC: gfortran-${{ matrix.gcc-version }} + PFUNIT_VERSION: 'v4.13.0' + PFUNIT_PATH: ${{ github.workspace }}/pFUnit + + steps: + - name: Install GCC toolchain + run: | + sudo apt-get update + sudo apt-get install -y gcc-${{ matrix.gcc-version }} g++-${{ matrix.gcc-version }} gfortran-${{ matrix.gcc-version }} + + - name: Checkout + uses: actions/checkout@v5 + + - name: Cache pFUnit + id: cache-pfunit + uses: actions/cache@v5 + with: + key: pfunit-${{ env.PFUNIT_VERSION }}-gcc-${{ matrix.gcc-version }} + path: ${{ env.PFUNIT_PATH }}/install + + - name: Build pFUnit + if: steps.cache-pfunit.outputs.cache-hit != 'true' + run: | + git clone --depth 1 --branch ${PFUNIT_VERSION} \ + https://github.com/Goddard-Fortran-Ecosystem/pFUnit.git ${PFUNIT_PATH} + + cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=${PFUNIT_PATH}/install \ + -B ${PFUNIT_PATH}/build \ + -S ${PFUNIT_PATH} + + cmake --build ${PFUNIT_PATH}/build --parallel $(nproc) + cmake --install ${PFUNIT_PATH}/build + + - name: Build tests + run: | + cmake \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_PREFIX_PATH=${PFUNIT_PATH}/install \ + -B tests/build \ + -S tests + + cmake --build tests/build --parallel $(nproc) + + - name: Run tests + run: | + ctest \ + --build-config Debug \ + --output-on-failure \ + --test-dir tests/build \ + --verbose \ + --output-log tests/build/unit-tests.log + + - name: Upload test log + if: always() + uses: actions/upload-artifact@v6 + with: + name: unit-tests-log-gcc-${{ matrix.gcc-version }} + path: tests/build/unit-tests.log + retention-days: 14 + if-no-files-found: ignore diff --git a/src/core_atmosphere/mpas_atm_core_interface.F b/src/core_atmosphere/mpas_atm_core_interface.F index 21c651905e..7c7dadb204 100644 --- a/src/core_atmosphere/mpas_atm_core_interface.F +++ b/src/core_atmosphere/mpas_atm_core_interface.F @@ -99,7 +99,7 @@ end subroutine atm_setup_domain !> not allocated until after this routine has been called. ! !----------------------------------------------------------------------- - function atm_setup_packages(configs, streamInfo, packages, iocontext) result(ierr) + function atm_setup_packages(configs, streamInfo, packages, iocontext, dminfo) result(ierr) use mpas_dmpar use mpas_derived_types, only : mpas_pool_type, mpas_io_context_type, MPAS_streamInfo_type @@ -116,6 +116,7 @@ function atm_setup_packages(configs, streamInfo, packages, iocontext) result(ier type (MPAS_streamInfo_type), intent(inout) :: streamInfo type (mpas_pool_type), intent(inout) :: packages type (mpas_io_context_type), intent(inout) :: iocontext + type (dm_info), intent(in) :: dminfo integer :: ierr logical, pointer :: iauActive diff --git a/src/core_init_atmosphere/Registry.xml b/src/core_init_atmosphere/Registry.xml index 831901565a..160774e61c 100644 --- a/src/core_init_atmosphere/Registry.xml +++ b/src/core_init_atmosphere/Registry.xml @@ -217,6 +217,11 @@ description="Whether to use specific-humidity as the first-guess moisture variable. If this option is False, relative humidity will be used." possible_values="true or false"/> + + @@ -385,8 +390,10 @@ - + + + @@ -599,7 +606,7 @@ - + @@ -1025,15 +1032,15 @@ + packages="microphysics_aerosols"/> + packages="microphysics_aerosols"/> + packages="microphysics_aerosols"/> + description="Cloud water mixing ratio" + packages="qc"/> + description="Rain water mixing ratio" + packages="qr"/> + packages="microphysics_aerosols"/> + packages="microphysics_aerosols"/> + @@ -1203,18 +1213,20 @@ description="Water vapor mixing ratio on lateral boundary cells"/> + description="Cloud water mixing ratio on lateral boundary cells" + packages="qc"/> + description="Rain water mixing ratio on lateral boundary cells" + packages="qr"/> + packages="microphysics_aerosols"/> + packages="microphysics_aerosols"/> diff --git a/src/core_init_atmosphere/mpas_init_atm_core_interface.F b/src/core_init_atmosphere/mpas_init_atm_core_interface.F index f277a4a72f..d41d515d64 100644 --- a/src/core_init_atmosphere/mpas_init_atm_core_interface.F +++ b/src/core_init_atmosphere/mpas_init_atm_core_interface.F @@ -14,6 +14,9 @@ module init_atm_core_interface use mpas_io_units use mpas_log, only : mpas_log_write + private :: setup_hydrometeor_packages, & + mpas_split_string_new + contains @@ -100,7 +103,7 @@ end subroutine init_atm_setup_domain !> not allocated until after this routine has been called. ! !----------------------------------------------------------------------- - function init_atm_setup_packages(configs, streamInfo, packages, iocontext) result(ierr) + function init_atm_setup_packages(configs, streamInfo, packages, iocontext, dminfo) result(ierr) use mpas_derived_types, only : mpas_pool_type, mpas_io_context_type, MPAS_streamInfo_type use mpas_pool_routines, only : mpas_pool_get_config, mpas_pool_get_package @@ -111,6 +114,7 @@ function init_atm_setup_packages(configs, streamInfo, packages, iocontext) resul type (MPAS_streamInfo_type), intent(inout) :: streamInfo type (mpas_pool_type), intent(inout) :: packages type (mpas_io_context_type), intent(inout) :: iocontext + type (dm_info), intent(in) :: dminfo integer :: ierr logical :: lexist @@ -120,7 +124,7 @@ function init_atm_setup_packages(configs, streamInfo, packages, iocontext) resul logical, pointer :: config_native_gwd_static, config_static_interp, config_vertical_grid, config_met_interp logical, pointer :: config_native_gwd_gsl_static logical, pointer :: first_guess_field - logical, pointer :: mp_thompson_aers_in + logical, pointer :: microphysics_aerosols integer, pointer :: config_init_case logical, pointer :: noahmp, config_noahmp_static @@ -167,8 +171,8 @@ function init_atm_setup_packages(configs, streamInfo, packages, iocontext) resul nullify(met_stage_out) call mpas_pool_get_package(packages, 'met_stage_outActive', met_stage_out) - nullify(mp_thompson_aers_in) - call mpas_pool_get_package(packages, 'mp_thompson_aers_inActive', mp_thompson_aers_in) + nullify(microphysics_aerosols) + call mpas_pool_get_package(packages, 'microphysics_aerosolsActive', microphysics_aerosols) if (.not. associated(initial_conds) .or. & .not. associated(sfc_update) .or. & @@ -179,7 +183,7 @@ function init_atm_setup_packages(configs, streamInfo, packages, iocontext) resul .not. associated(vertical_stage_out) .or. & .not. associated(met_stage_in) .or. & .not. associated(met_stage_out) .or. & - .not. associated(mp_thompson_aers_in)) then + .not. associated(microphysics_aerosols)) then call mpas_log_write('********************************************************************************', messageType=MPAS_LOG_ERR) call mpas_log_write('* Error while setting up packages for init_atmosphere core.', messageType=MPAS_LOG_ERR) call mpas_log_write('********************************************************************************', messageType=MPAS_LOG_ERR) @@ -197,12 +201,12 @@ function init_atm_setup_packages(configs, streamInfo, packages, iocontext) resul if (config_init_case == 9) then lbcs = .true. - mp_thompson_aers_in = .false. + microphysics_aerosols = .false. inquire(file="QNWFA_QNIFA_SIGMA_MONTHLY.dat",exist=lexist) - if(lexist) mp_thompson_aers_in = .true. + if(lexist) microphysics_aerosols = .true. else lbcs = .false. - mp_thompson_aers_in = .false. + microphysics_aerosols = .false. end if if (config_init_case == 7) then @@ -226,9 +230,9 @@ function init_atm_setup_packages(configs, streamInfo, packages, iocontext) resul (.not. config_vertical_grid) met_stage_out = config_met_interp - mp_thompson_aers_in = .false. + microphysics_aerosols = .false. inquire(file="QNWFA_QNIFA_SIGMA_MONTHLY.dat",exist=lexist) - if((lexist .and. met_stage_out) .or. (lexist .and. met_stage_in)) mp_thompson_aers_in = .true. + if((lexist .and. met_stage_out) .or. (lexist .and. met_stage_in)) microphysics_aerosols = .true. else if (config_init_case == 8) then gwd_stage_in = .false. @@ -252,9 +256,9 @@ function init_atm_setup_packages(configs, streamInfo, packages, iocontext) resul met_stage_in = .true. met_stage_out = .true. - mp_thompson_aers_in = .false. + microphysics_aerosols = .false. inquire(file="QNWFA_QNIFA_SIGMA_MONTHLY.dat",exist=lexist) - if((lexist .and. met_stage_out) .or. (lexist .and. met_stage_in)) mp_thompson_aers_in = .true. + if((lexist .and. met_stage_out) .or. (lexist .and. met_stage_in)) microphysics_aerosols = .true. initial_conds = .false. ! Also, turn off the initial_conds package to avoid writing the IC "output" stream @@ -315,9 +319,202 @@ function init_atm_setup_packages(configs, streamInfo, packages, iocontext) resul return end if + if (config_init_case == 7 .or. config_init_case == 9) then + if (met_stage_out) then + call setup_hydrometeor_packages(packages, configs, dminfo, ierr) + end if + end if + end function init_atm_setup_packages + !*********************************************************************** + ! + ! function setup_hydrometeor_packages + ! + !> \brief Set up packages for hydrometeors based on contents of intermediate file + !> \author Michael Duda + !> \date 27 May 2026 + !> \details + !> This routine is responsible for setting up packages for hydrometeor species (qc, + !> qr, etc.) based on the value of the 'config_active_hydrometeors' namelist option, + !> and possibly on the contents of the initial intermediate file (i.e., the + !> intermediate file valid at the start time of a simulation; the assumption is + !> that all intermediate files contain the same set of fields). + !> + !> 1) If config_active_hydrometeors is set to 'detect_automatically', packages will + !> be set up by scanning through the fields in the initial intermediate file, + !> with packages being activated only for those hydrometeors that are present in + !> the file. + !> + !> 2) Alternatively, and to avoid the need to scan through all fields in the + !> initial intermediate file, if config_active_hydrometeors is set to a + !> semicolon-separated list of zero or more hydrometeor names (e.g., 'qc;qr'), + !> packages for those specified hydrometeors will be active, and the packages + !> for all unspecified hydrometeors will be inactive. + !> + !> If the setup of all hydrometeor packages is successful, a value of 0 is returned + !> in the ierr output argument; otherwise, a non-zero value is returned. + ! + !----------------------------------------------------------------------- + subroutine setup_hydrometeor_packages(packages, configs, dminfo, ierr) + + use init_atm_read_met, only : read_met_init, read_next_met_field, read_met_close, met_data + use mpas_timer, only : mpas_timer_start, mpas_timer_stop + + implicit none + + type (mpas_pool_type), intent(inout) :: packages + type (mpas_pool_type), intent(in) :: configs + type (dm_info), intent(in) :: dminfo + integer, intent(out) :: ierr + + character(len=StrKIND), pointer :: config_active_hydrometeors + character(len=StrKIND), pointer :: config_met_prefix, config_start_time + integer :: istatus + type (met_data) :: field + + integer :: i + character(len=:), dimension(:), allocatable :: hydrometeors + + logical, pointer :: qc, qr + + + call mpas_timer_start('setup_hydrometeor_packages') + + ierr = 0 + + call mpas_pool_get_config(configs, 'config_active_hydrometeors', config_active_hydrometeors) + + if (.not. associated(config_active_hydrometeors)) then + call mpas_log_write( & + "The namelist option 'config_active_hydrometeors' could not be found when setting up hydrometeor packages.", & + messageType=MPAS_LOG_ERR) + + ierr = 1 + call mpas_timer_stop('setup_hydrometeor_packages') + return + end if + + call mpas_pool_get_package(packages, 'qcActive', qc) + call mpas_pool_get_package(packages, 'qrActive', qr) + + if (.not. associated(qc) & + .or. .not. associated(qr) & + ) then + + call mpas_log_write('One or more packages could not be found when setting up hydrometeor packages.', & + messageType=MPAS_LOG_ERR) + + ierr = 1 + call mpas_timer_stop('setup_hydrometeor_packages') + return + end if + + qc = .false. + qr = .false. + + if (trim(config_active_hydrometeors) == 'detect_automatically') then + + call mpas_pool_get_config(configs, 'config_met_prefix', config_met_prefix) + call mpas_pool_get_config(configs, 'config_start_time', config_start_time) + + if (.not. associated(config_met_prefix) & + .or. .not. associated(config_start_time) & + ) then + + call mpas_log_write( & + 'One or more namelist options could not be found when setting up hydrometeor packages.', & + messageType=MPAS_LOG_ERR) + + ierr = 1 + call mpas_timer_stop('setup_hydrometeor_packages') + return + end if + + call mpas_log_write('Setting up hydrometeor packages from '//trim(config_met_prefix)//':'//config_start_time(1:13)) + + if (dminfo % my_proc_id == IO_NODE) then + call read_met_init(trim(config_met_prefix), .false., config_start_time(1:13), ierr) + + if (ierr == 0) then + call read_next_met_field(field, istatus) + else + call mpas_log_write('Could not open intermediate file ' & + //trim(config_met_prefix)//':'//config_start_time(1:13) & + //' to set up hydrometeor packages.', & + messageType=MPAS_LOG_ERR) + istatus = 1 + end if + + do while (istatus == 0) + if (trim(field % field) == 'QC') then + qc = .true. + else if (trim(field % field) == 'QR') then + qr = .true. + end if + + deallocate(field % slab) + + ! If all packages are true, there is no point in scanning through the rest + ! of the intermediate file, since this loop over fields can only switch packages + ! from false to true. + if (qc .and. qr) exit + + call read_next_met_field(field, istatus) + end do + + call read_met_close() + + call mpas_dmpar_bcast_int(dminfo, ierr) + call mpas_dmpar_bcast_logical(dminfo, qc) + call mpas_dmpar_bcast_logical(dminfo, qr) + else + call mpas_dmpar_bcast_int(dminfo, ierr) + call mpas_dmpar_bcast_logical(dminfo, qc) + call mpas_dmpar_bcast_logical(dminfo, qr) + end if + + else + + call mpas_log_write('Setting up hydrometeor packages from config_active_hydrometeors list: ' & + //trim(config_active_hydrometeors)) + + call mpas_split_string_new(trim(config_active_hydrometeors), ';', hydrometeors) + + do i = 1, size(hydrometeors) + select case (trim(hydrometeors(i))) + case ('qc') + qc = .true. + case ('qr') + qr = .true. + case default + call mpas_log_write('Unrecognized hydrometeor '//trim(hydrometeors(i)) & + //' found in config_active_hydrometeors', & + messageType=MPAS_LOG_WARN) + end select + end do + + deallocate(hydrometeors) + + end if + + if (ierr /= 0) then + call mpas_log_write('Failed to set up hydrometeor packages.', messageType=MPAS_LOG_ERR) + call mpas_timer_stop('setup_hydrometeor_packages') + return + end if + + call mpas_log_write(' QC = $l', logicArgs=[qc]) + call mpas_log_write(' QR = $l', logicArgs=[qr]) + call mpas_log_write('----- done setting up hydrometeor packages -----') + call mpas_log_write('') + + call mpas_timer_stop('setup_hydrometeor_packages') + + end subroutine setup_hydrometeor_packages + + !*********************************************************************** ! ! function init_atm_setup_clock @@ -522,6 +719,82 @@ function init_atm_setup_block(block) result(ierr) end function init_atm_setup_block + !----------------------------------------------------------------------- + ! routine mpas_split_string_new + ! + !> \brief Splits a string at a specified delimiter, returning an array of strings + !> \author Michael Duda + !> \date 28 May 2026 + !> \details + !> This routine takes as input a string and a delimiter character, and returns an + !> array of sub-strings from the input string that are separated by one or more + !> delimiter characters. + !> + !> If more than one delimiter character appears consecutively, no empty string + !> in the output subStrings array is generated. + !> + !> The length of the strings in the output subStrings argument is equal to the + !> length of the longest substring in the input string. + ! + !----------------------------------------------------------------------- + subroutine mpas_split_string_new(string, delimiter, subStrings) + + implicit none + + ! Arguments + character(len=*), intent(in) :: string + character, intent(in) :: delimiter + character(len=:), dimension(:), allocatable, intent(inout) :: subStrings + + ! Local variables + integer :: i, j, n_strs, strlen, max_strlen + + + i = 1 + n_strs = 0 + strlen = 0 + max_strlen = 1 + PARSE_LOOP: do while (i <= len(string)) + do while (string(i:i) == delimiter) + i = i + 1 + if (i > len(string)) exit PARSE_LOOP + end do + + n_strs = n_strs + 1 + strlen = 0 + do while (string(i:i) /= delimiter) + i = i + 1 + strlen = strlen + 1 + if (i > len(string)) exit + end do + max_strlen = max(strlen, max_strlen) + end do PARSE_LOOP + + if (allocated(subStrings)) deallocate(subStrings) + allocate(character(len=max_strlen) :: subStrings(n_strs)) + + i = 1 + n_strs = 0 + COPY_LOOP: do while (i <= len(string)) + do while (string(i:i) == delimiter) + i = i + 1 + if (i > len(string)) exit COPY_LOOP + end do + + n_strs = n_strs + 1 + j = 1 + do while (string(i:i) /= delimiter) + subStrings(n_strs)(j:j) = string(i:i) + i = i + 1 + j = j + 1 + if (i > len(string)) exit + end do + subStrings(n_strs)(j:max_strlen) = '' + end do COPY_LOOP + + end subroutine mpas_split_string_new + + #include "setup_immutable_streams.inc" #include "block_dimension_routines.inc" diff --git a/src/core_init_atmosphere/mpas_init_atm_read_met.F b/src/core_init_atmosphere/mpas_init_atm_read_met.F index 69c662766b..7b46c37d49 100644 --- a/src/core_init_atmosphere/mpas_init_atm_read_met.F +++ b/src/core_init_atmosphere/mpas_init_atm_read_met.F @@ -25,17 +25,37 @@ module init_atm_read_met ! Derived types type met_data - integer :: version, nx, ny, iproj - real (kind=real32) :: xfcst, xlvl, startlat, startlon, starti, startj, & - deltalat, deltalon, dx, dy, xlonc, & - truelat1, truelat2, earth_radius - real (kind=real32), pointer, dimension(:,:) :: slab - logical :: is_wind_grid_rel - character (len=9) :: field - character (len=24) :: hdate - character (len=25) :: units - character (len=32) :: map_source - character (len=46) :: desc + integer :: version = 5, & ! Format version (must =5 for WPS format) + nx = 0, & ! First (x) dimension of 2-d array 'slab' + ny = 0, & ! Second (y) dimension of 2-d array 'slab' + iproj = PROJ_LATLON ! Code for projection of data in array + + real (kind=real32) :: xfcst = 0.0_real32, & ! Forecast hour of data + xlvl = 0.0_real32, & ! Vertical level of data in 2-d array 'slab' + startlat = 0.0_real32, & ! Latitude of starting point (degrees) + startlon = 0.0_real32, & ! Longitude of starting point (degrees) + starti = 1.0_real32, & ! Starting x-/i-index + startj = 1.0_real32, & ! Starting y-/j-index + deltalat = 0.0_real32, & ! Grid spacing (degrees) in meridional direction + deltalon = 0.0_real32, & ! Grid spacing (degrees) in zonal direction + dx = 0.0_real32, & ! Grid spacing (km) in x-direction + dy = 0.0_real32, & ! Grid spacing (km) in y-direction + xlonc = 0.0_real32, & ! Standard longitude of projection + truelat1 = 0.0_real32, & ! First true latitude of projection + truelat2 = 0.0_real32, & ! Second true latitude of projection + earth_radius = EARTH_RADIUS_M / 1000.0_real32 ! Earth radius (km) + + real (kind=real32), pointer, dimension(:,:) :: slab => null() ! 2-d array of data + + logical :: is_wind_grid_rel = .false. ! Flag indicating whether winds are + ! relative to source grid (.true.) or + ! relative to earth (.false.) + + character (len=9) :: field = '' ! Name of the field + character (len=24) :: hdate = '' ! Valid date for data YYYY:MM:DD_HH:mm:ss + character (len=25) :: units = '' ! Units of data + character (len=32) :: map_source = '' ! Source model / originating center + character (len=46) :: desc = '' ! Short description of data end type met_data diff --git a/src/core_landice/mode_forward/mpas_li_core_interface.F b/src/core_landice/mode_forward/mpas_li_core_interface.F index e003bceb21..a091c4ea07 100644 --- a/src/core_landice/mode_forward/mpas_li_core_interface.F +++ b/src/core_landice/mode_forward/mpas_li_core_interface.F @@ -90,13 +90,14 @@ end subroutine li_setup_domain!}}} !> *not* allocated until after this routine is called. ! !----------------------------------------------------------------------- - function li_setup_packages(configPool, streamInfo, packagePool, iocontext) result(ierr) + function li_setup_packages(configPool, streamInfo, packagePool, iocontext, dminfo) result(ierr) implicit none type (mpas_pool_type), intent(inout) :: configPool type (MPAS_streamInfo_type), intent(inout) :: streamInfo type (mpas_pool_type), intent(inout) :: packagePool type (mpas_io_context_type), intent(inout) :: iocontext + type (dm_info), intent(in) :: dminfo integer :: ierr ! Local variables diff --git a/src/core_ocean/driver/mpas_ocn_core_interface.F b/src/core_ocean/driver/mpas_ocn_core_interface.F index 28609948d3..593b108c3a 100644 --- a/src/core_ocean/driver/mpas_ocn_core_interface.F +++ b/src/core_ocean/driver/mpas_ocn_core_interface.F @@ -95,7 +95,7 @@ end subroutine ocn_setup_domain!}}} !> *not* allocated until after this routine is called. ! !----------------------------------------------------------------------- - function ocn_setup_packages(configPool, streamInfo, packagePool, iocontext) result(ierr)!{{{ + function ocn_setup_packages(configPool, streamInfo, packagePool, iocontext, dminfo) result(ierr)!{{{ use ocn_analysis_driver @@ -103,6 +103,7 @@ function ocn_setup_packages(configPool, streamInfo, packagePool, iocontext) resu type (MPAS_streamInfo_type), intent(inout) :: streamInfo type (mpas_pool_type), intent(inout) :: packagePool type (mpas_io_context_type), intent(inout) :: iocontext + type (dm_info), intent(in) :: dminfo integer :: ierr diff --git a/src/core_seaice/model_forward/mpas_seaice_core_interface.F b/src/core_seaice/model_forward/mpas_seaice_core_interface.F index 82c8c0f2b0..90f2a22478 100644 --- a/src/core_seaice/model_forward/mpas_seaice_core_interface.F +++ b/src/core_seaice/model_forward/mpas_seaice_core_interface.F @@ -88,7 +88,7 @@ end subroutine seaice_setup_domain!}}} !> *not* allocated until after this routine is called. ! !----------------------------------------------------------------------- - function seaice_setup_packages(configPool, streamInfo, packagePool, iocontext) result(ierr)!{{{ + function seaice_setup_packages(configPool, streamInfo, packagePool, iocontext, dminfo) result(ierr)!{{{ use mpas_derived_types @@ -98,6 +98,7 @@ function seaice_setup_packages(configPool, streamInfo, packagePool, iocontext) r type (MPAS_streamInfo_type), intent(inout) :: streamInfo type (mpas_pool_type), intent(inout) :: packagePool type (mpas_io_context_type), intent(inout) :: iocontext + type (dm_info), intent(in) :: dminfo integer :: ierr diff --git a/src/core_sw/mpas_sw_core_interface.F b/src/core_sw/mpas_sw_core_interface.F index 04df23f19d..608576c09f 100644 --- a/src/core_sw/mpas_sw_core_interface.F +++ b/src/core_sw/mpas_sw_core_interface.F @@ -89,7 +89,7 @@ end subroutine sw_setup_domain!}}} !> *not* allocated until after this routine is called. ! !----------------------------------------------------------------------- - function sw_setup_packages(configPool, streamInfo, packagePool, iocontext) result(ierr)!{{{ + function sw_setup_packages(configPool, streamInfo, packagePool, iocontext, dminfo) result(ierr)!{{{ use mpas_derived_types @@ -99,6 +99,7 @@ function sw_setup_packages(configPool, streamInfo, packagePool, iocontext) resul type (MPAS_streamInfo_type), intent(inout) :: streamInfo type (mpas_pool_type), intent(inout) :: packagePool type (mpas_io_context_type), intent(inout) :: iocontext + type (dm_info), intent(in) :: dminfo integer :: ierr ierr = 0 diff --git a/src/core_test/mpas_test_core_interface.F b/src/core_test/mpas_test_core_interface.F index e600824bc4..cbbee6040a 100644 --- a/src/core_test/mpas_test_core_interface.F +++ b/src/core_test/mpas_test_core_interface.F @@ -89,7 +89,7 @@ end subroutine test_setup_domain!}}} !> *not* allocated until after this routine is called. ! !----------------------------------------------------------------------- - function test_setup_packages(configPool, streamInfo, packagePool, iocontext) result(ierr)!{{{ + function test_setup_packages(configPool, streamInfo, packagePool, iocontext, dminfo) result(ierr)!{{{ use mpas_derived_types @@ -99,6 +99,7 @@ function test_setup_packages(configPool, streamInfo, packagePool, iocontext) res type (MPAS_streamInfo_type), intent(inout) :: streamInfo type (mpas_pool_type), intent(inout) :: packagePool type (mpas_io_context_type), intent(inout) :: iocontext + type (dm_info), intent(in) :: dminfo integer :: ierr ierr = 0 diff --git a/src/driver/mpas_subdriver.F b/src/driver/mpas_subdriver.F index 68adea9dc3..80be1a4e8f 100644 --- a/src/driver/mpas_subdriver.F +++ b/src/driver/mpas_subdriver.F @@ -275,7 +275,7 @@ end subroutine xml_stream_get_attributes end if ierr = domain_ptr % core % setup_packages(domain_ptr % configs, domain_ptr % streamInfo, domain_ptr % packages, & - domain_ptr % iocontext) + domain_ptr % iocontext, domain_ptr % dminfo) if ( ierr /= 0 ) then call mpas_log_write('Package setup failed for core '//trim(domain_ptr % core % coreName), messageType=MPAS_LOG_CRIT) end if diff --git a/src/external/ezxml/ezxml.c b/src/external/ezxml/ezxml.c index 91ae2bc1d9..88f0002b0c 100644 --- a/src/external/ezxml/ezxml.c +++ b/src/external/ezxml/ezxml.c @@ -481,6 +481,7 @@ ezxml_t ezxml_parse_str(char *s, size_t len) int l, i, j; root->m = s; + root->len = -1; // so we know to free s in ezxml_free() if (! len) return ezxml_err(root, NULL, "root tag missing"); root->u = ezxml_str2utf8(&s, &len); // convert utf-16 to utf-8 root->e = (root->s = s) + len; // record start and end of work area diff --git a/src/framework/mpas_block_creator.F b/src/framework/mpas_block_creator.F index e9fea7253f..73b45fe8de 100644 --- a/src/framework/mpas_block_creator.F +++ b/src/framework/mpas_block_creator.F @@ -249,6 +249,8 @@ subroutine mpas_block_creator_build_0halo_cell_fields(nHalos, indexToCellIDBlock call mpas_dmpar_alltoall_field(cellsOnCellBlock, cellsOnCell_0Halo, sendingHaloLayers) call mpas_dmpar_alltoall_field(verticesOnCellBlock, verticesOnCell_0Halo, sendingHaloLayers) call mpas_dmpar_alltoall_field(edgesOnCellBlock, edgesOnCell_0Halo, sendingHaloLayers) + + deallocate(sendingHaloLayers) end subroutine mpas_block_creator_build_0halo_cell_fields!}}} !*********************************************************************** diff --git a/src/framework/mpas_core_types.inc b/src/framework/mpas_core_types.inc index 15a9866ccd..3789b8eb53 100644 --- a/src/framework/mpas_core_types.inc +++ b/src/framework/mpas_core_types.inc @@ -21,15 +21,17 @@ end interface abstract interface - function mpas_setup_packages_function(configs, streamInfo, packages, iocontext) result(iErr) + function mpas_setup_packages_function(configs, streamInfo, packages, iocontext, dminfo) result(iErr) import mpas_pool_type import mpas_io_context_type import mpas_streaminfo_type + import dm_info type (mpas_pool_type), intent(inout) :: configs type (mpas_streaminfo_type), intent(inout) :: streamInfo type (mpas_pool_type), intent(inout) :: packages type (mpas_io_context_type), intent(inout) :: iocontext + type (dm_info), intent(in) :: dminfo integer :: iErr end function mpas_setup_packages_function end interface diff --git a/src/framework/mpas_domain_routines.F b/src/framework/mpas_domain_routines.F index 5d6c563cf7..7d2c289c12 100644 --- a/src/framework/mpas_domain_routines.F +++ b/src/framework/mpas_domain_routines.F @@ -46,10 +46,7 @@ subroutine mpas_allocate_domain(dom)!{{{ allocate(dom % dminfo) nullify(dom % blocklist) - allocate(dom % configs) - allocate(dom % packages) allocate(dom % clock) - allocate(dom % streamManager) allocate(dom % ioContext) call mpas_pool_create_pool(dom % configs) @@ -87,9 +84,6 @@ subroutine mpas_allocate_block(nHaloLayers, b, dom, blockID) !{{{ b % domain => dom - allocate(b % structs) - allocate(b % dimensions) - allocate(b % allFields) call mpas_pool_create_pool(b % structs) call mpas_pool_create_pool(b % dimensions) call mpas_pool_create_pool(b % allFields) @@ -138,6 +132,8 @@ subroutine mpas_deallocate_domain(dom)!{{{ deallocate(dom % clock) deallocate(dom % ioContext) + deallocate(dom % dminfo) + end subroutine mpas_deallocate_domain!}}} @@ -167,17 +163,17 @@ subroutine mpas_deallocate_block(b)!{{{ call mpas_pool_destroy_pool(b % structs) call mpas_pool_destroy_pool(b % dimensions) - deallocate(b % parinfo % cellsToSend) - deallocate(b % parinfo % cellsToRecv) - deallocate(b % parinfo % cellsToCopy) + call mpas_dmpar_destroy_mulithalo_exchange_list(b % parinfo % cellsToSend) + call mpas_dmpar_destroy_mulithalo_exchange_list(b % parinfo % cellsToRecv) + call mpas_dmpar_destroy_mulithalo_exchange_list(b % parinfo % cellsToCopy) - deallocate(b % parinfo % edgesToSend) - deallocate(b % parinfo % edgesToRecv) - deallocate(b % parinfo % edgesToCopy) + call mpas_dmpar_destroy_mulithalo_exchange_list(b % parinfo % edgesToSend) + call mpas_dmpar_destroy_mulithalo_exchange_list(b % parinfo % edgesToRecv) + call mpas_dmpar_destroy_mulithalo_exchange_list(b % parinfo % edgesToCopy) - deallocate(b % parinfo % verticesToSend) - deallocate(b % parinfo % verticesToRecv) - deallocate(b % parinfo % verticesToCopy) + call mpas_dmpar_destroy_mulithalo_exchange_list(b % parinfo % verticesToSend) + call mpas_dmpar_destroy_mulithalo_exchange_list(b % parinfo % verticesToRecv) + call mpas_dmpar_destroy_mulithalo_exchange_list(b % parinfo % verticesToCopy) deallocate(b % parinfo) diff --git a/src/framework/mpas_framework.F b/src/framework/mpas_framework.F index 10367a3d9c..3aa4ad7b2f 100644 --- a/src/framework/mpas_framework.F +++ b/src/framework/mpas_framework.F @@ -65,7 +65,6 @@ subroutine mpas_framework_init_phase1(dminfo, external_comm)!{{{ integer, intent(in), optional :: external_comm #endif - allocate(dminfo) call mpas_dmpar_init(dminfo, external_comm) end subroutine mpas_framework_init_phase1!}}} @@ -182,14 +181,14 @@ subroutine mpas_framework_finalize(dminfo, domain, io_system)!{{{ call MPAS_io_finalize(domain % ioContext, .false.) - call mpas_deallocate_domain(domain) - call mpas_dmpar_finalize(dminfo) call mpas_finish_block_proc_list(dminfo) call mpas_timekeeping_finalize() + call mpas_deallocate_domain(domain) + end subroutine mpas_framework_finalize!}}} diff --git a/src/framework/mpas_io.F b/src/framework/mpas_io.F index 5b3b5642bb..724e71b802 100644 --- a/src/framework/mpas_io.F +++ b/src/framework/mpas_io.F @@ -210,7 +210,6 @@ subroutine MPAS_io_init(ioContext, io_task_count, io_task_stride, io_system, ier #endif #ifdef MPAS_SMIOL_SUPPORT - allocate(ioContext % smiol_context) #ifdef MPAS_USE_MPI_F08 local_ierr = SMIOLf_init(ioContext % dminfo % comm % mpi_val, & #else @@ -1256,6 +1255,7 @@ subroutine MPAS_io_def_var(handle, fieldname, fieldtype, dimnames, precision, ie ! TODO: Can we get the dimension sizes to see whether they match those from the file? end if + if (associated(inq_dimnames)) deallocate(inq_dimnames) return end if @@ -6335,6 +6335,7 @@ subroutine MPAS_io_close(handle, ierr) if (attlist_del % atthandle % attType == MPAS_ATT_INTA) deallocate(attlist_del % atthandle % attValueIntA) if (attlist_del % atthandle % attType == MPAS_ATT_REALA) deallocate(attlist_del % atthandle % attValueRealA) deallocate(attlist_del % atthandle) + deallocate(attlist_del) end do nullify(fieldlist_del % fieldhandle % attlist_head) nullify(fieldlist_del % fieldhandle % attlist_tail) @@ -6342,6 +6343,7 @@ subroutine MPAS_io_close(handle, ierr) deallocate(fieldlist_del % fieldhandle % dims) deallocate(fieldlist_del % fieldhandle) + deallocate(fieldlist_del) end do nullify(handle % fieldlist_head) nullify(handle % fieldlist_tail) @@ -6351,6 +6353,7 @@ subroutine MPAS_io_close(handle, ierr) dimlist_del => dimlist_ptr dimlist_ptr => dimlist_ptr % next deallocate(dimlist_del % dimhandle) + deallocate(dimlist_del) end do nullify(handle % dimlist_head) nullify(handle % dimlist_tail) @@ -6362,6 +6365,7 @@ subroutine MPAS_io_close(handle, ierr) if (attlist_del % atthandle % attType == MPAS_ATT_INTA) deallocate(attlist_del % atthandle % attValueIntA) if (attlist_del % atthandle % attType == MPAS_ATT_REALA) deallocate(attlist_del % atthandle % attValueRealA) deallocate(attlist_del % atthandle) + deallocate(attlist_del) end do nullify(handle % attlist_head) nullify(handle % attlist_tail) diff --git a/src/framework/mpas_pool_routines.F b/src/framework/mpas_pool_routines.F index 80339e3112..ad7df945b3 100644 --- a/src/framework/mpas_pool_routines.F +++ b/src/framework/mpas_pool_routines.F @@ -229,6 +229,14 @@ recursive subroutine mpas_pool_destroy_pool(inPool)!{{{ deallocate(ptr % data % simple_int) end if + else if (ptr % contentsType == MPAS_POOL_PACKAGE) then + + dptr => ptr % data + + if (dptr % contentsType == MPAS_POOL_LOGICAL) then + deallocate(dptr % simple_logical) + end if + else if (ptr % contentsType == MPAS_POOL_CONFIG) then dptr => ptr % data @@ -5795,6 +5803,7 @@ logical function pool_remove_member(inPool, key, memType)!{{{ end if !TODO: are there cases where we need to delete more data here? + if (associated(ptr_prev % data)) deallocate(ptr_prev % data) deallocate(ptr_prev) end if pool_remove_member = .true. @@ -5828,6 +5837,7 @@ logical function pool_remove_member(inPool, key, memType)!{{{ end if !TODO: are there cases where we need to delete more data here? + if (associated(ptr % data)) deallocate(ptr % data) deallocate(ptr) end if pool_remove_member = .true. diff --git a/src/framework/mpas_stream_manager.F b/src/framework/mpas_stream_manager.F index 8a4a1b4ad3..609663ed58 100644 --- a/src/framework/mpas_stream_manager.F +++ b/src/framework/mpas_stream_manager.F @@ -245,6 +245,19 @@ subroutine MPAS_stream_mgr_finalize(manager, ierr)!{{{ STREAM_ERROR_WRITE('Problems while destroying stream list') end if + ! + ! Remove all items from manager % alarms_in(put) list + ! + stream_cursor => manager % alarms_in % head + do while (associated(stream_cursor)) + call MPAS_stream_list_destroy(stream_cursor % streamList, ierr=err_local) + if (err_local /= MPAS_STREAM_LIST_NOERR) then + if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR + STREAM_ERROR_WRITE('Problems while destroying input alarms item') + end if + stream_cursor => stream_cursor % next + end do + ! ! Free up list of input alarms ! @@ -254,6 +267,19 @@ subroutine MPAS_stream_mgr_finalize(manager, ierr)!{{{ STREAM_ERROR_WRITE('Problems while destroying input alarms list') end if + ! + ! Remove all items from manager % alarms_out(put) list + ! + stream_cursor => manager % alarms_out % head + do while (associated(stream_cursor)) + call MPAS_stream_list_destroy(stream_cursor % streamList, ierr=err_local) + if (err_local /= MPAS_STREAM_LIST_NOERR) then + if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR + STREAM_ERROR_WRITE('Problems while destroying output alarms list') + end if + stream_cursor => stream_cursor % next + end do + ! ! Free up list of output alarms ! diff --git a/src/framework/regex_matching.c b/src/framework/regex_matching.c index d37a23436d..17136507a7 100644 --- a/src/framework/regex_matching.c +++ b/src/framework/regex_matching.c @@ -1,14 +1,71 @@ #include #include #include +#include #define MAX_LEN 1024 -void check_regex_match(const char * pattern, const char * str, int *imatch){ +static const char *BRE_SPECIAL_CHARS = ".[\\*^$"; + +/****************************************************************************** + * + * possible_bre + * + * Determine whether an input string may be a Basic Regular Expression (BRE). + * This function may return false positives (i.e., a return value indicating + * that a string is a BRE, when in fact the string is not a BRE), but it will + * never return a false negative (i.e., a return value indicating that a string + * is not a BRE, when in fact the string is a BRE). + * + * Inputs: + * s - a null-terminated string + * + * Return value: 1 if the input string 's' is possibly a BRE and 0 otherwise. + * + ******************************************************************************/ +static int possible_bre(const char *s) +{ + for (; *s != '\0'; s++) { + if (strchr(BRE_SPECIAL_CHARS, *s) != NULL) { + return 1; + } + } + return 0; +} + + +/****************************************************************************** + * + * check_regex_match + * + * Determine whether an input string 'str' matches the Basic Regular Expression + * (BRE) 'pattern'. + * + * Inputs: + * pattern - a null-terminated string that may contain any valid BRE, + * including a simple string + * str - a null-terminated string to be checked against pattern + * + * Return value: 1 if the input string str matches the BRE pattern, + * 0 if the input string does not match the BRE pattern, and + * -1 if an error occurred. + * + ******************************************************************************/ +void check_regex_match(const char * pattern, const char * str, int *imatch) +{ regex_t regex; char bracketed_pattern[MAX_LEN]; int ierr, len; + /* + * If pattern is a simple string and not a basic regular expression, + * a string comparison will suffice + */ + if (!possible_bre(pattern)) { + *imatch = (strcmp(pattern, str) == 0) ? 1 : 0; + return; + } + *imatch = 0; len = snprintf(bracketed_pattern, 1024, "^%s$", pattern); if ( len >= MAX_LEN ) { @@ -34,4 +91,3 @@ void check_regex_match(const char * pattern, const char * str, int *imatch){ *imatch = -1; } } - diff --git a/src/framework/xml_stream_parser.c b/src/framework/xml_stream_parser.c index 00b22fd009..470cc0fe63 100644 --- a/src/framework/xml_stream_parser.c +++ b/src/framework/xml_stream_parser.c @@ -1057,7 +1057,7 @@ void xml_stream_parser(char *fname, void *manager, int *mpi_comm, int *status) const char *streamID2, *interval_in2, *interval_out2; char interval_name[256]; char match_stream_name[256]; - char *packages, *package; + char *packages, *packages_ptr, *package; char filename_interval_string[256]; char ref_time_local[256]; char rec_intv_local[256]; @@ -1379,6 +1379,7 @@ void xml_stream_parser(char *fname, void *manager, int *mpi_comm, int *status) if (packagelist != NULL) { packages = strdup(packagelist); + packages_ptr = packages; package = strsep(&packages, ";"); stream_mgr_add_pkg_c(manager, streamID, package, &err); @@ -1403,7 +1404,7 @@ void xml_stream_parser(char *fname, void *manager, int *mpi_comm, int *status) } } - free(packages); + free(packages_ptr); } } @@ -1688,6 +1689,7 @@ void xml_stream_parser(char *fname, void *manager, int *mpi_comm, int *status) if (packagelist != NULL) { packages = strdup(packagelist); + packages_ptr = packages; package = strsep(&packages, ";"); stream_mgr_add_pkg_c(manager, streamID, package, &err); @@ -1712,7 +1714,7 @@ void xml_stream_parser(char *fname, void *manager, int *mpi_comm, int *status) } } - free(packages); + free(packages_ptr); } for (varfile_xml = ezxml_child(stream_xml, "file"); varfile_xml; varfile_xml = ezxml_next(varfile_xml)) { diff --git a/src/tools/registry/gen_inc.c b/src/tools/registry/gen_inc.c index bee81db012..562c53af3d 100644 --- a/src/tools/registry/gen_inc.c +++ b/src/tools/registry/gen_inc.c @@ -1065,7 +1065,7 @@ int parse_dimensions_from_registry(ezxml_t registry)/*{{{*/ fortprintf(fd, "\n"); - fortprintf(fd, "call mpas_log_write('Assigning remaining dimensions from definitions in Registry.xml ...')\n"); + fortprintf(fd, " call mpas_log_write('Assigning remaining dimensions from definitions in Registry.xml ...')\n"); for (dims_xml = ezxml_child(registry, "dims"); dims_xml; dims_xml = dims_xml->next) { for (dim_xml = ezxml_child(dims_xml, "dim"); dim_xml; dim_xml = dim_xml->next) { @@ -1081,14 +1081,17 @@ int parse_dimensions_from_registry(ezxml_t registry)/*{{{*/ if(strncmp(dimdef, "namelist:", 9) == 0){ snprintf(option_name, 1024, "%s", (dimdef)+9); fortprintf(fd, " %s = %s\n", dimname, option_name); - fortprintf(fd, "call mpas_log_write(' %s = $i (%s)', intArgs=(/%s/))\n", dimname, option_name, option_name); + fortprintf(fd, " call mpas_log_write(' %s = $i (%s)', intArgs=(/%s/))\n", dimname, option_name, option_name); } else { fortprintf(fd, " %s = %s\n", dimname, dimdef); - fortprintf(fd, "call mpas_log_write(' %s = $i', intArgs=(/%s/))\n", dimname, dimdef); + fortprintf(fd, " call mpas_log_write(' %s = $i', intArgs=(/%s/))\n", dimname, dimdef); } fortprintf(fd, " call mpas_pool_add_dimension(dimensionPool, '%s', %s)\n", dimname, dimname); - fortprintf(fd, " else if ( %s == MPAS_MISSING_DIM ) then\n", dimname, dimname); + fortprintf(fd, " deallocate(%s)\n", dimname); + fortprintf(fd, " ! Now let %s point to pool memory so the dimension can be referenced later in this routine as needed\n", dimname); + fortprintf(fd, " call mpas_pool_get_dimension(dimensionPool, '%s', %s)\n", dimname, dimname); + fortprintf(fd, " else if ( %s == MPAS_MISSING_DIM ) then\n", dimname, dimname); // Namelist defined dimension if(strncmp(dimdef, "namelist:", 9) == 0){ snprintf(option_name, 1024, "%s", (dimdef)+9); @@ -1097,12 +1100,15 @@ int parse_dimensions_from_registry(ezxml_t registry)/*{{{*/ fortprintf(fd, " %s = %s\n", dimname, dimdef); } - fortprintf(fd, " end if\n\n"); + fortprintf(fd, " end if\n\n"); } else { fortprintf(fd, " if ( .not. associated(%s) ) then\n", dimname); fortprintf(fd, " allocate(%s)\n", dimname); fortprintf(fd, " %s = MPAS_MISSING_DIM\n", dimname); fortprintf(fd, " call mpas_pool_add_dimension(dimensionPool, '%s', %s)\n", dimname, dimname); + fortprintf(fd, " deallocate(%s)\n", dimname); + fortprintf(fd, " ! Now let %s point to pool memory so the dimension can be referenced later in this routine as needed\n", dimname); + fortprintf(fd, " call mpas_pool_get_dimension(dimensionPool, '%s', %s)\n", dimname, dimname); fortprintf(fd, " end if\n\n"); } } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 0000000000..3e8f2a9fee --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,40 @@ +cmake_minimum_required(VERSION 3.21) + +project(mpas_unit_tests + VERSION 0.1.0 + DESCRIPTION "Unit tests for standalone MPAS-Model procedures" + LANGUAGES Fortran +) + +# Build a small library from MPAS source files that have testable, +# standalone procedures with minimal dependencies. +# Add source files here as more tests are developed. +set(MPAS_SRC ${CMAKE_CURRENT_SOURCE_DIR}/../src) + +add_library(mpas_testable_procedures) +target_sources(mpas_testable_procedures + PRIVATE + ${MPAS_SRC}/framework/mpas_kind_types.F + ${MPAS_SRC}/operators/mpas_spline_interpolation.F +) +target_include_directories(mpas_testable_procedures + INTERFACE ${CMAKE_CURRENT_BINARY_DIR} +) + +# MPAS .F files are free-form Fortran with C preprocessor directives. +# gfortran treats .F as fixed-form by default, so we must override. +target_compile_options(mpas_testable_procedures + PRIVATE + $<$:-cpp -ffree-form -fbacktrace -ffree-line-length-none> + $<$:-fpp -free -traceback> + $<$:-fpp -free -traceback> +) + +if(PROJECT_IS_TOP_LEVEL) + include(CTest) + enable_testing() +endif() + +if(BUILD_TESTING) + add_subdirectory(unit) +endif() diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt new file mode 100644 index 0000000000..68777daffb --- /dev/null +++ b/tests/unit/CMakeLists.txt @@ -0,0 +1,13 @@ +find_package(PFUNIT REQUIRED) + +# Spline interpolation tests +add_pfunit_ctest(test_spline_interpolation + TEST_SOURCES test_spline_interpolation.pf + LINK_LIBRARIES mpas_testable_procedures +) +target_compile_options(test_spline_interpolation + PRIVATE + $<$:-fbacktrace -ffree-line-length-none> + $<$:-traceback> + $<$:-traceback> +) diff --git a/tests/unit/test_spline_interpolation.pf b/tests/unit/test_spline_interpolation.pf new file mode 100644 index 0000000000..ae61d26479 --- /dev/null +++ b/tests/unit/test_spline_interpolation.pf @@ -0,0 +1,200 @@ +!----------------------------------------------------------------------- +! Unit tests for mpas_spline_interpolation module +! +! Tests cubic spline and linear interpolation routines against +! known analytic functions. Adapted from the testing patterns in +! ESCOMP/CAM-SIMA MPAS dynamical core CI. +!----------------------------------------------------------------------- + +module test_spline_interpolation + + use funit + use mpas_kind_types, only: RKIND + use mpas_spline_interpolation, only: & + mpas_cubic_spline_coefficients, & + mpas_interpolate_cubic_spline, & + mpas_integrate_cubic_spline, & + mpas_interpolate_linear + + implicit none + + real(kind=RKIND), parameter :: tol = 1.0e-10_RKIND + +contains + + !--------------------------------------------------------------------- + ! Linear interpolation tests + !--------------------------------------------------------------------- + + @test + subroutine test_linear_interp_identity() + ! Interpolating at the same nodes should return the original values. + ! mpas_interpolate_linear uses strict < on x(kIn+1), so it cannot + ! interpolate at x(n). We test n-1 interior+left-boundary nodes. + integer, parameter :: n = 5 + real(kind=RKIND) :: x(n), y(n), xOut(n-1), yOut(n-1) + integer :: i + + x = [1.0_RKIND, 2.0_RKIND, 3.0_RKIND, 4.0_RKIND, 5.0_RKIND] + y = [10.0_RKIND, 20.0_RKIND, 30.0_RKIND, 40.0_RKIND, 50.0_RKIND] + xOut = x(1:n-1) + + call mpas_interpolate_linear(x, y, n, xOut, yOut, n-1) + + do i = 1, n-1 + @assertEqual(y(i), yOut(i), tol) + end do + end subroutine test_linear_interp_identity + + @test + subroutine test_linear_interp_midpoints() + ! Midpoint interpolation of a linear function should be exact + integer, parameter :: n = 3, nOut = 2 + real(kind=RKIND) :: x(n), y(n), xOut(nOut), yOut(nOut) + + x = [0.0_RKIND, 1.0_RKIND, 2.0_RKIND] + y = [0.0_RKIND, 2.0_RKIND, 4.0_RKIND] + xOut = [0.5_RKIND, 1.5_RKIND] + + call mpas_interpolate_linear(x, y, n, xOut, yOut, nOut) + + @assertEqual(1.0_RKIND, yOut(1), tol) + @assertEqual(3.0_RKIND, yOut(2), tol) + end subroutine test_linear_interp_midpoints + + @test + subroutine test_linear_interp_quadratic_function() + ! Linear interpolation of y = x^2 at midpoints + integer, parameter :: n = 5, nOut = 4 + real(kind=RKIND) :: x(n), y(n), xOut(nOut), yOut(nOut) + real(kind=RKIND) :: expected + integer :: i + + x = [0.0_RKIND, 1.0_RKIND, 2.0_RKIND, 3.0_RKIND, 4.0_RKIND] + y = [0.0_RKIND, 1.0_RKIND, 4.0_RKIND, 9.0_RKIND, 16.0_RKIND] + xOut = [0.5_RKIND, 1.5_RKIND, 2.5_RKIND, 3.5_RKIND] + + call mpas_interpolate_linear(x, y, n, xOut, yOut, nOut) + + ! Linear interpolation of x^2 at midpoints: (x_i^2 + x_{i+1}^2) / 2 + do i = 1, nOut + expected = (x(i)**2 + x(i+1)**2) / 2.0_RKIND + @assertEqual(expected, yOut(i), tol) + end do + end subroutine test_linear_interp_quadratic_function + + !--------------------------------------------------------------------- + ! Cubic spline tests + !--------------------------------------------------------------------- + + @test + subroutine test_cubic_spline_linear_function() + ! Cubic spline of a linear function should have zero second derivatives + integer, parameter :: n = 5 + real(kind=RKIND) :: x(n), y(n), y2(n) + + x = [1.0_RKIND, 2.0_RKIND, 3.0_RKIND, 4.0_RKIND, 5.0_RKIND] + y = [2.0_RKIND, 4.0_RKIND, 6.0_RKIND, 8.0_RKIND, 10.0_RKIND] + + call mpas_cubic_spline_coefficients(x, y, n, y2) + + @assertEqual(0.0_RKIND, y2(1), tol) + @assertEqual(0.0_RKIND, y2(n), tol) + end subroutine test_cubic_spline_linear_function + + @test + subroutine test_cubic_spline_reproduces_nodes() + ! Interpolating at the original nodes should return the original values. + ! mpas_interpolate_cubic_spline uses strict < on x(kIn+1), so it cannot + ! interpolate at x(n). We test n-1 interior+left-boundary nodes. + integer, parameter :: n = 6 + real(kind=RKIND) :: x(n), y(n), y2(n) + real(kind=RKIND) :: xOut(n-1), yOut(n-1) + integer :: i + + x = [0.0_RKIND, 1.0_RKIND, 2.0_RKIND, 3.0_RKIND, 4.0_RKIND, 5.0_RKIND] + do i = 1, n + y(i) = sin(x(i)) + end do + xOut = x(1:n-1) + + call mpas_cubic_spline_coefficients(x, y, n, y2) + call mpas_interpolate_cubic_spline(x, y, y2, n, xOut, yOut, n-1) + + do i = 1, n-1 + @assertEqual(y(i), yOut(i), tol) + end do + end subroutine test_cubic_spline_reproduces_nodes + + @test + subroutine test_cubic_spline_accuracy_sine() + ! Cubic spline of sin(x) should be accurate to ~O(h^4) + integer, parameter :: n = 11, nOut = 5 + real(kind=RKIND) :: x(n), y(n), y2(n) + real(kind=RKIND) :: xOut(nOut), yOut(nOut) + real(kind=RKIND) :: h, maxErr + integer :: i + + ! Nodes from 0 to pi with h = pi/10 + h = acos(-1.0_RKIND) / real(n - 1, RKIND) + do i = 1, n + x(i) = real(i - 1, RKIND) * h + y(i) = sin(x(i)) + end do + + ! Test at midpoints between nodes 3-7 (away from boundaries) + do i = 1, nOut + xOut(i) = x(i + 3) + 0.5_RKIND * h + end do + + call mpas_cubic_spline_coefficients(x, y, n, y2) + call mpas_interpolate_cubic_spline(x, y, y2, n, xOut, yOut, nOut) + + maxErr = 0.0_RKIND + do i = 1, nOut + maxErr = max(maxErr, abs(yOut(i) - sin(xOut(i)))) + end do + + ! With h ~ 0.31, cubic spline error should be well under 1e-4 + @assertTrue(maxErr < 1.0e-4_RKIND, 'Cubic spline of sin(x) error too large') + end subroutine test_cubic_spline_accuracy_sine + + @test + subroutine test_cubic_spline_integrate_linear() + ! Integral of a linear function f(x) = 2x from 0 to 3 should be 9 + integer, parameter :: n = 4 + real(kind=RKIND) :: x(n), y(n), y2(n) + real(kind=RKIND) :: integral + + x = [0.0_RKIND, 1.0_RKIND, 2.0_RKIND, 3.0_RKIND] + y = [0.0_RKIND, 2.0_RKIND, 4.0_RKIND, 6.0_RKIND] + + call mpas_cubic_spline_coefficients(x, y, n, y2) + call mpas_integrate_cubic_spline(x, y, y2, n, 0.0_RKIND, 3.0_RKIND, integral) + + @assertEqual(9.0_RKIND, integral, tol) + end subroutine test_cubic_spline_integrate_linear + + @test + subroutine test_cubic_spline_integrate_sine() + ! Integral of sin(x) from 0 to pi should be 2.0 + integer, parameter :: n = 21 + real(kind=RKIND) :: x(n), y(n), y2(n) + real(kind=RKIND) :: integral, pi_val + integer :: i + + pi_val = acos(-1.0_RKIND) + do i = 1, n + x(i) = real(i - 1, RKIND) * pi_val / real(n - 1, RKIND) + y(i) = sin(x(i)) + end do + + call mpas_cubic_spline_coefficients(x, y, n, y2) + call mpas_integrate_cubic_spline(x, y, y2, n, 0.0_RKIND, pi_val, integral) + + ! Analytic integral of sin(x) from 0 to pi = 2.0 + ! With 21 nodes (h~0.157), spline integration error is ~1.7e-6 + @assertEqual(2.0_RKIND, integral, 1.0e-5_RKIND) + end subroutine test_cubic_spline_integrate_sine + +end module test_spline_interpolation