E3SM-Project · Copilot · Aug 30, 2026 · Aug 30, 2026 · Aug 30, 2026 · Aug 31, 2026
diff --git a/.github/workflows/ghci-snl.yaml b/.github/workflows/ghci-snl.yaml
@@ -25,11 +25,24 @@ concurrency:
 
 jobs:
   base:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.runs_on }}
     permissions:
       contents: read
       packages: write
       id-token: write
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: x86_64
+            runs_on: ubuntu-latest
+            spack_arch: linux-rhel9-x86_64
+            tag_suffix: '-x86_64'
+          - arch: aarch64
+            runs_on: ubuntu-22.04-arm
+            spack_arch: linux-rhel9-aarch64
+            tag_suffix: '-aarch64'
+    name: base-${{ matrix.arch }}
 
     steps:
       - name: Checkout
@@ -55,12 +68,12 @@ jobs:
         with:
           images: ghcr.io/${{ github.repository_owner }}/e3sm-ghci-snl
           tags: |
-            type=raw,value=base,enable=${{ github.ref == 'refs/heads/main' }}
-            type=ref,event=pr,prefix=base-pr-
-            type=ref,event=branch,prefix=base-
-            type=match,pattern=ghci-snl-(.*),group=1,prefix=base-
+            type=raw,value=base${{ matrix.tag_suffix }},enable=${{ github.ref == 'refs/heads/main' }}
+            type=ref,event=pr,prefix=base${{ matrix.tag_suffix }}-pr-
+            type=ref,event=branch,prefix=base${{ matrix.tag_suffix }}-
+            type=match,pattern=ghci-snl-(.*),group=1,prefix=base${{ matrix.tag_suffix }}-
           labels: |
-            org.opencontainers.image.description=Base image for E3SM ghci-snl containers used for testing on GitHub actions
+            org.opencontainers.image.description=Base image (${{ matrix.arch }}) for E3SM ghci-snl containers used for testing on GitHub actions
 
       - name: Push
         uses: docker/build-push-action@v6
@@ -70,11 +83,13 @@ jobs:
           push: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          build-args: |
+            SPACK_ARCH=${{ matrix.spack_arch }}
+          cache-from: type=gha,scope=base-${{ matrix.arch }}
+          cache-to: type=gha,scope=base-${{ matrix.arch }},mode=max
 
   compiler:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.runs_on }}
     needs: base
     permissions:
       contents: read
@@ -87,10 +102,22 @@ jobs:
           - compiler: gnu
             spack_compiler: gcc
             version: 13.3.0
+            arch: x86_64
+            runs_on: ubuntu-latest
+            tag_suffix: ''
           - compiler: intel
             spack_compiler: oneapi
             version: 2024.1.0
-    name: ${{ matrix.compiler }}
+            arch: x86_64
+            runs_on: ubuntu-latest
+            tag_suffix: ''
+          - compiler: gnu
+            spack_compiler: gcc
+            version: 13.3.0
+            arch: aarch64
+            runs_on: ubuntu-22.04-arm
+            tag_suffix: '-aarch64'
+    name: ${{ matrix.compiler }}-${{ matrix.arch }}
 
     steps:
       - name: Checkout
@@ -116,12 +143,12 @@ jobs:
         with:
           images: ghcr.io/${{ github.repository_owner }}/e3sm-ghci-snl
           tags: |
-            type=raw,value=${{ matrix.compiler }}-${{ matrix.version }},enable=${{ github.ref == 'refs/heads/main' }}
-            type=ref,event=pr,prefix=${{ matrix.compiler }}-${{ matrix.version }}-pr-
-            type=ref,event=branch,prefix=${{ matrix.compiler }}-${{ matrix.version }}-
-            type=match,pattern=ghci-snl-(.*),group=1,prefix=${{ matrix.compiler }}-${{ matrix.version }}-
+            type=raw,value=${{ matrix.compiler }}-${{ matrix.version }}${{ matrix.tag_suffix }},enable=${{ github.ref == 'refs/heads/main' }}
+            type=ref,event=pr,prefix=${{ matrix.compiler }}-${{ matrix.version }}${{ matrix.tag_suffix }}-pr-
+            type=ref,event=branch,prefix=${{ matrix.compiler }}-${{ matrix.version }}${{ matrix.tag_suffix }}-
+            type=match,pattern=ghci-snl-(.*),group=1,prefix=${{ matrix.compiler }}-${{ matrix.version }}${{ matrix.tag_suffix }}-
           labels: |
-            org.opencontainers.image.description=${{ matrix.compiler }} compiler base image for E3SM ghci-snl images used for testing on GitHub actions
+            org.opencontainers.image.description=${{ matrix.compiler }} compiler base image (${{ matrix.arch }}) for E3SM ghci-snl images used for testing on GitHub actions
 
       - name: Push
         uses: docker/build-push-action@v6
@@ -134,12 +161,12 @@ jobs:
           build-args: |
             COMPILER=${{ matrix.spack_compiler }}
             COMPILER_VERSION=${{ matrix.version }}
-            BASE_TAG=${{ github.event_name == 'pull_request' && format('base-pr-{0}', github.event.pull_request.number) || 'base' }}
-          cache-from: type=gha,scope=${{ matrix.compiler }}
-          cache-to: type=gha,scope=${{ matrix.compiler }},mode=max
+            BASE_TAG=${{ github.event_name == 'pull_request' && format('base{0}-pr-{1}', matrix.tag_suffix, github.event.pull_request.number) || format('base{0}', matrix.tag_suffix) }}
+          cache-from: type=gha,scope=${{ matrix.compiler }}-${{ matrix.arch }}
+          cache-to: type=gha,scope=${{ matrix.compiler }}-${{ matrix.arch }},mode=max
 
   final-env:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.runs_on }}
     needs: compiler
     permissions:
       contents: read
@@ -150,14 +177,41 @@ jobs:
       matrix:
         include:
           - compiler: gnu
+            dir: gnu-cpu-env
             tag: gnu-cpu-env
             version: 13.3.0
+            arch: x86_64
+            runs_on: ubuntu-latest
+            tag_suffix: ''
           - compiler: gnu
+            dir: gnu-cuda-env
             tag: gnu-cuda-env
             version: 13.3.0
+            arch: x86_64
+            runs_on: ubuntu-latest
+            tag_suffix: ''
           - compiler: intel
+            dir: intel-cpu-env
             tag: intel-cpu-env
             version: 2024.1.0
+            arch: x86_64
+            runs_on: ubuntu-latest
+            tag_suffix: ''
+          - compiler: gnu
+            dir: gnu-cpu-env
+            tag: gnu-cpu-env-aarch64
+            version: 13.3.0
+            arch: aarch64
+            runs_on: ubuntu-22.04-arm
+            tag_suffix: '-aarch64'
+          - compiler: gnu
+            dir: gnu-cuda-env-aarch64
+            tag: gnu-cuda-env-aarch64
+            version: 13.3.0
+            arch: aarch64
+            runs_on: ubuntu-22.04-arm
+            tag_suffix: '-aarch64'
+    name: ${{ matrix.tag }}
 
     steps:
       - name: Checkout
@@ -188,17 +242,17 @@ jobs:
             type=ref,event=branch,prefix=${{ matrix.tag }}-
             type=match,pattern=ghci-snl-(.*),group=1,prefix=${{ matrix.tag }}-
           labels: |
-            org.opencontainers.image.description=${{ matrix.tag }} image for E3SM ghci-snl testing on CPU on GitHub actions
+            org.opencontainers.image.description=${{ matrix.tag }} image (${{ matrix.arch }}) for E3SM ghci-snl testing on GitHub actions
 
       - name: Push
         uses: docker/build-push-action@v6
         with:
-          context: ghci-snl/final/${{ matrix.tag }}
-          file: ghci-snl/final/${{ matrix.tag }}/Dockerfile
+          context: ghci-snl/final/${{ matrix.dir }}
+          file: ghci-snl/final/${{ matrix.dir }}/Dockerfile
           push: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           build-args: |
-            BASE_TAG=${{ matrix.compiler }}-${{ matrix.version }}${{ github.event_name == 'pull_request' && format('-pr-{0}', github.event.pull_request.number) || '' }}
+            BASE_TAG=${{ matrix.compiler }}-${{ matrix.version }}${{ matrix.tag_suffix }}${{ github.event_name == 'pull_request' && format('-pr-{0}', github.event.pull_request.number) || '' }}
           cache-from: type=gha,scope=${{ matrix.tag }}
           cache-to: type=gha,scope=${{ matrix.tag }},mode=max
diff --git a/ghci-snl/README b/ghci-snl/README
@@ -28,6 +28,25 @@ podman run --rm -it \
   ghcr.io/E3SM-Project/e3sm-ghci-snl:gnu-cpu-env /bin/bash
 ```
 
+## Available images
+
+Each image is published for `x86_64`; the GNU-based ones are also published for
+`aarch64` (native builds, no emulation), under an `-aarch64`-suffixed tag:
+
+| Tag                      | Arch      | Compiler | Notes                                             |
+|--------------------------|-----------|----------|----------------------------------------------------|
+| `gnu-cpu-env`            | x86_64    | GNU      | CPU-only stack                                     |
+| `gnu-cpu-env-aarch64`    | aarch64   | GNU      | CPU-only stack; runs well on macOS M1/Apple Silicon |
+| `gnu-cuda-env`           | x86_64    | GNU      | CUDA 12.4 stack                                    |
+| `gnu-cuda-env-aarch64`   | aarch64   | GNU      | CUDA 13.0 stack; targets NVIDIA aarch64 ("superchip") systems such as Grace Hopper/Blackwell |
+| `intel-cpu-env`          | x86_64    | Intel    | CPU-only stack (Intel oneAPI is x86_64-only)        |
+
+To pull an aarch64 image explicitly (e.g. on an Apple Silicon Mac via Podman machine, or on an NVIDIA Grace-based host):
+
+```bash
+podman pull ghcr.io/E3SM-Project/e3sm-ghci-snl:gnu-cpu-env-aarch64
+```
+
 ## Important Storage & Permission Guidelines
 
 - **Input data** (`inputdata`): Required for downloading and reading datasets.
@@ -56,6 +75,16 @@ pip install <package-name>
 
 If you need to test with a different compiler or library version (for example, upgrading from GCC 13.3.0 to GCC 14), you can rebuild the container stack locally using custom build arguments:
 
+- **(aarch64 only) Build the base image for your target arch**:
+  The `x86_64` base image is built by default; for `aarch64` targets, override `SPACK_ARCH`:
+
+```bash
+podman build \\
+  --build-arg SPACK_ARCH=linux-rhel9-aarch64 \\
+  --tag e3sm-base:aarch64 \\
+  ghci-snl/base/
+```
+
 - **Build the custom compiler container**:
   Navigate to the repository root and build the compiler image specifying your target version:
 

diff --git a/ghci-snl/base/Dockerfile b/ghci-snl/base/Dockerfile
@@ -75,8 +75,10 @@ ENV LANG=en_US.UTF-8 \
 
 # 7. Setup SPACK
 ARG SPACK_VER=v0.23.1
+# Override at build time (e.g. --build-arg SPACK_ARCH=linux-rhel9-aarch64) for aarch64 images
+ARG SPACK_ARCH=linux-rhel9-x86_64
 ENV SPACK_ROOT=/spack \
-    SPACK_ARCH=linux-rhel9-x86_64 \
+    SPACK_ARCH=${SPACK_ARCH} \
     SPACK_SYSTEM_CONFIG_PATH=/spack-configuration \
     PATH=/spack/bin:$PATH
 

diff --git a/ghci-snl/final/gnu-cpu-env/Dockerfile b/ghci-snl/final/gnu-cpu-env/Dockerfile
@@ -21,6 +21,24 @@ RUN source /spack/share/spack/setup-env.sh && \
 RUN source /spack/share/spack/setup-env.sh && \
     spack install hdf5 netcdf-c+mpi netcdf-fortran parallel-netcdf %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH}
 
+# MOAB's "zoltan" TPL, as fetched by the vendor install-bootstrap.sh script below from
+# its default (circa-2013) dist tarball mirror, ships a bundled config.guess/
+# config.sub that predates the aarch64 GNU triplet, so its ./configure fails
+# immediately on aarch64 -- and the vendor script has no remediation pattern that
+# recognizes that failure, so it just aborts. We can't patch that third-party
+# script, so work around it here instead: pre-build zoltan via spack (whose
+# package.py always reruns autoreconf, regenerating a current, aarch64-aware
+# config.guess/config.sub) while spack is still available in this image, matching
+# the same version/MPI/link flags the vendor script would have used itself. The
+# vendor script's own TPL-reuse feature (REUSE_TPLS=yes by default) will find this
+# already built at the path it expects and skip building zoltan itself. Guarded to
+# aarch64 only: x86_64 is unaffected since its older bundled triplet files already
+# recognize x86_64.
+source /spack/share/spack/setup-env.sh && \
+spack install --reuse zoltan@3.901+mpi+fortran~shared %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} ^openmpi@4.1.4 && \
+mkdir -p /projects/e3sm/software/moab/tpls && \
+ln -s "$(spack location -i zoltan@3.901 %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} ^openmpi@4.1.4)" /projects/e3sm/software/moab/tpls/zoltan; \
+
 # Debugging libs
 RUN source /spack/share/spack/setup-env.sh && \
     spack install valgrind@3.20~mpi gdb@13.1 %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH}

diff --git a/ghci-snl/final/gnu-cuda-env-aarch64/Dockerfile b/ghci-snl/final/gnu-cuda-env-aarch64/Dockerfile
@@ -0,0 +1,88 @@
+ARG BASE_TAG=UNSET
+ARG BASE_IMAGE=ghcr.io/e3sm-project/e3sm-ghci-snl
+FROM ${BASE_IMAGE}:${BASE_TAG}
+
+# aarch64 equivalent of final/gnu-cuda-env, meant to run on NVIDIA aarch64
+# ("superchip") systems, e.g. Grace Hopper/Grace Blackwell.
+#
+# NOTE: spack@0.23.1 (pinned in base/Dockerfile) only packages CUDA up to 12.6,
+# so instead of `spack install cuda@...`, the toolkit below is installed straight
+# from NVIDIA's official RHEL9/SBSA repository, which already offers CUDA 13.
+
+# Install packages in chunks, so we can tweak versions of some pkg without
+# rebuilding the whole image. Hence, put "stable" pkgs first to minimize rebuild time
+# NOTE: the env vars COMPILER, COMPILER_VERSION, and SPACK_ARCH are defined in the base image
+
+# Common utilities
+RUN source /spack/share/spack/setup-env.sh && \
+    spack install yaml-cpp cmake %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH}
+
+# Common scientific libs (non-MPI dependent)
+RUN source /spack/share/spack/setup-env.sh && \
+    spack install boost netlib-lapack %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH}
+
+# CUDA (NVIDIA's RHEL9/SBSA repo, since spack@0.23.1 tops out at CUDA 12.6)
+# NOTE: install the compiler+libraries components only, NOT the "cuda-toolkit"
+# meta-package: the latter pulls in cuda-tools -> cuda-visual-tools ->
+# cuda-nsight-systems, which requires desktop/GUI libs (libOpenGL, libxcb-*,
+# libxkbcommon-x11) that aren't available (and aren't wanted) in this minimal,
+# headless container.
+RUN curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/rhel9/sbsa/cuda-rhel9.repo \
+        -o /etc/yum.repos.d/cuda-rhel9.repo && \
+    rpm --import "$(sed -n 's/^gpgkey=//p' /etc/yum.repos.d/cuda-rhel9.repo | head -1)" && \
+    microdnf install -y cuda-compiler-13-0 cuda-libraries-13-0 cuda-libraries-devel-13-0 && \
+    microdnf clean all && \
+    rm -rf /var/cache/dnf
+
+# MPI
+RUN source /spack/share/spack/setup-env.sh && \
+    spack install mpich@4.1.1 %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH}
+
+# Netcdf libraries
+RUN source /spack/share/spack/setup-env.sh && \
+    spack install netcdf-c+mpi netcdf-fortran parallel-netcdf %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH}
+
+# Make sure the modules for the installed packages are all available
+RUN source /spack/share/spack/setup-env.sh && \
+    spack module lmod refresh -y --delete-tree
+
+# Generate the profile.d script to load them all at bash startup
+RUN echo "module load yaml-cpp cmake" >> /etc/profile.d/eamxx-env.sh
+RUN echo "module load boost netlib-lapack" >> /etc/profile.d/eamxx-env.sh
+# CUDA isn't a spack/lmod module here (see note above), so export its env by hand
+RUN echo "export CUDA_HOME=/usr/local/cuda" >> /etc/profile.d/eamxx-env.sh && \
+    echo "export PATH=\$CUDA_HOME/bin:\$PATH" >> /etc/profile.d/eamxx-env.sh && \
+    echo "export LD_LIBRARY_PATH=\$CUDA_HOME/lib64\${LD_LIBRARY_PATH:+:\$LD_LIBRARY_PATH}" >> /etc/profile.d/eamxx-env.sh
+RUN echo "module load mpich" >> /etc/profile.d/eamxx-env.sh
+RUN echo "module load netcdf-c netcdf-fortran parallel-netcdf" >> /etc/profile.d/eamxx-env.sh
+
+# Install numpy/cupy. Unlike x86_64, we use the prebuilt cupy-cuda13x wheel (built
+# and shipped by the cupy team specifically for CUDA 13) instead of building the
+# plain "cupy" sdist from source, so there's no need for the stub-linking trick.
+RUN source /etc/profile.d/modules.sh        &&  \
+    source /etc/profile.d/eamxx-env.sh &&  \
+    python3 -m pip install numpy cupy-cuda13x
+
+# Install pytorch
+RUN source /etc/profile.d/modules.sh        &&  \
+    source /etc/profile.d/eamxx-env.sh &&  \
+    python3 -m pip install                      \
+            --extra-index-url https://download.pytorch.org/whl/cu130 \
+            torch==2.13.0+cu130
+
+# Whomever will run the container will be able to pip install pkgs in the venv
+# After all, the container is ephemeral/disposable, so there's no persistent state
+RUN chmod -R a+rwX /projects/e3sm/software/eamxx-venv
+
+# Clean up garbage packages that were only needed to BUILD other pkgs (not to USE them)
+RUN source /spack/share/spack/setup-env.sh && spack gc -y
+
+# Remove spack repo (except .git, so you can restore it) and spack buildcache (to keep image size under control)
+RUN rm -rf /spack-buildcache && \
+    rm -rf /spack/*
+
+# CIME tests need a bunch of perl
+RUN microdnf install -y perl
+
+ENTRYPOINT []
+CMD [ "/bin/bash", "--login" ]