From 78820d227c92044a2867f40ab80ecb0104ef2313 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 30 Aug 2026 01:13:52 +0000 Subject: [PATCH 1/6] Add aarch64 variants for ghci-snl gnu-cpu-env and gnu-cuda-env Co-authored-by: mahf708 <122953255+mahf708@users.noreply.github.com> --- .github/workflows/ghci-snl.yaml | 100 ++++++++++++++---- ghci-snl/README | 29 +++++ ghci-snl/base/Dockerfile | 4 +- .../final/gnu-cuda-env-aarch64/Dockerfile | 83 +++++++++++++++ 4 files changed, 192 insertions(+), 24 deletions(-) create mode 100644 ghci-snl/final/gnu-cuda-env-aarch64/Dockerfile diff --git a/.github/workflows/ghci-snl.yaml b/.github/workflows/ghci-snl.yaml index cda4651..0519175 100644 --- a/.github/workflows/ghci-snl.yaml +++ b/.github/workflows/ghci-snl.yaml @@ -25,11 +25,24 @@ concurrency: jobs: base: - runs-on: ubuntu-latest + runs-on: ${{ matrix.runs_on }} permissions: contents: read packages: write id-token: write + strategy: + fail-fast: false + matrix: + include: + - arch: x86_64 + runs_on: ubuntu-latest + spack_arch: linux-rhel9-x86_64 + tag_suffix: '' + - arch: aarch64 + runs_on: ubuntu-22.04-arm + spack_arch: linux-rhel9-aarch64 + tag_suffix: '-aarch64' + name: base-${{ matrix.arch }} steps: - name: Checkout @@ -55,12 +68,12 @@ jobs: with: images: ghcr.io/${{ github.repository_owner }}/e3sm-ghci-snl tags: | - type=raw,value=base,enable=${{ github.ref == 'refs/heads/main' }} - type=ref,event=pr,prefix=base-pr- - type=ref,event=branch,prefix=base- - type=match,pattern=ghci-snl-(.*),group=1,prefix=base- + type=raw,value=base${{ matrix.tag_suffix }},enable=${{ github.ref == 'refs/heads/main' }} + type=ref,event=pr,prefix=base${{ matrix.tag_suffix }}-pr- + type=ref,event=branch,prefix=base${{ matrix.tag_suffix }}- + type=match,pattern=ghci-snl-(.*),group=1,prefix=base${{ matrix.tag_suffix }}- labels: | - org.opencontainers.image.description=Base image for E3SM ghci-snl containers used for testing on GitHub actions + org.opencontainers.image.description=Base image (${{ matrix.arch }}) for E3SM ghci-snl containers used for testing on GitHub actions - name: Push uses: docker/build-push-action@v6 @@ -70,11 +83,13 @@ jobs: push: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max + build-args: | + SPACK_ARCH=${{ matrix.spack_arch }} + cache-from: type=gha,scope=base-${{ matrix.arch }} + cache-to: type=gha,scope=base-${{ matrix.arch }},mode=max compiler: - runs-on: ubuntu-latest + runs-on: ${{ matrix.runs_on }} needs: base permissions: contents: read @@ -87,10 +102,22 @@ jobs: - compiler: gnu spack_compiler: gcc version: 13.3.0 + arch: x86_64 + runs_on: ubuntu-latest + tag_suffix: '' - compiler: intel spack_compiler: oneapi version: 2024.1.0 - name: ${{ matrix.compiler }} + arch: x86_64 + runs_on: ubuntu-latest + tag_suffix: '' + - compiler: gnu + spack_compiler: gcc + version: 13.3.0 + arch: aarch64 + runs_on: ubuntu-22.04-arm + tag_suffix: '-aarch64' + name: ${{ matrix.compiler }}-${{ matrix.arch }} steps: - name: Checkout @@ -116,12 +143,12 @@ jobs: with: images: ghcr.io/${{ github.repository_owner }}/e3sm-ghci-snl tags: | - type=raw,value=${{ matrix.compiler }}-${{ matrix.version }},enable=${{ github.ref == 'refs/heads/main' }} - type=ref,event=pr,prefix=${{ matrix.compiler }}-${{ matrix.version }}-pr- - type=ref,event=branch,prefix=${{ matrix.compiler }}-${{ matrix.version }}- - type=match,pattern=ghci-snl-(.*),group=1,prefix=${{ matrix.compiler }}-${{ matrix.version }}- + type=raw,value=${{ matrix.compiler }}-${{ matrix.version }}${{ matrix.tag_suffix }},enable=${{ github.ref == 'refs/heads/main' }} + type=ref,event=pr,prefix=${{ matrix.compiler }}-${{ matrix.version }}${{ matrix.tag_suffix }}-pr- + type=ref,event=branch,prefix=${{ matrix.compiler }}-${{ matrix.version }}${{ matrix.tag_suffix }}- + type=match,pattern=ghci-snl-(.*),group=1,prefix=${{ matrix.compiler }}-${{ matrix.version }}${{ matrix.tag_suffix }}- labels: | - org.opencontainers.image.description=${{ matrix.compiler }} compiler base image for E3SM ghci-snl images used for testing on GitHub actions + org.opencontainers.image.description=${{ matrix.compiler }} compiler base image (${{ matrix.arch }}) for E3SM ghci-snl images used for testing on GitHub actions - name: Push uses: docker/build-push-action@v6 @@ -134,12 +161,12 @@ jobs: build-args: | COMPILER=${{ matrix.spack_compiler }} COMPILER_VERSION=${{ matrix.version }} - BASE_TAG=${{ github.event_name == 'pull_request' && format('base-pr-{0}', github.event.pull_request.number) || 'base' }} - cache-from: type=gha,scope=${{ matrix.compiler }} - cache-to: type=gha,scope=${{ matrix.compiler }},mode=max + BASE_TAG=${{ github.event_name == 'pull_request' && format('base{0}-pr-{1}', matrix.tag_suffix, github.event.pull_request.number) || format('base{0}', matrix.tag_suffix) }} + cache-from: type=gha,scope=${{ matrix.compiler }}-${{ matrix.arch }} + cache-to: type=gha,scope=${{ matrix.compiler }}-${{ matrix.arch }},mode=max final-env: - runs-on: ubuntu-latest + runs-on: ${{ matrix.runs_on }} needs: compiler permissions: contents: read @@ -150,14 +177,41 @@ jobs: matrix: include: - compiler: gnu + dir: gnu-cpu-env tag: gnu-cpu-env version: 13.3.0 + arch: x86_64 + runs_on: ubuntu-latest + tag_suffix: '' - compiler: gnu + dir: gnu-cuda-env tag: gnu-cuda-env version: 13.3.0 + arch: x86_64 + runs_on: ubuntu-latest + tag_suffix: '' - compiler: intel + dir: intel-cpu-env tag: intel-cpu-env version: 2024.1.0 + arch: x86_64 + runs_on: ubuntu-latest + tag_suffix: '' + - compiler: gnu + dir: gnu-cpu-env + tag: gnu-cpu-env-aarch64 + version: 13.3.0 + arch: aarch64 + runs_on: ubuntu-22.04-arm + tag_suffix: '-aarch64' + - compiler: gnu + dir: gnu-cuda-env-aarch64 + tag: gnu-cuda-env-aarch64 + version: 13.3.0 + arch: aarch64 + runs_on: ubuntu-22.04-arm + tag_suffix: '-aarch64' + name: ${{ matrix.tag }} steps: - name: Checkout @@ -188,17 +242,17 @@ jobs: type=ref,event=branch,prefix=${{ matrix.tag }}- type=match,pattern=ghci-snl-(.*),group=1,prefix=${{ matrix.tag }}- labels: | - org.opencontainers.image.description=${{ matrix.tag }} image for E3SM ghci-snl testing on CPU on GitHub actions + org.opencontainers.image.description=${{ matrix.tag }} image (${{ matrix.arch }}) for E3SM ghci-snl testing on GitHub actions - name: Push uses: docker/build-push-action@v6 with: - context: ghci-snl/final/${{ matrix.tag }} - file: ghci-snl/final/${{ matrix.tag }}/Dockerfile + context: ghci-snl/final/${{ matrix.dir }} + file: ghci-snl/final/${{ matrix.dir }}/Dockerfile push: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BASE_TAG=${{ matrix.compiler }}-${{ matrix.version }}${{ github.event_name == 'pull_request' && format('-pr-{0}', github.event.pull_request.number) || '' }} + BASE_TAG=${{ matrix.compiler }}-${{ matrix.version }}${{ matrix.tag_suffix }}${{ github.event_name == 'pull_request' && format('-pr-{0}', github.event.pull_request.number) || '' }} cache-from: type=gha,scope=${{ matrix.tag }} cache-to: type=gha,scope=${{ matrix.tag }},mode=max diff --git a/ghci-snl/README b/ghci-snl/README index f139cc6..9e99971 100644 --- a/ghci-snl/README +++ b/ghci-snl/README @@ -28,6 +28,25 @@ podman run --rm -it \ ghcr.io/E3SM-Project/e3sm-ghci-snl:gnu-cpu-env /bin/bash ``` +## Available images + +Each image is published for `x86_64`; the GNU-based ones are also published for +`aarch64` (native builds, no emulation), under an `-aarch64`-suffixed tag: + +| Tag | Arch | Compiler | Notes | +|--------------------------|-----------|----------|----------------------------------------------------| +| `gnu-cpu-env` | x86_64 | GNU | CPU-only stack | +| `gnu-cpu-env-aarch64` | aarch64 | GNU | CPU-only stack; runs well on macOS M1/Apple Silicon | +| `gnu-cuda-env` | x86_64 | GNU | CUDA 12.4 stack | +| `gnu-cuda-env-aarch64` | aarch64 | GNU | CUDA 13.0 stack; targets NVIDIA aarch64 ("superchip") systems such as Grace Hopper/Blackwell | +| `intel-cpu-env` | x86_64 | Intel | CPU-only stack (Intel oneAPI is x86_64-only) | + +To pull an aarch64 image explicitly (e.g. on an Apple Silicon Mac via Podman machine, or on an NVIDIA Grace-based host): + +```bash +podman pull ghcr.io/E3SM-Project/e3sm-ghci-snl:gnu-cpu-env-aarch64 +``` + ## Important Storage & Permission Guidelines - **Input data** (`inputdata`): Required for downloading and reading datasets. @@ -56,6 +75,16 @@ pip install If you need to test with a different compiler or library version (for example, upgrading from GCC 13.3.0 to GCC 14), you can rebuild the container stack locally using custom build arguments: +- **(aarch64 only) Build the base image for your target arch**: + The `x86_64` base image is built by default; for `aarch64` targets, override `SPACK_ARCH`: + +```bash +podman build \\ + --build-arg SPACK_ARCH=linux-rhel9-aarch64 \\ + --tag e3sm-base:aarch64 \\ + ghci-snl/base/ +``` + - **Build the custom compiler container**: Navigate to the repository root and build the compiler image specifying your target version: diff --git a/ghci-snl/base/Dockerfile b/ghci-snl/base/Dockerfile index 61c3e24..f5a3aaa 100644 --- a/ghci-snl/base/Dockerfile +++ b/ghci-snl/base/Dockerfile @@ -75,8 +75,10 @@ ENV LANG=en_US.UTF-8 \ # 7. Setup SPACK ARG SPACK_VER=v0.23.1 +# Override at build time (e.g. --build-arg SPACK_ARCH=linux-rhel9-aarch64) for aarch64 images +ARG SPACK_ARCH=linux-rhel9-x86_64 ENV SPACK_ROOT=/spack \ - SPACK_ARCH=linux-rhel9-x86_64 \ + SPACK_ARCH=${SPACK_ARCH} \ SPACK_SYSTEM_CONFIG_PATH=/spack-configuration \ PATH=/spack/bin:$PATH diff --git a/ghci-snl/final/gnu-cuda-env-aarch64/Dockerfile b/ghci-snl/final/gnu-cuda-env-aarch64/Dockerfile new file mode 100644 index 0000000..8b880c9 --- /dev/null +++ b/ghci-snl/final/gnu-cuda-env-aarch64/Dockerfile @@ -0,0 +1,83 @@ +ARG BASE_TAG=UNSET +ARG BASE_IMAGE=ghcr.io/e3sm-project/e3sm-ghci-snl +FROM ${BASE_IMAGE}:${BASE_TAG} + +# aarch64 equivalent of final/gnu-cuda-env, meant to run on NVIDIA aarch64 +# ("superchip") systems, e.g. Grace Hopper/Grace Blackwell. +# +# NOTE: spack@0.23.1 (pinned in base/Dockerfile) only packages CUDA up to 12.6, +# so instead of `spack install cuda@...`, the toolkit below is installed straight +# from NVIDIA's official RHEL9/SBSA repository, which already offers CUDA 13. + +# Install packages in chunks, so we can tweak versions of some pkg without +# rebuilding the whole image. Hence, put "stable" pkgs first to minimize rebuild time +# NOTE: the env vars COMPILER, COMPILER_VERSION, and SPACK_ARCH are defined in the base image + +# Common utilities +RUN source /spack/share/spack/setup-env.sh && \ + spack install yaml-cpp cmake %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} + +# Common scientific libs (non-MPI dependent) +RUN source /spack/share/spack/setup-env.sh && \ + spack install boost netlib-lapack %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} + +# CUDA (NVIDIA's RHEL9/SBSA repo, since spack@0.23.1 tops out at CUDA 12.6) +RUN curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/rhel9/sbsa/cuda-rhel9.repo \ + -o /etc/yum.repos.d/cuda-rhel9.repo && \ + rpm --import "$(sed -n 's/^gpgkey=//p' /etc/yum.repos.d/cuda-rhel9.repo | head -1)" && \ + microdnf install -y cuda-toolkit-13-0 && \ + microdnf clean all && \ + rm -rf /var/cache/dnf + +# MPI +RUN source /spack/share/spack/setup-env.sh && \ + spack install mpich@4.1.1 %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} + +# Netcdf libraries +RUN source /spack/share/spack/setup-env.sh && \ + spack install netcdf-c+mpi netcdf-fortran parallel-netcdf %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} + +# Make sure the modules for the installed packages are all available +RUN source /spack/share/spack/setup-env.sh && \ + spack module lmod refresh -y --delete-tree + +# Generate the profile.d script to load them all at bash startup +RUN echo "module load yaml-cpp cmake" >> /etc/profile.d/eamxx-env.sh +RUN echo "module load boost netlib-lapack" >> /etc/profile.d/eamxx-env.sh +# CUDA isn't a spack/lmod module here (see note above), so export its env by hand +RUN echo "export CUDA_HOME=/usr/local/cuda" >> /etc/profile.d/eamxx-env.sh && \ + echo "export PATH=\$CUDA_HOME/bin:\$PATH" >> /etc/profile.d/eamxx-env.sh && \ + echo "export LD_LIBRARY_PATH=\$CUDA_HOME/lib64:\$LD_LIBRARY_PATH" >> /etc/profile.d/eamxx-env.sh +RUN echo "module load mpich" >> /etc/profile.d/eamxx-env.sh +RUN echo "module load netcdf-c netcdf-fortran parallel-netcdf" >> /etc/profile.d/eamxx-env.sh + +# Install numpy/cupy. Unlike x86_64, we use the prebuilt cupy-cuda13x wheel (built +# and shipped by the cupy team specifically for CUDA 13) instead of building the +# plain "cupy" sdist from source, so there's no need for the stub-linking trick. +RUN source /etc/profile.d/modules.sh && \ + source /etc/profile.d/eamxx-env.sh && \ + python3 -m pip install numpy cupy-cuda13x + +# Install pytorch +RUN source /etc/profile.d/modules.sh && \ + source /etc/profile.d/eamxx-env.sh && \ + python3 -m pip install \ + --extra-index-url https://download.pytorch.org/whl/cu130 \ + torch==2.13.0+cu130 + +# Whomever will run the container will be able to pip install pkgs in the venv +# After all, the container is ephemeral/disposable, so there's no persistent state +RUN chmod -R a+rwX /projects/e3sm/software/eamxx-venv + +# Clean up garbage packages that were only needed to BUILD other pkgs (not to USE them) +RUN source /spack/share/spack/setup-env.sh && spack gc -y + +# Remove spack repo (except .git, so you can restore it) and spack buildcache (to keep image size under control) +RUN rm -rf /spack-buildcache && \ + rm -rf /spack/* + +# CIME tests need a bunch of perl +RUN microdnf install -y perl + +ENTRYPOINT [] +CMD [ "/bin/bash", "--login" ] From 018172cac394d66ec06fa2a61323d8efb9db5391 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 30 Aug 2026 04:11:16 +0000 Subject: [PATCH 2/6] Fix gnu-cuda-env-aarch64: avoid cuda-toolkit meta-package (pulls in GUI nsight-systems deps) Co-authored-by: mahf708 <122953255+mahf708@users.noreply.github.com> --- ghci-snl/final/gnu-cuda-env-aarch64/Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ghci-snl/final/gnu-cuda-env-aarch64/Dockerfile b/ghci-snl/final/gnu-cuda-env-aarch64/Dockerfile index 8b880c9..6e3815d 100644 --- a/ghci-snl/final/gnu-cuda-env-aarch64/Dockerfile +++ b/ghci-snl/final/gnu-cuda-env-aarch64/Dockerfile @@ -22,10 +22,15 @@ RUN source /spack/share/spack/setup-env.sh && \ spack install boost netlib-lapack %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} # CUDA (NVIDIA's RHEL9/SBSA repo, since spack@0.23.1 tops out at CUDA 12.6) +# NOTE: install the compiler+libraries components only, NOT the "cuda-toolkit" +# meta-package: the latter pulls in cuda-tools -> cuda-visual-tools -> +# cuda-nsight-systems, which requires desktop/GUI libs (libOpenGL, libxcb-*, +# libxkbcommon-x11) that aren't available (and aren't wanted) in this minimal, +# headless container. RUN curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/rhel9/sbsa/cuda-rhel9.repo \ -o /etc/yum.repos.d/cuda-rhel9.repo && \ rpm --import "$(sed -n 's/^gpgkey=//p' /etc/yum.repos.d/cuda-rhel9.repo | head -1)" && \ - microdnf install -y cuda-toolkit-13-0 && \ + microdnf install -y cuda-compiler-13-0 cuda-libraries-13-0 cuda-libraries-devel-13-0 && \ microdnf clean all && \ rm -rf /var/cache/dnf From 587512b7366522e38b2760af2770f4138b9089b4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 30 Aug 2026 04:41:18 +0000 Subject: [PATCH 3/6] Work around MOAB zoltan aarch64 build failure via spack pre-install Co-authored-by: mahf708 <122953255+mahf708@users.noreply.github.com> --- ghci-snl/final/gnu-cpu-env/Dockerfile | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ghci-snl/final/gnu-cpu-env/Dockerfile b/ghci-snl/final/gnu-cpu-env/Dockerfile index 9868e72..294527f 100644 --- a/ghci-snl/final/gnu-cpu-env/Dockerfile +++ b/ghci-snl/final/gnu-cpu-env/Dockerfile @@ -21,6 +21,26 @@ RUN source /spack/share/spack/setup-env.sh && \ RUN source /spack/share/spack/setup-env.sh && \ spack install hdf5 netcdf-c+mpi netcdf-fortran parallel-netcdf %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} +# MOAB's "zoltan" TPL, as fetched by the vendor install-bootstrap.sh script below from +# its default (circa-2013) dist tarball mirror, ships a bundled config.guess/ +# config.sub that predates the aarch64 GNU triplet, so its ./configure fails +# immediately on aarch64 -- and the vendor script has no remediation pattern that +# recognizes that failure, so it just aborts. We can't patch that third-party +# script, so work around it here instead: pre-build zoltan via spack (whose +# package.py always reruns autoreconf, regenerating a current, aarch64-aware +# config.guess/config.sub) while spack is still available in this image, matching +# the same version/MPI/link flags the vendor script would have used itself. The +# vendor script's own TPL-reuse feature (REUSE_TPLS=yes by default) will find this +# already built at the path it expects and skip building zoltan itself. Guarded to +# aarch64 only: x86_64 is unaffected since its older bundled triplet files already +# recognize x86_64. +RUN if [[ "$SPACK_ARCH" == *aarch64* ]]; then \ + source /spack/share/spack/setup-env.sh && \ + spack install --reuse zoltan@3.901+mpi+fortran~shared %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} ^openmpi@4.1.4 && \ + mkdir -p /projects/e3sm/software/moab/tpls && \ + ln -s "$(spack location -i zoltan@3.901 %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} ^openmpi@4.1.4)" /projects/e3sm/software/moab/tpls/zoltan; \ + fi + # Debugging libs RUN source /spack/share/spack/setup-env.sh && \ spack install valgrind@3.20~mpi gdb@13.1 %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} From 6f56013267bb88b0d420ed0a4762420616a9ea79 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Mon, 31 Aug 2026 17:40:43 -0400 Subject: [PATCH 4/6] response to feedback --- .github/workflows/ghci-snl.yaml | 2 +- ghci-snl/final/gnu-cpu-env/Dockerfile | 10 ++++------ ghci-snl/final/gnu-cuda-env-aarch64/Dockerfile | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ghci-snl.yaml b/.github/workflows/ghci-snl.yaml index 0519175..dc7bbd3 100644 --- a/.github/workflows/ghci-snl.yaml +++ b/.github/workflows/ghci-snl.yaml @@ -37,7 +37,7 @@ jobs: - arch: x86_64 runs_on: ubuntu-latest spack_arch: linux-rhel9-x86_64 - tag_suffix: '' + tag_suffix: '-x86_64' - arch: aarch64 runs_on: ubuntu-22.04-arm spack_arch: linux-rhel9-aarch64 diff --git a/ghci-snl/final/gnu-cpu-env/Dockerfile b/ghci-snl/final/gnu-cpu-env/Dockerfile index 294527f..d27d211 100644 --- a/ghci-snl/final/gnu-cpu-env/Dockerfile +++ b/ghci-snl/final/gnu-cpu-env/Dockerfile @@ -34,12 +34,10 @@ RUN source /spack/share/spack/setup-env.sh && \ # already built at the path it expects and skip building zoltan itself. Guarded to # aarch64 only: x86_64 is unaffected since its older bundled triplet files already # recognize x86_64. -RUN if [[ "$SPACK_ARCH" == *aarch64* ]]; then \ - source /spack/share/spack/setup-env.sh && \ - spack install --reuse zoltan@3.901+mpi+fortran~shared %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} ^openmpi@4.1.4 && \ - mkdir -p /projects/e3sm/software/moab/tpls && \ - ln -s "$(spack location -i zoltan@3.901 %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} ^openmpi@4.1.4)" /projects/e3sm/software/moab/tpls/zoltan; \ - fi +source /spack/share/spack/setup-env.sh && \ +spack install --reuse zoltan@3.901+mpi+fortran~shared %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} ^openmpi@4.1.4 && \ +mkdir -p /projects/e3sm/software/moab/tpls && \ +ln -s "$(spack location -i zoltan@3.901 %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} ^openmpi@4.1.4)" /projects/e3sm/software/moab/tpls/zoltan; \ # Debugging libs RUN source /spack/share/spack/setup-env.sh && \ diff --git a/ghci-snl/final/gnu-cuda-env-aarch64/Dockerfile b/ghci-snl/final/gnu-cuda-env-aarch64/Dockerfile index 6e3815d..f06dc13 100644 --- a/ghci-snl/final/gnu-cuda-env-aarch64/Dockerfile +++ b/ghci-snl/final/gnu-cuda-env-aarch64/Dockerfile @@ -52,7 +52,7 @@ RUN echo "module load boost netlib-lapack" >> /etc/profile.d/eamxx-env.sh # CUDA isn't a spack/lmod module here (see note above), so export its env by hand RUN echo "export CUDA_HOME=/usr/local/cuda" >> /etc/profile.d/eamxx-env.sh && \ echo "export PATH=\$CUDA_HOME/bin:\$PATH" >> /etc/profile.d/eamxx-env.sh && \ - echo "export LD_LIBRARY_PATH=\$CUDA_HOME/lib64:\$LD_LIBRARY_PATH" >> /etc/profile.d/eamxx-env.sh + echo "export LD_LIBRARY_PATH=\$CUDA_HOME/lib64\${LD_LIBRARY_PATH:+:\$LD_LIBRARY_PATH}" >> /etc/profile.d/eamxx-env.sh RUN echo "module load mpich" >> /etc/profile.d/eamxx-env.sh RUN echo "module load netcdf-c netcdf-fortran parallel-netcdf" >> /etc/profile.d/eamxx-env.sh From 37ae97d4662cc40d53aa9d362e660accee2c1b90 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Mon, 31 Aug 2026 19:04:19 -0400 Subject: [PATCH 5/6] fix syntax --- ghci-snl/final/gnu-cpu-env/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ghci-snl/final/gnu-cpu-env/Dockerfile b/ghci-snl/final/gnu-cpu-env/Dockerfile index d27d211..b1c6e49 100644 --- a/ghci-snl/final/gnu-cpu-env/Dockerfile +++ b/ghci-snl/final/gnu-cpu-env/Dockerfile @@ -37,7 +37,7 @@ RUN source /spack/share/spack/setup-env.sh && \ source /spack/share/spack/setup-env.sh && \ spack install --reuse zoltan@3.901+mpi+fortran~shared %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} ^openmpi@4.1.4 && \ mkdir -p /projects/e3sm/software/moab/tpls && \ -ln -s "$(spack location -i zoltan@3.901 %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} ^openmpi@4.1.4)" /projects/e3sm/software/moab/tpls/zoltan; \ +ln -s "$(spack location -i zoltan@3.901 %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} ^openmpi@4.1.4)" /projects/e3sm/software/moab/tpls/zoltan # Debugging libs RUN source /spack/share/spack/setup-env.sh && \ From 2b93a3cd53e5202f576dd5c190ed31af8d37da3e Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Mon, 31 Aug 2026 19:10:52 -0400 Subject: [PATCH 6/6] syntax again --- ghci-snl/final/gnu-cpu-env/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ghci-snl/final/gnu-cpu-env/Dockerfile b/ghci-snl/final/gnu-cpu-env/Dockerfile index b1c6e49..4682604 100644 --- a/ghci-snl/final/gnu-cpu-env/Dockerfile +++ b/ghci-snl/final/gnu-cpu-env/Dockerfile @@ -34,7 +34,7 @@ RUN source /spack/share/spack/setup-env.sh && \ # already built at the path it expects and skip building zoltan itself. Guarded to # aarch64 only: x86_64 is unaffected since its older bundled triplet files already # recognize x86_64. -source /spack/share/spack/setup-env.sh && \ +RUN source /spack/share/spack/setup-env.sh && \ spack install --reuse zoltan@3.901+mpi+fortran~shared %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} ^openmpi@4.1.4 && \ mkdir -p /projects/e3sm/software/moab/tpls && \ ln -s "$(spack location -i zoltan@3.901 %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} ^openmpi@4.1.4)" /projects/e3sm/software/moab/tpls/zoltan