diff --git a/.github/workflows/ghci-snl.yaml b/.github/workflows/ghci-snl.yaml index cda4651..dc7bbd3 100644 --- a/.github/workflows/ghci-snl.yaml +++ b/.github/workflows/ghci-snl.yaml @@ -25,11 +25,24 @@ concurrency: jobs: base: - runs-on: ubuntu-latest + runs-on: ${{ matrix.runs_on }} permissions: contents: read packages: write id-token: write + strategy: + fail-fast: false + matrix: + include: + - arch: x86_64 + runs_on: ubuntu-latest + spack_arch: linux-rhel9-x86_64 + tag_suffix: '-x86_64' + - arch: aarch64 + runs_on: ubuntu-22.04-arm + spack_arch: linux-rhel9-aarch64 + tag_suffix: '-aarch64' + name: base-${{ matrix.arch }} steps: - name: Checkout @@ -55,12 +68,12 @@ jobs: with: images: ghcr.io/${{ github.repository_owner }}/e3sm-ghci-snl tags: | - type=raw,value=base,enable=${{ github.ref == 'refs/heads/main' }} - type=ref,event=pr,prefix=base-pr- - type=ref,event=branch,prefix=base- - type=match,pattern=ghci-snl-(.*),group=1,prefix=base- + type=raw,value=base${{ matrix.tag_suffix }},enable=${{ github.ref == 'refs/heads/main' }} + type=ref,event=pr,prefix=base${{ matrix.tag_suffix }}-pr- + type=ref,event=branch,prefix=base${{ matrix.tag_suffix }}- + type=match,pattern=ghci-snl-(.*),group=1,prefix=base${{ matrix.tag_suffix }}- labels: | - org.opencontainers.image.description=Base image for E3SM ghci-snl containers used for testing on GitHub actions + org.opencontainers.image.description=Base image (${{ matrix.arch }}) for E3SM ghci-snl containers used for testing on GitHub actions - name: Push uses: docker/build-push-action@v6 @@ -70,11 +83,13 @@ jobs: push: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max + build-args: | + SPACK_ARCH=${{ matrix.spack_arch }} + cache-from: type=gha,scope=base-${{ matrix.arch }} + cache-to: type=gha,scope=base-${{ matrix.arch }},mode=max compiler: - runs-on: ubuntu-latest + runs-on: ${{ matrix.runs_on }} needs: base permissions: contents: read @@ -87,10 +102,22 @@ jobs: - compiler: gnu spack_compiler: gcc version: 13.3.0 + arch: x86_64 + runs_on: ubuntu-latest + tag_suffix: '' - compiler: intel spack_compiler: oneapi version: 2024.1.0 - name: ${{ matrix.compiler }} + arch: x86_64 + runs_on: ubuntu-latest + tag_suffix: '' + - compiler: gnu + spack_compiler: gcc + version: 13.3.0 + arch: aarch64 + runs_on: ubuntu-22.04-arm + tag_suffix: '-aarch64' + name: ${{ matrix.compiler }}-${{ matrix.arch }} steps: - name: Checkout @@ -116,12 +143,12 @@ jobs: with: images: ghcr.io/${{ github.repository_owner }}/e3sm-ghci-snl tags: | - type=raw,value=${{ matrix.compiler }}-${{ matrix.version }},enable=${{ github.ref == 'refs/heads/main' }} - type=ref,event=pr,prefix=${{ matrix.compiler }}-${{ matrix.version }}-pr- - type=ref,event=branch,prefix=${{ matrix.compiler }}-${{ matrix.version }}- - type=match,pattern=ghci-snl-(.*),group=1,prefix=${{ matrix.compiler }}-${{ matrix.version }}- + type=raw,value=${{ matrix.compiler }}-${{ matrix.version }}${{ matrix.tag_suffix }},enable=${{ github.ref == 'refs/heads/main' }} + type=ref,event=pr,prefix=${{ matrix.compiler }}-${{ matrix.version }}${{ matrix.tag_suffix }}-pr- + type=ref,event=branch,prefix=${{ matrix.compiler }}-${{ matrix.version }}${{ matrix.tag_suffix }}- + type=match,pattern=ghci-snl-(.*),group=1,prefix=${{ matrix.compiler }}-${{ matrix.version }}${{ matrix.tag_suffix }}- labels: | - org.opencontainers.image.description=${{ matrix.compiler }} compiler base image for E3SM ghci-snl images used for testing on GitHub actions + org.opencontainers.image.description=${{ matrix.compiler }} compiler base image (${{ matrix.arch }}) for E3SM ghci-snl images used for testing on GitHub actions - name: Push uses: docker/build-push-action@v6 @@ -134,12 +161,12 @@ jobs: build-args: | COMPILER=${{ matrix.spack_compiler }} COMPILER_VERSION=${{ matrix.version }} - BASE_TAG=${{ github.event_name == 'pull_request' && format('base-pr-{0}', github.event.pull_request.number) || 'base' }} - cache-from: type=gha,scope=${{ matrix.compiler }} - cache-to: type=gha,scope=${{ matrix.compiler }},mode=max + BASE_TAG=${{ github.event_name == 'pull_request' && format('base{0}-pr-{1}', matrix.tag_suffix, github.event.pull_request.number) || format('base{0}', matrix.tag_suffix) }} + cache-from: type=gha,scope=${{ matrix.compiler }}-${{ matrix.arch }} + cache-to: type=gha,scope=${{ matrix.compiler }}-${{ matrix.arch }},mode=max final-env: - runs-on: ubuntu-latest + runs-on: ${{ matrix.runs_on }} needs: compiler permissions: contents: read @@ -150,14 +177,41 @@ jobs: matrix: include: - compiler: gnu + dir: gnu-cpu-env tag: gnu-cpu-env version: 13.3.0 + arch: x86_64 + runs_on: ubuntu-latest + tag_suffix: '' - compiler: gnu + dir: gnu-cuda-env tag: gnu-cuda-env version: 13.3.0 + arch: x86_64 + runs_on: ubuntu-latest + tag_suffix: '' - compiler: intel + dir: intel-cpu-env tag: intel-cpu-env version: 2024.1.0 + arch: x86_64 + runs_on: ubuntu-latest + tag_suffix: '' + - compiler: gnu + dir: gnu-cpu-env + tag: gnu-cpu-env-aarch64 + version: 13.3.0 + arch: aarch64 + runs_on: ubuntu-22.04-arm + tag_suffix: '-aarch64' + - compiler: gnu + dir: gnu-cuda-env-aarch64 + tag: gnu-cuda-env-aarch64 + version: 13.3.0 + arch: aarch64 + runs_on: ubuntu-22.04-arm + tag_suffix: '-aarch64' + name: ${{ matrix.tag }} steps: - name: Checkout @@ -188,17 +242,17 @@ jobs: type=ref,event=branch,prefix=${{ matrix.tag }}- type=match,pattern=ghci-snl-(.*),group=1,prefix=${{ matrix.tag }}- labels: | - org.opencontainers.image.description=${{ matrix.tag }} image for E3SM ghci-snl testing on CPU on GitHub actions + org.opencontainers.image.description=${{ matrix.tag }} image (${{ matrix.arch }}) for E3SM ghci-snl testing on GitHub actions - name: Push uses: docker/build-push-action@v6 with: - context: ghci-snl/final/${{ matrix.tag }} - file: ghci-snl/final/${{ matrix.tag }}/Dockerfile + context: ghci-snl/final/${{ matrix.dir }} + file: ghci-snl/final/${{ matrix.dir }}/Dockerfile push: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BASE_TAG=${{ matrix.compiler }}-${{ matrix.version }}${{ github.event_name == 'pull_request' && format('-pr-{0}', github.event.pull_request.number) || '' }} + BASE_TAG=${{ matrix.compiler }}-${{ matrix.version }}${{ matrix.tag_suffix }}${{ github.event_name == 'pull_request' && format('-pr-{0}', github.event.pull_request.number) || '' }} cache-from: type=gha,scope=${{ matrix.tag }} cache-to: type=gha,scope=${{ matrix.tag }},mode=max diff --git a/ghci-snl/README b/ghci-snl/README index f139cc6..9e99971 100644 --- a/ghci-snl/README +++ b/ghci-snl/README @@ -28,6 +28,25 @@ podman run --rm -it \ ghcr.io/E3SM-Project/e3sm-ghci-snl:gnu-cpu-env /bin/bash ``` +## Available images + +Each image is published for `x86_64`; the GNU-based ones are also published for +`aarch64` (native builds, no emulation), under an `-aarch64`-suffixed tag: + +| Tag | Arch | Compiler | Notes | +|--------------------------|-----------|----------|----------------------------------------------------| +| `gnu-cpu-env` | x86_64 | GNU | CPU-only stack | +| `gnu-cpu-env-aarch64` | aarch64 | GNU | CPU-only stack; runs well on macOS M1/Apple Silicon | +| `gnu-cuda-env` | x86_64 | GNU | CUDA 12.4 stack | +| `gnu-cuda-env-aarch64` | aarch64 | GNU | CUDA 13.0 stack; targets NVIDIA aarch64 ("superchip") systems such as Grace Hopper/Blackwell | +| `intel-cpu-env` | x86_64 | Intel | CPU-only stack (Intel oneAPI is x86_64-only) | + +To pull an aarch64 image explicitly (e.g. on an Apple Silicon Mac via Podman machine, or on an NVIDIA Grace-based host): + +```bash +podman pull ghcr.io/E3SM-Project/e3sm-ghci-snl:gnu-cpu-env-aarch64 +``` + ## Important Storage & Permission Guidelines - **Input data** (`inputdata`): Required for downloading and reading datasets. @@ -56,6 +75,16 @@ pip install If you need to test with a different compiler or library version (for example, upgrading from GCC 13.3.0 to GCC 14), you can rebuild the container stack locally using custom build arguments: +- **(aarch64 only) Build the base image for your target arch**: + The `x86_64` base image is built by default; for `aarch64` targets, override `SPACK_ARCH`: + +```bash +podman build \\ + --build-arg SPACK_ARCH=linux-rhel9-aarch64 \\ + --tag e3sm-base:aarch64 \\ + ghci-snl/base/ +``` + - **Build the custom compiler container**: Navigate to the repository root and build the compiler image specifying your target version: diff --git a/ghci-snl/base/Dockerfile b/ghci-snl/base/Dockerfile index 61c3e24..f5a3aaa 100644 --- a/ghci-snl/base/Dockerfile +++ b/ghci-snl/base/Dockerfile @@ -75,8 +75,10 @@ ENV LANG=en_US.UTF-8 \ # 7. Setup SPACK ARG SPACK_VER=v0.23.1 +# Override at build time (e.g. --build-arg SPACK_ARCH=linux-rhel9-aarch64) for aarch64 images +ARG SPACK_ARCH=linux-rhel9-x86_64 ENV SPACK_ROOT=/spack \ - SPACK_ARCH=linux-rhel9-x86_64 \ + SPACK_ARCH=${SPACK_ARCH} \ SPACK_SYSTEM_CONFIG_PATH=/spack-configuration \ PATH=/spack/bin:$PATH diff --git a/ghci-snl/final/gnu-cpu-env/Dockerfile b/ghci-snl/final/gnu-cpu-env/Dockerfile index 9868e72..4682604 100644 --- a/ghci-snl/final/gnu-cpu-env/Dockerfile +++ b/ghci-snl/final/gnu-cpu-env/Dockerfile @@ -21,6 +21,24 @@ RUN source /spack/share/spack/setup-env.sh && \ RUN source /spack/share/spack/setup-env.sh && \ spack install hdf5 netcdf-c+mpi netcdf-fortran parallel-netcdf %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} +# MOAB's "zoltan" TPL, as fetched by the vendor install-bootstrap.sh script below from +# its default (circa-2013) dist tarball mirror, ships a bundled config.guess/ +# config.sub that predates the aarch64 GNU triplet, so its ./configure fails +# immediately on aarch64 -- and the vendor script has no remediation pattern that +# recognizes that failure, so it just aborts. We can't patch that third-party +# script, so work around it here instead: pre-build zoltan via spack (whose +# package.py always reruns autoreconf, regenerating a current, aarch64-aware +# config.guess/config.sub) while spack is still available in this image, matching +# the same version/MPI/link flags the vendor script would have used itself. The +# vendor script's own TPL-reuse feature (REUSE_TPLS=yes by default) will find this +# already built at the path it expects and skip building zoltan itself. Guarded to +# aarch64 only: x86_64 is unaffected since its older bundled triplet files already +# recognize x86_64. +RUN source /spack/share/spack/setup-env.sh && \ +spack install --reuse zoltan@3.901+mpi+fortran~shared %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} ^openmpi@4.1.4 && \ +mkdir -p /projects/e3sm/software/moab/tpls && \ +ln -s "$(spack location -i zoltan@3.901 %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} ^openmpi@4.1.4)" /projects/e3sm/software/moab/tpls/zoltan + # Debugging libs RUN source /spack/share/spack/setup-env.sh && \ spack install valgrind@3.20~mpi gdb@13.1 %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} diff --git a/ghci-snl/final/gnu-cuda-env-aarch64/Dockerfile b/ghci-snl/final/gnu-cuda-env-aarch64/Dockerfile new file mode 100644 index 0000000..f06dc13 --- /dev/null +++ b/ghci-snl/final/gnu-cuda-env-aarch64/Dockerfile @@ -0,0 +1,88 @@ +ARG BASE_TAG=UNSET +ARG BASE_IMAGE=ghcr.io/e3sm-project/e3sm-ghci-snl +FROM ${BASE_IMAGE}:${BASE_TAG} + +# aarch64 equivalent of final/gnu-cuda-env, meant to run on NVIDIA aarch64 +# ("superchip") systems, e.g. Grace Hopper/Grace Blackwell. +# +# NOTE: spack@0.23.1 (pinned in base/Dockerfile) only packages CUDA up to 12.6, +# so instead of `spack install cuda@...`, the toolkit below is installed straight +# from NVIDIA's official RHEL9/SBSA repository, which already offers CUDA 13. + +# Install packages in chunks, so we can tweak versions of some pkg without +# rebuilding the whole image. Hence, put "stable" pkgs first to minimize rebuild time +# NOTE: the env vars COMPILER, COMPILER_VERSION, and SPACK_ARCH are defined in the base image + +# Common utilities +RUN source /spack/share/spack/setup-env.sh && \ + spack install yaml-cpp cmake %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} + +# Common scientific libs (non-MPI dependent) +RUN source /spack/share/spack/setup-env.sh && \ + spack install boost netlib-lapack %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} + +# CUDA (NVIDIA's RHEL9/SBSA repo, since spack@0.23.1 tops out at CUDA 12.6) +# NOTE: install the compiler+libraries components only, NOT the "cuda-toolkit" +# meta-package: the latter pulls in cuda-tools -> cuda-visual-tools -> +# cuda-nsight-systems, which requires desktop/GUI libs (libOpenGL, libxcb-*, +# libxkbcommon-x11) that aren't available (and aren't wanted) in this minimal, +# headless container. +RUN curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/rhel9/sbsa/cuda-rhel9.repo \ + -o /etc/yum.repos.d/cuda-rhel9.repo && \ + rpm --import "$(sed -n 's/^gpgkey=//p' /etc/yum.repos.d/cuda-rhel9.repo | head -1)" && \ + microdnf install -y cuda-compiler-13-0 cuda-libraries-13-0 cuda-libraries-devel-13-0 && \ + microdnf clean all && \ + rm -rf /var/cache/dnf + +# MPI +RUN source /spack/share/spack/setup-env.sh && \ + spack install mpich@4.1.1 %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} + +# Netcdf libraries +RUN source /spack/share/spack/setup-env.sh && \ + spack install netcdf-c+mpi netcdf-fortran parallel-netcdf %${COMPILER}@${COMPILER_VERSION} arch=${SPACK_ARCH} + +# Make sure the modules for the installed packages are all available +RUN source /spack/share/spack/setup-env.sh && \ + spack module lmod refresh -y --delete-tree + +# Generate the profile.d script to load them all at bash startup +RUN echo "module load yaml-cpp cmake" >> /etc/profile.d/eamxx-env.sh +RUN echo "module load boost netlib-lapack" >> /etc/profile.d/eamxx-env.sh +# CUDA isn't a spack/lmod module here (see note above), so export its env by hand +RUN echo "export CUDA_HOME=/usr/local/cuda" >> /etc/profile.d/eamxx-env.sh && \ + echo "export PATH=\$CUDA_HOME/bin:\$PATH" >> /etc/profile.d/eamxx-env.sh && \ + echo "export LD_LIBRARY_PATH=\$CUDA_HOME/lib64\${LD_LIBRARY_PATH:+:\$LD_LIBRARY_PATH}" >> /etc/profile.d/eamxx-env.sh +RUN echo "module load mpich" >> /etc/profile.d/eamxx-env.sh +RUN echo "module load netcdf-c netcdf-fortran parallel-netcdf" >> /etc/profile.d/eamxx-env.sh + +# Install numpy/cupy. Unlike x86_64, we use the prebuilt cupy-cuda13x wheel (built +# and shipped by the cupy team specifically for CUDA 13) instead of building the +# plain "cupy" sdist from source, so there's no need for the stub-linking trick. +RUN source /etc/profile.d/modules.sh && \ + source /etc/profile.d/eamxx-env.sh && \ + python3 -m pip install numpy cupy-cuda13x + +# Install pytorch +RUN source /etc/profile.d/modules.sh && \ + source /etc/profile.d/eamxx-env.sh && \ + python3 -m pip install \ + --extra-index-url https://download.pytorch.org/whl/cu130 \ + torch==2.13.0+cu130 + +# Whomever will run the container will be able to pip install pkgs in the venv +# After all, the container is ephemeral/disposable, so there's no persistent state +RUN chmod -R a+rwX /projects/e3sm/software/eamxx-venv + +# Clean up garbage packages that were only needed to BUILD other pkgs (not to USE them) +RUN source /spack/share/spack/setup-env.sh && spack gc -y + +# Remove spack repo (except .git, so you can restore it) and spack buildcache (to keep image size under control) +RUN rm -rf /spack-buildcache && \ + rm -rf /spack/* + +# CIME tests need a bunch of perl +RUN microdnf install -y perl + +ENTRYPOINT [] +CMD [ "/bin/bash", "--login" ]