From f26cba017e0b6ee37de9644d0a32b4919545378c Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Wed, 26 Aug 2026 07:47:32 -0700 Subject: [PATCH] Set OMEGA_ARCH from the E3SM machine configuration * Derive GPU backend from `USE_CUDA`, `USE_HIP`, `USE_SYCL`, or threading settings * Forward `Kokkos_ARCH_*` and `Kokkos_ENABLE_*` options from `KOKKOS_OPTIONS` * Fail GPU builds when no Kokkos architecture is configured * Preserve EAMxx-provided `Kokkos::kokkos` when available * Fix `CASEROOT` handling for coupled and standalone builds * Handle bare and namespaced dependency targets safely * Use EKAT to build Kokkos in coupled builds without EAMxx * Add `BUILD_CONFIG_TEST` to verify `OMEGA_ARCH` matches the Kokkos execution space --- components/omega/OmegaBuild.cmake | 175 +++++++++++++----- components/omega/external/CMakeLists.txt | 60 +++++- components/omega/test/CMakeLists.txt | 11 ++ .../omega/test/infra/BuildConfigTest.cpp | 157 ++++++++++++++++ 4 files changed, 351 insertions(+), 52 deletions(-) create mode 100644 components/omega/test/infra/BuildConfigTest.cpp diff --git a/components/omega/OmegaBuild.cmake b/components/omega/OmegaBuild.cmake index 1474b3e898c9..972fbf6a403d 100644 --- a/components/omega/OmegaBuild.cmake +++ b/components/omega/OmegaBuild.cmake @@ -18,8 +18,6 @@ set(E3SM_CIME_ROOT "${E3SM_ROOT}/cime") set(E3SM_CIMECONFIG_ROOT "${E3SM_ROOT}/cime_config") set(E3SM_EXTERNALS_ROOT "${E3SM_ROOT}/externals") -set(CASEROOT "${OMEGA_BUILD_DIR}/e3smcase") - ########################### # Macros # ########################### @@ -164,6 +162,10 @@ endmacro() # and detect OMEGA_ARCH and compilers macro(init_standalone_build) + # A standalone build has no E3SM case to read machine settings from, so it + # creates a throwaway one (see read_cime_config) and points CASEROOT at it. + set(CASEROOT "${OMEGA_BUILD_DIR}/e3smcase") + # get cime configuration read_cime_config() @@ -342,31 +344,10 @@ macro(init_standalone_build) set(CMAKE_C_COMPILER ${OMEGA_C_COMPILER}) set(CMAKE_Fortran_COMPILER ${OMEGA_Fortran_COMPILER}) -# TODO: do we want to use these variables? -# # Set compiler and linker flags -# if (CXXFLAGS) -# separate_arguments(_CXXFLAGS NATIVE_COMMAND ${CXXFLAGS}) -# list(APPEND OMEGA_CXX_FLAGS ${_CXXFLAGS}) -# endif() -# -# if (LDFLAGS) -# separate_arguments(_LDFLAGS NATIVE_COMMAND ${LDFLAGS}) -# list(APPEND OMEGA_LINK_OPTIONS ${_LDFLAGS}) -# endif() -# -# if (SLIBS) -# separate_arguments(_SLIBS NATIVE_COMMAND ${SLIBS}) -# list(APPEND OMEGA_LINK_OPTIONS ${_SLIBS}) -# endif() - if(OMEGA_CXX_FLAGS) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OMEGA_CXX_FLAGS}") endif() -# if(OMEGA_EXE_LINKER_FLAGS) -# set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OMEGA_EXE_LINKER_FLAGS}") -# endif() - # set CXX compiler *before* calling CMake project() if("${OMEGA_ARCH}" STREQUAL "CUDA") @@ -520,6 +501,19 @@ macro(setup_standalone_build) endmacro() +# Recover per-machine settings from the case's Macros.cmake that E3SM does not +# make visible to Omega. +function(omega_read_e3sm_macros) + + include("${CASEROOT}/Macros.cmake") + + set(KOKKOS_OPTIONS "${KOKKOS_OPTIONS}" PARENT_SCOPE) + + # USE_SYCL must come from here too. + set(USE_SYCL "${USE_SYCL}" PARENT_SCOPE) + +endfunction() + # set build-control-variables for e3sm build macro(setup_e3sm_build) @@ -527,11 +521,40 @@ macro(setup_e3sm_build) set(OMEGA_CXX_COMPILER ${CMAKE_CXX_COMPILER}) - #TODO: set OMEGA_ARCH according to E3SM variables - set(OMEGA_ARCH "") + # Recover the per-machine settings that E3SM does not propagate into this + # scope (KOKKOS_OPTIONS, USE_SYCL). This must run BEFORE the arch detection + # below, which reads USE_SYCL. CASEROOT here is the real case root CIME passed + # in with -DCASEROOT=. + if(CASEROOT AND EXISTS "${CASEROOT}/Macros.cmake") + omega_read_e3sm_macros() + endif() + + # Detect OMEGA_ARCH from the E3SM/CIME build variables when not provided. + # USE_CUDA/USE_HIP/USE_SYCL are set by the GPU machine cmake_macros + if(NOT DEFINED OMEGA_ARCH OR "${OMEGA_ARCH}" STREQUAL "") + if(USE_CUDA) + set(OMEGA_ARCH "CUDA") + + elseif(USE_HIP) + set(OMEGA_ARCH "HIP") + + elseif(USE_SYCL) + set(OMEGA_ARCH "SYCL") + + elseif(compile_threaded) + set(OMEGA_ARCH "OPENMP") + + else() + set(OMEGA_ARCH "SERIAL") + + endif() + endif() + set(OMEGA_BUILD_MODE "E3SM") message(STATUS "OMEGA_CXX_COMPILER = ${OMEGA_CXX_COMPILER}") + message(STATUS "OMEGA_ARCH = ${OMEGA_ARCH}") + message(STATUS "OMEGA_KOKKOS_OPTIONS = ${KOKKOS_OPTIONS}") endmacro() @@ -637,33 +660,97 @@ macro(update_variables) option(OMEGA_CUDA_MALLOC_ASYNC "Enable CUDA async support (default OFF)." OFF) set(OMEGA_TARGET_DEVICE FALSE) - - if("${OMEGA_ARCH}" STREQUAL "CUDA") - option(Kokkos_ENABLE_CUDA "" ON) - option(Kokkos_ENABLE_CUDA_LAMBDA "" ON) + if("${OMEGA_ARCH}" STREQUAL "CUDA" OR + "${OMEGA_ARCH}" STREQUAL "HIP" OR + "${OMEGA_ARCH}" STREQUAL "SYCL") set(OMEGA_TARGET_DEVICE TRUE) - option(Kokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC "" OFF) - set(Kokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC ${OMEGA_CUDA_MALLOC_ASYNC} CACHE BOOL "" FORCE) + endif() - elseif("${OMEGA_ARCH}" STREQUAL "HIP") - option(Kokkos_ENABLE_HIP "" ON) - set(OMEGA_TARGET_DEVICE TRUE) + # In a coupled build that also includes EAMxx, EAMxx has already created the + # Kokkos::kokkos target with the correct per-machine architecture, backend + # and (for CUDA) compiler launcher. Omega then reuses that target as-is and + # must not re-set any Kokkos_* options. + if(NOT TARGET Kokkos::kokkos) + + # In E3SM mode, reuse the per-machine Kokkos settings that the CIME machine + # configuration already provides through KOKKOS_OPTIONS (set by + # cime_config/machines/cmake_macros/.cmake and consumed unchanged + # by EAMxx/EKAT). + set(_OMEGA_KOKKOS_ARCH_SET FALSE) + if("${OMEGA_BUILD_MODE}" STREQUAL "E3SM" AND KOKKOS_OPTIONS) + string(REPLACE " " ";" _OmegaKokkosOpts "${KOKKOS_OPTIONS}") + foreach(_OmegaKopt ${_OmegaKokkosOpts}) + string(REGEX MATCH + "(Kokkos_(ARCH|ENABLE)_[A-Za-z0-9_]+)=([A-Za-z0-9_]+)" + _OmegaKmatch "${_OmegaKopt}") + if(CMAKE_MATCH_1) + set(_OmegaKvar "${CMAKE_MATCH_1}") + set(_OmegaKkind "${CMAKE_MATCH_2}") + set(_OmegaKval "${CMAKE_MATCH_3}") + option(${_OmegaKvar} "" ${_OmegaKval}) + if("${_OmegaKkind}" STREQUAL "ARCH") + if(_OmegaKval) # value form: On/ON/TRUE -> true, OFF -> false + set(_OMEGA_KOKKOS_ARCH_SET TRUE) + endif() + endif() + endif() + endforeach() - elseif("${OMEGA_ARCH}" STREQUAL "SYCL") - option(Kokkos_ENABLE_SYCL "" ON) - set(OMEGA_TARGET_DEVICE TRUE) + # Kokkos treats a variable literally named KOKKOS_OPTIONS as a DEPRECATED + # option list and hard-errors on it (kokkos_functions.cmake + # kokkos_deprecated_list, reached from kokkos_setup_build_environment). + unset(KOKKOS_OPTIONS) + endif() + # Enable the Kokkos backend that matches OMEGA_ARCH. option() is a no-op + # when the backend was already enabled by the KOKKOS_OPTIONS above. + if("${OMEGA_ARCH}" STREQUAL "CUDA") + option(Kokkos_ENABLE_CUDA "" ON) + option(Kokkos_ENABLE_CUDA_LAMBDA "" ON) + option(Kokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC "" OFF) + set(Kokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC ${OMEGA_CUDA_MALLOC_ASYNC} + CACHE BOOL "" FORCE) - elseif("${OMEGA_ARCH}" STREQUAL "OPENMP") - option(Kokkos_ENABLE_OPENMP "" ON) + elseif("${OMEGA_ARCH}" STREQUAL "HIP") + option(Kokkos_ENABLE_HIP "" ON) - elseif("${OMEGA_ARCH}" STREQUAL "THREADS") - option(Kokkos_ENABLE_THREADS "" ON) + elseif("${OMEGA_ARCH}" STREQUAL "SYCL") + option(Kokkos_ENABLE_SYCL "" ON) - else() - set(OMEGA_ARCH "SERIAL") - option(Kokkos_ENABLE_SERIAL "" ON) + elseif("${OMEGA_ARCH}" STREQUAL "OPENMP") + option(Kokkos_ENABLE_OPENMP "" ON) + + elseif("${OMEGA_ARCH}" STREQUAL "THREADS") + option(Kokkos_ENABLE_THREADS "" ON) + + else() + set(OMEGA_ARCH "SERIAL") + option(Kokkos_ENABLE_SERIAL "" ON) + + endif() + # Fail loudly if Omega must build its own Kokkos for a GPU but no Kokkos + # architecture was selected (e.g. a machine whose cmake_macros do not carry + # the arch in KOKKOS_OPTIONS). + if("${OMEGA_BUILD_MODE}" STREQUAL "E3SM" AND OMEGA_TARGET_DEVICE AND + NOT _OMEGA_KOKKOS_ARCH_SET) + message(FATAL_ERROR + "OMEGA_ARCH=${OMEGA_ARCH} requests a GPU build but no Kokkos_ARCH_* " + "was provided. Omega is building its own Kokkos here because the " + "Kokkos::kokkos target does not already exist (no EAMxx in this case). " + "On machine '${MACH}' the GPU architecture is expected in KOKKOS_OPTIONS " + "(cime_config/machines/cmake_macros/); add the appropriate Kokkos_ARCH_* " + "there, or include EAMxx so Omega reuses its Kokkos.") + endif() + + endif() + + # Belt and braces: drop KOKKOS_OPTIONS in E3SM mode even on the branch where + # Omega reused an existing Kokkos::kokkos (EAMxx present) and so never entered + # the parse above. Kokkos hard-errors on this deprecated variable name, and + # build_eamxx()'s own unset is function-local and does not reach this scope. + if("${OMEGA_BUILD_MODE}" STREQUAL "E3SM") + unset(KOKKOS_OPTIONS) endif() add_definitions(-DOMEGA_ENABLE_${OMEGA_ARCH}) diff --git a/components/omega/external/CMakeLists.txt b/components/omega/external/CMakeLists.txt index 0e565a5763ba..99b060297a0d 100644 --- a/components/omega/external/CMakeLists.txt +++ b/components/omega/external/CMakeLists.txt @@ -22,30 +22,63 @@ set_target_properties(gswteos-10 PROPERTIES ) # Add the spdlog library -if (NOT TARGET spdlog::spdlog) +# Guard on both of bare and namespaced names. +if (NOT TARGET spdlog AND NOT TARGET spdlog::spdlog) add_subdirectory( ${OMEGA_SOURCE_DIR}/external/spdlog ${CMAKE_CURRENT_BINARY_DIR}/external/spdlog ) endif() +# Omega links the bare name (src/CMakeLists.txt), so guarantee it exists. +if (NOT TARGET spdlog) + add_library(spdlog INTERFACE) + target_link_libraries(spdlog INTERFACE spdlog::spdlog) +endif() + # Add the yaml-cpp library -if (NOT TARGET yaml-cpp::yaml-cpp) +if (NOT TARGET yaml-cpp AND NOT TARGET yaml-cpp::yaml-cpp) add_subdirectory( ${OMEGA_SOURCE_DIR}/external/yaml-cpp ${CMAKE_CURRENT_BINARY_DIR}/external/yaml-cpp ) endif() -# Add the Kokkos library +# Omega links the bare name (src/CMakeLists.txt) +if (NOT TARGET yaml-cpp) + add_library(yaml-cpp INTERFACE) + target_link_libraries(yaml-cpp INTERFACE yaml-cpp::yaml-cpp) +endif() + +# Add the Kokkos library. +# Reuse-if-present: when Omega is built alongside EAMxx (or a shared EKAT), +# the Kokkos::kokkos target already exists with the correct per-machine +# architecture and compiler configuration, so Omega simply reuses it and this +# block is skipped entirely. Otherwise Omega must build Kokkos itself. +if (NOT TARGET Kokkos::kokkos AND NOT TARGET kokkos) + if ("${OMEGA_BUILD_MODE}" STREQUAL "E3SM") + list(APPEND CMAKE_MODULE_PATH + ${E3SM_EXTERNALS_ROOT}/ekat/cmake + ${E3SM_EXTERNALS_ROOT}/ekat/cmake/tpls + ) + include(EkatBuildKokkos) + else() + add_subdirectory( + ${E3SM_EXTERNALS_ROOT}/ekat/extern/kokkos + ${CMAKE_CURRENT_BINARY_DIR}/ekat/extern/kokkos + ) + endif() +endif() + +# Omega links the namespaced name (src/CMakeLists.txt). if (NOT TARGET Kokkos::kokkos) - add_subdirectory( - ${E3SM_EXTERNALS_ROOT}/ekat/extern/kokkos - ${CMAKE_CURRENT_BINARY_DIR}/ekat/extern/kokkos - ) + add_library(Kokkos::kokkos ALIAS kokkos) endif() # Add the Scorpio library +# Deliberately left as a single-name guard: unlike the libraries above, "pioc" +# has no namespaced alias to also test, so one name already covers every path +# that supplies it. if (NOT TARGET pioc) # forward env. variables to Scorpio build @@ -80,6 +113,7 @@ if (NOT TARGET pioc) endif() # Add E3SM GPTL library +# Also a single-name guard: "gptl" has no namespaced alias anywhere. if (NOT TARGET gptl) add_subdirectory( ${E3SM_ROOT}/share/timing @@ -88,6 +122,8 @@ if (NOT TARGET gptl) endif() # Add the Pacer library +# Single-name guard: no E3SM-side "pacer" target exists anywhere, so Omega +# always builds it. if (NOT TARGET pacer) add_library(pacer ${E3SM_ROOT}/share/pacer/Pacer.cpp) @@ -139,7 +175,10 @@ if(GKlib_FOUND) endif() # Add the cpptrace library -if (NOT TARGET cpptrace::cpptrace) +# Note the real target is named "cpptrace-lib", not "cpptrace" +# (external/cpptrace/CMakeLists.txt sets target_name to avoid colliding with +# its own PROJECT_NAME); cpptrace::cpptrace is an ALIAS of it. +if (NOT TARGET cpptrace::cpptrace AND NOT TARGET cpptrace-lib) # Unset PACKAGE_VERSION before adding cpptrace to prevent ADIOS2's # PACKAGE_VERSION (e.g. 2.10.2) from leaking into libdwarf's configure_file unset(PACKAGE_VERSION) @@ -149,3 +188,8 @@ if (NOT TARGET cpptrace::cpptrace) ${CMAKE_CURRENT_BINARY_DIR}/external/cpptrace ) endif() + +# Omega links the namespaced name (src/CMakeLists.txt). +if (NOT TARGET cpptrace::cpptrace) + add_library(cpptrace::cpptrace ALIAS cpptrace-lib) +endif() diff --git a/components/omega/test/CMakeLists.txt b/components/omega/test/CMakeLists.txt index 20fe917bdd48..98be34936ac8 100644 --- a/components/omega/test/CMakeLists.txt +++ b/components/omega/test/CMakeLists.txt @@ -480,6 +480,17 @@ add_omega_test( "-n;1;" ) +########################## +# Build configuration test +########################## + +add_omega_test( + BUILD_CONFIG_TEST + testBuildConfig.exe + infra/BuildConfigTest.cpp + "-n;1" +) + ################## # Driver test ################## diff --git a/components/omega/test/infra/BuildConfigTest.cpp b/components/omega/test/infra/BuildConfigTest.cpp new file mode 100644 index 000000000000..0d310640fa7a --- /dev/null +++ b/components/omega/test/infra/BuildConfigTest.cpp @@ -0,0 +1,157 @@ +//===-- Test driver for OMEGA build configuration ----------------*- C++ -*-===/ +// +/// \file +/// \brief Test driver for OMEGA build configuration +/// +/// This driver checks that the architecture Omega was configured with agrees +/// with the Kokkos backend it was actually built against. OMEGA_ARCH is chosen +/// by the build system - from the CIME machine settings in an E3SM build, or +/// from the standalone configuration - and it drives both the OMEGA_ENABLE_* +/// and OMEGA_TARGET_DEVICE macros and the Kokkos backend selection. Nothing at +/// runtime re-derives it, so a mismatch between the two is silent: Omega would +/// believe it is running on one architecture while Kokkos executes on another. +/// That is the failure this test exists to catch, in particular a host-only +/// Kokkos paired with a GPU OMEGA_ARCH or the reverse. +/// +// +//===-----------------------------------------------------------------------===/ + +#include "DataTypes.h" +#include "Error.h" +#include "Logging.h" +#include "MachEnv.h" +#include "OmegaKokkos.h" +#include "Pacer.h" +#include "mpi.h" + +#include +#include + +using namespace OMEGA; + +// OMEGA_ARCH is defined as a bare token (e.g. -DOMEGA_ARCH=CUDA) so it must be +// stringified before it can be compared. +#define OMEGA_STR_HELPER(Arg) #Arg +#define OMEGA_STR(Arg) OMEGA_STR_HELPER(Arg) + +//------------------------------------------------------------------------------ +// Returns the Kokkos execution space name expected for a given OMEGA_ARCH. +// These are the names Kokkos itself reports, so the comparison stays valid +// across Kokkos versions that rename the underlying types. + +std::string expectedKokkosSpace(const std::string &Arch // [in] OMEGA_ARCH value +) { + + if (Arch == "CUDA") + return "Cuda"; + if (Arch == "HIP") + return "HIP"; + if (Arch == "SYCL") + return "SYCL"; + if (Arch == "OPENMP") + return "OpenMP"; + if (Arch == "SERIAL") + return "Serial"; + + return ""; + +} // end expectedKokkosSpace + +//------------------------------------------------------------------------------ +// The test driver for the build configuration. +// +int main(int argc, char *argv[]) { + + // Initialize the global MPI environment + MPI_Init(&argc, &argv); + Kokkos::initialize(); + Pacer::initialize(MPI_COMM_WORLD); + Pacer::setPrefix("Omega:"); + + // These are needed to set up logging for output + MachEnv::init(MPI_COMM_WORLD); + MachEnv *DefEnv = MachEnv::getDefault(); + initLogging(DefEnv); + LOG_INFO("----- Build Configuration Unit Testing -----"); + + { + const std::string ArchName = OMEGA_STR(OMEGA_ARCH); + const std::string KokkosName{Kokkos::DefaultExecutionSpace::name()}; + + LOG_INFO("BuildConfigTest: OMEGA_ARCH = {}", ArchName); + LOG_INFO("BuildConfigTest: Kokkos default execution space = {}", + KokkosName); + + // OMEGA_ARCH must be one of the architectures the build system knows how + // to configure. An empty or unrecognized value means the build system + // failed to determine one. + const std::string ExpectedSpace = expectedKokkosSpace(ArchName); + if (ExpectedSpace.empty()) + ABORT_ERROR("BuildConfigTest: FAIL OMEGA_ARCH '{}' is not one of " + "CUDA, HIP, SYCL, OPENMP, SERIAL", + ArchName); + + // The matching OMEGA_ENABLE_ macro must have been defined, since + // the source uses it to select architecture-specific code paths. + std::string EnabledArch; +#ifdef OMEGA_ENABLE_CUDA + EnabledArch = "CUDA"; +#endif +#ifdef OMEGA_ENABLE_HIP + EnabledArch = "HIP"; +#endif +#ifdef OMEGA_ENABLE_SYCL + EnabledArch = "SYCL"; +#endif +#ifdef OMEGA_ENABLE_OPENMP + EnabledArch = "OPENMP"; +#endif +#ifdef OMEGA_ENABLE_SERIAL + EnabledArch = "SERIAL"; +#endif + + if (EnabledArch != ArchName) + ABORT_ERROR("BuildConfigTest: FAIL OMEGA_ARCH is '{}' but the macro " + "OMEGA_ENABLE_{} was not the one defined (got '{}')", + ArchName, ArchName, EnabledArch); + + // Kokkos must actually be running on the architecture Omega was told it + // would use. This is the check that catches a host-only Kokkos in a build + // that believes it is on a GPU, or the reverse. + if (KokkosName != ExpectedSpace) + ABORT_ERROR("BuildConfigTest: FAIL OMEGA_ARCH '{}' expects Kokkos " + "execution space '{}' but Kokkos is using '{}'", + ArchName, ExpectedSpace, KokkosName); + + // OMEGA_TARGET_DEVICE must be set for, and only for, a build whose + // default execution space has its own memory space. Omega uses it to + // decide whether device arrays need explicit host mirrors. + constexpr bool KokkosIsDevice = + !std::is_same_v; +#ifdef OMEGA_TARGET_DEVICE + constexpr bool OmegaIsDevice = true; +#else + constexpr bool OmegaIsDevice = false; +#endif + + if (OmegaIsDevice != KokkosIsDevice) + ABORT_ERROR("BuildConfigTest: FAIL OMEGA_TARGET_DEVICE is {} but the " + "Kokkos default execution space '{}' is {} device space", + OmegaIsDevice ? "defined" : "not defined", KokkosName, + KokkosIsDevice ? "a" : "not a"); + + LOG_INFO("BuildConfigTest: OMEGA_TARGET_DEVICE = {}", + OmegaIsDevice ? "defined" : "not defined"); + } + + LOG_INFO("----- Build Configuration Unit Tests Successful -----"); + Pacer::finalize(); + Kokkos::finalize(); + MPI_Barrier(MPI_COMM_WORLD); + MPI_Finalize(); + + return 0; // if we made it here, return successfully + +} // end of main +//===-----------------------------------------------------------------------===/