From 3561d0d05d790f6ca6aa06a0b3e28b4101c3ca83 Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Wed, 26 Aug 2026 07:23:14 -0700 Subject: [PATCH] Use driver-owned PIO settings for coupled Omega initialization * Pass `IOBaseTask` and `IORearranger` from CIME through the Fortran/C++ interface * Add `IO::init(IOInitParams)` while preserving configuration-based initialization for standalone runs * Block `IO.IOBaseTask` and `IO.IORearranger` overrides in `omega_buildnml` * Log the PIO settings used to initialize SCORPIO * Add `IO_INIT_PARAMS_TEST` to verify driver-provided settings reach SCORPIO --- .../tests/test_validate_user_overrides.py | 28 ++++ .../cime_config/omega_buildnml/validate.py | 5 + components/omega/src/base/IO.cpp | 105 +++++++++--- components/omega/src/base/IO.h | 19 +++ .../src/drivers/coupled/ocn_comp_mct.F90 | 13 +- .../drivers/coupled/omega_cxx2f_interface.cpp | 13 +- .../drivers/coupled/omega_f2cxx_interface.F90 | 8 +- components/omega/src/ocn/OceanDriver.h | 13 +- components/omega/src/ocn/OceanInit.cpp | 26 ++- components/omega/test/CMakeLists.txt | 11 ++ .../omega/test/base/IOInitParamsTest.cpp | 152 ++++++++++++++++++ 11 files changed, 355 insertions(+), 38 deletions(-) create mode 100644 components/omega/test/base/IOInitParamsTest.cpp diff --git a/components/omega/cime_config/omega_buildnml/tests/test_validate_user_overrides.py b/components/omega/cime_config/omega_buildnml/tests/test_validate_user_overrides.py index 7ca59a65e545..60b502385c7c 100644 --- a/components/omega/cime_config/omega_buildnml/tests/test_validate_user_overrides.py +++ b/components/omega/cime_config/omega_buildnml/tests/test_validate_user_overrides.py @@ -13,6 +13,13 @@ def defaults(): "StartTime": "0001-01-01_00:00:00", }, "Tendencies": {"SurfaceTracerRestoringEnable": False}, + "IO": { + "IOTasks": 1, + "IOStride": 1, + "IOBaseTask": 0, + "IORearranger": "box", + "IODefaultFormat": "pnetcdf", + }, "IOStreams": { "InitialState": {"Filename": "ocean.nc"}, "History": {"Freq": 1, "FreqUnits": "months"}, @@ -85,3 +92,24 @@ def test_overriding_an_existing_stream_is_allowed(defaults): validated = validate_user_overrides(user_overrides, defaults) assert validated == user_overrides + + +@pytest.mark.parametrize("option", ["IOBaseTask", "IORearranger"]) +def test_driver_owned_io_options_are_rejected(option, defaults): + """ + The base IO task and rearranger are owned by the driver (CIME/shr_pio), + so a user may not override them in ``user_nl_omega``. + """ + user_overrides = {"IO": {option: 4 if option == "IOBaseTask" else "subset"}} + + with pytest.raises(ValueError, match="cannot be overridden"): + validate_user_overrides(user_overrides, defaults) + + +def test_component_configurable_io_options_are_allowed(defaults): + """``IOTasks`` and ``IOStride`` remain component-configurable.""" + user_overrides = {"IO": {"IOTasks": 8, "IOStride": 2}} + + validated = validate_user_overrides(user_overrides, defaults) + + assert validated == user_overrides diff --git a/components/omega/cime_config/omega_buildnml/validate.py b/components/omega/cime_config/omega_buildnml/validate.py index 514c024c4170..69afeb43de5b 100644 --- a/components/omega/cime_config/omega_buildnml/validate.py +++ b/components/omega/cime_config/omega_buildnml/validate.py @@ -47,6 +47,11 @@ "TimeIntegration.RunDuration", # calendar must agree with the CIME ``CALENDAR`` setting "TimeIntegration.CalendarType", + # base IO task and rearranger are owned by the driver (CIME/shr_pio) + # so they stay consistent with the rest of the case. IOTasks and + # IOStride remain component-configurable. + "IO.IOBaseTask", + "IO.IORearranger", } ) diff --git a/components/omega/src/base/IO.cpp b/components/omega/src/base/IO.cpp index 86a6ef14be53..ff5613051339 100644 --- a/components/omega/src/base/IO.cpp +++ b/components/omega/src/base/IO.cpp @@ -51,6 +51,24 @@ RearrFromString(const std::string &Rearr // [in] choice of IO rearranger } // End RearrFromString +//------------------------------------------------------------------------------ +// Converts the PIO rearranger enum back to the name used in the config file. +// Only used for log messages. RearrMap is searched rather than adding a second +// table so the two cannot drift apart. "default" is skipped because it aliases +// "box" and the recognisable name is wanted here. Falls back to the numeric +// value for anything not in the map. +static std::string RearrToString(const Rearranger Rearr // [in] rearranger enum +) { + + for (const auto &Entry : RearrMap) { + if (Entry.second == Rearr && Entry.first != "default") + return Entry.first; + } + + return std::to_string(static_cast(Rearr)); + +} // End RearrToString + //------------------------------------------------------------------------------ // Converts string choice for File Format to an enum FileFmt @@ -125,11 +143,54 @@ IfExists IfExistsFromString( // Methods //------------------------------------------------------------------------------ // Initializes the IO system based on configuration inputs and -// default MPI communicator +// default MPI communicator. The base task and rearranger are read from the +// Omega configuration and forwarded to the driver-owned overload below. void init(const MPI_Comm &InComm // [in] MPI communicator to use ) { - // Retrieve parallel IO parameters from the Omega configuration + // Retrieve the driver-owned parallel IO parameters from the Omega + // configuration. These are read here for standalone runs and unit tests + // where there is no coupler to supply them. + + Error Err; + Config *OmegaConfig = Config::getOmegaConfig(); + OMEGA_REQUIRE(OmegaConfig, "Null OmegaConfig pointer in IO::init"); + + // Read IO subconfiguration + Config IOConfig("IO"); + Err = OmegaConfig->get(IOConfig); + CHECK_ERROR_ABORT(Err, "IO: IO group not found in input Config"); + + // Read parallel IO settings - default to single-task if config + // values do not exist + int IOBaseTask = 0; + std::string InRearranger = "box"; + + Err = IOConfig.get("IOBaseTask", IOBaseTask); + CHECK_ERROR_WARN(Err, "IO: IOBaseTask not found in Config - using {}", + IOBaseTask); + + Err = IOConfig.get("IORearranger", InRearranger); + CHECK_ERROR_WARN(Err, "IO: Rearranger not found in Config - using {}", + InRearranger); + + IOInitParams IOParams{IOBaseTask, RearrFromString(InRearranger)}; + init(InComm, IOParams); + + return; + +} // end init + +//------------------------------------------------------------------------------ +// Initializes the IO system using driver-owned IO parameters for the base +// task and rearranger. The number of IO tasks and the IO stride are still +// read from the Omega configuration. +void init(const MPI_Comm &InComm, // [in] MPI communicator to use + const IOInitParams &IOParams // [in] driver-owned IO parameters +) { + + // Retrieve the remaining parallel IO parameters from the Omega + // configuration Error Err; Config *OmegaConfig = Config::getOmegaConfig(); @@ -147,12 +208,10 @@ void init(const MPI_Comm &InComm // [in] MPI communicator to use InFileFmt); DefaultFileFmt = FileFmtFromString(InFileFmt); - // Read parallel IO settings - default to single-task if config - // values do not exist - int NumIOTasks = 1; - int IOStride = 1; - int IOBaseTask = 0; - std::string InRearranger = "box"; + // Read component-configurable parallel IO settings - default to + // single-task if config values do not exist + int NumIOTasks = 1; + int IOStride = 1; Err = IOConfig.get("IOTasks", NumIOTasks); CHECK_ERROR_WARN(Err, "IO: NumIOTasks not found in Config - using {}", @@ -162,22 +221,26 @@ void init(const MPI_Comm &InComm // [in] MPI communicator to use CHECK_ERROR_WARN(Err, "IO: IOStride not found in Config - using {}", IOStride); - Err = IOConfig.get("IOBaseTask", IOBaseTask); - CHECK_ERROR_WARN(Err, "IO: IOBaseTask not found in Config - using {}", - IOBaseTask); - - Err = IOConfig.get("IORearranger", InRearranger); - CHECK_ERROR_WARN(Err, "IO: Rearranger not found in Config - using {}", - InRearranger); - Rearranger Rearrange = RearrFromString(InRearranger); - - // Call PIO routine to initialize - DefaultRearr = Rearrange; - int PIOErr = PIOc_Init_Intracomm(InComm, NumIOTasks, IOStride, IOBaseTask, - Rearrange, &SysID); + // Base task and rearranger are supplied by the caller (driver-owned in a + // coupled run) rather than read from the component config. + DefaultRearr = IOParams.IORearranger; + int PIOErr = + PIOc_Init_Intracomm(InComm, NumIOTasks, IOStride, IOParams.IOBaseTask, + IOParams.IORearranger, &SysID); if (PIOErr != 0) ABORT_ERROR("IO::init: Error initializing SCORPIO"); + // Report the settings SCORPIO was actually initialized with. IOBaseTask and + // IORearranger are the interesting ones: in a coupled run they come from the + // driver (shr_pio_getioroot/shr_pio_getrearranger) rather than from Omega's + // config, and without this message there is no way to observe which values + // were used, since they are passed straight into PIOc_Init_Intracomm. + LOG_INFO( + "IO::init: IOTasks={} IOStride={} IOBaseTask={} IORearranger={} ({})", + NumIOTasks, IOStride, IOParams.IOBaseTask, + RearrToString(IOParams.IORearranger), + static_cast(IOParams.IORearranger)); + return; } // end init diff --git a/components/omega/src/base/IO.h b/components/omega/src/base/IO.h index e0de7cb91437..b2f0c29456e0 100644 --- a/components/omega/src/base/IO.h +++ b/components/omega/src/base/IO.h @@ -163,6 +163,16 @@ IfExists IfExistsFromString( const std::string &IfExists ///< [in] choice of behavior on file existence ); +/// Parameters for initializing the IO subsystem that are owned by the +/// driver/coupler rather than by the Omega component config. In a coupled +/// run the coupler (via CIME/shr_pio) chooses the base IO task and the +/// rearranger so that Omega's parallel IO layout is consistent with the +/// rest of the E3SM case. +struct IOInitParams { + int IOBaseTask; ///< base (root) MPI task for IO + Rearranger IORearranger; ///< parallel IO rearranger algorithm +}; + // Methods /// Initializes the IO system based on configuration inputs and @@ -170,6 +180,15 @@ IfExists IfExistsFromString( void init(const MPI_Comm &InComm ///< [in] MPI communicator to use ); +/// Initializes the IO system using driver-owned IO parameters for the base +/// task and rearranger. The number of IO tasks and the IO stride are still +/// read from the Omega config; only the base task and rearranger are taken +/// from the supplied parameters. Used in coupled runs where the coupler owns +/// these settings. +void init(const MPI_Comm &InComm, ///< [in] MPI communicator to use + const IOInitParams &IOParams ///< [in] driver-owned IO parameters +); + /// This routine opens a file for reading. The filename with full path must be /// supplied and a FileID is returned to be used by other IO functions. /// The format of the file is assumed to be the default defined on init diff --git a/components/omega/src/drivers/coupled/ocn_comp_mct.F90 b/components/omega/src/drivers/coupled/ocn_comp_mct.F90 index 7f5e5e7fc21a..4250cfd8aec7 100644 --- a/components/omega/src/drivers/coupled/ocn_comp_mct.F90 +++ b/components/omega/src/drivers/coupled/ocn_comp_mct.F90 @@ -69,6 +69,7 @@ subroutine ocn_init_mct(EClock, cdata, x2o, o2x, NLFilename) use shr_sys_mod, only: shr_sys_flush use shr_cal_mod, only: shr_cal_noleap, shr_cal_gregorian use shr_file_mod, only: shr_file_getunit, shr_file_setIO + use shr_pio_mod, only: shr_pio_getioroot, shr_pio_getrearranger ! !INPUT/OUTPUT PARAMETERS: type(ESMF_Clock), intent(inout) :: EClock @@ -97,6 +98,8 @@ subroutine ocn_init_mct(EClock, cdata, x2o, o2x, NLFilename) coupling_time_step, case_start_tod, case_start_ymd, cur_tod, cur_ymd integer(kind=c_int) :: start_type_c integer(kind=c_int) :: layout + integer(kind=c_int) :: io_base_task ! driver-owned base (root) IO task + integer(kind=c_int) :: io_rearranger ! driver-owned PIO rearranger character(kind=c_char, len=CL), target :: calendar_c character(kind=c_char, len=CL), target :: ocn_log_fname_c @@ -186,6 +189,12 @@ subroutine ocn_init_mct(EClock, cdata, x2o, o2x, NLFilename) ! populate the import/export field name and index arrays call omega_set_cpl_indices() + ! The base IO task and rearranger are owned by the driver/coupler (set + ! by CIME via shr_pio) so they stay consistent with the rest of the + ! case; they are passed to Omega rather than read from omega.yml. + io_base_task = shr_pio_getioroot(OCN_ID) + io_rearranger = shr_pio_getrearranger(OCN_ID) + #ifdef HAVE_MOAB layout = omega_get_layout_moab() #else @@ -209,7 +218,9 @@ subroutine ocn_init_mct(EClock, cdata, x2o, o2x, NLFilename) c_loc(import_field_names), & c_loc(export_field_names), & c_loc(import_field_indices), & - c_loc(export_field_indices) & + c_loc(export_field_indices), & + io_base_task, & + io_rearranger & ) !------------------------------------------------------------------------- diff --git a/components/omega/src/drivers/coupled/omega_cxx2f_interface.cpp b/components/omega/src/drivers/coupled/omega_cxx2f_interface.cpp index 2347521336f6..5aa5cf436d94 100644 --- a/components/omega/src/drivers/coupled/omega_cxx2f_interface.cpp +++ b/components/omega/src/drivers/coupled/omega_cxx2f_interface.cpp @@ -3,6 +3,7 @@ // //===----------------------------------------------------------------------===// #include "DataTypes.h" +#include "IO.h" #include "Logging.h" #include "MachEnv.h" #include "OceanDriver.h" @@ -60,7 +61,9 @@ void omega_ocn_init1( const char *ImportFieldNames, // [in] array of import field names const char *ExportFieldNames, // [in] array of export field names const int *ImportFieldIndices, // [in] array of import field indices - const int *ExportFieldIndices // [in] array of export field indices + const int *ExportFieldIndices, // [in] array of export field indices + const int IOBaseTask, // [in] driver-owned base (root) IO task + const int IORearranger // [in] driver-owned PIO rearranger (int) ) { // Create the C MPI_Comm from the Fortran one @@ -103,8 +106,14 @@ void omega_ocn_init1( NCouplerImports, NCouplerExports, ImportIdxMap, ExportIdxMap, CouplingInterval, OMEGA::CouplingLayout::MCT}; + // The base IO task and rearranger are owned by the driver/coupler (via + // CIME/shr_pio). The rearranger int uses the same PIO_REARR_* values as + // Omega's IO::Rearranger enum (box = 1, subset = 2). + OMEGA::IO::IOInitParams IOParams{ + IOBaseTask, static_cast(IORearranger)}; + OMEGA::ocnInit1(Comm, OcnID, YamlConfigFile, OcnLogFile, StartTypeEnum, - TimeParams, CouplingParams); + TimeParams, CouplingParams, IOParams); Pacer::stop("Init1", 0); diff --git a/components/omega/src/drivers/coupled/omega_f2cxx_interface.F90 b/components/omega/src/drivers/coupled/omega_f2cxx_interface.F90 index 7f00263fffa7..53102d4756d8 100644 --- a/components/omega/src/drivers/coupled/omega_f2cxx_interface.F90 +++ b/components/omega/src/drivers/coupled/omega_f2cxx_interface.F90 @@ -27,7 +27,9 @@ subroutine omega_ocn_init1( & import_field_names, & export_field_names, & import_field_indices, & - export_field_indices) bind(c) + export_field_indices, & + io_base_task, & + io_rearranger) bind(c) use, intrinsic :: iso_c_binding, only: c_int, c_char, c_ptr @@ -43,7 +45,9 @@ subroutine omega_ocn_init1( & n_coupler_imports, & n_coupler_exports, & n_omega_imports, & - n_omega_exports + n_omega_exports, & + io_base_task, & + io_rearranger character(kind=c_char), target, intent(in) :: & yaml_config_name, ocn_log_name, calendar_name diff --git a/components/omega/src/ocn/OceanDriver.h b/components/omega/src/ocn/OceanDriver.h index 5410585f73ab..95eda572b6de 100644 --- a/components/omega/src/ocn/OceanDriver.h +++ b/components/omega/src/ocn/OceanDriver.h @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "Config.h" +#include "IO.h" #include "SfcCoupling.h" #include "TimeMgr.h" #include "TimeStepper.h" @@ -40,9 +41,10 @@ int ocnInit1( const int OcnId, ///< [in] mct comp id for ocean const std::string &ConfigFile, ///< [in] path to yaml config file const std::string &LogFile, ///< [in] path to log file - const StartType StartType, ///< [in] simulation start type - const TimeInitParams &TimeParams, ///< [in] time parameters - const CouplingInitParams &CouplingParams ///< [in] coupling parameters + const StartType StartType, ///< [in] simulation start type + const TimeInitParams &TimeParams, ///< [in] time parameters + const CouplingInitParams &CouplingParams, ///< [in] coupling parameters + const IO::IOInitParams &IOParams ///< [in] driver-owned IO params ); /// Coupled init phase 2: runs once the coupler has allocated its MCT buffers; @@ -63,9 +65,10 @@ int ocnFinalize(const TimeInstant &CurrTime); /// Initialize Omega modules needed to run ocean model int initOmegaModules(MPI_Comm Comm); -/// Initialize Omega modules with coupler-provided time parameters +/// Initialize Omega modules with coupler-provided time and IO parameters int initOmegaModules(MPI_Comm Comm, const TimeInitParams &TParams, - const CouplingInitParams &CParams); + const CouplingInitParams &CParams, + const IO::IOInitParams &IOParams); /// Update Halo/Host arrays with new state, auxiliary state, and tracer fields int initUpdateHaloAndHostArrays(); diff --git a/components/omega/src/ocn/OceanInit.cpp b/components/omega/src/ocn/OceanInit.cpp index 472725f07dc4..82135bfc71b8 100644 --- a/components/omega/src/ocn/OceanInit.cpp +++ b/components/omega/src/ocn/OceanInit.cpp @@ -37,6 +37,8 @@ #include "mpi.h" +#include + namespace OMEGA { // Convienvence converter of an int to a StartType enum, with error checking @@ -174,7 +176,8 @@ int ocnInit1(MPI_Comm Comm, ///< [in] ocean MPI communicator const std::string &LogFile, ///< [in] path to log file const StartType StartType, ///< [in] simulation start type const TimeInitParams &TimeParams, ///< [in] simulation start time - const CouplingInitParams &CouplingParams ///< [in] coupler info + const CouplingInitParams &CouplingParams, ///< [in] coupler info + const IO::IOInitParams &IOParams ///< [in] driver-owned IO params ) { I4 Err = 0; // return error code @@ -191,7 +194,7 @@ int ocnInit1(MPI_Comm Comm, ///< [in] ocean MPI communicator readTimingConfig(OmegaConfig); // initialize remaining Omega modules - Err = initOmegaModules(Comm, TimeParams, CouplingParams); + Err = initOmegaModules(Comm, TimeParams, CouplingParams, IOParams); if (Err != 0) ABORT_ERROR("ocnInit: Error initializing Omega modules"); @@ -261,8 +264,12 @@ int ocnInit2(const Real *CplToOcnData, Real *OcnToCplData) { // Call init routines for remaining Omega modules // Internal helper — all module init after TimeStepper::init1 is called. -// Called by both initOmegaModules overloads. -static int initOmegaModulesImpl(MPI_Comm Comm) { +// Called by both initOmegaModules overloads. When IOParams is provided (the +// coupled path) the IO base task and rearranger come from the driver; +// otherwise they are read from the Omega config. +static int initOmegaModulesImpl( + MPI_Comm Comm, + const std::optional &IOParams = std::nullopt) { // error and return codes int Err = 0; @@ -274,7 +281,11 @@ static int initOmegaModulesImpl(MPI_Comm Comm) { // of each file, only creates streams from Config IOStream::init(ModelClock); - IO::init(Comm); + if (IOParams.has_value()) { + IO::init(Comm, IOParams.value()); + } else { + IO::init(Comm); + } Field::init(ModelClock); Decomp::init(); @@ -324,12 +335,13 @@ int initOmegaModules(MPI_Comm Comm) { } int initOmegaModules(MPI_Comm Comm, const TimeInitParams &TParams, - const CouplingInitParams &CParams) { + const CouplingInitParams &CParams, + const IO::IOInitParams &IOParams) { int Err = 0; // Initialize time stepper (phase 1) using coupler provided time parameters // Calendar should have already been initalized TimeStepper::init1(TParams); - Err = initOmegaModulesImpl(Comm); + Err = initOmegaModulesImpl(Comm, IOParams); SfcCoupling::init(CParams); return Err; diff --git a/components/omega/test/CMakeLists.txt b/components/omega/test/CMakeLists.txt index 20fe917bdd48..f6b77e561d81 100644 --- a/components/omega/test/CMakeLists.txt +++ b/components/omega/test/CMakeLists.txt @@ -276,6 +276,17 @@ add_omega_test( "-n;8" ) +############################## +# Driver-owned IO params test +############################## + +add_omega_test( + IO_INIT_PARAMS_TEST + testIOInitParams.exe + base/IOInitParamsTest.cpp + "-n;8" +) + ################## # Config test ################## diff --git a/components/omega/test/base/IOInitParamsTest.cpp b/components/omega/test/base/IOInitParamsTest.cpp new file mode 100644 index 000000000000..e2c250fa9ea1 --- /dev/null +++ b/components/omega/test/base/IOInitParamsTest.cpp @@ -0,0 +1,152 @@ +//===-- Test driver for OMEGA driver-owned IO parameters ---------*- C++ -*-===/ +// +/// \file +/// \brief Test driver for OMEGA driver-owned IO parameters +/// +/// This driver tests the IO::init overload that takes IOInitParams. In a +/// coupled run the IO base task and rearranger belong to the driver, which +/// supplies them from shr_pio_getioroot and shr_pio_getrearranger, while the +/// remaining IO settings still come from the Omega configuration. This test +/// passes values that deliberately differ from the ones in the config file and +/// checks that the supplied values are the ones actually used, so that a +/// regression which silently fell back to the config values would fail here. +/// +// +//===-----------------------------------------------------------------------===/ + +#include "IO.h" +#include "Config.h" +#include "DataTypes.h" +#include "Error.h" +#include "Logging.h" +#include "MachEnv.h" +#include "Pacer.h" +#include "mpi.h" + +#include + +using namespace OMEGA; + +//------------------------------------------------------------------------------ +// The test driver for driver-owned IO parameters. +// +int main(int argc, char *argv[]) { + + // Initialize the global MPI environment + MPI_Init(&argc, &argv); + Kokkos::initialize(); + Pacer::initialize(MPI_COMM_WORLD); + Pacer::setPrefix("Omega:"); + + { + Error Err; + + // Initialize the Machine Environment class and retrieve the default + // environment and communicator + MachEnv::init(MPI_COMM_WORLD); + MachEnv *DefEnv = MachEnv::getDefault(); + MPI_Comm DefComm = DefEnv->getComm(); + I4 MyTask = DefEnv->getMyTask(); + I4 NumTasks = DefEnv->getNumTasks(); + + // Initialize the Logging system + initLogging(DefEnv); + LOG_INFO("----- Driver-owned IO Parameters Unit Testing -----"); + + // This test needs a task the base task can be moved to + if (NumTasks < 2) + ABORT_ERROR("IOInitParamsTest: FAIL test requires at least 2 tasks, " + "got {}", + NumTasks); + + // Open config file + Config("Omega"); + Config::readAll("omega.yml"); + Config *OmegaConfig = Config::getOmegaConfig(); + + // Read the IO settings the configuration file supplies. These are the + // values IO::init would use if the supplied parameters were ignored. + Config IOConfig("IO"); + Err = OmegaConfig->get(IOConfig); + CHECK_ERROR_ABORT(Err, "IOInitParamsTest: FAIL IO group not found in " + "config"); + + I4 ConfigBaseTask = 0; + Err = IOConfig.get("IOBaseTask", ConfigBaseTask); + CHECK_ERROR_ABORT(Err, "IOInitParamsTest: FAIL IOBaseTask not found in " + "config"); + + std::string ConfigRearrName = "box"; + Err = IOConfig.get("IORearranger", ConfigRearrName); + CHECK_ERROR_ABORT(Err, "IOInitParamsTest: FAIL IORearranger not found in " + "config"); + IO::Rearranger ConfigRearr = IO::RearrFromString(ConfigRearrName); + + // Choose parameters that differ from the config values, so that the two + // sources can be told apart. Without this the test would pass whether or + // not the supplied values were honored. + I4 DriverBaseTask = (ConfigBaseTask == 0) ? 1 : 0; + IO::Rearranger DriverRearr = + (ConfigRearr == IO::RearrSubset) ? IO::RearrBox : IO::RearrSubset; + + LOG_INFO("IOInitParamsTest: config supplies IOBaseTask={} IORearranger={}", + ConfigBaseTask, static_cast(ConfigRearr)); + LOG_INFO("IOInitParamsTest: driver supplies IOBaseTask={} IORearranger={}", + DriverBaseTask, static_cast(DriverRearr)); + + // Initialize IO with the driver-owned parameters + IO::IOInitParams DriverParams{DriverBaseTask, DriverRearr}; + IO::init(DefComm, DriverParams); + + // The rearranger actually in use is recorded in DefaultRearr + if (IO::DefaultRearr != DriverRearr) + ABORT_ERROR("IOInitParamsTest: FAIL rearranger is {} but the driver " + "supplied {}", + static_cast(IO::DefaultRearr), + static_cast(DriverRearr)); + + if (IO::DefaultRearr == ConfigRearr) + ABORT_ERROR("IOInitParamsTest: FAIL rearranger fell back to the " + "config value {}", + static_cast(ConfigRearr)); + + // Ask SCORPIO which tasks it made IO tasks. The lowest of them is the + // base task, whatever the task count and stride happen to be, so this + // checks the supplied base task reached PIOc_Init_Intracomm rather than + // only being stored somewhere. + bool IsIOTask = false; + int PIOErr = PIOc_iam_iotask(IO::SysID, &IsIOTask); + if (PIOErr != PIO_NOERR) + ABORT_ERROR("IOInitParamsTest: FAIL could not query SCORPIO IO tasks"); + + I4 MyIORank = IsIOTask ? MyTask : NumTasks; + I4 MinIORank = NumTasks; + MPI_Allreduce(&MyIORank, &MinIORank, 1, MPI_INT, MPI_MIN, DefComm); + + if (MinIORank == NumTasks) + ABORT_ERROR("IOInitParamsTest: FAIL SCORPIO reports no IO tasks"); + + if (MinIORank != DriverBaseTask) + ABORT_ERROR("IOInitParamsTest: FAIL lowest SCORPIO IO task is {} but " + "the driver supplied base task {}", + MinIORank, DriverBaseTask); + + LOG_INFO("IOInitParamsTest: lowest SCORPIO IO task is {} as supplied", + MinIORank); + + // Exit environments + MachEnv::removeAll(); + + LOG_INFO("IOInitParamsTest: Successful completion"); + } + + LOG_INFO("----- Driver-owned IO Parameters Unit Tests Successful -----"); + Pacer::finalize(); + Kokkos::finalize(); + MPI_Barrier(MPI_COMM_WORLD); + MPI_Finalize(); + + return 0; // if we made it here, return successfully + +} // end of main +//===-----------------------------------------------------------------------===/