diff --git a/CMakeLists.txt b/CMakeLists.txt index 28d71f09b0..613deebfa2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,12 +20,13 @@ cmake_policy(SET CMP0074 NEW) # find_package() uses upper-case _ROOT variables in conjunction with CMP0074. # https://cmake.org/cmake/help/latest/policy/CMP0144.html if(POLICY CMP0144) - cmake_policy(SET CMP0144 NEW) +cmake_policy(SET CMP0144 NEW) endif() # Project Variables =========================================================== set(NGEN_ROOT_DIR "${CMAKE_CURRENT_LIST_DIR}") set(NGEN_SRC_DIR "${NGEN_ROOT_DIR}/src") +set(NGEN_NCF_DIR "${NGEN_ROOT_DIR}/src/netcdf") set(NGEN_INC_DIR "${NGEN_ROOT_DIR}/include") set(NGEN_EXT_DIR "${NGEN_ROOT_DIR}/extern") set(NGEN_MOD_DIR "${NGEN_ROOT_DIR}/cmake") @@ -85,28 +86,28 @@ cmake_dependent_option(NGEN_WITH_EXTERN_NOAH_OWP_MODULAR "Build with extern-dist # Allow enabling of all extern models dependent on NGen options given. option(NGEN_WITH_EXTERN_ALL OFF) if(NGEN_WITH_EXTERN_ALL) - set(NGEN_WITH_EXTERN_SLOTH ON) +set(NGEN_WITH_EXTERN_SLOTH ON) - # Ensure C support is enabled - if(NGEN_WITH_BMI_C) - set(NGEN_WITH_EXTERN_TOPMODEL ON) - set(NGEN_WITH_EXTERN_CFE ON) - set(NGEN_WITH_EXTERN_PET ON) - else() - message(FATAL_ERROR "NGEN_WITH_EXTERN_ALL set to ${NGEN_WITH_EXTERN_ALL}, but NGEN_WITH_BMI_C set to ${NGEN_WITH_BMI_C}") - endif() - - # Ensure Fortran support is enabled - if(NGEN_WITH_BMI_FORTRAN) - set(NGEN_WITH_EXTERN_NOAH_OWP_MODULAR ON) - else() - message(FATAL_ERROR "NGEN_WITH_EXTERN_ALL set to ${NGEN_WITH_EXTERN_ALL}, but NGEN_WITH_BMI_FORTRAN set to ${NGEN_WITH_BMI_FORTRAN}") - endif() - - # Ensure Python support is enabled - if(NOT NGEN_WITH_PYTHON) - message(FATAL_ERROR "NGEN_WITH_EXTERN_ALL set to ${NGEN_WITH_EXTERN_ALL}, but NGEN_WITH_PYTHON set to ${NGEN_WITH_PYTHON}") - endif() +# Ensure C support is enabled +if(NGEN_WITH_BMI_C) +set(NGEN_WITH_EXTERN_TOPMODEL ON) +set(NGEN_WITH_EXTERN_CFE ON) +set(NGEN_WITH_EXTERN_PET ON) + else() +message(FATAL_ERROR "NGEN_WITH_EXTERN_ALL set to ${NGEN_WITH_EXTERN_ALL}, but NGEN_WITH_BMI_C set to ${NGEN_WITH_BMI_C}") + endif() + +# Ensure Fortran support is enabled +if(NGEN_WITH_BMI_FORTRAN) +set(NGEN_WITH_EXTERN_NOAH_OWP_MODULAR ON) + else() +message(FATAL_ERROR "NGEN_WITH_EXTERN_ALL set to ${NGEN_WITH_EXTERN_ALL}, but NGEN_WITH_BMI_FORTRAN set to ${NGEN_WITH_BMI_FORTRAN}") + endif() + +# Ensure Python support is enabled +if(NOT NGEN_WITH_PYTHON) +message(FATAL_ERROR "NGEN_WITH_EXTERN_ALL set to ${NGEN_WITH_EXTERN_ALL}, but NGEN_WITH_PYTHON set to ${NGEN_WITH_PYTHON}") + endif() endif() # Project ===================================================================== @@ -119,6 +120,8 @@ set(CMAKE_CXX_VISIBILITY_PRESET hidden) # Define project version and use via generated config header project(ngen VERSION 0.3.0) +set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) +set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) add_executable(ngen "${NGEN_SRC_DIR}/NGen.cpp") @@ -126,15 +129,15 @@ add_executable(ngen "${NGEN_SRC_DIR}/NGen.cpp") find_package(ewts CONFIG REQUIRED) if(DEFINED ewts_VERSION AND NOT "${ewts_VERSION}" STREQUAL "") - set(NGEN_EWTS_VERSION "${ewts_VERSION}") +set(NGEN_EWTS_VERSION "${ewts_VERSION}") else() - set(NGEN_EWTS_VERSION "") +set(NGEN_EWTS_VERSION "") endif() if(DEFINED EWTS_NGWPC_VERSION AND NOT "${EWTS_NGWPC_VERSION}" STREQUAL "") - set(NGEN_EWTS_NGWPC_VERSION "${EWTS_NGWPC_VERSION}") +set(NGEN_EWTS_NGWPC_VERSION "${EWTS_NGWPC_VERSION}") else() - set(NGEN_EWTS_NGWPC_VERSION "") +set(NGEN_EWTS_NGWPC_VERSION "") endif() get_filename_component(EWTS_PREFIX "${ewts_DIR}" DIRECTORY) @@ -151,40 +154,42 @@ target_link_libraries(ngen PRIVATE ewts::ewts_ngen_bridge) # ----------------------------------------------------------------------------- # Check if NGen is the main project (e.g. development environment) if(PROJECT_NAME STREQUAL CMAKE_PROJECT_NAME) - set(NGEN_IS_MAIN_PROJECT ON) +set(NGEN_IS_MAIN_PROJECT ON) else() - set(NGEN_IS_MAIN_PROJECT OFF) +set(NGEN_IS_MAIN_PROJECT OFF) endif() # ----------------------------------------------------------------------------- # If Coverage is enabled, include coverage module and # set compiler flags/linkage to accomodate coverage. if(NGEN_WITH_COVERAGE) - include(CodeCoverage) - append_coverage_compiler_flags() +include(CodeCoverage) +append_coverage_compiler_flags() endif() # ----------------------------------------------------------------------------- # If MPI support is enabled, set the C++ compiler to "mpicxx" if(NGEN_WITH_MPI) - find_package(MPI REQUIRED) +find_package(MPI REQUIRED) +set(CMAKE_C_COMPILER ${MPI_C_COMPILER} CACHE FILEPATH "" FORCE) +set(CMAKE_CXX_COMPILER ${MPI_CXX_COMPILER} CACHE FILEPATH "" FORCE) endif() # ----------------------------------------------------------------------------- # TODO: Used for compatibility on Hera. Refactor this in the future. # https://www.intel.com/content/www/us/en/develop/documentation/oneapi-dpcpp-cpp-compiler-dev-guide-and-reference/top/compiler-setup/use-the-command-line/use-cmake-with-the-compiler.html if(INTEL_DPCPP) - find_package(IntelDPCPP REQUIRED) +find_package(IntelDPCPP REQUIRED) endif() # ----------------------------------------------------------------------------- # Account for different OS and how that impacts shared lib file names if(WIN32) - message(FATAL_ERROR "Windows platforms are not currently supported") +message(FATAL_ERROR "Windows platforms are not currently supported") elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin") - set(NGEN_SHARED_LIB_EXTENSION "dylib") +set(NGEN_SHARED_LIB_EXTENSION "dylib") else() - set(NGEN_SHARED_LIB_EXTENSION "so") +set(NGEN_SHARED_LIB_EXTENSION "so") endif() add_compile_definitions(NGEN_SHARED_LIB_EXTENSION) @@ -194,124 +199,106 @@ set(Boost_USE_STATIC_LIBS OFF) set(Boost_USE_MULTITHREADED ON) set(Boost_USE_STATIC_RUNTIME OFF) if(CMAKE_CXX_STANDARD LESS 17) - # requires non-header filesystem for state saving if C++ 11 or lower - find_package(Boost 1.86.0 REQUIRED COMPONENTS system filesystem) +# requires non-header filesystem for state saving if C++ 11 or lower +find_package(Boost 1.86.0 REQUIRED COMPONENTS system filesystem) else() - find_package(Boost 1.86.0 REQUIRED) +find_package(Boost 1.86.0 REQUIRED) endif() # ----------------------------------------------------------------------------- if(NGEN_WITH_SQLITE) - find_package(SQLite3 REQUIRED) - add_compile_definitions(NGEN_WITH_SQLITE3) +find_package(SQLite3 REQUIRED) +add_compile_definitions(NGEN_WITH_SQLITE3) endif() # ----------------------------------------------------------------------------- if(NGEN_WITH_UDUNITS) - find_package(UDUNITS2 REQUIRED) - add_compile_definitions(NGEN_WITH_UDUNITS) +find_package(UDUNITS2 REQUIRED) +add_compile_definitions(NGEN_WITH_UDUNITS) endif() # ----------------------------------------------------------------------------- -if(NGEN_WITH_NETCDF) - find_package(NetCDF COMPONENTS CXX) - - if(NOT NetCDF_CXX_FOUND) - find_package(NetCDF REQUIRED) - message(INFO " Using internal copy of netcdf-cxx4") - add_external_subdirectory( - SOURCE "${NGEN_EXT_DIR}/netcdf-cxx4" - OUTPUT "${PROJECT_BINARY_DIR}/extern-libs/netcdf-cxx4" - GIT_UPDATE "${NGEN_EXT_DIR}/netcdf-cxx4/netcdf-cxx4" - IMPORTS netcdf-cxx4 - ) - set(NetCDF_CXX_INCLUDE_DIR "${NGEN_EXT_DIR}/netcdf-cxx4/netcdf-cxx4/cxx4") - set(NetCDF_CXX_LIBRARY "${PROJECT_BINARY_DIR}/extern-libs/netcdf-cxx4/libnetcdf-cxx4.a") - target_link_libraries(NetCDF INTERFACE netcdf-cxx4) - endif() -endif() - if(NGEN_WITH_EXTERN_SLOTH) - # This is set because SLOTH will link to gtest by default, - # which causes a configure error. - set(PACKAGE_TESTS OFF CACHE BOOL "SLoTH testing") - add_external_subdirectory( - SOURCE "${NGEN_EXT_DIR}/sloth" - OUTPUT "${NGEN_EXT_DIR}/sloth/cmake_build" - GIT_UPDATE "${NGEN_EXT_DIR}/sloth" - IMPORTS slothmodel - ) - set_target_properties(slothmodel PROPERTIES CXX_VISIBILITY_PRESET default) +# This is set because SLOTH will link to gtest by default, +# which causes a configure error. +set(PACKAGE_TESTS OFF CACHE BOOL "SLoTH testing") +add_external_subdirectory( +SOURCE "${NGEN_EXT_DIR}/sloth" +OUTPUT "${NGEN_EXT_DIR}/sloth/cmake_build" +GIT_UPDATE "${NGEN_EXT_DIR}/sloth" +IMPORTS slothmodel +) +set_target_properties(slothmodel PROPERTIES CXX_VISIBILITY_PRESET default) endif() # ----------------------------------------------------------------------------- # Handle several steps for BMI C library logic and dependencies, at top level, if functionality is turned on if(NGEN_WITH_BMI_C) - if(NGEN_WITH_EXTERN_TOPMODEL) - add_external_subdirectory( - SOURCE "${NGEN_EXT_DIR}/topmodel" - OUTPUT "${NGEN_EXT_DIR}/topmodel/cmake_build" - GIT_UPDATE "${NGEN_EXT_DIR}/topmodel/topmodel" - IMPORTS topmodelbmi - ) - endif() +if(NGEN_WITH_EXTERN_TOPMODEL) +add_external_subdirectory( +SOURCE "${NGEN_EXT_DIR}/topmodel" +OUTPUT "${NGEN_EXT_DIR}/topmodel/cmake_build" +GIT_UPDATE "${NGEN_EXT_DIR}/topmodel/topmodel" +IMPORTS topmodelbmi +) + endif() - if(NGEN_WITH_EXTERN_CFE) - add_external_subdirectory( - SOURCE "${NGEN_EXT_DIR}/cfe" - OUTPUT "${NGEN_EXT_DIR}/cfe/cmake_build" - GIT_UPDATE "${NGEN_EXT_DIR}/cfe/cfe" - IMPORTS cfebmi - ) - endif() +if(NGEN_WITH_EXTERN_CFE) +add_external_subdirectory( +SOURCE "${NGEN_EXT_DIR}/cfe" +OUTPUT "${NGEN_EXT_DIR}/cfe/cmake_build" +GIT_UPDATE "${NGEN_EXT_DIR}/cfe/cfe" +IMPORTS cfebmi +) + endif() - if(NGEN_WITH_EXTERN_PET) - add_external_subdirectory( - SOURCE "${NGEN_EXT_DIR}/evapotranspiration/evapotranspiration" - OUTPUT "${NGEN_EXT_DIR}/evapotranspiration/evapotranspiration/cmake_build" - GIT_UPDATE "${NGEN_EXT_DIR}/evapotranspiration/evapotranspiration" - IMPORTS petbmi - ) - endif() +if(NGEN_WITH_EXTERN_PET) +add_external_subdirectory( +SOURCE "${NGEN_EXT_DIR}/evapotranspiration/evapotranspiration" +OUTPUT "${NGEN_EXT_DIR}/evapotranspiration/evapotranspiration/cmake_build" +GIT_UPDATE "${NGEN_EXT_DIR}/evapotranspiration/evapotranspiration" +IMPORTS petbmi +) + endif() endif() # ----------------------------------------------------------------------------- # Configure whether Fortran BMI functionality is active if(NGEN_WITH_BMI_FORTRAN) - add_external_subdirectory( - SOURCE "${NGEN_EXT_DIR}/iso_c_fortran_bmi" - OUTPUT "${NGEN_EXT_DIR}/iso_c_fortran_bmi/cmake_build" - IMPORTS iso_c_bmi - ) +add_external_subdirectory( +SOURCE "${NGEN_EXT_DIR}/iso_c_fortran_bmi" +OUTPUT "${NGEN_EXT_DIR}/iso_c_fortran_bmi/cmake_build" +IMPORTS iso_c_bmi +) - if(NGEN_WITH_EXTERN_NOAH_OWP_MODULAR) - add_external_subdirectory( - SOURCE "${NGEN_EXT_DIR}/noah-owp-modular" - OUTPUT "${NGEN_EXT_DIR}/noah-owp-modular/cmake_build" - GIT_UPDATE "${NGEN_EXT_DIR}/noah-owp-modular/noah-owp-modular" - IMPORTS surfacebmi - ) - endif() +if(NGEN_WITH_EXTERN_NOAH_OWP_MODULAR) +add_external_subdirectory( +SOURCE "${NGEN_EXT_DIR}/noah-owp-modular" +OUTPUT "${NGEN_EXT_DIR}/noah-owp-modular/cmake_build" +GIT_UPDATE "${NGEN_EXT_DIR}/noah-owp-modular/noah-owp-modular" +IMPORTS surfacebmi +) + endif() endif() # ----------------------------------------------------------------------------- if(NGEN_WITH_PYTHON) - find_package(Python 3.6.8 REQUIRED COMPONENTS Interpreter Development NumPy) - if(${Python_NumPy_VERSION} VERSION_GREATER_EQUAL 2) - message(FATAL_ERROR "Found numpy version \"${Python_NumPy_VERSION}\"; numpy>=2.0.0 is not currently supported") - endif() - set(PYTHON_EXECUTABLE ${Python_EXECUTABLE}) # Case-sensitive difference - add_subdirectory(extern/pybind11 pybind11) +find_package(Python 3.6.8 REQUIRED COMPONENTS Interpreter Development NumPy) +if(${Python_NumPy_VERSION} VERSION_GREATER_EQUAL 2) +message(FATAL_ERROR "Found numpy version \"${Python_NumPy_VERSION}\"; numpy>=2.0.0 is not currently supported") + endif() +set(PYTHON_EXECUTABLE ${Python_EXECUTABLE}) # Case-sensitive difference +add_subdirectory(extern/pybind11 pybind11) endif() # ----------------------------------------------------------------------------- if(NGEN_QUIET) - set(UDUNITS_QUIET true) - add_compile_definitions(NGEN_QUIET) +set(UDUNITS_QUIET true) +add_compile_definitions(NGEN_QUIET) endif() if(UDUNITS_QUIET) - add_compile_definitions(UDUNITS_QUIET) +add_compile_definitions(UDUNITS_QUIET) endif() # ----------------------------------------------------------------------------- @@ -324,23 +311,23 @@ add_library(NGen::config_header ALIAS ngen_config_header) target_include_directories(ngen_config_header INTERFACE "${CMAKE_CURRENT_BINARY_DIR}/include") if(NGEN_WITH_EXTERN_SLOTH) - add_dependencies(ngen slothmodel) +add_dependencies(ngen slothmodel) endif() if(NGEN_WITH_EXTERN_TOPMODEL) - add_dependencies(ngen topmodelbmi) +add_dependencies(ngen topmodelbmi) endif() if(NGEN_WITH_EXTERN_CFE) - add_dependencies(ngen cfebmi) +add_dependencies(ngen cfebmi) endif() if(NGEN_WITH_EXTERN_PET) - add_dependencies(ngen petbmi) +add_dependencies(ngen petbmi) endif() if(NGEN_WITH_EXTERN_NOAH_OWP_MODULAR) - add_dependencies(ngen surfacebmi) +add_dependencies(ngen surfacebmi) endif() target_include_directories(ngen PUBLIC "${NGEN_INC_DIR}") @@ -359,29 +346,29 @@ add_subdirectory("src/utilities/python") add_subdirectory("src/utilities/bmi") target_link_libraries(ngen - PUBLIC - NGen::config_header - NGen::core - NGen::core_catchment - NGen::core_nexus - NGen::geojson - NGen::realizations_catchment - NGen::forcing - NGen::core_mediator - NGen::logging - NGen::parallel - NGen::state_save_restore - NGen::bmi_protocols +PUBLIC +NGen::config_header +NGen::core +NGen::core_catchment +NGen::core_nexus +NGen::geojson +NGen::realizations_catchment +NGen::forcing +NGen::core_mediator +NGen::logging +NGen::parallel +NGen::state_save_restore +NGen::bmi_protocols ) if(NGEN_WITH_SQLITE) - add_subdirectory("src/geopackage") - target_link_libraries(ngen PUBLIC NGen::geopackage) +add_subdirectory("src/geopackage") +target_link_libraries(ngen PUBLIC NGen::geopackage) endif() if(NGEN_WITH_ROUTING) - add_subdirectory("src/routing") - target_link_libraries(ngen PUBLIC NGen::routing) +add_subdirectory("src/routing") +target_link_libraries(ngen PUBLIC NGen::routing) endif() add_executable(partitionGenerator src/partitionGenerator.cpp) @@ -389,19 +376,84 @@ target_link_libraries(partitionGenerator PRIVATE ewts::ewts_ngen_bridge) target_link_libraries(partitionGenerator PUBLIC NGen::logging) target_include_directories(partitionGenerator PUBLIC "${PROJECT_BINARY_DIR}/include") if(NGEN_WITH_SQLITE) - target_include_directories(partitionGenerator PUBLIC AFTER "${NGEN_INC_DIR}/geopackage") +target_include_directories(partitionGenerator PUBLIC AFTER "${NGEN_INC_DIR}/geopackage") +endif() + +if(NGEN_WITH_NETCDF) + if(NGEN_WITH_MPI) + set(HDF5_INSTALL "$ENV{HOME}/MyInstalls/hdf5_mpi_parallel" CACHE PATH "Path to Parallel HDF5 installation") + set(NETCDF_INSTALL "$ENV{HOME}/MyInstalls/netcdf_mpi_parallel" CACHE PATH "Path to Parallel NetCDF installation") + set(NETCDF_LIB_DIR "$ENV{HOME}/MyInstalls/netcdf_mpi_parallel/lib") + set(HDF5_LIB_DIR "$ENV{HOME}/MyInstalls/hdf5_mpi_parallel/lib") + else() + set(HDF5_INSTALL "$ENV{HOME}/MyInstalls/hdf5_serial" CACHE PATH "Path to Serial HDF5 installation") + set(NETCDF_INSTALL "$ENV{HOME}/MyInstalls/netcdf_serial" CACHE PATH "Path to Serial NetCDF installation") + set(NETCDF_LIB_DIR "$ENV{HOME}/MyInstalls/netcdf_serial/lib") + set(HDF5_LIB_DIR "$ENV{HOME}/MyInstalls/hdf5_serial/lib") + endif() + set(WRAPPER_SOURCES + ${NGEN_SRC_DIR}/netcdf/NetCDFManager.cpp + ${NGEN_SRC_DIR}/netcdf/NetCDFFile.cpp + ${NGEN_SRC_DIR}/netcdf/NetCDFVar.cpp + ) + add_library(NetCDFManager STATIC ${WRAPPER_SOURCES}) + set_target_properties(NetCDFManager PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) + target_include_directories(NetCDFManager PUBLIC + ${EWTS_PREFIX}/include + ${NGEN_INC_DIR} + ${NGEN_INC_DIR}/netcdf + ${NGEN_INC_DIR}/core + ${NGEN_INC_DIR}/core/catchment + ${NGEN_INC_DIR}/realizations/catchment + ${NGEN_INC_DIR}/simulation_time + ${NGEN_INC_DIR}/utilities + ${NGEN_INC_DIR}/geojson + ${NGEN_INC_DIR}/forcing + ${NGEN_INC_DIR}/core/mediator + ${Python_INCLUDE_DIRS} + ${pybind11_INCLUDE_DIR} + ${NETCDF_INSTALL}/include + ${HDF5_INSTALL}/include + $<$:${MPI_CXX_INCLUDE_DIRS}> + ) + target_link_libraries(NetCDFManager PUBLIC NGen::config_header) + # the order for the custom netcdf and hdf5 libraries is important. + target_link_libraries(NetCDFManager PRIVATE + ${NETCDF_INSTALL}/lib/libnetcdf.so + ${HDF5_INSTALL}/lib/libhdf5_hl.so + ${HDF5_INSTALL}/lib/libhdf5.so + $<$:MPI::MPI_CXX> + ) + target_compile_definitions(NetCDFManager PUBLIC NGEN_WITH_NETCDF) + if(NGEN_WITH_MPI) + target_compile_definitions(NetCDFManager PUBLIC NGEN_WITH_MPI) + endif() + target_link_libraries(ngen PRIVATE NetCDFManager) + # set target properties to the actual path so that it is propagated during runtime. + set_target_properties(ngen PROPERTIES + BUILD_WITH_INSTALL_RPATH FALSE + SKIP_BUILD_RPATH FALSE + BUILD_RPATH "${NETCDF_LIB_DIR};${HDF5_LIB_DIR}" + ) + # The compiler automatically converts classic RPATH entries into modern RUNPATH entries. + # By adding the -Wl,--disable-new-dtags flag + # we force the system to use traditional RPATH rules instead of RUNPATH. + target_link_options(ngen PRIVATE + "-Wl,-rpath,${NETCDF_LIB_DIR}:${HDF5_LIB_DIR}" + "-Wl,--disable-new-dtags" + ) endif() target_link_libraries(partitionGenerator PUBLIC NGen::core NGen::geojson) if(NGEN_WITH_SQLITE) - target_link_libraries(partitionGenerator PUBLIC NGen::geopackage) +target_link_libraries(partitionGenerator PUBLIC NGen::geopackage) endif() # For automated testing with Google Test if(NGEN_WITH_TESTS) - include(CTest) # calls enable_testing() - include(GoogleTest) - add_subdirectory(test) +include(CTest) # calls enable_testing() +include(GoogleTest) +add_subdirectory(test) endif() # ----------------------------------------------------------------------------- @@ -412,27 +464,27 @@ endif() macro(ngen_multiline_message) set(messages "${ARGN}") foreach(msg IN LISTS messages) - set(_CONF "${NGEN_CONF_SUMMARY}") - string(APPEND _CONF "${msg}\n") - set(NGEN_CONF_SUMMARY "${_CONF}") +set(_CONF "${NGEN_CONF_SUMMARY}") +string(APPEND _CONF "${msg}\n") +set(NGEN_CONF_SUMMARY "${_CONF}") - message(STATUS "${msg}") +message(STATUS "${msg}") endforeach() endmacro() # Syntax: ngen_dependent_multiline_message( "" "" ...) macro(ngen_dependent_multiline_message VARIABLE) if(${VARIABLE}) - ngen_multiline_message(${ARGN}) +ngen_multiline_message(${ARGN}) endif() endmacro() # This is used purely for configuration output, # and does not affect the virtual environment used. if(DEFINED ENV{VIRTUAL_ENV}) - set(NGEN_CONFIGURED_VENV "$ENV{VIRTUAL_ENV}") +set(NGEN_CONFIGURED_VENV "$ENV{VIRTUAL_ENV}") else() - set(NGEN_CONFIGURED_VENV "") +set(NGEN_CONFIGURED_VENV "") endif() # MPI CXX libraries output both C and CXX libs @@ -536,4 +588,4 @@ configure_file("${NGEN_INC_DIR}/NGenConfig.h.in" "${CMAKE_CURRENT_BINARY_DIR}/in include(GNUInstallDirs) install(TARGETS ngen OPTIONAL) -install(TARGETS partitionGenerator OPTIONAL) +install(TARGETS partitionGenerator OPTIONAL) \ No newline at end of file diff --git a/cmake/FindNetCDF.cmake b/cmake/FindNetCDF.cmake.bak similarity index 100% rename from cmake/FindNetCDF.cmake rename to cmake/FindNetCDF.cmake.bak diff --git a/extern/LASAM b/extern/LASAM index 0fcbf84200..764dc82e8f 160000 --- a/extern/LASAM +++ b/extern/LASAM @@ -1 +1 @@ -Subproject commit 0fcbf842009bd7bd8929e2c83aba4fa3b97f4de6 +Subproject commit 764dc82e8fb5a160e646a8f0fde35f964fd42234 diff --git a/extern/SoilFreezeThaw/SoilFreezeThaw b/extern/SoilFreezeThaw/SoilFreezeThaw index c45d48f5f4..815a970af3 160000 --- a/extern/SoilFreezeThaw/SoilFreezeThaw +++ b/extern/SoilFreezeThaw/SoilFreezeThaw @@ -1 +1 @@ -Subproject commit c45d48f5f402dd65e8da0efa31b586ed5faf33c5 +Subproject commit 815a970af30e407785bc767a4a5f5dffa2708eb5 diff --git a/extern/SoilMoistureProfiles/SoilMoistureProfiles b/extern/SoilMoistureProfiles/SoilMoistureProfiles index 3118a7b9c2..d29f268223 160000 --- a/extern/SoilMoistureProfiles/SoilMoistureProfiles +++ b/extern/SoilMoistureProfiles/SoilMoistureProfiles @@ -1 +1 @@ -Subproject commit 3118a7b9c2098168a123dd65e58f377f84218c95 +Subproject commit d29f26822391740549642a9818065ea296688b2e diff --git a/extern/lstm b/extern/lstm index b3ccaeb458..f2b2d3c33a 160000 --- a/extern/lstm +++ b/extern/lstm @@ -1 +1 @@ -Subproject commit b3ccaeb45897693d7ebb05dc38788a5766a6921e +Subproject commit f2b2d3c33abe19faac8d95a0e09765221f0676ee diff --git a/extern/sac-sma/sac-sma b/extern/sac-sma/sac-sma index 4aa0d804f7..a5afd2ad0b 160000 --- a/extern/sac-sma/sac-sma +++ b/extern/sac-sma/sac-sma @@ -1 +1 @@ -Subproject commit 4aa0d804f707f324d5b17143f350e24a52e6e2b5 +Subproject commit a5afd2ad0b48fcdecd960dff3fc20c362ed7d8ad diff --git a/extern/snow17 b/extern/snow17 index f224c0aaea..8d10da02a0 160000 --- a/extern/snow17 +++ b/extern/snow17 @@ -1 +1 @@ -Subproject commit f224c0aaeaebc6980ea403fe72093219b6d0019c +Subproject commit 8d10da02a0fa0155d962b65068775990be71d56b diff --git a/extern/t-route b/extern/t-route index 6c16005256..625eb805e7 160000 --- a/extern/t-route +++ b/extern/t-route @@ -1 +1 @@ -Subproject commit 6c160052568f9871dc7f1dac06446dc427a49047 +Subproject commit 625eb805e74b8fbaae749d6bd8393f321eeac5e1 diff --git a/extern/ueb-bmi b/extern/ueb-bmi index d18ddb425d..26f76de81c 160000 --- a/extern/ueb-bmi +++ b/extern/ueb-bmi @@ -1 +1 @@ -Subproject commit d18ddb425d456c37fc049d909da9752b66db7695 +Subproject commit 26f76de81c34cb27da41d33f9131ad26da45db65 diff --git a/include/core/Layer.hpp b/include/core/Layer.hpp index 643f1b6970..1f230dd61a 100644 --- a/include/core/Layer.hpp +++ b/include/core/Layer.hpp @@ -9,6 +9,7 @@ #include "State_Exception.hpp" #include "geojson/FeatureBuilder.hpp" #include +#include namespace hy_features { @@ -116,6 +117,7 @@ namespace ngen virtual void save_state_snapshot(std::shared_ptr snapshot_saver); virtual void load_state_snapshot(std::shared_ptr snapshot_loader); virtual void load_hot_start(std::shared_ptr snapshot_loader); + std::map get_catchment_output_data_for_timestep(); protected: @@ -128,6 +130,7 @@ namespace ngen //TODO is this really required at the top level? or can this be moved to SurfaceLayer? const geojson::GeoJSON catchment_data; long output_time_index; + std::map catchment_output_values; }; } diff --git a/include/core/NetCDFCreator.hpp b/include/core/NetCDFCreator.hpp new file mode 100644 index 0000000000..e5a1fe425a --- /dev/null +++ b/include/core/NetCDFCreator.hpp @@ -0,0 +1,47 @@ +#ifndef NGEN_NETCDF_CREATOR_HPP +#define NGEN_NETCDF_CREATOR_HPP + +#include + +#if NGEN_WITH_NETCDF +#include +#include +#include +#include + +namespace netCDF { + class NcVar; + class NcFile; +} +class NetCDFCreator +{ +public: + NetCDFCreator(std::shared_ptr manager, + const std::string& output_name, Simulation_Time const& sim_time, int mpi_rank, int mpi_num_procs); + NetCDFCreator() = delete; + ~NetCDFCreator(); + + void write_simulations_response_from_formulation(size_t time_index, std::map catchment_output_values); + + netCDF::NcFile& get_ncfile(); + +protected: + void add_output_variable_info_from_formulation(); + + void retrieve_output_variables_mpi(); + + std::vector string_split(std::string str, char delimiter); + + bool create_ncfile(); + + void close_ncfile(); + +private: + std::shared_ptr catchmentNcFile; + std::shared_ptr manager_; + std::shared_ptr sim_time_; + std::vector catchments; + std::vector nc_output_variables; +}; +#endif // NGEN_WITH_NETCDF +#endif // NGEN_NETCDF_CREATOR_HPP \ No newline at end of file diff --git a/include/core/NgenSimulation.hpp b/include/core/NgenSimulation.hpp index 62744e4588..c29f21426c 100644 --- a/include/core/NgenSimulation.hpp +++ b/include/core/NgenSimulation.hpp @@ -1,20 +1,32 @@ #ifndef NGENSIMULATION_HPP #define NGENSIMULATION_HPP -#include +#if NGEN_WITH_MPI + #include +#endif +#include #include +#include #include +#if NGEN_WITH_NETCDF + #include "netcdf/NetCDFManager.hpp" +#endif + + namespace hy_features { class HY_Features; class HY_Features_MPI; } - class State_Snapshot_Saver; class State_Snapshot_Loader; +// #if NGEN_WITH_NETCDF +// class NetCDFManager; +// #endif + #if NGEN_WITH_ROUTING #include "bmi/Bmi_Py_Adapter.hpp" #endif // NGEN_WITH_ROUTING @@ -79,6 +91,7 @@ class NgenSimulation void save_end_of_run(std::shared_ptr snapshot_saver); // Load a snapshot of the end of a previous run. This will create a T-Route python adapter if the loader finds a unit for it and the config path is not empty. void load_hot_start(std::shared_ptr snapshot_loader, const std::string &t_route_config_file_with_path); + void create_netcdf_writer(std::shared_ptr manager, std::string nc_output_file_name, int mpi_rank, int mpi_num_procs); private: void advance_models_one_output_step(); @@ -125,6 +138,16 @@ class NgenSimulation // Serialization template will be defined and instantiated in the .cpp file template void serialize(Archive& ar); + + //Pointer to netcdfcreator to write simulation output per timestep. + //std::unique_ptr nc_writer_; +#if NGEN_WITH_MPI + MPI_Comm mpi_comm_; +#endif + +#if NGEN_WITH_NETCDF + std::unique_ptr nc_manager_; +#endif }; #endif diff --git a/include/forcing/NetCDFPerFeatureDataProvider.hpp b/include/forcing/NetCDFPerFeatureDataProvider.hpp index 5719bc32e2..c9a1311c33 100644 --- a/include/forcing/NetCDFPerFeatureDataProvider.hpp +++ b/include/forcing/NetCDFPerFeatureDataProvider.hpp @@ -7,7 +7,7 @@ #include "GenericDataProvider.hpp" #include "DataProviderSelectors.hpp" - +//#include #include #include #include @@ -25,10 +25,10 @@ #include "AorcForcing.hpp" -namespace netCDF { - class NcVar; - class NcFile; -} +// Forward declarations +class NetCDFManager; +class NetCDFFile; +class NetCDFVar; namespace data_access { @@ -132,15 +132,15 @@ namespace data_access utils::StreamHandler log_stream; std::string file_path; - std::shared_ptr nc_file; + std::shared_ptr nc_manager; - std::map ncvar_cache; + std::map> ncvar_cache; std::map units_cache; boost::compute::detail::lru_cache>> value_cache; size_t cache_slice_t_size = 1; size_t cache_slice_c_size = 1; - const netCDF::NcVar& get_ncvar(const std::string& name); + std::shared_ptr get_ncvar(const std::string& name) const; const std::string& get_ncvar_units(const std::string& name) const; diff --git a/include/netcdf/NetCDFFile.hpp b/include/netcdf/NetCDFFile.hpp new file mode 100644 index 0000000000..68f7301823 --- /dev/null +++ b/include/netcdf/NetCDFFile.hpp @@ -0,0 +1,87 @@ +#ifndef NetCDFFILE_HPP +#define NetCDFFILE_HPP + +//#include + +#if NGEN_WITH_NETCDF +#include +#include +#include +#include +#if NGEN_WITH_MPI +#include +#define _PARALLEL4 +#include +#else +#include +#endif +#include "NetCDFVar.hpp" + +class NetCDFFile { +public: + + NetCDFFile(const std::string& filename, bool write_only, bool is_mpi); + + ~NetCDFFile(); + void load_variables(); + std::vector list_variables() const; + std::shared_ptr get_ncvar(const std::string& name) const; + int get_ncid() const { return ncid_; } + + // Dimension handling + int add_dimension(const std::string& name, size_t len); + size_t get_dim_size(const std::string& name) const; + int get_dim_id(const std::string& name) const; + + // Variable handling + int get_next_varid() { return next_varid_++; } + void add_ncvar(std::shared_ptr var); + + // Add a variable to the file + void add_variable(const std::string& name, nc_type type, const std::vector& dims, const std::vector& dim_names); + + // Write data to a variable + template + void write_variable_data(const std::string& name, const std::vector& data, size_t start_index = 0); + + int write_data_to_ncvar(int ncid, int varid, const std::vector& start, + const std::vector& count, const std::vector& data); + + int write_data_to_ncvar(int ncid, int varid, const std::vector& start, + const std::vector& count, const std::vector& data); + + int write_data_to_ncvar(int ncid, int varid, const std::vector& start, + const std::vector& count, const std::vector& data); + + void write_catchment_output_data(const std::string& name, std::vector start, + std::vector count, const double& data); + + //Add attribute to a variable + void write_attribute_to_ncvar(const std::string& name, const std::string& attName, const std::string& attValue); + + void close_file(); + + void end_def_mode(); + +private: + std::string nc_file_name_; + bool read_only_; + int ncid_; + int next_varid_ = 0; + bool is_mpi_; + std::vector> variables_; + std::map dims_; + std::map> variables_map_; + + // Compute chunk offsets/counts for first dimension + // void computeStartCount(size_t total_size, size_t& start, size_t& count); +#if NGEN_WITH_MPI + MPI_Comm comm_; +#endif + bool parallel_; + + // Helper to get variable index by name + std::shared_ptr get_ncvar_by_name(const std::string& name) const; +}; +#endif // NGEN_WITH_NETCDF +#endif // NETCDFFILE_HPP \ No newline at end of file diff --git a/include/netcdf/NetCDFManager.hpp b/include/netcdf/NetCDFManager.hpp new file mode 100644 index 0000000000..313d4941f0 --- /dev/null +++ b/include/netcdf/NetCDFManager.hpp @@ -0,0 +1,126 @@ +#ifndef NETCDFMANAGER_HPP +#define NETCDFMANAGER_HPP + +//#include + + +#if NGEN_WITH_NETCDF +#if NGEN_WITH_MPI + #include + #define _PARALLEL4 +#endif +#include "NetCDFFile.hpp" +#include "NetCDFVar.hpp" +#include +#include +#include +#include +//#include "realizations/catchment/Formulation_Manager.hpp" +//#include "simulation_time/Simulation_Time.hpp" +//#include "Catchment_Formulation.hpp" + +namespace realization { + class Formulation_Manager; + class Catchment_Formulation; +} +class Simulation_Time; + +class NetCDFManager +{ +public: + NetCDFManager(std::shared_ptr manager, + const std::string& output_name, Simulation_Time const& sim_time, int mpi_rank, int mpi_num_procs); + + //NetCDFManager(const std::string& output_name, int mpi_rank, int mpi_num_procs); + + // Constructor for read-only NetCDF (no MPI needed) + NetCDFManager(const std::string& filename, bool read_only); + + // Constructor for writing (can be parallel if size>1) + // NetCDFManager(const std::string& filename, int rank, int size); + + // Default constructor for mdframe + NetCDFManager(); + + // File operations + int create_file(const std::string& filename); + void open_file(); + void close_file(); + + //set up netcdf dimensions and variables + void define_catchment_netcdf_components(); + + // List variable names + std::vector list_variables() const; + + //Get NetCDFFile handle + NetCDFFile* get_file_handle() {return nc_file_.get();} + + // Access NetCDFVar by name + std::shared_ptr get_ncvar_by_name(const std::string& name) const; + + // Attribute access + std::vector list_attributes(const std::string& var_name) const; + std::string get_string_attribute(const std::string& var_name, const std::string& att_name) const; + int get_int_attribute(const std::string& var_name, const std::string& att_name) const; + double get_double_attribute(const std::string& var_name, const std::string& att_name) const; + + // Add a dimension + int add_dimension(const std::string& name, size_t len); + + // Add a variable + void add_variable(const std::string& var_name, nc_type type, const std::vector& dims, const std::vector& dim_names); + + // Add variables to the file (for writing) + void add_output_variable_data_from_formulation(); + + // Add catchment output data to the file (for writing) + void write_simulations_response_from_formulation(size_t time_index, std::map catchment_output_values); + + // Create NetCDF file with time and entity dimensions, and multiple variables + void create_timeslice(size_t num_entities, size_t num_timesteps, + const std::vector& var_names); + + // Write a block of timesteps for all entities for a variable + template + void write_timeslice(const std::string& var_name, size_t time_start, + size_t time_count, const std::vector& data); + + // Get the time range assigned to this rank + void get_local_time_range(size_t& start, size_t& count) const; + + size_t get_chunk_start() const {return chunk_start_; } + + size_t get_chunk_count() const {return chunk_count_; } + + void prepare_data_chunks(std::map catchment_output_vals); + + void write_timestep_data_to_netcdf(size_t time_index); + + ~NetCDFManager(); + +private: + bool read_only_; + std::string nc_filename_; + std::unique_ptr nc_file_; + std::vector vars_; + std::shared_ptr manager_; + std::shared_ptr sim_time_; + size_t chunk_start_ = 0; + size_t chunk_count_ = 0; + size_t num_entities_; + size_t num_timesteps_; + std::vector catchments_; + std::map> variables_map_; + std::vector nc_output_variables_; + std::vector> data_chunks_; + +#if NGEN_WITH_MPI + MPI_Comm comm_; +#endif + int rank_ = 0; + int num_procs_ = 1; + bool is_mpi_ = false; +}; +#endif // NGEN_WITH_NETCDF +#endif // NETCDFMANAGER_HPP \ No newline at end of file diff --git a/include/netcdf/NetCDFVar.hpp b/include/netcdf/NetCDFVar.hpp new file mode 100644 index 0000000000..e07823dd22 --- /dev/null +++ b/include/netcdf/NetCDFVar.hpp @@ -0,0 +1,93 @@ +#ifndef NETCDFVAR_HPP +#define NETCDFVAR_HPP + +//#include + +#if NGEN_WITH_NETCDF +#include +#include +#include +#include +#include + +#if NGEN_WITH_MPI +#include +#include +#else +#include +#endif + +class NetCDFVar { +public: + NetCDFVar(const std::string& name, nc_type type, const std::vector& dims, + const std::vector& dim_names, int varid, int ncid); + + const std::string& get_name() const; + nc_type get_type() const; + const std::vector& get_dims() const; + const std::vector& get_dim_names() const; + size_t get_dim_size(const std::string& dim_name) const; + size_t get_dim_size(size_t idx) const; + size_t get_dim_count() const; + int get_varid() const; + size_t get_total_size() const; + std::vector get_string_array_values() const; + std::vector get_time_values() const; + int get_int_value_at_index(const std::vector& index) const; + double get_dbl_value_at_index(const std::vector& index) const; + std::string get_str_value_at_index(const std::vector& index) const; + void read_slice(const std::vector& start, const std::vector& count, double* data) const; + + void add_attribute(const std::string& att_name, const std::string& att_value); + void add_attribute(const std::string& att_name, int att_value); + void add_attribute(const std::string& att_name, double att_value); + void build_variables_index(size_t num_items); + + std::vector list_attributes() const; + std::string get_string_attribute(const std::string& att_name) const; + int get_int_attribute(const std::string& att_name) const; + double get_double_attribute(const std::string& att_name) const; + size_t get_variable_index(const std::string& name) const; + + void write_timesliced_data(size_t timestep, size_t slice_start, size_t slice_count, const double* data); + + //Implementing this single-element write function for mdframe_netcdf_test + template + void write_single_value(const std::vector& block_array, const T& value) const + { + std::vector start(block_array.begin(), block_array.end()); + std::vector count(block_array.size(), 1); + + int retval = 0; + + if (std::is_same::value) { + retval = nc_put_vara_int(ncid_, varid_, start.data(), count.data(), &value); + } + else if (std::is_same::value) { + retval = nc_put_vara_float(ncid_, varid_, start.data(), count.data(), &value); + } + else if (std::is_same::value) { + retval = nc_put_vara_double(ncid_, varid_, start.data(), count.data(), &value); + } + else { + throw std::runtime_error("Unsupported type for write_single_value"); + } + + if (retval != NC_NOERR) throw std::runtime_error(std::string("Erorr in writing single value: ") + nc_strerror(retval)); + } + + private: + std::string name_; + nc_type type_; + std::vector dims_; + std::vector dim_names_; + int varid_; + int ncid_; + + std::map attributes_str_; + std::map attributes_int_; + std::map attributes_double_; + std::unordered_map variable_index_; +}; +#endif // NGEN_WITH_NETCDF +#endif // NETCDFVAR_HPP \ No newline at end of file diff --git a/include/realizations/catchment/Bmi_Formulation.hpp b/include/realizations/catchment/Bmi_Formulation.hpp index 91236fea42..78cae3ee88 100644 --- a/include/realizations/catchment/Bmi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Formulation.hpp @@ -207,6 +207,15 @@ namespace realization { return REQUIRED_PARAMETERS; } + /** + * Get the units for the output variables organized as a vector of strings. + * + * @return The units for the output variables organized as a vector. + */ + const std::vector &get_output_variable_units() const { + return output_variable_units; + } + virtual bool is_bmi_input_variable(const std::string &var_name) const = 0; /** @@ -265,6 +274,10 @@ namespace realization { output_header_fields = output_headers; } + void set_output_variable_units(const std::vector &output_units) { + output_variable_units = output_units; + } + /** * Set the names of variables in formulation output. * @@ -291,6 +304,11 @@ namespace realization { * the BMI module output variables accessible to the instance. */ std::vector output_variable_names; + /** + * Output units corresponding to the variables output by the realization, as defined in + * `output_variables`. + */ + std::vector output_variable_units; /** The degree of precision in output values when converting to text. */ int output_precision; diff --git a/include/realizations/catchment/Formulation_Manager.hpp b/include/realizations/catchment/Formulation_Manager.hpp index 746129b90e..fe7c02cdfc 100644 --- a/include/realizations/catchment/Formulation_Manager.hpp +++ b/include/realizations/catchment/Formulation_Manager.hpp @@ -237,6 +237,10 @@ namespace realization { return this->formulations.at(id); } + std::map> get_all_formulations() const { + return this->formulations; + } + std::shared_ptr get_domain_formulation(long id) const { return this->domain_formulations.at(id); } diff --git a/src/NGen.cpp b/src/NGen.cpp index 24dac00d07..4a44d3b869 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -11,6 +11,10 @@ #include #include +#if NGEN_WITH_NETCDF +#include +#endif + #if NGEN_WITH_SQLITE3 #include #endif @@ -487,7 +491,7 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { } else { catchment_collection = geojson::read(catchmentDataFile, catchment_subset_ids); } - + std::cout << catchment_collection << std::endl; for (auto& feature : *catchment_collection) { // feature->set_id(feature->get_property("id").as_string()); nexus_collection->add_feature(feature); @@ -689,7 +693,13 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { std::move(nexus_indexes), mpi_rank, mpi_num_procs); - + #if NGEN_WITH_NETCDF + #if NGEN_WITH_MPI + LOG("Under MPI mode, a NetCDF file for catchment output values is not created due to limitations in NetCDFCxx4 library.", LogLevel::INFO); + #else + simulation->create_netcdf_writer(manager, "catchment_output", mpi_rank, mpi_num_procs); //create a NetCDF file only for Non-MPI run. + #endif //NGEN_WITH_MPI + #endif //NGEN_WITH_NETCDF auto time_done_init = std::chrono::steady_clock::now(); std::chrono::duration time_elapsed_init = time_done_init - time_start; LOG("[TIMING]: Init: " + std::to_string(time_elapsed_init.count()), LogLevel::INFO); @@ -703,7 +713,9 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { } } + simulation->run_catchments(); + #if NGEN_WITH_MPI MPI_Barrier(MPI_COMM_WORLD); diff --git a/src/NetCDFCreator.cpp b/src/NetCDFCreator.cpp new file mode 100644 index 0000000000..3dd8d7b0e2 --- /dev/null +++ b/src/NetCDFCreator.cpp @@ -0,0 +1,208 @@ +#include + +#if NGEN_WITH_NETCDF +#include +#include +#include "ewts_ngen/logger.hpp" + +#if NGEN_WITH_MPI +#include +#include "parallel_utils.h" +#endif + +NetCDFCreator::NetCDFCreator(std::shared_ptr manager, + const std::string& output_name,Simulation_Time const& sim_time, int mpi_rank, int mpi_num_procs) +{ + manager_ = manager; + sim_time_ = std::make_shared(sim_time); + + //Check if there is a need to create a netcdf file (whether output variables are mentioned in realization) + if (create_ncfile()){ + try{ + catchments.reserve(manager_->get_size()); + for (auto const& formulation_info : manager->get_all_formulations()) + { + std::string catchm = formulation_info.first; + catchments.push_back(catchm); + } + + #if NGEN_WITH_MPI + if (mpi_num_procs > 1){ + std::vector all_catchments = catchments; + all_catchments = parallel::gather_strings(catchments, mpi_rank, mpi_num_procs); + if (mpi_rank == 0){ + std::sort(all_catchments.begin(), all_catchments.end()); + all_catchments.erase( + std::unique(all_catchments.begin(), all_catchments.end()), + all_catchments.end() + ); + } + catchments = parallel::broadcast_strings(all_catchments, mpi_rank, mpi_num_procs); + } + #endif + std::string ncOutputFileName = manager->get_output_root() + output_name + ".nc"; + if(mpi_rank == 0){ + catchmentNcFile = std::make_shared(ncOutputFileName, netCDF::NcFile::replace); + if(!catchmentNcFile){ + LOG("Catchment output netcdf file creation failed: " + ncOutputFileName, LogLevel::FATAL); + throw std::runtime_error("Catchment output netcdf file creation failed: " + ncOutputFileName); + } + //Add dimension and coordinate variable for time + //TO DO: create a separate function if this action is initiated from outside the class. + //TO DO: convert seconds epoch to minutes or days? + int num_timesteps = sim_time_->get_total_output_times(); + auto time_dim = catchmentNcFile->addDim("time", num_timesteps); + auto time_var = catchmentNcFile->addVar("time", NC_INT, time_dim); + time_var.putAtt("units", "Seconds since 1970-01-01 00:00:00"); + time_var.putAtt("calendar", "gregorian"); + std::vector time_epoch_seconds(num_timesteps); + time_epoch_seconds[0] = sim_time_->get_current_epoch_time(); + for(int time_index = 1; time_index < num_timesteps; time_index++) + { + sim_time_->advance_timestep(); + time_epoch_seconds[time_index] = sim_time_->get_current_epoch_time(); + } + time_var.putVar(time_epoch_seconds.data()); + + //Add dimension and coordinate variable for catchments + //TO DO: create a separate function if this action is initiated from outside the class. + auto catchments_dim = catchmentNcFile->addDim("catchments", catchments.size()); + auto catchments_var = catchmentNcFile->addVar("catchments", NC_STRING, catchments_dim); + catchments_var.putAtt("Catchment ID", "Catchment identifier in input"); + int item_index = 0; + std::vector index; + index.resize(1); + for (auto const& catchment : catchments) + { + index[0] = item_index; + catchments_var.putVar(index,catchment); + item_index++; + } + + //Add output data variables information such as headers, variable names, units to netcdf + //TO DO: change scope of this function if this is initiated from outside the class. + add_output_variable_info_from_formulation(); + + //Add global attributes + catchmentNcFile->putAtt("title", "NextGen Catchment Output Data"); + catchmentNcFile->putAtt("description", "NetCDF file containing catchment-level output data from NextGen simulation"); + catchmentNcFile->putAtt("institution", "NOAA"); + } + + #if NGEN_WITH_MPI + if (mpi_num_procs > 1) MPI_Barrier(MPI_COMM_WORLD); + #endif + + if (mpi_rank != 0){ + catchmentNcFile = std::make_shared(ncOutputFileName, netCDF::NcFile::write); + retrieve_output_variables_mpi(); + } + } + catch (const netCDF::exceptions::NcException& e){ + LOG(std::string("Error in catchments NetCDF initiation: ") + e.what(), LogLevel::FATAL); + throw std::runtime_error(std::string("Error in catchments NetCDF initiation: ") + e.what()); + } + } + else{ + LOG("No output variables/headers information provided in the realization config. A NetCDF file for the catchment output will not be created.", LogLevel::INFO); + } +} + +void NetCDFCreator::add_output_variable_info_from_formulation() +{ + typename std::map>::const_iterator it = manager_->begin(); + const auto& catchment_info = *it; + auto r_c = std::dynamic_pointer_cast(catchment_info.second); + if(r_c->get_output_header_count() > 0){ + std::vectoroutput_variables = r_c->get_output_variable_names(); + std::vectoroutput_headers = r_c->get_output_header_fields(); + std::vectoroutput_units = r_c->get_output_variable_units(); + nc_output_variables.resize(output_variables.size()); + + std::vector dims = {catchmentNcFile->getDim("time"), catchmentNcFile->getDim("catchments")}; + for(int index = 0; index < output_variables.size(); index ++){ + nc_output_variables[index] = catchmentNcFile->addVar(output_headers[index], NC_DOUBLE, dims); + nc_output_variables[index].putAtt("variable name", output_variables[index]); + nc_output_variables[index].putAtt("variable units", output_units[index]); + nc_output_variables[index].putAtt("_FillValue", NC_DOUBLE, -1.0); //TO DO: Change to another value, if recommended. + nc_output_variables[index].putAtt("missing_value", NC_DOUBLE, -2.0); //TO DO: Change to another value, if recommended. + } + } +} + +void NetCDFCreator::retrieve_output_variables_mpi() +{ + typename std::map>::const_iterator it = manager_->begin(); + const auto& catchment_info = *it; + auto r_c = std::dynamic_pointer_cast(catchment_info.second); + if(r_c->get_output_header_count() > 0){ + std::vectoroutput_headers = r_c->get_output_header_fields(); + nc_output_variables.resize(output_headers.size()); + + std::vector dims = {catchmentNcFile->getDim("time"), catchmentNcFile->getDim("catchments")}; + for(int index = 0; index < output_headers.size(); index ++){ + nc_output_variables[index] = catchmentNcFile->getVar(output_headers[index]); + } + } +} + +void NetCDFCreator::write_simulations_response_from_formulation(size_t time_index, std::map catchment_output_values) +{ + for (auto const& catchment_val : catchment_output_values) + { + std::string catchment_id = catchment_val.first; + + //iterate through catchment dimension to find the index of the catchment for writing. + //also split the comma separated outputs string to a vector of double values + std::vector catchment_output; + std::vector count = {1, 1}; + for (size_t c_index = 0; c_index < catchments.size(); ++c_index) + { + if(catchments[c_index] == catchment_id){ + catchment_output = string_split(catchment_val.second, ','); + std::vector start = {time_index, c_index}; + for(int var_index = 0; var_index < nc_output_variables.size(); ++var_index) + { + nc_output_variables[var_index].putVar(start, count, &catchment_output[var_index]); + } + break; + } + } + } +} + +std::vector NetCDFCreator::string_split(std::string str, char delimiter) +{ + std::stringstream ss(str); + std::vector res; + std::string token; + while (getline(ss, token, delimiter)) { //will return full string if no comma (or single output variable). + res.push_back(std::stod(token)); + } + return res; +} + +bool NetCDFCreator::create_ncfile() +{ + typename std::map>::const_iterator it = manager_->begin(); + const auto& catchment_info = *it; + auto r_c = std::dynamic_pointer_cast(catchment_info.second); + return (r_c->get_output_header_count() > 0) ? true : false; +} + +netCDF::NcFile& NetCDFCreator::get_ncfile(){ + return *catchmentNcFile; +} + +void NetCDFCreator::close_ncfile(){ + if (catchmentNcFile != nullptr) { + catchmentNcFile->close(); + } + catchmentNcFile = nullptr; +} + +NetCDFCreator::~NetCDFCreator() +{ + close_ncfile(); +} +#endif // NGEN_WITH_NETCDF \ No newline at end of file diff --git a/src/core/Layer.cpp b/src/core/Layer.cpp index 37021b506a..828099c81e 100644 --- a/src/core/Layer.cpp +++ b/src/core/Layer.cpp @@ -63,6 +63,13 @@ void ngen::Layer::update_models(boost::span catchment_outflows, std::string output = std::to_string(output_time_index)+","+current_timestamp+","+ r_c->get_output_line_for_timestep(output_time_index)+"\n"; r_c->write_output(output); + + //capture all the output values for this timestep to write to netcdf in non-MPI run. + #if NGEN_WITH_NETCDF + #if !NGEN_WITH_MPI + catchment_output_values[id] = r_c->get_output_line_for_timestep(output_time_index); + #endif //NGEN_WITH_MPI + #endif //NGEN_WITH_NETCDF } //TODO put this somewhere else. For now, just trying to ensure we get m^3/s into nexus output double area_sq_km; @@ -149,3 +156,6 @@ void ngen::Layer::load_hot_start(std::shared_ptr snapshot r_c->load_hot_start(snapshot_loader); } } +std::map ngen::Layer::get_catchment_output_data_for_timestep(){ + return catchment_output_values; +} diff --git a/src/core/NgenSimulation.cpp b/src/core/NgenSimulation.cpp index f2caad5831..a66f3127f0 100644 --- a/src/core/NgenSimulation.cpp +++ b/src/core/NgenSimulation.cpp @@ -24,7 +24,7 @@ NgenSimulation::NgenSimulation( std::unordered_map nexus_indexes, int mpi_rank, int mpi_num_procs - ) + ) : simulation_step_(0) , sim_time_(std::make_shared(sim_time)) , layers_(std::move(layers)) @@ -43,7 +43,6 @@ void NgenSimulation::run_catchments() { // Now loop some time, iterate catchments, do stuff for total number of output times auto num_times = get_num_output_times(); - for (; simulation_step_ < num_times; simulation_step_++) { // Make room for this output step's results catchment_outflows_.resize(catchment_outflows_.size() + catchment_indexes_.size(), 0.0); @@ -107,6 +106,22 @@ void NgenSimulation::advance_models_one_output_step() nexus_indexes_, simulation_step_ ); // assume update_models() calls time->advance_timestep() + + // After updating the layer, get the output data for that timestep and write to netcdf. This is currently + //set up only for a non-MPI run. + #if NGEN_WITH_NETCDF +// #if !NGEN_WITH_MPI + std::map catchment_output_vals = layer->get_catchment_output_data_for_timestep(); + size_t chunk_start = nc_manager_->get_chunk_start(); + size_t chunk_count = nc_manager_->get_chunk_count(); + LOG("Chunk start: " + std::to_string(chunk_start) + "; Chunk count: " + + std::to_string(chunk_count) + " for rank: " + std::to_string(mpi_rank_), LogLevel::DEBUG); + nc_manager_->prepare_data_chunks(catchment_output_vals); + //nc_manager_->write_simulations_response_from_formulation(simulation_step_,catchment_output_vals); + nc_manager_->write_timestep_data_to_netcdf(simulation_step_); +// #endif //NGEN_WITH_MPI + #endif //NGEN_WITH_NETCDF + prev_layer_time = layer_next_time; } else { layer_min_next_time = prev_layer_time = layer->current_timestep_epoch_time(); @@ -387,3 +402,10 @@ void NgenSimulation::serialize(Archive& ar) { ar & nexus_indexes_; ar & nexus_downstream_flows_; } + +void NgenSimulation::create_netcdf_writer(std::shared_ptr manager, std::string nc_output_file_name, int mpi_rank, int mpi_num_procs) +{ +#if NGEN_WITH_NETCDF + this->nc_manager_ = std::make_unique(manager, nc_output_file_name, *sim_time_, mpi_rank, mpi_num_procs); +#endif +} diff --git a/src/forcing/CMakeLists.txt b/src/forcing/CMakeLists.txt index b02e3c4ff8..445c505254 100644 --- a/src/forcing/CMakeLists.txt +++ b/src/forcing/CMakeLists.txt @@ -6,10 +6,12 @@ find_package(Threads REQUIRED) target_include_directories(forcing PUBLIC ${PROJECT_SOURCE_DIR}/include + {PROJECT_SOURCE_DIR}/include/netcdf ${PROJECT_SOURCE_DIR}/include/core ${PROJECT_SOURCE_DIR}/include/core/mediator ${PROJECT_SOURCE_DIR}/include/utilities ${PROJECT_SOURCE_DIR}/include/forcing + {CMAKE_SOURCE_DIR}/include ) target_link_libraries(forcing PUBLIC @@ -25,7 +27,7 @@ target_sources(forcing PRIVATE "${CMAKE_CURRENT_LIST_DIR}/NullForcingProvider.cp if(NGEN_WITH_NETCDF) target_sources(forcing PRIVATE "${CMAKE_CURRENT_LIST_DIR}/NetCDFPerFeatureDataProvider.cpp") - target_link_libraries(forcing PUBLIC NetCDF) + target_link_libraries(forcing PRIVATE NetCDFManager) endif() if(NGEN_WITH_PYTHON) @@ -37,4 +39,4 @@ if(NGEN_WITH_PYTHON) target_link_libraries(forcing PUBLIC NGen::ngen_bmi) endif() -#target_compile_options(forcing PUBLIC -std=c++14 -Wall) +#target_compile_definitions(forcing PUBLIC $<$:NGEN_WITH_NETCDF>) diff --git a/src/forcing/NetCDFPerFeatureDataProvider.cpp b/src/forcing/NetCDFPerFeatureDataProvider.cpp index 9940521252..2a27ab17ce 100644 --- a/src/forcing/NetCDFPerFeatureDataProvider.cpp +++ b/src/forcing/NetCDFPerFeatureDataProvider.cpp @@ -2,8 +2,10 @@ #if NGEN_WITH_NETCDF #include "NetCDFPerFeatureDataProvider.hpp" - -#include +#include "netcdf/NetCDFManager.hpp" +#include "netcdf/NetCDFFile.hpp" +#include "netcdf/NetCDFVar.hpp" +#include #include #include #include @@ -51,34 +53,26 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat //nc_set_chunk_cache(sizep, nelemsp, preemptionp); //open the file - nc_file = std::make_shared(input_path, netCDF::NcFile::read); + nc_manager = std::make_shared(input_path, true); + nc_manager->open_file(); //nc_get_chunk_cache(&sizep, &nelemsp, &preemptionp); //std::cout << "Chunk cache parameters: "<getVars(); + std::vector var_set = nc_manager->list_variables(); // populate the ncvar and units caches... - std::for_each(var_set.begin(), var_set.end(), [&](const auto& element) + for (const auto& var_name : var_set) { - std::string var_name = element.first; - auto ncvar = nc_file->getVar(var_name); + auto ncvar = nc_manager->get_ncvar_by_name(var_name); variable_names.push_back(var_name); ncvar_cache.emplace(var_name,ncvar); - std::string native_units; + std::string native_units = ""; try { - auto units_att = ncvar.getAtt("units"); - if ( units_att.isNull() ) - { - native_units = ""; - } - else - { - units_att.getValues(native_units); - } + native_units = ncvar->get_string_attribute("units"); } catch(...) { @@ -95,79 +89,71 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat } units_cache[var_name] = native_units; - }); + } // read the variable ids - auto ids = nc_file->getVar("ids"); - auto id_dim_count = ids.getDimCount(); - + auto ids = nc_manager->get_ncvar_by_name("ids"); + const auto& ids_dims = ids->get_dims(); + // some sanity checks - if ( id_dim_count > 1) + if (ids_dims.empty()) { - std::string msg = "Provided NetCDF file has an \"ids\" variable with more than 1 dimension"; + std::string msg = "Provided NetCDF file has a NULL dimension for variable \"ids\""; LOG(LogLevel::FATAL, msg); throw std::runtime_error(msg); } - - auto id_dim = ids.getDim(0); - - if (id_dim.isNull() ) + if ( ids_dims.size() != 1) { - std::string msg = "Provided NetCDF file has a NULL dimension for variable \"ids\""; + std::string msg = "Provided NetCDF file has an \"ids\" variable that is not a 1D array of catchments"; LOG(LogLevel::FATAL, msg); throw std::runtime_error(msg); } - auto num_ids = id_dim.getSize(); + auto num_ids = ids_dims[0]; + if (num_ids == 0) + { + std::string msg = "Provided NetCDF file has no catchments listed for variable \"ids\""; + LOG(LogLevel::FATAL, msg); + throw std::runtime_error(msg); + } //TODO: split into smaller slices if num_ids is large. cache_slice_c_size = num_ids; - // allocate an array of character pointers - std::vector< char* > string_buffers(num_ids); - - // read the id strings - ids.getVar(&string_buffers[0]); - - // initalize the map of catchment-name to offset location and free the strings allocated by the C library + // initalize the map of catchment-name to offset location size_t loc = 0; - for_each( string_buffers.begin(), string_buffers.end(), [&](char* str) + for(const auto& id : ids->get_string_array_values()) { - loc_ids.push_back(str); - id_pos[str] = loc++; - }); - - // correct string release - nc_free_string(num_ids,&string_buffers[0]); + loc_ids.push_back(id); + id_pos[id] = loc++; + } // Get the time variable - getVar collects all values at once and stores in memory // Extremely large timespans could be problematic, but for ngen use cases, this should not be a problem - auto time_var = nc_file->getVar("Time"); + auto time_var = nc_manager->get_ncvar_by_name("Time"); // Get the size of the time dimension - size_t num_times = nc_file->getDim("time").getSize(); + size_t num_times = time_var->get_dims().size(); - std::vector raw_time(num_times); + std::vector raw_time; try { - auto dim_count = time_var.getDimCount(); + auto dim_count = time_var->get_dim_count(); // Old-format files have dimensions (catchment, time), new-format // files generated by the forcings engine have just (time) if (dim_count == 2) { - if (time_var.getDim(0).getName() != "catchment-id" || time_var.getDim(1).getName() != "time") { + if (time_var->get_dim_names()[0] != "catchment-id" || time_var->get_dim_names()[1] != "time") { std::string message = "In NetCDF file '" + input_path + "', 'Time' variable dimensions don't match expectations"; std::string throw_msg; throw_msg.assign(message); LOG(throw_msg, LogLevel::WARNING); throw std::runtime_error(throw_msg); } - time_var.getVar({0ul, 0ul}, {1ul, num_times}, raw_time.data()); - } else if (dim_count == 1) { - time_var.getVar({0ul}, {num_times}, raw_time.data()); - } else { + } else if (dim_count != 1) { throw std::runtime_error("Unexpected " + std::to_string(dim_count) + " dimensions on Time variable in NetCDF file '" + input_path + "'"); } + raw_time = time_var->get_time_values(); } catch(const std::exception& e) { netcdf_ss << "Error reading time variable: " << e.what() << std::endl; LOG(netcdf_ss.str(), LogLevel::WARNING); netcdf_ss.str(""); @@ -176,8 +162,8 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat std::string time_units; try { - time_var.getAtt("units").getValues(time_units); - } catch(const netCDF::exceptions::NcException& e) { + time_units = time_var->get_string_attribute("units"); + } catch(const std::runtime_error& e) { netcdf_ss << "Error reading time units: " << e.what() << std::endl; LOG(netcdf_ss.str(), LogLevel::WARNING); netcdf_ss.str(""); netcdf_ss << "Warning: Using default time units (seconds since epoch)" << std::endl; @@ -192,7 +178,6 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat auto since_index = time_units.find("since"); if (since_index != std::string::npos) { time_base_unit = time_units.substr(0, since_index - 1); - std::string datetime_str = time_units.substr(since_index + 6); std::tm tm = {}; std::istringstream ss(datetime_str); @@ -274,10 +259,10 @@ NetCDFPerFeatureDataProvider::~NetCDFPerFeatureDataProvider() = default; void NetCDFPerFeatureDataProvider::finalize() { - if (nc_file != nullptr) { - nc_file->close(); + if (nc_manager != nullptr) { + nc_manager->close_file(); } - nc_file = nullptr; + nc_manager = nullptr; } boost::span NetCDFPerFeatureDataProvider::get_available_variable_names() const @@ -375,7 +360,7 @@ double NetCDFPerFeatureDataProvider::get_value(const CatchmentAggrDataSelector& size_t cache_slices_t_n = (read_len + cache_slice_t_size - 1) / cache_slice_t_size; // Ceiling division to ensure remainders have a slice - auto dims = ncvar.getDims(); + std::vector dims = ncvar->get_dim_names(); int dim_time, dim_catchment; if (dims.size() != 2) { @@ -384,11 +369,11 @@ double NetCDFPerFeatureDataProvider::get_value(const CatchmentAggrDataSelector& LOG(throw_msg, LogLevel::WARNING); throw std::runtime_error(throw_msg); } - if (dims[0].getName() == "time" && dims[1].getName() == "catchment-id") { + if (dims[0] == "time" && dims[1] == "catchment-id") { // Forcings Engine NetCDF output case dim_time = 0; dim_catchment = 1; - } else if (dims[1].getName() == "time" && dims[0].getName() == "catchment-id") { + } else if (dims[1] == "time" && dims[0] == "catchment-id") { // Classic NetCDF file case dim_time = 1; dim_catchment = 0; @@ -399,14 +384,14 @@ double NetCDFPerFeatureDataProvider::get_value(const CatchmentAggrDataSelector& throw std::runtime_error(throw_msg); } - size_t time_dim_size = dims[dim_time].getSize(); - size_t catchment_dim_size = dims[dim_catchment].getSize(); + size_t time_dim_size = ncvar->get_dim_size(dim_time); + size_t catchment_dim_size = ncvar->get_dim_size(dim_catchment); for( size_t i = 0; i < cache_slices_t_n; i++ ) { std::shared_ptr> cached; size_t cache_t_idx = idx1 + i * cache_slice_t_size; size_t slice_size = std::min(cache_slice_t_size, time_dim_size - cache_t_idx); - std::string key = ncvar.getName() + "|" + std::to_string(cache_t_idx) + "|" + std::to_string(slice_size); + std::string key = ncvar->get_name() + "|" + std::to_string(cache_t_idx) + "|" + std::to_string(slice_size); if(value_cache.contains(key)){ cached = value_cache.get(key).get(); @@ -423,9 +408,9 @@ double NetCDFPerFeatureDataProvider::get_value(const CatchmentAggrDataSelector& var_index_map[dim_catchment] = 1; try { - ncvar.getVar(start,count, {1l, 1l}, var_index_map, cached->data()); + ncvar->read_slice(start,count,cached->data()); value_cache.insert(key, cached); - } catch (netCDF::exceptions::NcException& e) { + } catch (const std::runtime_error& e) { netcdf_ss << "NetCDF exception: " << e.what() << std::endl; log_stream << netcdf_ss.str(); LOG(netcdf_ss.str(), LogLevel::WARNING); netcdf_ss.str(""); @@ -515,7 +500,7 @@ std::vector NetCDFPerFeatureDataProvider::get_values(const CatchmentAggr // private: -const netCDF::NcVar& NetCDFPerFeatureDataProvider::get_ncvar(const std::string& name){ +std::shared_ptr NetCDFPerFeatureDataProvider::get_ncvar(const std::string& name) const{ auto cache_hit = ncvar_cache.find(name); if(cache_hit != ncvar_cache.end()){ return cache_hit->second; diff --git a/src/netcdf/NetCDFFile.cpp b/src/netcdf/NetCDFFile.cpp new file mode 100644 index 0000000000..c1deb5dde3 --- /dev/null +++ b/src/netcdf/NetCDFFile.cpp @@ -0,0 +1,275 @@ +#if NGEN_WITH_MPI + #define _PARALLEL4 +#endif + +#if NGEN_WITH_NETCDF +#include "NetCDFFile.hpp" +#include "ewts_ngen/logger.hpp" +#include +#include +#include +#include +#include + + +NetCDFFile::NetCDFFile(const std::string& filename, bool write_only, bool is_mpi) + : nc_file_name_(filename), is_mpi_(is_mpi) +{ + int retval; + int mode = NC_NETCDF4; +#if NGEN_WITH_MPI + if(is_mpi_){ + if(write_only){ + retval = nc_create_par(nc_file_name_.c_str(), NC_NETCDF4 | NC_MPIIO | NC_CLOBBER, + MPI_COMM_WORLD, MPI_INFO_NULL, &ncid_); + } + else{ + retval = nc_open_par(nc_file_name_.c_str(), NC_WRITE | NC_MPIIO, + MPI_COMM_WORLD, MPI_INFO_NULL, &ncid_); + } + } + else +#endif + { + if(write_only){ + LOG("Attempting to create NetCDF file", LogLevel::DEBUG); + retval = nc_create(nc_file_name_.c_str(), NC_NETCDF4 | NC_CLOBBER, &ncid_); + } + else{ + read_only_ = true; + LOG("Attempting to open NetCDF file", LogLevel::DEBUG); + retval = nc_open(nc_file_name_.c_str(), NC_NOWRITE, &ncid_); + } + } + + if(retval){ + throw std::runtime_error("Failed creating/opening NetCDF file: " + std::string(nc_strerror(retval))); + } +} + +// Add a new dimension +int NetCDFFile::add_dimension(const std::string& name, size_t len) { + int dimid; + int retval = nc_def_dim(ncid_, name.c_str(), len, &dimid); + if (retval) throw std::runtime_error("Error defining dimension " + name + ": " + nc_strerror(retval)); + dims_[name] = dimid; + return dimid; +} + +void NetCDFFile::add_variable(const std::string& name, nc_type type, const std::vector& dims, const std::vector& dim_names) { + int varid; + int retval = nc_def_var(ncid_, name.c_str(), type, dims.size(), dims.data(), &varid); + if (retval != NC_NOERR) throw std::runtime_error(nc_strerror(retval)); + // set fill value and missing value for NC_DOUBLE + if (type == NC_DOUBLE) { + double fill_missing_val = -1.0; + nc_def_var_fill(ncid_, varid, 0, &fill_missing_val); + nc_put_att_double(ncid_, varid, "missing_value", NC_DOUBLE, 1, &fill_missing_val); + } + auto var = std::make_shared(name, type, dims, dim_names, varid, ncid_); + add_ncvar(var); +} + +// Get dimension length by name +size_t NetCDFFile::get_dim_size(const std::string& name) const { + auto it = dims_.find(name); + if(it == dims_.end()) return -1; + + size_t len; + int retval = nc_inq_dimlen(ncid_, it->second, &len); + if(retval != NC_NOERR) return -1; + return len; +} + +int NetCDFFile::get_dim_id(const std::string& name) const { + auto it = dims_.find(name); + if(it == dims_.end()) return -1; + return it->second; +} + +// Add a variable +void NetCDFFile::add_ncvar(std::shared_ptr var) { + variables_map_[var->get_name()] = var; +} + +void NetCDFFile::write_catchment_output_data(const std::string& name, std::vector start, + std::vector count, const double& data) +{ + auto var = get_ncvar_by_name(name); + if (!var) throw std::runtime_error("Variable not found: " + name); + int retval = nc_put_vara_double(ncid_, var->get_varid(), start.data(), count.data(), &data); + LOG("Added value for catchments", LogLevel::INFO); + if (retval != NC_NOERR) { + throw std::runtime_error("Error writing value: " + std::string(nc_strerror(retval))); + } +} + +int NetCDFFile::write_data_to_ncvar(int ncid_, int varid, const std::vector& start, + const std::vector& count, const std::vector& data) { + return nc_put_vara_double(ncid_, varid, start.data(), count.data(), data.data()); +} + +int NetCDFFile::write_data_to_ncvar(int ncid_, int varid, const std::vector& start, + const std::vector& count, const std::vector& data) { + return nc_put_vara_int(ncid_, varid, start.data(), count.data(), data.data()); +} + +int NetCDFFile::write_data_to_ncvar(int ncid_, int varid, const std::vector& start, + const std::vector& count, const std::vector& data) { + std::vector cstrs(data.size()); + for (size_t i = 0; i < data.size(); ++i) { + cstrs[i] = data[i].c_str(); + } + return nc_put_vara_string(ncid_, varid, start.data(), count.data(), cstrs.data()); +} + +template +void NetCDFFile::write_variable_data(const std::string& name, const std::vector& data, size_t start_index) { + auto var = get_ncvar_by_name(name); + if (!var) throw std::runtime_error("Variable not found: " + name); + std::vector start = {start_index}; + std::vector count = {data.size()}; + + #if NGEN_WITH_MPI + if (comm_ != MPI_COMM_NULL) { + int retval = nc_var_par_access(ncid_, var->get_varid(), NC_COLLECTIVE); + if (retval != NC_NOERR) throw std::runtime_error(nc_strerror(retval)); + } + #endif + + int retval = NC_NOERR; + retval = write_data_to_ncvar(ncid_, var->get_varid(), start, count, data); + if (retval != NC_NOERR) throw std::runtime_error(nc_strerror(retval)); + + // We need to construct a map of variable value and the index in the nc file. + // this will be used while writing the output values. + // We need this only for the coordinates variables like catchments. + nc_type data_type; + retval = nc_inq_vartype(ncid_, var->get_varid(), &data_type); + if (retval != NC_NOERR) throw std::runtime_error(nc_strerror(retval)); + + if (data_type == NC_STRING) { + var->build_variables_index(data.size()); + } +} + +void NetCDFFile::write_attribute_to_ncvar(const std::string& name, const std::string& attName, const std::string& attValue) { + auto var = get_ncvar_by_name(name); + if (!var) throw std::runtime_error("Variable not found: " + name); + double value; + size_t pos; + bool is_numeric = true; + try { + value = std::stod(attValue, &pos); + } + catch (...) { + is_numeric = false; + } + if (pos == attValue.size()){ //needed to ensure that the entire string is a numeric value. + var->add_attribute(attName, value); + } + else{ + var->add_attribute(attName, attValue); + } +} + +std::shared_ptr NetCDFFile::get_ncvar(const std::string& name) const +{ + return get_ncvar_by_name(name); +} + +void NetCDFFile::end_def_mode() +{ + int retval = nc_enddef(ncid_); + if(retval != NC_NOERR) + throw std::runtime_error(std::string("Error switching to NetCDF data mode: ") + nc_strerror(retval)); +} + +std::shared_ptr NetCDFFile::get_ncvar_by_name(const std::string& name) const { + auto it = variables_map_.find(name); + if (it != variables_map_.end()) return it->second; + return nullptr; +} + +void NetCDFFile::load_variables() +{ + int retval; + int mode = read_only_ ? NC_NOWRITE : NC_WRITE; + + if (retval != NC_NOERR) { + throw std::runtime_error(std::string("Failed to open NetCDF file: ") + nc_strerror(retval)); + } + + // Load dimensions + int num_dims; + retval = nc_inq_ndims(ncid_, &num_dims); + if (retval != NC_NOERR) + throw std::runtime_error(std::string("Number of Dimensions: ") + nc_strerror(retval)); + + for (int i = 0; i < num_dims; ++i) { + char dim_name[NC_MAX_NAME + 1]; + size_t dim_len; + retval = nc_inq_dim(ncid_, i, dim_name, &dim_len); + if (retval == NC_NOERR) { + dims_[dim_name] = i; + } + } + + // Load variables + int num_variables; + retval = nc_inq_nvars(ncid_, &num_variables); + if (retval != NC_NOERR) + throw std::runtime_error(std::string("Number of Variables: ") + nc_strerror(retval)); + + for (int i = 0; i < num_variables; ++i) { + char var_name[NC_MAX_NAME + 1]; + retval = nc_inq_varname(ncid_, i, var_name); + if (retval != NC_NOERR) continue; + + nc_type type; + int num_dims_var; + int dim_ids[NC_MAX_DIMS]; + + retval = nc_inq_var(ncid_, i, nullptr, &type, &num_dims_var, dim_ids, nullptr); + if (retval != NC_NOERR) continue; + + // Collect dim names + std::vector dim_names; + std::vector dim_ids_var; + for (int d = 0; d < num_dims_var; ++d) { + dim_ids_var.push_back(dim_ids[d]); + char dim_name[NC_MAX_NAME + 1]; + nc_inq_dim(ncid_, dim_ids[d], dim_name, nullptr); + dim_names.push_back(dim_name); + } + + // Create shared_ptr NcVar + auto nc_var = std::make_shared(var_name, type,dim_ids_var, dim_names, i, ncid_); + variables_.push_back(nc_var); + variables_map_[var_name] = nc_var; + } +} + +std::vector NetCDFFile::list_variables() const +{ + std::vector names; + for(const auto& var : variables_) { + names.push_back(var->get_name()); + } + return names; +} + +void NetCDFFile::close_file() { + if (ncid_ >= 0) nc_close(ncid_); +} + +NetCDFFile::~NetCDFFile() { + close_file(); +} + +template void NetCDFFile::write_variable_data(const std::string& name, const std::vector& data, size_t start_index); +template void NetCDFFile::write_variable_data(const std::string& name, const std::vector& data, size_t start_index); +template void NetCDFFile::write_variable_data(const std::string& name, const std::vector& data, size_t start_index); +//template void NetCDFFile::writeAllVariablesDistributed(const std::vector>&); +//template void NetCDFFile::writeAllVariablesDistributed(const std::vector>&); +#endif // NGEN_WITH_NETCDF \ No newline at end of file diff --git a/src/netcdf/NetCDFManager.cpp b/src/netcdf/NetCDFManager.cpp new file mode 100644 index 0000000000..f629e8b0fa --- /dev/null +++ b/src/netcdf/NetCDFManager.cpp @@ -0,0 +1,408 @@ +#if NGEN_WITH_NETCDF +#include "NetCDFManager.hpp" +#include "Formulation_Manager.hpp" +#include "Catchment_Formulation.hpp" +#include "simulation_time/Simulation_Time.hpp" +#include "ewts_ngen/logger.hpp" +#include +#include + +NetCDFManager::NetCDFManager(std::shared_ptr manager, + const std::string& output_name, Simulation_Time const& sim_time, int mpi_rank, int mpi_num_procs) +{ + manager_ = manager; + sim_time_ = std::make_shared(sim_time); + nc_filename_ = manager_->get_output_root() + output_name + ".nc"; +#if NGEN_WITH_MPI + rank_ = mpi_rank; + num_procs_ = mpi_num_procs; + is_mpi_ = (num_procs_ > 1); + comm_ = (num_procs_ > 1) ? MPI_COMM_WORLD : MPI_COMM_NULL; +#else + is_mpi_ = false; +#endif + nc_file_ = std::make_unique(nc_filename_, true, is_mpi_); + define_catchment_netcdf_components(); +} + +int NetCDFManager::create_file(const std::string& filename) +{ +#if NGEN_WITH_MPI + if (num_procs_ > 1) { + // MPI-enabled NetCDF + is_mpi_ = true; + nc_file_ = std::make_unique(filename, true, is_mpi_); + }else{ + is_mpi_ = false; + } +#endif + nc_filename_ = filename; + nc_file_ = std::make_unique(nc_filename_, true, is_mpi_); + if (!nc_file_) { + throw std::runtime_error("Failed to create NetCDF file: " + filename); + } + return nc_file_ ->get_ncid(); +} + +void NetCDFManager::define_catchment_netcdf_components() +{ + std::string name; + int dim_id; + std::vector dims; + std::vector names; + + //add time dimension and variable + try + { + name = "time"; + int num_timesteps = sim_time_->get_total_output_times(); + dim_id = add_dimension(name, num_timesteps); + dims = {dim_id}; + names = {name}; + add_variable(name, NC_INT, dims, names); + + //add timestep values and attributes for time + std::vector time_epoch_seconds(num_timesteps); + time_epoch_seconds[0] = sim_time_->get_current_epoch_time(); + for(int time_index = 1; time_index < num_timesteps; time_index++) + { + sim_time_->advance_timestep(); + time_epoch_seconds[time_index] = sim_time_->get_current_epoch_time(); + } + nc_file_->write_variable_data(name, time_epoch_seconds); + nc_file_->write_attribute_to_ncvar(name, "units", "Seconds since 1970-01-01 00:00:00"); + nc_file_->write_attribute_to_ncvar(name, "calendar", "gregorian"); + } + catch(const std::runtime_error& e) + { + LOG(std::string("Error in adding time information to catchment NetCDF: ") + e.what(), LogLevel::FATAL); + throw std::runtime_error(std::string("Error in adding time information to catchment NetCDF: ") + e.what()); + } + + //add catchments dimension and variable + try + { + name = "catchments"; + dim_id = add_dimension(name, manager_->get_size()); + dims = {dim_id}; + names = {name}; + add_variable(name, NC_STRING, dims, names); + + //add catchment IDs and attributes for catchments + int num_catchments = manager_->get_size(); + catchments_.reserve(num_catchments); + for (auto const& formulation_info : manager_->get_all_formulations()) + { + std::string catchm = formulation_info.first; + if (catchm.compare(0, 3, "cat") != 0) { + catchm = "cat-" + catchm; + } + catchments_.push_back(catchm); + } + nc_file_->write_variable_data(name, catchments_); + nc_file_->write_attribute_to_ncvar(name, "Catchment ID", "Catchment identifier in input"); + + //Compute writing chunks for catchments + size_t base = num_catchments / num_procs_; + size_t rem = num_catchments % num_procs_; + chunk_count_ = base + (rank_ < rem ? 1 : 0); + chunk_start_ = rank_ * base + std::min(static_cast(rank_), rem); + LOG("in NcManager: Chunk start: " + std::to_string(chunk_start_) + "; Chunk count: " + + std::to_string(chunk_count_) + " for rank: " + std::to_string(rank_), LogLevel::DEBUG); + + } + catch(const std::runtime_error& e) + { + LOG(std::string("Error in adding catchment IDs to catchment NetCDF: ") + e.what(), LogLevel::FATAL); + throw std::runtime_error(std::string("Error in adding catchment IDs to catchment NetCDF: ") + e.what()); + } + + // Add realization config output variables + try + { + add_output_variable_data_from_formulation(); + } + catch(const std::runtime_error& e) + { + LOG(std::string("Error in adding output variables information to catchment NetCDF: ") + e.what(), LogLevel::FATAL); + throw std::runtime_error(std::string("Error in adding output variables information to catchment NetCDF: ") + e.what()); + } + + // Resize the 2D array/vector that will hold the output data for each timestep. + int num_variables = nc_output_variables_.size(); + if (num_variables > 0){ + data_chunks_.resize(num_variables); + for(size_t v = 0; v < num_variables; ++v) + { + data_chunks_[v].resize(chunk_count_); + } + } + + // Switch to data mode for writing + nc_file_->end_def_mode(); +} + +NetCDFManager::NetCDFManager(const std::string& filename, bool read_only) + : read_only_(true) +{ + if (read_only_){ + nc_file_ = std::make_unique(filename, !read_only, false); + } + else{ + throw std::runtime_error("Write only non-MPI function not implemented."); + } +#if NGEN_WITH_MPI + comm_ = MPI_COMM_NULL; + rank_ = 0; + num_procs_ = 1; +#endif +} + +NetCDFManager::NetCDFManager() +{} + +int NetCDFManager::add_dimension(const std::string& name, size_t len){ + return nc_file_->add_dimension(name, len); +} + +void NetCDFManager::add_variable(const std::string& var_name, nc_type type, const std::vector& dims, const std::vector& dim_names){ + nc_file_->add_variable(var_name, type, dims, dim_names); +} + +void NetCDFManager::add_output_variable_data_from_formulation() +{ + typename std::map>::const_iterator it = manager_->begin(); + const auto& catchment_info = *it; + auto r_c = std::dynamic_pointer_cast(catchment_info.second); + if(r_c->get_output_header_count() > 0){ + std::vectoroutput_variables = r_c->get_output_variable_names(); + std::vectoroutput_headers = r_c->get_output_header_fields(); + std::vectoroutput_units = r_c->get_output_variable_units(); + nc_output_variables_.reserve(output_variables.size()); + std::vector dims_2D = {nc_file_->get_dim_id("time"), nc_file_->get_dim_id("catchments")}; + std::vector dim_names{"time", "catchments"}; + + for(int index = 0; index < output_variables.size(); index ++){ + nc_file_->add_variable(output_headers[index], NC_DOUBLE, dims_2D, dim_names); + nc_file_->write_attribute_to_ncvar(output_headers[index], "variable name", output_variables[index]); + nc_file_->write_attribute_to_ncvar(output_headers[index], "variable units", output_units[index]); + nc_file_->write_attribute_to_ncvar(output_headers[index], "_FillValue", "-1.0"); + nc_file_->write_attribute_to_ncvar(output_headers[index], "missing_value", "-1.0"); + nc_output_variables_.push_back(output_headers[index]); + } + } +} + +static std::vector string_split(std::string str, char delimiter) +{ + std::stringstream ss(str); + std::vector res; + std::string token; + while (getline(ss, token, delimiter)) { //will return full string if no comma (or single output variable). + res.push_back(std::stod(token)); + } + return res; +} + +void NetCDFManager::prepare_data_chunks(std::map catchment_output_vals) +{ + int num_variables = data_chunks_.size(); + if (num_variables > 0){ // if number of output variables is greater than zero + size_t index = 0; + for(auto it = catchment_output_vals.begin(); it != catchment_output_vals.end(); ++it) + { + if(index < chunk_start_) { index++; continue; } // skip catchments not for this rank + if(index >= chunk_start_ + chunk_count_) break; // done with this rank. + std::vector catchment_output = string_split(it->second, ','); + for(size_t var_index = 0; var_index < num_variables; ++var_index){ + data_chunks_[var_index][index - chunk_start_] = catchment_output[var_index]; + // LOG("Index value: " + std::to_string(index) + "; Inserted value " + std::to_string(catchment_output[var_index]) + " to " + + // std::to_string(var_index) + " for catchment index: " + std::to_string(index - chunk_start_), LogLevel::DEBUG); + } + index++; + } + // LOG("Number of Variables: " + std::to_string(data_chunks_.size()) + "; Number of catchments: " + + // std::to_string(data_chunks_[0].size()) + " for rank: " + std::to_string(rank_), LogLevel::DEBUG); + } +} + +void NetCDFManager::write_simulations_response_from_formulation(size_t time_index, std::map catchment_output_values) +{ + LOG("Write simulation response for timestep: " + std::to_string(time_index), LogLevel::INFO); + auto var = nc_file_->get_ncvar("catchments"); + if (!var) throw std::runtime_error("Catchments variable not found"); + for (auto const& catchment_val : catchment_output_values) + { + std::string catchment_id = catchment_val.first; + LOG("Getting Catchment index for: " + catchment_id, LogLevel::INFO); + size_t catchment_index = var->get_variable_index(catchment_id); + LOG("Catchment index for: " + catchment_id + " is: " + std::to_string(time_index), LogLevel::INFO); + if (catchment_index < 0) { + throw std::runtime_error("Catchment not found in NetCDF: " + catchment_id); + } + std::vector start = {time_index, catchment_index}; + std::vector count = {1, 1}; + std::vector catchment_output = string_split(catchment_val.second, ','); + LOG("Time index: " + std::to_string(time_index) + " has values: " + catchment_val.second, LogLevel::INFO); + for(int var_index = 0; var_index < nc_output_variables_.size(); ++var_index) + { + LOG("Writing output data for: " + nc_output_variables_[var_index], LogLevel::INFO); + nc_file_ ->write_catchment_output_data(nc_output_variables_[var_index], start, count, catchment_output[var_index]); + } + } +} + +void NetCDFManager::write_timestep_data_to_netcdf(size_t time_index) +{ + if(nc_output_variables_.size() > 0){ + if(nc_output_variables_.size() != data_chunks_.size()){ + LOG("Number of Variables: " + std::to_string(nc_output_variables_.size()) + "; Chunk size: " + + std::to_string(data_chunks_.size()) + " for rank: " + std::to_string(rank_), LogLevel::DEBUG); + throw std::runtime_error("Mismatch between variable names and the size of the incoming values."); + } + for(size_t i = 0; i < nc_output_variables_.size(); ++i) + { + const std::string& name = nc_output_variables_[i]; + const std::vector& data = data_chunks_[i]; + + if(chunk_count_ == 0) continue; //nothing to write + + if(data.size() != chunk_count_) + throw std::runtime_error("Data chunk size mismatch for " + name); + + auto var = nc_file_->get_ncvar(name); + if (!var) throw std::runtime_error("Catchments variable not found"); + var->write_timesliced_data(time_index, chunk_start_, chunk_count_, data.data()); + } + } +} + +std::vector NetCDFManager::list_variables() const +{ + if(!nc_file_) return {}; + return nc_file_->list_variables(); +} + +std::shared_ptr NetCDFManager::get_ncvar_by_name(const std::string& name) const +{ + if(!nc_file_) return nullptr; + return nc_file_->get_ncvar(name); +} + +// Attribute access +std::vector NetCDFManager::list_attributes(const std::string& var_name) const +{ + auto var = nc_file_->get_ncvar(var_name); + return var->list_attributes(); +} + +std::string NetCDFManager::get_string_attribute(const std::string& var_name, const std::string& att_name) const +{ + auto var = nc_file_->get_ncvar(var_name); + return var->get_string_attribute(att_name); +} + +int NetCDFManager::get_int_attribute(const std::string& var_name, const std::string& att_name) const +{ + auto var = nc_file_->get_ncvar(var_name); + return var->get_int_attribute(att_name); +} + +double NetCDFManager::get_double_attribute(const std::string& var_name, const std::string& att_name) const +{ + auto var = nc_file_->get_ncvar(var_name); + return var->get_double_attribute(att_name); +} + +void NetCDFManager::open_file() +{ + if(nc_file_) { + nc_file_->load_variables(); + } else { + throw std::runtime_error("NcFile not initialized!"); + } +} + +void NetCDFManager::close_file() +{ + nc_file_->close_file(); +} + +void NetCDFManager::create_timeslice(size_t num_entities, size_t num_timesteps, + const std::vector& var_names) { + + num_entities_ = num_entities; + num_timesteps_ = num_timesteps; + + // Define dimensions + nc_file_->add_dimension("time", num_timesteps_); + nc_file_->add_dimension("entity", num_entities_); + + // Create variables + for (const auto& name : var_names) { + std::vector dims = {num_timesteps_, num_entities_}; + std::vector dim_names = {"time", "entity"}; + auto var = std::make_shared( + name, NC_DOUBLE, dims, dim_names, + nc_file_->get_next_varid(), nc_file_->get_ncid()); + nc_file_->add_ncvar(var); + variables_map_[name] = var; + +#if USE_MPI + nc_var_par_access(nc_file_->get_ncid(), var->get_varid(), NC_COLLECTIVE); +#endif + } +} + +// Partition time dimension across ranks +void NetCDFManager::get_local_time_range(size_t& start, size_t& count) const { +#if USE_MPI + size_t base = num_timesteps_ / size_; + size_t remainder = num_timesteps_ % size_; + start = rank_ * base + std::min(static_cast(rank_), remainder); + count = base + (rank_ < static_cast(remainder) ? 1 : 0); +#else + start = 0; + count = num_timesteps_; +#endif +} + +template +void NetCDFManager::write_timeslice(const std::string& var_name, size_t time_start, + size_t time_count, const std::vector& data) { + auto it = variables_map_.find(var_name); + if (it == variables_map_.end()) + throw std::runtime_error("Variable not found: " + var_name); + + auto var = it->second; + + std::vector start = {time_start, 0}; + std::vector count = {time_count, var->get_dim_size(1)}; + +#if USE_MPI + if(rank_ >= 0 && size_ > 1) { + int retval = nc_put_vara_double(nc_file_->get_ncid(), var->get_varid(), + start.data(), count.data(), + data.data()); + if(retval) throw std::runtime_error("Error writing variable: " + std::string(nc_strerror(retval))); + } +#else + int retval = nc_put_vara_double(nc_file_->get_ncid(), var->get_varid(), + start.data(), count.data(), + data.data()); + if(retval) throw std::runtime_error("Error writing variable: " + std::string(nc_strerror(retval))); +#endif +} + + + +NetCDFManager::~NetCDFManager() { + #if NGEN_WITH_MPI + if(num_procs_ > 1 && comm_ != MPI_COMM_NULL) { + MPI_Comm_free(&comm_); + } + #endif +} +// Explicit template instantiation for double +template void NetCDFManager::write_timeslice(const std::string&, size_t, size_t, const std::vector&); +#endif // NGEN_WITH_NETCDF \ No newline at end of file diff --git a/src/netcdf/NetCDFVar.cpp b/src/netcdf/NetCDFVar.cpp new file mode 100644 index 0000000000..9218168acc --- /dev/null +++ b/src/netcdf/NetCDFVar.cpp @@ -0,0 +1,267 @@ +#if NGEN_WITH_NETCDF +#include "NetCDFVar.hpp" +#include "ewts_ngen/logger.hpp" +#include +#include +#include +#include + +NetCDFVar::NetCDFVar(const std::string& name, nc_type type, const std::vector& dims, + const std::vector& dim_names,int varid, int ncid) + : name_(name), type_(type), dims_(dims), dim_names_(dim_names), varid_(varid), ncid_(ncid) +{} + +const std::string& NetCDFVar::get_name() const { return name_; } +nc_type NetCDFVar::get_type() const { return type_; } +const std::vector& NetCDFVar::get_dims() const { return dims_; } +const std::vector& NetCDFVar::get_dim_names() const { return dim_names_; } +size_t NetCDFVar::get_dim_count() const { return dims_.size(); } +int NetCDFVar::get_varid() const { return varid_; } + +size_t NetCDFVar::get_dim_size(const std::string& dim_name) const { + for (size_t i = 0; i < dim_names_.size(); ++i) { + if (dim_names_[i] == dim_name) { + return dims_[i]; + } + } + throw std::runtime_error( + "Dimension '" + dim_name + "' not found in variable '" + name_ + "'" + ); +} + +size_t NetCDFVar::get_dim_size(size_t idx) const { + if (idx >= dims_.size()) { + throw std::out_of_range( + "Dimension index " + std::to_string(idx) + + " out of range for variable '" + name_ + "'" + ); + } + return dims_[idx]; +} +size_t NetCDFVar::get_total_size() const { + return std::accumulate(dims_.begin(), dims_.end(), size_t{1}, std::multiplies()); +} + +std::vector NetCDFVar::get_string_array_values() const { + + // Safety check for zero dimensions + if (dims_.empty()) { + throw std::runtime_error("Variable '" + name_ + "' has no dimensions"); + } + std::vector array_items; + if (type_ == NC_STRING) { + if (dims_.size() != 1) { + throw std::runtime_error("NC_STRING variable must be 1D"); + } + size_t items_count = dims_[0]; + char** raw_strings = nullptr; + int retval = nc_get_var_string(ncid_, varid_, raw_strings); + if (retval != NC_NOERR) { + throw std::runtime_error(nc_strerror(retval)); + } + array_items.reserve(items_count); + for (size_t i = 0; i < items_count; ++i) { + array_items.emplace_back(raw_strings[i] ? raw_strings[i] : ""); + } + nc_free_string(items_count, raw_strings); + } + else if (type_ == NC_CHAR) { + if (dims_.size() != 2) { + throw std::runtime_error("NC_CHAR string variable must be 2D"); + } + size_t num_strings = dims_[0]; + size_t str_len = dims_[1]; + std::vector buffer(num_strings * str_len); + int retval = nc_get_var_text(ncid_, varid_, buffer.data()); + if (retval != NC_NOERR) { + throw std::runtime_error(nc_strerror(retval)); + } + array_items.reserve(num_strings); + for (size_t i = 0; i < num_strings; ++i) { + const char* start = buffer.data() + (i * str_len); + std::string s(start, str_len); + size_t end = s.find_last_not_of(" \0"); // trim trailing spaces/nulls + if (end != std::string::npos) { + s.erase(end + 1); + } else { + s.clear(); + } + array_items.push_back(s); + } + } + if (array_items.empty()) { + throw std::runtime_error("Unsupported variable type called for this function."); + } + return array_items; +} + +std::vector NetCDFVar::get_time_values() const +{ + // Safety checks for dimensions and data type + if (dims_.empty()) { + throw std::runtime_error("Variable '" + name_ + "' has no dimensions"); + } + if (type_ != NC_DOUBLE) { + throw std::runtime_error("Variable '" + name_ + "' is not NC_DOUBLE"); + } + + size_t time_count = 0; + if (dims_.size() == 1) { + time_count = dims_[0]; // (time) + + } + else if (dims_.size() == 2) { + time_count = dims_[1]; // (entity, time) + } else { + throw std::runtime_error("Unsupported dimensions for time variable '" + name_ + "'"); + } + if (time_count == 0) { + return {}; + } + + std::vector time_values(time_count); + int retval = NC_NOERR; + if (dims_.size() == 1) { + retval = nc_get_var_double(ncid_, varid_, time_values.data()); + } else { + // Read first entity slice: + // (entity=0, all times) + std::vector start = {0, 0}; + std::vector count = {1, time_count}; + retval = nc_get_vara_double(ncid_, varid_, start.data(), count.data(), time_values.data()); + } + if (retval != NC_NOERR) { + throw std::runtime_error(nc_strerror(retval)); + } + return time_values; +} + +int NetCDFVar::get_int_value_at_index(const std::vector& index) const { + int value = -1; + if (index.empty()) { + return value; + } + int retval = nc_get_var1_int(ncid_, varid_, index.data(), &value); + if (retval != NC_NOERR) { + throw std::runtime_error(nc_strerror(retval)); + } + return value; +} + +double NetCDFVar::get_dbl_value_at_index(const std::vector& index) const { + double value = -1; + if (index.empty()) { + return value; + } + int retval = nc_get_var1_double(ncid_, varid_, index.data(), &value); + if (retval != NC_NOERR) { + throw std::runtime_error(nc_strerror(retval)); + } + return value; +} + +std::string NetCDFVar::get_str_value_at_index(const std::vector& index) const { + char* value = nullptr; + std::string str_value; + int retval = nc_get_var1_string(ncid_, varid_, index.data(), &value); + if (retval == NC_NOERR && value != nullptr) { + str_value = std::string(value); + nc_free_string(1, &value); // free allocated string + } + return str_value; +} + +void NetCDFVar::add_attribute(const std::string& att_name, const std::string& att_value) { + int retval = nc_put_att_text(ncid_, varid_, att_name.c_str(), att_value.size(), att_value.c_str()); + if (retval != NC_NOERR) throw std::runtime_error(nc_strerror(retval)); + attributes_str_[att_name] = att_value; +} + +void NetCDFVar::add_attribute(const std::string& att_name, int att_value) { + int retval = nc_put_att_int(ncid_, varid_, att_name.c_str(), NC_INT, 1, &att_value); + if (retval != NC_NOERR) throw std::runtime_error(nc_strerror(retval)); + attributes_int_[att_name] = att_value; +} + +void NetCDFVar::add_attribute(const std::string& att_name, double att_value) { + int retval = nc_put_att_double(ncid_, varid_, att_name.c_str(), NC_DOUBLE, 1, &att_value); + if (retval != NC_NOERR) throw std::runtime_error(nc_strerror(retval)); + attributes_double_[att_name] = att_value; +} + +// List attributes +std::vector NetCDFVar::list_attributes() const { + std::vector names; + for(const auto& a : attributes_str_) names.push_back(a.first); + for(const auto& a : attributes_int_) names.push_back(a.first); + for(const auto& a : attributes_double_) names.push_back(a.first); + return names; +} + +// Access attribute values +std::string NetCDFVar::get_string_attribute(const std::string& att_name) const { + auto it = attributes_str_.find(att_name); + if(it != attributes_str_.end()) return it->second; + throw std::runtime_error("String attribute not found: " + att_name); +} + +int NetCDFVar::get_int_attribute(const std::string& att_name) const { + auto it = attributes_int_.find(att_name); + if(it != attributes_int_.end()) return it->second; + throw std::runtime_error("Int attribute not found: " + att_name); +} + +double NetCDFVar::get_double_attribute(const std::string& att_name) const { + auto it = attributes_double_.find(att_name); + if(it != attributes_double_.end()) return it->second; + throw std::runtime_error("Double attribute not found: " + att_name); +} + +size_t NetCDFVar::get_variable_index(const std::string& name) const +{ + auto it = variable_index_.find(name); + if (it == variable_index_.end()) { + throw std::runtime_error(std::string("Variable not found in NetCDF: ") + name); + } + return it->second; +} + +void NetCDFVar::build_variables_index(size_t num_items) +{ + std::vector data(num_items); + nc_get_var_string(ncid_, varid_, data.data()); + LOG("Number of catchments: " + std::to_string(num_items), LogLevel::INFO); + for (size_t index = 0; index < num_items; ++index) { + std::string key = data[index]; //required to prevent heap corruption while freeing memory later + variable_index_[key] = index; + } + nc_free_string(dims_[0], data.data()); +} + +void NetCDFVar::read_slice(const std::vector& start, const std::vector& count, double* data) const +{ + int retval = NC_NOERR; + retval = nc_get_vara_double(ncid_, varid_, start.data(), count.data(), data); + if (retval != NC_NOERR) { + throw std::runtime_error(std::string("NetCDF read error: ") + nc_strerror(retval)); + } +} + +void NetCDFVar::write_timesliced_data(size_t timestep, size_t slice_start, size_t slice_count, const double* data) +{ + std::vector start = {timestep, slice_start}; + std::vector count = {1, slice_count}; + int retval; + +#if NGEN_WITH_MPI + retval = nc_var_par_access(ncid_, varid_, NC_COLLECTIVE); + if(retval!= NC_NOERR) { + throw std::runtime_error(std::string("Failed to set up parallel access: ") + nc_strerror(retval)); + } +#endif + retval = nc_put_vara_double(ncid_, varid_, start.data(), count.data(), data); + if (retval != NC_NOERR) { + throw std::runtime_error("Error writing value in NetCDF: " + std::string(nc_strerror(retval))); + } +} +#endif // NGEN_WITH_NETCDF \ No newline at end of file diff --git a/src/realizations/catchment/Bmi_Module_Formulation.cpp b/src/realizations/catchment/Bmi_Module_Formulation.cpp index 359a257f88..1195e61916 100644 --- a/src/realizations/catchment/Bmi_Module_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Module_Formulation.cpp @@ -616,6 +616,7 @@ namespace realization { output_var_units[i] = get_provider_units_for_variable(names[i]); } } + set_output_variable_units(output_var_units); //check if output variable indices (for vector variables) are specified in config. If not, default to zero (first index). if(output_var_indices.size() == 0){ diff --git a/src/realizations/catchment/Bmi_Multi_Formulation.cpp b/src/realizations/catchment/Bmi_Multi_Formulation.cpp index ed5b296193..a5d2d8cb36 100644 --- a/src/realizations/catchment/Bmi_Multi_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Multi_Formulation.cpp @@ -282,6 +282,7 @@ void Bmi_Multi_Formulation::create_multi_formulation(geojson::PropertyMap proper output_var_units[i] = get_provider_units_for_variable(names[i]); } } + set_output_variable_units(output_var_units); //check if output variable indices (for vector variables) are specified in config. If not, default to zero (first index). if(output_var_indices.size() == 0){ @@ -462,7 +463,6 @@ std::string Bmi_Multi_Formulation::get_output_line_for_timestep(int timestep, st if (timestep != (next_time_step_index - 1)) { throw std::invalid_argument("Only current time step valid when getting multi-module BMI formulation output"); } - // Start by first checking whether we are just using the last module's values if (is_out_vars_from_last_mod) { // The default behavior, which means we either diff --git a/src/utilities/mdframe/CMakeLists.txt b/src/utilities/mdframe/CMakeLists.txt index a254268f54..1770770716 100644 --- a/src/utilities/mdframe/CMakeLists.txt +++ b/src/utilities/mdframe/CMakeLists.txt @@ -10,5 +10,5 @@ NGen::logging target_link_libraries(mdframe PUBLIC ewts::ewts_ngen_bridge) if(NGEN_WITH_NETCDF) - target_link_libraries(mdframe PUBLIC NetCDF) + target_link_libraries(mdframe PUBLIC NetCDFManager) endif() diff --git a/src/utilities/mdframe/handler_netcdf.cpp b/src/utilities/mdframe/handler_netcdf.cpp index 0858a080d8..fae78eba78 100644 --- a/src/utilities/mdframe/handler_netcdf.cpp +++ b/src/utilities/mdframe/handler_netcdf.cpp @@ -4,7 +4,10 @@ #if NGEN_WITH_NETCDF -#include +#include +#include "netcdf/NetCDFManager.hpp" +#include "netcdf/NetCDFFile.hpp" +#include "netcdf/NetCDFVar.hpp" namespace ngen { @@ -12,16 +15,19 @@ namespace visitors { struct mdarray_netcdf_putvar : public boost::static_visitor { + std::shared_ptr var; + mdarray_netcdf_putvar(std::shared_ptr v) : var(v) {} + template - void operator()(const mdarray& arr, netCDF::NcVar& var) + void operator()(const mdarray& arr) const { typename mdarray::size_type rank = arr.rank(); std::vector::size_type> expanded_index(rank); - + for (auto it = arr.begin(); it != arr.end(); it++) { it.mdindex(expanded_index); - var.putVar(expanded_index, *it); + //var.putVar(expanded_index, *it); } } @@ -31,47 +37,50 @@ struct mdarray_netcdf_putvar : public boost::static_visitor void mdframe::to_netcdf(const std::string& path) const { - netCDF::NcFile output{path, netCDF::NcFile::replace}; - - std::unordered_map dimmap; - std::unordered_map varmap; + NetCDFManager nc_manager; + int ncid = nc_manager.create_file(path); + NetCDFFile* nc_file = nc_manager.get_file_handle(); + std::unordered_map dimmap; for (const auto& dim : this->m_dimensions) - dimmap[dim.name()] = output.addDim(dim.name(), dim.size()); + dimmap[dim.name()] = nc_manager.add_dimension(dim.name(), dim.size()); + + std::unordered_map> varmap; for (const auto& pair : this->m_variables) { - netCDF::NcType* type = nullptr; - decltype(auto) var = pair.second; - switch (var.values().which()) { - case 0: // int - type = &netCDF::ncInt; - break; + nc_type type; // NetCDF C type + auto& var = pair.second; - case 1: // float - type = &netCDF::ncFloat; + switch (var.values().which()) { + case 0: + type = NC_INT; break; - - case 2: // double - default: - type = &netCDF::ncDouble; + case 1: + type = NC_FLOAT; + break; + case 2: + default: type = NC_DOUBLE; break; } - - std::vector dimensions; - dimensions.reserve(var.rank()); + std::vector dim_ids; + std::vector dim_names; + dim_ids.reserve(var.rank()); + dim_names.reserve(var.rank()); for (const auto& dimname : var.dimensions()) - dimensions.push_back(dimmap[dimname]); - - const auto& nc_var = output.addVar(var.name(), *type, dimensions); - varmap[var.name()] = nc_var; + { + auto it = dimmap.find(dimname); + if(it == dimmap.end()) + throw std::runtime_error("NetCDF dimension not found: " + dimname); - auto visitor = std::bind( - visitors::mdarray_netcdf_putvar{}, - std::placeholders::_1, - nc_var - ); + dim_ids.push_back(it->second); + dim_names.push_back(it->first); + } - var.values().apply_visitor(visitor); + nc_manager.add_variable(var.name(), type, dim_ids, dim_names); + const auto& nc_var = nc_manager.get_ncvar_by_name(var.name()); + varmap[var.name()] = nc_var; + visitors::mdarray_netcdf_putvar visitor(nc_var); + boost::apply_visitor(visitor, var.values()); } } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a6b26ad200..d389ac1ab1 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -61,6 +61,7 @@ endif() add_executable(test_all) target_link_libraries(test_all PRIVATE gtest gtest_main) +target_link_libraries(test_all PRIVATE NetCDFManager) set_target_properties(test_all PROPERTIES FOLDER test) target_include_directories(test_all PRIVATE ${CMAKE_CURRENT_LIST_DIR}/bmi) @@ -460,7 +461,12 @@ ngen_add_test( NGen::geojson NGen::logging NGen::core_mediator + NGen::realizations_catchment + NGen::forcing NGen::ngen_bmi + NetCDFManager + REQUIRES + NGEN_WITH_NETCDF ) ########################### Netcdf Forcing Tests @@ -479,6 +485,26 @@ ngen_add_test( NGEN_WITH_NETCDF ) +########################## NetCDFCreator Class Tests +# ngen_add_test( +# test_netcdf_creator +# OBJECTS +# core/NetCDFCreatorTest.cpp +# LIBRARIES +# NGen::core +# NGen::geopackage +# NGen::geojson +# NGen::logging +# NGen::core_catchment +# NGen::core_mediator +# NGen::realizations_catchment +# NGen::forcing +# NGen::ngen_bmi +# NetCDFCreator +# REQUIRES +# NGEN_WITH_NETCDF +# ) + ########################## Primary Combined Unit Test Target ngen_add_test( test_unit diff --git a/test/core/NetCDFCreatorTest.cpp b/test/core/NetCDFCreatorTest.cpp new file mode 100644 index 0000000000..4eac0391cb --- /dev/null +++ b/test/core/NetCDFCreatorTest.cpp @@ -0,0 +1,293 @@ +#include + + +#if NGEN_WITH_NETCDF +#include "gtest/gtest.h" +#include +#include +#include "Bmi_Testing_Util.hpp" +#include "FileChecker.h" +#include +#include +#include +#include "ewts_ngen/logger.hpp" + +class NetCDFCreatorTest : public ::testing::Test { + protected: + std::stringstream stream_; + std::shared_ptr manager_; + geojson::GeoJSON fabric = std::make_shared(); + std::shared_ptr sim_time_; + std::map> calculated_results; + std::unique_ptr nc_creator; + + typedef struct tm time_type; + std::shared_ptr start_date_time; + std::shared_ptr end_date_time; + + void SetUp() override { + SetupFormulationManager(); + } + + void TearDown() override { + + } + + //Construct a Formulation Manager and Simulation Time objects + void SetupFormulationManager() { + stream_ << fix_paths(example_config); + boost::property_tree::ptree realization_config; + boost::property_tree::json_parser::read_json(stream_, realization_config); + manager_ = std::make_shared(realization_config); + + auto possible_simulation_time = realization_config.get_child_optional("time"); + if (!possible_simulation_time) { + std::string throw_msg; throw_msg.assign("ERROR: No simulation time period defined."); + LOG(throw_msg, LogLevel::WARNING); + throw std::runtime_error(throw_msg); + } + + auto simulation_time_config = realization::config::Time(*possible_simulation_time).make_params(); + sim_time_ = std::make_shared(simulation_time_config); + + std::ostream* raw_pointer = &std::cout; + std::shared_ptr s_ptr(raw_pointer, [](void*) {}); + utils::StreamHandler catchment_output(s_ptr); + + this->add_feature("cat-52"); + this->add_feature("cat-67"); + manager_->read(simulation_time_config, fabric, catchment_output); + } + + std::string fix_paths(std::string json) + { + std::vector forcing_paths = { + "./data/forcing/cat-52_2015-12-01 00_00_00_2015-12-30 23_00_00.csv", + "./data/forcing/cat-67_2015-12-01 00_00_00_2015-12-30 23_00_00.csv", + }; + std::vector v = {}; + for(unsigned int i = 0; i < path_options.size(); i++){ + v.push_back( path_options[i] + "data/forcing" ); + } + std::string dir = utils::FileChecker::find_first_readable(v); + if(dir != ""){ + std::string remove = "\"./data/forcing/\""; + std::string replace = "\""+dir+"\""; + boost::replace_all(json, remove , replace); + } + + //BMI_CPP_INIT_DIR_PATH + replace_paths(json, "{{BMI_CPP_INIT_DIR_PATH}}", "data/bmi/test_bmi_cpp"); + //EXTERN_DIR_PATH + replace_paths(json, "{{EXTERN_LIB_DIR_PATH}}", "extern/test_bmi_cpp/cmake_build/"); + + for (unsigned int i = 0; i < forcing_paths.size(); i++) { + if(json.find(forcing_paths[i]) == std::string::npos){ + continue; + } + std::vector v = {}; + for (unsigned int j = 0; j < path_options.size(); j++) { + v.push_back(path_options[j] + forcing_paths[i]); + } + std::string right_path = utils::FileChecker::find_first_readable(v); + if(right_path != forcing_paths[i]){ + std::cerr<<"Replacing "< v{path_options.size()}; + for(unsigned int i = 0; i < path_options.size(); i++) + v[i] = path_options[i] + replacement; + + const std::string dir = utils::FileChecker::find_first_readable(v); + if (dir == "") return; + + boost::replace_all(input, pattern, dir); + } + + void add_feature(std::string id) + { + geojson::three_dimensional_coordinates three_dimensions { + { + {1.0, 2.0}, + {3.0, 4.0}, + {5.0, 6.0} + }, + { + {7.0, 8.0}, + {9.0, 10.0}, + {11.0, 12.0} + } + }; + std::vector bounding_box{1.0, 2.0}; + geojson::PropertyMap properties{}; + + geojson::Feature feature = std::make_shared(geojson::PolygonFeature( + geojson::polygon(three_dimensions), + id, + properties + )); + + fabric->add_feature(feature); + } + + + std::vector path_options = { + "", + "../", + "../../", + "./test/", + "../test/", + "../../test/" + + }; + + const std::string example_config = + "{" + " \"global\": {" + " \"formulations\": [" + " {" + " \"name\": \"bmi_multi\"," + " \"params\": {" + " \"model_type_name\": \"bmi_multi_c++\"," + " \"forcing_file\": \"\"," + " \"init_config\": \"\"," + " \"allow_exceed_end_time\": true," + " \"main_output_variable\": \"OUTPUT_VAR_4\"," + " \"uses_forcing_file\": false," + " \"modules\": [" + " {" + " \"name\": \"bmi_c++\"," + " \"params\": {" + " \"model_type_name\": \"test_bmi_c++\"," + " \"library_file\": \"{{EXTERN_LIB_DIR_PATH}}" BMI_TEST_CPP_LIB_NAME "\"," + " \"init_config\": \"{{BMI_CPP_INIT_DIR_PATH}}/test_bmi_cpp_config_2.txt\"," + " \"allow_exceed_end_time\": true," + " \"main_output_variable\": \"OUTPUT_VAR_4\"," + " \"uses_forcing_file\": false," + " \"model_params\": {" + " \"MODEL_VAR_1\": {" + " \"source\": \"hydrofabric\"," + " \"from\": \"val\"" + " }," + " \"MODEL_VAR_2\": {" + " \"source\": \"hydrofabric\"" + " }" + " }," + " \"" BMI_REALIZATION_CFG_PARAM_OPT__VAR_STD_NAMES "\": {" + " \"INPUT_VAR_1\": \"APCP_surface\"," + " \"INPUT_VAR_2\": \"APCP_surface\"" + " }" + " }" + " }" + " ]" + " }" + " }" + " ]," + " \"forcing\": { " + " \"file_pattern\": \".*{{id}}.*.csv\"," + " \"path\": \"./data/forcing/\"," + " \"provider\": \"CsvPerFeature\"" + " }" + " }," + " \"time\": {" + " \"start_time\": \"2015-12-01 00:00:00\"," + " \"end_time\": \"2015-12-30 23:00:00\"," + " \"output_interval\": 3600" + " }" + "}"; +}; + +TEST_F(NetCDFCreatorTest, TestCatchmentIdentifiers) +{ + nc_creator = std::make_unique(manager_,"catchment_test", *sim_time_, 0, 1); + netCDF::NcFile& ncFile = nc_creator->get_ncfile(); + netCDF::NcVar catchments_var = ncFile.getVar("catchments"); + std::vector nc_dims = catchments_var.getDims(); + size_t len = nc_dims[0].getSize(); + int item_index = 0; + std::vector index; + index.resize(1); + std::vector catchments; + std::string catchment; + for(size_t i = 0; i < len; ++i){ + index[0] = item_index; + catchments_var.getVar(index, &catchment); + catchments.push_back(catchment.c_str()); + item_index++; + } + //delete the netcdf file that was created once the information is obtained. + ncFile.close(); + std::string ncOutputFileName = manager_->get_output_root() + "catchment_test.nc"; + const char* nc_filename = ncOutputFileName.c_str(); + int result = std::remove(nc_filename); + + ASSERT_EQ(catchments.size(), 2); + ASSERT_TRUE(std::find(catchments.begin(), catchments.end(), "cat-52") != catchments.end()); + ASSERT_TRUE(std::find(catchments.begin(), catchments.end(), "cat-67") != catchments.end()); +} + +TEST_F(NetCDFCreatorTest, TestOutputValues) +{ + nc_creator = std::make_unique(manager_,"catchment_test", *sim_time_, 0, 1); //create netcdf file or replace this with an existing file? + + //write outputs to netcdf at time index 0 for cat-52 + std::map catchment_output_values; + auto c_form = std::dynamic_pointer_cast(manager_->get_formulation("cat-52")); + double resp = c_form->get_response(0, 3600); + std::string out_resp = c_form->get_output_line_for_timestep(0); + catchment_output_values["cat-52"] = out_resp; + nc_creator->write_simulations_response_from_formulation(0, catchment_output_values); + + + auto r_c = std::dynamic_pointer_cast(manager_->get_formulation("cat-52")); + if(r_c->get_output_header_count() > 0){ + std::vectoroutput_variables = r_c->get_output_variable_names(); + std::vectoroutput_headers = r_c->get_output_header_fields(); + std::vectoroutput_units = r_c->get_output_variable_units(); + + netCDF::NcFile& ncFile = nc_creator->get_ncfile(); //get a handle to the netcdf file object + + //find the index for "cat-52" and retrieve the values for the output variables + double catchment_output_nc; + std::string output_str; + netCDF::NcVar catchments_var = ncFile.getVar("catchments"); + std::vector nc_dims = catchments_var.getDims(); + size_t len = nc_dims[0].getSize(); + size_t catchment_index = 0; + std::vector index; + index.resize(1); + std::vector catchments; + std::string catchment; + for(size_t i = 0; i < len; ++i){ + index[0] = catchment_index; + catchments_var.getVar(index, &catchment); + std::string catchm = catchment.c_str(); + if(catchm == "cat-52"){ + //use the catchment_index to query the value from the netcdf file and write it to a string. + std::vector start = {0, catchment_index}; + std::vector count = {1, 1}; + for(int var_index = 0; var_index < output_variables.size(); var_index ++){ + netCDF::NcVar out_var = ncFile.getVar(output_headers[var_index]); + out_var.getVar(start, count, &catchment_output_nc); + output_str += (output_str.empty() ? "" : ",") + std::to_string(catchment_output_nc); + } + } + catchment_index++; + } + + //delete the netcdf file that was created once the information is obtained. + ncFile.close(); + std::string ncOutputFileName = manager_->get_output_root() + "catchment_test.nc"; + const char* nc_filename = ncOutputFileName.c_str(); + int result = std::remove(nc_filename); + + ASSERT_EQ(out_resp, output_str); + } +} +#endif diff --git a/test/utils/mdframe_netcdf_Test.cpp b/test/utils/mdframe_netcdf_Test.cpp index 722cf9e0d0..e1696da379 100644 --- a/test/utils/mdframe_netcdf_Test.cpp +++ b/test/utils/mdframe_netcdf_Test.cpp @@ -3,7 +3,9 @@ #include "gtest/gtest.h" #if NGEN_WITH_NETCDF -#include +#include "netcdf/NetCDFManager.hpp" +#include "netcdf/NetCDFFile.hpp" +#include "netcdf/NetCDFVar.hpp" #endif #include "mdframe.hpp" @@ -61,40 +63,41 @@ TEST_F(mdframe_netcdf_Test, io_netcdf) df.to_netcdf(this->path); - netCDF::NcFile ex; - ex.open(this->path, netCDF::NcFile::read); + NetCDFFile ex(this->path, false, false); - const auto xdim = ex.getDim("x"); - ASSERT_FALSE(xdim.isNull()); - ASSERT_EQ(xdim.getSize(), 2); + int dim_id = ex.get_dim_id("x"); + ASSERT_FALSE(dim_id == -1); + ASSERT_EQ(ex.get_dim_size("x"), 2); - const auto ydim = ex.getDim("y"); - ASSERT_FALSE(ydim.isNull()); - ASSERT_EQ(ydim.getSize(), 2); + dim_id = ex.get_dim_id("y"); + ASSERT_FALSE(dim_id == -1); + ASSERT_EQ(ex.get_dim_size("y"), 2); - const auto xvar = ex.getVar("x"); - const auto yvar = ex.getVar("y"); - const auto vvar = ex.getVar("v"); + auto xvar = ex.get_ncvar("x"); + auto yvar = ex.get_ncvar("y"); + auto vvar = ex.get_ncvar("v"); - EXPECT_EQ(xvar.getDimCount(), 1); - EXPECT_EQ(yvar.getDimCount(), 1); - EXPECT_EQ(vvar.getDimCount(), 2); + EXPECT_EQ(xvar->get_dim_count(), 1); + EXPECT_EQ(yvar->get_dim_count(), 1); + EXPECT_EQ(vvar->get_dim_count(), 2); int xval = 0; int yval = 0; double vval = 0; for (size_t x = 0; x < 2; x++) { - xvar.getVar({ x }, &xval); + std::vector index = {x}; + int xval = xvar->get_int_value_at_index(index); EXPECT_EQ(xval, x + 1); for (size_t y = 0; y < 2; y++) { - yvar.getVar({ y }, &yval); + index = {y}; + int yval = yvar->get_int_value_at_index(index); EXPECT_EQ(yval, y + 1); - - vvar.getVar({ x, y}, &vval); + index = {x, y}; + double vval = vvar->get_dbl_value_at_index(index); EXPECT_EQ(vval, (x + 1) * (y + 1)); } } - ex.close(); + if (ex.get_ncid() >= 0) nc_close(ex.get_ncid()); #endif }