From 59831b47565a3f54f71f479ac0b302f1eea19b3a Mon Sep 17 00:00:00 2001 From: "siva.selvanathan" Date: Fri, 24 Apr 2026 16:55:12 -0400 Subject: [PATCH 1/8] Squashed commit of the following: commit 971c8308df7899c1bbf2f82a25c562c535965a74 Author: siva.selvanathan Added checks for MPI and output variables in realization to prevent NetCDF creation. Added appropriate log messages. commit b82797bd6df4f95e6dc5031258d21bdb6dbb1478 Author: siva.selvanathan Date: Mon Jan 26 21:46:41 2026 -0500 Added a couple of macro If blocks to prevent attempts to write to a single NetCDF in parallel fashion. commit cf5e694ce01e6ad9255563d41d287fa0ac6df674 Author: siva.selvanathan Date: Mon Jan 26 14:52:17 2026 -0500 Included num_proc if clause to prevent MPI Barrier being called for a single process. Also, removed an unused function from the test class. commit f4b881946abf9fe1184d60743524f77ea63d8240 Author: siva.selvanathan Date: Mon Jan 26 11:29:36 2026 -0500 Unit tests for netcdf creators. This build does not support MPI functionality. commit fff50aa1c7f7dfc99d35076c2b8a0a9890d5153d Author: siva.selvanathan Date: Tue Jan 13 21:18:32 2026 -0500 Added dependent libraries to test_ngen_simulation for a successful build. commit 915c0c3de7c0c22f33f7bdb97dbf1126d2c8d539 Author: siva.selvanathan Date: Tue Jan 13 17:24:56 2026 -0500 Changes made to enable MPI for netcdf writing. commit fc5eec51a73cafb0fd1e45430e6a9d60d7225258 Author: siva.selvanathan Date: Mon Jan 12 09:19:57 2026 -0500 Made changes to enable MPI for catchment NetCDF creation. Build is still failing. commit 43fb963507109029e919acbc2be4e79d85c9946a Author: siva.selvanathan Date: Sun Jan 11 21:11:14 2026 -0500 Added pybind include directory to check pipeline build commit 53f70227d0ec20d2c39932b11ef54c69ec9c8a57 Author: siva.selvanathan Date: Sun Jan 11 17:14:34 2026 -0500 NetCDF for catchment outputs funcitonality implemented. commit 74a624ab9f80193093478ca0902d46dd5ac9c3f6 Author: siva.selvanathan Date: Wed Dec 24 09:17:12 2025 -0500 Included proper headers and target directories to CMakeFile to make the program compilable. commit 3dcac1a054298369ded1b2f59dc7c3bca7ae17b1 Author: siva.selvanathan Date: Tue Dec 23 14:26:04 2025 -0500 Removed all class variables and constructor implementation for NetCDFCreator. Currently, the code is skeletal with the focus on getting a successful compile. commit 4358f6ac021d569867f29333a42a235c23047cc3 Merge: 6fcb1806c b44c038e0 Author: siva.selvanathan Date: Thu Dec 18 13:44:24 2025 -0500 Merge branch 'ssn_9011_netcdf_for_catchments' of https://github.com/NGWPC/ngen into ssn_9011_netcdf_for_catchments commit 6fcb1806cd85b24c94d07fdd897b62fd5d89ab13 Author: siva.selvanathan Date: Thu Dec 18 12:26:03 2025 -0500 first commit with new netcdfcreator class commit b44c038e047b20049190d5e4a44f211c66a29498 Author: siva.selvanathan Date: Thu Dec 18 12:26:03 2025 -0500 first commit with new netcdfcreator class --- CMakeLists.txt | 21 ++ include/core/Layer.hpp | 3 + include/core/NetCDFCreator.hpp | 47 +++ include/core/NgenSimulation.hpp | 9 + .../catchment/Bmi_Formulation.hpp | 18 ++ .../catchment/Formulation_Manager.hpp | 4 + src/NGen.cpp | 14 +- src/NetCDFCreator.cpp | 208 +++++++++++++ src/core/Layer.cpp | 10 + src/core/NgenSimulation.cpp | 18 +- .../catchment/Bmi_Module_Formulation.cpp | 1 + .../catchment/Bmi_Multi_Formulation.cpp | 2 +- test/CMakeLists.txt | 26 ++ test/core/NetCDFCreatorTest.cpp | 293 ++++++++++++++++++ 14 files changed, 670 insertions(+), 4 deletions(-) create mode 100644 include/core/NetCDFCreator.hpp create mode 100644 src/NetCDFCreator.cpp create mode 100644 test/core/NetCDFCreatorTest.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 28d71f09b0..fa8829162f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -392,6 +392,27 @@ if(NGEN_WITH_SQLITE) target_include_directories(partitionGenerator PUBLIC AFTER "${NGEN_INC_DIR}/geopackage") endif() +if(NGEN_WITH_NETCDF) + add_library(NetCDFCreator STATIC "${NGEN_SRC_DIR}/NetCDFCreator.cpp") + target_include_directories(NetCDFCreator PUBLIC + ${NGEN_INC_DIR} + ${NGEN_INC_DIR}/core + ${NGEN_INC_DIR}/core/catchment + ${NGEN_INC_DIR}/realizations/catchment + ${NGEN_INC_DIR}/simulation_time + ${NGEN_INC_DIR}/utilities + ${NGEN_INC_DIR}/geojson + ${NGEN_INC_DIR}/forcing + ${NGEN_INC_DIR}/core/mediator + ${Python_INCLUDE_DIRS} + ${pybind11_INCLUDE_DIR} + ${MPI_CXX_INCLUDE_DIRS} + ) + target_link_libraries(NetCDFCreator PUBLIC NGen::config_header) + target_link_libraries(NetCDFCreator PRIVATE NetCDF) + target_link_libraries(ngen PRIVATE NetCDFCreator) +endif() + target_link_libraries(partitionGenerator PUBLIC NGen::core NGen::geojson) if(NGEN_WITH_SQLITE) target_link_libraries(partitionGenerator PUBLIC NGen::geopackage) diff --git a/include/core/Layer.hpp b/include/core/Layer.hpp index 643f1b6970..1f230dd61a 100644 --- a/include/core/Layer.hpp +++ b/include/core/Layer.hpp @@ -9,6 +9,7 @@ #include "State_Exception.hpp" #include "geojson/FeatureBuilder.hpp" #include +#include namespace hy_features { @@ -116,6 +117,7 @@ namespace ngen virtual void save_state_snapshot(std::shared_ptr snapshot_saver); virtual void load_state_snapshot(std::shared_ptr snapshot_loader); virtual void load_hot_start(std::shared_ptr snapshot_loader); + std::map get_catchment_output_data_for_timestep(); protected: @@ -128,6 +130,7 @@ namespace ngen //TODO is this really required at the top level? or can this be moved to SurfaceLayer? const geojson::GeoJSON catchment_data; long output_time_index; + std::map catchment_output_values; }; } diff --git a/include/core/NetCDFCreator.hpp b/include/core/NetCDFCreator.hpp new file mode 100644 index 0000000000..e5a1fe425a --- /dev/null +++ b/include/core/NetCDFCreator.hpp @@ -0,0 +1,47 @@ +#ifndef NGEN_NETCDF_CREATOR_HPP +#define NGEN_NETCDF_CREATOR_HPP + +#include + +#if NGEN_WITH_NETCDF +#include +#include +#include +#include + +namespace netCDF { + class NcVar; + class NcFile; +} +class NetCDFCreator +{ +public: + NetCDFCreator(std::shared_ptr manager, + const std::string& output_name, Simulation_Time const& sim_time, int mpi_rank, int mpi_num_procs); + NetCDFCreator() = delete; + ~NetCDFCreator(); + + void write_simulations_response_from_formulation(size_t time_index, std::map catchment_output_values); + + netCDF::NcFile& get_ncfile(); + +protected: + void add_output_variable_info_from_formulation(); + + void retrieve_output_variables_mpi(); + + std::vector string_split(std::string str, char delimiter); + + bool create_ncfile(); + + void close_ncfile(); + +private: + std::shared_ptr catchmentNcFile; + std::shared_ptr manager_; + std::shared_ptr sim_time_; + std::vector catchments; + std::vector nc_output_variables; +}; +#endif // NGEN_WITH_NETCDF +#endif // NGEN_NETCDF_CREATOR_HPP \ No newline at end of file diff --git a/include/core/NgenSimulation.hpp b/include/core/NgenSimulation.hpp index 62744e4588..d6b4714931 100644 --- a/include/core/NgenSimulation.hpp +++ b/include/core/NgenSimulation.hpp @@ -4,8 +4,13 @@ #include #include +#include #include +#if NGEN_WITH_NETCDF + #include +#endif + namespace hy_features { class HY_Features; @@ -79,6 +84,7 @@ class NgenSimulation void save_end_of_run(std::shared_ptr snapshot_saver); // Load a snapshot of the end of a previous run. This will create a T-Route python adapter if the loader finds a unit for it and the config path is not empty. void load_hot_start(std::shared_ptr snapshot_loader, const std::string &t_route_config_file_with_path); + void create_netcdf_writer(std::shared_ptr manager, std::string nc_output_file_name, int mpi_rank, int mpi_num_procs); private: void advance_models_one_output_step(); @@ -125,6 +131,9 @@ class NgenSimulation // Serialization template will be defined and instantiated in the .cpp file template void serialize(Archive& ar); + + //Pointer to netcdfcreator to write simulation output per timestep. + std::unique_ptr nc_writer_; }; #endif diff --git a/include/realizations/catchment/Bmi_Formulation.hpp b/include/realizations/catchment/Bmi_Formulation.hpp index 91236fea42..78cae3ee88 100644 --- a/include/realizations/catchment/Bmi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Formulation.hpp @@ -207,6 +207,15 @@ namespace realization { return REQUIRED_PARAMETERS; } + /** + * Get the units for the output variables organized as a vector of strings. + * + * @return The units for the output variables organized as a vector. + */ + const std::vector &get_output_variable_units() const { + return output_variable_units; + } + virtual bool is_bmi_input_variable(const std::string &var_name) const = 0; /** @@ -265,6 +274,10 @@ namespace realization { output_header_fields = output_headers; } + void set_output_variable_units(const std::vector &output_units) { + output_variable_units = output_units; + } + /** * Set the names of variables in formulation output. * @@ -291,6 +304,11 @@ namespace realization { * the BMI module output variables accessible to the instance. */ std::vector output_variable_names; + /** + * Output units corresponding to the variables output by the realization, as defined in + * `output_variables`. + */ + std::vector output_variable_units; /** The degree of precision in output values when converting to text. */ int output_precision; diff --git a/include/realizations/catchment/Formulation_Manager.hpp b/include/realizations/catchment/Formulation_Manager.hpp index 746129b90e..fe7c02cdfc 100644 --- a/include/realizations/catchment/Formulation_Manager.hpp +++ b/include/realizations/catchment/Formulation_Manager.hpp @@ -237,6 +237,10 @@ namespace realization { return this->formulations.at(id); } + std::map> get_all_formulations() const { + return this->formulations; + } + std::shared_ptr get_domain_formulation(long id) const { return this->domain_formulations.at(id); } diff --git a/src/NGen.cpp b/src/NGen.cpp index 24dac00d07..f0831d6ddf 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -11,6 +11,10 @@ #include #include +#if NGEN_WITH_NETCDF +#include +#endif + #if NGEN_WITH_SQLITE3 #include #endif @@ -689,7 +693,13 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { std::move(nexus_indexes), mpi_rank, mpi_num_procs); - + #if NGEN_WITH_NETCDF + #if NGEN_WITH_MPI + LOG("Under MPI mode, a NetCDF file for catchment output values is not created due to limitations in NetCDFCxx4 library.", LogLevel::INFO); + #else + simulation->create_netcdf_writer(manager, "catchment_output", mpi_rank, mpi_num_procs); //create a NetCDF file only for Non-MPI run. + #endif //NGEN_WITH_MPI + #endif //NGEN_WITH_NETCDF auto time_done_init = std::chrono::steady_clock::now(); std::chrono::duration time_elapsed_init = time_done_init - time_start; LOG("[TIMING]: Init: " + std::to_string(time_elapsed_init.count()), LogLevel::INFO); @@ -703,7 +713,9 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { } } + simulation->run_catchments(); + #if NGEN_WITH_MPI MPI_Barrier(MPI_COMM_WORLD); diff --git a/src/NetCDFCreator.cpp b/src/NetCDFCreator.cpp new file mode 100644 index 0000000000..e8cd56c391 --- /dev/null +++ b/src/NetCDFCreator.cpp @@ -0,0 +1,208 @@ +#include + +#if NGEN_WITH_NETCDF +#include +#include +#include + +#if NGEN_WITH_MPI +#include +#include "parallel_utils.h" +#endif + +NetCDFCreator::NetCDFCreator(std::shared_ptr manager, + const std::string& output_name,Simulation_Time const& sim_time, int mpi_rank, int mpi_num_procs) +{ + manager_ = manager; + sim_time_ = std::make_shared(sim_time); + + //Check if there is a need to create a netcdf file (whether output variables are mentioned in realization) + if (create_ncfile()){ + try{ + catchments.reserve(manager_->get_size()); + for (auto const& formulation_info : manager->get_all_formulations()) + { + std::string catchm = formulation_info.first; + catchments.push_back(catchm); + } + + #if NGEN_WITH_MPI + if (mpi_num_procs > 1){ + std::vector all_catchments = catchments; + all_catchments = parallel::gather_strings(catchments, mpi_rank, mpi_num_procs); + if (mpi_rank == 0){ + std::sort(all_catchments.begin(), all_catchments.end()); + all_catchments.erase( + std::unique(all_catchments.begin(), all_catchments.end()), + all_catchments.end() + ); + } + catchments = parallel::broadcast_strings(all_catchments, mpi_rank, mpi_num_procs); + } + #endif + std::string ncOutputFileName = manager->get_output_root() + output_name + ".nc"; + if(mpi_rank == 0){ + catchmentNcFile = std::make_shared(ncOutputFileName, netCDF::NcFile::replace); + if(!catchmentNcFile){ + LOG("Catchment output netcdf file creation failed: " + ncOutputFileName, LogLevel::FATAL); + throw std::runtime_error("Catchment output netcdf file creation failed: " + ncOutputFileName); + } + //Add dimension and coordinate variable for time + //TO DO: create a separate function if this action is initiated from outside the class. + //TO DO: convert seconds epoch to minutes or days? + int num_timesteps = sim_time_->get_total_output_times(); + auto time_dim = catchmentNcFile->addDim("time", num_timesteps); + auto time_var = catchmentNcFile->addVar("time", NC_INT, time_dim); + time_var.putAtt("units", "Seconds since 1970-01-01 00:00:00"); + time_var.putAtt("calendar", "gregorian"); + std::vector time_epoch_seconds(num_timesteps); + time_epoch_seconds[0] = sim_time_->get_current_epoch_time(); + for(int time_index = 1; time_index < num_timesteps; time_index++) + { + sim_time_->advance_timestep(); + time_epoch_seconds[time_index] = sim_time_->get_current_epoch_time(); + } + time_var.putVar(time_epoch_seconds.data()); + + //Add dimension and coordinate variable for catchments + //TO DO: create a separate function if this action is initiated from outside the class. + auto catchments_dim = catchmentNcFile->addDim("catchments", catchments.size()); + auto catchments_var = catchmentNcFile->addVar("catchments", NC_STRING, catchments_dim); + catchments_var.putAtt("Catchment ID", "Catchment identifier in input"); + int item_index = 0; + std::vector index; + index.resize(1); + for (auto const& catchment : catchments) + { + index[0] = item_index; + catchments_var.putVar(index,catchment); + item_index++; + } + + //Add output data variables information such as headers, variable names, units to netcdf + //TO DO: change scope of this function if this is initiated from outside the class. + add_output_variable_info_from_formulation(); + + //Add global attributes + catchmentNcFile->putAtt("title", "NextGen Catchment Output Data"); + catchmentNcFile->putAtt("description", "NetCDF file containing catchment-level output data from NextGen simulation"); + catchmentNcFile->putAtt("institution", "NOAA"); + } + + #if NGEN_WITH_MPI + if (mpi_num_procs > 1) MPI_Barrier(MPI_COMM_WORLD); + #endif + + if (mpi_rank != 0){ + catchmentNcFile = std::make_shared(ncOutputFileName, netCDF::NcFile::write); + retrieve_output_variables_mpi(); + } + } + catch (const netCDF::exceptions::NcException& e){ + LOG(std::string("Error in catchments NetCDF initiation: ") + e.what(), LogLevel::FATAL); + throw std::runtime_error(std::string("Error in catchments NetCDF initiation: ") + e.what()); + } + } + else{ + LOG("No output variables/headers information provided in the realization config. A NetCDF file for the catchment output will not be created.", LogLevel::INFO); + } +} + +void NetCDFCreator::add_output_variable_info_from_formulation() +{ + typename std::map>::const_iterator it = manager_->begin(); + const auto& catchment_info = *it; + auto r_c = std::dynamic_pointer_cast(catchment_info.second); + if(r_c->get_output_header_count() > 0){ + std::vectoroutput_variables = r_c->get_output_variable_names(); + std::vectoroutput_headers = r_c->get_output_header_fields(); + std::vectoroutput_units = r_c->get_output_variable_units(); + nc_output_variables.resize(output_variables.size()); + + std::vector dims = {catchmentNcFile->getDim("time"), catchmentNcFile->getDim("catchments")}; + for(int index = 0; index < output_variables.size(); index ++){ + nc_output_variables[index] = catchmentNcFile->addVar(output_headers[index], NC_DOUBLE, dims); + nc_output_variables[index].putAtt("variable name", output_variables[index]); + nc_output_variables[index].putAtt("variable units", output_units[index]); + nc_output_variables[index].putAtt("_FillValue", NC_DOUBLE, -1.0); //TO DO: Change to another value, if recommended. + nc_output_variables[index].putAtt("missing_value", NC_DOUBLE, -2.0); //TO DO: Change to another value, if recommended. + } + } +} + +void NetCDFCreator::retrieve_output_variables_mpi() +{ + typename std::map>::const_iterator it = manager_->begin(); + const auto& catchment_info = *it; + auto r_c = std::dynamic_pointer_cast(catchment_info.second); + if(r_c->get_output_header_count() > 0){ + std::vectoroutput_headers = r_c->get_output_header_fields(); + nc_output_variables.resize(output_headers.size()); + + std::vector dims = {catchmentNcFile->getDim("time"), catchmentNcFile->getDim("catchments")}; + for(int index = 0; index < output_headers.size(); index ++){ + nc_output_variables[index] = catchmentNcFile->getVar(output_headers[index]); + } + } +} + +void NetCDFCreator::write_simulations_response_from_formulation(size_t time_index, std::map catchment_output_values) +{ + for (auto const& catchment_val : catchment_output_values) + { + std::string catchment_id = catchment_val.first; + + //iterate through catchment dimension to find the index of the catchment for writing. + //also split the comma separated outputs string to a vector of double values + std::vector catchment_output; + std::vector count = {1, 1}; + for (size_t c_index = 0; c_index < catchments.size(); ++c_index) + { + if(catchments[c_index] == catchment_id){ + catchment_output = string_split(catchment_val.second, ','); + std::vector start = {time_index, c_index}; + for(int var_index = 0; var_index < nc_output_variables.size(); ++var_index) + { + nc_output_variables[var_index].putVar(start, count, &catchment_output[var_index]); + } + break; + } + } + } +} + +std::vector NetCDFCreator::string_split(std::string str, char delimiter) +{ + std::stringstream ss(str); + std::vector res; + std::string token; + while (getline(ss, token, delimiter)) { //will return full string if no comma (or single output variable). + res.push_back(std::stod(token)); + } + return res; +} + +bool NetCDFCreator::create_ncfile() +{ + typename std::map>::const_iterator it = manager_->begin(); + const auto& catchment_info = *it; + auto r_c = std::dynamic_pointer_cast(catchment_info.second); + return (r_c->get_output_header_count() > 0) ? true : false; +} + +netCDF::NcFile& NetCDFCreator::get_ncfile(){ + return *catchmentNcFile; +} + +void NetCDFCreator::close_ncfile(){ + if (catchmentNcFile != nullptr) { + catchmentNcFile->close(); + } + catchmentNcFile = nullptr; +} + +NetCDFCreator::~NetCDFCreator() +{ + close_ncfile(); +} +#endif // NGEN_WITH_NETCDF \ No newline at end of file diff --git a/src/core/Layer.cpp b/src/core/Layer.cpp index 37021b506a..828099c81e 100644 --- a/src/core/Layer.cpp +++ b/src/core/Layer.cpp @@ -63,6 +63,13 @@ void ngen::Layer::update_models(boost::span catchment_outflows, std::string output = std::to_string(output_time_index)+","+current_timestamp+","+ r_c->get_output_line_for_timestep(output_time_index)+"\n"; r_c->write_output(output); + + //capture all the output values for this timestep to write to netcdf in non-MPI run. + #if NGEN_WITH_NETCDF + #if !NGEN_WITH_MPI + catchment_output_values[id] = r_c->get_output_line_for_timestep(output_time_index); + #endif //NGEN_WITH_MPI + #endif //NGEN_WITH_NETCDF } //TODO put this somewhere else. For now, just trying to ensure we get m^3/s into nexus output double area_sq_km; @@ -149,3 +156,6 @@ void ngen::Layer::load_hot_start(std::shared_ptr snapshot r_c->load_hot_start(snapshot_loader); } } +std::map ngen::Layer::get_catchment_output_data_for_timestep(){ + return catchment_output_values; +} diff --git a/src/core/NgenSimulation.cpp b/src/core/NgenSimulation.cpp index f2caad5831..20108a4300 100644 --- a/src/core/NgenSimulation.cpp +++ b/src/core/NgenSimulation.cpp @@ -24,7 +24,7 @@ NgenSimulation::NgenSimulation( std::unordered_map nexus_indexes, int mpi_rank, int mpi_num_procs - ) + ) : simulation_step_(0) , sim_time_(std::make_shared(sim_time)) , layers_(std::move(layers)) @@ -43,7 +43,6 @@ void NgenSimulation::run_catchments() { // Now loop some time, iterate catchments, do stuff for total number of output times auto num_times = get_num_output_times(); - for (; simulation_step_ < num_times; simulation_step_++) { // Make room for this output step's results catchment_outflows_.resize(catchment_outflows_.size() + catchment_indexes_.size(), 0.0); @@ -107,6 +106,16 @@ void NgenSimulation::advance_models_one_output_step() nexus_indexes_, simulation_step_ ); // assume update_models() calls time->advance_timestep() + + // After updating the layer, get the output data for that timestep and write to netcdf. This is currently + //set up only for a non-MPI run. + #if NGEN_WITH_NETCDF + #if !NGEN_WITH_MPI + std::map catchment_output_vals = layer->get_catchment_output_data_for_timestep(); + nc_writer_->write_simulations_response_from_formulation(simulation_step_,catchment_output_vals); + #endif //NGEN_WITH_MPI + #endif //NGEN_WITH_NETCDF + prev_layer_time = layer_next_time; } else { layer_min_next_time = prev_layer_time = layer->current_timestep_epoch_time(); @@ -387,3 +396,8 @@ void NgenSimulation::serialize(Archive& ar) { ar & nexus_indexes_; ar & nexus_downstream_flows_; } + +void NgenSimulation::create_netcdf_writer(std::shared_ptr manager, std::string nc_output_file_name, int mpi_rank, int mpi_num_procs) +{ + this->nc_writer_ = std::make_unique(manager,nc_output_file_name,*sim_time_, mpi_rank, mpi_num_procs_); +} diff --git a/src/realizations/catchment/Bmi_Module_Formulation.cpp b/src/realizations/catchment/Bmi_Module_Formulation.cpp index 359a257f88..1195e61916 100644 --- a/src/realizations/catchment/Bmi_Module_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Module_Formulation.cpp @@ -616,6 +616,7 @@ namespace realization { output_var_units[i] = get_provider_units_for_variable(names[i]); } } + set_output_variable_units(output_var_units); //check if output variable indices (for vector variables) are specified in config. If not, default to zero (first index). if(output_var_indices.size() == 0){ diff --git a/src/realizations/catchment/Bmi_Multi_Formulation.cpp b/src/realizations/catchment/Bmi_Multi_Formulation.cpp index ed5b296193..a5d2d8cb36 100644 --- a/src/realizations/catchment/Bmi_Multi_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Multi_Formulation.cpp @@ -282,6 +282,7 @@ void Bmi_Multi_Formulation::create_multi_formulation(geojson::PropertyMap proper output_var_units[i] = get_provider_units_for_variable(names[i]); } } + set_output_variable_units(output_var_units); //check if output variable indices (for vector variables) are specified in config. If not, default to zero (first index). if(output_var_indices.size() == 0){ @@ -462,7 +463,6 @@ std::string Bmi_Multi_Formulation::get_output_line_for_timestep(int timestep, st if (timestep != (next_time_step_index - 1)) { throw std::invalid_argument("Only current time step valid when getting multi-module BMI formulation output"); } - // Start by first checking whether we are just using the last module's values if (is_out_vars_from_last_mod) { // The default behavior, which means we either diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a6b26ad200..04533d7e25 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -61,6 +61,7 @@ endif() add_executable(test_all) target_link_libraries(test_all PRIVATE gtest gtest_main) +target_link_libraries(test_all PRIVATE NetCDFCreator) set_target_properties(test_all PROPERTIES FOLDER test) target_include_directories(test_all PRIVATE ${CMAKE_CURRENT_LIST_DIR}/bmi) @@ -460,7 +461,12 @@ ngen_add_test( NGen::geojson NGen::logging NGen::core_mediator + NGen::realizations_catchment + NGen::forcing NGen::ngen_bmi + NetCDFCreator + REQUIRES + NGEN_WITH_NETCDF ) ########################### Netcdf Forcing Tests @@ -479,6 +485,26 @@ ngen_add_test( NGEN_WITH_NETCDF ) +########################## NetCDFCreator Class Tests +ngen_add_test( + test_netcdf_creator + OBJECTS + core/NetCDFCreatorTest.cpp + LIBRARIES + NGen::core + NGen::geopackage + NGen::geojson + NGen::logging + NGen::core_catchment + NGen::core_mediator + NGen::realizations_catchment + NGen::forcing + NGen::ngen_bmi + NetCDFCreator + REQUIRES + NGEN_WITH_NETCDF +) + ########################## Primary Combined Unit Test Target ngen_add_test( test_unit diff --git a/test/core/NetCDFCreatorTest.cpp b/test/core/NetCDFCreatorTest.cpp new file mode 100644 index 0000000000..f99323dca0 --- /dev/null +++ b/test/core/NetCDFCreatorTest.cpp @@ -0,0 +1,293 @@ +#include + + +#if NGEN_WITH_NETCDF +#include "gtest/gtest.h" +#include +#include +#include "Bmi_Testing_Util.hpp" +#include "FileChecker.h" +#include +#include +#include +#include + +class NetCDFCreatorTest : public ::testing::Test { + protected: + std::stringstream stream_; + std::shared_ptr manager_; + geojson::GeoJSON fabric = std::make_shared(); + std::shared_ptr sim_time_; + std::map> calculated_results; + std::unique_ptr nc_creator; + + typedef struct tm time_type; + std::shared_ptr start_date_time; + std::shared_ptr end_date_time; + + void SetUp() override { + SetupFormulationManager(); + } + + void TearDown() override { + + } + + //Construct a Formulation Manager and Simulation Time objects + void SetupFormulationManager() { + stream_ << fix_paths(example_config); + boost::property_tree::ptree realization_config; + boost::property_tree::json_parser::read_json(stream_, realization_config); + manager_ = std::make_shared(realization_config); + + auto possible_simulation_time = realization_config.get_child_optional("time"); + if (!possible_simulation_time) { + std::string throw_msg; throw_msg.assign("ERROR: No simulation time period defined."); + LOG(throw_msg, LogLevel::WARNING); + throw std::runtime_error(throw_msg); + } + + auto simulation_time_config = realization::config::Time(*possible_simulation_time).make_params(); + sim_time_ = std::make_shared(simulation_time_config); + + std::ostream* raw_pointer = &std::cout; + std::shared_ptr s_ptr(raw_pointer, [](void*) {}); + utils::StreamHandler catchment_output(s_ptr); + + this->add_feature("cat-52"); + this->add_feature("cat-67"); + manager_->read(simulation_time_config, fabric, catchment_output); + } + + std::string fix_paths(std::string json) + { + std::vector forcing_paths = { + "./data/forcing/cat-52_2015-12-01 00_00_00_2015-12-30 23_00_00.csv", + "./data/forcing/cat-67_2015-12-01 00_00_00_2015-12-30 23_00_00.csv", + }; + std::vector v = {}; + for(unsigned int i = 0; i < path_options.size(); i++){ + v.push_back( path_options[i] + "data/forcing" ); + } + std::string dir = utils::FileChecker::find_first_readable(v); + if(dir != ""){ + std::string remove = "\"./data/forcing/\""; + std::string replace = "\""+dir+"\""; + boost::replace_all(json, remove , replace); + } + + //BMI_CPP_INIT_DIR_PATH + replace_paths(json, "{{BMI_CPP_INIT_DIR_PATH}}", "data/bmi/test_bmi_cpp"); + //EXTERN_DIR_PATH + replace_paths(json, "{{EXTERN_LIB_DIR_PATH}}", "extern/test_bmi_cpp/cmake_build/"); + + for (unsigned int i = 0; i < forcing_paths.size(); i++) { + if(json.find(forcing_paths[i]) == std::string::npos){ + continue; + } + std::vector v = {}; + for (unsigned int j = 0; j < path_options.size(); j++) { + v.push_back(path_options[j] + forcing_paths[i]); + } + std::string right_path = utils::FileChecker::find_first_readable(v); + if(right_path != forcing_paths[i]){ + std::cerr<<"Replacing "< v{path_options.size()}; + for(unsigned int i = 0; i < path_options.size(); i++) + v[i] = path_options[i] + replacement; + + const std::string dir = utils::FileChecker::find_first_readable(v); + if (dir == "") return; + + boost::replace_all(input, pattern, dir); + } + + void add_feature(std::string id) + { + geojson::three_dimensional_coordinates three_dimensions { + { + {1.0, 2.0}, + {3.0, 4.0}, + {5.0, 6.0} + }, + { + {7.0, 8.0}, + {9.0, 10.0}, + {11.0, 12.0} + } + }; + std::vector bounding_box{1.0, 2.0}; + geojson::PropertyMap properties{}; + + geojson::Feature feature = std::make_shared(geojson::PolygonFeature( + geojson::polygon(three_dimensions), + id, + properties + )); + + fabric->add_feature(feature); + } + + + std::vector path_options = { + "", + "../", + "../../", + "./test/", + "../test/", + "../../test/" + + }; + + const std::string example_config = + "{" + " \"global\": {" + " \"formulations\": [" + " {" + " \"name\": \"bmi_multi\"," + " \"params\": {" + " \"model_type_name\": \"bmi_multi_c++\"," + " \"forcing_file\": \"\"," + " \"init_config\": \"\"," + " \"allow_exceed_end_time\": true," + " \"main_output_variable\": \"OUTPUT_VAR_4\"," + " \"uses_forcing_file\": false," + " \"modules\": [" + " {" + " \"name\": \"bmi_c++\"," + " \"params\": {" + " \"model_type_name\": \"test_bmi_c++\"," + " \"library_file\": \"{{EXTERN_LIB_DIR_PATH}}" BMI_TEST_CPP_LIB_NAME "\"," + " \"init_config\": \"{{BMI_CPP_INIT_DIR_PATH}}/test_bmi_cpp_config_2.txt\"," + " \"allow_exceed_end_time\": true," + " \"main_output_variable\": \"OUTPUT_VAR_4\"," + " \"uses_forcing_file\": false," + " \"model_params\": {" + " \"MODEL_VAR_1\": {" + " \"source\": \"hydrofabric\"," + " \"from\": \"val\"" + " }," + " \"MODEL_VAR_2\": {" + " \"source\": \"hydrofabric\"" + " }" + " }," + " \"" BMI_REALIZATION_CFG_PARAM_OPT__VAR_STD_NAMES "\": {" + " \"INPUT_VAR_1\": \"APCP_surface\"," + " \"INPUT_VAR_2\": \"APCP_surface\"" + " }" + " }" + " }" + " ]" + " }" + " }" + " ]," + " \"forcing\": { " + " \"file_pattern\": \".*{{id}}.*.csv\"," + " \"path\": \"./data/forcing/\"," + " \"provider\": \"CsvPerFeature\"" + " }" + " }," + " \"time\": {" + " \"start_time\": \"2015-12-01 00:00:00\"," + " \"end_time\": \"2015-12-30 23:00:00\"," + " \"output_interval\": 3600" + " }" + "}"; +}; + +TEST_F(NetCDFCreatorTest, TestCatchmentIdentifiers) +{ + nc_creator = std::make_unique(manager_,"catchment_test", *sim_time_, 0, 1); + netCDF::NcFile& ncFile = nc_creator->get_ncfile(); + netCDF::NcVar catchments_var = ncFile.getVar("catchments"); + std::vector nc_dims = catchments_var.getDims(); + size_t len = nc_dims[0].getSize(); + int item_index = 0; + std::vector index; + index.resize(1); + std::vector catchments; + std::string catchment; + for(size_t i = 0; i < len; ++i){ + index[0] = item_index; + catchments_var.getVar(index, &catchment); + catchments.push_back(catchment.c_str()); + item_index++; + } + //delete the netcdf file that was created once the information is obtained. + ncFile.close(); + std::string ncOutputFileName = manager_->get_output_root() + "catchment_test.nc"; + const char* nc_filename = ncOutputFileName.c_str(); + int result = std::remove(nc_filename); + + ASSERT_EQ(catchments.size(), 2); + ASSERT_TRUE(std::find(catchments.begin(), catchments.end(), "cat-52") != catchments.end()); + ASSERT_TRUE(std::find(catchments.begin(), catchments.end(), "cat-67") != catchments.end()); +} + +TEST_F(NetCDFCreatorTest, TestOutputValues) +{ + nc_creator = std::make_unique(manager_,"catchment_test", *sim_time_, 0, 1); //create netcdf file or replace this with an existing file? + + //write outputs to netcdf at time index 0 for cat-52 + std::map catchment_output_values; + auto c_form = std::dynamic_pointer_cast(manager_->get_formulation("cat-52")); + double resp = c_form->get_response(0, 3600); + std::string out_resp = c_form->get_output_line_for_timestep(0); + catchment_output_values["cat-52"] = out_resp; + nc_creator->write_simulations_response_from_formulation(0, catchment_output_values); + + + auto r_c = std::dynamic_pointer_cast(manager_->get_formulation("cat-52")); + if(r_c->get_output_header_count() > 0){ + std::vectoroutput_variables = r_c->get_output_variable_names(); + std::vectoroutput_headers = r_c->get_output_header_fields(); + std::vectoroutput_units = r_c->get_output_variable_units(); + + netCDF::NcFile& ncFile = nc_creator->get_ncfile(); //get a handle to the netcdf file object + + //find the index for "cat-52" and retrieve the values for the output variables + double catchment_output_nc; + std::string output_str; + netCDF::NcVar catchments_var = ncFile.getVar("catchments"); + std::vector nc_dims = catchments_var.getDims(); + size_t len = nc_dims[0].getSize(); + size_t catchment_index = 0; + std::vector index; + index.resize(1); + std::vector catchments; + std::string catchment; + for(size_t i = 0; i < len; ++i){ + index[0] = catchment_index; + catchments_var.getVar(index, &catchment); + std::string catchm = catchment.c_str(); + if(catchm == "cat-52"){ + //use the catchment_index to query the value from the netcdf file and write it to a string. + std::vector start = {0, catchment_index}; + std::vector count = {1, 1}; + for(int var_index = 0; var_index < output_variables.size(); var_index ++){ + netCDF::NcVar out_var = ncFile.getVar(output_headers[var_index]); + out_var.getVar(start, count, &catchment_output_nc); + output_str += (output_str.empty() ? "" : ",") + std::to_string(catchment_output_nc); + } + } + catchment_index++; + } + + //delete the netcdf file that was created once the information is obtained. + ncFile.close(); + std::string ncOutputFileName = manager_->get_output_root() + "catchment_test.nc"; + const char* nc_filename = ncOutputFileName.c_str(); + int result = std::remove(nc_filename); + + ASSERT_EQ(out_resp, output_str); + } +} +#endif From 59c5357e4f08a5118f8eedac09da958015362da5 Mon Sep 17 00:00:00 2001 From: "siva.selvanathan" Date: Wed, 29 Apr 2026 09:27:22 -0400 Subject: [PATCH 2/8] Made updates to CmakeLists and netcdf creator class to accomodate ewts updates with logger. --- CMakeLists.txt | 3 +++ src/NetCDFCreator.cpp | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fa8829162f..13bf14d03c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -394,7 +394,10 @@ endif() if(NGEN_WITH_NETCDF) add_library(NetCDFCreator STATIC "${NGEN_SRC_DIR}/NetCDFCreator.cpp") + set_target_properties(NetCDFCreator PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib + ) target_include_directories(NetCDFCreator PUBLIC + ${EWTS_PREFIX}/include ${NGEN_INC_DIR} ${NGEN_INC_DIR}/core ${NGEN_INC_DIR}/core/catchment diff --git a/src/NetCDFCreator.cpp b/src/NetCDFCreator.cpp index e8cd56c391..3dd8d7b0e2 100644 --- a/src/NetCDFCreator.cpp +++ b/src/NetCDFCreator.cpp @@ -3,7 +3,7 @@ #if NGEN_WITH_NETCDF #include #include -#include +#include "ewts_ngen/logger.hpp" #if NGEN_WITH_MPI #include From 0dd2a9e9bcccf4236e1c9784bfe5d8736a411446 Mon Sep 17 00:00:00 2001 From: "siva.selvanathan" Date: Thu, 30 Apr 2026 10:48:03 -0400 Subject: [PATCH 3/8] Changed logger path to ewts logger in the unit test cpp file. --- test/core/NetCDFCreatorTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core/NetCDFCreatorTest.cpp b/test/core/NetCDFCreatorTest.cpp index f99323dca0..4eac0391cb 100644 --- a/test/core/NetCDFCreatorTest.cpp +++ b/test/core/NetCDFCreatorTest.cpp @@ -10,7 +10,7 @@ #include #include #include -#include +#include "ewts_ngen/logger.hpp" class NetCDFCreatorTest : public ::testing::Test { protected: From dc9e1417507e179da181765ac0950190cb6145f1 Mon Sep 17 00:00:00 2001 From: "siva.selvanathan" Date: Wed, 13 May 2026 13:00:44 -0400 Subject: [PATCH 4/8] Commit with C++ wrappers for using NetCDF C library. It also includes changes to NgenSimulation and NetCDF Data provider. The old C++ class NetCDFCreator hasn't been removed yet. --- CMakeLists.txt | 354 ++++++++++-------- include/core/NgenSimulation.hpp | 22 +- .../forcing/NetCDFPerFeatureDataProvider.hpp | 16 +- include/netcdf/NetCDFFile.hpp | 102 +++++ include/netcdf/NetCDFManager.hpp | 108 ++++++ include/netcdf/NetCDFVar.hpp | 52 +++ src/NGen.cpp | 4 +- src/core/NgenSimulation.cpp | 13 +- src/forcing/NetCDFPerFeatureDataProvider.cpp | 119 +++--- src/netcdf/NetCDFFile.cpp | 354 ++++++++++++++++++ src/netcdf/NetCDFManager.cpp | 295 +++++++++++++++ src/netcdf/NetCDFVar.cpp | 186 +++++++++ 12 files changed, 1382 insertions(+), 243 deletions(-) create mode 100644 include/netcdf/NetCDFFile.hpp create mode 100644 include/netcdf/NetCDFManager.hpp create mode 100644 include/netcdf/NetCDFVar.hpp create mode 100644 src/netcdf/NetCDFFile.cpp create mode 100644 src/netcdf/NetCDFManager.cpp create mode 100644 src/netcdf/NetCDFVar.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 13bf14d03c..f471abd588 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,12 +20,13 @@ cmake_policy(SET CMP0074 NEW) # find_package() uses upper-case _ROOT variables in conjunction with CMP0074. # https://cmake.org/cmake/help/latest/policy/CMP0144.html if(POLICY CMP0144) - cmake_policy(SET CMP0144 NEW) +cmake_policy(SET CMP0144 NEW) endif() # Project Variables =========================================================== set(NGEN_ROOT_DIR "${CMAKE_CURRENT_LIST_DIR}") set(NGEN_SRC_DIR "${NGEN_ROOT_DIR}/src") +set(NGEN_NCF_DIR "${NGEN_ROOT_DIR}/src/netcdf") set(NGEN_INC_DIR "${NGEN_ROOT_DIR}/include") set(NGEN_EXT_DIR "${NGEN_ROOT_DIR}/extern") set(NGEN_MOD_DIR "${NGEN_ROOT_DIR}/cmake") @@ -85,28 +86,28 @@ cmake_dependent_option(NGEN_WITH_EXTERN_NOAH_OWP_MODULAR "Build with extern-dist # Allow enabling of all extern models dependent on NGen options given. option(NGEN_WITH_EXTERN_ALL OFF) if(NGEN_WITH_EXTERN_ALL) - set(NGEN_WITH_EXTERN_SLOTH ON) - - # Ensure C support is enabled - if(NGEN_WITH_BMI_C) - set(NGEN_WITH_EXTERN_TOPMODEL ON) - set(NGEN_WITH_EXTERN_CFE ON) - set(NGEN_WITH_EXTERN_PET ON) - else() - message(FATAL_ERROR "NGEN_WITH_EXTERN_ALL set to ${NGEN_WITH_EXTERN_ALL}, but NGEN_WITH_BMI_C set to ${NGEN_WITH_BMI_C}") - endif() - - # Ensure Fortran support is enabled - if(NGEN_WITH_BMI_FORTRAN) - set(NGEN_WITH_EXTERN_NOAH_OWP_MODULAR ON) - else() - message(FATAL_ERROR "NGEN_WITH_EXTERN_ALL set to ${NGEN_WITH_EXTERN_ALL}, but NGEN_WITH_BMI_FORTRAN set to ${NGEN_WITH_BMI_FORTRAN}") - endif() - - # Ensure Python support is enabled - if(NOT NGEN_WITH_PYTHON) - message(FATAL_ERROR "NGEN_WITH_EXTERN_ALL set to ${NGEN_WITH_EXTERN_ALL}, but NGEN_WITH_PYTHON set to ${NGEN_WITH_PYTHON}") - endif() +set(NGEN_WITH_EXTERN_SLOTH ON) + +# Ensure C support is enabled +if(NGEN_WITH_BMI_C) +set(NGEN_WITH_EXTERN_TOPMODEL ON) +set(NGEN_WITH_EXTERN_CFE ON) +set(NGEN_WITH_EXTERN_PET ON) + else() +message(FATAL_ERROR "NGEN_WITH_EXTERN_ALL set to ${NGEN_WITH_EXTERN_ALL}, but NGEN_WITH_BMI_C set to ${NGEN_WITH_BMI_C}") + endif() + +# Ensure Fortran support is enabled +if(NGEN_WITH_BMI_FORTRAN) +set(NGEN_WITH_EXTERN_NOAH_OWP_MODULAR ON) + else() +message(FATAL_ERROR "NGEN_WITH_EXTERN_ALL set to ${NGEN_WITH_EXTERN_ALL}, but NGEN_WITH_BMI_FORTRAN set to ${NGEN_WITH_BMI_FORTRAN}") + endif() + +# Ensure Python support is enabled +if(NOT NGEN_WITH_PYTHON) +message(FATAL_ERROR "NGEN_WITH_EXTERN_ALL set to ${NGEN_WITH_EXTERN_ALL}, but NGEN_WITH_PYTHON set to ${NGEN_WITH_PYTHON}") + endif() endif() # Project ===================================================================== @@ -119,6 +120,8 @@ set(CMAKE_CXX_VISIBILITY_PRESET hidden) # Define project version and use via generated config header project(ngen VERSION 0.3.0) +set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) +set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) add_executable(ngen "${NGEN_SRC_DIR}/NGen.cpp") @@ -126,15 +129,15 @@ add_executable(ngen "${NGEN_SRC_DIR}/NGen.cpp") find_package(ewts CONFIG REQUIRED) if(DEFINED ewts_VERSION AND NOT "${ewts_VERSION}" STREQUAL "") - set(NGEN_EWTS_VERSION "${ewts_VERSION}") +set(NGEN_EWTS_VERSION "${ewts_VERSION}") else() - set(NGEN_EWTS_VERSION "") +set(NGEN_EWTS_VERSION "") endif() if(DEFINED EWTS_NGWPC_VERSION AND NOT "${EWTS_NGWPC_VERSION}" STREQUAL "") - set(NGEN_EWTS_NGWPC_VERSION "${EWTS_NGWPC_VERSION}") +set(NGEN_EWTS_NGWPC_VERSION "${EWTS_NGWPC_VERSION}") else() - set(NGEN_EWTS_NGWPC_VERSION "") +set(NGEN_EWTS_NGWPC_VERSION "") endif() get_filename_component(EWTS_PREFIX "${ewts_DIR}" DIRECTORY) @@ -151,40 +154,40 @@ target_link_libraries(ngen PRIVATE ewts::ewts_ngen_bridge) # ----------------------------------------------------------------------------- # Check if NGen is the main project (e.g. development environment) if(PROJECT_NAME STREQUAL CMAKE_PROJECT_NAME) - set(NGEN_IS_MAIN_PROJECT ON) +set(NGEN_IS_MAIN_PROJECT ON) else() - set(NGEN_IS_MAIN_PROJECT OFF) +set(NGEN_IS_MAIN_PROJECT OFF) endif() # ----------------------------------------------------------------------------- # If Coverage is enabled, include coverage module and # set compiler flags/linkage to accomodate coverage. if(NGEN_WITH_COVERAGE) - include(CodeCoverage) - append_coverage_compiler_flags() +include(CodeCoverage) +append_coverage_compiler_flags() endif() # ----------------------------------------------------------------------------- # If MPI support is enabled, set the C++ compiler to "mpicxx" if(NGEN_WITH_MPI) - find_package(MPI REQUIRED) +find_package(MPI REQUIRED) endif() # ----------------------------------------------------------------------------- # TODO: Used for compatibility on Hera. Refactor this in the future. # https://www.intel.com/content/www/us/en/develop/documentation/oneapi-dpcpp-cpp-compiler-dev-guide-and-reference/top/compiler-setup/use-the-command-line/use-cmake-with-the-compiler.html if(INTEL_DPCPP) - find_package(IntelDPCPP REQUIRED) +find_package(IntelDPCPP REQUIRED) endif() # ----------------------------------------------------------------------------- # Account for different OS and how that impacts shared lib file names if(WIN32) - message(FATAL_ERROR "Windows platforms are not currently supported") +message(FATAL_ERROR "Windows platforms are not currently supported") elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin") - set(NGEN_SHARED_LIB_EXTENSION "dylib") +set(NGEN_SHARED_LIB_EXTENSION "dylib") else() - set(NGEN_SHARED_LIB_EXTENSION "so") +set(NGEN_SHARED_LIB_EXTENSION "so") endif() add_compile_definitions(NGEN_SHARED_LIB_EXTENSION) @@ -194,124 +197,124 @@ set(Boost_USE_STATIC_LIBS OFF) set(Boost_USE_MULTITHREADED ON) set(Boost_USE_STATIC_RUNTIME OFF) if(CMAKE_CXX_STANDARD LESS 17) - # requires non-header filesystem for state saving if C++ 11 or lower - find_package(Boost 1.86.0 REQUIRED COMPONENTS system filesystem) +# requires non-header filesystem for state saving if C++ 11 or lower +find_package(Boost 1.86.0 REQUIRED COMPONENTS system filesystem) else() - find_package(Boost 1.86.0 REQUIRED) +find_package(Boost 1.86.0 REQUIRED) endif() # ----------------------------------------------------------------------------- if(NGEN_WITH_SQLITE) - find_package(SQLite3 REQUIRED) - add_compile_definitions(NGEN_WITH_SQLITE3) +find_package(SQLite3 REQUIRED) +add_compile_definitions(NGEN_WITH_SQLITE3) endif() # ----------------------------------------------------------------------------- if(NGEN_WITH_UDUNITS) - find_package(UDUNITS2 REQUIRED) - add_compile_definitions(NGEN_WITH_UDUNITS) +find_package(UDUNITS2 REQUIRED) +add_compile_definitions(NGEN_WITH_UDUNITS) endif() # ----------------------------------------------------------------------------- -if(NGEN_WITH_NETCDF) - find_package(NetCDF COMPONENTS CXX) - - if(NOT NetCDF_CXX_FOUND) - find_package(NetCDF REQUIRED) - message(INFO " Using internal copy of netcdf-cxx4") - add_external_subdirectory( - SOURCE "${NGEN_EXT_DIR}/netcdf-cxx4" - OUTPUT "${PROJECT_BINARY_DIR}/extern-libs/netcdf-cxx4" - GIT_UPDATE "${NGEN_EXT_DIR}/netcdf-cxx4/netcdf-cxx4" - IMPORTS netcdf-cxx4 - ) - set(NetCDF_CXX_INCLUDE_DIR "${NGEN_EXT_DIR}/netcdf-cxx4/netcdf-cxx4/cxx4") - set(NetCDF_CXX_LIBRARY "${PROJECT_BINARY_DIR}/extern-libs/netcdf-cxx4/libnetcdf-cxx4.a") - target_link_libraries(NetCDF INTERFACE netcdf-cxx4) - endif() -endif() +# if(NGEN_WITH_NETCDF) +# find_package(NetCDF COMPONENTS CXX) + +# if(NOT NetCDF_CXX_FOUND) +# find_package(NetCDF REQUIRED) +# message(INFO " Using internal copy of netcdf-cxx4") +# add_external_subdirectory( +# SOURCE "${NGEN_EXT_DIR}/netcdf-cxx4" +# OUTPUT "${PROJECT_BINARY_DIR}/extern-libs/netcdf-cxx4" +# GIT_UPDATE "${NGEN_EXT_DIR}/netcdf-cxx4/netcdf-cxx4" +# IMPORTS netcdf-cxx4 +# ) +# set(NetCDF_CXX_INCLUDE_DIR "${NGEN_EXT_DIR}/netcdf-cxx4/netcdf-cxx4/cxx4") +# set(NetCDF_CXX_LIBRARY "${PROJECT_BINARY_DIR}/extern-libs/netcdf-cxx4/libnetcdf-cxx4.a") +# target_link_libraries(NetCDF INTERFACE netcdf-cxx4) +# endif() +# endif() if(NGEN_WITH_EXTERN_SLOTH) - # This is set because SLOTH will link to gtest by default, - # which causes a configure error. - set(PACKAGE_TESTS OFF CACHE BOOL "SLoTH testing") - add_external_subdirectory( - SOURCE "${NGEN_EXT_DIR}/sloth" - OUTPUT "${NGEN_EXT_DIR}/sloth/cmake_build" - GIT_UPDATE "${NGEN_EXT_DIR}/sloth" - IMPORTS slothmodel - ) - set_target_properties(slothmodel PROPERTIES CXX_VISIBILITY_PRESET default) +# This is set because SLOTH will link to gtest by default, +# which causes a configure error. +set(PACKAGE_TESTS OFF CACHE BOOL "SLoTH testing") +add_external_subdirectory( +SOURCE "${NGEN_EXT_DIR}/sloth" +OUTPUT "${NGEN_EXT_DIR}/sloth/cmake_build" +GIT_UPDATE "${NGEN_EXT_DIR}/sloth" +IMPORTS slothmodel +) +set_target_properties(slothmodel PROPERTIES CXX_VISIBILITY_PRESET default) endif() # ----------------------------------------------------------------------------- # Handle several steps for BMI C library logic and dependencies, at top level, if functionality is turned on if(NGEN_WITH_BMI_C) - if(NGEN_WITH_EXTERN_TOPMODEL) - add_external_subdirectory( - SOURCE "${NGEN_EXT_DIR}/topmodel" - OUTPUT "${NGEN_EXT_DIR}/topmodel/cmake_build" - GIT_UPDATE "${NGEN_EXT_DIR}/topmodel/topmodel" - IMPORTS topmodelbmi - ) - endif() - - if(NGEN_WITH_EXTERN_CFE) - add_external_subdirectory( - SOURCE "${NGEN_EXT_DIR}/cfe" - OUTPUT "${NGEN_EXT_DIR}/cfe/cmake_build" - GIT_UPDATE "${NGEN_EXT_DIR}/cfe/cfe" - IMPORTS cfebmi - ) - endif() - - if(NGEN_WITH_EXTERN_PET) - add_external_subdirectory( - SOURCE "${NGEN_EXT_DIR}/evapotranspiration/evapotranspiration" - OUTPUT "${NGEN_EXT_DIR}/evapotranspiration/evapotranspiration/cmake_build" - GIT_UPDATE "${NGEN_EXT_DIR}/evapotranspiration/evapotranspiration" - IMPORTS petbmi - ) - endif() +if(NGEN_WITH_EXTERN_TOPMODEL) +add_external_subdirectory( +SOURCE "${NGEN_EXT_DIR}/topmodel" +OUTPUT "${NGEN_EXT_DIR}/topmodel/cmake_build" +GIT_UPDATE "${NGEN_EXT_DIR}/topmodel/topmodel" +IMPORTS topmodelbmi +) + endif() + +if(NGEN_WITH_EXTERN_CFE) +add_external_subdirectory( +SOURCE "${NGEN_EXT_DIR}/cfe" +OUTPUT "${NGEN_EXT_DIR}/cfe/cmake_build" +GIT_UPDATE "${NGEN_EXT_DIR}/cfe/cfe" +IMPORTS cfebmi +) + endif() + +if(NGEN_WITH_EXTERN_PET) +add_external_subdirectory( +SOURCE "${NGEN_EXT_DIR}/evapotranspiration/evapotranspiration" +OUTPUT "${NGEN_EXT_DIR}/evapotranspiration/evapotranspiration/cmake_build" +GIT_UPDATE "${NGEN_EXT_DIR}/evapotranspiration/evapotranspiration" +IMPORTS petbmi +) + endif() endif() # ----------------------------------------------------------------------------- # Configure whether Fortran BMI functionality is active if(NGEN_WITH_BMI_FORTRAN) - add_external_subdirectory( - SOURCE "${NGEN_EXT_DIR}/iso_c_fortran_bmi" - OUTPUT "${NGEN_EXT_DIR}/iso_c_fortran_bmi/cmake_build" - IMPORTS iso_c_bmi - ) +add_external_subdirectory( +SOURCE "${NGEN_EXT_DIR}/iso_c_fortran_bmi" +OUTPUT "${NGEN_EXT_DIR}/iso_c_fortran_bmi/cmake_build" +IMPORTS iso_c_bmi +) - if(NGEN_WITH_EXTERN_NOAH_OWP_MODULAR) - add_external_subdirectory( - SOURCE "${NGEN_EXT_DIR}/noah-owp-modular" - OUTPUT "${NGEN_EXT_DIR}/noah-owp-modular/cmake_build" - GIT_UPDATE "${NGEN_EXT_DIR}/noah-owp-modular/noah-owp-modular" - IMPORTS surfacebmi - ) - endif() +if(NGEN_WITH_EXTERN_NOAH_OWP_MODULAR) +add_external_subdirectory( +SOURCE "${NGEN_EXT_DIR}/noah-owp-modular" +OUTPUT "${NGEN_EXT_DIR}/noah-owp-modular/cmake_build" +GIT_UPDATE "${NGEN_EXT_DIR}/noah-owp-modular/noah-owp-modular" +IMPORTS surfacebmi +) + endif() endif() # ----------------------------------------------------------------------------- if(NGEN_WITH_PYTHON) - find_package(Python 3.6.8 REQUIRED COMPONENTS Interpreter Development NumPy) - if(${Python_NumPy_VERSION} VERSION_GREATER_EQUAL 2) - message(FATAL_ERROR "Found numpy version \"${Python_NumPy_VERSION}\"; numpy>=2.0.0 is not currently supported") - endif() - set(PYTHON_EXECUTABLE ${Python_EXECUTABLE}) # Case-sensitive difference - add_subdirectory(extern/pybind11 pybind11) +find_package(Python 3.6.8 REQUIRED COMPONENTS Interpreter Development NumPy) +if(${Python_NumPy_VERSION} VERSION_GREATER_EQUAL 2) +message(FATAL_ERROR "Found numpy version \"${Python_NumPy_VERSION}\"; numpy>=2.0.0 is not currently supported") + endif() +set(PYTHON_EXECUTABLE ${Python_EXECUTABLE}) # Case-sensitive difference +add_subdirectory(extern/pybind11 pybind11) endif() # ----------------------------------------------------------------------------- if(NGEN_QUIET) - set(UDUNITS_QUIET true) - add_compile_definitions(NGEN_QUIET) +set(UDUNITS_QUIET true) +add_compile_definitions(NGEN_QUIET) endif() if(UDUNITS_QUIET) - add_compile_definitions(UDUNITS_QUIET) +add_compile_definitions(UDUNITS_QUIET) endif() # ----------------------------------------------------------------------------- @@ -324,23 +327,23 @@ add_library(NGen::config_header ALIAS ngen_config_header) target_include_directories(ngen_config_header INTERFACE "${CMAKE_CURRENT_BINARY_DIR}/include") if(NGEN_WITH_EXTERN_SLOTH) - add_dependencies(ngen slothmodel) +add_dependencies(ngen slothmodel) endif() if(NGEN_WITH_EXTERN_TOPMODEL) - add_dependencies(ngen topmodelbmi) +add_dependencies(ngen topmodelbmi) endif() if(NGEN_WITH_EXTERN_CFE) - add_dependencies(ngen cfebmi) +add_dependencies(ngen cfebmi) endif() if(NGEN_WITH_EXTERN_PET) - add_dependencies(ngen petbmi) +add_dependencies(ngen petbmi) endif() if(NGEN_WITH_EXTERN_NOAH_OWP_MODULAR) - add_dependencies(ngen surfacebmi) +add_dependencies(ngen surfacebmi) endif() target_include_directories(ngen PUBLIC "${NGEN_INC_DIR}") @@ -359,29 +362,29 @@ add_subdirectory("src/utilities/python") add_subdirectory("src/utilities/bmi") target_link_libraries(ngen - PUBLIC - NGen::config_header - NGen::core - NGen::core_catchment - NGen::core_nexus - NGen::geojson - NGen::realizations_catchment - NGen::forcing - NGen::core_mediator - NGen::logging - NGen::parallel - NGen::state_save_restore - NGen::bmi_protocols +PUBLIC +NGen::config_header +NGen::core +NGen::core_catchment +NGen::core_nexus +NGen::geojson +NGen::realizations_catchment +NGen::forcing +NGen::core_mediator +NGen::logging +NGen::parallel +NGen::state_save_restore +NGen::bmi_protocols ) if(NGEN_WITH_SQLITE) - add_subdirectory("src/geopackage") - target_link_libraries(ngen PUBLIC NGen::geopackage) +add_subdirectory("src/geopackage") +target_link_libraries(ngen PUBLIC NGen::geopackage) endif() if(NGEN_WITH_ROUTING) - add_subdirectory("src/routing") - target_link_libraries(ngen PUBLIC NGen::routing) +add_subdirectory("src/routing") +target_link_libraries(ngen PUBLIC NGen::routing) endif() add_executable(partitionGenerator src/partitionGenerator.cpp) @@ -389,16 +392,25 @@ target_link_libraries(partitionGenerator PRIVATE ewts::ewts_ngen_bridge) target_link_libraries(partitionGenerator PUBLIC NGen::logging) target_include_directories(partitionGenerator PUBLIC "${PROJECT_BINARY_DIR}/include") if(NGEN_WITH_SQLITE) - target_include_directories(partitionGenerator PUBLIC AFTER "${NGEN_INC_DIR}/geopackage") +target_include_directories(partitionGenerator PUBLIC AFTER "${NGEN_INC_DIR}/geopackage") endif() if(NGEN_WITH_NETCDF) - add_library(NetCDFCreator STATIC "${NGEN_SRC_DIR}/NetCDFCreator.cpp") - set_target_properties(NetCDFCreator PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib + set(HDF5_INSTALL "$ENV{HOME}/MyInstalls/hdf5_serial" CACHE PATH "Path to Serial HDF5 installation") + set(NETCDF_INSTALL "$ENV{HOME}/MyInstalls/netcdf_serial" CACHE PATH "Path to Serial NetCDF installation") + set(NETCDF_LIB_DIR "$ENV{HOME}/MyInstalls/netcdf_serial/lib") + set(HDF5_LIB_DIR "$ENV{HOME}/MyInstalls/hdf5_serial/lib") + set(WRAPPER_SOURCES + ${NGEN_SRC_DIR}/netcdf/NetCDFManager.cpp + ${NGEN_SRC_DIR}/netcdf/NetCDFFile.cpp + ${NGEN_SRC_DIR}/netcdf/NetCDFVar.cpp ) - target_include_directories(NetCDFCreator PUBLIC + add_library(NetCDFManager STATIC ${WRAPPER_SOURCES}) + set_target_properties(NetCDFManager PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) + target_include_directories(NetCDFManager PUBLIC ${EWTS_PREFIX}/include ${NGEN_INC_DIR} + ${NGEN_INC_DIR}/netcdf ${NGEN_INC_DIR}/core ${NGEN_INC_DIR}/core/catchment ${NGEN_INC_DIR}/realizations/catchment @@ -409,23 +421,45 @@ if(NGEN_WITH_NETCDF) ${NGEN_INC_DIR}/core/mediator ${Python_INCLUDE_DIRS} ${pybind11_INCLUDE_DIR} - ${MPI_CXX_INCLUDE_DIRS} + ${NETCDF_INSTALL}/include + ${HDF5_INSTALL}/include + $<$:${MPI_CXX_INCLUDE_DIRS}> + ) + target_link_libraries(NetCDFManager PUBLIC NGen::config_header) + # the order for the custom netcdf and hdf5 libraries is important. + target_link_libraries(NetCDFManager PRIVATE + ${NETCDF_INSTALL}/lib/libnetcdf.so + ${HDF5_INSTALL}/lib/libhdf5_hl.so + ${HDF5_INSTALL}/lib/libhdf5.so + $<$:MPI::MPI_CXX> + ) + target_compile_definitions(NetCDFManager PUBLIC NGEN_WITH_NETCDF) + target_link_libraries(ngen PRIVATE NetCDFManager) + # set target properties to the actual path so that it is propagated during runtime. + set_target_properties(ngen PROPERTIES + BUILD_WITH_INSTALL_RPATH FALSE + SKIP_BUILD_RPATH FALSE + BUILD_RPATH "${NETCDF_LIB_DIR};${HDF5_LIB_DIR}" + ) + # The compiler automatically converts classic RPATH entries into modern RUNPATH entries. + # By adding the -Wl,--disable-new-dtags flag + # we force the system to use traditional RPATH rules instead of RUNPATH. + target_link_options(ngen PRIVATE + "-Wl,-rpath,${NETCDF_LIB_DIR}:${HDF5_LIB_DIR}" + "-Wl,--disable-new-dtags" ) - target_link_libraries(NetCDFCreator PUBLIC NGen::config_header) - target_link_libraries(NetCDFCreator PRIVATE NetCDF) - target_link_libraries(ngen PRIVATE NetCDFCreator) endif() target_link_libraries(partitionGenerator PUBLIC NGen::core NGen::geojson) if(NGEN_WITH_SQLITE) - target_link_libraries(partitionGenerator PUBLIC NGen::geopackage) +target_link_libraries(partitionGenerator PUBLIC NGen::geopackage) endif() # For automated testing with Google Test if(NGEN_WITH_TESTS) - include(CTest) # calls enable_testing() - include(GoogleTest) - add_subdirectory(test) +include(CTest) # calls enable_testing() +include(GoogleTest) +add_subdirectory(test) endif() # ----------------------------------------------------------------------------- @@ -436,27 +470,27 @@ endif() macro(ngen_multiline_message) set(messages "${ARGN}") foreach(msg IN LISTS messages) - set(_CONF "${NGEN_CONF_SUMMARY}") - string(APPEND _CONF "${msg}\n") - set(NGEN_CONF_SUMMARY "${_CONF}") +set(_CONF "${NGEN_CONF_SUMMARY}") +string(APPEND _CONF "${msg}\n") +set(NGEN_CONF_SUMMARY "${_CONF}") - message(STATUS "${msg}") +message(STATUS "${msg}") endforeach() endmacro() # Syntax: ngen_dependent_multiline_message( "" "" ...) macro(ngen_dependent_multiline_message VARIABLE) if(${VARIABLE}) - ngen_multiline_message(${ARGN}) +ngen_multiline_message(${ARGN}) endif() endmacro() # This is used purely for configuration output, # and does not affect the virtual environment used. if(DEFINED ENV{VIRTUAL_ENV}) - set(NGEN_CONFIGURED_VENV "$ENV{VIRTUAL_ENV}") +set(NGEN_CONFIGURED_VENV "$ENV{VIRTUAL_ENV}") else() - set(NGEN_CONFIGURED_VENV "") +set(NGEN_CONFIGURED_VENV "") endif() # MPI CXX libraries output both C and CXX libs @@ -560,4 +594,4 @@ configure_file("${NGEN_INC_DIR}/NGenConfig.h.in" "${CMAKE_CURRENT_BINARY_DIR}/in include(GNUInstallDirs) install(TARGETS ngen OPTIONAL) -install(TARGETS partitionGenerator OPTIONAL) +install(TARGETS partitionGenerator OPTIONAL) \ No newline at end of file diff --git a/include/core/NgenSimulation.hpp b/include/core/NgenSimulation.hpp index d6b4714931..c29f21426c 100644 --- a/include/core/NgenSimulation.hpp +++ b/include/core/NgenSimulation.hpp @@ -1,25 +1,32 @@ #ifndef NGENSIMULATION_HPP #define NGENSIMULATION_HPP -#include +#if NGEN_WITH_MPI + #include +#endif +#include #include #include #include #if NGEN_WITH_NETCDF - #include + #include "netcdf/NetCDFManager.hpp" #endif + namespace hy_features { class HY_Features; class HY_Features_MPI; } - class State_Snapshot_Saver; class State_Snapshot_Loader; +// #if NGEN_WITH_NETCDF +// class NetCDFManager; +// #endif + #if NGEN_WITH_ROUTING #include "bmi/Bmi_Py_Adapter.hpp" #endif // NGEN_WITH_ROUTING @@ -133,7 +140,14 @@ class NgenSimulation void serialize(Archive& ar); //Pointer to netcdfcreator to write simulation output per timestep. - std::unique_ptr nc_writer_; + //std::unique_ptr nc_writer_; +#if NGEN_WITH_MPI + MPI_Comm mpi_comm_; +#endif + +#if NGEN_WITH_NETCDF + std::unique_ptr nc_manager_; +#endif }; #endif diff --git a/include/forcing/NetCDFPerFeatureDataProvider.hpp b/include/forcing/NetCDFPerFeatureDataProvider.hpp index 5719bc32e2..c9a1311c33 100644 --- a/include/forcing/NetCDFPerFeatureDataProvider.hpp +++ b/include/forcing/NetCDFPerFeatureDataProvider.hpp @@ -7,7 +7,7 @@ #include "GenericDataProvider.hpp" #include "DataProviderSelectors.hpp" - +//#include #include #include #include @@ -25,10 +25,10 @@ #include "AorcForcing.hpp" -namespace netCDF { - class NcVar; - class NcFile; -} +// Forward declarations +class NetCDFManager; +class NetCDFFile; +class NetCDFVar; namespace data_access { @@ -132,15 +132,15 @@ namespace data_access utils::StreamHandler log_stream; std::string file_path; - std::shared_ptr nc_file; + std::shared_ptr nc_manager; - std::map ncvar_cache; + std::map> ncvar_cache; std::map units_cache; boost::compute::detail::lru_cache>> value_cache; size_t cache_slice_t_size = 1; size_t cache_slice_c_size = 1; - const netCDF::NcVar& get_ncvar(const std::string& name); + std::shared_ptr get_ncvar(const std::string& name) const; const std::string& get_ncvar_units(const std::string& name) const; diff --git a/include/netcdf/NetCDFFile.hpp b/include/netcdf/NetCDFFile.hpp new file mode 100644 index 0000000000..376be0d0bc --- /dev/null +++ b/include/netcdf/NetCDFFile.hpp @@ -0,0 +1,102 @@ +#ifndef NetCDFFILE_HPP +#define NetCDFFILE_HPP + +//#include + +#if NGEN_WITH_MPI + #include + #define _PARALLEL4 +#endif + + +#if NGEN_WITH_NETCDF +#include +#include +#include +#include +#include +#include "NetCDFVar.hpp" + +class NetCDFFile { +public: +#if NGEN_WITH_MPI + NetCDFFile(const std::string& filename, bool write_only, MPI_Comm comm); +#endif + + // Serial constructor for write only + NetCDFFile(const std::string& filename, bool write_only); + + // Serial read-only mode for NetCDFPerFeatureDataProvider + NetCDFFile(const std::string &filename); + + ~NetCDFFile(); + void load_variables(); + std::vector list_variables() const; + std::shared_ptr get_ncvar(const std::string& name) const; + + // Dimension handling + int add_dimension(const std::string& name, size_t len); + size_t get_dim_size(const std::string& name) const; + int get_dim_id(const std::string& name) const; + + // Variable handling + int get_next_varid() { return next_varid_++; } + void add_ncvar(std::shared_ptr var); + + // Add a variable to the file + void add_variable(const std::string& name, nc_type type, const std::vector& dims, const std::vector& dim_names); + + // Write data to a variable + template + void write_variable_data(const std::string& name, const std::vector& data, size_t start_index = 0); + + int write_data_to_ncvar(int ncid, int varid, const std::vector& start, + const std::vector& count, const std::vector& data); + + int write_data_to_ncvar(int ncid, int varid, const std::vector& start, + const std::vector& count, const std::vector& data); + + int write_data_to_ncvar(int ncid, int varid, const std::vector& start, + const std::vector& count, const std::vector& data); + + void write_catchment_output_data(const std::string& name, std::vector start, + std::vector count, const double& data); + + //Add attribute to a variable + void write_attribute_to_ncvar(const std::string& name, const std::string& attName, const std::string& attValue); + + + + // // Write all variables in parallel + // template + // void writeAllVariables(const std::vector>& all_data); + + // // MPI-safe write (each rank writes its portion) + // template + // void writeAllVariablesDistributed(const std::vector>& all_data); + + void close_file(); + + int get_ncid() const { return ncid_; } + +private: + std::string nc_file_name_; + bool read_only_; + int ncid_; + int next_varid_ = 0; + std::vector> variables_; + std::map dims_; + std::map> variables_map_; + + // Compute chunk offsets/counts for first dimension + // void computeStartCount(size_t total_size, size_t& start, size_t& count); +#if NGEN_WITH_MPI + MPI_Comm comm_; +#endif + bool parallel_; + + // Helper to get variable index by name + std::shared_ptr get_ncvar_by_name(const std::string& name); +}; +#endif // NGEN_WITH_NETCDF +#endif // NETCDFFILE_HPP \ No newline at end of file diff --git a/include/netcdf/NetCDFManager.hpp b/include/netcdf/NetCDFManager.hpp new file mode 100644 index 0000000000..00670143a3 --- /dev/null +++ b/include/netcdf/NetCDFManager.hpp @@ -0,0 +1,108 @@ +#ifndef NETCDFMANAGER_HPP +#define NETCDFMANAGER_HPP + +//#include +#if NGEN_WITH_MPI + #include + #define _PARALLEL4 +#endif + +#if NGEN_WITH_NETCDF +#include "NetCDFFile.hpp" +#include "NetCDFVar.hpp" +#include +#include +#include +#include +//#include "realizations/catchment/Formulation_Manager.hpp" +//#include "simulation_time/Simulation_Time.hpp" +//#include "Catchment_Formulation.hpp" + +namespace realization { + class Formulation_Manager; + class Catchment_Formulation; +} +class Simulation_Time; + +class NetCDFManager +{ +public: + NetCDFManager(std::shared_ptr manager, + const std::string& output_name, Simulation_Time const& sim_time, int mpi_rank, int mpi_num_procs); + + //NetCDFManager(const std::string& output_name, int mpi_rank, int mpi_num_procs); + + // Constructor for read-only NetCDF (no MPI needed) + NetCDFManager(const std::string& filename, bool read_only); + + // Constructor for writing (can be parallel if size>1) + // NetCDFManager(const std::string& filename, int rank, int size); + + // Default constructor for mdframe + NetCDFManager(); + + // File operations + void create_file(const std::string& filename); + void open_file(); + void close_file(); + + // List variable names + std::vector list_variables() const; + + // Access NetCDFVar by name + std::shared_ptr get_ncvar_by_name(const std::string& name) const; + + // Attribute access + std::vector list_attributes(const std::string& var_name) const; + std::string get_string_attribute(const std::string& var_name, const std::string& att_name) const; + int get_int_attribute(const std::string& var_name, const std::string& att_name) const; + double get_double_attribute(const std::string& var_name, const std::string& att_name) const; + + // Add a dimension + int add_dimension(const std::string& name, size_t len); + + // Add a variable + void add_variable(const std::string& var_name, nc_type type, const std::vector& dims, const std::vector& dim_names); + + // Add a variable to the file (for writing) + void add_output_variable_data_from_formulation(); + + // Add catchment output data to the file (for writing) + void write_simulations_response_from_formulation(size_t time_index, std::map catchment_output_values); + + // Create NetCDF file with time and entity dimensions, and multiple variables + void create_timeslice(size_t num_entities, size_t num_timesteps, + const std::vector& var_names); + + // Write a block of timesteps for all entities for a variable + template + void write_timeslice(const std::string& var_name, size_t time_start, + size_t time_count, const std::vector& data); + + // Get the time range assigned to this rank + void get_local_time_range(size_t& start, size_t& count) const; + + + ~NetCDFManager(); +private: + int rank_; + int num_procs_; + bool read_only_; + std::string filename_; + std::shared_ptr nc_file_; + std::vector vars_; + std::shared_ptr manager_; + std::shared_ptr sim_time_; + size_t num_entities_; + size_t num_timesteps_; + std::vector catchments_; + std::map> variables_map_; + std::vector nc_output_variables; + +#if NGEN_WITH_MPI + MPI_Comm comm_; +#endif + bool is_mpi_; +}; +#endif // NGEN_WITH_NETCDF +#endif // NETCDFMANAGER_HPP \ No newline at end of file diff --git a/include/netcdf/NetCDFVar.hpp b/include/netcdf/NetCDFVar.hpp new file mode 100644 index 0000000000..5263245dc9 --- /dev/null +++ b/include/netcdf/NetCDFVar.hpp @@ -0,0 +1,52 @@ +#ifndef NETCDFVAR_HPP +#define NETCDFVAR_HPP + +//#include + +#if NGEN_WITH_NETCDF +#include +#include +#include +#include + +class NetCDFVar { +public: + NetCDFVar(const std::string& name, nc_type type, const std::vector& dims, + const std::vector& dim_names, int varid, int ncid); + + const std::string& get_name() const; + nc_type get_type() const; + const std::vector& get_dims() const; + const std::vector& get_dim_names() const; + size_t get_dim_size(const std::string& dim_name) const; + size_t get_dim_size(size_t idx) const; + size_t get_dim_count() const; + int get_varid() const; + size_t get_total_size() const; + std::vector get_string_array_values() const; + std::vector get_time_values() const; + void read_slice(const std::vector& start, const std::vector& count, double* data) const; + + void add_attribute(const std::string& att_name, const std::string& att_value); + void add_attribute(const std::string& att_name, int att_value); + void add_attribute(const std::string& att_name, double att_value); + + std::vector list_attributes() const; + std::string get_string_attribute(const std::string& att_name) const; + int get_int_attribute(const std::string& att_name) const; + double get_double_attribute(const std::string& att_name) const; + + private: + std::string name_; + nc_type type_; + std::vector dims_; + std::vector dim_names_; + int varid_; + int ncid_; + + std::map attributes_str_; + std::map attributes_int_; + std::map attributes_double_; +}; +#endif // NGEN_WITH_NETCDF +#endif // NETCDFVAR_HPP \ No newline at end of file diff --git a/src/NGen.cpp b/src/NGen.cpp index f0831d6ddf..4a44d3b869 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -12,7 +12,7 @@ #include #if NGEN_WITH_NETCDF -#include +#include #endif #if NGEN_WITH_SQLITE3 @@ -491,7 +491,7 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { } else { catchment_collection = geojson::read(catchmentDataFile, catchment_subset_ids); } - + std::cout << catchment_collection << std::endl; for (auto& feature : *catchment_collection) { // feature->set_id(feature->get_property("id").as_string()); nexus_collection->add_feature(feature); diff --git a/src/core/NgenSimulation.cpp b/src/core/NgenSimulation.cpp index 20108a4300..c3db0e5d8a 100644 --- a/src/core/NgenSimulation.cpp +++ b/src/core/NgenSimulation.cpp @@ -8,6 +8,10 @@ #include "HY_Features.hpp" #endif +// #if NGEN_WITH_NETCDF +// #include "netcdf/NetCDFManager.hpp" +// #endif + #include "state_save_restore/State_Save_Utils.hpp" #include "state_save_restore/State_Save_Restore.hpp" #include "parallel_utils.h" @@ -112,7 +116,8 @@ void NgenSimulation::advance_models_one_output_step() #if NGEN_WITH_NETCDF #if !NGEN_WITH_MPI std::map catchment_output_vals = layer->get_catchment_output_data_for_timestep(); - nc_writer_->write_simulations_response_from_formulation(simulation_step_,catchment_output_vals); + //nc_writer_->write_simulations_response_from_formulation(simulation_step_,catchment_output_vals); + nc_manager_->write_simulations_response_from_formulation(simulation_step_,catchment_output_vals); #endif //NGEN_WITH_MPI #endif //NGEN_WITH_NETCDF @@ -399,5 +404,9 @@ void NgenSimulation::serialize(Archive& ar) { void NgenSimulation::create_netcdf_writer(std::shared_ptr manager, std::string nc_output_file_name, int mpi_rank, int mpi_num_procs) { - this->nc_writer_ = std::make_unique(manager,nc_output_file_name,*sim_time_, mpi_rank, mpi_num_procs_); + //this->nc_writer_ = std::make_unique(manager,nc_output_file_name,*sim_time_, mpi_rank, mpi_num_procs); +#if NGEN_WITH_NETCDF + this->nc_manager_ = std::make_unique(manager, nc_output_file_name, *sim_time_, mpi_rank, mpi_num_procs); +#endif + //nc_manager_ = std::make_unique(nc_output_file_name, mpi_rank, mpi_num_procs); } diff --git a/src/forcing/NetCDFPerFeatureDataProvider.cpp b/src/forcing/NetCDFPerFeatureDataProvider.cpp index 9940521252..2a27ab17ce 100644 --- a/src/forcing/NetCDFPerFeatureDataProvider.cpp +++ b/src/forcing/NetCDFPerFeatureDataProvider.cpp @@ -2,8 +2,10 @@ #if NGEN_WITH_NETCDF #include "NetCDFPerFeatureDataProvider.hpp" - -#include +#include "netcdf/NetCDFManager.hpp" +#include "netcdf/NetCDFFile.hpp" +#include "netcdf/NetCDFVar.hpp" +#include #include #include #include @@ -51,34 +53,26 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat //nc_set_chunk_cache(sizep, nelemsp, preemptionp); //open the file - nc_file = std::make_shared(input_path, netCDF::NcFile::read); + nc_manager = std::make_shared(input_path, true); + nc_manager->open_file(); //nc_get_chunk_cache(&sizep, &nelemsp, &preemptionp); //std::cout << "Chunk cache parameters: "<getVars(); + std::vector var_set = nc_manager->list_variables(); // populate the ncvar and units caches... - std::for_each(var_set.begin(), var_set.end(), [&](const auto& element) + for (const auto& var_name : var_set) { - std::string var_name = element.first; - auto ncvar = nc_file->getVar(var_name); + auto ncvar = nc_manager->get_ncvar_by_name(var_name); variable_names.push_back(var_name); ncvar_cache.emplace(var_name,ncvar); - std::string native_units; + std::string native_units = ""; try { - auto units_att = ncvar.getAtt("units"); - if ( units_att.isNull() ) - { - native_units = ""; - } - else - { - units_att.getValues(native_units); - } + native_units = ncvar->get_string_attribute("units"); } catch(...) { @@ -95,79 +89,71 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat } units_cache[var_name] = native_units; - }); + } // read the variable ids - auto ids = nc_file->getVar("ids"); - auto id_dim_count = ids.getDimCount(); - + auto ids = nc_manager->get_ncvar_by_name("ids"); + const auto& ids_dims = ids->get_dims(); + // some sanity checks - if ( id_dim_count > 1) + if (ids_dims.empty()) { - std::string msg = "Provided NetCDF file has an \"ids\" variable with more than 1 dimension"; + std::string msg = "Provided NetCDF file has a NULL dimension for variable \"ids\""; LOG(LogLevel::FATAL, msg); throw std::runtime_error(msg); } - - auto id_dim = ids.getDim(0); - - if (id_dim.isNull() ) + if ( ids_dims.size() != 1) { - std::string msg = "Provided NetCDF file has a NULL dimension for variable \"ids\""; + std::string msg = "Provided NetCDF file has an \"ids\" variable that is not a 1D array of catchments"; LOG(LogLevel::FATAL, msg); throw std::runtime_error(msg); } - auto num_ids = id_dim.getSize(); + auto num_ids = ids_dims[0]; + if (num_ids == 0) + { + std::string msg = "Provided NetCDF file has no catchments listed for variable \"ids\""; + LOG(LogLevel::FATAL, msg); + throw std::runtime_error(msg); + } //TODO: split into smaller slices if num_ids is large. cache_slice_c_size = num_ids; - // allocate an array of character pointers - std::vector< char* > string_buffers(num_ids); - - // read the id strings - ids.getVar(&string_buffers[0]); - - // initalize the map of catchment-name to offset location and free the strings allocated by the C library + // initalize the map of catchment-name to offset location size_t loc = 0; - for_each( string_buffers.begin(), string_buffers.end(), [&](char* str) + for(const auto& id : ids->get_string_array_values()) { - loc_ids.push_back(str); - id_pos[str] = loc++; - }); - - // correct string release - nc_free_string(num_ids,&string_buffers[0]); + loc_ids.push_back(id); + id_pos[id] = loc++; + } // Get the time variable - getVar collects all values at once and stores in memory // Extremely large timespans could be problematic, but for ngen use cases, this should not be a problem - auto time_var = nc_file->getVar("Time"); + auto time_var = nc_manager->get_ncvar_by_name("Time"); // Get the size of the time dimension - size_t num_times = nc_file->getDim("time").getSize(); + size_t num_times = time_var->get_dims().size(); - std::vector raw_time(num_times); + std::vector raw_time; try { - auto dim_count = time_var.getDimCount(); + auto dim_count = time_var->get_dim_count(); // Old-format files have dimensions (catchment, time), new-format // files generated by the forcings engine have just (time) if (dim_count == 2) { - if (time_var.getDim(0).getName() != "catchment-id" || time_var.getDim(1).getName() != "time") { + if (time_var->get_dim_names()[0] != "catchment-id" || time_var->get_dim_names()[1] != "time") { std::string message = "In NetCDF file '" + input_path + "', 'Time' variable dimensions don't match expectations"; std::string throw_msg; throw_msg.assign(message); LOG(throw_msg, LogLevel::WARNING); throw std::runtime_error(throw_msg); } - time_var.getVar({0ul, 0ul}, {1ul, num_times}, raw_time.data()); - } else if (dim_count == 1) { - time_var.getVar({0ul}, {num_times}, raw_time.data()); - } else { + } else if (dim_count != 1) { throw std::runtime_error("Unexpected " + std::to_string(dim_count) + " dimensions on Time variable in NetCDF file '" + input_path + "'"); } + raw_time = time_var->get_time_values(); } catch(const std::exception& e) { netcdf_ss << "Error reading time variable: " << e.what() << std::endl; LOG(netcdf_ss.str(), LogLevel::WARNING); netcdf_ss.str(""); @@ -176,8 +162,8 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat std::string time_units; try { - time_var.getAtt("units").getValues(time_units); - } catch(const netCDF::exceptions::NcException& e) { + time_units = time_var->get_string_attribute("units"); + } catch(const std::runtime_error& e) { netcdf_ss << "Error reading time units: " << e.what() << std::endl; LOG(netcdf_ss.str(), LogLevel::WARNING); netcdf_ss.str(""); netcdf_ss << "Warning: Using default time units (seconds since epoch)" << std::endl; @@ -192,7 +178,6 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat auto since_index = time_units.find("since"); if (since_index != std::string::npos) { time_base_unit = time_units.substr(0, since_index - 1); - std::string datetime_str = time_units.substr(since_index + 6); std::tm tm = {}; std::istringstream ss(datetime_str); @@ -274,10 +259,10 @@ NetCDFPerFeatureDataProvider::~NetCDFPerFeatureDataProvider() = default; void NetCDFPerFeatureDataProvider::finalize() { - if (nc_file != nullptr) { - nc_file->close(); + if (nc_manager != nullptr) { + nc_manager->close_file(); } - nc_file = nullptr; + nc_manager = nullptr; } boost::span NetCDFPerFeatureDataProvider::get_available_variable_names() const @@ -375,7 +360,7 @@ double NetCDFPerFeatureDataProvider::get_value(const CatchmentAggrDataSelector& size_t cache_slices_t_n = (read_len + cache_slice_t_size - 1) / cache_slice_t_size; // Ceiling division to ensure remainders have a slice - auto dims = ncvar.getDims(); + std::vector dims = ncvar->get_dim_names(); int dim_time, dim_catchment; if (dims.size() != 2) { @@ -384,11 +369,11 @@ double NetCDFPerFeatureDataProvider::get_value(const CatchmentAggrDataSelector& LOG(throw_msg, LogLevel::WARNING); throw std::runtime_error(throw_msg); } - if (dims[0].getName() == "time" && dims[1].getName() == "catchment-id") { + if (dims[0] == "time" && dims[1] == "catchment-id") { // Forcings Engine NetCDF output case dim_time = 0; dim_catchment = 1; - } else if (dims[1].getName() == "time" && dims[0].getName() == "catchment-id") { + } else if (dims[1] == "time" && dims[0] == "catchment-id") { // Classic NetCDF file case dim_time = 1; dim_catchment = 0; @@ -399,14 +384,14 @@ double NetCDFPerFeatureDataProvider::get_value(const CatchmentAggrDataSelector& throw std::runtime_error(throw_msg); } - size_t time_dim_size = dims[dim_time].getSize(); - size_t catchment_dim_size = dims[dim_catchment].getSize(); + size_t time_dim_size = ncvar->get_dim_size(dim_time); + size_t catchment_dim_size = ncvar->get_dim_size(dim_catchment); for( size_t i = 0; i < cache_slices_t_n; i++ ) { std::shared_ptr> cached; size_t cache_t_idx = idx1 + i * cache_slice_t_size; size_t slice_size = std::min(cache_slice_t_size, time_dim_size - cache_t_idx); - std::string key = ncvar.getName() + "|" + std::to_string(cache_t_idx) + "|" + std::to_string(slice_size); + std::string key = ncvar->get_name() + "|" + std::to_string(cache_t_idx) + "|" + std::to_string(slice_size); if(value_cache.contains(key)){ cached = value_cache.get(key).get(); @@ -423,9 +408,9 @@ double NetCDFPerFeatureDataProvider::get_value(const CatchmentAggrDataSelector& var_index_map[dim_catchment] = 1; try { - ncvar.getVar(start,count, {1l, 1l}, var_index_map, cached->data()); + ncvar->read_slice(start,count,cached->data()); value_cache.insert(key, cached); - } catch (netCDF::exceptions::NcException& e) { + } catch (const std::runtime_error& e) { netcdf_ss << "NetCDF exception: " << e.what() << std::endl; log_stream << netcdf_ss.str(); LOG(netcdf_ss.str(), LogLevel::WARNING); netcdf_ss.str(""); @@ -515,7 +500,7 @@ std::vector NetCDFPerFeatureDataProvider::get_values(const CatchmentAggr // private: -const netCDF::NcVar& NetCDFPerFeatureDataProvider::get_ncvar(const std::string& name){ +std::shared_ptr NetCDFPerFeatureDataProvider::get_ncvar(const std::string& name) const{ auto cache_hit = ncvar_cache.find(name); if(cache_hit != ncvar_cache.end()){ return cache_hit->second; diff --git a/src/netcdf/NetCDFFile.cpp b/src/netcdf/NetCDFFile.cpp new file mode 100644 index 0000000000..1539e92fd9 --- /dev/null +++ b/src/netcdf/NetCDFFile.cpp @@ -0,0 +1,354 @@ +#if NGEN_WITH_MPI + #define _PARALLEL4 +#endif + +#if NGEN_WITH_NETCDF +#include "NetCDFFile.hpp" +#include +#include +#include +#include +#include + +#if NGEN_WITH_MPI +NetCDFFile::NetCDFFile(const std::string& filename, bool write_only, MPI_Comm comm) + : nc_file_name_(filename), comm_(comm) +{ + int retval; + int mode = NC_NETCDF4; + if (write_only) mode |= NC_CLOBBER; + + if ((retval = nc_create_par(filename.c_str(), mode, comm_, MPI_INFO_NULL, &ncid_))) + throw std::runtime_error("Error creating NetCDF file: " + std::string(nc_strerror(retval))); +} +#else +NetCDFFile::NetCDFFile(const std::string& full_filename, bool write_only) +{ + int retval; + int mode = NC_NETCDF4; + if (write_only) mode |= NC_CLOBBER; + if ((retval = nc_create(full_filename.c_str(), mode, &ncid_))) + throw std::runtime_error("Error creating NetCDF file: " + std::string(nc_strerror(retval))); + +} +#endif + +NetCDFFile::NetCDFFile(const std::string &filename) +{ + nc_file_name_ = filename; + read_only_ = true; +#if NGEN_WITH_MPI + comm_ = MPI_COMM_NULL; +#endif + load_variables(); +} + +// Add a new dimension +int NetCDFFile::add_dimension(const std::string& name, size_t len) { + int dimid; + int retval = nc_def_dim(ncid_, name.c_str(), len, &dimid); + if (retval) throw std::runtime_error("Error defining dimension " + name + ": " + nc_strerror(retval)); + dims_[name] = dimid; + return dimid; +} + +void NetCDFFile::add_variable(const std::string& name, nc_type type, const std::vector& dims, const std::vector& dim_names) { + int varid; + int retval = nc_def_var(ncid_, name.c_str(), type, dims.size(), dims.data(), &varid); + if (retval != NC_NOERR) throw std::runtime_error(nc_strerror(retval)); + // set fill value and missing value for NC_DOUBLE + if (type = NC_DOUBLE) { + double fill_missing_val = -1.0; + nc_def_var_fill(ncid_, varid, 0, &fill_missing_val); + nc_put_att_double(ncid_, varid, "missing_value", NC_DOUBLE, 1, &fill_missing_val); + } + auto var = std::make_shared(name, type, dims, dim_names, varid, ncid_); + add_ncvar(var); +} + +// Get dimension length by name +size_t NetCDFFile::get_dim_size(const std::string& name) const { + auto it = dims_.find(name); + if(it == dims_.end()) throw std::runtime_error("Dimension not found: " + name); + + size_t len; + int retval = nc_inq_dimlen(ncid_, it->second, &len); + if(retval) throw std::runtime_error("Error getting dimension length for " + name + ": " + nc_strerror(retval)); + return len; +} + +int NetCDFFile::get_dim_id(const std::string& name) const { + auto it = dims_.find(name); + if(it == dims_.end()) throw std::runtime_error("Dimension not found: " + name); + return it->second; +} + +// Add a variable +void NetCDFFile::add_ncvar(std::shared_ptr var) { + variables_map_[var->get_name()] = var; +} + + void NetCDFFile::write_catchment_output_data(const std::string& name, std::vector start, + std::vector count, const double& data) +{ + auto var = get_ncvar_by_name(name); + if (!var) throw std::runtime_error("Variable not found: " + name); + int retval = nc_put_vara_double(ncid_, var->get_varid(), start.data(), count.data(), &data); + if (retval != NC_NOERR) { + throw std::runtime_error("Error writing value: " + std::string(nc_strerror(retval))); + } +} + +int NetCDFFile::write_data_to_ncvar(int ncid_, int varid, const std::vector& start, + const std::vector& count, const std::vector& data) { + return nc_put_vara_double(ncid_, varid, start.data(), count.data(), data.data()); +} + +int NetCDFFile::write_data_to_ncvar(int ncid_, int varid, const std::vector& start, + const std::vector& count, const std::vector& data) { + return nc_put_vara_int(ncid_, varid, start.data(), count.data(), data.data()); +} + +int NetCDFFile::write_data_to_ncvar(int ncid_, int varid, const std::vector& start, + const std::vector& count, const std::vector& data) { + std::vector cstrs(data.size()); + for (size_t i = 0; i < data.size(); ++i) { + cstrs[i] = data[i].c_str(); + } + return nc_put_vara_string(ncid_, varid, start.data(), count.data(), cstrs.data()); +} + +template +void NetCDFFile::write_variable_data(const std::string& name, const std::vector& data, size_t start_index) { + auto var = get_ncvar_by_name(name); + if (!var) throw std::runtime_error("Variable not found: " + name); + std::vector start = {start_index}; + std::vector count = {data.size()}; + + #if NGEN_WITH_MPI + if (comm_ != MPI_COMM_NULL) { + int retval = nc_var_par_access(ncid_, var->getVarId(), NC_COLLECTIVE); + if (retval != NC_NOERR) throw std::runtime_error(nc_strerror(retval)); + } + #endif + + int retval = NC_NOERR; + retval = write_data_to_ncvar(ncid_, var->get_varid(), start, count, data); + + if (retval != NC_NOERR) throw std::runtime_error(nc_strerror(retval)); +} + +void NetCDFFile::write_attribute_to_ncvar(const std::string& name, const std::string& attName, const std::string& attValue) { + auto var = get_ncvar_by_name(name); + if (!var) throw std::runtime_error("Variable not found: " + name); + var->add_attribute(attName, attValue); +} + +// template +// void NcFile::writeAllVariables(const std::vector>& all_data) { +// if (all_data.size() != variables_.size()) +// throw std::runtime_error("Mismatch between variables and data"); + +// for (size_t v = 0; v < variables_.size(); ++v) { +// auto var = variables_[v]; +// const auto& data = all_data[v]; + +// #if NGEN_WITH_MPI +// if (comm_ != MPI_COMM_NULL && num_procs_ > 1) { +// int retval = nc_var_par_access(ncid_, var->getVarId(), NC_COLLECTIVE); +// if (retval != NC_NOERR) throw std::runtime_error(nc_strerror(retval)); +// } +// #endif + +// int retval = NC_NOERR; +// if constexpr (std::is_same_v) { +// retval = nc_put_var_double(ncid_, var->getVarId(), data.data()); +// } else if constexpr (std::is_same_v) { +// retval = nc_put_var_int(ncid_, var->getVarId(), data.data()); +// } else { +// static_assert(sizeof(T) == 0, "Unsupported data type"); +// } + +// if (retval != NC_NOERR) throw std::runtime_error(nc_strerror(retval)); +// } +// } + +// template +// void NetCDFFile::writeAllVariablesDistributed(const std::vector>& all_data) { +// if (all_data.size() != variables_.size()) +// throw std::runtime_error("Mismatch between variables and data"); + +// for (size_t v = 0; v < variables_.size(); ++v) { +// auto var = variables_[v]; +// const auto& data = all_data[v]; + +// // Compute first-dimension chunking +// size_t first_dim_size = var->getDims()[0]; +// size_t start0, count0; +// computeStartCount(first_dim_size, start0, count0); + +// std::vector start(var->getDims().size(), 0); +// std::vector count = var->getDims(); +// start[0] = start0; +// count[0] = count0; + +// #if NGEN_WITH_MPI +// if (comm_ != MPI_COMM_NULL && num_procs_ > 1) { +// int retval = nc_var_par_access(ncid_, var->getVarId(), NC_COLLECTIVE); +// if (retval != NC_NOERR) throw std::runtime_error(nc_strerror(retval)); +// } +// #endif + +// int retval = NC_NOERR; +// if constexpr (std::is_same_v) { +// retval = nc_put_vara_double(ncid_, var->getVarId(), start.data(), count.data(), +// &data[start0 * (data.size()/first_dim_size)]); +// } else if constexpr (std::is_same_v) { +// retval = nc_put_vara_int(ncid_, var->getVarId(), start.data(), count.data(), +// &data[start0 * (data.size()/first_dim_size)]); +// } else { +// static_assert(sizeof(T) == 0, "Unsupported data type"); +// } + +// if (retval != NC_NOERR) throw std::runtime_error(nc_strerror(retval)); +// } +// } + +std::shared_ptr NetCDFFile::get_ncvar_by_name(const std::string& name) { + auto it = variables_map_.find(name); + if (it != variables_map_.end()) return it->second; + return nullptr; +} + +// void NetCDFFile::computeStartCount(size_t total_size, size_t& start, size_t& count) { +// #if NGEN_WITH_MPI +// if (comm_ == MPI_COMM_NULL || num_procs_ == 1) { +// start = 0; +// count = total_size; +// } else { +// size_t chunk = total_size / num_procs_; +// start = rank_ * chunk; +// count = (rank_ == num_procs_ - 1) ? (total_size - start) : chunk; +// } +// #else +// start = 0; +// count = total_size; +// #endif +// } + +void NetCDFFile::load_variables() +{ + int retval; + +#if NGEN_WITH_MPI + if(comm_ != MPI_COMM_NULL) { + if((retval = nc_open_par(ncFileName.c_str(), NC_NOWRITE, comm_, MPI_INFO_NULL, &ncid_))) { + throw std::runtime_error("Cannot open NetCDF (MPI): " + std::string(nc_strerror(retval))); + } + } else +#endif + { + if((retval = nc_open(nc_file_name_.c_str(), NC_NOWRITE, &ncid_))) { + throw std::runtime_error("Cannot open NetCDF: " + std::string(nc_strerror(retval))); + } + } + + int num_variables; + if((retval = nc_inq_nvars(ncid_, &num_variables))) { + nc_close(ncid_); + throw std::runtime_error("Cannot get number of variables: " + std::string(nc_strerror(retval))); + } + + variables_.clear(); + + for(int varid = 0; varid < num_variables; ++varid) { + char name[NC_MAX_NAME + 1]; + nc_type type; + int num_dims, dim_ids[NC_MAX_VAR_DIMS], num_attributes; + + if((retval = nc_inq_var(ncid_, varid, name, &type, &num_dims, dim_ids, &num_attributes))) { + std::cerr << "Warning: could not get var info for varid " << varid + << ": " << nc_strerror(retval) << std::endl; + continue; + } + + std::vector dims(num_dims); + std::vector dim_names; + for(int d=0; d(name, type, dims, dim_names, varid, ncid_); + + // load attributes + for(int attid=0; attid buffer(att_len + 1, '\0'); + nc_get_att_text(ncid_, varid, attname, buffer.data()); + var->add_attribute(attname, std::string(buffer.data(), att_len)); + } else if(att_type == NC_INT) { + int val; + nc_get_att_int(ncid_, varid, attname, &val); + var->add_attribute(attname, val); + } else if(att_type == NC_DOUBLE) { + double val; + nc_get_att_double(ncid_, varid, attname, &val); + var->add_attribute(attname, val); + } + // extend for other types if needed (float, long, etc.) + } + + variables_.push_back(var); + } + nc_close(ncid_); +} + +std::vector NetCDFFile::list_variables() const +{ + std::vector names; + for(const auto& var : variables_) { + names.push_back(var->get_name()); + } + return names; +} + +std::shared_ptr NetCDFFile::get_ncvar(const std::string& name) const +{ + for(const auto& var : variables_) { + if(var->get_name() == name) return var; + } + return nullptr; +} + +void NetCDFFile::close_file() { + if (ncid_ >= 0) nc_close(ncid_); +} + +NetCDFFile::~NetCDFFile() { + close_file(); +} + +template void NetCDFFile::write_variable_data(const std::string& name, const std::vector& data, size_t start_index); +template void NetCDFFile::write_variable_data(const std::string& name, const std::vector& data, size_t start_index); +template void NetCDFFile::write_variable_data(const std::string& name, const std::vector& data, size_t start_index); +//template void NetCDFFile::writeAllVariablesDistributed(const std::vector>&); +//template void NetCDFFile::writeAllVariablesDistributed(const std::vector>&); +#endif // NGEN_WITH_NETCDF \ No newline at end of file diff --git a/src/netcdf/NetCDFManager.cpp b/src/netcdf/NetCDFManager.cpp new file mode 100644 index 0000000000..2ee86e9f8b --- /dev/null +++ b/src/netcdf/NetCDFManager.cpp @@ -0,0 +1,295 @@ +#if NGEN_WITH_NETCDF +#include "NetCDFManager.hpp" +#include "Formulation_Manager.hpp" +#include "Catchment_Formulation.hpp" +#include "simulation_time/Simulation_Time.hpp" +//#include "ewts_ngen/logger.hpp" +#include +#include + +NetCDFManager::NetCDFManager(std::shared_ptr manager, + const std::string& output_name, Simulation_Time const& sim_time, int mpi_rank, int mpi_num_procs) +{ + manager_ = manager; + sim_time_ = std::make_shared(sim_time); + rank_ = mpi_rank; + rank_ = rank_ + 1; + num_procs_ = mpi_num_procs; + is_mpi_ = (num_procs_ > 1); + std::string ncOutputFileName = manager_->get_output_root() + output_name + ".nc"; +#if NGEN_WITH_MPI + comm_ = (num_procs_ > 1) ? MPI_COMM_WORLD : MPI_COMM_NULL; + nc_file_ = std::make_shared(filename_, true, comm_); +#else + nc_file_ = std::make_shared(ncOutputFileName, true); + + //add time dimension and variable + std::string name = "time"; + int num_timesteps = sim_time_->get_total_output_times(); + int dim_id = add_dimension(name, num_timesteps); + std::vector dims = {dim_id}; + std::vector names = {name}; + add_variable(name, NC_INT, dims, names); + + //add timestep values and attributes for time + std::vector time_epoch_seconds(num_timesteps); + time_epoch_seconds[0] = sim_time_->get_current_epoch_time(); + for(int time_index = 1; time_index < num_timesteps; time_index++) + { + sim_time_->advance_timestep(); + time_epoch_seconds[time_index] = sim_time_->get_current_epoch_time(); + } + nc_file_->write_variable_data(name, time_epoch_seconds); + nc_file_->write_attribute_to_ncvar(name, "units", "Seconds since 1970-01-01 00:00:00"); + nc_file_->write_attribute_to_ncvar(name, "calendar", "gregorian"); + + //add catchments dimension and variable + name = "catchments"; + dim_id = add_dimension(name, manager_->get_size()); + dims = {dim_id}; + names = {name}; + add_variable(name, NC_STRING, dims, names); + + //add catchment IDs and attributes for catchments + catchments_.reserve(manager_->get_size()); + for (auto const& formulation_info : manager_->get_all_formulations()) + { + std::string catchm = formulation_info.first; + catchments_.push_back(catchm); + } + nc_file_->write_variable_data(name, catchments_); + nc_file_->write_attribute_to_ncvar(name, "Catchment ID", "Catchment identifier in input"); +#endif +} + +NetCDFManager::NetCDFManager(const std::string& filename, bool read_only) + : filename_(filename), read_only_(true) +{ + if (read_only){ + nc_file_ = std::make_shared(filename_); + } + else{ + std::string ncOutputFileName = manager_->get_output_root() + filename + ".nc"; + nc_file_ = std::make_shared(filename_, !read_only); + } +#if NGEN_WITH_MPI + comm_ = MPI_COMM_NULL; + rank_ = 0; + num_procs_ = 1; +#endif +} + +NetCDFManager::NetCDFManager() +{} + +void NetCDFManager::create_file(const std::string& filename) +{ +#if NGEN_WITH_MPI + if (num_procs_ > 1) { + // MPI-enabled NetCDF + nc_file_ = std::make_shared(filename, /*write=*/true, comm_); + } else { + // Serial NetCDF + nc_file_ = std::make_shared(filename, /*write=*/true); + } +#else + // Serial NetCDF + nc_file_ = std::make_shared(filename, /*write=*/true); +#endif + if (!nc_file_) { + throw std::runtime_error("Failed to create NetCDF file: " + filename); + } +} + +int NetCDFManager::add_dimension(const std::string& name, size_t len){ + return nc_file_->add_dimension(name, len); +} + +void NetCDFManager::add_variable(const std::string& var_name, nc_type type, const std::vector& dims, const std::vector& dim_names){ + return nc_file_->add_variable(var_name, NC_INT, dims, dim_names); +} +void NetCDFManager::add_output_variable_data_from_formulation() +{ + typename std::map>::const_iterator it = manager_->begin(); + const auto& catchment_info = *it; + auto r_c = std::dynamic_pointer_cast(catchment_info.second); + if(r_c->get_output_header_count() > 0){ + std::vectoroutput_variables = r_c->get_output_variable_names(); + std::vectoroutput_headers = r_c->get_output_header_fields(); + std::vectoroutput_units = r_c->get_output_variable_units(); + nc_output_variables.resize(output_variables.size()); + std::vector dims_2D = {nc_file_->get_dim_id("time"), nc_file_->get_dim_id("catchments")}; + std::vector dim_names{"time", "catchments"}; + + for(int index = 0; index < output_variables.size(); index ++){ + nc_file_->add_variable(output_headers[index], NC_DOUBLE, dims_2D, dim_names); + nc_file_->write_attribute_to_ncvar(output_headers[index], "variable name", output_variables[index]); + nc_file_->write_attribute_to_ncvar(output_headers[index], "variable units", output_units[index]); + nc_file_->write_attribute_to_ncvar(output_headers[index], "_FillValue", "-1.0"); + nc_file_->write_attribute_to_ncvar(output_headers[index], "missing_value", "-1.0"); + nc_output_variables.push_back(output_headers[index]); + } + } +} + +static std::vector string_split(std::string str, char delimiter) +{ + std::stringstream ss(str); + std::vector res; + std::string token; + while (getline(ss, token, delimiter)) { //will return full string if no comma (or single output variable). + res.push_back(std::stod(token)); + } + return res; +} + +void NetCDFManager::write_simulations_response_from_formulation(size_t time_index, std::map catchment_output_values) +{ + for (auto const& catchment_val : catchment_output_values) + { + std::string catchment_id = catchment_val.first; + + int catchment_index = nc_file_->get_dim_id(catchment_id); + if (catchment_index < 0) { + throw std::runtime_error("Catchment not found in NetCDF: " + catchment_id); + } + size_t c_index = catchment_index; + std::vector start = {time_index, c_index}; + std::vector count = {1, 1}; + std::vector catchment_output = string_split(catchment_val.second, ','); + for(int var_index = 0; var_index < nc_output_variables.size(); ++var_index) + { + nc_file_ ->write_catchment_output_data(nc_output_variables[var_index], start, count, catchment_output[var_index]); + } + } +} + +std::vector NetCDFManager::list_variables() const +{ + if(!nc_file_) return {}; + return nc_file_->list_variables(); +} + +std::shared_ptr NetCDFManager::get_ncvar_by_name(const std::string& name) const +{ + if(!nc_file_) return nullptr; + return nc_file_->get_ncvar(name); +} + +// Attribute access +std::vector NetCDFManager::list_attributes(const std::string& var_name) const +{ + auto var = nc_file_->get_ncvar(var_name); + return var->list_attributes(); +} + +std::string NetCDFManager::get_string_attribute(const std::string& var_name, const std::string& att_name) const +{ + auto var = nc_file_->get_ncvar(var_name); + return var->get_string_attribute(att_name); +} + +int NetCDFManager::get_int_attribute(const std::string& var_name, const std::string& att_name) const +{ + auto var = nc_file_->get_ncvar(var_name); + return var->get_int_attribute(att_name); +} + +double NetCDFManager::get_double_attribute(const std::string& var_name, const std::string& att_name) const +{ + auto var = nc_file_->get_ncvar(var_name); + return var->get_double_attribute(att_name); +} + +void NetCDFManager::open_file() +{ + if(nc_file_) { + nc_file_->load_variables(); + } else { + throw std::runtime_error("NcFile not initialized!"); + } +} + +void NetCDFManager::close_file() +{ + nc_file_->close_file(); +} + +void NetCDFManager::create_timeslice(size_t num_entities, size_t num_timesteps, + const std::vector& var_names) { + + num_entities_ = num_entities; + num_timesteps_ = num_timesteps; + + // Define dimensions + nc_file_->add_dimension("time", num_timesteps_); + nc_file_->add_dimension("entity", num_entities_); + + // Create variables + for (const auto& name : var_names) { + std::vector dims = {num_timesteps_, num_entities_}; + std::vector dim_names = {"time", "entity"}; + auto var = std::make_shared( + name, NC_DOUBLE, dims, dim_names, + nc_file_->get_next_varid(), nc_file_->get_ncid()); + nc_file_->add_ncvar(var); + variables_map_[name] = var; + +#if USE_MPI + nc_var_par_access(nc_file_->get_ncid(), var->get_varid(), NC_COLLECTIVE); +#endif + } +} + +// Partition time dimension across ranks +void NetCDFManager::get_local_time_range(size_t& start, size_t& count) const { +#if USE_MPI + size_t base = num_timesteps_ / size_; + size_t remainder = num_timesteps_ % size_; + start = rank_ * base + std::min(static_cast(rank_), remainder); + count = base + (rank_ < static_cast(remainder) ? 1 : 0); +#else + start = 0; + count = num_timesteps_; +#endif +} + +template +void NetCDFManager::write_timeslice(const std::string& var_name, size_t time_start, + size_t time_count, const std::vector& data) { + auto it = variables_map_.find(var_name); + if (it == variables_map_.end()) + throw std::runtime_error("Variable not found: " + var_name); + + auto var = it->second; + + std::vector start = {time_start, 0}; + std::vector count = {time_count, var->get_dim_size(1)}; + +#if USE_MPI + if(rank_ >= 0 && size_ > 1) { + int retval = nc_put_vara_double(nc_file_->get_ncid(), var->get_varid(), + start.data(), count.data(), + data.data()); + if(retval) throw std::runtime_error("Error writing variable: " + std::string(nc_strerror(retval))); + } +#else + int retval = nc_put_vara_double(nc_file_->get_ncid(), var->get_varid(), + start.data(), count.data(), + data.data()); + if(retval) throw std::runtime_error("Error writing variable: " + std::string(nc_strerror(retval))); +#endif +} + + + +NetCDFManager::~NetCDFManager() { + #if NGEN_WITH_MPI + if(num_procs_ > 1 && comm_ != MPI_COMM_NULL) { + MPI_Comm_free(&comm_); + } + #endif +} +// Explicit template instantiation for double +template void NetCDFManager::write_timeslice(const std::string&, size_t, size_t, const std::vector&); +#endif // NGEN_WITH_NETCDF \ No newline at end of file diff --git a/src/netcdf/NetCDFVar.cpp b/src/netcdf/NetCDFVar.cpp new file mode 100644 index 0000000000..206b637cf7 --- /dev/null +++ b/src/netcdf/NetCDFVar.cpp @@ -0,0 +1,186 @@ +#if NGEN_WITH_NETCDF +#include "NetCDFVar.hpp" +#include +#include +#include +#include + +NetCDFVar::NetCDFVar(const std::string& name, nc_type type, const std::vector& dims, + const std::vector& dim_names,int varid, int ncid) + : name_(name), type_(type), dims_(dims), dim_names_(dim_names), varid_(varid), ncid_(ncid) +{} + +const std::string& NetCDFVar::get_name() const { return name_; } +nc_type NetCDFVar::get_type() const { return type_; } +const std::vector& NetCDFVar::get_dims() const { return dims_; } +const std::vector& NetCDFVar::get_dim_names() const { return dim_names_; } +size_t NetCDFVar::get_dim_count() const { return dims_.size(); } +int NetCDFVar::get_varid() const { return varid_; } + +size_t NetCDFVar::get_dim_size(const std::string& dim_name) const { + for (size_t i = 0; i < dim_names_.size(); ++i) { + if (dim_names_[i] == dim_name) { + return dims_[i]; + } + } + throw std::runtime_error( + "Dimension '" + dim_name + "' not found in variable '" + name_ + "'" + ); +} + +size_t NetCDFVar::get_dim_size(size_t idx) const { + if (idx >= dims_.size()) { + throw std::out_of_range( + "Dimension index " + std::to_string(idx) + + " out of range for variable '" + name_ + "'" + ); + } + return dims_[idx]; +} +size_t NetCDFVar::get_total_size() const { + return std::accumulate(dims_.begin(), dims_.end(), size_t{1}, std::multiplies()); +} + +std::vector NetCDFVar::get_string_array_values() const { + + // Safety check for zero dimensions + if (dims_.empty()) { + throw std::runtime_error("Variable '" + name_ + "' has no dimensions"); + } + std::vector array_items; + if (type_ == NC_STRING) { + if (dims_.size() != 1) { + throw std::runtime_error("NC_STRING variable must be 1D"); + } + size_t items_count = dims_[0]; + char** raw_strings = nullptr; + int retval = nc_get_var_string(ncid_, varid_, raw_strings); + if (retval != NC_NOERR) { + throw std::runtime_error(nc_strerror(retval)); + } + array_items.reserve(items_count); + for (size_t i = 0; i < items_count; ++i) { + array_items.emplace_back(raw_strings[i] ? raw_strings[i] : ""); + } + nc_free_string(items_count, raw_strings); + } + else if (type_ == NC_CHAR) { + if (dims_.size() != 2) { + throw std::runtime_error("NC_CHAR string variable must be 2D"); + } + size_t num_strings = dims_[0]; + size_t str_len = dims_[1]; + std::vector buffer(num_strings * str_len); + int retval = nc_get_var_text(ncid_, varid_, buffer.data()); + if (retval != NC_NOERR) { + throw std::runtime_error(nc_strerror(retval)); + } + array_items.reserve(num_strings); + for (size_t i = 0; i < num_strings; ++i) { + const char* start = buffer.data() + (i * str_len); + std::string s(start, str_len); + size_t end = s.find_last_not_of(" \0"); // trim trailing spaces/nulls + if (end != std::string::npos) { + s.erase(end + 1); + } else { + s.clear(); + } + array_items.push_back(s); + } + } + if (array_items.empty()) { + throw std::runtime_error("Unsupported variable type called for this function."); + } + return array_items; +} + +std::vector NetCDFVar::get_time_values() const +{ + // Safety checks for dimensions and data type + if (dims_.empty()) { + throw std::runtime_error("Variable '" + name_ + "' has no dimensions"); + } + if (type_ != NC_DOUBLE) { + throw std::runtime_error("Variable '" + name_ + "' is not NC_DOUBLE"); + } + + size_t time_count = 0; + if (dims_.size() == 1) { + time_count = dims_[0]; // (time) + + } + else if (dims_.size() == 2) { + time_count = dims_[1]; // (entity, time) + } else { + throw std::runtime_error("Unsupported dimensions for time variable '" + name_ + "'"); + } + if (time_count == 0) { + return {}; + } + + std::vector time_values(time_count); + int retval = NC_NOERR; + if (dims_.size() == 1) { + retval = nc_get_var_double(ncid_, varid_, time_values.data()); + } else { + // Read first entity slice: + // (entity=0, all times) + std::vector start = {0, 0}; + std::vector count = {1, time_count}; + retval = nc_get_vara_double(ncid_, varid_, start.data(), count.data(), time_values.data()); + } + if (retval != NC_NOERR) { + throw std::runtime_error(nc_strerror(retval)); + } + return time_values; +} + +void NetCDFVar::add_attribute(const std::string& att_name, const std::string& att_value) { + attributes_str_[att_name] = att_value; +} + +void NetCDFVar::add_attribute(const std::string& att_name, int att_value) { + attributes_int_[att_name] = att_value; +} + +void NetCDFVar::add_attribute(const std::string& att_name, double att_value) { + attributes_double_[att_name] = att_value; +} + +// List attributes +std::vector NetCDFVar::list_attributes() const { + std::vector names; + for(const auto& a : attributes_str_) names.push_back(a.first); + for(const auto& a : attributes_int_) names.push_back(a.first); + for(const auto& a : attributes_double_) names.push_back(a.first); + return names; +} + +// Access attribute values +std::string NetCDFVar::get_string_attribute(const std::string& att_name) const { + auto it = attributes_str_.find(att_name); + if(it != attributes_str_.end()) return it->second; + throw std::runtime_error("String attribute not found: " + att_name); +} + +int NetCDFVar::get_int_attribute(const std::string& att_name) const { + auto it = attributes_int_.find(att_name); + if(it != attributes_int_.end()) return it->second; + throw std::runtime_error("Int attribute not found: " + att_name); +} + +double NetCDFVar::get_double_attribute(const std::string& att_name) const { + auto it = attributes_double_.find(att_name); + if(it != attributes_double_.end()) return it->second; + throw std::runtime_error("Double attribute not found: " + att_name); +} + +void NetCDFVar::read_slice(const std::vector& start, const std::vector& count, double* data) const +{ + int retval = NC_NOERR; + retval = nc_get_vara_double(ncid_, varid_, start.data(), count.data(), data); + if (retval != NC_NOERR) { + throw std::runtime_error(std::string("NetCDF read error: ") + nc_strerror(retval)); + } +} +#endif // NGEN_WITH_NETCDF \ No newline at end of file From 8e1625e28988328bf626181225365acca179b155 Mon Sep 17 00:00:00 2001 From: "siva.selvanathan" Date: Wed, 13 May 2026 13:08:07 -0400 Subject: [PATCH 5/8] Added more updates due to C library. --- src/forcing/CMakeLists.txt | 6 ++++-- src/utilities/mdframe/CMakeLists.txt | 2 +- src/utilities/mdframe/handler_netcdf.cpp | 14 +++++++++++++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/forcing/CMakeLists.txt b/src/forcing/CMakeLists.txt index b02e3c4ff8..445c505254 100644 --- a/src/forcing/CMakeLists.txt +++ b/src/forcing/CMakeLists.txt @@ -6,10 +6,12 @@ find_package(Threads REQUIRED) target_include_directories(forcing PUBLIC ${PROJECT_SOURCE_DIR}/include + {PROJECT_SOURCE_DIR}/include/netcdf ${PROJECT_SOURCE_DIR}/include/core ${PROJECT_SOURCE_DIR}/include/core/mediator ${PROJECT_SOURCE_DIR}/include/utilities ${PROJECT_SOURCE_DIR}/include/forcing + {CMAKE_SOURCE_DIR}/include ) target_link_libraries(forcing PUBLIC @@ -25,7 +27,7 @@ target_sources(forcing PRIVATE "${CMAKE_CURRENT_LIST_DIR}/NullForcingProvider.cp if(NGEN_WITH_NETCDF) target_sources(forcing PRIVATE "${CMAKE_CURRENT_LIST_DIR}/NetCDFPerFeatureDataProvider.cpp") - target_link_libraries(forcing PUBLIC NetCDF) + target_link_libraries(forcing PRIVATE NetCDFManager) endif() if(NGEN_WITH_PYTHON) @@ -37,4 +39,4 @@ if(NGEN_WITH_PYTHON) target_link_libraries(forcing PUBLIC NGen::ngen_bmi) endif() -#target_compile_options(forcing PUBLIC -std=c++14 -Wall) +#target_compile_definitions(forcing PUBLIC $<$:NGEN_WITH_NETCDF>) diff --git a/src/utilities/mdframe/CMakeLists.txt b/src/utilities/mdframe/CMakeLists.txt index a254268f54..1770770716 100644 --- a/src/utilities/mdframe/CMakeLists.txt +++ b/src/utilities/mdframe/CMakeLists.txt @@ -10,5 +10,5 @@ NGen::logging target_link_libraries(mdframe PUBLIC ewts::ewts_ngen_bridge) if(NGEN_WITH_NETCDF) - target_link_libraries(mdframe PUBLIC NetCDF) + target_link_libraries(mdframe PUBLIC NetCDFManager) endif() diff --git a/src/utilities/mdframe/handler_netcdf.cpp b/src/utilities/mdframe/handler_netcdf.cpp index 0858a080d8..6141c56e9d 100644 --- a/src/utilities/mdframe/handler_netcdf.cpp +++ b/src/utilities/mdframe/handler_netcdf.cpp @@ -4,7 +4,10 @@ #if NGEN_WITH_NETCDF -#include +//#include +#include "netcdf/NetCDFManager.hpp" +#include "netcdf/NetCDFFile.hpp" +#include "netcdf/NetCDFVar.hpp" namespace ngen { @@ -31,6 +34,15 @@ struct mdarray_netcdf_putvar : public boost::static_visitor void mdframe::to_netcdf(const std::string& path) const { + nc_manager = new NetCDFManager(); + nc_manager.createFile(path); + + std::unordered_map dimmap; + for (const auto& dim : this->m_dimensions) + dimmap[dim.name()] = nc_manager.add_dimension(dim.name(), dim.size()); + + std::unordered_map varmap; + netCDF::NcFile output{path, netCDF::NcFile::replace}; std::unordered_map dimmap; From 448df0f52f2b832cc78441681c22c2b09efd724f Mon Sep 17 00:00:00 2001 From: "siva.selvanathan" Date: Wed, 13 May 2026 13:10:23 -0400 Subject: [PATCH 6/8] Adding a bak file generated --- cmake/{FindNetCDF.cmake => FindNetCDF.cmake.bak} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename cmake/{FindNetCDF.cmake => FindNetCDF.cmake.bak} (100%) diff --git a/cmake/FindNetCDF.cmake b/cmake/FindNetCDF.cmake.bak similarity index 100% rename from cmake/FindNetCDF.cmake rename to cmake/FindNetCDF.cmake.bak From f84be25ccc99a5a306ecbc24767534969becbbe5 Mon Sep 17 00:00:00 2001 From: "siva.selvanathan" Date: Wed, 13 May 2026 13:21:05 -0400 Subject: [PATCH 7/8] submodule hashes --- extern/LASAM | 2 +- extern/SoilFreezeThaw/SoilFreezeThaw | 2 +- extern/SoilMoistureProfiles/SoilMoistureProfiles | 2 +- extern/lstm | 2 +- extern/sac-sma/sac-sma | 2 +- extern/snow17 | 2 +- extern/t-route | 2 +- extern/ueb-bmi | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/extern/LASAM b/extern/LASAM index 0fcbf84200..764dc82e8f 160000 --- a/extern/LASAM +++ b/extern/LASAM @@ -1 +1 @@ -Subproject commit 0fcbf842009bd7bd8929e2c83aba4fa3b97f4de6 +Subproject commit 764dc82e8fb5a160e646a8f0fde35f964fd42234 diff --git a/extern/SoilFreezeThaw/SoilFreezeThaw b/extern/SoilFreezeThaw/SoilFreezeThaw index c45d48f5f4..815a970af3 160000 --- a/extern/SoilFreezeThaw/SoilFreezeThaw +++ b/extern/SoilFreezeThaw/SoilFreezeThaw @@ -1 +1 @@ -Subproject commit c45d48f5f402dd65e8da0efa31b586ed5faf33c5 +Subproject commit 815a970af30e407785bc767a4a5f5dffa2708eb5 diff --git a/extern/SoilMoistureProfiles/SoilMoistureProfiles b/extern/SoilMoistureProfiles/SoilMoistureProfiles index 3118a7b9c2..d29f268223 160000 --- a/extern/SoilMoistureProfiles/SoilMoistureProfiles +++ b/extern/SoilMoistureProfiles/SoilMoistureProfiles @@ -1 +1 @@ -Subproject commit 3118a7b9c2098168a123dd65e58f377f84218c95 +Subproject commit d29f26822391740549642a9818065ea296688b2e diff --git a/extern/lstm b/extern/lstm index b3ccaeb458..f2b2d3c33a 160000 --- a/extern/lstm +++ b/extern/lstm @@ -1 +1 @@ -Subproject commit b3ccaeb45897693d7ebb05dc38788a5766a6921e +Subproject commit f2b2d3c33abe19faac8d95a0e09765221f0676ee diff --git a/extern/sac-sma/sac-sma b/extern/sac-sma/sac-sma index 4aa0d804f7..a5afd2ad0b 160000 --- a/extern/sac-sma/sac-sma +++ b/extern/sac-sma/sac-sma @@ -1 +1 @@ -Subproject commit 4aa0d804f707f324d5b17143f350e24a52e6e2b5 +Subproject commit a5afd2ad0b48fcdecd960dff3fc20c362ed7d8ad diff --git a/extern/snow17 b/extern/snow17 index f224c0aaea..8d10da02a0 160000 --- a/extern/snow17 +++ b/extern/snow17 @@ -1 +1 @@ -Subproject commit f224c0aaeaebc6980ea403fe72093219b6d0019c +Subproject commit 8d10da02a0fa0155d962b65068775990be71d56b diff --git a/extern/t-route b/extern/t-route index 6c16005256..625eb805e7 160000 --- a/extern/t-route +++ b/extern/t-route @@ -1 +1 @@ -Subproject commit 6c160052568f9871dc7f1dac06446dc427a49047 +Subproject commit 625eb805e74b8fbaae749d6bd8393f321eeac5e1 diff --git a/extern/ueb-bmi b/extern/ueb-bmi index d18ddb425d..26f76de81c 160000 --- a/extern/ueb-bmi +++ b/extern/ueb-bmi @@ -1 +1 @@ -Subproject commit d18ddb425d456c37fc049d909da9752b66db7695 +Subproject commit 26f76de81c34cb27da41d33f9131ad26da45db65 From 220c800feca0f488fdccf02ed64b1cbedc918642 Mon Sep 17 00:00:00 2001 From: "siva.selvanathan" Date: Tue, 19 May 2026 00:15:01 -0400 Subject: [PATCH 8/8] Commit with successful build for ngen with MPI on and off. Test executable fails due to shared library issue. --- CMakeLists.txt | 38 ++- include/netcdf/NetCDFFile.hpp | 37 +-- include/netcdf/NetCDFManager.hpp | 38 ++- include/netcdf/NetCDFVar.hpp | 41 +++ src/core/NgenSimulation.cpp | 19 +- src/netcdf/NetCDFFile.cpp | 303 +++++++++-------------- src/netcdf/NetCDFManager.cpp | 251 +++++++++++++------ src/netcdf/NetCDFVar.cpp | 81 ++++++ src/utilities/mdframe/handler_netcdf.cpp | 81 +++--- test/CMakeLists.txt | 40 +-- test/utils/mdframe_netcdf_Test.cpp | 43 ++-- 11 files changed, 562 insertions(+), 410 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f471abd588..613deebfa2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -171,6 +171,8 @@ endif() # If MPI support is enabled, set the C++ compiler to "mpicxx" if(NGEN_WITH_MPI) find_package(MPI REQUIRED) +set(CMAKE_C_COMPILER ${MPI_C_COMPILER} CACHE FILEPATH "" FORCE) +set(CMAKE_CXX_COMPILER ${MPI_CXX_COMPILER} CACHE FILEPATH "" FORCE) endif() # ----------------------------------------------------------------------------- @@ -216,24 +218,6 @@ add_compile_definitions(NGEN_WITH_UDUNITS) endif() # ----------------------------------------------------------------------------- -# if(NGEN_WITH_NETCDF) -# find_package(NetCDF COMPONENTS CXX) - -# if(NOT NetCDF_CXX_FOUND) -# find_package(NetCDF REQUIRED) -# message(INFO " Using internal copy of netcdf-cxx4") -# add_external_subdirectory( -# SOURCE "${NGEN_EXT_DIR}/netcdf-cxx4" -# OUTPUT "${PROJECT_BINARY_DIR}/extern-libs/netcdf-cxx4" -# GIT_UPDATE "${NGEN_EXT_DIR}/netcdf-cxx4/netcdf-cxx4" -# IMPORTS netcdf-cxx4 -# ) -# set(NetCDF_CXX_INCLUDE_DIR "${NGEN_EXT_DIR}/netcdf-cxx4/netcdf-cxx4/cxx4") -# set(NetCDF_CXX_LIBRARY "${PROJECT_BINARY_DIR}/extern-libs/netcdf-cxx4/libnetcdf-cxx4.a") -# target_link_libraries(NetCDF INTERFACE netcdf-cxx4) -# endif() -# endif() - if(NGEN_WITH_EXTERN_SLOTH) # This is set because SLOTH will link to gtest by default, # which causes a configure error. @@ -396,10 +380,17 @@ target_include_directories(partitionGenerator PUBLIC AFTER "${NGEN_INC_DIR}/geop endif() if(NGEN_WITH_NETCDF) - set(HDF5_INSTALL "$ENV{HOME}/MyInstalls/hdf5_serial" CACHE PATH "Path to Serial HDF5 installation") - set(NETCDF_INSTALL "$ENV{HOME}/MyInstalls/netcdf_serial" CACHE PATH "Path to Serial NetCDF installation") - set(NETCDF_LIB_DIR "$ENV{HOME}/MyInstalls/netcdf_serial/lib") - set(HDF5_LIB_DIR "$ENV{HOME}/MyInstalls/hdf5_serial/lib") + if(NGEN_WITH_MPI) + set(HDF5_INSTALL "$ENV{HOME}/MyInstalls/hdf5_mpi_parallel" CACHE PATH "Path to Parallel HDF5 installation") + set(NETCDF_INSTALL "$ENV{HOME}/MyInstalls/netcdf_mpi_parallel" CACHE PATH "Path to Parallel NetCDF installation") + set(NETCDF_LIB_DIR "$ENV{HOME}/MyInstalls/netcdf_mpi_parallel/lib") + set(HDF5_LIB_DIR "$ENV{HOME}/MyInstalls/hdf5_mpi_parallel/lib") + else() + set(HDF5_INSTALL "$ENV{HOME}/MyInstalls/hdf5_serial" CACHE PATH "Path to Serial HDF5 installation") + set(NETCDF_INSTALL "$ENV{HOME}/MyInstalls/netcdf_serial" CACHE PATH "Path to Serial NetCDF installation") + set(NETCDF_LIB_DIR "$ENV{HOME}/MyInstalls/netcdf_serial/lib") + set(HDF5_LIB_DIR "$ENV{HOME}/MyInstalls/hdf5_serial/lib") + endif() set(WRAPPER_SOURCES ${NGEN_SRC_DIR}/netcdf/NetCDFManager.cpp ${NGEN_SRC_DIR}/netcdf/NetCDFFile.cpp @@ -434,6 +425,9 @@ if(NGEN_WITH_NETCDF) $<$:MPI::MPI_CXX> ) target_compile_definitions(NetCDFManager PUBLIC NGEN_WITH_NETCDF) + if(NGEN_WITH_MPI) + target_compile_definitions(NetCDFManager PUBLIC NGEN_WITH_MPI) + endif() target_link_libraries(ngen PRIVATE NetCDFManager) # set target properties to the actual path so that it is propagated during runtime. set_target_properties(ngen PROPERTIES diff --git a/include/netcdf/NetCDFFile.hpp b/include/netcdf/NetCDFFile.hpp index 376be0d0bc..68f7301823 100644 --- a/include/netcdf/NetCDFFile.hpp +++ b/include/netcdf/NetCDFFile.hpp @@ -3,36 +3,30 @@ //#include -#if NGEN_WITH_MPI - #include - #define _PARALLEL4 -#endif - - #if NGEN_WITH_NETCDF #include #include #include #include +#if NGEN_WITH_MPI +#include +#define _PARALLEL4 +#include +#else #include +#endif #include "NetCDFVar.hpp" class NetCDFFile { public: -#if NGEN_WITH_MPI - NetCDFFile(const std::string& filename, bool write_only, MPI_Comm comm); -#endif - // Serial constructor for write only - NetCDFFile(const std::string& filename, bool write_only); - - // Serial read-only mode for NetCDFPerFeatureDataProvider - NetCDFFile(const std::string &filename); + NetCDFFile(const std::string& filename, bool write_only, bool is_mpi); ~NetCDFFile(); void load_variables(); std::vector list_variables() const; std::shared_ptr get_ncvar(const std::string& name) const; + int get_ncid() const { return ncid_; } // Dimension handling int add_dimension(const std::string& name, size_t len); @@ -65,25 +59,16 @@ class NetCDFFile { //Add attribute to a variable void write_attribute_to_ncvar(const std::string& name, const std::string& attName, const std::string& attValue); - - - // // Write all variables in parallel - // template - // void writeAllVariables(const std::vector>& all_data); - - // // MPI-safe write (each rank writes its portion) - // template - // void writeAllVariablesDistributed(const std::vector>& all_data); - void close_file(); - int get_ncid() const { return ncid_; } + void end_def_mode(); private: std::string nc_file_name_; bool read_only_; int ncid_; int next_varid_ = 0; + bool is_mpi_; std::vector> variables_; std::map dims_; std::map> variables_map_; @@ -96,7 +81,7 @@ class NetCDFFile { bool parallel_; // Helper to get variable index by name - std::shared_ptr get_ncvar_by_name(const std::string& name); + std::shared_ptr get_ncvar_by_name(const std::string& name) const; }; #endif // NGEN_WITH_NETCDF #endif // NETCDFFILE_HPP \ No newline at end of file diff --git a/include/netcdf/NetCDFManager.hpp b/include/netcdf/NetCDFManager.hpp index 00670143a3..313d4941f0 100644 --- a/include/netcdf/NetCDFManager.hpp +++ b/include/netcdf/NetCDFManager.hpp @@ -2,12 +2,13 @@ #define NETCDFMANAGER_HPP //#include + + +#if NGEN_WITH_NETCDF #if NGEN_WITH_MPI #include #define _PARALLEL4 #endif - -#if NGEN_WITH_NETCDF #include "NetCDFFile.hpp" #include "NetCDFVar.hpp" #include @@ -42,13 +43,19 @@ class NetCDFManager NetCDFManager(); // File operations - void create_file(const std::string& filename); + int create_file(const std::string& filename); void open_file(); void close_file(); + //set up netcdf dimensions and variables + void define_catchment_netcdf_components(); + // List variable names std::vector list_variables() const; + //Get NetCDFFile handle + NetCDFFile* get_file_handle() {return nc_file_.get();} + // Access NetCDFVar by name std::shared_ptr get_ncvar_by_name(const std::string& name) const; @@ -64,7 +71,7 @@ class NetCDFManager // Add a variable void add_variable(const std::string& var_name, nc_type type, const std::vector& dims, const std::vector& dim_names); - // Add a variable to the file (for writing) + // Add variables to the file (for writing) void add_output_variable_data_from_formulation(); // Add catchment output data to the file (for writing) @@ -82,27 +89,38 @@ class NetCDFManager // Get the time range assigned to this rank void get_local_time_range(size_t& start, size_t& count) const; + size_t get_chunk_start() const {return chunk_start_; } + + size_t get_chunk_count() const {return chunk_count_; } + + void prepare_data_chunks(std::map catchment_output_vals); + + void write_timestep_data_to_netcdf(size_t time_index); ~NetCDFManager(); + private: - int rank_; - int num_procs_; bool read_only_; - std::string filename_; - std::shared_ptr nc_file_; + std::string nc_filename_; + std::unique_ptr nc_file_; std::vector vars_; std::shared_ptr manager_; std::shared_ptr sim_time_; + size_t chunk_start_ = 0; + size_t chunk_count_ = 0; size_t num_entities_; size_t num_timesteps_; std::vector catchments_; std::map> variables_map_; - std::vector nc_output_variables; + std::vector nc_output_variables_; + std::vector> data_chunks_; #if NGEN_WITH_MPI MPI_Comm comm_; #endif - bool is_mpi_; + int rank_ = 0; + int num_procs_ = 1; + bool is_mpi_ = false; }; #endif // NGEN_WITH_NETCDF #endif // NETCDFMANAGER_HPP \ No newline at end of file diff --git a/include/netcdf/NetCDFVar.hpp b/include/netcdf/NetCDFVar.hpp index 5263245dc9..e07823dd22 100644 --- a/include/netcdf/NetCDFVar.hpp +++ b/include/netcdf/NetCDFVar.hpp @@ -5,9 +5,17 @@ #if NGEN_WITH_NETCDF #include +#include #include #include +#include + +#if NGEN_WITH_MPI +#include +#include +#else #include +#endif class NetCDFVar { public: @@ -25,16 +33,48 @@ class NetCDFVar { size_t get_total_size() const; std::vector get_string_array_values() const; std::vector get_time_values() const; + int get_int_value_at_index(const std::vector& index) const; + double get_dbl_value_at_index(const std::vector& index) const; + std::string get_str_value_at_index(const std::vector& index) const; void read_slice(const std::vector& start, const std::vector& count, double* data) const; void add_attribute(const std::string& att_name, const std::string& att_value); void add_attribute(const std::string& att_name, int att_value); void add_attribute(const std::string& att_name, double att_value); + void build_variables_index(size_t num_items); std::vector list_attributes() const; std::string get_string_attribute(const std::string& att_name) const; int get_int_attribute(const std::string& att_name) const; double get_double_attribute(const std::string& att_name) const; + size_t get_variable_index(const std::string& name) const; + + void write_timesliced_data(size_t timestep, size_t slice_start, size_t slice_count, const double* data); + + //Implementing this single-element write function for mdframe_netcdf_test + template + void write_single_value(const std::vector& block_array, const T& value) const + { + std::vector start(block_array.begin(), block_array.end()); + std::vector count(block_array.size(), 1); + + int retval = 0; + + if (std::is_same::value) { + retval = nc_put_vara_int(ncid_, varid_, start.data(), count.data(), &value); + } + else if (std::is_same::value) { + retval = nc_put_vara_float(ncid_, varid_, start.data(), count.data(), &value); + } + else if (std::is_same::value) { + retval = nc_put_vara_double(ncid_, varid_, start.data(), count.data(), &value); + } + else { + throw std::runtime_error("Unsupported type for write_single_value"); + } + + if (retval != NC_NOERR) throw std::runtime_error(std::string("Erorr in writing single value: ") + nc_strerror(retval)); + } private: std::string name_; @@ -47,6 +87,7 @@ class NetCDFVar { std::map attributes_str_; std::map attributes_int_; std::map attributes_double_; + std::unordered_map variable_index_; }; #endif // NGEN_WITH_NETCDF #endif // NETCDFVAR_HPP \ No newline at end of file diff --git a/src/core/NgenSimulation.cpp b/src/core/NgenSimulation.cpp index c3db0e5d8a..a66f3127f0 100644 --- a/src/core/NgenSimulation.cpp +++ b/src/core/NgenSimulation.cpp @@ -8,10 +8,6 @@ #include "HY_Features.hpp" #endif -// #if NGEN_WITH_NETCDF -// #include "netcdf/NetCDFManager.hpp" -// #endif - #include "state_save_restore/State_Save_Utils.hpp" #include "state_save_restore/State_Save_Restore.hpp" #include "parallel_utils.h" @@ -114,11 +110,16 @@ void NgenSimulation::advance_models_one_output_step() // After updating the layer, get the output data for that timestep and write to netcdf. This is currently //set up only for a non-MPI run. #if NGEN_WITH_NETCDF - #if !NGEN_WITH_MPI +// #if !NGEN_WITH_MPI std::map catchment_output_vals = layer->get_catchment_output_data_for_timestep(); - //nc_writer_->write_simulations_response_from_formulation(simulation_step_,catchment_output_vals); - nc_manager_->write_simulations_response_from_formulation(simulation_step_,catchment_output_vals); - #endif //NGEN_WITH_MPI + size_t chunk_start = nc_manager_->get_chunk_start(); + size_t chunk_count = nc_manager_->get_chunk_count(); + LOG("Chunk start: " + std::to_string(chunk_start) + "; Chunk count: " + + std::to_string(chunk_count) + " for rank: " + std::to_string(mpi_rank_), LogLevel::DEBUG); + nc_manager_->prepare_data_chunks(catchment_output_vals); + //nc_manager_->write_simulations_response_from_formulation(simulation_step_,catchment_output_vals); + nc_manager_->write_timestep_data_to_netcdf(simulation_step_); +// #endif //NGEN_WITH_MPI #endif //NGEN_WITH_NETCDF prev_layer_time = layer_next_time; @@ -404,9 +405,7 @@ void NgenSimulation::serialize(Archive& ar) { void NgenSimulation::create_netcdf_writer(std::shared_ptr manager, std::string nc_output_file_name, int mpi_rank, int mpi_num_procs) { - //this->nc_writer_ = std::make_unique(manager,nc_output_file_name,*sim_time_, mpi_rank, mpi_num_procs); #if NGEN_WITH_NETCDF this->nc_manager_ = std::make_unique(manager, nc_output_file_name, *sim_time_, mpi_rank, mpi_num_procs); #endif - //nc_manager_ = std::make_unique(nc_output_file_name, mpi_rank, mpi_num_procs); } diff --git a/src/netcdf/NetCDFFile.cpp b/src/netcdf/NetCDFFile.cpp index 1539e92fd9..c1deb5dde3 100644 --- a/src/netcdf/NetCDFFile.cpp +++ b/src/netcdf/NetCDFFile.cpp @@ -4,43 +4,47 @@ #if NGEN_WITH_NETCDF #include "NetCDFFile.hpp" +#include "ewts_ngen/logger.hpp" #include #include #include #include #include -#if NGEN_WITH_MPI -NetCDFFile::NetCDFFile(const std::string& filename, bool write_only, MPI_Comm comm) - : nc_file_name_(filename), comm_(comm) -{ - int retval; - int mode = NC_NETCDF4; - if (write_only) mode |= NC_CLOBBER; - if ((retval = nc_create_par(filename.c_str(), mode, comm_, MPI_INFO_NULL, &ncid_))) - throw std::runtime_error("Error creating NetCDF file: " + std::string(nc_strerror(retval))); -} -#else -NetCDFFile::NetCDFFile(const std::string& full_filename, bool write_only) +NetCDFFile::NetCDFFile(const std::string& filename, bool write_only, bool is_mpi) + : nc_file_name_(filename), is_mpi_(is_mpi) { int retval; int mode = NC_NETCDF4; - if (write_only) mode |= NC_CLOBBER; - if ((retval = nc_create(full_filename.c_str(), mode, &ncid_))) - throw std::runtime_error("Error creating NetCDF file: " + std::string(nc_strerror(retval))); - -} -#endif - -NetCDFFile::NetCDFFile(const std::string &filename) -{ - nc_file_name_ = filename; - read_only_ = true; #if NGEN_WITH_MPI - comm_ = MPI_COMM_NULL; + if(is_mpi_){ + if(write_only){ + retval = nc_create_par(nc_file_name_.c_str(), NC_NETCDF4 | NC_MPIIO | NC_CLOBBER, + MPI_COMM_WORLD, MPI_INFO_NULL, &ncid_); + } + else{ + retval = nc_open_par(nc_file_name_.c_str(), NC_WRITE | NC_MPIIO, + MPI_COMM_WORLD, MPI_INFO_NULL, &ncid_); + } + } + else #endif - load_variables(); + { + if(write_only){ + LOG("Attempting to create NetCDF file", LogLevel::DEBUG); + retval = nc_create(nc_file_name_.c_str(), NC_NETCDF4 | NC_CLOBBER, &ncid_); + } + else{ + read_only_ = true; + LOG("Attempting to open NetCDF file", LogLevel::DEBUG); + retval = nc_open(nc_file_name_.c_str(), NC_NOWRITE, &ncid_); + } + } + + if(retval){ + throw std::runtime_error("Failed creating/opening NetCDF file: " + std::string(nc_strerror(retval))); + } } // Add a new dimension @@ -57,7 +61,7 @@ void NetCDFFile::add_variable(const std::string& name, nc_type type, const std:: int retval = nc_def_var(ncid_, name.c_str(), type, dims.size(), dims.data(), &varid); if (retval != NC_NOERR) throw std::runtime_error(nc_strerror(retval)); // set fill value and missing value for NC_DOUBLE - if (type = NC_DOUBLE) { + if (type == NC_DOUBLE) { double fill_missing_val = -1.0; nc_def_var_fill(ncid_, varid, 0, &fill_missing_val); nc_put_att_double(ncid_, varid, "missing_value", NC_DOUBLE, 1, &fill_missing_val); @@ -69,17 +73,17 @@ void NetCDFFile::add_variable(const std::string& name, nc_type type, const std:: // Get dimension length by name size_t NetCDFFile::get_dim_size(const std::string& name) const { auto it = dims_.find(name); - if(it == dims_.end()) throw std::runtime_error("Dimension not found: " + name); + if(it == dims_.end()) return -1; size_t len; int retval = nc_inq_dimlen(ncid_, it->second, &len); - if(retval) throw std::runtime_error("Error getting dimension length for " + name + ": " + nc_strerror(retval)); + if(retval != NC_NOERR) return -1; return len; } int NetCDFFile::get_dim_id(const std::string& name) const { auto it = dims_.find(name); - if(it == dims_.end()) throw std::runtime_error("Dimension not found: " + name); + if(it == dims_.end()) return -1; return it->second; } @@ -88,12 +92,13 @@ void NetCDFFile::add_ncvar(std::shared_ptr var) { variables_map_[var->get_name()] = var; } - void NetCDFFile::write_catchment_output_data(const std::string& name, std::vector start, +void NetCDFFile::write_catchment_output_data(const std::string& name, std::vector start, std::vector count, const double& data) { auto var = get_ncvar_by_name(name); if (!var) throw std::runtime_error("Variable not found: " + name); int retval = nc_put_vara_double(ncid_, var->get_varid(), start.data(), count.data(), &data); + LOG("Added value for catchments", LogLevel::INFO); if (retval != NC_NOERR) { throw std::runtime_error("Error writing value: " + std::string(nc_strerror(retval))); } @@ -127,198 +132,122 @@ void NetCDFFile::write_variable_data(const std::string& name, const std::vector< #if NGEN_WITH_MPI if (comm_ != MPI_COMM_NULL) { - int retval = nc_var_par_access(ncid_, var->getVarId(), NC_COLLECTIVE); + int retval = nc_var_par_access(ncid_, var->get_varid(), NC_COLLECTIVE); if (retval != NC_NOERR) throw std::runtime_error(nc_strerror(retval)); } #endif int retval = NC_NOERR; retval = write_data_to_ncvar(ncid_, var->get_varid(), start, count, data); + if (retval != NC_NOERR) throw std::runtime_error(nc_strerror(retval)); + // We need to construct a map of variable value and the index in the nc file. + // this will be used while writing the output values. + // We need this only for the coordinates variables like catchments. + nc_type data_type; + retval = nc_inq_vartype(ncid_, var->get_varid(), &data_type); if (retval != NC_NOERR) throw std::runtime_error(nc_strerror(retval)); + + if (data_type == NC_STRING) { + var->build_variables_index(data.size()); + } } void NetCDFFile::write_attribute_to_ncvar(const std::string& name, const std::string& attName, const std::string& attValue) { auto var = get_ncvar_by_name(name); if (!var) throw std::runtime_error("Variable not found: " + name); - var->add_attribute(attName, attValue); + double value; + size_t pos; + bool is_numeric = true; + try { + value = std::stod(attValue, &pos); + } + catch (...) { + is_numeric = false; + } + if (pos == attValue.size()){ //needed to ensure that the entire string is a numeric value. + var->add_attribute(attName, value); + } + else{ + var->add_attribute(attName, attValue); + } } -// template -// void NcFile::writeAllVariables(const std::vector>& all_data) { -// if (all_data.size() != variables_.size()) -// throw std::runtime_error("Mismatch between variables and data"); - -// for (size_t v = 0; v < variables_.size(); ++v) { -// auto var = variables_[v]; -// const auto& data = all_data[v]; - -// #if NGEN_WITH_MPI -// if (comm_ != MPI_COMM_NULL && num_procs_ > 1) { -// int retval = nc_var_par_access(ncid_, var->getVarId(), NC_COLLECTIVE); -// if (retval != NC_NOERR) throw std::runtime_error(nc_strerror(retval)); -// } -// #endif - -// int retval = NC_NOERR; -// if constexpr (std::is_same_v) { -// retval = nc_put_var_double(ncid_, var->getVarId(), data.data()); -// } else if constexpr (std::is_same_v) { -// retval = nc_put_var_int(ncid_, var->getVarId(), data.data()); -// } else { -// static_assert(sizeof(T) == 0, "Unsupported data type"); -// } - -// if (retval != NC_NOERR) throw std::runtime_error(nc_strerror(retval)); -// } -// } - -// template -// void NetCDFFile::writeAllVariablesDistributed(const std::vector>& all_data) { -// if (all_data.size() != variables_.size()) -// throw std::runtime_error("Mismatch between variables and data"); - -// for (size_t v = 0; v < variables_.size(); ++v) { -// auto var = variables_[v]; -// const auto& data = all_data[v]; - -// // Compute first-dimension chunking -// size_t first_dim_size = var->getDims()[0]; -// size_t start0, count0; -// computeStartCount(first_dim_size, start0, count0); - -// std::vector start(var->getDims().size(), 0); -// std::vector count = var->getDims(); -// start[0] = start0; -// count[0] = count0; - -// #if NGEN_WITH_MPI -// if (comm_ != MPI_COMM_NULL && num_procs_ > 1) { -// int retval = nc_var_par_access(ncid_, var->getVarId(), NC_COLLECTIVE); -// if (retval != NC_NOERR) throw std::runtime_error(nc_strerror(retval)); -// } -// #endif - -// int retval = NC_NOERR; -// if constexpr (std::is_same_v) { -// retval = nc_put_vara_double(ncid_, var->getVarId(), start.data(), count.data(), -// &data[start0 * (data.size()/first_dim_size)]); -// } else if constexpr (std::is_same_v) { -// retval = nc_put_vara_int(ncid_, var->getVarId(), start.data(), count.data(), -// &data[start0 * (data.size()/first_dim_size)]); -// } else { -// static_assert(sizeof(T) == 0, "Unsupported data type"); -// } - -// if (retval != NC_NOERR) throw std::runtime_error(nc_strerror(retval)); -// } -// } - -std::shared_ptr NetCDFFile::get_ncvar_by_name(const std::string& name) { +std::shared_ptr NetCDFFile::get_ncvar(const std::string& name) const +{ + return get_ncvar_by_name(name); +} + +void NetCDFFile::end_def_mode() +{ + int retval = nc_enddef(ncid_); + if(retval != NC_NOERR) + throw std::runtime_error(std::string("Error switching to NetCDF data mode: ") + nc_strerror(retval)); +} + +std::shared_ptr NetCDFFile::get_ncvar_by_name(const std::string& name) const { auto it = variables_map_.find(name); if (it != variables_map_.end()) return it->second; return nullptr; } -// void NetCDFFile::computeStartCount(size_t total_size, size_t& start, size_t& count) { -// #if NGEN_WITH_MPI -// if (comm_ == MPI_COMM_NULL || num_procs_ == 1) { -// start = 0; -// count = total_size; -// } else { -// size_t chunk = total_size / num_procs_; -// start = rank_ * chunk; -// count = (rank_ == num_procs_ - 1) ? (total_size - start) : chunk; -// } -// #else -// start = 0; -// count = total_size; -// #endif -// } - void NetCDFFile::load_variables() { int retval; + int mode = read_only_ ? NC_NOWRITE : NC_WRITE; -#if NGEN_WITH_MPI - if(comm_ != MPI_COMM_NULL) { - if((retval = nc_open_par(ncFileName.c_str(), NC_NOWRITE, comm_, MPI_INFO_NULL, &ncid_))) { - throw std::runtime_error("Cannot open NetCDF (MPI): " + std::string(nc_strerror(retval))); - } - } else -#endif - { - if((retval = nc_open(nc_file_name_.c_str(), NC_NOWRITE, &ncid_))) { - throw std::runtime_error("Cannot open NetCDF: " + std::string(nc_strerror(retval))); + if (retval != NC_NOERR) { + throw std::runtime_error(std::string("Failed to open NetCDF file: ") + nc_strerror(retval)); + } + + // Load dimensions + int num_dims; + retval = nc_inq_ndims(ncid_, &num_dims); + if (retval != NC_NOERR) + throw std::runtime_error(std::string("Number of Dimensions: ") + nc_strerror(retval)); + + for (int i = 0; i < num_dims; ++i) { + char dim_name[NC_MAX_NAME + 1]; + size_t dim_len; + retval = nc_inq_dim(ncid_, i, dim_name, &dim_len); + if (retval == NC_NOERR) { + dims_[dim_name] = i; } } + // Load variables int num_variables; - if((retval = nc_inq_nvars(ncid_, &num_variables))) { - nc_close(ncid_); - throw std::runtime_error("Cannot get number of variables: " + std::string(nc_strerror(retval))); - } + retval = nc_inq_nvars(ncid_, &num_variables); + if (retval != NC_NOERR) + throw std::runtime_error(std::string("Number of Variables: ") + nc_strerror(retval)); - variables_.clear(); + for (int i = 0; i < num_variables; ++i) { + char var_name[NC_MAX_NAME + 1]; + retval = nc_inq_varname(ncid_, i, var_name); + if (retval != NC_NOERR) continue; - for(int varid = 0; varid < num_variables; ++varid) { - char name[NC_MAX_NAME + 1]; nc_type type; - int num_dims, dim_ids[NC_MAX_VAR_DIMS], num_attributes; + int num_dims_var; + int dim_ids[NC_MAX_DIMS]; - if((retval = nc_inq_var(ncid_, varid, name, &type, &num_dims, dim_ids, &num_attributes))) { - std::cerr << "Warning: could not get var info for varid " << varid - << ": " << nc_strerror(retval) << std::endl; - continue; - } + retval = nc_inq_var(ncid_, i, nullptr, &type, &num_dims_var, dim_ids, nullptr); + if (retval != NC_NOERR) continue; - std::vector dims(num_dims); + // Collect dim names std::vector dim_names; - for(int d=0; d dim_ids_var; + for (int d = 0; d < num_dims_var; ++d) { + dim_ids_var.push_back(dim_ids[d]); char dim_name[NC_MAX_NAME + 1]; - if((retval = nc_inq_dim(ncid_, dim_ids[d], dim_name, &len))) { - std::cerr << "Warning: could not get dim length for dimid " << dim_ids[d] - << ": " << nc_strerror(retval) << std::endl; - len = 0; - } - dims[d] = len; + nc_inq_dim(ncid_, dim_ids[d], dim_name, nullptr); dim_names.push_back(dim_name); } - auto var = std::make_shared(name, type, dims, dim_names, varid, ncid_); - - // load attributes - for(int attid=0; attid buffer(att_len + 1, '\0'); - nc_get_att_text(ncid_, varid, attname, buffer.data()); - var->add_attribute(attname, std::string(buffer.data(), att_len)); - } else if(att_type == NC_INT) { - int val; - nc_get_att_int(ncid_, varid, attname, &val); - var->add_attribute(attname, val); - } else if(att_type == NC_DOUBLE) { - double val; - nc_get_att_double(ncid_, varid, attname, &val); - var->add_attribute(attname, val); - } - // extend for other types if needed (float, long, etc.) - } - - variables_.push_back(var); + // Create shared_ptr NcVar + auto nc_var = std::make_shared(var_name, type,dim_ids_var, dim_names, i, ncid_); + variables_.push_back(nc_var); + variables_map_[var_name] = nc_var; } - nc_close(ncid_); } std::vector NetCDFFile::list_variables() const @@ -330,14 +259,6 @@ std::vector NetCDFFile::list_variables() const return names; } -std::shared_ptr NetCDFFile::get_ncvar(const std::string& name) const -{ - for(const auto& var : variables_) { - if(var->get_name() == name) return var; - } - return nullptr; -} - void NetCDFFile::close_file() { if (ncid_ >= 0) nc_close(ncid_); } diff --git a/src/netcdf/NetCDFManager.cpp b/src/netcdf/NetCDFManager.cpp index 2ee86e9f8b..f629e8b0fa 100644 --- a/src/netcdf/NetCDFManager.cpp +++ b/src/netcdf/NetCDFManager.cpp @@ -3,7 +3,7 @@ #include "Formulation_Manager.hpp" #include "Catchment_Formulation.hpp" #include "simulation_time/Simulation_Time.hpp" -//#include "ewts_ngen/logger.hpp" +#include "ewts_ngen/logger.hpp" #include #include @@ -12,65 +12,144 @@ NetCDFManager::NetCDFManager(std::shared_ptr m { manager_ = manager; sim_time_ = std::make_shared(sim_time); + nc_filename_ = manager_->get_output_root() + output_name + ".nc"; +#if NGEN_WITH_MPI rank_ = mpi_rank; - rank_ = rank_ + 1; num_procs_ = mpi_num_procs; is_mpi_ = (num_procs_ > 1); - std::string ncOutputFileName = manager_->get_output_root() + output_name + ".nc"; -#if NGEN_WITH_MPI comm_ = (num_procs_ > 1) ? MPI_COMM_WORLD : MPI_COMM_NULL; - nc_file_ = std::make_shared(filename_, true, comm_); #else - nc_file_ = std::make_shared(ncOutputFileName, true); + is_mpi_ = false; +#endif + nc_file_ = std::make_unique(nc_filename_, true, is_mpi_); + define_catchment_netcdf_components(); +} + +int NetCDFManager::create_file(const std::string& filename) +{ +#if NGEN_WITH_MPI + if (num_procs_ > 1) { + // MPI-enabled NetCDF + is_mpi_ = true; + nc_file_ = std::make_unique(filename, true, is_mpi_); + }else{ + is_mpi_ = false; + } +#endif + nc_filename_ = filename; + nc_file_ = std::make_unique(nc_filename_, true, is_mpi_); + if (!nc_file_) { + throw std::runtime_error("Failed to create NetCDF file: " + filename); + } + return nc_file_ ->get_ncid(); +} + +void NetCDFManager::define_catchment_netcdf_components() +{ + std::string name; + int dim_id; + std::vector dims; + std::vector names; //add time dimension and variable - std::string name = "time"; - int num_timesteps = sim_time_->get_total_output_times(); - int dim_id = add_dimension(name, num_timesteps); - std::vector dims = {dim_id}; - std::vector names = {name}; - add_variable(name, NC_INT, dims, names); - - //add timestep values and attributes for time - std::vector time_epoch_seconds(num_timesteps); - time_epoch_seconds[0] = sim_time_->get_current_epoch_time(); - for(int time_index = 1; time_index < num_timesteps; time_index++) + try { - sim_time_->advance_timestep(); - time_epoch_seconds[time_index] = sim_time_->get_current_epoch_time(); + name = "time"; + int num_timesteps = sim_time_->get_total_output_times(); + dim_id = add_dimension(name, num_timesteps); + dims = {dim_id}; + names = {name}; + add_variable(name, NC_INT, dims, names); + + //add timestep values and attributes for time + std::vector time_epoch_seconds(num_timesteps); + time_epoch_seconds[0] = sim_time_->get_current_epoch_time(); + for(int time_index = 1; time_index < num_timesteps; time_index++) + { + sim_time_->advance_timestep(); + time_epoch_seconds[time_index] = sim_time_->get_current_epoch_time(); + } + nc_file_->write_variable_data(name, time_epoch_seconds); + nc_file_->write_attribute_to_ncvar(name, "units", "Seconds since 1970-01-01 00:00:00"); + nc_file_->write_attribute_to_ncvar(name, "calendar", "gregorian"); + } + catch(const std::runtime_error& e) + { + LOG(std::string("Error in adding time information to catchment NetCDF: ") + e.what(), LogLevel::FATAL); + throw std::runtime_error(std::string("Error in adding time information to catchment NetCDF: ") + e.what()); } - nc_file_->write_variable_data(name, time_epoch_seconds); - nc_file_->write_attribute_to_ncvar(name, "units", "Seconds since 1970-01-01 00:00:00"); - nc_file_->write_attribute_to_ncvar(name, "calendar", "gregorian"); //add catchments dimension and variable - name = "catchments"; - dim_id = add_dimension(name, manager_->get_size()); - dims = {dim_id}; - names = {name}; - add_variable(name, NC_STRING, dims, names); - - //add catchment IDs and attributes for catchments - catchments_.reserve(manager_->get_size()); - for (auto const& formulation_info : manager_->get_all_formulations()) + try + { + name = "catchments"; + dim_id = add_dimension(name, manager_->get_size()); + dims = {dim_id}; + names = {name}; + add_variable(name, NC_STRING, dims, names); + + //add catchment IDs and attributes for catchments + int num_catchments = manager_->get_size(); + catchments_.reserve(num_catchments); + for (auto const& formulation_info : manager_->get_all_formulations()) + { + std::string catchm = formulation_info.first; + if (catchm.compare(0, 3, "cat") != 0) { + catchm = "cat-" + catchm; + } + catchments_.push_back(catchm); + } + nc_file_->write_variable_data(name, catchments_); + nc_file_->write_attribute_to_ncvar(name, "Catchment ID", "Catchment identifier in input"); + + //Compute writing chunks for catchments + size_t base = num_catchments / num_procs_; + size_t rem = num_catchments % num_procs_; + chunk_count_ = base + (rank_ < rem ? 1 : 0); + chunk_start_ = rank_ * base + std::min(static_cast(rank_), rem); + LOG("in NcManager: Chunk start: " + std::to_string(chunk_start_) + "; Chunk count: " + + std::to_string(chunk_count_) + " for rank: " + std::to_string(rank_), LogLevel::DEBUG); + + } + catch(const std::runtime_error& e) { - std::string catchm = formulation_info.first; - catchments_.push_back(catchm); + LOG(std::string("Error in adding catchment IDs to catchment NetCDF: ") + e.what(), LogLevel::FATAL); + throw std::runtime_error(std::string("Error in adding catchment IDs to catchment NetCDF: ") + e.what()); } - nc_file_->write_variable_data(name, catchments_); - nc_file_->write_attribute_to_ncvar(name, "Catchment ID", "Catchment identifier in input"); -#endif + + // Add realization config output variables + try + { + add_output_variable_data_from_formulation(); + } + catch(const std::runtime_error& e) + { + LOG(std::string("Error in adding output variables information to catchment NetCDF: ") + e.what(), LogLevel::FATAL); + throw std::runtime_error(std::string("Error in adding output variables information to catchment NetCDF: ") + e.what()); + } + + // Resize the 2D array/vector that will hold the output data for each timestep. + int num_variables = nc_output_variables_.size(); + if (num_variables > 0){ + data_chunks_.resize(num_variables); + for(size_t v = 0; v < num_variables; ++v) + { + data_chunks_[v].resize(chunk_count_); + } + } + + // Switch to data mode for writing + nc_file_->end_def_mode(); } NetCDFManager::NetCDFManager(const std::string& filename, bool read_only) - : filename_(filename), read_only_(true) + : read_only_(true) { - if (read_only){ - nc_file_ = std::make_shared(filename_); + if (read_only_){ + nc_file_ = std::make_unique(filename, !read_only, false); } else{ - std::string ncOutputFileName = manager_->get_output_root() + filename + ".nc"; - nc_file_ = std::make_shared(filename_, !read_only); + throw std::runtime_error("Write only non-MPI function not implemented."); } #if NGEN_WITH_MPI comm_ = MPI_COMM_NULL; @@ -82,32 +161,14 @@ NetCDFManager::NetCDFManager(const std::string& filename, bool read_only) NetCDFManager::NetCDFManager() {} -void NetCDFManager::create_file(const std::string& filename) -{ -#if NGEN_WITH_MPI - if (num_procs_ > 1) { - // MPI-enabled NetCDF - nc_file_ = std::make_shared(filename, /*write=*/true, comm_); - } else { - // Serial NetCDF - nc_file_ = std::make_shared(filename, /*write=*/true); - } -#else - // Serial NetCDF - nc_file_ = std::make_shared(filename, /*write=*/true); -#endif - if (!nc_file_) { - throw std::runtime_error("Failed to create NetCDF file: " + filename); - } -} - int NetCDFManager::add_dimension(const std::string& name, size_t len){ return nc_file_->add_dimension(name, len); } void NetCDFManager::add_variable(const std::string& var_name, nc_type type, const std::vector& dims, const std::vector& dim_names){ - return nc_file_->add_variable(var_name, NC_INT, dims, dim_names); + nc_file_->add_variable(var_name, type, dims, dim_names); } + void NetCDFManager::add_output_variable_data_from_formulation() { typename std::map>::const_iterator it = manager_->begin(); @@ -117,7 +178,7 @@ void NetCDFManager::add_output_variable_data_from_formulation() std::vectoroutput_variables = r_c->get_output_variable_names(); std::vectoroutput_headers = r_c->get_output_header_fields(); std::vectoroutput_units = r_c->get_output_variable_units(); - nc_output_variables.resize(output_variables.size()); + nc_output_variables_.reserve(output_variables.size()); std::vector dims_2D = {nc_file_->get_dim_id("time"), nc_file_->get_dim_id("catchments")}; std::vector dim_names{"time", "catchments"}; @@ -127,7 +188,7 @@ void NetCDFManager::add_output_variable_data_from_formulation() nc_file_->write_attribute_to_ncvar(output_headers[index], "variable units", output_units[index]); nc_file_->write_attribute_to_ncvar(output_headers[index], "_FillValue", "-1.0"); nc_file_->write_attribute_to_ncvar(output_headers[index], "missing_value", "-1.0"); - nc_output_variables.push_back(output_headers[index]); + nc_output_variables_.push_back(output_headers[index]); } } } @@ -143,23 +204,75 @@ static std::vector string_split(std::string str, char delimiter) return res; } +void NetCDFManager::prepare_data_chunks(std::map catchment_output_vals) +{ + int num_variables = data_chunks_.size(); + if (num_variables > 0){ // if number of output variables is greater than zero + size_t index = 0; + for(auto it = catchment_output_vals.begin(); it != catchment_output_vals.end(); ++it) + { + if(index < chunk_start_) { index++; continue; } // skip catchments not for this rank + if(index >= chunk_start_ + chunk_count_) break; // done with this rank. + std::vector catchment_output = string_split(it->second, ','); + for(size_t var_index = 0; var_index < num_variables; ++var_index){ + data_chunks_[var_index][index - chunk_start_] = catchment_output[var_index]; + // LOG("Index value: " + std::to_string(index) + "; Inserted value " + std::to_string(catchment_output[var_index]) + " to " + + // std::to_string(var_index) + " for catchment index: " + std::to_string(index - chunk_start_), LogLevel::DEBUG); + } + index++; + } + // LOG("Number of Variables: " + std::to_string(data_chunks_.size()) + "; Number of catchments: " + + // std::to_string(data_chunks_[0].size()) + " for rank: " + std::to_string(rank_), LogLevel::DEBUG); + } +} + void NetCDFManager::write_simulations_response_from_formulation(size_t time_index, std::map catchment_output_values) { + LOG("Write simulation response for timestep: " + std::to_string(time_index), LogLevel::INFO); + auto var = nc_file_->get_ncvar("catchments"); + if (!var) throw std::runtime_error("Catchments variable not found"); for (auto const& catchment_val : catchment_output_values) { std::string catchment_id = catchment_val.first; - - int catchment_index = nc_file_->get_dim_id(catchment_id); + LOG("Getting Catchment index for: " + catchment_id, LogLevel::INFO); + size_t catchment_index = var->get_variable_index(catchment_id); + LOG("Catchment index for: " + catchment_id + " is: " + std::to_string(time_index), LogLevel::INFO); if (catchment_index < 0) { throw std::runtime_error("Catchment not found in NetCDF: " + catchment_id); } - size_t c_index = catchment_index; - std::vector start = {time_index, c_index}; + std::vector start = {time_index, catchment_index}; std::vector count = {1, 1}; std::vector catchment_output = string_split(catchment_val.second, ','); - for(int var_index = 0; var_index < nc_output_variables.size(); ++var_index) + LOG("Time index: " + std::to_string(time_index) + " has values: " + catchment_val.second, LogLevel::INFO); + for(int var_index = 0; var_index < nc_output_variables_.size(); ++var_index) { - nc_file_ ->write_catchment_output_data(nc_output_variables[var_index], start, count, catchment_output[var_index]); + LOG("Writing output data for: " + nc_output_variables_[var_index], LogLevel::INFO); + nc_file_ ->write_catchment_output_data(nc_output_variables_[var_index], start, count, catchment_output[var_index]); + } + } +} + +void NetCDFManager::write_timestep_data_to_netcdf(size_t time_index) +{ + if(nc_output_variables_.size() > 0){ + if(nc_output_variables_.size() != data_chunks_.size()){ + LOG("Number of Variables: " + std::to_string(nc_output_variables_.size()) + "; Chunk size: " + + std::to_string(data_chunks_.size()) + " for rank: " + std::to_string(rank_), LogLevel::DEBUG); + throw std::runtime_error("Mismatch between variable names and the size of the incoming values."); + } + for(size_t i = 0; i < nc_output_variables_.size(); ++i) + { + const std::string& name = nc_output_variables_[i]; + const std::vector& data = data_chunks_[i]; + + if(chunk_count_ == 0) continue; //nothing to write + + if(data.size() != chunk_count_) + throw std::runtime_error("Data chunk size mismatch for " + name); + + auto var = nc_file_->get_ncvar(name); + if (!var) throw std::runtime_error("Catchments variable not found"); + var->write_timesliced_data(time_index, chunk_start_, chunk_count_, data.data()); } } } diff --git a/src/netcdf/NetCDFVar.cpp b/src/netcdf/NetCDFVar.cpp index 206b637cf7..9218168acc 100644 --- a/src/netcdf/NetCDFVar.cpp +++ b/src/netcdf/NetCDFVar.cpp @@ -1,5 +1,6 @@ #if NGEN_WITH_NETCDF #include "NetCDFVar.hpp" +#include "ewts_ngen/logger.hpp" #include #include #include @@ -135,15 +136,56 @@ std::vector NetCDFVar::get_time_values() const return time_values; } +int NetCDFVar::get_int_value_at_index(const std::vector& index) const { + int value = -1; + if (index.empty()) { + return value; + } + int retval = nc_get_var1_int(ncid_, varid_, index.data(), &value); + if (retval != NC_NOERR) { + throw std::runtime_error(nc_strerror(retval)); + } + return value; +} + +double NetCDFVar::get_dbl_value_at_index(const std::vector& index) const { + double value = -1; + if (index.empty()) { + return value; + } + int retval = nc_get_var1_double(ncid_, varid_, index.data(), &value); + if (retval != NC_NOERR) { + throw std::runtime_error(nc_strerror(retval)); + } + return value; +} + +std::string NetCDFVar::get_str_value_at_index(const std::vector& index) const { + char* value = nullptr; + std::string str_value; + int retval = nc_get_var1_string(ncid_, varid_, index.data(), &value); + if (retval == NC_NOERR && value != nullptr) { + str_value = std::string(value); + nc_free_string(1, &value); // free allocated string + } + return str_value; +} + void NetCDFVar::add_attribute(const std::string& att_name, const std::string& att_value) { + int retval = nc_put_att_text(ncid_, varid_, att_name.c_str(), att_value.size(), att_value.c_str()); + if (retval != NC_NOERR) throw std::runtime_error(nc_strerror(retval)); attributes_str_[att_name] = att_value; } void NetCDFVar::add_attribute(const std::string& att_name, int att_value) { + int retval = nc_put_att_int(ncid_, varid_, att_name.c_str(), NC_INT, 1, &att_value); + if (retval != NC_NOERR) throw std::runtime_error(nc_strerror(retval)); attributes_int_[att_name] = att_value; } void NetCDFVar::add_attribute(const std::string& att_name, double att_value) { + int retval = nc_put_att_double(ncid_, varid_, att_name.c_str(), NC_DOUBLE, 1, &att_value); + if (retval != NC_NOERR) throw std::runtime_error(nc_strerror(retval)); attributes_double_[att_name] = att_value; } @@ -175,6 +217,27 @@ double NetCDFVar::get_double_attribute(const std::string& att_name) const { throw std::runtime_error("Double attribute not found: " + att_name); } +size_t NetCDFVar::get_variable_index(const std::string& name) const +{ + auto it = variable_index_.find(name); + if (it == variable_index_.end()) { + throw std::runtime_error(std::string("Variable not found in NetCDF: ") + name); + } + return it->second; +} + +void NetCDFVar::build_variables_index(size_t num_items) +{ + std::vector data(num_items); + nc_get_var_string(ncid_, varid_, data.data()); + LOG("Number of catchments: " + std::to_string(num_items), LogLevel::INFO); + for (size_t index = 0; index < num_items; ++index) { + std::string key = data[index]; //required to prevent heap corruption while freeing memory later + variable_index_[key] = index; + } + nc_free_string(dims_[0], data.data()); +} + void NetCDFVar::read_slice(const std::vector& start, const std::vector& count, double* data) const { int retval = NC_NOERR; @@ -183,4 +246,22 @@ void NetCDFVar::read_slice(const std::vector& start, const std::vector start = {timestep, slice_start}; + std::vector count = {1, slice_count}; + int retval; + +#if NGEN_WITH_MPI + retval = nc_var_par_access(ncid_, varid_, NC_COLLECTIVE); + if(retval!= NC_NOERR) { + throw std::runtime_error(std::string("Failed to set up parallel access: ") + nc_strerror(retval)); + } +#endif + retval = nc_put_vara_double(ncid_, varid_, start.data(), count.data(), data); + if (retval != NC_NOERR) { + throw std::runtime_error("Error writing value in NetCDF: " + std::string(nc_strerror(retval))); + } +} #endif // NGEN_WITH_NETCDF \ No newline at end of file diff --git a/src/utilities/mdframe/handler_netcdf.cpp b/src/utilities/mdframe/handler_netcdf.cpp index 6141c56e9d..fae78eba78 100644 --- a/src/utilities/mdframe/handler_netcdf.cpp +++ b/src/utilities/mdframe/handler_netcdf.cpp @@ -4,7 +4,7 @@ #if NGEN_WITH_NETCDF -//#include +#include #include "netcdf/NetCDFManager.hpp" #include "netcdf/NetCDFFile.hpp" #include "netcdf/NetCDFVar.hpp" @@ -15,16 +15,19 @@ namespace visitors { struct mdarray_netcdf_putvar : public boost::static_visitor { + std::shared_ptr var; + mdarray_netcdf_putvar(std::shared_ptr v) : var(v) {} + template - void operator()(const mdarray& arr, netCDF::NcVar& var) + void operator()(const mdarray& arr) const { typename mdarray::size_type rank = arr.rank(); std::vector::size_type> expanded_index(rank); - + for (auto it = arr.begin(); it != arr.end(); it++) { it.mdindex(expanded_index); - var.putVar(expanded_index, *it); + //var.putVar(expanded_index, *it); } } @@ -34,56 +37,50 @@ struct mdarray_netcdf_putvar : public boost::static_visitor void mdframe::to_netcdf(const std::string& path) const { - nc_manager = new NetCDFManager(); - nc_manager.createFile(path); + NetCDFManager nc_manager; + int ncid = nc_manager.create_file(path); + NetCDFFile* nc_file = nc_manager.get_file_handle(); - std::unordered_map dimmap; + std::unordered_map dimmap; for (const auto& dim : this->m_dimensions) dimmap[dim.name()] = nc_manager.add_dimension(dim.name(), dim.size()); - - std::unordered_map varmap; - - netCDF::NcFile output{path, netCDF::NcFile::replace}; - - std::unordered_map dimmap; - std::unordered_map varmap; - - for (const auto& dim : this->m_dimensions) - dimmap[dim.name()] = output.addDim(dim.name(), dim.size()); + + std::unordered_map> varmap; for (const auto& pair : this->m_variables) { - netCDF::NcType* type = nullptr; - decltype(auto) var = pair.second; - switch (var.values().which()) { - case 0: // int - type = &netCDF::ncInt; - break; + nc_type type; // NetCDF C type + auto& var = pair.second; - case 1: // float - type = &netCDF::ncFloat; + switch (var.values().which()) { + case 0: + type = NC_INT; break; - - case 2: // double - default: - type = &netCDF::ncDouble; + case 1: + type = NC_FLOAT; + break; + case 2: + default: type = NC_DOUBLE; break; } - - std::vector dimensions; - dimensions.reserve(var.rank()); + std::vector dim_ids; + std::vector dim_names; + dim_ids.reserve(var.rank()); + dim_names.reserve(var.rank()); for (const auto& dimname : var.dimensions()) - dimensions.push_back(dimmap[dimname]); + { + auto it = dimmap.find(dimname); + if(it == dimmap.end()) + throw std::runtime_error("NetCDF dimension not found: " + dimname); - const auto& nc_var = output.addVar(var.name(), *type, dimensions); - varmap[var.name()] = nc_var; - - auto visitor = std::bind( - visitors::mdarray_netcdf_putvar{}, - std::placeholders::_1, - nc_var - ); + dim_ids.push_back(it->second); + dim_names.push_back(it->first); + } - var.values().apply_visitor(visitor); + nc_manager.add_variable(var.name(), type, dim_ids, dim_names); + const auto& nc_var = nc_manager.get_ncvar_by_name(var.name()); + varmap[var.name()] = nc_var; + visitors::mdarray_netcdf_putvar visitor(nc_var); + boost::apply_visitor(visitor, var.values()); } } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 04533d7e25..d389ac1ab1 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -61,7 +61,7 @@ endif() add_executable(test_all) target_link_libraries(test_all PRIVATE gtest gtest_main) -target_link_libraries(test_all PRIVATE NetCDFCreator) +target_link_libraries(test_all PRIVATE NetCDFManager) set_target_properties(test_all PROPERTIES FOLDER test) target_include_directories(test_all PRIVATE ${CMAKE_CURRENT_LIST_DIR}/bmi) @@ -464,7 +464,7 @@ ngen_add_test( NGen::realizations_catchment NGen::forcing NGen::ngen_bmi - NetCDFCreator + NetCDFManager REQUIRES NGEN_WITH_NETCDF ) @@ -486,24 +486,24 @@ ngen_add_test( ) ########################## NetCDFCreator Class Tests -ngen_add_test( - test_netcdf_creator - OBJECTS - core/NetCDFCreatorTest.cpp - LIBRARIES - NGen::core - NGen::geopackage - NGen::geojson - NGen::logging - NGen::core_catchment - NGen::core_mediator - NGen::realizations_catchment - NGen::forcing - NGen::ngen_bmi - NetCDFCreator - REQUIRES - NGEN_WITH_NETCDF -) +# ngen_add_test( +# test_netcdf_creator +# OBJECTS +# core/NetCDFCreatorTest.cpp +# LIBRARIES +# NGen::core +# NGen::geopackage +# NGen::geojson +# NGen::logging +# NGen::core_catchment +# NGen::core_mediator +# NGen::realizations_catchment +# NGen::forcing +# NGen::ngen_bmi +# NetCDFCreator +# REQUIRES +# NGEN_WITH_NETCDF +# ) ########################## Primary Combined Unit Test Target ngen_add_test( diff --git a/test/utils/mdframe_netcdf_Test.cpp b/test/utils/mdframe_netcdf_Test.cpp index 722cf9e0d0..e1696da379 100644 --- a/test/utils/mdframe_netcdf_Test.cpp +++ b/test/utils/mdframe_netcdf_Test.cpp @@ -3,7 +3,9 @@ #include "gtest/gtest.h" #if NGEN_WITH_NETCDF -#include +#include "netcdf/NetCDFManager.hpp" +#include "netcdf/NetCDFFile.hpp" +#include "netcdf/NetCDFVar.hpp" #endif #include "mdframe.hpp" @@ -61,40 +63,41 @@ TEST_F(mdframe_netcdf_Test, io_netcdf) df.to_netcdf(this->path); - netCDF::NcFile ex; - ex.open(this->path, netCDF::NcFile::read); + NetCDFFile ex(this->path, false, false); - const auto xdim = ex.getDim("x"); - ASSERT_FALSE(xdim.isNull()); - ASSERT_EQ(xdim.getSize(), 2); + int dim_id = ex.get_dim_id("x"); + ASSERT_FALSE(dim_id == -1); + ASSERT_EQ(ex.get_dim_size("x"), 2); - const auto ydim = ex.getDim("y"); - ASSERT_FALSE(ydim.isNull()); - ASSERT_EQ(ydim.getSize(), 2); + dim_id = ex.get_dim_id("y"); + ASSERT_FALSE(dim_id == -1); + ASSERT_EQ(ex.get_dim_size("y"), 2); - const auto xvar = ex.getVar("x"); - const auto yvar = ex.getVar("y"); - const auto vvar = ex.getVar("v"); + auto xvar = ex.get_ncvar("x"); + auto yvar = ex.get_ncvar("y"); + auto vvar = ex.get_ncvar("v"); - EXPECT_EQ(xvar.getDimCount(), 1); - EXPECT_EQ(yvar.getDimCount(), 1); - EXPECT_EQ(vvar.getDimCount(), 2); + EXPECT_EQ(xvar->get_dim_count(), 1); + EXPECT_EQ(yvar->get_dim_count(), 1); + EXPECT_EQ(vvar->get_dim_count(), 2); int xval = 0; int yval = 0; double vval = 0; for (size_t x = 0; x < 2; x++) { - xvar.getVar({ x }, &xval); + std::vector index = {x}; + int xval = xvar->get_int_value_at_index(index); EXPECT_EQ(xval, x + 1); for (size_t y = 0; y < 2; y++) { - yvar.getVar({ y }, &yval); + index = {y}; + int yval = yvar->get_int_value_at_index(index); EXPECT_EQ(yval, y + 1); - - vvar.getVar({ x, y}, &vval); + index = {x, y}; + double vval = vvar->get_dbl_value_at_index(index); EXPECT_EQ(vval, (x + 1) * (y + 1)); } } - ex.close(); + if (ex.get_ncid() >= 0) nc_close(ex.get_ncid()); #endif }