From 2ec46226699c38374977bb47aa5c86d9e7c0f32f Mon Sep 17 00:00:00 2001 From: Mohammed Karim Date: Mon, 20 Apr 2026 14:07:08 -0700 Subject: [PATCH 1/9] provide time info from realization file --- include/bmi/Bmi_Py_Adapter.hpp | 4 + src/bmi/Bmi_Py_Adapter.cpp | 47 ++++++++ .../catchment/Bmi_Module_Formulation.cpp | 105 ++++++++++++++---- 3 files changed, 135 insertions(+), 21 deletions(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index 3e47fc2d16..dc81726e94 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -195,6 +195,10 @@ namespace models { get_grid_coordinates("get_grid_z", grid, 2, z); } + void ApplyRealizationTimeConfig(const std::string& start_time_iso, + const std::string& end_time_iso, + double dt_seconds); + double GetStartTime() override; std::string GetTimeUnits() override; diff --git a/src/bmi/Bmi_Py_Adapter.cpp b/src/bmi/Bmi_Py_Adapter.cpp index d4d667fedc..8829c2f0f3 100644 --- a/src/bmi/Bmi_Py_Adapter.cpp +++ b/src/bmi/Bmi_Py_Adapter.cpp @@ -236,4 +236,51 @@ void Bmi_Py_Adapter::UpdateUntil(double time) { bmi_model->attr("update_until")(time); } +void Bmi_Py_Adapter::ApplyRealizationTimeConfig(const std::string& start_time_iso, + const std::string& end_time_iso, + double dt_seconds) { + if (bmi_model == nullptr) { + throw std::runtime_error("Bmi_Py_Adapter cannot apply realization time config: Python BMI model is null."); + } + + py::object& model = *bmi_model; + + if (!py::hasattr(model, "adapter_set_realization_times")) { + throw std::runtime_error( + "Bmi_Py_Adapter cannot apply realization time config: Python BMI model '" + model_name + + "' does not define adapter_set_realization_times(start, end)." + ); + } + + model.attr("adapter_set_realization_times")(start_time_iso, end_time_iso); + + if (py::hasattr(model, "_timestep_size_s")) { + model.attr("_timestep_size_s") = py::float_(dt_seconds); + } + + if (py::hasattr(model, "dt")) { + model.attr("dt") = py::float_(dt_seconds); + } + + if (py::hasattr(model, "days_per_dt")) { + model.attr("days_per_dt") = py::float_(dt_seconds / 86400.0); + } + + if (py::hasattr(model, "_apply_realization_time_from_strings")) { + model.attr("_apply_realization_time_from_strings")(); + } + else { + throw std::runtime_error( + "Bmi_Py_Adapter cannot apply realization time config: Python BMI model '" + model_name + + "' does not define _apply_realization_time_from_strings()." + ); + } + + std::string msg = "Applied realization time config to Python BMI model '" + model_name + + "': start='" + start_time_iso + + "', end='" + end_time_iso + + "', dt_seconds=" + std::to_string(dt_seconds); + LOG(LogLevel::INFO, msg); +} + #endif //NGEN_WITH_PYTHON diff --git a/src/realizations/catchment/Bmi_Module_Formulation.cpp b/src/realizations/catchment/Bmi_Module_Formulation.cpp index 359a257f88..9fab6c0302 100644 --- a/src/realizations/catchment/Bmi_Module_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Module_Formulation.cpp @@ -6,8 +6,29 @@ #include "state_save_restore/State_Save_Utils.hpp" #include +#include +#include +#include +#include + +#if NGEN_WITH_PYTHON +#include "bmi/Bmi_Py_Adapter.hpp" +#endif + + std::stringstream bmiform_ss; +static std::string format_iso_like_utc_datetime(time_t epoch_time) { + std::tm* t = std::gmtime(&epoch_time); + if (t == nullptr) { + throw std::runtime_error("Unable to convert epoch time to UTC calendar time."); + } + + std::ostringstream oss; + oss << std::put_time(t, "%Y-%m-%d %H:%M:%S"); + return oss.str(); +} + namespace realization { void Bmi_Module_Formulation::create_formulation(boost::property_tree::ptree &config, geojson::PropertyMap *global) { geojson::PropertyMap options = this->interpret_parameters(config, global); @@ -426,15 +447,22 @@ namespace realization { return bmi_model_start_time_forcing_offset_s; } + void Bmi_Module_Formulation::inner_create_formulation(geojson::PropertyMap properties, bool needs_param_validation) { if (needs_param_validation) { validate_parameters(properties); } + // Required parameters first set_bmi_init_config(properties.at(BMI_REALIZATION_CFG_PARAM_REQ__INIT_CONFIG).as_string()); set_bmi_main_output_var(properties.at(BMI_REALIZATION_CFG_PARAM_REQ__MAIN_OUT_VAR).as_string()); set_model_type_name(properties.at(BMI_REALIZATION_CFG_PARAM_REQ__MODEL_TYPE).as_string()); + const std::string model_type_name = + boost::algorithm::to_lower_copy( + properties.at(BMI_REALIZATION_CFG_PARAM_REQ__MODEL_TYPE).as_string() + ); + // Then optional ... auto uses_forcings_it = properties.find(BMI_REALIZATION_CFG_PARAM_OPT__USES_FORCINGS); @@ -469,6 +497,41 @@ namespace realization { // now construct the adapter and init the model set_bmi_model(construct_model(properties)); +#if NGEN_WITH_PYTHON + if (model_type_name.find("topoflow") != std::string::npos) { + auto py_bmi_model = std::dynamic_pointer_cast(get_bmi_model()); + if (py_bmi_model == nullptr) { + throw std::runtime_error( + "TopoFlow-Glacier formulation did not construct a Python BMI adapter for catchment '" + + this->get_id() + "'."); + } + + const time_t realization_start_time = forcing->get_data_start_time(); + const time_t realization_end_time = forcing->get_data_stop_time(); + const long realization_dt_seconds = forcing->record_duration(); + + if (realization_dt_seconds <= 0) { + throw std::runtime_error( + "TopoFlow-Glacier forcing record duration is invalid for catchment '" + this->get_id() + "'."); + } + + const std::string start_time_iso = format_iso_like_utc_datetime(realization_start_time); + const std::string end_time_iso = format_iso_like_utc_datetime(realization_end_time); + + std::stringstream ss; + ss << "Applying TopoFlow-Glacier realization time config for catchment '" << this->get_id() + << "': start='" << start_time_iso + << "', end='" << end_time_iso + << "', dt_seconds=" << realization_dt_seconds << std::endl; + LOG(ss.str(), LogLevel::INFO); + + py_bmi_model->ApplyRealizationTimeConfig( + start_time_iso, + end_time_iso, + static_cast(realization_dt_seconds)); + } +#endif + //Check if any parameter values need to be set on the BMI model, //and set them before it is run set_initial_bmi_parameters(properties); @@ -484,14 +547,14 @@ namespace realization { std::vector out_vars_json_list = out_var_it->second.as_list(); //Check if the first item is an object type or string type. //string type: old format; object type: new format - if (out_vars_json_list.size() > 0){ + if (out_vars_json_list.size() > 0) { std::string item_type = get_propertytype_name(out_vars_json_list[0].get_type()); - if (item_type == "String"){ + if (item_type == "String") { set_realization_file_format(true); } } std::vector out_vars(out_vars_json_list.size()); - if (is_realization_legacy_format()){ + if (is_realization_legacy_format()) { for (int i = 0; i < out_vars_json_list.size(); ++i) { out_vars[i] = out_vars_json_list[i].as_string(); } @@ -499,17 +562,17 @@ namespace realization { if (out_vars.size() == 1 && out_vars[0].empty()) out_vars.pop_back(); } - else{ + else { out_headers.resize(out_vars_json_list.size()); //assumption: number of vars = number of headers output_var_units.resize(out_vars_json_list.size()); //assumption: number of vars = number of units - output_var_indices.resize(out_vars_json_list.size(), 0); + output_var_indices.resize(out_vars_json_list.size(), 0); for (int i = 0; i < out_vars_json_list.size(); ++i) { out_vars[i] = out_vars_json_list[i].at("name").as_string(); - if(out_vars_json_list[i].has_key("header")){ + if (out_vars_json_list[i].has_key("header")) { //indicates that a valid header is provided out_headers[i] = out_vars_json_list[i].at("header").as_string(); } - else{ + else { //indicates that header is not provided. The error actually returns a string. //in such cases, we assign variable name to the header. out_headers[i] = out_vars[i]; @@ -517,15 +580,15 @@ namespace realization { ss << "Header not provided for '" << out_vars[i] << "'. Using the variable name as header." << std::endl; LOG(ss.str(), LogLevel::INFO); ss.str(""); } - if(out_vars_json_list[i].has_key("units")){ + if (out_vars_json_list[i].has_key("units")) { //indicates that a valid unit is provided output_var_units[i] = out_vars_json_list[i].at("units").as_string(); } - else{ - LOG("Units not provided for '" + out_vars[i] + "' in the realization file.",LogLevel::WARNING); - output_var_units[i] = ""; //add an empty entry and populate it with BMI native units later. + else { + LOG("Units not provided for '" + out_vars[i] + "' in the realization file.", LogLevel::WARNING); + output_var_units[i] = ""; //add an empty entry and populate it with BMI native units later. } - if(out_vars_json_list[i].has_key("index")){ + if (out_vars_json_list[i].has_key("index")) { //indicates that a valid index is provided output_var_indices[i] = stoi(out_vars_json_list[i].at("index").as_string()); } @@ -533,8 +596,7 @@ namespace realization { //check if the units can be parsed correctly and write a warning message std::stringstream ss; for (const std::string& out_unit : output_var_units) { - if (!UnitsHelper::can_parse(out_unit)) - { + if (!UnitsHelper::can_parse(out_unit)) { ss << "Unable to parse '" << out_unit << "' in units value." << std::endl; LOG(ss.str(), LogLevel::WARNING); ss.str(""); } @@ -557,7 +619,7 @@ namespace realization { // Output header fields, if present auto out_headers_it = properties.find(BMI_REALIZATION_CFG_PARAM_OPT__OUT_HEADER_FIELDS); - if(is_realization_legacy_format()){ + if (is_realization_legacy_format()) { if (out_headers_it != properties.end() && get_output_variable_names().size() > 0) { std::vector out_headers_json_list = out_var_it->second.as_list(); std::vector out_headers(out_headers_json_list.size()); @@ -570,9 +632,9 @@ namespace realization { set_output_header_fields(get_output_variable_names()); } } - else{ + else { if (out_headers_it != properties.end()) { - //indicates that the new json format has legacy headers format in the realization. + //indicates that the new json format has legacy headers format in the realization. //put out a message that this is ignored. LOG("Deprecated output_header_fields item found in realization file ignored.", LogLevel::WARNING); } @@ -594,7 +656,7 @@ namespace realization { for (const std::string &output_var_name : get_bmi_model()->GetOutputVarNames()) { available_forcings.push_back(output_var_name); available_forcing_units[output_var_name] = get_bmi_model()->GetVarUnits(output_var_name); - if (bmi_var_names_map.find(output_var_name) != bmi_var_names_map.end()){ + if (bmi_var_names_map.find(output_var_name) != bmi_var_names_map.end()) { available_forcings.push_back(bmi_var_names_map[output_var_name]); available_forcing_units[bmi_var_names_map[output_var_name]] = get_bmi_model()->GetVarUnits(output_var_name); //units come from the model output variable. } @@ -607,21 +669,22 @@ namespace realization { //check if units have not been specified. If not, default to native units. std::string blank_string = ""; auto &names = get_output_variable_names(); - if(output_var_units.size() == 0){ + if (output_var_units.size() == 0) { output_var_units.resize(names.size(), blank_string); } for (int i = 0; i < names.size(); ++i) { - if (output_var_units[i] == blank_string){ + if (output_var_units[i] == blank_string) { output_var_units[i] = get_provider_units_for_variable(names[i]); } } //check if output variable indices (for vector variables) are specified in config. If not, default to zero (first index). - if(output_var_indices.size() == 0){ + if (output_var_indices.size() == 0) { output_var_indices.resize(names.size(), 0); } } + /** * @brief Template function for copying iterator range into contiguous array. * From 704734ab55930be97dff4eadeb0d0e764e283c83 Mon Sep 17 00:00:00 2001 From: Mohammed Karim Date: Mon, 20 Apr 2026 18:27:02 -0700 Subject: [PATCH 2/9] NOAH-OWP-MODULAR Updates --- .../catchment/Bmi_Module_Formulation.cpp | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/realizations/catchment/Bmi_Module_Formulation.cpp b/src/realizations/catchment/Bmi_Module_Formulation.cpp index 9fab6c0302..f0701fd844 100644 --- a/src/realizations/catchment/Bmi_Module_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Module_Formulation.cpp @@ -497,6 +497,34 @@ namespace realization { // now construct the adapter and init the model set_bmi_model(construct_model(properties)); +#if NGEN_WITH_BMI_FORTRAN + if (model_type_name.find("noah") != std::string::npos) { + const time_t realization_start_time = forcing->get_data_start_time(); + const time_t realization_end_time = forcing->get_data_stop_time(); + const long realization_dt_seconds = forcing->record_duration(); + + if (realization_dt_seconds <= 0) { + throw std::runtime_error( + "Noah-OWP forcing record duration is invalid for catchment '" + this->get_id() + "'."); + } + + const double start_time_value = static_cast(realization_start_time); + const double end_time_value = static_cast(realization_end_time); + const double dt_seconds_value = static_cast(realization_dt_seconds); + + std::stringstream ss; + ss << "Applying Noah-OWP realization time config for catchment '" << this->get_id() + << "': start_utime=" << realization_start_time + << ", end_utime=" << realization_end_time + << ", dt_seconds=" << realization_dt_seconds << std::endl; + LOG(ss.str(), LogLevel::INFO); + + get_bmi_model()->SetValue("ngen_realization_start_time", (void *)&start_time_value); + get_bmi_model()->SetValue("ngen_realization_end_time", (void *)&end_time_value); + get_bmi_model()->SetValue("ngen_realization_dt", (void *)&dt_seconds_value); + } +#endif + #if NGEN_WITH_PYTHON if (model_type_name.find("topoflow") != std::string::npos) { auto py_bmi_model = std::dynamic_pointer_cast(get_bmi_model()); From f6e2e2f6381d524dc439afb72b74372dc7e8fab1 Mon Sep 17 00:00:00 2001 From: Mohammed Karim Date: Tue, 21 Apr 2026 15:41:45 -0700 Subject: [PATCH 3/9] UEB Fix --- .../catchment/Bmi_Module_Formulation.cpp | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/realizations/catchment/Bmi_Module_Formulation.cpp b/src/realizations/catchment/Bmi_Module_Formulation.cpp index f0701fd844..f340a690a4 100644 --- a/src/realizations/catchment/Bmi_Module_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Module_Formulation.cpp @@ -525,6 +525,32 @@ namespace realization { } #endif + if (model_type_name.find("ueb") != std::string::npos) { + const time_t realization_start_time = forcing->get_data_start_time(); + const time_t realization_end_time = forcing->get_data_stop_time(); + const long realization_dt_seconds = forcing->record_duration(); + + if (realization_dt_seconds <= 0) { + throw std::runtime_error( + "UEB forcing record duration is invalid for catchment '" + this->get_id() + "'."); + } + + const double start_time_value = static_cast(realization_start_time); + const double end_time_value = static_cast(realization_end_time); + const double dt_seconds_value = static_cast(realization_dt_seconds); + + std::stringstream ss; + ss << "Applying UEB realization time config for catchment '" << this->get_id() + << "': start_utime=" << realization_start_time + << ", end_utime=" << realization_end_time + << ", dt_seconds=" << realization_dt_seconds << std::endl; + LOG(ss.str(), LogLevel::INFO); + + get_bmi_model()->SetValue("ngen_realization_start_time", (void *)&start_time_value); + get_bmi_model()->SetValue("ngen_realization_end_time", (void *)&end_time_value); + get_bmi_model()->SetValue("ngen_realization_dt", (void *)&dt_seconds_value); + } + #if NGEN_WITH_PYTHON if (model_type_name.find("topoflow") != std::string::npos) { auto py_bmi_model = std::dynamic_pointer_cast(get_bmi_model()); @@ -713,6 +739,7 @@ namespace realization { } } + /** * @brief Template function for copying iterator range into contiguous array. * From 934bf2c78006f9d1162a8aa57b0df3b198ef57bf Mon Sep 17 00:00:00 2001 From: Mohammed Karim Date: Fri, 1 May 2026 10:45:00 -0700 Subject: [PATCH 4/9] making model agnostic --- .../catchment/Bmi_Multi_Formulation.hpp | 10 ++ .../catchment/Bmi_Module_Formulation.cpp | 125 +++++++----------- 2 files changed, 57 insertions(+), 78 deletions(-) diff --git a/include/realizations/catchment/Bmi_Multi_Formulation.hpp b/include/realizations/catchment/Bmi_Multi_Formulation.hpp index 1eb2487917..7c8fe00425 100644 --- a/include/realizations/catchment/Bmi_Multi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Multi_Formulation.hpp @@ -25,6 +25,12 @@ class Bmi_Cpp_Multi_Array_Test; namespace realization { + static bool is_ngen_realization_time_input(const std::string& var_name) { + return var_name == "ngen_realization_start_time" || + var_name == "ngen_realization_end_time" || + var_name == "ngen_realization_dt"; + } + /** * Abstraction of a formulation with multiple backing model object that implements the BMI. */ @@ -636,6 +642,10 @@ namespace realization { std::shared_ptr> var_aliases; var_aliases = std::make_shared>(std::map()); for (const std::string &var_name : mod->get_bmi_input_variables()) { + if (is_ngen_realization_time_input(var_name)) { + continue; + } + std::string framework_alias = mod->get_config_mapped_variable_name(var_name); (*var_aliases)[framework_alias] = var_name; // If framework_name is not yet in collection from which we have available data sources ... diff --git a/src/realizations/catchment/Bmi_Module_Formulation.cpp b/src/realizations/catchment/Bmi_Module_Formulation.cpp index f340a690a4..9c4d315d98 100644 --- a/src/realizations/catchment/Bmi_Module_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Module_Formulation.cpp @@ -29,6 +29,13 @@ static std::string format_iso_like_utc_datetime(time_t epoch_time) { return oss.str(); } +static bool is_ngen_realization_time_input(const std::string& var_name) { + return var_name == "ngen_realization_start_time" || + var_name == "ngen_realization_end_time" || + var_name == "ngen_realization_dt"; +} + + namespace realization { void Bmi_Module_Formulation::create_formulation(boost::property_tree::ptree &config, geojson::PropertyMap *global) { geojson::PropertyMap options = this->interpret_parameters(config, global); @@ -493,98 +500,53 @@ namespace realization { } } - // Do this next, since after checking whether other input variables are present in the properties, we can // now construct the adapter and init the model set_bmi_model(construct_model(properties)); -#if NGEN_WITH_BMI_FORTRAN - if (model_type_name.find("noah") != std::string::npos) { - const time_t realization_start_time = forcing->get_data_start_time(); - const time_t realization_end_time = forcing->get_data_stop_time(); - const long realization_dt_seconds = forcing->record_duration(); - - if (realization_dt_seconds <= 0) { - throw std::runtime_error( - "Noah-OWP forcing record duration is invalid for catchment '" + this->get_id() + "'."); - } - - const double start_time_value = static_cast(realization_start_time); - const double end_time_value = static_cast(realization_end_time); - const double dt_seconds_value = static_cast(realization_dt_seconds); + { + auto bmi_model = get_bmi_model(); - std::stringstream ss; - ss << "Applying Noah-OWP realization time config for catchment '" << this->get_id() - << "': start_utime=" << realization_start_time - << ", end_utime=" << realization_end_time - << ", dt_seconds=" << realization_dt_seconds << std::endl; - LOG(ss.str(), LogLevel::INFO); + if (bmi_model != nullptr) { - get_bmi_model()->SetValue("ngen_realization_start_time", (void *)&start_time_value); - get_bmi_model()->SetValue("ngen_realization_end_time", (void *)&end_time_value); - get_bmi_model()->SetValue("ngen_realization_dt", (void *)&dt_seconds_value); - } -#endif + std::vector input_vars = bmi_model->GetInputVarNames(); - if (model_type_name.find("ueb") != std::string::npos) { - const time_t realization_start_time = forcing->get_data_start_time(); - const time_t realization_end_time = forcing->get_data_stop_time(); - const long realization_dt_seconds = forcing->record_duration(); - - if (realization_dt_seconds <= 0) { - throw std::runtime_error( - "UEB forcing record duration is invalid for catchment '" + this->get_id() + "'."); - } + const std::string start_var = "ngen_realization_start_time"; + const std::string end_var = "ngen_realization_end_time"; + const std::string dt_var = "ngen_realization_dt"; - const double start_time_value = static_cast(realization_start_time); - const double end_time_value = static_cast(realization_end_time); - const double dt_seconds_value = static_cast(realization_dt_seconds); + bool has_start = std::find(input_vars.begin(), input_vars.end(), start_var) != input_vars.end(); + bool has_end = std::find(input_vars.begin(), input_vars.end(), end_var) != input_vars.end(); + bool has_dt = std::find(input_vars.begin(), input_vars.end(), dt_var) != input_vars.end(); - std::stringstream ss; - ss << "Applying UEB realization time config for catchment '" << this->get_id() - << "': start_utime=" << realization_start_time - << ", end_utime=" << realization_end_time - << ", dt_seconds=" << realization_dt_seconds << std::endl; - LOG(ss.str(), LogLevel::INFO); + if (has_start && has_end && has_dt) { - get_bmi_model()->SetValue("ngen_realization_start_time", (void *)&start_time_value); - get_bmi_model()->SetValue("ngen_realization_end_time", (void *)&end_time_value); - get_bmi_model()->SetValue("ngen_realization_dt", (void *)&dt_seconds_value); - } + const time_t realization_start_time = forcing->get_data_start_time(); + const time_t realization_end_time = forcing->get_data_stop_time(); + const long realization_dt_seconds = forcing->record_duration(); -#if NGEN_WITH_PYTHON - if (model_type_name.find("topoflow") != std::string::npos) { - auto py_bmi_model = std::dynamic_pointer_cast(get_bmi_model()); - if (py_bmi_model == nullptr) { - throw std::runtime_error( - "TopoFlow-Glacier formulation did not construct a Python BMI adapter for catchment '" + - this->get_id() + "'."); - } + if (realization_dt_seconds <= 0) { + throw std::runtime_error( + "Forcing record duration is invalid for catchment '" + this->get_id() + "'." + ); + } - const time_t realization_start_time = forcing->get_data_start_time(); - const time_t realization_end_time = forcing->get_data_stop_time(); - const long realization_dt_seconds = forcing->record_duration(); + const double start_time_value = static_cast(realization_start_time); + const double end_time_value = static_cast(realization_end_time); + const double dt_seconds_value = static_cast(realization_dt_seconds); - if (realization_dt_seconds <= 0) { - throw std::runtime_error( - "TopoFlow-Glacier forcing record duration is invalid for catchment '" + this->get_id() + "'."); + std::stringstream ss; + ss << "Applying realization time config via BMI inputs for catchment '" << this->get_id() + << "': start_utime=" << realization_start_time + << ", end_utime=" << realization_end_time + << ", dt_seconds=" << realization_dt_seconds << std::endl; + LOG(ss.str(), LogLevel::INFO); + + bmi_model->SetValue(start_var, (void *)&start_time_value); + bmi_model->SetValue(end_var, (void *)&end_time_value); + bmi_model->SetValue(dt_var, (void *)&dt_seconds_value); + } } - - const std::string start_time_iso = format_iso_like_utc_datetime(realization_start_time); - const std::string end_time_iso = format_iso_like_utc_datetime(realization_end_time); - - std::stringstream ss; - ss << "Applying TopoFlow-Glacier realization time config for catchment '" << this->get_id() - << "': start='" << start_time_iso - << "', end='" << end_time_iso - << "', dt_seconds=" << realization_dt_seconds << std::endl; - LOG(ss.str(), LogLevel::INFO); - - py_bmi_model->ApplyRealizationTimeConfig( - start_time_iso, - end_time_iso, - static_cast(realization_dt_seconds)); } -#endif //Check if any parameter values need to be set on the BMI model, //and set them before it is run @@ -1003,6 +965,9 @@ namespace realization { time_t model_epoch_time = convert_model_time(model_init_time) + get_bmi_model_start_time_forcing_offset_s(); for (std::string & var_name : in_var_names) { + if (is_ngen_realization_time_input(var_name)) { + continue; + } data_access::GenericDataProvider *provider; std::string var_map_alias = get_config_mapped_variable_name(var_name); if (input_forcing_providers.find(var_map_alias) != input_forcing_providers.end()) { @@ -1090,6 +1055,9 @@ namespace realization { inputs << "Input variables were as follows:"; for (std::string & var_name : in_var_names) { + if (is_ngen_realization_time_input(var_name)) { + continue; + } data_access::GenericDataProvider *provider; std::string var_map_alias = get_config_mapped_variable_name(var_name); if (input_forcing_providers.find(var_map_alias) != input_forcing_providers.end()) { @@ -1237,4 +1205,5 @@ namespace realization { void* _; // this pointer will be unused by SetValue bmi->SetValue(StateSaveNames::FREE, _); } + } From 62e76883c847fb4fb5679039a2745c1b1d4a1c3f Mon Sep 17 00:00:00 2001 From: Mohammed Karim Date: Fri, 15 May 2026 09:59:51 -0700 Subject: [PATCH 5/9] removed updates in Bmi_Py_Adapter as recommended by Kyle and Ian --- include/bmi/Bmi_Py_Adapter.hpp | 4 - .../catchment/Bmi_Multi_Formulation.hpp | 12 +- src/bmi/Bmi_Py_Adapter.cpp | 47 ------- .../catchment/Bmi_Module_Formulation.cpp | 129 ++++-------------- 4 files changed, 36 insertions(+), 156 deletions(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index dc81726e94..3e47fc2d16 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -195,10 +195,6 @@ namespace models { get_grid_coordinates("get_grid_z", grid, 2, z); } - void ApplyRealizationTimeConfig(const std::string& start_time_iso, - const std::string& end_time_iso, - double dt_seconds); - double GetStartTime() override; std::string GetTimeUnits() override; diff --git a/include/realizations/catchment/Bmi_Multi_Formulation.hpp b/include/realizations/catchment/Bmi_Multi_Formulation.hpp index 78f94b72e0..1f2fdcf5a2 100644 --- a/include/realizations/catchment/Bmi_Multi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Multi_Formulation.hpp @@ -631,24 +631,22 @@ namespace realization { std::shared_ptr wfp = std::make_shared(this); std::shared_ptr mod = std::make_shared(identifier, wfp, output); - // Since this is a nested formulation, support usage of the '{{id}}' syntax for init config file paths. Catchment_Formulation::config_pattern_substitution(properties, BMI_REALIZATION_CFG_PARAM_REQ__INIT_CONFIG, "{{id}}", Catchment_Formulation::config_pattern_id_replacement(id)); - // Call create_formulation to perform the rest of the typical initialization steps for the formulation. mod->create_formulation(properties); - // Set this up for placing in the module_variable_maps member variable std::shared_ptr> var_aliases; var_aliases = std::make_shared>(std::map()); + for (const std::string &var_name : mod->get_bmi_input_variables()) { - if (is_ngen_realization_time_input(var_name)) { + if (var_name == "ngen_current_time") { continue; } std::string framework_alias = mod->get_config_mapped_variable_name(var_name); (*var_aliases)[framework_alias] = var_name; - // If framework_name is not yet in collection from which we have available data sources ... + if (availableData.count(framework_alias) != 1) { setup_nested_deferred_provider(var_name, framework_alias, mod, mod_index); } @@ -658,10 +656,10 @@ namespace realization { } } - // Also add the output variable aliases for (const std::string &var_name : mod->get_bmi_output_variables()) { std::string framework_alias = mod->get_config_mapped_variable_name(var_name); (*var_aliases)[framework_alias] = var_name; + if (availableData.count(framework_alias) > 0) { std::string throw_msg; throw_msg.assign( "Multi BMI cannot be created with module " + mod->get_model_type_name() + @@ -671,9 +669,11 @@ namespace realization { LOG(throw_msg, LogLevel::WARNING); throw std::runtime_error(throw_msg); } + availableData[framework_alias] = mod; available_forcing_units[framework_alias] = mod->get_provider_units_for_variable(framework_alias); } + module_variable_maps[mod_index] = var_aliases; return mod; } diff --git a/src/bmi/Bmi_Py_Adapter.cpp b/src/bmi/Bmi_Py_Adapter.cpp index 8829c2f0f3..d4d667fedc 100644 --- a/src/bmi/Bmi_Py_Adapter.cpp +++ b/src/bmi/Bmi_Py_Adapter.cpp @@ -236,51 +236,4 @@ void Bmi_Py_Adapter::UpdateUntil(double time) { bmi_model->attr("update_until")(time); } -void Bmi_Py_Adapter::ApplyRealizationTimeConfig(const std::string& start_time_iso, - const std::string& end_time_iso, - double dt_seconds) { - if (bmi_model == nullptr) { - throw std::runtime_error("Bmi_Py_Adapter cannot apply realization time config: Python BMI model is null."); - } - - py::object& model = *bmi_model; - - if (!py::hasattr(model, "adapter_set_realization_times")) { - throw std::runtime_error( - "Bmi_Py_Adapter cannot apply realization time config: Python BMI model '" + model_name + - "' does not define adapter_set_realization_times(start, end)." - ); - } - - model.attr("adapter_set_realization_times")(start_time_iso, end_time_iso); - - if (py::hasattr(model, "_timestep_size_s")) { - model.attr("_timestep_size_s") = py::float_(dt_seconds); - } - - if (py::hasattr(model, "dt")) { - model.attr("dt") = py::float_(dt_seconds); - } - - if (py::hasattr(model, "days_per_dt")) { - model.attr("days_per_dt") = py::float_(dt_seconds / 86400.0); - } - - if (py::hasattr(model, "_apply_realization_time_from_strings")) { - model.attr("_apply_realization_time_from_strings")(); - } - else { - throw std::runtime_error( - "Bmi_Py_Adapter cannot apply realization time config: Python BMI model '" + model_name + - "' does not define _apply_realization_time_from_strings()." - ); - } - - std::string msg = "Applied realization time config to Python BMI model '" + model_name + - "': start='" + start_time_iso + - "', end='" + end_time_iso + - "', dt_seconds=" + std::to_string(dt_seconds); - LOG(LogLevel::INFO, msg); -} - #endif //NGEN_WITH_PYTHON diff --git a/src/realizations/catchment/Bmi_Module_Formulation.cpp b/src/realizations/catchment/Bmi_Module_Formulation.cpp index 9c4d315d98..5edaf52519 100644 --- a/src/realizations/catchment/Bmi_Module_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Module_Formulation.cpp @@ -18,24 +18,12 @@ std::stringstream bmiform_ss; -static std::string format_iso_like_utc_datetime(time_t epoch_time) { - std::tm* t = std::gmtime(&epoch_time); - if (t == nullptr) { - throw std::runtime_error("Unable to convert epoch time to UTC calendar time."); - } - - std::ostringstream oss; - oss << std::put_time(t, "%Y-%m-%d %H:%M:%S"); - return oss.str(); -} - static bool is_ngen_realization_time_input(const std::string& var_name) { return var_name == "ngen_realization_start_time" || var_name == "ngen_realization_end_time" || var_name == "ngen_realization_dt"; } - namespace realization { void Bmi_Module_Formulation::create_formulation(boost::property_tree::ptree &config, geojson::PropertyMap *global) { geojson::PropertyMap options = this->interpret_parameters(config, global); @@ -503,51 +491,6 @@ namespace realization { // now construct the adapter and init the model set_bmi_model(construct_model(properties)); - { - auto bmi_model = get_bmi_model(); - - if (bmi_model != nullptr) { - - std::vector input_vars = bmi_model->GetInputVarNames(); - - const std::string start_var = "ngen_realization_start_time"; - const std::string end_var = "ngen_realization_end_time"; - const std::string dt_var = "ngen_realization_dt"; - - bool has_start = std::find(input_vars.begin(), input_vars.end(), start_var) != input_vars.end(); - bool has_end = std::find(input_vars.begin(), input_vars.end(), end_var) != input_vars.end(); - bool has_dt = std::find(input_vars.begin(), input_vars.end(), dt_var) != input_vars.end(); - - if (has_start && has_end && has_dt) { - - const time_t realization_start_time = forcing->get_data_start_time(); - const time_t realization_end_time = forcing->get_data_stop_time(); - const long realization_dt_seconds = forcing->record_duration(); - - if (realization_dt_seconds <= 0) { - throw std::runtime_error( - "Forcing record duration is invalid for catchment '" + this->get_id() + "'." - ); - } - - const double start_time_value = static_cast(realization_start_time); - const double end_time_value = static_cast(realization_end_time); - const double dt_seconds_value = static_cast(realization_dt_seconds); - - std::stringstream ss; - ss << "Applying realization time config via BMI inputs for catchment '" << this->get_id() - << "': start_utime=" << realization_start_time - << ", end_utime=" << realization_end_time - << ", dt_seconds=" << realization_dt_seconds << std::endl; - LOG(ss.str(), LogLevel::INFO); - - bmi_model->SetValue(start_var, (void *)&start_time_value); - bmi_model->SetValue(end_var, (void *)&end_time_value); - bmi_model->SetValue(dt_var, (void *)&dt_seconds_value); - } - } - } - //Check if any parameter values need to be set on the BMI model, //and set them before it is run set_initial_bmi_parameters(properties); @@ -960,14 +903,15 @@ namespace realization { "': no logic for converting value to variable's type."); } - void Bmi_Module_Formulation::set_model_inputs_prior_to_update(const double &model_init_time, time_step_t t_delta) { + void Bmi_Module_Formulation::set_model_inputs_prior_to_update(const double &model_init_time, time_step_t t_delta) { std::vector in_var_names = get_bmi_model()->GetInputVarNames(); time_t model_epoch_time = convert_model_time(model_init_time) + get_bmi_model_start_time_forcing_offset_s(); for (std::string & var_name : in_var_names) { - if (is_ngen_realization_time_input(var_name)) { + if (is_ngen_realization_time_input(var_name)) { continue; - } + } + data_access::GenericDataProvider *provider; std::string var_map_alias = get_config_mapped_variable_name(var_name); if (input_forcing_providers.find(var_map_alias) != input_forcing_providers.end()) { @@ -980,54 +924,46 @@ namespace realization { provider = forcing.get(); } - // TODO: probably need to actually allow this by default and warn, but have config option to activate - // this type of behavior - // TODO: account for arrays later int nbytes = get_bmi_model()->GetVarNbytes(var_name); int varItemSize = get_bmi_model()->GetVarItemsize(var_name); int numItems = nbytes / varItemSize; assert(nbytes % varItemSize == 0); std::shared_ptr value_ptr; - // Finally, use the value obtained to set the model input std::string type = get_bmi_model()->get_analogous_cxx_type(get_bmi_model()->GetVarType(var_name), varItemSize); - // Minimal change: normalize requested units (treat ""/none as dimensionless "1") std::string consumer_units = get_bmi_model()->GetVarUnits(var_name); if (consumer_units.empty() || consumer_units == "none") consumer_units = "1"; if (numItems != 1) { - //more than a single value needed for var_name - auto values = provider->get_values(CatchmentAggrDataSelector(this->get_catchment_id(),var_map_alias, model_epoch_time, t_delta, - consumer_units, 0)); - //need to marshal data types to the receiver as well - //this could be done a little more elegantly if the provider interface were - //"type aware", but for now, this will do (but requires yet another copy) - if(values.size() == 1){ - //FIXME this isn't generic broadcasting, but works for scalar implementations + auto values = provider->get_values(CatchmentAggrDataSelector(this->get_catchment_id(), var_map_alias, model_epoch_time, t_delta, + consumer_units, 0)); + + if (values.size() == 1) { #ifndef NGEN_QUIET std::stringstream ss; - ss << "WARN: broadcasting variable '" << var_name << "' from scalar to expected array\n";; + ss << "WARN: broadcasting variable '" << var_name << "' from scalar to expected array\n"; LOG(ss.str(), LogLevel::SEVERE); ss.str(""); #endif values.resize(numItems, values[0]); - } else if (values.size() != numItems) { + } + else if (values.size() != numItems) { throw std::runtime_error("Mismatch in item count for variable '" + var_name + "': model expects " + std::to_string(numItems) + ", provider returned " + std::to_string(values.size()) + " items\n"); - } - value_ptr = get_values_as_type( type, values.begin(), values.end() ); - } else { + value_ptr = get_values_as_type(type, values.begin(), values.end()); + } + else { try { - //scalar value - double value = provider->get_value(CatchmentAggrDataSelector(this->get_catchment_id(),var_map_alias, model_epoch_time, t_delta, + double value = provider->get_value(CatchmentAggrDataSelector(this->get_catchment_id(), var_map_alias, model_epoch_time, t_delta, consumer_units, 0)); value_ptr = get_value_as_type(type, value); - } catch (data_access::unit_conversion_exception &uce) { + } + catch (data_access::unit_conversion_exception &uce) { data_access::unit_error_log_key key{get_id(), var_map_alias, uce.provider_model_name, uce.provider_bmi_var_name, uce.what()}; auto ret = data_access::unit_errors_reported.insert(key); bool new_error = ret.second; @@ -1045,19 +981,21 @@ namespace realization { value_ptr = get_value_as_type(type, uce.unconverted_values[0]); } } + get_bmi_model()->SetValue(var_name, value_ptr.get()); } } - + void Bmi_Module_Formulation::append_model_inputs_to_stream(const double &model_init_time, time_step_t t_delta, std::stringstream &inputs) { std::vector in_var_names = get_bmi_model()->GetInputVarNames(); time_t model_epoch_time = convert_model_time(model_init_time) + get_bmi_model_start_time_forcing_offset_s(); inputs << "Input variables were as follows:"; for (std::string & var_name : in_var_names) { - if (is_ngen_realization_time_input(var_name)) { + if (is_ngen_realization_time_input(var_name)) { continue; - } + } + data_access::GenericDataProvider *provider; std::string var_map_alias = get_config_mapped_variable_name(var_name); if (input_forcing_providers.find(var_map_alias) != input_forcing_providers.end()) { @@ -1070,31 +1008,24 @@ namespace realization { provider = forcing.get(); } - // TODO: probably need to actually allow this by default and warn, but have config option to activate - // this type of behavior - // TODO: account for arrays later int nbytes = get_bmi_model()->GetVarNbytes(var_name); int varItemSize = get_bmi_model()->GetVarItemsize(var_name); int numItems = nbytes / varItemSize; std::shared_ptr value_ptr; - // Finally, use the value obtained to set the model input std::string type = get_bmi_model()->get_analogous_cxx_type(get_bmi_model()->GetVarType(var_name), varItemSize); - + inputs << "\n" << var_map_alias << " = "; if (numItems != 1) { - //more than a single value needed for var_name - auto values = provider->get_values(CatchmentAggrDataSelector(this->get_catchment_id(),var_map_alias, model_epoch_time, t_delta, - get_bmi_model()->GetVarUnits(var_name), 0)); - value_ptr = get_values_as_type( type, values.begin(), values.end() ); - // array like input: precipitation_mm_per_h = [0.2, 0.8, 1.8] + auto values = provider->get_values(CatchmentAggrDataSelector(this->get_catchment_id(), var_map_alias, model_epoch_time, t_delta, + get_bmi_model()->GetVarUnits(var_name), 0)); + value_ptr = get_values_as_type(type, values.begin(), values.end()); this->append_inputs(type, value_ptr, numItems, inputs); - - } else { - //scalar value - double value = provider->get_value(CatchmentAggrDataSelector(this->get_catchment_id(),var_map_alias, model_epoch_time, t_delta, - get_bmi_model()->GetVarUnits(var_name), 0)); + } + else { + double value = provider->get_value(CatchmentAggrDataSelector(this->get_catchment_id(), var_map_alias, model_epoch_time, t_delta, + get_bmi_model()->GetVarUnits(var_name), 0)); this->append_input(type, value, inputs); } } From 72609becb33cc2023c7d0ccb3aa6a04800a42e32 Mon Sep 17 00:00:00 2001 From: Mohammed Karim Date: Fri, 15 May 2026 11:09:55 -0700 Subject: [PATCH 6/9] update --- include/realizations/catchment/Bmi_Multi_Formulation.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/realizations/catchment/Bmi_Multi_Formulation.hpp b/include/realizations/catchment/Bmi_Multi_Formulation.hpp index 1f2fdcf5a2..cbc0252640 100644 --- a/include/realizations/catchment/Bmi_Multi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Multi_Formulation.hpp @@ -640,7 +640,7 @@ namespace realization { var_aliases = std::make_shared>(std::map()); for (const std::string &var_name : mod->get_bmi_input_variables()) { - if (var_name == "ngen_current_time") { + if (is_ngen_realization_time_input(var_name)) { continue; } From 056cc51da4fbe0b5033b2c1e9945ac92ac471765 Mon Sep 17 00:00:00 2001 From: Mohammed Karim Date: Fri, 15 May 2026 11:42:56 -0700 Subject: [PATCH 7/9] cleanup --- include/realizations/catchment/Bmi_Module_Formulation.hpp | 5 +++++ include/realizations/catchment/Bmi_Multi_Formulation.hpp | 7 ------- src/realizations/catchment/Bmi_Module_Formulation.cpp | 5 ----- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index 318fafe064..19aa15a68e 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -21,6 +21,11 @@ class Bmi_C_Pet_IT; class Bmi_Cpp_Multi_Array_Test; namespace realization { + static bool is_ngen_realization_time_input(const std::string& var_name) { + return var_name == "ngen_realization_start_time" || + var_name == "ngen_realization_end_time" || + var_name == "ngen_realization_dt"; + } /** * Abstraction of a formulation with a single backing model object that implements the BMI. diff --git a/include/realizations/catchment/Bmi_Multi_Formulation.hpp b/include/realizations/catchment/Bmi_Multi_Formulation.hpp index cbc0252640..712173b918 100644 --- a/include/realizations/catchment/Bmi_Multi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Multi_Formulation.hpp @@ -24,13 +24,6 @@ class Bmi_Multi_Formulation_Test; class Bmi_Cpp_Multi_Array_Test; namespace realization { - - static bool is_ngen_realization_time_input(const std::string& var_name) { - return var_name == "ngen_realization_start_time" || - var_name == "ngen_realization_end_time" || - var_name == "ngen_realization_dt"; - } - /** * Abstraction of a formulation with multiple backing model object that implements the BMI. */ diff --git a/src/realizations/catchment/Bmi_Module_Formulation.cpp b/src/realizations/catchment/Bmi_Module_Formulation.cpp index 5edaf52519..cdca00e758 100644 --- a/src/realizations/catchment/Bmi_Module_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Module_Formulation.cpp @@ -11,11 +11,6 @@ #include #include -#if NGEN_WITH_PYTHON -#include "bmi/Bmi_Py_Adapter.hpp" -#endif - - std::stringstream bmiform_ss; static bool is_ngen_realization_time_input(const std::string& var_name) { From 3c97a8a161e19848e47bf1ef12f0b52b0ce27c58 Mon Sep 17 00:00:00 2001 From: Mohammed Karim Date: Fri, 15 May 2026 12:22:25 -0700 Subject: [PATCH 8/9] riverting extra spaces --- .../catchment/Bmi_Multi_Formulation.hpp | 9 +- .../catchment/Bmi_Module_Formulation.cpp | 84 +++++++++++-------- 2 files changed, 52 insertions(+), 41 deletions(-) diff --git a/include/realizations/catchment/Bmi_Multi_Formulation.hpp b/include/realizations/catchment/Bmi_Multi_Formulation.hpp index 712173b918..38fcea7ec2 100644 --- a/include/realizations/catchment/Bmi_Multi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Multi_Formulation.hpp @@ -24,6 +24,7 @@ class Bmi_Multi_Formulation_Test; class Bmi_Cpp_Multi_Array_Test; namespace realization { + /** * Abstraction of a formulation with multiple backing model object that implements the BMI. */ @@ -624,14 +625,16 @@ namespace realization { std::shared_ptr wfp = std::make_shared(this); std::shared_ptr mod = std::make_shared(identifier, wfp, output); + // Since this is a nested formulation, support usage of the '{{id}}' syntax for init config file paths. Catchment_Formulation::config_pattern_substitution(properties, BMI_REALIZATION_CFG_PARAM_REQ__INIT_CONFIG, "{{id}}", Catchment_Formulation::config_pattern_id_replacement(id)); + // Call create_formulation to perform the rest of the typical initialization steps for the formulation. mod->create_formulation(properties); + // Set this up for placing in the module_variable_maps member variable std::shared_ptr> var_aliases; var_aliases = std::make_shared>(std::map()); - for (const std::string &var_name : mod->get_bmi_input_variables()) { if (is_ngen_realization_time_input(var_name)) { continue; @@ -649,10 +652,10 @@ namespace realization { } } + // Also add the output variable aliases for (const std::string &var_name : mod->get_bmi_output_variables()) { std::string framework_alias = mod->get_config_mapped_variable_name(var_name); (*var_aliases)[framework_alias] = var_name; - if (availableData.count(framework_alias) > 0) { std::string throw_msg; throw_msg.assign( "Multi BMI cannot be created with module " + mod->get_model_type_name() + @@ -662,11 +665,9 @@ namespace realization { LOG(throw_msg, LogLevel::WARNING); throw std::runtime_error(throw_msg); } - availableData[framework_alias] = mod; available_forcing_units[framework_alias] = mod->get_provider_units_for_variable(framework_alias); } - module_variable_maps[mod_index] = var_aliases; return mod; } diff --git a/src/realizations/catchment/Bmi_Module_Formulation.cpp b/src/realizations/catchment/Bmi_Module_Formulation.cpp index cdca00e758..1cc292051e 100644 --- a/src/realizations/catchment/Bmi_Module_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Module_Formulation.cpp @@ -437,12 +437,10 @@ namespace realization { return bmi_model_start_time_forcing_offset_s; } - void Bmi_Module_Formulation::inner_create_formulation(geojson::PropertyMap properties, bool needs_param_validation) { if (needs_param_validation) { validate_parameters(properties); } - // Required parameters first set_bmi_init_config(properties.at(BMI_REALIZATION_CFG_PARAM_REQ__INIT_CONFIG).as_string()); set_bmi_main_output_var(properties.at(BMI_REALIZATION_CFG_PARAM_REQ__MAIN_OUT_VAR).as_string()); @@ -483,6 +481,7 @@ namespace realization { } } + // Do this next, since after checking whether other input variables are present in the properties, we can // now construct the adapter and init the model set_bmi_model(construct_model(properties)); @@ -501,14 +500,14 @@ namespace realization { std::vector out_vars_json_list = out_var_it->second.as_list(); //Check if the first item is an object type or string type. //string type: old format; object type: new format - if (out_vars_json_list.size() > 0) { + if (out_vars_json_list.size() > 0){ std::string item_type = get_propertytype_name(out_vars_json_list[0].get_type()); - if (item_type == "String") { + if (item_type == "String"){ set_realization_file_format(true); } } std::vector out_vars(out_vars_json_list.size()); - if (is_realization_legacy_format()) { + if (is_realization_legacy_format()){ for (int i = 0; i < out_vars_json_list.size(); ++i) { out_vars[i] = out_vars_json_list[i].as_string(); } @@ -516,17 +515,17 @@ namespace realization { if (out_vars.size() == 1 && out_vars[0].empty()) out_vars.pop_back(); } - else { + else{ out_headers.resize(out_vars_json_list.size()); //assumption: number of vars = number of headers output_var_units.resize(out_vars_json_list.size()); //assumption: number of vars = number of units - output_var_indices.resize(out_vars_json_list.size(), 0); + output_var_indices.resize(out_vars_json_list.size(), 0); for (int i = 0; i < out_vars_json_list.size(); ++i) { out_vars[i] = out_vars_json_list[i].at("name").as_string(); - if (out_vars_json_list[i].has_key("header")) { + if(out_vars_json_list[i].has_key("header")){ //indicates that a valid header is provided out_headers[i] = out_vars_json_list[i].at("header").as_string(); } - else { + else{ //indicates that header is not provided. The error actually returns a string. //in such cases, we assign variable name to the header. out_headers[i] = out_vars[i]; @@ -534,15 +533,15 @@ namespace realization { ss << "Header not provided for '" << out_vars[i] << "'. Using the variable name as header." << std::endl; LOG(ss.str(), LogLevel::INFO); ss.str(""); } - if (out_vars_json_list[i].has_key("units")) { + if(out_vars_json_list[i].has_key("units")) { //indicates that a valid unit is provided output_var_units[i] = out_vars_json_list[i].at("units").as_string(); } - else { - LOG("Units not provided for '" + out_vars[i] + "' in the realization file.", LogLevel::WARNING); - output_var_units[i] = ""; //add an empty entry and populate it with BMI native units later. + else{ + LOG("Units not provided for '" + out_vars[i] + "' in the realization file.", LogLevel::WARNING); + output_var_units[i] = ""; //add an empty entry and populate it with BMI native units later. } - if (out_vars_json_list[i].has_key("index")) { + if(out_vars_json_list[i].has_key("index")){ //indicates that a valid index is provided output_var_indices[i] = stoi(out_vars_json_list[i].at("index").as_string()); } @@ -550,7 +549,8 @@ namespace realization { //check if the units can be parsed correctly and write a warning message std::stringstream ss; for (const std::string& out_unit : output_var_units) { - if (!UnitsHelper::can_parse(out_unit)) { + if (!UnitsHelper::can_parse(out_unit)) + { ss << "Unable to parse '" << out_unit << "' in units value." << std::endl; LOG(ss.str(), LogLevel::WARNING); ss.str(""); } @@ -573,7 +573,7 @@ namespace realization { // Output header fields, if present auto out_headers_it = properties.find(BMI_REALIZATION_CFG_PARAM_OPT__OUT_HEADER_FIELDS); - if (is_realization_legacy_format()) { + if(is_realization_legacy_format()){ if (out_headers_it != properties.end() && get_output_variable_names().size() > 0) { std::vector out_headers_json_list = out_var_it->second.as_list(); std::vector out_headers(out_headers_json_list.size()); @@ -586,9 +586,9 @@ namespace realization { set_output_header_fields(get_output_variable_names()); } } - else { + else{ if (out_headers_it != properties.end()) { - //indicates that the new json format has legacy headers format in the realization. + //indicates that the new json format has legacy headers format in the realization. //put out a message that this is ignored. LOG("Deprecated output_header_fields item found in realization file ignored.", LogLevel::WARNING); } @@ -610,7 +610,7 @@ namespace realization { for (const std::string &output_var_name : get_bmi_model()->GetOutputVarNames()) { available_forcings.push_back(output_var_name); available_forcing_units[output_var_name] = get_bmi_model()->GetVarUnits(output_var_name); - if (bmi_var_names_map.find(output_var_name) != bmi_var_names_map.end()) { + if (bmi_var_names_map.find(output_var_name) != bmi_var_names_map.end()){ available_forcings.push_back(bmi_var_names_map[output_var_name]); available_forcing_units[bmi_var_names_map[output_var_name]] = get_bmi_model()->GetVarUnits(output_var_name); //units come from the model output variable. } @@ -623,23 +623,21 @@ namespace realization { //check if units have not been specified. If not, default to native units. std::string blank_string = ""; auto &names = get_output_variable_names(); - if (output_var_units.size() == 0) { + if(output_var_units.size() == 0){ output_var_units.resize(names.size(), blank_string); } for (int i = 0; i < names.size(); ++i) { - if (output_var_units[i] == blank_string) { + if (output_var_units[i] == blank_string){ output_var_units[i] = get_provider_units_for_variable(names[i]); } } //check if output variable indices (for vector variables) are specified in config. If not, default to zero (first index). - if (output_var_indices.size() == 0) { + if(output_var_indices.size() == 0){ output_var_indices.resize(names.size(), 0); } } - - /** * @brief Template function for copying iterator range into contiguous array. * @@ -898,7 +896,7 @@ namespace realization { "': no logic for converting value to variable's type."); } - void Bmi_Module_Formulation::set_model_inputs_prior_to_update(const double &model_init_time, time_step_t t_delta) { + void Bmi_Module_Formulation::set_model_inputs_prior_to_update(const double &model_init_time, time_step_t t_delta) { std::vector in_var_names = get_bmi_model()->GetInputVarNames(); time_t model_epoch_time = convert_model_time(model_init_time) + get_bmi_model_start_time_forcing_offset_s(); @@ -919,23 +917,31 @@ namespace realization { provider = forcing.get(); } + // TODO: probably need to actually allow this by default and warn, but have config option to activate + // this type of behavior + // TODO: account for arrays later int nbytes = get_bmi_model()->GetVarNbytes(var_name); int varItemSize = get_bmi_model()->GetVarItemsize(var_name); int numItems = nbytes / varItemSize; assert(nbytes % varItemSize == 0); std::shared_ptr value_ptr; + // Finally, use the value obtained to set the model input std::string type = get_bmi_model()->get_analogous_cxx_type(get_bmi_model()->GetVarType(var_name), varItemSize); + // Minimal change: normalize requested units (treat ""/none as dimensionless "1") std::string consumer_units = get_bmi_model()->GetVarUnits(var_name); if (consumer_units.empty() || consumer_units == "none") consumer_units = "1"; if (numItems != 1) { + //more than a single value needed for var_name auto values = provider->get_values(CatchmentAggrDataSelector(this->get_catchment_id(), var_map_alias, model_epoch_time, t_delta, consumer_units, 0)); - + //need to marshal data types to the receiver as well + //this could be done a little more elegantly if the provider interface were + //"type aware", but for now, this will do (but requires yet another copy) if (values.size() == 1) { #ifndef NGEN_QUIET std::stringstream ss; @@ -943,22 +949,22 @@ namespace realization { LOG(ss.str(), LogLevel::SEVERE); ss.str(""); #endif values.resize(numItems, values[0]); - } - else if (values.size() != numItems) { + } else if (values.size() != numItems) { throw std::runtime_error("Mismatch in item count for variable '" + var_name + "': model expects " + std::to_string(numItems) + ", provider returned " + std::to_string(values.size()) + " items\n"); + } value_ptr = get_values_as_type(type, values.begin(), values.end()); } else { try { + //scalar value double value = provider->get_value(CatchmentAggrDataSelector(this->get_catchment_id(), var_map_alias, model_epoch_time, t_delta, consumer_units, 0)); value_ptr = get_value_as_type(type, value); - } - catch (data_access::unit_conversion_exception &uce) { + } catch (data_access::unit_conversion_exception &uce) { data_access::unit_error_log_key key{get_id(), var_map_alias, uce.provider_model_name, uce.provider_bmi_var_name, uce.what()}; auto ret = data_access::unit_errors_reported.insert(key); bool new_error = ret.second; @@ -976,11 +982,10 @@ namespace realization { value_ptr = get_value_as_type(type, uce.unconverted_values[0]); } } - get_bmi_model()->SetValue(var_name, value_ptr.get()); } } - + void Bmi_Module_Formulation::append_model_inputs_to_stream(const double &model_init_time, time_step_t t_delta, std::stringstream &inputs) { std::vector in_var_names = get_bmi_model()->GetInputVarNames(); time_t model_epoch_time = convert_model_time(model_init_time) + get_bmi_model_start_time_forcing_offset_s(); @@ -1003,22 +1008,28 @@ namespace realization { provider = forcing.get(); } + // TODO: probably need to actually allow this by default and warn, but have config option to activate + // this type of behavior + // TODO: account for arrays later int nbytes = get_bmi_model()->GetVarNbytes(var_name); int varItemSize = get_bmi_model()->GetVarItemsize(var_name); int numItems = nbytes / varItemSize; + // Finally, use the value obtained to set the model input std::shared_ptr value_ptr; std::string type = get_bmi_model()->get_analogous_cxx_type(get_bmi_model()->GetVarType(var_name), varItemSize); - + inputs << "\n" << var_map_alias << " = "; if (numItems != 1) { + //more than a single value needed for var_name auto values = provider->get_values(CatchmentAggrDataSelector(this->get_catchment_id(), var_map_alias, model_epoch_time, t_delta, - get_bmi_model()->GetVarUnits(var_name), 0)); + get_bmi_model()->GetVarUnits(var_name), 0)); value_ptr = get_values_as_type(type, values.begin(), values.end()); + // array like input: precipitation_mm_per_h = [0.2, 0.8, 1.8] this->append_inputs(type, value_ptr, numItems, inputs); - } - else { + } else { + //scalar value double value = provider->get_value(CatchmentAggrDataSelector(this->get_catchment_id(), var_map_alias, model_epoch_time, t_delta, get_bmi_model()->GetVarUnits(var_name), 0)); this->append_input(type, value, inputs); @@ -1131,5 +1142,4 @@ namespace realization { void* _; // this pointer will be unused by SetValue bmi->SetValue(StateSaveNames::FREE, _); } - } From 4cf61cf2233389c0b1185dd7a4f2217780d6bad0 Mon Sep 17 00:00:00 2001 From: Mohammed Karim Date: Fri, 15 May 2026 13:58:49 -0700 Subject: [PATCH 9/9] setting realization time --- .../catchment/Bmi_Module_Formulation.hpp | 6 +- .../catchment/Bmi_Multi_Formulation.hpp | 13 +- .../catchment/Bmi_Module_Formulation.cpp | 124 ++++++++++++------ 3 files changed, 96 insertions(+), 47 deletions(-) diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index 19aa15a68e..151e9958ab 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -313,7 +313,6 @@ namespace realization { } protected: - /** * @brief Get correct BMI variable name, which may be the output or something mapped to this output. * @@ -398,6 +397,11 @@ namespace realization { */ void set_initial_bmi_parameters(geojson::PropertyMap properties); + /** + * If supported by the BMI module, pass realization timing metadata through legal BMI SetValue calls. + */ + void set_realization_time_inputs(); + /** * Test whether backing model has fixed time step size. * diff --git a/include/realizations/catchment/Bmi_Multi_Formulation.hpp b/include/realizations/catchment/Bmi_Multi_Formulation.hpp index 38fcea7ec2..6738def708 100644 --- a/include/realizations/catchment/Bmi_Multi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Multi_Formulation.hpp @@ -625,24 +625,23 @@ namespace realization { std::shared_ptr wfp = std::make_shared(this); std::shared_ptr mod = std::make_shared(identifier, wfp, output); - // Since this is a nested formulation, support usage of the '{{id}}' syntax for init config file paths. + // Since this is a nested formulation, support usage of the '{{id}}' syntax for init config file paths. Catchment_Formulation::config_pattern_substitution(properties, BMI_REALIZATION_CFG_PARAM_REQ__INIT_CONFIG, "{{id}}", Catchment_Formulation::config_pattern_id_replacement(id)); - // Call create_formulation to perform the rest of the typical initialization steps for the formulation. + // Call create_formulation to perform the rest of the typical initialization steps for the formulation. mod->create_formulation(properties); - // Set this up for placing in the module_variable_maps member variable + // Set this up for placing in the module_variable_maps member variable std::shared_ptr> var_aliases; var_aliases = std::make_shared>(std::map()); for (const std::string &var_name : mod->get_bmi_input_variables()) { - if (is_ngen_realization_time_input(var_name)) { + if (is_ngen_realization_time_input(var_name)) { continue; } - std::string framework_alias = mod->get_config_mapped_variable_name(var_name); (*var_aliases)[framework_alias] = var_name; - + // If framework_name is not yet in collection from which we have available data sources ... if (availableData.count(framework_alias) != 1) { setup_nested_deferred_provider(var_name, framework_alias, mod, mod_index); } @@ -652,7 +651,7 @@ namespace realization { } } - // Also add the output variable aliases + // Also add the output variable aliases for (const std::string &var_name : mod->get_bmi_output_variables()) { std::string framework_alias = mod->get_config_mapped_variable_name(var_name); (*var_aliases)[framework_alias] = var_name; diff --git a/src/realizations/catchment/Bmi_Module_Formulation.cpp b/src/realizations/catchment/Bmi_Module_Formulation.cpp index 1cc292051e..ce92bb2499 100644 --- a/src/realizations/catchment/Bmi_Module_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Module_Formulation.cpp @@ -13,12 +13,6 @@ std::stringstream bmiform_ss; -static bool is_ngen_realization_time_input(const std::string& var_name) { - return var_name == "ngen_realization_start_time" || - var_name == "ngen_realization_end_time" || - var_name == "ngen_realization_dt"; -} - namespace realization { void Bmi_Module_Formulation::create_formulation(boost::property_tree::ptree &config, geojson::PropertyMap *global) { geojson::PropertyMap options = this->interpret_parameters(config, global); @@ -437,6 +431,56 @@ namespace realization { return bmi_model_start_time_forcing_offset_s; } + void Bmi_Module_Formulation::set_realization_time_inputs() { + auto model = get_bmi_model(); + if (model == nullptr) { + return; + } + + std::vector input_vars = model->GetInputVarNames(); + + const bool has_start = + std::find(input_vars.begin(), input_vars.end(), "ngen_realization_start_time") != input_vars.end(); + const bool has_end = + std::find(input_vars.begin(), input_vars.end(), "ngen_realization_end_time") != input_vars.end(); + const bool has_dt = + std::find(input_vars.begin(), input_vars.end(), "ngen_realization_dt") != input_vars.end(); + + if (!has_start && !has_end && !has_dt) { + return; + } + + const double start_time_value = static_cast(forcing->get_data_start_time()); + const double end_time_value = static_cast(forcing->get_data_stop_time()); + const double dt_value = static_cast(forcing->record_duration()); + + if (dt_value <= 0.0) { + throw std::runtime_error( + "Invalid realization record duration while setting BMI realization time inputs for catchment '" + + this->get_id() + "'." + ); + } + + if (has_start) { + model->SetValue("ngen_realization_start_time", (void *)&start_time_value); + } + + if (has_end) { + model->SetValue("ngen_realization_end_time", (void *)&end_time_value); + } + + if (has_dt) { + model->SetValue("ngen_realization_dt", (void *)&dt_value); + } + + std::stringstream ss; + ss << "Applied realization time inputs via BMI SetValue for catchment '" << this->get_id() + << "': start_utime=" << static_cast(start_time_value) + << ", end_utime=" << static_cast(end_time_value) + << ", dt_seconds=" << static_cast(dt_value); + LOG(ss.str(), LogLevel::INFO); + } + void Bmi_Module_Formulation::inner_create_formulation(geojson::PropertyMap properties, bool needs_param_validation) { if (needs_param_validation) { validate_parameters(properties); @@ -446,7 +490,7 @@ namespace realization { set_bmi_main_output_var(properties.at(BMI_REALIZATION_CFG_PARAM_REQ__MAIN_OUT_VAR).as_string()); set_model_type_name(properties.at(BMI_REALIZATION_CFG_PARAM_REQ__MODEL_TYPE).as_string()); - const std::string model_type_name = + const std::string model_type_name = boost::algorithm::to_lower_copy( properties.at(BMI_REALIZATION_CFG_PARAM_REQ__MODEL_TYPE).as_string() ); @@ -481,7 +525,7 @@ namespace realization { } } - // Do this next, since after checking whether other input variables are present in the properties, we can + // Do this next, since after checking whether other input variables are present in the properties, we can // now construct the adapter and init the model set_bmi_model(construct_model(properties)); @@ -489,6 +533,9 @@ namespace realization { //and set them before it is run set_initial_bmi_parameters(properties); + // Pass realization timing to BMI modules that explicitly advertise these inputs. + set_realization_time_inputs(); + // Make sure that this is able to interpret model time and convert to real time, since BMI model time is // usually starting at 0 and just counting up determine_model_time_offset(); @@ -533,12 +580,12 @@ namespace realization { ss << "Header not provided for '" << out_vars[i] << "'. Using the variable name as header." << std::endl; LOG(ss.str(), LogLevel::INFO); ss.str(""); } - if(out_vars_json_list[i].has_key("units")) { + if(out_vars_json_list[i].has_key("units")){ //indicates that a valid unit is provided output_var_units[i] = out_vars_json_list[i].at("units").as_string(); } else{ - LOG("Units not provided for '" + out_vars[i] + "' in the realization file.", LogLevel::WARNING); + LOG("Units not provided for '" + out_vars[i] + "' in the realization file.",LogLevel::WARNING); output_var_units[i] = ""; //add an empty entry and populate it with BMI native units later. } if(out_vars_json_list[i].has_key("index")){ @@ -550,7 +597,7 @@ namespace realization { std::stringstream ss; for (const std::string& out_unit : output_var_units) { if (!UnitsHelper::can_parse(out_unit)) - { + { ss << "Unable to parse '" << out_unit << "' in units value." << std::endl; LOG(ss.str(), LogLevel::WARNING); ss.str(""); } @@ -896,15 +943,14 @@ namespace realization { "': no logic for converting value to variable's type."); } - void Bmi_Module_Formulation::set_model_inputs_prior_to_update(const double &model_init_time, time_step_t t_delta) { + void Bmi_Module_Formulation::set_model_inputs_prior_to_update(const double &model_init_time, time_step_t t_delta) { std::vector in_var_names = get_bmi_model()->GetInputVarNames(); time_t model_epoch_time = convert_model_time(model_init_time) + get_bmi_model_start_time_forcing_offset_s(); for (std::string & var_name : in_var_names) { - if (is_ngen_realization_time_input(var_name)) { + if (is_ngen_realization_time_input(var_name)) { continue; } - data_access::GenericDataProvider *provider; std::string var_map_alias = get_config_mapped_variable_name(var_name); if (input_forcing_providers.find(var_map_alias) != input_forcing_providers.end()) { @@ -917,7 +963,7 @@ namespace realization { provider = forcing.get(); } - // TODO: probably need to actually allow this by default and warn, but have config option to activate + // TODO: probably need to actually allow this by default and warn, but have config option to activate // this type of behavior // TODO: account for arrays later int nbytes = get_bmi_model()->GetVarNbytes(var_name); @@ -926,26 +972,27 @@ namespace realization { assert(nbytes % varItemSize == 0); std::shared_ptr value_ptr; - // Finally, use the value obtained to set the model input + // Finally, use the value obtained to set the model input std::string type = get_bmi_model()->get_analogous_cxx_type(get_bmi_model()->GetVarType(var_name), varItemSize); - // Minimal change: normalize requested units (treat ""/none as dimensionless "1") + // Minimal change: normalize requested units (treat ""/none as dimensionless "1") std::string consumer_units = get_bmi_model()->GetVarUnits(var_name); if (consumer_units.empty() || consumer_units == "none") consumer_units = "1"; if (numItems != 1) { - //more than a single value needed for var_name - auto values = provider->get_values(CatchmentAggrDataSelector(this->get_catchment_id(), var_map_alias, model_epoch_time, t_delta, - consumer_units, 0)); - //need to marshal data types to the receiver as well + //more than a single value needed for var_name + auto values = provider->get_values(CatchmentAggrDataSelector(this->get_catchment_id(),var_map_alias, model_epoch_time, t_delta, + consumer_units, 0)); + //need to marshal data types to the receiver as well //this could be done a little more elegantly if the provider interface were //"type aware", but for now, this will do (but requires yet another copy) - if (values.size() == 1) { + if(values.size() == 1){ + //FIXME this isn't generic broadcasting, but works for scalar implementations #ifndef NGEN_QUIET std::stringstream ss; - ss << "WARN: broadcasting variable '" << var_name << "' from scalar to expected array\n"; + ss << "WARN: broadcasting variable '" << var_name << "' from scalar to expected array\n";; LOG(ss.str(), LogLevel::SEVERE); ss.str(""); #endif values.resize(numItems, values[0]); @@ -955,13 +1002,12 @@ namespace realization { " items\n"); } + value_ptr = get_values_as_type( type, values.begin(), values.end() ); - value_ptr = get_values_as_type(type, values.begin(), values.end()); - } - else { + } else { try { - //scalar value - double value = provider->get_value(CatchmentAggrDataSelector(this->get_catchment_id(), var_map_alias, model_epoch_time, t_delta, + //scalar value + double value = provider->get_value(CatchmentAggrDataSelector(this->get_catchment_id(),var_map_alias, model_epoch_time, t_delta, consumer_units, 0)); value_ptr = get_value_as_type(type, value); } catch (data_access::unit_conversion_exception &uce) { @@ -992,10 +1038,9 @@ namespace realization { inputs << "Input variables were as follows:"; for (std::string & var_name : in_var_names) { - if (is_ngen_realization_time_input(var_name)) { + if (is_ngen_realization_time_input(var_name)) { continue; } - data_access::GenericDataProvider *provider; std::string var_map_alias = get_config_mapped_variable_name(var_name); if (input_forcing_providers.find(var_map_alias) != input_forcing_providers.end()) { @@ -1008,30 +1053,31 @@ namespace realization { provider = forcing.get(); } - // TODO: probably need to actually allow this by default and warn, but have config option to activate + // TODO: probably need to actually allow this by default and warn, but have config option to activate // this type of behavior // TODO: account for arrays later int nbytes = get_bmi_model()->GetVarNbytes(var_name); int varItemSize = get_bmi_model()->GetVarItemsize(var_name); int numItems = nbytes / varItemSize; - // Finally, use the value obtained to set the model input std::shared_ptr value_ptr; + // Finally, use the value obtained to set the model input std::string type = get_bmi_model()->get_analogous_cxx_type(get_bmi_model()->GetVarType(var_name), varItemSize); inputs << "\n" << var_map_alias << " = "; if (numItems != 1) { - //more than a single value needed for var_name - auto values = provider->get_values(CatchmentAggrDataSelector(this->get_catchment_id(), var_map_alias, model_epoch_time, t_delta, + //more than a single value needed for var_name + auto values = provider->get_values(CatchmentAggrDataSelector(this->get_catchment_id(),var_map_alias, model_epoch_time, t_delta, get_bmi_model()->GetVarUnits(var_name), 0)); - value_ptr = get_values_as_type(type, values.begin(), values.end()); - // array like input: precipitation_mm_per_h = [0.2, 0.8, 1.8] + value_ptr = get_values_as_type( type, values.begin(), values.end() ); + // array like input: precipitation_mm_per_h = [0.2, 0.8, 1.8] this->append_inputs(type, value_ptr, numItems, inputs); + } else { - //scalar value - double value = provider->get_value(CatchmentAggrDataSelector(this->get_catchment_id(), var_map_alias, model_epoch_time, t_delta, - get_bmi_model()->GetVarUnits(var_name), 0)); + //scalar value + double value = provider->get_value(CatchmentAggrDataSelector(this->get_catchment_id(),var_map_alias, model_epoch_time, t_delta, + get_bmi_model()->GetVarUnits(var_name), 0)); this->append_input(type, value, inputs); } }