Kaz integration

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.1.6"
+current_version = "0.1.11"
 commit = false
 tag = false
 parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)"

.gitignore

@@ -11,11 +11,16 @@ __pycache__/
 # Project specific
 /runs
 /lightning_logs
-/configs
+/configs/*
+!/configs/examples/
 /.serena
 /st_forecast/data_utils/__pycache__
 /st_forecast/data_utils/__pycache__
 /st_forecast/data_utils/__pycache__
 /old_configs
 /st_forecast/data_utils/__pycache__
 /comparison_output
+/sapphire_ref
+/intermediate_results_kaz
+/memory
+/kaz_export

README.md

@@ -63,7 +63,7 @@ result = train_model(
 External applications (e.g., the SAPPHIRE forecast system) use `SapphirePredictor` to run predictions from a trained model folder:
 
 ```python
-from st_forecast.operational.sapphire_predictor import SapphirePredictor
+from st_forecast import SapphirePredictor
 
 predictor = SapphirePredictor("runs/my_model")
 

configs/examples/rnn_lstm_caravan.json

@@ -0,0 +1,78 @@
+{
+  "region": "lamah_ce",
+  "data_source": "caravan",
+
+  "caravan": {
+    "base_path": "/path/to/Caravan/Caravan-nc",
+    "region": "lamah",
+    "n_basins": 10,
+    "variable_mapping": {
+      "streamflow": "discharge",
+      "total_precipitation_sum": "P",
+      "temperature_2m_mean": "T",
+      "potential_evaporation_sum_FAO_PENMAN_MONTEITH": "PET",
+      "snow_depth_water_equivalent_mean": "SWE"
+    }
+  },
+
+  "model_type": "RNN",
+  "model_name": "RNN_LSTM_LAMAH",
+  "encoder": "month",
+  "input_chunk_length": 30,
+  "forecast_horizon": 10,
+
+  "exog_features": ["P", "T", "PET", "SWE"],
+  "past_features": [],
+  "future_features": ["P", "T", "PET"],
+  "static_features": [],
+
+  "moving_windows": [3, 5, 10],
+  "shift_moving_avg": false,
+
+  "scale_discharge_bool": true,
+  "global_scaling": true,
+  "transform_log": false,
+  "transform_mm": false,
+
+  "discharge_scaler_type": "standard",
+  "forcing_scaler_type": "standard",
+  "static_scaler_type": "minmax",
+
+  "hidden_size": 128,
+  "rnn_model": "LSTM",
+  "n_rnn_layers": 1,
+  "training_length": 50,
+  "dropout": 0.2,
+  "n_epochs": 30,
+  "weight_decay": 1e-5,
+  "lr": 0.0005,
+  "batch_size": 1028,
+
+  "factor": 0.5,
+  "patience": 3,
+  "early_stopping_patience": 5,
+
+  "scheduler_type": "cosine",
+  "scheduler_t_max": 30,
+  "scheduler_eta_min": 5e-5,
+
+  "early_stopping_enabled": true,
+  "early_stopping_min_delta": 0.001,
+
+  "likelihood_type": "quantile",
+  "gradient_clip_val": 0.1,
+  "accelerator": "auto",
+  "checkpoint_strategy": "last",
+  "mc_dropout": true,
+
+  "quantiles": [0.05, 0.10, 0.25, 0.50, 0.75, 0.90, 0.95],
+  "num_samples": 100,
+
+  "work_dir": "/path/to/checkpoints/",
+
+  "train_years": [1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005],
+  "val_years": [2005, 2006, 2007, 2008],
+  "test_years": [2009, 2010, 2011, 2012, 2013, 2014, 2015],
+
+  "use_only_natural_rivers": false
+}

configs/tft_kgz.json → configs/examples/tft_sapphire.json

@@ -1,19 +1,20 @@
 {
   "region": "KGZ",
 
-  "path_to_operational_forcing": "../../../data/kyg_data_forecast_tools/intermediate_data/control_member_forcing",
-  "path_to_hindcast_forcing": "../../../data/kyg_data_forecast_tools/intermediate_data/hindcast_forcing",
+  "path_to_operational_forcing": "../../../data/xy_data_forecast_tools/intermediate_data/control_member_forcing",
+  "path_to_hindcast_forcing": "../../../data/xy_data_forecast_tools/intermediate_data/hindcast_forcing",
   "HRU_forcing": "00003",
-  "path_rivers": "../../../data/kyg_data_forecast_tools/intermediate_data/runoff_day.csv",
-  "path_static": "../../../data/kyg_data_forecast_tools/config/models_and_scalers/static_features/ML_basin_attributes_v2.csv",
-  "path_to_sla": "../../../data/sla_silvan/fsc_sla_timeseries_gapfilled.csv",
-  "path_to_nir": "../../../data/sla_silvan/meanNIR_TS_allBasins.csv",
-  "path_to_swe": "../../../data/kyg_data_forecast_tools/intermediate_data/snow_data/SWE",
-  "path_to_hs": "../../../data/kyg_data_forecast_tools/intermediate_data/snow_data/HS",
-  "path_to_rof": "../../../data/kyg_data_forecast_tools/intermediate_data/snow_data/RoF",
+  "path_rivers": "../../../data/xy_data_forecast_tools/intermediate_data/runoff_day.csv",
+  "path_static": "../../../data/xy_data_forecast_tools/config/models_and_scalers/static_features/ML_basin_attributes_v2.csv",
+  "path_to_sla": "../../../data/sla/fsc_sla_timeseries_gapfilled.csv",
+  "path_to_nir": "../../../data/sla/meanNIR_TS_allBasins.csv",
+  "path_to_swe": "../../../data/xy_data_forecast_tools/intermediate_data/snow_data/SWE",
+  "path_to_hs": "../../../data/xy_data_forecast_tools/intermediate_data/snow_data/HS",
+  "path_to_rof": "../../../data/xy_data_forecast_tools/intermediate_data/snow_data/RoF",
 
   "model_type": "TFT",
   "model_name": "TFT_KGZ",
+  "encoder": "month",
   "input_chunk_length": 30,
   "forecast_horizon": 10,
 
@@ -46,9 +47,6 @@
   "early_stopping_patience": 10,
 
   "scheduler_type": "reduce_on_plateau",
-  "scheduler_step_size": 10,
-  "scheduler_t_max": 30,
-  "scheduler_eta_min": 1e-5,
 
   "early_stopping_enabled": true,
   "early_stopping_min_delta": 0.001,
@@ -60,17 +58,16 @@
   "checkpoint_strategy": "last",
   "mc_dropout": true,
 
-
   "quantiles": [0.05, 0.10, 0.25, 0.50, 0.75, 0.90, 0.95],
   "num_samples": 200,
 
-  "work_dir": "/Users/sandrohunziker/Documents/Sapphire_Logs/",
+  "work_dir": "/path/to/checkpoints/",
 
   "train_years": [2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016],
   "val_years": [2017, 2018, 2019, 2020, 2021],
   "test_years": [2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025],
 
-  "codes_to_remove": [15049, 16093, 16681, 15960, 15954, 15213, 15217, 15261, 15292],
-  "unnatural_rivers": [15069, 15070, 15102, 15259, 17462, 16135, 16143, 16153, 16159],
+  "codes_to_remove": [],
+  "unnatural_rivers": [],
   "use_only_natural_rivers": false
 }

configs/examples/tide_caravan.json

@@ -0,0 +1,78 @@
+{
+  "region": "lamah_ce",
+  "data_source": "caravan",
+
+  "caravan": {
+    "base_path": "/path/to/Caravan/Caravan-nc",
+    "region": "lamah",
+    "n_basins": 10,
+    "variable_mapping": {
+      "streamflow": "discharge",
+      "total_precipitation_sum": "P",
+      "temperature_2m_mean": "T",
+      "potential_evaporation_sum_FAO_PENMAN_MONTEITH": "PET",
+      "snow_depth_water_equivalent_mean": "SWE"
+    }
+  },
+
+  "model_type": "TiDE",
+  "model_name": "TiDE_LAMAH",
+  "encoder": "month",
+  "input_chunk_length": 30,
+  "forecast_horizon": 10,
+
+  "exog_features": ["P", "T", "PET", "SWE"],
+  "past_features": ["moving_avr_dis_3", "moving_avr_dis_5", "moving_avr_dis_10"],
+  "future_features": ["P", "T", "PET", "SWE"],
+  "static_features": [],
+
+  "moving_windows": [3, 5, 10],
+  "shift_moving_avg": false,
+
+  "scale_discharge_bool": true,
+  "global_scaling": true,
+  "transform_log": false,
+  "transform_mm": false,
+
+  "discharge_scaler_type": "standard",
+  "forcing_scaler_type": "standard",
+  "static_scaler_type": "minmax",
+
+  "n_epochs": 30,
+  "weight_decay": 4.35e-05,
+  "lr": 0.0005,
+  "hidden_size": 64,
+  "dropout": 0.5,
+  "num_layers": 1,
+  "temporal_decoder_hidden_size": 32,
+  "decoder_output_dim": 16,
+  "batch_size": 1028,
+
+  "factor": 0.5,
+  "patience": 3,
+  "early_stopping_patience": 10,
+
+  "scheduler_type": "cosine",
+  "scheduler_t_max": 20,
+  "scheduler_eta_min": 5e-5,
+
+  "early_stopping_enabled": true,
+  "early_stopping_min_delta": 0.001,
+
+  "likelihood_type": "quantile",
+  "gradient_clip_val": 0.1,
+  "accelerator": "auto",
+  "checkpoint_strategy": "last",
+  "mc_dropout": true,
+
+  "quantiles": [0.05, 0.10, 0.25, 0.50, 0.75, 0.90, 0.95],
+  "num_samples": 200,
+
+  "work_dir": "/path/to/checkpoints/",
+
+  "train_years": [1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005],
+  "val_years": [2005, 2006, 2007, 2008],
+  "test_years": [2009, 2010, 2011, 2012, 2013, 2014, 2015],
+
+  "use_only_natural_rivers": false
+}

configs/examples/tsmixer_sapphire.json

@@ -0,0 +1,78 @@
+{
+  "region": "KGZ",
+
+  "path_to_operational_forcing": "../../../data/xy_data_forecast_tools/intermediate_data/control_member_forcing",
+  "path_to_hindcast_forcing": "../../../data/xy_data_forecast_tools/intermediate_data/hindcast_forcing",
+  "HRU_forcing": "00003",
+  "path_rivers": "../../../data/xy_data_forecast_tools/intermediate_data/runoff_day.csv",
+  "path_static": "../../../data/xy_data_forecast_tools/config/models_and_scalers/static_features/ML_basin_attributes_v2.csv",
+  "path_to_sla": "../../../data/sla/fsc_sla_timeseries_gapfilled.csv",
+  "path_to_nir": "../../../data/sla/meanNIR_TS_allBasins.csv",
+  "path_to_swe": "../../../data/xy_data_forecast_tools/intermediate_data/snow_data/SWE",
+  "path_to_hs": "../../../data/xy_data_forecast_tools/intermediate_data/snow_data/HS",
+  "path_to_rof": "../../../data/xy_data_forecast_tools/intermediate_data/snow_data/RoF",
+
+  "model_type": "TSMixer",
+  "model_name": "TSMixer_KGZ",
+  "encoder": "month",
+  "input_chunk_length": 30,
+  "forecast_horizon": 10,
+
+  "exog_features": ["P", "T", "PET", "ROF", "SWE", "daylight_hours"],
+  "past_features": ["moving_avr_dis_3", "moving_avr_dis_5", "moving_avr_dis_10"],
+  "future_features": ["P", "T", "PET", "daylight_hours"],
+
+  "moving_windows": [3, 5, 10],
+  "shift_moving_avg": false,
+
+  "scale_discharge_bool": true,
+  "global_scaling": true,
+  "transform_log": false,
+  "transform_mm": true,
+
+  "discharge_scaler_type": "standard",
+  "forcing_scaler_type": "standard",
+  "static_scaler_type": "standard",
+
+  "n_epochs": 40,
+  "weight_decay": 1e-5,
+  "lr": 0.0005,
+  "hidden_size": 64,
+  "dropout": 0.2,
+  "num_blocks": 2,
+  "ff_size": 64,
+  "activation": "ReLU",
+  "normalize_before": false,
+  "norm_type": "LayerNorm",
+  "batch_size": 2048,
+
+  "factor": 0.5,
+  "patience": 3,
+  "early_stopping_patience": 10,
+
+  "scheduler_type": "cosine",
+  "scheduler_t_max": 20,
+  "scheduler_eta_min": 5e-5,
+
+  "early_stopping_enabled": true,
+  "early_stopping_min_delta": 0.001,
+
+  "likelihood_type": "quantile",
+  "gradient_clip_val": 0.1,
+  "accelerator": "auto",
+  "checkpoint_strategy": "last",
+  "mc_dropout": true,
+
+  "quantiles": [0.05, 0.10, 0.25, 0.50, 0.75, 0.90, 0.95],
+  "num_samples": 200,
+
+  "work_dir": "/path/to/checkpoints/",
+
+  "train_years": [2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016],
+  "val_years": [2017, 2018, 2019, 2020, 2021],
+  "test_years": [2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025],
+
+  "codes_to_remove": [],
+  "unnatural_rivers": [],
+  "use_only_natural_rivers": false
+}